rebase & fix conflicts

.github/workflows/build-appimage.yml

@@ -1,6 +1,7 @@
 name: Build AppImage
 
 on:
+  workflow_dispatch:
   push:
     branches:
       - 'main'
@@ -15,7 +16,7 @@ env:
   
 jobs:
   build_appimage:
-    runs-on: ubuntu-22.04
+    runs-on: ubuntu-24.04
     defaults:
       run:
         working-directory: ./wayvr

dash-frontend/assets/lang/en.json

@@ -63,6 +63,8 @@
 		"KEYBOARD_MIDDLE_CLICK": "Keyboard middle click",
 		"KEYBOARD_MIDDLE_CLICK_HELP": "Modifier to use when typing\nwith purple laser",
 		"KEYBOARD_SOUND_ENABLED": "Keyboard sounds",
+		"KEYBOARD_SWIPE_TO_TYPE_ENABLED": "Keyboard swipe to type",
+		"KEYBOARD_SWIPE_TO_TYPE_ENABLED_HELP": "Only works for English! Results will be drastically worse on non qwerty keyboards.",
 		"LANGUAGE": "Language",
 		"LEFT_HANDED_MOUSE": "Left-handed mouse",
 		"LEFT_HANDED_MOUSE_HELP": "Use this if mouse buttons are swapped",

dash-frontend/src/tab/settings/mod.rs

@@ -231,6 +231,7 @@ enum SettingType {
 	InvertScrollDirectionY,
 	KeyboardMiddleClick,
 	KeyboardSoundEnabled,
+	KeyboardSwipeToTypeEnabled,
 	Language,
 	LeftHandedMouse,
 	LongPressDuration,
@@ -264,6 +265,7 @@ impl SettingType {
 			Self::NotificationsEnabled => &mut config.notifications_enabled,
 			Self::NotificationsSoundEnabled => &mut config.notifications_sound_enabled,
 			Self::KeyboardSoundEnabled => &mut config.keyboard_sound_enabled,
+			Self::KeyboardSwipeToTypeEnabled => &mut config.keyboard_swipe_to_type_enabled,
 			Self::UprightScreenFix => &mut config.upright_screen_fix,
 			Self::DoubleCursorFix => &mut config.double_cursor_fix,
 			Self::SetsOnWatch => &mut config.sets_on_watch,
@@ -381,6 +383,7 @@ impl SettingType {
 			Self::InvertScrollDirectionY => Ok("APP_SETTINGS.INVERT_SCROLL_DIRECTION_Y"),
 			Self::KeyboardMiddleClick => Ok("APP_SETTINGS.KEYBOARD_MIDDLE_CLICK"),
 			Self::KeyboardSoundEnabled => Ok("APP_SETTINGS.KEYBOARD_SOUND_ENABLED"),
+			Self::KeyboardSwipeToTypeEnabled => Ok("APP_SETTINGS.KEYBOARD_SWIPE_TO_TYPE_ENABLED"),
 			Self::Language => Ok("APP_SETTINGS.LANGUAGE"),
 			Self::LeftHandedMouse => Ok("APP_SETTINGS.LEFT_HANDED_MOUSE"),
 			Self::LongPressDuration => Ok("APP_SETTINGS.LONG_PRESS_DURATION"),
@@ -417,6 +420,7 @@ impl SettingType {
 			Self::GridOpacity => Some("APP_SETTINGS.GRID_OPACITY_HELP"),
 			Self::HandsfreePointer => Some("APP_SETTINGS.HANDSFREE_POINTER_HELP"),
 			Self::KeyboardMiddleClick => Some("APP_SETTINGS.KEYBOARD_MIDDLE_CLICK_HELP"),
+			Self::KeyboardSwipeToTypeEnabled => Some("APP_SETTINGS.KEYBOARD_SWIPE_TO_TYPE_ENABLED_HELP"),
 			Self::LeftHandedMouse => Some("APP_SETTINGS.LEFT_HANDED_MOUSE_HELP"),
 			Self::ScreenRenderDown => Some("APP_SETTINGS.SCREEN_RENDER_DOWN_HELP"),
 			Self::UprightScreenFix => Some("APP_SETTINGS.UPRIGHT_SCREEN_FIX_HELP"),

dash-frontend/src/tab/settings/tab_features.rs

@@ -1,6 +1,6 @@
 use crate::tab::settings::{
+	macros::{options_category, options_checkbox, options_slider_f32, MacroParams},
 	SettingType,
-	macros::{MacroParams, options_category, options_checkbox, options_slider_f32},
 };
 use wgui::layout::WidgetID;
 
@@ -9,6 +9,7 @@ pub fn mount(mp: &mut MacroParams, parent: WidgetID) -> anyhow::Result<()> {
 	options_checkbox(mp, c, SettingType::NotificationsEnabled)?;
 	options_checkbox(mp, c, SettingType::NotificationsSoundEnabled)?;
 	options_checkbox(mp, c, SettingType::KeyboardSoundEnabled)?;
+	options_checkbox(mp, c, SettingType::KeyboardSwipeToTypeEnabled)?;
 	options_checkbox(mp, c, SettingType::SpaceDragUnlocked)?;
 	options_checkbox(mp, c, SettingType::SpaceRotateUnlocked)?;
 	options_slider_f32(mp, c, SettingType::SpaceDragMultiplier, -10.0, 10.0, 0.5)?;

wayvr/Cargo.toml

@@ -90,6 +90,9 @@ xcb = { version = "1.6.0", features = [
   "as-raw-xcb-connection",
 ], optional = true }
 xkbcommon = { version = "0.8.0" }  # 0.9.0 breaks keymap import on some distros
+codes-iso-639 = "0.1.5"
+arboard = { version="3.6.1", features = ["wayland-data-control", "wl-clipboard-rs"] }
+super-swipe-type = "0.2.2"
 
 [build-dependencies]
 regex.workspace = true

wayvr/src/assets/gui/keyboard.xml

@@ -13,6 +13,11 @@
     width="${width}" height="${height}" min_width="${width}" min_height="${height}" max_width="${width}" max_height="${height}"
   />
 
+  <macro name="prediction_rect"
+    margin="2" padding="8 16" overflow="visible" box_sizing="border_box"
+    border_color="~color_accent_translucent" border="2" round="6" color="~color_accent_40" color2="~color_accent_10" gradient="vertical"
+    align_items="center" justify_content="center" />
+
   <template name="VerticalSeparator">
     <rectangle width="2" height="100%" color="~color_accent" />
   </template>
@@ -90,6 +95,13 @@
     </div>
   </template>
 
+  <!-- Word prediction key for swipe typing -->
+  <template name="KeyPrediction">
+    <rectangle id="${id}" macro="prediction_rect" height="60" min_height="60" flex_grow="1" flex_shrink="1" flex_basis="0">
+      <label text="${text}" size="16" flex_wrap="wrap" />
+    </rectangle>
+  </template>
+
   <macro name="button_style" border="2" border_color="~color_accent_translucent" color="~color_bg" round="6"
     align_items="center" justify_content="center" padding="6" width="60" height="60" overflow="visible"/>
 
@@ -231,7 +243,10 @@
           </div>
         </div>
       </rectangle>
-      <div width="100%" height="13" interactable="0" />
+      <div width="100%" height="11" interactable="0" />
+      <rectangle id="swipe_predictions_root" macro="bg_rect" padding="10" align_items="center" justify_content="space_between" flex_direction="row" min_height="60">
+      </rectangle>
+      <div width="100%" height="11" interactable="0" />
       <rectangle id="keyboard_root" macro="bg_rect" flex_direction="column" padding="10">
       </rectangle>
     </div>

wayvr/src/backend/wayvr/client.rs

@@ -7,10 +7,12 @@ use smithay::{
     reexports::wayland_server,
     utils::SerialCounter,
 };
+use smithay::input::Seat;
+use smithay::wayland::selection::data_device::set_data_device_selection;
 use xkbcommon::xkb;
 
 use crate::backend::wayvr::{ExternalProcessRequest, WayVRTask};
-
+use crate::backend::wayvr::comp::Application;
 use super::{
     ProcessWayVREnv,
     comp::{self, ClientState},
@@ -26,6 +28,7 @@ pub struct WayVRCompositor {
     pub state: comp::Application,
     pub seat_keyboard: KeyboardHandle<comp::Application>,
     pub seat_pointer: PointerHandle<comp::Application>,
+    pub seat: Seat<comp::Application>,
     pub serial_counter: SerialCounter,
     pub wayland_env: super::WaylandEnv,
 
@@ -68,6 +71,7 @@ impl WayVRCompositor {
         display: wayland_server::Display<comp::Application>,
         seat_keyboard: KeyboardHandle<comp::Application>,
         seat_pointer: PointerHandle<comp::Application>,
+        seat: Seat<comp::Application>,
     ) -> anyhow::Result<Self> {
         let (wayland_env, listener) = create_wayland_listener()?;
 
@@ -81,6 +85,7 @@ impl WayVRCompositor {
             serial_counter: SerialCounter::new(),
             clients: Vec::new(),
             toplevel_surf_count: 0,
+            seat,
         })
     }
 
@@ -200,6 +205,14 @@ impl WayVRCompositor {
         );
     }
 
+    pub fn set_clipboard_text(&mut self, text: String) {
+        set_data_device_selection::<Application>(
+            &self.state.display_handle,
+            &self.seat,
+            vec!["text/plain;charset=utf-8".to_string(), "text/plain".to_string()],
+            text.as_bytes().into(),
+        );
+    }
     pub fn set_keymap(&mut self, keymap: &xkb::Keymap) -> anyhow::Result<()> {
         // Smithay only accepts keymaps in a string form due to thread safety concerns
         self.seat_keyboard

wayvr/src/backend/wayvr/mod.rs

@@ -241,7 +241,13 @@ impl WvrServerState {
         };
 
         Ok(Self {
-            manager: client::WayVRCompositor::new(state, display, seat_keyboard, seat_pointer)?,
+            manager: client::WayVRCompositor::new(
+                state,
+                display,
+                seat_keyboard,
+                seat_pointer,
+                seat,
+            )?,
             processes: ProcessVec::new(),
             wm: window::WindowManager::new(),
             ticks: 0,
@@ -608,6 +614,9 @@ impl WvrServerState {
     pub fn send_key(&mut self, virtual_key: u32, down: bool) {
         self.manager.send_key(virtual_key, down);
     }
+    pub fn set_clipboard_text(&mut self, text: String) {
+        self.manager.set_clipboard_text(text);
+    }
 
     pub fn set_keymap(&mut self, keymap: &xkb::Keymap) -> anyhow::Result<()> {
         self.manager.set_keymap(keymap)

wayvr/src/overlays/keyboard/builder.rs

@@ -1,5 +1,4 @@
 use std::{collections::HashMap, rc::Rc, time::Duration};
-
 use crate::{
     app_misc,
     gui::{
@@ -11,8 +10,9 @@ use crate::{
     subsystem::hid::XkbKeymap,
     windowing::backend::OverlayEventData,
 };
-use anyhow::Context;
+use anyhow::{bail, Context};
 use glam::{FloatExt, Mat4, Vec2, vec2, vec3};
+use slotmap::Key;
 use wgui::{
     animation::{Animation, AnimationEasing},
     assets::AssetPath,
@@ -25,11 +25,10 @@ use wgui::{
     taffy::{self, prelude::length},
     widget::{EventResult, div::WidgetDiv, rectangle::WidgetRectangle},
 };
-
+use wgui::event::StyleSetRequest;
-use super::{
+use wgui::layout::LayoutTask;
-    KeyButtonData, KeyState, KeyboardState, handle_press, handle_release,
+use wgui::taffy::Display;
-    layout::{self, KeyCapType},
+use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion, init_swipe_type_manager};
-};
 
 const PIXELS_PER_UNIT: f32 = 60.;
 
@@ -41,6 +40,147 @@ fn new_doc_params(panel: &mut GuiPanel<KeyboardState>) -> ParseDocumentParams<'s
     }
 }
 
+pub(super) fn update_swipe_prediction_bar(
+    panel: &mut GuiPanel<KeyboardState>,
+    app: &mut AppState
+) -> anyhow::Result<bool> {
+    let mut elements_changed = false;
+
+    let (accent_color, anim_mult) = {
+        let theme = &app.wgui_theme;
+        (theme.accent_color, theme.animation_mult)
+    };
+
+    if let Some(recv) = panel.state.swipe_candidate_receiver.as_mut()
+    && let Ok(candidates) = recv.try_recv() {
+
+        let predictions_root = panel.parser_state
+            .get_widget_id("swipe_predictions_root")
+            .unwrap_or_default();
+
+        if predictions_root.is_null() {
+            return Ok(elements_changed)
+        }
+        let doc_params = new_doc_params(panel);
+
+        panel.layout.remove_children(predictions_root);
+
+        let Some(new_suggestions) = candidates else {
+            return Ok(elements_changed)
+        };
+
+        let mut iter = new_suggestions.iter();
+        let Some(best_prediction) = iter.next() else {
+            bail!("not enough swipe predictions");
+        };
+        if let Some(manager) = panel.state.swipe_typing_manager.as_mut() {
+            manager.select_word(best_prediction, app, panel.state.modifiers);
+        }
+        for (i, prediction) in iter.enumerate() {
+            let mut params = HashMap::new();
+            let id: Rc<str> = Rc::from(format!("Prediction-{i}"));
+            params.insert("id".into(), id.clone());
+            params.insert("text".into(), prediction.clone().into());
+
+            panel.parser_state.instantiate_template(
+                &doc_params,
+                "KeyPrediction",
+                &mut panel.layout,
+                predictions_root,
+                params
+            )?;
+
+            if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) {
+                let key_state = {
+                    let rect = panel
+                        .layout
+                        .state
+                        .widgets
+                        .get_as::<WidgetRectangle>(widget_id)
+                        .unwrap(); // want panic
+
+                    Rc::new(KeyState {
+                        // fake button state just so we get key state for anims
+                        button_state: KeyButtonData::Modifier {
+                            modifier: 0,
+                            sticky: core::cell::Cell::new(false),
+                        },
+                        color: rect.params.color,
+                        color2: rect.params.color2,
+                        base_border_color: rect.params.border_color,
+                        cur_border_color: rect.params.border_color.into(),
+                        border: rect.params.border,
+                        drawn_state: false.into(),
+                    })
+                };
+                panel.add_event_listener(
+                    widget_id,
+                    EventListenerKind::MousePress,
+                    Box::new({
+                        let k = key_state.clone();
+                        let pred = prediction.clone();
+                        move |common, data, app, state| {
+                            if let Some(manager) = state.swipe_typing_manager.as_mut() {
+                                manager.select_alternate_prediction(&pred, app, state.modifiers);
+                                on_press_anim(k.clone(), common, data)
+                            }
+                            Ok(EventResult::Pass)
+                        }
+                    })
+                );
+                panel.add_event_listener(
+                    widget_id,
+                    EventListenerKind::MouseEnter,
+                    Box::new({
+                        let k = key_state.clone();
+                        move |common, data, _app, _state| {
+                            on_enter_anim(
+                                k.clone(),
+                                common,
+                                data,
+                                accent_color,
+                                anim_mult,
+                                0.0,
+                            );
+                            Ok(EventResult::Pass)
+                        }
+                    })
+                );
+                panel.add_event_listener(
+                    widget_id,
+                    EventListenerKind::MouseLeave,
+                    Box::new({
+                        let k = key_state.clone();
+                        move |common, data, _app, _state | {
+                            on_leave_anim(
+                                k.clone(),
+                                common,
+                                data,
+                                accent_color,
+                                anim_mult,
+                                0.0,
+                            );
+                            Ok(EventResult::Pass)
+                        }
+                    })
+                );
+                panel.add_event_listener(
+                    widget_id,
+                    EventListenerKind::MouseRelease,
+                    Box::new({
+                        let k = key_state.clone();
+                        move |common, data, _app, _state| {
+                            on_release_anim(k.clone(), common, data);
+                            Ok(EventResult::Pass)
+                        }
+                    })
+                );
+            }
+        }
+        elements_changed = true;
+    }
+    Ok(elements_changed)
+}
 #[allow(clippy::too_many_lines, clippy::significant_drop_tightening)]
 pub(super) fn create_keyboard_panel(
     app: &mut AppState,
@@ -113,7 +253,7 @@ pub(super) fn create_keyboard_panel(
             params.insert(Rc::from("width"), Rc::from(key_width.to_string()));
             params.insert(Rc::from("height"), Rc::from(key_height.to_string()));
 
-            let mut label = key.label.into_iter();
+            let mut label = key.label.clone().into_iter();
             label
                 .next()
                 .and_then(|s| params.insert("text".into(), s.into()));
@@ -169,6 +309,9 @@ pub(super) fn create_keyboard_panel(
                     })
                 };
 
+                let key_cap_type: Rc<KeyCapType> = Rc::from(key.cap_type);
+                let key_label: Rc<Vec<String>> = Rc::from(key.label);
+
                 let width_mul = 1. / my_size_f32;
 
                 panel.add_event_listener(
@@ -186,6 +329,8 @@ pub(super) fn create_keyboard_panel(
                                 anim_mult,
                                 width_mul,
                             );
+
+
                             Ok(EventResult::Pass)
                         }
                     }),
@@ -205,6 +350,7 @@ pub(super) fn create_keyboard_panel(
                                 anim_mult,
                                 width_mul,
                             );
+
                             Ok(EventResult::Pass)
                         }
                     }),
@@ -214,26 +360,49 @@ pub(super) fn create_keyboard_panel(
                     EventListenerKind::MousePress,
                     Box::new({
                         let k = key_state.clone();
+                        let k_label = key_label.clone();
+                        let k_cap_type = key_cap_type.clone();
                         move |common, data, app, state| {
                             let CallbackMetadata::MouseButton(button) = data.metadata else {
                                 panic!("CallbackMetadata should contain MouseButton!");
                             };
+                            let within_key_pos = data.metadata.get_mouse_pos_normalized(&common.alterables.transform_stack);
 
-                            handle_press(app, &k, state, button);
+                            handle_press(app, &k, &k_label, &k_cap_type, &within_key_pos, state, button, button.device);
                             on_press_anim(k.clone(), common, data);
                             Ok(EventResult::Pass)
                         }
                     }),
                 );
+                panel.add_event_listener(
+                    widget_id,
+                    EventListenerKind::MouseMotion,
+                    Box::new({
+                        let k = key_state.clone();
+                        let k_label = key_label.clone();
+                        let k_cap_type = key_cap_type.clone();
+                        move |common, data, _app, state| {
+                            let within_key_pos = data.metadata.get_mouse_pos_normalized(&common.alterables.transform_stack);
+                            let CallbackMetadata::MousePosition(position) = data.metadata else {
+                                panic!("CallbackMetadata should contain MousePosition!");
+                            };
+
+                            handle_mouse_motion(&k, &k_label, &k_cap_type, state, &within_key_pos, position.device);
+                            Ok(EventResult::Pass)
+                        }
+                    }),
+                );
                 panel.add_event_listener(
                     widget_id,
                     EventListenerKind::MouseRelease,
                     Box::new({
                         let k = key_state.clone();
+                        let k_cap_type = key_cap_type.clone();
                         move |common, data, app, state| {
-                            if handle_release(app, &k, state) {
+                            if handle_release(app, &k, &k_cap_type, state) {
                                 on_release_anim(k.clone(), common, data);
                             }
+
                             Ok(EventResult::Pass)
                         }
                     }),
@@ -307,6 +476,38 @@ pub(super) fn create_keyboard_panel(
                             elems_changed = true;
                         }
                     }
+                    if !app.session.config.keyboard_swipe_to_type_enabled {
+                        panel.state.swipe_typing_manager = None;
+                        panel.state.swipe_candidate_receiver = None;
+
+                        let predictions_root = panel.parser_state
+                            .get_widget_id("swipe_predictions_root")
+                            .unwrap_or_default();
+
+                        if !predictions_root.is_null() {
+                            panel.layout.remove_children(predictions_root);
+
+                            panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
+                                predictions_root,
+                                StyleSetRequest::Display(Display::None),
+                            ));
+
+                        }
+                    }
+                    if app.session.config.keyboard_swipe_to_type_enabled && panel.state.swipe_typing_manager.is_none() {
+                        init_swipe_type_manager(&mut panel.state);
+
+                        let predictions_root = panel.parser_state
+                            .get_widget_id("swipe_predictions_root")
+                            .unwrap_or_default();
+
+                        if !predictions_root.is_null() {
+                            panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
+                                predictions_root,
+                                StyleSetRequest::Display(Display::Flex),
+                            ));
+                        }
+                    }
                 }
 
                 OverlayEventData::CustomCommand { element, command } => {

wayvr/src/overlays/keyboard/layout.rs

@@ -2,7 +2,6 @@ use std::{collections::HashMap, str::FromStr, sync::LazyLock};
 
 use regex::Regex;
 use serde::{Deserialize, Serialize};
-
 use crate::{
     config::{ConfigType, load_known_yaml},
     subsystem::hid::{
@@ -230,7 +229,7 @@ pub(super) struct KeyData {
     pub(super) cap_type: KeyCapType,
 }
 
-#[derive(Debug)]
+#[derive(Debug, Eq, PartialEq)]
 pub enum KeyCapType {
     /// Label an SVG
     Special,

wayvr/src/overlays/keyboard/mod.rs

@@ -1,10 +1,6 @@
-use std::{
+use crate::overlays::keyboard::builder::update_swipe_prediction_bar;
-    cell::Cell,
+use crate::overlays::keyboard::layout::KeyCapType;
-    collections::HashMap,
+use crate::overlays::keyboard::swipe_type::SwipeTypingManager;
-    process::{Child, Command},
-    sync::atomic::Ordering,
-};
-
 use crate::{
     KEYMAP_CHANGE,
     backend::{
@@ -27,9 +23,20 @@ use crate::{
     },
 };
 use anyhow::Context;
-use glam::{Affine3A, Quat, Vec3, vec3};
+use glam::{Affine3A, Quat, Vec2, Vec3, vec3};
 use regex::Regex;
-use slotmap::{SlotMap, new_key_type};
+use slotmap::{Key, SlotMap, new_key_type};
+use std::sync::mpsc::Receiver;
+use std::{
+    cell::Cell,
+    collections::HashMap,
+    process::{Child, Command},
+    sync::atomic::Ordering,
+};
+use wgui::event::StyleSetRequest;
+use wgui::layout::LayoutTask;
+use wgui::parser::Fetchable;
+use wgui::taffy::Display;
 use wgui::{
     drawing,
     event::{InternalStateChangeEvent, MouseButtonEvent, MouseButtonIndex},
@@ -42,6 +49,7 @@ use wlx_common::{
 
 pub mod builder;
 mod layout;
+mod swipe_type;
 
 pub const KEYBOARD_NAME: &str = "kbd";
 const AUTO_RELEASE_MODS: [KeyModifier; 5] = [SHIFT, CTRL, ALT, SUPER, META];
@@ -56,6 +64,8 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
         overlay_list: OverlayList::default(),
         set_list: SetList::default(),
         clock_12h: app.session.config.clock_12h,
+        swipe_typing_manager: None,
+        swipe_candidate_receiver: None,
     };
 
     let auto_labels = layout.auto_labels.unwrap_or(true);
@@ -111,7 +121,7 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
             transform: Affine3A::from_scale_rotation_translation(
                 Vec3::ONE * width,
                 Quat::from_rotation_x(-10f32.to_radians()),
-                vec3(0.0, -0.65, -0.5),
+                vec3(0.0, -0.69, -0.5),
             ),
             ..OverlayWindowState::default()
         },
@@ -119,6 +129,32 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
     })
 }
 
+pub(self) fn init_swipe_type_manager(state: &mut KeyboardState) {
+    match SwipeTypingManager::new() {
+        Ok((engine, receiver)) => {
+            state.swipe_typing_manager = Some(engine);
+            state.swipe_candidate_receiver = Some(receiver);
+        }
+        Err(e) => {
+            log::error!("Error occurred while trying to load swipe engine: {}", e);
+        }
+    };
+}
+
+pub(self) fn hide_swipe_type_manager(panel: &mut GuiPanel<KeyboardState>) {
+    let predictions_root = panel
+        .parser_state
+        .get_widget_id("swipe_predictions_root")
+        .unwrap_or_default();
+
+    if !predictions_root.is_null() {
+        panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
+            predictions_root,
+            StyleSetRequest::Display(Display::None),
+        ));
+    }
+}
+
 const fn alt_modifier_to_key(m: AltModifier) -> KeyModifier {
     match m {
         AltModifier::Shift => SHIFT,
@@ -150,8 +186,18 @@ impl KeyboardBackend {
         keymap: Option<&XkbKeymap>,
         app: &mut AppState,
     ) -> anyhow::Result<KeyboardPanelKey> {
-        let panel =
+        let mut state = self.default_state.take();
-            create_keyboard_panel(app, keymap, self.default_state.take(), &self.wlx_layout)?;
+
+        if app.session.config.keyboard_swipe_to_type_enabled {
+            init_swipe_type_manager(&mut state);
+        }
+
+        log::info!("swipe engine created");
+        let mut panel = create_keyboard_panel(app, keymap, state, &self.wlx_layout)?;
+
+        if !app.session.config.keyboard_swipe_to_type_enabled {
+            hide_swipe_type_manager(&mut panel);
+        }
 
         let id = self.layout_panels.insert(panel);
         if let Some(layout_name) = keymap.and_then(|k| k.get_name()) {
@@ -176,30 +222,38 @@ impl KeyboardBackend {
             if self.active_layout.eq(new_key) {
                 return Ok(false);
             }
-            self.internal_switch_keymap(*new_key);
+            self.internal_switch_keymap(*new_key, app);
         } else {
             let new_key = self.add_new_keymap(Some(keymap), app)?;
-            self.internal_switch_keymap(new_key);
+            self.internal_switch_keymap(new_key, app);
         }
         app.tasks
             .enqueue(TaskType::Overlay(OverlayTask::KeyboardChanged));
         Ok(true)
     }
 
-    fn internal_switch_keymap(&mut self, new_key: KeyboardPanelKey) {
+    fn internal_switch_keymap(&mut self, new_key: KeyboardPanelKey, app: &AppState) {
-        let state_from = self
+        let mut state_from = self
             .layout_panels
             .get_mut(self.active_layout)
             .unwrap()
             .state
             .take();
 
+        if app.session.config.keyboard_swipe_to_type_enabled {
+            init_swipe_type_manager(&mut state_from);
+        }
+
         self.active_layout = new_key;
 
         self.layout_panels
             .get_mut(self.active_layout)
             .unwrap()
             .state = state_from;
+
+        if !app.session.config.keyboard_swipe_to_type_enabled {
+            hide_swipe_type_manager(self.layout_panels.get_mut(self.active_layout).unwrap())
+        }
     }
 
     fn get_effective_keymap(&mut self) -> anyhow::Result<XkbKeymap> {
@@ -233,6 +287,13 @@ impl KeyboardBackend {
         }
     }
 
+    fn update_swipe_prediction_bar(&mut self, app: &mut AppState) -> anyhow::Result<()> {
+        if update_swipe_prediction_bar(self.panel(), app)? {
+            self.panel().process_custom_elems(app);
+        }
+        Ok(())
+    }
+
     fn auto_switch_keymap(&mut self, app: &mut AppState) -> anyhow::Result<bool> {
         let keymap = self.get_effective_keymap()?;
         app.hid_provider
@@ -266,6 +327,7 @@ impl OverlayBackend for KeyboardBackend {
                 });
             }
         }
+        self.update_swipe_prediction_bar(app)?;
         self.panel().should_render(app)
     }
     fn render(&mut self, app: &mut AppState, rdr: &mut RenderResources) -> anyhow::Result<()> {
@@ -327,6 +389,8 @@ struct KeyboardState {
     overlay_list: OverlayList,
     set_list: SetList,
     clock_12h: bool,
+    swipe_typing_manager: Option<SwipeTypingManager>,
+    swipe_candidate_receiver: Option<Receiver<Option<Vec<String>>>>,
 }
 
 macro_rules! take_and_leave_default {
@@ -346,6 +410,8 @@ impl KeyboardState {
             overlay_list: OverlayList::default(),
             set_list: SetList::default(),
             clock_12h: self.clock_12h,
+            swipe_typing_manager: None,
+            swipe_candidate_receiver: None,
         }
     }
 }
@@ -386,26 +452,75 @@ enum KeyButtonData {
     },
 }
 
+fn handle_mouse_motion(
+    key: &KeyState,
+    key_label: &Vec<String>,
+    key_cap_type: &KeyCapType,
+    keyboard: &mut KeyboardState,
+    within_key_pos: &Option<Vec2>,
+    device: usize,
+) {
+    if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut()
+        && *key_cap_type == KeyCapType::Letter
+    {
+        if !swipe_manager.is_current_swipe_empty() {
+            match &key.button_state {
+                KeyButtonData::Key { vk, pressed } => {
+                    if let Some(pos) = within_key_pos {
+                        // check because mouse motion can trigger despite hover being false
+                        if pos.x >= 0.0 && pos.x <= 1.0 && pos.y >= 0.0 && pos.y <= 1.0 {
+                            if let Some(label) = key_label.first() {
+                                swipe_manager.add_swipe(
+                                    pos,
+                                    label.chars().next().unwrap_or_default(),
+                                    device,
+                                );
+                            }
+                        }
+                    }
+                }
+                _ => {}
+            }
+        }
+    }
+}
 fn handle_press(
     app: &mut AppState,
     key: &KeyState,
+    key_label: &Vec<String>,
+    key_cap_type: &KeyCapType,
+    within_key_pos: &Option<Vec2>,
     keyboard: &mut KeyboardState,
     button: MouseButtonEvent,
+    device: usize,
 ) {
     match &key.button_state {
         KeyButtonData::Key { vk, pressed } => {
-            keyboard.modifiers |= match button.index {
+            if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut()
-                MouseButtonIndex::Right => SHIFT,
+                && *key_cap_type == KeyCapType::Letter
-                MouseButtonIndex::Middle => keyboard.alt_modifier,
+            {
-                _ => 0,
+                if let Some(pos) = within_key_pos {
-            };
+                    if let Some(label) = key_label.first() {
-
+                        swipe_manager.add_swipe(
-            app.hid_provider
+                            pos,
-                .set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
+                            label.chars().next().unwrap_or_default(),
-            app.hid_provider
+                            device,
-                .send_key_routed(app.wvr_server.as_mut(), *vk, true);
+                        );
-            pressed.set(true);
+                    }
-            play_key_click(app);
+                }
+            } else {
+                keyboard.modifiers |= match button.index {
+                    MouseButtonIndex::Right => SHIFT,
+                    MouseButtonIndex::Middle => keyboard.alt_modifier,
+                    _ => 0,
+                };
+                app.hid_provider
+                    .set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
+                app.hid_provider
+                    .send_key_routed(app.wvr_server.as_mut(), *vk, true);
+                pressed.set(true);
+                play_key_click(app);
+            }
         }
         KeyButtonData::Modifier { modifier, sticky } => {
             sticky.set(keyboard.modifiers & *modifier == 0);
@@ -435,20 +550,51 @@ fn handle_press(
     }
 }
 
-fn handle_release(app: &mut AppState, key: &KeyState, keyboard: &mut KeyboardState) -> bool {
+fn handle_release(
+    app: &mut AppState,
+    key: &KeyState,
+    k_cap_type: &KeyCapType,
+    keyboard: &mut KeyboardState,
+) -> bool {
     match &key.button_state {
         KeyButtonData::Key { vk, pressed } => {
-            pressed.set(false);
+            if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut()
+                && *k_cap_type == KeyCapType::Letter
+            {
+                if swipe_manager.did_swipe_leave_first_key() {
+                    match swipe_manager.predict() {
+                        Ok(()) => {}
+                        Err(e) => {
+                            log::error!("{}", e)
+                        }
+                    }
+                } else {
+                    // pointer must have been released on the same key it was pressed on
+                    swipe_manager.reset(); // drop swipe tracking that was started on press
 
-            for m in &AUTO_RELEASE_MODS {
+                    app.hid_provider
-                if keyboard.modifiers & *m != 0 {
+                        .send_key_routed(app.wvr_server.as_mut(), *vk, true);
-                    keyboard.modifiers &= !*m;
+                    pressed.set(true);
+                    app.hid_provider
+                        .send_key_routed(app.wvr_server.as_mut(), *vk, false);
+                    play_key_click(app);
                 }
+            } else {
+                if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() {
+                    swipe_manager.reset();
+                }
+                pressed.set(false);
+
+                for m in &AUTO_RELEASE_MODS {
+                    if keyboard.modifiers & *m != 0 {
+                        keyboard.modifiers &= !*m;
+                    }
+                }
+                app.hid_provider
+                    .send_key_routed(app.wvr_server.as_mut(), *vk, false);
+                app.hid_provider
+                    .set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
             }
-            app.hid_provider
-                .send_key_routed(app.wvr_server.as_mut(), *vk, false);
-            app.hid_provider
-                .set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
             true
         }
         KeyButtonData::Modifier { modifier, sticky } => {

wayvr/src/overlays/keyboard/swipe_type.rs

@@ -0,0 +1,238 @@
+use crate::state::AppState;
+use crate::subsystem::hid::{KeyModifier, VirtualKey, CTRL};
+use anyhow::{bail};
+use arboard::Clipboard;
+use glam::Vec2;
+use std::mem;
+use std::sync::mpsc::{sync_channel, Receiver, SyncSender, channel, Sender};
+use std::thread::{self, JoinHandle};
+use std::time::Instant;
+use super_swipe_type::keyboard_manager::QwertyKeyboardGrid;
+use super_swipe_type::swipe_orchestrator::SwipeOrchestrator;
+use super_swipe_type::{SwipePoint};
+use crate::subsystem::input::KeyboardFocus;
+
+const PREDICTION_SUGGESTION_COUNT: usize = 5;
+
+enum PredictionTask {
+    Predict {
+        swipe: Vec<SwipePoint>,
+        last_word: Option<String>,
+    },
+    Shutdown,
+}
+
+pub struct SwipeTypingManager {
+    keyboard_gird: QwertyKeyboardGrid,
+    current_swipe: Vec<SwipePoint>,
+    swipe_candidate_sender: SyncSender<Option<Vec<String>>>,
+    prediction_task_sender: Sender<PredictionTask>,
+    worker_thread: Option<JoinHandle<()>>,
+    swipe_start_time: Option<Instant>,
+    clipboard: Clipboard,
+    swipe_left_first_key: bool,
+    first_swipe_char: char,
+    current_swipe_device: Option<usize>,
+    last_swiped_word: Option<String>,
+}
+
+impl SwipeTypingManager {
+    pub fn select_alternate_prediction(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
+        Self::undo(app, original_keyboard_mods);
+        self.select_word(word, app, original_keyboard_mods);
+    }
+
+    pub fn select_word(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
+        self.last_swiped_word = Some(word.clone());
+        let text_to_paste = format!("{word} ");
+
+        match app.hid_provider.keyboard_focus {
+            KeyboardFocus::PhysicalScreen => {
+                if let Ok(_) = self.clipboard.set_text(text_to_paste) {
+                    Self::paste(app, original_keyboard_mods);
+                }
+            },
+            KeyboardFocus::WayVR => {
+                if let Some(wvr_server) = app.wvr_server.as_mut() {
+                    wvr_server.set_clipboard_text(text_to_paste);
+                    Self::paste(app, original_keyboard_mods);
+                }
+            },
+        }
+
+    }
+
+    fn undo(app: &mut AppState, original_keyboard_mods: KeyModifier) {
+        app.hid_provider
+            .set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
+        app.hid_provider
+            .send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, true);
+        app.hid_provider
+            .send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, false);
+        app.hid_provider
+            .set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
+    }
+
+    fn paste(app: &mut AppState, original_keyboard_mods: KeyModifier) {
+        app.hid_provider
+            .set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
+        app.hid_provider
+            .send_key_routed(app.wvr_server.as_mut(), VirtualKey::V, true);
+        app.hid_provider
+            .send_key_routed(app.wvr_server.as_mut(), VirtualKey::V, false);
+        app.hid_provider
+            .set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
+    }
+    pub fn new() -> anyhow::Result<(SwipeTypingManager, Receiver<Option<Vec<String>>>)> {
+        let (candidate_sender, candidate_receiver) = sync_channel(1);
+        let (task_sender, task_receiver) = channel::<PredictionTask>();
+
+        // Spawn persistent worker thread
+        let worker_candidate_sender = candidate_sender.clone();
+        let worker_thread = thread::spawn(move || {
+            let mut swipe_engine = match SwipeOrchestrator::new() {
+                Ok(engine) => engine,
+                Err(e) => {
+                    log::error!("Failed to initialize SwipeOrchestrator: {}", e);
+                    return;
+                }
+            };
+
+            while let Ok(task) = task_receiver.recv() {
+                match task {
+                    PredictionTask::Predict { swipe, last_word } => {
+                        match swipe_engine.predict(swipe, &last_word) {
+                            Ok(candidates) => {
+                                let words: Vec<String> = candidates
+                                    .into_iter()
+                                    .take(PREDICTION_SUGGESTION_COUNT)
+                                    .map(|c| c.word)
+                                    .collect();
+
+                                let _ = worker_candidate_sender.send(Some(words));
+                            }
+                            Err(e) => {
+                                log::error!("Prediction failed: {}", e);
+                            }
+                        }
+                    }
+                    PredictionTask::Shutdown => break,
+                }
+            }
+        });
+
+        Ok((
+            Self {
+                keyboard_gird: QwertyKeyboardGrid::new(),
+                current_swipe: Vec::new(),
+                swipe_candidate_sender: candidate_sender,
+                prediction_task_sender: task_sender,
+                worker_thread: Some(worker_thread),
+                swipe_start_time: None,
+                clipboard: Clipboard::new()?,
+                swipe_left_first_key: false,
+                first_swipe_char: char::default(),
+                current_swipe_device: None,
+                last_swiped_word: None,
+            },
+            candidate_receiver,
+        ))
+    }
+
+    pub fn predict(&mut self) -> anyhow::Result<()> {
+        if self.is_current_swipe_empty() {
+            bail!("nothing to predict");
+        }
+
+        let current_swipe = mem::take(&mut self.current_swipe);
+        let last_word = self.last_swiped_word.clone();
+        self.reset_swipe();
+
+        self.prediction_task_sender
+            .send(PredictionTask::Predict {
+                swipe: current_swipe,
+                last_word,
+            })?;
+
+        Ok(())
+    }
+
+    pub fn reset(&mut self) {
+        self.reset_swipe();
+        let _ = self.swipe_candidate_sender.send(None);
+        self.last_swiped_word = None;
+    }
+
+    fn reset_swipe(&mut self) {
+        self.swipe_start_time = None;
+        self.current_swipe = Vec::new();
+        self.first_swipe_char = char::default();
+        self.swipe_left_first_key = false;
+        self.current_swipe_device = None;
+    }
+
+    fn start_swipe(&mut self, key_label: char, device: usize) -> Instant {
+        let now = Instant::now();
+        self.swipe_start_time = Some(now);
+        self.first_swipe_char = key_label.to_ascii_lowercase();
+        self.current_swipe_device = Some(device);
+        now
+    }
+
+    pub fn did_swipe_leave_first_key(&self) -> bool {
+        self.swipe_left_first_key
+    }
+
+    pub fn is_current_swipe_empty(&self) -> bool {
+        self.current_swipe.is_empty()
+    }
+
+    pub fn add_swipe(&mut self, within_key_pos_normalized: &Vec2, key_label: char, device: usize) {
+        if let Some(pos) = self.keyboard_gird.key_positions.get(&key_label.to_ascii_lowercase()) {
+            if let Some(current_device) = self.current_swipe_device {
+                if current_device != device {
+                    return;
+                }
+            }
+
+            if self.first_swipe_char != char::default()
+                && self.first_swipe_char != key_label.to_ascii_lowercase()
+            {
+                self.swipe_left_first_key = true;
+            }
+
+            let key_pos = Vec2 {
+                x: pos.x as f32,
+                y: pos.y as f32,
+            };
+
+            let start_time = match self.swipe_start_time {
+                Some(time) => time,
+                None => self.start_swipe(key_label, device),
+            };
+
+            let within_key_pos_from_center = Vec2 {
+                x: within_key_pos_normalized.x - 0.5,
+                y: within_key_pos_normalized.y - 0.5,
+            };
+            let key_dimensions = Vec2 {
+                x: QwertyKeyboardGrid::get_key_width() as f32,
+                y: QwertyKeyboardGrid::get_key_height() as f32,
+            };
+
+            let point = within_key_pos_from_center * key_dimensions + key_pos;
+            let duration = Instant::now().duration_since(start_time).mul_f32(0.8); // multiply by .8 because library is trained on mobile swipes which happen on a smaller keyboard and are faster
+            self.current_swipe
+                .push(SwipePoint::new(point.x.into(), point.y.into(), duration))
+        }
+    }
+}
+
+impl Drop for SwipeTypingManager {
+    fn drop(&mut self) {
+        let _ = self.prediction_task_sender.send(PredictionTask::Shutdown);
+        if let Some(handle) = self.worker_thread.take() {
+            let _ = handle.join();
+        }
+    }
+}

wayvr/src/subsystem/hid/mod.rs

@@ -253,6 +253,7 @@ impl HidProvider for UInputProvider {
             log::error!("send_key: {res}");
         }
     }
+
     fn set_desktop_extent(&mut self, extent: Vec2) {
         self.desktop_extent = extent;
     }
@@ -337,6 +338,7 @@ pub const META: KeyModifier = 0x80;
 #[allow(non_camel_case_types)]
 #[repr(u16)]
 #[derive(Debug, Deserialize, PartialEq, Eq, Clone, Copy, IntegerId, EnumString, EnumIter)]
+#[derive(Hash)]
 pub enum VirtualKey {
     Escape = 9,
     N1, // number row

wgui/src/event.rs

@@ -215,6 +215,10 @@ impl CallbackMetadata {
 		let mouse_pos_abs = self.get_mouse_pos_absolute()?;
 		Some(mouse_pos_abs - transform_stack.get().abs_pos)
 	}
+	pub fn get_mouse_pos_normalized(&self, transform_stack: &TransformStack) -> Option<Vec2> {
+		let pos_relative = self.get_mouse_pos_relative(transform_stack)?;
+		Some(pos_relative/transform_stack.parent().raw_dim)
+	}
 }
 
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]

wlx-common/src/config.rs

@@ -327,4 +327,7 @@ pub struct GeneralConfig {
 
 	#[serde(default = "def_one")]
 	pub grid_opacity: f32,
+
+	#[serde(default = "def_false")]
+	pub keyboard_swipe_to_type_enabled: bool,
 }