From 0ee76718324f79c9ebe17905a53619c8cd31af14 Mon Sep 17 00:00:00 2001 From: Diamond White Date: Sun, 15 Mar 2026 12:32:16 +0000 Subject: [PATCH] process swipes on separate thread to not block ui --- wayvr/src/overlays/keyboard/builder.rs | 257 +++++++++++----------- wayvr/src/overlays/keyboard/mod.rs | 45 ++-- wayvr/src/overlays/keyboard/swipe_type.rs | 178 ++++++++++----- 3 files changed, 276 insertions(+), 204 deletions(-) diff --git a/wayvr/src/overlays/keyboard/builder.rs b/wayvr/src/overlays/keyboard/builder.rs index a3749356..87e41a4f 100644 --- a/wayvr/src/overlays/keyboard/builder.rs +++ b/wayvr/src/overlays/keyboard/builder.rs @@ -10,7 +10,7 @@ use crate::{ subsystem::hid::XkbKeymap, windowing::backend::OverlayEventData, }; -use anyhow::Context; +use anyhow::{bail, Context}; use glam::{FloatExt, Mat4, Vec2, vec2, vec3}; use slotmap::Key; use wgui::{ @@ -29,7 +29,7 @@ use wgui::event::StyleSetRequest; use wgui::layout::LayoutTask; use wgui::taffy::Display; use crate::overlays::keyboard::swipe_type::SwipeTypingManager; -use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion}; +use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion, init_swipe_type_manager}; const PIXELS_PER_UNIT: f32 = 60.; @@ -52,128 +52,134 @@ pub(super) fn update_swipe_prediction_bar( (theme.accent_color, theme.animation_mult) }; - let new_swipe_candidates; - if let Some(manager) = panel.state.swipe_typing_manager.as_mut() - && manager.swipe_candidates != panel.state.swipe_bar_candidates { - panel.state.swipe_bar_candidates = manager.swipe_candidates.clone(); - new_swipe_candidates = manager.swipe_candidates.clone(); - } else { - return Ok(elements_changed) - } + if let Some(recv) = panel.state.swipe_candidate_receiver.as_mut() + && let Ok(candidates) = recv.try_recv() { - let predictions_root = panel.parser_state - .get_widget_id("swipe_predictions_root") - .unwrap_or_default(); + let predictions_root = panel.parser_state + .get_widget_id("swipe_predictions_root") + .unwrap_or_default(); - if predictions_root.is_null() { - return Ok(elements_changed) - } - let doc_params = new_doc_params(panel); - - panel.layout.remove_children(predictions_root); - - for (i, prediction) in new_swipe_candidates.iter().enumerate() { - let mut params = HashMap::new(); - let id: Rc = Rc::from(format!("Prediction-{i}")); - params.insert("id".into(), id.clone()); - params.insert("text".into(), prediction.clone().into()); - - panel.parser_state.instantiate_template( - &doc_params, - "KeyPrediction", - &mut panel.layout, - predictions_root, - params - )?; - - if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) { - let key_state = { - let rect = panel - .layout - .state - .widgets - .get_as::(widget_id) - .unwrap(); // want panic - - Rc::new(KeyState { - // fake button state just so we get key state for anims - button_state: KeyButtonData::Modifier { - modifier: 0, - sticky: core::cell::Cell::new(false), - }, - color: rect.params.color, - color2: rect.params.color2, - base_border_color: rect.params.border_color, - cur_border_color: rect.params.border_color.into(), - border: rect.params.border, - drawn_state: false.into(), - }) - }; - panel.add_event_listener( - widget_id, - EventListenerKind::MousePress, - Box::new({ - let k = key_state.clone(); - let pred = prediction.clone(); - move |common, data, app, state| { - if let Some(manager) = state.swipe_typing_manager.as_mut() { - manager.select_alternate_prediction(&pred, app, state.modifiers); - on_press_anim(k.clone(), common, data) - } - Ok(EventResult::Pass) - } - }) - ); - panel.add_event_listener( - widget_id, - EventListenerKind::MouseEnter, - Box::new({ - let k = key_state.clone(); - move |common, data, _app, _state| { - on_enter_anim( - k.clone(), - common, - data, - accent_color, - anim_mult, - 0.0, - ); - Ok(EventResult::Pass) - } - }) - ); - panel.add_event_listener( - widget_id, - EventListenerKind::MouseLeave, - Box::new({ - let k = key_state.clone(); - move |common, data, _app, _state | { - on_leave_anim( - k.clone(), - common, - data, - accent_color, - anim_mult, - 0.0, - ); - Ok(EventResult::Pass) - } - }) - ); - panel.add_event_listener( - widget_id, - EventListenerKind::MouseRelease, - Box::new({ - let k = key_state.clone(); - move |common, data, _app, _state| { - on_release_anim(k.clone(), common, data); - Ok(EventResult::Pass) - } - }) - ); + if predictions_root.is_null() { + return Ok(elements_changed) } + let doc_params = new_doc_params(panel); + + panel.layout.remove_children(predictions_root); + + let Some(new_suggestions) = candidates else { + return Ok(elements_changed) + }; + + let mut iter = new_suggestions.iter(); + let Some(best_prediction) = iter.next() else { + bail!("not enough swipe predictions"); + }; + if let Some(manager) = panel.state.swipe_typing_manager.as_mut() { + manager.select_word(best_prediction, app, panel.state.modifiers); + } + for (i, prediction) in iter.enumerate() { + let mut params = HashMap::new(); + let id: Rc = Rc::from(format!("Prediction-{i}")); + params.insert("id".into(), id.clone()); + params.insert("text".into(), prediction.clone().into()); + + panel.parser_state.instantiate_template( + &doc_params, + "KeyPrediction", + &mut panel.layout, + predictions_root, + params + )?; + + if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) { + let key_state = { + let rect = panel + .layout + .state + .widgets + .get_as::(widget_id) + .unwrap(); // want panic + + Rc::new(KeyState { + // fake button state just so we get key state for anims + button_state: KeyButtonData::Modifier { + modifier: 0, + sticky: core::cell::Cell::new(false), + }, + color: rect.params.color, + color2: rect.params.color2, + base_border_color: rect.params.border_color, + cur_border_color: rect.params.border_color.into(), + border: rect.params.border, + drawn_state: false.into(), + }) + }; + panel.add_event_listener( + widget_id, + EventListenerKind::MousePress, + Box::new({ + let k = key_state.clone(); + let pred = prediction.clone(); + move |common, data, app, state| { + if let Some(manager) = state.swipe_typing_manager.as_mut() { + manager.select_alternate_prediction(&pred, app, state.modifiers); + on_press_anim(k.clone(), common, data) + } + Ok(EventResult::Pass) + } + }) + ); + panel.add_event_listener( + widget_id, + EventListenerKind::MouseEnter, + Box::new({ + let k = key_state.clone(); + move |common, data, _app, _state| { + on_enter_anim( + k.clone(), + common, + data, + accent_color, + anim_mult, + 0.0, + ); + Ok(EventResult::Pass) + } + }) + ); + panel.add_event_listener( + widget_id, + EventListenerKind::MouseLeave, + Box::new({ + let k = key_state.clone(); + move |common, data, _app, _state | { + on_leave_anim( + k.clone(), + common, + data, + accent_color, + anim_mult, + 0.0, + ); + Ok(EventResult::Pass) + } + }) + ); + panel.add_event_listener( + widget_id, + EventListenerKind::MouseRelease, + Box::new({ + let k = key_state.clone(); + move |common, data, _app, _state| { + on_release_anim(k.clone(), common, data); + Ok(EventResult::Pass) + } + }) + ); + } + } + elements_changed = true; } - elements_changed = true; Ok(elements_changed) } #[allow(clippy::too_many_lines, clippy::significant_drop_tightening)] @@ -473,6 +479,8 @@ pub(super) fn create_keyboard_panel( } if !app.session.config.swipe_to_type_enabled { panel.state.swipe_typing_manager = None; + panel.state.swipe_candidate_receiver = None; + let predictions_root = panel.parser_state .get_widget_id("swipe_predictions_root") .unwrap_or_default(); @@ -488,13 +496,8 @@ pub(super) fn create_keyboard_panel( } } if app.session.config.swipe_to_type_enabled && panel.state.swipe_typing_manager.is_none() { - panel.state.swipe_typing_manager = match SwipeTypingManager::new() { - Ok(engine) => Some(engine), - Err(e) => { - log::error!("Error occurred while trying to load swipe engine: {:?}", e); - None - } - }; + init_swipe_type_manager(&mut panel.state); + let predictions_root = panel.parser_state .get_widget_id("swipe_predictions_root") .unwrap_or_default(); diff --git a/wayvr/src/overlays/keyboard/mod.rs b/wayvr/src/overlays/keyboard/mod.rs index 7475acf8..e98d4333 100644 --- a/wayvr/src/overlays/keyboard/mod.rs +++ b/wayvr/src/overlays/keyboard/mod.rs @@ -4,7 +4,7 @@ use std::{ process::{Child, Command}, sync::atomic::Ordering, }; - +use std::sync::mpsc::Receiver; use crate::{ KEYMAP_CHANGE, backend::{ @@ -61,7 +61,7 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result anyhow::Result { + state.swipe_typing_manager = Some(engine); + state.swipe_candidate_receiver = Some(receiver); + }, + Err(e) => { + log::error!("Error occured while trying to load swipe engine: {:?}", e); + } + }; +} fn alt_modifier_to_key(m: AltModifier) -> KeyModifier { match m { AltModifier::Shift => SHIFT, @@ -158,13 +168,7 @@ impl KeyboardBackend { ) -> anyhow::Result { let mut state = self.default_state.take(); - state.swipe_typing_manager = match SwipeTypingManager::new() { - Ok(engine) => Some(engine), - Err(e) => { - log::error!("Error occured while trying to load swipe engine: {:?}", e); - None - } - }; + init_swipe_type_manager(&mut state); log::info!("swipe engine created"); let panel = @@ -179,6 +183,7 @@ impl KeyboardBackend { Ok(id) } + fn switch_keymap(&mut self, keymap: &XkbKeymap, app: &mut AppState) -> anyhow::Result { if !self.wlx_layout.auto_labels.unwrap_or(true) { return Ok(false); @@ -211,13 +216,8 @@ impl KeyboardBackend { .state .take(); - state_from.swipe_typing_manager = match SwipeTypingManager::new() { - Ok(engine) => Some(engine), - Err(e) => { - log::error!("Error occured while trying to load swipe engine: {:?}", e); - None - } - }; + init_swipe_type_manager(&mut state_from); + self.active_layout = new_key; self.layout_panels @@ -360,7 +360,7 @@ struct KeyboardState { set_list: SetList, clock_12h: bool, swipe_typing_manager: Option, - swipe_bar_candidates: Vec, + swipe_candidate_receiver: Option>>> } macro_rules! take_and_leave_default { @@ -381,7 +381,7 @@ impl KeyboardState { set_list: SetList::default(), clock_12h: self.clock_12h, swipe_typing_manager: None, - swipe_bar_candidates: Vec::new(), + swipe_candidate_receiver: None, } } } @@ -515,14 +515,9 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k match &key.button_state { KeyButtonData::Key { vk, pressed } => { if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() && *k_cap_type == KeyCapType::Letter { - println!("some"); if swipe_manager.did_swipe_leave_first_key() { - println!("left first"); match swipe_manager.predict() { - Ok(top_prediction) => { - println!("best prediction for swipe: {top_prediction}"); - swipe_manager.select_word(&top_prediction, app, keyboard.modifiers); - }, + Ok(()) => {}, Err(e) => { log::error!("{}", e) } diff --git a/wayvr/src/overlays/keyboard/swipe_type.rs b/wayvr/src/overlays/keyboard/swipe_type.rs index 1929dc8c..7132ff2c 100644 --- a/wayvr/src/overlays/keyboard/swipe_type.rs +++ b/wayvr/src/overlays/keyboard/swipe_type.rs @@ -4,37 +4,50 @@ use anyhow::{anyhow, bail}; use arboard::Clipboard; use glam::Vec2; use std::mem; +use std::sync::mpsc::{sync_channel, Receiver, SyncSender, channel, Sender}; +use std::thread::{self, JoinHandle}; use std::time::Instant; use super_swipe_type::keyboard_manager::QwertyKeyboardGrid; use super_swipe_type::swipe_orchestrator::SwipeOrchestrator; use super_swipe_type::{SwipeCandidate, SwipePoint}; -const PREDICTION_SUGGESTION_COUNT: usize = 4; +const PREDICTION_SUGGESTION_COUNT: usize = 5; + +enum PredictionTask { + Predict { + swipe: Vec, + last_word: Option, + }, + Shutdown, +} + pub struct SwipeTypingManager { - swipe_engine: SwipeOrchestrator, keyboard_gird: QwertyKeyboardGrid, current_swipe: Vec, - pub swipe_candidates: Vec, + swipe_candidate_sender: SyncSender>>, + prediction_task_sender: Sender, + worker_thread: Option>, swipe_start_time: Option, clipboard: Clipboard, swipe_left_first_key: bool, first_swipe_char: char, - current_swipe_device: Option, // the pointer that started this swipe - last_swiped_word: Option + current_swipe_device: Option, + last_swiped_word: Option, } -impl SwipeTypingManager { +impl SwipeTypingManager { pub fn select_alternate_prediction(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) { Self::undo_paste(app, original_keyboard_mods); self.select_word(word, app, original_keyboard_mods); } - /// Attempts to "type" the word by copying to clipboard and pasting + pub fn select_word(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) { self.last_swiped_word = Some(word.clone()); - if let Ok(_) =self.copy_text_to_primary_clipboard(word.as_ref()) { + if let Ok(_) = self.copy_text_to_primary_clipboard(word.as_ref()) { Self::paste(app, original_keyboard_mods); } } + fn undo_paste(app: &mut AppState, original_keyboard_mods: KeyModifier) { app.hid_provider .set_modifiers_routed(app.wvr_server.as_mut(), CTRL); @@ -45,6 +58,7 @@ impl SwipeTypingManager { app.hid_provider .set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods); } + fn paste(app: &mut AppState, original_keyboard_mods: KeyModifier) { app.hid_provider .set_modifiers_routed(app.wvr_server.as_mut(), CTRL); @@ -55,46 +69,91 @@ impl SwipeTypingManager { app.hid_provider .set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods); } + fn copy_text_to_primary_clipboard(&mut self, text: &str) -> Result<(), arboard::Error> { self.clipboard.set_text(format!("{text} ")) } - pub fn new() -> anyhow::Result { - // todo: use the layout_name to choose a sensible language for the swipe engine - Ok(Self { - swipe_engine: SwipeOrchestrator::new()?, - keyboard_gird: QwertyKeyboardGrid::new(), // contains a hashmap where the vector2 is the center pos of each key in querty - current_swipe: Vec::new(), - swipe_candidates: Vec::new(), - swipe_start_time: None, - clipboard: Clipboard::new()?, - swipe_left_first_key: false, - first_swipe_char: char::default(), - current_swipe_device: None, - last_swiped_word: None - }) + pub fn new() -> anyhow::Result<(SwipeTypingManager, Receiver>>)> { + let (candidate_sender, candidate_receiver) = sync_channel(1); + let (task_sender, task_receiver) = channel::(); + + // Spawn persistent worker thread + let worker_candidate_sender = candidate_sender.clone(); + let worker_thread = thread::spawn(move || { + let mut swipe_engine = match SwipeOrchestrator::new() { + Ok(engine) => engine, + Err(e) => { + log::error!("Failed to initialize SwipeOrchestrator: {}", e); + return; + } + }; + + while let Ok(task) = task_receiver.recv() { + match task { + PredictionTask::Predict { swipe, last_word } => { + match swipe_engine.predict(swipe, &last_word) { + Ok(candidates) => { + let words: Vec = candidates + .iter() + .take(PREDICTION_SUGGESTION_COUNT) + .map(|c| c.word.clone()) + .collect(); + + let _ = worker_candidate_sender.send(Some(words)); + } + Err(e) => { + log::error!("Prediction failed: {}", e); + } + } + } + PredictionTask::Shutdown => break, + } + } + }); + + Ok(( + Self { + keyboard_gird: QwertyKeyboardGrid::new(), + current_swipe: Vec::new(), + swipe_candidate_sender: candidate_sender, + prediction_task_sender: task_sender, + worker_thread: Some(worker_thread), + swipe_start_time: None, + clipboard: Clipboard::new()?, + swipe_left_first_key: false, + first_swipe_char: char::default(), + current_swipe_device: None, + last_swiped_word: None, + }, + candidate_receiver, + )) } - pub fn predict(&mut self) -> anyhow::Result{ + + pub fn predict(&mut self) -> anyhow::Result<()> { if self.is_current_swipe_empty() { bail!("nothing to predict"); - } else { - let current_swipe = mem::take(&mut self.current_swipe); - self.reset_swipe(); - - let candidates: Vec = self.swipe_engine.predict(current_swipe, &self.last_swiped_word)?; - - let mut iter = candidates.iter(); - let top_choice = iter.next().ok_or(anyhow!("No swipe candidates generated"))?.word.clone(); - self.swipe_candidates = iter.take(PREDICTION_SUGGESTION_COUNT).map(|c| c.word.clone()).collect(); - - Ok(top_choice) } + + let current_swipe = mem::take(&mut self.current_swipe); + let last_word = self.last_swiped_word.clone(); + self.reset_swipe(); + + self.prediction_task_sender + .send(PredictionTask::Predict { + swipe: current_swipe, + last_word, + })?; + + Ok(()) } + pub fn reset(&mut self) { self.reset_swipe(); + let _ = self.swipe_candidate_sender.send(None); self.last_swiped_word = None; - self.swipe_candidates = Vec::new(); } + fn reset_swipe(&mut self) { self.swipe_start_time = None; self.current_swipe = Vec::new(); @@ -102,36 +161,45 @@ impl SwipeTypingManager { self.swipe_left_first_key = false; self.current_swipe_device = None; } - fn start_swipe(&mut self, key_label: char, device: usize) -> Instant{ + + fn start_swipe(&mut self, key_label: char, device: usize) -> Instant { let now = Instant::now(); self.swipe_start_time = Some(now); self.first_swipe_char = key_label.to_ascii_lowercase(); self.current_swipe_device = Some(device); - self.swipe_candidates = Vec::new(); now } + pub fn did_swipe_leave_first_key(&self) -> bool { self.swipe_left_first_key } + pub fn is_current_swipe_empty(&self) -> bool { self.current_swipe.is_empty() } + pub fn add_swipe(&mut self, within_key_pos_normalized: &Vec2, key_label: char, device: usize) { if let Some(pos) = self.keyboard_gird.key_positions.get(&key_label.to_ascii_lowercase()) { - if self.first_swipe_char != char::default() && self.first_swipe_char != key_label.to_ascii_lowercase() { + if self.first_swipe_char != char::default() + && self.first_swipe_char != key_label.to_ascii_lowercase() + { self.swipe_left_first_key = true; } - let key_pos = Vec2{x: pos.x as f32, y: pos.y as f32 }; - //println!("char : {key_label} at pos: {key_pos}"); - let start_time = match self.swipe_start_time { - Some(time) => time, - None => self.start_swipe(key_label, device) + let key_pos = Vec2 { + x: pos.x as f32, + y: pos.y as f32, }; - // only allow the pointer that started the swipe to contribute to it - if let Some(current_device) = self.current_swipe_device && current_device != device{ - return; + let start_time = match self.swipe_start_time { + Some(time) => time, + None => self.start_swipe(key_label, device), + }; + + if let Some(current_device) = self.current_swipe_device { + if current_device != device { + return; + } } let within_key_pos_from_center = Vec2 { @@ -143,13 +211,19 @@ impl SwipeTypingManager { y: QwertyKeyboardGrid::get_key_width() as f32, }; - let point = within_key_pos_from_center*key_dimensions + key_pos; + let point = within_key_pos_from_center * key_dimensions + key_pos; let duration = Instant::now().duration_since(start_time); - self.current_swipe.push(SwipePoint::new(point.x.into(), point.y.into(), duration)) + self.current_swipe + .push(SwipePoint::new(point.x.into(), point.y.into(), duration)) + } + } +} + +impl Drop for SwipeTypingManager { + fn drop(&mut self) { + let _ = self.prediction_task_sender.send(PredictionTask::Shutdown); + if let Some(handle) = self.worker_thread.take() { + let _ = handle.join(); } - } - - - }