diff --git a/wayvr/src/assets/gui/keyboard.xml b/wayvr/src/assets/gui/keyboard.xml index 4d6fc321..ceea4027 100644 --- a/wayvr/src/assets/gui/keyboard.xml +++ b/wayvr/src/assets/gui/keyboard.xml @@ -13,6 +13,11 @@ width="${width}" height="${height}" min_width="${width}" min_height="${height}" max_width="${width}" max_height="${height}" /> + + @@ -90,6 +95,13 @@ + + + @@ -232,6 +244,9 @@
+ + +
diff --git a/wayvr/src/overlays/keyboard/builder.rs b/wayvr/src/overlays/keyboard/builder.rs index fad16dca..efd1e44c 100644 --- a/wayvr/src/overlays/keyboard/builder.rs +++ b/wayvr/src/overlays/keyboard/builder.rs @@ -1,5 +1,4 @@ use std::{collections::HashMap, rc::Rc, time::Duration}; - use crate::{ app_misc, gui::{ @@ -12,8 +11,8 @@ use crate::{ windowing::backend::OverlayEventData, }; use anyhow::Context; -use dbus::arg::Append; use glam::{FloatExt, Mat4, Vec2, vec2, vec3}; +use slotmap::Key; use wgui::{ animation::{Animation, AnimationEasing}, assets::AssetPath, @@ -26,7 +25,10 @@ use wgui::{ taffy::{self, prelude::length}, widget::{EventResult, div::WidgetDiv, rectangle::WidgetRectangle}, }; -use crate::overlays::keyboard::layout::KeyData; +use wgui::event::StyleSetRequest; +use wgui::layout::LayoutTask; +use wgui::taffy::Display; +use crate::overlays::keyboard::swipe_type::SwipeTypingManager; use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion}; const PIXELS_PER_UNIT: f32 = 60.; @@ -39,6 +41,141 @@ fn new_doc_params(panel: &mut GuiPanel) -> ParseDocumentParams<'s } } +pub(super) fn update_swipe_prediction_bar( + panel: &mut GuiPanel, + app: &mut AppState +) -> anyhow::Result { + let mut elements_changed = false; + + let (accent_color, anim_mult) = { + let theme = &app.wgui_theme; + (theme.accent_color, theme.animation_mult) + }; + + let new_swipe_candidates; + if let Some(manager) = panel.state.swipe_typing_manager.as_mut() + && manager.swipe_candidates != panel.state.swipe_bar_candidates { + panel.state.swipe_bar_candidates = manager.swipe_candidates.clone(); + new_swipe_candidates = manager.swipe_candidates.clone(); + } else { + return Ok(elements_changed) + } + + let predictions_root = panel.parser_state + .get_widget_id("swipe_predictions_root") + .unwrap_or_default(); + + if predictions_root.is_null() { + return Ok(elements_changed) + } + let doc_params = new_doc_params(panel); + + panel.layout.remove_children(predictions_root); + + for (i, prediction) in new_swipe_candidates.iter().enumerate() { + let mut params = HashMap::new(); + let id: Rc = Rc::from(format!("Prediction-{i}")); + params.insert("id".into(), id.clone()); + params.insert("text".into(), prediction.clone().into()); + + panel.parser_state.instantiate_template( + &doc_params, + "KeyPrediction", + &mut panel.layout, + predictions_root, + params + )?; + + if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) { + let key_state = { + let rect = panel + .layout + .state + .widgets + .get_as::(widget_id) + .unwrap(); // want panic + + Rc::new(KeyState { + // fake button state just so we get key state for anims + button_state: KeyButtonData::Modifier { + modifier: 0, + sticky: core::cell::Cell::new(false), + }, + color: rect.params.color, + color2: rect.params.color2, + base_border_color: rect.params.border_color, + cur_border_color: rect.params.border_color.into(), + border: rect.params.border, + drawn_state: false.into(), + }) + }; + panel.add_event_listener( + widget_id, + EventListenerKind::MousePress, + Box::new({ + let k = key_state.clone(); + let pred = prediction.clone(); + move |common, data, app, state| { + if let Some(manager) = state.swipe_typing_manager.as_mut() { + manager.select_alternate_prediction(&pred, app, state.modifiers); + on_press_anim(k.clone(), common, data) + } + Ok(EventResult::Pass) + } + }) + ); + panel.add_event_listener( + widget_id, + EventListenerKind::MouseEnter, + Box::new({ + let k = key_state.clone(); + move |common, data, _app, _state| { + on_enter_anim( + k.clone(), + common, + data, + accent_color, + anim_mult, + 0.0, + ); + Ok(EventResult::Pass) + } + }) + ); + panel.add_event_listener( + widget_id, + EventListenerKind::MouseLeave, + Box::new({ + let k = key_state.clone(); + move |common, data, _app, _state | { + on_leave_anim( + k.clone(), + common, + data, + accent_color, + anim_mult, + 0.0, + ); + Ok(EventResult::Pass) + } + }) + ); + panel.add_event_listener( + widget_id, + EventListenerKind::MouseRelease, + Box::new({ + let k = key_state.clone(); + move |common, data, _app, _state| { + on_release_anim(k.clone(), common, data); + Ok(EventResult::Pass) + } + }) + ); + } + } + elements_changed = true; + Ok(elements_changed) +} #[allow(clippy::too_many_lines, clippy::significant_drop_tightening)] pub(super) fn create_keyboard_panel( app: &mut AppState, @@ -331,6 +468,42 @@ pub(super) fn create_keyboard_panel( elems_changed = true; } } + if !app.session.config.swipe_to_type_enabled { + panel.state.swipe_typing_manager = None; + let predictions_root = panel.parser_state + .get_widget_id("swipe_predictions_root") + .unwrap_or_default(); + + if !predictions_root.is_null() { + panel.layout.remove_children(predictions_root); + + panel.layout.tasks.push(LayoutTask::SetWidgetStyle( + predictions_root, + StyleSetRequest::Display(Display::None), + )); + + } + } + if app.session.config.swipe_to_type_enabled && panel.state.swipe_typing_manager.is_none() { + panel.state.swipe_typing_manager = match SwipeTypingManager::new() { + Ok(engine) => Some(engine), + Err(e) => { + log::error!("Error occurred while trying to load swipe engine: {:?}", e); + None + } + }; + let predictions_root = panel.parser_state + .get_widget_id("swipe_predictions_root") + .unwrap_or_default(); + + if !predictions_root.is_null() { + panel.layout.tasks.push(LayoutTask::SetWidgetStyle( + predictions_root, + StyleSetRequest::Display(Display::Flex), + )); + + } + } } OverlayEventData::CustomCommand { element, command } => { diff --git a/wayvr/src/overlays/keyboard/mod.rs b/wayvr/src/overlays/keyboard/mod.rs index 9ab63a11..6c1177a1 100644 --- a/wayvr/src/overlays/keyboard/mod.rs +++ b/wayvr/src/overlays/keyboard/mod.rs @@ -27,7 +27,6 @@ use crate::{ }, }; use anyhow::Context; -use arboard::Clipboard; use glam::{Affine3A, Quat, Vec3, vec3, Vec2}; use regex::Regex; use slotmap::{SlotMap, new_key_type}; @@ -40,7 +39,7 @@ use wlx_common::{ config::AltModifier, overlays::{BackendAttrib, BackendAttribValue}, }; -use codes_iso_639::part_1::LanguageCode; +use crate::overlays::keyboard::builder::update_swipe_prediction_bar; use crate::overlays::keyboard::layout::KeyCapType; use crate::overlays::keyboard::swipe_type::SwipeTypingManager; @@ -62,6 +61,7 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result anyhow::Result anyhow::Result<()> { + if update_swipe_prediction_bar(self.panel(), app)? { + self.panel().process_custom_elems(app); + } + Ok(()) + } + fn auto_switch_keymap(&mut self, app: &mut AppState) -> anyhow::Result { let keymap = self.get_effective_keymap()?; app.hid_provider @@ -290,6 +297,7 @@ impl OverlayBackend for KeyboardBackend { }); } } + self.update_swipe_prediction_bar(app)?; self.panel().should_render(app) } fn render(&mut self, app: &mut AppState, rdr: &mut RenderResources) -> anyhow::Result<()> { @@ -352,6 +360,7 @@ struct KeyboardState { set_list: SetList, clock_12h: bool, swipe_typing_manager: Option, + swipe_bar_candidates: Vec, } macro_rules! take_and_leave_default { @@ -372,6 +381,7 @@ impl KeyboardState { set_list: SetList::default(), clock_12h: self.clock_12h, swipe_typing_manager: None, + swipe_bar_candidates: Vec::new(), } } } @@ -497,13 +507,13 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k match &key.button_state { KeyButtonData::Key { vk, pressed } => { if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() && *k_cap_type == KeyCapType::Letter { - if swipe_manager.swipe_left_first_key() { + println!("some"); + if swipe_manager.did_swipe_leave_first_key() { + println!("left first"); match swipe_manager.predict() { - Ok(predictions) => { - let best_prediction = predictions.first().unwrap(); - println!("best prediction for swipe: {best_prediction}"); - - swipe_manager.select_word(&best_prediction, app, keyboard.modifiers); + Ok(top_prediction) => { + println!("best prediction for swipe: {top_prediction}"); + swipe_manager.select_word(&top_prediction, app, keyboard.modifiers); }, Err(e) => { log::error!("{}", e) @@ -511,8 +521,9 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k } } else { // pointer must have been released on the same key it was pressed on + println!("swipe reset"); swipe_manager.reset(); // drop swipe tracking that was started on press - + app.hid_provider .send_key_routed(app.wvr_server.as_mut(), *vk, true); pressed.set(true); @@ -522,6 +533,9 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k } } else { + if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() { + swipe_manager.reset(); + } pressed.set(false); for m in &AUTO_RELEASE_MODS { diff --git a/wayvr/src/overlays/keyboard/swipe_type.rs b/wayvr/src/overlays/keyboard/swipe_type.rs index ca25c945..871f3d84 100644 --- a/wayvr/src/overlays/keyboard/swipe_type.rs +++ b/wayvr/src/overlays/keyboard/swipe_type.rs @@ -1,6 +1,6 @@ use crate::state::AppState; use crate::subsystem::hid::{KeyModifier, VirtualKey, CTRL}; -use anyhow::bail; +use anyhow::{anyhow, bail}; use arboard::Clipboard; use glam::Vec2; use std::mem; @@ -9,10 +9,12 @@ use super_swipe_type::keyboard_manager::QwertyKeyboardGrid; use super_swipe_type::swipe_orchestrator::SwipeOrchestrator; use super_swipe_type::{SwipeCandidate, SwipePoint}; +const PREDICTION_SUGGESTION_COUNT: usize = 4; pub struct SwipeTypingManager { swipe_engine: SwipeOrchestrator, keyboard_gird: QwertyKeyboardGrid, current_swipe: Vec, + pub swipe_candidates: Vec, swipe_start_time: Option, clipboard: Clipboard, swipe_left_first_key: bool, @@ -21,6 +23,10 @@ pub struct SwipeTypingManager { } impl SwipeTypingManager { + pub fn select_alternate_prediction(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) { + Self::undo_paste(app, original_keyboard_mods); + self.select_word(word, app, original_keyboard_mods); + } /// Attempts to "type" the word by copying to clipboard and pasting pub fn select_word(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) { self.last_swiped_word = Some(word.clone()); @@ -28,6 +34,16 @@ impl SwipeTypingManager { Self::paste(app, original_keyboard_mods); } } + fn undo_paste(app: &mut AppState, original_keyboard_mods: KeyModifier) { + app.hid_provider + .set_modifiers_routed(app.wvr_server.as_mut(), CTRL); + app.hid_provider + .send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, true); + app.hid_provider + .send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, false); + app.hid_provider + .set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods); + } fn paste(app: &mut AppState, original_keyboard_mods: KeyModifier) { app.hid_provider .set_modifiers_routed(app.wvr_server.as_mut(), CTRL); @@ -39,7 +55,6 @@ impl SwipeTypingManager { .set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods); } fn copy_text_to_primary_clipboard(&mut self, text: &str) -> Result<(), arboard::Error> { - println!("{}", std::env::var("WAYLAND_DISPLAY").unwrap()); self.clipboard.set_text(format!("{text} ")) } pub fn new() -> anyhow::Result { @@ -49,6 +64,7 @@ impl SwipeTypingManager { swipe_engine: SwipeOrchestrator::new()?, keyboard_gird: QwertyKeyboardGrid::new(), // contains a hashmap where the vector2 is the center pos of each key in querty current_swipe: Vec::new(), + swipe_candidates: Vec::new(), swipe_start_time: None, clipboard: Clipboard::new()?, swipe_left_first_key: false, @@ -56,25 +72,41 @@ impl SwipeTypingManager { last_swiped_word: None }) } - pub fn predict(&mut self) -> anyhow::Result>{ + pub fn predict(&mut self) -> anyhow::Result{ if self.is_current_swipe_empty() { bail!("nothing to predict"); } else { let current_swipe = mem::take(&mut self.current_swipe); - self.reset(); + self.reset_swipe(); let candidates: Vec = self.swipe_engine.predict(current_swipe, &self.last_swiped_word)?; - Ok(candidates.iter().map(|c| c.word.clone()).collect()) + + let mut iter = candidates.iter(); + let top_choice = iter.next().ok_or(anyhow!("No swipe candidates generated"))?.word.clone(); + self.swipe_candidates = iter.take(PREDICTION_SUGGESTION_COUNT).map(|c| c.word.clone()).collect(); + + Ok(top_choice) } } pub fn reset(&mut self) { + self.reset_swipe(); self.last_swiped_word = None; + self.swipe_candidates = Vec::new(); + } + fn reset_swipe(&mut self) { self.swipe_start_time = None; self.current_swipe = Vec::new(); self.first_swipe_char = char::default(); self.swipe_left_first_key = false; } - pub fn swipe_left_first_key(&self) -> bool { + fn start_swipe(&mut self, key_label: char) -> Instant{ + let now = Instant::now(); + self.swipe_start_time = Some(now); + self.first_swipe_char = key_label.to_ascii_lowercase(); + self.swipe_candidates = Vec::new(); + now + } + pub fn did_swipe_leave_first_key(&self) -> bool { self.swipe_left_first_key } pub fn is_current_swipe_empty(&self) -> bool { @@ -85,17 +117,12 @@ impl SwipeTypingManager { if self.first_swipe_char != char::default() && self.first_swipe_char != key_label.to_ascii_lowercase() { self.swipe_left_first_key = true; } + let key_pos = Vec2{x: pos.x as f32, y: pos.y as f32 }; - println!("char : {key_label} at pos: {key_pos}"); + //println!("char : {key_label} at pos: {key_pos}"); let start_time = match self.swipe_start_time { Some(time) => time, - None => { - // this is the first swipe entry - let now = Instant::now(); - self.swipe_start_time = Some(now); - self.first_swipe_char = key_label.to_ascii_lowercase(); - now - } + None => self.start_swipe(key_label) }; let within_key_pos_from_center = Vec2 { diff --git a/wlx-common/src/config.rs b/wlx-common/src/config.rs index 8ab23297..c578ef18 100644 --- a/wlx-common/src/config.rs +++ b/wlx-common/src/config.rs @@ -327,4 +327,7 @@ pub struct GeneralConfig { #[serde(default = "def_one")] pub grid_opacity: f32, + + #[serde(default = "def_true")] + pub swipe_to_type_enabled: bool, }