RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

process swipes on separate thread to not block ui

This commit is contained in:
Diamond White 2026-03-15 12:32:16 +00:00
parent f8f38f6adf
commit 0ee7671832
3 changed files with 276 additions and 204 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

257
wayvr/src/overlays/keyboard/builder.rs
View File

@ -10,7 +10,7 @@ use crate::{
subsystem::hid::XkbKeymap,
windowing::backend::OverlayEventData,
};
use anyhow::Context;
use anyhow::{bail, Context};
use glam::{FloatExt, Mat4, Vec2, vec2, vec3};
use slotmap::Key;
use wgui::{
@ -29,7 +29,7 @@ use wgui::event::StyleSetRequest;
use wgui::layout::LayoutTask;
use wgui::taffy::Display;
use crate::overlays::keyboard::swipe_type::SwipeTypingManager;
use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion};
use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion, init_swipe_type_manager};
const PIXELS_PER_UNIT: f32 = 60.;
@ -52,128 +52,134 @@ pub(super) fn update_swipe_prediction_bar(
(theme.accent_color, theme.animation_mult)
};
let new_swipe_candidates;
if let Some(manager) = panel.state.swipe_typing_manager.as_mut()
&& manager.swipe_candidates != panel.state.swipe_bar_candidates {
panel.state.swipe_bar_candidates = manager.swipe_candidates.clone();
new_swipe_candidates = manager.swipe_candidates.clone();
} else {
return Ok(elements_changed)
}
if let Some(recv) = panel.state.swipe_candidate_receiver.as_mut()
&& let Ok(candidates) = recv.try_recv() {
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if predictions_root.is_null() {
return Ok(elements_changed)
}
let doc_params = new_doc_params(panel);
panel.layout.remove_children(predictions_root);
for (i, prediction) in new_swipe_candidates.iter().enumerate() {
let mut params = HashMap::new();
let id: Rc<str> = Rc::from(format!("Prediction-{i}"));
params.insert("id".into(), id.clone());
params.insert("text".into(), prediction.clone().into());
panel.parser_state.instantiate_template(
&doc_params,
"KeyPrediction",
&mut panel.layout,
predictions_root,
params
)?;
if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) {
let key_state = {
let rect = panel
.layout
.state
.widgets
.get_as::<WidgetRectangle>(widget_id)
.unwrap(); // want panic
Rc::new(KeyState {
// fake button state just so we get key state for anims
button_state: KeyButtonData::Modifier {
modifier: 0,
sticky: core::cell::Cell::new(false),
},
color: rect.params.color,
color2: rect.params.color2,
base_border_color: rect.params.border_color,
cur_border_color: rect.params.border_color.into(),
border: rect.params.border,
drawn_state: false.into(),
})
};
panel.add_event_listener(
widget_id,
EventListenerKind::MousePress,
Box::new({
let k = key_state.clone();
let pred = prediction.clone();
move |common, data, app, state| {
if let Some(manager) = state.swipe_typing_manager.as_mut() {
manager.select_alternate_prediction(&pred, app, state.modifiers);
on_press_anim(k.clone(), common, data)
}
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseEnter,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_enter_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseLeave,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state | {
on_leave_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseRelease,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_release_anim(k.clone(), common, data);
Ok(EventResult::Pass)
}
})
);
if predictions_root.is_null() {
return Ok(elements_changed)
}
let doc_params = new_doc_params(panel);
panel.layout.remove_children(predictions_root);
let Some(new_suggestions) = candidates else {
return Ok(elements_changed)
};
let mut iter = new_suggestions.iter();
let Some(best_prediction) = iter.next() else {
bail!("not enough swipe predictions");
};
if let Some(manager) = panel.state.swipe_typing_manager.as_mut() {
manager.select_word(best_prediction, app, panel.state.modifiers);
}
for (i, prediction) in iter.enumerate() {
let mut params = HashMap::new();
let id: Rc<str> = Rc::from(format!("Prediction-{i}"));
params.insert("id".into(), id.clone());
params.insert("text".into(), prediction.clone().into());
panel.parser_state.instantiate_template(
&doc_params,
"KeyPrediction",
&mut panel.layout,
predictions_root,
params
)?;
if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) {
let key_state = {
let rect = panel
.layout
.state
.widgets
.get_as::<WidgetRectangle>(widget_id)
.unwrap(); // want panic
Rc::new(KeyState {
// fake button state just so we get key state for anims
button_state: KeyButtonData::Modifier {
modifier: 0,
sticky: core::cell::Cell::new(false),
},
color: rect.params.color,
color2: rect.params.color2,
base_border_color: rect.params.border_color,
cur_border_color: rect.params.border_color.into(),
border: rect.params.border,
drawn_state: false.into(),
})
};
panel.add_event_listener(
widget_id,
EventListenerKind::MousePress,
Box::new({
let k = key_state.clone();
let pred = prediction.clone();
move |common, data, app, state| {
if let Some(manager) = state.swipe_typing_manager.as_mut() {
manager.select_alternate_prediction(&pred, app, state.modifiers);
on_press_anim(k.clone(), common, data)
}
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseEnter,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_enter_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseLeave,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state | {
on_leave_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseRelease,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_release_anim(k.clone(), common, data);
Ok(EventResult::Pass)
}
})
);
}
}
elements_changed = true;
}
elements_changed = true;
Ok(elements_changed)
}
#[allow(clippy::too_many_lines, clippy::significant_drop_tightening)]
@ -473,6 +479,8 @@ pub(super) fn create_keyboard_panel(
}
if !app.session.config.swipe_to_type_enabled {
panel.state.swipe_typing_manager = None;
panel.state.swipe_candidate_receiver = None;
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
@ -488,13 +496,8 @@ pub(super) fn create_keyboard_panel(
}
}
if app.session.config.swipe_to_type_enabled && panel.state.swipe_typing_manager.is_none() {
panel.state.swipe_typing_manager = match SwipeTypingManager::new() {
Ok(engine) => Some(engine),
Err(e) => {
log::error!("Error occurred while trying to load swipe engine: {:?}", e);
None
}
};
init_swipe_type_manager(&mut panel.state);
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();

45
wayvr/src/overlays/keyboard/mod.rs
View File

@ -4,7 +4,7 @@ use std::{
process::{Child, Command},
sync::atomic::Ordering,
};
use std::sync::mpsc::Receiver;
use crate::{
KEYMAP_CHANGE,
backend::{
@ -61,7 +61,7 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
set_list: SetList::default(),
clock_12h: app.session.config.clock_12h,
swipe_typing_manager: None,
swipe_bar_candidates: Vec::new(),
swipe_candidate_receiver: None,
};
let auto_labels = layout.auto_labels.unwrap_or(true);
@ -124,7 +124,17 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
..OverlayWindowConfig::from_backend(Box::new(backend))
})
}
pub fn init_swipe_type_manager(state: &mut KeyboardState) {
match SwipeTypingManager::new() {
Ok((engine, receiver)) => {
state.swipe_typing_manager = Some(engine);
state.swipe_candidate_receiver = Some(receiver);
},
Err(e) => {
log::error!("Error occured while trying to load swipe engine: {:?}", e);
}
};
}
fn alt_modifier_to_key(m: AltModifier) -> KeyModifier {
match m {
AltModifier::Shift => SHIFT,
@ -158,13 +168,7 @@ impl KeyboardBackend {
) -> anyhow::Result<KeyboardPanelKey> {
let mut state = self.default_state.take();
state.swipe_typing_manager = match SwipeTypingManager::new() {
Ok(engine) => Some(engine),
Err(e) => {
log::error!("Error occured while trying to load swipe engine: {:?}", e);
None
}
};
init_swipe_type_manager(&mut state);
log::info!("swipe engine created");
let panel =
@ -179,6 +183,7 @@ impl KeyboardBackend {
Ok(id)
}
fn switch_keymap(&mut self, keymap: &XkbKeymap, app: &mut AppState) -> anyhow::Result<bool> {
if !self.wlx_layout.auto_labels.unwrap_or(true) {
return Ok(false);
@ -211,13 +216,8 @@ impl KeyboardBackend {
.state
.take();
state_from.swipe_typing_manager = match SwipeTypingManager::new() {
Ok(engine) => Some(engine),
Err(e) => {
log::error!("Error occured while trying to load swipe engine: {:?}", e);
None
}
};
init_swipe_type_manager(&mut state_from);
self.active_layout = new_key;
self.layout_panels
@ -360,7 +360,7 @@ struct KeyboardState {
set_list: SetList,
clock_12h: bool,
swipe_typing_manager: Option<SwipeTypingManager>,
swipe_bar_candidates: Vec<String>,
swipe_candidate_receiver: Option<Receiver<Option<Vec<String>>>>
}
macro_rules! take_and_leave_default {
@ -381,7 +381,7 @@ impl KeyboardState {
set_list: SetList::default(),
clock_12h: self.clock_12h,
swipe_typing_manager: None,
swipe_bar_candidates: Vec::new(),
swipe_candidate_receiver: None,
}
}
}
@ -515,14 +515,9 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() && *k_cap_type == KeyCapType::Letter {
println!("some");
if swipe_manager.did_swipe_leave_first_key() {
println!("left first");
match swipe_manager.predict() {
Ok(top_prediction) => {
println!("best prediction for swipe: {top_prediction}");
swipe_manager.select_word(&top_prediction, app, keyboard.modifiers);
},
Ok(()) => {},
Err(e) => {
log::error!("{}", e)
}

178
wayvr/src/overlays/keyboard/swipe_type.rs
View File

@ -4,37 +4,50 @@ use anyhow::{anyhow, bail};
use arboard::Clipboard;
use glam::Vec2;
use std::mem;
use std::sync::mpsc::{sync_channel, Receiver, SyncSender, channel, Sender};
use std::thread::{self, JoinHandle};
use std::time::Instant;
use super_swipe_type::keyboard_manager::QwertyKeyboardGrid;
use super_swipe_type::swipe_orchestrator::SwipeOrchestrator;
use super_swipe_type::{SwipeCandidate, SwipePoint};
const PREDICTION_SUGGESTION_COUNT: usize = 4;
const PREDICTION_SUGGESTION_COUNT: usize = 5;
enum PredictionTask {
Predict {
swipe: Vec<SwipePoint>,
last_word: Option<String>,
},
Shutdown,
}
pub struct SwipeTypingManager {
swipe_engine: SwipeOrchestrator,
keyboard_gird: QwertyKeyboardGrid,
current_swipe: Vec<SwipePoint>,
pub swipe_candidates: Vec<String>,
swipe_candidate_sender: SyncSender<Option<Vec<String>>>,
prediction_task_sender: Sender<PredictionTask>,
worker_thread: Option<JoinHandle<()>>,
swipe_start_time: Option<Instant>,
clipboard: Clipboard,
swipe_left_first_key: bool,
first_swipe_char: char,
current_swipe_device: Option<usize>, // the pointer that started this swipe
last_swiped_word: Option<String>
current_swipe_device: Option<usize>,
last_swiped_word: Option<String>,
}
impl SwipeTypingManager {
impl SwipeTypingManager {
pub fn select_alternate_prediction(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
Self::undo_paste(app, original_keyboard_mods);
self.select_word(word, app, original_keyboard_mods);
}
/// Attempts to "type" the word by copying to clipboard and pasting
pub fn select_word(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
self.last_swiped_word = Some(word.clone());
if let Ok(_) =self.copy_text_to_primary_clipboard(word.as_ref()) {
if let Ok(_) = self.copy_text_to_primary_clipboard(word.as_ref()) {
Self::paste(app, original_keyboard_mods);
}
}
fn undo_paste(app: &mut AppState, original_keyboard_mods: KeyModifier) {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
@ -45,6 +58,7 @@ impl SwipeTypingManager {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
}
fn paste(app: &mut AppState, original_keyboard_mods: KeyModifier) {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
@ -55,46 +69,91 @@ impl SwipeTypingManager {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
}
fn copy_text_to_primary_clipboard(&mut self, text: &str) -> Result<(), arboard::Error> {
self.clipboard.set_text(format!("{text} "))
}
pub fn new() -> anyhow::Result<SwipeTypingManager> {
// todo: use the layout_name to choose a sensible language for the swipe engine
Ok(Self {
swipe_engine: SwipeOrchestrator::new()?,
keyboard_gird: QwertyKeyboardGrid::new(), // contains a hashmap<char, vector2> where the vector2 is the center pos of each key in querty
current_swipe: Vec::new(),
swipe_candidates: Vec::new(),
swipe_start_time: None,
clipboard: Clipboard::new()?,
swipe_left_first_key: false,
first_swipe_char: char::default(),
current_swipe_device: None,
last_swiped_word: None
})
pub fn new() -> anyhow::Result<(SwipeTypingManager, Receiver<Option<Vec<String>>>)> {
let (candidate_sender, candidate_receiver) = sync_channel(1);
let (task_sender, task_receiver) = channel::<PredictionTask>();
// Spawn persistent worker thread
let worker_candidate_sender = candidate_sender.clone();
let worker_thread = thread::spawn(move || {
let mut swipe_engine = match SwipeOrchestrator::new() {
Ok(engine) => engine,
Err(e) => {
log::error!("Failed to initialize SwipeOrchestrator: {}", e);
return;
}
};
while let Ok(task) = task_receiver.recv() {
match task {
PredictionTask::Predict { swipe, last_word } => {
match swipe_engine.predict(swipe, &last_word) {
Ok(candidates) => {
let words: Vec<String> = candidates
.iter()
.take(PREDICTION_SUGGESTION_COUNT)
.map(|c| c.word.clone())
.collect();
let _ = worker_candidate_sender.send(Some(words));
}
Err(e) => {
log::error!("Prediction failed: {}", e);
}
}
}
PredictionTask::Shutdown => break,
}
}
});
Ok((
Self {
keyboard_gird: QwertyKeyboardGrid::new(),
current_swipe: Vec::new(),
swipe_candidate_sender: candidate_sender,
prediction_task_sender: task_sender,
worker_thread: Some(worker_thread),
swipe_start_time: None,
clipboard: Clipboard::new()?,
swipe_left_first_key: false,
first_swipe_char: char::default(),
current_swipe_device: None,
last_swiped_word: None,
},
candidate_receiver,
))
}
pub fn predict(&mut self) -> anyhow::Result<String>{
pub fn predict(&mut self) -> anyhow::Result<()> {
if self.is_current_swipe_empty() {
bail!("nothing to predict");
} else {
let current_swipe = mem::take(&mut self.current_swipe);
self.reset_swipe();
let candidates: Vec<SwipeCandidate> = self.swipe_engine.predict(current_swipe, &self.last_swiped_word)?;
let mut iter = candidates.iter();
let top_choice = iter.next().ok_or(anyhow!("No swipe candidates generated"))?.word.clone();
self.swipe_candidates = iter.take(PREDICTION_SUGGESTION_COUNT).map(|c| c.word.clone()).collect();
Ok(top_choice)
}
let current_swipe = mem::take(&mut self.current_swipe);
let last_word = self.last_swiped_word.clone();
self.reset_swipe();
self.prediction_task_sender
.send(PredictionTask::Predict {
swipe: current_swipe,
last_word,
})?;
Ok(())
}
pub fn reset(&mut self) {
self.reset_swipe();
let _ = self.swipe_candidate_sender.send(None);
self.last_swiped_word = None;
self.swipe_candidates = Vec::new();
}
fn reset_swipe(&mut self) {
self.swipe_start_time = None;
self.current_swipe = Vec::new();
@ -102,36 +161,45 @@ impl SwipeTypingManager {
self.swipe_left_first_key = false;
self.current_swipe_device = None;
}
fn start_swipe(&mut self, key_label: char, device: usize) -> Instant{
fn start_swipe(&mut self, key_label: char, device: usize) -> Instant {
let now = Instant::now();
self.swipe_start_time = Some(now);
self.first_swipe_char = key_label.to_ascii_lowercase();
self.current_swipe_device = Some(device);
self.swipe_candidates = Vec::new();
now
}
pub fn did_swipe_leave_first_key(&self) -> bool {
self.swipe_left_first_key
}
pub fn is_current_swipe_empty(&self) -> bool {
self.current_swipe.is_empty()
}
pub fn add_swipe(&mut self, within_key_pos_normalized: &Vec2, key_label: char, device: usize) {
if let Some(pos) = self.keyboard_gird.key_positions.get(&key_label.to_ascii_lowercase()) {
if self.first_swipe_char != char::default() && self.first_swipe_char != key_label.to_ascii_lowercase() {
if self.first_swipe_char != char::default()
&& self.first_swipe_char != key_label.to_ascii_lowercase()
{
self.swipe_left_first_key = true;
}
let key_pos = Vec2{x: pos.x as f32, y: pos.y as f32 };
//println!("char : {key_label} at pos: {key_pos}");
let start_time = match self.swipe_start_time {
Some(time) => time,
None => self.start_swipe(key_label, device)
let key_pos = Vec2 {
x: pos.x as f32,
y: pos.y as f32,
};
// only allow the pointer that started the swipe to contribute to it
if let Some(current_device) = self.current_swipe_device && current_device != device{
return;
let start_time = match self.swipe_start_time {
Some(time) => time,
None => self.start_swipe(key_label, device),
};
if let Some(current_device) = self.current_swipe_device {
if current_device != device {
return;
}
}
let within_key_pos_from_center = Vec2 {
@ -143,13 +211,19 @@ impl SwipeTypingManager {
y: QwertyKeyboardGrid::get_key_width() as f32,
};
let point = within_key_pos_from_center*key_dimensions + key_pos;
let point = within_key_pos_from_center * key_dimensions + key_pos;
let duration = Instant::now().duration_since(start_time);
self.current_swipe.push(SwipePoint::new(point.x.into(), point.y.into(), duration))
self.current_swipe
.push(SwipePoint::new(point.x.into(), point.y.into(), duration))
}
}
}
impl Drop for SwipeTypingManager {
fn drop(&mut self) {
let _ = self.prediction_task_sender.send(PredictionTask::Shutdown);
if let Some(handle) = self.worker_thread.take() {
let _ = handle.join();
}
}
}