RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

use new prediction engine, move swipe type related logic to its own module

This commit is contained in:
Diamond White 2026-03-12 03:06:27 +00:00
parent e45756d787
commit 84b43f30dc
7 changed files with 709 additions and 228 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

641
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

3
wayvr/Cargo.toml
View File

@ -87,10 +87,9 @@ xcb = { version = "1.6.0", optional = true, features = [
"as-raw-xcb-connection",
] }
xkbcommon = { version = "0.8.0" } # 0.9.0 breaks keymap import on some distros
super-swipe-engine = {git="https://github.com/oneshinyboi/swipeType", tag="v0.1.11"}
codes-iso-639 = "0.1.5"
swipe-types = "0.1.6"
arboard = { version="3.6.1", features = ["wayland-data-control", "wl-clipboard-rs"] }
super-swipe-type = "0.1.2"
[build-dependencies]

25
wayvr/src/overlays/keyboard/builder.rs
View File

@ -27,7 +27,7 @@ use wgui::{
widget::{EventResult, div::WidgetDiv, rectangle::WidgetRectangle},
};
use crate::overlays::keyboard::layout::KeyData;
use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_enter};
use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion};
const PIXELS_PER_UNIT: f32 = 60.;
@ -179,8 +179,6 @@ pub(super) fn create_keyboard_panel(
EventListenerKind::MouseEnter,
Box::new({
let k = key_state.clone();
let k_label = key_label.clone();
let k_cap_type = key_cap_type.clone();
move |common, data, _app, state| {
common.alterables.trigger_haptics();
on_enter_anim(
@ -192,7 +190,7 @@ pub(super) fn create_keyboard_panel(
width_mul,
);
handle_enter(&k, &k_label, &k_cap_type, state);
Ok(EventResult::Pass)
}
}),
@ -228,19 +226,34 @@ pub(super) fn create_keyboard_panel(
let CallbackMetadata::MouseButton(button) = data.metadata else {
panic!("CallbackMetadata should contain MouseButton!");
};
let within_key_pos = data.metadata.get_mouse_pos_normalized(&common.alterables.transform_stack);
handle_press(app, &k, &k_cap_type, &k_label, state, button);
handle_press(app, &k, &k_label, &k_cap_type, &within_key_pos, state, button);
on_press_anim(k.clone(), common, data);
Ok(EventResult::Pass)
}
}),
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseMotion,
Box::new({
let k = key_state.clone();
let k_label = key_label.clone();
let k_cap_type = key_cap_type.clone();
move |common, data, app, state| {
let within_key_pos = data.metadata.get_mouse_pos_normalized(&common.alterables.transform_stack);
handle_mouse_motion(&k, &k_label, &k_cap_type, state, &within_key_pos);
Ok(EventResult::Pass)
}
}),
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseRelease,
Box::new({
let k = key_state.clone();
let k_label = key_label.clone();
let k_cap_type = key_cap_type.clone();
move |common, data, app, state| {
if handle_release(app, &k, &k_cap_type, state) {

1
wayvr/src/overlays/keyboard/layout.rs
View File

@ -3,7 +3,6 @@ use std::{collections::HashMap, str::FromStr, sync::LazyLock};
use regex::Regex;
use serde::{Deserialize, Serialize};
use smithay::reexports::wayland_protocols::wp::pointer_gestures::zv1::server::zwp_pointer_gesture_swipe_v1::ZwpPointerGestureSwipeV1;
use swipe_types::types::Point;
use crate::{
config::{ConfigType, load_known_yaml},
subsystem::hid::{

103
wayvr/src/overlays/keyboard/mod.rs
View File

@ -28,10 +28,9 @@ use crate::{
};
use anyhow::Context;
use arboard::Clipboard;
use glam::{Affine3A, Quat, Vec3, vec3};
use glam::{Affine3A, Quat, Vec3, vec3, Vec2};
use regex::Regex;
use slotmap::{SlotMap, new_key_type};
use super_swipe_engine::SwipeEngine;
use wgui::{
drawing,
event::{InternalStateChangeEvent, MouseButtonEvent, MouseButtonIndex},
@ -43,7 +42,7 @@ use wlx_common::{
};
use codes_iso_639::part_1::LanguageCode;
use crate::overlays::keyboard::layout::KeyCapType;
use crate::overlays::keyboard::swipe_type::{copy_text_to_primary_clipboard, create_new_swipe_engine};
use crate::overlays::keyboard::swipe_type::SwipeTypingManager;
pub mod builder;
mod layout;
@ -62,12 +61,7 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
overlay_list: OverlayList::default(),
set_list: SetList::default(),
clock_12h: app.session.config.clock_12h,
swipe_engine: None,
current_swipe_input: String::new(),
is_swiping: false,
last_pressed_key_label: String::new(),
clipboard: Clipboard::new()?,
last_swiped_word: None
swipe_typing_manager: None,
};
let auto_labels = layout.auto_labels.unwrap_or(true);
@ -164,7 +158,7 @@ impl KeyboardBackend {
) -> anyhow::Result<KeyboardPanelKey> {
let mut state = self.default_state.take();
state.swipe_engine = match create_new_swipe_engine(&keymap, &self.wlx_layout) {
state.swipe_typing_manager = match SwipeTypingManager::new() {
Ok(engine) => Some(engine),
Err(e) => {
log::error!("Error occured while trying to load swipe engine: {:?}", e);
@ -217,7 +211,7 @@ impl KeyboardBackend {
.state
.take();
state_from.swipe_engine = match create_new_swipe_engine(&Some(keymap), &self.wlx_layout) {
state_from.swipe_typing_manager = match SwipeTypingManager::new() {
Ok(engine) => Some(engine),
Err(e) => {
log::error!("Error occured while trying to load swipe engine: {:?}", e);
@ -357,15 +351,7 @@ struct KeyboardState {
overlay_list: OverlayList,
set_list: SetList,
clock_12h: bool,
// todo move all this swipe stuff into its own class
swipe_engine: Option<SwipeEngine>,
current_swipe_input: String,
last_pressed_key_label: String,
is_swiping: bool,
clipboard: Clipboard,
last_swiped_word: Option<String>
swipe_typing_manager: Option<SwipeTypingManager>,
}
macro_rules! take_and_leave_default {
@ -385,12 +371,7 @@ impl KeyboardState {
overlay_list: OverlayList::default(),
set_list: SetList::default(),
clock_12h: self.clock_12h,
swipe_engine: None,
current_swipe_input: String::new(),
is_swiping: false,
last_pressed_key_label: String::new(),
clipboard: Clipboard::new().unwrap(),
last_swiped_word: None
swipe_typing_manager: None,
}
}
}
@ -431,37 +412,44 @@ enum KeyButtonData {
},
}
fn handle_enter(key: &KeyState, key_label: &Vec<String>, key_cap_type: &KeyCapType, keyboard: &mut KeyboardState) {
if let Some(_) = keyboard.swipe_engine.as_ref() && *key_cap_type == KeyCapType::Letter {
if *key_label.iter().next().unwrap() != keyboard.last_pressed_key_label {
keyboard.is_swiping = true;
}
if keyboard.is_swiping {
fn handle_mouse_motion(key: &KeyState, key_label: &Vec<String>, key_cap_type: &KeyCapType, keyboard: &mut KeyboardState, within_key_pos: &Option<Vec2>) {
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() && *key_cap_type == KeyCapType::Letter {
if !swipe_manager.is_current_swipe_empty() {
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
keyboard.current_swipe_input.push_str(&*key_label.iter().next().unwrap().to_ascii_lowercase())
if let Some(pos) = within_key_pos {
// check because mouse motion can trigger despite hover being false
if pos.x >= 0.0 && pos.x <= 1.0 && pos.y >= 0.0 && pos.y <= 1.0 {
if let Some(label) = key_label.first() {
swipe_manager.add_swipe(pos, label.chars().next().unwrap_or_default());
}
}
}
}
_ => {}
}
}
}
}
fn handle_press(
app: &mut AppState,
key: &KeyState,
key_cap_type: &KeyCapType,
key_label: &Vec<String>,
key_cap_type: &KeyCapType,
within_key_pos: &Option<Vec2>,
keyboard: &mut KeyboardState,
button: MouseButtonEvent,
) {
keyboard.is_swiping = false;
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
if let Some(_) = keyboard.swipe_engine.as_ref() && *key_cap_type == KeyCapType::Letter {
let actual_label = key_label.iter().next().unwrap();
keyboard.last_pressed_key_label = actual_label.clone();
keyboard.current_swipe_input.clear();
keyboard.current_swipe_input.push_str(&*actual_label.to_ascii_lowercase())
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() && *key_cap_type == KeyCapType::Letter {
if let Some(pos) = within_key_pos {
if let Some(label) = key_label.first() {
swipe_manager.add_swipe(pos, label.chars().next().unwrap_or_default());
}
}
}
else {
keyboard.modifiers |= match button.index {
@ -508,29 +496,23 @@ fn handle_press(
fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, keyboard: &mut KeyboardState) -> bool {
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
if let Some(engine) = keyboard.swipe_engine.as_ref() && *k_cap_type == KeyCapType::Letter {
if keyboard.is_swiping {
if !keyboard.current_swipe_input.is_empty() {
let prediction = engine.predict(&*keyboard.current_swipe_input, keyboard.last_swiped_word.as_ref().map(|x| x.as_str()), 5);
keyboard.current_swipe_input.clear();
println!("swipe path: {}", keyboard.current_swipe_input);
println!("{:?}", prediction);
let best_prediction = prediction.first().unwrap().word.as_ref();
copy_text_to_primary_clipboard(best_prediction, &mut keyboard.clipboard);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), SHIFT);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::Insert, true);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::Insert, false);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
keyboard.last_swiped_word = Some(best_prediction.parse().unwrap())
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() && *k_cap_type == KeyCapType::Letter {
if swipe_manager.swipe_left_first_key() {
match swipe_manager.predict() {
Ok(predictions) => {
let best_prediction = predictions.first().unwrap();
println!("best prediction for swipe: {best_prediction}");
swipe_manager.select_word(&best_prediction, app, keyboard.modifiers);
},
Err(e) => {
log::error!("{}", e)
}
}
}
else { // pointer must have been released on the same key it was pressed on
swipe_manager.reset(); // drop swipe tracking that was started on press
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, true);
pressed.set(true);
@ -538,7 +520,6 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k
.send_key_routed(app.wvr_server.as_mut(), *vk, false);
play_key_click(app);
}
}
else {
pressed.set(false);

160
wayvr/src/overlays/keyboard/swipe_type.rs
View File

@ -1,53 +1,119 @@
use std::collections::HashMap;
use arboard::{Clipboard, LinuxClipboardKind, SetExtLinux};
use codes_iso_639::part_1::LanguageCode;
use futures::TryFutureExt;
use strum::IntoEnumIterator;
use super_swipe_engine::{EngineLoadError, SwipeEngine};
use swipe_types::types::Point;
use crate::overlays::keyboard::layout;
use crate::overlays::keyboard::layout::KeyCapType;
use crate::subsystem::hid::{get_key_type, KeyType, VirtualKey, XkbKeymap};
use crate::state::AppState;
use crate::subsystem::hid::{KeyModifier, VirtualKey, CTRL};
use anyhow::bail;
use arboard::Clipboard;
use glam::Vec2;
use std::mem;
use std::time::Instant;
use super_swipe_type::keyboard_manager::QwertyKeyboardGrid;
use super_swipe_type::swipe_orchestrator::SwipeOrchestrator;
use super_swipe_type::{SwipeCandidate, SwipePoint};
pub fn copy_text_to_primary_clipboard(text: &str, clip: &mut Clipboard) {
println!("{}", std::env::var("WAYLAND_DISPLAY").unwrap());
clip.set_text(format!("{text} ")).unwrap();
pub struct SwipeTypingManager {
swipe_engine: SwipeOrchestrator,
keyboard_gird: QwertyKeyboardGrid,
current_swipe: Vec<SwipePoint>,
swipe_start_time: Option<Instant>,
clipboard: Clipboard,
swipe_left_first_key: bool,
first_swipe_char: char,
last_swiped_word: Option<String>
}
pub fn create_new_swipe_engine(keymap: &Option<&XkbKeymap>, layout: &layout::Layout) -> Result<SwipeEngine, EngineLoadError> {
let layout_name = keymap.and_then(|k| k.get_name()).unwrap_or("us");
let point_map = build_key_to_char_point_map(*keymap, layout);
impl SwipeTypingManager {
// todo: use the layout_name to choose a sensible language for the swipe engine
SwipeEngine::new(LanguageCode::En, Some(point_map))
}
fn build_key_to_char_point_map(keymap: Option<&XkbKeymap>, layout: &layout::Layout) -> HashMap<char, Point> {
let mut map = HashMap::new();
let has_altgr = keymap.as_ref().is_some_and(|m| XkbKeymap::has_altgr(m));
let mut pos_x: f32 = 0.0;
let mut pos_y: f32 = 0.0;
for (row_idx, row) in layout.main_layout.iter().enumerate() {
for (col_idx, col) in row.iter().enumerate() {
let label = layout.get_key_data(keymap, has_altgr, col_idx, row_idx);
if let Some(label) = label {
match label.cap_type {
KeyCapType::Letter => {
if let Some(char) = label.label.iter().next() {
let point = Point {x: pos_x as f64, y: pos_y as f64};
println!("char: {} \n point: {:?}\n",char, point);
map.insert(char.to_ascii_lowercase().chars().next().unwrap(), point);
}
}
_ => {}
}
}
pos_x += layout.key_sizes[row_idx][col_idx];
/// Attempts to "type" the word by copying to clipboard and pasting
pub fn select_word(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
self.last_swiped_word = Some(word.clone());
if let Ok(_) =self.copy_text_to_primary_clipboard(word.as_ref()) {
Self::paste(app, original_keyboard_mods);
}
pos_y += 1.0;
pos_x = 0.0;
}
map
fn paste(app: &mut AppState, original_keyboard_mods: KeyModifier) {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::V, true);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::V, false);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
}
fn copy_text_to_primary_clipboard(&mut self, text: &str) -> Result<(), arboard::Error> {
println!("{}", std::env::var("WAYLAND_DISPLAY").unwrap());
self.clipboard.set_text(format!("{text} "))
}
pub fn new() -> anyhow::Result<SwipeTypingManager> {
// todo: use the layout_name to choose a sensible language for the swipe engine
Ok(Self {
swipe_engine: SwipeOrchestrator::new()?,
keyboard_gird: QwertyKeyboardGrid::new(), // contains a hashmap<char, vector2> where the vector2 is the center pos of each key in querty
current_swipe: Vec::new(),
swipe_start_time: None,
clipboard: Clipboard::new()?,
swipe_left_first_key: false,
first_swipe_char: char::default(),
last_swiped_word: None
})
}
pub fn predict(&mut self) -> anyhow::Result<Vec<String>>{
if self.is_current_swipe_empty() {
bail!("nothing to predict");
} else {
let current_swipe = mem::take(&mut self.current_swipe);
self.reset();
let candidates: Vec<SwipeCandidate> = self.swipe_engine.predict(current_swipe, &self.last_swiped_word)?;
Ok(candidates.iter().map(|c| c.word.clone()).collect())
}
}
pub fn reset(&mut self) {
self.last_swiped_word = None;
self.swipe_start_time = None;
self.current_swipe = Vec::new();
self.first_swipe_char = char::default();
self.swipe_left_first_key = false;
}
pub fn swipe_left_first_key(&self) -> bool {
self.swipe_left_first_key
}
pub fn is_current_swipe_empty(&self) -> bool {
self.current_swipe.is_empty()
}
pub fn add_swipe(&mut self, within_key_pos_normalized: &Vec2, key_label: char) {
if let Some(pos) = self.keyboard_gird.key_positions.get(&key_label.to_ascii_lowercase()) {
if self.first_swipe_char != char::default() && self.first_swipe_char != key_label.to_ascii_lowercase() {
self.swipe_left_first_key = true;
}
let key_pos = Vec2{x: pos.x as f32, y: pos.y as f32 };
println!("char : {key_label} at pos: {key_pos}");
let start_time = match self.swipe_start_time {
Some(time) => time,
None => {
// this is the first swipe entry
let now = Instant::now();
self.swipe_start_time = Some(now);
self.first_swipe_char = key_label.to_ascii_lowercase();
now
}
};
let within_key_pos_from_center = Vec2 {
x: within_key_pos_normalized.x - 0.5,
y: 0.5 - within_key_pos_normalized.y,
};
let key_dimensions = Vec2 {
x: QwertyKeyboardGrid::get_key_width() as f32,
y: QwertyKeyboardGrid::get_key_width() as f32,
};
let point = within_key_pos_from_center*key_dimensions + key_pos;
let duration = Instant::now().duration_since(start_time);
self.current_swipe.push(SwipePoint::new(point.x.into(), point.y.into(), duration))
}
}
}

4
wgui/src/event.rs
View File

@ -219,6 +219,10 @@ impl CallbackMetadata {
let mouse_pos_abs = self.get_mouse_pos_absolute()?;
Some(mouse_pos_abs - transform_stack.get().abs_pos)
}
pub fn get_mouse_pos_normalized(&self, transform_stack: &TransformStack) -> Option<Vec2> {
let pos_relative = self.get_mouse_pos_relative(transform_stack)?;
Some(pos_relative/transform_stack.parent().raw_dim)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]