RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: swipe to type (#451) 

* start to integrate swipetype with keyboardstate

* successful debug implementation

* change to git dependency

* add manual trigger to app image build to test

* use latest swipeType lib to test including binaries

* use clipboard to paste swipe output

* use new prediction engine, move swipe type related logic to its own module

* update appimage bulid to support building ort runtime

* implement swipe selection ui in keyboard overlay

* only add motion data for the device that started the swipe

* process swipes on separate thread to not block ui

* add toggle to settings.rs

* default swipe to type config to false

* remove println, minor fix

* use correct key_dimensions

* use latest swipe_type library version

* optionally use wayvr compositor clipboard when pasting

* remove unused imports

* fix typo in log::error

* fix within key positioning

* run swipes on AltGr keycaps
This commit is contained in:
Diamond White 2026-04-10 00:46:22 -05:00 committed by GitHub
parent 18f99b5daf
commit db1675d399
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
18 changed files with 1758 additions and 65 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
.github/workflows/build-appimage.yml vendored
View File

@ -1,6 +1,7 @@
name: Build AppImage
on:
workflow_dispatch:
push:
branches:
- 'main'
@ -15,7 +16,7 @@ env:
jobs:
build_appimage:
runs-on: ubuntu-22.04
runs-on: ubuntu-24.04
defaults:
run:
working-directory: ./wayvr

1099
Cargo.lock generated
View File

File diff suppressed because it is too large Load Diff

2
dash-frontend/assets/lang/en.json
View File

@ -63,6 +63,8 @@
"KEYBOARD_MIDDLE_CLICK": "Keyboard middle click",
"KEYBOARD_MIDDLE_CLICK_HELP": "Modifier to use when typing\nwith purple laser",
"KEYBOARD_SOUND_ENABLED": "Keyboard sounds",
"KEYBOARD_SWIPE_TO_TYPE_ENABLED": "Keyboard swipe to type",
"KEYBOARD_SWIPE_TO_TYPE_ENABLED_HELP": "Only works for English! Results will be drastically worse on non qwerty keyboards.",
"LANGUAGE": "Language",
"LEFT_HANDED_MOUSE": "Left-handed mouse",
"LEFT_HANDED_MOUSE_HELP": "Use this if mouse buttons are swapped",

4
dash-frontend/src/tab/settings/mod.rs
View File

@ -231,6 +231,7 @@ enum SettingType {
InvertScrollDirectionY,
KeyboardMiddleClick,
KeyboardSoundEnabled,
KeyboardSwipeToTypeEnabled,
Language,
LeftHandedMouse,
LongPressDuration,
@ -283,6 +284,7 @@ impl SettingType {
Self::HideUsername => &mut config.hide_username,
Self::OpaqueBackground => &mut config.opaque_background,
Self::XwaylandByDefault => &mut config.xwayland_by_default,
Self::KeyboardSwipeToTypeEnabled => &mut config.keyboard_swipe_to_type_enabled,
_ => panic!("Requested bool for non-bool SettingType"),
}
}
@ -381,6 +383,7 @@ impl SettingType {
Self::InvertScrollDirectionY => Ok("APP_SETTINGS.INVERT_SCROLL_DIRECTION_Y"),
Self::KeyboardMiddleClick => Ok("APP_SETTINGS.KEYBOARD_MIDDLE_CLICK"),
Self::KeyboardSoundEnabled => Ok("APP_SETTINGS.KEYBOARD_SOUND_ENABLED"),
Self::KeyboardSwipeToTypeEnabled => Ok("APP_SETTINGS.KEYBOARD_SWIPE_TO_TYPE_ENABLED"),
Self::Language => Ok("APP_SETTINGS.LANGUAGE"),
Self::LeftHandedMouse => Ok("APP_SETTINGS.LEFT_HANDED_MOUSE"),
Self::LongPressDuration => Ok("APP_SETTINGS.LONG_PRESS_DURATION"),
@ -417,6 +420,7 @@ impl SettingType {
Self::GridOpacity => Some("APP_SETTINGS.GRID_OPACITY_HELP"),
Self::HandsfreePointer => Some("APP_SETTINGS.HANDSFREE_POINTER_HELP"),
Self::KeyboardMiddleClick => Some("APP_SETTINGS.KEYBOARD_MIDDLE_CLICK_HELP"),
Self::KeyboardSwipeToTypeEnabled => Some("APP_SETTINGS.KEYBOARD_SWIPE_TO_TYPE_ENABLED_HELP"),
Self::LeftHandedMouse => Some("APP_SETTINGS.LEFT_HANDED_MOUSE_HELP"),
Self::ScreenRenderDown => Some("APP_SETTINGS.SCREEN_RENDER_DOWN_HELP"),
Self::UprightScreenFix => Some("APP_SETTINGS.UPRIGHT_SCREEN_FIX_HELP"),

1
dash-frontend/src/tab/settings/tab_features.rs
View File

@ -9,6 +9,7 @@ pub fn mount(mp: &mut MacroParams, parent: WidgetID) -> anyhow::Result<()> {
options_checkbox(mp, c, SettingType::NotificationsEnabled)?;
options_checkbox(mp, c, SettingType::NotificationsSoundEnabled)?;
options_checkbox(mp, c, SettingType::KeyboardSoundEnabled)?;
options_checkbox(mp, c, SettingType::KeyboardSwipeToTypeEnabled)?;
options_checkbox(mp, c, SettingType::SpaceDragUnlocked)?;
options_checkbox(mp, c, SettingType::SpaceRotateUnlocked)?;
options_slider_f32(mp, c, SettingType::SpaceDragMultiplier, -10.0, 10.0, 0.5)?;

2
wayvr/Cargo.toml
View File

@ -90,6 +90,8 @@ xcb = { version = "1.6.0", features = [
"as-raw-xcb-connection",
], optional = true }
xkbcommon = { version = "0.8.0" } # 0.9.0 breaks keymap import on some distros
arboard = { version="3.6.1", features = ["wayland-data-control", "wl-clipboard-rs"] }
super-swipe-type = "0.2.2"
[build-dependencies]
regex.workspace = true

17
wayvr/src/assets/gui/keyboard.xml
View File

@ -13,6 +13,11 @@
width="${width}" height="${height}" min_width="${width}" min_height="${height}" max_width="${width}" max_height="${height}"
/>
<macro name="prediction_rect"
margin="2" padding="8 16" overflow="visible" box_sizing="border_box"
border_color="~color_accent_translucent" border="2" round="6" color="~color_accent_40" color2="~color_accent_10" gradient="vertical"
align_items="center" justify_content="center" />
<template name="VerticalSeparator">
<rectangle width="2" height="100%" color="~color_accent" />
</template>
@ -90,6 +95,13 @@
</div>
</template>
<!-- Word prediction key for swipe typing -->
<template name="KeyPrediction">
<rectangle id="${id}" macro="prediction_rect" height="60" min_height="60" flex_grow="1" flex_shrink="1" flex_basis="0">
<label text="${text}" size="16" flex_wrap="wrap" />
</rectangle>
</template>
<macro name="button_style" border="2" border_color="~color_accent_translucent" color="~color_bg" round="6"
align_items="center" justify_content="center" padding="6" width="60" height="60" overflow="visible"/>
@ -231,7 +243,10 @@
</div>
</div>
</rectangle>
<div width="100%" height="13" interactable="0" />
<div width="100%" height="11" interactable="0" />
<rectangle id="swipe_predictions_root" macro="bg_rect" padding="10" align_items="center" justify_content="space_between" flex_direction="row" min_height="60">
</rectangle>
<div width="100%" height="11" interactable="0" />
<rectangle id="keyboard_root" macro="bg_rect" flex_direction="column" padding="10">
</rectangle>
</div>

15
wayvr/src/backend/wayvr/client.rs
View File

@ -7,10 +7,12 @@ use smithay::{
reexports::wayland_server,
utils::SerialCounter,
};
use smithay::input::Seat;
use smithay::wayland::selection::data_device::set_data_device_selection;
use xkbcommon::xkb;
use crate::backend::wayvr::{ExternalProcessRequest, WayVRTask};
use crate::backend::wayvr::comp::Application;
use super::{
ProcessWayVREnv,
comp::{self, ClientState},
@ -26,6 +28,7 @@ pub struct WayVRCompositor {
pub state: comp::Application,
pub seat_keyboard: KeyboardHandle<comp::Application>,
pub seat_pointer: PointerHandle<comp::Application>,
pub seat: Seat<comp::Application>,
pub serial_counter: SerialCounter,
pub wayland_env: super::WaylandEnv,
@ -68,6 +71,7 @@ impl WayVRCompositor {
display: wayland_server::Display<comp::Application>,
seat_keyboard: KeyboardHandle<comp::Application>,
seat_pointer: PointerHandle<comp::Application>,
seat: Seat<comp::Application>,
) -> anyhow::Result<Self> {
let (wayland_env, listener) = create_wayland_listener()?;
@ -81,6 +85,7 @@ impl WayVRCompositor {
serial_counter: SerialCounter::new(),
clients: Vec::new(),
toplevel_surf_count: 0,
seat,
})
}
@ -200,6 +205,14 @@ impl WayVRCompositor {
);
}
pub fn set_clipboard_text(&mut self, text: String) {
set_data_device_selection::<Application>(
&self.state.display_handle,
&self.seat,
vec!["text/plain;charset=utf-8".to_string(), "text/plain".to_string()],
text.as_bytes().into(),
);
}
pub fn set_keymap(&mut self, keymap: &xkb::Keymap) -> anyhow::Result<()> {
// Smithay only accepts keymaps in a string form due to thread safety concerns
self.seat_keyboard

5
wayvr/src/backend/wayvr/mod.rs
View File

@ -241,7 +241,7 @@ impl WvrServerState {
};
Ok(Self {
manager: client::WayVRCompositor::new(state, display, seat_keyboard, seat_pointer)?,
manager: client::WayVRCompositor::new(state, display, seat_keyboard, seat_pointer, seat)?,
processes: ProcessVec::new(),
wm: window::WindowManager::new(),
ticks: 0,
@ -608,6 +608,9 @@ impl WvrServerState {
pub fn send_key(&mut self, virtual_key: u32, down: bool) {
self.manager.send_key(virtual_key, down);
}
pub fn set_clipboard_text(&mut self, text: String) {
self.manager.set_clipboard_text(text);
}
pub fn set_keymap(&mut self, keymap: &xkb::Keymap) -> anyhow::Result<()> {
self.manager.set_keymap(keymap)

2
wayvr/src/config.rs
View File

@ -140,6 +140,7 @@ pub struct AutoSettings {
pub context_menu_hold_and_release: bool,
pub capture_method: CaptureMethod,
pub keyboard_middle_click_mode: AltModifier,
pub keyboard_swipe_to_type_enabled: bool,
pub autostart_apps: Vec<WvrProcessLaunchParams>,
pub handsfree_pointer: HandsfreePointer,
pub language: Option<Language>,
@ -191,6 +192,7 @@ pub fn save_settings(config: &GeneralConfig) -> anyhow::Result<()> {
context_menu_hold_and_release: config.context_menu_hold_and_release,
capture_method: config.capture_method,
keyboard_middle_click_mode: config.keyboard_middle_click_mode,
keyboard_swipe_to_type_enabled: config.keyboard_swipe_to_type_enabled,
autostart_apps: config.autostart_apps.clone(),
handsfree_pointer: config.handsfree_pointer,
language: config.language,

1
wayvr/src/main.rs
View File

@ -281,6 +281,7 @@ fn logging_init(args: &mut Args) {
.from_env_lossy()
.add_directive("symphonia_core::probe=warn".parse().unwrap())
.add_directive("symphonia_bundle_mp3=warn".parse().unwrap())
.add_directive("ort=warn".parse().unwrap())
.add_directive("zbus=warn".parse().unwrap())
.add_directive("usvg=error".parse().unwrap())
.add_directive("resvg=error".parse().unwrap())

221
wayvr/src/overlays/keyboard/builder.rs
View File

@ -1,5 +1,4 @@
use std::{collections::HashMap, rc::Rc, time::Duration};
use crate::{
app_misc,
gui::{
@ -11,8 +10,9 @@ use crate::{
subsystem::hid::XkbKeymap,
windowing::backend::OverlayEventData,
};
use anyhow::Context;
use anyhow::{bail, Context};
use glam::{FloatExt, Mat4, Vec2, vec2, vec3};
use slotmap::Key;
use wgui::{
animation::{Animation, AnimationEasing},
assets::AssetPath,
@ -25,11 +25,10 @@ use wgui::{
taffy::{self, prelude::length},
widget::{EventResult, div::WidgetDiv, rectangle::WidgetRectangle},
};
use super::{
KeyButtonData, KeyState, KeyboardState, handle_press, handle_release,
layout::{self, KeyCapType},
};
use wgui::event::StyleSetRequest;
use wgui::layout::LayoutTask;
use wgui::taffy::Display;
use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion, init_swipe_type_manager};
const PIXELS_PER_UNIT: f32 = 60.;
@ -41,6 +40,147 @@ fn new_doc_params(panel: &mut GuiPanel<KeyboardState>) -> ParseDocumentParams<'s
}
}
pub(super) fn update_swipe_prediction_bar(
panel: &mut GuiPanel<KeyboardState>,
app: &mut AppState
) -> anyhow::Result<bool> {
let mut elements_changed = false;
let (accent_color, anim_mult) = {
let theme = &app.wgui_theme;
(theme.accent_color, theme.animation_mult)
};
if let Some(recv) = panel.state.swipe_candidate_receiver.as_mut()
&& let Ok(candidates) = recv.try_recv() {
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if predictions_root.is_null() {
return Ok(elements_changed)
}
let doc_params = new_doc_params(panel);
panel.layout.remove_children(predictions_root);
let Some(new_suggestions) = candidates else {
return Ok(elements_changed)
};
let mut iter = new_suggestions.iter();
let Some(best_prediction) = iter.next() else {
bail!("not enough swipe predictions");
};
if let Some(manager) = panel.state.swipe_typing_manager.as_mut() {
manager.select_word(best_prediction, app, panel.state.modifiers);
}
for (i, prediction) in iter.enumerate() {
let mut params = HashMap::new();
let id: Rc<str> = Rc::from(format!("Prediction-{i}"));
params.insert("id".into(), id.clone());
params.insert("text".into(), prediction.clone().into());
panel.parser_state.instantiate_template(
&doc_params,
"KeyPrediction",
&mut panel.layout,
predictions_root,
params
)?;
if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) {
let key_state = {
let rect = panel
.layout
.state
.widgets
.get_as::<WidgetRectangle>(widget_id)
.unwrap(); // want panic
Rc::new(KeyState {
// fake button state just so we get key state for anims
button_state: KeyButtonData::Modifier {
modifier: 0,
sticky: core::cell::Cell::new(false),
},
color: rect.params.color,
color2: rect.params.color2,
base_border_color: rect.params.border_color,
cur_border_color: rect.params.border_color.into(),
border: rect.params.border,
drawn_state: false.into(),
})
};
panel.add_event_listener(
widget_id,
EventListenerKind::MousePress,
Box::new({
let k = key_state.clone();
let pred = prediction.clone();
move |common, data, app, state| {
if let Some(manager) = state.swipe_typing_manager.as_mut() {
manager.select_alternate_prediction(&pred, app, state.modifiers);
on_press_anim(k.clone(), common, data)
}
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseEnter,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_enter_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseLeave,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state | {
on_leave_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseRelease,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_release_anim(k.clone(), common, data);
Ok(EventResult::Pass)
}
})
);
}
}
elements_changed = true;
}
Ok(elements_changed)
}
#[allow(clippy::too_many_lines, clippy::significant_drop_tightening)]
pub(super) fn create_keyboard_panel(
app: &mut AppState,
@ -113,7 +253,7 @@ pub(super) fn create_keyboard_panel(
params.insert(Rc::from("width"), Rc::from(key_width.to_string()));
params.insert(Rc::from("height"), Rc::from(key_height.to_string()));
let mut label = key.label.into_iter();
let mut label = key.label.clone().into_iter();
label
.next()
.and_then(|s| params.insert("text".into(), s.into()));
@ -169,6 +309,9 @@ pub(super) fn create_keyboard_panel(
})
};
let key_cap_type: Rc<KeyCapType> = Rc::from(key.cap_type);
let key_label: Rc<Vec<String>> = Rc::from(key.label);
let width_mul = 1. / my_size_f32;
panel.add_event_listener(
@ -186,6 +329,8 @@ pub(super) fn create_keyboard_panel(
anim_mult,
width_mul,
);
Ok(EventResult::Pass)
}
}),
@ -205,6 +350,7 @@ pub(super) fn create_keyboard_panel(
anim_mult,
width_mul,
);
Ok(EventResult::Pass)
}
}),
@ -214,26 +360,49 @@ pub(super) fn create_keyboard_panel(
EventListenerKind::MousePress,
Box::new({
let k = key_state.clone();
let k_label = key_label.clone();
let k_cap_type = key_cap_type.clone();
move |common, data, app, state| {
let CallbackMetadata::MouseButton(button) = data.metadata else {
panic!("CallbackMetadata should contain MouseButton!");
};
let within_key_pos = data.metadata.get_mouse_pos_normalized(&common.alterables.transform_stack);
handle_press(app, &k, state, button);
handle_press(app, &k, &k_label, &k_cap_type, &within_key_pos, state, button, button.device);
on_press_anim(k.clone(), common, data);
Ok(EventResult::Pass)
}
}),
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseMotion,
Box::new({
let k = key_state.clone();
let k_label = key_label.clone();
let k_cap_type = key_cap_type.clone();
move |common, data, _app, state| {
let within_key_pos = data.metadata.get_mouse_pos_normalized(&common.alterables.transform_stack);
let CallbackMetadata::MousePosition(position) = data.metadata else {
panic!("CallbackMetadata should contain MousePosition!");
};
handle_mouse_motion(&k, &k_label, &k_cap_type, state, &within_key_pos, position.device);
Ok(EventResult::Pass)
}
}),
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseRelease,
Box::new({
let k = key_state.clone();
let k_cap_type = key_cap_type.clone();
move |common, data, app, state| {
if handle_release(app, &k, state) {
if handle_release(app, &k, &k_cap_type, state) {
on_release_anim(k.clone(), common, data);
}
Ok(EventResult::Pass)
}
}),
@ -307,6 +476,38 @@ pub(super) fn create_keyboard_panel(
elems_changed = true;
}
}
if !app.session.config.keyboard_swipe_to_type_enabled {
panel.state.swipe_typing_manager = None;
panel.state.swipe_candidate_receiver = None;
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if !predictions_root.is_null() {
panel.layout.remove_children(predictions_root);
panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
predictions_root,
StyleSetRequest::Display(Display::None),
));
}
}
if app.session.config.keyboard_swipe_to_type_enabled && panel.state.swipe_typing_manager.is_none() {
init_swipe_type_manager(&mut panel.state);
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if !predictions_root.is_null() {
panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
predictions_root,
StyleSetRequest::Display(Display::Flex),
));
}
}
}
OverlayEventData::CustomCommand { element, command } => {

3
wayvr/src/overlays/keyboard/layout.rs
View File

@ -2,7 +2,6 @@ use std::{collections::HashMap, str::FromStr, sync::LazyLock};
use regex::Regex;
use serde::{Deserialize, Serialize};
use crate::{
config::{ConfigType, load_known_yaml},
subsystem::hid::{
@ -230,7 +229,7 @@ pub(super) struct KeyData {
pub(super) cap_type: KeyCapType,
}
#[derive(Debug)]
#[derive(Debug, Eq, PartialEq)]
pub enum KeyCapType {
/// Label an SVG
Special,

201
wayvr/src/overlays/keyboard/mod.rs
View File

@ -4,7 +4,7 @@ use std::{
process::{Child, Command},
sync::atomic::Ordering,
};
use std::sync::mpsc::Receiver;
use crate::{
KEYMAP_CHANGE,
backend::{
@ -27,21 +27,29 @@ use crate::{
},
};
use anyhow::Context;
use glam::{Affine3A, Quat, Vec3, vec3};
use glam::{Affine3A, Quat, Vec3, vec3, Vec2};
use regex::Regex;
use slotmap::{SlotMap, new_key_type};
use slotmap::{SlotMap, new_key_type, Key};
use wgui::{
drawing,
event::{InternalStateChangeEvent, MouseButtonEvent, MouseButtonIndex},
};
use wgui::event::StyleSetRequest;
use wgui::layout::{LayoutTask};
use wgui::parser::Fetchable;
use wgui::taffy::Display;
use wlx_common::windowing::{OverlayWindowState, Positioning};
use wlx_common::{
config::AltModifier,
overlays::{BackendAttrib, BackendAttribValue},
};
use crate::overlays::keyboard::builder::update_swipe_prediction_bar;
use crate::overlays::keyboard::layout::KeyCapType;
use crate::overlays::keyboard::swipe_type::SwipeTypingManager;
pub mod builder;
mod layout;
mod swipe_type;
pub const KEYBOARD_NAME: &str = "kbd";
const AUTO_RELEASE_MODS: [KeyModifier; 5] = [SHIFT, CTRL, ALT, SUPER, META];
@ -56,6 +64,8 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
overlay_list: OverlayList::default(),
set_list: SetList::default(),
clock_12h: app.session.config.clock_12h,
swipe_typing_manager: None,
swipe_candidate_receiver: None,
};
let auto_labels = layout.auto_labels.unwrap_or(true);
@ -111,14 +121,35 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
transform: Affine3A::from_scale_rotation_translation(
Vec3::ONE * width,
Quat::from_rotation_x(-10f32.to_radians()),
vec3(0.0, -0.65, -0.5),
vec3(0.0, -0.69, -0.5),
),
..OverlayWindowState::default()
},
..OverlayWindowConfig::from_backend(Box::new(backend))
})
}
pub(self) fn init_swipe_type_manager(state: &mut KeyboardState) {
match SwipeTypingManager::new() {
Ok((engine, receiver)) => {
state.swipe_typing_manager = Some(engine);
state.swipe_candidate_receiver = Some(receiver);
},
Err(e) => {
log::error!("Error occurred while trying to load swipe engine: {}", e);
}
};
}
pub(self) fn hide_swipe_type_manager(panel: &mut GuiPanel<KeyboardState>) {
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if !predictions_root.is_null() {
panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
predictions_root,
StyleSetRequest::Display(Display::None),
));
}
}
const fn alt_modifier_to_key(m: AltModifier) -> KeyModifier {
match m {
AltModifier::Shift => SHIFT,
@ -150,8 +181,19 @@ impl KeyboardBackend {
keymap: Option<&XkbKeymap>,
app: &mut AppState,
) -> anyhow::Result<KeyboardPanelKey> {
let panel =
create_keyboard_panel(app, keymap, self.default_state.take(), &self.wlx_layout)?;
let mut state = self.default_state.take();
if app.session.config.keyboard_swipe_to_type_enabled {
init_swipe_type_manager(&mut state);
}
log::info!("swipe engine created");
let mut panel =
create_keyboard_panel(app, keymap, state, &self.wlx_layout)?;
if !app.session.config.keyboard_swipe_to_type_enabled {
hide_swipe_type_manager(&mut panel);
}
let id = self.layout_panels.insert(panel);
if let Some(layout_name) = keymap.and_then(|k| k.get_name()) {
@ -162,6 +204,7 @@ impl KeyboardBackend {
Ok(id)
}
fn switch_keymap(&mut self, keymap: &XkbKeymap, app: &mut AppState) -> anyhow::Result<bool> {
if !self.wlx_layout.auto_labels.unwrap_or(true) {
return Ok(false);
@ -176,30 +219,41 @@ impl KeyboardBackend {
if self.active_layout.eq(new_key) {
return Ok(false);
}
self.internal_switch_keymap(*new_key);
self.internal_switch_keymap(*new_key, app);
} else {
let new_key = self.add_new_keymap(Some(keymap), app)?;
self.internal_switch_keymap(new_key);
self.internal_switch_keymap(new_key, app);
}
app.tasks
.enqueue(TaskType::Overlay(OverlayTask::KeyboardChanged));
Ok(true)
}
fn internal_switch_keymap(&mut self, new_key: KeyboardPanelKey) {
let state_from = self
fn internal_switch_keymap(&mut self, new_key: KeyboardPanelKey, app: &AppState) {
let mut state_from = self
.layout_panels
.get_mut(self.active_layout)
.unwrap()
.state
.take();
if app.session.config.keyboard_swipe_to_type_enabled {
init_swipe_type_manager(&mut state_from);
}
self.active_layout = new_key;
self.layout_panels
.get_mut(self.active_layout)
.unwrap()
.state = state_from;
if !app.session.config.keyboard_swipe_to_type_enabled {
hide_swipe_type_manager(self.layout_panels
.get_mut(self.active_layout)
.unwrap()
)
}
}
fn get_effective_keymap(&mut self) -> anyhow::Result<XkbKeymap> {
@ -233,6 +287,13 @@ impl KeyboardBackend {
}
}
fn update_swipe_prediction_bar(&mut self, app: &mut AppState) -> anyhow::Result<()> {
if update_swipe_prediction_bar(self.panel(), app)? {
self.panel().process_custom_elems(app);
}
Ok(())
}
fn auto_switch_keymap(&mut self, app: &mut AppState) -> anyhow::Result<bool> {
let keymap = self.get_effective_keymap()?;
app.hid_provider
@ -266,6 +327,7 @@ impl OverlayBackend for KeyboardBackend {
});
}
}
self.update_swipe_prediction_bar(app)?;
self.panel().should_render(app)
}
fn render(&mut self, app: &mut AppState, rdr: &mut RenderResources) -> anyhow::Result<()> {
@ -327,6 +389,8 @@ struct KeyboardState {
overlay_list: OverlayList,
set_list: SetList,
clock_12h: bool,
swipe_typing_manager: Option<SwipeTypingManager>,
swipe_candidate_receiver: Option<Receiver<Option<Vec<String>>>>
}
macro_rules! take_and_leave_default {
@ -346,6 +410,8 @@ impl KeyboardState {
overlay_list: OverlayList::default(),
set_list: SetList::default(),
clock_12h: self.clock_12h,
swipe_typing_manager: None,
swipe_candidate_receiver: None,
}
}
}
@ -386,26 +452,70 @@ enum KeyButtonData {
},
}
fn handle_mouse_motion(
key: &KeyState,
key_label: &Vec<String>,
key_cap_type: &KeyCapType,
keyboard: &mut KeyboardState,
within_key_pos: &Option<Vec2>,
device: usize
) {
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut()
&& matches!(*key_cap_type, KeyCapType::Letter | KeyCapType::LetterAltGr)
{
if !swipe_manager.is_current_swipe_empty() {
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
if let Some(pos) = within_key_pos {
// check because mouse motion can trigger despite hover being false
if pos.x >= 0.0 && pos.x <= 1.0 && pos.y >= 0.0 && pos.y <= 1.0 {
if let Some(label) = key_label.first() {
swipe_manager.add_swipe(pos, label.chars().next().unwrap_or_default(), device);
}
}
}
}
_ => {}
}
}
}
}
fn handle_press(
app: &mut AppState,
key: &KeyState,
key_label: &Vec<String>,
key_cap_type: &KeyCapType,
within_key_pos: &Option<Vec2>,
keyboard: &mut KeyboardState,
button: MouseButtonEvent,
device: usize
) {
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
keyboard.modifiers |= match button.index {
MouseButtonIndex::Right => SHIFT,
MouseButtonIndex::Middle => keyboard.alt_modifier,
_ => 0,
};
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, true);
pressed.set(true);
play_key_click(app);
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut()
&& matches!(*key_cap_type, KeyCapType::Letter | KeyCapType::LetterAltGr)
{
if let Some(pos) = within_key_pos {
if let Some(label) = key_label.first() {
swipe_manager.add_swipe(pos, label.chars().next().unwrap_or_default(), device);
}
}
}
else {
keyboard.modifiers |= match button.index {
MouseButtonIndex::Right => SHIFT,
MouseButtonIndex::Middle => keyboard.alt_modifier,
_ => 0,
};
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, true);
pressed.set(true);
play_key_click(app);
}
}
KeyButtonData::Modifier { modifier, sticky } => {
sticky.set(keyboard.modifiers & *modifier == 0);
@ -435,20 +545,47 @@ fn handle_press(
}
}
fn handle_release(app: &mut AppState, key: &KeyState, keyboard: &mut KeyboardState) -> bool {
fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, keyboard: &mut KeyboardState) -> bool {
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
pressed.set(false);
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut()
&& matches!(*k_cap_type, KeyCapType::Letter | KeyCapType::LetterAltGr)
{
if swipe_manager.did_swipe_leave_first_key() {
match swipe_manager.predict() {
Ok(()) => {},
Err(e) => {
log::error!("{}", e)
}
}
}
else { // pointer must have been released on the same key it was pressed on
swipe_manager.reset(); // drop swipe tracking that was started on press
for m in &AUTO_RELEASE_MODS {
if keyboard.modifiers & *m != 0 {
keyboard.modifiers &= !*m;
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, true);
pressed.set(true);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, false);
play_key_click(app);
}
}
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, false);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
else {
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() {
swipe_manager.reset();
}
pressed.set(false);
for m in &AUTO_RELEASE_MODS {
if keyboard.modifiers & *m != 0 {
keyboard.modifiers &= !*m;
}
}
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, false);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), keyboard.modifiers);
}
true
}
KeyButtonData::Modifier { modifier, sticky } => {

238
wayvr/src/overlays/keyboard/swipe_type.rs Normal file
View File

@ -0,0 +1,238 @@
use crate::state::AppState;
use crate::subsystem::hid::{KeyModifier, VirtualKey, CTRL};
use anyhow::{bail};
use arboard::Clipboard;
use glam::Vec2;
use std::mem;
use std::sync::mpsc::{sync_channel, Receiver, SyncSender, channel, Sender};
use std::thread::{self, JoinHandle};
use std::time::Instant;
use super_swipe_type::keyboard_manager::QwertyKeyboardGrid;
use super_swipe_type::swipe_orchestrator::SwipeOrchestrator;
use super_swipe_type::{SwipePoint};
use crate::subsystem::input::KeyboardFocus;
const PREDICTION_SUGGESTION_COUNT: usize = 5;
enum PredictionTask {
Predict {
swipe: Vec<SwipePoint>,
last_word: Option<String>,
},
Shutdown,
}
pub struct SwipeTypingManager {
keyboard_gird: QwertyKeyboardGrid,
current_swipe: Vec<SwipePoint>,
swipe_candidate_sender: SyncSender<Option<Vec<String>>>,
prediction_task_sender: Sender<PredictionTask>,
worker_thread: Option<JoinHandle<()>>,
swipe_start_time: Option<Instant>,
clipboard: Clipboard,
swipe_left_first_key: bool,
first_swipe_char: char,
current_swipe_device: Option<usize>,
last_swiped_word: Option<String>,
}
impl SwipeTypingManager {
pub fn select_alternate_prediction(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
Self::undo(app, original_keyboard_mods);
self.select_word(word, app, original_keyboard_mods);
}
pub fn select_word(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
self.last_swiped_word = Some(word.clone());
let text_to_paste = format!("{word} ");
match app.hid_provider.keyboard_focus {
KeyboardFocus::PhysicalScreen => {
if let Ok(_) = self.clipboard.set_text(text_to_paste) {
Self::paste(app, original_keyboard_mods);
}
},
KeyboardFocus::WayVR => {
if let Some(wvr_server) = app.wvr_server.as_mut() {
wvr_server.set_clipboard_text(text_to_paste);
Self::paste(app, original_keyboard_mods);
}
},
}
}
fn undo(app: &mut AppState, original_keyboard_mods: KeyModifier) {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, true);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, false);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
}
fn paste(app: &mut AppState, original_keyboard_mods: KeyModifier) {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::V, true);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::V, false);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
}
pub fn new() -> anyhow::Result<(SwipeTypingManager, Receiver<Option<Vec<String>>>)> {
let (candidate_sender, candidate_receiver) = sync_channel(1);
let (task_sender, task_receiver) = channel::<PredictionTask>();
// Spawn persistent worker thread
let worker_candidate_sender = candidate_sender.clone();
let worker_thread = thread::spawn(move || {
let mut swipe_engine = match SwipeOrchestrator::new() {
Ok(engine) => engine,
Err(e) => {
log::error!("Failed to initialize SwipeOrchestrator: {}", e);
return;
}
};
while let Ok(task) = task_receiver.recv() {
match task {
PredictionTask::Predict { swipe, last_word } => {
match swipe_engine.predict(swipe, &last_word) {
Ok(candidates) => {
let words: Vec<String> = candidates
.into_iter()
.take(PREDICTION_SUGGESTION_COUNT)
.map(|c| c.word)
.collect();
let _ = worker_candidate_sender.send(Some(words));
}
Err(e) => {
log::error!("Prediction failed: {}", e);
}
}
}
PredictionTask::Shutdown => break,
}
}
});
Ok((
Self {
keyboard_gird: QwertyKeyboardGrid::new(),
current_swipe: Vec::new(),
swipe_candidate_sender: candidate_sender,
prediction_task_sender: task_sender,
worker_thread: Some(worker_thread),
swipe_start_time: None,
clipboard: Clipboard::new()?,
swipe_left_first_key: false,
first_swipe_char: char::default(),
current_swipe_device: None,
last_swiped_word: None,
},
candidate_receiver,
))
}
pub fn predict(&mut self) -> anyhow::Result<()> {
if self.is_current_swipe_empty() {
bail!("nothing to predict");
}
let current_swipe = mem::take(&mut self.current_swipe);
let last_word = self.last_swiped_word.clone();
self.reset_swipe();
self.prediction_task_sender
.send(PredictionTask::Predict {
swipe: current_swipe,
last_word,
})?;
Ok(())
}
pub fn reset(&mut self) {
self.reset_swipe();
let _ = self.swipe_candidate_sender.send(None);
self.last_swiped_word = None;
}
fn reset_swipe(&mut self) {
self.swipe_start_time = None;
self.current_swipe = Vec::new();
self.first_swipe_char = char::default();
self.swipe_left_first_key = false;
self.current_swipe_device = None;
}
fn start_swipe(&mut self, key_label: char, device: usize) -> Instant {
let now = Instant::now();
self.swipe_start_time = Some(now);
self.first_swipe_char = key_label.to_ascii_lowercase();
self.current_swipe_device = Some(device);
now
}
pub fn did_swipe_leave_first_key(&self) -> bool {
self.swipe_left_first_key
}
pub fn is_current_swipe_empty(&self) -> bool {
self.current_swipe.is_empty()
}
pub fn add_swipe(&mut self, within_key_pos_normalized: &Vec2, key_label: char, device: usize) {
if let Some(pos) = self.keyboard_gird.key_positions.get(&key_label.to_ascii_lowercase()) {
if let Some(current_device) = self.current_swipe_device {
if current_device != device {
return;
}
}
if self.first_swipe_char != char::default()
&& self.first_swipe_char != key_label.to_ascii_lowercase()
{
self.swipe_left_first_key = true;
}
let key_pos = Vec2 {
x: pos.x as f32,
y: pos.y as f32,
};
let start_time = match self.swipe_start_time {
Some(time) => time,
None => self.start_swipe(key_label, device),
};
let within_key_pos_from_center = Vec2 {
x: within_key_pos_normalized.x - 0.5,
y: within_key_pos_normalized.y - 0.5,
};
let key_dimensions = Vec2 {
x: QwertyKeyboardGrid::get_key_width() as f32,
y: QwertyKeyboardGrid::get_key_height() as f32,
};
let point = within_key_pos_from_center * key_dimensions + key_pos;
let duration = Instant::now().duration_since(start_time).mul_f32(0.8); // multiply by .8 because library is trained on mobile swipes which happen on a smaller keyboard and are faster
self.current_swipe
.push(SwipePoint::new(point.x.into(), point.y.into(), duration))
}
}
}
impl Drop for SwipeTypingManager {
fn drop(&mut self) {
let _ = self.prediction_task_sender.send(PredictionTask::Shutdown);
if let Some(handle) = self.worker_thread.take() {
let _ = handle.join();
}
}
}

2
wayvr/src/subsystem/hid/mod.rs
View File

@ -253,6 +253,7 @@ impl HidProvider for UInputProvider {
log::error!("send_key: {res}");
}
}
fn set_desktop_extent(&mut self, extent: Vec2) {
self.desktop_extent = extent;
}
@ -337,6 +338,7 @@ pub const META: KeyModifier = 0x80;
#[allow(non_camel_case_types)]
#[repr(u16)]
#[derive(Debug, Deserialize, PartialEq, Eq, Clone, Copy, IntegerId, EnumString, EnumIter)]
#[derive(Hash)]
pub enum VirtualKey {
Escape = 9,
N1, // number row

4
wgui/src/event.rs
View File

@ -215,6 +215,10 @@ impl CallbackMetadata {
let mouse_pos_abs = self.get_mouse_pos_absolute()?;
Some(mouse_pos_abs - transform_stack.get().abs_pos)
}
pub fn get_mouse_pos_normalized(&self, transform_stack: &TransformStack) -> Option<Vec2> {
let pos_relative = self.get_mouse_pos_relative(transform_stack)?;
Some(pos_relative/transform_stack.parent().raw_dim)
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]

3
wlx-common/src/config.rs
View File

@ -327,4 +327,7 @@ pub struct GeneralConfig {
#[serde(default = "def_one")]
pub grid_opacity: f32,
#[serde(default = "def_false")]
pub keyboard_swipe_to_type_enabled: bool,
}