RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

implement swipe selection ui in keyboard overlay

This commit is contained in:
Diamond White 2026-03-15 10:43:28 +00:00
parent fb4e4d59ff
commit 74e8c4b588
5 changed files with 259 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
wayvr/src/assets/gui/keyboard.xml
View File

@ -13,6 +13,11 @@
width="${width}" height="${height}" min_width="${width}" min_height="${height}" max_width="${width}" max_height="${height}"
/>
<macro name="prediction_rect"
margin="2" padding="8 16" overflow="visible" box_sizing="border_box"
border_color="~color_accent_translucent" border="2" round="6" color="~color_accent_40" color2="~color_accent_10" gradient="vertical"
align_items="center" justify_content="center" />
<template name="VerticalSeparator">
<rectangle width="2" height="100%" color="~color_accent" />
</template>
@ -90,6 +95,13 @@
</div>
</template>
<!-- Word prediction key for swipe typing -->
<template name="KeyPrediction">
<rectangle id="${id}" macro="prediction_rect" height="60" min_height="60" flex_grow="1" flex_shrink="1" flex_basis="0">
<label text="${text}" size="16" wrap="true" />
</rectangle>
</template>
<macro name="button_style" border="2" border_color="~color_accent_translucent" color="~color_bg" round="6"
align_items="center" justify_content="center" padding="6" width="60" height="60" overflow="visible"/>
@ -232,6 +244,9 @@
</div>
</rectangle>
<div width="100%" height="13" interactable="0" />
<rectangle id="swipe_predictions_root" macro="bg_rect" padding="10" align_items="center" justify_content="space_between" flex_direction="row" min_height="60">
</rectangle>
<div width="100%" height="7" interactable="0" />
<rectangle id="keyboard_root" macro="bg_rect" flex_direction="column" padding="10">
</rectangle>
</div>

179
wayvr/src/overlays/keyboard/builder.rs
View File

@ -1,5 +1,4 @@
use std::{collections::HashMap, rc::Rc, time::Duration};
use crate::{
app_misc,
gui::{
@ -12,8 +11,8 @@ use crate::{
windowing::backend::OverlayEventData,
};
use anyhow::Context;
use dbus::arg::Append;
use glam::{FloatExt, Mat4, Vec2, vec2, vec3};
use slotmap::Key;
use wgui::{
animation::{Animation, AnimationEasing},
assets::AssetPath,
@ -26,7 +25,10 @@ use wgui::{
taffy::{self, prelude::length},
widget::{EventResult, div::WidgetDiv, rectangle::WidgetRectangle},
};
use crate::overlays::keyboard::layout::KeyData;
use wgui::event::StyleSetRequest;
use wgui::layout::LayoutTask;
use wgui::taffy::Display;
use crate::overlays::keyboard::swipe_type::SwipeTypingManager;
use super::{KeyButtonData, KeyState, KeyboardState, handle_press, handle_release, layout::{self, KeyCapType}, handle_mouse_motion};
const PIXELS_PER_UNIT: f32 = 60.;
@ -39,6 +41,141 @@ fn new_doc_params(panel: &mut GuiPanel<KeyboardState>) -> ParseDocumentParams<'s
}
}
pub(super) fn update_swipe_prediction_bar(
panel: &mut GuiPanel<KeyboardState>,
app: &mut AppState
) -> anyhow::Result<bool> {
let mut elements_changed = false;
let (accent_color, anim_mult) = {
let theme = &app.wgui_theme;
(theme.accent_color, theme.animation_mult)
};
let new_swipe_candidates;
if let Some(manager) = panel.state.swipe_typing_manager.as_mut()
&& manager.swipe_candidates != panel.state.swipe_bar_candidates {
panel.state.swipe_bar_candidates = manager.swipe_candidates.clone();
new_swipe_candidates = manager.swipe_candidates.clone();
} else {
return Ok(elements_changed)
}
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if predictions_root.is_null() {
return Ok(elements_changed)
}
let doc_params = new_doc_params(panel);
panel.layout.remove_children(predictions_root);
for (i, prediction) in new_swipe_candidates.iter().enumerate() {
let mut params = HashMap::new();
let id: Rc<str> = Rc::from(format!("Prediction-{i}"));
params.insert("id".into(), id.clone());
params.insert("text".into(), prediction.clone().into());
panel.parser_state.instantiate_template(
&doc_params,
"KeyPrediction",
&mut panel.layout,
predictions_root,
params
)?;
if let Ok(widget_id) = panel.parser_state.get_widget_id(&id) {
let key_state = {
let rect = panel
.layout
.state
.widgets
.get_as::<WidgetRectangle>(widget_id)
.unwrap(); // want panic
Rc::new(KeyState {
// fake button state just so we get key state for anims
button_state: KeyButtonData::Modifier {
modifier: 0,
sticky: core::cell::Cell::new(false),
},
color: rect.params.color,
color2: rect.params.color2,
base_border_color: rect.params.border_color,
cur_border_color: rect.params.border_color.into(),
border: rect.params.border,
drawn_state: false.into(),
})
};
panel.add_event_listener(
widget_id,
EventListenerKind::MousePress,
Box::new({
let k = key_state.clone();
let pred = prediction.clone();
move |common, data, app, state| {
if let Some(manager) = state.swipe_typing_manager.as_mut() {
manager.select_alternate_prediction(&pred, app, state.modifiers);
on_press_anim(k.clone(), common, data)
}
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseEnter,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_enter_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseLeave,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state | {
on_leave_anim(
k.clone(),
common,
data,
accent_color,
anim_mult,
0.0,
);
Ok(EventResult::Pass)
}
})
);
panel.add_event_listener(
widget_id,
EventListenerKind::MouseRelease,
Box::new({
let k = key_state.clone();
move |common, data, _app, _state| {
on_release_anim(k.clone(), common, data);
Ok(EventResult::Pass)
}
})
);
}
}
elements_changed = true;
Ok(elements_changed)
}
#[allow(clippy::too_many_lines, clippy::significant_drop_tightening)]
pub(super) fn create_keyboard_panel(
app: &mut AppState,
@ -331,6 +468,42 @@ pub(super) fn create_keyboard_panel(
elems_changed = true;
}
}
if !app.session.config.swipe_to_type_enabled {
panel.state.swipe_typing_manager = None;
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if !predictions_root.is_null() {
panel.layout.remove_children(predictions_root);
panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
predictions_root,
StyleSetRequest::Display(Display::None),
));
}
}
if app.session.config.swipe_to_type_enabled && panel.state.swipe_typing_manager.is_none() {
panel.state.swipe_typing_manager = match SwipeTypingManager::new() {
Ok(engine) => Some(engine),
Err(e) => {
log::error!("Error occurred while trying to load swipe engine: {:?}", e);
None
}
};
let predictions_root = panel.parser_state
.get_widget_id("swipe_predictions_root")
.unwrap_or_default();
if !predictions_root.is_null() {
panel.layout.tasks.push(LayoutTask::SetWidgetStyle(
predictions_root,
StyleSetRequest::Display(Display::Flex),
));
}
}
}
OverlayEventData::CustomCommand { element, command } => {

34
wayvr/src/overlays/keyboard/mod.rs
View File

@ -27,7 +27,6 @@ use crate::{
},
};
use anyhow::Context;
use arboard::Clipboard;
use glam::{Affine3A, Quat, Vec3, vec3, Vec2};
use regex::Regex;
use slotmap::{SlotMap, new_key_type};
@ -40,7 +39,7 @@ use wlx_common::{
config::AltModifier,
overlays::{BackendAttrib, BackendAttribValue},
};
use codes_iso_639::part_1::LanguageCode;
use crate::overlays::keyboard::builder::update_swipe_prediction_bar;
use crate::overlays::keyboard::layout::KeyCapType;
use crate::overlays::keyboard::swipe_type::SwipeTypingManager;
@ -62,6 +61,7 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
set_list: SetList::default(),
clock_12h: app.session.config.clock_12h,
swipe_typing_manager: None,
swipe_bar_candidates: Vec::new(),
};
let auto_labels = layout.auto_labels.unwrap_or(true);
@ -117,7 +117,7 @@ pub fn create_keyboard(app: &mut AppState, wayland: bool) -> anyhow::Result<Over
transform: Affine3A::from_scale_rotation_translation(
Vec3::ONE * width,
Quat::from_rotation_x(-10f32.to_radians()),
vec3(0.0, -0.65, -0.5),
vec3(0.0, -0.69, -0.5),
),
..OverlayWindowState::default()
},
@ -257,6 +257,13 @@ impl KeyboardBackend {
}
}
fn update_swipe_prediction_bar(&mut self, app: &mut AppState) -> anyhow::Result<()> {
if update_swipe_prediction_bar(self.panel(), app)? {
self.panel().process_custom_elems(app);
}
Ok(())
}
fn auto_switch_keymap(&mut self, app: &mut AppState) -> anyhow::Result<bool> {
let keymap = self.get_effective_keymap()?;
app.hid_provider
@ -290,6 +297,7 @@ impl OverlayBackend for KeyboardBackend {
});
}
}
self.update_swipe_prediction_bar(app)?;
self.panel().should_render(app)
}
fn render(&mut self, app: &mut AppState, rdr: &mut RenderResources) -> anyhow::Result<()> {
@ -352,6 +360,7 @@ struct KeyboardState {
set_list: SetList,
clock_12h: bool,
swipe_typing_manager: Option<SwipeTypingManager>,
swipe_bar_candidates: Vec<String>,
}
macro_rules! take_and_leave_default {
@ -372,6 +381,7 @@ impl KeyboardState {
set_list: SetList::default(),
clock_12h: self.clock_12h,
swipe_typing_manager: None,
swipe_bar_candidates: Vec::new(),
}
}
}
@ -497,13 +507,13 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k
match &key.button_state {
KeyButtonData::Key { vk, pressed } => {
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() && *k_cap_type == KeyCapType::Letter {
if swipe_manager.swipe_left_first_key() {
println!("some");
if swipe_manager.did_swipe_leave_first_key() {
println!("left first");
match swipe_manager.predict() {
Ok(predictions) => {
let best_prediction = predictions.first().unwrap();
println!("best prediction for swipe: {best_prediction}");
swipe_manager.select_word(&best_prediction, app, keyboard.modifiers);
Ok(top_prediction) => {
println!("best prediction for swipe: {top_prediction}");
swipe_manager.select_word(&top_prediction, app, keyboard.modifiers);
},
Err(e) => {
log::error!("{}", e)
@ -511,8 +521,9 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k
}
}
else { // pointer must have been released on the same key it was pressed on
println!("swipe reset");
swipe_manager.reset(); // drop swipe tracking that was started on press
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), *vk, true);
pressed.set(true);
@ -522,6 +533,9 @@ fn handle_release(app: &mut AppState, key: &KeyState, k_cap_type: &KeyCapType, k
}
}
else {
if let Some(swipe_manager) = keyboard.swipe_typing_manager.as_mut() {
swipe_manager.reset();
}
pressed.set(false);
for m in &AUTO_RELEASE_MODS {

55
wayvr/src/overlays/keyboard/swipe_type.rs
View File

@ -1,6 +1,6 @@
use crate::state::AppState;
use crate::subsystem::hid::{KeyModifier, VirtualKey, CTRL};
use anyhow::bail;
use anyhow::{anyhow, bail};
use arboard::Clipboard;
use glam::Vec2;
use std::mem;
@ -9,10 +9,12 @@ use super_swipe_type::keyboard_manager::QwertyKeyboardGrid;
use super_swipe_type::swipe_orchestrator::SwipeOrchestrator;
use super_swipe_type::{SwipeCandidate, SwipePoint};
const PREDICTION_SUGGESTION_COUNT: usize = 4;
pub struct SwipeTypingManager {
swipe_engine: SwipeOrchestrator,
keyboard_gird: QwertyKeyboardGrid,
current_swipe: Vec<SwipePoint>,
pub swipe_candidates: Vec<String>,
swipe_start_time: Option<Instant>,
clipboard: Clipboard,
swipe_left_first_key: bool,
@ -21,6 +23,10 @@ pub struct SwipeTypingManager {
}
impl SwipeTypingManager {
pub fn select_alternate_prediction(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
Self::undo_paste(app, original_keyboard_mods);
self.select_word(word, app, original_keyboard_mods);
}
/// Attempts to "type" the word by copying to clipboard and pasting
pub fn select_word(&mut self, word: &String, app: &mut AppState, original_keyboard_mods: KeyModifier) {
self.last_swiped_word = Some(word.clone());
@ -28,6 +34,16 @@ impl SwipeTypingManager {
Self::paste(app, original_keyboard_mods);
}
}
fn undo_paste(app: &mut AppState, original_keyboard_mods: KeyModifier) {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, true);
app.hid_provider
.send_key_routed(app.wvr_server.as_mut(), VirtualKey::Z, false);
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
}
fn paste(app: &mut AppState, original_keyboard_mods: KeyModifier) {
app.hid_provider
.set_modifiers_routed(app.wvr_server.as_mut(), CTRL);
@ -39,7 +55,6 @@ impl SwipeTypingManager {
.set_modifiers_routed(app.wvr_server.as_mut(), original_keyboard_mods);
}
fn copy_text_to_primary_clipboard(&mut self, text: &str) -> Result<(), arboard::Error> {
println!("{}", std::env::var("WAYLAND_DISPLAY").unwrap());
self.clipboard.set_text(format!("{text} "))
}
pub fn new() -> anyhow::Result<SwipeTypingManager> {
@ -49,6 +64,7 @@ impl SwipeTypingManager {
swipe_engine: SwipeOrchestrator::new()?,
keyboard_gird: QwertyKeyboardGrid::new(), // contains a hashmap<char, vector2> where the vector2 is the center pos of each key in querty
current_swipe: Vec::new(),
swipe_candidates: Vec::new(),
swipe_start_time: None,
clipboard: Clipboard::new()?,
swipe_left_first_key: false,
@ -56,25 +72,41 @@ impl SwipeTypingManager {
last_swiped_word: None
})
}
pub fn predict(&mut self) -> anyhow::Result<Vec<String>>{
pub fn predict(&mut self) -> anyhow::Result<String>{
if self.is_current_swipe_empty() {
bail!("nothing to predict");
} else {
let current_swipe = mem::take(&mut self.current_swipe);
self.reset();
self.reset_swipe();
let candidates: Vec<SwipeCandidate> = self.swipe_engine.predict(current_swipe, &self.last_swiped_word)?;
Ok(candidates.iter().map(|c| c.word.clone()).collect())
let mut iter = candidates.iter();
let top_choice = iter.next().ok_or(anyhow!("No swipe candidates generated"))?.word.clone();
self.swipe_candidates = iter.take(PREDICTION_SUGGESTION_COUNT).map(|c| c.word.clone()).collect();
Ok(top_choice)
}
}
pub fn reset(&mut self) {
self.reset_swipe();
self.last_swiped_word = None;
self.swipe_candidates = Vec::new();
}
fn reset_swipe(&mut self) {
self.swipe_start_time = None;
self.current_swipe = Vec::new();
self.first_swipe_char = char::default();
self.swipe_left_first_key = false;
}
pub fn swipe_left_first_key(&self) -> bool {
fn start_swipe(&mut self, key_label: char) -> Instant{
let now = Instant::now();
self.swipe_start_time = Some(now);
self.first_swipe_char = key_label.to_ascii_lowercase();
self.swipe_candidates = Vec::new();
now
}
pub fn did_swipe_leave_first_key(&self) -> bool {
self.swipe_left_first_key
}
pub fn is_current_swipe_empty(&self) -> bool {
@ -85,17 +117,12 @@ impl SwipeTypingManager {
if self.first_swipe_char != char::default() && self.first_swipe_char != key_label.to_ascii_lowercase() {
self.swipe_left_first_key = true;
}
let key_pos = Vec2{x: pos.x as f32, y: pos.y as f32 };
println!("char : {key_label} at pos: {key_pos}");
//println!("char : {key_label} at pos: {key_pos}");
let start_time = match self.swipe_start_time {
Some(time) => time,
None => {
// this is the first swipe entry
let now = Instant::now();
self.swipe_start_time = Some(now);
self.first_swipe_char = key_label.to_ascii_lowercase();
now
}
None => self.start_swipe(key_label)
};
let within_key_pos_from_center = Vec2 {

3
wlx-common/src/config.rs
View File

@ -327,4 +327,7 @@ pub struct GeneralConfig {
#[serde(default = "def_one")]
pub grid_opacity: f32,
#[serde(default = "def_true")]
pub swipe_to_type_enabled: bool,
}