From 82f53e66681a0222b151799730303deed32db47c Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Sun, 4 Feb 2024 14:23:28 +0100 Subject: [PATCH] rework interactions --- README.md | 12 ++- src/backend/common.rs | 5 +- src/backend/input.rs | 39 +++++---- src/backend/openvr/lines.rs | 3 +- src/backend/openvr/mod.rs | 4 + src/backend/openxr/lines.rs | 1 + src/backend/openxr/mod.rs | 4 + src/backend/overlay.rs | 19 +++- src/overlays/keyboard.rs | 7 +- src/overlays/mod.rs | 1 + src/overlays/screen.rs | 2 + src/overlays/toast.rs | 34 ++++++++ src/overlays/watch.rs | 168 +++++++++++++++++++++--------------- src/res/config.yaml | 2 - src/state.rs | 2 - 15 files changed, 201 insertions(+), 102 deletions(-) create mode 100644 src/overlays/toast.rs diff --git a/README.md b/README.md index f99aaf73..de7e0003 100644 --- a/README.md +++ b/README.md @@ -34,13 +34,21 @@ cargo run --release You'll see a screen and keyboard. You can turn these on and off using the watch on your left wrist. -Right click: Touch (do not push down) the B/Y button on your controller to get a YELLOW laser, and pull the trigger. +Right click: Touch (do not push down) the B/Y button on your controller to get a ORANGE laser, and pull the trigger. Middle click: Same as right click, but A/X to get a PURPLE laser. Move screen: Point your laser on the screen and then grab using grip. Adjust distance using stick up/down while gripping. -Resize screen: While grabbing, touch B/Y to get a YELLOW laser and use stick up/down (this is wonky, I'll come up with a better one!) +Resize screen: While grabbing, pull trigger to get a RED laser and use stick up/down + +Reset size/position: Click the button corresponding to the screen or keyboard on your watch, hold for 3s, then release. + +Show/hide: Quickly hide or show your selection of screens by double-tapping B/Y on your left controller. + +Lock a screen in place: On your non-watch hand, touch B/Y to get an ORANGE laser and click the screen's button on your watch. You will no longer be able to grab the screen, it will not re-center in front of you when show, nor it will react to the show/hide shortcut. + +Make a screen non-clickable: On your non-watch hand, touch A/X to get a PURPLE laser and click the screen's button on your watch. You will no longer get a laser when pointing to that screen. Repeat to toggle back off. # Known Issues diff --git a/src/backend/common.rs b/src/backend/common.rs index 183ba7c5..97d19357 100644 --- a/src/backend/common.rs +++ b/src/backend/common.rs @@ -11,6 +11,7 @@ use crate::{ overlays::{ keyboard::create_keyboard, screen::{get_screens_wayland, get_screens_x11}, + toast::Toast, watch::create_watch, }, state::AppState, @@ -106,7 +107,7 @@ where self.overlays.values_mut().for_each(|o| { if o.state.show_hide { o.state.want_visible = !any_shown; - if o.state.want_visible && app.session.recenter_on_show { + if o.state.want_visible && o.state.recenter { o.state.reset(app, false); } } @@ -114,6 +115,7 @@ where } } +#[derive(Clone)] pub enum OverlaySelector { Id(usize), Name(Arc), @@ -147,6 +149,7 @@ pub enum TaskType { OverlaySelector, Box, ), + Toast(Toast), } pub struct TaskContainer { diff --git a/src/backend/input.rs b/src/backend/input.rs index 3fa42665..b7e7c124 100644 --- a/src/backend/input.rs +++ b/src/backend/input.rs @@ -239,6 +239,7 @@ pub enum PointerMode { Left, Right, Middle, + Special, } pub fn interact( @@ -371,7 +372,7 @@ impl Pointer { let mut hits: SmallVec<[RayHit; 8]> = smallvec!(); for overlay in overlays.iter() { - if !overlay.state.want_visible { + if !overlay.state.want_visible || !overlay.state.interactable { continue; } @@ -435,30 +436,32 @@ impl Pointer { O: Default, { if self.now.grab { - if self.now.click && !self.before.click { - log::warn!("todo: click-while-grabbed"); - } - let grab_data = self.interaction.grabbed.as_mut().unwrap(); - match self.interaction.mode { - PointerMode::Left => { - grab_data.offset.z -= self.now.scroll * 0.05; - } - _ => { - overlay.state.transform.matrix3 = overlay - .state - .transform - .matrix3 - .mul_scalar(1.0 + 0.01 * self.now.scroll); + if self.now.click { + self.interaction.mode = PointerMode::Special; + let cur_scale = overlay.state.transform.x_axis.length(); + if cur_scale < 0.1 && self.now.scroll > 0.0 { + return; + } else if cur_scale > 20. && self.now.scroll < 0.0 { + return; } + + overlay.state.transform.matrix3 = overlay + .state + .transform + .matrix3 + .mul_scalar(1.0 - 0.025 * self.now.scroll); + } else { + grab_data.offset.z -= self.now.scroll * 0.05; } overlay.state.transform.translation = self.pose.transform_point3a(grab_data.offset); overlay.state.realign(hmd); overlay.state.dirty = true; } else { - overlay.state.spawn_point = hmd - .inverse() - .transform_point3a(overlay.state.transform.translation); + overlay.state.saved_point = Some( + hmd.inverse() + .transform_point3a(overlay.state.transform.translation), + ); self.interaction.grabbed = None; log::info!("Hand {}: dropped {}", self.idx, overlay.state.name); } diff --git a/src/backend/openvr/lines.rs b/src/backend/openvr/lines.rs index 6eb90997..5f505210 100644 --- a/src/backend/openvr/lines.rs +++ b/src/backend/openvr/lines.rs @@ -21,7 +21,7 @@ static AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(1); pub(super) struct LinePool { lines: IdMap>, view: Arc, - colors: [Vec4; 4], + colors: [Vec4; 5], } impl LinePool { @@ -52,6 +52,7 @@ impl LinePool { Vec4::new(0., 0.375, 0.5, 1.), Vec4::new(0.69, 0.188, 0., 1.), Vec4::new(0.375, 0., 0.5, 1.), + Vec4::new(1., 0., 0., 1.), ], } } diff --git a/src/backend/openvr/mod.rs b/src/backend/openvr/mod.rs index 85143448..301060e9 100644 --- a/src/backend/openvr/mod.rs +++ b/src/backend/openvr/mod.rs @@ -140,6 +140,10 @@ pub fn openvr_run(running: Arc) -> Result<(), BackendError> { f(&mut state, &mut o.state); } } + TaskType::Toast(t) => { + // TODO toasts + log::info!("Toast: {} {}", t.title, t.body); + } } } diff --git a/src/backend/openxr/lines.rs b/src/backend/openxr/lines.rs index f4d63193..53b7212b 100644 --- a/src/backend/openxr/lines.rs +++ b/src/backend/openxr/lines.rs @@ -36,6 +36,7 @@ impl LinePool { [0x00, 0x60, 0x80, 0xff], [0xB0, 0x30, 0x00, 0xff], [0x60, 0x00, 0x80, 0xff], + [0xff, 0x00, 0x00, 0xff], ]; let views = colors diff --git a/src/backend/openxr/mod.rs b/src/backend/openxr/mod.rs index ec212ce8..8183b172 100644 --- a/src/backend/openxr/mod.rs +++ b/src/backend/openxr/mod.rs @@ -186,6 +186,10 @@ pub fn openxr_run(running: Arc) -> Result<(), BackendError> { f(&mut app_state, &mut o.state); } } + TaskType::Toast(t) => { + // TODO toasts + log::info!("Toast: {} {}", t.title, t.body); + } } } diff --git a/src/backend/overlay.rs b/src/backend/overlay.rs index c0ea5a73..ae5a04f1 100644 --- a/src/backend/overlay.rs +++ b/src/backend/overlay.rs @@ -24,8 +24,11 @@ pub struct OverlayState { pub want_visible: bool, pub show_hide: bool, pub grabbable: bool, + pub interactable: bool, + pub recenter: bool, pub dirty: bool, pub transform: Affine3A, + pub saved_point: Option, pub spawn_scale: f32, // aka width pub spawn_point: Vec3A, pub spawn_rotation: Quat, @@ -43,8 +46,11 @@ impl Default for OverlayState { want_visible: false, show_hide: false, grabbable: false, + recenter: false, + interactable: false, dirty: true, relative_to: RelativeTo::None, + saved_point: None, spawn_scale: 1.0, spawn_point: Vec3A::NEG_Z, spawn_rotation: Quat::IDENTITY, @@ -90,26 +96,31 @@ impl OverlayState { pub fn auto_movement(&mut self, app: &mut AppState) { if let Some(parent) = self.parent_transform(app) { + let point = self.saved_point.unwrap_or(self.spawn_point); self.transform = parent * Affine3A::from_scale_rotation_translation( Vec3::ONE * self.spawn_scale, self.spawn_rotation * Quat::from_rotation_x(f32::to_radians(-180.0)) * Quat::from_rotation_z(f32::to_radians(180.0)), - self.spawn_point.into(), + point.into(), ); self.dirty = true; } } - pub fn reset(&mut self, app: &mut AppState, reset_scale: bool) { - let translation = app.input_state.hmd.transform_point3a(self.spawn_point); - let scale = if reset_scale { + pub fn reset(&mut self, app: &mut AppState, hard_reset: bool) { + let scale = if hard_reset { + self.saved_point = None; self.spawn_scale } else { self.transform.x_axis.length() }; + + let point = self.saved_point.unwrap_or(self.spawn_point); + + let translation = app.input_state.hmd.transform_point3a(point); self.transform = Affine3A::from_scale_rotation_translation( Vec3::ONE * scale, Quat::IDENTITY, diff --git a/src/overlays/keyboard.rs b/src/overlays/keyboard.rs index acee2783..4671b1d5 100644 --- a/src/overlays/keyboard.rs +++ b/src/overlays/keyboard.rs @@ -3,7 +3,6 @@ use std::{ io::Cursor, process::{Child, Command}, str::FromStr, - sync::Arc, }; use crate::{ @@ -26,6 +25,8 @@ const PIXELS_PER_UNIT: f32 = 80.; const BUTTON_PADDING: f32 = 4.; const AUTO_RELEASE_MODS: [KeyModifier; 5] = [SHIFT, CTRL, ALT, SUPER, META]; +pub const KEYBOARD_NAME: &str = "kbd"; + pub fn create_keyboard(app: &AppState) -> OverlayData where O: Default, @@ -116,9 +117,11 @@ where OverlayData { state: OverlayState { - name: Arc::from("kbd"), + name: KEYBOARD_NAME.into(), size: (size.x as _, size.y as _), grabbable: true, + recenter: true, + interactable: true, spawn_scale: width, spawn_point: vec3a(0., -0.5, -1.), interaction_transform, diff --git a/src/overlays/mod.rs b/src/overlays/mod.rs index 01e06979..e246d7b8 100644 --- a/src/overlays/mod.rs +++ b/src/overlays/mod.rs @@ -1,3 +1,4 @@ pub mod keyboard; pub mod screen; +pub mod toast; pub mod watch; diff --git a/src/overlays/screen.rs b/src/overlays/screen.rs index cda6edc2..ab6ddd35 100644 --- a/src/overlays/screen.rs +++ b/src/overlays/screen.rs @@ -378,6 +378,8 @@ where size, want_visible: session.show_screens.iter().any(|s| s == &*output.name), grabbable: true, + recenter: true, + interactable: true, spawn_scale: 1.5 * session.config.desktop_view_scale, spawn_point: vec3a(0., 0.5, -1.), spawn_rotation: Quat::from_axis_angle(axis, angle), diff --git a/src/overlays/toast.rs b/src/overlays/toast.rs new file mode 100644 index 00000000..1067258c --- /dev/null +++ b/src/overlays/toast.rs @@ -0,0 +1,34 @@ +use std::sync::Arc; + +pub struct Toast { + pub title: Arc, + pub body: Arc, + pub opacity: f32, + pub timeout: f32, + pub sound: bool, +} + +#[allow(dead_code)] +impl Toast { + pub fn new(title: Arc, body: Arc) -> Self { + Toast { + title, + body, + opacity: 1.0, + timeout: 3.0, + sound: false, + } + } + pub fn with_timeout(mut self, timeout: f32) -> Self { + self.timeout = timeout; + self + } + pub fn with_opacity(mut self, opacity: f32) -> Self { + self.opacity = opacity; + self + } + pub fn with_sound(mut self) -> Self { + self.sound = true; + self + } +} diff --git a/src/overlays/watch.rs b/src/overlays/watch.rs index 20594f1e..8d2a72b9 100644 --- a/src/overlays/watch.rs +++ b/src/overlays/watch.rs @@ -6,12 +6,15 @@ use glam::{vec2, Affine2}; use crate::{ backend::{ common::{OverlaySelector, TaskType}, + input::PointerMode, overlay::{OverlayData, OverlayState, RelativeTo}, }, - gui::{color_parse, CanvasBuilder}, + gui::{color_parse, CanvasBuilder, Control}, state::AppState, }; +use super::{keyboard::KEYBOARD_NAME, toast::Toast}; + pub const WATCH_NAME: &str = "watch"; pub fn create_watch(state: &AppState, screens: &[OverlayData]) -> OverlayData @@ -79,44 +82,21 @@ where let button_width = 360. / num_buttons as f32; let mut button_x = 40.; - let keyboard = canvas.button(button_x + 2., 162., button_width - 4., 36., "kbd".into()); + let keyboard = canvas.button( + button_x + 2., + 162., + button_width - 4., + 36., + KEYBOARD_NAME.into(), + ); keyboard.state = Some(WatchButtonState { pressed_at: Instant::now(), - scr_idx: 0, + overlay: OverlaySelector::Name(KEYBOARD_NAME.into()), + mode: PointerMode::Left, }); - keyboard.on_press = Some(|control, _data, _app, _| { - if let Some(state) = control.state.as_mut() { - state.pressed_at = Instant::now(); - } - }); - keyboard.on_release = Some(|control, _data, app| { - if let Some(state) = control.state.as_ref() { - if Instant::now() - .saturating_duration_since(state.pressed_at) - .as_millis() - < 2000 - { - app.tasks.enqueue(TaskType::Overlay( - OverlaySelector::Name("kbd".into()), - Box::new(|app, o| { - o.show_hide = !o.show_hide; - o.want_visible = o.show_hide; - if app.session.recenter_on_show { - o.reset(app, false); - } - }), - )); - } else { - app.tasks.enqueue(TaskType::Overlay( - OverlaySelector::Name("kbd".into()), - Box::new(|app, o| { - o.reset(app, true); - }), - )); - } - } - }); + keyboard.on_press = Some(overlay_button_dn); + keyboard.on_release = Some(overlay_button_up); button_x += button_width; canvas.bg_color = color_parse("#405060"); @@ -131,42 +111,12 @@ where ); button.state = Some(WatchButtonState { pressed_at: Instant::now(), - scr_idx: screen.state.id, + overlay: OverlaySelector::Id(screen.state.id), + mode: PointerMode::Left, }); - button.on_press = Some(|control, _data, _app, _| { - if let Some(state) = control.state.as_mut() { - state.pressed_at = Instant::now(); - } - }); - button.on_release = Some(|control, _data, app| { - if let Some(state) = control.state.as_ref() { - let scr_idx = state.scr_idx; - if Instant::now() - .saturating_duration_since(state.pressed_at) - .as_millis() - < 2000 - { - app.tasks.enqueue(TaskType::Overlay( - OverlaySelector::Id(scr_idx), - Box::new(|app, o| { - o.show_hide = !o.show_hide; - o.want_visible = o.show_hide; - if app.session.recenter_on_show { - o.reset(app, false); - } - }), - )); - } else { - app.tasks.enqueue(TaskType::Overlay( - OverlaySelector::Id(scr_idx), - Box::new(|app, o| { - o.reset(app, true); - }), - )); - } - } - }); + button.on_press = Some(overlay_button_dn); + button.on_release = Some(overlay_button_up); button_x += button_width; } let interaction_transform = @@ -179,6 +129,7 @@ where name: WATCH_NAME.into(), size: (400, 200), want_visible: true, + interactable: true, spawn_scale: 0.11 * state.session.config.watch_scale, spawn_point: state.session.watch_pos.into(), spawn_rotation: state.session.watch_rot, @@ -193,5 +144,82 @@ where struct WatchButtonState { pressed_at: Instant, - scr_idx: usize, + mode: PointerMode, + overlay: OverlaySelector, +} + +fn overlay_button_dn( + control: &mut Control<(), WatchButtonState>, + _: &mut (), + _: &mut AppState, + mode: PointerMode, +) { + if let Some(state) = control.state.as_mut() { + state.pressed_at = Instant::now(); + state.mode = mode; + } +} + +fn overlay_button_up(control: &mut Control<(), WatchButtonState>, _: &mut (), app: &mut AppState) { + if let Some(state) = control.state.as_ref() { + let selector = state.overlay.clone(); + if Instant::now() + .saturating_duration_since(state.pressed_at) + .as_millis() + < 2000 + { + match state.mode { + PointerMode::Left => { + app.tasks.enqueue(TaskType::Overlay( + selector, + Box::new(|app, o| { + o.want_visible = !o.want_visible; + if o.recenter { + o.show_hide = o.want_visible; + o.reset(app, false); + } + }), + )); + } + PointerMode::Right => { + app.tasks.enqueue(TaskType::Overlay( + selector, + Box::new(|app, o| { + o.recenter = !o.recenter; + o.grabbable = o.recenter; + o.show_hide = o.recenter; + if !o.recenter { + app.tasks.enqueue(TaskType::Toast(Toast::new( + format!("{} is now locked in place!", o.name).into(), + "Right-click again to toggle.".into(), + ))) + } + }), + )); + } + PointerMode::Middle => { + app.tasks.enqueue(TaskType::Overlay( + selector, + Box::new(|app, o| { + o.interactable = !o.interactable; + if !o.interactable { + app.tasks.enqueue(TaskType::Toast(Toast::new( + format!("{} is now non-interactable!", o.name).into(), + "Middle-click again to toggle.".into(), + ))) + } + }), + )); + } + _ => {} + } + } else { + app.tasks.enqueue(TaskType::Overlay( + selector, + Box::new(|app, o| { + o.reset(app, true); + }), + )); + } + } } diff --git a/src/res/config.yaml b/src/res/config.yaml index 1b3634f9..523a5f47 100644 --- a/src/res/config.yaml +++ b/src/res/config.yaml @@ -3,8 +3,6 @@ # Default: 300 click_freeze_time_ms: 300 -recenter_on_show: true - # Default: true keyboard_sound_enabled: true diff --git a/src/state.rs b/src/state.rs index 856b5b5d..493f9044 100644 --- a/src/state.rs +++ b/src/state.rs @@ -77,7 +77,6 @@ pub struct AppSession { pub config: GeneralConfig, pub show_screens: Vec, - pub recenter_on_show: bool, pub watch_hand: usize, pub watch_pos: Vec3, @@ -103,7 +102,6 @@ impl AppSession { config_root_path, config, show_screens: vec!["DP-3".to_string()], - recenter_on_show: true, capture_method: "auto".to_string(), primary_hand: 1, watch_hand: 0,