From 341ee333dc255e1c68c0ef4e534dcb61a6fd94e3 Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Fri, 3 Jul 2026 22:26:17 +0900 Subject: [PATCH] whisper transcription working --- Cargo.lock | 12 + dash-frontend/src/tab/settings/mod.rs | 7 +- .../src/tab/settings/tab_features.rs | 8 +- dash-frontend/src/util/whisper.rs | 6 - wayvr/Cargo.toml | 1 + wayvr/src/assets/gui/whisper.xml | 36 ++ wayvr/src/assets/icons/mic.svg | 1 + wayvr/src/assets/icons/paste-go.svg | 1 + wayvr/src/assets/lang/en.json | 7 + wayvr/src/backend/wayvr/comp.rs | 12 +- wayvr/src/backend/wayvr/mod.rs | 5 + wayvr/src/gui/panel/overlay_list.rs | 2 +- wayvr/src/overlays/mod.rs | 2 + wayvr/src/overlays/whisper.rs | 256 ++++++++++++++ wayvr/src/subsystem/clipboard/mod.rs | 8 + wayvr/src/subsystem/clipboard/wl.rs | 23 ++ wayvr/src/subsystem/clipboard/x11.rs | 329 ++++++++++++++++++ wayvr/src/subsystem/mod.rs | 3 + wayvr/src/windowing/manager.rs | 7 + wayvr/src/windowing/window.rs | 3 + wlx-capture/src/wayland.rs | 1 + 21 files changed, 720 insertions(+), 10 deletions(-) create mode 100644 wayvr/src/assets/gui/whisper.xml create mode 100644 wayvr/src/assets/icons/mic.svg create mode 100644 wayvr/src/assets/icons/paste-go.svg create mode 100644 wayvr/src/overlays/whisper.rs create mode 100644 wayvr/src/subsystem/clipboard/mod.rs create mode 100644 wayvr/src/subsystem/clipboard/wl.rs create mode 100644 wayvr/src/subsystem/clipboard/x11.rs diff --git a/Cargo.lock b/Cargo.lock index 641fbb61..8db3d3b1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5447,6 +5447,17 @@ dependencies = [ "xkeysym", ] +[[package]] +name = "smithay-clipboard" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71704c03f739f7745053bde45fa203a46c58d25bc5c4efba1d9a60e9dba81226" +dependencies = [ + "libc", + "smithay-client-toolkit 0.20.0", + "wayland-backend", +] + [[package]] name = "smol" version = "2.0.2" @@ -6633,6 +6644,7 @@ dependencies = [ "slotmap", "smallvec", "smithay", + "smithay-clipboard", "smol", "strum", "sysinfo", diff --git a/dash-frontend/src/tab/settings/mod.rs b/dash-frontend/src/tab/settings/mod.rs index dfef32b0..56fcacfb 100644 --- a/dash-frontend/src/tab/settings/mod.rs +++ b/dash-frontend/src/tab/settings/mod.rs @@ -112,6 +112,7 @@ trait SettingsTab { &mut self, _action: Rc, _config: &mut GeneralConfig, + _change_kind: &mut Option, _layout: &mut Layout, _state: &mut ParserState, ) -> anyhow::Result<()> { @@ -250,7 +251,11 @@ impl Tab for TabSettings { && let Some(tab) = self.current_tab.as_mut() { let config = frontend.interface.general_config(data); - tab.context_menu_custom(name, config, &mut frontend.layout, &mut self.state)?; + let mut change_kind : Option = None; + tab.context_menu_custom(name, config, &mut change_kind, &mut frontend.layout, &mut self.state)?; + if let Some(change_kind) = change_kind { + frontend.interface.config_changed(data, change_kind); + } } else if let (Some(setting), Some(id), Some(value), Some(text), Some(translated)) = { let mut s = name.splitn(5, ';'); (s.next(), s.next(), s.next(), s.next(), s.next()) diff --git a/dash-frontend/src/tab/settings/tab_features.rs b/dash-frontend/src/tab/settings/tab_features.rs index c6d0d01b..75c8db7f 100644 --- a/dash-frontend/src/tab/settings/tab_features.rs +++ b/dash-frontend/src/tab/settings/tab_features.rs @@ -11,7 +11,7 @@ use wgui::{ widget::label::WidgetLabel, windowing::context_menu::{self}, }; -use wlx_common::{async_executor::AsyncExecutor, config::GeneralConfig}; +use wlx_common::{async_executor::AsyncExecutor, config::GeneralConfig, dash_interface::ConfigChangeKind}; use crate::{ frontend::FrontendTasks, @@ -61,6 +61,7 @@ impl SettingsTab for State { if !whisper_model_path(model.file_name).exists() { // download failed, set to selection to none par.general_config.whisper_model = "".into(); + par.config_change_kind.replace(ConfigChangeKind::Other); // reload the tab self.parent_tasks.push(ParentTask::SetTab(TabNameEnum::Features)); } @@ -76,6 +77,7 @@ impl SettingsTab for State { Task::WhisperDownloadDone => { if let Some(model) = self.pending_download.take() { par.general_config.whisper_model = model.file_name.into(); + par.config_change_kind.replace(ConfigChangeKind::Other); // reload tab so that the downloaded checkmarks get populated self.parent_tasks.push(ParentTask::SetTab(TabNameEnum::Features)); @@ -98,6 +100,7 @@ impl SettingsTab for State { &mut self, action: Rc, config: &mut GeneralConfig, + change_kind: &mut Option, layout: &mut wgui::layout::Layout, state: &mut wgui::parser::ParserState, ) -> anyhow::Result<()> { @@ -115,9 +118,12 @@ impl SettingsTab for State { config.whisper_model = model.file_name.into(); if !whisper_model_path(model.file_name).exists() { self.show_whisper_model_dialog_box_download(model)?; + } else { + change_kind.replace(ConfigChangeKind::Other); } } else { config.whisper_model = "".into(); + change_kind.replace(ConfigChangeKind::Other); // re-init the whole tab if whisper_any_models_downloaded().unwrap_or_default() { self.show_whisper_model_dialog_box_cleanup()?; diff --git a/dash-frontend/src/util/whisper.rs b/dash-frontend/src/util/whisper.rs index 823f5b07..c6920268 100644 --- a/dash-frontend/src/util/whisper.rs +++ b/dash-frontend/src/util/whisper.rs @@ -6,7 +6,6 @@ pub struct WhisperModel { pub file_name: &'static str, pub display_name: &'static str, pub url: &'static str, - pub hash: &'static str, } pub const WHISPER_MODELS: &[WhisperModel] = &[ @@ -14,31 +13,26 @@ pub const WHISPER_MODELS: &[WhisperModel] = &[ file_name: "ggml-base-q8_0.bin", display_name: "Base Q8 (78MiB)", url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q8_0.bin", - hash: "7bb89bb49ed6955013b166f1b6a6c04584a20fbe", }, WhisperModel { file_name: "ggml-small-q8_0.bin", display_name: "Small Q8 (252MiB)", url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q8_0.bin", - hash: "bcad8a2083f4e53d648d586b7dbc0cd673d8afad", }, WhisperModel { file_name: "ggml-large-v3-turbo-q5_0.bin", display_name: "Turbo Q5 (574MiB)", url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin", - hash: "e050f7970618a659205450ad97eb95a18d69c9ee", }, WhisperModel { file_name: "ggml-large-v3-turbo-q8_0.bin", display_name: "Turbo Q8 (874MiB)", url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q8_0.bin", - hash: "01bf15bedffe9f39d65c1b6ff9b687ea91f59e0e", }, WhisperModel { file_name: "ggml-large-v3-turbo.bin", display_name: "Turbo (1.5GiB)", url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin", - hash: "4af2b29d7ec73d781377bfd1758ca957a807e941", }, ]; diff --git a/wayvr/Cargo.toml b/wayvr/Cargo.toml index a4bdf1ac..087d9393 100644 --- a/wayvr/Cargo.toml +++ b/wayvr/Cargo.toml @@ -93,6 +93,7 @@ xcb = { version = "1.6.0", features = [ ], optional = true } xkbcommon = { version = "0.8.0" } # 0.9.0 breaks keymap import on some distros whisper-rs = { version = "0.16.0", features = ["vulkan"], optional = true } +smithay-clipboard = "0.7.3" [build-dependencies] regex.workspace = true diff --git a/wayvr/src/assets/gui/whisper.xml b/wayvr/src/assets/gui/whisper.xml new file mode 100644 index 00000000..28218b87 --- /dev/null +++ b/wayvr/src/assets/gui/whisper.xml @@ -0,0 +1,36 @@ + + + + + + +
+ +
+ + +
+
+ + +
+ +
+ +
+
+ +
+ + + diff --git a/wayvr/src/assets/icons/mic.svg b/wayvr/src/assets/icons/mic.svg new file mode 100644 index 00000000..809dfe32 --- /dev/null +++ b/wayvr/src/assets/icons/mic.svg @@ -0,0 +1 @@ + diff --git a/wayvr/src/assets/icons/paste-go.svg b/wayvr/src/assets/icons/paste-go.svg new file mode 100644 index 00000000..c9f49c44 --- /dev/null +++ b/wayvr/src/assets/icons/paste-go.svg @@ -0,0 +1 @@ + diff --git a/wayvr/src/assets/lang/en.json b/wayvr/src/assets/lang/en.json index b3a3070a..dc4b7ee0 100644 --- a/wayvr/src/assets/lang/en.json +++ b/wayvr/src/assets/lang/en.json @@ -112,5 +112,12 @@ "RECENTER": "Recenter playspace", "SWITCH_TO_SET": "Switch to set", "TOGGLE_FOR_CURRENT_SET": "Toggle for current set" + }, + "WHISPER": { + "TRANSCRIBE": "Press and hold to transcribe text", + "PASTE": "Paste transcription to focused overlay", + "UNLOAD": "Unload model from VRAM", + "MODEL_NOT_DOWNLOADED": "Speech-to-text not set up!", + "DOWNLOAD_GUIDANCE": "Set up speech-to-text under settings, features!" } } diff --git a/wayvr/src/backend/wayvr/comp.rs b/wayvr/src/backend/wayvr/comp.rs index 1a80a1ec..6b630628 100644 --- a/wayvr/src/backend/wayvr/comp.rs +++ b/wayvr/src/backend/wayvr/comp.rs @@ -42,7 +42,7 @@ use smithay::wayland::compositor::{self, BufferAssignment, SurfaceAttributes, se use smithay::wayland::selection::data_device::{ ClientDndGrabHandler, DataDeviceHandler, DataDeviceState, ServerDndGrabHandler, - set_data_device_focus, + set_data_device_focus, set_data_device_selection, }; use smithay::wayland::selection::{self, SelectionHandler}; use smithay::wayland::shell::xdg::{ @@ -63,6 +63,7 @@ pub struct Application { pub dmabuf_state: (DmabufState, DmabufGlobal, Option), pub compositor: compositor::CompositorState, pub xdg_shell: XdgShellState, + pub seat: Seat, pub seat_state: SeatState, pub shm: ShmState, pub data_device: DataDeviceState, @@ -81,6 +82,15 @@ impl Application { self.image_importer.cleanup(); } + pub fn set_clipboard_text(&mut self, content: &str) { + let mime_types = vec![ + "text/plain;charset=utf-8".to_string(), + "text/plain".to_string(), + ]; + let user_data = Arc::from(content.as_bytes()); + set_data_device_selection(&self.display_handle, &self.seat, mime_types, user_data); + } + fn popups_commit(&mut self, surface: &WlSurface) { self.popup_manager.commit(surface); diff --git a/wayvr/src/backend/wayvr/mod.rs b/wayvr/src/backend/wayvr/mod.rs index 5d83f02b..b33f68df 100644 --- a/wayvr/src/backend/wayvr/mod.rs +++ b/wayvr/src/backend/wayvr/mod.rs @@ -227,6 +227,7 @@ impl WvrServerState { display_handle: dh, compositor, xdg_shell, + seat, seat_state, shm, data_device, @@ -626,6 +627,10 @@ impl WvrServerState { self.cur_modifiers = modifiers; } + pub fn set_clipboard(&mut self, content: &str) { + self.manager.state.set_clipboard_text(content); + } + pub fn add_external_process(&mut self, pid: u32) -> process::ProcessHandle { self.processes .add(process::Process::External(process::ExternalProcess { pid })) diff --git a/wayvr/src/gui/panel/overlay_list.rs b/wayvr/src/gui/panel/overlay_list.rs index 329165ad..3601fe54 100644 --- a/wayvr/src/gui/panel/overlay_list.rs +++ b/wayvr/src/gui/panel/overlay_list.rs @@ -59,7 +59,7 @@ impl OverlayList { ); ("Mirror", panels_root) } - OverlayCategory::Panel => { + OverlayCategory::Panel | OverlayCategory::BuiltInPanel => { let icon: Rc = if let Some(icon) = meta.icon.as_ref() { icon.to_string().into() } else { diff --git a/wayvr/src/overlays/mod.rs b/wayvr/src/overlays/mod.rs index e499de1c..b8d8a9e4 100644 --- a/wayvr/src/overlays/mod.rs +++ b/wayvr/src/overlays/mod.rs @@ -7,3 +7,5 @@ pub mod screen; pub mod toast; pub mod watch; pub mod wayvr; +#[cfg(feature = "whisper")] +pub mod whisper; diff --git a/wayvr/src/overlays/whisper.rs b/wayvr/src/overlays/whisper.rs new file mode 100644 index 00000000..b94a1466 --- /dev/null +++ b/wayvr/src/overlays/whisper.rs @@ -0,0 +1,256 @@ +use std::{path::Path, rc::Rc, time::Duration}; + +use glam::{Affine3A, Quat, Vec3, vec3}; +use wgui::{ + components::button::ComponentButton, + event::EventCallback, + i18n::Translation, + log::LogErr, + parser::Fetchable, + widget::{EventResult, label::WidgetLabel}, +}; +use wlx_common::{ + data_dir, + overlays::{BackendAttrib, BackendAttribValue, ToastTopic}, + windowing::{OverlayWindowState, Positioning}, +}; + +use crate::{ + gui::{ + panel::{ + GuiPanel, NewGuiPanelParams, OnCustomAttribFunc, + button::{BUTTON_EVENT_SUFFIX, BUTTON_EVENTS}, + }, + timer::GuiTimer, + }, + overlays::toast::Toast, + state::AppState, + subsystem::{ + clipboard::{ClipboardProvider, wl::WlClipboardProvider, x11::X11ClipboardProvider}, + hid::VirtualKey, + input::KeyboardFocus, + whisper_stt::WhisperStt, + }, + windowing::window::{OverlayCategory, OverlayWindowConfig}, +}; + +pub const WHISPER_NAME: &str = "Speech-to-Text"; + +#[derive(Default)] +struct WhisperState { + whisper_sst: Option, + clipboard_provider: Option>, + last_transcription: Option>, +} + +pub fn create_whisper( + app: &mut AppState, + headless: bool, + wayland: bool, +) -> anyhow::Result { + // let clipboard_provider: Option> = match (headless, wayland) { + // (true, _) => None, + // (false, true) => WlClipboardProvider::new() + // .log_err("Could not create Wayland clipboard provider") + // .ok() + // .map(|p| Box::new(p) as Box), + // (false, false) => X11ClipboardProvider::new() + // .log_err("Could not create X11 clipboard provider") + // .ok() + // .map(|p| Box::new(p) as Box), + // }; + + let state = WhisperState { + clipboard_provider: None, + ..Default::default() + }; + let xml = "gui/whisper.xml"; + + let on_custom_attrib: OnCustomAttribFunc = Box::new(move |layout, parser, attribs, _app| { + let Ok(button) = + parser.fetch_component_from_widget_id_as::(attribs.widget_id) + else { + return; + }; + + for (name, kind, test_button, test_duration) in &BUTTON_EVENTS { + for suffix in BUTTON_EVENT_SUFFIX { + let name = &format!("{name}{suffix}"); + let Some(action) = attribs.get_value(name) else { + break; + }; + + let mut args = action.split_whitespace(); + let Some(command) = args.next() else { + continue; + }; + + let button = button.clone(); + + let callback: EventCallback = match command { + "::WhisperTranscribeStart" => Box::new(move |_common, data, app, state| { + if !test_button(data) || !test_duration(&button, app) { + return Ok(EventResult::Pass); + } + + if let Some(whisper) = state.whisper_sst.as_mut() { + let _ = whisper + .ptt_start() + .log_err("Could not start Whisper transcription"); + + return Ok(EventResult::Consumed); + } + + let model_path = data_dir::get_path("whisper") + .join(app.session.config.whisper_model.as_ref()); + if model_path.is_file() { + state.whisper_sst = WhisperStt::new(model_path) + .log_err("Could not create STT provider") + .ok(); + } else { + Toast::new( + ToastTopic::System, + "WHISPER.MODEL_NOT_DOWNLOADED".into(), + "WHISPER.DOWNLOAD_GUIDANCE".into(), + ) + .with_timeout(5.) + .with_sound(true) + .submit(app); + } + + Ok(EventResult::Consumed) + }), + "::WhisperTranscribeStop" => Box::new(move |_common, data, app, state| { + if !test_button(data) || !test_duration(&button, app) { + return Ok(EventResult::Pass); + } + + if let Some(whisper) = state.whisper_sst.as_mut() { + let _ = whisper + .ptt_end() + .log_err("Could not stop Whisper transcription"); + } + Ok(EventResult::Consumed) + }), + "::WhisperPaste" => Box::new(move |_common, data, app, state| { + if !test_button(data) || !test_duration(&button, app) { + return Ok(EventResult::Pass); + } + + let Some(transcription) = state.last_transcription.as_ref() else { + return Ok(EventResult::Consumed); + }; + + let mut success = false; + + match app.hid_provider.keyboard_focus { + KeyboardFocus::WayVR => { + if let Some(wvr) = app.wvr_server.as_mut() { + wvr.set_clipboard(transcription.as_ref()); + success = true; + } + } + KeyboardFocus::PhysicalScreen => { + if let Some(clip) = state.clipboard_provider.as_mut() { + clip.set_clipboard_utf8(transcription.as_ref()); + success = true; + } + } + } + + // send ctrl-v + if success { + app.hid_provider.send_key_routed( + app.wvr_server.as_mut(), + VirtualKey::RCtrl, + true, + ); + app.hid_provider.send_key_routed( + app.wvr_server.as_mut(), + VirtualKey::V, + true, + ); + app.hid_provider.send_key_routed( + app.wvr_server.as_mut(), + VirtualKey::RCtrl, + false, + ); + } + + Ok(EventResult::Consumed) + }), + "::WhisperUnload" => Box::new(move |_common, data, app, state| { + if !test_button(data) || !test_duration(&button, app) { + return Ok(EventResult::Pass); + } + + state.whisper_sst = None; + + Ok(EventResult::Consumed) + }), + _ => return, + }; + + let id = layout.add_event_listener(attribs.widget_id, *kind, callback); + log::debug!("Registered {action} on {:?} as {id:?}", attribs.widget_id); + } + } + }); + + let params = NewGuiPanelParams { + on_custom_attrib: Some(on_custom_attrib), + ..Default::default() + }; + + let mut panel = GuiPanel::new_from_template(app, xml, state, params)?; + panel.extra_attribs.insert( + BackendAttrib::Icon, + BackendAttribValue::Icon("icons/mic.svg".into()), + ); + + let label = panel.parser_state.get_widget_id("transcription")?; + + panel + .timers + .push(GuiTimer::new(Duration::from_millis(100), 0)); + + let on_label_tick: EventCallback = + Box::new(move |common, data, _app, state| { + if let Some(whisper_stt) = state.whisper_sst.as_mut() + && let Some(text) = whisper_stt.take_transcription() + { + let text: Rc = text.into(); + state.last_transcription = Some(text.clone()); + + let label = data.obj.get_as_mut::().unwrap(); + label.set_text(common, Translation::from_raw_text_rc(text)); + } + + Ok(EventResult::Pass) + }); + + panel.layout.add_event_listener( + label, + wgui::event::EventListenerKind::InternalStateChange, + on_label_tick, + ); + + panel.update_layout(app)?; + + Ok(OverlayWindowConfig { + name: WHISPER_NAME.into(), + default_state: OverlayWindowState { + interactable: true, + grabbable: true, + transform: Affine3A::from_scale_rotation_translation( + Vec3::ONE * 0.5, + Quat::IDENTITY, + vec3(0.0, 0.0, -0.6), + ), + positioning: Positioning::Anchored, + ..OverlayWindowState::default() + }, + category: OverlayCategory::BuiltInPanel, + ..OverlayWindowConfig::from_backend(Box::new(panel)) + }) +} diff --git a/wayvr/src/subsystem/clipboard/mod.rs b/wayvr/src/subsystem/clipboard/mod.rs new file mode 100644 index 00000000..a19b9e22 --- /dev/null +++ b/wayvr/src/subsystem/clipboard/mod.rs @@ -0,0 +1,8 @@ +#[cfg(feature = "wayland")] +pub mod wl; +#[cfg(feature = "x11")] +pub mod x11; + +pub trait ClipboardProvider { + fn set_clipboard_utf8(&mut self, content: &str); +} diff --git a/wayvr/src/subsystem/clipboard/wl.rs b/wayvr/src/subsystem/clipboard/wl.rs new file mode 100644 index 00000000..6be6c4e9 --- /dev/null +++ b/wayvr/src/subsystem/clipboard/wl.rs @@ -0,0 +1,23 @@ +use smithay_clipboard::Clipboard; +use wlx_capture::wayland::smithay_client_toolkit::reexports::client::{Connection, Proxy}; + +use crate::subsystem::clipboard::ClipboardProvider; + +pub struct WlClipboardProvider { + clipboard: Clipboard, +} + +impl WlClipboardProvider { + pub fn new() -> anyhow::Result { + let connection = Connection::connect_to_env()?; + let clipboard = unsafe { Clipboard::new(connection.display().id().as_ptr() as *mut _) }; + + Ok(Self { clipboard }) + } +} + +impl ClipboardProvider for WlClipboardProvider { + fn set_clipboard_utf8(&mut self, content: &str) { + self.clipboard.store(content.to_owned()); + } +} diff --git a/wayvr/src/subsystem/clipboard/x11.rs b/wayvr/src/subsystem/clipboard/x11.rs new file mode 100644 index 00000000..5cd9e783 --- /dev/null +++ b/wayvr/src/subsystem/clipboard/x11.rs @@ -0,0 +1,329 @@ +use std::{ + error::Error, + sync::{ + atomic::Ordering, + mpsc::{self, Receiver, RecvTimeoutError, Sender}, + }, + thread::{self, JoinHandle}, + time::Duration, +}; + +use xcb::{Xid, x}; + +use crate::{RUNNING, subsystem::clipboard::ClipboardProvider}; + +pub struct X11ClipboardProvider { + tx: Sender, + worker: Option>, +} + +impl X11ClipboardProvider { + pub fn new() -> Result> { + let (conn, screen_num) = xcb::Connection::connect(None)?; + let runtime = ClipboardRuntime::new(conn, screen_num)?; + + let (tx, rx) = mpsc::channel(); + + let worker = thread::spawn(move || { + runtime.run(rx); + }); + + Ok(Self { + tx, + worker: Some(worker), + }) + } +} + +impl ClipboardProvider for X11ClipboardProvider { + fn set_clipboard_utf8(&mut self, content: &str) { + let (ack_tx, ack_rx) = mpsc::channel(); + + if self + .tx + .send(Command::Set(content.as_bytes().to_vec(), ack_tx)) + .is_ok() + { + // make sure the X server has received the ownership request before returning + let _ = ack_rx.recv(); + } + } +} + +impl Drop for X11ClipboardProvider { + fn drop(&mut self) { + let _ = self.tx.send(Command::Shutdown); + + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } +} + +enum Command { + Set(Vec, Sender<()>), + Shutdown, +} + +#[derive(Clone, Copy)] +struct Atoms { + clipboard: x::Atom, + utf8_string: x::Atom, + targets: x::Atom, + text: x::Atom, + text_plain: x::Atom, + text_plain_utf8: x::Atom, +} + +struct ClipboardRuntime { + conn: xcb::Connection, + window: x::Window, + atoms: Atoms, + content: Vec, + owns_clipboard: bool, +} + +impl ClipboardRuntime { + fn new( + conn: xcb::Connection, + screen_num: i32, + ) -> Result> { + let atoms = Atoms { + clipboard: intern_atom(&conn, b"CLIPBOARD")?, + utf8_string: intern_atom(&conn, b"UTF8_STRING")?, + targets: intern_atom(&conn, b"TARGETS")?, + text: intern_atom(&conn, b"TEXT")?, + text_plain: intern_atom(&conn, b"text/plain")?, + text_plain_utf8: intern_atom(&conn, b"text/plain;charset=utf-8")?, + }; + + let setup = conn.get_setup(); + let screen = setup.roots().nth(screen_num as usize).ok_or_else(|| { + std::io::Error::new(std::io::ErrorKind::NotFound, "X11 screen not found") + })?; + + let window: x::Window = conn.generate_id(); + + conn.send_and_check_request(&x::CreateWindow { + depth: x::COPY_FROM_PARENT as u8, + wid: window, + parent: screen.root(), + x: 0, + y: 0, + width: 1, + height: 1, + border_width: 0, + class: x::WindowClass::InputOutput, + visual: screen.root_visual(), + value_list: &[], + })?; + + conn.flush()?; + + Ok(Self { + conn, + window, + atoms, + content: Vec::new(), + owns_clipboard: false, + }) + } + + fn run(mut self, rx: Receiver) { + while RUNNING.load(Ordering::Relaxed) { + self.drain_x_events(); + + match rx.recv_timeout(Duration::from_millis(10)) { + Ok(command) => { + if !self.handle_command(command) { + break; + } + + while let Ok(command) = rx.try_recv() { + if !self.handle_command(command) { + return; + } + } + } + Err(RecvTimeoutError::Timeout) => {} + Err(RecvTimeoutError::Disconnected) => break, + } + } + } + + fn handle_command(&mut self, command: Command) -> bool { + match command { + Command::Set(content, ack) => { + self.content = content; + self.become_clipboard_owner(); + let _ = ack.send(()); + true + } + Command::Shutdown => { + self.release_clipboard_if_owned(); + false + } + } + } + + fn become_clipboard_owner(&mut self) { + self.conn.send_request(&x::SetSelectionOwner { + owner: self.window, + selection: self.atoms.clipboard, + time: x::CURRENT_TIME, + }); + + self.owns_clipboard = true; + let _ = self.conn.flush(); + } + + fn release_clipboard_if_owned(&mut self) { + if !self.owns_clipboard { + return; + } + + self.conn.send_request(&x::SetSelectionOwner { + owner: x::Window::none(), + selection: self.atoms.clipboard, + time: x::CURRENT_TIME, + }); + + self.owns_clipboard = false; + let _ = self.conn.flush(); + } + + fn drain_x_events(&mut self) { + loop { + match self.conn.poll_for_event() { + Ok(Some(event)) => self.handle_x_event(event), + Ok(None) => break, + Err(xcb::Error::Protocol(_)) => continue, + Err(xcb::Error::Connection(_)) => break, + } + } + } + + fn handle_x_event(&mut self, event: xcb::Event) { + match event { + xcb::Event::X(x::Event::SelectionRequest(req)) => { + self.handle_selection_request(&req); + } + xcb::Event::X(x::Event::SelectionClear(ev)) => { + if ev.selection() == self.atoms.clipboard { + self.owns_clipboard = false; + } + } + _ => {} + } + } + + fn handle_selection_request(&self, req: &x::SelectionRequestEvent) { + if !self.owns_clipboard || req.selection() != self.atoms.clipboard { + self.send_selection_notify(req, x::Atom::none()); + return; + } + + let property = if req.property().is_none() { + // compatibility with old clients that use XCB_NONE for property + req.target() + } else { + req.property() + }; + + let ok = if req.target() == self.atoms.targets { + self.write_targets(req.requestor(), property) + } else if let Some(type_atom) = self.type_for_text_target(req.target()) { + self.write_text(req.requestor(), property, type_atom) + } else { + false + }; + + self.send_selection_notify(req, if ok { property } else { x::Atom::none() }); + } + + fn supported_targets(&self) -> Vec { + let mut targets = vec![ + self.atoms.targets, + self.atoms.utf8_string, + self.atoms.text_plain_utf8, + self.atoms.text_plain, + ]; + + // STRING/TEXT are not UTF-8 targets. Only advertise them when the + // content is representable as plain ASCII. + if self.content.is_ascii() { + targets.push(self.atoms.text); + targets.push(x::ATOM_STRING); + } + + targets + } + + fn type_for_text_target(&self, target: x::Atom) -> Option { + if target == self.atoms.utf8_string + || target == self.atoms.text_plain_utf8 + || target == self.atoms.text_plain + { + Some(target) + } else if self.content.is_ascii() && (target == self.atoms.text || target == x::ATOM_STRING) + { + Some(target) + } else { + None + } + } + + fn write_targets(&self, requestor: x::Window, property: x::Atom) -> bool { + let targets = self.supported_targets(); + + self.conn + .send_and_check_request(&x::ChangeProperty { + mode: x::PropMode::Replace, + window: requestor, + property, + r#type: x::ATOM_ATOM, + data: targets.as_slice(), + }) + .is_ok() + } + + fn write_text(&self, requestor: x::Window, property: x::Atom, type_atom: x::Atom) -> bool { + self.conn + .send_and_check_request(&x::ChangeProperty { + mode: x::PropMode::Replace, + window: requestor, + property, + r#type: type_atom, + data: self.content.as_slice(), + }) + .is_ok() + } + + fn send_selection_notify(&self, req: &x::SelectionRequestEvent, property: x::Atom) { + let event = x::SelectionNotifyEvent::new( + req.time(), + req.requestor(), + req.selection(), + req.target(), + property, + ); + + let _ = self.conn.send_and_check_request(&x::SendEvent { + propagate: false, + destination: x::SendEventDest::Window(req.requestor()), + event_mask: x::EventMask::NO_EVENT, + event: &event, + }); + + let _ = self.conn.flush(); + } +} + +fn intern_atom(conn: &xcb::Connection, name: &[u8]) -> xcb::Result { + let cookie = conn.send_request(&x::InternAtom { + only_if_exists: false, + name, + }); + + Ok(conn.wait_for_reply(cookie)?.atom()) +} diff --git a/wayvr/src/subsystem/mod.rs b/wayvr/src/subsystem/mod.rs index 1243d666..5909c060 100644 --- a/wayvr/src/subsystem/mod.rs +++ b/wayvr/src/subsystem/mod.rs @@ -3,6 +3,9 @@ pub mod hid; pub mod input; pub mod notifications; +#[cfg(feature = "whisper")] +pub mod clipboard; + #[cfg(feature = "whisper")] pub mod whisper_stt; diff --git a/wayvr/src/windowing/manager.rs b/wayvr/src/windowing/manager.rs index 16777e76..2104b80b 100644 --- a/wayvr/src/windowing/manager.rs +++ b/wayvr/src/windowing/manager.rs @@ -136,6 +136,13 @@ where let grab_help = OverlayWindowData::from_config(create_grab_help(app)?); me.add(grab_help, app); + #[cfg(feature = "whisper")] + { + use crate::overlays::whisper::create_whisper; + let whisper = OverlayWindowData::from_config(create_whisper(app, headless, wayland)?); + me.add(whisper, app); + } + let custom_panels = app.session.config.custom_panels.clone(); for name in custom_panels { let Some(panel) = create_custom(app, name) else { diff --git a/wayvr/src/windowing/window.rs b/wayvr/src/windowing/window.rs index 6d025821..cda85cab 100644 --- a/wayvr/src/windowing/window.rs +++ b/wayvr/src/windowing/window.rs @@ -57,7 +57,10 @@ pub enum OverlayCategory { Internal, Keyboard, Dashboard, + /// User-provided custom panels Panel, + /// Same as Panel but does not support Modify and Reload + BuiltInPanel, Screen, Mirror, WayVR, diff --git a/wlx-capture/src/wayland.rs b/wlx-capture/src/wayland.rs index 4ad7c74c..4c2c8be4 100644 --- a/wlx-capture/src/wayland.rs +++ b/wlx-capture/src/wayland.rs @@ -17,6 +17,7 @@ use smithay_client_toolkit::reexports::{ }, }; +pub use smithay_client_toolkit; pub use wayland_client; use wayland_client::{ Connection, Dispatch, EventQueue, Proxy, QueueHandle,