whisper working

Cargo.lock

@@ -6608,6 +6608,8 @@ dependencies = [
  "anyhow",
  "ash",
  "bytes",
+ "calloop 0.14.4",
+ "calloop-wayland-source 0.4.1",
  "chrono",
  "chrono-tz",
  "clap",
@@ -6655,6 +6657,7 @@ dependencies = [
  "vulkano",
  "vulkano-shaders",
  "wayland-client",
+ "wayland-protocols",
  "wayland-protocols-misc",
  "wayvr-ipc",
  "wgui",

Cargo.toml

@@ -34,7 +34,8 @@ vulkano = { version = "0.35.2", default-features = false, features = [
   "macros",
 ] }
 vulkano-shaders = "0.35.0"
-wayland-client = { version = "0.31.11" }
+wayland-client = { version = "0.31" }
+wayland-protocols = { version = "0.32" }
 wayland-protocols-misc = { version = "0.3.12" }
 xdg = "3.0.0"
 

wayvr/Cargo.toml

@@ -86,6 +86,7 @@ tracing = "0.1.43"
 tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }
 uuid = { workspace = true }
 wayland-client = { workspace = true }
+wayland-protocols = { workspace = true }
 wayland-protocols-misc = { workspace = true }
 winit = { version = "0.30.12", optional = true }
 xcb = { version = "1.6.0", features = [
@@ -95,7 +96,11 @@ xkbcommon = { version = "0.8.0" } # 0.9.0 breaks keymap import on some distros
 whisper-rs = { version = "0.16.0", features = ["vulkan"], optional = true }
 smithay-clipboard = "0.7.3"
 
+calloop = { version = "0.14", optional = true }
+calloop-wayland-source = { version = "0.4", optional = true }
+
 [build-dependencies]
+
 regex.workspace = true
 
 [features]
@@ -108,6 +113,6 @@ osc = ["dep:rosc"]
 pipewire = ["wlx-capture/pipewire"]
 uidev = ["dep:winit"]
 wayland = ["pipewire", "wlx-capture/wlr", "xkbcommon/wayland"]
-whisper = ["dep:pipewire", "dep:whisper-rs"]
+whisper = ["dep:pipewire", "dep:whisper-rs", "dep:calloop", "dep:calloop-wayland-source"]
 x11 = ["dep:xcb", "wlx-capture/xshm", "xkbcommon/x11"]
 xcb = ["dep:xcb"]

wayvr/src/overlays/whisper.rs

@@ -1,4 +1,4 @@
-use std::{path::Path, rc::Rc, time::Duration};
+use std::{rc::Rc, time::Duration};
 
 use glam::{Affine3A, Quat, Vec3, vec3};
 use wgui::{
@@ -27,7 +27,7 @@ use crate::{
     overlays::toast::Toast,
     state::AppState,
     subsystem::{
-        clipboard::{ClipboardProvider, wl::WlClipboardProvider, x11::X11ClipboardProvider},
+        clipboard::{self, ClipboardProvider},
         hid::VirtualKey,
         input::KeyboardFocus,
         whisper_stt::WhisperStt,
@@ -76,20 +76,22 @@ pub fn create_whisper(
     headless: bool,
     wayland: bool,
 ) -> anyhow::Result<OverlayWindowConfig> {
-    // let clipboard_provider: Option<Box<dyn ClipboardProvider>> = match (headless, wayland) {
+    let clipboard_provider: Option<Box<dyn ClipboardProvider>> = match (headless, wayland) {
-    //     (true, _) => None,
+        #[cfg(feature = "wayland")]
-    //     (false, true) => WlClipboardProvider::new()
+        (false, true) => clipboard::wl::Provider::new()
-    //         .log_err("Could not create Wayland clipboard provider")
+            .log_err("Could not create Wayland clipboard provider")
-    //         .ok()
+            .ok()
-    //         .map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
+            .map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
-    //     (false, false) => X11ClipboardProvider::new()
+        #[cfg(feature = "x11")]
-    //         .log_err("Could not create X11 clipboard provider")
+        (false, false) => clipboard::x11::Provider::new()
-    //         .ok()
+            .log_err("Could not create X11 clipboard provider")
-    //         .map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
+            .ok()
-    // };
+            .map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
+        _ => None,
+    };
 
     let state = WhisperState {
-        clipboard_provider: None,
+        clipboard_provider,
         ..Default::default()
     };
     let xml = "gui/whisper.xml";
@@ -203,6 +205,11 @@ pub fn create_whisper(
                                 VirtualKey::V,
                                 true,
                             );
+                            app.hid_provider.send_key_routed(
+                                app.wvr_server.as_mut(),
+                                VirtualKey::V,
+                                false,
+                            );
                             app.hid_provider.send_key_routed(
                                 app.wvr_server.as_mut(),
                                 VirtualKey::RCtrl,
@@ -310,16 +317,19 @@ pub fn create_whisper(
 
     panel.update_layout(app)?;
 
+    let transform = Affine3A::from_cols_array_2d(&[
+        [0.49993715, -0.00020921684, -0.008030709],
+        [-0.0021463279, 0.47818363, -0.14607349],
+        [0.007741399, 0.1460891, 0.478121],
+        [-0.021562248, -0.40786624, -0.3346647],
+    ]);
+
     Ok(OverlayWindowConfig {
         name: WHISPER_NAME.into(),
         default_state: OverlayWindowState {
             interactable: true,
             grabbable: true,
-            transform: Affine3A::from_scale_rotation_translation(
+            transform,
-                Vec3::ONE * 0.5,
-                Quat::IDENTITY,
-                vec3(0.0, 0.0, -0.6),
-            ),
             positioning: Positioning::Anchored,
             ..OverlayWindowState::default()
         },

wayvr/src/subsystem/clipboard/wl.rs

@@ -1,23 +1,349 @@
-use smithay_clipboard::Clipboard;
+use std::{
-use wlx_capture::wayland::smithay_client_toolkit::reexports::client::{Connection, Proxy};
+    fs::File,
+    io::Write,
+    sync::mpsc::{self, SyncSender},
+    thread::{self, JoinHandle},
+};
+
+use anyhow::Context;
+use calloop::{
+    EventLoop,
+    channel::{Channel, Event as ChannelEvent, Sender, channel},
+};
+use calloop_wayland_source::WaylandSource;
+use wayland_client::{
+    Connection, Dispatch, QueueHandle,
+    globals::{GlobalListContents, registry_queue_init},
+    protocol::{
+        wl_registry,
+        wl_seat::{self, WlSeat},
+    },
+};
+use wayland_protocols::ext::data_control::v1::client::{
+    ext_data_control_device_v1::{self, ExtDataControlDeviceV1},
+    ext_data_control_manager_v1::{self, ExtDataControlManagerV1},
+    ext_data_control_offer_v1::{self, ExtDataControlOfferV1},
+    ext_data_control_source_v1::{self, ExtDataControlSourceV1},
+};
 
 use crate::subsystem::clipboard::ClipboardProvider;
 
-pub struct WlClipboardProvider {
+const TEXT_MIME_TYPES: &[&str] = &[
-    clipboard: Clipboard,
+    "text/plain;charset=utf-8",
+    "text/plain;charset=UTF-8",
+    "text/plain",
+    "UTF8_STRING",
+    "TEXT",
+    "STRING",
+];
+
+pub struct Provider {
+    tx: Sender<Command>,
+    _thread: JoinHandle<()>,
 }
 
-impl WlClipboardProvider {
+enum Command {
+    SetText(String),
+    Shutdown,
+}
+
+struct ActiveSource {
+    source: ExtDataControlSourceV1,
+    bytes: Vec<u8>,
+}
+
+struct State {
+    conn: Connection,
+    qh: QueueHandle<State>,
+    manager: ExtDataControlManagerV1,
+    device: ExtDataControlDeviceV1,
+    active_sources: Vec<ActiveSource>,
+    ignored_offers: Vec<ExtDataControlOfferV1>,
+    stop: bool,
+}
+
+impl Provider {
     pub fn new() -> anyhow::Result<Self> {
-        let connection = Connection::connect_to_env()?;
+        let (tx, rx) = channel::<Command>();
-        let clipboard = unsafe { Clipboard::new(connection.display().id().as_ptr() as *mut _) };
+        let (ready_tx, ready_rx) = mpsc::sync_channel::<Result<(), String>>(1);
 
-        Ok(Self { clipboard })
+        let thread = thread::Builder::new()
+            .name("wayland-clipboard-ext-data-control".to_owned())
+            .spawn(move || run_worker(rx, ready_tx))
+            .context("failed to spawn Wayland clipboard worker thread")?;
+
+        match ready_rx
+            .recv()
+            .context("Wayland clipboard worker exited before reporting readiness")?
+        {
+            Ok(()) => Ok(Self {
+                tx,
+                _thread: thread,
+            }),
+            Err(err) => anyhow::bail!("{}", err),
+        }
     }
 }
 
-impl ClipboardProvider for WlClipboardProvider {
+impl ClipboardProvider for Provider {
     fn set_clipboard_utf8(&mut self, content: &str) {
-        self.clipboard.store(content.to_owned());
+        let _ = self.tx.send(Command::SetText(content.to_owned()));
+    }
+}
+
+impl Drop for Provider {
+    fn drop(&mut self) {
+        let _ = self.tx.send(Command::Shutdown);
+    }
+}
+
+fn run_worker(rx: Channel<Command>, ready: SyncSender<Result<(), String>>) {
+    macro_rules! init_or_report {
+        ($expr:expr, $context:literal) => {
+            match $expr {
+                Ok(value) => value,
+                Err(err) => {
+                    let _ = ready.send(Err(format!("{}: {}", $context, err)));
+                    return;
+                }
+            }
+        };
+    }
+
+    let conn = init_or_report!(
+        Connection::connect_to_env(),
+        "connect to Wayland compositor"
+    );
+
+    let (globals, event_queue) =
+        init_or_report!(registry_queue_init::<State>(&conn), "read Wayland globals");
+
+    let qh = event_queue.handle();
+
+    let mut event_loop =
+        init_or_report!(EventLoop::<State>::try_new(), "create calloop event loop");
+
+    init_or_report!(
+        WaylandSource::new(conn.clone(), event_queue).insert(event_loop.handle()),
+        "attach Wayland queue to event loop"
+    );
+
+    let manager: ExtDataControlManagerV1 = init_or_report!(
+        globals.bind(&qh, 1..=1, ()),
+        "bind ext_data_control_manager_v1"
+    );
+
+    let seat: WlSeat = init_or_report!(globals.bind(&qh, 1..=9, ()), "bind wl_seat");
+
+    let device = manager.get_data_device(&seat, &qh, ());
+
+    let mut state = State {
+        conn,
+        qh,
+        manager,
+        device,
+        active_sources: Vec::new(),
+        ignored_offers: Vec::new(),
+        stop: false,
+    };
+
+    init_or_report!(
+        event_loop
+            .handle()
+            .insert_source(rx, |event, _metadata, state| match event {
+                ChannelEvent::Msg(Command::SetText(text)) => {
+                    state.set_clipboard_text(text);
+                }
+                ChannelEvent::Msg(Command::Shutdown) | ChannelEvent::Closed => {
+                    state.stop = true;
+                }
+            }),
+        "attach clipboard command channel to event loop"
+    );
+
+    if let Err(err) = state.conn.flush() {
+        let _ = ready.send(Err(format!("flush initial Wayland requests: {err}")));
+        return;
+    }
+
+    let _ = ready.send(Ok(()));
+
+    while !state.stop {
+        if let Err(err) = event_loop.dispatch(None, &mut state) {
+            eprintln!("Wayland clipboard event loop failed: {err}");
+            break;
+        }
+
+        if let Err(err) = state.conn.flush() {
+            eprintln!("Wayland clipboard flush failed: {err}");
+            break;
+        }
+    }
+}
+
+impl State {
+    fn set_clipboard_text(&mut self, content: String) {
+        let source = self.manager.create_data_source(&self.qh, ());
+
+        for mime_type in TEXT_MIME_TYPES {
+            source.offer((*mime_type).to_owned());
+        }
+
+        self.device.set_selection(Some(&source));
+
+        let previous_sources = std::mem::take(&mut self.active_sources);
+
+        self.active_sources.push(ActiveSource {
+            source,
+            bytes: content.into_bytes(),
+        });
+
+        for old in previous_sources {
+            old.source.destroy();
+        }
+
+        let _ = self.conn.flush();
+    }
+}
+
+impl Dispatch<wl_registry::WlRegistry, GlobalListContents> for State {
+    fn event(
+        _state: &mut Self,
+        _registry: &wl_registry::WlRegistry,
+        _event: wl_registry::Event,
+        _data: &GlobalListContents,
+        _conn: &Connection,
+        _qh: &QueueHandle<Self>,
+    ) {
+    }
+}
+
+impl Dispatch<WlSeat, ()> for State {
+    fn event(
+        _state: &mut Self,
+        _seat: &WlSeat,
+        _event: wl_seat::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qh: &QueueHandle<Self>,
+    ) {
+    }
+}
+
+impl Dispatch<ExtDataControlManagerV1, ()> for State {
+    fn event(
+        _state: &mut Self,
+        _manager: &ExtDataControlManagerV1,
+        _event: ext_data_control_manager_v1::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qh: &QueueHandle<Self>,
+    ) {
+    }
+}
+
+impl Dispatch<ExtDataControlDeviceV1, ()> for State {
+    fn event(
+        state: &mut Self,
+        device: &ExtDataControlDeviceV1,
+        event: ext_data_control_device_v1::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qh: &QueueHandle<Self>,
+    ) {
+        match event {
+            ext_data_control_device_v1::Event::DataOffer { id } => {
+                state.ignored_offers.push(id);
+            }
+            ext_data_control_device_v1::Event::Selection { id }
+            | ext_data_control_device_v1::Event::PrimarySelection { id } => {
+                if let Some(offer) = id {
+                    if let Some(pos) = state
+                        .ignored_offers
+                        .iter()
+                        .position(|stored| *stored == offer)
+                    {
+                        state.ignored_offers.swap_remove(pos);
+                    }
+
+                    offer.destroy();
+                }
+            }
+            ext_data_control_device_v1::Event::Finished => {
+                device.destroy();
+                state.stop = true;
+            }
+            _ => {}
+        }
+    }
+
+    fn event_created_child(
+        opcode: u16,
+        qh: &QueueHandle<Self>,
+    ) -> std::sync::Arc<dyn wayland_client::backend::ObjectData> {
+        if opcode == ext_data_control_device_v1::EVT_DATA_OFFER_OPCODE {
+            qh.make_data::<ExtDataControlOfferV1, _>(())
+        } else {
+            unreachable!("unknown ext_data_control_device_v1 child event opcode: {opcode}")
+        }
+    }
+}
+
+impl Dispatch<ExtDataControlOfferV1, ()> for State {
+    fn event(
+        _state: &mut Self,
+        _offer: &ExtDataControlOfferV1,
+        _event: ext_data_control_offer_v1::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qh: &QueueHandle<Self>,
+    ) {
+    }
+}
+
+impl Dispatch<ExtDataControlSourceV1, ()> for State {
+    fn event(
+        state: &mut Self,
+        source: &ExtDataControlSourceV1,
+        event: ext_data_control_source_v1::Event,
+        _data: &(),
+        _conn: &Connection,
+        _qh: &QueueHandle<Self>,
+    ) {
+        match event {
+            ext_data_control_source_v1::Event::Send { mime_type, fd } => {
+                if !TEXT_MIME_TYPES.contains(&mime_type.as_str()) {
+                    return;
+                }
+
+                let Some(active) = state
+                    .active_sources
+                    .iter()
+                    .find(|active| active.source == *source)
+                else {
+                    return;
+                };
+
+                let mut file = File::from(fd);
+
+                if let Err(err) = file.write_all(&active.bytes) {
+                    eprintln!("failed to write clipboard data to Wayland fd: {err}");
+                    return;
+                }
+
+                let _ = file.flush();
+            }
+            ext_data_control_source_v1::Event::Cancelled => {
+                if let Some(pos) = state
+                    .active_sources
+                    .iter()
+                    .position(|active| active.source == *source)
+                {
+                    let active = state.active_sources.swap_remove(pos);
+                    active.source.destroy();
+                }
+            }
+            _ => {}
+        }
     }
 }

wayvr/src/subsystem/clipboard/x11.rs

@@ -12,12 +12,12 @@ use xcb::{Xid, x};
 
 use crate::{RUNNING, subsystem::clipboard::ClipboardProvider};
 
-pub struct X11ClipboardProvider {
+pub struct Provider {
     tx: Sender<Command>,
     worker: Option<JoinHandle<()>>,
 }
 
-impl X11ClipboardProvider {
+impl Provider {
     pub fn new() -> Result<Self, Box<dyn Error + Send + Sync + 'static>> {
         let (conn, screen_num) = xcb::Connection::connect(None)?;
         let runtime = ClipboardRuntime::new(conn, screen_num)?;
@@ -35,7 +35,7 @@ impl X11ClipboardProvider {
     }
 }
 
-impl ClipboardProvider for X11ClipboardProvider {
+impl ClipboardProvider for Provider {
     fn set_clipboard_utf8(&mut self, content: &str) {
         let (ack_tx, ack_rx) = mpsc::channel();
 
@@ -50,7 +50,7 @@ impl ClipboardProvider for X11ClipboardProvider {
     }
 }
 
-impl Drop for X11ClipboardProvider {
+impl Drop for Provider {
     fn drop(&mut self) {
         let _ = self.tx.send(Command::Shutdown);
 

wayvr/src/subsystem/hid/wayland.rs

@@ -1,5 +1,5 @@
 use anyhow::Context;
-use wlx_capture::wayland::wayland_client::{
+use wayland_client::{
     Connection, Dispatch, Proxy, QueueHandle,
     globals::{GlobalListContents, registry_queue_init},
     protocol::{

wlx-capture/src/wayland.rs

@@ -17,8 +17,6 @@ use smithay_client_toolkit::reexports::{
     },
 };
 
-pub use smithay_client_toolkit;
-pub use wayland_client;
 use wayland_client::{
     Connection, Dispatch, EventQueue, Proxy, QueueHandle,
     backend::WaylandError,