RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

whisper working

This commit is contained in:
galister 2026-07-04 00:33:06 +09:00
parent dd9838f5ff
commit 6dd624e046
8 changed files with 381 additions and 38 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
Cargo.lock generated
View File

@ -6608,6 +6608,8 @@ dependencies = [
"anyhow",
"ash",
"bytes",
"calloop 0.14.4",
"calloop-wayland-source 0.4.1",
"chrono",
"chrono-tz",
"clap",
@ -6655,6 +6657,7 @@ dependencies = [
"vulkano",
"vulkano-shaders",
"wayland-client",
"wayland-protocols",
"wayland-protocols-misc",
"wayvr-ipc",
"wgui",

3
Cargo.toml
View File

@ -34,7 +34,8 @@ vulkano = { version = "0.35.2", default-features = false, features = [
"macros",
] }
vulkano-shaders = "0.35.0"
wayland-client = { version = "0.31.11" }
wayland-client = { version = "0.31" }
wayland-protocols = { version = "0.32" }
wayland-protocols-misc = { version = "0.3.12" }
xdg = "3.0.0"

7
wayvr/Cargo.toml
View File

@ -86,6 +86,7 @@ tracing = "0.1.43"
tracing-subscriber = { version = "0.3.22", features = ["env-filter"] }
uuid = { workspace = true }
wayland-client = { workspace = true }
wayland-protocols = { workspace = true }
wayland-protocols-misc = { workspace = true }
winit = { version = "0.30.12", optional = true }
xcb = { version = "1.6.0", features = [
@ -95,7 +96,11 @@ xkbcommon = { version = "0.8.0" } # 0.9.0 breaks keymap import on some distros
whisper-rs = { version = "0.16.0", features = ["vulkan"], optional = true }
smithay-clipboard = "0.7.3"
calloop = { version = "0.14", optional = true }
calloop-wayland-source = { version = "0.4", optional = true }
[build-dependencies]
regex.workspace = true
[features]
@ -108,6 +113,6 @@ osc = ["dep:rosc"]
pipewire = ["wlx-capture/pipewire"]
uidev = ["dep:winit"]
wayland = ["pipewire", "wlx-capture/wlr", "xkbcommon/wayland"]
whisper = ["dep:pipewire", "dep:whisper-rs"]
whisper = ["dep:pipewire", "dep:whisper-rs", "dep:calloop", "dep:calloop-wayland-source"]
x11 = ["dep:xcb", "wlx-capture/xshm", "xkbcommon/x11"]
xcb = ["dep:xcb"]

48
wayvr/src/overlays/whisper.rs
View File

@ -1,4 +1,4 @@
use std::{path::Path, rc::Rc, time::Duration};
use std::{rc::Rc, time::Duration};
use glam::{Affine3A, Quat, Vec3, vec3};
use wgui::{
@ -27,7 +27,7 @@ use crate::{
overlays::toast::Toast,
state::AppState,
subsystem::{
clipboard::{ClipboardProvider, wl::WlClipboardProvider, x11::X11ClipboardProvider},
clipboard::{self, ClipboardProvider},
hid::VirtualKey,
input::KeyboardFocus,
whisper_stt::WhisperStt,
@ -76,20 +76,22 @@ pub fn create_whisper(
headless: bool,
wayland: bool,
) -> anyhow::Result<OverlayWindowConfig> {
// let clipboard_provider: Option<Box<dyn ClipboardProvider>> = match (headless, wayland) {
// (true, _) => None,
// (false, true) => WlClipboardProvider::new()
// .log_err("Could not create Wayland clipboard provider")
// .ok()
// .map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
// (false, false) => X11ClipboardProvider::new()
// .log_err("Could not create X11 clipboard provider")
// .ok()
// .map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
// };
let clipboard_provider: Option<Box<dyn ClipboardProvider>> = match (headless, wayland) {
#[cfg(feature = "wayland")]
(false, true) => clipboard::wl::Provider::new()
.log_err("Could not create Wayland clipboard provider")
.ok()
.map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
#[cfg(feature = "x11")]
(false, false) => clipboard::x11::Provider::new()
.log_err("Could not create X11 clipboard provider")
.ok()
.map(|p| Box::new(p) as Box<dyn ClipboardProvider>),
_ => None,
};
let state = WhisperState {
clipboard_provider: None,
clipboard_provider,
..Default::default()
};
let xml = "gui/whisper.xml";
@ -203,6 +205,11 @@ pub fn create_whisper(
VirtualKey::V,
true,
);
app.hid_provider.send_key_routed(
app.wvr_server.as_mut(),
VirtualKey::V,
false,
);
app.hid_provider.send_key_routed(
app.wvr_server.as_mut(),
VirtualKey::RCtrl,
@ -310,16 +317,19 @@ pub fn create_whisper(
panel.update_layout(app)?;
let transform = Affine3A::from_cols_array_2d(&[
[0.49993715, -0.00020921684, -0.008030709],
[-0.0021463279, 0.47818363, -0.14607349],
[0.007741399, 0.1460891, 0.478121],
[-0.021562248, -0.40786624, -0.3346647],
]);
Ok(OverlayWindowConfig {
name: WHISPER_NAME.into(),
default_state: OverlayWindowState {
interactable: true,
grabbable: true,
transform: Affine3A::from_scale_rotation_translation(
Vec3::ONE * 0.5,
Quat::IDENTITY,
vec3(0.0, 0.0, -0.6),
),
transform,
positioning: Positioning::Anchored,
..OverlayWindowState::default()
},

346
wayvr/src/subsystem/clipboard/wl.rs
View File

@ -1,23 +1,349 @@
use smithay_clipboard::Clipboard;
use wlx_capture::wayland::smithay_client_toolkit::reexports::client::{Connection, Proxy};
use std::{
fs::File,
io::Write,
sync::mpsc::{self, SyncSender},
thread::{self, JoinHandle},
};
use anyhow::Context;
use calloop::{
EventLoop,
channel::{Channel, Event as ChannelEvent, Sender, channel},
};
use calloop_wayland_source::WaylandSource;
use wayland_client::{
Connection, Dispatch, QueueHandle,
globals::{GlobalListContents, registry_queue_init},
protocol::{
wl_registry,
wl_seat::{self, WlSeat},
},
};
use wayland_protocols::ext::data_control::v1::client::{
ext_data_control_device_v1::{self, ExtDataControlDeviceV1},
ext_data_control_manager_v1::{self, ExtDataControlManagerV1},
ext_data_control_offer_v1::{self, ExtDataControlOfferV1},
ext_data_control_source_v1::{self, ExtDataControlSourceV1},
};
use crate::subsystem::clipboard::ClipboardProvider;
pub struct WlClipboardProvider {
clipboard: Clipboard,
const TEXT_MIME_TYPES: &[&str] = &[
"text/plain;charset=utf-8",
"text/plain;charset=UTF-8",
"text/plain",
"UTF8_STRING",
"TEXT",
"STRING",
];
pub struct Provider {
tx: Sender<Command>,
_thread: JoinHandle<()>,
}
impl WlClipboardProvider {
enum Command {
SetText(String),
Shutdown,
}
struct ActiveSource {
source: ExtDataControlSourceV1,
bytes: Vec<u8>,
}
struct State {
conn: Connection,
qh: QueueHandle<State>,
manager: ExtDataControlManagerV1,
device: ExtDataControlDeviceV1,
active_sources: Vec<ActiveSource>,
ignored_offers: Vec<ExtDataControlOfferV1>,
stop: bool,
}
impl Provider {
pub fn new() -> anyhow::Result<Self> {
let connection = Connection::connect_to_env()?;
let clipboard = unsafe { Clipboard::new(connection.display().id().as_ptr() as *mut _) };
let (tx, rx) = channel::<Command>();
let (ready_tx, ready_rx) = mpsc::sync_channel::<Result<(), String>>(1);
Ok(Self { clipboard })
let thread = thread::Builder::new()
.name("wayland-clipboard-ext-data-control".to_owned())
.spawn(move || run_worker(rx, ready_tx))
.context("failed to spawn Wayland clipboard worker thread")?;
match ready_rx
.recv()
.context("Wayland clipboard worker exited before reporting readiness")?
{
Ok(()) => Ok(Self {
tx,
_thread: thread,
}),
Err(err) => anyhow::bail!("{}", err),
}
}
}
impl ClipboardProvider for WlClipboardProvider {
impl ClipboardProvider for Provider {
fn set_clipboard_utf8(&mut self, content: &str) {
self.clipboard.store(content.to_owned());
let _ = self.tx.send(Command::SetText(content.to_owned()));
}
}
impl Drop for Provider {
fn drop(&mut self) {
let _ = self.tx.send(Command::Shutdown);
}
}
fn run_worker(rx: Channel<Command>, ready: SyncSender<Result<(), String>>) {
macro_rules! init_or_report {
($expr:expr, $context:literal) => {
match $expr {
Ok(value) => value,
Err(err) => {
let _ = ready.send(Err(format!("{}: {}", $context, err)));
return;
}
}
};
}
let conn = init_or_report!(
Connection::connect_to_env(),
"connect to Wayland compositor"
);
let (globals, event_queue) =
init_or_report!(registry_queue_init::<State>(&conn), "read Wayland globals");
let qh = event_queue.handle();
let mut event_loop =
init_or_report!(EventLoop::<State>::try_new(), "create calloop event loop");
init_or_report!(
WaylandSource::new(conn.clone(), event_queue).insert(event_loop.handle()),
"attach Wayland queue to event loop"
);
let manager: ExtDataControlManagerV1 = init_or_report!(
globals.bind(&qh, 1..=1, ()),
"bind ext_data_control_manager_v1"
);
let seat: WlSeat = init_or_report!(globals.bind(&qh, 1..=9, ()), "bind wl_seat");
let device = manager.get_data_device(&seat, &qh, ());
let mut state = State {
conn,
qh,
manager,
device,
active_sources: Vec::new(),
ignored_offers: Vec::new(),
stop: false,
};
init_or_report!(
event_loop
.handle()
.insert_source(rx, |event, _metadata, state| match event {
ChannelEvent::Msg(Command::SetText(text)) => {
state.set_clipboard_text(text);
}
ChannelEvent::Msg(Command::Shutdown) | ChannelEvent::Closed => {
state.stop = true;
}
}),
"attach clipboard command channel to event loop"
);
if let Err(err) = state.conn.flush() {
let _ = ready.send(Err(format!("flush initial Wayland requests: {err}")));
return;
}
let _ = ready.send(Ok(()));
while !state.stop {
if let Err(err) = event_loop.dispatch(None, &mut state) {
eprintln!("Wayland clipboard event loop failed: {err}");
break;
}
if let Err(err) = state.conn.flush() {
eprintln!("Wayland clipboard flush failed: {err}");
break;
}
}
}
impl State {
fn set_clipboard_text(&mut self, content: String) {
let source = self.manager.create_data_source(&self.qh, ());
for mime_type in TEXT_MIME_TYPES {
source.offer((*mime_type).to_owned());
}
self.device.set_selection(Some(&source));
let previous_sources = std::mem::take(&mut self.active_sources);
self.active_sources.push(ActiveSource {
source,
bytes: content.into_bytes(),
});
for old in previous_sources {
old.source.destroy();
}
let _ = self.conn.flush();
}
}
impl Dispatch<wl_registry::WlRegistry, GlobalListContents> for State {
fn event(
_state: &mut Self,
_registry: &wl_registry::WlRegistry,
_event: wl_registry::Event,
_data: &GlobalListContents,
_conn: &Connection,
_qh: &QueueHandle<Self>,
) {
}
}
impl Dispatch<WlSeat, ()> for State {
fn event(
_state: &mut Self,
_seat: &WlSeat,
_event: wl_seat::Event,
_data: &(),
_conn: &Connection,
_qh: &QueueHandle<Self>,
) {
}
}
impl Dispatch<ExtDataControlManagerV1, ()> for State {
fn event(
_state: &mut Self,
_manager: &ExtDataControlManagerV1,
_event: ext_data_control_manager_v1::Event,
_data: &(),
_conn: &Connection,
_qh: &QueueHandle<Self>,
) {
}
}
impl Dispatch<ExtDataControlDeviceV1, ()> for State {
fn event(
state: &mut Self,
device: &ExtDataControlDeviceV1,
event: ext_data_control_device_v1::Event,
_data: &(),
_conn: &Connection,
_qh: &QueueHandle<Self>,
) {
match event {
ext_data_control_device_v1::Event::DataOffer { id } => {
state.ignored_offers.push(id);
}
ext_data_control_device_v1::Event::Selection { id }
| ext_data_control_device_v1::Event::PrimarySelection { id } => {
if let Some(offer) = id {
if let Some(pos) = state
.ignored_offers
.iter()
.position(|stored| *stored == offer)
{
state.ignored_offers.swap_remove(pos);
}
offer.destroy();
}
}
ext_data_control_device_v1::Event::Finished => {
device.destroy();
state.stop = true;
}
_ => {}
}
}
fn event_created_child(
opcode: u16,
qh: &QueueHandle<Self>,
) -> std::sync::Arc<dyn wayland_client::backend::ObjectData> {
if opcode == ext_data_control_device_v1::EVT_DATA_OFFER_OPCODE {
qh.make_data::<ExtDataControlOfferV1, _>(())
} else {
unreachable!("unknown ext_data_control_device_v1 child event opcode: {opcode}")
}
}
}
impl Dispatch<ExtDataControlOfferV1, ()> for State {
fn event(
_state: &mut Self,
_offer: &ExtDataControlOfferV1,
_event: ext_data_control_offer_v1::Event,
_data: &(),
_conn: &Connection,
_qh: &QueueHandle<Self>,
) {
}
}
impl Dispatch<ExtDataControlSourceV1, ()> for State {
fn event(
state: &mut Self,
source: &ExtDataControlSourceV1,
event: ext_data_control_source_v1::Event,
_data: &(),
_conn: &Connection,
_qh: &QueueHandle<Self>,
) {
match event {
ext_data_control_source_v1::Event::Send { mime_type, fd } => {
if !TEXT_MIME_TYPES.contains(&mime_type.as_str()) {
return;
}
let Some(active) = state
.active_sources
.iter()
.find(|active| active.source == *source)
else {
return;
};
let mut file = File::from(fd);
if let Err(err) = file.write_all(&active.bytes) {
eprintln!("failed to write clipboard data to Wayland fd: {err}");
return;
}
let _ = file.flush();
}
ext_data_control_source_v1::Event::Cancelled => {
if let Some(pos) = state
.active_sources
.iter()
.position(|active| active.source == *source)
{
let active = state.active_sources.swap_remove(pos);
active.source.destroy();
}
}
_ => {}
}
}
}

8
wayvr/src/subsystem/clipboard/x11.rs
View File

@ -12,12 +12,12 @@ use xcb::{Xid, x};
use crate::{RUNNING, subsystem::clipboard::ClipboardProvider};
pub struct X11ClipboardProvider {
pub struct Provider {
tx: Sender<Command>,
worker: Option<JoinHandle<()>>,
}
impl X11ClipboardProvider {
impl Provider {
pub fn new() -> Result<Self, Box<dyn Error + Send + Sync + 'static>> {
let (conn, screen_num) = xcb::Connection::connect(None)?;
let runtime = ClipboardRuntime::new(conn, screen_num)?;
@ -35,7 +35,7 @@ impl X11ClipboardProvider {
}
}
impl ClipboardProvider for X11ClipboardProvider {
impl ClipboardProvider for Provider {
fn set_clipboard_utf8(&mut self, content: &str) {
let (ack_tx, ack_rx) = mpsc::channel();
@ -50,7 +50,7 @@ impl ClipboardProvider for X11ClipboardProvider {
}
}
impl Drop for X11ClipboardProvider {
impl Drop for Provider {
fn drop(&mut self) {
let _ = self.tx.send(Command::Shutdown);

2
wayvr/src/subsystem/hid/wayland.rs
View File

@ -1,5 +1,5 @@
use anyhow::Context;
use wlx_capture::wayland::wayland_client::{
use wayland_client::{
Connection, Dispatch, Proxy, QueueHandle,
globals::{GlobalListContents, registry_queue_init},
protocol::{

2
wlx-capture/src/wayland.rs
View File

@ -17,8 +17,6 @@ use smithay_client_toolkit::reexports::{
},
};
pub use smithay_client_toolkit;
pub use wayland_client;
use wayland_client::{
Connection, Dispatch, EventQueue, Proxy, QueueHandle,
backend::WaylandError,