From 21a05046015fc072a90c2730b3eed6eb5794ddc3 Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Fri, 3 Jul 2026 15:17:39 +0900 Subject: [PATCH] wip: whisper stt --- Cargo.lock | 53 ++- Cargo.toml | 1 + wayvr/Cargo.toml | 5 +- wayvr/src/subsystem/mod.rs | 3 + wayvr/src/subsystem/whisper_stt.rs | 712 +++++++++++++++++++++++++++++ wlx-capture/Cargo.toml | 5 +- 6 files changed, 766 insertions(+), 13 deletions(-) create mode 100644 wayvr/src/subsystem/whisper_stt.rs diff --git a/Cargo.lock b/Cargo.lock index 9646ed23..4ce2e84f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -690,6 +690,8 @@ dependencies = [ "cexpr", "clang-sys", "itertools 0.13.0", + "log", + "prettyplease", "proc-macro2", "quote", "regex", @@ -1997,6 +1999,12 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "futures" version = "0.3.32" @@ -3015,8 +3023,9 @@ dependencies = [ [[package]] name = "libspa" -version = "0.9.2" -source = "git+https://gitlab.freedesktop.org/pipewire/pipewire-rs.git?rev=670b0aabf04263ece5dc1e752e5d64aeae3056e8#670b0aabf04263ece5dc1e752e5d64aeae3056e8" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2909f3be29d674e7f10604aff18d1bbe1bb03c4cd61c8a8ba19c0b1d162f7d4e" dependencies = [ "bitflags 2.12.1", "cc", @@ -3030,8 +3039,9 @@ dependencies = [ [[package]] name = "libspa-sys" -version = "0.9.2" -source = "git+https://gitlab.freedesktop.org/pipewire/pipewire-rs.git?rev=670b0aabf04263ece5dc1e752e5d64aeae3056e8#670b0aabf04263ece5dc1e752e5d64aeae3056e8" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "69ad52764fca54818486f3cf75afec844d1f1a1568c24dcee25d41b1ab007dda" dependencies = [ "bindgen", "cc", @@ -4111,8 +4121,9 @@ dependencies = [ [[package]] name = "pipewire" -version = "0.9.2" -source = "git+https://gitlab.freedesktop.org/pipewire/pipewire-rs.git?rev=670b0aabf04263ece5dc1e752e5d64aeae3056e8#670b0aabf04263ece5dc1e752e5d64aeae3056e8" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8585aba8a52ad74ccc633b8e293c1dc4277976bd5d510b925533f34fd6685f38" dependencies = [ "bitflags 2.12.1", "libc", @@ -4124,8 +4135,9 @@ dependencies = [ [[package]] name = "pipewire-sys" -version = "0.9.2" -source = "git+https://gitlab.freedesktop.org/pipewire/pipewire-rs.git?rev=670b0aabf04263ece5dc1e752e5d64aeae3056e8#670b0aabf04263ece5dc1e752e5d64aeae3056e8" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2089f245b548723e60325773c27f586b7a2372c79ea941b246cd0d654706adc" dependencies = [ "bindgen", "libspa-sys", @@ -6571,6 +6583,7 @@ dependencies = [ "mint", "openxr", "ovr_overlay", + "pipewire", "prost", "pure-rust-locales", "regex", @@ -6597,6 +6610,7 @@ dependencies = [ "wayland-protocols-misc", "wayvr-ipc", "wgui", + "whisper-rs", "winit", "wlx-capture", "wlx-common", @@ -6690,6 +6704,29 @@ dependencies = [ "vulkano-shaders", ] +[[package]] +name = "whisper-rs" +version = "0.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2088172d00f936c348d6a72f488dc2660ab3f507263a195df308a3c2383229f6" +dependencies = [ + "libc", + "whisper-rs-sys", +] + +[[package]] +name = "whisper-rs-sys" +version = "0.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6986c0fe081241d391f09b9a071fbcbb59720c3563628c3c829057cf69f2a56f" +dependencies = [ + "bindgen", + "cfg-if", + "cmake", + "fs_extra", + "semver", +] + [[package]] name = "widestring" version = "1.2.1" diff --git a/Cargo.toml b/Cargo.toml index 64970436..6a4acded 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -20,6 +20,7 @@ glam = { version = "0.30.9", features = ["mint", "serde"] } idmap = "0.2.2" idmap-derive = "0.2.22" log = "0.4.29" +pipewire = { version = "0.10", features = ["v0_3_33", "v0_3_65"] } regex = "1.12.2" rust-embed = "8.9.0" serde = { version = "1", features = ["derive"] } diff --git a/wayvr/Cargo.toml b/wayvr/Cargo.toml index c4f389f3..a4bdf1ac 100644 --- a/wayvr/Cargo.toml +++ b/wayvr/Cargo.toml @@ -32,6 +32,7 @@ glam = { workspace = true, features = ["mint", "serde"] } idmap = { workspace = true, features = ["serde"] } idmap-derive.workspace = true log.workspace = true +pipewire = { workspace = true, optional = true } regex.workspace = true rust-embed.workspace = true serde = { workspace = true, features = ["rc"] } @@ -91,12 +92,13 @@ xcb = { version = "1.6.0", features = [ "as-raw-xcb-connection", ], optional = true } xkbcommon = { version = "0.8.0" } # 0.9.0 breaks keymap import on some distros +whisper-rs = { version = "0.16.0", features = ["vulkan"], optional = true } [build-dependencies] regex.workspace = true [features] -default = ["openvr", "openxr", "osc", "wayland", "x11"] +default = ["openvr", "openxr", "osc", "wayland", "whisper", "x11"] as-raw-xcb-connection = [] feat-monado-metrics = [] openvr = ["dep:json", "dep:ovr_overlay"] @@ -105,5 +107,6 @@ osc = ["dep:rosc"] pipewire = ["wlx-capture/pipewire"] uidev = ["dep:winit"] wayland = ["pipewire", "wlx-capture/wlr", "xkbcommon/wayland"] +whisper = ["dep:pipewire", "dep:whisper-rs"] x11 = ["dep:xcb", "wlx-capture/xshm", "xkbcommon/x11"] xcb = ["dep:xcb"] diff --git a/wayvr/src/subsystem/mod.rs b/wayvr/src/subsystem/mod.rs index 04d997b9..1243d666 100644 --- a/wayvr/src/subsystem/mod.rs +++ b/wayvr/src/subsystem/mod.rs @@ -3,6 +3,9 @@ pub mod hid; pub mod input; pub mod notifications; +#[cfg(feature = "whisper")] +pub mod whisper_stt; + #[cfg(feature = "osc")] pub mod osc; diff --git a/wayvr/src/subsystem/whisper_stt.rs b/wayvr/src/subsystem/whisper_stt.rs new file mode 100644 index 00000000..d1b23dcb --- /dev/null +++ b/wayvr/src/subsystem/whisper_stt.rs @@ -0,0 +1,712 @@ +// whisper_stt.rs + +use std::{ + fmt, + path::{Path, PathBuf}, + sync::{Arc, mpsc}, + thread::{self, JoinHandle}, + time::{Duration, Instant}, +}; + +use pipewire as pw; +use pw::{properties::properties, spa}; +use spa::{ + param::format::{MediaSubtype, MediaType}, + param::format_utils, + pod::Pod, +}; +use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; + +pub struct WhisperModel { + pub name: &'static str, + pub url: &'static str, + pub hash: &'static str, +} + +pub const WHISPER_MODELS: &[WhisperModel] = &[ + WhisperModel { + name: "Base Q8 (78MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q8_0.bin", + hash: "7bb89bb49ed6955013b166f1b6a6c04584a20fbe", + }, + WhisperModel { + name: "Small Q8 (252MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q8_0.bin", + hash: "bcad8a2083f4e53d648d586b7dbc0cd673d8afad", + }, + WhisperModel { + name: "Turbo Q5 (574MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin", + hash: "e050f7970618a659205450ad97eb95a18d69c9ee", + }, + WhisperModel { + name: "Turbo Q8 (874MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q8_0.bin", + hash: "01bf15bedffe9f39d65c1b6ff9b687ea91f59e0e", + }, + WhisperModel { + name: "Turbo (1.5GiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin", + hash: "4af2b29d7ec73d781377bfd1758ca957a807e941", + }, +]; + +const WHISPER_SAMPLE_RATE: usize = 16_000; +const MAX_DURATION: Duration = Duration::from_secs(30); + +#[derive(Clone, Debug)] +pub struct WhisperSttConfig { + pub model_path: PathBuf, + pub language: Option, + + pub initial_prompt: Option, + pub n_threads: i32, + + /// lower values reduce release-time lag but cost more CPU/GPU + pub partial_decode_interval_ms: u64, + + /// ignore extremely short accidental taps + pub min_audio_ms: u64, + + /// mic object name from `pw-dump`, None for the default + pub pipewire_target_object: Option, + + pub use_gpu: bool, + pub gpu_device: i32, + pub flash_attn: bool, +} + +impl WhisperSttConfig { + pub fn new(model_path: impl AsRef) -> Self { + let n_threads = std::thread::available_parallelism() + .map(|n| n.get().min(4) as i32) + .unwrap_or(4); + + Self { + model_path: model_path.as_ref().to_path_buf(), + language: None, + initial_prompt: None, + n_threads, + partial_decode_interval_ms: 700, + min_audio_ms: 250, + pipewire_target_object: None, + use_gpu: true, + gpu_device: 0, + flash_attn: false, + } + } +} + +#[derive(Debug)] +pub enum WhisperSttError { + ModelLoad(String), + Whisper(String), + PipeWire(String), + CaptureInit(String), + ThreadSpawn(String), + CaptureThreadPanicked, + AlreadyRecording, + NotRecording, +} + +impl fmt::Display for WhisperSttError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::ModelLoad(e) => write!(f, "failed to load whisper model: {e}"), + Self::Whisper(e) => write!(f, "whisper error: {e}"), + Self::PipeWire(e) => write!(f, "pipewire error: {e}"), + Self::CaptureInit(e) => write!(f, "failed to initialize capture: {e}"), + Self::ThreadSpawn(e) => write!(f, "failed to spawn thread: {e}"), + Self::CaptureThreadPanicked => write!(f, "capture thread panicked"), + Self::AlreadyRecording => write!(f, "PTT is already active"), + Self::NotRecording => write!(f, "PTT is not active"), + } + } +} + +impl std::error::Error for WhisperSttError {} + +struct StopCapture; + +struct CaptureSession { + stop_tx: pw::channel::Sender, + capture_thread: Option>, + recognizer_thread: Option>, + deadline: Instant, +} + +pub struct WhisperStt { + config: WhisperSttConfig, + ctx: Arc, + + active: Option, + finished_recognizers: Vec>, + + completed_rx: mpsc::Receiver>, + completed_tx: mpsc::Sender>, + + last_error: Option, +} + +impl WhisperStt { + pub fn new(model_path: impl AsRef) -> Result { + Self::init(WhisperSttConfig::new(model_path)) + } + + pub fn init(config: WhisperSttConfig) -> Result { + let mut ctx_params = WhisperContextParameters::default(); + ctx_params.use_gpu = config.use_gpu; + ctx_params.gpu_device = config.gpu_device; + ctx_params.flash_attn = config.flash_attn; + + let ctx = WhisperContext::new_with_params(&config.model_path, ctx_params) + .map_err(|e| WhisperSttError::ModelLoad(e.to_string()))?; + + let (completed_tx, completed_rx) = mpsc::channel(); + + Ok(Self { + config, + ctx: Arc::new(ctx), + active: None, + finished_recognizers: Vec::new(), + completed_rx, + completed_tx, + last_error: None, + }) + } + + /// starts a fresh pw capture stream and a transcription worker + pub fn ptt_start(&mut self) -> Result<(), WhisperSttError> { + self.reap_finished_recognizers(); + + if self.active.is_some() { + return Err(WhisperSttError::AlreadyRecording); + } + + let (audio_tx, audio_rx) = mpsc::channel::>(); + let (ready_tx, ready_rx) = mpsc::channel::>(); + let (stop_tx, stop_rx) = pw::channel::channel::(); + + let recognizer_thread = spawn_recognizer_thread( + Arc::clone(&self.ctx), + self.config.clone(), + audio_rx, + self.completed_tx.clone(), + )?; + + let target_object = self.config.pipewire_target_object.clone(); + + let capture_thread = thread::Builder::new() + .name("whisper-stt-pipewire-capture".to_string()) + .spawn(move || { + pipewire_capture_thread(audio_tx, stop_rx, target_object, ready_tx); + }) + .map_err(|e| WhisperSttError::ThreadSpawn(e.to_string()))?; + + match ready_rx.recv() { + Ok(Ok(())) => { + self.active = Some(CaptureSession { + stop_tx, + capture_thread: Some(capture_thread), + recognizer_thread: Some(recognizer_thread), + deadline: Instant::now() + MAX_DURATION, + }); + + Ok(()) + } + Ok(Err(e)) => { + let _ = stop_tx.send(StopCapture); + let _ = capture_thread.join(); + let _ = recognizer_thread.join(); + + Err(WhisperSttError::CaptureInit(e)) + } + Err(e) => { + let _ = stop_tx.send(StopCapture); + let _ = capture_thread.join(); + let _ = recognizer_thread.join(); + + Err(WhisperSttError::CaptureInit(e.to_string())) + } + } + } + + fn stop_active_capture(&mut self) -> Result<(), WhisperSttError> { + let Some(mut session) = self.active.take() else { + return Err(WhisperSttError::NotRecording); + }; + + let _ = session.stop_tx.send(StopCapture); + + let capture_result = if let Some(capture_thread) = session.capture_thread.take() { + capture_thread + .join() + .map_err(|_| WhisperSttError::CaptureThreadPanicked) + } else { + Ok(()) + }; + + if let Some(recognizer_thread) = session.recognizer_thread.take() { + self.finished_recognizers.push(recognizer_thread); + } + + capture_result + } + + fn drain_completed_transcriptions(&mut self) -> Option { + let mut latest = None; + + while let Ok(result) = self.completed_rx.try_recv() { + match result { + Ok(text) => { + let text = normalize_transcript(text); + if !text.is_empty() { + latest = Some(text); + } + } + Err(e) => { + self.last_error = Some(e); + } + } + } + + latest + } + + /// stops the pw stream & finalizes recognition asynchronously + /// poll `take_transcription()` from your main loop to receive transcription + pub fn ptt_end(&mut self) -> Result<(), WhisperSttError> { + self.stop_active_capture() + } + + pub fn take_transcription(&mut self) -> Option { + self.reap_finished_recognizers(); + + let latest = self.drain_completed_transcriptions(); + + if latest.is_some() { + return latest; + } + + // been recording for too long, force send a stop signal + if self + .active + .as_ref() + .is_some_and(|session| Instant::now() >= session.deadline) + { + if let Err(e) = self.stop_active_capture() { + self.last_error = Some(e.to_string()); + } + } + + return None; + } + + pub fn take_error(&mut self) -> Option { + self.last_error.take() + } + + pub fn is_recording(&self) -> bool { + self.active.is_some() + } + + fn reap_finished_recognizers(&mut self) { + let mut i = 0; + + while i < self.finished_recognizers.len() { + if self.finished_recognizers[i].is_finished() { + let handle = self.finished_recognizers.swap_remove(i); + let _ = handle.join(); + } else { + i += 1; + } + } + } +} + +impl Drop for WhisperStt { + fn drop(&mut self) { + if self.active.is_some() { + let _ = self.ptt_end(); + } + + for handle in self.finished_recognizers.drain(..) { + let _ = handle.join(); + } + } +} + +fn spawn_recognizer_thread( + ctx: Arc, + config: WhisperSttConfig, + audio_rx: mpsc::Receiver>, + completed_tx: mpsc::Sender>, +) -> Result, WhisperSttError> { + thread::Builder::new() + .name("whisper-stt-recognizer".to_string()) + .spawn(move || { + recognizer_thread(ctx, config, audio_rx, completed_tx); + }) + .map_err(|e| WhisperSttError::ThreadSpawn(e.to_string())) +} + +fn recognizer_thread( + ctx: Arc, + config: WhisperSttConfig, + audio_rx: mpsc::Receiver>, + completed_tx: mpsc::Sender>, +) { + let partial_stride_samples = + ms_to_samples(config.partial_decode_interval_ms).max(WHISPER_SAMPLE_RATE / 4); + let min_samples = ms_to_samples(config.min_audio_ms); + + let mut audio = Vec::::new(); + let mut last_decoded_len = 0usize; + let mut latest_partial = String::new(); + + while let Ok(chunk) = audio_rx.recv() { + if chunk.is_empty() { + continue; + } + + audio.extend_from_slice(&chunk); + + let enough_new_audio = + audio.len().saturating_sub(last_decoded_len) >= partial_stride_samples; + + if audio.len() >= min_samples && enough_new_audio { + match transcribe_audio(&ctx, &config, &audio) { + Ok(text) => { + latest_partial = text; + last_decoded_len = audio.len(); + } + Err(_) => { + // Do not fail the session on a speculative decode. + // The final decode after PTT end gets reported. + } + } + } + } + + if audio.len() < min_samples { + let _ = completed_tx.send(Ok(String::new())); + return; + } + + match transcribe_audio(&ctx, &config, &audio) { + Ok(text) => { + let _ = completed_tx.send(Ok(text)); + } + Err(e) if !latest_partial.trim().is_empty() => { + // Prefer a recent partial over losing the utterance completely. + let _ = completed_tx.send(Ok(latest_partial)); + let _ = completed_tx.send(Err(e.to_string())); + } + Err(e) => { + let _ = completed_tx.send(Err(e.to_string())); + } + } +} + +fn transcribe_audio( + ctx: &WhisperContext, + config: &WhisperSttConfig, + audio: &[f32], +) -> Result { + let mut params = FullParams::new(SamplingStrategy::Greedy { best_of: 1 }); + + params.set_n_threads(config.n_threads); + params.set_language(config.language.as_deref()); + params.set_no_timestamps(true); + params.set_print_special(false); + params.set_print_progress(false); + params.set_print_realtime(false); + params.set_print_timestamps(false); + + if let Some(prompt) = config.initial_prompt.as_deref() { + params.set_initial_prompt(prompt); + } + + let mut state = ctx + .create_state() + .map_err(|e| WhisperSttError::Whisper(e.to_string()))?; + + state + .full(params, audio) + .map_err(|e| WhisperSttError::Whisper(e.to_string()))?; + + let text = state + .as_iter() + .map(|segment| segment.to_string()) + .collect::>() + .join(""); + + Ok(normalize_transcript(text)) +} + +fn pipewire_capture_thread( + audio_tx: mpsc::Sender>, + stop_rx: pw::channel::Receiver, + target_object: Option, + ready_tx: mpsc::Sender>, +) { + let mut ready_tx = Some(ready_tx); + + let result = run_pipewire_capture(audio_tx, stop_rx, target_object, &mut ready_tx); + + if let Err(e) = result { + if let Some(ready_tx) = ready_tx.take() { + let _ = ready_tx.send(Err(e.to_string())); + } + } +} + +fn run_pipewire_capture( + audio_tx: mpsc::Sender>, + stop_rx: pw::channel::Receiver, + target_object: Option, + ready_tx: &mut Option>>, +) -> Result<(), WhisperSttError> { + pw::init(); + + let mainloop = pw::main_loop::MainLoopRc::new(None) + .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + + let context = pw::context::ContextRc::new(&mainloop, None) + .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + + let core = context + .connect_rc(None) + .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + + let _stop_receiver = stop_rx.attach(mainloop.loop_(), { + let mainloop = mainloop.clone(); + move |_| { + mainloop.quit(); + } + }); + + let mut props = properties! { + *pw::keys::MEDIA_TYPE => "Audio", + *pw::keys::MEDIA_CATEGORY => "Capture", + *pw::keys::MEDIA_ROLE => "Communication", + *pw::keys::APP_NAME => "WhisperStt", + }; + + if let Some(target_object) = target_object { + props.insert(*pw::keys::TARGET_OBJECT, target_object); + } + + let stream = pw::stream::StreamBox::new(&core, "WhisperStt microphone capture", props) + .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + + let user_data = AudioCaptureUserData::default(); + let audio_tx_for_callback = audio_tx.clone(); + + let _listener = stream + .add_local_listener_with_user_data(user_data) + .param_changed(|_, user_data, id, param| { + let Some(param) = param else { + return; + }; + + if id != pw::spa::param::ParamType::Format.as_raw() { + return; + } + + let Ok((media_type, media_subtype)) = format_utils::parse_format(param) else { + return; + }; + + if media_type != MediaType::Audio || media_subtype != MediaSubtype::Raw { + return; + } + + let _ = user_data.format.parse(param); + }) + .process(move |stream, user_data| { + let Some(mut buffer) = stream.dequeue_buffer() else { + return; + }; + + let datas = buffer.datas_mut(); + if datas.is_empty() { + return; + } + + let data = &mut datas[0]; + + let chunk = data.chunk(); + + let offset = chunk.offset() as usize; + let size = chunk.size() as usize; + + let Some(bytes) = data.data() else { + return; + }; + + if offset >= bytes.len() { + return; + } + + let end = offset.saturating_add(size).min(bytes.len()); + let bytes = &bytes[offset..end]; + + let channels = (user_data.format.channels() as usize).max(1); + let input_rate = { + let rate = user_data.format.rate() as usize; + if rate == 0 { 48_000 } else { rate } + }; + + let resampled = user_data + .resampler + .push_interleaved_f32le_mono_16k(bytes, channels, input_rate); + + if !resampled.is_empty() { + let _ = audio_tx_for_callback.send(resampled); + } + }) + .register() + .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + + let mut audio_info = spa::param::audio::AudioInfoRaw::new(); + audio_info.set_format(spa::param::audio::AudioFormat::F32LE); + + let obj = pw::spa::pod::Object { + type_: pw::spa::utils::SpaTypes::ObjectParamFormat.as_raw(), + id: pw::spa::param::ParamType::EnumFormat.as_raw(), + properties: audio_info.into(), + }; + + let values: Vec = pw::spa::pod::serialize::PodSerializer::serialize( + std::io::Cursor::new(Vec::new()), + &pw::spa::pod::Value::Object(obj), + ) + .map_err(|e| WhisperSttError::PipeWire(e.to_string()))? + .0 + .into_inner(); + + let pod = Pod::from_bytes(&values).ok_or_else(|| { + WhisperSttError::PipeWire("failed to parse serialized PipeWire pod".to_string()) + })?; + + let mut params = [pod]; + + stream + .connect( + spa::utils::Direction::Input, + None, + pw::stream::StreamFlags::AUTOCONNECT + | pw::stream::StreamFlags::MAP_BUFFERS + | pw::stream::StreamFlags::RT_PROCESS, + &mut params, + ) + .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + + if let Some(ready_tx) = ready_tx.take() { + let _ = ready_tx.send(Ok(())); + } + + mainloop.run(); + + Ok(()) +} + +#[derive(Default)] +struct AudioCaptureUserData { + format: spa::param::audio::AudioInfoRaw, + resampler: StreamingResampler, +} + +#[derive(Default)] +struct StreamingResampler { + pending: Vec, + position: f64, + input_rate: usize, +} + +impl StreamingResampler { + fn push_interleaved_f32le_mono_16k( + &mut self, + bytes: &[u8], + channels: usize, + input_rate: usize, + ) -> Vec { + if channels == 0 || input_rate == 0 { + return Vec::new(); + } + + if self.input_rate != input_rate { + self.pending.clear(); + self.position = 0.0; + self.input_rate = input_rate; + } + + let frame_bytes = channels * std::mem::size_of::(); + if frame_bytes == 0 { + return Vec::new(); + } + + let frames = bytes.len() / frame_bytes; + if frames == 0 { + return Vec::new(); + } + + let mut mono = Vec::with_capacity(frames); + + for frame in 0..frames { + let frame_start = frame * frame_bytes; + let mut sum = 0.0f32; + + for ch in 0..channels { + let sample_start = frame_start + ch * 4; + + let sample = f32::from_le_bytes([ + bytes[sample_start], + bytes[sample_start + 1], + bytes[sample_start + 2], + bytes[sample_start + 3], + ]); + + sum += sample; + } + + mono.push(sum / channels as f32); + } + + self.pending.extend_from_slice(&mono); + + let step = input_rate as f64 / WHISPER_SAMPLE_RATE as f64; + let mut out = Vec::with_capacity( + ((self.pending.len() as f64 - self.position) / step).max(0.0) as usize, + ); + + while self.position + 1.0 < self.pending.len() as f64 { + let i = self.position.floor() as usize; + let frac = (self.position - i as f64) as f32; + + let a = self.pending[i]; + let b = self.pending[i + 1]; + + out.push(a + (b - a) * frac); + + self.position += step; + } + + let drop_count = self.position.floor() as usize; + if drop_count > 0 { + self.pending.drain(..drop_count); + self.position -= drop_count as f64; + } + + out + } +} + +fn ms_to_samples(ms: u64) -> usize { + ((ms as usize) * WHISPER_SAMPLE_RATE) / 1000 +} + +fn normalize_transcript(text: String) -> String { + text.split_whitespace().collect::>().join(" ") +} diff --git a/wlx-capture/Cargo.toml b/wlx-capture/Cargo.toml index 5294556e..b338f85d 100644 --- a/wlx-capture/Cargo.toml +++ b/wlx-capture/Cargo.toml @@ -32,10 +32,7 @@ drm-fourcc = "2.2.0" idmap = { workspace = true } libc = "0.2.178" log = { workspace = true } -pipewire = { git = "https://gitlab.freedesktop.org/pipewire/pipewire-rs.git", rev = "670b0aabf04263ece5dc1e752e5d64aeae3056e8", features = [ - "v0_3_33", "v0_3_65", -], optional = true } -#pipewire = { version = "0.8.0", features = ["v0_3_33"], optional = true } +pipewire = { workspace = true, optional = true } rxscreen = { version = "0.1.7", features = [ "shm", "xrandr",