From 5ec1866613f15c51e38c6bf75c3d6a99377f60bb Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Fri, 3 Jul 2026 19:06:36 +0900 Subject: [PATCH] whisper to use rodio --- Cargo.lock | 123 ++++++------ wayvr/src/subsystem/whisper_stt.rs | 291 +++++++++++------------------ wlx-common/Cargo.toml | 6 +- wlx-common/src/audio.rs | 8 +- 4 files changed, 178 insertions(+), 250 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 4ce2e84f..641fbb61 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -87,9 +87,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "alsa" -version = "0.9.1" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43" +checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d" dependencies = [ "alsa-sys", "bitflags 2.12.1", @@ -99,9 +99,9 @@ dependencies = [ [[package]] name = "alsa-sys" -version = "0.3.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527" +checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04" dependencies = [ "libc", "pkg-config", @@ -748,6 +748,15 @@ dependencies = [ "objc2 0.5.2", ] +[[package]] +name = "block2" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5" +dependencies = [ + "objc2 0.6.4", +] + [[package]] name = "blocking" version = "1.6.2" @@ -1180,11 +1189,11 @@ dependencies = [ [[package]] name = "coreaudio-rs" -version = "0.13.0" +version = "0.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aae284fbaf7d27aa0e292f7677dfbe26503b0d555026f702940805a630eac17" +checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58" dependencies = [ - "bitflags 1.3.2", + "bitflags 2.12.1", "libc", "objc2-audio-toolbox", "objc2-core-audio", @@ -1218,9 +1227,9 @@ dependencies = [ [[package]] name = "cpal" -version = "0.16.0" +version = "0.17.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbd307f43cc2a697e2d1f8bc7a1d824b5269e052209e28883e5bc04d095aaa3f" +checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7" dependencies = [ "alsa", "coreaudio-rs", @@ -1233,13 +1242,17 @@ dependencies = [ "ndk-context", "num-derive", "num-traits", + "objc2 0.6.4", "objc2-audio-toolbox", + "objc2-avf-audio", "objc2-core-audio", "objc2-core-audio-types", + "objc2-core-foundation", + "objc2-foundation 0.3.2", "wasm-bindgen", "wasm-bindgen-futures", "web-sys", - "windows 0.54.0", + "windows", ] [[package]] @@ -3135,9 +3148,9 @@ dependencies = [ [[package]] name = "mach2" -version = "0.4.3" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" +checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea" dependencies = [ "libc", ] @@ -3502,7 +3515,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "libc", "objc2 0.5.2", "objc2-core-data", @@ -3526,6 +3539,16 @@ dependencies = [ "objc2-foundation 0.3.2", ] +[[package]] +name = "objc2-avf-audio" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be" +dependencies = [ + "objc2 0.6.4", + "objc2-foundation 0.3.2", +] + [[package]] name = "objc2-cloud-kit" version = "0.2.2" @@ -3533,7 +3556,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-core-location", "objc2-foundation 0.2.2", @@ -3545,7 +3568,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a5ff520e9c33812fd374d8deecef01d4a840e7b41862d849513de77e44aa4889" dependencies = [ - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", ] @@ -3560,6 +3583,7 @@ dependencies = [ "objc2 0.6.4", "objc2-core-audio-types", "objc2-core-foundation", + "objc2-foundation 0.3.2", ] [[package]] @@ -3579,7 +3603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", ] @@ -3591,7 +3615,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536" dependencies = [ "bitflags 2.12.1", + "block2 0.6.2", "dispatch2", + "libc", "objc2 0.6.4", ] @@ -3601,7 +3627,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80" dependencies = [ - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", "objc2-metal 0.2.2", @@ -3613,7 +3639,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "000cfee34e683244f284252ee206a27953279d370e309649dc3ee317b37e5781" dependencies = [ - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-contacts", "objc2-foundation 0.2.2", @@ -3632,7 +3658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "dispatch", "libc", "objc2 0.5.2", @@ -3645,6 +3671,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272" dependencies = [ "bitflags 2.12.1", + "block2 0.6.2", + "libc", "objc2 0.6.4", "objc2-core-foundation", ] @@ -3665,7 +3693,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398" dependencies = [ - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-app-kit", "objc2-foundation 0.2.2", @@ -3678,7 +3706,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", ] @@ -3701,7 +3729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", "objc2-metal 0.2.2", @@ -3737,7 +3765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-cloud-kit", "objc2-core-data", @@ -3757,7 +3785,7 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "44fa5f9748dbfe1ca6c0b79ad20725a11eca7c2218bceb4b005cb1be26273bfe" dependencies = [ - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-foundation 0.2.2", ] @@ -3769,7 +3797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3" dependencies = [ "bitflags 2.12.1", - "block2", + "block2 0.5.1", "objc2 0.5.2", "objc2-core-location", "objc2-foundation 0.2.2", @@ -4772,15 +4800,17 @@ dependencies = [ [[package]] name = "rodio" -version = "0.21.1" +version = "0.22.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e40ecf59e742e03336be6a3d53755e789fd05a059fa22dfa0ed624722319e183" +checksum = "d0a536bb79db59098ef71a4dd4246c02eb87b316deceb1b68e0cde7167ec01eb" dependencies = [ "cpal", "dasp_sample", "hound", "num-rational", + "rtrb", "symphonia", + "thiserror 2.0.18", ] [[package]] @@ -4832,6 +4862,12 @@ dependencies = [ "memchr", ] +[[package]] +name = "rtrb" +version = "0.3.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4ade083ccbb4bf536df69d1f6432cc23deb7acccff86b183f3923a6fd56a1153" + [[package]] name = "rust-embed" version = "8.11.0" @@ -5637,7 +5673,7 @@ dependencies = [ "ntapi", "objc2-core-foundation", "objc2-io-kit", - "windows 0.61.3", + "windows", ] [[package]] @@ -6764,16 +6800,6 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49" -dependencies = [ - "windows-core 0.54.0", - "windows-targets 0.52.6", -] - [[package]] name = "windows" version = "0.61.3" @@ -6796,16 +6822,6 @@ dependencies = [ "windows-core 0.61.2", ] -[[package]] -name = "windows-core" -version = "0.54.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65" -dependencies = [ - "windows-result 0.1.2", - "windows-targets 0.52.6", -] - [[package]] name = "windows-core" version = "0.61.2" @@ -6887,15 +6903,6 @@ dependencies = [ "windows-link 0.1.3", ] -[[package]] -name = "windows-result" -version = "0.1.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-result" version = "0.3.4" @@ -7108,7 +7115,7 @@ dependencies = [ "android-activity", "atomic-waker", "bitflags 2.12.1", - "block2", + "block2 0.5.1", "bytemuck", "calloop 0.13.0", "cfg_aliases", diff --git a/wayvr/src/subsystem/whisper_stt.rs b/wayvr/src/subsystem/whisper_stt.rs index b5a36931..f0499931 100644 --- a/wayvr/src/subsystem/whisper_stt.rs +++ b/wayvr/src/subsystem/whisper_stt.rs @@ -1,5 +1,3 @@ -// whisper_stt.rs - use std::{ fmt, path::{Path, PathBuf}, @@ -8,14 +6,11 @@ use std::{ time::{Duration, Instant}, }; -use pipewire as pw; -use pw::{properties::properties, spa}; -use spa::{ - param::format::{MediaSubtype, MediaType}, - param::format_utils, - pod::Pod, -}; use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; +use wlx_common::audio::rodio::{ + Source, + microphone::{MicrophoneBuilder, available_inputs}, +}; const WHISPER_SAMPLE_RATE: usize = 16_000; const MAX_DURATION: Duration = Duration::from_secs(30); @@ -34,8 +29,8 @@ pub struct WhisperSttConfig { /// ignore extremely short accidental taps pub min_audio_ms: u64, - /// mic object name from `pw-dump`, None for the default - pub pipewire_target_object: Option, + /// force a specific recording device; see `rodio::microphone::available_inputs()` + pub rodio_input_device_name: Option, pub use_gpu: bool, pub gpu_device: i32, @@ -55,7 +50,7 @@ impl WhisperSttConfig { n_threads, partial_decode_interval_ms: 700, min_audio_ms: 250, - pipewire_target_object: None, + rodio_input_device_name: None, use_gpu: true, gpu_device: 0, flash_attn: false, @@ -67,7 +62,7 @@ impl WhisperSttConfig { pub enum WhisperSttError { ModelLoad(String), Whisper(String), - PipeWire(String), + Rodio(String), CaptureInit(String), ThreadSpawn(String), CaptureThreadPanicked, @@ -80,7 +75,7 @@ impl fmt::Display for WhisperSttError { match self { Self::ModelLoad(e) => write!(f, "failed to load whisper model: {e}"), Self::Whisper(e) => write!(f, "whisper error: {e}"), - Self::PipeWire(e) => write!(f, "pipewire error: {e}"), + Self::Rodio(e) => write!(f, "rodio error: {e}"), Self::CaptureInit(e) => write!(f, "failed to initialize capture: {e}"), Self::ThreadSpawn(e) => write!(f, "failed to spawn thread: {e}"), Self::CaptureThreadPanicked => write!(f, "capture thread panicked"), @@ -95,7 +90,7 @@ impl std::error::Error for WhisperSttError {} struct StopCapture; struct CaptureSession { - stop_tx: pw::channel::Sender, + stop_tx: mpsc::Sender, capture_thread: Option>, recognizer_thread: Option>, deadline: Instant, @@ -141,7 +136,7 @@ impl WhisperStt { }) } - /// starts a fresh pw capture stream and a transcription worker + /// starts a fresh capture stream and a transcription worker pub fn ptt_start(&mut self) -> Result<(), WhisperSttError> { self.reap_finished_recognizers(); @@ -151,7 +146,7 @@ impl WhisperStt { let (audio_tx, audio_rx) = mpsc::channel::>(); let (ready_tx, ready_rx) = mpsc::channel::>(); - let (stop_tx, stop_rx) = pw::channel::channel::(); + let (stop_tx, stop_rx) = mpsc::channel::(); let recognizer_thread = spawn_recognizer_thread( Arc::clone(&self.ctx), @@ -160,12 +155,12 @@ impl WhisperStt { self.completed_tx.clone(), )?; - let target_object = self.config.pipewire_target_object.clone(); + let input_device_name = self.config.rodio_input_device_name.clone(); let capture_thread = thread::Builder::new() - .name("whisper-stt-pipewire-capture".to_string()) + .name("whisper-stt-rodio-capture".to_string()) .spawn(move || { - pipewire_capture_thread(audio_tx, stop_rx, target_object, ready_tx); + rodio_capture_thread(audio_tx, stop_rx, input_device_name, ready_tx); }) .map_err(|e| WhisperSttError::ThreadSpawn(e.to_string()))?; @@ -265,7 +260,7 @@ impl WhisperStt { } } - return None; + None } pub fn take_error(&mut self) -> Option { @@ -347,8 +342,8 @@ fn recognizer_thread( last_decoded_len = audio.len(); } Err(_) => { - // Do not fail the session on a speculative decode. - // The final decode after PTT end gets reported. + // do not fail the session on a speculative decode + // the final decode after PTT end gets reported } } } @@ -410,15 +405,15 @@ fn transcribe_audio( Ok(normalize_transcript(text)) } -fn pipewire_capture_thread( +fn rodio_capture_thread( audio_tx: mpsc::Sender>, - stop_rx: pw::channel::Receiver, - target_object: Option, + stop_rx: mpsc::Receiver, + input_device_name: Option, ready_tx: mpsc::Sender>, ) { let mut ready_tx = Some(ready_tx); - let result = run_pipewire_capture(audio_tx, stop_rx, target_object, &mut ready_tx); + let result = run_rodio_capture(audio_tx, stop_rx, input_device_name, &mut ready_tx); if let Err(e) = result { if let Some(ready_tx) = ready_tx.take() { @@ -427,163 +422,105 @@ fn pipewire_capture_thread( } } -fn run_pipewire_capture( +fn run_rodio_capture( audio_tx: mpsc::Sender>, - stop_rx: pw::channel::Receiver, - target_object: Option, + stop_rx: mpsc::Receiver, + input_device_name: Option, ready_tx: &mut Option>>, ) -> Result<(), WhisperSttError> { - pw::init(); + let builder = MicrophoneBuilder::new(); - let mainloop = pw::main_loop::MainLoopRc::new(None) - .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + let builder = if let Some(input_device_name) = input_device_name { + let inputs = available_inputs().map_err(|e| WhisperSttError::Rodio(e.to_string()))?; + let input_device_name_lower = input_device_name.to_lowercase(); - let context = pw::context::ContextRc::new(&mainloop, None) - .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + let input = inputs + .into_iter() + .find(|input| { + input + .to_string() + .to_lowercase() + .contains(&input_device_name_lower) + }) + .ok_or_else(|| { + WhisperSttError::Rodio(format!( + "no rodio input device matched {input_device_name:?}" + )) + })?; - let core = context - .connect_rc(None) - .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; - - let _stop_receiver = stop_rx.attach(mainloop.loop_(), { - let mainloop = mainloop.clone(); - move |_| { - mainloop.quit(); - } - }); - - let mut props = properties! { - *pw::keys::MEDIA_TYPE => "Audio", - *pw::keys::MEDIA_CATEGORY => "Capture", - *pw::keys::MEDIA_ROLE => "Communication", - *pw::keys::APP_NAME => "WhisperStt", + builder + .device(input) + .map_err(|e| WhisperSttError::Rodio(e.to_string()))? + } else { + builder + .default_device() + .map_err(|e| WhisperSttError::Rodio(e.to_string()))? }; - if let Some(target_object) = target_object { - props.insert(*pw::keys::TARGET_OBJECT, target_object); - } + let builder = builder + .default_config() + .map_err(|e| WhisperSttError::Rodio(e.to_string()))? + .prefer_channel_counts([ + 1.try_into().expect("not zero"), + 2.try_into().expect("not zero"), + ]) + .prefer_sample_rates([ + 16_000.try_into().expect("not zero"), + 32_000.try_into().expect("not zero"), + 48_000.try_into().expect("not zero"), + ]) + .prefer_buffer_sizes(512..); - let stream = pw::stream::StreamBox::new(&core, "WhisperStt microphone capture", props) - .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + let mut mic = builder + .open_stream() + .map_err(|e| WhisperSttError::Rodio(e.to_string()))?; - let user_data = AudioCaptureUserData::default(); - let audio_tx_for_callback = audio_tx.clone(); - - let _listener = stream - .add_local_listener_with_user_data(user_data) - .param_changed(|_, user_data, id, param| { - let Some(param) = param else { - return; - }; - - if id != pw::spa::param::ParamType::Format.as_raw() { - return; - } - - let Ok((media_type, media_subtype)) = format_utils::parse_format(param) else { - return; - }; - - if media_type != MediaType::Audio || media_subtype != MediaSubtype::Raw { - return; - } - - let _ = user_data.format.parse(param); - }) - .process(move |stream, user_data| { - let Some(mut buffer) = stream.dequeue_buffer() else { - return; - }; - - let datas = buffer.datas_mut(); - if datas.is_empty() { - return; - } - - let data = &mut datas[0]; - - let chunk = data.chunk(); - - let offset = chunk.offset() as usize; - let size = chunk.size() as usize; - - let Some(bytes) = data.data() else { - return; - }; - - if offset >= bytes.len() { - return; - } - - let end = offset.saturating_add(size).min(bytes.len()); - let bytes = &bytes[offset..end]; - - let channels = (user_data.format.channels() as usize).max(1); - let input_rate = { - let rate = user_data.format.rate() as usize; - if rate == 0 { 48_000 } else { rate } - }; - - let resampled = user_data - .resampler - .push_interleaved_f32le_mono_16k(bytes, channels, input_rate); - - if !resampled.is_empty() { - let _ = audio_tx_for_callback.send(resampled); - } - }) - .register() - .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; - - let mut audio_info = spa::param::audio::AudioInfoRaw::new(); - audio_info.set_format(spa::param::audio::AudioFormat::F32LE); - - let obj = pw::spa::pod::Object { - type_: pw::spa::utils::SpaTypes::ObjectParamFormat.as_raw(), - id: pw::spa::param::ParamType::EnumFormat.as_raw(), - properties: audio_info.into(), - }; - - let values: Vec = pw::spa::pod::serialize::PodSerializer::serialize( - std::io::Cursor::new(Vec::new()), - &pw::spa::pod::Value::Object(obj), - ) - .map_err(|e| WhisperSttError::PipeWire(e.to_string()))? - .0 - .into_inner(); - - let pod = Pod::from_bytes(&values).ok_or_else(|| { - WhisperSttError::PipeWire("failed to parse serialized PipeWire pod".to_string()) - })?; - - let mut params = [pod]; - - stream - .connect( - spa::utils::Direction::Input, - None, - pw::stream::StreamFlags::AUTOCONNECT - | pw::stream::StreamFlags::MAP_BUFFERS - | pw::stream::StreamFlags::RT_PROCESS, - &mut params, - ) - .map_err(|e| WhisperSttError::PipeWire(e.to_string()))?; + let channels = mic.channels().get() as usize; + let input_rate = mic.sample_rate().get() as usize; if let Some(ready_tx) = ready_tx.take() { let _ = ready_tx.send(Ok(())); } - mainloop.run(); + let mut resampler = StreamingResampler::default(); + let mut interleaved = Vec::new(); + + // ~20 ms of input frames; whisper still receives 16 kHz mono chunks + let chunk_input_samples = ((input_rate / 50).max(1)) * channels.max(1); + + 'capture: loop { + if stop_rx.try_recv().is_ok() { + break; + } + + interleaved.clear(); + + while interleaved.len() < chunk_input_samples { + if stop_rx.try_recv().is_ok() { + break 'capture; + } + + let Some(sample) = mic.next() else { + return Err(WhisperSttError::Rodio( + "microphone stream ended unexpectedly".to_string(), + )); + }; + + // Rodio's default sample type is f32. This cast also keeps the code + // compiling if the crate is built with rodio's `64bit` feature. + interleaved.push(sample as f32); + } + + let resampled = resampler.push_interleaved_mono_16k(&interleaved, channels, input_rate); + + if !resampled.is_empty() && audio_tx.send(resampled).is_err() { + break; + } + } Ok(()) } -#[derive(Default)] -struct AudioCaptureUserData { - format: spa::param::audio::AudioInfoRaw, - resampler: StreamingResampler, -} - #[derive(Default)] struct StreamingResampler { pending: Vec, @@ -592,9 +529,9 @@ struct StreamingResampler { } impl StreamingResampler { - fn push_interleaved_f32le_mono_16k( + fn push_interleaved_mono_16k( &mut self, - bytes: &[u8], + samples: &[f32], channels: usize, input_rate: usize, ) -> Vec { @@ -608,12 +545,7 @@ impl StreamingResampler { self.input_rate = input_rate; } - let frame_bytes = channels * std::mem::size_of::(); - if frame_bytes == 0 { - return Vec::new(); - } - - let frames = bytes.len() / frame_bytes; + let frames = samples.len() / channels; if frames == 0 { return Vec::new(); } @@ -621,20 +553,11 @@ impl StreamingResampler { let mut mono = Vec::with_capacity(frames); for frame in 0..frames { - let frame_start = frame * frame_bytes; + let frame_start = frame * channels; let mut sum = 0.0f32; for ch in 0..channels { - let sample_start = frame_start + ch * 4; - - let sample = f32::from_le_bytes([ - bytes[sample_start], - bytes[sample_start + 1], - bytes[sample_start + 2], - bytes[sample_start + 3], - ]); - - sum += sample; + sum += samples[frame_start + ch]; } mono.push(sum / channels as f32); diff --git a/wlx-common/Cargo.toml b/wlx-common/Cargo.toml index c98de46c..da4f3665 100644 --- a/wlx-common/Cargo.toml +++ b/wlx-common/Cargo.toml @@ -25,11 +25,7 @@ xdg.workspace = true chrono = "0.4.42" smol = { workspace = true } wgui = { path = "../wgui/" } -rodio = { version = "0.21.1", default-features = false, features = [ - "playback", - "mp3", - "hound", -] } +rodio = { version = "0.22.2", default-features = false, features = ["playback", "mp3", "hound", "recording"] } glob = "0.3.3" walkdir = "2.5.0" rust-ini = "0.21.3" diff --git a/wlx-common/src/audio.rs b/wlx-common/src/audio.rs index 946d47ab..00b0d646 100644 --- a/wlx-common/src/audio.rs +++ b/wlx-common/src/audio.rs @@ -5,8 +5,10 @@ use wgui::{assets::AssetProvider, sound::WguiSoundType}; use std::io::Read; +pub use rodio; + pub struct AudioSystem { - audio_stream: Option, + audio_stream: Option, first_try: bool, } @@ -100,10 +102,10 @@ impl AudioSystem { } } - fn get_handle(&mut self) -> Option<&rodio::OutputStream> { + fn get_handle(&mut self) -> Option<&rodio::MixerDeviceSink> { if self.audio_stream.is_none() && self.first_try { self.first_try = false; - if let Ok(stream) = rodio::OutputStreamBuilder::open_default_stream() { + if let Ok(stream) = rodio::DeviceSinkBuilder::open_default_sink() { self.audio_stream = Some(stream); } else { log::error!("Failed to open audio stream. Audio will not work.");