mirror of https://github.com/wayvr-org/wayvr.git
whisper to use rodio
This commit is contained in:
galister
parent
d6713176c6
commit
893df87dec
|
|
@ -87,9 +87,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
|||
|
||||
[[package]]
|
||||
name = "alsa"
|
||||
version = "0.9.1"
|
||||
version = "0.11.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed7572b7ba83a31e20d1b48970ee402d2e3e0537dcfe0a3ff4d6eb7508617d43"
|
||||
checksum = "812947049edcd670a82cd5c73c3661d2e58468577ba8489de58e1a73c04cbd5d"
|
||||
dependencies = [
|
||||
"alsa-sys",
|
||||
"bitflags 2.12.1",
|
||||
|
|
@ -99,9 +99,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "alsa-sys"
|
||||
version = "0.3.1"
|
||||
version = "0.4.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "db8fee663d06c4e303404ef5f40488a53e062f89ba8bfed81f42325aafad1527"
|
||||
checksum = "ad7569085a265dd3f607ebecce7458eaab2132a84393534c95b18dcbc3f31e04"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"pkg-config",
|
||||
|
|
@ -748,6 +748,15 @@ dependencies = [
|
|||
"objc2 0.5.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block2"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5"
|
||||
dependencies = [
|
||||
"objc2 0.6.4",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blocking"
|
||||
version = "1.6.2"
|
||||
|
|
@ -1180,11 +1189,11 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "coreaudio-rs"
|
||||
version = "0.13.0"
|
||||
version = "0.14.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1aae284fbaf7d27aa0e292f7677dfbe26503b0d555026f702940805a630eac17"
|
||||
checksum = "7d5d7dca3ebcf65a035582c9ad4385371a9d9ee6537474d2a278f4e1e475bb58"
|
||||
dependencies = [
|
||||
"bitflags 1.3.2",
|
||||
"bitflags 2.12.1",
|
||||
"libc",
|
||||
"objc2-audio-toolbox",
|
||||
"objc2-core-audio",
|
||||
|
|
@ -1218,9 +1227,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "cpal"
|
||||
version = "0.16.0"
|
||||
version = "0.17.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbd307f43cc2a697e2d1f8bc7a1d824b5269e052209e28883e5bc04d095aaa3f"
|
||||
checksum = "d8942da362c0f0d895d7cac616263f2f9424edc5687364dfd1d25ef7eba506d7"
|
||||
dependencies = [
|
||||
"alsa",
|
||||
"coreaudio-rs",
|
||||
|
|
@ -1233,13 +1242,17 @@ dependencies = [
|
|||
"ndk-context",
|
||||
"num-derive",
|
||||
"num-traits",
|
||||
"objc2 0.6.4",
|
||||
"objc2-audio-toolbox",
|
||||
"objc2-avf-audio",
|
||||
"objc2-core-audio",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation 0.3.2",
|
||||
"wasm-bindgen",
|
||||
"wasm-bindgen-futures",
|
||||
"web-sys",
|
||||
"windows 0.54.0",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3135,9 +3148,9 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "mach2"
|
||||
version = "0.4.3"
|
||||
version = "0.5.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44"
|
||||
checksum = "6a1b95cd5421ec55b445b5ae102f5ea0e768de1f82bd3001e11f426c269c3aea"
|
||||
dependencies = [
|
||||
"libc",
|
||||
]
|
||||
|
|
@ -3502,7 +3515,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "e4e89ad9e3d7d297152b17d39ed92cd50ca8063a89a9fa569046d41568891eff"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"libc",
|
||||
"objc2 0.5.2",
|
||||
"objc2-core-data",
|
||||
|
|
@ -3526,6 +3539,16 @@ dependencies = [
|
|||
"objc2-foundation 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-avf-audio"
|
||||
version = "0.3.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "13a380031deed8e99db00065c45937da434ca987c034e13b87e4441f9e4090be"
|
||||
dependencies = [
|
||||
"objc2 0.6.4",
|
||||
"objc2-foundation 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-cloud-kit"
|
||||
version = "0.2.2"
|
||||
|
|
@ -3533,7 +3556,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "74dd3b56391c7a0596a295029734d3c1c5e7e510a4cb30245f8221ccea96b009"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-core-location",
|
||||
"objc2-foundation 0.2.2",
|
||||
|
|
@ -3545,7 +3568,7 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5ff520e9c33812fd374d8deecef01d4a840e7b41862d849513de77e44aa4889"
|
||||
dependencies = [
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-foundation 0.2.2",
|
||||
]
|
||||
|
|
@ -3560,6 +3583,7 @@ dependencies = [
|
|||
"objc2 0.6.4",
|
||||
"objc2-core-audio-types",
|
||||
"objc2-core-foundation",
|
||||
"objc2-foundation 0.3.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -3579,7 +3603,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "617fbf49e071c178c0b24c080767db52958f716d9eabdf0890523aeae54773ef"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-foundation 0.2.2",
|
||||
]
|
||||
|
|
@ -3591,7 +3615,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2 0.6.2",
|
||||
"dispatch2",
|
||||
"libc",
|
||||
"objc2 0.6.4",
|
||||
]
|
||||
|
||||
|
|
@ -3601,7 +3627,7 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "55260963a527c99f1819c4f8e3b47fe04f9650694ef348ffd2227e8196d34c80"
|
||||
dependencies = [
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-foundation 0.2.2",
|
||||
"objc2-metal 0.2.2",
|
||||
|
|
@ -3613,7 +3639,7 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "000cfee34e683244f284252ee206a27953279d370e309649dc3ee317b37e5781"
|
||||
dependencies = [
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-contacts",
|
||||
"objc2-foundation 0.2.2",
|
||||
|
|
@ -3632,7 +3658,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "0ee638a5da3799329310ad4cfa62fbf045d5f56e3ef5ba4149e7452dcf89d5a8"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"dispatch",
|
||||
"libc",
|
||||
"objc2 0.5.2",
|
||||
|
|
@ -3645,6 +3671,8 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "e3e0adef53c21f888deb4fa59fc59f7eb17404926ee8a6f59f5df0fd7f9f3272"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2 0.6.2",
|
||||
"libc",
|
||||
"objc2 0.6.4",
|
||||
"objc2-core-foundation",
|
||||
]
|
||||
|
|
@ -3665,7 +3693,7 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1a1ae721c5e35be65f01a03b6d2ac13a54cb4fa70d8a5da293d7b0020261398"
|
||||
dependencies = [
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-app-kit",
|
||||
"objc2-foundation 0.2.2",
|
||||
|
|
@ -3678,7 +3706,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "dd0cba1276f6023976a406a14ffa85e1fdd19df6b0f737b063b95f6c8c7aadd6"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-foundation 0.2.2",
|
||||
]
|
||||
|
|
@ -3701,7 +3729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "e42bee7bff906b14b167da2bac5efe6b6a07e6f7c0a21a7308d40c960242dc7a"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-foundation 0.2.2",
|
||||
"objc2-metal 0.2.2",
|
||||
|
|
@ -3737,7 +3765,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "b8bb46798b20cd6b91cbd113524c490f1686f4c4e8f49502431415f3512e2b6f"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-cloud-kit",
|
||||
"objc2-core-data",
|
||||
|
|
@ -3757,7 +3785,7 @@ version = "0.2.2"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "44fa5f9748dbfe1ca6c0b79ad20725a11eca7c2218bceb4b005cb1be26273bfe"
|
||||
dependencies = [
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-foundation 0.2.2",
|
||||
]
|
||||
|
|
@ -3769,7 +3797,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
|||
checksum = "76cfcbf642358e8689af64cee815d139339f3ed8ad05103ed5eaf73db8d84cb3"
|
||||
dependencies = [
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"objc2 0.5.2",
|
||||
"objc2-core-location",
|
||||
"objc2-foundation 0.2.2",
|
||||
|
|
@ -4772,15 +4800,17 @@ dependencies = [
|
|||
|
||||
[[package]]
|
||||
name = "rodio"
|
||||
version = "0.21.1"
|
||||
version = "0.22.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e40ecf59e742e03336be6a3d53755e789fd05a059fa22dfa0ed624722319e183"
|
||||
checksum = "d0a536bb79db59098ef71a4dd4246c02eb87b316deceb1b68e0cde7167ec01eb"
|
||||
dependencies = [
|
||||
"cpal",
|
||||
"dasp_sample",
|
||||
"hound",
|
||||
"num-rational",
|
||||
"rtrb",
|
||||
"symphonia",
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -4832,6 +4862,12 @@ dependencies = [
|
|||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rtrb"
|
||||
version = "0.3.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ade083ccbb4bf536df69d1f6432cc23deb7acccff86b183f3923a6fd56a1153"
|
||||
|
||||
[[package]]
|
||||
name = "rust-embed"
|
||||
version = "8.11.0"
|
||||
|
|
@ -5637,7 +5673,7 @@ dependencies = [
|
|||
"ntapi",
|
||||
"objc2-core-foundation",
|
||||
"objc2-io-kit",
|
||||
"windows 0.61.3",
|
||||
"windows",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
|
@ -6764,16 +6800,6 @@ version = "0.4.0"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.54.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9252e5725dbed82865af151df558e754e4a3c2c30818359eb17465f1346a1b49"
|
||||
dependencies = [
|
||||
"windows-core 0.54.0",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows"
|
||||
version = "0.61.3"
|
||||
|
|
@ -6796,16 +6822,6 @@ dependencies = [
|
|||
"windows-core 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.54.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12661b9c89351d684a50a8a643ce5f608e20243b9fb84687800163429f161d65"
|
||||
dependencies = [
|
||||
"windows-result 0.1.2",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-core"
|
||||
version = "0.61.2"
|
||||
|
|
@ -6887,15 +6903,6 @@ dependencies = [
|
|||
"windows-link 0.1.3",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.1.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5e383302e8ec8515204254685643de10811af0ed97ea37210dc26fb0032647f8"
|
||||
dependencies = [
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "windows-result"
|
||||
version = "0.3.4"
|
||||
|
|
@ -7108,7 +7115,7 @@ dependencies = [
|
|||
"android-activity",
|
||||
"atomic-waker",
|
||||
"bitflags 2.12.1",
|
||||
"block2",
|
||||
"block2 0.5.1",
|
||||
"bytemuck",
|
||||
"calloop 0.13.0",
|
||||
"cfg_aliases",
|
||||
|
|
|
|||
|
|
@ -1,5 +1,3 @@
|
|||
// whisper_stt.rs
|
||||
|
||||
use std::{
|
||||
fmt,
|
||||
path::{Path, PathBuf},
|
||||
|
|
@ -8,14 +6,11 @@ use std::{
|
|||
time::{Duration, Instant},
|
||||
};
|
||||
|
||||
use pipewire as pw;
|
||||
use pw::{properties::properties, spa};
|
||||
use spa::{
|
||||
param::format::{MediaSubtype, MediaType},
|
||||
param::format_utils,
|
||||
pod::Pod,
|
||||
};
|
||||
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
|
||||
use wlx_common::audio::rodio::{
|
||||
Source,
|
||||
microphone::{MicrophoneBuilder, available_inputs},
|
||||
};
|
||||
|
||||
const WHISPER_SAMPLE_RATE: usize = 16_000;
|
||||
const MAX_DURATION: Duration = Duration::from_secs(30);
|
||||
|
|
@ -34,8 +29,8 @@ pub struct WhisperSttConfig {
|
|||
/// ignore extremely short accidental taps
|
||||
pub min_audio_ms: u64,
|
||||
|
||||
/// mic object name from `pw-dump`, None for the default
|
||||
pub pipewire_target_object: Option<String>,
|
||||
/// force a specific recording device; see `rodio::microphone::available_inputs()`
|
||||
pub rodio_input_device_name: Option<String>,
|
||||
|
||||
pub use_gpu: bool,
|
||||
pub gpu_device: i32,
|
||||
|
|
@ -55,7 +50,7 @@ impl WhisperSttConfig {
|
|||
n_threads,
|
||||
partial_decode_interval_ms: 700,
|
||||
min_audio_ms: 250,
|
||||
pipewire_target_object: None,
|
||||
rodio_input_device_name: None,
|
||||
use_gpu: true,
|
||||
gpu_device: 0,
|
||||
flash_attn: false,
|
||||
|
|
@ -67,7 +62,7 @@ impl WhisperSttConfig {
|
|||
pub enum WhisperSttError {
|
||||
ModelLoad(String),
|
||||
Whisper(String),
|
||||
PipeWire(String),
|
||||
Rodio(String),
|
||||
CaptureInit(String),
|
||||
ThreadSpawn(String),
|
||||
CaptureThreadPanicked,
|
||||
|
|
@ -80,7 +75,7 @@ impl fmt::Display for WhisperSttError {
|
|||
match self {
|
||||
Self::ModelLoad(e) => write!(f, "failed to load whisper model: {e}"),
|
||||
Self::Whisper(e) => write!(f, "whisper error: {e}"),
|
||||
Self::PipeWire(e) => write!(f, "pipewire error: {e}"),
|
||||
Self::Rodio(e) => write!(f, "rodio error: {e}"),
|
||||
Self::CaptureInit(e) => write!(f, "failed to initialize capture: {e}"),
|
||||
Self::ThreadSpawn(e) => write!(f, "failed to spawn thread: {e}"),
|
||||
Self::CaptureThreadPanicked => write!(f, "capture thread panicked"),
|
||||
|
|
@ -95,7 +90,7 @@ impl std::error::Error for WhisperSttError {}
|
|||
struct StopCapture;
|
||||
|
||||
struct CaptureSession {
|
||||
stop_tx: pw::channel::Sender<StopCapture>,
|
||||
stop_tx: mpsc::Sender<StopCapture>,
|
||||
capture_thread: Option<JoinHandle<()>>,
|
||||
recognizer_thread: Option<JoinHandle<()>>,
|
||||
deadline: Instant,
|
||||
|
|
@ -141,7 +136,7 @@ impl WhisperStt {
|
|||
})
|
||||
}
|
||||
|
||||
/// starts a fresh pw capture stream and a transcription worker
|
||||
/// starts a fresh capture stream and a transcription worker
|
||||
pub fn ptt_start(&mut self) -> Result<(), WhisperSttError> {
|
||||
self.reap_finished_recognizers();
|
||||
|
||||
|
|
@ -151,7 +146,7 @@ impl WhisperStt {
|
|||
|
||||
let (audio_tx, audio_rx) = mpsc::channel::<Vec<f32>>();
|
||||
let (ready_tx, ready_rx) = mpsc::channel::<Result<(), String>>();
|
||||
let (stop_tx, stop_rx) = pw::channel::channel::<StopCapture>();
|
||||
let (stop_tx, stop_rx) = mpsc::channel::<StopCapture>();
|
||||
|
||||
let recognizer_thread = spawn_recognizer_thread(
|
||||
Arc::clone(&self.ctx),
|
||||
|
|
@ -160,12 +155,12 @@ impl WhisperStt {
|
|||
self.completed_tx.clone(),
|
||||
)?;
|
||||
|
||||
let target_object = self.config.pipewire_target_object.clone();
|
||||
let input_device_name = self.config.rodio_input_device_name.clone();
|
||||
|
||||
let capture_thread = thread::Builder::new()
|
||||
.name("whisper-stt-pipewire-capture".to_string())
|
||||
.name("whisper-stt-rodio-capture".to_string())
|
||||
.spawn(move || {
|
||||
pipewire_capture_thread(audio_tx, stop_rx, target_object, ready_tx);
|
||||
rodio_capture_thread(audio_tx, stop_rx, input_device_name, ready_tx);
|
||||
})
|
||||
.map_err(|e| WhisperSttError::ThreadSpawn(e.to_string()))?;
|
||||
|
||||
|
|
@ -265,7 +260,7 @@ impl WhisperStt {
|
|||
}
|
||||
}
|
||||
|
||||
return None;
|
||||
None
|
||||
}
|
||||
|
||||
pub fn take_error(&mut self) -> Option<String> {
|
||||
|
|
@ -347,8 +342,8 @@ fn recognizer_thread(
|
|||
last_decoded_len = audio.len();
|
||||
}
|
||||
Err(_) => {
|
||||
// Do not fail the session on a speculative decode.
|
||||
// The final decode after PTT end gets reported.
|
||||
// do not fail the session on a speculative decode
|
||||
// the final decode after PTT end gets reported
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -410,15 +405,15 @@ fn transcribe_audio(
|
|||
Ok(normalize_transcript(text))
|
||||
}
|
||||
|
||||
fn pipewire_capture_thread(
|
||||
fn rodio_capture_thread(
|
||||
audio_tx: mpsc::Sender<Vec<f32>>,
|
||||
stop_rx: pw::channel::Receiver<StopCapture>,
|
||||
target_object: Option<String>,
|
||||
stop_rx: mpsc::Receiver<StopCapture>,
|
||||
input_device_name: Option<String>,
|
||||
ready_tx: mpsc::Sender<Result<(), String>>,
|
||||
) {
|
||||
let mut ready_tx = Some(ready_tx);
|
||||
|
||||
let result = run_pipewire_capture(audio_tx, stop_rx, target_object, &mut ready_tx);
|
||||
let result = run_rodio_capture(audio_tx, stop_rx, input_device_name, &mut ready_tx);
|
||||
|
||||
if let Err(e) = result {
|
||||
if let Some(ready_tx) = ready_tx.take() {
|
||||
|
|
@ -427,163 +422,105 @@ fn pipewire_capture_thread(
|
|||
}
|
||||
}
|
||||
|
||||
fn run_pipewire_capture(
|
||||
fn run_rodio_capture(
|
||||
audio_tx: mpsc::Sender<Vec<f32>>,
|
||||
stop_rx: pw::channel::Receiver<StopCapture>,
|
||||
target_object: Option<String>,
|
||||
stop_rx: mpsc::Receiver<StopCapture>,
|
||||
input_device_name: Option<String>,
|
||||
ready_tx: &mut Option<mpsc::Sender<Result<(), String>>>,
|
||||
) -> Result<(), WhisperSttError> {
|
||||
pw::init();
|
||||
let builder = MicrophoneBuilder::new();
|
||||
|
||||
let mainloop = pw::main_loop::MainLoopRc::new(None)
|
||||
.map_err(|e| WhisperSttError::PipeWire(e.to_string()))?;
|
||||
let builder = if let Some(input_device_name) = input_device_name {
|
||||
let inputs = available_inputs().map_err(|e| WhisperSttError::Rodio(e.to_string()))?;
|
||||
let input_device_name_lower = input_device_name.to_lowercase();
|
||||
|
||||
let context = pw::context::ContextRc::new(&mainloop, None)
|
||||
.map_err(|e| WhisperSttError::PipeWire(e.to_string()))?;
|
||||
let input = inputs
|
||||
.into_iter()
|
||||
.find(|input| {
|
||||
input
|
||||
.to_string()
|
||||
.to_lowercase()
|
||||
.contains(&input_device_name_lower)
|
||||
})
|
||||
.ok_or_else(|| {
|
||||
WhisperSttError::Rodio(format!(
|
||||
"no rodio input device matched {input_device_name:?}"
|
||||
))
|
||||
})?;
|
||||
|
||||
let core = context
|
||||
.connect_rc(None)
|
||||
.map_err(|e| WhisperSttError::PipeWire(e.to_string()))?;
|
||||
|
||||
let _stop_receiver = stop_rx.attach(mainloop.loop_(), {
|
||||
let mainloop = mainloop.clone();
|
||||
move |_| {
|
||||
mainloop.quit();
|
||||
}
|
||||
});
|
||||
|
||||
let mut props = properties! {
|
||||
*pw::keys::MEDIA_TYPE => "Audio",
|
||||
*pw::keys::MEDIA_CATEGORY => "Capture",
|
||||
*pw::keys::MEDIA_ROLE => "Communication",
|
||||
*pw::keys::APP_NAME => "WhisperStt",
|
||||
builder
|
||||
.device(input)
|
||||
.map_err(|e| WhisperSttError::Rodio(e.to_string()))?
|
||||
} else {
|
||||
builder
|
||||
.default_device()
|
||||
.map_err(|e| WhisperSttError::Rodio(e.to_string()))?
|
||||
};
|
||||
|
||||
if let Some(target_object) = target_object {
|
||||
props.insert(*pw::keys::TARGET_OBJECT, target_object);
|
||||
}
|
||||
let builder = builder
|
||||
.default_config()
|
||||
.map_err(|e| WhisperSttError::Rodio(e.to_string()))?
|
||||
.prefer_channel_counts([
|
||||
1.try_into().expect("not zero"),
|
||||
2.try_into().expect("not zero"),
|
||||
])
|
||||
.prefer_sample_rates([
|
||||
16_000.try_into().expect("not zero"),
|
||||
32_000.try_into().expect("not zero"),
|
||||
48_000.try_into().expect("not zero"),
|
||||
])
|
||||
.prefer_buffer_sizes(512..);
|
||||
|
||||
let stream = pw::stream::StreamBox::new(&core, "WhisperStt microphone capture", props)
|
||||
.map_err(|e| WhisperSttError::PipeWire(e.to_string()))?;
|
||||
let mut mic = builder
|
||||
.open_stream()
|
||||
.map_err(|e| WhisperSttError::Rodio(e.to_string()))?;
|
||||
|
||||
let user_data = AudioCaptureUserData::default();
|
||||
let audio_tx_for_callback = audio_tx.clone();
|
||||
|
||||
let _listener = stream
|
||||
.add_local_listener_with_user_data(user_data)
|
||||
.param_changed(|_, user_data, id, param| {
|
||||
let Some(param) = param else {
|
||||
return;
|
||||
};
|
||||
|
||||
if id != pw::spa::param::ParamType::Format.as_raw() {
|
||||
return;
|
||||
}
|
||||
|
||||
let Ok((media_type, media_subtype)) = format_utils::parse_format(param) else {
|
||||
return;
|
||||
};
|
||||
|
||||
if media_type != MediaType::Audio || media_subtype != MediaSubtype::Raw {
|
||||
return;
|
||||
}
|
||||
|
||||
let _ = user_data.format.parse(param);
|
||||
})
|
||||
.process(move |stream, user_data| {
|
||||
let Some(mut buffer) = stream.dequeue_buffer() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let datas = buffer.datas_mut();
|
||||
if datas.is_empty() {
|
||||
return;
|
||||
}
|
||||
|
||||
let data = &mut datas[0];
|
||||
|
||||
let chunk = data.chunk();
|
||||
|
||||
let offset = chunk.offset() as usize;
|
||||
let size = chunk.size() as usize;
|
||||
|
||||
let Some(bytes) = data.data() else {
|
||||
return;
|
||||
};
|
||||
|
||||
if offset >= bytes.len() {
|
||||
return;
|
||||
}
|
||||
|
||||
let end = offset.saturating_add(size).min(bytes.len());
|
||||
let bytes = &bytes[offset..end];
|
||||
|
||||
let channels = (user_data.format.channels() as usize).max(1);
|
||||
let input_rate = {
|
||||
let rate = user_data.format.rate() as usize;
|
||||
if rate == 0 { 48_000 } else { rate }
|
||||
};
|
||||
|
||||
let resampled = user_data
|
||||
.resampler
|
||||
.push_interleaved_f32le_mono_16k(bytes, channels, input_rate);
|
||||
|
||||
if !resampled.is_empty() {
|
||||
let _ = audio_tx_for_callback.send(resampled);
|
||||
}
|
||||
})
|
||||
.register()
|
||||
.map_err(|e| WhisperSttError::PipeWire(e.to_string()))?;
|
||||
|
||||
let mut audio_info = spa::param::audio::AudioInfoRaw::new();
|
||||
audio_info.set_format(spa::param::audio::AudioFormat::F32LE);
|
||||
|
||||
let obj = pw::spa::pod::Object {
|
||||
type_: pw::spa::utils::SpaTypes::ObjectParamFormat.as_raw(),
|
||||
id: pw::spa::param::ParamType::EnumFormat.as_raw(),
|
||||
properties: audio_info.into(),
|
||||
};
|
||||
|
||||
let values: Vec<u8> = pw::spa::pod::serialize::PodSerializer::serialize(
|
||||
std::io::Cursor::new(Vec::new()),
|
||||
&pw::spa::pod::Value::Object(obj),
|
||||
)
|
||||
.map_err(|e| WhisperSttError::PipeWire(e.to_string()))?
|
||||
.0
|
||||
.into_inner();
|
||||
|
||||
let pod = Pod::from_bytes(&values).ok_or_else(|| {
|
||||
WhisperSttError::PipeWire("failed to parse serialized PipeWire pod".to_string())
|
||||
})?;
|
||||
|
||||
let mut params = [pod];
|
||||
|
||||
stream
|
||||
.connect(
|
||||
spa::utils::Direction::Input,
|
||||
None,
|
||||
pw::stream::StreamFlags::AUTOCONNECT
|
||||
| pw::stream::StreamFlags::MAP_BUFFERS
|
||||
| pw::stream::StreamFlags::RT_PROCESS,
|
||||
&mut params,
|
||||
)
|
||||
.map_err(|e| WhisperSttError::PipeWire(e.to_string()))?;
|
||||
let channels = mic.channels().get() as usize;
|
||||
let input_rate = mic.sample_rate().get() as usize;
|
||||
|
||||
if let Some(ready_tx) = ready_tx.take() {
|
||||
let _ = ready_tx.send(Ok(()));
|
||||
}
|
||||
|
||||
mainloop.run();
|
||||
let mut resampler = StreamingResampler::default();
|
||||
let mut interleaved = Vec::new();
|
||||
|
||||
// ~20 ms of input frames; whisper still receives 16 kHz mono chunks
|
||||
let chunk_input_samples = ((input_rate / 50).max(1)) * channels.max(1);
|
||||
|
||||
'capture: loop {
|
||||
if stop_rx.try_recv().is_ok() {
|
||||
break;
|
||||
}
|
||||
|
||||
interleaved.clear();
|
||||
|
||||
while interleaved.len() < chunk_input_samples {
|
||||
if stop_rx.try_recv().is_ok() {
|
||||
break 'capture;
|
||||
}
|
||||
|
||||
let Some(sample) = mic.next() else {
|
||||
return Err(WhisperSttError::Rodio(
|
||||
"microphone stream ended unexpectedly".to_string(),
|
||||
));
|
||||
};
|
||||
|
||||
// Rodio's default sample type is f32. This cast also keeps the code
|
||||
// compiling if the crate is built with rodio's `64bit` feature.
|
||||
interleaved.push(sample as f32);
|
||||
}
|
||||
|
||||
let resampled = resampler.push_interleaved_mono_16k(&interleaved, channels, input_rate);
|
||||
|
||||
if !resampled.is_empty() && audio_tx.send(resampled).is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct AudioCaptureUserData {
|
||||
format: spa::param::audio::AudioInfoRaw,
|
||||
resampler: StreamingResampler,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct StreamingResampler {
|
||||
pending: Vec<f32>,
|
||||
|
|
@ -592,9 +529,9 @@ struct StreamingResampler {
|
|||
}
|
||||
|
||||
impl StreamingResampler {
|
||||
fn push_interleaved_f32le_mono_16k(
|
||||
fn push_interleaved_mono_16k(
|
||||
&mut self,
|
||||
bytes: &[u8],
|
||||
samples: &[f32],
|
||||
channels: usize,
|
||||
input_rate: usize,
|
||||
) -> Vec<f32> {
|
||||
|
|
@ -608,12 +545,7 @@ impl StreamingResampler {
|
|||
self.input_rate = input_rate;
|
||||
}
|
||||
|
||||
let frame_bytes = channels * std::mem::size_of::<f32>();
|
||||
if frame_bytes == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
||||
let frames = bytes.len() / frame_bytes;
|
||||
let frames = samples.len() / channels;
|
||||
if frames == 0 {
|
||||
return Vec::new();
|
||||
}
|
||||
|
|
@ -621,20 +553,11 @@ impl StreamingResampler {
|
|||
let mut mono = Vec::with_capacity(frames);
|
||||
|
||||
for frame in 0..frames {
|
||||
let frame_start = frame * frame_bytes;
|
||||
let frame_start = frame * channels;
|
||||
let mut sum = 0.0f32;
|
||||
|
||||
for ch in 0..channels {
|
||||
let sample_start = frame_start + ch * 4;
|
||||
|
||||
let sample = f32::from_le_bytes([
|
||||
bytes[sample_start],
|
||||
bytes[sample_start + 1],
|
||||
bytes[sample_start + 2],
|
||||
bytes[sample_start + 3],
|
||||
]);
|
||||
|
||||
sum += sample;
|
||||
sum += samples[frame_start + ch];
|
||||
}
|
||||
|
||||
mono.push(sum / channels as f32);
|
||||
|
|
|
|||
|
|
@ -25,11 +25,7 @@ xdg.workspace = true
|
|||
chrono = "0.4.42"
|
||||
smol = { workspace = true }
|
||||
wgui = { path = "../wgui/" }
|
||||
rodio = { version = "0.21.1", default-features = false, features = [
|
||||
"playback",
|
||||
"mp3",
|
||||
"hound",
|
||||
] }
|
||||
rodio = { version = "0.22.2", default-features = false, features = ["playback", "mp3", "hound", "recording"] }
|
||||
glob = "0.3.3"
|
||||
walkdir = "2.5.0"
|
||||
rust-ini = "0.21.3"
|
||||
|
|
|
|||
|
|
@ -5,8 +5,10 @@ use wgui::{assets::AssetProvider, sound::WguiSoundType};
|
|||
|
||||
use std::io::Read;
|
||||
|
||||
pub use rodio;
|
||||
|
||||
pub struct AudioSystem {
|
||||
audio_stream: Option<rodio::OutputStream>,
|
||||
audio_stream: Option<rodio::MixerDeviceSink>,
|
||||
first_try: bool,
|
||||
}
|
||||
|
||||
|
|
@ -100,10 +102,10 @@ impl AudioSystem {
|
|||
}
|
||||
}
|
||||
|
||||
fn get_handle(&mut self) -> Option<&rodio::OutputStream> {
|
||||
fn get_handle(&mut self) -> Option<&rodio::MixerDeviceSink> {
|
||||
if self.audio_stream.is_none() && self.first_try {
|
||||
self.first_try = false;
|
||||
if let Ok(stream) = rodio::OutputStreamBuilder::open_default_stream() {
|
||||
if let Ok(stream) = rodio::DeviceSinkBuilder::open_default_sink() {
|
||||
self.audio_stream = Some(stream);
|
||||
} else {
|
||||
log::error!("Failed to open audio stream. Audio will not work.");
|
||||
|
|
|
|||
Loading…
Reference in New Issue