From 0b2c3052f67ab669e7929451d4e8af2ed73e180e Mon Sep 17 00:00:00 2001 From: galister <22305755+galister@users.noreply.github.com> Date: Fri, 3 Jul 2026 17:57:02 +0900 Subject: [PATCH] wip: whisper stt #2 --- dash-frontend/assets/lang/en.json | 7 + dash-frontend/src/tab/settings/mod.rs | 47 ++- .../src/tab/settings/tab_features.rs | 281 +++++++++++++++++- dash-frontend/src/util/mod.rs | 1 + dash-frontend/src/util/whisper.rs | 81 +++++ dash-frontend/src/views/download_file.rs | 2 +- wayvr/src/config.rs | 2 + wayvr/src/subsystem/whisper_stt.rs | 34 --- wlx-common/src/config.rs | 3 + wlx-common/src/data_dir.rs | 20 ++ wlx-common/src/lib.rs | 1 + 11 files changed, 422 insertions(+), 57 deletions(-) create mode 100644 dash-frontend/src/util/whisper.rs create mode 100644 wlx-common/src/data_dir.rs diff --git a/dash-frontend/assets/lang/en.json b/dash-frontend/assets/lang/en.json index 568d55fc..6398977a 100644 --- a/dash-frontend/assets/lang/en.json +++ b/dash-frontend/assets/lang/en.json @@ -193,6 +193,12 @@ "USE_SKYBOX_HELP": "Show a skybox if there's no scene app or passthrough", "WATCH_VIEW_ANGLE": "Watch view angles", "WATCH_VIEW_ANGLE_HELP": "Control how the watch fades away", + "WHISPER": { + "NEED_TO_DOWNLOAD_MODEL": "The selected model needs to be downloaded.\nDownload it right now?", + "REMOVE_UNUSED_MODELS": "You are turning off the speech-to-text feature.\nWould you like to remove downloaded Whisper models to free up space?" + }, + "WHISPER_MODEL": "Speech-to-text model", + "WHISPER_MODEL_HELP": "Whisper model to use.\nLarger models produce more accurate\nresults, but consume more VRAM.", "GRID_OPACITY": "Floor grid opacity", "GRID_OPACITY_HELP": "Opacity of the floor grid when the skybox is enabled", "XWAYLAND_BY_DEFAULT": "Run apps in Compatibility mode by default", @@ -227,6 +233,7 @@ "BUTTON_TOOLTIP": "Donate via Open Collective", "CURRENT_SUPPORTERS": "Current Supporters" }, + "DOWNLOAD": "Download", "DOWNLOADER": "Downloader", "WELCOME": { "ALL_SET": "You're all set!", diff --git a/dash-frontend/src/tab/settings/mod.rs b/dash-frontend/src/tab/settings/mod.rs index afebc9db..2f467da1 100644 --- a/dash-frontend/src/tab/settings/mod.rs +++ b/dash-frontend/src/tab/settings/mod.rs @@ -28,7 +28,7 @@ use wlx_common::{ use crate::{ frontend::{Frontend, FrontendTask, FrontendTasks}, - tab::{Tab, TabType, settings::macros::MacroParams}, + tab::{settings::macros::MacroParams, Tab, TabType}, views::ViewUpdateParams, }; @@ -107,6 +107,16 @@ trait SettingsTab { fn setting_updated(&mut self, _sup: &mut SettingUpdatedParams) -> anyhow::Result<()> { Ok(()) } + + fn context_menu_custom( + &mut self, + _action: Rc, + _config: &mut GeneralConfig, + _layout: &mut Layout, + _state: &mut ParserState, + ) -> anyhow::Result<()> { + Ok(()) + } } pub struct TabSettings { @@ -235,27 +245,32 @@ impl Tab for TabSettings { } // Dropdown handling - if let TickResult::Action(name) = self.context_menu.tick(&mut frontend.layout, &mut self.state)? - && let (Some(setting), Some(id), Some(value), Some(text), Some(translated)) = { + if let TickResult::Action(name) = self.context_menu.tick(&mut frontend.layout, &mut self.state)? { + if name.starts_with(';') && let Some(tab) = self.current_tab.as_mut() { + let config = frontend.interface.general_config(data); + tab.context_menu_custom(name, config, &mut frontend.layout, &mut self.state)?; + + } else if let (Some(setting), Some(id), Some(value), Some(text), Some(translated)) = { let mut s = name.splitn(5, ';'); (s.next(), s.next(), s.next(), s.next(), s.next()) } { - let mut common = frontend.layout.common(); - let mut label = self - .state - .fetch_widget_as::(common.state, &format!("{id}_value"))?; + let mut common = frontend.layout.common(); + let mut label = self + .state + .fetch_widget_as::(common.state, &format!("{id}_value"))?; - let translation = Translation { - text: text.into(), - translated: translated == "1", - }; + let translation = Translation { + text: text.into(), + translated: translated == "1", + }; - label.set_text(&mut common, translation); + label.set_text(&mut common, translation); - let setting = SettingType::from_str(setting).expect("Invalid Enum string"); - let config = frontend.interface.general_config(data); - setting.set_enum(config, value); - changed = Some(ConfigChangeKind::OverlayConfig); + let setting = SettingType::from_str(setting).expect("Invalid Enum string"); + let config = frontend.interface.general_config(data); + setting.set_enum(config, value); + changed = Some(ConfigChangeKind::OverlayConfig); + } } // Notify overlays of the change diff --git a/dash-frontend/src/tab/settings/tab_features.rs b/dash-frontend/src/tab/settings/tab_features.rs index 96d40fc3..4be0cc83 100644 --- a/dash-frontend/src/tab/settings/tab_features.rs +++ b/dash-frontend/src/tab/settings/tab_features.rs @@ -1,15 +1,117 @@ -use crate::tab::settings::{ - SettingType, SettingsMountParams, SettingsTab, - macros::{options_category, options_checkbox, options_range_f32}, +use std::rc::Rc; + +use wgui::{ + components::button::{ButtonClickEvent, ComponentButton}, globals::WguiGlobals, i18n::Translation, layout::WidgetID, log::LogErr, parser::{Fetchable, TemplateParams}, task::Tasks, widget::label::WidgetLabel, windowing::context_menu::{self} +}; +use wlx_common::{async_executor::AsyncExecutor, config::GeneralConfig}; + +use crate::{ + frontend::FrontendTasks, + tab::settings::{ + SettingType, SettingsMountParams, SettingsTab, TabNameEnum, Task as ParentTask, horiz_cell, macros::{MacroParams, options_category, options_checkbox, options_range_f32} + }, + util::{ + popup_manager::PopupHolder, + whisper::{WHISPER_MODELS, WhisperModel, whisper_any_models_downloaded, whisper_delete_all_models, whisper_model_from_name, whisper_model_path}, + }, + views::{self, ViewUpdateParams}, }; -pub struct State {} +#[derive(Clone)] +enum Task { + WhisperDownloadClosed, + WhisperRemoveUnused, + WhisperDownload(&'static WhisperModel), + WhisperDownloadDone, + CloseDialog, +} -impl SettingsTab for State {} +pub struct State { + popup_download: PopupHolder, + popup_dialog: PopupHolder, + frontend_tasks: FrontendTasks, + globals: WguiGlobals, + tasks: Tasks, + parent_tasks: Tasks, + pending_download: Option<&'static WhisperModel>, +} + +impl SettingsTab for State { + fn update(&mut self, par: &mut ViewUpdateParams) -> anyhow::Result<()> { + self.popup_download.update(par)?; + self.popup_dialog.update(par)?; + + for task in self.tasks.drain() { + match task { + Task::WhisperDownloadClosed => { + if let Some(model) = self.pending_download.take() { + if !whisper_model_path(model.file_name).exists() { + // download failed, set to selection to none + par.general_config.whisper_model = "".into(); + // re-init the whole tab + self.parent_tasks.push(ParentTask::SetTab(TabNameEnum::Features)); + } + } + } + Task::WhisperRemoveUnused => { + let _ = whisper_delete_all_models().log_err("could not remove whisper models"); + }, + Task::WhisperDownload(model) => { + self.pending_download = Some(model); + self.show_whisper_download_dialog(model, par.executor.clone()); + }, + Task::WhisperDownloadDone => { + // we're happy + }, + Task::CloseDialog => { + let close_dialog = self.popup_dialog.get_close_callback(par.layout); + close_dialog(); + } + } + } + + Ok(()) + } + + fn context_menu_custom( + &mut self, + action: Rc, + config: &mut GeneralConfig, + layout: &mut wgui::layout::Layout, + state: &mut wgui::parser::ParserState, + ) -> anyhow::Result<()> { + if let (Some(_), Some(id), Some(model_name)) = { + let mut s = action.splitn(3, ';'); + (s.next(), s.next(), s.next()) + } { + if !model_name.is_empty() && let Some(model) = whisper_model_from_name(model_name) + { + let mut common = layout.common(); + let mut label = state.fetch_widget_as::(common.state, &format!("{id}_value"))?; + label.set_text(&mut common, Translation::from_raw_text(model.display_name)); + + config.whisper_model = action.as_ref().into(); + + self.show_whisper_model_dialog_box_download(model)?; + } else if whisper_any_models_downloaded().unwrap_or_default() { + self.show_whisper_model_dialog_box_cleanup()?; + } + } + + Ok(()) + } +} impl State { pub fn mount(par: SettingsMountParams) -> anyhow::Result { + let tasks = Tasks::::new(); + let popup_download = PopupHolder::::default(); + let popup_dialog = PopupHolder::::default(); + let c = options_category(par.mp, par.id_parent, "APP_SETTINGS.FEATURES", "dashboard/options.svg")?; + + whisper_models_dropdown(par.mp, c)?; + options_checkbox(par.mp, c, SettingType::NotificationsEnabled)?; options_checkbox(par.mp, c, SettingType::NotificationsSoundEnabled)?; options_checkbox(par.mp, c, SettingType::KeyboardSoundEnabled)?; @@ -32,6 +134,173 @@ impl State { 1.0, 0.1, )?; - Ok(State {}) + Ok(State { + tasks, + parent_tasks: par.mp.tasks.clone(), + popup_download, + popup_dialog, + frontend_tasks: par.frontend_tasks.clone(), + globals: par.mp.doc_params.globals.clone(), + pending_download: None, + }) + } + + fn show_whisper_download_dialog(&mut self, model: &WhisperModel, executor: AsyncExecutor) { + views::download_file::mount_popup( + self.popup_download.clone(), + self.frontend_tasks.clone(), + self.tasks.make_callback_box(Task::WhisperDownloadClosed), + views::download_file::Params { + globals: self.globals.clone(), + executor, + target_path: whisper_model_path(model.file_name), + url: model.url.into(), + on_downloaded: self.tasks.make_callback_box(Task::WhisperDownloadDone), + }, + ); + } + + fn show_whisper_model_dialog_box_download(&mut self, model: &'static WhisperModel) -> anyhow::Result<()> { + const ACTION_DOWNLOAD: &str = "download"; + const ACTION_CANCEL: &str = "cancel"; + + self.pending_download = Some(model); + + let tasks = self.tasks.clone(); + views::dialog_box::mount_popup( + self.popup_dialog.clone(), + self.frontend_tasks.clone(), + views::dialog_box::Params { + globals: self.globals.clone(), + message: Translation::from_translation_key("APP_SETTINGS.WHISPER.NEED_TO_DOWNLOAD_MODEL"), + entries: vec![ + views::dialog_box::ButtonEntry { + content: Translation::from_translation_key("APP_SETTINGS.CANCEL"), + icon: "dashboard/close.svg", + action: ACTION_CANCEL, + }, + views::dialog_box::ButtonEntry { + content: Translation::from_translation_key("DOWNLOAD"), + icon: "dashboard/download.svg", + action: ACTION_DOWNLOAD, + }, + ], + on_action_click: Box::new(move |action| match action { + ACTION_DOWNLOAD => { + tasks.push(Task::WhisperDownload(model)); + } + ACTION_CANCEL => { + tasks.push(Task::CloseDialog); + + // treat as failed download + tasks.push(Task::WhisperDownloadClosed); + } + _ => unreachable!(), + }), + }, + ); + + Ok(()) + } + + fn show_whisper_model_dialog_box_cleanup(&mut self) -> anyhow::Result<()> { + const ACTION_REMOVE: &str = "remove"; + const ACTION_CANCEL: &str = "cancel"; + + let tasks = self.tasks.clone(); + + views::dialog_box::mount_popup( + self.popup_dialog.clone(), + self.frontend_tasks.clone(), + views::dialog_box::Params { + globals: self.globals.clone(), + message: Translation::from_translation_key("APP_SETTINGS.WHISPER.REMOVE_UNUSED_MODELS"), + entries: vec![ + views::dialog_box::ButtonEntry { + content: Translation::from_translation_key("APP_SETTINGS.CANCEL"), + icon: "dashboard/close.svg", + action: ACTION_CANCEL, + }, + views::dialog_box::ButtonEntry { + content: Translation::from_translation_key("REMOVE"), + icon: "dashboard/trash.svg", + action: ACTION_REMOVE, + }, + ], + on_action_click: Box::new(move |action| match action { + ACTION_REMOVE => { + tasks.push(Task::WhisperRemoveUnused); + } + ACTION_CANCEL => { + tasks.push(Task::CloseDialog); + } + _ => unreachable!(), + }), + }, + ); + + Ok(()) } } + +fn whisper_models_dropdown(mp: &mut MacroParams, parent: WidgetID) -> anyhow::Result<()> { + let id = mp.idx.to_string(); + mp.idx += 1; + + let parent = horiz_cell(mp.layout, parent)?; + + let current_file = mp.config.whisper_model.as_ref(); + let current_translation = whisper_model_from_name(current_file) + .map_or(Translation::from_translation_key("APP_SETTINGS.OPTION.NONE"), |x| { + Translation::from_raw_text(x.display_name) + }); + + let mut params = TemplateParams::new(); + params.insert("id", &id); + params.insert("translation", "APP_SETTINGS.WHISPER_MODEL".into()); + params.insert("tooltip", "APP_SETTINGS.WHISPER_MODEL_HELP".into()); + + mp.parser_state + .instantiate_template(mp.doc_params, "DropdownButton", mp.layout, parent, params)?; + + { + let mut label = mp + .parser_state + .fetch_widget_as::(&mp.layout.state, &format!("{id}_value"))?; + label.set_text_simple(&mut mp.layout.state.globals.get(), current_translation); + } + + let btn = mp.parser_state.fetch_component_as::(&id)?; + btn.on_click(Rc::new({ + let parent_tasks = mp.tasks.clone(); + move |_common, e: ButtonClickEvent| { + let mut cells = WHISPER_MODELS + .iter() + .map(|item| context_menu::Cell { + action_name: Some(format!(";{id};{}", item.file_name).into()), + title: Translation::from_raw_text(item.display_name), + tooltip: None, + attribs: vec![], + }) + .collect::>(); + + cells.insert( + 0, + context_menu::Cell { + action_name: Some(format!(";{id};").into()), + title: Translation::from_translation_key("APP_SETTINGS.OPTION.NONE"), + tooltip: None, + attribs: vec![], + }, + ); + + parent_tasks.push(ParentTask::OpenContextMenu( + e.mouse_pos_absolute.unwrap_or_default(), + cells, + )); + Ok(()) + } + })); + + Ok(()) +} diff --git a/dash-frontend/src/util/mod.rs b/dash-frontend/src/util/mod.rs index 2dd7db67..7eda5e07 100644 --- a/dash-frontend/src/util/mod.rs +++ b/dash-frontend/src/util/mod.rs @@ -6,3 +6,4 @@ pub mod popup_manager; pub mod steam_utils; pub mod toast_manager; pub mod wgui_simple; +pub mod whisper; diff --git a/dash-frontend/src/util/whisper.rs b/dash-frontend/src/util/whisper.rs new file mode 100644 index 00000000..823f5b07 --- /dev/null +++ b/dash-frontend/src/util/whisper.rs @@ -0,0 +1,81 @@ +use std::{fs, io, path::PathBuf}; + +use wlx_common::data_dir; + +pub struct WhisperModel { + pub file_name: &'static str, + pub display_name: &'static str, + pub url: &'static str, + pub hash: &'static str, +} + +pub const WHISPER_MODELS: &[WhisperModel] = &[ + WhisperModel { + file_name: "ggml-base-q8_0.bin", + display_name: "Base Q8 (78MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q8_0.bin", + hash: "7bb89bb49ed6955013b166f1b6a6c04584a20fbe", + }, + WhisperModel { + file_name: "ggml-small-q8_0.bin", + display_name: "Small Q8 (252MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q8_0.bin", + hash: "bcad8a2083f4e53d648d586b7dbc0cd673d8afad", + }, + WhisperModel { + file_name: "ggml-large-v3-turbo-q5_0.bin", + display_name: "Turbo Q5 (574MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin", + hash: "e050f7970618a659205450ad97eb95a18d69c9ee", + }, + WhisperModel { + file_name: "ggml-large-v3-turbo-q8_0.bin", + display_name: "Turbo Q8 (874MiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q8_0.bin", + hash: "01bf15bedffe9f39d65c1b6ff9b687ea91f59e0e", + }, + WhisperModel { + file_name: "ggml-large-v3-turbo.bin", + display_name: "Turbo (1.5GiB)", + url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin", + hash: "4af2b29d7ec73d781377bfd1758ca957a807e941", + }, +]; + +pub fn whisper_model_from_name(file_name: &str) -> Option<&'static WhisperModel> { + WHISPER_MODELS.iter().find(|x| x.file_name == file_name) +} + +pub fn whisper_model_folder() -> PathBuf { + data_dir::get_path("whisper") +} + +pub fn whisper_model_path(file_name: &str) -> PathBuf { + whisper_model_folder().join(file_name) +} + +pub fn whisper_any_models_downloaded() -> io::Result { + let path = whisper_model_folder(); + if !path.is_dir() { + return Ok(false); + } + Ok(fs::read_dir(path)?.count() > 0) +} + +pub fn whisper_delete_all_models() -> io::Result<()> { + let path = whisper_model_folder(); + if !path.is_dir() { + return Ok(()); + } + + for entry in fs::read_dir(path)? { + let entry = entry?; + let file_type = entry.file_type()?; + + if file_type.is_file() || file_type.is_symlink() { + fs::remove_file(entry.path())?; + } + } + + Ok(()) +} diff --git a/dash-frontend/src/views/download_file.rs b/dash-frontend/src/views/download_file.rs index c2502451..fb5d51e3 100644 --- a/dash-frontend/src/views/download_file.rs +++ b/dash-frontend/src/views/download_file.rs @@ -220,7 +220,7 @@ impl View { tasks.push(Task::SetStatusText(String::from("Writing to file..."))); - // create skymaps directory if it doesn't exist yet + // create parent directory if it doesn't exist yet if let Some(parent) = target_path.parent() { handle_async_result( "Directory creation failed", diff --git a/wayvr/src/config.rs b/wayvr/src/config.rs index 5229906b..f9255988 100644 --- a/wayvr/src/config.rs +++ b/wayvr/src/config.rs @@ -184,6 +184,7 @@ pub struct AutoSettings { pub chroma_key_params: ChromaKeyParams, pub input_emulation_method: InputEmulationMethod, pub tutorial_graduated: bool, + pub whisper_model: Arc, } fn get_settings_path() -> PathBuf { @@ -246,6 +247,7 @@ pub fn save_settings(config: &GeneralConfig) -> anyhow::Result<()> { chroma_key_params: config.chroma_key_params.clone(), input_emulation_method: config.input_emulation_method, tutorial_graduated: config.tutorial_graduated, + whisper_model: config.whisper_model.clone(), }; let json = serde_json::to_string_pretty(&conf).unwrap(); // want panic diff --git a/wayvr/src/subsystem/whisper_stt.rs b/wayvr/src/subsystem/whisper_stt.rs index d1b23dcb..b5a36931 100644 --- a/wayvr/src/subsystem/whisper_stt.rs +++ b/wayvr/src/subsystem/whisper_stt.rs @@ -17,40 +17,6 @@ use spa::{ }; use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters}; -pub struct WhisperModel { - pub name: &'static str, - pub url: &'static str, - pub hash: &'static str, -} - -pub const WHISPER_MODELS: &[WhisperModel] = &[ - WhisperModel { - name: "Base Q8 (78MiB)", - url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q8_0.bin", - hash: "7bb89bb49ed6955013b166f1b6a6c04584a20fbe", - }, - WhisperModel { - name: "Small Q8 (252MiB)", - url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q8_0.bin", - hash: "bcad8a2083f4e53d648d586b7dbc0cd673d8afad", - }, - WhisperModel { - name: "Turbo Q5 (574MiB)", - url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin", - hash: "e050f7970618a659205450ad97eb95a18d69c9ee", - }, - WhisperModel { - name: "Turbo Q8 (874MiB)", - url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q8_0.bin", - hash: "01bf15bedffe9f39d65c1b6ff9b687ea91f59e0e", - }, - WhisperModel { - name: "Turbo (1.5GiB)", - url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin", - hash: "4af2b29d7ec73d781377bfd1758ca957a807e941", - }, -]; - const WHISPER_SAMPLE_RATE: usize = 16_000; const MAX_DURATION: Duration = Duration::from_secs(30); diff --git a/wlx-common/src/config.rs b/wlx-common/src/config.rs index d0745fc5..2d7ec386 100644 --- a/wlx-common/src/config.rs +++ b/wlx-common/src/config.rs @@ -430,4 +430,7 @@ pub struct GeneralConfig { #[serde(default)] pub tutorial_graduated: bool, + + #[serde(default = "def_empty")] + pub whisper_model: Arc, } diff --git a/wlx-common/src/data_dir.rs b/wlx-common/src/data_dir.rs new file mode 100644 index 00000000..c843284d --- /dev/null +++ b/wlx-common/src/data_dir.rs @@ -0,0 +1,20 @@ +use std::{path::PathBuf, sync::LazyLock}; + +const FALLBACK_DATA_PATH: &str = "/tmp/wayvr_data"; + +static DATA_ROOT_PATH: LazyLock = LazyLock::new(|| { + if let Some(mut dir) = xdg::BaseDirectories::new().get_data_home() { + dir.push("wayvr"); + return dir; + } + log::error!("Err: Failed to find data path, using {FALLBACK_DATA_PATH}"); + PathBuf::from(FALLBACK_DATA_PATH) +}); + +fn get_data_root() -> PathBuf { + DATA_ROOT_PATH.clone() +} + +pub fn get_path(data_path: &str) -> PathBuf { + get_data_root().join(data_path) +} diff --git a/wlx-common/src/lib.rs b/wlx-common/src/lib.rs index fa81cfa1..09fe1a07 100644 --- a/wlx-common/src/lib.rs +++ b/wlx-common/src/lib.rs @@ -5,6 +5,7 @@ pub mod cache_dir; pub mod common; pub mod config; pub mod config_io; +pub mod data_dir; pub mod dash_interface; pub mod dash_interface_emulated; pub mod desktop_finder;