RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wip: whisper stt #2

This commit is contained in:
galister 2026-07-03 17:57:02 +09:00
parent 21a0504601
commit 1b8c800de5
11 changed files with 422 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
dash-frontend/assets/lang/en.json
View File

@ -193,6 +193,12 @@
"USE_SKYBOX_HELP": "Show a skybox if there's no scene app or passthrough",
"WATCH_VIEW_ANGLE": "Watch view angles",
"WATCH_VIEW_ANGLE_HELP": "Control how the watch fades away",
"WHISPER": {
"NEED_TO_DOWNLOAD_MODEL": "The selected model needs to be downloaded.\nDownload it right now?",
"REMOVE_UNUSED_MODELS": "You are turning off the speech-to-text feature.\nWould you like to remove downloaded Whisper models to free up space?"
},
"WHISPER_MODEL": "Speech-to-text model",
"WHISPER_MODEL_HELP": "Whisper model to use.\nLarger models produce more accurate\nresults, but consume more VRAM.",
"GRID_OPACITY": "Floor grid opacity",
"GRID_OPACITY_HELP": "Opacity of the floor grid when the skybox is enabled",
"XWAYLAND_BY_DEFAULT": "Run apps in Compatibility mode by default",
@ -227,6 +233,7 @@
"BUTTON_TOOLTIP": "Donate via Open Collective",
"CURRENT_SUPPORTERS": "Current Supporters"
},
"DOWNLOAD": "Download",
"DOWNLOADER": "Downloader",
"WELCOME": {
"ALL_SET": "You're all set!",

47
dash-frontend/src/tab/settings/mod.rs
View File

@ -28,7 +28,7 @@ use wlx_common::{
use crate::{
frontend::{Frontend, FrontendTask, FrontendTasks},
tab::{Tab, TabType, settings::macros::MacroParams},
tab::{settings::macros::MacroParams, Tab, TabType},
views::ViewUpdateParams,
};
@ -107,6 +107,16 @@ trait SettingsTab {
fn setting_updated(&mut self, _sup: &mut SettingUpdatedParams) -> anyhow::Result<()> {
Ok(())
}
fn context_menu_custom(
&mut self,
_action: Rc<str>,
_config: &mut GeneralConfig,
_layout: &mut Layout,
_state: &mut ParserState,
) -> anyhow::Result<()> {
Ok(())
}
}
pub struct TabSettings<T> {
@ -235,27 +245,32 @@ impl<T> Tab<T> for TabSettings<T> {
}
// Dropdown handling
if let TickResult::Action(name) = self.context_menu.tick(&mut frontend.layout, &mut self.state)?
&& let (Some(setting), Some(id), Some(value), Some(text), Some(translated)) = {
if let TickResult::Action(name) = self.context_menu.tick(&mut frontend.layout, &mut self.state)? {
if name.starts_with(';') && let Some(tab) = self.current_tab.as_mut() {
let config = frontend.interface.general_config(data);
tab.context_menu_custom(name, config, &mut frontend.layout, &mut self.state)?;
} else if let (Some(setting), Some(id), Some(value), Some(text), Some(translated)) = {
let mut s = name.splitn(5, ';');
(s.next(), s.next(), s.next(), s.next(), s.next())
} {
let mut common = frontend.layout.common();
let mut label = self
.state
.fetch_widget_as::<WidgetLabel>(common.state, &format!("{id}_value"))?;
let mut common = frontend.layout.common();
let mut label = self
.state
.fetch_widget_as::<WidgetLabel>(common.state, &format!("{id}_value"))?;
let translation = Translation {
text: text.into(),
translated: translated == "1",
};
let translation = Translation {
text: text.into(),
translated: translated == "1",
};
label.set_text(&mut common, translation);
label.set_text(&mut common, translation);
let setting = SettingType::from_str(setting).expect("Invalid Enum string");
let config = frontend.interface.general_config(data);
setting.set_enum(config, value);
changed = Some(ConfigChangeKind::OverlayConfig);
let setting = SettingType::from_str(setting).expect("Invalid Enum string");
let config = frontend.interface.general_config(data);
setting.set_enum(config, value);
changed = Some(ConfigChangeKind::OverlayConfig);
}
}
// Notify overlays of the change

281
dash-frontend/src/tab/settings/tab_features.rs
View File

@ -1,15 +1,117 @@
use crate::tab::settings::{
SettingType, SettingsMountParams, SettingsTab,
macros::{options_category, options_checkbox, options_range_f32},
use std::rc::Rc;
use wgui::{
components::button::{ButtonClickEvent, ComponentButton}, globals::WguiGlobals, i18n::Translation, layout::WidgetID, log::LogErr, parser::{Fetchable, TemplateParams}, task::Tasks, widget::label::WidgetLabel, windowing::context_menu::{self}
};
use wlx_common::{async_executor::AsyncExecutor, config::GeneralConfig};
use crate::{
frontend::FrontendTasks,
tab::settings::{
SettingType, SettingsMountParams, SettingsTab, TabNameEnum, Task as ParentTask, horiz_cell, macros::{MacroParams, options_category, options_checkbox, options_range_f32}
},
util::{
popup_manager::PopupHolder,
whisper::{WHISPER_MODELS, WhisperModel, whisper_any_models_downloaded, whisper_delete_all_models, whisper_model_from_name, whisper_model_path},
},
views::{self, ViewUpdateParams},
};
pub struct State {}
#[derive(Clone)]
enum Task {
WhisperDownloadClosed,
WhisperRemoveUnused,
WhisperDownload(&'static WhisperModel),
WhisperDownloadDone,
CloseDialog,
}
impl SettingsTab for State {}
pub struct State {
popup_download: PopupHolder<views::download_file::View>,
popup_dialog: PopupHolder<views::dialog_box::View>,
frontend_tasks: FrontendTasks,
globals: WguiGlobals,
tasks: Tasks<Task>,
parent_tasks: Tasks<ParentTask>,
pending_download: Option<&'static WhisperModel>,
}
impl SettingsTab for State {
fn update(&mut self, par: &mut ViewUpdateParams) -> anyhow::Result<()> {
self.popup_download.update(par)?;
self.popup_dialog.update(par)?;
for task in self.tasks.drain() {
match task {
Task::WhisperDownloadClosed => {
if let Some(model) = self.pending_download.take() {
if !whisper_model_path(model.file_name).exists() {
// download failed, set to selection to none
par.general_config.whisper_model = "".into();
// re-init the whole tab
self.parent_tasks.push(ParentTask::SetTab(TabNameEnum::Features));
}
}
}
Task::WhisperRemoveUnused => {
let _ = whisper_delete_all_models().log_err("could not remove whisper models");
},
Task::WhisperDownload(model) => {
self.pending_download = Some(model);
self.show_whisper_download_dialog(model, par.executor.clone());
},
Task::WhisperDownloadDone => {
// we're happy
},
Task::CloseDialog => {
let close_dialog = self.popup_dialog.get_close_callback(par.layout);
close_dialog();
}
}
}
Ok(())
}
fn context_menu_custom(
&mut self,
action: Rc<str>,
config: &mut GeneralConfig,
layout: &mut wgui::layout::Layout,
state: &mut wgui::parser::ParserState,
) -> anyhow::Result<()> {
if let (Some(_), Some(id), Some(model_name)) = {
let mut s = action.splitn(3, ';');
(s.next(), s.next(), s.next())
} {
if !model_name.is_empty() && let Some(model) = whisper_model_from_name(model_name)
{
let mut common = layout.common();
let mut label = state.fetch_widget_as::<WidgetLabel>(common.state, &format!("{id}_value"))?;
label.set_text(&mut common, Translation::from_raw_text(model.display_name));
config.whisper_model = action.as_ref().into();
self.show_whisper_model_dialog_box_download(model)?;
} else if whisper_any_models_downloaded().unwrap_or_default() {
self.show_whisper_model_dialog_box_cleanup()?;
}
}
Ok(())
}
}
impl State {
pub fn mount(par: SettingsMountParams) -> anyhow::Result<State> {
let tasks = Tasks::<Task>::new();
let popup_download = PopupHolder::<views::download_file::View>::default();
let popup_dialog = PopupHolder::<views::dialog_box::View>::default();
let c = options_category(par.mp, par.id_parent, "APP_SETTINGS.FEATURES", "dashboard/options.svg")?;
whisper_models_dropdown(par.mp, c)?;
options_checkbox(par.mp, c, SettingType::NotificationsEnabled)?;
options_checkbox(par.mp, c, SettingType::NotificationsSoundEnabled)?;
options_checkbox(par.mp, c, SettingType::KeyboardSoundEnabled)?;
@ -32,6 +134,173 @@ impl State {
1.0,
0.1,
)?;
Ok(State {})
Ok(State {
tasks,
parent_tasks: par.mp.tasks.clone(),
popup_download,
popup_dialog,
frontend_tasks: par.frontend_tasks.clone(),
globals: par.mp.doc_params.globals.clone(),
pending_download: None,
})
}
fn show_whisper_download_dialog(&mut self, model: &WhisperModel, executor: AsyncExecutor) {
views::download_file::mount_popup(
self.popup_download.clone(),
self.frontend_tasks.clone(),
self.tasks.make_callback_box(Task::WhisperDownloadClosed),
views::download_file::Params {
globals: self.globals.clone(),
executor,
target_path: whisper_model_path(model.file_name),
url: model.url.into(),
on_downloaded: self.tasks.make_callback_box(Task::WhisperDownloadDone),
},
);
}
fn show_whisper_model_dialog_box_download(&mut self, model: &'static WhisperModel) -> anyhow::Result<()> {
const ACTION_DOWNLOAD: &str = "download";
const ACTION_CANCEL: &str = "cancel";
self.pending_download = Some(model);
let tasks = self.tasks.clone();
views::dialog_box::mount_popup(
self.popup_dialog.clone(),
self.frontend_tasks.clone(),
views::dialog_box::Params {
globals: self.globals.clone(),
message: Translation::from_translation_key("APP_SETTINGS.WHISPER.NEED_TO_DOWNLOAD_MODEL"),
entries: vec![
views::dialog_box::ButtonEntry {
content: Translation::from_translation_key("APP_SETTINGS.CANCEL"),
icon: "dashboard/close.svg",
action: ACTION_CANCEL,
},
views::dialog_box::ButtonEntry {
content: Translation::from_translation_key("DOWNLOAD"),
icon: "dashboard/download.svg",
action: ACTION_DOWNLOAD,
},
],
on_action_click: Box::new(move |action| match action {
ACTION_DOWNLOAD => {
tasks.push(Task::WhisperDownload(model));
}
ACTION_CANCEL => {
tasks.push(Task::CloseDialog);
// treat as failed download
tasks.push(Task::WhisperDownloadClosed);
}
_ => unreachable!(),
}),
},
);
Ok(())
}
fn show_whisper_model_dialog_box_cleanup(&mut self) -> anyhow::Result<()> {
const ACTION_REMOVE: &str = "remove";
const ACTION_CANCEL: &str = "cancel";
let tasks = self.tasks.clone();
views::dialog_box::mount_popup(
self.popup_dialog.clone(),
self.frontend_tasks.clone(),
views::dialog_box::Params {
globals: self.globals.clone(),
message: Translation::from_translation_key("APP_SETTINGS.WHISPER.REMOVE_UNUSED_MODELS"),
entries: vec![
views::dialog_box::ButtonEntry {
content: Translation::from_translation_key("APP_SETTINGS.CANCEL"),
icon: "dashboard/close.svg",
action: ACTION_CANCEL,
},
views::dialog_box::ButtonEntry {
content: Translation::from_translation_key("REMOVE"),
icon: "dashboard/trash.svg",
action: ACTION_REMOVE,
},
],
on_action_click: Box::new(move |action| match action {
ACTION_REMOVE => {
tasks.push(Task::WhisperRemoveUnused);
}
ACTION_CANCEL => {
tasks.push(Task::CloseDialog);
}
_ => unreachable!(),
}),
},
);
Ok(())
}
}
fn whisper_models_dropdown(mp: &mut MacroParams, parent: WidgetID) -> anyhow::Result<()> {
let id = mp.idx.to_string();
mp.idx += 1;
let parent = horiz_cell(mp.layout, parent)?;
let current_file = mp.config.whisper_model.as_ref();
let current_translation = whisper_model_from_name(current_file)
.map_or(Translation::from_translation_key("APP_SETTINGS.OPTION.NONE"), |x| {
Translation::from_raw_text(x.display_name)
});
let mut params = TemplateParams::new();
params.insert("id", &id);
params.insert("translation", "APP_SETTINGS.WHISPER_MODEL".into());
params.insert("tooltip", "APP_SETTINGS.WHISPER_MODEL_HELP".into());
mp.parser_state
.instantiate_template(mp.doc_params, "DropdownButton", mp.layout, parent, params)?;
{
let mut label = mp
.parser_state
.fetch_widget_as::<WidgetLabel>(&mp.layout.state, &format!("{id}_value"))?;
label.set_text_simple(&mut mp.layout.state.globals.get(), current_translation);
}
let btn = mp.parser_state.fetch_component_as::<ComponentButton>(&id)?;
btn.on_click(Rc::new({
let parent_tasks = mp.tasks.clone();
move |_common, e: ButtonClickEvent| {
let mut cells = WHISPER_MODELS
.iter()
.map(|item| context_menu::Cell {
action_name: Some(format!(";{id};{}", item.file_name).into()),
title: Translation::from_raw_text(item.display_name),
tooltip: None,
attribs: vec![],
})
.collect::<Vec<_>>();
cells.insert(
0,
context_menu::Cell {
action_name: Some(format!(";{id};").into()),
title: Translation::from_translation_key("APP_SETTINGS.OPTION.NONE"),
tooltip: None,
attribs: vec![],
},
);
parent_tasks.push(ParentTask::OpenContextMenu(
e.mouse_pos_absolute.unwrap_or_default(),
cells,
));
Ok(())
}
}));
Ok(())
}

1
dash-frontend/src/util/mod.rs
View File

@ -6,3 +6,4 @@ pub mod popup_manager;
pub mod steam_utils;
pub mod toast_manager;
pub mod wgui_simple;
pub mod whisper;

81
dash-frontend/src/util/whisper.rs Normal file
View File

@ -0,0 +1,81 @@
use std::{fs, io, path::PathBuf};
use wlx_common::data_dir;
pub struct WhisperModel {
pub file_name: &'static str,
pub display_name: &'static str,
pub url: &'static str,
pub hash: &'static str,
}
pub const WHISPER_MODELS: &[WhisperModel] = &[
WhisperModel {
file_name: "ggml-base-q8_0.bin",
display_name: "Base Q8 (78MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q8_0.bin",
hash: "7bb89bb49ed6955013b166f1b6a6c04584a20fbe",
},
WhisperModel {
file_name: "ggml-small-q8_0.bin",
display_name: "Small Q8 (252MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q8_0.bin",
hash: "bcad8a2083f4e53d648d586b7dbc0cd673d8afad",
},
WhisperModel {
file_name: "ggml-large-v3-turbo-q5_0.bin",
display_name: "Turbo Q5 (574MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin",
hash: "e050f7970618a659205450ad97eb95a18d69c9ee",
},
WhisperModel {
file_name: "ggml-large-v3-turbo-q8_0.bin",
display_name: "Turbo Q8 (874MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q8_0.bin",
hash: "01bf15bedffe9f39d65c1b6ff9b687ea91f59e0e",
},
WhisperModel {
file_name: "ggml-large-v3-turbo.bin",
display_name: "Turbo (1.5GiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin",
hash: "4af2b29d7ec73d781377bfd1758ca957a807e941",
},
];
pub fn whisper_model_from_name(file_name: &str) -> Option<&'static WhisperModel> {
WHISPER_MODELS.iter().find(|x| x.file_name == file_name)
}
pub fn whisper_model_folder() -> PathBuf {
data_dir::get_path("whisper")
}
pub fn whisper_model_path(file_name: &str) -> PathBuf {
whisper_model_folder().join(file_name)
}
pub fn whisper_any_models_downloaded() -> io::Result<bool> {
let path = whisper_model_folder();
if !path.is_dir() {
return Ok(false);
}
Ok(fs::read_dir(path)?.count() > 0)
}
pub fn whisper_delete_all_models() -> io::Result<()> {
let path = whisper_model_folder();
if !path.is_dir() {
return Ok(());
}
for entry in fs::read_dir(path)? {
let entry = entry?;
let file_type = entry.file_type()?;
if file_type.is_file() || file_type.is_symlink() {
fs::remove_file(entry.path())?;
}
}
Ok(())
}

2
dash-frontend/src/views/download_file.rs
View File

@ -220,7 +220,7 @@ impl View {
tasks.push(Task::SetStatusText(String::from("Writing to file...")));
// create skymaps directory if it doesn't exist yet
// create parent directory if it doesn't exist yet
if let Some(parent) = target_path.parent() {
handle_async_result(
"Directory creation failed",

2
wayvr/src/config.rs
View File

@ -184,6 +184,7 @@ pub struct AutoSettings {
pub chroma_key_params: ChromaKeyParams,
pub input_emulation_method: InputEmulationMethod,
pub tutorial_graduated: bool,
pub whisper_model: Arc<str>,
}
fn get_settings_path() -> PathBuf {
@ -246,6 +247,7 @@ pub fn save_settings(config: &GeneralConfig) -> anyhow::Result<()> {
chroma_key_params: config.chroma_key_params.clone(),
input_emulation_method: config.input_emulation_method,
tutorial_graduated: config.tutorial_graduated,
whisper_model: config.whisper_model.clone(),
};
let json = serde_json::to_string_pretty(&conf).unwrap(); // want panic

34
wayvr/src/subsystem/whisper_stt.rs
View File

@ -17,40 +17,6 @@ use spa::{
};
use whisper_rs::{FullParams, SamplingStrategy, WhisperContext, WhisperContextParameters};
pub struct WhisperModel {
pub name: &'static str,
pub url: &'static str,
pub hash: &'static str,
}
pub const WHISPER_MODELS: &[WhisperModel] = &[
WhisperModel {
name: "Base Q8 (78MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base-q8_0.bin",
hash: "7bb89bb49ed6955013b166f1b6a6c04584a20fbe",
},
WhisperModel {
name: "Small Q8 (252MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-small-q8_0.bin",
hash: "bcad8a2083f4e53d648d586b7dbc0cd673d8afad",
},
WhisperModel {
name: "Turbo Q5 (574MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q5_0.bin",
hash: "e050f7970618a659205450ad97eb95a18d69c9ee",
},
WhisperModel {
name: "Turbo Q8 (874MiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo-q8_0.bin",
hash: "01bf15bedffe9f39d65c1b6ff9b687ea91f59e0e",
},
WhisperModel {
name: "Turbo (1.5GiB)",
url: "https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin",
hash: "4af2b29d7ec73d781377bfd1758ca957a807e941",
},
];
const WHISPER_SAMPLE_RATE: usize = 16_000;
const MAX_DURATION: Duration = Duration::from_secs(30);

3
wlx-common/src/config.rs
View File

@ -430,4 +430,7 @@ pub struct GeneralConfig {
#[serde(default)]
pub tutorial_graduated: bool,
#[serde(default = "def_empty")]
pub whisper_model: Arc<str>,
}

20
wlx-common/src/data_dir.rs Normal file
View File

@ -0,0 +1,20 @@
use std::{path::PathBuf, sync::LazyLock};
const FALLBACK_DATA_PATH: &str = "/tmp/wayvr_data";
static DATA_ROOT_PATH: LazyLock<PathBuf> = LazyLock::new(|| {
if let Some(mut dir) = xdg::BaseDirectories::new().get_data_home() {
dir.push("wayvr");
return dir;
}
log::error!("Err: Failed to find data path, using {FALLBACK_DATA_PATH}");
PathBuf::from(FALLBACK_DATA_PATH)
});
fn get_data_root() -> PathBuf {
DATA_ROOT_PATH.clone()
}
pub fn get_path(data_path: &str) -> PathBuf {
get_data_root().join(data_path)
}

1
wlx-common/src/lib.rs
View File

@ -5,6 +5,7 @@ pub mod cache_dir;
pub mod common;
pub mod config;
pub mod config_io;
pub mod data_dir;
pub mod dash_interface;
pub mod dash_interface_emulated;
pub mod desktop_finder;