RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

whisper ui tweaks + osc send

This commit is contained in:
galister 2026-07-03 23:30:48 +09:00
parent 341ee333dc
commit a54d1a02bb
5 changed files with 138 additions and 56 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

15
wayvr/src/assets/gui/whisper.xml
View File

@ -5,21 +5,24 @@
<macro name="bg_rect" width="100%" color="~color_bg" round="10" border="2" border_color="~color_accent" />
<elements>
<div flex_direction="column" interactable="0" width="600" height="200">
<div flex_direction="column" width="600" >
<rectangle macro="bg_rect" padding="10" align_items="center" justify_content="space_between">
<div gap="10">
<Button macro="button_style" _press="::WhisperTranscribeStart" _release="::WhisperTranscribeStop" tooltip="WHISPER.TRANSCRIBE">
<sprite width="38" height="38" color="~color_text" src="icons/mic.svg" />
</Button>
<Button macro="button_style" _press="::WhisperPaste" tooltip="WHISPER.PASTE">
<sprite width="38" height="38" color="~color_text" src_builtin="icons/paste.svg" />
</Button>
<Button macro="button_style" _press="::WhisperPasteAndGo" tooltip="WHISPER.PASTE_AND_GO">
<sprite width="38" height="38" color="~color_text" src_builtin="icons/paste-go.svg" />
</Button>
<Button macro="button_style" _press="::WhisperSendOSC" tooltip="WHISPER.SEND_OSC" id="btn_osc_send">
<sprite width="38" height="38" color="~color_text" src_builtin="icons/chat.svg" />
</Button>
</div>
<div gap="10">
<Button macro="button_style" _press="::WhisperUnload" tooltip="WHISPER.UNLOAD">
<sprite width="38" height="38" color="~color_text" src_builtin="watch/settings.svg" />
</Button>
<Button macro="button_style" _press="::OverlayToggle whisper">
<Button macro="button_style" _press="::WhisperUnloadAndClose" tooltip="WHISPER.UNLOAD_AND_CLOSE">
<sprite width="38" height="38" color="~color_text" src_builtin="edit/close.svg" />
</Button>
</div>
@ -27,7 +30,7 @@
<div width="100%" height="13" interactable="0" />
<rectangle macro="bg_rect" flex_direction="column" padding="10">
<div>
<label id="transcription" wrap="1" size="20" padding_left="16" padding_right="16" text="Press and hold the microphone button to start transcribing..." />
<label id="transcription" wrap="1" size="20" padding_left="16" padding_right="16" translation="WHISPER.INITIAL_TEXT" />
</div>
</rectangle>
</div>

1
wayvr/src/assets/icons/chat.svg Normal file
View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24"><!-- Icon from Material Symbols by Google - https://github.com/google/material-design-icons/blob/master/LICENSE --><path fill="white" d="M2 22V4q0-.825.588-1.412T4 2h16q.825 0 1.413.588T22 4v12q0 .825-.587 1.413T20 18H6zm4-8h8v-2H6zm0-3h12V9H6zm0-3h12V6H6z"/></svg>
Side by Side

After

Width:  |  Height:  |  Size: 349 B

1
wayvr/src/assets/icons/paste.svg Normal file
View File

@ -0,0 +1 @@
<svg xmlns="http://www.w3.org/2000/svg" width="32" height="32" viewBox="0 0 24 24"><!-- Icon from Material Symbols by Google - https://github.com/google/material-design-icons/blob/master/LICENSE --><path fill="white" d="M5 21q-.825 0-1.412-.587T3 19V5q0-.825.588-1.412T5 3h4.175q.275-.875 1.075-1.437T12 1q1 0 1.788.563T14.85 3H19q.825 0 1.413.588T21 5v14q0 .825-.587 1.413T19 21zm0-2h14V5h-2v3H7V5H5zm7-14q.425 0 .713-.288T13 4t-.288-.712T12 3t-.712.288T11 4t.288.713T12 5"/></svg>
Side by Side

After

Width:  |  Height:  |  Size: 483 B

10
wayvr/src/assets/lang/en.json
View File

@ -115,9 +115,13 @@
},
"WHISPER": {
"TRANSCRIBE": "Press and hold to transcribe text",
"PASTE": "Paste transcription to focused overlay",
"UNLOAD": "Unload model from VRAM",
"PASTE": "Copy transcription to clipboard",
"PASTE_AND_GO": "Copy transcription and send Ctrl-V to focused app",
"SEND_OSC": "Send transcription to OSC chatbox",
"UNLOAD_AND_CLOSE": "Unload model and close",
"MODEL_NOT_DOWNLOADED": "Speech-to-text not set up!",
"DOWNLOAD_GUIDANCE": "Set up speech-to-text under settings, features!"
"DOWNLOAD_GUIDANCE": "Set up speech-to-text under settings, features!",
"INITIAL_TEXT": "Press and hold the mic button to start transcribing...",
"INIT_ERROR": "Error while starting Whisper engine"
}
}

167
wayvr/src/overlays/whisper.rs
View File

@ -16,6 +16,7 @@ use wlx_common::{
};
use crate::{
backend::task::{OverlayTask, TaskType, ToggleMode},
gui::{
panel::{
GuiPanel, NewGuiPanelParams, OnCustomAttribFunc,
@ -31,10 +32,13 @@ use crate::{
input::KeyboardFocus,
whisper_stt::WhisperStt,
},
windowing::window::{OverlayCategory, OverlayWindowConfig},
windowing::{
OverlaySelector,
window::{OverlayCategory, OverlayWindowConfig},
},
};
pub const WHISPER_NAME: &str = "Speech-to-Text";
pub const WHISPER_NAME: &str = "whisper";
#[derive(Default)]
struct WhisperState {
@ -43,6 +47,30 @@ struct WhisperState {
last_transcription: Option<Rc<str>>,
}
impl WhisperState {
fn set_clipboard_text(&mut self, app: &mut AppState) -> bool {
let Some(text) = self.last_transcription.as_ref() else {
return false;
};
match app.hid_provider.keyboard_focus {
KeyboardFocus::WayVR => {
if let Some(wvr) = app.wvr_server.as_mut() {
wvr.set_clipboard(text);
return true;
}
}
KeyboardFocus::PhysicalScreen => {
if let Some(clip) = self.clipboard_provider.as_mut() {
clip.set_clipboard_utf8(text);
return true;
}
}
}
return false;
}
}
pub fn create_whisper(
app: &mut AppState,
headless: bool,
@ -93,30 +121,47 @@ pub fn create_whisper(
return Ok(EventResult::Pass);
}
if let Some(whisper) = state.whisper_sst.as_mut() {
let _ = whisper
.ptt_start()
.log_err("Could not start Whisper transcription");
let whisper = match state.whisper_sst.as_mut() {
Some(x) => x,
None => {
let model_path = data_dir::get_path("whisper")
.join(app.session.config.whisper_model.as_ref());
if model_path.is_file() {
state.whisper_sst = match WhisperStt::new(model_path)
.log_err("Error while starting Whisper engine")
{
Ok(x) => Some(x),
Err(e) => {
Toast::new(
ToastTopic::System,
"WHISPER.INIT_ERROR".into(),
e.to_string(),
)
.with_timeout(5.)
.with_sound(true)
.submit(app);
return Ok(EventResult::Consumed);
}
}
} else {
Toast::new(
ToastTopic::System,
"WHISPER.MODEL_NOT_DOWNLOADED".into(),
"WHISPER.DOWNLOAD_GUIDANCE".into(),
)
.with_timeout(5.)
.with_sound(true)
.submit(app);
return Ok(EventResult::Consumed);
}
return Ok(EventResult::Consumed);
}
state.whisper_sst.as_mut().unwrap()
}
};
let model_path = data_dir::get_path("whisper")
.join(app.session.config.whisper_model.as_ref());
if model_path.is_file() {
state.whisper_sst = WhisperStt::new(model_path)
.log_err("Could not create STT provider")
.ok();
} else {
Toast::new(
ToastTopic::System,
"WHISPER.MODEL_NOT_DOWNLOADED".into(),
"WHISPER.DOWNLOAD_GUIDANCE".into(),
)
.with_timeout(5.)
.with_sound(true)
.submit(app);
}
let _ = whisper
.ptt_start()
.log_err("Could not start Whisper transcription");
Ok(EventResult::Consumed)
}),
@ -137,29 +182,17 @@ pub fn create_whisper(
return Ok(EventResult::Pass);
}
let Some(transcription) = state.last_transcription.as_ref() else {
return Ok(EventResult::Consumed);
};
state.set_clipboard_text(app);
let mut success = false;
match app.hid_provider.keyboard_focus {
KeyboardFocus::WayVR => {
if let Some(wvr) = app.wvr_server.as_mut() {
wvr.set_clipboard(transcription.as_ref());
success = true;
}
}
KeyboardFocus::PhysicalScreen => {
if let Some(clip) = state.clipboard_provider.as_mut() {
clip.set_clipboard_utf8(transcription.as_ref());
success = true;
}
}
Ok(EventResult::Consumed)
}),
"::WhisperPasteAndGo" => Box::new(move |_common, data, app, state| {
if !test_button(data) || !test_duration(&button, app) {
return Ok(EventResult::Pass);
}
// send ctrl-v
if success {
if state.set_clipboard_text(app) {
// send ctrl-v
app.hid_provider.send_key_routed(
app.wvr_server.as_mut(),
VirtualKey::RCtrl,
@ -179,12 +212,38 @@ pub fn create_whisper(
Ok(EventResult::Consumed)
}),
"::WhisperUnload" => Box::new(move |_common, data, app, state| {
#[cfg(feature = "osc")]
"::WhisperSendOSC" => Box::new(move |_common, data, app, state| {
if !test_button(data) || !test_duration(&button, app) {
return Ok(EventResult::Pass);
}
if let Some(text) = state.last_transcription.as_ref()
&& let Some(osc) = app.osc_sender.as_mut()
{
use rosc::OscType;
let _ = osc
.send_message(
"/chatbox/input".into(),
vec![OscType::String(text.to_string()), OscType::Bool(true)],
)
.log_err("Could not send OSC message");
}
Ok(EventResult::Consumed)
}),
"::WhisperUnloadAndClose" => Box::new(move |_common, data, app, state| {
if !test_button(data) || !test_duration(&button, app) {
return Ok(EventResult::Pass);
}
state.whisper_sst = None;
app.tasks
.enqueue(TaskType::Overlay(OverlayTask::ToggleOverlay(
OverlaySelector::Name(WHISPER_NAME.into()),
ToggleMode::EnsureOff,
)));
Ok(EventResult::Consumed)
}),
@ -208,6 +267,22 @@ pub fn create_whisper(
BackendAttribValue::Icon("icons/mic.svg".into()),
);
#[cfg(not(feature = "osc"))]
{
use wgui::event::{CallbackDataCommon, StyleSetRequest};
let osc_button = panel
.parser_state
.fetch_component_as::<ComponentButton>("btn_osc_send")?;
let common = CallbackDataCommon {
state: &panel.layout.state,
alterables: &mut panel.layout.alterables,
};
common.alterables.set_style(
osc_button.get_rect(),
StyleSetRequest::Display(wgui::taffy::Display::None),
);
}
let label = panel.parser_state.get_widget_id("transcription")?;
panel
@ -221,11 +296,9 @@ pub fn create_whisper(
{
let text: Rc<str> = text.into();
state.last_transcription = Some(text.clone());
let label = data.obj.get_as_mut::<WidgetLabel>().unwrap();
label.set_text(common, Translation::from_raw_text_rc(text));
}
Ok(EventResult::Pass)
});