passthru windows

This commit is contained in:
galister 2026-07-30 00:08:24 +09:00
parent ee1d2aaec2
commit 0962f84c90
19 changed files with 387 additions and 36 deletions

View File

@ -154,7 +154,7 @@
<!-- title text="${name}" /-->
<cell translation="BAR.TOGGLE_VISIBILITY" _press="::OverlayToggle ${name}" />
<cell translation="BAR.RESET_POSITION" _press="::OverlayReset ${name}" />
<cell translation="BAR.CLOSE_MIRROR" _press="::OverlayDrop ${name}" />
<cell translation="BAR.CLOSE_OVERLAY" _press="::OverlayDrop ${name}" />
</context_menu>
</blueprint>
@ -182,6 +182,7 @@
<cell translation="BAR.EDIT_MODE_TOGGLE" _press="::EditToggle" _press2="::ContextMenuClose" />
<cell translation="BAR.HANDSFREE.TITLE" _press="::ContextMenuOpen menu_handsfree" />
<cell translation="BAR.ADD_MIRROR" _press="::NewMirror" />
<cell translation="BAR.ADD_PASSTHRU" _press="::NewPassthru" />
<cell translation="BAR.ADD_NEW_SET" _press="::AddSet" />
<cell translation="BAR.DELETE_CURRENT_SET" _press="::DeleteSet" />
</context_menu>

View File

@ -10,7 +10,6 @@
"TOGGLE_VISIBILITY": "Sichtbarkeit umschalten",
"RESET_POSITION": "Position zurücksetzen",
"RELOAD_FROM_DISK": "XML-Datei von der Festplatte neu laden",
"CLOSE_MIRROR": "Spiegel schließen",
"CLOSE_APP": "App schließen",
"FORCE_CLOSE_APP": "App zwangsweise schließen",
"HANDSFREE": {
@ -20,7 +19,9 @@
"EYE_TRACKING": "Auge + Kneifen",
"EYE_ONLY": "Augenlaser",
"HMD_ONLY": "HMD-Laser"
}
},
"ADD_PASSTHRU": "Passthrough-Fenster hinzufügen",
"CLOSE_OVERLAY": "Overlay schließen"
},
"WATCH": {
"RECENTER": "Spielbereich neu zentrieren",

View File

@ -5,8 +5,9 @@
"BAR": {
"ADD_MIRROR": "Add a new mirror overlay",
"ADD_NEW_SET": "Add new set",
"ADD_PASSTHRU": "Add a passthrough window",
"CLOSE_APP": "Close app",
"CLOSE_MIRROR": "Close mirror",
"CLOSE_OVERLAY": "Close overlay",
"DELETE_CURRENT_SET": "Delete current set",
"EDIT_MODE_TOGGLE": "Toggle edit mode",
"FORCE_CLOSE_APP": "Force close app",

View File

@ -10,7 +10,6 @@
"TOGGLE_VISIBILITY": "Alternar visibilidad",
"RESET_POSITION": "Restablecer posición",
"RELOAD_FROM_DISK": "Volver a cargar XML desde el disco",
"CLOSE_MIRROR": "Cerrar espejo",
"CLOSE_APP": "Cerrar aplicación",
"FORCE_CLOSE_APP": "Forzar cierre de la aplicación",
"HANDSFREE": {
@ -20,7 +19,9 @@
"EYE_TRACKING": "Ojo + pellizco",
"EYE_ONLY": "Láser ocular",
"HMD_ONLY": "Láser del HMD"
}
},
"ADD_PASSTHRU": "Añadir una ventana de passthrough",
"CLOSE_OVERLAY": "Cerrar superposición"
},
"WATCH": {
"RECENTER": "Recentrar el área de juego",

View File

@ -10,8 +10,6 @@
"TOGGLE_VISIBILITY": "Alterna visibilità",
"RESET_POSITION": "Ripristina posizione",
"RELOAD_FROM_DISK": "Ricarica XML dal disco",
"CLOSE_MIRROR": "Chiudi mirror",
"CLOSE_APP": "Chiudi app",
"FORCE_CLOSE_APP": "Chiudi forzatamente l'app",
"HANDSFREE": {
"TITLE": "Modalità a mani libere",
@ -20,7 +18,10 @@
"EYE_TRACKING": "Occhio + pizzico",
"EYE_ONLY": "Laser oculare",
"HMD_ONLY": "Laser HMD"
}
},
"ADD_PASSTHRU": "Aggiungi una finestra passthrough",
"CLOSE_APP": "Chiudi app",
"CLOSE_OVERLAY": "Chiudi overlay"
},
"DEFAULT": "Predefinito",
"DISABLED": "Disabilitato",

View File

@ -10,7 +10,6 @@
"TOGGLE_VISIBILITY": "表示/非表示の切り替え",
"RESET_POSITION": "位置をリセット",
"RELOAD_FROM_DISK": "ディスクからXMLを再読み込み",
"CLOSE_MIRROR": "ミラーを閉じる",
"CLOSE_APP": "アプリを閉じる",
"FORCE_CLOSE_APP": "アプリを強制終了",
"HANDSFREE": {
@ -20,7 +19,9 @@
"EYE_TRACKING": "視線 + ピンチ",
"EYE_ONLY": "アイルーザー",
"HMD_ONLY": "HMDレーザー"
}
},
"ADD_PASSTHRU": "パススルーウィンドウを追加",
"CLOSE_OVERLAY": "オーバーレイを閉じる"
},
"WATCH": {
"RECENTER": "プレイスペースをリセンター",

View File

@ -10,7 +10,6 @@
"TOGGLE_VISIBILITY": "Przełącz widoczność",
"RESET_POSITION": "Zresetuj pozycję",
"RELOAD_FROM_DISK": "Przeładuj XML z dysku",
"CLOSE_MIRROR": "Zamknij lustro",
"CLOSE_APP": "Zamknij aplikację",
"FORCE_CLOSE_APP": "Wymuś zamknięcie aplikacji",
"HANDSFREE": {
@ -20,7 +19,9 @@
"EYE_TRACKING": "Oko + uszczypnięcie",
"EYE_ONLY": "Laser wzrokowy",
"HMD_ONLY": "Laser w HMD"
}
},
"ADD_PASSTHRU": "Dodaj okno passthrough",
"CLOSE_OVERLAY": "Zamknij nakładkę"
},
"WATCH": {
"RECENTER": "Wyśrodkuj przestrzeń gry",

View File

@ -10,7 +10,6 @@
"TOGGLE_VISIBILITY": "切换可见性",
"RESET_POSITION": "重置位置",
"RELOAD_FROM_DISK": "从磁盘重新加载 XML",
"CLOSE_MIRROR": "关闭镜像",
"CLOSE_APP": "关闭应用",
"FORCE_CLOSE_APP": "强制关闭应用",
"HANDSFREE": {
@ -20,7 +19,9 @@
"EYE_TRACKING": "眼动 + 捏合",
"EYE_ONLY": "眼睛激光",
"HMD_ONLY": "头显射线"
}
},
"ADD_PASSTHRU": "添加透视窗口",
"CLOSE_OVERLAY": "关闭叠加层"
},
"DEFAULT": "默认",
"DISABLED": "已禁用",

View File

@ -1,3 +1,5 @@
use std::ffi::CStr;
use anyhow::{bail, ensure};
use glam::{Affine3A, Quat, Vec3, Vec3A};
use openxr::{self as xr, SessionCreateFlags, Version, sys::Handle};
@ -41,6 +43,12 @@ pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
} else {
log::warn!("Missing EXT_dpad_binding extension.");
}
if available_extensions.fb_composition_layer_alpha_blend {
enabled_extensions.fb_composition_layer_alpha_blend = true;
} else {
log::warn!("Missing XR_FB_composition_layer_alpha_blend extension.");
}
if available_extensions.ext_samsung_odyssey_controller {
enabled_extensions.ext_samsung_odyssey_controller = true;
} else {
@ -118,6 +126,22 @@ pub(super) fn init_xr() -> Result<(xr::Instance, xr::SystemId), anyhow::Error> {
instance_props.runtime_version
);
let enabled_extensions = enabled_extensions
.names()
.into_iter()
.filter_map(|name| {
Some(
CStr::from_bytes_with_nul(name)
.ok()?
.to_string_lossy()
.into_owned(),
)
})
.collect::<Vec<_>>()
.join("\n");
log::info!("OpenXR extensions:\n{enabled_extensions}");
let Ok(system) = xr_instance.system(xr::FormFactor::HEAD_MOUNTED_DISPLAY) else {
bail!("Failed to access OpenXR HMD system.");
};

View File

@ -11,7 +11,7 @@ use crate::{
swapchain::{SwapchainOpts, WlxSwapchainImage, create_swapchain},
},
state::AppState,
windowing::window::OverlayWindowData,
windowing::window::{OverlayCategory, OverlayWindowData},
};
#[derive(Default)]
@ -21,6 +21,7 @@ pub struct OpenXrOverlayData {
pub(super) init: bool,
pub(super) cur_visible: bool,
color_bias_khr: Option<Box<xr::sys::CompositionLayerColorScaleBiasKHR>>,
layer_alpha_blend_fb: Option<Box<xr::sys::CompositionLayerAlphaBlendFB>>,
}
impl OverlayWindowData<OpenXrOverlayData> {
@ -104,12 +105,34 @@ impl OverlayWindowData<OpenXrOverlayData> {
(major, major / aspect_ratio)
};
let flags = if state.additive {
CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA
} else {
CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA
| CompositionLayerFlags::UNPREMULTIPLIED_ALPHA
};
let mut flags = CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA;
if state.additive {
flags |= CompositionLayerFlags::UNPREMULTIPLIED_ALPHA;
}
if xr
.instance
.exts()
.fb_composition_layer_alpha_blend
.is_some()
&& self.config.category == OverlayCategory::Passthru
&& self.data.layer_alpha_blend_fb.is_none()
&& !self.config.editing
{
self.data.layer_alpha_blend_fb =
Some(Box::new(openxr::sys::CompositionLayerAlphaBlendFB {
ty: xr::sys::CompositionLayerAlphaBlendFB::TYPE,
next: std::ptr::null_mut(),
src_factor_color: xr::BlendFactorFB::ZERO,
dst_factor_color: xr::BlendFactorFB::SRC_ALPHA,
src_factor_alpha: xr::BlendFactorFB::ONE,
dst_factor_alpha: xr::BlendFactorFB::ZERO,
}));
flags &= !CompositionLayerFlags::BLEND_TEXTURE_SOURCE_ALPHA;
} else if self.config.editing {
self.data.layer_alpha_blend_fb = None;
}
if let Some(curvature) = state.curvature {
let radius = scale_x / (2.0 * PI * curvature);
@ -132,6 +155,12 @@ impl OverlayWindowData<OpenXrOverlayData> {
.central_angle(angle)
.aspect_ratio(aspect_ratio);
if let Some(layer_alpha_blend_fb) = self.data.layer_alpha_blend_fb.as_mut() {
unsafe {
let raw = next_chain_insert!(cylinder, layer_alpha_blend_fb);
cylinder = xr::CompositionLayerCylinderKHR::from_raw(raw);
}
}
if let Some(color_bias_khr) = self.data.color_bias_khr.as_mut() {
unsafe {
let raw = next_chain_insert!(cylinder, color_bias_khr);
@ -157,6 +186,12 @@ impl OverlayWindowData<OpenXrOverlayData> {
height: scale_y,
});
if let Some(layer_alpha_blend_fb) = self.data.layer_alpha_blend_fb.as_mut() {
unsafe {
let raw = next_chain_insert!(quad, layer_alpha_blend_fb);
quad = xr::CompositionLayerQuad::from_raw(raw);
}
}
if let Some(color_bias_khr) = self.data.color_bias_khr.as_mut() {
unsafe {
let raw = next_chain_insert!(quad, color_bias_khr);

View File

@ -10,7 +10,10 @@ use serde::Deserialize;
use crate::{
backend::input,
state::AppState,
windowing::{OverlaySelector, window::OverlayWindowConfig},
windowing::{
OverlaySelector,
window::{OverlayCategory, OverlayWindowConfig},
},
};
static TASK_AUTO_INCREMENT: AtomicUsize = AtomicUsize::new(0);
@ -123,7 +126,7 @@ pub enum OverlayTask {
ToggleEditMode,
ToggleDashboard,
ShowHide,
CleanupMirrors,
CleanupOverlays(OverlayCategory),
GlobalChange(GlobalChange),
Modify(OverlaySelector, Box<ModifyOverlayTask>),
Spawn(OverlaySelector, SpawnPos, Box<CreateOverlayTask>),

View File

@ -552,7 +552,35 @@ pub(super) fn setup_custom_button<S: 'static>(
}
app.tasks
.enqueue(TaskType::Overlay(OverlayTask::CleanupMirrors));
.enqueue(TaskType::Overlay(OverlayTask::CleanupOverlays(
OverlayCategory::Mirror,
)));
Ok(EventResult::Consumed)
}),
"::NewPassthru" => Box::new(move |_common, data, app, _| {
if !test_button(data) || !test_duration(&button, app) {
return Ok(EventResult::Pass);
}
let name = crate::overlays::passthrough::new_passtrhu_name();
app.tasks.enqueue(TaskType::Overlay(OverlayTask::Spawn(
OverlaySelector::Name(name.clone()),
SpawnPos::Spread,
Box::new(move |app| {
Some(crate::overlays::passthrough::new_passthru(name, app))
}),
)));
Ok(EventResult::Consumed)
}),
"::CleanupPassthrus" => Box::new(move |_common, data, app, _| {
if !test_button(data) || !test_duration(&button, app) {
return Ok(EventResult::Pass);
}
app.tasks
.enqueue(TaskType::Overlay(OverlayTask::CleanupOverlays(
OverlayCategory::Passthru,
)));
Ok(EventResult::Consumed)
}),
"::PlayspaceReset" => Box::new(move |_common, data, app, _| {

View File

@ -59,6 +59,13 @@ impl OverlayList {
);
("Mirror", panels_root)
}
OverlayCategory::Passthru => {
params.insert_rc(
"display",
(*meta.name).chars().last().unwrap().to_string().into(),
);
("Mirror", panels_root)
}
OverlayCategory::Panel | OverlayCategory::BuiltInPanel => {
let icon: Rc<str> = if let Some(icon) = meta.icon.as_ref() {
icon.to_string().into()

View File

@ -3,6 +3,7 @@ pub mod custom;
pub mod dashboard;
pub mod edit;
pub mod keyboard;
pub mod passthrough;
pub mod screen;
pub mod toast;
pub mod watch;

View File

@ -0,0 +1,243 @@
use std::sync::{
Arc,
atomic::{AtomicUsize, Ordering},
};
use glam::{Affine2, Affine3A, Quat, Vec3};
use slotmap::Key;
use vulkano::buffer::BufferUsage;
use wgui::{
color::WguiColorName,
gfx::pipeline::{WGfxPipeline, WPipelineCreateInfo},
};
use wlx_common::{
overlays::{BackendAttrib, BackendAttribValue},
windowing::{OverlayWindowState, Positioning},
};
use crate::{
backend::task::{OverlayTask, TaskType},
graphics::Vert2Uv,
state::AppState,
windowing::{
OverlayID, OverlaySelector,
backend::{FrameMeta, OverlayBackend, OverlayEventData, ShouldRender, ui_transform},
overlay_scale_from_extent,
window::{OverlayCategory, OverlayWindowConfig},
},
};
static PASSTHRU_COUNTER: AtomicUsize = AtomicUsize::new(1);
pub fn new_passtrhu_name() -> Arc<str> {
format!("P-{}", PASSTHRU_COUNTER.fetch_add(1, Ordering::Relaxed)).into()
}
pub fn new_passthru(name: Arc<str>, app: &AppState) -> OverlayWindowConfig {
OverlayWindowConfig {
name,
default_state: OverlayWindowState {
transform: Affine3A::from_scale_rotation_translation(
Vec3::ONE * 0.5,
Quat::IDENTITY,
Vec3::new(0., 0., -0.5),
),
grabbable: true,
interactable: true,
positioning: Positioning::Static,
..OverlayWindowState::default()
},
global: true,
show_on_spawn: true,
category: OverlayCategory::Passthru,
..OverlayWindowConfig::from_backend(Box::new(PassthruBackend::new(app)))
}
}
struct PassthruBackend {
frame_meta: FrameMeta,
rendered: bool,
interaction_transform: Affine2,
scale: f32,
overlay_id: OverlayID,
}
const DEFAULT_EXTENT: [u32; 2] = [512, 512];
impl PassthruBackend {
fn new(app: &AppState) -> Self {
let (scale, _) = overlay_scale_from_extent(DEFAULT_EXTENT);
Self {
frame_meta: FrameMeta {
extent: DEFAULT_EXTENT,
format: app.gfx.surface_format,
..Default::default()
},
interaction_transform: ui_transform(DEFAULT_EXTENT),
rendered: false,
scale,
overlay_id: OverlayID::null(),
}
}
}
impl OverlayBackend for PassthruBackend {
fn init(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
Ok(())
}
fn pause(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
Ok(())
}
fn resume(&mut self, _app: &mut AppState) -> anyhow::Result<()> {
Ok(())
}
fn should_render(&mut self, _app: &mut AppState) -> anyhow::Result<ShouldRender> {
Ok(if self.rendered {
ShouldRender::Can
} else {
ShouldRender::Should
})
}
fn render(
&mut self,
app: &mut AppState,
rdr: &mut crate::windowing::backend::RenderResources,
) -> anyhow::Result<()> {
// this is heavy, but only done once
let pipeline: Arc<WGfxPipeline<Vert2Uv>> = app.gfx.create_pipeline(
app.gfx_extras.shaders.get("vert_quad").unwrap(), // want panic
app.gfx_extras.shaders.get("frag_color").unwrap(), // want panic
WPipelineCreateInfo::new(app.gfx.surface_format),
)?;
let color = WguiColorName::Primary
.to_wgui_color()
.resolve(&app.wgui_globals.get().palette);
let buf_color = app.gfx.new_buffer(
BufferUsage::TRANSFER_DST | BufferUsage::UNIFORM_BUFFER,
color.with_alpha(1.0).as_arr().iter(),
)?;
let set0 = pipeline.buffer(0, buf_color.clone())?;
let extentf32 = [
self.frame_meta.extent[0] as f32,
self.frame_meta.extent[1] as f32,
];
let pass = pipeline.create_pass(
extentf32,
[0.0, 0.0],
app.gfx_extras.quad_verts.clone(),
0..4,
0..1,
vec![set0],
&Default::default(),
)?;
rdr.cmd_buf_single().run_ref(&pass)?;
let buf_color = app.gfx.new_buffer(
BufferUsage::TRANSFER_DST | BufferUsage::UNIFORM_BUFFER,
[0.0, 0.0, 0.0, 1.0].iter(),
)?;
let set0 = pipeline.buffer(0, buf_color.clone())?;
let pass = pipeline.create_pass(
[extentf32[0] - 8.0, extentf32[1] - 8.0],
[4.0, 4.0],
app.gfx_extras.quad_verts.clone(),
0..4,
0..1,
vec![set0],
&Default::default(),
)?;
rdr.cmd_buf_single().run_ref(&pass)?;
//self.rendered = true;
Ok(())
}
fn frame_meta(&mut self) -> Option<FrameMeta> {
Some(self.frame_meta)
}
fn notify(&mut self, app: &mut AppState, event_data: OverlayEventData) -> anyhow::Result<()> {
match event_data {
OverlayEventData::IdAssigned(id) => self.overlay_id = id,
OverlayEventData::ResizeRequest(new_size) => {
self.frame_meta.extent =
[new_size[0].clamp(256, 1024), new_size[1].clamp(256, 1024)];
self.interaction_transform = ui_transform(self.frame_meta.extent);
let old_scale = self.scale;
let (new_scale, _) = overlay_scale_from_extent(self.frame_meta.extent);
self.scale = new_scale;
if new_scale.abs() > f32::EPSILON {
let scale_delta = new_scale / old_scale;
if (scale_delta - 1.0).abs() > f32::EPSILON {
app.tasks.enqueue(TaskType::Overlay(OverlayTask::Modify(
OverlaySelector::Id(self.overlay_id),
Box::new(move |_app, owc| {
if let Some(state) = owc.active_state.as_mut() {
if let Some(saved) = state.saved_transform.as_mut() {
saved.matrix3 = saved.matrix3.mul_scalar(scale_delta);
}
state.transform.matrix3 =
state.transform.matrix3.mul_scalar(scale_delta);
}
}),
)));
}
}
self.rendered = false;
}
_ => {}
}
Ok(())
}
fn on_hover(
&mut self,
_app: &mut AppState,
_hit: &crate::backend::input::PointerHit,
) -> crate::backend::input::HoverResult {
crate::backend::input::HoverResult::consume()
}
fn on_left(&mut self, _app: &mut AppState, _pointer: usize) {}
fn on_pointer(
&mut self,
_app: &mut AppState,
_hit: &crate::backend::input::PointerHit,
_pressed: bool,
) {
}
fn on_scroll(
&mut self,
_app: &mut AppState,
_hit: &crate::backend::input::PointerHit,
_delta: crate::subsystem::hid::WheelDelta,
) {
}
fn get_interaction_transform(&mut self) -> Option<Affine2> {
Some(self.interaction_transform)
}
fn get_attrib(
&self,
attrib: wlx_common::overlays::BackendAttrib,
) -> Option<wlx_common::overlays::BackendAttribValue> {
match attrib {
BackendAttrib::Resizable => Some(BackendAttribValue::Resizable(true)),
_ => None,
}
}
fn set_attrib(
&mut self,
_app: &mut AppState,
_value: wlx_common::overlays::BackendAttribValue,
) -> bool {
false
}
}

View File

@ -64,6 +64,7 @@ use crate::{
FrameMeta, OverlayBackend, OverlayEventData, RenderResources, ShouldRender,
ui_transform,
},
overlay_scale_from_extent,
window::{OverlayCategory, OverlayWindowConfig},
},
};
@ -1078,12 +1079,3 @@ impl OverlayBackend for WvrWindowBackend {
}
}
}
fn overlay_scale_from_extent(size: [u32; 2]) -> (f32, f32) {
const RELATIVE_SIZE: f32 = 1440.0; // sqrt(1920 * 1080)
let w = size[0].max(1) as f32;
let h = size[1].max(1) as f32;
((w * h).sqrt() / RELATIVE_SIZE, w / 1920.0)
}

View File

@ -347,10 +347,10 @@ where
.notify(app, OverlayEventData::ColorPaletteRefresh);
}
}
OverlayTask::CleanupMirrors => {
OverlayTask::CleanupOverlays(category) => {
let mut ids_to_remove = vec![];
for (oid, o) in &self.overlays {
if !matches!(o.config.category, OverlayCategory::Mirror) {
if o.config.category == category {
continue;
}
if o.config.active_state.is_some() {

View File

@ -49,3 +49,12 @@ pub fn snap_upright(transform: Affine3A, up_dir: Vec3A) -> Affine3A {
transform
}
}
pub fn overlay_scale_from_extent(size: [u32; 2]) -> (f32, f32) {
const RELATIVE_SIZE: f32 = 1440.0; // sqrt(1920 * 1080)
let w = size[0].max(1) as f32;
let h = size[1].max(1) as f32;
((w * h).sqrt() / RELATIVE_SIZE, w / 1920.0)
}

View File

@ -64,6 +64,7 @@ pub enum OverlayCategory {
Screen,
Mirror,
WayVR,
Passthru,
}
pub struct OverlayWindowConfig {