From 8c4a3b46cbeaf0efbb8a7775903e9648e26637d0 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Wed, 18 Sep 2024 13:58:26 -0700 Subject: [PATCH 01/20] WIP --- ui/src/connecteddevice.py | 12 ++++++ ui/src/gtk/connected-device.ui | 13 +++++++ ui/src/meson.build | 1 + ui/src/virtualdisplay.py | 71 ++++++++++++++++++++++++++++++++++ 4 files changed, 97 insertions(+) create mode 100644 ui/src/virtualdisplay.py diff --git a/ui/src/connecteddevice.py b/ui/src/connecteddevice.py index a1883ba..3693da2 100644 --- a/ui/src/connecteddevice.py +++ b/ui/src/connecteddevice.py @@ -4,10 +4,13 @@ from .license import BREEZY_GNOME_FEATURES from .settingsmanager import SettingsManager from .shortcutdialog import bind_shortcut_settings from .statemanager import StateManager +from .virtualdisplay import VirtualMonitor from .xrdriveripc import XRDriverIPC import gettext +import logging _ = gettext.gettext +logger = logging.getLogger('breezy_ui') @Gtk.Template(resource_path='/com/xronlinux/BreezyDesktop/gtk/connected-device.ui') class ConnectedDevice(Gtk.Box): @@ -28,6 +31,7 @@ class ConnectedDevice(Gtk.Box): widescreen_mode_switch = Gtk.Template.Child() widescreen_mode_row = Gtk.Template.Child() curved_display_switch = Gtk.Template.Child() + add_virtual_display_button = Gtk.Template.Child() set_toggle_display_distance_start_button = Gtk.Template.Child() set_toggle_display_distance_end_button = Gtk.Template.Child() reassign_recenter_display_shortcut_button = Gtk.Template.Child() @@ -55,6 +59,7 @@ class ConnectedDevice(Gtk.Box): self.follow_mode_switch, self.follow_threshold_scale, self.curved_display_switch, + # self.add_virtual_display_button, self.set_toggle_display_distance_start_button, self.set_toggle_display_distance_end_button, self.movement_look_ahead_scale @@ -87,6 +92,7 @@ class ConnectedDevice(Gtk.Box): self.set_toggle_display_distance_start_button, self.set_toggle_display_distance_end_button ]) + self.add_virtual_display_button.connect('clicked', self.on_add_virtual_display) self.state_manager = StateManager.get_instance() self.state_manager.bind_property('follow-mode', self.follow_mode_switch, 'active', GObject.BindingFlags.DEFAULT) @@ -166,6 +172,12 @@ class ConnectedDevice(Gtk.Box): for widget in widgets: widget.connect('clicked', lambda *args, widget=widget: on_set_display_distance_toggle(widget)) reload_display_distance_toggle_button(widget) + + def on_add_virtual_display(self, widget): + VirtualMonitor(1920, 1080, self.on_virtual_display_ready).create() + + def on_virtual_display_ready(self): + logger.info("Virtual display ready") def _on_widget_destroy(self, widget): self.state_manager.unbind_property('follow-mode', self.follow_mode_switch, 'active') diff --git a/ui/src/gtk/connected-device.ui b/ui/src/gtk/connected-device.ui index 3a73559..2cec2c6 100644 --- a/ui/src/gtk/connected-device.ui +++ b/ui/src/gtk/connected-device.ui @@ -84,6 +84,19 @@ + + + Virtual monitors + 2 + + + add-virtual-display + 3 + Add + + + + diff --git a/ui/src/meson.build b/ui/src/meson.build index cb1ecbc..8272bfa 100644 --- a/ui/src/meson.build +++ b/ui/src/meson.build @@ -46,6 +46,7 @@ breezydesktop_sources = [ 'shortcutdialog.py', 'statemanager.py', 'time.py', + 'virtualdisplay.py', 'verify.py', 'window.py' ] diff --git a/ui/src/virtualdisplay.py b/ui/src/virtualdisplay.py new file mode 100644 index 0000000..b57f473 --- /dev/null +++ b/ui/src/virtualdisplay.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 + +import logging +import sys +import signal +import pydbus +import gi +gi.require_version('Gst', '1.0') +from gi.repository import GLib, GObject, Gst + +logger = logging.getLogger('breezy_ui') + +screen_cast_iface = 'org.gnome.Mutter.ScreenCast' +screen_cast_session_iface = 'org.gnome.Mutter.ScreenCast.Session' +screen_cast_stream_iface = 'org.gnome.Mutter.ScreenCast.Session' +gst_pipeline_format = "pipewiresrc path=%u ! video/x-raw,max-framerate=120/1,width=%d,height=%d ! videoconvert ! fakesink sync=false" + + +def _screen_cast_session(): + bus = pydbus.SessionBus() + screen_cast = bus.get(screen_cast_iface, '/org/gnome/Mutter/ScreenCast') + session_path = screen_cast.CreateSession([]) + logger.info("session path: %s" % session_path) + screen_cast_session = bus.get(screen_cast_iface, session_path) + + return screen_cast_session + +class VirtualMonitor: + def __init__(self, width, height, on_ready_cb): + self.width = width + self.height = height + self.on_ready_cb = on_ready_cb + + Gst.init(None) + + def create(self): + session = _screen_cast_session() + stream_path = session.RecordVirtual({ + 'is-platform': GLib.Variant.new_boolean(True), + }) + logger.info("stream path: %s" % stream_path) + bus = pydbus.SessionBus() + self.stream = bus.get(screen_cast_iface, stream_path) + + self.stream.onPipeWireStreamAdded = self._on_pipewire_stream_added + + session.Start() + + def terminate(self): + if self.stream is not None: + self.stream.Stop() + + if self.pipeline is not None: + self.pipeline.send_event(Gst.Event.new_eos()) + self.pipeline.set_state(Gst.State.NULL) + + def _on_message(self, bus, message): + type = message.type + logger.info("message type: %s" % type) + if type == Gst.MessageType.EOS or type == Gst.MessageType.ERROR: + self.terminate() + + def _on_pipewire_stream_added(self, node_id): + logger.info("pipe wire stream added: %u" % node_id) + + self.pipeline = Gst.parse_launch(gst_pipeline_format % (node_id, self.width, self.height)) + self.pipeline.set_state(Gst.State.PLAYING) + self.pipeline.get_bus().connect('message', self._on_message) + self.pipeline.set_state(Gst.State.PAUSED) + + self.on_ready_cb() \ No newline at end of file From 8af11e12725344d14e93d2033e12f5a488a4dfe2 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Fri, 20 Sep 2024 11:24:46 -0700 Subject: [PATCH 02/20] WIP --- gnome/bin/dev/use_local_extension.sh | 1 + gnome/src/extension.js | 8 +++----- gnome/src/xrEffect.js | 3 ++- ui/src/virtualdisplay.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/gnome/bin/dev/use_local_extension.sh b/gnome/bin/dev/use_local_extension.sh index f959f13..12465b3 100755 --- a/gnome/bin/dev/use_local_extension.sh +++ b/gnome/bin/dev/use_local_extension.sh @@ -7,6 +7,7 @@ if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME="$USER_HOME/.local/share" fi DATA_DIR="$XDG_DATA_HOME/breezy_gnome" +mkdir -p $DATA_DIR # if $XDG_DATA_HOME/gnome-shell/extensions/breezydesktop@xronlinux.com exists extension_path="$XDG_DATA_HOME/gnome-shell/extensions/breezydesktop@xronlinux.com" diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 5f62bfd..6dd5dcc 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -131,8 +131,8 @@ export default class BreezyDesktopExtension extends Extension { try { Globals.logger.log_debug('BreezyDesktopExtension _find_supported_monitor'); const target_monitor = this._monitor_manager.getMonitorPropertiesList()?.find( - monitor => SUPPORTED_MONITOR_PRODUCTS.includes(monitor.product) || - this.settings.get_string('custom-monitor-product') === monitor.product); + monitor => monitor && (SUPPORTED_MONITOR_PRODUCTS.includes(monitor.product) || + this.settings.get_string('custom-monitor-product') === monitor.product)); if (target_monitor !== undefined) { Globals.logger.log(`Identified supported monitor: ${target_monitor.product} on ${target_monitor.connector}`); return { @@ -250,12 +250,10 @@ export default class BreezyDesktopExtension extends Extension { this._cursor_manager = new CursorManager(Main.layoutManager.uiGroup, refreshRate); this._cursor_manager.enable(); - this._overlay = new St.Bin(); + this._overlay = new St.Bin({ style: 'background-color: rgba(0, 0, 0, 1);'}); this._overlay.opacity = 255; this._overlay.set_position(targetMonitor.x, targetMonitor.y); this._overlay.set_size(targetMonitor.width, targetMonitor.height); - Globals.logger.log_debug(`BreezyDesktopExtension _effect_enable overlay size: \ - ${targetMonitor.width}x${targetMonitor.height} at ${targetMonitor.x},${targetMonitor.y}`); const overlayContent = new Clutter.Actor({clip_to_allocation: true}); const uiClone = new Clutter.Clone({ source: Main.layoutManager.uiGroup, clip_to_allocation: true }); diff --git a/gnome/src/xrEffect.js b/gnome/src/xrEffect.js index 6b1421b..1e9704b 100644 --- a/gnome/src/xrEffect.js +++ b/gnome/src/xrEffect.js @@ -6,6 +6,7 @@ import GObject from 'gi://GObject'; import Shell from 'gi://Shell'; import Globals from './globals.js'; +import * as Main from 'resource:///org/gnome/shell/ui/main.js'; import { dataViewEnd, @@ -209,7 +210,7 @@ function setIntermittentUniformVariables() { setSingleFloat(this, 'sideview_display_size', 1.0); this.set_uniform_float(shaderUniformLocations['display_resolution'], 2, displayRes); - this.set_uniform_float(shaderUniformLocations['source_to_display_ratio'], 2, [this.target_monitor.width/displayRes[0], this.target_monitor.height/displayRes[1]]); + this.set_uniform_float(shaderUniformLocations['source_to_display_ratio'], 2, [Main.layoutManager.uiGroup.width/displayRes[0], Main.layoutManager.uiGroup.height/displayRes[1]]); } else if (dataView.byteLength !== 0) { throw new Error(`Invalid dataView.byteLength: ${dataView.byteLength} !== ${DATA_VIEW_LENGTH}`); } diff --git a/ui/src/virtualdisplay.py b/ui/src/virtualdisplay.py index b57f473..3ef9969 100644 --- a/ui/src/virtualdisplay.py +++ b/ui/src/virtualdisplay.py @@ -13,7 +13,7 @@ logger = logging.getLogger('breezy_ui') screen_cast_iface = 'org.gnome.Mutter.ScreenCast' screen_cast_session_iface = 'org.gnome.Mutter.ScreenCast.Session' screen_cast_stream_iface = 'org.gnome.Mutter.ScreenCast.Session' -gst_pipeline_format = "pipewiresrc path=%u ! video/x-raw,max-framerate=120/1,width=%d,height=%d ! videoconvert ! fakesink sync=false" +gst_pipeline_format = "pipewiresrc path=%u ! video/x-raw,max-framerate=120/1,width=%d,height=%d ! fakesink sync=false" def _screen_cast_session(): From 0c93705def31ea6dd1420bb81bbe65380233fcda Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Sun, 29 Sep 2024 11:57:40 -0700 Subject: [PATCH 03/20] WIP --- gnome/src/extension.js | 13 +++++++------ gnome/src/xrEffect.js | 1 + 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 6dd5dcc..f6daed7 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -256,10 +256,10 @@ export default class BreezyDesktopExtension extends Extension { this._overlay.set_size(targetMonitor.width, targetMonitor.height); const overlayContent = new Clutter.Actor({clip_to_allocation: true}); - const uiClone = new Clutter.Clone({ source: Main.layoutManager.uiGroup, clip_to_allocation: true }); - uiClone.x = -targetMonitor.x; - uiClone.y = -targetMonitor.y; - overlayContent.add_child(uiClone); + this._ui_clone = new Clutter.Clone({ source: Main.layoutManager.uiGroup }); + this._ui_clone.x = -targetMonitor.x; + this._ui_clone.y = -targetMonitor.y; + overlayContent.add_child(this._ui_clone); this._overlay.set_child(overlayContent); @@ -308,7 +308,7 @@ export default class BreezyDesktopExtension extends Extension { this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); - this._overlay.add_effect_with_name('xr-desktop', this._xr_effect); + this._ui_clone.add_effect_with_name('xr-desktop', this._xr_effect); Meta.disable_unredirect_for_display(global.display); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); @@ -520,7 +520,8 @@ export default class BreezyDesktopExtension extends Extension { } if (this._overlay) { if (this._xr_effect) this._xr_effect.cleanup(); - this._overlay.remove_effect_by_name('xr-desktop'); + if (this._ui_clone) this._ui_clone.remove_effect_by_name('xr-desktop'); + this._ui_clone = null; global.stage.remove_child(this._overlay); this._overlay.destroy(); diff --git a/gnome/src/xrEffect.js b/gnome/src/xrEffect.js index 1e9704b..b7c4393 100644 --- a/gnome/src/xrEffect.js +++ b/gnome/src/xrEffect.js @@ -210,6 +210,7 @@ function setIntermittentUniformVariables() { setSingleFloat(this, 'sideview_display_size', 1.0); this.set_uniform_float(shaderUniformLocations['display_resolution'], 2, displayRes); + Globals.logger.log_debug(`Source resolution ${Main.layoutManager.uiGroup.width}x${Main.layoutManager.uiGroup.height}`); this.set_uniform_float(shaderUniformLocations['source_to_display_ratio'], 2, [Main.layoutManager.uiGroup.width/displayRes[0], Main.layoutManager.uiGroup.height/displayRes[1]]); } else if (dataView.byteLength !== 0) { throw new Error(`Invalid dataView.byteLength: ${dataView.byteLength} !== ${DATA_VIEW_LENGTH}`); From 8dd76e849201037a8f27d5cbd87e961154f056d7 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Sun, 6 Oct 2024 23:13:46 -0700 Subject: [PATCH 04/20] Add support for texcoord visible area property that allows us to focus the effect on just a small area relative to the whole desktop texture --- gnome/src/xrEffect.js | 11 ++++++++++- modules/sombrero | 2 +- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/gnome/src/xrEffect.js b/gnome/src/xrEffect.js index 43af7b0..73dc927 100644 --- a/gnome/src/xrEffect.js +++ b/gnome/src/xrEffect.js @@ -68,6 +68,7 @@ const shaderUniformLocations = { 'fov_widths': null, 'display_resolution': null, 'source_to_display_ratio': null, + 'texcoord_visible_area': null, 'curved_display': null, // only used by the reshade integration, but needs to be set to a default value by this effect @@ -167,6 +168,13 @@ function setIntermittentUniformVariables() { texcoordXLimitsRight[1] = 0.75; } } + const texcoordVisibleArea = [ + this.target_monitor.x / Main.layoutManager.uiGroup.width, + this.target_monitor.y / Main.layoutManager.uiGroup.height, + (this.target_monitor.x + this.target_monitor.width) / Main.layoutManager.uiGroup.width, + (this.target_monitor.y + this.target_monitor.height) / Main.layoutManager.uiGroup.height + ] + const lensVector = [lensDistanceRatio, lensFromCenter, 0.0]; const lensVectorRight = [lensDistanceRatio, -lensFromCenter, 0.0]; @@ -185,9 +193,10 @@ function setIntermittentUniformVariables() { setSingleFloat(this, 'half_fov_y_rads', halfFovYRads); this.set_uniform_float(shaderUniformLocations['fov_half_widths'], 2, fovHalfWidths); this.set_uniform_float(shaderUniformLocations['fov_widths'], 2, fovWidths); - setSingleFloat(this, 'curved_display', this.curved_display ? 1.0 : 0.0); this.set_uniform_float(shaderUniformLocations['texcoord_x_limits'], 2, texcoordXLimits); this.set_uniform_float(shaderUniformLocations['texcoord_x_limits_r'], 2, texcoordXLimitsRight); + this.set_uniform_float(shaderUniformLocations['texcoord_visible_area'], 4, texcoordVisibleArea); + setSingleFloat(this, 'curved_display', this.curved_display ? 1.0 : 0.0); this.set_uniform_float(shaderUniformLocations['lens_vector'], 3, lensVector); this.set_uniform_float(shaderUniformLocations['lens_vector_r'], 3, lensVectorRight); } diff --git a/modules/sombrero b/modules/sombrero index d270ebf..ad688e3 160000 --- a/modules/sombrero +++ b/modules/sombrero @@ -1 +1 @@ -Subproject commit d270ebfd2e3202133fea75e1513f1571960bdafd +Subproject commit ad688e3ea77fd58e44952d2ee10820146da734e3 From 60aa409c8fc105f944ca4b8c951eed9aba0a7fd8 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Mon, 7 Oct 2024 12:46:27 -0700 Subject: [PATCH 05/20] Remove hardcoded main actor from xrEffect --- gnome/src/xrEffect.js | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/gnome/src/xrEffect.js b/gnome/src/xrEffect.js index 73dc927..1fd74c5 100644 --- a/gnome/src/xrEffect.js +++ b/gnome/src/xrEffect.js @@ -6,7 +6,6 @@ import GObject from 'gi://GObject'; import Shell from 'gi://Shell'; import Globals from './globals.js'; -import * as Main from 'resource:///org/gnome/shell/ui/main.js'; import { dataViewEnd, @@ -134,6 +133,7 @@ function setIntermittentUniformVariables() { const displayRes = dataViewUint32Array(dataView, DISPLAY_RES); const sbsEnabled = dataViewUint8(dataView, SBS_ENABLED) !== 0; + const texture_actor = this.get_actor(); if (enabled) { const displayFov = dataViewFloat(dataView, DISPLAY_FOV); @@ -168,11 +168,15 @@ function setIntermittentUniformVariables() { texcoordXLimitsRight[1] = 0.75; } } + const monitor_coords_relative = [texture_actor.x - this.target_monitor.x, texture_actor.y - this.target_monitor.y]; + + Globals.logger.log(`texture_actor: ${texture_actor.x}, ${texture_actor.y}, ${texture_actor.width}, ${texture_actor.height}`); + const texcoordVisibleArea = [ - this.target_monitor.x / Main.layoutManager.uiGroup.width, - this.target_monitor.y / Main.layoutManager.uiGroup.height, - (this.target_monitor.x + this.target_monitor.width) / Main.layoutManager.uiGroup.width, - (this.target_monitor.y + this.target_monitor.height) / Main.layoutManager.uiGroup.height + monitor_coords_relative[0] / texture_actor.width, + monitor_coords_relative[1] / texture_actor.height, + (monitor_coords_relative[0] + this.target_monitor.width) / texture_actor.width, + (monitor_coords_relative[1] + this.target_monitor.height) / texture_actor.height ] const lensVector = [lensDistanceRatio, lensFromCenter, 0.0]; @@ -219,8 +223,8 @@ function setIntermittentUniformVariables() { setSingleFloat(this, 'sideview_display_size', 1.0); this.set_uniform_float(shaderUniformLocations['display_resolution'], 2, displayRes); - Globals.logger.log_debug(`Source resolution ${Main.layoutManager.uiGroup.width}x${Main.layoutManager.uiGroup.height}`); - this.set_uniform_float(shaderUniformLocations['source_to_display_ratio'], 2, [Main.layoutManager.uiGroup.width/displayRes[0], Main.layoutManager.uiGroup.height/displayRes[1]]); + Globals.logger.log_debug(`Source resolution ${texture_actor.width}x${texture_actor.height}`); + this.set_uniform_float(shaderUniformLocations['source_to_display_ratio'], 2, [texture_actor.width/displayRes[0], texture_actor.height/displayRes[1]]); } else if (dataView.byteLength !== 0) { throw new Error(`Invalid dataView.byteLength: ${dataView.byteLength} !== ${DATA_VIEW_LENGTH}`); } From dbef7f5c80bf71197b6648fc0603de6262b297d3 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Wed, 9 Oct 2024 08:42:59 -0700 Subject: [PATCH 06/20] Improved visible region logic --- gnome/src/extension.js | 7 ++++++- gnome/src/xrEffect.js | 19 ++++++++++++------- 2 files changed, 18 insertions(+), 8 deletions(-) diff --git a/gnome/src/extension.js b/gnome/src/extension.js index f6daed7..c148ded 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -255,8 +255,9 @@ export default class BreezyDesktopExtension extends Extension { this._overlay.set_position(targetMonitor.x, targetMonitor.y); this._overlay.set_size(targetMonitor.width, targetMonitor.height); + const textureSourceActor = Main.layoutManager.uiGroup; const overlayContent = new Clutter.Actor({clip_to_allocation: true}); - this._ui_clone = new Clutter.Clone({ source: Main.layoutManager.uiGroup }); + this._ui_clone = new Clutter.Clone({ source: textureSourceActor }); this._ui_clone.x = -targetMonitor.x; this._ui_clone.y = -targetMonitor.y; overlayContent.add_child(this._ui_clone); @@ -280,6 +281,10 @@ export default class BreezyDesktopExtension extends Extension { this._xr_effect = new XREffect({ target_monitor: targetMonitor, target_framerate: refreshRate, + texture_monitor_position: { + x: targetMonitor.x - textureSourceActor.x, + y: targetMonitor.y - textureSourceActor.y + }, display_distance: this.settings.get_double('display-distance'), toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), diff --git a/gnome/src/xrEffect.js b/gnome/src/xrEffect.js index 1fd74c5..18dab0c 100644 --- a/gnome/src/xrEffect.js +++ b/gnome/src/xrEffect.js @@ -168,15 +168,14 @@ function setIntermittentUniformVariables() { texcoordXLimitsRight[1] = 0.75; } } - const monitor_coords_relative = [texture_actor.x - this.target_monitor.x, texture_actor.y - this.target_monitor.y]; Globals.logger.log(`texture_actor: ${texture_actor.x}, ${texture_actor.y}, ${texture_actor.width}, ${texture_actor.height}`); const texcoordVisibleArea = [ - monitor_coords_relative[0] / texture_actor.width, - monitor_coords_relative[1] / texture_actor.height, - (monitor_coords_relative[0] + this.target_monitor.width) / texture_actor.width, - (monitor_coords_relative[1] + this.target_monitor.height) / texture_actor.height + this.texture_monitor_position.x / texture_actor.width, + this.texture_monitor_position.y / texture_actor.height, + (this.texture_monitor_position.x + this.target_monitor.width) / texture_actor.width, + (this.texture_monitor_position.y + this.target_monitor.height) / texture_actor.height ] const lensVector = [lensDistanceRatio, lensFromCenter, 0.0]; @@ -184,8 +183,8 @@ function setIntermittentUniformVariables() { // our overlay doesn't quite cover the full screen texture, which allows us to see some of the real desktop // underneath, so we trim three pixels around the entire edge of the texture - const trimWidthPercent = 3.0 / this.target_monitor.width; - const trimHeightPercent = 3.0 / this.target_monitor.height; + const trimWidthPercent = 3.0 / texture_actor.width; + const trimHeightPercent = 3.0 / texture_actor.height; // all these values are transferred directly, unmodified from the driver transferUniformFloat(this, 'look_ahead_cfg', dataView, LOOK_AHEAD_CFG); @@ -268,6 +267,12 @@ export const XREffect = GObject.registerClass({ 'Target framerate for this effect', GObject.ParamFlags.READWRITE, 30, 240, 60 ), + 'texture-monitor-position': GObject.ParamSpec.jsobject( + 'texture-monitor-position', + 'Texture Monitor Position', + 'Coordinates of the monitor relative to the target actor texture', + GObject.ParamFlags.READWRITE + ), 'display-distance': GObject.ParamSpec.double( 'display-distance', 'Display Distance', From f4e081cd738c21f4b9cda6b14c713466777a5a66 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Thu, 19 Dec 2024 12:03:29 -0800 Subject: [PATCH 07/20] WIP --- gnome/src/customeffect.js | 71 +++++++++ gnome/src/extension.js | 140 +++++++++-------- gnome/src/testactor.js | 309 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 457 insertions(+), 63 deletions(-) create mode 100644 gnome/src/customeffect.js create mode 100644 gnome/src/testactor.js diff --git a/gnome/src/customeffect.js b/gnome/src/customeffect.js new file mode 100644 index 0000000..0df7fd6 --- /dev/null +++ b/gnome/src/customeffect.js @@ -0,0 +1,71 @@ +const { Clutter, GLib, GObject } = imports.gi; + +export const CustomEffect = GObject.registerClass({ + Properties: { + 'fov-degrees': GObject.ParamSpec.double( + 'fov-degrees', + 'FOV Degrees', + 'Diagonal field-of-view in degrees', + GObject.ParamFlags.READWRITE, + 1.0, + 179.0, + 60.0 + ) + } +}, class Customffect extends Clutter.ShaderEffect { + _init(params = {}) { + super._init(params); + + this.fov_degrees = params['fov-degrees'] || 60.0; + this.connect('notify::fov-degrees', this._updateMatrices.bind(this)); + + // Set up the vertex shader + this.set_shader_source(Clutter.ShaderType.VERTEX, ` + uniform mat4 viewMatrix; + uniform mat4 projectionMatrix; + uniform vec4 quaternion; + + vec3 applyQuaternionToVector(vec3 v, vec4 q) { + return v + 2.0 * cross(q.xyz, cross(q.xyz, v) + q.w * v); + } + + void main() { + // First apply the view matrix to position the vertex in camera space + vec4 viewPosition = viewMatrix * vec4(gl_Vertex.xyz, 1.0); + // Then apply the quaternion rotation + vec3 transformedPosition = applyQuaternionToVector(viewPosition.xyz, quaternion); + // Finally apply the projection matrix + gl_Position = projectionMatrix * vec4(transformedPosition, 1.0); + gl_TexCoord[0] = gl_MultiTexCoord0; + } + `); + + // Initialize with the current matrices + this._updateMatrices(); + } + + _updateMatrices() { + let aspect = this.get_parent().width / this.get_parent().height; + let fov = this.fov_degrees * Math.PI / 180.0; + let near = 0.1; + let far = 100.0; + let top = Math.tan(fov / 2.0) * near; + let bottom = -top; + let right = top * aspect; + let left = -right; + + let projectionMatrix = GLib.Matrix.init_frustum(left, right, bottom, top, near, far); + let viewMatrix = GLib.Matrix.init_identity(); + + // Calculate the appropriate Z-distance based on FOV + let distance = -1.0 / Math.tan(fov / 2.0); + viewMatrix = viewMatrix.translate(0, 0, distance); + + this.set_shader_uniform_value('projectionMatrix', new Clutter.ShaderValue({matrix: projectionMatrix})); + this.set_shader_uniform_value('viewMatrix', new Clutter.ShaderValue({matrix: viewMatrix})); + } + + set_quaternion(quat) { + this.set_shader_uniform_value('quaternion', new Clutter.ShaderValue({vector4: quat})); + } +}); diff --git a/gnome/src/extension.js b/gnome/src/extension.js index c148ded..9073a28 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -9,6 +9,7 @@ import { CursorManager } from './cursormanager.js'; import Globals from './globals.js'; import { Logger } from './logger.js'; import { MonitorManager } from './monitormanager.js'; +import { TestActorEffect, TestActor } from './testactor.js'; import { isValidKeepAlive } from './time.js'; import { IPC_FILE_PATH, XREffect } from './xrEffect.js'; @@ -47,7 +48,7 @@ export default class BreezyDesktopExtension extends Extension { this._follow_threshold_connection = null; this._widescreen_mode_settings_connection = null; this._widescreen_mode_effect_state_connection = null; - this._supported_device_detected_connected = null; + this._supported_device_detected_connection = null; this._start_binding = null; this._end_binding = null; this._curved_display_binding = null; @@ -91,7 +92,7 @@ export default class BreezyDesktopExtension extends Extension { this._setup(); } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension enable ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension enable ${e.message}\n${e.stack}`); } } @@ -120,7 +121,7 @@ export default class BreezyDesktopExtension extends Extension { return GLib.SOURCE_CONTINUE; } } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _poll_for_ready ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _poll_for_ready ${e.message}\n${e.stack}`); this._running_poller_id = undefined; return GLib.SOURCE_REMOVE; } @@ -157,7 +158,7 @@ export default class BreezyDesktopExtension extends Extension { Globals.logger.log_debug('BreezyDesktopExtension _find_supported_monitor - No supported monitor found'); return null; } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _find_supported_monitor ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _find_supported_monitor ${e.message}\n${e.stack}`); return null; } } @@ -199,11 +200,7 @@ export default class BreezyDesktopExtension extends Extension { this._poll_for_ready(); } } else { - if (!target_monitor) { - Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, no supported monitor found`); - } else { - Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, target monitor found, waiting for poller to pick it up`); - } + Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, driver running: ${this._check_driver_running()}, target_monitor found: ${!!target_monitor}`); } } @@ -218,7 +215,7 @@ export default class BreezyDesktopExtension extends Extension { return isValidKeepAlive(file_modified_time, true); } } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _check_driver_running ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _check_driver_running ${e.message}\n${e.stack}`); } return false; @@ -250,17 +247,24 @@ export default class BreezyDesktopExtension extends Extension { this._cursor_manager = new CursorManager(Main.layoutManager.uiGroup, refreshRate); this._cursor_manager.enable(); - this._overlay = new St.Bin({ style: 'background-color: rgba(0, 0, 0, 1);'}); + this._overlay = new St.Bin({ style: 'background-color: rgba(0, 0, 0, 1);', reactive: false, clip_to_allocation: true }); this._overlay.opacity = 255; this._overlay.set_position(targetMonitor.x, targetMonitor.y); this._overlay.set_size(targetMonitor.width, targetMonitor.height); + // this._overlay.inhibit_culling(); - const textureSourceActor = Main.layoutManager.uiGroup; - const overlayContent = new Clutter.Actor({clip_to_allocation: true}); - this._ui_clone = new Clutter.Clone({ source: textureSourceActor }); - this._ui_clone.x = -targetMonitor.x; - this._ui_clone.y = -targetMonitor.y; - overlayContent.add_child(this._ui_clone); + // const textureSourceActor = Main.layoutManager.uiGroup; + const overlayContent = new TestActor({ + monitors: [], + quaternion: { + x: 0.094, y: 0.079, z: 0.094, w: 0.988 + }, + fov_degrees: 46.0, + width: 100, + height: 100, + 'z-position': 0 + }); + // overlayContent.inhibit_culling(); this._overlay.set_child(overlayContent); @@ -278,49 +282,59 @@ export default class BreezyDesktopExtension extends Extension { this._handle_sibling_update.bind(this), ); - this._xr_effect = new XREffect({ - target_monitor: targetMonitor, - target_framerate: refreshRate, - texture_monitor_position: { - x: targetMonitor.x - textureSourceActor.x, - y: targetMonitor.y - textureSourceActor.y - }, - display_distance: this.settings.get_double('display-distance'), - toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), - toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), - look_ahead_override: this.settings.get_int('look-ahead-override'), - disable_anti_aliasing: this.settings.get_boolean('disable-anti-aliasing') - }); + // this._xr_effect = new XREffect({ + // target_monitor: targetMonitor, + // target_framerate: refreshRate, + // texture_monitor_position: { + // // x: targetMonitor.x - textureSourceActor.x, + // // y: targetMonitor.y - textureSourceActor.y + // x: 0, + // y: 0 + // }, + // display_distance: this.settings.get_double('display-distance'), + // toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), + // toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), + // look_ahead_override: this.settings.get_int('look-ahead-override'), + // disable_anti_aliasing: this.settings.get_boolean('disable-anti-aliasing') + // }); + // this._xr_effect = new TestActorEffect({ + // quaternion: { + // x: 0.094, y: 0.079, z: 0.094, w: 0.988 + // }, + // fov_degrees: 46.0, + // width: targetMonitor.width, + // height: targetMonitor.height + // }); this._update_follow_threshold(this.settings); // this gets triggered before _effect_enable if in fast-sbs-mode-switching mode - if (!this.settings.get_boolean('fast-sbs-mode-switching')) - this._update_widescreen_mode_from_settings(this.settings); + // if (!this.settings.get_boolean('fast-sbs-mode-switching')) + // this._update_widescreen_mode_from_settings(this.settings); - this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); - this._supported_device_detected_connected = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); + // this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); + // this._supported_device_detected_connection = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); - this._distance_binding = this.settings.bind('display-distance', this._xr_effect, 'display-distance', Gio.SettingsBindFlags.DEFAULT) + // this._distance_binding = this.settings.bind('display-distance', this._xr_effect, 'display-distance', Gio.SettingsBindFlags.DEFAULT) this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)) this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)) - this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) - this._start_binding = this.settings.bind('toggle-display-distance-start', this._xr_effect, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) - this._end_binding = this.settings.bind('toggle-display-distance-end', this._xr_effect, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) - this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) - this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); - this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); - this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); + // this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) + // this._start_binding = this.settings.bind('toggle-display-distance-start', this._xr_effect, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) + // this._end_binding = this.settings.bind('toggle-display-distance-end', this._xr_effect, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) + // this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) + // this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); + // this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); + // this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); - this._ui_clone.add_effect_with_name('xr-desktop', this._xr_effect); + // this._ui_clone.add_effect_with_name('xr-desktop', this._xr_effect); Meta.disable_unredirect_for_display(global.display); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); - this._add_settings_keybinding('toggle-display-distance-shortcut', this._xr_effect._change_distance.bind(this._xr_effect)); + // this._add_settings_keybinding('toggle-display-distance-shortcut', this._xr_effect._change_distance.bind(this._xr_effect)); this._add_settings_keybinding('toggle-follow-shortcut', this._toggle_follow_mode.bind(this)); } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); this._effect_disable(); } } @@ -356,11 +370,11 @@ export default class BreezyDesktopExtension extends Extension { bind_to_function ); } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _add_settings_keybinding settings binding lambda ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _add_settings_keybinding settings binding lambda ${e.message}\n${e.stack}`); } }); } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _add_settings_keybinding ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _add_settings_keybinding ${e.message}\n${e.stack}`); } } @@ -371,7 +385,7 @@ export default class BreezyDesktopExtension extends Extension { stream.write(`${key}=${value}`, null); stream.close(null); } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _write_control ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _write_control ${e.message}\n${e.stack}`); } } @@ -395,7 +409,7 @@ export default class BreezyDesktopExtension extends Extension { } } } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _read_state ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _read_state ${e.message}\n${e.stack}`); } return state; } @@ -524,7 +538,7 @@ export default class BreezyDesktopExtension extends Extension { this._actor_removed_connection = null; } if (this._overlay) { - if (this._xr_effect) this._xr_effect.cleanup(); + // if (this._xr_effect) this._xr_effect.cleanup(); if (this._ui_clone) this._ui_clone.remove_effect_by_name('xr-desktop'); this._ui_clone = null; @@ -572,17 +586,17 @@ export default class BreezyDesktopExtension extends Extension { this.settings.unbind(this._disable_anti_aliasing_binding); this._disable_anti_aliasing_binding = null; } - if (this._xr_effect) { - if (this._widescreen_mode_effect_state_connection) { - this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); - this._widescreen_mode_effect_state_connection = null; - } - if (this._supported_device_detected_connected) { - this._xr_effect.disconnect(this._supported_device_detected_connected); - this._supported_device_detected_connected = null; - } - this._xr_effect = null; - } + // if (this._xr_effect) { + // if (this._widescreen_mode_effect_state_connection) { + // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); + // this._widescreen_mode_effect_state_connection = null; + // } + // if (this._supported_device_detected_connection) { + // this._xr_effect.disconnect(this._supported_device_detected_connection); + // this._supported_device_detected_connection = null; + // } + // this._xr_effect = null; + // } if (this._cursor_manager) { this._cursor_manager.disable(); this._cursor_manager = null; @@ -595,7 +609,7 @@ export default class BreezyDesktopExtension extends Extension { this._write_control('sbs_mode', 'disable'); } } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension _effect_disable ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_disable ${e.message}\n${e.stack}`); } } @@ -618,7 +632,7 @@ export default class BreezyDesktopExtension extends Extension { this._monitor_manager = null; } } catch (e) { - Globals.logger.log(`ERROR: BreezyDesktopExtension disable ${e.message}\n${e.stack}`); + Globals.logger.log(`[ERROR] BreezyDesktopExtension disable ${e.message}\n${e.stack}`); } } } diff --git a/gnome/src/testactor.js b/gnome/src/testactor.js new file mode 100644 index 0000000..a1751ad --- /dev/null +++ b/gnome/src/testactor.js @@ -0,0 +1,309 @@ + +import Clutter from 'gi://Clutter' +import Cogl from 'gi://Cogl'; +import GObject from 'gi://GObject'; +import Shell from 'gi://Shell'; +import * as Main from 'resource:///org/gnome/shell/ui/main.js'; + +import Globals from './globals.js'; + +export const TestActorEffect = GObject.registerClass({ + Properties: { + 'quaternion': GObject.ParamSpec.jsobject( + 'quaternion', + 'Quaternion', + 'Camera orientation quaternion', + GObject.ParamFlags.READWRITE + ), + 'fov-degrees': GObject.ParamSpec.double( + 'fov-degrees', + 'FOV Degrees', + 'Field of view in degrees', + GObject.ParamFlags.READWRITE, + 30.0, 100.0, 46.0 + ), + 'width': GObject.ParamSpec.int( + 'width', + 'Width', + 'Width of the viewport', + GObject.ParamFlags.READWRITE, + 1, 10000, 1920 + ), + 'height': GObject.ParamSpec.int( + 'height', + 'Height', + 'Height of the viewport', + GObject.ParamFlags.READWRITE, + 1, 10000, 1080 + ) + } +}, class TestActorEffect extends Shell.GLSLEffect { + constructor(params = {}) { + super(params); + + + // Compute the projection matrix + let aspectRatio = this.width / this.height; + let fovRadians = this.fov_degrees * (Math.PI / 180); + let near = 0.1; + let far = 1000.0; + + let projectionMatrix = this._computeProjectionMatrix(fovRadians, aspectRatio, near, far); + Globals.logger.log(JSON.stringify(projectionMatrix)); + + // Compute the view matrix from the quaternion + let viewMatrix = this._computeViewMatrixFromQuaternion(this.quaternion); + Globals.logger.log(JSON.stringify(viewMatrix)); + + let rotationMatrix = this._createRotationMatrix(this.quaternion); + Globals.logger.log(JSON.stringify(rotationMatrix)); + } + + _computeProjectionMatrix(fovRadians, aspect, near, far) { + let f = 1.0 / Math.tan(fovRadians / 2); + let nf = 1 / (near - far); + + let projectionMatrix = [ + f / aspect, 0, 0, 0, + 0, f, 0, 0, + 0, 0, (far + near) * nf, -1, + 0, 0, (2 * far * near) * nf, 0 + ]; + + return projectionMatrix; + } + + _computeViewMatrixFromQuaternion(q) { + let x = q.x, y = q.y, z = q.z, w = q.w; + + let x2 = x + x; + let y2 = y + y; + let z2 = z + z; + + let xx = x * x2; + let xy = x * y2; + let xz = x * z2; + let yy = y * y2; + let yz = y * z2; + let zz = z * z2; + let wx = w * x2; + let wy = w * y2; + let wz = w * z2; + + let viewMatrix = [ + 1 - (yy + zz), xy - wz, xz + wy, 0, + xy + wz, 1 - (xx + zz), yz - wx, 0, + xz - wy, yz + wx, 1 - (xx + yy), 0, + 0, 0, 0, 1 + ]; + + // Invert the view matrix (since it's from camera space) + // For rotation matrices, the inverse is the transpose + let inverseViewMatrix = [ + viewMatrix[0], viewMatrix[4], viewMatrix[8], 0, + viewMatrix[1], viewMatrix[5], viewMatrix[9], 0, + viewMatrix[2], viewMatrix[6], viewMatrix[10], 0, + 0, 0, 0, 1 + ]; + + return viewMatrix; + } + + _createRotationMatrix(q) { + // Normalize the quaternion + const len = Math.sqrt( + q.x * q.x + + q.y * q.y + + q.z * q.z + + q.w * q.w + ); + const x = q.x / len; + const y = q.y / len; + const z = q.z / len; + const w = q.w / len; + + // Compute matrix elements + const x2 = x * x; + const y2 = y * y; + const z2 = z * z; + const xy = x * y; + const xz = x * z; + const yz = y * z; + const wx = w * x; + const wy = w * y; + const wz = w * z; + + // Create rotation matrix + return [ + 1.0 - 2.0 * (y2 + z2), // m00 + 2.0 * (xy - wz), // m01 + 2.0 * (xz + wy), // m02 + 0.0, // m03 + + 2.0 * (xy + wz), // m10 + 1.0 - 2.0 * (x2 + z2), // m11 + 2.0 * (yz - wx), // m12 + 0.0, // m13 + + 2.0 * (xz - wy), // m20 + 2.0 * (yz + wx), // m21 + 1.0 - 2.0 * (x2 + y2), // m22 + 0.0, // m23 + + 0.0, // m30 + 0.0, // m31 + 0.0, // m32 + 1.0 // m33 + ]; + } + + vfunc_build_pipeline() { + const declarations = ` + uniform mat4 u_rotation_matrix; + uniform mat4 u_view_matrix; + uniform mat4 u_projection_matrix; + `; + + const main = ` + vec4 world_pos = cogl_position_in; + world_pos = u_rotation_matrix * world_pos; + world_pos = cogl_modelview_matrix * world_pos; + cogl_position_out = cogl_projection_matrix * world_pos; + cogl_tex_coord_out[0] = cogl_tex_coord_in; + ` + + this.add_glsl_snippet(Shell.SnippetHook.VERTEX, declarations, main, false); + } + + vfunc_paint_target(node, paintContext) { + // Compute the projection matrix + let aspectRatio = this.width / this.height; + let fovRadians = this.fov_degrees * (Math.PI / 180); + let near = 0.1; + let far = 1000.0; + + let projectionMatrix = this._computeProjectionMatrix(fovRadians, aspectRatio, near, far); + + // Compute the view matrix from the quaternion + let viewMatrix = this._computeViewMatrixFromQuaternion(this.quaternion); + + // Set up the uniforms + this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projectionMatrix); + this.set_uniform_matrix(this.get_uniform_location("u_view_matrix"), false, 4, viewMatrix); + this.set_uniform_matrix(this.get_uniform_location("u_rotation_matrix"), false, 4, this._createRotationMatrix(this.quaternion)); + + this.get_pipeline().set_layer_filters( + 0, + Cogl.PipelineFilter.LINEAR_MIPMAP_LINEAR, + Cogl.PipelineFilter.LINEAR + ); + + super.vfunc_paint_target(node, paintContext); + } +}); + +export const TestActor = GObject.registerClass({ + Properties: { + 'monitors': GObject.ParamSpec.jsobject( + 'monitors', + 'Monitors', + 'Array of monitor indexes', + GObject.ParamFlags.READWRITE + ), + 'quaternion': GObject.ParamSpec.jsobject( + 'quaternion', + 'Quaternion', + 'Camera orientation quaternion', + GObject.ParamFlags.READWRITE + ), + 'fov-degrees': GObject.ParamSpec.double( + 'fov-degrees', + 'FOV Degrees', + 'Field of view in degrees', + GObject.ParamFlags.READWRITE, + 30.0, 100.0, 46.0 + ), + 'width': GObject.ParamSpec.int( + 'width', + 'Width', + 'Width of the viewport', + GObject.ParamFlags.READWRITE, + 1, 10000, 1920 + ), + 'height': GObject.ParamSpec.int( + 'height', + 'Height', + 'Height of the viewport', + GObject.ParamFlags.READWRITE, + 1, 10000, 1080 + ) + } +}, class TestActor extends Clutter.Actor { + constructor(params = {}) { + super({...params}); + + // Set the size of the viewport (implicitly provides aspect ratio) + // You can set the size when adding this actor to the stage + // this.set_size(this.width, this.height); + + // Create the monitor actors + this._createMonitorActors(); + + // Apply the shader effect to this viewport actor + // this._applyShaderEffect(); + } + + _createMonitorActors() { + Main.layoutManager.monitors.forEach((monitor, index) => { + // if (index === 0) return; + Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); + + const containerActor = new Clutter.Actor({ + x: -monitor.x, + y: monitor.y, + 'z-position': -500, + width: monitor.width, + height: monitor.height, + reactive: false + }); + // Create a clone of the stage content for this monitor + const monitorClone = new Clutter.Clone({ + source: Main.layoutManager.uiGroup, + reactive: false + }); + + monitorClone.x = -monitor.x; + // monitorActor.y = 0; + + // Set the size and position of the clone to match the monitor + // monitorActor.set_size(monitor.width, monitor.height); + + // // Apply clipping to show only this monitor's area + monitorClone.set_clip(monitor.x, 0, monitor.width, monitor.height); + + // Position the monitor actor within the 3D scene + // monitorActor.set_position(0, 0); + + // // For 3D positioning, we might want to center the monitors around (0,0,0) + // // Adjust positions accordingly + // monitorActor.set_translation(monitor.x, monitor.y, 1.0); + + // Add the monitor actor to the scene + containerActor.add_child(monitorClone); + containerActor.add_effect_with_name('viewport-effect', new TestActorEffect({ + quaternion: this.quaternion, + fov_degrees: this.fov_degrees, + width: this.width, + height: this.height + })); + this.add_child(containerActor); + }); + } + + // _applyShaderEffect() { + // const glslEffect = + + // // Apply the shader effect to this viewport actor + // this.add_effect_with_name('viewport-effect', glslEffect); + // } +}); \ No newline at end of file From ebc3910c9dd9103dbfaeb0ec192c355bca29116d Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Tue, 14 Jan 2025 16:11:39 -0800 Subject: [PATCH 08/20] WIP --- gnome/src/devicedatastream.js | 174 ++++++++++ gnome/src/extension.js | 148 ++++----- gnome/src/monitormanager.js | 2 +- gnome/src/testactor.js | 606 +++++++++++++++++++++++----------- 4 files changed, 663 insertions(+), 267 deletions(-) create mode 100644 gnome/src/devicedatastream.js diff --git a/gnome/src/devicedatastream.js b/gnome/src/devicedatastream.js new file mode 100644 index 0000000..37d8d5a --- /dev/null +++ b/gnome/src/devicedatastream.js @@ -0,0 +1,174 @@ +import Gio from 'gi://Gio'; +import GObject from 'gi://GObject'; + +import Globals from './globals.js'; +import { + dataViewEnd, + dataViewUint8, + dataViewBigUint, + dataViewUint32Array, + dataViewUint8Array, + dataViewFloat, + dataViewFloatArray, + BOOL_SIZE, + FLOAT_SIZE, + UINT_SIZE, + UINT8_SIZE +} from "./ipc.js"; +import { isValidKeepAlive, getEpochSec, toSec } from "./time.js"; + +const IPC_FILE_PATH = "/dev/shm/breezy_desktop_imu"; +const KEEPALIVE_REFRESH_INTERVAL_SEC = 1; + +// the driver should be using the same data layout version +const DATA_LAYOUT_VERSION = 3; + +// DataView info: [offset, size, count] +const VERSION = [0, UINT8_SIZE, 1]; +const ENABLED = [dataViewEnd(VERSION), BOOL_SIZE, 1]; +const LOOK_AHEAD_CFG = [dataViewEnd(ENABLED), FLOAT_SIZE, 4]; +const DISPLAY_RES = [dataViewEnd(LOOK_AHEAD_CFG), UINT_SIZE, 2]; +const DISPLAY_FOV = [dataViewEnd(DISPLAY_RES), FLOAT_SIZE, 1]; +const LENS_DISTANCE_RATIO = [dataViewEnd(DISPLAY_FOV), FLOAT_SIZE, 1]; +const SBS_ENABLED = [dataViewEnd(LENS_DISTANCE_RATIO), BOOL_SIZE, 1]; +const CUSTOM_BANNER_ENABLED = [dataViewEnd(SBS_ENABLED), BOOL_SIZE, 1]; +const EPOCH_MS = [dataViewEnd(CUSTOM_BANNER_ENABLED), UINT_SIZE, 2]; +const IMU_QUAT_DATA = [dataViewEnd(EPOCH_MS), FLOAT_SIZE, 16]; +const IMU_PARITY_BYTE = [dataViewEnd(IMU_QUAT_DATA), UINT8_SIZE, 1]; +const DATA_VIEW_LENGTH = dataViewEnd(IMU_PARITY_BYTE); + +function checkParityByte(dataView) { + const parityByte = dataViewUint8(dataView, IMU_PARITY_BYTE); + let parity = 0; + const epochUint8 = dataViewUint8Array(dataView, EPOCH_MS); + const imuDataUint8 = dataViewUint8Array(dataView, IMU_QUAT_DATA); + for (let i = 0; i < epochUint8.length; i++) { + parity ^= epochUint8[i]; + } + for (let i = 0; i < imuDataUint8.length; i++) { + parity ^= imuDataUint8[i]; + } + return parityByte === parity; +} + +export const DeviceDataStream = GObject.registerClass({ + Properties: { + 'supported-device-connected': GObject.ParamSpec.boolean( + 'supported-device-connected', + 'Supported device connected', + 'Whether a supported device is connected', + GObject.ParamFlags.READWRITE, + false + ), + 'widescreen-mode-state': GObject.ParamSpec.boolean( + 'widescreen-mode-state', + 'Widescreen mode state', + 'The state of widescreen mode from the perspective of the driver', + GObject.ParamFlags.READWRITE, + false + ), + 'quaternion': GObject.ParamSpec.jsobject( + 'quaternion', + 'Quaternion', + 'Camera orientation quaternion', + GObject.ParamFlags.READWRITE + ), + } +}, class DeviceDataStream extends GObject.Object { + constructor(params = {}) { + super(params); + this.supported_device_connected = false; + this._ipc_file = Gio.file_new_for_path(IPC_FILE_PATH); + this._running = false; + this._device_data = null; + } + + start() { + this._running = true; + this._poll(); + } + + stop() { + this._running = false; + } + + // polling is just intended to keep supported_device_connected current, anything needing up-to-date imu data should + // trigger a refresh with the default flag + _poll() { + if (this._running) { + this.refresh_data(true); + setTimeout(this._poll.bind(this), 1000); + } + } + + // Refresh the data from the IPC file. if keepalive_only is true, we'll only check and update supported_device_connected if it + // hasn't been checked within KEEPALIVE_REFRESH_INTERVAL_SEC. + refresh_data(keepalive_only = false) { + if (!this._device_data?.imuData || !keepalive_only || getEpochSec() - this._device_data.imuDateMs > KEEPALIVE_REFRESH_INTERVAL_SEC) { + let data = this._ipc_file.load_contents(null); + if (data[0]) { + let buffer = new Uint8Array(data[1]).buffer; + let dataView = new DataView(buffer); + if (dataView.byteLength === DATA_VIEW_LENGTH) { + const imuDateMs = dataViewBigUint(dataView, EPOCH_MS); + const validKeepalive = isValidKeepAlive(toSec(imuDateMs)); + const imuData = dataViewFloatArray(dataView, IMU_QUAT_DATA); + const imuResetState = validKeepalive && imuData[0] === 0.0 && imuData[1] === 0.0 && imuData[2] === 0.0 && imuData[3] === 1.0; + const version = dataViewUint8(dataView, VERSION); + const enabled = dataViewUint8(dataView, ENABLED) !== 0 && version === DATA_LAYOUT_VERSION && validKeepalive; + const sbsEnabled = dataViewUint8(dataView, SBS_ENABLED) !== 0; + + // update the widescreen property if the state changes while still enabled, trigger "notify::" events + if (enabled && this.widescreen_mode_state !== sbsEnabled) this.widescreen_mode_state = sbsEnabled; + + if (!this._device_data) { + this._device_data = { + version, + enabled, + imuResetState, + displayRes: dataViewUint32Array(dataView, DISPLAY_RES), + sbsEnabled, + displayFov: dataViewFloat(dataView, DISPLAY_FOV), + lookAheadCfg: dataViewFloatArray(dataView, LOOK_AHEAD_CFG), + }; + } + + let success = keepalive_only; + let attempts = 0; + while (!success && attempts < 3) { + if (dataView.byteLength === DATA_VIEW_LENGTH) { + if (checkParityByte(dataView)) { + this._device_data.imuData = imuData; + this._device_data.imuDateMs = imuDateMs; + this.quaternion = { + x: imuData[0], + y: imuData[1], + z: imuData[2], + w: imuData[3] + }; + success = true; + } + } else if (dataView.byteLength !== 0) { + Globals.logger.log(`[ERROR] Invalid dataView.byteLength: ${dataView.byteLength} !== ${DATA_VIEW_LENGTH}`) + } + + if (!success && ++attempts < 3) { + data = this._ipc_file.load_contents(null); + if (data[0]) { + buffer = new Uint8Array(data[1]).buffer; + dataView = new DataView(buffer); + } + } + } + + if (success) { + // update the supported device connected property if the state changes, trigger "notify::" events + if (this.supported_device_connected !== validKeepalive) this.supported_device_connected = validKeepalive; + } + } + } else { + this.supported_device_connected = false; + } + } + } +}); \ No newline at end of file diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 9073a28..01b45b1 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -1,11 +1,13 @@ import Clutter from 'gi://Clutter' import Gio from 'gi://Gio'; import GLib from 'gi://GLib'; +import GObject from 'gi://GObject'; import Meta from 'gi://Meta'; import Shell from 'gi://Shell'; import St from 'gi://St'; import { CursorManager } from './cursormanager.js'; +import { DeviceDataStream } from './devicedatastream.js'; import Globals from './globals.js'; import { Logger } from './logger.js'; import { MonitorManager } from './monitormanager.js'; @@ -38,8 +40,9 @@ export default class BreezyDesktopExtension extends Extension { // Set/destroyed by enable/disable this._cursor_manager = null; + this._device_data_stream = null; this._monitor_manager = null; - this._xr_effect = null; + this.overlay_content = null; this._overlay = null; this._target_monitor = null; this._is_effect_running = false; @@ -76,6 +79,9 @@ export default class BreezyDesktopExtension extends Extension { Globals.extension_dir = this.path; this.settings.bind('debug', Globals.logger, 'debug', Gio.SettingsBindFlags.DEFAULT); + this._device_data_stream = new DeviceDataStream(); + this._device_data_stream.start(); + this._monitor_manager = new MonitorManager({ use_optimal_monitor_config: this.settings.get_boolean('use-optimal-monitor-config'), headset_as_primary: this.settings.get_boolean('headset-as-primary'), @@ -107,7 +113,7 @@ export default class BreezyDesktopExtension extends Extension { return GLib.SOURCE_REMOVE; } - if (this._check_driver_running() && target_monitor) { + if (this._device_data_stream.supported_device_connected && target_monitor) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes, // so we can remove this timeout_add no matter what. @@ -118,6 +124,7 @@ export default class BreezyDesktopExtension extends Extension { this._running_poller_id = undefined; return GLib.SOURCE_REMOVE; } else { + Globals.logger.log_debug(`BreezyDesktopExtension _poll_for_ready - device connected: ${this._device_data_stream.supported_device_connected}, target_monitor: ${!!target_monitor}`); return GLib.SOURCE_CONTINUE; } } catch (e) { @@ -186,7 +193,7 @@ export default class BreezyDesktopExtension extends Extension { if (target_monitor && this._running_poller_id === undefined) { this._target_monitor = target_monitor; - if (this._check_driver_running()) { + if (this._device_data_stream.supported_device_connected) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes if (this._target_monitor_ready(target_monitor)) { @@ -200,27 +207,10 @@ export default class BreezyDesktopExtension extends Extension { this._poll_for_ready(); } } else { - Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, driver running: ${this._check_driver_running()}, target_monitor found: ${!!target_monitor}`); + Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, device connected: ${this._device_data_stream.supported_device_connected}, target_monitor found: ${!!target_monitor}`); } } - _check_driver_running() { - try { - if (!Globals.ipc_file) Globals.ipc_file = Gio.file_new_for_path(IPC_FILE_PATH); - if (Globals.ipc_file.query_exists(null)) { - const file_info = Globals.ipc_file.query_info(Gio.FILE_ATTRIBUTE_TIME_MODIFIED, Gio.FileQueryInfoFlags.NONE, null); - const file_modified_time = file_info.get_attribute_uint64(Gio.FILE_ATTRIBUTE_TIME_MODIFIED); - - // when the driver is running, the IMU file is updated at least 60x per second, do a strict check - return isValidKeepAlive(file_modified_time, true); - } - } catch (e) { - Globals.logger.log(`[ERROR] BreezyDesktopExtension _check_driver_running ${e.message}\n${e.stack}`); - } - - return false; - } - _needs_widescreen_monitor_update() { Globals.logger.log_debug('BreezyDesktopExtension _needs_widescreen_monitor_update'); const state = this._read_state(); @@ -251,22 +241,21 @@ export default class BreezyDesktopExtension extends Extension { this._overlay.opacity = 255; this._overlay.set_position(targetMonitor.x, targetMonitor.y); this._overlay.set_size(targetMonitor.width, targetMonitor.height); - // this._overlay.inhibit_culling(); // const textureSourceActor = Main.layoutManager.uiGroup; - const overlayContent = new TestActor({ + this.overlay_content = new TestActor({ monitors: [], - quaternion: { - x: 0.094, y: 0.079, z: 0.094, w: 0.988 - }, fov_degrees: 46.0, - width: 100, - height: 100, - 'z-position': 0 + // width: 100, + // height: 100, + width: targetMonitor.width, + height: targetMonitor.height, + display_distance: this.settings.get_double('display-distance'), + toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), + toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end') }); - // overlayContent.inhibit_culling(); - this._overlay.set_child(overlayContent); + this._overlay.set_child(this.overlay_content); Shell.util_set_hidden_from_pick(this._overlay, true); global.stage.add_child(this._overlay); @@ -281,30 +270,6 @@ export default class BreezyDesktopExtension extends Extension { clutterContainer ? 'actor-removed' : 'child-removed', this._handle_sibling_update.bind(this), ); - - // this._xr_effect = new XREffect({ - // target_monitor: targetMonitor, - // target_framerate: refreshRate, - // texture_monitor_position: { - // // x: targetMonitor.x - textureSourceActor.x, - // // y: targetMonitor.y - textureSourceActor.y - // x: 0, - // y: 0 - // }, - // display_distance: this.settings.get_double('display-distance'), - // toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), - // toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), - // look_ahead_override: this.settings.get_int('look-ahead-override'), - // disable_anti_aliasing: this.settings.get_boolean('disable-anti-aliasing') - // }); - // this._xr_effect = new TestActorEffect({ - // quaternion: { - // x: 0.094, y: 0.079, z: 0.094, w: 0.988 - // }, - // fov_degrees: 46.0, - // width: targetMonitor.width, - // height: targetMonitor.height - // }); this._update_follow_threshold(this.settings); @@ -314,24 +279,35 @@ export default class BreezyDesktopExtension extends Extension { // this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); // this._supported_device_detected_connection = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); + this.overlay_content.renderMonitors(); + this._data_stream_connection = this._device_data_stream.bind_property( + 'quaternion', + this.overlay_content, + 'quaternion', + GObject.BindingFlags.DEFAULT + ); - // this._distance_binding = this.settings.bind('display-distance', this._xr_effect, 'display-distance', Gio.SettingsBindFlags.DEFAULT) - this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)) - this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)) + this._distance_binding = this.settings.bind('display-distance', this.overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); + this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)); + this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)); // this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) - // this._start_binding = this.settings.bind('toggle-display-distance-start', this._xr_effect, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) - // this._end_binding = this.settings.bind('toggle-display-distance-end', this._xr_effect, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) + this._start_binding = this.settings.bind('toggle-display-distance-start', this.overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) + this._end_binding = this.settings.bind('toggle-display-distance-end', this.overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) // this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) // this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); // this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); // this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); - // this._ui_clone.add_effect_with_name('xr-desktop', this._xr_effect); Meta.disable_unredirect_for_display(global.display); + + global.stage.connect('before-paint', (() => { + this._device_data_stream.refresh_data(); + this._overlay.queue_redraw(); + }).bind(this)); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); - // this._add_settings_keybinding('toggle-display-distance-shortcut', this._xr_effect._change_distance.bind(this._xr_effect)); + this._add_settings_keybinding('toggle-display-distance-shortcut', this.overlay_content._change_distance.bind(this.overlay_content)); this._add_settings_keybinding('toggle-follow-shortcut', this._toggle_follow_mode.bind(this)); } catch (e) { Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); @@ -462,12 +438,12 @@ export default class BreezyDesktopExtension extends Extension { } _update_widescreen_mode_from_settings(settings, event) { - const value = settings.get_boolean('widescreen-mode'); - Globals.logger.log_debug(`BreezyDesktopExtension _update_widescreen_mode_from_settings ${value}`); - if (value !== undefined && value !== this._xr_effect.widescreen_mode_state) { - this._request_sbs_mode_change(value); - } else - Globals.logger.log_debug('effect.widescreen_mode_state already matched setting'); + // const value = settings.get_boolean('widescreen-mode'); + // Globals.logger.log_debug(`BreezyDesktopExtension _update_widescreen_mode_from_settings ${value}`); + // if (value !== undefined && value !== this._xr_effect.widescreen_mode_state) { + // this._request_sbs_mode_change(value); + // } else + // Globals.logger.log_debug('effect.widescreen_mode_state already matched setting'); } _update_widescreen_mode_from_state(effect, _pspec) { @@ -538,9 +514,7 @@ export default class BreezyDesktopExtension extends Extension { this._actor_removed_connection = null; } if (this._overlay) { - // if (this._xr_effect) this._xr_effect.cleanup(); - if (this._ui_clone) this._ui_clone.remove_effect_by_name('xr-desktop'); - this._ui_clone = null; + this.overlay_content = null; global.stage.remove_child(this._overlay); this._overlay.destroy(); @@ -554,6 +528,10 @@ export default class BreezyDesktopExtension extends Extension { this.settings.disconnect(this._distance_connection); this._distance_connection = null; } + if (this._data_stream_connection) { + this._device_data_stream.unbind(this._data_stream_connection); + this._data_stream_connection = null; + } if (this._follow_threshold_connection) { this.settings.disconnect(this._follow_threshold_connection); this._follow_threshold_connection = null; @@ -586,17 +564,17 @@ export default class BreezyDesktopExtension extends Extension { this.settings.unbind(this._disable_anti_aliasing_binding); this._disable_anti_aliasing_binding = null; } - // if (this._xr_effect) { - // if (this._widescreen_mode_effect_state_connection) { - // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); - // this._widescreen_mode_effect_state_connection = null; - // } - // if (this._supported_device_detected_connection) { - // this._xr_effect.disconnect(this._supported_device_detected_connection); - // this._supported_device_detected_connection = null; - // } - // this._xr_effect = null; - // } + if (this.overlay_content) { + // if (this._widescreen_mode_effect_state_connection) { + // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); + // this._widescreen_mode_effect_state_connection = null; + // } + // if (this._supported_device_detected_connection) { + // this._xr_effect.disconnect(this._supported_device_detected_connection); + // this._supported_device_detected_connection = null; + // } + this.overlay_content = null; + } if (this._cursor_manager) { this._cursor_manager.disable(); this._cursor_manager = null; @@ -618,6 +596,12 @@ export default class BreezyDesktopExtension extends Extension { Globals.logger.log_debug('BreezyDesktopExtension disable'); this._effect_disable(); this._target_monitor = null; + + if (this._device_data_stream) { + this._device_data_stream.stop(); + this._device_data_stream = null; + } + if (this._monitor_manager) { if (this._optimal_monitor_config_binding) { this.settings.unbind(this._optimal_monitor_config_binding); diff --git a/gnome/src/monitormanager.js b/gnome/src/monitormanager.js index ca8a6a5..b0ceea6 100644 --- a/gnome/src/monitormanager.js +++ b/gnome/src/monitormanager.js @@ -35,7 +35,7 @@ function getDisplayConfigProxy(extPath) { xml = new TextDecoder().decode(bytes); } } catch (e) { - Globals.logger.log('ERROR: failed to load DisplayConfig interface XML'); + Globals.logger.log('[ERROR] failed to load DisplayConfig interface XML'); throw e; } cachedDisplayConfigProxy = Gio.DBusProxy.makeProxyWrapper(xml); diff --git a/gnome/src/testactor.js b/gnome/src/testactor.js index a1751ad..86064c7 100644 --- a/gnome/src/testactor.js +++ b/gnome/src/testactor.js @@ -1,14 +1,193 @@ - import Clutter from 'gi://Clutter' import Cogl from 'gi://Cogl'; +import GLib from 'gi://GLib'; import GObject from 'gi://GObject'; import Shell from 'gi://Shell'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; import Globals from './globals.js'; +function applyQuaternionToVector(vector, quaternion) { + const t = [ + 2.0 * (quaternion[1] * vector[2] - quaternion[2] * vector[1]), + 2.0 * (quaternion[2] * vector[0] - quaternion[0] * vector[2]), + 2.0 * (quaternion[0] * vector[1] - quaternion[1] * vector[0]) + ]; + return [ + vector[0] + quaternion[3] * t[0] + quaternion[1] * t[2] - quaternion[2] * t[1], + vector[1] + quaternion[3] * t[1] + quaternion[2] * t[0] - quaternion[0] * t[2], + vector[2] + quaternion[3] * t[2] + quaternion[0] * t[1] - quaternion[1] * t[0] + ]; +} + +/** + * Find the vector in the array that's closest to the quaternion rotation + * + * @param {number[]} quaternion - Reference quaternion [w, x, y, z] + * @param {number[][]} vectors - Array of vectors [x, y, z] to search from + * @returns {number} Index of the closest vector, if it surpasses the previous closest index by a certain margin, otherwise the previous index + */ +function findClosestVector(quaternion, vectors, previousClosestIndex) { + + const lookVector = [1.0, 0.0, 0.0]; // NWU vector pointing to the center of the screen + const rotatedLookVector = applyQuaternionToVector(lookVector, [quaternion.x, quaternion.y, quaternion.z, quaternion.w]); + Globals.logger.log(`\t\t\tQuaternion: ${JSON.stringify(quaternion)}`); + Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); + + let closestIndex = -1; + let closestDistance = Infinity; + let previousDistance = Infinity; + + // find the vector closest to the rotated look vector + vectors.forEach((vector, index) => { + const distance = Math.acos( + Math.min(1.0, Math.max(-1.0, vector[0] * rotatedLookVector[0] + vector[1] * rotatedLookVector[1] + vector[2] * rotatedLookVector[2])) + ); + + if (previousClosestIndex === index) { + previousDistance = distance; + } + + Globals.logger.log(`\t\t\tMonitor ${index} distance: ${distance}`); + if (distance < closestDistance) { + closestIndex = index; + closestDistance = distance; + } + }); + + Globals.logger.log(`\t\t\tClosest monitor: ${closestIndex}, distance: ${closestDistance}`); + + // only switch if the closest monitor is greater than the previous closest by 25% + if (previousClosestIndex !== undefined && closestIndex !== previousClosestIndex && closestDistance * 1.25 > previousDistance) { + return previousClosestIndex; + } + + return closestIndex; +} + +function degreesToRadians(degrees) { + return degrees * Math.PI / 180.0; +} + +function radiansToDegrees(radians) { + return radians * 180.0 / Math.PI; +} + +/*** + * @returns {Object} - containing `center` and `end` radians + */ +function monitorWrap(radiusPixels, previousMonitorEndRadians, monitorPixels) { + const monitorHalfPixels = monitorPixels / 2; + const monitorHalfRadians = Math.asin(monitorHalfPixels / radiusPixels); + const centerRadians = previousMonitorEndRadians + monitorHalfRadians; + return { + center: centerRadians, + end: centerRadians + monitorHalfRadians + } +} + +/** + * Convert the given monitor details into NWU vectors pointing to the center of each monitor. + * + * @param {Object} fovDetails - contains reference fovDegrees (diagonal), widthPixels, heightPixels + * @param {Object[]} monitorDetailsList - contains x, y, width, height (coordinates from top-left) + * @param {string} monitorWrappingScheme - horizontal, vertical, none + * @returns {number[]} - Vector [x, y, z] + */ +function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme) { + const aspect = fovDetails.widthPixels / fovDetails.heightPixels; + const fovVerticalRadians = degreesToRadians(fovDetails.fovDegrees / Math.sqrt(1 + aspect * aspect)); + + // NWU vectors pointing to the center of the screen for each monitor + const monitorVectors = []; + + if (monitorWrappingScheme === 'horizontal') { + // monitors wrap around us horizontally + const fovHorizontalRadians = fovVerticalRadians * aspect; + + // radius is the hypothenuse of the triangle where the opposite side is half the width of the reference fov screen + const radius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); + + let previousMonitorEndRadians = -fovHorizontalRadians / 2; + monitorDetailsList.forEach(monitorDetails => { + const monitorWrapDetails = monitorWrap(radius, previousMonitorEndRadians, monitorDetails.width); + previousMonitorEndRadians = monitorWrapDetails.end; + + monitorVectors.push([ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + radius * Math.cos(monitorWrapDetails.center), + + // west is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -radius * Math.sin(monitorWrapDetails.center), + + // up is flat when wrapping horizontally + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ]); + }); + } else if (monitorWrappingScheme === 'vertical') { + // monitors wrap around us vertically + + // radius is the hypothenuse of the triangle where the opposite side is half the height of the reference fov screen + const radius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); + + let previousMonitorEndRadians = -fovVerticalRadians / 2; + monitorDetailsList.forEach(monitorDetails => { + const monitorWrapDetails = monitorWrap(radius, previousMonitorEndRadians, monitorDetails.height); + previousMonitorEndRadians = monitorWrapDetails.end; + + monitorVectors.push([ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + radius * Math.cos(monitorWrapDetails.center), + + // west is flat when wrapping vertically + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + + // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -radius * Math.sin(monitorWrapDetails.center) + ]); + }); + } else { + // monitors make a flat wall in front of us, no wrapping + monitorDetailsList.forEach(monitorDetails => { + monitorVectors.push([ + fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2), + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ]); + }); + } + + return monitorVectors; +} + +function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { + if (monitorWrappingScheme === 'horizontal') { + // monitors wrap around us horizontally + return { + angle: radiansToDegrees(Math.atan2(vector[1], vector[0])), + axis: Clutter.RotateAxis.Y_AXIS + }; + } else if (monitorWrappingScheme === 'vertical') { + // monitors wrap around us vertically + return { + angle: radiansToDegrees(Math.atan2(vector[2], vector[0])), + axis: Clutter.RotateAxis.X_AXIS + } + } else { + // no rotation + return undefined; + } +} + export const TestActorEffect = GObject.registerClass({ Properties: { + 'monitor-index': GObject.ParamSpec.int( + 'monitor-index', + 'Monitor Index', + 'Index of the monitor that this effect is applied to', + GObject.ParamFlags.READWRITE, + 0, 100, 0 + ), 'quaternion': GObject.ParamSpec.jsobject( 'quaternion', 'Quaternion', @@ -35,140 +214,126 @@ export const TestActorEffect = GObject.registerClass({ 'Height of the viewport', GObject.ParamFlags.READWRITE, 1, 10000, 1080 - ) + ), + 'monitor-wrapping-scheme': GObject.ParamSpec.string( + 'monitor-wrapping-scheme', + 'Monitor Wrapping Scheme', + 'How the monitors are wrapped around the viewport', + GObject.ParamFlags.READWRITE, + 'horizontal', ['horizontal', 'vertical', 'none'] + ), + 'focused-monitor-index': GObject.ParamSpec.int( + 'focused-monitor-index', + 'Focused Monitor Index', + 'Index of the monitor that is currently focused', + GObject.ParamFlags.READWRITE, + 0, 100, 0 + ), + 'display-distance': GObject.ParamSpec.double( + 'display-distance', + 'Display Distance', + 'Distance of the display from the camera', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.0 + ), + 'toggle-display-distance-start': GObject.ParamSpec.double( + 'toggle-display-distance-start', + 'Display distance start', + 'Start distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), + 'toggle-display-distance-end': GObject.ParamSpec.double( + 'toggle-display-distance-end', + 'Display distance end', + 'End distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), } }, class TestActorEffect extends Shell.GLSLEffect { - constructor(params = {}) { - super(params); + perspective(fovDiagonalRadians, aspect, near, far) { + // compute horizontal fov given diagonal fov and aspect ratio + const h = Math.sqrt(aspect * aspect + 1); + const fovRadians = fovDiagonalRadians / h * aspect; + console.log(`fovRadians: ${fovRadians}`); - // Compute the projection matrix - let aspectRatio = this.width / this.height; - let fovRadians = this.fov_degrees * (Math.PI / 180); - let near = 0.1; - let far = 1000.0; - - let projectionMatrix = this._computeProjectionMatrix(fovRadians, aspectRatio, near, far); - Globals.logger.log(JSON.stringify(projectionMatrix)); - - // Compute the view matrix from the quaternion - let viewMatrix = this._computeViewMatrixFromQuaternion(this.quaternion); - Globals.logger.log(JSON.stringify(viewMatrix)); - - let rotationMatrix = this._createRotationMatrix(this.quaternion); - Globals.logger.log(JSON.stringify(rotationMatrix)); - } - - _computeProjectionMatrix(fovRadians, aspect, near, far) { - let f = 1.0 / Math.tan(fovRadians / 2); - let nf = 1 / (near - far); - - let projectionMatrix = [ - f / aspect, 0, 0, 0, - 0, f, 0, 0, - 0, 0, (far + near) * nf, -1, - 0, 0, (2 * far * near) * nf, 0 - ]; - - return projectionMatrix; - } - - _computeViewMatrixFromQuaternion(q) { - let x = q.x, y = q.y, z = q.z, w = q.w; - - let x2 = x + x; - let y2 = y + y; - let z2 = z + z; - - let xx = x * x2; - let xy = x * y2; - let xz = x * z2; - let yy = y * y2; - let yz = y * z2; - let zz = z * z2; - let wx = w * x2; - let wy = w * y2; - let wz = w * z2; - - let viewMatrix = [ - 1 - (yy + zz), xy - wz, xz + wy, 0, - xy + wz, 1 - (xx + zz), yz - wx, 0, - xz - wy, yz + wx, 1 - (xx + yy), 0, - 0, 0, 0, 1 - ]; - - // Invert the view matrix (since it's from camera space) - // For rotation matrices, the inverse is the transpose - let inverseViewMatrix = [ - viewMatrix[0], viewMatrix[4], viewMatrix[8], 0, - viewMatrix[1], viewMatrix[5], viewMatrix[9], 0, - viewMatrix[2], viewMatrix[6], viewMatrix[10], 0, - 0, 0, 0, 1 - ]; - - return viewMatrix; - } - - _createRotationMatrix(q) { - // Normalize the quaternion - const len = Math.sqrt( - q.x * q.x + - q.y * q.y + - q.z * q.z + - q.w * q.w - ); - const x = q.x / len; - const y = q.y / len; - const z = q.z / len; - const w = q.w / len; - - // Compute matrix elements - const x2 = x * x; - const y2 = y * y; - const z2 = z * z; - const xy = x * y; - const xz = x * z; - const yz = y * z; - const wx = w * x; - const wy = w * y; - const wz = w * z; - - // Create rotation matrix + const f = 1.0 / Math.tan(fovRadians / 2.0); + const range = far - near; + return [ - 1.0 - 2.0 * (y2 + z2), // m00 - 2.0 * (xy - wz), // m01 - 2.0 * (xz + wy), // m02 - 0.0, // m03 - - 2.0 * (xy + wz), // m10 - 1.0 - 2.0 * (x2 + z2), // m11 - 2.0 * (yz - wx), // m12 - 0.0, // m13 - - 2.0 * (xz - wy), // m20 - 2.0 * (yz + wx), // m21 - 1.0 - 2.0 * (x2 + y2), // m22 - 0.0, // m23 - - 0.0, // m30 - 0.0, // m31 - 0.0, // m32 - 1.0 // m33 + f / aspect, 0, 0, 0, + 0, f, 0, 0, + 0, 0, - (far + near) / range, -1, + 0, 0, - (2.0 * near * far) / range, 0 ]; } vfunc_build_pipeline() { const declarations = ` - uniform mat4 u_rotation_matrix; - uniform mat4 u_view_matrix; + uniform vec4 u_quaternion; uniform mat4 u_projection_matrix; + uniform float u_display_north_offset; + + vec4 applyQuaternionToVector(vec4 v, vec4 q) { + vec3 t = 2.0 * cross(q.xyz, v.xyz); + vec3 rotated = v.xyz + q.w * t + cross(q.xyz, t); + return vec4(rotated, v.w); + } `; const main = ` vec4 world_pos = cogl_position_in; - world_pos = u_rotation_matrix * world_pos; + + // // move pixel space to texcoord space + // world_pos.x = (world_pos.x / 192.0); + // world_pos.y = (world_pos.y / 108.0); + + // float displayAspectRatio = 1920.0 / 1080.0; + // float diagToVertRatio = sqrt(pow(displayAspectRatio, 2) + 1); + // float halfFovZRads = radians(46.0 / diagToVertRatio) / 2.0; + // float halfFovYRads = halfFovZRads * displayAspectRatio; + // vec2 fovHalfWidths = vec2(tan(halfFovYRads), tan(halfFovZRads)); + // vec2 fovWidths = fovHalfWidths * 2.0; + + // float vec_y = -world_pos.x * fovWidths.x + fovHalfWidths.x; + // float vec_z = -world_pos.y * fovWidths.y + fovHalfWidths.y; + // vec4 look_vector = vec4(1.0, vec_y, vec_z, 1.0); + // // vec3 rotated_vector = applyQuaternionToVector(look_vector, u_quaternion).xyz; + // vec3 rotated_vector = look_vector.xyz; + + // // scale back to the screen distance + // rotated_vector /= rotated_vector.x; + // cogl_position_out = vec4( + // ((fovHalfWidths.x - rotated_vector.y) / fovWidths.x) * 2.0 - 1.0, + // ((fovHalfWidths.y - rotated_vector.z) / fovWidths.y) * 2.0 - 1.0, + // 0.0, + // 1.0 + // ); + + // float z_orig = world_pos.z; + // world_pos.z -= z_orig / 1920.0; + // world_pos.x /= 2.0; + // world_pos *= u_display_north_offset; + world_pos = applyQuaternionToVector(world_pos, u_quaternion); + // world_pos /= u_display_north_offset; + // world_pos.x *= 2.0; + // world_pos.z += z_orig / 1920.0; world_pos = cogl_modelview_matrix * world_pos; cogl_position_out = cogl_projection_matrix * world_pos; + + // cogl_position_out.x = world_pos.x / 103.4; + // cogl_position_out.y = world_pos.y / 29.075; + // cogl_position_out.z = -1.0; + // cogl_position_out.w = 1.0; + cogl_tex_coord_out[0] = cogl_tex_coord_in; ` @@ -176,21 +341,23 @@ export const TestActorEffect = GObject.registerClass({ } vfunc_paint_target(node, paintContext) { - // Compute the projection matrix - let aspectRatio = this.width / this.height; - let fovRadians = this.fov_degrees * (Math.PI / 180); - let near = 0.1; - let far = 1000.0; + if (!this._initialized) { + const aspect = this.get_actor().width / this.get_actor().height; + const projection_matrix = this.perspective( + this.fov_degrees * Math.PI / 180.0, + aspect, + 0.0001, + 1000.0 + ); + Globals.logger.log(`aspect: ${aspect}, fov: ${this.fov_degrees}, width: ${this.get_actor().width}, height: ${this.get_actor().height}, projection matrix: ${JSON.stringify(projection_matrix)}`); + this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projection_matrix); + this._initialized = true; + } - let projectionMatrix = this._computeProjectionMatrix(fovRadians, aspectRatio, near, far); + this.set_uniform_float(this.get_uniform_location("u_display_north_offset"), 1, [this.focused_monitor_index === this.monitor_index ? this.display_distance : this.toggle_display_distance_start]); - // Compute the view matrix from the quaternion - let viewMatrix = this._computeViewMatrixFromQuaternion(this.quaternion); - - // Set up the uniforms - this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projectionMatrix); - this.set_uniform_matrix(this.get_uniform_location("u_view_matrix"), false, 4, viewMatrix); - this.set_uniform_matrix(this.get_uniform_location("u_rotation_matrix"), false, 4, this._createRotationMatrix(this.quaternion)); + // NUW to east-up-south conversion, inverted + this.set_uniform_float(this.get_uniform_location("u_quaternion"), 4, [this.quaternion.y, -this.quaternion.z, this.quaternion.x, this.quaternion.w]); this.get_pipeline().set_layer_filters( 0, @@ -223,87 +390,158 @@ export const TestActor = GObject.registerClass({ GObject.ParamFlags.READWRITE, 30.0, 100.0, 46.0 ), - 'width': GObject.ParamSpec.int( - 'width', - 'Width', - 'Width of the viewport', + 'focused-monitor-index': GObject.ParamSpec.int( + 'focused-monitor-index', + 'Focused Monitor Index', + 'Index of the monitor that is currently focused', GObject.ParamFlags.READWRITE, - 1, 10000, 1920 + 0, 100, 0 ), - 'height': GObject.ParamSpec.int( - 'height', - 'Height', - 'Height of the viewport', + 'display-size': GObject.ParamSpec.double( + 'display-size', + 'Display size', + 'Size of the display', GObject.ParamFlags.READWRITE, - 1, 10000, 1080 - ) + 0.2, + 2.5, + 1.0 + ), + 'display-distance': GObject.ParamSpec.double( + 'display-distance', + 'Display Distance', + 'Distance of the display from the camera', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.0 + ), + 'toggle-display-distance-start': GObject.ParamSpec.double( + 'toggle-display-distance-start', + 'Display distance start', + 'Start distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), + 'toggle-display-distance-end': GObject.ParamSpec.double( + 'toggle-display-distance-end', + 'Display distance end', + 'End distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), } }, class TestActor extends Clutter.Actor { - constructor(params = {}) { - super({...params}); - - // Set the size of the viewport (implicitly provides aspect ratio) - // You can set the size when adding this actor to the stage - // this.set_size(this.width, this.height); - - // Create the monitor actors - this._createMonitorActors(); - - // Apply the shader effect to this viewport actor - // this._applyShaderEffect(); - } - - _createMonitorActors() { - Main.layoutManager.monitors.forEach((monitor, index) => { + renderMonitors() { + this.monitorsAsVectors = monitorsToVectors( + { + fovDegrees: this.fov_degrees, + widthPixels: this.width, + heightPixels: this.height + }, + Main.layoutManager.monitors.map(monitor => ({ + x: monitor.x, + y: monitor.y, + width: monitor.width, + height: monitor.height + })), + 'horizontal' + ); + this.monitorAsNormalizedVectors = this.monitorsAsVectors.map(vector => { + const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); + return [vector[0] / length, vector[1] / length, vector[2] / length]; + }); + + Main.layoutManager.monitors.forEach(((monitor, index) => { // if (index === 0) return; Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); + // this is in NWU coordinates + const monitorVector = this.monitorsAsVectors[index]; + const monitorRotation = monitorVectorToRotationAngle(monitorVector, 'horizontal'); + Globals.logger.log_debug(`\t\t\tMonitor ${index} vector: ${monitorVector} rotation: ${JSON.stringify(monitorRotation)}`); + + // actor coordinates are east-up-south const containerActor = new Clutter.Actor({ - x: -monitor.x, - y: monitor.y, - 'z-position': -500, + x: -monitorVector[1], + y: -monitorVector[2], + 'z-position': -monitorVector[0], width: monitor.width, height: monitor.height, reactive: false }); + // Create a clone of the stage content for this monitor const monitorClone = new Clutter.Clone({ source: Main.layoutManager.uiGroup, reactive: false }); - monitorClone.x = -monitor.x; + monitorClone.x = -containerActor.x; // monitorActor.y = 0; - - // Set the size and position of the clone to match the monitor - // monitorActor.set_size(monitor.width, monitor.height); - - // // Apply clipping to show only this monitor's area monitorClone.set_clip(monitor.x, 0, monitor.width, monitor.height); - // Position the monitor actor within the 3D scene - // monitorActor.set_position(0, 0); - - // // For 3D positioning, we might want to center the monitors around (0,0,0) - // // Adjust positions accordingly - // monitorActor.set_translation(monitor.x, monitor.y, 1.0); - // Add the monitor actor to the scene containerActor.add_child(monitorClone); - containerActor.add_effect_with_name('viewport-effect', new TestActorEffect({ + containerActor.set_pivot_point(0.5, 0.5); + containerActor.set_rotation_angle(monitorRotation.axis, monitorRotation.angle); + const effect = new TestActorEffect({ quaternion: this.quaternion, fov_degrees: this.fov_degrees, - width: this.width, - height: this.height - })); + monitor_index: index, + display_distance: this.toggle_display_distance_start + }); + containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); - }); + this.bind_property('quaternion', effect, 'quaternion', GObject.BindingFlags.DEFAULT); + this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); + this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); + }).bind(this)); + + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 1000, (() => { + if (this.quaternion) { + const closestMonitorIndex = findClosestVector(this.quaternion, this.monitorAsNormalizedVectors, this.closestMonitorIndex); + + // only switch if the closest monitor is greater than the previous closest by 25% + if (this.closestMonitorIndex === undefined || this.closestMonitorIndex !== closestMonitorIndex) { + Globals.logger.log(`Switching to monitor ${closestMonitorIndex}`); + this.closestMonitorIndex = closestMonitorIndex; + } + } + + return GLib.SOURCE_CONTINUE; + }).bind(this)); + + this._distance_ease_timeline = null; + this.connect('notify::toggle-display-distance-start', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::toggle-display-distance-end', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::display-distance', this._handle_display_distance_properties_change.bind(this)); + this._handle_display_distance_properties_change(); + } + + _handle_display_distance_properties_change() { + const distance_from_end = Math.abs(this.display_distance - this.toggle_display_distance_end); + const distance_from_start = Math.abs(this.display_distance - this.toggle_display_distance_start); + this._is_display_distance_at_end = distance_from_end < distance_from_start; } - // _applyShaderEffect() { - // const glslEffect = + _change_distance() { + if (this._distance_ease_timeline?.is_playing()) this._distance_ease_timeline.stop(); - // // Apply the shader effect to this viewport actor - // this.add_effect_with_name('viewport-effect', glslEffect); - // } + this._distance_ease_start = this.display_distance; + this._distance_ease_timeline = Clutter.Timeline.new_for_actor(this, 250); + + const toggle_display_distance_target = this._is_display_distance_at_end ? + this.toggle_display_distance_start : this.toggle_display_distance_end; + this._distance_ease_timeline.connect('new-frame', () => { + this.display_distance = this._distance_ease_start + + this._distance_ease_timeline.get_progress() * + (toggle_display_distance_target - this._distance_ease_start); + }); + + this._distance_ease_timeline.start(); + } }); \ No newline at end of file From 80f54f5297fe6bb8bf003ad4b2aa293088e1ad11 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Fri, 17 Jan 2025 14:34:15 -0800 Subject: [PATCH 09/20] WIP --- gnome/src/extension.js | 2 +- gnome/src/testactor.js | 245 +++++++++++++++++++++++++---------------- 2 files changed, 152 insertions(+), 95 deletions(-) diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 01b45b1..9f81e5a 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -529,7 +529,7 @@ export default class BreezyDesktopExtension extends Extension { this._distance_connection = null; } if (this._data_stream_connection) { - this._device_data_stream.unbind(this._data_stream_connection); + this._data_stream_connection.unbind(); this._data_stream_connection = null; } if (this._follow_threshold_connection) { diff --git a/gnome/src/testactor.js b/gnome/src/testactor.js index 86064c7..bc6f0fa 100644 --- a/gnome/src/testactor.js +++ b/gnome/src/testactor.js @@ -31,7 +31,6 @@ function findClosestVector(quaternion, vectors, previousClosestIndex) { const lookVector = [1.0, 0.0, 0.0]; // NWU vector pointing to the center of the screen const rotatedLookVector = applyQuaternionToVector(lookVector, [quaternion.x, quaternion.y, quaternion.z, quaternion.w]); - Globals.logger.log(`\t\t\tQuaternion: ${JSON.stringify(quaternion)}`); Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); let closestIndex = -1; @@ -81,23 +80,28 @@ function monitorWrap(radiusPixels, previousMonitorEndRadians, monitorPixels) { const monitorHalfRadians = Math.asin(monitorHalfPixels / radiusPixels); const centerRadians = previousMonitorEndRadians + monitorHalfRadians; return { + begin: previousMonitorEndRadians, center: centerRadians, end: centerRadians + monitorHalfRadians } } /** - * Convert the given monitor details into NWU vectors pointing to the center of each monitor. + * Convert the given monitor details into NWU vectors describing the center of the fully placed monitor, + * and the top-left of the partially placed monitor (minus only a single-axis rotation) * * @param {Object} fovDetails - contains reference fovDegrees (diagonal), widthPixels, heightPixels * @param {Object[]} monitorDetailsList - contains x, y, width, height (coordinates from top-left) * @param {string} monitorWrappingScheme - horizontal, vertical, none - * @returns {number[]} - Vector [x, y, z] + * @returns {Object[]} - contains NWU vectors pointing to `topLeftNoRotate` and `center` of each monitor */ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme) { const aspect = fovDetails.widthPixels / fovDetails.heightPixels; const fovVerticalRadians = degreesToRadians(fovDetails.fovDegrees / Math.sqrt(1 + aspect * aspect)); + // distance needed for the FOV-sized monitor to fill up the screen + const centerRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); + // NWU vectors pointing to the center of the screen for each monitor const monitorVectors = []; @@ -105,55 +109,76 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // monitors wrap around us horizontally const fovHorizontalRadians = fovVerticalRadians * aspect; - // radius is the hypothenuse of the triangle where the opposite side is half the width of the reference fov screen - const radius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); + // distance to a horizontal edge is the hypothenuse of the triangle where the opposite side is half the width of the reference fov screen + const edgeRadius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); let previousMonitorEndRadians = -fovHorizontalRadians / 2; monitorDetailsList.forEach(monitorDetails => { - const monitorWrapDetails = monitorWrap(radius, previousMonitorEndRadians, monitorDetails.width); + const monitorWrapDetails = monitorWrap(edgeRadius, previousMonitorEndRadians, monitorDetails.width); previousMonitorEndRadians = monitorWrapDetails.end; - monitorVectors.push([ - // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians - radius * Math.cos(monitorWrapDetails.center), + monitorVectors.push({ + topLeftNoRotate: [ + centerRadius, + fovDetails.widthPixels / 2, + -(monitorDetails.y - fovDetails.heightPixels / 2) + ], + center: [ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + centerRadius * Math.cos(monitorWrapDetails.center), - // west is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians - -radius * Math.sin(monitorWrapDetails.center), + // west is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -centerRadius * Math.sin(monitorWrapDetails.center), - // up is flat when wrapping horizontally - -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) - ]); + // up is flat when wrapping horizontally + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ] + }); }); } else if (monitorWrappingScheme === 'vertical') { // monitors wrap around us vertically - // radius is the hypothenuse of the triangle where the opposite side is half the height of the reference fov screen - const radius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); + // distance to a vertical edge is the hypothenuse of the triangle where the opposite side is half the height of the reference fov screen + const edgeRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); let previousMonitorEndRadians = -fovVerticalRadians / 2; monitorDetailsList.forEach(monitorDetails => { - const monitorWrapDetails = monitorWrap(radius, previousMonitorEndRadians, monitorDetails.height); + const monitorWrapDetails = monitorWrap(edgeRadius, previousMonitorEndRadians, monitorDetails.height); previousMonitorEndRadians = monitorWrapDetails.end; - monitorVectors.push([ - // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians - radius * Math.cos(monitorWrapDetails.center), + monitorVectors.push({ + topLeftNoRotate: [ + centerRadius, + -(monitorDetails.x - fovDetails.widthPixels / 2), + fovDetails.heightPixels / 2 + ], + center: [ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + centerRadius * Math.cos(monitorWrapDetails.center), - // west is flat when wrapping vertically - -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + // west is flat when wrapping vertically + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), - // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians - -radius * Math.sin(monitorWrapDetails.center) - ]); + // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -centerRadius * Math.sin(monitorWrapDetails.center) + ] + }); }); } else { // monitors make a flat wall in front of us, no wrapping monitorDetailsList.forEach(monitorDetails => { - monitorVectors.push([ - fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2), - -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), - -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) - ]); + monitorVectors.push({ + topLeftNoRotate: [ + centerRadius, + -(monitorDetails.x - fovDetails.widthPixels / 2), + -(monitorDetails.y - fovDetails.heightPixels / 2) + ], + center: [ + centerRadius, + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ] + }); }); } @@ -164,13 +189,13 @@ function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { if (monitorWrappingScheme === 'horizontal') { // monitors wrap around us horizontally return { - angle: radiansToDegrees(Math.atan2(vector[1], vector[0])), + angle: Math.atan2(vector[1], vector[0]), axis: Clutter.RotateAxis.Y_AXIS }; } else if (monitorWrappingScheme === 'vertical') { // monitors wrap around us vertically return { - angle: radiansToDegrees(Math.atan2(vector[2], vector[0])), + angle: Math.atan2(vector[2], vector[0]), axis: Clutter.RotateAxis.X_AXIS } } else { @@ -222,6 +247,13 @@ export const TestActorEffect = GObject.registerClass({ GObject.ParamFlags.READWRITE, 'horizontal', ['horizontal', 'vertical', 'none'] ), + 'monitor-wrapping-rotation-radians': GObject.ParamSpec.double( + 'monitor-wrapping-rotation-radians', + 'Monitor Wrapping Rotation Radians', + 'Rotation of the monitor wrapping around the viewport', + GObject.ParamFlags.READWRITE, + -360.0, 360.0, 0.0 + ), 'focused-monitor-index': GObject.ParamSpec.int( 'focused-monitor-index', 'Focused Monitor Index', @@ -234,9 +266,9 @@ export const TestActorEffect = GObject.registerClass({ 'Display Distance', 'Distance of the display from the camera', GObject.ParamFlags.READWRITE, - 0.2, - 2.5, - 1.0 + 0.0, + 10000.0, + 2900.0 ), 'toggle-display-distance-start': GObject.ParamSpec.double( 'toggle-display-distance-start', @@ -256,6 +288,12 @@ export const TestActorEffect = GObject.registerClass({ 2.5, 1.05 ), + 'actor-to-display-ratios': GObject.ParamSpec.jsobject( + 'actor-to-display-ratios', + 'Actor to Display Ratios', + 'Ratios to convert actor coordinates to display coordinates', + GObject.ParamFlags.READWRITE + ) } }, class TestActorEffect extends Shell.GLSLEffect { perspective(fovDiagonalRadians, aspect, near, far) { @@ -281,58 +319,66 @@ export const TestActorEffect = GObject.registerClass({ uniform vec4 u_quaternion; uniform mat4 u_projection_matrix; uniform float u_display_north_offset; + uniform float u_rotation_x_radians; + uniform float u_rotation_y_radians; + uniform float u_aspect_ratio; + + // for some reason the vector positions are relative to the width and height of the uiGroup actor + uniform vec2 u_actor_to_display_ratios; + + // constants that help me adjust CoGL vector positions so their components are at the ratios intended, for proper rotation + float cogl_position_width = 51.7; // no idea... + float cogl_z_factor = 2.5; // no idea... vec4 applyQuaternionToVector(vec4 v, vec4 q) { vec3 t = 2.0 * cross(q.xyz, v.xyz); vec3 rotated = v.xyz + q.w * t + cross(q.xyz, t); return vec4(rotated, v.w); } + + vec4 applyXRotationToVector(vec4 v, float angle) { + float c = cos(angle); + float s = sin(angle); + return vec4(v.x, v.y * c - v.z * s, v.y * s + v.z * c, v.w); + } + + vec4 applyYRotationToVector(vec4 v, float angle) { + float c = cos(angle); + float s = sin(angle); + return vec4(v.x * c + v.z * s, v.y, v.z * c - v.x * s, v.w); + } `; const main = ` vec4 world_pos = cogl_position_in; - // // move pixel space to texcoord space - // world_pos.x = (world_pos.x / 192.0); - // world_pos.y = (world_pos.y / 108.0); + float cogl_position_height = cogl_position_width / u_aspect_ratio; + float position_width_adjustment_count = u_actor_to_display_ratios.x - 1.0; + float position_height_adjustment_count = u_actor_to_display_ratios.y - 1.0; - // float displayAspectRatio = 1920.0 / 1080.0; - // float diagToVertRatio = sqrt(pow(displayAspectRatio, 2) + 1); - // float halfFovZRads = radians(46.0 / diagToVertRatio) / 2.0; - // float halfFovYRads = halfFovZRads * displayAspectRatio; - // vec2 fovHalfWidths = vec2(tan(halfFovYRads), tan(halfFovZRads)); - // vec2 fovWidths = fovHalfWidths * 2.0; + world_pos.z /= cogl_z_factor; - // float vec_y = -world_pos.x * fovWidths.x + fovHalfWidths.x; - // float vec_z = -world_pos.y * fovWidths.y + fovHalfWidths.y; - // vec4 look_vector = vec4(1.0, vec_y, vec_z, 1.0); - // // vec3 rotated_vector = applyQuaternionToVector(look_vector, u_quaternion).xyz; - // vec3 rotated_vector = look_vector.xyz; + // if the perspective includes more than just our actor, move vertices towards the center of the perspective so they'll be properly rotated + world_pos.x += position_width_adjustment_count * cogl_position_width; + world_pos.y += position_height_adjustment_count * cogl_position_height; - // // scale back to the screen distance - // rotated_vector /= rotated_vector.x; - // cogl_position_out = vec4( - // ((fovHalfWidths.x - rotated_vector.y) / fovWidths.x) * 2.0 - 1.0, - // ((fovHalfWidths.y - rotated_vector.z) / fovWidths.y) * 2.0 - 1.0, - // 0.0, - // 1.0 - // ); - - // float z_orig = world_pos.z; - // world_pos.z -= z_orig / 1920.0; - // world_pos.x /= 2.0; - // world_pos *= u_display_north_offset; + world_pos.z *= u_aspect_ratio; + world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); + world_pos = applyYRotationToVector(world_pos, u_rotation_y_radians); world_pos = applyQuaternionToVector(world_pos, u_quaternion); - // world_pos /= u_display_north_offset; - // world_pos.x *= 2.0; - // world_pos.z += z_orig / 1920.0; - world_pos = cogl_modelview_matrix * world_pos; - cogl_position_out = cogl_projection_matrix * world_pos; + world_pos.z /= u_aspect_ratio; - // cogl_position_out.x = world_pos.x / 103.4; - // cogl_position_out.y = world_pos.y / 29.075; - // cogl_position_out.z = -1.0; - // cogl_position_out.w = 1.0; + world_pos.x /= u_actor_to_display_ratios.x; + world_pos.y /= u_actor_to_display_ratios.y; + + world_pos = u_projection_matrix * world_pos; + + // if the perspective includes more than just our actor, move the vertices back to just the area we can see. + // this needs to be done after the projection matrix multiplication so it will be projected as if centered in our vision + world_pos.x -= 0.5 * position_width_adjustment_count * world_pos.w; + world_pos.y -= 0.5 * position_height_adjustment_count * world_pos.w; + + cogl_position_out = world_pos; cogl_tex_coord_out[0] = cogl_tex_coord_in; ` @@ -351,12 +397,16 @@ export const TestActorEffect = GObject.registerClass({ ); Globals.logger.log(`aspect: ${aspect}, fov: ${this.fov_degrees}, width: ${this.get_actor().width}, height: ${this.get_actor().height}, projection matrix: ${JSON.stringify(projection_matrix)}`); this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projection_matrix); + this.set_uniform_float(this.get_uniform_location("u_rotation_x_radians"), 1, [this.monitor_wrapping_scheme === 'vertical' ? this.monitor_wrapping_rotation_radians : 0.0]); + this.set_uniform_float(this.get_uniform_location("u_rotation_y_radians"), 1, [this.monitor_wrapping_scheme === 'horizontal' ? this.monitor_wrapping_rotation_radians : 0.0]); + this.set_uniform_float(this.get_uniform_location("u_aspect_ratio"), 1, [aspect]); + this.set_uniform_float(this.get_uniform_location("u_actor_to_display_ratios"), 2, this.actor_to_display_ratios); this._initialized = true; } - this.set_uniform_float(this.get_uniform_location("u_display_north_offset"), 1, [this.focused_monitor_index === this.monitor_index ? this.display_distance : this.toggle_display_distance_start]); + this.set_uniform_float(this.get_uniform_location("u_display_north_offset"), 1, [this.display_distance]); - // NUW to east-up-south conversion, inverted + // NWU to east-up-south conversion, inverted this.set_uniform_float(this.get_uniform_location("u_quaternion"), 4, [this.quaternion.y, -this.quaternion.z, this.quaternion.x, this.quaternion.w]); this.get_pipeline().set_layer_filters( @@ -450,58 +500,65 @@ export const TestActor = GObject.registerClass({ })), 'horizontal' ); - this.monitorAsNormalizedVectors = this.monitorsAsVectors.map(vector => { + + // normalize the center vectors + this.monitorAsNormalizedVectors = this.monitorsAsVectors.map(monitorVectors => { + const vector = monitorVectors.center; const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); return [vector[0] / length, vector[1] / length, vector[2] / length]; }); + + const actorToDisplayRatios = [ + Main.layoutManager.uiGroup.width / this.width, + Main.layoutManager.uiGroup.height / this.height + ]; Main.layoutManager.monitors.forEach(((monitor, index) => { // if (index === 0) return; Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); // this is in NWU coordinates - const monitorVector = this.monitorsAsVectors[index]; - const monitorRotation = monitorVectorToRotationAngle(monitorVector, 'horizontal'); - Globals.logger.log_debug(`\t\t\tMonitor ${index} vector: ${monitorVector} rotation: ${JSON.stringify(monitorRotation)}`); + const noRotationVector = this.monitorsAsVectors[index].topLeftNoRotate; + Globals.logger.log_debug(`\t\t\tMonitor ${index} vectors: ${JSON.stringify(this.monitorsAsVectors[index])}`); // actor coordinates are east-up-south const containerActor = new Clutter.Actor({ - x: -monitorVector[1], - y: -monitorVector[2], - 'z-position': -monitorVector[0], + x: -noRotationVector[1], + y: -noRotationVector[2], + 'z-position': -noRotationVector[0], width: monitor.width, height: monitor.height, - reactive: false + reactive: false, }); // Create a clone of the stage content for this monitor const monitorClone = new Clutter.Clone({ source: Main.layoutManager.uiGroup, - reactive: false + reactive: false, + x: -containerActor.x - monitor.x, + y: -containerActor.y - monitor.y }); - - monitorClone.x = -containerActor.x; - // monitorActor.y = 0; - monitorClone.set_clip(monitor.x, 0, monitor.width, monitor.height); + monitorClone.set_clip(monitor.x, monitor.y, monitor.width, monitor.height); // Add the monitor actor to the scene containerActor.add_child(monitorClone); - containerActor.set_pivot_point(0.5, 0.5); - containerActor.set_rotation_angle(monitorRotation.axis, monitorRotation.angle); const effect = new TestActorEffect({ quaternion: this.quaternion, fov_degrees: this.fov_degrees, monitor_index: index, - display_distance: this.toggle_display_distance_start + display_distance: noRotationVector[0], + monitor_wrapping_scheme: 'horizontal', + monitor_wrapping_rotation_radians: monitorVectorToRotationAngle(this.monitorsAsVectors[index].center, 'horizontal').angle, + actor_to_display_ratios: actorToDisplayRatios }); containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); this.bind_property('quaternion', effect, 'quaternion', GObject.BindingFlags.DEFAULT); this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); - this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); + // this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); }).bind(this)); - GLib.timeout_add(GLib.PRIORITY_DEFAULT, 1000, (() => { + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 500, (() => { if (this.quaternion) { const closestMonitorIndex = findClosestVector(this.quaternion, this.monitorAsNormalizedVectors, this.closestMonitorIndex); @@ -516,9 +573,9 @@ export const TestActor = GObject.registerClass({ }).bind(this)); this._distance_ease_timeline = null; - this.connect('notify::toggle-display-distance-start', this._handle_display_distance_properties_change.bind(this)); - this.connect('notify::toggle-display-distance-end', this._handle_display_distance_properties_change.bind(this)); - this.connect('notify::display-distance', this._handle_display_distance_properties_change.bind(this)); + // this.connect('notify::toggle-display-distance-start', this._handle_display_distance_properties_change.bind(this)); + // this.connect('notify::toggle-display-distance-end', this._handle_display_distance_properties_change.bind(this)); + // this.connect('notify::display-distance', this._handle_display_distance_properties_change.bind(this)); this._handle_display_distance_properties_change(); } From 439d2fccce2eda46c056f52f31751fcede7128fc Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Wed, 22 Jan 2025 18:19:43 -0800 Subject: [PATCH 10/20] WIP --- gnome/src/devicedatastream.js | 18 ++- gnome/src/extension.js | 55 +++++---- gnome/src/testactor.js | 224 ++++++++++++++++++++++++++-------- 3 files changed, 211 insertions(+), 86 deletions(-) diff --git a/gnome/src/devicedatastream.js b/gnome/src/devicedatastream.js index 37d8d5a..ed7ac0d 100644 --- a/gnome/src/devicedatastream.js +++ b/gnome/src/devicedatastream.js @@ -67,12 +67,12 @@ export const DeviceDataStream = GObject.registerClass({ GObject.ParamFlags.READWRITE, false ), - 'quaternion': GObject.ParamSpec.jsobject( - 'quaternion', - 'Quaternion', - 'Camera orientation quaternion', + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', GObject.ParamFlags.READWRITE - ), + ) } }, class DeviceDataStream extends GObject.Object { constructor(params = {}) { @@ -140,11 +140,9 @@ export const DeviceDataStream = GObject.registerClass({ if (checkParityByte(dataView)) { this._device_data.imuData = imuData; this._device_data.imuDateMs = imuDateMs; - this.quaternion = { - x: imuData[0], - y: imuData[1], - z: imuData[2], - w: imuData[3] + this.imu_snapshots = { + imu_data: imuData, + timestamp_ms: imuDateMs }; success = true; } diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 9f81e5a..5751821 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -42,7 +42,7 @@ export default class BreezyDesktopExtension extends Extension { this._cursor_manager = null; this._device_data_stream = null; this._monitor_manager = null; - this.overlay_content = null; + this._overlay_content = null; this._overlay = null; this._target_monitor = null; this._is_effect_running = false; @@ -62,6 +62,8 @@ export default class BreezyDesktopExtension extends Extension { this._headset_as_primary_binding = null; this._actor_added_connection = null; this._actor_removed_connection = null; + this._data_stream_connection = null; + this._stage_redraw_connection = null; if (!Globals.logger) { Globals.logger = new Logger({ @@ -243,7 +245,7 @@ export default class BreezyDesktopExtension extends Extension { this._overlay.set_size(targetMonitor.width, targetMonitor.height); // const textureSourceActor = Main.layoutManager.uiGroup; - this.overlay_content = new TestActor({ + this._overlay_content = new TestActor({ monitors: [], fov_degrees: 46.0, // width: 100, @@ -255,7 +257,7 @@ export default class BreezyDesktopExtension extends Extension { toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end') }); - this._overlay.set_child(this.overlay_content); + this._overlay.set_child(this._overlay_content); Shell.util_set_hidden_from_pick(this._overlay, true); global.stage.add_child(this._overlay); @@ -279,21 +281,21 @@ export default class BreezyDesktopExtension extends Extension { // this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); // this._supported_device_detected_connection = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); - this.overlay_content.renderMonitors(); + this._overlay_content.renderMonitors(); this._data_stream_connection = this._device_data_stream.bind_property( - 'quaternion', - this.overlay_content, - 'quaternion', + 'imu-snapshots', + this._overlay_content, + 'imu-snapshots', GObject.BindingFlags.DEFAULT ); - this._distance_binding = this.settings.bind('display-distance', this.overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); + this._distance_binding = this.settings.bind('display-distance', this._overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)); this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)); // this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) - this._start_binding = this.settings.bind('toggle-display-distance-start', this.overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) - this._end_binding = this.settings.bind('toggle-display-distance-end', this.overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) + this._start_binding = this.settings.bind('toggle-display-distance-start', this._overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) + this._end_binding = this.settings.bind('toggle-display-distance-end', this._overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) // this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) // this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); // this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); @@ -301,13 +303,13 @@ export default class BreezyDesktopExtension extends Extension { Meta.disable_unredirect_for_display(global.display); - global.stage.connect('before-paint', (() => { + this._stage_redraw_connection = global.stage.connect('before-paint', (() => { this._device_data_stream.refresh_data(); this._overlay.queue_redraw(); }).bind(this)); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); - this._add_settings_keybinding('toggle-display-distance-shortcut', this.overlay_content._change_distance.bind(this.overlay_content)); + this._add_settings_keybinding('toggle-display-distance-shortcut', this._overlay_content._change_distance.bind(this._overlay_content)); this._add_settings_keybinding('toggle-follow-shortcut', this._toggle_follow_mode.bind(this)); } catch (e) { Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); @@ -504,6 +506,11 @@ export default class BreezyDesktopExtension extends Extension { Main.wm.removeKeybinding('toggle-display-distance-shortcut'); Main.wm.removeKeybinding('toggle-follow-shortcut'); Meta.enable_unredirect_for_display(global.display); + + if (this._stage_redraw_connection) { + global.stage.disconnect(this._stage_redraw_connection); + this._stage_redraw_connection = null; + } if (this._actor_added_connection) { global.stage.disconnect(this._actor_added_connection); @@ -514,7 +521,18 @@ export default class BreezyDesktopExtension extends Extension { this._actor_removed_connection = null; } if (this._overlay) { - this.overlay_content = null; + if (this._overlay_content) { + // if (this._widescreen_mode_effect_state_connection) { + // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); + // this._widescreen_mode_effect_state_connection = null; + // } + // if (this._supported_device_detected_connection) { + // this._xr_effect.disconnect(this._supported_device_detected_connection); + // this._supported_device_detected_connection = null; + // } + this._overlay_content.destroy(); + this._overlay_content = null; + } global.stage.remove_child(this._overlay); this._overlay.destroy(); @@ -564,17 +582,6 @@ export default class BreezyDesktopExtension extends Extension { this.settings.unbind(this._disable_anti_aliasing_binding); this._disable_anti_aliasing_binding = null; } - if (this.overlay_content) { - // if (this._widescreen_mode_effect_state_connection) { - // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); - // this._widescreen_mode_effect_state_connection = null; - // } - // if (this._supported_device_detected_connection) { - // this._xr_effect.disconnect(this._supported_device_detected_connection); - // this._supported_device_detected_connection = null; - // } - this.overlay_content = null; - } if (this._cursor_manager) { this._cursor_manager.disable(); this._cursor_manager = null; diff --git a/gnome/src/testactor.js b/gnome/src/testactor.js index bc6f0fa..80d30d1 100644 --- a/gnome/src/testactor.js +++ b/gnome/src/testactor.js @@ -23,15 +23,15 @@ function applyQuaternionToVector(vector, quaternion) { /** * Find the vector in the array that's closest to the quaternion rotation * - * @param {number[]} quaternion - Reference quaternion [w, x, y, z] + * @param {number[]} quaternion - Reference quaternion [x, y, z, w] * @param {number[][]} vectors - Array of vectors [x, y, z] to search from * @returns {number} Index of the closest vector, if it surpasses the previous closest index by a certain margin, otherwise the previous index */ function findClosestVector(quaternion, vectors, previousClosestIndex) { const lookVector = [1.0, 0.0, 0.0]; // NWU vector pointing to the center of the screen - const rotatedLookVector = applyQuaternionToVector(lookVector, [quaternion.x, quaternion.y, quaternion.z, quaternion.w]); - Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); + const rotatedLookVector = applyQuaternionToVector(lookVector, quaternion); + // Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); let closestIndex = -1; let closestDistance = Infinity; @@ -47,14 +47,14 @@ function findClosestVector(quaternion, vectors, previousClosestIndex) { previousDistance = distance; } - Globals.logger.log(`\t\t\tMonitor ${index} distance: ${distance}`); + // Globals.logger.log(`\t\t\tMonitor ${index} distance: ${distance}`); if (distance < closestDistance) { closestIndex = index; closestDistance = distance; } }); - Globals.logger.log(`\t\t\tClosest monitor: ${closestIndex}, distance: ${closestDistance}`); + // Globals.logger.log(`\t\t\tClosest monitor: ${closestIndex}, distance: ${closestDistance}`); // only switch if the closest monitor is greater than the previous closest by 25% if (previousClosestIndex !== undefined && closestIndex !== previousClosestIndex && closestDistance * 1.25 > previousDistance) { @@ -73,16 +73,34 @@ function radiansToDegrees(radians) { } /*** - * @returns {Object} - containing `center` and `end` radians + * @returns {Object} - containing `start`, `center`, and `end` radians for rotating the given monitor */ -function monitorWrap(radiusPixels, previousMonitorEndRadians, monitorPixels) { - const monitorHalfPixels = monitorPixels / 2; - const monitorHalfRadians = Math.asin(monitorHalfPixels / radiusPixels); - const centerRadians = previousMonitorEndRadians + monitorHalfRadians; +function monitorWrap(cachedMonitorWrap, radiusPixels, monitorBeginPixel, monitorLengthPixels) { + let closestWrap = cachedMonitorWrap.reduce((previous, current) => { + return (!previous || Math.abs(current.pixel - monitorBeginPixel) < Math.abs(previous.pixel - monitorBeginPixel)) ? current : previous; + }, undefined); + + if (closestWrap.pixel !== monitorBeginPixel) { + // there's a gap between the cached wrap value and this one + const gapPixels = monitorBeginPixel - closestWrap.pixel; + const gapHalfRadians = Math.asin(gapPixels / 2 / radiusPixels); + const gapRadians = gapHalfRadians * 2; + + // update the closestWrap value and cache it + closestWrap = { pixel: monitorBeginPixel, radians: closestWrap.radians + gapRadians }; + cachedMonitorWrap.push(closestWrap); + } + + const monitorHalfRadians = Math.asin(monitorLengthPixels / 2 / radiusPixels); + const centerRadians = closestWrap.radians + monitorHalfRadians; + const endRadians = centerRadians + monitorHalfRadians; + + // since we're computing the end values for this monitor, cache them too in case they line up with a future monitor + cachedMonitorWrap.push({ pixel: monitorBeginPixel + monitorLengthPixels, radians: endRadians }); return { - begin: previousMonitorEndRadians, + begin: closestWrap.radians, center: centerRadians, - end: centerRadians + monitorHalfRadians + end: endRadians } } @@ -93,17 +111,18 @@ function monitorWrap(radiusPixels, previousMonitorEndRadians, monitorPixels) { * @param {Object} fovDetails - contains reference fovDegrees (diagonal), widthPixels, heightPixels * @param {Object[]} monitorDetailsList - contains x, y, width, height (coordinates from top-left) * @param {string} monitorWrappingScheme - horizontal, vertical, none - * @returns {Object[]} - contains NWU vectors pointing to `topLeftNoRotate` and `center` of each monitor + * @returns {Object[]} - contains NWU vectors pointing to `topLeftNoRotate` and `center` of each monitor + * and a `rotation` angle for the given wrapping scheme */ -function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme) { +function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingScheme) { const aspect = fovDetails.widthPixels / fovDetails.heightPixels; const fovVerticalRadians = degreesToRadians(fovDetails.fovDegrees / Math.sqrt(1 + aspect * aspect)); // distance needed for the FOV-sized monitor to fill up the screen const centerRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); - // NWU vectors pointing to the center of the screen for each monitor - const monitorVectors = []; + const monitorPlacements = []; + const cachedMonitorWrap = []; if (monitorWrappingScheme === 'horizontal') { // monitors wrap around us horizontally @@ -112,12 +131,11 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // distance to a horizontal edge is the hypothenuse of the triangle where the opposite side is half the width of the reference fov screen const edgeRadius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); - let previousMonitorEndRadians = -fovHorizontalRadians / 2; + cachedMonitorWrap.push({ pixel: 0, radians: -fovHorizontalRadians / 2 }); monitorDetailsList.forEach(monitorDetails => { - const monitorWrapDetails = monitorWrap(edgeRadius, previousMonitorEndRadians, monitorDetails.width); - previousMonitorEndRadians = monitorWrapDetails.end; + const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.x, monitorDetails.width); - monitorVectors.push({ + monitorPlacements.push({ topLeftNoRotate: [ centerRadius, fovDetails.widthPixels / 2, @@ -132,7 +150,8 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // up is flat when wrapping horizontally -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) - ] + ], + rotationAngleRadians: -monitorWrapDetails.center }); }); } else if (monitorWrappingScheme === 'vertical') { @@ -141,12 +160,11 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // distance to a vertical edge is the hypothenuse of the triangle where the opposite side is half the height of the reference fov screen const edgeRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); - let previousMonitorEndRadians = -fovVerticalRadians / 2; + cachedMonitorWrap.push({ pixel: 0, radians: -fovVerticalRadians / 2 }); monitorDetailsList.forEach(monitorDetails => { - const monitorWrapDetails = monitorWrap(edgeRadius, previousMonitorEndRadians, monitorDetails.height); - previousMonitorEndRadians = monitorWrapDetails.end; + const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.y, monitorDetails.height); - monitorVectors.push({ + monitorPlacements.push({ topLeftNoRotate: [ centerRadius, -(monitorDetails.x - fovDetails.widthPixels / 2), @@ -161,13 +179,14 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians -centerRadius * Math.sin(monitorWrapDetails.center) - ] + ], + rotationAngleRadians: -monitorWrapDetails.center }); }); } else { // monitors make a flat wall in front of us, no wrapping monitorDetailsList.forEach(monitorDetails => { - monitorVectors.push({ + monitorPlacements.push({ topLeftNoRotate: [ centerRadius, -(monitorDetails.x - fovDetails.widthPixels / 2), @@ -177,12 +196,13 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme centerRadius, -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) - ] + ], + rotationAngleRadians: 0 }); }); } - return monitorVectors; + return monitorPlacements; } function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { @@ -204,6 +224,17 @@ function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { } } +// how far to look ahead is how old the IMU data is plus a constant that is either the default for this device or an override +function lookAheadMS(imuDateMs, override) { + // how stale the imu data is + const dataAge = Date.now() - imuDateMs; + + // if (override === -1) + // return lookAheadCfg[0] + dataAge; + + return override + dataAge; +} + export const TestActorEffect = GObject.registerClass({ Properties: { 'monitor-index': GObject.ParamSpec.int( @@ -213,10 +244,10 @@ export const TestActorEffect = GObject.registerClass({ GObject.ParamFlags.READWRITE, 0, 100, 0 ), - 'quaternion': GObject.ParamSpec.jsobject( - 'quaternion', - 'Quaternion', - 'Camera orientation quaternion', + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', GObject.ParamFlags.READWRITE ), 'fov-degrees': GObject.ParamSpec.double( @@ -316,7 +347,8 @@ export const TestActorEffect = GObject.registerClass({ vfunc_build_pipeline() { const declarations = ` - uniform vec4 u_quaternion; + uniform mat4 u_imu_data; + uniform float u_look_ahead_ms; uniform mat4 u_projection_matrix; uniform float u_display_north_offset; uniform float u_rotation_x_radians; @@ -330,6 +362,88 @@ export const TestActorEffect = GObject.registerClass({ float cogl_position_width = 51.7; // no idea... float cogl_z_factor = 2.5; // no idea... + float vectorLength(vec3 v) { + return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); + } + + float quaternionLength(vec4 q) { + return sqrt(q.x * q.x + q.y * q.y + q.z * q.z + q.w * q.w); + } + + vec4 quatMul(vec4 q1, vec4 q2) { + return vec4( + q1.w * q2.x + q1.x * q2.w + q1.y * q2.z - q1.z * q2.y, // x + q1.w * q2.y - q1.x * q2.z + q1.y * q2.w + q1.z * q2.x, // y + q1.w * q2.z + q1.x * q2.y - q1.y * q2.x + q1.z * q2.w, // z + q1.w * q2.w - q1.x * q2.x - q1.y * q2.y - q1.z * q2.z // w + ); + } + + vec4 quatConjugate(vec4 q) { + return vec4(-q.xyz, q.w); + } + + vec4 quatExp(vec4 q) { + float vLength = vectorLength(q.xyz); + float expW = exp(q.w); + + if (vLength < 0.000001) { + return vec4(0.0, 0.0, 0.0, expW); + } + + float scale = expW * sin(vLength) / vLength; + return vec4(q.xyz * scale, expW * cos(vLength)); + } + + vec4 quatLog(vec4 q) { + float qLength = quaternionLength(q); + float vLength = vectorLength(q.xyz); + + if (vLength < 0.000001) { + return vec4(0.0, 0.0, 0.0, log(qLength)); + } + + float scale = acos(clamp(q.w / qLength, -1.0, 1.0)) / vLength; + return vec4(q.xyz * scale, log(qLength)); + } + + vec4 computeQuaternionVelocity(vec4 q1, vec4 q2, float milliseconds) { + // Normalize input quaternions + q1 = normalize(q1); + q2 = normalize(q2); + + // Compute difference quaternion (q2 * q1^-1) + vec4 diffQ = quatMul(q2, quatConjugate(q1)); + + // Ensure we take the shortest path + if (diffQ.w < 0.0) { + diffQ = -diffQ; + } + + // Take the log and scale by time + return quatLog(diffQ) / milliseconds; + } + + vec4 extrapolateRotation(vec4 initialQuat, vec4 velocity, float deltaTimeMs) { + // Scale velocity by time + vec4 scaledVelocity = velocity * deltaTimeMs; + + // Compute the exponential + vec4 deltaRotation = quatExp(scaledVelocity); + + // Apply to initial quaternion + return normalize(quatMul(deltaRotation, initialQuat)); + } + + vec4 imuDataToLookAheadQuaternion(mat4 imuData, float lookAheadMS) { + // last row of matrix contains imu timestamps, subtract the second column from the first + float imuDeltaTime = imuData[3][0] - imuData[3][1]; + + // rotation per ms + vec4 velocity = computeQuaternionVelocity(imuData[0], imuData[1], imuDeltaTime); + return extrapolateRotation(imuData[0], velocity, lookAheadMS); + } + vec4 applyQuaternionToVector(vec4 v, vec4 q) { vec3 t = 2.0 * cross(q.xyz, v.xyz); vec3 rotated = v.xyz + q.w * t + cross(q.xyz, t); @@ -347,10 +461,15 @@ export const TestActorEffect = GObject.registerClass({ float s = sin(angle); return vec4(v.x * c + v.z * s, v.y, v.z * c - v.x * s, v.w); } + + vec4 nwuToESU(vec4 v) { + return vec4(-v.y, v.z, -v.x, v.w); + } `; const main = ` vec4 world_pos = cogl_position_in; + vec4 look_ahead_quaternion = nwuToESU(imuDataToLookAheadQuaternion(u_imu_data, u_look_ahead_ms)); float cogl_position_height = cogl_position_width / u_aspect_ratio; float position_width_adjustment_count = u_actor_to_display_ratios.x - 1.0; @@ -365,7 +484,7 @@ export const TestActorEffect = GObject.registerClass({ world_pos.z *= u_aspect_ratio; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); world_pos = applyYRotationToVector(world_pos, u_rotation_y_radians); - world_pos = applyQuaternionToVector(world_pos, u_quaternion); + world_pos = applyQuaternionToVector(world_pos, quatConjugate(look_ahead_quaternion)); world_pos.z /= u_aspect_ratio; world_pos.x /= u_actor_to_display_ratios.x; @@ -395,7 +514,6 @@ export const TestActorEffect = GObject.registerClass({ 0.0001, 1000.0 ); - Globals.logger.log(`aspect: ${aspect}, fov: ${this.fov_degrees}, width: ${this.get_actor().width}, height: ${this.get_actor().height}, projection matrix: ${JSON.stringify(projection_matrix)}`); this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projection_matrix); this.set_uniform_float(this.get_uniform_location("u_rotation_x_radians"), 1, [this.monitor_wrapping_scheme === 'vertical' ? this.monitor_wrapping_rotation_radians : 0.0]); this.set_uniform_float(this.get_uniform_location("u_rotation_y_radians"), 1, [this.monitor_wrapping_scheme === 'horizontal' ? this.monitor_wrapping_rotation_radians : 0.0]); @@ -404,10 +522,9 @@ export const TestActorEffect = GObject.registerClass({ this._initialized = true; } + this.set_uniform_float(this.get_uniform_location('u_look_ahead_ms'), 1, [lookAheadMS(this.imu_snapshots.timestamp_ms, 0)]); this.set_uniform_float(this.get_uniform_location("u_display_north_offset"), 1, [this.display_distance]); - - // NWU to east-up-south conversion, inverted - this.set_uniform_float(this.get_uniform_location("u_quaternion"), 4, [this.quaternion.y, -this.quaternion.z, this.quaternion.x, this.quaternion.w]); + this.set_uniform_matrix(this.get_uniform_location("u_imu_data"), false, 4, this.imu_snapshots.imu_data); this.get_pipeline().set_layer_filters( 0, @@ -427,10 +544,10 @@ export const TestActor = GObject.registerClass({ 'Array of monitor indexes', GObject.ParamFlags.READWRITE ), - 'quaternion': GObject.ParamSpec.jsobject( - 'quaternion', - 'Quaternion', - 'Camera orientation quaternion', + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', GObject.ParamFlags.READWRITE ), 'fov-degrees': GObject.ParamSpec.double( @@ -486,7 +603,7 @@ export const TestActor = GObject.registerClass({ } }, class TestActor extends Clutter.Actor { renderMonitors() { - this.monitorsAsVectors = monitorsToVectors( + this._monitorPlacements = monitorsToPlacements( { fovDegrees: this.fov_degrees, widthPixels: this.width, @@ -502,7 +619,7 @@ export const TestActor = GObject.registerClass({ ); // normalize the center vectors - this.monitorAsNormalizedVectors = this.monitorsAsVectors.map(monitorVectors => { + this._monitorsAsNormalizedVectors = this._monitorPlacements.map(monitorVectors => { const vector = monitorVectors.center; const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); return [vector[0] / length, vector[1] / length, vector[2] / length]; @@ -518,8 +635,8 @@ export const TestActor = GObject.registerClass({ Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); // this is in NWU coordinates - const noRotationVector = this.monitorsAsVectors[index].topLeftNoRotate; - Globals.logger.log_debug(`\t\t\tMonitor ${index} vectors: ${JSON.stringify(this.monitorsAsVectors[index])}`); + const noRotationVector = this._monitorPlacements[index].topLeftNoRotate; + Globals.logger.log_debug(`\t\t\tMonitor ${index} vectors: ${JSON.stringify(this._monitorPlacements[index])}`); // actor coordinates are east-up-south const containerActor = new Clutter.Actor({ @@ -543,24 +660,27 @@ export const TestActor = GObject.registerClass({ // Add the monitor actor to the scene containerActor.add_child(monitorClone); const effect = new TestActorEffect({ - quaternion: this.quaternion, + imu_snapshots: this.imu_snapshots, fov_degrees: this.fov_degrees, monitor_index: index, display_distance: noRotationVector[0], monitor_wrapping_scheme: 'horizontal', - monitor_wrapping_rotation_radians: monitorVectorToRotationAngle(this.monitorsAsVectors[index].center, 'horizontal').angle, + monitor_wrapping_rotation_radians: this._monitorPlacements[index].rotationAngleRadians, actor_to_display_ratios: actorToDisplayRatios }); containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); - this.bind_property('quaternion', effect, 'quaternion', GObject.BindingFlags.DEFAULT); + this.bind_property('imu-snapshots', effect, 'imu-snapshots', GObject.BindingFlags.DEFAULT); this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); // this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); }).bind(this)); GLib.timeout_add(GLib.PRIORITY_DEFAULT, 500, (() => { - if (this.quaternion) { - const closestMonitorIndex = findClosestVector(this.quaternion, this.monitorAsNormalizedVectors, this.closestMonitorIndex); + if (this.imu_snapshots) { + const closestMonitorIndex = findClosestVector( + this.imu_snapshots.imu_data.splice(0, 4), + this._monitorsAsNormalizedVectors, this.closestMonitorIndex + ); // only switch if the closest monitor is greater than the previous closest by 25% if (this.closestMonitorIndex === undefined || this.closestMonitorIndex !== closestMonitorIndex) { From d0864c1a3b025337185575a9d616db48dcf314ff Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Fri, 24 Jan 2025 15:08:33 -0800 Subject: [PATCH 11/20] WIP --- gnome/src/extension.js | 85 ++++----- gnome/src/globals.js | 3 +- .../{testactor.js => virtualmonitorsactor.js} | 163 +++++++++++++----- 3 files changed, 155 insertions(+), 96 deletions(-) rename gnome/src/{testactor.js => virtualmonitorsactor.js} (84%) diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 5751821..5d3c5e7 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -11,9 +11,7 @@ import { DeviceDataStream } from './devicedatastream.js'; import Globals from './globals.js'; import { Logger } from './logger.js'; import { MonitorManager } from './monitormanager.js'; -import { TestActorEffect, TestActor } from './testactor.js'; -import { isValidKeepAlive } from './time.js'; -import { IPC_FILE_PATH, XREffect } from './xrEffect.js'; +import { VirtualMonitorsActor } from './virtualmonitorsactor.js'; import {Extension} from 'resource:///org/gnome/shell/extensions/extension.js'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; @@ -40,7 +38,6 @@ export default class BreezyDesktopExtension extends Extension { // Set/destroyed by enable/disable this._cursor_manager = null; - this._device_data_stream = null; this._monitor_manager = null; this._overlay_content = null; this._overlay = null; @@ -72,6 +69,10 @@ export default class BreezyDesktopExtension extends Extension { }); Globals.logger.logVersion(); } + + if (!Globals.data_stream) { + Globals.data_stream = new DeviceDataStream(); + } } enable() { @@ -81,8 +82,7 @@ export default class BreezyDesktopExtension extends Extension { Globals.extension_dir = this.path; this.settings.bind('debug', Globals.logger, 'debug', Gio.SettingsBindFlags.DEFAULT); - this._device_data_stream = new DeviceDataStream(); - this._device_data_stream.start(); + Globals.data_stream.start(); this._monitor_manager = new MonitorManager({ use_optimal_monitor_config: this.settings.get_boolean('use-optimal-monitor-config'), @@ -115,7 +115,7 @@ export default class BreezyDesktopExtension extends Extension { return GLib.SOURCE_REMOVE; } - if (this._device_data_stream.supported_device_connected && target_monitor) { + if (Globals.data_stream.supported_device_connected && target_monitor) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes, // so we can remove this timeout_add no matter what. @@ -126,7 +126,7 @@ export default class BreezyDesktopExtension extends Extension { this._running_poller_id = undefined; return GLib.SOURCE_REMOVE; } else { - Globals.logger.log_debug(`BreezyDesktopExtension _poll_for_ready - device connected: ${this._device_data_stream.supported_device_connected}, target_monitor: ${!!target_monitor}`); + Globals.logger.log_debug(`BreezyDesktopExtension _poll_for_ready - device connected: ${Globals.data_stream.supported_device_connected}, target_monitor: ${!!target_monitor}`); return GLib.SOURCE_CONTINUE; } } catch (e) { @@ -195,7 +195,7 @@ export default class BreezyDesktopExtension extends Extension { if (target_monitor && this._running_poller_id === undefined) { this._target_monitor = target_monitor; - if (this._device_data_stream.supported_device_connected) { + if (Globals.data_stream.supported_device_connected) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes if (this._target_monitor_ready(target_monitor)) { @@ -209,7 +209,7 @@ export default class BreezyDesktopExtension extends Extension { this._poll_for_ready(); } } else { - Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, device connected: ${this._device_data_stream.supported_device_connected}, target_monitor found: ${!!target_monitor}`); + Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, device connected: ${Globals.data_stream.supported_device_connected}, target_monitor found: ${!!target_monitor}`); } } @@ -245,16 +245,16 @@ export default class BreezyDesktopExtension extends Extension { this._overlay.set_size(targetMonitor.width, targetMonitor.height); // const textureSourceActor = Main.layoutManager.uiGroup; - this._overlay_content = new TestActor({ + Globals.data_stream.refresh_data(); + this._overlay_content = new VirtualMonitorsActor({ monitors: [], fov_degrees: 46.0, - // width: 100, - // height: 100, width: targetMonitor.width, height: targetMonitor.height, display_distance: this.settings.get_double('display-distance'), toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), - toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end') + toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), + imu_snapshots: Globals.data_stream.imu_snapshots }); this._overlay.set_child(this._overlay_content); @@ -282,12 +282,6 @@ export default class BreezyDesktopExtension extends Extension { // this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); // this._supported_device_detected_connection = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); this._overlay_content.renderMonitors(); - this._data_stream_connection = this._device_data_stream.bind_property( - 'imu-snapshots', - this._overlay_content, - 'imu-snapshots', - GObject.BindingFlags.DEFAULT - ); this._distance_binding = this.settings.bind('display-distance', this._overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)); @@ -295,18 +289,14 @@ export default class BreezyDesktopExtension extends Extension { // this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) this._start_binding = this.settings.bind('toggle-display-distance-start', this._overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) - this._end_binding = this.settings.bind('toggle-display-distance-end', this._overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) + this._end_binding = this.settings.bind('toggle-display-distance-end', this._overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT); + this._display_size_binding = this.settings.bind('display-size', this._overlay_content, 'display-size', Gio.SettingsBindFlags.DEFAULT); // this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) // this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); // this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); // this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); Meta.disable_unredirect_for_display(global.display); - - this._stage_redraw_connection = global.stage.connect('before-paint', (() => { - this._device_data_stream.refresh_data(); - this._overlay.queue_redraw(); - }).bind(this)); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); this._add_settings_keybinding('toggle-display-distance-shortcut', this._overlay_content._change_distance.bind(this._overlay_content)); @@ -520,24 +510,6 @@ export default class BreezyDesktopExtension extends Extension { global.stage.disconnect(this._actor_removed_connection); this._actor_removed_connection = null; } - if (this._overlay) { - if (this._overlay_content) { - // if (this._widescreen_mode_effect_state_connection) { - // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); - // this._widescreen_mode_effect_state_connection = null; - // } - // if (this._supported_device_detected_connection) { - // this._xr_effect.disconnect(this._supported_device_detected_connection); - // this._supported_device_detected_connection = null; - // } - this._overlay_content.destroy(); - this._overlay_content = null; - } - - global.stage.remove_child(this._overlay); - this._overlay.destroy(); - this._overlay = null; - } if (this._distance_binding) { this.settings.unbind(this._distance_binding); this._distance_binding = null; @@ -582,6 +554,24 @@ export default class BreezyDesktopExtension extends Extension { this.settings.unbind(this._disable_anti_aliasing_binding); this._disable_anti_aliasing_binding = null; } + if (this._overlay) { + if (this._overlay_content) { + // if (this._widescreen_mode_effect_state_connection) { + // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); + // this._widescreen_mode_effect_state_connection = null; + // } + // if (this._supported_device_detected_connection) { + // this._xr_effect.disconnect(this._supported_device_detected_connection); + // this._supported_device_detected_connection = null; + // } + this._overlay_content.destroy(); + this._overlay_content = null; + } + + global.stage.remove_child(this._overlay); + this._overlay.destroy(); + this._overlay = null; + } if (this._cursor_manager) { this._cursor_manager.disable(); this._cursor_manager = null; @@ -601,14 +591,11 @@ export default class BreezyDesktopExtension extends Extension { disable() { try { Globals.logger.log_debug('BreezyDesktopExtension disable'); + Globals.data_stream.stop(); + this._effect_disable(); this._target_monitor = null; - if (this._device_data_stream) { - this._device_data_stream.stop(); - this._device_data_stream = null; - } - if (this._monitor_manager) { if (this._optimal_monitor_config_binding) { this.settings.unbind(this._optimal_monitor_config_binding); diff --git a/gnome/src/globals.js b/gnome/src/globals.js index 124d2e1..f11451c 100644 --- a/gnome/src/globals.js +++ b/gnome/src/globals.js @@ -1,6 +1,7 @@ const Globals = { logger: null, ipc_file: null, // Gio.File instance, file exists if set - extension_dir: null // string path + extension_dir: null, // string path + data_stream: null, // DeviceDataStream instance } export default Globals; \ No newline at end of file diff --git a/gnome/src/testactor.js b/gnome/src/virtualmonitorsactor.js similarity index 84% rename from gnome/src/testactor.js rename to gnome/src/virtualmonitorsactor.js index 80d30d1..f8db098 100644 --- a/gnome/src/testactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -235,7 +235,7 @@ function lookAheadMS(imuDateMs, override) { return override + dataAge; } -export const TestActorEffect = GObject.registerClass({ +export const VirtualMonitorEffect = GObject.registerClass({ Properties: { 'monitor-index': GObject.ParamSpec.int( 'monitor-index', @@ -298,35 +298,90 @@ export const TestActorEffect = GObject.registerClass({ 'Distance of the display from the camera', GObject.ParamFlags.READWRITE, 0.0, - 10000.0, + 2.5, + 1.0 + ), + 'display-distance-z': GObject.ParamSpec.double( + 'display-distance-z', + 'Display Distance z-position', + 'Distance of the display from the camera in the z-axis', + GObject.ParamFlags.READWRITE, + 0.0, + 10000.0, 2900.0 ), - 'toggle-display-distance-start': GObject.ParamSpec.double( - 'toggle-display-distance-start', - 'Display distance start', - 'Start distance when using the "change distance" shortcut.', + 'display-distance-default': GObject.ParamSpec.double( + 'display-distance-default', + 'Display distance default', + 'Distance to use when not explicitly set, or when reset', GObject.ParamFlags.READWRITE, 0.2, 2.5, - 1.05 - ), - 'toggle-display-distance-end': GObject.ParamSpec.double( - 'toggle-display-distance-end', - 'Display distance end', - 'End distance when using the "change distance" shortcut.', - GObject.ParamFlags.READWRITE, - 0.2, - 2.5, - 1.05 + 1.0 ), 'actor-to-display-ratios': GObject.ParamSpec.jsobject( 'actor-to-display-ratios', 'Actor to Display Ratios', 'Ratios to convert actor coordinates to display coordinates', GObject.ParamFlags.READWRITE + ), + 'monitor-actor': GObject.ParamSpec.object( + 'monitor-actor', + 'Monitor Actor', + 'The actor that represents the monitor', + GObject.ParamFlags.READWRITE, + Clutter.Actor.$gtype + ), + 'is-closest': GObject.ParamSpec.boolean( + 'is-closest', + 'Is Closest', + 'Whether this monitor is the closest to the camera', + GObject.ParamFlags.READWRITE, + false ) } -}, class TestActorEffect extends Shell.GLSLEffect { +}, class VirtualMonitorEffect extends Shell.GLSLEffect { + constructor(params = {}) { + super(params); + + this._current_display_distance = this._is_focused() ? this.display_distance : this.display_distance_default; + + this.connect('notify::display-distance', this._update_display_distance.bind(this)); + this.connect('notify::focused-monitor-index', this._update_display_distance.bind(this)); + } + + _is_focused() { + return this.focused_monitor_index === this.monitor_index; + } + + _update_display_distance() { + const desired_distance = this._is_focused() ? this.display_distance : this.display_distance_default; + if (this._distance_ease_timeline?.is_playing()) { + // we're already easing towards the desired distance, do nothing + if (this._distance_ease_target === desired_distance) return; + + this._distance_ease_timeline.stop(); + } + + const mid_distance = (this.display_distance_default + desired_distance) / 2; + + this._distance_ease_start = this._current_display_distance; + this._distance_ease_timeline = Clutter.Timeline.new_for_actor(this.get_actor(), 250); + + this._distance_ease_target = desired_distance; + this._distance_ease_timeline.connect('new-frame', (() => { + this._current_display_distance = this._distance_ease_start + + this._distance_ease_timeline.get_progress() * + (this._distance_ease_target - this._distance_ease_start); + this.is_closest = this._current_display_distance < mid_distance; + }).bind(this)); + + this._distance_ease_timeline.start(); + + this.monitor_actor.set_z_position(this.monitor_index); + this.monitor_actor.queue_redraw(); + } + perspective(fovDiagonalRadians, aspect, near, far) { // compute horizontal fov given diagonal fov and aspect ratio const h = Math.sqrt(aspect * aspect + 1); @@ -350,7 +405,7 @@ export const TestActorEffect = GObject.registerClass({ uniform mat4 u_imu_data; uniform float u_look_ahead_ms; uniform mat4 u_projection_matrix; - uniform float u_display_north_offset; + uniform float u_display_distance; uniform float u_rotation_x_radians; uniform float u_rotation_y_radians; uniform float u_aspect_ratio; @@ -360,7 +415,7 @@ export const TestActorEffect = GObject.registerClass({ // constants that help me adjust CoGL vector positions so their components are at the ratios intended, for proper rotation float cogl_position_width = 51.7; // no idea... - float cogl_z_factor = 2.5; // no idea... + float cogl_z_factor = 34.66; // no idea... float vectorLength(vec3 v) { return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); @@ -475,7 +530,7 @@ export const TestActorEffect = GObject.registerClass({ float position_width_adjustment_count = u_actor_to_display_ratios.x - 1.0; float position_height_adjustment_count = u_actor_to_display_ratios.y - 1.0; - world_pos.z /= cogl_z_factor; + world_pos.z = - u_display_distance / cogl_z_factor; // if the perspective includes more than just our actor, move vertices towards the center of the perspective so they'll be properly rotated world_pos.x += position_width_adjustment_count * cogl_position_width; @@ -491,6 +546,7 @@ export const TestActorEffect = GObject.registerClass({ world_pos.y /= u_actor_to_display_ratios.y; world_pos = u_projection_matrix * world_pos; + world_pos /= u_display_distance; // if the perspective includes more than just our actor, move the vertices back to just the area we can see. // this needs to be done after the projection matrix multiplication so it will be projected as if centered in our vision @@ -523,7 +579,8 @@ export const TestActorEffect = GObject.registerClass({ } this.set_uniform_float(this.get_uniform_location('u_look_ahead_ms'), 1, [lookAheadMS(this.imu_snapshots.timestamp_ms, 0)]); - this.set_uniform_float(this.get_uniform_location("u_display_north_offset"), 1, [this.display_distance]); + // Globals.logger.log(`\t\t\tDisplay distance: ${this._current_display_distance * this.display_distance_z}`); + this.set_uniform_float(this.get_uniform_location("u_display_distance"), 1, [this._current_display_distance * this.display_distance_z]); this.set_uniform_matrix(this.get_uniform_location("u_imu_data"), false, 4, this.imu_snapshots.imu_data); this.get_pipeline().set_layer_filters( @@ -536,7 +593,7 @@ export const TestActorEffect = GObject.registerClass({ } }); -export const TestActor = GObject.registerClass({ +export const VirtualMonitorsActor = GObject.registerClass({ Properties: { 'monitors': GObject.ParamSpec.jsobject( 'monitors', @@ -580,7 +637,7 @@ export const TestActor = GObject.registerClass({ GObject.ParamFlags.READWRITE, 0.2, 2.5, - 1.0 + 1.05 ), 'toggle-display-distance-start': GObject.ParamSpec.double( 'toggle-display-distance-start', @@ -601,7 +658,7 @@ export const TestActor = GObject.registerClass({ 1.05 ), } -}, class TestActor extends Clutter.Actor { +}, class VirtualMonitorsActor extends Clutter.Actor { renderMonitors() { this._monitorPlacements = monitorsToPlacements( { @@ -642,7 +699,8 @@ export const TestActor = GObject.registerClass({ const containerActor = new Clutter.Actor({ x: -noRotationVector[1], y: -noRotationVector[2], - 'z-position': -noRotationVector[0], + // ideally we would do this, but it causes blur, so we instead set the distance in the shader + // 'z-position': -noRotationVector[0], width: monitor.width, height: monitor.height, reactive: false, @@ -659,20 +717,28 @@ export const TestActor = GObject.registerClass({ // Add the monitor actor to the scene containerActor.add_child(monitorClone); - const effect = new TestActorEffect({ + const effect = new VirtualMonitorEffect({ imu_snapshots: this.imu_snapshots, fov_degrees: this.fov_degrees, monitor_index: index, - display_distance: noRotationVector[0], + display_distance_z: noRotationVector[0], + display_distance: this.display_distance, + display_distance_default: Math.max(this.toggle_display_distance_start, this.toggle_display_distance_end), monitor_wrapping_scheme: 'horizontal', monitor_wrapping_rotation_radians: this._monitorPlacements[index].rotationAngleRadians, - actor_to_display_ratios: actorToDisplayRatios + actor_to_display_ratios: actorToDisplayRatios, + monitor_actor: containerActor }); containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); this.bind_property('imu-snapshots', effect, 'imu-snapshots', GObject.BindingFlags.DEFAULT); this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); - // this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); + this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); + + // in addition to rendering distance property in the shader, the parent actor determines overlap based on child ordering + effect.connect('notify::is-closest', ((actor, _pspec) => { + if (actor.is_closest) this.set_child_above_sibling(containerActor, null); + }).bind(this)); }).bind(this)); GLib.timeout_add(GLib.PRIORITY_DEFAULT, 500, (() => { @@ -683,19 +749,28 @@ export const TestActor = GObject.registerClass({ ); // only switch if the closest monitor is greater than the previous closest by 25% - if (this.closestMonitorIndex === undefined || this.closestMonitorIndex !== closestMonitorIndex) { + if (closestMonitorIndex !== -1 && (this.focused_monitor_index === undefined || this.focused_monitor_index !== closestMonitorIndex)) { Globals.logger.log(`Switching to monitor ${closestMonitorIndex}`); - this.closestMonitorIndex = closestMonitorIndex; + this.focused_monitor_index = closestMonitorIndex; } } return GLib.SOURCE_CONTINUE; }).bind(this)); + this._redraw_timeline = Clutter.Timeline.new_for_actor(this, 1000); + this._redraw_timeline.connect('new-frame', (() => { + Globals.data_stream.refresh_data(); + this.imu_snapshots = Globals.data_stream.imu_snapshots; + this.queue_redraw(); + }).bind(this)); + this._redraw_timeline.set_repeat_count(-1); + this._redraw_timeline.start(); + this._distance_ease_timeline = null; - // this.connect('notify::toggle-display-distance-start', this._handle_display_distance_properties_change.bind(this)); - // this.connect('notify::toggle-display-distance-end', this._handle_display_distance_properties_change.bind(this)); - // this.connect('notify::display-distance', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::toggle-display-distance-start', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::toggle-display-distance-end', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::display-distance', this._handle_display_distance_properties_change.bind(this)); this._handle_display_distance_properties_change(); } @@ -706,19 +781,15 @@ export const TestActor = GObject.registerClass({ } _change_distance() { - if (this._distance_ease_timeline?.is_playing()) this._distance_ease_timeline.stop(); - - this._distance_ease_start = this.display_distance; - this._distance_ease_timeline = Clutter.Timeline.new_for_actor(this, 250); - - const toggle_display_distance_target = this._is_display_distance_at_end ? + this.display_distance = this._is_display_distance_at_end ? this.toggle_display_distance_start : this.toggle_display_distance_end; - this._distance_ease_timeline.connect('new-frame', () => { - this.display_distance = this._distance_ease_start + - this._distance_ease_timeline.get_progress() * - (toggle_display_distance_target - this._distance_ease_start); - }); + } - this._distance_ease_timeline.start(); + destroy() { + if (this._redraw_timeline) { + this._redraw_timeline.stop(); + this._redraw_timeline = null; + } + super.destroy(); } }); \ No newline at end of file From 0ddcce949462da886db84f9f60cb786972fdafae Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Mon, 27 Jan 2025 14:00:56 -0800 Subject: [PATCH 12/20] WIP --- gnome/src/virtualmonitorsactor.js | 63 +++++++++++++++++-------------- 1 file changed, 35 insertions(+), 28 deletions(-) diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index f8db098..e0bd4fc 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -325,13 +325,6 @@ export const VirtualMonitorEffect = GObject.registerClass({ 'Ratios to convert actor coordinates to display coordinates', GObject.ParamFlags.READWRITE ), - 'monitor-actor': GObject.ParamSpec.object( - 'monitor-actor', - 'Monitor Actor', - 'The actor that represents the monitor', - GObject.ParamFlags.READWRITE, - Clutter.Actor.$gtype - ), 'is-closest': GObject.ParamSpec.boolean( 'is-closest', 'Is Closest', @@ -365,21 +358,35 @@ export const VirtualMonitorEffect = GObject.registerClass({ const mid_distance = (this.display_distance_default + desired_distance) / 2; + // if we're the focused display, we'll double the timeline and wait for the first half to let other + // displays ease out first + this._distance_ease_focus = this._is_focused(); + const timeline_ms = this._distance_ease_focus ? 500 : 150; + this._distance_ease_start = this._current_display_distance; - this._distance_ease_timeline = Clutter.Timeline.new_for_actor(this.get_actor(), 250); + this._distance_ease_timeline = Clutter.Timeline.new_for_actor(this.get_actor(), timeline_ms); this._distance_ease_target = desired_distance; this._distance_ease_timeline.connect('new-frame', (() => { + let progress = this._distance_ease_timeline.get_progress(); + if (this._distance_ease_focus) { + // if we're the focused display, wait for the first half of the timeline to pass + if (progress < 0.5) return; + + // treat the second half of the timeline as its own full progression + progress = (progress - 0.5) * 2; + + // put this display in front as it starts to easy in + this.is_closest = true; + } else { + this.is_closest = false; + } + this._current_display_distance = this._distance_ease_start + - this._distance_ease_timeline.get_progress() * - (this._distance_ease_target - this._distance_ease_start); - this.is_closest = this._current_display_distance < mid_distance; + progress * (this._distance_ease_target - this._distance_ease_start); }).bind(this)); this._distance_ease_timeline.start(); - - this.monitor_actor.set_z_position(this.monitor_index); - this.monitor_actor.queue_redraw(); } perspective(fovDiagonalRadians, aspect, near, far) { @@ -408,14 +415,14 @@ export const VirtualMonitorEffect = GObject.registerClass({ uniform float u_display_distance; uniform float u_rotation_x_radians; uniform float u_rotation_y_radians; - uniform float u_aspect_ratio; + uniform vec2 u_display_resolution; // for some reason the vector positions are relative to the width and height of the uiGroup actor uniform vec2 u_actor_to_display_ratios; // constants that help me adjust CoGL vector positions so their components are at the ratios intended, for proper rotation - float cogl_position_width = 51.7; // no idea... - float cogl_z_factor = 34.66; // no idea... + float cogl_position_width_factor = 29.09; // no idea... + float cogl_z_factor = 55.41; // no idea... float vectorLength(vec3 v) { return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); @@ -525,33 +532,34 @@ export const VirtualMonitorEffect = GObject.registerClass({ const main = ` vec4 world_pos = cogl_position_in; vec4 look_ahead_quaternion = nwuToESU(imuDataToLookAheadQuaternion(u_imu_data, u_look_ahead_ms)); + float aspect_ratio = u_display_resolution.x / u_display_resolution.y; - float cogl_position_height = cogl_position_width / u_aspect_ratio; + float cogl_position_width = cogl_position_width_factor * aspect_ratio; + float cogl_position_height = cogl_position_width / aspect_ratio; float position_width_adjustment_count = u_actor_to_display_ratios.x - 1.0; float position_height_adjustment_count = u_actor_to_display_ratios.y - 1.0; - world_pos.z = - u_display_distance / cogl_z_factor; + world_pos.z = - u_display_distance * cogl_z_factor / u_display_resolution.x; // if the perspective includes more than just our actor, move vertices towards the center of the perspective so they'll be properly rotated world_pos.x += position_width_adjustment_count * cogl_position_width; world_pos.y += position_height_adjustment_count * cogl_position_height; - world_pos.z *= u_aspect_ratio; + world_pos.z *= aspect_ratio; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); world_pos = applyYRotationToVector(world_pos, u_rotation_y_radians); world_pos = applyQuaternionToVector(world_pos, quatConjugate(look_ahead_quaternion)); - world_pos.z /= u_aspect_ratio; + world_pos.z /= aspect_ratio; world_pos.x /= u_actor_to_display_ratios.x; world_pos.y /= u_actor_to_display_ratios.y; world_pos = u_projection_matrix * world_pos; - world_pos /= u_display_distance; // if the perspective includes more than just our actor, move the vertices back to just the area we can see. // this needs to be done after the projection matrix multiplication so it will be projected as if centered in our vision - world_pos.x -= 0.5 * position_width_adjustment_count * world_pos.w; - world_pos.y -= 0.5 * position_height_adjustment_count * world_pos.w; + world_pos.x -= (position_width_adjustment_count / u_actor_to_display_ratios.x) * world_pos.w; + world_pos.y -= (position_height_adjustment_count / u_actor_to_display_ratios.y) * world_pos.w; cogl_position_out = world_pos; @@ -573,7 +581,7 @@ export const VirtualMonitorEffect = GObject.registerClass({ this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projection_matrix); this.set_uniform_float(this.get_uniform_location("u_rotation_x_radians"), 1, [this.monitor_wrapping_scheme === 'vertical' ? this.monitor_wrapping_rotation_radians : 0.0]); this.set_uniform_float(this.get_uniform_location("u_rotation_y_radians"), 1, [this.monitor_wrapping_scheme === 'horizontal' ? this.monitor_wrapping_rotation_radians : 0.0]); - this.set_uniform_float(this.get_uniform_location("u_aspect_ratio"), 1, [aspect]); + this.set_uniform_float(this.get_uniform_location("u_display_resolution"), 2, [this.get_actor().width, this.get_actor().height]); this.set_uniform_float(this.get_uniform_location("u_actor_to_display_ratios"), 2, this.actor_to_display_ratios); this._initialized = true; } @@ -726,8 +734,7 @@ export const VirtualMonitorsActor = GObject.registerClass({ display_distance_default: Math.max(this.toggle_display_distance_start, this.toggle_display_distance_end), monitor_wrapping_scheme: 'horizontal', monitor_wrapping_rotation_radians: this._monitorPlacements[index].rotationAngleRadians, - actor_to_display_ratios: actorToDisplayRatios, - monitor_actor: containerActor + actor_to_display_ratios: actorToDisplayRatios }); containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); @@ -735,7 +742,7 @@ export const VirtualMonitorsActor = GObject.registerClass({ this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); - // in addition to rendering distance property in the shader, the parent actor determines overlap based on child ordering + // in addition to rendering distance properly in the shader, the parent actor determines overlap based on child ordering effect.connect('notify::is-closest', ((actor, _pspec) => { if (actor.is_closest) this.set_child_above_sibling(containerActor, null); }).bind(this)); From f2d448e513001db9b963bf67158fad673252f88c Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Tue, 28 Jan 2025 15:53:25 -0800 Subject: [PATCH 13/20] WIP --- gnome/src/monitormanager.js | 1 + gnome/src/virtualmonitorsactor.js | 37 ++++++++++++++++++------------- 2 files changed, 23 insertions(+), 15 deletions(-) diff --git a/gnome/src/monitormanager.js b/gnome/src/monitormanager.js index b0ceea6..3d45e88 100644 --- a/gnome/src/monitormanager.js +++ b/gnome/src/monitormanager.js @@ -58,6 +58,7 @@ function getMonitorConfig(displayConfigProxy, callback) { if (error) { callback(null, `GetResourcesRemote failed: ${error}`); } else { + Globals.logger.log_debug(`monitormanager.js getMonitorConfig GetResources result: ${JSON.stringify(result)}`); const monitors = []; for (let i = 0; i < result[2].length; i++) { const output = result[2][i]; diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index e0bd4fc..4119d56 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -132,21 +132,22 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch const edgeRadius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); cachedMonitorWrap.push({ pixel: 0, radians: -fovHorizontalRadians / 2 }); - monitorDetailsList.forEach(monitorDetails => { + monitorDetailsList.sort((a, b) => a.x - b.x).forEach(monitorDetails => { const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.x, monitorDetails.width); + const monitorCenterRadius = Math.sqrt(Math.pow(edgeRadius, 2) - Math.pow(monitorDetails.width / 2, 2)) monitorPlacements.push({ topLeftNoRotate: [ - centerRadius, + monitorCenterRadius, fovDetails.widthPixels / 2, -(monitorDetails.y - fovDetails.heightPixels / 2) ], center: [ // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians - centerRadius * Math.cos(monitorWrapDetails.center), + monitorCenterRadius * Math.cos(monitorWrapDetails.center), // west is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians - -centerRadius * Math.sin(monitorWrapDetails.center), + -monitorCenterRadius * Math.sin(monitorWrapDetails.center), // up is flat when wrapping horizontally -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) @@ -161,24 +162,25 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch const edgeRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); cachedMonitorWrap.push({ pixel: 0, radians: -fovVerticalRadians / 2 }); - monitorDetailsList.forEach(monitorDetails => { + monitorDetailsList.sort((a, b) => a.y - b.y).forEach(monitorDetails => { const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.y, monitorDetails.height); + const monitorCenterRadius = Math.sqrt(Math.pow(edgeRadius, 2) - Math.pow(monitorDetails.height / 2, 2)) ; monitorPlacements.push({ topLeftNoRotate: [ - centerRadius, + monitorCenterRadius, -(monitorDetails.x - fovDetails.widthPixels / 2), fovDetails.heightPixels / 2 ], center: [ // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians - centerRadius * Math.cos(monitorWrapDetails.center), + monitorCenterRadius * Math.cos(monitorWrapDetails.center), // west is flat when wrapping vertically -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians - -centerRadius * Math.sin(monitorWrapDetails.center) + -monitorCenterRadius * Math.sin(monitorWrapDetails.center) ], rotationAngleRadians: -monitorWrapDetails.center }); @@ -201,6 +203,7 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch }); }); } + Globals.logger.log_debug(`\t\t\tCached monitor wrap: ${JSON.stringify(cachedMonitorWrap)}`); return monitorPlacements; } @@ -421,8 +424,7 @@ export const VirtualMonitorEffect = GObject.registerClass({ uniform vec2 u_actor_to_display_ratios; // constants that help me adjust CoGL vector positions so their components are at the ratios intended, for proper rotation - float cogl_position_width_factor = 29.09; // no idea... - float cogl_z_factor = 55.41; // no idea... + float cogl_position_mystery_factor = 29.09; // no idea... float vectorLength(vec3 v) { return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); @@ -534,12 +536,12 @@ export const VirtualMonitorEffect = GObject.registerClass({ vec4 look_ahead_quaternion = nwuToESU(imuDataToLookAheadQuaternion(u_imu_data, u_look_ahead_ms)); float aspect_ratio = u_display_resolution.x / u_display_resolution.y; - float cogl_position_width = cogl_position_width_factor * aspect_ratio; + float cogl_position_width = cogl_position_mystery_factor * aspect_ratio; float cogl_position_height = cogl_position_width / aspect_ratio; float position_width_adjustment_count = u_actor_to_display_ratios.x - 1.0; float position_height_adjustment_count = u_actor_to_display_ratios.y - 1.0; - world_pos.z = - u_display_distance * cogl_z_factor / u_display_resolution.x; + world_pos.z = - u_display_distance * cogl_position_mystery_factor * 2 / u_display_resolution.x; // if the perspective includes more than just our actor, move vertices towards the center of the perspective so they'll be properly rotated world_pos.x += position_width_adjustment_count * cogl_position_width; @@ -689,13 +691,18 @@ export const VirtualMonitorsActor = GObject.registerClass({ const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); return [vector[0] / length, vector[1] / length, vector[2] / length]; }); + const monitors = Main.layoutManager.monitors; + const minMonitorX = Math.min(...monitors.map(monitor => monitor.x)); + const maxMonitorX = Math.max(...monitors.map(monitor => monitor.x + monitor.width)); + const minMonitorY = Math.min(...monitors.map(monitor => monitor.y)); + const maxMonitorY = Math.max(...monitors.map(monitor => monitor.y + monitor.height)); const actorToDisplayRatios = [ - Main.layoutManager.uiGroup.width / this.width, - Main.layoutManager.uiGroup.height / this.height + (maxMonitorX - minMonitorX) / this.width, + (maxMonitorY - minMonitorY) / this.height ]; - Main.layoutManager.monitors.forEach(((monitor, index) => { + monitors.forEach(((monitor, index) => { // if (index === 0) return; Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); From 1c5cafdb2fa19921902891829071b2abdbade22c Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Wed, 29 Jan 2025 15:40:58 -0800 Subject: [PATCH 14/20] WIP --- gnome/src/extension.js | 5 +- gnome/src/virtualmonitorsactor.js | 108 ++++++++++++++++++------------ 2 files changed, 69 insertions(+), 44 deletions(-) diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 5d3c5e7..d558708 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -248,9 +248,8 @@ export default class BreezyDesktopExtension extends Extension { Globals.data_stream.refresh_data(); this._overlay_content = new VirtualMonitorsActor({ monitors: [], - fov_degrees: 46.0, - width: targetMonitor.width, - height: targetMonitor.height, + fov_degrees: 46.0, + target_monitor: targetMonitor, display_distance: this.settings.get_double('display-distance'), toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index 4119d56..fca3a46 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -132,15 +132,15 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch const edgeRadius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); cachedMonitorWrap.push({ pixel: 0, radians: -fovHorizontalRadians / 2 }); - monitorDetailsList.sort((a, b) => a.x - b.x).forEach(monitorDetails => { + monitorDetailsList.forEach(monitorDetails => { const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.x, monitorDetails.width); const monitorCenterRadius = Math.sqrt(Math.pow(edgeRadius, 2) - Math.pow(monitorDetails.width / 2, 2)) monitorPlacements.push({ topLeftNoRotate: [ monitorCenterRadius, - fovDetails.widthPixels / 2, - -(monitorDetails.y - fovDetails.heightPixels / 2) + 0, + -monitorDetails.y ], center: [ // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians @@ -162,15 +162,15 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch const edgeRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); cachedMonitorWrap.push({ pixel: 0, radians: -fovVerticalRadians / 2 }); - monitorDetailsList.sort((a, b) => a.y - b.y).forEach(monitorDetails => { + monitorDetailsList.forEach(monitorDetails => { const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.y, monitorDetails.height); const monitorCenterRadius = Math.sqrt(Math.pow(edgeRadius, 2) - Math.pow(monitorDetails.height / 2, 2)) ; monitorPlacements.push({ topLeftNoRotate: [ monitorCenterRadius, - -(monitorDetails.x - fovDetails.widthPixels / 2), - fovDetails.heightPixels / 2 + -monitorDetails.x, + 0 ], center: [ // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians @@ -191,8 +191,8 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch monitorPlacements.push({ topLeftNoRotate: [ centerRadius, - -(monitorDetails.x - fovDetails.widthPixels / 2), - -(monitorDetails.y - fovDetails.heightPixels / 2) + -monitorDetails.x, + -monitorDetails.y ], center: [ centerRadius, @@ -304,14 +304,11 @@ export const VirtualMonitorEffect = GObject.registerClass({ 2.5, 1.0 ), - 'display-distance-z': GObject.ParamSpec.double( - 'display-distance-z', - 'Display Distance z-position', - 'Distance of the display from the camera in the z-axis', - GObject.ParamFlags.READWRITE, - 0.0, - 10000.0, - 2900.0 + 'display-position': GObject.ParamSpec.jsobject( + 'display-position', + 'Display Position', + 'Position of the display in COGL (ESU) coordinates', + GObject.ParamFlags.READWRITE ), 'display-distance-default': GObject.ParamSpec.double( 'display-distance-default', @@ -328,6 +325,12 @@ export const VirtualMonitorEffect = GObject.registerClass({ 'Ratios to convert actor coordinates to display coordinates', GObject.ParamFlags.READWRITE ), + 'actor-to-display-offsets': GObject.ParamSpec.jsobject( + 'actor-to-display-offsets', + 'Actor to Display Offsets', + 'Offsets to convert actor coordinates to display coordinates', + GObject.ParamFlags.READWRITE + ), 'is-closest': GObject.ParamSpec.boolean( 'is-closest', 'Is Closest', @@ -415,16 +418,17 @@ export const VirtualMonitorEffect = GObject.registerClass({ uniform mat4 u_imu_data; uniform float u_look_ahead_ms; uniform mat4 u_projection_matrix; - uniform float u_display_distance; + uniform vec3 u_display_position; uniform float u_rotation_x_radians; uniform float u_rotation_y_radians; uniform vec2 u_display_resolution; - // for some reason the vector positions are relative to the width and height of the uiGroup actor + // vector positions are relative to the width and height of the entire stage uniform vec2 u_actor_to_display_ratios; + uniform vec2 u_actor_to_display_offsets; - // constants that help me adjust CoGL vector positions so their components are at the ratios intended, for proper rotation - float cogl_position_mystery_factor = 29.09; // no idea... + // discovered through trial and error, no idea the significance + float cogl_position_mystery_factor = 29.09; float vectorLength(vec3 v) { return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); @@ -538,14 +542,14 @@ export const VirtualMonitorEffect = GObject.registerClass({ float cogl_position_width = cogl_position_mystery_factor * aspect_ratio; float cogl_position_height = cogl_position_width / aspect_ratio; - float position_width_adjustment_count = u_actor_to_display_ratios.x - 1.0; - float position_height_adjustment_count = u_actor_to_display_ratios.y - 1.0; - world_pos.z = - u_display_distance * cogl_position_mystery_factor * 2 / u_display_resolution.x; + world_pos.x -= u_display_position.x * cogl_position_width * 2 / u_display_resolution.x; + world_pos.y -= u_display_position.y * cogl_position_height * 2 / u_display_resolution.y; + world_pos.z = u_display_position.z * cogl_position_mystery_factor * 2 / u_display_resolution.x; // if the perspective includes more than just our actor, move vertices towards the center of the perspective so they'll be properly rotated - world_pos.x += position_width_adjustment_count * cogl_position_width; - world_pos.y += position_height_adjustment_count * cogl_position_height; + world_pos.x += u_actor_to_display_offsets.x * cogl_position_width; + world_pos.y += u_actor_to_display_offsets.y * cogl_position_height; world_pos.z *= aspect_ratio; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); @@ -560,8 +564,8 @@ export const VirtualMonitorEffect = GObject.registerClass({ // if the perspective includes more than just our actor, move the vertices back to just the area we can see. // this needs to be done after the projection matrix multiplication so it will be projected as if centered in our vision - world_pos.x -= (position_width_adjustment_count / u_actor_to_display_ratios.x) * world_pos.w; - world_pos.y -= (position_height_adjustment_count / u_actor_to_display_ratios.y) * world_pos.w; + world_pos.x -= (u_actor_to_display_offsets.x / u_actor_to_display_ratios.x) * world_pos.w; + world_pos.y -= (u_actor_to_display_offsets.y / u_actor_to_display_ratios.y) * world_pos.w; cogl_position_out = world_pos; @@ -585,12 +589,12 @@ export const VirtualMonitorEffect = GObject.registerClass({ this.set_uniform_float(this.get_uniform_location("u_rotation_y_radians"), 1, [this.monitor_wrapping_scheme === 'horizontal' ? this.monitor_wrapping_rotation_radians : 0.0]); this.set_uniform_float(this.get_uniform_location("u_display_resolution"), 2, [this.get_actor().width, this.get_actor().height]); this.set_uniform_float(this.get_uniform_location("u_actor_to_display_ratios"), 2, this.actor_to_display_ratios); + this.set_uniform_float(this.get_uniform_location("u_actor_to_display_offsets"), 2, this.actor_to_display_offsets); this._initialized = true; } this.set_uniform_float(this.get_uniform_location('u_look_ahead_ms'), 1, [lookAheadMS(this.imu_snapshots.timestamp_ms, 0)]); - // Globals.logger.log(`\t\t\tDisplay distance: ${this._current_display_distance * this.display_distance_z}`); - this.set_uniform_float(this.get_uniform_location("u_display_distance"), 1, [this._current_display_distance * this.display_distance_z]); + this.set_uniform_float(this.get_uniform_location("u_display_position"), 3, [this.display_position[0], this.display_position[1], this._current_display_distance * this.display_position[2]]); this.set_uniform_matrix(this.get_uniform_location("u_imu_data"), false, 4, this.imu_snapshots.imu_data); this.get_pipeline().set_layer_filters( @@ -611,6 +615,12 @@ export const VirtualMonitorsActor = GObject.registerClass({ 'Array of monitor indexes', GObject.ParamFlags.READWRITE ), + 'target-monitor': GObject.ParamSpec.jsobject( + 'target-monitor', + 'Target Monitor', + 'Details about the monitor being used as a viewport', + GObject.ParamFlags.READWRITE + ), 'imu-snapshots': GObject.ParamSpec.jsobject( 'imu-snapshots', 'IMU Snapshots', @@ -669,6 +679,13 @@ export const VirtualMonitorsActor = GObject.registerClass({ ), } }, class VirtualMonitorsActor extends Clutter.Actor { + constructor(params = {}) { + super(params); + + this.width = this.target_monitor.width; + this.height = this.target_monitor.height; + } + renderMonitors() { this._monitorPlacements = monitorsToPlacements( { @@ -697,10 +714,22 @@ export const VirtualMonitorsActor = GObject.registerClass({ const minMonitorY = Math.min(...monitors.map(monitor => monitor.y)); const maxMonitorY = Math.max(...monitors.map(monitor => monitor.y + monitor.height)); + const displayWidth = maxMonitorX - minMonitorX; + const displayHeight = maxMonitorY - minMonitorY; const actorToDisplayRatios = [ - (maxMonitorX - minMonitorX) / this.width, - (maxMonitorY - minMonitorY) / this.height + displayWidth / this.width, + displayHeight / this.height ]; + + // how far this viewport actor's center is from the center of the whole stage + const actorMidX = this.target_monitor.x + this.width / 2; + const actorMidY = this.target_monitor.y + this.height / 2; + const actorToDisplayOffsets = [ + (displayWidth / 2 - (actorMidX - minMonitorX)) * 2 / this.width, + (displayHeight / 2 - (actorMidY - minMonitorY)) * 2 / this.height + ]; + + Globals.logger.log_debug(`\t\t\tActor to display ratios: ${actorToDisplayRatios}, offsets: ${actorToDisplayOffsets}`); monitors.forEach(((monitor, index) => { // if (index === 0) return; @@ -712,12 +741,8 @@ export const VirtualMonitorsActor = GObject.registerClass({ // actor coordinates are east-up-south const containerActor = new Clutter.Actor({ - x: -noRotationVector[1], - y: -noRotationVector[2], - // ideally we would do this, but it causes blur, so we instead set the distance in the shader - // 'z-position': -noRotationVector[0], - width: monitor.width, - height: monitor.height, + width: this.width, + height: this.height, reactive: false, }); @@ -725,8 +750,8 @@ export const VirtualMonitorsActor = GObject.registerClass({ const monitorClone = new Clutter.Clone({ source: Main.layoutManager.uiGroup, reactive: false, - x: -containerActor.x - monitor.x, - y: -containerActor.y - monitor.y + x: -monitor.x, + y: -monitor.y }); monitorClone.set_clip(monitor.x, monitor.y, monitor.width, monitor.height); @@ -736,12 +761,13 @@ export const VirtualMonitorsActor = GObject.registerClass({ imu_snapshots: this.imu_snapshots, fov_degrees: this.fov_degrees, monitor_index: index, - display_distance_z: noRotationVector[0], + display_position: [-noRotationVector[1], -noRotationVector[2], -noRotationVector[0]], display_distance: this.display_distance, display_distance_default: Math.max(this.toggle_display_distance_start, this.toggle_display_distance_end), monitor_wrapping_scheme: 'horizontal', monitor_wrapping_rotation_radians: this._monitorPlacements[index].rotationAngleRadians, - actor_to_display_ratios: actorToDisplayRatios + actor_to_display_ratios: actorToDisplayRatios, + actor_to_display_offsets: actorToDisplayOffsets }); containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); From 21e448833aba9a288246edd56e6b42fb9419b7bd Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Thu, 30 Jan 2025 11:19:50 -0800 Subject: [PATCH 15/20] Fix scaling and perspective adjustments when monitors are vertically stacked --- gnome/src/virtualmonitorsactor.js | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index fca3a46..af65cd8 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -543,26 +543,26 @@ export const VirtualMonitorEffect = GObject.registerClass({ float cogl_position_width = cogl_position_mystery_factor * aspect_ratio; float cogl_position_height = cogl_position_width / aspect_ratio; - world_pos.x -= u_display_position.x * cogl_position_width * 2 / u_display_resolution.x; - world_pos.y -= u_display_position.y * cogl_position_height * 2 / u_display_resolution.y; + world_pos.x -= u_display_position.x * cogl_position_width * 2 / u_display_resolution.x / u_actor_to_display_ratios.y; + world_pos.y -= u_display_position.y * cogl_position_height * 2 / u_display_resolution.y / u_actor_to_display_ratios.y; world_pos.z = u_display_position.z * cogl_position_mystery_factor * 2 / u_display_resolution.x; - // if the perspective includes more than just our actor, move vertices towards the center of the perspective so they'll be properly rotated - world_pos.x += u_actor_to_display_offsets.x * cogl_position_width; - world_pos.y += u_actor_to_display_offsets.y * cogl_position_height; + // if the perspective includes more than just our viewport actor, move vertices towards the center of the perspective so they'll be properly rotated + world_pos.x += u_actor_to_display_offsets.x * cogl_position_width / u_actor_to_display_ratios.y; + world_pos.y += u_actor_to_display_offsets.y * cogl_position_height / u_actor_to_display_ratios.y; - world_pos.z *= aspect_ratio; + world_pos.z *= aspect_ratio / u_actor_to_display_ratios.y; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); world_pos = applyYRotationToVector(world_pos, u_rotation_y_radians); world_pos = applyQuaternionToVector(world_pos, quatConjugate(look_ahead_quaternion)); - world_pos.z /= aspect_ratio; + world_pos.z /= aspect_ratio / u_actor_to_display_ratios.y; - world_pos.x /= u_actor_to_display_ratios.x; - world_pos.y /= u_actor_to_display_ratios.y; + world_pos.x /= u_actor_to_display_ratios.x / u_actor_to_display_ratios.y; + // world_pos.y /= u_actor_to_display_ratios.y; world_pos = u_projection_matrix * world_pos; - // if the perspective includes more than just our actor, move the vertices back to just the area we can see. + // if the perspective includes more than just our viewport actor, move the vertices back to just the area we can see. // this needs to be done after the projection matrix multiplication so it will be projected as if centered in our vision world_pos.x -= (u_actor_to_display_offsets.x / u_actor_to_display_ratios.x) * world_pos.w; world_pos.y -= (u_actor_to_display_offsets.y / u_actor_to_display_ratios.y) * world_pos.w; From b55a5b0c62ba76fdc6d703e7c08ed8313a4fcc22 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Thu, 30 Jan 2025 12:26:19 -0800 Subject: [PATCH 16/20] Fix centering of different sized monitors --- gnome/src/virtualmonitorsactor.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index af65cd8..9dda1be 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -139,7 +139,7 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch monitorPlacements.push({ topLeftNoRotate: [ monitorCenterRadius, - 0, + -(monitorDetails.width - fovDetails.widthPixels) / 2, -monitorDetails.y ], center: [ @@ -170,7 +170,7 @@ function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingSch topLeftNoRotate: [ monitorCenterRadius, -monitorDetails.x, - 0 + -(monitorDetails.height - fovDetails.heightPixels) / 2 ], center: [ // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians From 39460a521fdb71d63d4846fe6c18053e3cc6869f Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Fri, 31 Jan 2025 14:10:55 -0800 Subject: [PATCH 17/20] Fix issue with vertical stretching --- gnome/src/virtualmonitorsactor.js | 5 ++--- ui/src/virtualdisplay.py | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index 9dda1be..29a3f90 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -549,7 +549,7 @@ export const VirtualMonitorEffect = GObject.registerClass({ // if the perspective includes more than just our viewport actor, move vertices towards the center of the perspective so they'll be properly rotated world_pos.x += u_actor_to_display_offsets.x * cogl_position_width / u_actor_to_display_ratios.y; - world_pos.y += u_actor_to_display_offsets.y * cogl_position_height / u_actor_to_display_ratios.y; + world_pos.y -= u_actor_to_display_offsets.y * cogl_position_height / u_actor_to_display_ratios.y; world_pos.z *= aspect_ratio / u_actor_to_display_ratios.y; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); @@ -558,14 +558,13 @@ export const VirtualMonitorEffect = GObject.registerClass({ world_pos.z /= aspect_ratio / u_actor_to_display_ratios.y; world_pos.x /= u_actor_to_display_ratios.x / u_actor_to_display_ratios.y; - // world_pos.y /= u_actor_to_display_ratios.y; world_pos = u_projection_matrix * world_pos; // if the perspective includes more than just our viewport actor, move the vertices back to just the area we can see. // this needs to be done after the projection matrix multiplication so it will be projected as if centered in our vision world_pos.x -= (u_actor_to_display_offsets.x / u_actor_to_display_ratios.x) * world_pos.w; - world_pos.y -= (u_actor_to_display_offsets.y / u_actor_to_display_ratios.y) * world_pos.w; + world_pos.y += (u_actor_to_display_offsets.y / u_actor_to_display_ratios.y) * world_pos.w; cogl_position_out = world_pos; diff --git a/ui/src/virtualdisplay.py b/ui/src/virtualdisplay.py index 3ef9969..1d09ab3 100644 --- a/ui/src/virtualdisplay.py +++ b/ui/src/virtualdisplay.py @@ -13,7 +13,7 @@ logger = logging.getLogger('breezy_ui') screen_cast_iface = 'org.gnome.Mutter.ScreenCast' screen_cast_session_iface = 'org.gnome.Mutter.ScreenCast.Session' screen_cast_stream_iface = 'org.gnome.Mutter.ScreenCast.Session' -gst_pipeline_format = "pipewiresrc path=%u ! video/x-raw,max-framerate=120/1,width=%d,height=%d ! fakesink sync=false" +gst_pipeline_format = "pipewiresrc path=%u ! video/x-raw,max-framerate=60/1,width=%d,height=%d ! fakesink sync=false" def _screen_cast_session(): From 3f73d2148d74a5f59db63164726028dac9f3865f Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Fri, 31 Jan 2025 16:11:01 -0800 Subject: [PATCH 18/20] Simplify width/height usage, cap forced redraws --- gnome/src/virtualmonitorsactor.js | 28 ++++++++++++++++++++-------- 1 file changed, 20 insertions(+), 8 deletions(-) diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index 29a3f90..2eab655 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -428,7 +428,7 @@ export const VirtualMonitorEffect = GObject.registerClass({ uniform vec2 u_actor_to_display_offsets; // discovered through trial and error, no idea the significance - float cogl_position_mystery_factor = 29.09; + float cogl_position_mystery_factor = 29.09 * 2; float vectorLength(vec3 v) { return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); @@ -540,16 +540,16 @@ export const VirtualMonitorEffect = GObject.registerClass({ vec4 look_ahead_quaternion = nwuToESU(imuDataToLookAheadQuaternion(u_imu_data, u_look_ahead_ms)); float aspect_ratio = u_display_resolution.x / u_display_resolution.y; - float cogl_position_width = cogl_position_mystery_factor * aspect_ratio; + float cogl_position_width = cogl_position_mystery_factor * aspect_ratio / u_actor_to_display_ratios.y; float cogl_position_height = cogl_position_width / aspect_ratio; - world_pos.x -= u_display_position.x * cogl_position_width * 2 / u_display_resolution.x / u_actor_to_display_ratios.y; - world_pos.y -= u_display_position.y * cogl_position_height * 2 / u_display_resolution.y / u_actor_to_display_ratios.y; - world_pos.z = u_display_position.z * cogl_position_mystery_factor * 2 / u_display_resolution.x; + world_pos.x -= u_display_position.x * cogl_position_width / u_display_resolution.x; + world_pos.y -= u_display_position.y * cogl_position_height/ u_display_resolution.y; + world_pos.z = u_display_position.z * cogl_position_mystery_factor / u_display_resolution.x; // if the perspective includes more than just our viewport actor, move vertices towards the center of the perspective so they'll be properly rotated - world_pos.x += u_actor_to_display_offsets.x * cogl_position_width / u_actor_to_display_ratios.y; - world_pos.y -= u_actor_to_display_offsets.y * cogl_position_height / u_actor_to_display_ratios.y; + world_pos.x += u_actor_to_display_offsets.x * cogl_position_width / 2; + world_pos.y -= u_actor_to_display_offsets.y * cogl_position_height / 2; world_pos.z *= aspect_ratio / u_actor_to_display_ratios.y; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); @@ -557,7 +557,7 @@ export const VirtualMonitorEffect = GObject.registerClass({ world_pos = applyQuaternionToVector(world_pos, quatConjugate(look_ahead_quaternion)); world_pos.z /= aspect_ratio / u_actor_to_display_ratios.y; - world_pos.x /= u_actor_to_display_ratios.x / u_actor_to_display_ratios.y; + world_pos.x *= u_actor_to_display_ratios.y / u_actor_to_display_ratios.x; world_pos = u_projection_matrix * world_pos; @@ -676,6 +676,13 @@ export const VirtualMonitorsActor = GObject.registerClass({ 2.5, 1.05 ), + 'target-framerate': GObject.ParamSpec.double( + 'target-framerate', + 'Target Framerate', + 'Target framerate for the virtual monitors', + GObject.ParamFlags.READWRITE, + 1.0, 120.0, 60.0 + ) } }, class VirtualMonitorsActor extends Clutter.Actor { constructor(params = {}) { @@ -683,6 +690,7 @@ export const VirtualMonitorsActor = GObject.registerClass({ this.width = this.target_monitor.width; this.height = this.target_monitor.height; + this._frametime_ms = Math.floor(1000 / (this.target_framerate ?? 60.0)); } renderMonitors() { @@ -799,9 +807,13 @@ export const VirtualMonitorsActor = GObject.registerClass({ this._redraw_timeline = Clutter.Timeline.new_for_actor(this, 1000); this._redraw_timeline.connect('new-frame', (() => { + // let's try to cap the forced redraw rate + if (this._last_redraw !== undefined && Date.now() - this._last_redraw < this._frametime_ms) return; + Globals.data_stream.refresh_data(); this.imu_snapshots = Globals.data_stream.imu_snapshots; this.queue_redraw(); + this._last_redraw = Date.now(); }).bind(this)); this._redraw_timeline.set_repeat_count(-1); this._redraw_timeline.start(); From 0a75f2f710299d94f022786dacd5e6f84ba1f527 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Mon, 3 Feb 2025 13:19:12 -0800 Subject: [PATCH 19/20] Incorporate the old look-ahead logic, no more jitters --- gnome/src/virtualmonitorsactor.js | 98 ++++++------------------------- 1 file changed, 18 insertions(+), 80 deletions(-) diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index 2eab655..6d060b9 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -430,88 +430,10 @@ export const VirtualMonitorEffect = GObject.registerClass({ // discovered through trial and error, no idea the significance float cogl_position_mystery_factor = 29.09 * 2; - float vectorLength(vec3 v) { - return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); - } - - float quaternionLength(vec4 q) { - return sqrt(q.x * q.x + q.y * q.y + q.z * q.z + q.w * q.w); - } - - vec4 quatMul(vec4 q1, vec4 q2) { - return vec4( - q1.w * q2.x + q1.x * q2.w + q1.y * q2.z - q1.z * q2.y, // x - q1.w * q2.y - q1.x * q2.z + q1.y * q2.w + q1.z * q2.x, // y - q1.w * q2.z + q1.x * q2.y - q1.y * q2.x + q1.z * q2.w, // z - q1.w * q2.w - q1.x * q2.x - q1.y * q2.y - q1.z * q2.z // w - ); - } - vec4 quatConjugate(vec4 q) { return vec4(-q.xyz, q.w); } - vec4 quatExp(vec4 q) { - float vLength = vectorLength(q.xyz); - float expW = exp(q.w); - - if (vLength < 0.000001) { - return vec4(0.0, 0.0, 0.0, expW); - } - - float scale = expW * sin(vLength) / vLength; - return vec4(q.xyz * scale, expW * cos(vLength)); - } - - vec4 quatLog(vec4 q) { - float qLength = quaternionLength(q); - float vLength = vectorLength(q.xyz); - - if (vLength < 0.000001) { - return vec4(0.0, 0.0, 0.0, log(qLength)); - } - - float scale = acos(clamp(q.w / qLength, -1.0, 1.0)) / vLength; - return vec4(q.xyz * scale, log(qLength)); - } - - vec4 computeQuaternionVelocity(vec4 q1, vec4 q2, float milliseconds) { - // Normalize input quaternions - q1 = normalize(q1); - q2 = normalize(q2); - - // Compute difference quaternion (q2 * q1^-1) - vec4 diffQ = quatMul(q2, quatConjugate(q1)); - - // Ensure we take the shortest path - if (diffQ.w < 0.0) { - diffQ = -diffQ; - } - - // Take the log and scale by time - return quatLog(diffQ) / milliseconds; - } - - vec4 extrapolateRotation(vec4 initialQuat, vec4 velocity, float deltaTimeMs) { - // Scale velocity by time - vec4 scaledVelocity = velocity * deltaTimeMs; - - // Compute the exponential - vec4 deltaRotation = quatExp(scaledVelocity); - - // Apply to initial quaternion - return normalize(quatMul(deltaRotation, initialQuat)); - } - - vec4 imuDataToLookAheadQuaternion(mat4 imuData, float lookAheadMS) { - // last row of matrix contains imu timestamps, subtract the second column from the first - float imuDeltaTime = imuData[3][0] - imuData[3][1]; - - // rotation per ms - vec4 velocity = computeQuaternionVelocity(imuData[0], imuData[1], imuDeltaTime); - return extrapolateRotation(imuData[0], velocity, lookAheadMS); - } - vec4 applyQuaternionToVector(vec4 v, vec4 q) { vec3 t = 2.0 * cross(q.xyz, v.xyz); vec3 rotated = v.xyz + q.w * t + cross(q.xyz, t); @@ -533,11 +455,22 @@ export const VirtualMonitorEffect = GObject.registerClass({ vec4 nwuToESU(vec4 v) { return vec4(-v.y, v.z, -v.x, v.w); } + + // returns the rate of change between the two vectors, in same time units as delta_time + // e.g. if delta_time is in ms, then the rate of change is "per ms" + vec3 rateOfChange(vec3 v1, vec3 v2, float delta_time) { + return (v1-v2) / delta_time; + } + + // attempt to figure out where the current position should be based on previous position and velocity. + // velocity and time values should use the same time units (secs, ms, etc...) + vec3 applyLookAhead(vec3 position, vec3 velocity, float t) { + return position + velocity * t; + } `; const main = ` vec4 world_pos = cogl_position_in; - vec4 look_ahead_quaternion = nwuToESU(imuDataToLookAheadQuaternion(u_imu_data, u_look_ahead_ms)); float aspect_ratio = u_display_resolution.x / u_display_resolution.y; float cogl_position_width = cogl_position_mystery_factor * aspect_ratio / u_actor_to_display_ratios.y; @@ -554,7 +487,12 @@ export const VirtualMonitorEffect = GObject.registerClass({ world_pos.z *= aspect_ratio / u_actor_to_display_ratios.y; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); world_pos = applyYRotationToVector(world_pos, u_rotation_y_radians); - world_pos = applyQuaternionToVector(world_pos, quatConjugate(look_ahead_quaternion)); + + vec3 rotated_vector_t0 = applyQuaternionToVector(world_pos, nwuToESU(quatConjugate(u_imu_data[0]))).xyz; + vec3 rotated_vector_t1 = applyQuaternionToVector(world_pos, nwuToESU(quatConjugate(u_imu_data[1]))).xyz; + float delta_time_t0 = u_imu_data[3][0] - u_imu_data[3][1]; + vec3 velocity_t0 = rateOfChange(rotated_vector_t0, rotated_vector_t1, delta_time_t0); + world_pos = vec4(applyLookAhead(rotated_vector_t0, velocity_t0, u_look_ahead_ms), world_pos.w); world_pos.z /= aspect_ratio / u_actor_to_display_ratios.y; world_pos.x *= u_actor_to_display_ratios.y / u_actor_to_display_ratios.x; From db3f59f7e739e4e1bfe9d5a750355e2edda6f8f2 Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Mon, 3 Feb 2025 16:29:04 -0800 Subject: [PATCH 20/20] Update monitor manager integration to allow for virtual monitor detection, update effect to only show virtual monitors --- gnome/src/extension.js | 28 ++++++++++++++++-- gnome/src/monitormanager.js | 47 +++++++++++++++---------------- gnome/src/virtualmonitorsactor.js | 16 +++++++---- 3 files changed, 59 insertions(+), 32 deletions(-) diff --git a/gnome/src/extension.js b/gnome/src/extension.js index d558708..99b0b41 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -17,6 +17,7 @@ import {Extension} from 'resource:///org/gnome/shell/extensions/extension.js'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; const NESTED_MONITOR_PRODUCT = 'MetaMonitor'; +const VIRTUAL_MONITOR_PRODUCT = 'Virtual remote monitor'; const SUPPORTED_MONITOR_PRODUCTS = [ 'VITURE', 'nreal air', @@ -137,7 +138,29 @@ export default class BreezyDesktopExtension extends Extension { }).bind(this)); } + _find_virtual_monitors() { + try { + Globals.logger.log_debug('BreezyDesktopExtension _find_virtual_monitors'); + const virtual_monitors = this._monitor_manager.getMonitorPropertiesList()?.filter( + monitor => monitor && monitor.product === VIRTUAL_MONITOR_PRODUCT); + if (virtual_monitors.length > 0) { + Globals.logger.log(`Found ${virtual_monitors.length} virtual monitors`); + return virtual_monitors.map(monitor => { + return this._monitor_manager.getMonitors()[monitor.index]; + }); + } + + Globals.logger.log_debug('BreezyDesktopExtension _find_virtual_monitors - No virtual monitors found'); + } catch (e) { + Globals.logger.log(`[ERROR] BreezyDesktopExtension _find_virtual_monitors ${e.message}\n${e.stack}`) + } + + return []; + } + _find_supported_monitor() { + if (!this._monitor_manager.getMonitorPropertiesList()) return null; + try { Globals.logger.log_debug('BreezyDesktopExtension _find_supported_monitor'); const target_monitor = this._monitor_manager.getMonitorPropertiesList()?.find( @@ -149,7 +172,8 @@ export default class BreezyDesktopExtension extends Extension { monitor: this._monitor_manager.getMonitors()[target_monitor.index], connector: target_monitor.connector, refreshRate: target_monitor.refreshRate, - is_dummy: target_monitor.product === NESTED_MONITOR_PRODUCT + is_dummy: target_monitor.product === NESTED_MONITOR_PRODUCT, + is_virtual: target_monitor.product === VIRTUAL_MONITOR_PRODUCT }; } @@ -247,7 +271,7 @@ export default class BreezyDesktopExtension extends Extension { // const textureSourceActor = Main.layoutManager.uiGroup; Globals.data_stream.refresh_data(); this._overlay_content = new VirtualMonitorsActor({ - monitors: [], + monitors: this._find_virtual_monitors(), fov_degrees: 46.0, target_monitor: targetMonitor, display_distance: this.settings.get_double('display-distance'), diff --git a/gnome/src/monitormanager.js b/gnome/src/monitormanager.js index 3d45e88..84d0aec 100644 --- a/gnome/src/monitormanager.js +++ b/gnome/src/monitormanager.js @@ -54,34 +54,28 @@ export function newDisplayConfig(extPath, callback) { } function getMonitorConfig(displayConfigProxy, callback) { - displayConfigProxy.GetResourcesRemote((result, error) => { + displayConfigProxy.GetCurrentStateRemote((result, error) => { if (error) { - callback(null, `GetResourcesRemote failed: ${error}`); + callback(null, `GetCurrentState failed: ${error}`); } else { - Globals.logger.log_debug(`monitormanager.js getMonitorConfig GetResources result: ${JSON.stringify(result)}`); - const monitors = []; - for (let i = 0; i < result[2].length; i++) { - const output = result[2][i]; - if (output.length <= 7) { - callback(null, 'Cannot get DisplayConfig: No properties on output #' + i); - return; - } - const props = output[7]; - const displayName = props['display-name'].get_string()[0]; - const connectorName = output[4]; - if (!displayName || displayName == '') { - const displayName = 'Monitor on output ' + connectorName; - } - const vendor = props['vendor'].get_string()[0]; - const product = props['product'].get_string()[0]; - const serial = props['serial'].get_string()[0]; + Globals.logger.log_debug(`monitormanager.js getMonitorConfig GetCurrentState result: ${JSON.stringify(result)}`); + + const allMonitors = []; + const [serial, monitors, logicalMonitors, properties] = result; + for (let monitor of monitors) { + const [details, modes, monProperties] = monitor; + const [connector, vendor, product, monitorSerial] = details; + const displayName = monProperties['display-name'].get_string()[0]; - // grab refresh rate from the modes array - const refreshRate = result[3][i][4]; - - monitors.push([displayName, connectorName, vendor, product, serial, refreshRate]); + for (let mode of modes) { + const [modeId, width, height, refreshRate, preferredScale, supportedScales, modeProperites] = mode; + const isCurrent = !!modeProperites['is-current']; + if (isCurrent) { + allMonitors.push([displayName, connector, vendor, product, serial, refreshRate]); + } + } } - callback(monitors, null); + callback(allMonitors, null); } }); } @@ -289,6 +283,7 @@ export const MonitorManager = GObject.registerClass({ // help prevent certain actions from taking place multiple times in the event of rapid monitor updates this._asyncRequestsInFlight = 0; this._configCheckRequestsCount = 0; + this._enabled = false; } enable() { @@ -303,12 +298,14 @@ export const MonitorManager = GObject.registerClass({ }).bind(this)); this._monitorsChangedConnection = Main.layoutManager.connect('monitors-changed', this._on_monitors_change.bind(this)); + this._enabled = true; } disable() { Globals.logger.log_debug('MonitorManager disable'); Main.layoutManager.disconnect(this._monitorsChangedConnection); + this._enabled = false; this._monitorsChangedConnection = null; this._displayConfigProxy = null; this._backendManager = null; @@ -384,6 +381,8 @@ export const MonitorManager = GObject.registerClass({ } _on_monitors_change() { + if (!this._enabled) return; + Globals.logger.log_debug('MonitorManager _on_monitors_change'); if (this._displayConfigProxy == null) { return; diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js index 6d060b9..f66bdf9 100644 --- a/gnome/src/virtualmonitorsactor.js +++ b/gnome/src/virtualmonitorsactor.js @@ -629,6 +629,10 @@ export const VirtualMonitorsActor = GObject.registerClass({ this.width = this.target_monitor.width; this.height = this.target_monitor.height; this._frametime_ms = Math.floor(1000 / (this.target_framerate ?? 60.0)); + this._all_monitors = [ + this.target_monitor, + ...this.monitors + ]; } renderMonitors() { @@ -638,7 +642,7 @@ export const VirtualMonitorsActor = GObject.registerClass({ widthPixels: this.width, heightPixels: this.height }, - Main.layoutManager.monitors.map(monitor => ({ + this._all_monitors.map(monitor => ({ x: monitor.x, y: monitor.y, width: monitor.width, @@ -653,14 +657,14 @@ export const VirtualMonitorsActor = GObject.registerClass({ const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); return [vector[0] / length, vector[1] / length, vector[2] / length]; }); - const monitors = Main.layoutManager.monitors; + const monitors = this._all_monitors; const minMonitorX = Math.min(...monitors.map(monitor => monitor.x)); const maxMonitorX = Math.max(...monitors.map(monitor => monitor.x + monitor.width)); const minMonitorY = Math.min(...monitors.map(monitor => monitor.y)); const maxMonitorY = Math.max(...monitors.map(monitor => monitor.y + monitor.height)); - const displayWidth = maxMonitorX - minMonitorX; - const displayHeight = maxMonitorY - minMonitorY; + const displayWidth = global.stage.width; + const displayHeight = global.stage.height; const actorToDisplayRatios = [ displayWidth / this.width, displayHeight / this.height @@ -670,8 +674,8 @@ export const VirtualMonitorsActor = GObject.registerClass({ const actorMidX = this.target_monitor.x + this.width / 2; const actorMidY = this.target_monitor.y + this.height / 2; const actorToDisplayOffsets = [ - (displayWidth / 2 - (actorMidX - minMonitorX)) * 2 / this.width, - (displayHeight / 2 - (actorMidY - minMonitorY)) * 2 / this.height + (displayWidth / 2 - (actorMidX - global.stage.x)) * 2 / this.width, + (displayHeight / 2 - (actorMidY - global.stage.y)) * 2 / this.height ]; Globals.logger.log_debug(`\t\t\tActor to display ratios: ${actorToDisplayRatios}, offsets: ${actorToDisplayOffsets}`);