diff --git a/gnome/bin/dev/use_local_extension.sh b/gnome/bin/dev/use_local_extension.sh index f959f13..12465b3 100755 --- a/gnome/bin/dev/use_local_extension.sh +++ b/gnome/bin/dev/use_local_extension.sh @@ -7,6 +7,7 @@ if [ -z "$XDG_DATA_HOME" ]; then XDG_DATA_HOME="$USER_HOME/.local/share" fi DATA_DIR="$XDG_DATA_HOME/breezy_gnome" +mkdir -p $DATA_DIR # if $XDG_DATA_HOME/gnome-shell/extensions/breezydesktop@xronlinux.com exists extension_path="$XDG_DATA_HOME/gnome-shell/extensions/breezydesktop@xronlinux.com" diff --git a/gnome/src/customeffect.js b/gnome/src/customeffect.js new file mode 100644 index 0000000..0df7fd6 --- /dev/null +++ b/gnome/src/customeffect.js @@ -0,0 +1,71 @@ +const { Clutter, GLib, GObject } = imports.gi; + +export const CustomEffect = GObject.registerClass({ + Properties: { + 'fov-degrees': GObject.ParamSpec.double( + 'fov-degrees', + 'FOV Degrees', + 'Diagonal field-of-view in degrees', + GObject.ParamFlags.READWRITE, + 1.0, + 179.0, + 60.0 + ) + } +}, class Customffect extends Clutter.ShaderEffect { + _init(params = {}) { + super._init(params); + + this.fov_degrees = params['fov-degrees'] || 60.0; + this.connect('notify::fov-degrees', this._updateMatrices.bind(this)); + + // Set up the vertex shader + this.set_shader_source(Clutter.ShaderType.VERTEX, ` + uniform mat4 viewMatrix; + uniform mat4 projectionMatrix; + uniform vec4 quaternion; + + vec3 applyQuaternionToVector(vec3 v, vec4 q) { + return v + 2.0 * cross(q.xyz, cross(q.xyz, v) + q.w * v); + } + + void main() { + // First apply the view matrix to position the vertex in camera space + vec4 viewPosition = viewMatrix * vec4(gl_Vertex.xyz, 1.0); + // Then apply the quaternion rotation + vec3 transformedPosition = applyQuaternionToVector(viewPosition.xyz, quaternion); + // Finally apply the projection matrix + gl_Position = projectionMatrix * vec4(transformedPosition, 1.0); + gl_TexCoord[0] = gl_MultiTexCoord0; + } + `); + + // Initialize with the current matrices + this._updateMatrices(); + } + + _updateMatrices() { + let aspect = this.get_parent().width / this.get_parent().height; + let fov = this.fov_degrees * Math.PI / 180.0; + let near = 0.1; + let far = 100.0; + let top = Math.tan(fov / 2.0) * near; + let bottom = -top; + let right = top * aspect; + let left = -right; + + let projectionMatrix = GLib.Matrix.init_frustum(left, right, bottom, top, near, far); + let viewMatrix = GLib.Matrix.init_identity(); + + // Calculate the appropriate Z-distance based on FOV + let distance = -1.0 / Math.tan(fov / 2.0); + viewMatrix = viewMatrix.translate(0, 0, distance); + + this.set_shader_uniform_value('projectionMatrix', new Clutter.ShaderValue({matrix: projectionMatrix})); + this.set_shader_uniform_value('viewMatrix', new Clutter.ShaderValue({matrix: viewMatrix})); + } + + set_quaternion(quat) { + this.set_shader_uniform_value('quaternion', new Clutter.ShaderValue({vector4: quat})); + } +}); diff --git a/gnome/src/devicedatastream.js b/gnome/src/devicedatastream.js new file mode 100644 index 0000000..ed7ac0d --- /dev/null +++ b/gnome/src/devicedatastream.js @@ -0,0 +1,172 @@ +import Gio from 'gi://Gio'; +import GObject from 'gi://GObject'; + +import Globals from './globals.js'; +import { + dataViewEnd, + dataViewUint8, + dataViewBigUint, + dataViewUint32Array, + dataViewUint8Array, + dataViewFloat, + dataViewFloatArray, + BOOL_SIZE, + FLOAT_SIZE, + UINT_SIZE, + UINT8_SIZE +} from "./ipc.js"; +import { isValidKeepAlive, getEpochSec, toSec } from "./time.js"; + +const IPC_FILE_PATH = "/dev/shm/breezy_desktop_imu"; +const KEEPALIVE_REFRESH_INTERVAL_SEC = 1; + +// the driver should be using the same data layout version +const DATA_LAYOUT_VERSION = 3; + +// DataView info: [offset, size, count] +const VERSION = [0, UINT8_SIZE, 1]; +const ENABLED = [dataViewEnd(VERSION), BOOL_SIZE, 1]; +const LOOK_AHEAD_CFG = [dataViewEnd(ENABLED), FLOAT_SIZE, 4]; +const DISPLAY_RES = [dataViewEnd(LOOK_AHEAD_CFG), UINT_SIZE, 2]; +const DISPLAY_FOV = [dataViewEnd(DISPLAY_RES), FLOAT_SIZE, 1]; +const LENS_DISTANCE_RATIO = [dataViewEnd(DISPLAY_FOV), FLOAT_SIZE, 1]; +const SBS_ENABLED = [dataViewEnd(LENS_DISTANCE_RATIO), BOOL_SIZE, 1]; +const CUSTOM_BANNER_ENABLED = [dataViewEnd(SBS_ENABLED), BOOL_SIZE, 1]; +const EPOCH_MS = [dataViewEnd(CUSTOM_BANNER_ENABLED), UINT_SIZE, 2]; +const IMU_QUAT_DATA = [dataViewEnd(EPOCH_MS), FLOAT_SIZE, 16]; +const IMU_PARITY_BYTE = [dataViewEnd(IMU_QUAT_DATA), UINT8_SIZE, 1]; +const DATA_VIEW_LENGTH = dataViewEnd(IMU_PARITY_BYTE); + +function checkParityByte(dataView) { + const parityByte = dataViewUint8(dataView, IMU_PARITY_BYTE); + let parity = 0; + const epochUint8 = dataViewUint8Array(dataView, EPOCH_MS); + const imuDataUint8 = dataViewUint8Array(dataView, IMU_QUAT_DATA); + for (let i = 0; i < epochUint8.length; i++) { + parity ^= epochUint8[i]; + } + for (let i = 0; i < imuDataUint8.length; i++) { + parity ^= imuDataUint8[i]; + } + return parityByte === parity; +} + +export const DeviceDataStream = GObject.registerClass({ + Properties: { + 'supported-device-connected': GObject.ParamSpec.boolean( + 'supported-device-connected', + 'Supported device connected', + 'Whether a supported device is connected', + GObject.ParamFlags.READWRITE, + false + ), + 'widescreen-mode-state': GObject.ParamSpec.boolean( + 'widescreen-mode-state', + 'Widescreen mode state', + 'The state of widescreen mode from the perspective of the driver', + GObject.ParamFlags.READWRITE, + false + ), + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', + GObject.ParamFlags.READWRITE + ) + } +}, class DeviceDataStream extends GObject.Object { + constructor(params = {}) { + super(params); + this.supported_device_connected = false; + this._ipc_file = Gio.file_new_for_path(IPC_FILE_PATH); + this._running = false; + this._device_data = null; + } + + start() { + this._running = true; + this._poll(); + } + + stop() { + this._running = false; + } + + // polling is just intended to keep supported_device_connected current, anything needing up-to-date imu data should + // trigger a refresh with the default flag + _poll() { + if (this._running) { + this.refresh_data(true); + setTimeout(this._poll.bind(this), 1000); + } + } + + // Refresh the data from the IPC file. if keepalive_only is true, we'll only check and update supported_device_connected if it + // hasn't been checked within KEEPALIVE_REFRESH_INTERVAL_SEC. + refresh_data(keepalive_only = false) { + if (!this._device_data?.imuData || !keepalive_only || getEpochSec() - this._device_data.imuDateMs > KEEPALIVE_REFRESH_INTERVAL_SEC) { + let data = this._ipc_file.load_contents(null); + if (data[0]) { + let buffer = new Uint8Array(data[1]).buffer; + let dataView = new DataView(buffer); + if (dataView.byteLength === DATA_VIEW_LENGTH) { + const imuDateMs = dataViewBigUint(dataView, EPOCH_MS); + const validKeepalive = isValidKeepAlive(toSec(imuDateMs)); + const imuData = dataViewFloatArray(dataView, IMU_QUAT_DATA); + const imuResetState = validKeepalive && imuData[0] === 0.0 && imuData[1] === 0.0 && imuData[2] === 0.0 && imuData[3] === 1.0; + const version = dataViewUint8(dataView, VERSION); + const enabled = dataViewUint8(dataView, ENABLED) !== 0 && version === DATA_LAYOUT_VERSION && validKeepalive; + const sbsEnabled = dataViewUint8(dataView, SBS_ENABLED) !== 0; + + // update the widescreen property if the state changes while still enabled, trigger "notify::" events + if (enabled && this.widescreen_mode_state !== sbsEnabled) this.widescreen_mode_state = sbsEnabled; + + if (!this._device_data) { + this._device_data = { + version, + enabled, + imuResetState, + displayRes: dataViewUint32Array(dataView, DISPLAY_RES), + sbsEnabled, + displayFov: dataViewFloat(dataView, DISPLAY_FOV), + lookAheadCfg: dataViewFloatArray(dataView, LOOK_AHEAD_CFG), + }; + } + + let success = keepalive_only; + let attempts = 0; + while (!success && attempts < 3) { + if (dataView.byteLength === DATA_VIEW_LENGTH) { + if (checkParityByte(dataView)) { + this._device_data.imuData = imuData; + this._device_data.imuDateMs = imuDateMs; + this.imu_snapshots = { + imu_data: imuData, + timestamp_ms: imuDateMs + }; + success = true; + } + } else if (dataView.byteLength !== 0) { + Globals.logger.log(`[ERROR] Invalid dataView.byteLength: ${dataView.byteLength} !== ${DATA_VIEW_LENGTH}`) + } + + if (!success && ++attempts < 3) { + data = this._ipc_file.load_contents(null); + if (data[0]) { + buffer = new Uint8Array(data[1]).buffer; + dataView = new DataView(buffer); + } + } + } + + if (success) { + // update the supported device connected property if the state changes, trigger "notify::" events + if (this.supported_device_connected !== validKeepalive) this.supported_device_connected = validKeepalive; + } + } + } else { + this.supported_device_connected = false; + } + } + } +}); \ No newline at end of file diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 08c4c5d..3c5e368 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -1,21 +1,23 @@ import Clutter from 'gi://Clutter' import Gio from 'gi://Gio'; import GLib from 'gi://GLib'; +import GObject from 'gi://GObject'; import Meta from 'gi://Meta'; import Shell from 'gi://Shell'; import { CursorManager } from './cursormanager.js'; +import { DeviceDataStream } from './devicedatastream.js'; import Globals from './globals.js'; import { Logger } from './logger.js'; import { MonitorManager } from './monitormanager.js'; import { Overlay } from './overlay.js'; -import { isValidKeepAlive } from './time.js'; -import { IPC_FILE_PATH, XREffect } from './xrEffect.js'; +import { VirtualMonitorsActor } from './virtualmonitorsactor.js'; import {Extension} from 'resource:///org/gnome/shell/extensions/extension.js'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; const NESTED_MONITOR_PRODUCT = 'MetaMonitor'; +const VIRTUAL_MONITOR_PRODUCT = 'Virtual remote monitor'; const SUPPORTED_MONITOR_PRODUCTS = [ 'VITURE', 'nreal air', @@ -42,7 +44,7 @@ export default class BreezyDesktopExtension extends Extension { // Set/destroyed by enable/disable this._cursor_manager = null; this._monitor_manager = null; - this._xr_effect = null; + this._overlay_content = null; this._overlay = null; this._target_monitor = null; this._is_effect_running = false; @@ -51,7 +53,7 @@ export default class BreezyDesktopExtension extends Extension { this._follow_threshold_connection = null; this._widescreen_mode_settings_connection = null; this._widescreen_mode_effect_state_connection = null; - this._supported_device_detected_connected = null; + this._supported_device_detected_connection = null; this._start_binding = null; this._end_binding = null; this._curved_display_binding = null; @@ -62,6 +64,8 @@ export default class BreezyDesktopExtension extends Extension { this._headset_as_primary_binding = null; this._actor_added_connection = null; this._actor_removed_connection = null; + this._data_stream_connection = null; + this._stage_redraw_connection = null; if (!Globals.logger) { Globals.logger = new Logger({ @@ -70,6 +74,10 @@ export default class BreezyDesktopExtension extends Extension { }); Globals.logger.logVersion(); } + + if (!Globals.data_stream) { + Globals.data_stream = new DeviceDataStream(); + } } enable() { @@ -79,6 +87,8 @@ export default class BreezyDesktopExtension extends Extension { Globals.extension_dir = this.path; this.settings.bind('debug', Globals.logger, 'debug', Gio.SettingsBindFlags.DEFAULT); + Globals.data_stream.start(); + this._monitor_manager = new MonitorManager({ use_optimal_monitor_config: this.settings.get_boolean('use-optimal-monitor-config'), headset_as_primary: this.settings.get_boolean('headset-as-primary'), @@ -119,7 +129,7 @@ export default class BreezyDesktopExtension extends Extension { return GLib.SOURCE_REMOVE; } - if (this._check_driver_running() && target_monitor) { + if (Globals.data_stream.supported_device_connected && target_monitor) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes, // so we can remove this timeout_add no matter what. @@ -130,6 +140,7 @@ export default class BreezyDesktopExtension extends Extension { this._running_poller_id = undefined; return GLib.SOURCE_REMOVE; } else { + Globals.logger.log_debug(`BreezyDesktopExtension _poll_for_ready - device connected: ${Globals.data_stream.supported_device_connected}, target_monitor: ${!!target_monitor}`); return GLib.SOURCE_CONTINUE; } } catch (e) { @@ -140,19 +151,42 @@ export default class BreezyDesktopExtension extends Extension { }).bind(this)); } + _find_virtual_monitors() { + try { + Globals.logger.log_debug('BreezyDesktopExtension _find_virtual_monitors'); + const virtual_monitors = this._monitor_manager.getMonitorPropertiesList()?.filter( + monitor => monitor && monitor.product === VIRTUAL_MONITOR_PRODUCT); + if (virtual_monitors.length > 0) { + Globals.logger.log(`Found ${virtual_monitors.length} virtual monitors`); + return virtual_monitors.map(monitor => { + return this._monitor_manager.getMonitors()[monitor.index]; + }); + } + + Globals.logger.log_debug('BreezyDesktopExtension _find_virtual_monitors - No virtual monitors found'); + } catch (e) { + Globals.logger.log(`[ERROR] BreezyDesktopExtension _find_virtual_monitors ${e.message}\n${e.stack}`) + } + + return []; + } + _find_supported_monitor() { + if (!this._monitor_manager.getMonitorPropertiesList()) return null; + try { Globals.logger.log_debug('BreezyDesktopExtension _find_supported_monitor'); const target_monitor = this._monitor_manager.getMonitorPropertiesList()?.find( - monitor => SUPPORTED_MONITOR_PRODUCTS.includes(monitor.product) || - this.settings.get_string('custom-monitor-product') === monitor.product); + monitor => monitor && (SUPPORTED_MONITOR_PRODUCTS.includes(monitor.product) || + this.settings.get_string('custom-monitor-product') === monitor.product)); if (target_monitor !== undefined) { Globals.logger.log(`Identified supported monitor: ${target_monitor.product} on ${target_monitor.connector}`); return { monitor: this._monitor_manager.getMonitors()[target_monitor.index], connector: target_monitor.connector, refreshRate: target_monitor.refreshRate, - is_dummy: target_monitor.product === NESTED_MONITOR_PRODUCT + is_dummy: target_monitor.product === NESTED_MONITOR_PRODUCT, + is_virtual: target_monitor.product === VIRTUAL_MONITOR_PRODUCT }; } @@ -200,7 +234,7 @@ export default class BreezyDesktopExtension extends Extension { if (target_monitor && this._running_poller_id === undefined) { this._target_monitor = target_monitor; - if (this._check_driver_running()) { + if (Globals.data_stream.supported_device_connected) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes if (this._target_monitor_ready(target_monitor)) { @@ -222,21 +256,6 @@ export default class BreezyDesktopExtension extends Extension { } } - _check_driver_running() { - try { - if (!Globals.ipc_file) Globals.ipc_file = Gio.file_new_for_path(IPC_FILE_PATH); - if (Globals.ipc_file.query_exists(null)) { - const file_info = Globals.ipc_file.query_info(Gio.FILE_ATTRIBUTE_TIME_MODIFIED, Gio.FileQueryInfoFlags.NONE, null); - const file_modified_time = file_info.get_attribute_uint64(Gio.FILE_ATTRIBUTE_TIME_MODIFIED); - return isValidKeepAlive(file_modified_time); - } - } catch (e) { - Globals.logger.log(`[ERROR] BreezyDesktopExtension _check_driver_running ${e.message}\n${e.stack}`); - } - - return false; - } - _needs_widescreen_monitor_update() { Globals.logger.log_debug('BreezyDesktopExtension _needs_widescreen_monitor_update'); const state = this._read_state(); @@ -275,46 +294,39 @@ export default class BreezyDesktopExtension extends Extension { clutterContainer ? 'actor-removed' : 'child-removed', this._handle_sibling_update.bind(this), ); - - this._xr_effect = new XREffect({ - target_monitor: targetMonitor, - target_framerate: refreshRate, - display_distance: this.settings.get_double('display-distance'), - toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), - toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), - look_ahead_override: this.settings.get_int('look-ahead-override'), - disable_anti_aliasing: this.settings.get_boolean('disable-anti-aliasing') - }); this._update_follow_threshold(this.settings); // this gets triggered before _effect_enable if in fast-sbs-mode-switching mode - if (!this.settings.get_boolean('fast-sbs-mode-switching')) - this._update_widescreen_mode_from_settings(this.settings); + // if (!this.settings.get_boolean('fast-sbs-mode-switching')) + // this._update_widescreen_mode_from_settings(this.settings); - this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); - this._supported_device_detected_connected = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); + // this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); + // this._supported_device_detected_connection = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); + this._overlay_content.renderMonitors(); - this._distance_binding = this.settings.bind('display-distance', this._xr_effect, 'display-distance', Gio.SettingsBindFlags.DEFAULT) - this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)) - this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)) + this._distance_binding = this.settings.bind('display-distance', this._overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); + this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)); + this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)); - this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) - this._start_binding = this.settings.bind('toggle-display-distance-start', this._xr_effect, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) - this._end_binding = this.settings.bind('toggle-display-distance-end', this._xr_effect, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) - this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) - this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); - this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); - this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); + // this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) + this._start_binding = this.settings.bind('toggle-display-distance-start', this._overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) + this._end_binding = this.settings.bind('toggle-display-distance-end', this._overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT); + this._display_size_binding = this.settings.bind('display-size', this._overlay_content, 'display-size', Gio.SettingsBindFlags.DEFAULT); + // this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) + // this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); + // this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); + // this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); this._overlay.mainActor().add_effect_with_name('xr-desktop', this._xr_effect); Meta.disable_unredirect_for_display(global.display); this._add_settings_keybinding('toggle-xr-effect-shortcut', this._toggle_xr_effect.bind(this)); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); - this._add_settings_keybinding('toggle-display-distance-shortcut', this._xr_effect._change_distance.bind(this._xr_effect)); + this._add_settings_keybinding('toggle-display-distance-shortcut', this._overlay_content._change_distance.bind(this._overlay_content)); this._add_settings_keybinding('toggle-follow-shortcut', this._toggle_follow_mode.bind(this)); } catch (e) { + Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); this._effect_disable(); } @@ -443,12 +455,12 @@ export default class BreezyDesktopExtension extends Extension { } _update_widescreen_mode_from_settings(settings, event) { - const value = settings.get_boolean('widescreen-mode'); - Globals.logger.log_debug(`BreezyDesktopExtension _update_widescreen_mode_from_settings ${value}`); - if (value !== undefined && value !== this._xr_effect.widescreen_mode_state) { - this._request_sbs_mode_change(value); - } else - Globals.logger.log_debug('effect.widescreen_mode_state already matched setting'); + // const value = settings.get_boolean('widescreen-mode'); + // Globals.logger.log_debug(`BreezyDesktopExtension _update_widescreen_mode_from_settings ${value}`); + // if (value !== undefined && value !== this._xr_effect.widescreen_mode_state) { + // this._request_sbs_mode_change(value); + // } else + // Globals.logger.log_debug('effect.widescreen_mode_state already matched setting'); } _update_widescreen_mode_from_state(effect, _pspec) { @@ -540,6 +552,11 @@ export default class BreezyDesktopExtension extends Extension { Main.wm.removeKeybinding('toggle-display-distance-shortcut'); Main.wm.removeKeybinding('toggle-follow-shortcut'); Meta.enable_unredirect_for_display(global.display); + + if (this._stage_redraw_connection) { + global.stage.disconnect(this._stage_redraw_connection); + this._stage_redraw_connection = null; + } if (this._actor_added_connection) { global.stage.disconnect(this._actor_added_connection); @@ -557,6 +574,10 @@ export default class BreezyDesktopExtension extends Extension { this.settings.disconnect(this._distance_connection); this._distance_connection = null; } + if (this._data_stream_connection) { + this._data_stream_connection.unbind(); + this._data_stream_connection = null; + } if (this._follow_threshold_connection) { this.settings.disconnect(this._follow_threshold_connection); this._follow_threshold_connection = null; @@ -589,6 +610,24 @@ export default class BreezyDesktopExtension extends Extension { this.settings.unbind(this._disable_anti_aliasing_binding); this._disable_anti_aliasing_binding = null; } + if (this._overlay) { + if (this._overlay_content) { + // if (this._widescreen_mode_effect_state_connection) { + // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); + // this._widescreen_mode_effect_state_connection = null; + // } + // if (this._supported_device_detected_connection) { + // this._xr_effect.disconnect(this._supported_device_detected_connection); + // this._supported_device_detected_connection = null; + // } + this._overlay_content.destroy(); + this._overlay_content = null; + } + global.stage.remove_child(this._overlay); + this._overlay.destroy(); + this._overlay = null; + + } if (this._xr_effect) { if (this._widescreen_mode_effect_state_connection) { this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); @@ -624,8 +663,11 @@ export default class BreezyDesktopExtension extends Extension { disable() { try { Globals.logger.log_debug('BreezyDesktopExtension disable'); + Globals.data_stream.stop(); + this._effect_disable(); this._target_monitor = null; + if (this._monitor_manager) { if (this._optimal_monitor_config_binding) { this.settings.unbind(this._optimal_monitor_config_binding); diff --git a/gnome/src/globals.js b/gnome/src/globals.js index 124d2e1..f11451c 100644 --- a/gnome/src/globals.js +++ b/gnome/src/globals.js @@ -1,6 +1,7 @@ const Globals = { logger: null, ipc_file: null, // Gio.File instance, file exists if set - extension_dir: null // string path + extension_dir: null, // string path + data_stream: null, // DeviceDataStream instance } export default Globals; \ No newline at end of file diff --git a/gnome/src/monitormanager.js b/gnome/src/monitormanager.js index 6cf5532..7826a17 100644 --- a/gnome/src/monitormanager.js +++ b/gnome/src/monitormanager.js @@ -35,7 +35,7 @@ function getDisplayConfigProxy(extPath) { xml = new TextDecoder().decode(bytes); } } catch (e) { - Globals.logger.log('ERROR: failed to load DisplayConfig interface XML'); + Globals.logger.log('[ERROR] failed to load DisplayConfig interface XML'); throw e; } cachedDisplayConfigProxy = Gio.DBusProxy.makeProxyWrapper(xml); @@ -54,33 +54,28 @@ export function newDisplayConfig(extPath, callback) { } function getMonitorConfig(displayConfigProxy, callback) { - displayConfigProxy.GetResourcesRemote((result, error) => { + displayConfigProxy.GetCurrentStateRemote((result, error) => { if (error) { - callback(null, `GetResourcesRemote failed: ${error}`); + callback(null, `GetCurrentState failed: ${error}`); } else { - const monitors = []; - for (let i = 0; i < result[2].length; i++) { - const output = result[2][i]; - if (output.length <= 7) { - callback(null, 'Cannot get DisplayConfig: No properties on output #' + i); - return; - } - const props = output[7]; - const displayName = props['display-name'].get_string()[0]; - const connectorName = output[4]; - if (!displayName || displayName == '') { - const displayName = 'Monitor on output ' + connectorName; - } - const vendor = props['vendor'].get_string()[0]; - const product = props['product'].get_string()[0]; - const serial = props['serial'].get_string()[0]; + Globals.logger.log_debug(`monitormanager.js getMonitorConfig GetCurrentState result: ${JSON.stringify(result)}`); + + const allMonitors = []; + const [serial, monitors, logicalMonitors, properties] = result; + for (let monitor of monitors) { + const [details, modes, monProperties] = monitor; + const [connector, vendor, product, monitorSerial] = details; + const displayName = monProperties['display-name'].get_string()[0]; - // grab refresh rate from the modes array - const refreshRate = result[3][i][4]; - - monitors.push([displayName, connectorName, vendor, product, serial, refreshRate]); + for (let mode of modes) { + const [modeId, width, height, refreshRate, preferredScale, supportedScales, modeProperites] = mode; + const isCurrent = !!modeProperites['is-current']; + if (isCurrent) { + allMonitors.push([displayName, connector, vendor, product, serial, refreshRate]); + } + } } - callback(monitors, null); + callback(allMonitors, null); } }); } @@ -287,6 +282,7 @@ export const MonitorManager = GObject.registerClass({ // help prevent certain actions from taking place multiple times in the event of rapid monitor updates this._asyncRequestsInFlight = 0; this._configCheckRequestsCount = 0; + this._enabled = false; } enable() { @@ -300,12 +296,14 @@ export const MonitorManager = GObject.registerClass({ }).bind(this)); this._monitorsChangedConnection = Main.layoutManager.connect('monitors-changed', this._on_monitors_change.bind(this)); + this._enabled = true; } disable() { Globals.logger.log_debug('MonitorManager disable'); Main.layoutManager.disconnect(this._monitorsChangedConnection); + this._enabled = false; this._monitorsChangedConnection = null; this._displayConfigProxy = null; this._monitorProperties = null; @@ -380,6 +378,8 @@ export const MonitorManager = GObject.registerClass({ } _on_monitors_change() { + if (!this._enabled) return; + Globals.logger.log_debug('MonitorManager _on_monitors_change'); if (this._displayConfigProxy == null) { return; diff --git a/gnome/src/virtualmonitorsactor.js b/gnome/src/virtualmonitorsactor.js new file mode 100644 index 0000000..f66bdf9 --- /dev/null +++ b/gnome/src/virtualmonitorsactor.js @@ -0,0 +1,788 @@ +import Clutter from 'gi://Clutter' +import Cogl from 'gi://Cogl'; +import GLib from 'gi://GLib'; +import GObject from 'gi://GObject'; +import Shell from 'gi://Shell'; +import * as Main from 'resource:///org/gnome/shell/ui/main.js'; + +import Globals from './globals.js'; + +function applyQuaternionToVector(vector, quaternion) { + const t = [ + 2.0 * (quaternion[1] * vector[2] - quaternion[2] * vector[1]), + 2.0 * (quaternion[2] * vector[0] - quaternion[0] * vector[2]), + 2.0 * (quaternion[0] * vector[1] - quaternion[1] * vector[0]) + ]; + return [ + vector[0] + quaternion[3] * t[0] + quaternion[1] * t[2] - quaternion[2] * t[1], + vector[1] + quaternion[3] * t[1] + quaternion[2] * t[0] - quaternion[0] * t[2], + vector[2] + quaternion[3] * t[2] + quaternion[0] * t[1] - quaternion[1] * t[0] + ]; +} + +/** + * Find the vector in the array that's closest to the quaternion rotation + * + * @param {number[]} quaternion - Reference quaternion [x, y, z, w] + * @param {number[][]} vectors - Array of vectors [x, y, z] to search from + * @returns {number} Index of the closest vector, if it surpasses the previous closest index by a certain margin, otherwise the previous index + */ +function findClosestVector(quaternion, vectors, previousClosestIndex) { + + const lookVector = [1.0, 0.0, 0.0]; // NWU vector pointing to the center of the screen + const rotatedLookVector = applyQuaternionToVector(lookVector, quaternion); + // Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); + + let closestIndex = -1; + let closestDistance = Infinity; + let previousDistance = Infinity; + + // find the vector closest to the rotated look vector + vectors.forEach((vector, index) => { + const distance = Math.acos( + Math.min(1.0, Math.max(-1.0, vector[0] * rotatedLookVector[0] + vector[1] * rotatedLookVector[1] + vector[2] * rotatedLookVector[2])) + ); + + if (previousClosestIndex === index) { + previousDistance = distance; + } + + // Globals.logger.log(`\t\t\tMonitor ${index} distance: ${distance}`); + if (distance < closestDistance) { + closestIndex = index; + closestDistance = distance; + } + }); + + // Globals.logger.log(`\t\t\tClosest monitor: ${closestIndex}, distance: ${closestDistance}`); + + // only switch if the closest monitor is greater than the previous closest by 25% + if (previousClosestIndex !== undefined && closestIndex !== previousClosestIndex && closestDistance * 1.25 > previousDistance) { + return previousClosestIndex; + } + + return closestIndex; +} + +function degreesToRadians(degrees) { + return degrees * Math.PI / 180.0; +} + +function radiansToDegrees(radians) { + return radians * 180.0 / Math.PI; +} + +/*** + * @returns {Object} - containing `start`, `center`, and `end` radians for rotating the given monitor + */ +function monitorWrap(cachedMonitorWrap, radiusPixels, monitorBeginPixel, monitorLengthPixels) { + let closestWrap = cachedMonitorWrap.reduce((previous, current) => { + return (!previous || Math.abs(current.pixel - monitorBeginPixel) < Math.abs(previous.pixel - monitorBeginPixel)) ? current : previous; + }, undefined); + + if (closestWrap.pixel !== monitorBeginPixel) { + // there's a gap between the cached wrap value and this one + const gapPixels = monitorBeginPixel - closestWrap.pixel; + const gapHalfRadians = Math.asin(gapPixels / 2 / radiusPixels); + const gapRadians = gapHalfRadians * 2; + + // update the closestWrap value and cache it + closestWrap = { pixel: monitorBeginPixel, radians: closestWrap.radians + gapRadians }; + cachedMonitorWrap.push(closestWrap); + } + + const monitorHalfRadians = Math.asin(monitorLengthPixels / 2 / radiusPixels); + const centerRadians = closestWrap.radians + monitorHalfRadians; + const endRadians = centerRadians + monitorHalfRadians; + + // since we're computing the end values for this monitor, cache them too in case they line up with a future monitor + cachedMonitorWrap.push({ pixel: monitorBeginPixel + monitorLengthPixels, radians: endRadians }); + return { + begin: closestWrap.radians, + center: centerRadians, + end: endRadians + } +} + +/** + * Convert the given monitor details into NWU vectors describing the center of the fully placed monitor, + * and the top-left of the partially placed monitor (minus only a single-axis rotation) + * + * @param {Object} fovDetails - contains reference fovDegrees (diagonal), widthPixels, heightPixels + * @param {Object[]} monitorDetailsList - contains x, y, width, height (coordinates from top-left) + * @param {string} monitorWrappingScheme - horizontal, vertical, none + * @returns {Object[]} - contains NWU vectors pointing to `topLeftNoRotate` and `center` of each monitor + * and a `rotation` angle for the given wrapping scheme + */ +function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingScheme) { + const aspect = fovDetails.widthPixels / fovDetails.heightPixels; + const fovVerticalRadians = degreesToRadians(fovDetails.fovDegrees / Math.sqrt(1 + aspect * aspect)); + + // distance needed for the FOV-sized monitor to fill up the screen + const centerRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); + + const monitorPlacements = []; + const cachedMonitorWrap = []; + + if (monitorWrappingScheme === 'horizontal') { + // monitors wrap around us horizontally + const fovHorizontalRadians = fovVerticalRadians * aspect; + + // distance to a horizontal edge is the hypothenuse of the triangle where the opposite side is half the width of the reference fov screen + const edgeRadius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); + + cachedMonitorWrap.push({ pixel: 0, radians: -fovHorizontalRadians / 2 }); + monitorDetailsList.forEach(monitorDetails => { + const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.x, monitorDetails.width); + const monitorCenterRadius = Math.sqrt(Math.pow(edgeRadius, 2) - Math.pow(monitorDetails.width / 2, 2)) + + monitorPlacements.push({ + topLeftNoRotate: [ + monitorCenterRadius, + -(monitorDetails.width - fovDetails.widthPixels) / 2, + -monitorDetails.y + ], + center: [ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + monitorCenterRadius * Math.cos(monitorWrapDetails.center), + + // west is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -monitorCenterRadius * Math.sin(monitorWrapDetails.center), + + // up is flat when wrapping horizontally + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ], + rotationAngleRadians: -monitorWrapDetails.center + }); + }); + } else if (monitorWrappingScheme === 'vertical') { + // monitors wrap around us vertically + + // distance to a vertical edge is the hypothenuse of the triangle where the opposite side is half the height of the reference fov screen + const edgeRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); + + cachedMonitorWrap.push({ pixel: 0, radians: -fovVerticalRadians / 2 }); + monitorDetailsList.forEach(monitorDetails => { + const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.y, monitorDetails.height); + const monitorCenterRadius = Math.sqrt(Math.pow(edgeRadius, 2) - Math.pow(monitorDetails.height / 2, 2)) ; + + monitorPlacements.push({ + topLeftNoRotate: [ + monitorCenterRadius, + -monitorDetails.x, + -(monitorDetails.height - fovDetails.heightPixels) / 2 + ], + center: [ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + monitorCenterRadius * Math.cos(monitorWrapDetails.center), + + // west is flat when wrapping vertically + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + + // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -monitorCenterRadius * Math.sin(monitorWrapDetails.center) + ], + rotationAngleRadians: -monitorWrapDetails.center + }); + }); + } else { + // monitors make a flat wall in front of us, no wrapping + monitorDetailsList.forEach(monitorDetails => { + monitorPlacements.push({ + topLeftNoRotate: [ + centerRadius, + -monitorDetails.x, + -monitorDetails.y + ], + center: [ + centerRadius, + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ], + rotationAngleRadians: 0 + }); + }); + } + Globals.logger.log_debug(`\t\t\tCached monitor wrap: ${JSON.stringify(cachedMonitorWrap)}`); + + return monitorPlacements; +} + +function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { + if (monitorWrappingScheme === 'horizontal') { + // monitors wrap around us horizontally + return { + angle: Math.atan2(vector[1], vector[0]), + axis: Clutter.RotateAxis.Y_AXIS + }; + } else if (monitorWrappingScheme === 'vertical') { + // monitors wrap around us vertically + return { + angle: Math.atan2(vector[2], vector[0]), + axis: Clutter.RotateAxis.X_AXIS + } + } else { + // no rotation + return undefined; + } +} + +// how far to look ahead is how old the IMU data is plus a constant that is either the default for this device or an override +function lookAheadMS(imuDateMs, override) { + // how stale the imu data is + const dataAge = Date.now() - imuDateMs; + + // if (override === -1) + // return lookAheadCfg[0] + dataAge; + + return override + dataAge; +} + +export const VirtualMonitorEffect = GObject.registerClass({ + Properties: { + 'monitor-index': GObject.ParamSpec.int( + 'monitor-index', + 'Monitor Index', + 'Index of the monitor that this effect is applied to', + GObject.ParamFlags.READWRITE, + 0, 100, 0 + ), + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', + GObject.ParamFlags.READWRITE + ), + 'fov-degrees': GObject.ParamSpec.double( + 'fov-degrees', + 'FOV Degrees', + 'Field of view in degrees', + GObject.ParamFlags.READWRITE, + 30.0, 100.0, 46.0 + ), + 'width': GObject.ParamSpec.int( + 'width', + 'Width', + 'Width of the viewport', + GObject.ParamFlags.READWRITE, + 1, 10000, 1920 + ), + 'height': GObject.ParamSpec.int( + 'height', + 'Height', + 'Height of the viewport', + GObject.ParamFlags.READWRITE, + 1, 10000, 1080 + ), + 'monitor-wrapping-scheme': GObject.ParamSpec.string( + 'monitor-wrapping-scheme', + 'Monitor Wrapping Scheme', + 'How the monitors are wrapped around the viewport', + GObject.ParamFlags.READWRITE, + 'horizontal', ['horizontal', 'vertical', 'none'] + ), + 'monitor-wrapping-rotation-radians': GObject.ParamSpec.double( + 'monitor-wrapping-rotation-radians', + 'Monitor Wrapping Rotation Radians', + 'Rotation of the monitor wrapping around the viewport', + GObject.ParamFlags.READWRITE, + -360.0, 360.0, 0.0 + ), + 'focused-monitor-index': GObject.ParamSpec.int( + 'focused-monitor-index', + 'Focused Monitor Index', + 'Index of the monitor that is currently focused', + GObject.ParamFlags.READWRITE, + 0, 100, 0 + ), + 'display-distance': GObject.ParamSpec.double( + 'display-distance', + 'Display Distance', + 'Distance of the display from the camera', + GObject.ParamFlags.READWRITE, + 0.0, + 2.5, + 1.0 + ), + 'display-position': GObject.ParamSpec.jsobject( + 'display-position', + 'Display Position', + 'Position of the display in COGL (ESU) coordinates', + GObject.ParamFlags.READWRITE + ), + 'display-distance-default': GObject.ParamSpec.double( + 'display-distance-default', + 'Display distance default', + 'Distance to use when not explicitly set, or when reset', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.0 + ), + 'actor-to-display-ratios': GObject.ParamSpec.jsobject( + 'actor-to-display-ratios', + 'Actor to Display Ratios', + 'Ratios to convert actor coordinates to display coordinates', + GObject.ParamFlags.READWRITE + ), + 'actor-to-display-offsets': GObject.ParamSpec.jsobject( + 'actor-to-display-offsets', + 'Actor to Display Offsets', + 'Offsets to convert actor coordinates to display coordinates', + GObject.ParamFlags.READWRITE + ), + 'is-closest': GObject.ParamSpec.boolean( + 'is-closest', + 'Is Closest', + 'Whether this monitor is the closest to the camera', + GObject.ParamFlags.READWRITE, + false + ) + } +}, class VirtualMonitorEffect extends Shell.GLSLEffect { + constructor(params = {}) { + super(params); + + this._current_display_distance = this._is_focused() ? this.display_distance : this.display_distance_default; + + this.connect('notify::display-distance', this._update_display_distance.bind(this)); + this.connect('notify::focused-monitor-index', this._update_display_distance.bind(this)); + } + + _is_focused() { + return this.focused_monitor_index === this.monitor_index; + } + + _update_display_distance() { + const desired_distance = this._is_focused() ? this.display_distance : this.display_distance_default; + if (this._distance_ease_timeline?.is_playing()) { + // we're already easing towards the desired distance, do nothing + if (this._distance_ease_target === desired_distance) return; + + this._distance_ease_timeline.stop(); + } + + const mid_distance = (this.display_distance_default + desired_distance) / 2; + + // if we're the focused display, we'll double the timeline and wait for the first half to let other + // displays ease out first + this._distance_ease_focus = this._is_focused(); + const timeline_ms = this._distance_ease_focus ? 500 : 150; + + this._distance_ease_start = this._current_display_distance; + this._distance_ease_timeline = Clutter.Timeline.new_for_actor(this.get_actor(), timeline_ms); + + this._distance_ease_target = desired_distance; + this._distance_ease_timeline.connect('new-frame', (() => { + let progress = this._distance_ease_timeline.get_progress(); + if (this._distance_ease_focus) { + // if we're the focused display, wait for the first half of the timeline to pass + if (progress < 0.5) return; + + // treat the second half of the timeline as its own full progression + progress = (progress - 0.5) * 2; + + // put this display in front as it starts to easy in + this.is_closest = true; + } else { + this.is_closest = false; + } + + this._current_display_distance = this._distance_ease_start + + progress * (this._distance_ease_target - this._distance_ease_start); + }).bind(this)); + + this._distance_ease_timeline.start(); + } + + perspective(fovDiagonalRadians, aspect, near, far) { + // compute horizontal fov given diagonal fov and aspect ratio + const h = Math.sqrt(aspect * aspect + 1); + + const fovRadians = fovDiagonalRadians / h * aspect; + console.log(`fovRadians: ${fovRadians}`); + + const f = 1.0 / Math.tan(fovRadians / 2.0); + const range = far - near; + + return [ + f / aspect, 0, 0, 0, + 0, f, 0, 0, + 0, 0, - (far + near) / range, -1, + 0, 0, - (2.0 * near * far) / range, 0 + ]; + } + + vfunc_build_pipeline() { + const declarations = ` + uniform mat4 u_imu_data; + uniform float u_look_ahead_ms; + uniform mat4 u_projection_matrix; + uniform vec3 u_display_position; + uniform float u_rotation_x_radians; + uniform float u_rotation_y_radians; + uniform vec2 u_display_resolution; + + // vector positions are relative to the width and height of the entire stage + uniform vec2 u_actor_to_display_ratios; + uniform vec2 u_actor_to_display_offsets; + + // discovered through trial and error, no idea the significance + float cogl_position_mystery_factor = 29.09 * 2; + + vec4 quatConjugate(vec4 q) { + return vec4(-q.xyz, q.w); + } + + vec4 applyQuaternionToVector(vec4 v, vec4 q) { + vec3 t = 2.0 * cross(q.xyz, v.xyz); + vec3 rotated = v.xyz + q.w * t + cross(q.xyz, t); + return vec4(rotated, v.w); + } + + vec4 applyXRotationToVector(vec4 v, float angle) { + float c = cos(angle); + float s = sin(angle); + return vec4(v.x, v.y * c - v.z * s, v.y * s + v.z * c, v.w); + } + + vec4 applyYRotationToVector(vec4 v, float angle) { + float c = cos(angle); + float s = sin(angle); + return vec4(v.x * c + v.z * s, v.y, v.z * c - v.x * s, v.w); + } + + vec4 nwuToESU(vec4 v) { + return vec4(-v.y, v.z, -v.x, v.w); + } + + // returns the rate of change between the two vectors, in same time units as delta_time + // e.g. if delta_time is in ms, then the rate of change is "per ms" + vec3 rateOfChange(vec3 v1, vec3 v2, float delta_time) { + return (v1-v2) / delta_time; + } + + // attempt to figure out where the current position should be based on previous position and velocity. + // velocity and time values should use the same time units (secs, ms, etc...) + vec3 applyLookAhead(vec3 position, vec3 velocity, float t) { + return position + velocity * t; + } + `; + + const main = ` + vec4 world_pos = cogl_position_in; + float aspect_ratio = u_display_resolution.x / u_display_resolution.y; + + float cogl_position_width = cogl_position_mystery_factor * aspect_ratio / u_actor_to_display_ratios.y; + float cogl_position_height = cogl_position_width / aspect_ratio; + + world_pos.x -= u_display_position.x * cogl_position_width / u_display_resolution.x; + world_pos.y -= u_display_position.y * cogl_position_height/ u_display_resolution.y; + world_pos.z = u_display_position.z * cogl_position_mystery_factor / u_display_resolution.x; + + // if the perspective includes more than just our viewport actor, move vertices towards the center of the perspective so they'll be properly rotated + world_pos.x += u_actor_to_display_offsets.x * cogl_position_width / 2; + world_pos.y -= u_actor_to_display_offsets.y * cogl_position_height / 2; + + world_pos.z *= aspect_ratio / u_actor_to_display_ratios.y; + world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); + world_pos = applyYRotationToVector(world_pos, u_rotation_y_radians); + + vec3 rotated_vector_t0 = applyQuaternionToVector(world_pos, nwuToESU(quatConjugate(u_imu_data[0]))).xyz; + vec3 rotated_vector_t1 = applyQuaternionToVector(world_pos, nwuToESU(quatConjugate(u_imu_data[1]))).xyz; + float delta_time_t0 = u_imu_data[3][0] - u_imu_data[3][1]; + vec3 velocity_t0 = rateOfChange(rotated_vector_t0, rotated_vector_t1, delta_time_t0); + world_pos = vec4(applyLookAhead(rotated_vector_t0, velocity_t0, u_look_ahead_ms), world_pos.w); + world_pos.z /= aspect_ratio / u_actor_to_display_ratios.y; + + world_pos.x *= u_actor_to_display_ratios.y / u_actor_to_display_ratios.x; + + world_pos = u_projection_matrix * world_pos; + + // if the perspective includes more than just our viewport actor, move the vertices back to just the area we can see. + // this needs to be done after the projection matrix multiplication so it will be projected as if centered in our vision + world_pos.x -= (u_actor_to_display_offsets.x / u_actor_to_display_ratios.x) * world_pos.w; + world_pos.y += (u_actor_to_display_offsets.y / u_actor_to_display_ratios.y) * world_pos.w; + + cogl_position_out = world_pos; + + cogl_tex_coord_out[0] = cogl_tex_coord_in; + ` + + this.add_glsl_snippet(Shell.SnippetHook.VERTEX, declarations, main, false); + } + + vfunc_paint_target(node, paintContext) { + if (!this._initialized) { + const aspect = this.get_actor().width / this.get_actor().height; + const projection_matrix = this.perspective( + this.fov_degrees * Math.PI / 180.0, + aspect, + 0.0001, + 1000.0 + ); + this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projection_matrix); + this.set_uniform_float(this.get_uniform_location("u_rotation_x_radians"), 1, [this.monitor_wrapping_scheme === 'vertical' ? this.monitor_wrapping_rotation_radians : 0.0]); + this.set_uniform_float(this.get_uniform_location("u_rotation_y_radians"), 1, [this.monitor_wrapping_scheme === 'horizontal' ? this.monitor_wrapping_rotation_radians : 0.0]); + this.set_uniform_float(this.get_uniform_location("u_display_resolution"), 2, [this.get_actor().width, this.get_actor().height]); + this.set_uniform_float(this.get_uniform_location("u_actor_to_display_ratios"), 2, this.actor_to_display_ratios); + this.set_uniform_float(this.get_uniform_location("u_actor_to_display_offsets"), 2, this.actor_to_display_offsets); + this._initialized = true; + } + + this.set_uniform_float(this.get_uniform_location('u_look_ahead_ms'), 1, [lookAheadMS(this.imu_snapshots.timestamp_ms, 0)]); + this.set_uniform_float(this.get_uniform_location("u_display_position"), 3, [this.display_position[0], this.display_position[1], this._current_display_distance * this.display_position[2]]); + this.set_uniform_matrix(this.get_uniform_location("u_imu_data"), false, 4, this.imu_snapshots.imu_data); + + this.get_pipeline().set_layer_filters( + 0, + Cogl.PipelineFilter.LINEAR_MIPMAP_LINEAR, + Cogl.PipelineFilter.LINEAR + ); + + super.vfunc_paint_target(node, paintContext); + } +}); + +export const VirtualMonitorsActor = GObject.registerClass({ + Properties: { + 'monitors': GObject.ParamSpec.jsobject( + 'monitors', + 'Monitors', + 'Array of monitor indexes', + GObject.ParamFlags.READWRITE + ), + 'target-monitor': GObject.ParamSpec.jsobject( + 'target-monitor', + 'Target Monitor', + 'Details about the monitor being used as a viewport', + GObject.ParamFlags.READWRITE + ), + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', + GObject.ParamFlags.READWRITE + ), + 'fov-degrees': GObject.ParamSpec.double( + 'fov-degrees', + 'FOV Degrees', + 'Field of view in degrees', + GObject.ParamFlags.READWRITE, + 30.0, 100.0, 46.0 + ), + 'focused-monitor-index': GObject.ParamSpec.int( + 'focused-monitor-index', + 'Focused Monitor Index', + 'Index of the monitor that is currently focused', + GObject.ParamFlags.READWRITE, + 0, 100, 0 + ), + 'display-size': GObject.ParamSpec.double( + 'display-size', + 'Display size', + 'Size of the display', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.0 + ), + 'display-distance': GObject.ParamSpec.double( + 'display-distance', + 'Display Distance', + 'Distance of the display from the camera', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), + 'toggle-display-distance-start': GObject.ParamSpec.double( + 'toggle-display-distance-start', + 'Display distance start', + 'Start distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), + 'toggle-display-distance-end': GObject.ParamSpec.double( + 'toggle-display-distance-end', + 'Display distance end', + 'End distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), + 'target-framerate': GObject.ParamSpec.double( + 'target-framerate', + 'Target Framerate', + 'Target framerate for the virtual monitors', + GObject.ParamFlags.READWRITE, + 1.0, 120.0, 60.0 + ) + } +}, class VirtualMonitorsActor extends Clutter.Actor { + constructor(params = {}) { + super(params); + + this.width = this.target_monitor.width; + this.height = this.target_monitor.height; + this._frametime_ms = Math.floor(1000 / (this.target_framerate ?? 60.0)); + this._all_monitors = [ + this.target_monitor, + ...this.monitors + ]; + } + + renderMonitors() { + this._monitorPlacements = monitorsToPlacements( + { + fovDegrees: this.fov_degrees, + widthPixels: this.width, + heightPixels: this.height + }, + this._all_monitors.map(monitor => ({ + x: monitor.x, + y: monitor.y, + width: monitor.width, + height: monitor.height + })), + 'horizontal' + ); + + // normalize the center vectors + this._monitorsAsNormalizedVectors = this._monitorPlacements.map(monitorVectors => { + const vector = monitorVectors.center; + const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); + return [vector[0] / length, vector[1] / length, vector[2] / length]; + }); + const monitors = this._all_monitors; + const minMonitorX = Math.min(...monitors.map(monitor => monitor.x)); + const maxMonitorX = Math.max(...monitors.map(monitor => monitor.x + monitor.width)); + const minMonitorY = Math.min(...monitors.map(monitor => monitor.y)); + const maxMonitorY = Math.max(...monitors.map(monitor => monitor.y + monitor.height)); + + const displayWidth = global.stage.width; + const displayHeight = global.stage.height; + const actorToDisplayRatios = [ + displayWidth / this.width, + displayHeight / this.height + ]; + + // how far this viewport actor's center is from the center of the whole stage + const actorMidX = this.target_monitor.x + this.width / 2; + const actorMidY = this.target_monitor.y + this.height / 2; + const actorToDisplayOffsets = [ + (displayWidth / 2 - (actorMidX - global.stage.x)) * 2 / this.width, + (displayHeight / 2 - (actorMidY - global.stage.y)) * 2 / this.height + ]; + + Globals.logger.log_debug(`\t\t\tActor to display ratios: ${actorToDisplayRatios}, offsets: ${actorToDisplayOffsets}`); + + monitors.forEach(((monitor, index) => { + // if (index === 0) return; + Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); + + // this is in NWU coordinates + const noRotationVector = this._monitorPlacements[index].topLeftNoRotate; + Globals.logger.log_debug(`\t\t\tMonitor ${index} vectors: ${JSON.stringify(this._monitorPlacements[index])}`); + + // actor coordinates are east-up-south + const containerActor = new Clutter.Actor({ + width: this.width, + height: this.height, + reactive: false, + }); + + // Create a clone of the stage content for this monitor + const monitorClone = new Clutter.Clone({ + source: Main.layoutManager.uiGroup, + reactive: false, + x: -monitor.x, + y: -monitor.y + }); + monitorClone.set_clip(monitor.x, monitor.y, monitor.width, monitor.height); + + // Add the monitor actor to the scene + containerActor.add_child(monitorClone); + const effect = new VirtualMonitorEffect({ + imu_snapshots: this.imu_snapshots, + fov_degrees: this.fov_degrees, + monitor_index: index, + display_position: [-noRotationVector[1], -noRotationVector[2], -noRotationVector[0]], + display_distance: this.display_distance, + display_distance_default: Math.max(this.toggle_display_distance_start, this.toggle_display_distance_end), + monitor_wrapping_scheme: 'horizontal', + monitor_wrapping_rotation_radians: this._monitorPlacements[index].rotationAngleRadians, + actor_to_display_ratios: actorToDisplayRatios, + actor_to_display_offsets: actorToDisplayOffsets + }); + containerActor.add_effect_with_name('viewport-effect', effect); + this.add_child(containerActor); + this.bind_property('imu-snapshots', effect, 'imu-snapshots', GObject.BindingFlags.DEFAULT); + this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); + this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); + + // in addition to rendering distance properly in the shader, the parent actor determines overlap based on child ordering + effect.connect('notify::is-closest', ((actor, _pspec) => { + if (actor.is_closest) this.set_child_above_sibling(containerActor, null); + }).bind(this)); + }).bind(this)); + + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 500, (() => { + if (this.imu_snapshots) { + const closestMonitorIndex = findClosestVector( + this.imu_snapshots.imu_data.splice(0, 4), + this._monitorsAsNormalizedVectors, this.closestMonitorIndex + ); + + // only switch if the closest monitor is greater than the previous closest by 25% + if (closestMonitorIndex !== -1 && (this.focused_monitor_index === undefined || this.focused_monitor_index !== closestMonitorIndex)) { + Globals.logger.log(`Switching to monitor ${closestMonitorIndex}`); + this.focused_monitor_index = closestMonitorIndex; + } + } + + return GLib.SOURCE_CONTINUE; + }).bind(this)); + + this._redraw_timeline = Clutter.Timeline.new_for_actor(this, 1000); + this._redraw_timeline.connect('new-frame', (() => { + // let's try to cap the forced redraw rate + if (this._last_redraw !== undefined && Date.now() - this._last_redraw < this._frametime_ms) return; + + Globals.data_stream.refresh_data(); + this.imu_snapshots = Globals.data_stream.imu_snapshots; + this.queue_redraw(); + this._last_redraw = Date.now(); + }).bind(this)); + this._redraw_timeline.set_repeat_count(-1); + this._redraw_timeline.start(); + + this._distance_ease_timeline = null; + this.connect('notify::toggle-display-distance-start', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::toggle-display-distance-end', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::display-distance', this._handle_display_distance_properties_change.bind(this)); + this._handle_display_distance_properties_change(); + } + + _handle_display_distance_properties_change() { + const distance_from_end = Math.abs(this.display_distance - this.toggle_display_distance_end); + const distance_from_start = Math.abs(this.display_distance - this.toggle_display_distance_start); + this._is_display_distance_at_end = distance_from_end < distance_from_start; + } + + _change_distance() { + this.display_distance = this._is_display_distance_at_end ? + this.toggle_display_distance_start : this.toggle_display_distance_end; + } + + destroy() { + if (this._redraw_timeline) { + this._redraw_timeline.stop(); + this._redraw_timeline = null; + } + super.destroy(); + } +}); \ No newline at end of file diff --git a/gnome/src/xrEffect.js b/gnome/src/xrEffect.js index 19533f1..18cc5fa 100644 --- a/gnome/src/xrEffect.js +++ b/gnome/src/xrEffect.js @@ -67,6 +67,7 @@ const shaderUniformLocations = { 'fov_widths': null, 'display_resolution': null, 'source_to_display_ratio': null, + 'texcoord_visible_area': null, 'curved_display': null, // only used by the reshade integration, but needs to be set to a default value by this effect @@ -132,6 +133,7 @@ function setIntermittentUniformVariables() { const displayRes = dataViewUint32Array(dataView, DISPLAY_RES); const sbsEnabled = dataViewUint8(dataView, SBS_ENABLED) !== 0; + const texture_actor = this.get_actor(); if (enabled) { const displayFov = dataViewFloat(dataView, DISPLAY_FOV); @@ -166,13 +168,23 @@ function setIntermittentUniformVariables() { texcoordXLimitsRight[1] = 0.75; } } + + Globals.logger.log(`texture_actor: ${texture_actor.x}, ${texture_actor.y}, ${texture_actor.width}, ${texture_actor.height}`); + + const texcoordVisibleArea = [ + this.texture_monitor_position.x / texture_actor.width, + this.texture_monitor_position.y / texture_actor.height, + (this.texture_monitor_position.x + this.target_monitor.width) / texture_actor.width, + (this.texture_monitor_position.y + this.target_monitor.height) / texture_actor.height + ] + const lensVector = [lensDistanceRatio, lensFromCenter, 0.0]; const lensVectorRight = [lensDistanceRatio, -lensFromCenter, 0.0]; // our overlay doesn't quite cover the full screen texture, which allows us to see some of the real desktop // underneath, so we trim three pixels around the entire edge of the texture - const trimWidthPercent = 3.0 / this.target_monitor.width; - const trimHeightPercent = 3.0 / this.target_monitor.height; + const trimWidthPercent = 3.0 / texture_actor.width; + const trimHeightPercent = 3.0 / texture_actor.height; // all these values are transferred directly, unmodified from the driver transferUniformFloat(this, 'look_ahead_cfg', dataView, LOOK_AHEAD_CFG); @@ -184,9 +196,10 @@ function setIntermittentUniformVariables() { setSingleFloat(this, 'half_fov_y_rads', halfFovYRads); this.set_uniform_float(shaderUniformLocations['fov_half_widths'], 2, fovHalfWidths); this.set_uniform_float(shaderUniformLocations['fov_widths'], 2, fovWidths); - setSingleFloat(this, 'curved_display', this.curved_display ? 1.0 : 0.0); this.set_uniform_float(shaderUniformLocations['texcoord_x_limits'], 2, texcoordXLimits); this.set_uniform_float(shaderUniformLocations['texcoord_x_limits_r'], 2, texcoordXLimitsRight); + this.set_uniform_float(shaderUniformLocations['texcoord_visible_area'], 4, texcoordVisibleArea); + setSingleFloat(this, 'curved_display', this.curved_display ? 1.0 : 0.0); this.set_uniform_float(shaderUniformLocations['lens_vector'], 3, lensVector); this.set_uniform_float(shaderUniformLocations['lens_vector_r'], 3, lensVectorRight); } @@ -209,7 +222,8 @@ function setIntermittentUniformVariables() { setSingleFloat(this, 'sideview_display_size', 1.0); this.set_uniform_float(shaderUniformLocations['display_resolution'], 2, displayRes); - this.set_uniform_float(shaderUniformLocations['source_to_display_ratio'], 2, [this.target_monitor.width/displayRes[0], this.target_monitor.height/displayRes[1]]); + Globals.logger.log_debug(`Source resolution ${texture_actor.width}x${texture_actor.height}`); + this.set_uniform_float(shaderUniformLocations['source_to_display_ratio'], 2, [texture_actor.width/displayRes[0], texture_actor.height/displayRes[1]]); } else if (dataView.byteLength !== 0) { throw new Error(`Invalid dataView.byteLength: ${dataView.byteLength} !== ${DATA_VIEW_LENGTH}`); } @@ -253,6 +267,12 @@ export const XREffect = GObject.registerClass({ 'Target framerate for this effect', GObject.ParamFlags.READWRITE, 30, 240, 60 ), + 'texture-monitor-position': GObject.ParamSpec.jsobject( + 'texture-monitor-position', + 'Texture Monitor Position', + 'Coordinates of the monitor relative to the target actor texture', + GObject.ParamFlags.READWRITE + ), 'display-distance': GObject.ParamSpec.double( 'display-distance', 'Display Distance', diff --git a/ui/src/connecteddevice.py b/ui/src/connecteddevice.py index efe638e..7fadbea 100644 --- a/ui/src/connecteddevice.py +++ b/ui/src/connecteddevice.py @@ -5,10 +5,13 @@ from .license import BREEZY_GNOME_FEATURES from .settingsmanager import SettingsManager from .shortcutdialog import bind_shortcut_settings from .statemanager import StateManager +from .virtualdisplay import VirtualMonitor from .xrdriveripc import XRDriverIPC import gettext +import logging _ = gettext.gettext +logger = logging.getLogger('breezy_ui') @Gtk.Template(resource_path='/com/xronlinux/BreezyDesktop/gtk/connected-device.ui') class ConnectedDevice(Gtk.Box): @@ -29,6 +32,7 @@ class ConnectedDevice(Gtk.Box): widescreen_mode_switch = Gtk.Template.Child() widescreen_mode_row = Gtk.Template.Child() curved_display_switch = Gtk.Template.Child() + add_virtual_display_button = Gtk.Template.Child() set_toggle_display_distance_start_button = Gtk.Template.Child() set_toggle_display_distance_end_button = Gtk.Template.Child() reassign_toggle_xr_effect_shortcut_button = Gtk.Template.Child() @@ -59,6 +63,7 @@ class ConnectedDevice(Gtk.Box): self.follow_mode_switch, self.follow_threshold_scale, self.curved_display_switch, + # self.add_virtual_display_button, self.set_toggle_display_distance_start_button, self.set_toggle_display_distance_end_button, self.movement_look_ahead_scale @@ -92,6 +97,7 @@ class ConnectedDevice(Gtk.Box): self.set_toggle_display_distance_start_button, self.set_toggle_display_distance_end_button ]) + self.add_virtual_display_button.connect('clicked', self.on_add_virtual_display) self.state_manager = StateManager.get_instance() self.state_manager.bind_property('follow-mode', self.follow_mode_switch, 'active', GObject.BindingFlags.DEFAULT) @@ -172,6 +178,12 @@ class ConnectedDevice(Gtk.Box): for widget in widgets: widget.connect('clicked', lambda *args, widget=widget: on_set_display_distance_toggle(widget)) reload_display_distance_toggle_button(widget) + + def on_add_virtual_display(self, widget): + VirtualMonitor(1920, 1080, self.on_virtual_display_ready).create() + + def on_virtual_display_ready(self): + logger.info("Virtual display ready") def _on_widget_destroy(self, widget): self.state_manager.unbind_property('follow-mode', self.follow_mode_switch, 'active') diff --git a/ui/src/gtk/connected-device.ui b/ui/src/gtk/connected-device.ui index 26f7063..cbfc7a8 100644 --- a/ui/src/gtk/connected-device.ui +++ b/ui/src/gtk/connected-device.ui @@ -84,6 +84,19 @@ + + + Virtual monitors + 2 + + + add-virtual-display + 3 + Add + + + + diff --git a/ui/src/meson.build b/ui/src/meson.build index 0a2b141..9768ce5 100644 --- a/ui/src/meson.build +++ b/ui/src/meson.build @@ -47,6 +47,7 @@ breezydesktop_sources = [ 'shortcutdialog.py', 'statemanager.py', 'time.py', + 'virtualdisplay.py', 'verify.py', 'window.py' ] diff --git a/ui/src/virtualdisplay.py b/ui/src/virtualdisplay.py new file mode 100644 index 0000000..1d09ab3 --- /dev/null +++ b/ui/src/virtualdisplay.py @@ -0,0 +1,71 @@ +#!/usr/bin/python3 + +import logging +import sys +import signal +import pydbus +import gi +gi.require_version('Gst', '1.0') +from gi.repository import GLib, GObject, Gst + +logger = logging.getLogger('breezy_ui') + +screen_cast_iface = 'org.gnome.Mutter.ScreenCast' +screen_cast_session_iface = 'org.gnome.Mutter.ScreenCast.Session' +screen_cast_stream_iface = 'org.gnome.Mutter.ScreenCast.Session' +gst_pipeline_format = "pipewiresrc path=%u ! video/x-raw,max-framerate=60/1,width=%d,height=%d ! fakesink sync=false" + + +def _screen_cast_session(): + bus = pydbus.SessionBus() + screen_cast = bus.get(screen_cast_iface, '/org/gnome/Mutter/ScreenCast') + session_path = screen_cast.CreateSession([]) + logger.info("session path: %s" % session_path) + screen_cast_session = bus.get(screen_cast_iface, session_path) + + return screen_cast_session + +class VirtualMonitor: + def __init__(self, width, height, on_ready_cb): + self.width = width + self.height = height + self.on_ready_cb = on_ready_cb + + Gst.init(None) + + def create(self): + session = _screen_cast_session() + stream_path = session.RecordVirtual({ + 'is-platform': GLib.Variant.new_boolean(True), + }) + logger.info("stream path: %s" % stream_path) + bus = pydbus.SessionBus() + self.stream = bus.get(screen_cast_iface, stream_path) + + self.stream.onPipeWireStreamAdded = self._on_pipewire_stream_added + + session.Start() + + def terminate(self): + if self.stream is not None: + self.stream.Stop() + + if self.pipeline is not None: + self.pipeline.send_event(Gst.Event.new_eos()) + self.pipeline.set_state(Gst.State.NULL) + + def _on_message(self, bus, message): + type = message.type + logger.info("message type: %s" % type) + if type == Gst.MessageType.EOS or type == Gst.MessageType.ERROR: + self.terminate() + + def _on_pipewire_stream_added(self, node_id): + logger.info("pipe wire stream added: %u" % node_id) + + self.pipeline = Gst.parse_launch(gst_pipeline_format % (node_id, self.width, self.height)) + self.pipeline.set_state(Gst.State.PLAYING) + self.pipeline.get_bus().connect('message', self._on_message) + self.pipeline.set_state(Gst.State.PAUSED) + + self.on_ready_cb() \ No newline at end of file