diff --git a/gnome/src/devicedatastream.js b/gnome/src/devicedatastream.js new file mode 100644 index 0000000..37d8d5a --- /dev/null +++ b/gnome/src/devicedatastream.js @@ -0,0 +1,174 @@ +import Gio from 'gi://Gio'; +import GObject from 'gi://GObject'; + +import Globals from './globals.js'; +import { + dataViewEnd, + dataViewUint8, + dataViewBigUint, + dataViewUint32Array, + dataViewUint8Array, + dataViewFloat, + dataViewFloatArray, + BOOL_SIZE, + FLOAT_SIZE, + UINT_SIZE, + UINT8_SIZE +} from "./ipc.js"; +import { isValidKeepAlive, getEpochSec, toSec } from "./time.js"; + +const IPC_FILE_PATH = "/dev/shm/breezy_desktop_imu"; +const KEEPALIVE_REFRESH_INTERVAL_SEC = 1; + +// the driver should be using the same data layout version +const DATA_LAYOUT_VERSION = 3; + +// DataView info: [offset, size, count] +const VERSION = [0, UINT8_SIZE, 1]; +const ENABLED = [dataViewEnd(VERSION), BOOL_SIZE, 1]; +const LOOK_AHEAD_CFG = [dataViewEnd(ENABLED), FLOAT_SIZE, 4]; +const DISPLAY_RES = [dataViewEnd(LOOK_AHEAD_CFG), UINT_SIZE, 2]; +const DISPLAY_FOV = [dataViewEnd(DISPLAY_RES), FLOAT_SIZE, 1]; +const LENS_DISTANCE_RATIO = [dataViewEnd(DISPLAY_FOV), FLOAT_SIZE, 1]; +const SBS_ENABLED = [dataViewEnd(LENS_DISTANCE_RATIO), BOOL_SIZE, 1]; +const CUSTOM_BANNER_ENABLED = [dataViewEnd(SBS_ENABLED), BOOL_SIZE, 1]; +const EPOCH_MS = [dataViewEnd(CUSTOM_BANNER_ENABLED), UINT_SIZE, 2]; +const IMU_QUAT_DATA = [dataViewEnd(EPOCH_MS), FLOAT_SIZE, 16]; +const IMU_PARITY_BYTE = [dataViewEnd(IMU_QUAT_DATA), UINT8_SIZE, 1]; +const DATA_VIEW_LENGTH = dataViewEnd(IMU_PARITY_BYTE); + +function checkParityByte(dataView) { + const parityByte = dataViewUint8(dataView, IMU_PARITY_BYTE); + let parity = 0; + const epochUint8 = dataViewUint8Array(dataView, EPOCH_MS); + const imuDataUint8 = dataViewUint8Array(dataView, IMU_QUAT_DATA); + for (let i = 0; i < epochUint8.length; i++) { + parity ^= epochUint8[i]; + } + for (let i = 0; i < imuDataUint8.length; i++) { + parity ^= imuDataUint8[i]; + } + return parityByte === parity; +} + +export const DeviceDataStream = GObject.registerClass({ + Properties: { + 'supported-device-connected': GObject.ParamSpec.boolean( + 'supported-device-connected', + 'Supported device connected', + 'Whether a supported device is connected', + GObject.ParamFlags.READWRITE, + false + ), + 'widescreen-mode-state': GObject.ParamSpec.boolean( + 'widescreen-mode-state', + 'Widescreen mode state', + 'The state of widescreen mode from the perspective of the driver', + GObject.ParamFlags.READWRITE, + false + ), + 'quaternion': GObject.ParamSpec.jsobject( + 'quaternion', + 'Quaternion', + 'Camera orientation quaternion', + GObject.ParamFlags.READWRITE + ), + } +}, class DeviceDataStream extends GObject.Object { + constructor(params = {}) { + super(params); + this.supported_device_connected = false; + this._ipc_file = Gio.file_new_for_path(IPC_FILE_PATH); + this._running = false; + this._device_data = null; + } + + start() { + this._running = true; + this._poll(); + } + + stop() { + this._running = false; + } + + // polling is just intended to keep supported_device_connected current, anything needing up-to-date imu data should + // trigger a refresh with the default flag + _poll() { + if (this._running) { + this.refresh_data(true); + setTimeout(this._poll.bind(this), 1000); + } + } + + // Refresh the data from the IPC file. if keepalive_only is true, we'll only check and update supported_device_connected if it + // hasn't been checked within KEEPALIVE_REFRESH_INTERVAL_SEC. + refresh_data(keepalive_only = false) { + if (!this._device_data?.imuData || !keepalive_only || getEpochSec() - this._device_data.imuDateMs > KEEPALIVE_REFRESH_INTERVAL_SEC) { + let data = this._ipc_file.load_contents(null); + if (data[0]) { + let buffer = new Uint8Array(data[1]).buffer; + let dataView = new DataView(buffer); + if (dataView.byteLength === DATA_VIEW_LENGTH) { + const imuDateMs = dataViewBigUint(dataView, EPOCH_MS); + const validKeepalive = isValidKeepAlive(toSec(imuDateMs)); + const imuData = dataViewFloatArray(dataView, IMU_QUAT_DATA); + const imuResetState = validKeepalive && imuData[0] === 0.0 && imuData[1] === 0.0 && imuData[2] === 0.0 && imuData[3] === 1.0; + const version = dataViewUint8(dataView, VERSION); + const enabled = dataViewUint8(dataView, ENABLED) !== 0 && version === DATA_LAYOUT_VERSION && validKeepalive; + const sbsEnabled = dataViewUint8(dataView, SBS_ENABLED) !== 0; + + // update the widescreen property if the state changes while still enabled, trigger "notify::" events + if (enabled && this.widescreen_mode_state !== sbsEnabled) this.widescreen_mode_state = sbsEnabled; + + if (!this._device_data) { + this._device_data = { + version, + enabled, + imuResetState, + displayRes: dataViewUint32Array(dataView, DISPLAY_RES), + sbsEnabled, + displayFov: dataViewFloat(dataView, DISPLAY_FOV), + lookAheadCfg: dataViewFloatArray(dataView, LOOK_AHEAD_CFG), + }; + } + + let success = keepalive_only; + let attempts = 0; + while (!success && attempts < 3) { + if (dataView.byteLength === DATA_VIEW_LENGTH) { + if (checkParityByte(dataView)) { + this._device_data.imuData = imuData; + this._device_data.imuDateMs = imuDateMs; + this.quaternion = { + x: imuData[0], + y: imuData[1], + z: imuData[2], + w: imuData[3] + }; + success = true; + } + } else if (dataView.byteLength !== 0) { + Globals.logger.log(`[ERROR] Invalid dataView.byteLength: ${dataView.byteLength} !== ${DATA_VIEW_LENGTH}`) + } + + if (!success && ++attempts < 3) { + data = this._ipc_file.load_contents(null); + if (data[0]) { + buffer = new Uint8Array(data[1]).buffer; + dataView = new DataView(buffer); + } + } + } + + if (success) { + // update the supported device connected property if the state changes, trigger "notify::" events + if (this.supported_device_connected !== validKeepalive) this.supported_device_connected = validKeepalive; + } + } + } else { + this.supported_device_connected = false; + } + } + } +}); \ No newline at end of file diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 9073a28..01b45b1 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -1,11 +1,13 @@ import Clutter from 'gi://Clutter' import Gio from 'gi://Gio'; import GLib from 'gi://GLib'; +import GObject from 'gi://GObject'; import Meta from 'gi://Meta'; import Shell from 'gi://Shell'; import St from 'gi://St'; import { CursorManager } from './cursormanager.js'; +import { DeviceDataStream } from './devicedatastream.js'; import Globals from './globals.js'; import { Logger } from './logger.js'; import { MonitorManager } from './monitormanager.js'; @@ -38,8 +40,9 @@ export default class BreezyDesktopExtension extends Extension { // Set/destroyed by enable/disable this._cursor_manager = null; + this._device_data_stream = null; this._monitor_manager = null; - this._xr_effect = null; + this.overlay_content = null; this._overlay = null; this._target_monitor = null; this._is_effect_running = false; @@ -76,6 +79,9 @@ export default class BreezyDesktopExtension extends Extension { Globals.extension_dir = this.path; this.settings.bind('debug', Globals.logger, 'debug', Gio.SettingsBindFlags.DEFAULT); + this._device_data_stream = new DeviceDataStream(); + this._device_data_stream.start(); + this._monitor_manager = new MonitorManager({ use_optimal_monitor_config: this.settings.get_boolean('use-optimal-monitor-config'), headset_as_primary: this.settings.get_boolean('headset-as-primary'), @@ -107,7 +113,7 @@ export default class BreezyDesktopExtension extends Extension { return GLib.SOURCE_REMOVE; } - if (this._check_driver_running() && target_monitor) { + if (this._device_data_stream.supported_device_connected && target_monitor) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes, // so we can remove this timeout_add no matter what. @@ -118,6 +124,7 @@ export default class BreezyDesktopExtension extends Extension { this._running_poller_id = undefined; return GLib.SOURCE_REMOVE; } else { + Globals.logger.log_debug(`BreezyDesktopExtension _poll_for_ready - device connected: ${this._device_data_stream.supported_device_connected}, target_monitor: ${!!target_monitor}`); return GLib.SOURCE_CONTINUE; } } catch (e) { @@ -186,7 +193,7 @@ export default class BreezyDesktopExtension extends Extension { if (target_monitor && this._running_poller_id === undefined) { this._target_monitor = target_monitor; - if (this._check_driver_running()) { + if (this._device_data_stream.supported_device_connected) { // Don't enable the effect yet if monitor updates are needed. // _setup will be triggered again since a !ready result means it will trigger monitor changes if (this._target_monitor_ready(target_monitor)) { @@ -200,27 +207,10 @@ export default class BreezyDesktopExtension extends Extension { this._poll_for_ready(); } } else { - Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, driver running: ${this._check_driver_running()}, target_monitor found: ${!!target_monitor}`); + Globals.logger.log_debug(`BreezyDesktopExtension _setup - Doing nothing, device connected: ${this._device_data_stream.supported_device_connected}, target_monitor found: ${!!target_monitor}`); } } - _check_driver_running() { - try { - if (!Globals.ipc_file) Globals.ipc_file = Gio.file_new_for_path(IPC_FILE_PATH); - if (Globals.ipc_file.query_exists(null)) { - const file_info = Globals.ipc_file.query_info(Gio.FILE_ATTRIBUTE_TIME_MODIFIED, Gio.FileQueryInfoFlags.NONE, null); - const file_modified_time = file_info.get_attribute_uint64(Gio.FILE_ATTRIBUTE_TIME_MODIFIED); - - // when the driver is running, the IMU file is updated at least 60x per second, do a strict check - return isValidKeepAlive(file_modified_time, true); - } - } catch (e) { - Globals.logger.log(`[ERROR] BreezyDesktopExtension _check_driver_running ${e.message}\n${e.stack}`); - } - - return false; - } - _needs_widescreen_monitor_update() { Globals.logger.log_debug('BreezyDesktopExtension _needs_widescreen_monitor_update'); const state = this._read_state(); @@ -251,22 +241,21 @@ export default class BreezyDesktopExtension extends Extension { this._overlay.opacity = 255; this._overlay.set_position(targetMonitor.x, targetMonitor.y); this._overlay.set_size(targetMonitor.width, targetMonitor.height); - // this._overlay.inhibit_culling(); // const textureSourceActor = Main.layoutManager.uiGroup; - const overlayContent = new TestActor({ + this.overlay_content = new TestActor({ monitors: [], - quaternion: { - x: 0.094, y: 0.079, z: 0.094, w: 0.988 - }, fov_degrees: 46.0, - width: 100, - height: 100, - 'z-position': 0 + // width: 100, + // height: 100, + width: targetMonitor.width, + height: targetMonitor.height, + display_distance: this.settings.get_double('display-distance'), + toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), + toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end') }); - // overlayContent.inhibit_culling(); - this._overlay.set_child(overlayContent); + this._overlay.set_child(this.overlay_content); Shell.util_set_hidden_from_pick(this._overlay, true); global.stage.add_child(this._overlay); @@ -281,30 +270,6 @@ export default class BreezyDesktopExtension extends Extension { clutterContainer ? 'actor-removed' : 'child-removed', this._handle_sibling_update.bind(this), ); - - // this._xr_effect = new XREffect({ - // target_monitor: targetMonitor, - // target_framerate: refreshRate, - // texture_monitor_position: { - // // x: targetMonitor.x - textureSourceActor.x, - // // y: targetMonitor.y - textureSourceActor.y - // x: 0, - // y: 0 - // }, - // display_distance: this.settings.get_double('display-distance'), - // toggle_display_distance_start: this.settings.get_double('toggle-display-distance-start'), - // toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end'), - // look_ahead_override: this.settings.get_int('look-ahead-override'), - // disable_anti_aliasing: this.settings.get_boolean('disable-anti-aliasing') - // }); - // this._xr_effect = new TestActorEffect({ - // quaternion: { - // x: 0.094, y: 0.079, z: 0.094, w: 0.988 - // }, - // fov_degrees: 46.0, - // width: targetMonitor.width, - // height: targetMonitor.height - // }); this._update_follow_threshold(this.settings); @@ -314,24 +279,35 @@ export default class BreezyDesktopExtension extends Extension { // this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); // this._supported_device_detected_connection = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); + this.overlay_content.renderMonitors(); + this._data_stream_connection = this._device_data_stream.bind_property( + 'quaternion', + this.overlay_content, + 'quaternion', + GObject.BindingFlags.DEFAULT + ); - // this._distance_binding = this.settings.bind('display-distance', this._xr_effect, 'display-distance', Gio.SettingsBindFlags.DEFAULT) - this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)) - this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)) + this._distance_binding = this.settings.bind('display-distance', this.overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); + this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)); + this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)); // this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) - // this._start_binding = this.settings.bind('toggle-display-distance-start', this._xr_effect, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) - // this._end_binding = this.settings.bind('toggle-display-distance-end', this._xr_effect, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) + this._start_binding = this.settings.bind('toggle-display-distance-start', this.overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) + this._end_binding = this.settings.bind('toggle-display-distance-end', this.overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) // this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) // this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); // this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); // this._disable_anti_aliasing_binding = this.settings.bind('disable-anti-aliasing', this._xr_effect, 'disable-anti-aliasing', Gio.SettingsBindFlags.DEFAULT); - // this._ui_clone.add_effect_with_name('xr-desktop', this._xr_effect); Meta.disable_unredirect_for_display(global.display); + + global.stage.connect('before-paint', (() => { + this._device_data_stream.refresh_data(); + this._overlay.queue_redraw(); + }).bind(this)); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); - // this._add_settings_keybinding('toggle-display-distance-shortcut', this._xr_effect._change_distance.bind(this._xr_effect)); + this._add_settings_keybinding('toggle-display-distance-shortcut', this.overlay_content._change_distance.bind(this.overlay_content)); this._add_settings_keybinding('toggle-follow-shortcut', this._toggle_follow_mode.bind(this)); } catch (e) { Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); @@ -462,12 +438,12 @@ export default class BreezyDesktopExtension extends Extension { } _update_widescreen_mode_from_settings(settings, event) { - const value = settings.get_boolean('widescreen-mode'); - Globals.logger.log_debug(`BreezyDesktopExtension _update_widescreen_mode_from_settings ${value}`); - if (value !== undefined && value !== this._xr_effect.widescreen_mode_state) { - this._request_sbs_mode_change(value); - } else - Globals.logger.log_debug('effect.widescreen_mode_state already matched setting'); + // const value = settings.get_boolean('widescreen-mode'); + // Globals.logger.log_debug(`BreezyDesktopExtension _update_widescreen_mode_from_settings ${value}`); + // if (value !== undefined && value !== this._xr_effect.widescreen_mode_state) { + // this._request_sbs_mode_change(value); + // } else + // Globals.logger.log_debug('effect.widescreen_mode_state already matched setting'); } _update_widescreen_mode_from_state(effect, _pspec) { @@ -538,9 +514,7 @@ export default class BreezyDesktopExtension extends Extension { this._actor_removed_connection = null; } if (this._overlay) { - // if (this._xr_effect) this._xr_effect.cleanup(); - if (this._ui_clone) this._ui_clone.remove_effect_by_name('xr-desktop'); - this._ui_clone = null; + this.overlay_content = null; global.stage.remove_child(this._overlay); this._overlay.destroy(); @@ -554,6 +528,10 @@ export default class BreezyDesktopExtension extends Extension { this.settings.disconnect(this._distance_connection); this._distance_connection = null; } + if (this._data_stream_connection) { + this._device_data_stream.unbind(this._data_stream_connection); + this._data_stream_connection = null; + } if (this._follow_threshold_connection) { this.settings.disconnect(this._follow_threshold_connection); this._follow_threshold_connection = null; @@ -586,17 +564,17 @@ export default class BreezyDesktopExtension extends Extension { this.settings.unbind(this._disable_anti_aliasing_binding); this._disable_anti_aliasing_binding = null; } - // if (this._xr_effect) { - // if (this._widescreen_mode_effect_state_connection) { - // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); - // this._widescreen_mode_effect_state_connection = null; - // } - // if (this._supported_device_detected_connection) { - // this._xr_effect.disconnect(this._supported_device_detected_connection); - // this._supported_device_detected_connection = null; - // } - // this._xr_effect = null; - // } + if (this.overlay_content) { + // if (this._widescreen_mode_effect_state_connection) { + // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); + // this._widescreen_mode_effect_state_connection = null; + // } + // if (this._supported_device_detected_connection) { + // this._xr_effect.disconnect(this._supported_device_detected_connection); + // this._supported_device_detected_connection = null; + // } + this.overlay_content = null; + } if (this._cursor_manager) { this._cursor_manager.disable(); this._cursor_manager = null; @@ -618,6 +596,12 @@ export default class BreezyDesktopExtension extends Extension { Globals.logger.log_debug('BreezyDesktopExtension disable'); this._effect_disable(); this._target_monitor = null; + + if (this._device_data_stream) { + this._device_data_stream.stop(); + this._device_data_stream = null; + } + if (this._monitor_manager) { if (this._optimal_monitor_config_binding) { this.settings.unbind(this._optimal_monitor_config_binding); diff --git a/gnome/src/monitormanager.js b/gnome/src/monitormanager.js index ca8a6a5..b0ceea6 100644 --- a/gnome/src/monitormanager.js +++ b/gnome/src/monitormanager.js @@ -35,7 +35,7 @@ function getDisplayConfigProxy(extPath) { xml = new TextDecoder().decode(bytes); } } catch (e) { - Globals.logger.log('ERROR: failed to load DisplayConfig interface XML'); + Globals.logger.log('[ERROR] failed to load DisplayConfig interface XML'); throw e; } cachedDisplayConfigProxy = Gio.DBusProxy.makeProxyWrapper(xml); diff --git a/gnome/src/testactor.js b/gnome/src/testactor.js index a1751ad..86064c7 100644 --- a/gnome/src/testactor.js +++ b/gnome/src/testactor.js @@ -1,14 +1,193 @@ - import Clutter from 'gi://Clutter' import Cogl from 'gi://Cogl'; +import GLib from 'gi://GLib'; import GObject from 'gi://GObject'; import Shell from 'gi://Shell'; import * as Main from 'resource:///org/gnome/shell/ui/main.js'; import Globals from './globals.js'; +function applyQuaternionToVector(vector, quaternion) { + const t = [ + 2.0 * (quaternion[1] * vector[2] - quaternion[2] * vector[1]), + 2.0 * (quaternion[2] * vector[0] - quaternion[0] * vector[2]), + 2.0 * (quaternion[0] * vector[1] - quaternion[1] * vector[0]) + ]; + return [ + vector[0] + quaternion[3] * t[0] + quaternion[1] * t[2] - quaternion[2] * t[1], + vector[1] + quaternion[3] * t[1] + quaternion[2] * t[0] - quaternion[0] * t[2], + vector[2] + quaternion[3] * t[2] + quaternion[0] * t[1] - quaternion[1] * t[0] + ]; +} + +/** + * Find the vector in the array that's closest to the quaternion rotation + * + * @param {number[]} quaternion - Reference quaternion [w, x, y, z] + * @param {number[][]} vectors - Array of vectors [x, y, z] to search from + * @returns {number} Index of the closest vector, if it surpasses the previous closest index by a certain margin, otherwise the previous index + */ +function findClosestVector(quaternion, vectors, previousClosestIndex) { + + const lookVector = [1.0, 0.0, 0.0]; // NWU vector pointing to the center of the screen + const rotatedLookVector = applyQuaternionToVector(lookVector, [quaternion.x, quaternion.y, quaternion.z, quaternion.w]); + Globals.logger.log(`\t\t\tQuaternion: ${JSON.stringify(quaternion)}`); + Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); + + let closestIndex = -1; + let closestDistance = Infinity; + let previousDistance = Infinity; + + // find the vector closest to the rotated look vector + vectors.forEach((vector, index) => { + const distance = Math.acos( + Math.min(1.0, Math.max(-1.0, vector[0] * rotatedLookVector[0] + vector[1] * rotatedLookVector[1] + vector[2] * rotatedLookVector[2])) + ); + + if (previousClosestIndex === index) { + previousDistance = distance; + } + + Globals.logger.log(`\t\t\tMonitor ${index} distance: ${distance}`); + if (distance < closestDistance) { + closestIndex = index; + closestDistance = distance; + } + }); + + Globals.logger.log(`\t\t\tClosest monitor: ${closestIndex}, distance: ${closestDistance}`); + + // only switch if the closest monitor is greater than the previous closest by 25% + if (previousClosestIndex !== undefined && closestIndex !== previousClosestIndex && closestDistance * 1.25 > previousDistance) { + return previousClosestIndex; + } + + return closestIndex; +} + +function degreesToRadians(degrees) { + return degrees * Math.PI / 180.0; +} + +function radiansToDegrees(radians) { + return radians * 180.0 / Math.PI; +} + +/*** + * @returns {Object} - containing `center` and `end` radians + */ +function monitorWrap(radiusPixels, previousMonitorEndRadians, monitorPixels) { + const monitorHalfPixels = monitorPixels / 2; + const monitorHalfRadians = Math.asin(monitorHalfPixels / radiusPixels); + const centerRadians = previousMonitorEndRadians + monitorHalfRadians; + return { + center: centerRadians, + end: centerRadians + monitorHalfRadians + } +} + +/** + * Convert the given monitor details into NWU vectors pointing to the center of each monitor. + * + * @param {Object} fovDetails - contains reference fovDegrees (diagonal), widthPixels, heightPixels + * @param {Object[]} monitorDetailsList - contains x, y, width, height (coordinates from top-left) + * @param {string} monitorWrappingScheme - horizontal, vertical, none + * @returns {number[]} - Vector [x, y, z] + */ +function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme) { + const aspect = fovDetails.widthPixels / fovDetails.heightPixels; + const fovVerticalRadians = degreesToRadians(fovDetails.fovDegrees / Math.sqrt(1 + aspect * aspect)); + + // NWU vectors pointing to the center of the screen for each monitor + const monitorVectors = []; + + if (monitorWrappingScheme === 'horizontal') { + // monitors wrap around us horizontally + const fovHorizontalRadians = fovVerticalRadians * aspect; + + // radius is the hypothenuse of the triangle where the opposite side is half the width of the reference fov screen + const radius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); + + let previousMonitorEndRadians = -fovHorizontalRadians / 2; + monitorDetailsList.forEach(monitorDetails => { + const monitorWrapDetails = monitorWrap(radius, previousMonitorEndRadians, monitorDetails.width); + previousMonitorEndRadians = monitorWrapDetails.end; + + monitorVectors.push([ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + radius * Math.cos(monitorWrapDetails.center), + + // west is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -radius * Math.sin(monitorWrapDetails.center), + + // up is flat when wrapping horizontally + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ]); + }); + } else if (monitorWrappingScheme === 'vertical') { + // monitors wrap around us vertically + + // radius is the hypothenuse of the triangle where the opposite side is half the height of the reference fov screen + const radius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); + + let previousMonitorEndRadians = -fovVerticalRadians / 2; + monitorDetailsList.forEach(monitorDetails => { + const monitorWrapDetails = monitorWrap(radius, previousMonitorEndRadians, monitorDetails.height); + previousMonitorEndRadians = monitorWrapDetails.end; + + monitorVectors.push([ + // north is adjacent where radius is the hypotenuse, using monitorWrapDetails.center as the radians + radius * Math.cos(monitorWrapDetails.center), + + // west is flat when wrapping vertically + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + + // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians + -radius * Math.sin(monitorWrapDetails.center) + ]); + }); + } else { + // monitors make a flat wall in front of us, no wrapping + monitorDetailsList.forEach(monitorDetails => { + monitorVectors.push([ + fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2), + -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), + -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) + ]); + }); + } + + return monitorVectors; +} + +function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { + if (monitorWrappingScheme === 'horizontal') { + // monitors wrap around us horizontally + return { + angle: radiansToDegrees(Math.atan2(vector[1], vector[0])), + axis: Clutter.RotateAxis.Y_AXIS + }; + } else if (monitorWrappingScheme === 'vertical') { + // monitors wrap around us vertically + return { + angle: radiansToDegrees(Math.atan2(vector[2], vector[0])), + axis: Clutter.RotateAxis.X_AXIS + } + } else { + // no rotation + return undefined; + } +} + export const TestActorEffect = GObject.registerClass({ Properties: { + 'monitor-index': GObject.ParamSpec.int( + 'monitor-index', + 'Monitor Index', + 'Index of the monitor that this effect is applied to', + GObject.ParamFlags.READWRITE, + 0, 100, 0 + ), 'quaternion': GObject.ParamSpec.jsobject( 'quaternion', 'Quaternion', @@ -35,140 +214,126 @@ export const TestActorEffect = GObject.registerClass({ 'Height of the viewport', GObject.ParamFlags.READWRITE, 1, 10000, 1080 - ) + ), + 'monitor-wrapping-scheme': GObject.ParamSpec.string( + 'monitor-wrapping-scheme', + 'Monitor Wrapping Scheme', + 'How the monitors are wrapped around the viewport', + GObject.ParamFlags.READWRITE, + 'horizontal', ['horizontal', 'vertical', 'none'] + ), + 'focused-monitor-index': GObject.ParamSpec.int( + 'focused-monitor-index', + 'Focused Monitor Index', + 'Index of the monitor that is currently focused', + GObject.ParamFlags.READWRITE, + 0, 100, 0 + ), + 'display-distance': GObject.ParamSpec.double( + 'display-distance', + 'Display Distance', + 'Distance of the display from the camera', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.0 + ), + 'toggle-display-distance-start': GObject.ParamSpec.double( + 'toggle-display-distance-start', + 'Display distance start', + 'Start distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), + 'toggle-display-distance-end': GObject.ParamSpec.double( + 'toggle-display-distance-end', + 'Display distance end', + 'End distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), } }, class TestActorEffect extends Shell.GLSLEffect { - constructor(params = {}) { - super(params); + perspective(fovDiagonalRadians, aspect, near, far) { + // compute horizontal fov given diagonal fov and aspect ratio + const h = Math.sqrt(aspect * aspect + 1); + const fovRadians = fovDiagonalRadians / h * aspect; + console.log(`fovRadians: ${fovRadians}`); - // Compute the projection matrix - let aspectRatio = this.width / this.height; - let fovRadians = this.fov_degrees * (Math.PI / 180); - let near = 0.1; - let far = 1000.0; - - let projectionMatrix = this._computeProjectionMatrix(fovRadians, aspectRatio, near, far); - Globals.logger.log(JSON.stringify(projectionMatrix)); - - // Compute the view matrix from the quaternion - let viewMatrix = this._computeViewMatrixFromQuaternion(this.quaternion); - Globals.logger.log(JSON.stringify(viewMatrix)); - - let rotationMatrix = this._createRotationMatrix(this.quaternion); - Globals.logger.log(JSON.stringify(rotationMatrix)); - } - - _computeProjectionMatrix(fovRadians, aspect, near, far) { - let f = 1.0 / Math.tan(fovRadians / 2); - let nf = 1 / (near - far); - - let projectionMatrix = [ - f / aspect, 0, 0, 0, - 0, f, 0, 0, - 0, 0, (far + near) * nf, -1, - 0, 0, (2 * far * near) * nf, 0 - ]; - - return projectionMatrix; - } - - _computeViewMatrixFromQuaternion(q) { - let x = q.x, y = q.y, z = q.z, w = q.w; - - let x2 = x + x; - let y2 = y + y; - let z2 = z + z; - - let xx = x * x2; - let xy = x * y2; - let xz = x * z2; - let yy = y * y2; - let yz = y * z2; - let zz = z * z2; - let wx = w * x2; - let wy = w * y2; - let wz = w * z2; - - let viewMatrix = [ - 1 - (yy + zz), xy - wz, xz + wy, 0, - xy + wz, 1 - (xx + zz), yz - wx, 0, - xz - wy, yz + wx, 1 - (xx + yy), 0, - 0, 0, 0, 1 - ]; - - // Invert the view matrix (since it's from camera space) - // For rotation matrices, the inverse is the transpose - let inverseViewMatrix = [ - viewMatrix[0], viewMatrix[4], viewMatrix[8], 0, - viewMatrix[1], viewMatrix[5], viewMatrix[9], 0, - viewMatrix[2], viewMatrix[6], viewMatrix[10], 0, - 0, 0, 0, 1 - ]; - - return viewMatrix; - } - - _createRotationMatrix(q) { - // Normalize the quaternion - const len = Math.sqrt( - q.x * q.x + - q.y * q.y + - q.z * q.z + - q.w * q.w - ); - const x = q.x / len; - const y = q.y / len; - const z = q.z / len; - const w = q.w / len; - - // Compute matrix elements - const x2 = x * x; - const y2 = y * y; - const z2 = z * z; - const xy = x * y; - const xz = x * z; - const yz = y * z; - const wx = w * x; - const wy = w * y; - const wz = w * z; - - // Create rotation matrix + const f = 1.0 / Math.tan(fovRadians / 2.0); + const range = far - near; + return [ - 1.0 - 2.0 * (y2 + z2), // m00 - 2.0 * (xy - wz), // m01 - 2.0 * (xz + wy), // m02 - 0.0, // m03 - - 2.0 * (xy + wz), // m10 - 1.0 - 2.0 * (x2 + z2), // m11 - 2.0 * (yz - wx), // m12 - 0.0, // m13 - - 2.0 * (xz - wy), // m20 - 2.0 * (yz + wx), // m21 - 1.0 - 2.0 * (x2 + y2), // m22 - 0.0, // m23 - - 0.0, // m30 - 0.0, // m31 - 0.0, // m32 - 1.0 // m33 + f / aspect, 0, 0, 0, + 0, f, 0, 0, + 0, 0, - (far + near) / range, -1, + 0, 0, - (2.0 * near * far) / range, 0 ]; } vfunc_build_pipeline() { const declarations = ` - uniform mat4 u_rotation_matrix; - uniform mat4 u_view_matrix; + uniform vec4 u_quaternion; uniform mat4 u_projection_matrix; + uniform float u_display_north_offset; + + vec4 applyQuaternionToVector(vec4 v, vec4 q) { + vec3 t = 2.0 * cross(q.xyz, v.xyz); + vec3 rotated = v.xyz + q.w * t + cross(q.xyz, t); + return vec4(rotated, v.w); + } `; const main = ` vec4 world_pos = cogl_position_in; - world_pos = u_rotation_matrix * world_pos; + + // // move pixel space to texcoord space + // world_pos.x = (world_pos.x / 192.0); + // world_pos.y = (world_pos.y / 108.0); + + // float displayAspectRatio = 1920.0 / 1080.0; + // float diagToVertRatio = sqrt(pow(displayAspectRatio, 2) + 1); + // float halfFovZRads = radians(46.0 / diagToVertRatio) / 2.0; + // float halfFovYRads = halfFovZRads * displayAspectRatio; + // vec2 fovHalfWidths = vec2(tan(halfFovYRads), tan(halfFovZRads)); + // vec2 fovWidths = fovHalfWidths * 2.0; + + // float vec_y = -world_pos.x * fovWidths.x + fovHalfWidths.x; + // float vec_z = -world_pos.y * fovWidths.y + fovHalfWidths.y; + // vec4 look_vector = vec4(1.0, vec_y, vec_z, 1.0); + // // vec3 rotated_vector = applyQuaternionToVector(look_vector, u_quaternion).xyz; + // vec3 rotated_vector = look_vector.xyz; + + // // scale back to the screen distance + // rotated_vector /= rotated_vector.x; + // cogl_position_out = vec4( + // ((fovHalfWidths.x - rotated_vector.y) / fovWidths.x) * 2.0 - 1.0, + // ((fovHalfWidths.y - rotated_vector.z) / fovWidths.y) * 2.0 - 1.0, + // 0.0, + // 1.0 + // ); + + // float z_orig = world_pos.z; + // world_pos.z -= z_orig / 1920.0; + // world_pos.x /= 2.0; + // world_pos *= u_display_north_offset; + world_pos = applyQuaternionToVector(world_pos, u_quaternion); + // world_pos /= u_display_north_offset; + // world_pos.x *= 2.0; + // world_pos.z += z_orig / 1920.0; world_pos = cogl_modelview_matrix * world_pos; cogl_position_out = cogl_projection_matrix * world_pos; + + // cogl_position_out.x = world_pos.x / 103.4; + // cogl_position_out.y = world_pos.y / 29.075; + // cogl_position_out.z = -1.0; + // cogl_position_out.w = 1.0; + cogl_tex_coord_out[0] = cogl_tex_coord_in; ` @@ -176,21 +341,23 @@ export const TestActorEffect = GObject.registerClass({ } vfunc_paint_target(node, paintContext) { - // Compute the projection matrix - let aspectRatio = this.width / this.height; - let fovRadians = this.fov_degrees * (Math.PI / 180); - let near = 0.1; - let far = 1000.0; + if (!this._initialized) { + const aspect = this.get_actor().width / this.get_actor().height; + const projection_matrix = this.perspective( + this.fov_degrees * Math.PI / 180.0, + aspect, + 0.0001, + 1000.0 + ); + Globals.logger.log(`aspect: ${aspect}, fov: ${this.fov_degrees}, width: ${this.get_actor().width}, height: ${this.get_actor().height}, projection matrix: ${JSON.stringify(projection_matrix)}`); + this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projection_matrix); + this._initialized = true; + } - let projectionMatrix = this._computeProjectionMatrix(fovRadians, aspectRatio, near, far); + this.set_uniform_float(this.get_uniform_location("u_display_north_offset"), 1, [this.focused_monitor_index === this.monitor_index ? this.display_distance : this.toggle_display_distance_start]); - // Compute the view matrix from the quaternion - let viewMatrix = this._computeViewMatrixFromQuaternion(this.quaternion); - - // Set up the uniforms - this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projectionMatrix); - this.set_uniform_matrix(this.get_uniform_location("u_view_matrix"), false, 4, viewMatrix); - this.set_uniform_matrix(this.get_uniform_location("u_rotation_matrix"), false, 4, this._createRotationMatrix(this.quaternion)); + // NUW to east-up-south conversion, inverted + this.set_uniform_float(this.get_uniform_location("u_quaternion"), 4, [this.quaternion.y, -this.quaternion.z, this.quaternion.x, this.quaternion.w]); this.get_pipeline().set_layer_filters( 0, @@ -223,87 +390,158 @@ export const TestActor = GObject.registerClass({ GObject.ParamFlags.READWRITE, 30.0, 100.0, 46.0 ), - 'width': GObject.ParamSpec.int( - 'width', - 'Width', - 'Width of the viewport', + 'focused-monitor-index': GObject.ParamSpec.int( + 'focused-monitor-index', + 'Focused Monitor Index', + 'Index of the monitor that is currently focused', GObject.ParamFlags.READWRITE, - 1, 10000, 1920 + 0, 100, 0 ), - 'height': GObject.ParamSpec.int( - 'height', - 'Height', - 'Height of the viewport', + 'display-size': GObject.ParamSpec.double( + 'display-size', + 'Display size', + 'Size of the display', GObject.ParamFlags.READWRITE, - 1, 10000, 1080 - ) + 0.2, + 2.5, + 1.0 + ), + 'display-distance': GObject.ParamSpec.double( + 'display-distance', + 'Display Distance', + 'Distance of the display from the camera', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.0 + ), + 'toggle-display-distance-start': GObject.ParamSpec.double( + 'toggle-display-distance-start', + 'Display distance start', + 'Start distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), + 'toggle-display-distance-end': GObject.ParamSpec.double( + 'toggle-display-distance-end', + 'Display distance end', + 'End distance when using the "change distance" shortcut.', + GObject.ParamFlags.READWRITE, + 0.2, + 2.5, + 1.05 + ), } }, class TestActor extends Clutter.Actor { - constructor(params = {}) { - super({...params}); - - // Set the size of the viewport (implicitly provides aspect ratio) - // You can set the size when adding this actor to the stage - // this.set_size(this.width, this.height); - - // Create the monitor actors - this._createMonitorActors(); - - // Apply the shader effect to this viewport actor - // this._applyShaderEffect(); - } - - _createMonitorActors() { - Main.layoutManager.monitors.forEach((monitor, index) => { + renderMonitors() { + this.monitorsAsVectors = monitorsToVectors( + { + fovDegrees: this.fov_degrees, + widthPixels: this.width, + heightPixels: this.height + }, + Main.layoutManager.monitors.map(monitor => ({ + x: monitor.x, + y: monitor.y, + width: monitor.width, + height: monitor.height + })), + 'horizontal' + ); + this.monitorAsNormalizedVectors = this.monitorsAsVectors.map(vector => { + const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); + return [vector[0] / length, vector[1] / length, vector[2] / length]; + }); + + Main.layoutManager.monitors.forEach(((monitor, index) => { // if (index === 0) return; Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); + // this is in NWU coordinates + const monitorVector = this.monitorsAsVectors[index]; + const monitorRotation = monitorVectorToRotationAngle(monitorVector, 'horizontal'); + Globals.logger.log_debug(`\t\t\tMonitor ${index} vector: ${monitorVector} rotation: ${JSON.stringify(monitorRotation)}`); + + // actor coordinates are east-up-south const containerActor = new Clutter.Actor({ - x: -monitor.x, - y: monitor.y, - 'z-position': -500, + x: -monitorVector[1], + y: -monitorVector[2], + 'z-position': -monitorVector[0], width: monitor.width, height: monitor.height, reactive: false }); + // Create a clone of the stage content for this monitor const monitorClone = new Clutter.Clone({ source: Main.layoutManager.uiGroup, reactive: false }); - monitorClone.x = -monitor.x; + monitorClone.x = -containerActor.x; // monitorActor.y = 0; - - // Set the size and position of the clone to match the monitor - // monitorActor.set_size(monitor.width, monitor.height); - - // // Apply clipping to show only this monitor's area monitorClone.set_clip(monitor.x, 0, monitor.width, monitor.height); - // Position the monitor actor within the 3D scene - // monitorActor.set_position(0, 0); - - // // For 3D positioning, we might want to center the monitors around (0,0,0) - // // Adjust positions accordingly - // monitorActor.set_translation(monitor.x, monitor.y, 1.0); - // Add the monitor actor to the scene containerActor.add_child(monitorClone); - containerActor.add_effect_with_name('viewport-effect', new TestActorEffect({ + containerActor.set_pivot_point(0.5, 0.5); + containerActor.set_rotation_angle(monitorRotation.axis, monitorRotation.angle); + const effect = new TestActorEffect({ quaternion: this.quaternion, fov_degrees: this.fov_degrees, - width: this.width, - height: this.height - })); + monitor_index: index, + display_distance: this.toggle_display_distance_start + }); + containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); - }); + this.bind_property('quaternion', effect, 'quaternion', GObject.BindingFlags.DEFAULT); + this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); + this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); + }).bind(this)); + + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 1000, (() => { + if (this.quaternion) { + const closestMonitorIndex = findClosestVector(this.quaternion, this.monitorAsNormalizedVectors, this.closestMonitorIndex); + + // only switch if the closest monitor is greater than the previous closest by 25% + if (this.closestMonitorIndex === undefined || this.closestMonitorIndex !== closestMonitorIndex) { + Globals.logger.log(`Switching to monitor ${closestMonitorIndex}`); + this.closestMonitorIndex = closestMonitorIndex; + } + } + + return GLib.SOURCE_CONTINUE; + }).bind(this)); + + this._distance_ease_timeline = null; + this.connect('notify::toggle-display-distance-start', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::toggle-display-distance-end', this._handle_display_distance_properties_change.bind(this)); + this.connect('notify::display-distance', this._handle_display_distance_properties_change.bind(this)); + this._handle_display_distance_properties_change(); + } + + _handle_display_distance_properties_change() { + const distance_from_end = Math.abs(this.display_distance - this.toggle_display_distance_end); + const distance_from_start = Math.abs(this.display_distance - this.toggle_display_distance_start); + this._is_display_distance_at_end = distance_from_end < distance_from_start; } - // _applyShaderEffect() { - // const glslEffect = + _change_distance() { + if (this._distance_ease_timeline?.is_playing()) this._distance_ease_timeline.stop(); - // // Apply the shader effect to this viewport actor - // this.add_effect_with_name('viewport-effect', glslEffect); - // } + this._distance_ease_start = this.display_distance; + this._distance_ease_timeline = Clutter.Timeline.new_for_actor(this, 250); + + const toggle_display_distance_target = this._is_display_distance_at_end ? + this.toggle_display_distance_start : this.toggle_display_distance_end; + this._distance_ease_timeline.connect('new-frame', () => { + this.display_distance = this._distance_ease_start + + this._distance_ease_timeline.get_progress() * + (toggle_display_distance_target - this._distance_ease_start); + }); + + this._distance_ease_timeline.start(); + } }); \ No newline at end of file