From 439d2fccce2eda46c056f52f31751fcede7128fc Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Wed, 22 Jan 2025 18:19:43 -0800 Subject: [PATCH] WIP --- gnome/src/devicedatastream.js | 18 ++- gnome/src/extension.js | 55 +++++---- gnome/src/testactor.js | 224 ++++++++++++++++++++++++++-------- 3 files changed, 211 insertions(+), 86 deletions(-) diff --git a/gnome/src/devicedatastream.js b/gnome/src/devicedatastream.js index 37d8d5a..ed7ac0d 100644 --- a/gnome/src/devicedatastream.js +++ b/gnome/src/devicedatastream.js @@ -67,12 +67,12 @@ export const DeviceDataStream = GObject.registerClass({ GObject.ParamFlags.READWRITE, false ), - 'quaternion': GObject.ParamSpec.jsobject( - 'quaternion', - 'Quaternion', - 'Camera orientation quaternion', + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', GObject.ParamFlags.READWRITE - ), + ) } }, class DeviceDataStream extends GObject.Object { constructor(params = {}) { @@ -140,11 +140,9 @@ export const DeviceDataStream = GObject.registerClass({ if (checkParityByte(dataView)) { this._device_data.imuData = imuData; this._device_data.imuDateMs = imuDateMs; - this.quaternion = { - x: imuData[0], - y: imuData[1], - z: imuData[2], - w: imuData[3] + this.imu_snapshots = { + imu_data: imuData, + timestamp_ms: imuDateMs }; success = true; } diff --git a/gnome/src/extension.js b/gnome/src/extension.js index 9f81e5a..5751821 100644 --- a/gnome/src/extension.js +++ b/gnome/src/extension.js @@ -42,7 +42,7 @@ export default class BreezyDesktopExtension extends Extension { this._cursor_manager = null; this._device_data_stream = null; this._monitor_manager = null; - this.overlay_content = null; + this._overlay_content = null; this._overlay = null; this._target_monitor = null; this._is_effect_running = false; @@ -62,6 +62,8 @@ export default class BreezyDesktopExtension extends Extension { this._headset_as_primary_binding = null; this._actor_added_connection = null; this._actor_removed_connection = null; + this._data_stream_connection = null; + this._stage_redraw_connection = null; if (!Globals.logger) { Globals.logger = new Logger({ @@ -243,7 +245,7 @@ export default class BreezyDesktopExtension extends Extension { this._overlay.set_size(targetMonitor.width, targetMonitor.height); // const textureSourceActor = Main.layoutManager.uiGroup; - this.overlay_content = new TestActor({ + this._overlay_content = new TestActor({ monitors: [], fov_degrees: 46.0, // width: 100, @@ -255,7 +257,7 @@ export default class BreezyDesktopExtension extends Extension { toggle_display_distance_end: this.settings.get_double('toggle-display-distance-end') }); - this._overlay.set_child(this.overlay_content); + this._overlay.set_child(this._overlay_content); Shell.util_set_hidden_from_pick(this._overlay, true); global.stage.add_child(this._overlay); @@ -279,21 +281,21 @@ export default class BreezyDesktopExtension extends Extension { // this._widescreen_mode_effect_state_connection = this._xr_effect.connect('notify::widescreen-mode-state', this._update_widescreen_mode_from_state.bind(this)); // this._supported_device_detected_connection = this._xr_effect.connect('notify::supported-device-detected', this._handle_supported_device_change.bind(this)); - this.overlay_content.renderMonitors(); + this._overlay_content.renderMonitors(); this._data_stream_connection = this._device_data_stream.bind_property( - 'quaternion', - this.overlay_content, - 'quaternion', + 'imu-snapshots', + this._overlay_content, + 'imu-snapshots', GObject.BindingFlags.DEFAULT ); - this._distance_binding = this.settings.bind('display-distance', this.overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); + this._distance_binding = this.settings.bind('display-distance', this._overlay_content, 'display-distance', Gio.SettingsBindFlags.DEFAULT); this._distance_connection = this.settings.connect('changed::display-distance', this._update_display_distance.bind(this)); this._follow_threshold_connection = this.settings.connect('changed::follow-threshold', this._update_follow_threshold.bind(this)); // this._widescreen_mode_settings_connection = this.settings.connect('changed::widescreen-mode', this._update_widescreen_mode_from_settings.bind(this)) - this._start_binding = this.settings.bind('toggle-display-distance-start', this.overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) - this._end_binding = this.settings.bind('toggle-display-distance-end', this.overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) + this._start_binding = this.settings.bind('toggle-display-distance-start', this._overlay_content, 'toggle-display-distance-start', Gio.SettingsBindFlags.DEFAULT) + this._end_binding = this.settings.bind('toggle-display-distance-end', this._overlay_content, 'toggle-display-distance-end', Gio.SettingsBindFlags.DEFAULT) // this._curved_display_binding = this.settings.bind('curved-display', this._xr_effect, 'curved-display', Gio.SettingsBindFlags.DEFAULT) // this._display_size_binding = this.settings.bind('display-size', this._xr_effect, 'display-size', Gio.SettingsBindFlags.DEFAULT); // this._look_ahead_override_binding = this.settings.bind('look-ahead-override', this._xr_effect, 'look-ahead-override', Gio.SettingsBindFlags.DEFAULT); @@ -301,13 +303,13 @@ export default class BreezyDesktopExtension extends Extension { Meta.disable_unredirect_for_display(global.display); - global.stage.connect('before-paint', (() => { + this._stage_redraw_connection = global.stage.connect('before-paint', (() => { this._device_data_stream.refresh_data(); this._overlay.queue_redraw(); }).bind(this)); this._add_settings_keybinding('recenter-display-shortcut', this._recenter_display.bind(this)); - this._add_settings_keybinding('toggle-display-distance-shortcut', this.overlay_content._change_distance.bind(this.overlay_content)); + this._add_settings_keybinding('toggle-display-distance-shortcut', this._overlay_content._change_distance.bind(this._overlay_content)); this._add_settings_keybinding('toggle-follow-shortcut', this._toggle_follow_mode.bind(this)); } catch (e) { Globals.logger.log(`[ERROR] BreezyDesktopExtension _effect_enable ${e.message}\n${e.stack}`); @@ -504,6 +506,11 @@ export default class BreezyDesktopExtension extends Extension { Main.wm.removeKeybinding('toggle-display-distance-shortcut'); Main.wm.removeKeybinding('toggle-follow-shortcut'); Meta.enable_unredirect_for_display(global.display); + + if (this._stage_redraw_connection) { + global.stage.disconnect(this._stage_redraw_connection); + this._stage_redraw_connection = null; + } if (this._actor_added_connection) { global.stage.disconnect(this._actor_added_connection); @@ -514,7 +521,18 @@ export default class BreezyDesktopExtension extends Extension { this._actor_removed_connection = null; } if (this._overlay) { - this.overlay_content = null; + if (this._overlay_content) { + // if (this._widescreen_mode_effect_state_connection) { + // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); + // this._widescreen_mode_effect_state_connection = null; + // } + // if (this._supported_device_detected_connection) { + // this._xr_effect.disconnect(this._supported_device_detected_connection); + // this._supported_device_detected_connection = null; + // } + this._overlay_content.destroy(); + this._overlay_content = null; + } global.stage.remove_child(this._overlay); this._overlay.destroy(); @@ -564,17 +582,6 @@ export default class BreezyDesktopExtension extends Extension { this.settings.unbind(this._disable_anti_aliasing_binding); this._disable_anti_aliasing_binding = null; } - if (this.overlay_content) { - // if (this._widescreen_mode_effect_state_connection) { - // this._xr_effect.disconnect(this._widescreen_mode_effect_state_connection); - // this._widescreen_mode_effect_state_connection = null; - // } - // if (this._supported_device_detected_connection) { - // this._xr_effect.disconnect(this._supported_device_detected_connection); - // this._supported_device_detected_connection = null; - // } - this.overlay_content = null; - } if (this._cursor_manager) { this._cursor_manager.disable(); this._cursor_manager = null; diff --git a/gnome/src/testactor.js b/gnome/src/testactor.js index bc6f0fa..80d30d1 100644 --- a/gnome/src/testactor.js +++ b/gnome/src/testactor.js @@ -23,15 +23,15 @@ function applyQuaternionToVector(vector, quaternion) { /** * Find the vector in the array that's closest to the quaternion rotation * - * @param {number[]} quaternion - Reference quaternion [w, x, y, z] + * @param {number[]} quaternion - Reference quaternion [x, y, z, w] * @param {number[][]} vectors - Array of vectors [x, y, z] to search from * @returns {number} Index of the closest vector, if it surpasses the previous closest index by a certain margin, otherwise the previous index */ function findClosestVector(quaternion, vectors, previousClosestIndex) { const lookVector = [1.0, 0.0, 0.0]; // NWU vector pointing to the center of the screen - const rotatedLookVector = applyQuaternionToVector(lookVector, [quaternion.x, quaternion.y, quaternion.z, quaternion.w]); - Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); + const rotatedLookVector = applyQuaternionToVector(lookVector, quaternion); + // Globals.logger.log(`\t\t\tRotated look vector: ${rotatedLookVector}`); let closestIndex = -1; let closestDistance = Infinity; @@ -47,14 +47,14 @@ function findClosestVector(quaternion, vectors, previousClosestIndex) { previousDistance = distance; } - Globals.logger.log(`\t\t\tMonitor ${index} distance: ${distance}`); + // Globals.logger.log(`\t\t\tMonitor ${index} distance: ${distance}`); if (distance < closestDistance) { closestIndex = index; closestDistance = distance; } }); - Globals.logger.log(`\t\t\tClosest monitor: ${closestIndex}, distance: ${closestDistance}`); + // Globals.logger.log(`\t\t\tClosest monitor: ${closestIndex}, distance: ${closestDistance}`); // only switch if the closest monitor is greater than the previous closest by 25% if (previousClosestIndex !== undefined && closestIndex !== previousClosestIndex && closestDistance * 1.25 > previousDistance) { @@ -73,16 +73,34 @@ function radiansToDegrees(radians) { } /*** - * @returns {Object} - containing `center` and `end` radians + * @returns {Object} - containing `start`, `center`, and `end` radians for rotating the given monitor */ -function monitorWrap(radiusPixels, previousMonitorEndRadians, monitorPixels) { - const monitorHalfPixels = monitorPixels / 2; - const monitorHalfRadians = Math.asin(monitorHalfPixels / radiusPixels); - const centerRadians = previousMonitorEndRadians + monitorHalfRadians; +function monitorWrap(cachedMonitorWrap, radiusPixels, monitorBeginPixel, monitorLengthPixels) { + let closestWrap = cachedMonitorWrap.reduce((previous, current) => { + return (!previous || Math.abs(current.pixel - monitorBeginPixel) < Math.abs(previous.pixel - monitorBeginPixel)) ? current : previous; + }, undefined); + + if (closestWrap.pixel !== monitorBeginPixel) { + // there's a gap between the cached wrap value and this one + const gapPixels = monitorBeginPixel - closestWrap.pixel; + const gapHalfRadians = Math.asin(gapPixels / 2 / radiusPixels); + const gapRadians = gapHalfRadians * 2; + + // update the closestWrap value and cache it + closestWrap = { pixel: monitorBeginPixel, radians: closestWrap.radians + gapRadians }; + cachedMonitorWrap.push(closestWrap); + } + + const monitorHalfRadians = Math.asin(monitorLengthPixels / 2 / radiusPixels); + const centerRadians = closestWrap.radians + monitorHalfRadians; + const endRadians = centerRadians + monitorHalfRadians; + + // since we're computing the end values for this monitor, cache them too in case they line up with a future monitor + cachedMonitorWrap.push({ pixel: monitorBeginPixel + monitorLengthPixels, radians: endRadians }); return { - begin: previousMonitorEndRadians, + begin: closestWrap.radians, center: centerRadians, - end: centerRadians + monitorHalfRadians + end: endRadians } } @@ -93,17 +111,18 @@ function monitorWrap(radiusPixels, previousMonitorEndRadians, monitorPixels) { * @param {Object} fovDetails - contains reference fovDegrees (diagonal), widthPixels, heightPixels * @param {Object[]} monitorDetailsList - contains x, y, width, height (coordinates from top-left) * @param {string} monitorWrappingScheme - horizontal, vertical, none - * @returns {Object[]} - contains NWU vectors pointing to `topLeftNoRotate` and `center` of each monitor + * @returns {Object[]} - contains NWU vectors pointing to `topLeftNoRotate` and `center` of each monitor + * and a `rotation` angle for the given wrapping scheme */ -function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme) { +function monitorsToPlacements(fovDetails, monitorDetailsList, monitorWrappingScheme) { const aspect = fovDetails.widthPixels / fovDetails.heightPixels; const fovVerticalRadians = degreesToRadians(fovDetails.fovDegrees / Math.sqrt(1 + aspect * aspect)); // distance needed for the FOV-sized monitor to fill up the screen const centerRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); - // NWU vectors pointing to the center of the screen for each monitor - const monitorVectors = []; + const monitorPlacements = []; + const cachedMonitorWrap = []; if (monitorWrappingScheme === 'horizontal') { // monitors wrap around us horizontally @@ -112,12 +131,11 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // distance to a horizontal edge is the hypothenuse of the triangle where the opposite side is half the width of the reference fov screen const edgeRadius = fovDetails.widthPixels / 2 / Math.sin(fovHorizontalRadians / 2); - let previousMonitorEndRadians = -fovHorizontalRadians / 2; + cachedMonitorWrap.push({ pixel: 0, radians: -fovHorizontalRadians / 2 }); monitorDetailsList.forEach(monitorDetails => { - const monitorWrapDetails = monitorWrap(edgeRadius, previousMonitorEndRadians, monitorDetails.width); - previousMonitorEndRadians = monitorWrapDetails.end; + const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.x, monitorDetails.width); - monitorVectors.push({ + monitorPlacements.push({ topLeftNoRotate: [ centerRadius, fovDetails.widthPixels / 2, @@ -132,7 +150,8 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // up is flat when wrapping horizontally -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) - ] + ], + rotationAngleRadians: -monitorWrapDetails.center }); }); } else if (monitorWrappingScheme === 'vertical') { @@ -141,12 +160,11 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // distance to a vertical edge is the hypothenuse of the triangle where the opposite side is half the height of the reference fov screen const edgeRadius = fovDetails.heightPixels / 2 / Math.sin(fovVerticalRadians / 2); - let previousMonitorEndRadians = -fovVerticalRadians / 2; + cachedMonitorWrap.push({ pixel: 0, radians: -fovVerticalRadians / 2 }); monitorDetailsList.forEach(monitorDetails => { - const monitorWrapDetails = monitorWrap(edgeRadius, previousMonitorEndRadians, monitorDetails.height); - previousMonitorEndRadians = monitorWrapDetails.end; + const monitorWrapDetails = monitorWrap(cachedMonitorWrap, edgeRadius, monitorDetails.y, monitorDetails.height); - monitorVectors.push({ + monitorPlacements.push({ topLeftNoRotate: [ centerRadius, -(monitorDetails.x - fovDetails.widthPixels / 2), @@ -161,13 +179,14 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme // up is opposite where radius is the hypotenuse, using monitorWrapDetails.center as the radians -centerRadius * Math.sin(monitorWrapDetails.center) - ] + ], + rotationAngleRadians: -monitorWrapDetails.center }); }); } else { // monitors make a flat wall in front of us, no wrapping monitorDetailsList.forEach(monitorDetails => { - monitorVectors.push({ + monitorPlacements.push({ topLeftNoRotate: [ centerRadius, -(monitorDetails.x - fovDetails.widthPixels / 2), @@ -177,12 +196,13 @@ function monitorsToVectors(fovDetails, monitorDetailsList, monitorWrappingScheme centerRadius, -(monitorDetails.x + monitorDetails.width / 2 - fovDetails.widthPixels / 2), -(monitorDetails.y + monitorDetails.height / 2 - fovDetails.heightPixels / 2) - ] + ], + rotationAngleRadians: 0 }); }); } - return monitorVectors; + return monitorPlacements; } function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { @@ -204,6 +224,17 @@ function monitorVectorToRotationAngle(vector, monitorWrappingScheme) { } } +// how far to look ahead is how old the IMU data is plus a constant that is either the default for this device or an override +function lookAheadMS(imuDateMs, override) { + // how stale the imu data is + const dataAge = Date.now() - imuDateMs; + + // if (override === -1) + // return lookAheadCfg[0] + dataAge; + + return override + dataAge; +} + export const TestActorEffect = GObject.registerClass({ Properties: { 'monitor-index': GObject.ParamSpec.int( @@ -213,10 +244,10 @@ export const TestActorEffect = GObject.registerClass({ GObject.ParamFlags.READWRITE, 0, 100, 0 ), - 'quaternion': GObject.ParamSpec.jsobject( - 'quaternion', - 'Quaternion', - 'Camera orientation quaternion', + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', GObject.ParamFlags.READWRITE ), 'fov-degrees': GObject.ParamSpec.double( @@ -316,7 +347,8 @@ export const TestActorEffect = GObject.registerClass({ vfunc_build_pipeline() { const declarations = ` - uniform vec4 u_quaternion; + uniform mat4 u_imu_data; + uniform float u_look_ahead_ms; uniform mat4 u_projection_matrix; uniform float u_display_north_offset; uniform float u_rotation_x_radians; @@ -330,6 +362,88 @@ export const TestActorEffect = GObject.registerClass({ float cogl_position_width = 51.7; // no idea... float cogl_z_factor = 2.5; // no idea... + float vectorLength(vec3 v) { + return sqrt(v.x * v.x + v.y * v.y + v.z * v.z); + } + + float quaternionLength(vec4 q) { + return sqrt(q.x * q.x + q.y * q.y + q.z * q.z + q.w * q.w); + } + + vec4 quatMul(vec4 q1, vec4 q2) { + return vec4( + q1.w * q2.x + q1.x * q2.w + q1.y * q2.z - q1.z * q2.y, // x + q1.w * q2.y - q1.x * q2.z + q1.y * q2.w + q1.z * q2.x, // y + q1.w * q2.z + q1.x * q2.y - q1.y * q2.x + q1.z * q2.w, // z + q1.w * q2.w - q1.x * q2.x - q1.y * q2.y - q1.z * q2.z // w + ); + } + + vec4 quatConjugate(vec4 q) { + return vec4(-q.xyz, q.w); + } + + vec4 quatExp(vec4 q) { + float vLength = vectorLength(q.xyz); + float expW = exp(q.w); + + if (vLength < 0.000001) { + return vec4(0.0, 0.0, 0.0, expW); + } + + float scale = expW * sin(vLength) / vLength; + return vec4(q.xyz * scale, expW * cos(vLength)); + } + + vec4 quatLog(vec4 q) { + float qLength = quaternionLength(q); + float vLength = vectorLength(q.xyz); + + if (vLength < 0.000001) { + return vec4(0.0, 0.0, 0.0, log(qLength)); + } + + float scale = acos(clamp(q.w / qLength, -1.0, 1.0)) / vLength; + return vec4(q.xyz * scale, log(qLength)); + } + + vec4 computeQuaternionVelocity(vec4 q1, vec4 q2, float milliseconds) { + // Normalize input quaternions + q1 = normalize(q1); + q2 = normalize(q2); + + // Compute difference quaternion (q2 * q1^-1) + vec4 diffQ = quatMul(q2, quatConjugate(q1)); + + // Ensure we take the shortest path + if (diffQ.w < 0.0) { + diffQ = -diffQ; + } + + // Take the log and scale by time + return quatLog(diffQ) / milliseconds; + } + + vec4 extrapolateRotation(vec4 initialQuat, vec4 velocity, float deltaTimeMs) { + // Scale velocity by time + vec4 scaledVelocity = velocity * deltaTimeMs; + + // Compute the exponential + vec4 deltaRotation = quatExp(scaledVelocity); + + // Apply to initial quaternion + return normalize(quatMul(deltaRotation, initialQuat)); + } + + vec4 imuDataToLookAheadQuaternion(mat4 imuData, float lookAheadMS) { + // last row of matrix contains imu timestamps, subtract the second column from the first + float imuDeltaTime = imuData[3][0] - imuData[3][1]; + + // rotation per ms + vec4 velocity = computeQuaternionVelocity(imuData[0], imuData[1], imuDeltaTime); + return extrapolateRotation(imuData[0], velocity, lookAheadMS); + } + vec4 applyQuaternionToVector(vec4 v, vec4 q) { vec3 t = 2.0 * cross(q.xyz, v.xyz); vec3 rotated = v.xyz + q.w * t + cross(q.xyz, t); @@ -347,10 +461,15 @@ export const TestActorEffect = GObject.registerClass({ float s = sin(angle); return vec4(v.x * c + v.z * s, v.y, v.z * c - v.x * s, v.w); } + + vec4 nwuToESU(vec4 v) { + return vec4(-v.y, v.z, -v.x, v.w); + } `; const main = ` vec4 world_pos = cogl_position_in; + vec4 look_ahead_quaternion = nwuToESU(imuDataToLookAheadQuaternion(u_imu_data, u_look_ahead_ms)); float cogl_position_height = cogl_position_width / u_aspect_ratio; float position_width_adjustment_count = u_actor_to_display_ratios.x - 1.0; @@ -365,7 +484,7 @@ export const TestActorEffect = GObject.registerClass({ world_pos.z *= u_aspect_ratio; world_pos = applyXRotationToVector(world_pos, u_rotation_x_radians); world_pos = applyYRotationToVector(world_pos, u_rotation_y_radians); - world_pos = applyQuaternionToVector(world_pos, u_quaternion); + world_pos = applyQuaternionToVector(world_pos, quatConjugate(look_ahead_quaternion)); world_pos.z /= u_aspect_ratio; world_pos.x /= u_actor_to_display_ratios.x; @@ -395,7 +514,6 @@ export const TestActorEffect = GObject.registerClass({ 0.0001, 1000.0 ); - Globals.logger.log(`aspect: ${aspect}, fov: ${this.fov_degrees}, width: ${this.get_actor().width}, height: ${this.get_actor().height}, projection matrix: ${JSON.stringify(projection_matrix)}`); this.set_uniform_matrix(this.get_uniform_location("u_projection_matrix"), false, 4, projection_matrix); this.set_uniform_float(this.get_uniform_location("u_rotation_x_radians"), 1, [this.monitor_wrapping_scheme === 'vertical' ? this.monitor_wrapping_rotation_radians : 0.0]); this.set_uniform_float(this.get_uniform_location("u_rotation_y_radians"), 1, [this.monitor_wrapping_scheme === 'horizontal' ? this.monitor_wrapping_rotation_radians : 0.0]); @@ -404,10 +522,9 @@ export const TestActorEffect = GObject.registerClass({ this._initialized = true; } + this.set_uniform_float(this.get_uniform_location('u_look_ahead_ms'), 1, [lookAheadMS(this.imu_snapshots.timestamp_ms, 0)]); this.set_uniform_float(this.get_uniform_location("u_display_north_offset"), 1, [this.display_distance]); - - // NWU to east-up-south conversion, inverted - this.set_uniform_float(this.get_uniform_location("u_quaternion"), 4, [this.quaternion.y, -this.quaternion.z, this.quaternion.x, this.quaternion.w]); + this.set_uniform_matrix(this.get_uniform_location("u_imu_data"), false, 4, this.imu_snapshots.imu_data); this.get_pipeline().set_layer_filters( 0, @@ -427,10 +544,10 @@ export const TestActor = GObject.registerClass({ 'Array of monitor indexes', GObject.ParamFlags.READWRITE ), - 'quaternion': GObject.ParamSpec.jsobject( - 'quaternion', - 'Quaternion', - 'Camera orientation quaternion', + 'imu-snapshots': GObject.ParamSpec.jsobject( + 'imu-snapshots', + 'IMU Snapshots', + 'Latest IMU quaternion snapshots and epoch timestamp for when it was collected', GObject.ParamFlags.READWRITE ), 'fov-degrees': GObject.ParamSpec.double( @@ -486,7 +603,7 @@ export const TestActor = GObject.registerClass({ } }, class TestActor extends Clutter.Actor { renderMonitors() { - this.monitorsAsVectors = monitorsToVectors( + this._monitorPlacements = monitorsToPlacements( { fovDegrees: this.fov_degrees, widthPixels: this.width, @@ -502,7 +619,7 @@ export const TestActor = GObject.registerClass({ ); // normalize the center vectors - this.monitorAsNormalizedVectors = this.monitorsAsVectors.map(monitorVectors => { + this._monitorsAsNormalizedVectors = this._monitorPlacements.map(monitorVectors => { const vector = monitorVectors.center; const length = Math.sqrt(vector[0] * vector[0] + vector[1] * vector[1] + vector[2] * vector[2]); return [vector[0] / length, vector[1] / length, vector[2] / length]; @@ -518,8 +635,8 @@ export const TestActor = GObject.registerClass({ Globals.logger.log(`\t\t\tMonitor ${index}: ${monitor.x}, ${monitor.y}, ${monitor.width}, ${monitor.height}`); // this is in NWU coordinates - const noRotationVector = this.monitorsAsVectors[index].topLeftNoRotate; - Globals.logger.log_debug(`\t\t\tMonitor ${index} vectors: ${JSON.stringify(this.monitorsAsVectors[index])}`); + const noRotationVector = this._monitorPlacements[index].topLeftNoRotate; + Globals.logger.log_debug(`\t\t\tMonitor ${index} vectors: ${JSON.stringify(this._monitorPlacements[index])}`); // actor coordinates are east-up-south const containerActor = new Clutter.Actor({ @@ -543,24 +660,27 @@ export const TestActor = GObject.registerClass({ // Add the monitor actor to the scene containerActor.add_child(monitorClone); const effect = new TestActorEffect({ - quaternion: this.quaternion, + imu_snapshots: this.imu_snapshots, fov_degrees: this.fov_degrees, monitor_index: index, display_distance: noRotationVector[0], monitor_wrapping_scheme: 'horizontal', - monitor_wrapping_rotation_radians: monitorVectorToRotationAngle(this.monitorsAsVectors[index].center, 'horizontal').angle, + monitor_wrapping_rotation_radians: this._monitorPlacements[index].rotationAngleRadians, actor_to_display_ratios: actorToDisplayRatios }); containerActor.add_effect_with_name('viewport-effect', effect); this.add_child(containerActor); - this.bind_property('quaternion', effect, 'quaternion', GObject.BindingFlags.DEFAULT); + this.bind_property('imu-snapshots', effect, 'imu-snapshots', GObject.BindingFlags.DEFAULT); this.bind_property('focused-monitor-index', effect, 'focused-monitor-index', GObject.BindingFlags.DEFAULT); // this.bind_property('display-distance', effect, 'display-distance', GObject.BindingFlags.DEFAULT); }).bind(this)); GLib.timeout_add(GLib.PRIORITY_DEFAULT, 500, (() => { - if (this.quaternion) { - const closestMonitorIndex = findClosestVector(this.quaternion, this.monitorAsNormalizedVectors, this.closestMonitorIndex); + if (this.imu_snapshots) { + const closestMonitorIndex = findClosestVector( + this.imu_snapshots.imu_data.splice(0, 4), + this._monitorsAsNormalizedVectors, this.closestMonitorIndex + ); // only switch if the closest monitor is greater than the previous closest by 25% if (this.closestMonitorIndex === undefined || this.closestMonitorIndex !== closestMonitorIndex) {