From 054296da565842eea1e780e6c43b1f48e78fb2fc Mon Sep 17 00:00:00 2001 From: Wayne Heaney <42350981+wheaney@users.noreply.github.com> Date: Wed, 22 Oct 2025 16:13:08 -0700 Subject: [PATCH] Add support for 6DoF (#138) * Pull in driver and sombrero updates for 6DoF * Update Breezy GNOME to support 6DoF position * Update Breezy KDE to support 6DoF position --- VERSION | 2 +- gnome/src/devicedatastream.js | 36 +++++--- gnome/src/virtualdisplayeffect.js | 35 +++++-- gnome/src/virtualdisplaysactor.js | 6 +- kwin/src/breezydesktopeffect.cpp | 147 ++++++++++++++++-------------- kwin/src/breezydesktopeffect.h | 35 +++---- kwin/src/qml/BreezyDesktop.qml | 18 ++-- kwin/src/qml/CameraController.qml | 32 +++---- kwin/src/qml/main.qml | 10 +- modules/XRLinuxDriver | 2 +- modules/sombrero | 2 +- ui/modules/PyXRLinuxDriverIPC | 2 +- 12 files changed, 182 insertions(+), 145 deletions(-) diff --git a/VERSION b/VERSION index 58073ef..fad066f 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -2.4.1 \ No newline at end of file +2.5.0 \ No newline at end of file diff --git a/gnome/src/devicedatastream.js b/gnome/src/devicedatastream.js index e396403..4d88ff7 100644 --- a/gnome/src/devicedatastream.js +++ b/gnome/src/devicedatastream.js @@ -21,7 +21,7 @@ const IPC_FILE_PATH = "/dev/shm/breezy_desktop_imu"; const KEEPALIVE_REFRESH_INTERVAL_SEC = 1; // the driver should be using the same data layout version -const DATA_LAYOUT_VERSION = 4; +const DATA_LAYOUT_VERSION = 5; // DataView info: [offset, size, count] const VERSION = [0, UINT8_SIZE, 1]; @@ -34,16 +34,17 @@ const SBS_ENABLED = [dataViewEnd(LENS_DISTANCE_RATIO), BOOL_SIZE, 1]; const CUSTOM_BANNER_ENABLED = [dataViewEnd(SBS_ENABLED), BOOL_SIZE, 1]; const SMOOTH_FOLLOW_ENABLED = [dataViewEnd(CUSTOM_BANNER_ENABLED), BOOL_SIZE, 1]; const SMOOTH_FOLLOW_ORIGIN_DATA = [dataViewEnd(SMOOTH_FOLLOW_ENABLED), FLOAT_SIZE, 16]; -const EPOCH_MS = [dataViewEnd(SMOOTH_FOLLOW_ORIGIN_DATA), UINT_SIZE, 2]; -const IMU_QUAT_DATA = [dataViewEnd(EPOCH_MS), FLOAT_SIZE, 16]; -const IMU_PARITY_BYTE = [dataViewEnd(IMU_QUAT_DATA), UINT8_SIZE, 1]; +const POSE_POSITION = [dataViewEnd(SMOOTH_FOLLOW_ORIGIN_DATA), FLOAT_SIZE, 3]; +const EPOCH_MS = [dataViewEnd(POSE_POSITION), UINT_SIZE, 2]; +const POSE_ORIENTATION = [dataViewEnd(EPOCH_MS), FLOAT_SIZE, 16]; +const IMU_PARITY_BYTE = [dataViewEnd(POSE_ORIENTATION), UINT8_SIZE, 1]; const DATA_VIEW_LENGTH = dataViewEnd(IMU_PARITY_BYTE); function checkParityByte(dataView) { const parityByte = dataViewUint8(dataView, IMU_PARITY_BYTE); let parity = 0; const epochUint8 = dataViewUint8Array(dataView, EPOCH_MS); - const imuDataUint8 = dataViewUint8Array(dataView, IMU_QUAT_DATA); + const imuDataUint8 = dataViewUint8Array(dataView, POSE_ORIENTATION); for (let i = 0; i < epochUint8.length; i++) { parity ^= epochUint8[i]; } @@ -210,10 +211,11 @@ export const DeviceDataStream = GObject.registerClass({ const validData = validKeepAlive && displayFov !== 0.0; const version = dataViewUint8(dataView, VERSION); const enabled = dataViewUint8(dataView, ENABLED) !== 0 && version === DATA_LAYOUT_VERSION && validData; - let imuData = dataViewFloatArray(dataView, IMU_QUAT_DATA); + let poseOrientation = dataViewFloatArray(dataView, POSE_ORIENTATION); + let posePosition = dataViewFloatArray(dataView, POSE_POSITION); let smoothFollowEnabled = !this.legacy_follow_mode && dataViewUint8(dataView, SMOOTH_FOLLOW_ENABLED) !== 0; let smoothFollowOrigin = dataViewFloatArray(dataView, SMOOTH_FOLLOW_ORIGIN_DATA); - const imuResetState = enabled && validData && imuData[0] === 0.0 && imuData[1] === 0.0 && imuData[2] === 0.0 && imuData[3] === 1.0; + const imuResetState = enabled && validData && poseOrientation[0] === 0.0 && poseOrientation[1] === 0.0 && poseOrientation[2] === 0.0 && poseOrientation[3] === 1.0; const customBannerEnabled = dataViewUint8(dataView, CUSTOM_BANNER_ENABLED) !== 0; const sbsEnabled = dataViewUint8(dataView, SBS_ENABLED) !== 0; @@ -261,7 +263,8 @@ export const DeviceDataStream = GObject.registerClass({ if (checkParityByte(dataView)) { this.imu_snapshots = { - imu_data: imuData, + pose_orientation: poseOrientation, + pose_position: posePosition, timestamp_ms: imuDateMs, smooth_follow_origin: smoothFollowOrigin }; @@ -277,7 +280,8 @@ export const DeviceDataStream = GObject.registerClass({ buffer = new Uint8Array(data[1]).buffer; dataView = new DataView(buffer); imuDateMs = dataViewBigUint(dataView, EPOCH_MS); - imuData = dataViewFloatArray(dataView, IMU_QUAT_DATA); + poseOrientation = dataViewFloatArray(dataView, POSE_ORIENTATION); + posePosition = dataViewFloatArray(dataView, POSE_POSITION); } } } @@ -311,15 +315,17 @@ export const DeviceDataStream = GObject.registerClass({ if (!keepalive_only) { this._counter = ((this._counter ?? -1)+1)%COUNTER_MAX; - const imuDataFirst = nextDebugIMUQuaternion(this._counter); - const imuData = [ - ...imuDataFirst, - ...imuDataFirst, - ...imuDataFirst, + const poseOrientationFirst = nextDebugIMUQuaternion(this._counter); + const poseOrientation = [ + ...poseOrientationFirst, + ...poseOrientationFirst, + ...poseOrientationFirst, 2.0, 1.0, 0.0, 0.0 ] + const posePosition = [0.0, 0.0, 0.0]; this.imu_snapshots = { - imu_data: imuData, + pose_orientation: poseOrientation, + pose_position: posePosition, timestamp_ms: Date.now(), smooth_follow_origin: [0.0, 0.0, 0.0, 1.0] }; diff --git a/gnome/src/virtualdisplayeffect.js b/gnome/src/virtualdisplayeffect.js index 0cb8a16..f4b29f5 100644 --- a/gnome/src/virtualdisplayeffect.js +++ b/gnome/src/virtualdisplayeffect.js @@ -417,7 +417,8 @@ export const VirtualDisplayEffect = GObject.registerClass({ vfunc_build_pipeline() { const declarations = ` uniform bool u_show_banner; - uniform mat4 u_imu_data; + uniform mat4 u_pose_orientation; + uniform vec3 u_pose_position; uniform float u_look_ahead_ms; uniform vec4 u_look_ahead_cfg; uniform mat4 u_projection_matrix; @@ -460,7 +461,11 @@ export const VirtualDisplayEffect = GObject.registerClass({ vec4 nwuToESU(vec4 v) { return vec4(-v.y, v.z, -v.x, v.w); } - + + vec3 nwuToESU(vec3 v) { + return vec3(-v.y, v.z, -v.x); + } + // returns the rate of change between the two vectors, in same time units as delta_time // e.g. if delta_time is in ms, then the rate of change is "per ms" vec3 rateOfChange(vec3 v1, vec3 v2, float delta_time) { @@ -487,14 +492,23 @@ export const VirtualDisplayEffect = GObject.registerClass({ if (!u_show_banner) { float aspect_ratio = u_display_resolution.x / u_display_resolution.y; + vec4 quat_t0 = nwuToESU(quatConjugate(u_pose_orientation[0])); + vec3 position_vector = applyQuaternionToVector(nwuToESU(u_pose_position), quat_t0); + vec3 final_lens_position = u_lens_vector + position_vector; + vec3 complete_vector = applyXRotationToVector(world_pos.xyz, u_rotation_x_radians); complete_vector = applyYRotationToVector(complete_vector, u_rotation_y_radians); - vec4 quat_t0 = nwuToESU(quatConjugate(u_imu_data[0])); vec3 rotated_vector_t0 = applyQuaternionToVector(complete_vector, quat_t0); - vec3 rotated_vector_t1 = applyQuaternionToVector(complete_vector, nwuToESU(quatConjugate(u_imu_data[1]))); - float delta_time_t0 = u_imu_data[3][0] - u_imu_data[3][1]; - vec3 velocity_t0 = rateOfChange(rotated_vector_t0, rotated_vector_t1, delta_time_t0); + vec3 rotated_vector_t1 = applyQuaternionToVector(complete_vector, nwuToESU(quatConjugate(u_pose_orientation[1]))); + float delta_time_t0 = u_pose_orientation[3][0] - u_pose_orientation[3][1]; + + // how quickly the vertex is moving relative to the camera + vec3 velocity_t0 = rateOfChange( + rotated_vector_t0 - final_lens_position, + rotated_vector_t1 - final_lens_position, + delta_time_t0 + ); // compute the capped look ahead with scanline adjustments float look_ahead_scanline_ms = u_look_ahead_ms == 0.0 ? 0.0 : vectorToScanline(u_fov_vertical_radians, rotated_vector_t0) * u_look_ahead_cfg[2]; @@ -502,7 +516,7 @@ export const VirtualDisplayEffect = GObject.registerClass({ vec3 look_ahead_vector = applyLookAhead(rotated_vector_t0, velocity_t0, effective_look_ahead_ms); - world_pos = vec4(look_ahead_vector - u_lens_vector, world_pos.w); + world_pos = vec4(look_ahead_vector - final_lens_position, world_pos.w); world_pos.z /= aspect_ratio / u_actor_to_display_ratios.y; @@ -557,9 +571,12 @@ export const VirtualDisplayEffect = GObject.registerClass({ this.set_uniform_float(this.get_uniform_location('u_look_ahead_ms'), 1, [0.0]); lookAheadSet = true; } - this.set_uniform_matrix(this.get_uniform_location("u_imu_data"), false, 4, this.imu_snapshots.imu_data); + const posePositionPixels = this.imu_snapshots.pose_position.map(coord => coord * this.fov_details.completeScreenDistancePixels); + this.set_uniform_matrix(this.get_uniform_location("u_pose_orientation"), false, 4, this.imu_snapshots.pose_orientation); + this.set_uniform_float(this.get_uniform_location("u_pose_position"), 3, posePositionPixels); } else { - this.set_uniform_matrix(this.get_uniform_location("u_imu_data"), false, 4, this.imu_snapshots.smooth_follow_origin); + this.set_uniform_matrix(this.get_uniform_location("u_pose_orientation"), false, 4, this.imu_snapshots.smooth_follow_origin); + this.set_uniform_float(this.get_uniform_location("u_pose_position"), 3, [0.0, 0.0, 0.0]); } if (!lookAheadSet) { this.set_uniform_float(this.get_uniform_location('u_look_ahead_ms'), 1, [lookAheadMS(this.imu_snapshots.timestamp_ms, Globals.data_stream.device_data.lookAheadCfg, this.look_ahead_override)]); diff --git a/gnome/src/virtualdisplaysactor.js b/gnome/src/virtualdisplaysactor.js index 0a27948..4f7d7db 100644 --- a/gnome/src/virtualdisplaysactor.js +++ b/gnome/src/virtualdisplaysactor.js @@ -785,12 +785,12 @@ export const VirtualDisplaysActor = GObject.registerClass({ (!this._smooth_follow_slerping || this.focused_monitor_index === -1)) { // if smooth follow is enabled, use the origin IMU data to inform the initial focused monitor // since it reflects where the user is looking in relation to the original monitor positions - const currentPoseQuat = this.smooth_follow_enabled ? + const currentOrientationQuat = this.smooth_follow_enabled ? this.imu_snapshots.smooth_follow_origin.splice(0, 4) : - this.imu_snapshots.imu_data.splice(0, 4); + this.imu_snapshots.pose_orientation.splice(0, 4); const focusedMonitorIndex = findFocusedMonitor( - currentPoseQuat, + currentOrientationQuat, this.monitor_placements.map(monitorVectors => monitorVectors.centerLook), this.focused_monitor_index, this.display_distance / this._display_distance_default(), diff --git a/kwin/src/breezydesktopeffect.cpp b/kwin/src/breezydesktopeffect.cpp index 723834b..d0f3a64 100644 --- a/kwin/src/breezydesktopeffect.cpp +++ b/kwin/src/breezydesktopeffect.cpp @@ -98,11 +98,12 @@ namespace DataView constexpr int CUSTOM_BANNER_ENABLED[3] = {dataViewEnd(SBS_ENABLED), BOOL_SIZE, 1}; constexpr int SMOOTH_FOLLOW_ENABLED[3] = {dataViewEnd(CUSTOM_BANNER_ENABLED), BOOL_SIZE, 1}; constexpr int SMOOTH_FOLLOW_ORIGIN_DATA[3] = {dataViewEnd(SMOOTH_FOLLOW_ENABLED), FLOAT_SIZE, 16}; - constexpr int IMU_DATE_MS[3] = {dataViewEnd(SMOOTH_FOLLOW_ORIGIN_DATA), UINT_SIZE, 2}; - constexpr int IMU_QUAT_ENTRIES = 4; - constexpr int IMU_QUAT_DATA[3] = {dataViewEnd(IMU_DATE_MS), FLOAT_SIZE, 4 * IMU_QUAT_ENTRIES}; - constexpr int IMU_PARITY_BYTE[3] = {dataViewEnd(IMU_QUAT_DATA), UINT8_SIZE, 1}; - constexpr int LENGTH = dataViewEnd(IMU_PARITY_BYTE); + constexpr int POSE_POSITION_DATA[3] = {dataViewEnd(SMOOTH_FOLLOW_ORIGIN_DATA), FLOAT_SIZE, 3}; + constexpr int POSE_DATE_MS[3] = {dataViewEnd(POSE_POSITION_DATA), UINT_SIZE, 2}; + constexpr int POSE_ORIENTATION_ENTRIES = 4; + constexpr int POSE_ORIENTATION_DATA[3] = {dataViewEnd(POSE_DATE_MS), FLOAT_SIZE, 4 * POSE_ORIENTATION_ENTRIES}; + constexpr int POSE_PARITY_BYTE[3] = {dataViewEnd(POSE_ORIENTATION_DATA), UINT8_SIZE, 1}; + constexpr int LENGTH = dataViewEnd(POSE_PARITY_BYTE); } namespace KWin @@ -142,7 +143,7 @@ BreezyDesktopEffect::BreezyDesktopEffect() setSource(QUrl::fromLocalFile(QStandardPaths::locate(QStandardPaths::GenericDataLocation, QStringLiteral("kwin/effects/breezy_desktop/qml/main.qml")))); - // Monitor the IMU file for changes, even if it doesn't exist at startup + // Monitor the IPC file for changes, even if it doesn't exist at startup m_shmDirectoryWatcher = new QFileSystemWatcher(this); m_shmDirectoryWatcher->addPath(DataView::SHM_DIR); @@ -151,14 +152,14 @@ BreezyDesktopEffect::BreezyDesktopEffect() // Setup file watcher with recreation detection auto setupFileWatcher = [this]() { if (QFile::exists(DataView::SHM_PATH) && ( - m_imuTimestamp == 0 || - QDateTime::currentMSecsSinceEpoch() - m_imuTimestamp > 50 || // file may have been deleted and recreated + m_poseTimestamp == 0 || + QDateTime::currentMSecsSinceEpoch() - m_poseTimestamp > 50 || // file may have been deleted and recreated !m_shmFileWatcher->files().contains(DataView::SHM_PATH) )) { m_shmFileWatcher->removePath(DataView::SHM_PATH); - disconnect(m_shmFileWatcher, &QFileSystemWatcher::fileChanged, this, &BreezyDesktopEffect::updateImuRotation); + disconnect(m_shmFileWatcher, &QFileSystemWatcher::fileChanged, this, &BreezyDesktopEffect::updatePoseOrientation); m_shmFileWatcher->addPath(DataView::SHM_PATH); - connect(m_shmFileWatcher, &QFileSystemWatcher::fileChanged, this, &BreezyDesktopEffect::updateImuRotation); + connect(m_shmFileWatcher, &QFileSystemWatcher::fileChanged, this, &BreezyDesktopEffect::updatePoseOrientation); } }; @@ -168,13 +169,13 @@ BreezyDesktopEffect::BreezyDesktopEffect() // Initial setup setupFileWatcher(); - m_imuWatchdogTimer = new QTimer(this); - m_imuWatchdogTimer->setInterval(1000); - connect(m_imuWatchdogTimer, &QTimer::timeout, this, [this]() { + m_watchdogTimer = new QTimer(this); + m_watchdogTimer->setInterval(1000); + connect(m_watchdogTimer, &QTimer::timeout, this, [this]() { if (!m_enabled) return; - this->updateImuRotation(); + this->updatePoseOrientation(); }); - m_imuWatchdogTimer->start(); + m_watchdogTimer->start(); m_cursorUpdateTimer = new QTimer(this); connect(m_cursorUpdateTimer, &QTimer::timeout, this, &BreezyDesktopEffect::updateCursorPos); @@ -206,10 +207,10 @@ BreezyDesktopEffect::~BreezyDesktopEffect() m_shmDirectoryWatcher->deleteLater(); m_shmDirectoryWatcher = nullptr; } - if (m_imuWatchdogTimer) { - m_imuWatchdogTimer->stop(); - m_imuWatchdogTimer->deleteLater(); - m_imuWatchdogTimer = nullptr; + if (m_watchdogTimer) { + m_watchdogTimer->stop(); + m_watchdogTimer->deleteLater(); + m_watchdogTimer = nullptr; } deactivate(); } @@ -439,20 +440,24 @@ void BreezyDesktopEffect::toggleSmoothFollow() { XRDriverIPC::instance().writeControlFlags(flags); } -bool BreezyDesktopEffect::imuResetState() const { - return m_imuResetState; +bool BreezyDesktopEffect::poseResetState() const { + return m_poseResetState; } -QList BreezyDesktopEffect::imuRotations() const { - return m_imuRotations; +QList BreezyDesktopEffect::poseOrientations() const { + return m_poseOrientations; } -quint32 BreezyDesktopEffect::imuTimeElapsedMs() const { - return m_imuTimeElapsedMs; +QVector3D BreezyDesktopEffect::posePosition() const { + return m_posePosition; } -quint64 BreezyDesktopEffect::imuTimestamp() const { - return m_imuTimestamp; +quint32 BreezyDesktopEffect::poseTimeElapsedMs() const { + return m_poseTimeElapsedMs; +} + +quint64 BreezyDesktopEffect::poseTimestamp() const { + return m_poseTimestamp; } QList BreezyDesktopEffect::lookAheadConfig() const { @@ -581,17 +586,17 @@ bool BreezyDesktopEffect::smoothFollowEnabled() const { } bool BreezyDesktopEffect::checkParityByte(const char* data) { - const uint8_t parityByte = static_cast(data[DataView::IMU_PARITY_BYTE[DataView::OFFSET_INDEX]]); + const uint8_t parityByte = static_cast(data[DataView::POSE_PARITY_BYTE[DataView::OFFSET_INDEX]]); uint8_t parity = 0; - const int dateBytes = DataView::IMU_DATE_MS[DataView::COUNT_INDEX] * DataView::IMU_DATE_MS[DataView::SIZE_INDEX]; + const int dateBytes = DataView::POSE_DATE_MS[DataView::COUNT_INDEX] * DataView::POSE_DATE_MS[DataView::SIZE_INDEX]; for (int i = 0; i < dateBytes; ++i) { - parity ^= static_cast(data[DataView::IMU_DATE_MS[DataView::OFFSET_INDEX] + i]); + parity ^= static_cast(data[DataView::POSE_DATE_MS[DataView::OFFSET_INDEX] + i]); } - const int quatBytes = DataView::IMU_QUAT_DATA[DataView::COUNT_INDEX] * DataView::IMU_QUAT_DATA[DataView::SIZE_INDEX]; + const int quatBytes = DataView::POSE_ORIENTATION_DATA[DataView::COUNT_INDEX] * DataView::POSE_ORIENTATION_DATA[DataView::SIZE_INDEX]; for (int i = 0; i < quatBytes; ++i) { - parity ^= static_cast(data[DataView::IMU_QUAT_DATA[DataView::OFFSET_INDEX] + i]); + parity ^= static_cast(data[DataView::POSE_ORIENTATION_DATA[DataView::OFFSET_INDEX] + i]); } return parityByte == parity; @@ -599,15 +604,15 @@ bool BreezyDesktopEffect::checkParityByte(const char* data) { static qint64 lastConfigUpdate = 0; static qint64 activatedAt = 0; -void BreezyDesktopEffect::updateImuRotation() { +void BreezyDesktopEffect::updatePoseOrientation() { // Reentrancy guard: if an update is already in progress, skip bool expected = false; - if (!m_imuUpdateInProgress.compare_exchange_strong(expected, true)) { + if (!m_poseUpdateInProgress.compare_exchange_strong(expected, true)) { return; } // destructor called on function exit, triggers reset of the flag - struct ResetFlag { std::atomic* f; ~ResetFlag(){ f->store(false); } } reset{&m_imuUpdateInProgress}; + struct ResetFlag { std::atomic* f; ~ResetFlag(){ f->store(false); } } reset{&m_poseUpdateInProgress}; const QString shmPath = QStringLiteral("/dev/shm/breezy_desktop_imu"); QFile shmFile(shmPath); @@ -623,9 +628,9 @@ void BreezyDesktopEffect::updateImuRotation() { uint8_t version = static_cast(data[DataView::VERSION[DataView::OFFSET_INDEX]]); uint8_t enabledFlag = static_cast(data[DataView::ENABLED[DataView::OFFSET_INDEX]]); - uint64_t imuDateMs; - memcpy(&imuDateMs, data + DataView::IMU_DATE_MS[DataView::OFFSET_INDEX], sizeof(imuDateMs)); - imuDateMs = qFromLittleEndian(imuDateMs); + uint64_t poseDateMs; + memcpy(&poseDateMs, data + DataView::POSE_DATE_MS[DataView::OFFSET_INDEX], sizeof(poseDateMs)); + poseDateMs = qFromLittleEndian(poseDateMs); const qint64 currentTimeMs = QDateTime::currentMSecsSinceEpoch(); const bool updateConfig = lastConfigUpdate == 0 || currentTimeMs - lastConfigUpdate > 1000; @@ -664,9 +669,9 @@ void BreezyDesktopEffect::updateImuRotation() { lastConfigUpdate = currentTimeMs; } - const bool validKeepAlive = (currentTimeMs - imuDateMs) < 5000; + const bool validKeepAlive = (currentTimeMs - poseDateMs) < 5000; const bool validData = validKeepAlive && m_diagonalFOV != 0.0f; - const uint8_t expectedVersion = 4; + const uint8_t expectedVersion = 5; bool enabledFlagSet = (enabledFlag != 0); bool validVersion = (version == expectedVersion); const bool wasEnabled = m_enabled; @@ -675,7 +680,7 @@ void BreezyDesktopEffect::updateImuRotation() { // give a grace period after enabling the effect if (wasEnabled && (currentTimeMs - activatedAt > 1000)) { qCCritical(KWIN_XR) << "\t\t\tBreezy - disabling effect; currentTimeMs:" << currentTimeMs - << "imuDateMs:" << imuDateMs + << "poseDateMs:" << poseDateMs << "enabledFlag:" << enabledFlag << "version:" << version << "diagonalFOV:" << m_diagonalFOV; @@ -686,7 +691,7 @@ void BreezyDesktopEffect::updateImuRotation() { } } else if (!wasEnabled) { qCCritical(KWIN_XR) << "\t\t\tBreezy - enabling effect; currentTimeMs:" << currentTimeMs - << "imuDateMs:" << imuDateMs + << "poseDateMs:" << poseDateMs << "enabledFlag:" << enabledFlag << "version:" << version << "diagonalFOV:" << m_diagonalFOV; @@ -698,50 +703,56 @@ void BreezyDesktopEffect::updateImuRotation() { if (updateConfig) Q_EMIT devicePropertiesChanged(); - float imuData[4 * DataView::IMU_QUAT_ENTRIES]; // 4 quaternion-sized rows - memcpy(imuData, data + DataView::IMU_QUAT_DATA[DataView::OFFSET_INDEX], sizeof(imuData)); - bool wasImuResetState = m_imuResetState; - m_imuResetState = (imuData[0] == 0.0f && imuData[1] == 0.0f && imuData[2] == 0.0f && imuData[3] == 1.0f); - if (m_imuResetState != wasImuResetState) { - if (m_imuResetState) recenter(); - Q_EMIT imuResetStateChanged(); + float posePositionData[3]; + memcpy(posePositionData, data + DataView::POSE_POSITION_DATA[DataView::OFFSET_INDEX], sizeof(posePositionData)); + + // convert NWU to EUS by passing position values: -y, z, -x + m_posePosition = QVector3D(-posePositionData[1], posePositionData[2], -posePositionData[0]); + + float poseOrientationData[4 * DataView::POSE_ORIENTATION_ENTRIES]; // 4 quaternion-sized rows + memcpy(poseOrientationData, data + DataView::POSE_ORIENTATION_DATA[DataView::OFFSET_INDEX], sizeof(poseOrientationData)); + bool wasPoseResetState = m_poseResetState; + m_poseResetState = (poseOrientationData[0] == 0.0f && poseOrientationData[1] == 0.0f && poseOrientationData[2] == 0.0f && poseOrientationData[3] == 1.0f); + if (m_poseResetState != wasPoseResetState) { + if (m_poseResetState) recenter(); + Q_EMIT poseResetStateChanged(); } - // convert NWU to EUS by passing root.rotation values: -y, z, -x - QQuaternion quatT0(imuData[3], -imuData[1], imuData[2], -imuData[0]); + // convert NWU to EUS by passing orientation values: -y, z, -x + QQuaternion quatT0(poseOrientationData[3], -poseOrientationData[1], poseOrientationData[2], -poseOrientationData[0]); - int imuDataOffset = DataView::IMU_QUAT_ENTRIES; - QQuaternion quatT1(imuData[imuDataOffset + 3], -imuData[imuDataOffset + 1], imuData[imuDataOffset + 2], -imuData[imuDataOffset + 0]); + int orientationDataOffset = DataView::POSE_ORIENTATION_ENTRIES; + QQuaternion quatT1(poseOrientationData[orientationDataOffset + 3], -poseOrientationData[orientationDataOffset + 1], poseOrientationData[orientationDataOffset + 2], -poseOrientationData[orientationDataOffset + 0]); - imuDataOffset += DataView::IMU_QUAT_ENTRIES; + orientationDataOffset += DataView::POSE_ORIENTATION_ENTRIES; // skip the 3rd quaternion - imuDataOffset += DataView::IMU_QUAT_ENTRIES; + orientationDataOffset += DataView::POSE_ORIENTATION_ENTRIES; - // set imuRotations to the last two rotations, leave out the elapsed time - m_imuRotations.clear(); - m_imuRotations.append(quatT0); - m_imuRotations.append(quatT1); + // set poseOrientations to the last two rotations, leave out the elapsed time + m_poseOrientations.clear(); + m_poseOrientations.append(quatT0); + m_poseOrientations.append(quatT1); // 4th row isn't actually a quaternion, it contains the timestamps for each of the 3 quaternions - // elapsed time between T0 and T1 is: imuData[0] - imuData[1] - m_imuTimeElapsedMs = static_cast(imuData[imuDataOffset + 0] - imuData[imuDataOffset + 1]); + // elapsed time between T0 and T1 is: poseOrientationData[0] - poseOrientationData[1] + m_poseTimeElapsedMs = static_cast(poseOrientationData[orientationDataOffset + 0] - poseOrientationData[orientationDataOffset + 1]); - m_imuTimestamp = imuDateMs; + m_poseTimestamp = poseDateMs; - float originData[4 * DataView::IMU_QUAT_ENTRIES]; // 4 quaternion-sized rows + float originData[4 * DataView::POSE_ORIENTATION_ENTRIES]; // 4 quaternion-sized rows memcpy(originData, data + DataView::SMOOTH_FOLLOW_ORIGIN_DATA[DataView::OFFSET_INDEX], sizeof(originData)); // convert NWU to EUS by passing root.rotation values: -y, z, -x QQuaternion sfQuatT0(originData[3], -originData[1], originData[2], -originData[0]); - int originDataOffset = DataView::IMU_QUAT_ENTRIES; + int originDataOffset = DataView::POSE_ORIENTATION_ENTRIES; QQuaternion sfQuatT1(originData[originDataOffset + 3], -originData[originDataOffset + 1], originData[originDataOffset + 2], -originData[originDataOffset + 0]); - originDataOffset += DataView::IMU_QUAT_ENTRIES; + originDataOffset += DataView::POSE_ORIENTATION_ENTRIES; // skip the 3rd quaternion - originDataOffset += DataView::IMU_QUAT_ENTRIES; + originDataOffset += DataView::POSE_ORIENTATION_ENTRIES; // set smoothFollowOrigin to the last two rotations, leave out the elapsed time m_smoothFollowOrigin.clear(); @@ -878,9 +889,9 @@ void BreezyDesktopEffect::evaluateCursorOnScreenState(const QPointF &prevPos, co const bool onScreen = m_effectOnScreenExpandedGeometry.contains(newPos.toPoint()) || m_effectOnScreenExpandedGeometry.contains(predicted.toPoint()); - if (m_enabled && !m_imuResetState && !m_cursorHidden && onScreen) { + if (m_enabled && !m_poseResetState && !m_cursorHidden && onScreen) { hideCursor(); - } else if (m_cursorHidden && (!m_enabled || m_imuResetState || !onScreen)) { + } else if (m_cursorHidden && (!m_enabled || m_poseResetState || !onScreen)) { showCursor(); } } diff --git a/kwin/src/breezydesktopeffect.h b/kwin/src/breezydesktopeffect.h index c98f817..328a67d 100644 --- a/kwin/src/breezydesktopeffect.h +++ b/kwin/src/breezydesktopeffect.h @@ -24,10 +24,11 @@ namespace KWin Q_PROPERTY(int effectTargetScreenIndex READ effectTargetScreenIndex WRITE setEffectTargetScreenIndex) Q_PROPERTY(bool zoomOnFocusEnabled READ isZoomOnFocusEnabled WRITE setZoomOnFocusEnabled NOTIFY zoomOnFocusChanged) Q_PROPERTY(int lookingAtScreenIndex READ lookingAtScreenIndex WRITE setLookingAtScreenIndex) - Q_PROPERTY(bool imuResetState READ imuResetState NOTIFY imuResetStateChanged) - Q_PROPERTY(QList imuRotations READ imuRotations) - Q_PROPERTY(quint32 imuTimeElapsedMs READ imuTimeElapsedMs) - Q_PROPERTY(quint64 imuTimestamp READ imuTimestamp) + Q_PROPERTY(bool poseResetState READ poseResetState NOTIFY poseResetStateChanged) + Q_PROPERTY(QList poseOrientations READ poseOrientations) + Q_PROPERTY(QVector3D posePosition READ posePosition) + Q_PROPERTY(quint32 poseTimeElapsedMs READ poseTimeElapsedMs) + Q_PROPERTY(quint64 poseTimestamp READ poseTimestamp) Q_PROPERTY(QString cursorImageSource READ cursorImageSource NOTIFY cursorImageSourceChanged) Q_PROPERTY(QSize cursorImageSize READ cursorImageSize NOTIFY cursorImageSourceChanged) Q_PROPERTY(QPointF cursorPos READ cursorPos NOTIFY cursorPosChanged) @@ -73,10 +74,11 @@ namespace KWin void setZoomOnFocusEnabled(bool enabled); int lookingAtScreenIndex() const { return m_lookingAtScreenIndex; } void setLookingAtScreenIndex(int index); - QList imuRotations() const; - quint32 imuTimeElapsedMs() const; - quint64 imuTimestamp() const; - bool imuResetState() const; + QList poseOrientations() const; + QVector3D posePosition() const; + quint32 poseTimeElapsedMs() const; + quint64 poseTimestamp() const; + bool poseResetState() const; QList lookAheadConfig() const; qreal lookAheadOverride() const; void setLookAheadOverride(qreal override); @@ -112,7 +114,7 @@ namespace KWin void disableDriver(); void toggle(); void addVirtualDisplay(QSize size); - void updateImuRotation(); + void updatePoseOrientation(); void updateCursorImage(); void updateCursorPos(); QVariantList listVirtualDisplays() const; @@ -129,7 +131,7 @@ namespace KWin void displayWrappingSchemeChanged(); void enabledStateChanged(); void zoomOnFocusChanged(); - void imuResetStateChanged(); + void poseResetStateChanged(); void sbsEnabledChanged(); void smoothFollowEnabledChanged(); void devicePropertiesChanged(); @@ -165,10 +167,11 @@ namespace KWin bool m_zoomOnFocusEnabled = false; int m_lookingAtScreenIndex = -1; int m_effectTargetScreenIndex = -1; - bool m_imuResetState; - QList m_imuRotations; - quint32 m_imuTimeElapsedMs; - quint64 m_imuTimestamp = 0; + bool m_poseResetState; + QList m_poseOrientations; + QVector3D m_posePosition; + quint32 m_poseTimeElapsedMs; + quint64 m_poseTimestamp = 0; QList m_lookAheadConfig; qreal m_lookAheadOverride = -1.0; // -1 = use device default QList m_displayResolution; @@ -183,8 +186,8 @@ namespace KWin bool m_cursorHidden = false; QPointF m_cursorPos; QTimer *m_cursorUpdateTimer = nullptr; - QTimer *m_imuWatchdogTimer = nullptr; - std::atomic m_imuUpdateInProgress{false}; + QTimer *m_watchdogTimer = nullptr; + std::atomic m_poseUpdateInProgress{false}; qreal m_focusedDisplayDistance = 0.85; qreal m_allDisplaysDistance = 1.05; qreal m_displaySpacing = 0.0; diff --git a/kwin/src/qml/BreezyDesktop.qml b/kwin/src/qml/BreezyDesktop.qml index c660827..5cf65ab 100644 --- a/kwin/src/qml/BreezyDesktop.qml +++ b/kwin/src/qml/BreezyDesktop.qml @@ -26,11 +26,11 @@ Node { } function updateFocus(smoothFollowEnabledChanged = false) { - const rotations = smoothFollowEnabled ? effect.smoothFollowOrigin : effect.imuRotations; - if (rotations && rotations.length > 0) { + const orientations = smoothFollowEnabled ? effect.smoothFollowOrigin : effect.poseOrientations; + if (orientations && orientations.length > 0) { let focusedIndex = -1; const lookingAtIndex = displays.findFocusedMonitor( - displays.eusToNwuQuat(rotations[0]), + displays.eusToNwuQuat(orientations[0]), breezyDesktop.monitorPlacements.map(monitorVectors => monitorVectors.centerLook), breezyDesktop.focusedMonitorIndex, smoothFollowEnabled, @@ -120,10 +120,10 @@ Node { } // smoothFollowOrigin is the rotation away from the original placement of the displays - // imuRotations is the smooth follow rotation relative to the camera (very near an identity quat) + // poseOrientations is the smooth follow rotation relative to the camera (very near an identity quat) // subtract the latter from the former to get the complete rotation function smoothFollowQuat() { - return effect.smoothFollowOrigin[0].times(effect.imuRotations[0].conjugated()); + return effect.smoothFollowOrigin[0].times(effect.poseOrientations[0].conjugated()); } function displaySmoothFollowVector(display, smoothFollowRotation) { @@ -192,9 +192,9 @@ Node { } } - // smoothFollowEnabled gets cleared before the IMU begins slerping back to the origin so we can't just - // switch off smooth follow logic based on this flag. Instead, we have to rely on - // smoothFollowTransitionProgress to determine how much of the IMU positions to apply. + // smoothFollowEnabled gets cleared before the orientation begins slerping back to the origin so we can't just + // switch off smooth follow logic based on this flag. Instead, we have to rely on + // smoothFollowTransitionProgress to determine how much of the orientations to apply. onSmoothFollowEnabledChanged: { updateFocus(true); } @@ -218,7 +218,7 @@ Node { // When smooth follow is running, we're updating the position of the display manually // on every frame (avoid binding to a function that uses non-notify effect properties - // imuRotations and smoothFollowOrigin). + // poseOrientations and smoothFollowOrigin). focusedDisplay.position = displayPosition(focusedDisplay, smoothFollowRotation); } else { focusedDisplay.rotation = Qt.quaternion(1, 0, 0, 0); diff --git a/kwin/src/qml/CameraController.qml b/kwin/src/qml/CameraController.qml index af00f15..886116c 100644 --- a/kwin/src/qml/CameraController.qml +++ b/kwin/src/qml/CameraController.qml @@ -22,17 +22,17 @@ Item { aspectRatio ); - // if true, then smoothFollowEnabled just cleared and the IMU data is slerping back, + // if true, then smoothFollowEnabled just cleared and the orientation data is slerping back, // continue to use the origin data for the duration of the Timer property bool smoothFollowDisabling: false property real clipNear: 10.0 property real clipFar: 10000.0 - function ratesOfChange(rotations) { - const e0 = rotations[0].toEulerAngles(); - const e1 = rotations[1].toEulerAngles(); - const dt = effect.imuTimeElapsedMs; + function ratesOfChange(orientations) { + const e0 = orientations[0].toEulerAngles(); + const e1 = orientations[1].toEulerAngles(); + const dt = effect.poseTimeElapsedMs; const yawDegrees = (e0.y - e1.y) / dt; const pitchDegrees = (e0.x - e1.x) / dt; const rollDegrees = (e0.z - e1.z) / dt; @@ -49,22 +49,22 @@ Item { }; } - function updateCamera(rotations, rates) { + function updateCamera(orientations, position, rates) { camera.eulerRotation = applyLookAhead( rates, lookAheadMS( - effect.imuTimestamp, + effect.poseTimestamp, effect.lookAheadConfig, effect.lookAheadOverride ) ); - camera.position = rotations[0].times(Qt.vector3d(0, 0, -fovDetails.lensDistancePixels)); + camera.position = position.times(fovDetails.completeScreenDistancePixels).plus(orientations[0].times(Qt.vector3d(0, 0, -fovDetails.lensDistancePixels))); } - // how far to look ahead is how old the IMU data is plus a constant that is either the default for this device or an override - function lookAheadMS(imuDateMs, lookAheadConfig, override) { - // how stale the imu data is - const dataAge = Date.now() - imuDateMs; + // how far to look ahead is how old the pose data is plus a constant that is either the default for this device or an override + function lookAheadMS(poseDateMs, lookAheadConfig, override) { + // how stale the pose data is + const dataAge = Date.now() - poseDateMs; const lookAheadConstant = lookAheadConfig[0]; const lookAheadMultiplier = lookAheadConfig[1]; @@ -148,10 +148,10 @@ Item { FrameAnimation { running: true onTriggered: { - const rotations = (effect.smoothFollowEnabled || smoothFollowDisabling) ? effect.smoothFollowOrigin : effect.imuRotations; - if (rotations && rotations.length > 0) { - const rates = ratesOfChange(rotations); - updateCamera(rotations, rates); + const orientations = (effect.smoothFollowEnabled || smoothFollowDisabling) ? effect.smoothFollowOrigin : effect.poseOrientations; + if (orientations && orientations.length > 0) { + const rates = ratesOfChange(orientations); + updateCamera(orientations, effect.posePosition, rates); applyRollingShutterShear(rates); } } diff --git a/kwin/src/qml/main.qml b/kwin/src/qml/main.qml index 0544ba5..2682d4a 100644 --- a/kwin/src/qml/main.qml +++ b/kwin/src/qml/main.qml @@ -93,14 +93,14 @@ Item { property bool targetScreenSupported: supportedModels.some(model => root.targetScreen.model.includes(model)) property bool targetScreenIsVirtual: targetScreen.name.includes("BreezyDesktop") - property bool imuResetState: effect.imuResetState + property bool poseResetState: effect.poseResetState property bool isEnabled: effect.isEnabled Component { id: desktopViewComponent SingleDesktopView { supportsXR: targetScreenSupported - showCalibratingBanner: isEnabled && imuResetState + showCalibratingBanner: isEnabled && poseResetState } } @@ -141,8 +141,8 @@ Item { } function checkLoadedComponent() { - console.log(`Breezy - checking screen ${targetScreen.model}: ${targetScreenSupported} ${targetScreenIsVirtual} ${isEnabled} ${imuResetState}`); - const show3DView = targetScreenSupported && isEnabled && !imuResetState; + console.log(`Breezy - checking screen ${targetScreen.model}: ${targetScreenSupported} ${targetScreenIsVirtual} ${isEnabled} ${poseResetState}`); + const show3DView = targetScreenSupported && isEnabled && !poseResetState; if (!targetScreenIsVirtual) viewLoader.sourceComponent = show3DView ? view3DComponent : desktopViewComponent; if (targetScreenSupported) effect.effectTargetScreenIndex = KWinComponents.Workspace.screens.indexOf(targetScreen); } @@ -151,7 +151,7 @@ Item { checkLoadedComponent(); } - onImuResetStateChanged: { + onPoseResetStateChanged: { checkLoadedComponent(); } diff --git a/modules/XRLinuxDriver b/modules/XRLinuxDriver index 3bab088..6432928 160000 --- a/modules/XRLinuxDriver +++ b/modules/XRLinuxDriver @@ -1 +1 @@ -Subproject commit 3bab088768849a3068146586fed60ef0cc7d506d +Subproject commit 6432928fe013298b82ce69f466cef615d3aad0dd diff --git a/modules/sombrero b/modules/sombrero index 712497b..40326b9 160000 --- a/modules/sombrero +++ b/modules/sombrero @@ -1 +1 @@ -Subproject commit 712497b4795dbcbaa4c6fdb60a82144cfde26086 +Subproject commit 40326b9ec0266352f24500f47693c06f39832509 diff --git a/ui/modules/PyXRLinuxDriverIPC b/ui/modules/PyXRLinuxDriverIPC index 0eb04ff..da173bd 160000 --- a/ui/modules/PyXRLinuxDriverIPC +++ b/ui/modules/PyXRLinuxDriverIPC @@ -1 +1 @@ -Subproject commit 0eb04ff4429ce7a025f126843cd0d3b24bc0d73e +Subproject commit da173bd9e0392aaeb2cb68a332e5d4a20dd4dae1