Look-ahead work in progress

kwin/src/breezydesktopeffect.cpp

@@ -56,11 +56,21 @@ BreezyDesktopEffect::BreezyDesktopEffect()
 
     setSource(QUrl::fromLocalFile(QStandardPaths::locate(QStandardPaths::GenericDataLocation, QStringLiteral("kwin/effects/breezy_desktop/qml/main.qml"))));
 
-    // Monitor the IMU file for changes
+    // Monitor the IMU file for changes, even if it doesn't exist at startup
     const QString shmPath = QStringLiteral("/dev/shm/breezy_desktop_imu");
-    m_xrRotationFileWatcher = new QFileSystemWatcher(this);
+    const QString shmDir = QStringLiteral("/dev/shm");
-    m_xrRotationFileWatcher->addPath(shmPath);
+    m_imuRotationFileWatcher = new QFileSystemWatcher(this);
-    connect(m_xrRotationFileWatcher, &QFileSystemWatcher::fileChanged, this, &BreezyDesktopEffect::updateXrRotation);
+    if (QFile::exists(shmPath)) {
+        m_imuRotationFileWatcher->addPath(shmPath);
+    } else {
+        m_imuRotationFileWatcher->addPath(shmDir);
+        connect(m_imuRotationFileWatcher, &QFileSystemWatcher::directoryChanged, this, [this, shmPath](const QString &) {
+            if (QFile::exists(shmPath) && !m_imuRotationFileWatcher->files().contains(shmPath)) {
+                m_imuRotationFileWatcher->addPath(shmPath);
+            }
+        });
+    }
+    connect(m_imuRotationFileWatcher, &QFileSystemWatcher::fileChanged, this, &BreezyDesktopEffect::updateImuRotation);
 
     m_cursorUpdateTimer = new QTimer(this);
     connect(m_cursorUpdateTimer, &QTimer::timeout, this, &BreezyDesktopEffect::updateCursorPos);
@@ -172,89 +182,91 @@ QColor BreezyDesktopEffect::backgroundColor() const
     return QColor(Qt::black);
 }
 
-QQuaternion BreezyDesktopEffect::xrRotation() const {
+QList<QQuaternion> BreezyDesktopEffect::imuRotations() const {
-    return m_xrRotation;
+    return m_imuRotations;
 }
 
-void BreezyDesktopEffect::updateXrRotation() {
+quint32 BreezyDesktopEffect::imuTimeElapsedMs() const {
+    return m_imuTimeElapsedMs;
+}
+
+quint64 BreezyDesktopEffect::imuTimestamp() const {
+    return m_imuTimestamp;
+}
+
+qreal BreezyDesktopEffect::lookAheadConstant() const {
+    return m_lookAheadConstant;
+}
+
+// TODO - can this be something callable from the camera qml code, so it's pulled only when needed?
+void BreezyDesktopEffect::updateImuRotation() {
     const QString shmPath = QStringLiteral("/dev/shm/breezy_desktop_imu");
     QFile shmFile(shmPath);
-    
     if (!shmFile.open(QIODevice::ReadOnly)) {
         return;
     }
-    
     QByteArray buffer = shmFile.readAll();
     shmFile.close();
-    
+    if (buffer.size() < 64) {
-    if (buffer.size() < 64) { // Minimum expected size based on the data structure
         return;
     }
-    
-    // Create a data view for reading binary data
     const char* data = buffer.constData();
-    // Use proper data positions based on the original GJS layout
+    quint8 version = static_cast<quint8>(data[0]);
-    // VERSION at offset 0, ENABLED at offset 1, etc.
+    quint8 enabledFlag = static_cast<quint8>(data[1]);
-    
+    float lookAheadCnst;
-    // Read version and enabled flags at their correct positions
+    memcpy(&lookAheadCnst, data + 2, sizeof(float));
-    quint8 version = static_cast<quint8>(data[0]); // VERSION at offset 0
-    quint8 enabledFlag = static_cast<quint8>(data[1]); // ENABLED at offset 1
-    
-    // DISPLAY_FOV is at offset: 1 + 1 + (4*4) + (4*2) = 26
     float displayFov;
     memcpy(&displayFov, data + 26, sizeof(float));
-    
-    // EPOCH_MS is at offset: 26 + 4 + 4 + 1 + 1 + 1 + (4*16) = 101
     quint64 imuDateMs;
     memcpy(&imuDateMs, data + 101, sizeof(quint64));
     imuDateMs = qFromLittleEndian(imuDateMs);
     
-    // IMU_QUAT_DATA is at offset: 101 + 8 = 109
-    float imuData[4];
-    memcpy(imuData, data + 109, sizeof(imuData));
-    
-    // Validate data
     const quint64 currentTimeMs = QDateTime::currentMSecsSinceEpoch();
-    const bool validKeepAlive = (currentTimeMs - imuDateMs) < 5000; // 5 second timeout
+    const bool validKeepAlive = (currentTimeMs - imuDateMs) < 5000;
     const bool validData = validKeepAlive && displayFov != 0.0f;
-    const quint8 expectedVersion = 4; // Define expected data layout version
+    const quint8 expectedVersion = 4;
     const bool enabled = (enabledFlag != 0) && (version == expectedVersion) && validData;
-    
     if (!enabled) {
         if (isRunning()) {
             qCCritical(KWIN_XR) << "\t\t\tBreezy - deactivate due to disabled";
             deactivate();
         }
-
         return;
     }
-    
+
-    // Check for reset state (identity quaternion)
+    float imuData[4];
-    const bool imuResetState = (imuData[0] == 0.0f && imuData[1] == 0.0f && 
+    memcpy(imuData, data + 109, sizeof(imuData));
-                               imuData[2] == 0.0f && imuData[3] == 1.0f);
+    const bool imuResetState = (imuData[0] == 0.0f && imuData[1] == 0.0f && imuData[2] == 0.0f && imuData[3] == 1.0f);
-    
     if (imuResetState) {
         if (isRunning()) {
             qCCritical(KWIN_XR) << "\t\t\tBreezy - deactivate due to reset state";
             deactivate();
         }
-
         return;
     }
-    
-    // Create quaternion (w, x, y, z)
-    QQuaternion quat(imuData[3], imuData[0], imuData[1], imuData[2]);
-    
-    if (quat != m_xrRotation) {
-        m_xrRotation = quat;
-        
-        if (!isRunning()) {
-            qCCritical(KWIN_XR) << "\t\t\tBreezy - activate";
-            activate();
-        }
 
-        Q_EMIT xrRotationChanged();
+    QQuaternion quatT0(imuData[3], imuData[0], imuData[1], imuData[2]);
+
+    memcpy(imuData, data + 109 + sizeof(imuData), sizeof(imuData));
+    QQuaternion quatT1(imuData[3], imuData[0], imuData[1], imuData[2]);
+
+    // set imuRotations to the last two rotations, leave out the elapsed time
+    m_imuRotations.clear();
+    m_imuRotations.append(quatT0);
+    m_imuRotations.append(quatT1);
+
+    // 3rd row is imuData at the 3rd timestamp, that is unused, 4th row contains the timestamps
+    memcpy(imuData, data + 109 + sizeof(imuData) * 3, sizeof(imuData));
+    // elapsed time between T0 and T1 is: imuData[0] - imuData[1]
+    m_imuTimeElapsedMs = static_cast<quint32>(imuData[0] - imuData[1]);
+
+    m_imuTimestamp = imuDateMs;
+    m_lookAheadConstant = lookAheadCnst;
+    if (!isRunning()) {
+        qCCritical(KWIN_XR) << "\t\t\tBreezy - activate";
+        activate();
     }
+    Q_EMIT imuRotationsChanged();
 }
 
 QString BreezyDesktopEffect::cursorImageSource() const

kwin/src/breezydesktopeffect.h

@@ -20,7 +20,10 @@ namespace KWin
         Q_PROPERTY(bool mouseInvertedY READ mouseInvertedY NOTIFY mouseInvertedYChanged)
         Q_PROPERTY(BackgroundMode backgroundMode READ backgroundMode NOTIFY backgroundModeChanged)
         Q_PROPERTY(QColor backgroundColor READ backgroundColor NOTIFY backgroundColorChanged)
-        Q_PROPERTY(QQuaternion xrRotation READ xrRotation NOTIFY xrRotationChanged)
+        Q_PROPERTY(QList<QQuaternion> imuRotations READ imuRotations NOTIFY imuRotationsChanged)
+        Q_PROPERTY(quint32 imuTimeElapsedMs READ imuTimeElapsedMs NOTIFY imuRotationsChanged)
+        Q_PROPERTY(quint64 imuTimestamp READ imuTimestamp NOTIFY imuRotationsChanged)
+        Q_PROPERTY(quint8 lookAheadConstant READ lookAheadConstant NOTIFY imuRotationsChanged)
         Q_PROPERTY(QString cursorImageSource READ cursorImageSource NOTIFY cursorImageChanged)
         Q_PROPERTY(QPointF cursorPos READ cursorPos NOTIFY cursorPosChanged)
 
@@ -51,13 +54,16 @@ namespace KWin
         void showCursor();
         void hideCursor();
 
-        QQuaternion xrRotation() const;
+        QList<QQuaternion> imuRotations() const;
+        quint32 imuTimeElapsedMs() const;
+        quint64 imuTimestamp() const;
+        qreal lookAheadConstant() const;
 
     public Q_SLOTS:
         void activate();
         void deactivate();
         void toggle();
-        void updateXrRotation();
+        void updateImuRotation();
         void updateCursorImage();
         void updateCursorPos();
 
@@ -70,7 +76,7 @@ namespace KWin
         void skyboxChanged();
         void backgroundModeChanged();
         void backgroundColorChanged();
-        void xrRotationChanged();
+        void imuRotationsChanged();
         void cursorImageChanged();
         void cursorPosChanged();
 
@@ -88,8 +94,11 @@ namespace KWin
         QString m_cursorImageSource;
         bool m_isMouseHidden = false;
 
-        QQuaternion m_xrRotation;
+        QList<QQuaternion> m_imuRotations;
-        QFileSystemWatcher *m_xrRotationFileWatcher = nullptr;
+        quint32 m_imuTimeElapsedMs;
+        quint64 m_imuTimestamp;
+        qreal m_lookAheadConstant = 10.0;
+        QFileSystemWatcher *m_imuRotationFileWatcher = nullptr;
         QPointF m_cursorPos;
         QTimer *m_cursorUpdateTimer = nullptr;
     };

kwin/src/qml/BreezyDesktop.qml

@@ -20,9 +20,10 @@ Node {
     required property real viewportWidth
     required property real viewportHeight
     property real distance: viewportWidth / (2 * Math.tan(Math.PI * viewportFOVHorizontal / 360))
-    property var screens: KWinComponents.Workspace.screens.filter(function(screen) {
+    property var screens: KWinComponents.Workspace.screens
-        return supportedModels.includes(screen.model);
+    // .filter(function(screen) {
-    })
+    //     return supportedModels.includes(screen.model);
+    // })
 
     // x value for placing the viewport in the middle of all screens
     property real screensXMid: {

kwin/src/qml/CameraController.qml

@@ -6,7 +6,7 @@ Item {
 
     required property Camera camera
 
-    property quaternion rotation: Quaternion.fromEulerAngles(0, 0, 0)
+    property vector3d rotation: Qt.vector3d(0, 0, 0)
     property real radius: 2000
 
     property real speed: 1
@@ -20,30 +20,64 @@ Item {
     onRadiusChanged: root.updateCamera();
 
     function updateCamera() {
-        // convert NWU to EUS by passing root.rotation values: w, -y, z, -x
-        let effectiveRotation = Qt.quaternion(root.rotation.scalar, -root.rotation.y, root.rotation.z, -root.rotation.x);
-        
-        const eulerRotation = effectiveRotation.toEulerAngles();
         const theta = 90 * Math.PI / 180;
         const phi = 0.0;
 
         camera.position = Qt.vector3d(radius * Math.sin(phi) * Math.sin(theta),
                                       radius * Math.cos(theta),
                                       radius * Math.cos(phi) * Math.sin(theta));
-        camera.rotation = effectiveRotation;
+        camera.eulerRotation = root.rotation;
     }
 
-    // Add property to receive XR rotation from effect
+    // how far to look ahead is how old the IMU data is plus a constant that is either the default for this device or an override
-    property quaternion xrRotation: effect.xrRotation
+    function lookAheadMS(imuDateMs, lookAheadConstant, override) {
-    property bool useXrRotation: true // Set to true to use XR rotation when available
+        // how stale the imu data is
+        const dataAge = Date.now() - imuDateMs;
 
-    Timer {
+        return (override === -1 ? lookAheadConstant : override) + dataAge;
-        interval: 16
+    }
-        repeat: true
+
+    function applyLookAhead(quatT0, quatT1, elapsedTimeMs, lookAheadMs) {
+        console.log(`Applying look-ahead with ${elapsedTimeMs} and ${lookAheadMs}`);
+        // convert both quats to euler angles
+        const eulerT0 = quatT0.toEulerAngles();
+        const eulerT1 = quatT1.toEulerAngles();
+
+        // compute the rate of change of the angles based on the elapsed time
+        const deltaX = (eulerT0.x - eulerT1.x);
+        const deltaY = (eulerT0.y - eulerT1.y);
+        const deltaZ = (eulerT0.z - eulerT1.z);
+
+        // how much of the delta to apply based on the look-ahead time
+        const timeConstant = lookAheadMs / elapsedTimeMs;
+
+        // compute the look-ahead angles and convert NWU to EUS by passing root.rotation values: -y, z, -x
+        return Qt.vector3d(
+            -eulerT0.y + deltaY * timeConstant,
+            eulerT0.z + deltaZ * timeConstant,
+            -eulerT0.x + deltaX * timeConstant
+        );
+    }
+
+    // Add property to receive IMU rotation snapshots from effect
+    property var imuRotations: effect.imuRotations
+    property int imuTimeElapsedMs: effect.imuTimeElapsedMs
+    property double imuTimestamp: effect.imuTimestamp  
+    property double lookAheadConstant: effect.lookAheadConstant
+    property bool useImuRotation: true // Set to true to use XR rotation when available
+
+    FrameAnimation {
         running: true
         onTriggered: {
-            if (useXrRotation && xrRotation.length() > 0) {
+            console.log("FrameAnimation triggered, updating camera rotation");
-                root.rotation = xrRotation;
+            if (root.useImuRotation && root.imuRotations && root.imuRotations.length > 0) {
+                console.log("Using IMU rotation for camera control");
+                root.rotation = applyLookAhead(
+                    root.imuRotations[0],
+                    root.imuRotations[1],
+                    root.imuTimeElapsedMs,
+                    lookAheadMS(root.imuTimestamp, root.lookAheadConstant, -1)
+                );
             }
         }
     }

kwin/src/qml/main.qml

@@ -24,35 +24,6 @@ Item {
         id: view
         anchors.fill: parent
 
-        Loader {
-            id: colorSceneEnvironment
-            active: effect.backgroundMode == CubeEffect.BackgroundMode.Color
-            sourceComponent: SceneEnvironment {
-                clearColor: effect.backgroundColor
-                backgroundMode: SceneEnvironment.Color
-            }
-        }
-
-        Loader {
-            id: skyboxSceneEnvironment
-            active: effect.backgroundMode == CubeEffect.BackgroundMode.Skybox
-            sourceComponent: SceneEnvironment {
-                backgroundMode: SceneEnvironment.SkyBox
-                lightProbe: Texture {
-                    source: effect.skybox
-                }
-            }
-        }
-
-        environment: {
-            switch (effect.backgroundMode) {
-            case CubeEffect.BackgroundMode.Skybox:
-                return skyboxSceneEnvironment.item;
-            case CubeEffect.BackgroundMode.Color:
-                return colorSceneEnvironment.item;
-            }
-        }
-
         PerspectiveCamera { 
             id: camera
             fieldOfView: 22.55
@@ -85,13 +56,6 @@ Item {
                     easing.type: Easing.OutCubic
                 }
             }
-
-            function rotateTo(desktop) {
-                if (rotationAnimation.running) {
-                    return;
-                }
-                rotation = Quaternion.fromEulerAngles(0, 0, 0);
-            }
         }
     }