diff --git a/wayvr/src/backend/openxr/playspace.rs b/wayvr/src/backend/openxr/playspace.rs
index 21457765..0bc370b8 100644
--- a/wayvr/src/backend/openxr/playspace.rs
+++ b/wayvr/src/backend/openxr/playspace.rs
@@ -82,10 +82,11 @@ impl PlayspaceMover {
                 return;
             }
 
+            let pose_quat = Quat::from_affine3(&data.pose);
             let new_hand =
-                Quat::from_affine3(&(data.pose * app.input_state.pointers[data.hand].raw_pose));
+                pose_quat * Quat::from_affine3(&app.input_state.pointers[data.hand].raw_pose);
 
-            let dq = new_hand * data.hand_pose.conjugate();
+            let dq = pose_quat.conjugate() * (new_hand * data.hand_pose.conjugate()) * pose_quat;
             let mut space_transform = if app.session.config.space_rotate_unlocked {
                 Affine3A::from_quat(dq)
             } else {