diff --git a/wayvr/src/backend/openvr/mod.rs b/wayvr/src/backend/openvr/mod.rs
index c1772578..e9406825 100644
--- a/wayvr/src/backend/openvr/mod.rs
+++ b/wayvr/src/backend/openvr/mod.rs
@@ -279,7 +279,7 @@ pub fn openvr_run(args: &Args) -> Result<(), BackendError> {
 
         overlays.values_mut().for_each(|o| o.config.tick(&mut app));
 
-        playspace.update(&mut chaperone_mgr, &mut overlays, &app);
+        playspace.update(&mut chaperone_mgr, &mut overlays, &mut app);
 
         current_lines.clear();
 
diff --git a/wayvr/src/backend/openvr/playspace.rs b/wayvr/src/backend/openvr/playspace.rs
index bedffe97..62458229 100644
--- a/wayvr/src/backend/openvr/playspace.rs
+++ b/wayvr/src/backend/openvr/playspace.rs
@@ -67,7 +67,7 @@ impl PlayspaceMover {
         &mut self,
         chaperone_mgr: &mut ChaperoneSetupManager,
         overlays: &mut OverlayWindowManager<OpenVrOverlayData>,
-        app: &AppState,
+        app: &mut AppState,
     ) {
         let universe = self.universe.clone();
 
@@ -146,9 +146,9 @@ impl PlayspaceMover {
             }
 
             let overlay_offset = data.pose.inverse().transform_vector3a(relative_pos) * -1.0;
-            playspace_common::shift_overlays(overlays, overlay_offset);
-
+            let before_pose = data.pose;
             data.pose.translation += relative_pos;
+            playspace_common::shift_world(overlays, &mut app.anchor, &before_pose, &data.pose);
             data.hand_pose = new_hand;
 
             if self.universe == ETrackingUniverseOrigin::TrackingUniverseStanding {
diff --git a/wayvr/src/backend/openxr/playspace.rs b/wayvr/src/backend/openxr/playspace.rs
index da896fca..8cc8fb30 100644
--- a/wayvr/src/backend/openxr/playspace.rs
+++ b/wayvr/src/backend/openxr/playspace.rs
@@ -174,10 +174,9 @@ impl PlayspaceMover {
                 return;
             }
 
-            let overlay_offset = data.pose.inverse().transform_vector3a(relative_pos) * -1.0;
-            playspace_common::shift_overlays(overlays, overlay_offset);
-
+            let before_pose = data.pose;
             data.pose.translation += relative_pos;
+            playspace_common::shift_world(overlays, &mut app.anchor, &before_pose, &data.pose);
             data.hand_pose = new_hand;
 
             apply_offset(data.pose, &mut monado.ipc);
@@ -218,12 +217,12 @@ impl PlayspaceMover {
             config: &app.session.config,
             floor_height: app.session.config.space_gravity_floor_height,
         }) {
-            apply_offset(
-                Affine3A::from_translation(res.playspace_pos.into()),
-                &mut monado.ipc,
-            );
+            let prev_playspace_pos = res.playspace_pos - res.playspace_pos_offset;
+            let before_pose = Affine3A::from_translation(prev_playspace_pos.into());
+            let after_pose = Affine3A::from_translation(res.playspace_pos.into());
+            apply_offset(after_pose, &mut monado.ipc);
 
-            playspace_common::shift_overlays(overlays, -res.playspace_pos_offset);
+            playspace_common::shift_world(overlays, &mut app.anchor, &before_pose, &after_pose);
         }
     }
 
diff --git a/wayvr/src/backend/playspace_common.rs b/wayvr/src/backend/playspace_common.rs
index ceac0e7e..a0501cfc 100644
--- a/wayvr/src/backend/playspace_common.rs
+++ b/wayvr/src/backend/playspace_common.rs
@@ -1,4 +1,4 @@
-use glam::Vec3A;
+use glam::{Affine3A, Vec3A};
 use wlx_common::config::GeneralConfig;
 
 use crate::windowing::manager::OverlayWindowManager;
@@ -15,17 +15,20 @@ pub struct SpaceGravity {
     space_pos: Vec3A,
 }
 
-pub fn shift_overlays<OverlayData>(
+pub fn shift_world<OverlayData>(
     overlays: &mut OverlayWindowManager<OverlayData>,
-    overlay_offset: Vec3A,
+    anchor: &mut Affine3A,
+    before: &Affine3A,
+    after: &Affine3A,
 ) {
+    let correction = after.inverse() * before;
+    *anchor = correction * *anchor;
+
     overlays.values_mut().for_each(|overlay| {
         let Some(state) = overlay.config.active_state.as_mut() else {
             return;
         };
-        if state.positioning.moves_with_space() {
-            state.transform.translation += overlay_offset;
-        }
+        state.transform = correction * state.transform;
         overlay.config.dirty = true;
     });
 }
diff --git a/wlx-common/src/windowing.rs b/wlx-common/src/windowing.rs
index 139d6dab..16130de6 100644
--- a/wlx-common/src/windowing.rs
+++ b/wlx-common/src/windowing.rs
@@ -28,9 +28,6 @@ pub enum Positioning {
 }
 
 impl Positioning {
-	pub const fn moves_with_space(self) -> bool {
-		matches!(self, Self::Floating | Self::Anchored | Self::Static)
-	}
 	pub const fn get_lerp(self) -> Option<f32> {
 		match self {
 			Self::FollowHead { lerp } => Some(lerp),