GNOME 45 WIP

gnome/breezydesktop@org.xronlinux/IMUAdjust.frag

@@ -0,0 +1,187 @@
+#version 330 core
+
+uniform bool enabled;
+uniform bool show_banner;
+uniform sampler2D uDesktopTexture;
+uniform mat4 imu_quat_data;
+uniform vec4 look_ahead_cfg;
+uniform float display_zoom;
+uniform float display_north_offset;
+uniform float lens_distance_ratio;
+uniform bool sbs_enabled;
+uniform bool sbs_content;
+uniform bool sbs_mode_stretched;
+uniform bool custom_banner_enabled;
+uniform float stage_aspect_ratio;
+uniform float display_aspect_ratio;
+uniform float half_fov_z_rads;
+uniform float half_fov_y_rads;
+uniform float screen_distance;
+uniform float frametime;
+
+float look_ahead_ms_cap = 45.0;
+
+vec4 quatMul(vec4 q1, vec4 q2) {
+    vec3 u = vec3(q1.x, q1.y, q1.z);
+    float s = q1.w;
+    vec3 v = vec3(q2.x, q2.y, q2.z);
+    float t = q2.w;
+    return vec4(s*v + t*u + cross(u, v), s*t - dot(u, v));
+}
+
+vec4 quatConj(vec4 q) {
+	return vec4(-q.x, -q.y, -q.z, q.w);
+}
+
+vec3 applyQuaternionToVector(vec4 q, vec3 v) {
+	vec4 p = quatMul(quatMul(q, vec4(v, 0)), quatConj(q));
+	return p.xyz;
+}
+
+const int day_in_seconds = 24 * 60 * 60;
+
+vec3 applyLookAhead(
+	in vec3 position,
+	in vec3 velocity,
+	in vec3 accel,
+	in float t,
+	in float t_squared) {
+	return position + velocity * t + 0.5 * accel * t_squared;
+}
+
+vec3 rateOfChange(
+	in vec3 v1,
+	in vec3 v2,
+	in float delta_time) {
+	return (v1 - v2) / delta_time;
+}
+
+bool isKeepaliveRecent(
+	in vec4 currentDate,
+	in vec4 keepAliveDate) {
+	float _174 = currentDate.w + float(day_in_seconds);
+	float _176 = _174 - keepAliveDate.w;
+	float _178 = mod(_176, float(day_in_seconds));
+	float _179 = abs(_178);
+	bool _181 = _179 <= 5.00000000e+00;
+	return _181;
+}
+
+void PS_IMU_Transform(vec4 pos, vec2 texcoord, out vec4 color) {
+	float texcoord_x_min = 0.0;
+	float texcoord_x_max = 1.0;
+	float lens_y_offset = 0.0;
+	float lens_z_offset = 0.0;
+	float aspect_ratio = stage_aspect_ratio;
+
+	if (enabled && sbs_enabled) {
+        bool right_display = texcoord.x > 0.5;
+        aspect_ratio /= 2;
+
+        lens_y_offset = lens_distance_ratio / 3;
+        if (right_display) lens_y_offset = -lens_y_offset;
+        if (sbs_content) {
+            // source video is SBS, left-half of the screen goes to the left lens, right-half to the right lens
+            if (right_display)
+                texcoord_x_min = 0.5;
+            else
+                texcoord_x_max = 0.5;
+        }
+        if (!sbs_mode_stretched) {
+            // if the content isn't stretched, assume it's centered in the middle 50% of the screen
+            texcoord_x_min = max(0.25, texcoord_x_min);
+            texcoord_x_max = min(0.75, texcoord_x_max);
+        }
+
+        // translate the texcoord respresenting the current lens's half of the screen to a full-screen texcoord
+        texcoord.x = (texcoord.x - (right_display ? 0.5 : 0.0)) * 2;
+    }
+
+	if (!enabled || show_banner) {
+		// vec2 banner_size = vec2(800.0 / ReShade::ScreenSize.x, 200.0 / ReShade::ScreenSize.y); // Assuming ScreenWidth and ScreenHeight are defined
+
+        // if (show_banner &&
+        //     texcoord.x >= banner_position.x - banner_size.x / 2 &&
+        //     texcoord.x <= banner_position.x + banner_size.x / 2 &&
+        //     texcoord.y >= banner_position.y - banner_size.y / 2 &&
+        //     texcoord.y <= banner_position.y + banner_size.y / 2)
+        // {
+        //     vec2 banner_texcoord = (texcoord - (banner_position - banner_size / 2)) / banner_size;
+        //     if (custom_banner_enabled) {
+        //         color = tex2D(customBannerSampler, banner_texcoord);
+        //     } else {
+        //         color = tex2D(calibratingSampler, banner_texcoord);
+        //     }
+        // } else {
+            // adjust texcoord back to the range that describes where the content is displayed
+            float texcoord_width = texcoord_x_max - texcoord_x_min;
+            texcoord.x = texcoord.x * texcoord_width + texcoord_x_min;
+
+			color = texture2D(uDesktopTexture, texcoord);
+        // }
+	} else {
+		float lens_fov_z_offset_rads = atan(lens_z_offset/screen_distance);
+		float fov_z_pos = tan(half_fov_z_rads - lens_fov_z_offset_rads) * screen_distance;
+		float fov_z_neg = -tan(half_fov_z_rads + lens_fov_z_offset_rads) * screen_distance;
+		float fov_z_width = fov_z_pos - fov_z_neg;
+
+		float lens_fov_y_offset_rads = atan(lens_y_offset/screen_distance);
+		float fov_y_pos = tan(half_fov_y_rads - lens_fov_y_offset_rads) * screen_distance;
+		float fov_y_neg = -tan(half_fov_y_rads + lens_fov_y_offset_rads) * screen_distance;
+		float fov_y_width = fov_y_pos - fov_y_neg;
+		float vec_x = screen_distance;
+		float vec_y = -texcoord.x * fov_y_width + fov_y_pos;
+		float vec_z = -texcoord.y * fov_z_width + fov_z_pos;
+		vec3 texcoord_vector = vec3(vec_x, vec_y, vec_z);
+		vec3 lens_vector = vec3(lens_distance_ratio, lens_y_offset, lens_z_offset);
+
+        // then rotate the vector using each of the snapshots provided
+        vec3 rotated_vector_t0 = applyQuaternionToVector(imu_quat_data[0], texcoord_vector);
+        vec3 rotated_vector_t1 = applyQuaternionToVector(imu_quat_data[1], texcoord_vector);
+        vec3 rotated_vector_t2 = applyQuaternionToVector(imu_quat_data[2], texcoord_vector);
+        vec3 rotated_lens_vector = applyQuaternionToVector(imu_quat_data[0], lens_vector);
+
+        // compute the two velocities (units/ms) as change in the 3 rotation snapshots
+        float delta_time_t0 = imu_quat_data[3].x - imu_quat_data[3].y;
+        vec3 velocity_t0 = rateOfChange(rotated_vector_t0, rotated_vector_t1, delta_time_t0);
+        vec3 velocity_t1 = rateOfChange(rotated_vector_t1, rotated_vector_t2, imu_quat_data[3].y - imu_quat_data[3].z);
+
+        // and then the acceleration (units/ms^2) as the change in velocities
+        vec3 accel_t0 = rateOfChange(velocity_t0, velocity_t1, delta_time_t0);
+
+        // allows for the bottom and top of the screen to have different look-ahead values
+        float look_ahead_scanline_adjust = texcoord.y * look_ahead_cfg.z;
+
+        // use the 4th value of the look-ahead config to cap the look-ahead value
+        float look_ahead_ms = min(min(look_ahead_cfg.x + frametime * look_ahead_cfg.y, look_ahead_cfg.w), look_ahead_ms_cap) + look_ahead_scanline_adjust;
+        float look_ahead_ms_squared = pow(look_ahead_ms, 2);
+
+        // apply most recent velocity and acceleration to most recent position to get a predicted position
+        vec3 res = applyLookAhead(rotated_vector_t0, velocity_t0, accel_t0, look_ahead_ms, look_ahead_ms_squared);
+
+		bool looking_behind = res.x < 0.0;
+
+        // divide all values by x to scale the magnitude so x is exactly 1, and multiply by the final display distance
+        // so the vector is pointing at a coordinate on the screen
+        float display_distance = (sbs_enabled ? display_north_offset : 1.0) - rotated_lens_vector.x;
+        res *= display_distance/res.x;
+
+        // adjust x and y by how much our lens moved from its original offset
+        res += rotated_lens_vector - lens_vector;
+
+		// deconstruct the rotated and scaled vector back to a texcoord (just inverse operations of the first conversion
+		// above)
+		texcoord.x = (fov_y_pos - res.y) / fov_y_width;
+		texcoord.y = (fov_z_pos - res.z) / fov_z_width;
+
+		// apply the screen offsets now
+		float texcoord_width = texcoord_x_max - texcoord_x_min;
+		texcoord.x = texcoord.x * texcoord_width + texcoord_x_min;
+
+		if (looking_behind || texcoord.x < texcoord_x_min || texcoord.y < 0.0 || texcoord.x > texcoord_x_max || texcoord.y > 1.0 || texcoord.x <= 0.005 && texcoord.y <= 0.005) {
+			color = vec4(0, 0, 0, 1);
+		} else {
+			color = texture2D(uDesktopTexture, texcoord);
+		}
+	}
+}

gnome/breezydesktop@org.xronlinux/extension.js

@@ -0,0 +1,262 @@
+import Gio from 'gi://Gio';
+import GLib from 'gi://GLib';
+import GObject from 'gi://GObject';
+import Shell from 'gi://Shell';
+import Meta from 'gi://Meta';
+
+import ExtensionUtils from 'gi://ExtensionUtils';
+import Main from 'gi://Main';
+import PanelMenu from 'gi://PanelMenu';
+
+const UINT8_SIZE = 1;
+const BOOL_SIZE = UINT8_SIZE;
+const UINT_SIZE = 4;
+const FLOAT_SIZE = 4;
+
+const DATA_VIEW_INFO_OFFSET_INDEX = 0;
+const DATA_VIEW_INFO_SIZE_INDEX = 1;
+const DATA_VIEW_INFO_COUNT_INDEX = 2;
+
+// computes the end offset, exclusive
+function dataViewEnd(dataViewInfo) {
+    return dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX] + dataViewInfo[DATA_VIEW_INFO_SIZE_INDEX] * dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX];
+}
+
+// the driver should be using the same data layout version
+const DATA_LAYOUT_VERSION = 1;
+
+// DataView info: [offset, size, count]
+const VERSION = [0, UINT8_SIZE, 1];
+const ENABLED = [dataViewEnd(VERSION), BOOL_SIZE, 1];
+const EPOCH_SEC = [dataViewEnd(ENABLED), UINT_SIZE, 1];
+const LOOK_AHEAD_CFG = [dataViewEnd(EPOCH_SEC), FLOAT_SIZE, 4];
+const DISPLAY_RES = [dataViewEnd(LOOK_AHEAD_CFG), UINT_SIZE, 2];
+const DISPLAY_FOV = [dataViewEnd(DISPLAY_RES), FLOAT_SIZE, 1];
+const DISPLAY_ZOOM = [dataViewEnd(DISPLAY_FOV), FLOAT_SIZE, 1];
+const DISPLAY_NORTH_OFFSET = [dataViewEnd(DISPLAY_ZOOM), FLOAT_SIZE, 1];
+const LENS_DISTANCE_RATIO = [dataViewEnd(DISPLAY_NORTH_OFFSET), FLOAT_SIZE, 1];
+const SBS_ENABLED = [dataViewEnd(LENS_DISTANCE_RATIO), BOOL_SIZE, 1];
+const SBS_CONTENT = [dataViewEnd(SBS_ENABLED), BOOL_SIZE, 1];
+const SBS_MODE_STRETCHED = [dataViewEnd(SBS_CONTENT), BOOL_SIZE, 1];
+const CUSTOM_BANNER_ENABLED = [dataViewEnd(SBS_MODE_STRETCHED), BOOL_SIZE, 1];
+const IMU_QUAT_DATA = [dataViewEnd(CUSTOM_BANNER_ENABLED), FLOAT_SIZE, 16];
+
+// cached after first retrieval
+const shaderUniformLocations = {
+    'enabled': null,
+    'show_banner': null,
+    'imu_quat_data': null,
+    'look_ahead_cfg': null,
+    'stage_aspect_ratio': null,
+    'display_aspect_ratio': null,
+    'display_zoom': null,
+    'display_north_offset': null,
+    'lens_distance_ratio': null,
+    'sbs_enabled': null,
+    'sbs_content': null,
+    'sbs_mode_stretched': null,
+    'custom_banner_enabled': null,
+    'half_fov_z_rads': null,
+    'half_fov_y_rads': null,
+    'screen_distance': null,
+    'frametime': null
+};
+
+function dataViewUint8(dataView, dataViewInfo) {
+    return dataView.getUint8(dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX]);
+}
+
+function dataViewUint(dataView, dataViewInfo) {
+    return dataView.getUint32(dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX], true);
+}
+
+function dataViewUintArray(dataView, dataViewInfo) {
+    const uintArray = []
+    let offset = dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX];
+    for (let i = 0; i < dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX]; i++) {
+        uintArray.push(dataView.getUint32(offset, true));
+        offset += UINT_SIZE;
+    }
+    return uintArray;
+}
+
+function dataViewFloat(dataView, dataViewInfo) {
+    return dataView.getFloat32(dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX], true);
+}
+
+function dataViewFloatArray(dataView, dataViewInfo) {
+    const floatArray = []
+    let offset = dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX];
+    for (let i = 0; i < dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX]; i++) {
+        floatArray.push(dataView.getFloat32(offset, true));
+        offset += FLOAT_SIZE;
+    }
+    return floatArray;
+}
+
+
+function getShaderSource(path) {
+    const file = Gio.file_new_for_path(path);
+    const data = file.load_contents(null);
+
+    // version string helps with linting, but GNOME extension doesn't like it, so remove it if it's there
+    return data[1].toString().replace(/^#version .*$/gm, '') + '\n';
+}
+
+function transferUniformBoolean(effect, locationName, dataView, dataViewInfo) {
+    // GLSL bool is a float under the hood, evaluates false if 0 or 0.0, true otherwise
+    effect.set_uniform_float(locationName, 1, [dataViewUint8(dataView, dataViewInfo)]);
+}
+
+function setUniformFloat(effect, locationName, dataViewInfo, value) {
+    effect.set_uniform_float(shaderUniformLocations[locationName], dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX], value);
+}
+
+function transferUniformFloat(effect, locationName, dataView, dataViewInfo) {
+    setUniformFloat(effect, locationName, dataViewInfo, dataViewFloatArray(dataView, dataViewInfo));
+}
+
+function setSingleFloat(effect, locationName, value) {
+    effect.set_uniform_float(shaderUniformLocations[locationName], 1, [value]);
+}
+
+function setUniformMatrix(effect, locationName, components, dataView, dataViewInfo) {
+    const numValues = dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX];
+    if (numValues / components !== components) {
+        throw new Error('Invalid matrix size');
+    }
+
+    const floatArray = [].fill(0, 0, numValues);
+    let offset = dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX];
+    for (let i = 0; i < numValues; i++) {
+        // GLSL uses column-major order, so we need to transpose the matrix
+        const row = i % components;
+        const column = Math.floor(i / components);
+
+        floatArray[row * components + column] = dataView.getFloat32(offset, true);
+        offset += FLOAT_SIZE;
+    }
+    effect.set_uniform_matrix(shaderUniformLocations[locationName], true, components, floatArray);
+}
+
+function getEpochSec() {
+    return Math.floor(Date.now() / 1000);
+}
+
+function degreeToRadian(degree) {
+    return degree * Math.PI / 180;
+}
+
+
+
+// most uniforms don't change frequently, this function should be called periodically
+function setIntermittentUniformVariables() {
+    const dataView = this._dataView;
+    const version = dataViewUint8(dataView, VERSION);
+    const date = dataViewUint(dataView, EPOCH_SEC);
+    const validKeepalive = Math.abs(getEpochSec() - date) < 5;
+    const imuData = dataViewFloatArray(dataView, IMU_QUAT_DATA);
+    const imuResetState = imuData[0] === 0.0 && imuData[1] === 0.0 && imuData[2] === 0.0 && imuData[3] === 1.0;
+    const enabled = dataViewUint8(dataView, ENABLED) !== 0 && version === DATA_LAYOUT_VERSION && validKeepalive && !imuResetState;
+
+    if (enabled) {
+        const displayRes = dataViewUintArray(dataView, DISPLAY_RES);
+        const displayFov = dataViewFloat(dataView, DISPLAY_FOV);
+        const lensDistanceRatio = dataViewFloat(dataView, LENS_DISTANCE_RATIO);
+
+        // compute these values once, they only change when the XR device changes
+        const displayAspectRatio = displayRes[0] / displayRes[1];
+        const stageAspectRatio = global.stage.get_width() / global.stage.get_height();
+        const diagToVertRatio = Math.sqrt(Math.pow(stageAspectRatio, 2) + 1);
+        const halfFovZRads = degreeToRadian(displayFov / diagToVertRatio) / 2;
+        const halfFovYRads = halfFovZRads * stageAspectRatio;
+        const screenDistance = 1.0 - lensDistanceRatio;
+        
+        // all these values are transferred directly, unmodified from the driver
+        transferUniformFloat(this, 'look_ahead_cfg', dataView, LOOK_AHEAD_CFG);
+        transferUniformFloat(this, 'display_zoom', dataView, DISPLAY_ZOOM);
+        transferUniformFloat(this, 'display_north_offset', dataView, DISPLAY_NORTH_OFFSET);
+        transferUniformFloat(this, 'lens_distance_ratio', dataView, LENS_DISTANCE_RATIO);
+        transferUniformBoolean(this, 'sbs_enabled', dataView, SBS_ENABLED);
+        transferUniformBoolean(this, 'sbs_content', dataView, SBS_CONTENT);
+        transferUniformBoolean(this, 'sbs_mode_stretched', dataView, SBS_MODE_STRETCHED);
+        transferUniformBoolean(this, 'custom_banner_enabled', dataView, CUSTOM_BANNER_ENABLED);
+
+        // computed values with no dataViewInfo, so we set these manually
+        setSingleFloat(this, 'show_banner', imuResetState);
+        setSingleFloat(this, 'stage_aspect_ratio', stageAspectRatio);
+        setSingleFloat(this, 'display_aspect_ratio', displayAspectRatio);
+        setSingleFloat(this, 'half_fov_z_rads', halfFovZRads);
+        setSingleFloat(this, 'half_fov_y_rads', halfFovYRads);
+        setSingleFloat(this, 'screen_distance', screenDistance);
+        setSingleFloat(this, 'frametime', this._frametime);
+    }
+    setSingleFloat(this, 'enabled', enabled);
+}
+
+
+export default class ExampleExtension extends Extension {
+    enable() {
+        var XREffect = GObject.registerClass({}, class XREffect extends Shell.GLSLEffect {
+            vfunc_build_pipeline() {
+                const shaderPath = GLib.getenv('BREEZY_GNOME_SHADER_PATH');
+                const code = getShaderSource(shaderPath);
+                const main = 'PS_IMU_Transform(vec4(0, 0, 0, 0), cogl_tex_coord_in[0].xy, cogl_color_out);';
+                this.add_glsl_snippet(Shell.SnippetHook.FRAGMENT, code, main, false);
+
+                this._frametime = 10; // 100 FPS
+            }
+
+            vfunc_paint_target(node, paintContext) {
+              if (!this._initialized) {
+                this._shared_mem_file = Gio.file_new_for_path("/dev/shm/imu_data");
+              }
+
+              const data = this._shared_mem_file.load_contents(null);
+              if (data[0]) {
+                const buffer = new Uint8Array(data[1]).buffer;
+                this._dataView = new DataView(buffer);
+                var repaintNeeded = false;
+                if (!this._initialized) {
+                    this.set_uniform_float(this.get_uniform_location('uDesktopTexture'), 1, [0]);
+
+                    // iterate over shaderUniformLocations keys and set the uniform locations
+                    for (let key in shaderUniformLocations) {
+                        shaderUniformLocations[key] = this.get_uniform_location(key);
+                    }
+                    this.setIntermittentUniformVariables = setIntermittentUniformVariables.bind(this);
+                    this.setIntermittentUniformVariables();
+
+                    GLib.timeout_add(GLib.PRIORITY_DEFAULT, this._frametime, () => {
+                        repaintNeeded = true;
+                        this.queue_repaint();
+                        return GLib.SOURCE_CONTINUE;
+                    });
+
+                    GLib.timeout_add(GLib.PRIORITY_DEFAULT, 250, (() => {
+                        this.setIntermittentUniformVariables();
+                        return GLib.SOURCE_CONTINUE;
+                    }).bind(this));
+                    Meta.CursorTracker.get_for_display(global.display).set_pointer_visible(true);
+                    this._initialized = true;
+                }
+
+                setUniformMatrix(this, 'imu_quat_data', 4, this._dataView, IMU_QUAT_DATA);
+                
+                // if (repaintNeeded) {
+                  super.vfunc_paint_target(node, paintContext);
+                // }
+              }
+            }
+        });
+
+        global.stage.add_effect(new XREffect());
+    }
+
+    disable() {
+    }
+}
+
+function init() {
+    return new Extension();
+}

gnome/breezydesktop@org.xronlinux/metadata.json

@@ -0,0 +1,9 @@
+{
+    "uuid": "breezydesktop@org.xronlinux",
+    "name": "Breezy GNOME",
+    "description": "XR virtual desktop for Linux.",
+    "shell-version": [
+        "45", "46"
+    ],
+    "url": "https://github.com/wheaney/breezy-desktop"
+}