From 4f80efdf751beeeccb917455d6ff9bdda78ecafc Mon Sep 17 00:00:00 2001 From: wheaney <42350981+wheaney@users.noreply.github.com> Date: Wed, 20 Mar 2024 22:01:18 -0700 Subject: [PATCH] Working barebones shared and data transfer --- .../IMUAdjust.frag | 159 ++++++++++++ .../breezydesktop@org.xronlinux/extension.js | 234 ++++++++++++++++-- 2 files changed, 379 insertions(+), 14 deletions(-) create mode 100644 gnome/breezydesktop@org.xronlinux/IMUAdjust.frag diff --git a/gnome/breezydesktop@org.xronlinux/IMUAdjust.frag b/gnome/breezydesktop@org.xronlinux/IMUAdjust.frag new file mode 100644 index 0000000..0dcbe75 --- /dev/null +++ b/gnome/breezydesktop@org.xronlinux/IMUAdjust.frag @@ -0,0 +1,159 @@ +#version 330 core + +uniform sampler2D uDesktopTexture; +uniform mat4 g_imu_quat_data; + +vec4 quatMul(vec4 q1, vec4 q2) { + vec3 u = vec3(q1.x, q1.y, q1.z); + float s = q1.w; + vec3 v = vec3(q2.x, q2.y, q2.z); + float t = q2.w; + return vec4(s*v + t*u + cross(u, v), s*t - dot(u, v)); +} + +vec4 quatConj(vec4 q) { + return vec4(-q.x, -q.y, -q.z, q.w); +} + +vec3 applyQuaternionToVector(vec4 q, vec3 v) { + vec4 p = quatMul(quatMul(q, vec4(v, 0)), quatConj(q)); + return p.xyz; +} + +const int day_in_seconds = 24 * 60 * 60; + +vec3 applyLookAhead( + in vec3 position, + in vec3 velocity, + in vec3 accel, + in float t, + in float t_squared) { + vec3 _91 = velocity * t; + vec3 _92 = position + _91; + vec3 _94 = vec3(5.00000000e-01, 5.00000000e-01, 5.00000000e-01) * accel; + vec3 _96 = _94 * t_squared; + vec3 _97 = _92 + _96; + return _97; +} + +vec4 quatMul( + in vec4 q1, + in vec4 q2) { + vec3 _105 = vec3(q1.x, q1.y, q1.z); + vec3 u = _105; + float s = q1.w; + vec3 _112 = vec3(q2.x, q2.y, q2.z); + vec3 v = _112; + float t_115 = q2.w; + vec3 _117 = s * v; + vec3 _119 = t_115 * u; + vec3 _120 = _117 + _119; + vec3 _121 = cross(u, v); + vec3 _122 = _120 + _121; + float _123 = s * t_115; + float _124 = dot(u, v); + float _125 = _123 - _124; + vec4 _129 = vec4(_122.x, _122.y, _122.z, _125); + return _129; +} + +vec4 quatConj( + in vec4 q) { + float _134 = -(q.x); + float _136 = -(q.y); + float _138 = -(q.z); + vec4 _140 = vec4(_134, _136, _138, q.w); + return _140; +} + +vec3 applyQuaternionToVector( + in vec4 q, + in vec3 v) { + vec4 _149 = vec4(v.x, v.y, v.z, 0.00000000e+00); + vec4 _150; + vec4 _151; + _150 = q; + _151 = _149; + vec4 _152 = quatMul(_150, _151); + vec4 _153; + _153 = q; + vec4 _154 = quatConj(_153); + vec4 _155; + vec4 _156; + _155 = _152; + _156 = _154; + vec4 _157 = quatMul(_155, _156); + vec4 p = _157; + return p.xyz; +} + +vec3 rateOfChange( + in vec3 v1, + in vec3 v2, + in float delta_time) { + vec3 _165 = v1 - v2; + vec3 _167 = _165 / delta_time; + return _167; +} + +bool isKeepaliveRecent( + in vec4 currentDate, + in vec4 keepAliveDate) { + float _174 = currentDate.w + float(day_in_seconds); + float _176 = _174 - keepAliveDate.w; + float _178 = mod(_176, float(day_in_seconds)); + float _179 = abs(_178); + bool _181 = _179 <= 5.00000000e+00; + return _181; +} + +void PS_IMU_Transform(vec4 pos, vec2 texcoord, out vec4 color) { + float texcoord_x_min = 0.0; + float texcoord_x_max = 1.0; + vec2 screen_size = vec2(1920, 1080); + float lens_y_offset = 0.0; + float lens_z_offset = 0.0; + + float screen_aspect_ratio = screen_size.x / screen_size.y; + float native_aspect_ratio = screen_aspect_ratio; + + float diag_to_vert_ratio = sqrt(screen_aspect_ratio * screen_aspect_ratio + 1.0); + float half_fov_z_rads = radians(46.0 / diag_to_vert_ratio)/2.0; + float half_fov_y_rads = half_fov_z_rads * screen_aspect_ratio; + + float screen_distance = 1.0 - 0.05; + + float lens_fov_z_offset_rads = atan(lens_z_offset/screen_distance); + float fov_z_pos = tan(half_fov_z_rads - lens_fov_z_offset_rads) * screen_distance; + float fov_z_neg = -tan(half_fov_z_rads + lens_fov_z_offset_rads) * screen_distance; + float fov_z_width = fov_z_pos - fov_z_neg; + + float lens_fov_y_offset_rads = atan(lens_y_offset/screen_distance); + float fov_y_pos = tan(half_fov_y_rads - lens_fov_y_offset_rads) * screen_distance; + float fov_y_neg = -tan(half_fov_y_rads + lens_fov_y_offset_rads) * screen_distance; + float fov_y_width = fov_y_pos - fov_y_neg; + float vec_x = screen_distance; + float vec_y = -texcoord.x * fov_y_width + fov_y_pos; + float vec_z = -texcoord.y * fov_z_width + fov_z_pos; + vec3 texcoord_vector = vec3(vec_x, vec_y, vec_z); + vec3 lens_vector = vec3(0.05, lens_y_offset, lens_z_offset); + + vec3 res = applyQuaternionToVector(g_imu_quat_data[0], texcoord_vector); + + bool looking_behind = res.x < 0.0; + + // deconstruct the rotated and scaled vector back to a texcoord (just inverse operations of the first conversion + // above) + texcoord.x = (fov_y_pos - res.y) / fov_y_width; + texcoord.y = (fov_z_pos - res.z) / fov_z_width; + + // apply the screen offsets now + float texcoord_width = texcoord_x_max - texcoord_x_min; + texcoord.x = texcoord.x * texcoord_width + texcoord_x_min; + + if (looking_behind || texcoord.x < texcoord_x_min || texcoord.y < 0.0 || texcoord.x > texcoord_x_max || texcoord.y > 1.0 || texcoord.x <= 0.005 && texcoord.y <= 0.005) { + color = vec4(0, 0, 0, 1); + } else { + color = texture2D(uDesktopTexture, texcoord); + } +} \ No newline at end of file diff --git a/gnome/breezydesktop@org.xronlinux/extension.js b/gnome/breezydesktop@org.xronlinux/extension.js index 00abeb7..8e9afe2 100644 --- a/gnome/breezydesktop@org.xronlinux/extension.js +++ b/gnome/breezydesktop@org.xronlinux/extension.js @@ -2,42 +2,248 @@ const Lang = imports.lang; const St = imports.gi.St; const Clutter = imports.gi.Clutter; const Gio = imports.gi.Gio; +const GLib = imports.gi.GLib; const GObject = imports.gi.GObject; const Cogl = imports.gi.Cogl; const Shell = imports.gi.Shell; +const Meta = imports.gi.Meta; const ExtensionUtils = imports.misc.extensionUtils; const Me = ExtensionUtils.getCurrentExtension(); const Main = imports.ui.main; const PanelMenu = imports.ui.panelMenu; +const UINT8_SIZE = 1; +const BOOL_SIZE = UINT8_SIZE; +const UINT_SIZE = 4; +const FLOAT_SIZE = 4; + +// end offset, exclusive +const DATA_VIEW_INFO_OFFSET_INDEX = 0; +const DATA_VIEW_INFO_SIZE_INDEX = 1; +const DATA_VIEW_INFO_COUNT_INDEX = 1; +function dataViewEnd(dataViewInfo) { + return dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX] + dataViewInfo[DATA_VIEW_INFO_SIZE_INDEX] * dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX]; +} + +// the driver should be using the same data layout version +const DATA_LAYOUT_VERSION = 1; + +// DataView info: [offset, size, count] +const VERSION = [0, UINT8_SIZE, 1]; +const ENABLED = [dataViewEnd(VERSION), BOOL_SIZE, 1]; +const EPOCH_SEC_OFFSET = [dataViewEnd(VERSION), UINT_SIZE, 1]; +const LOOK_AHEAD_CFG = [dataViewEnd(EPOCH_SEC_OFFSET), FLOAT_SIZE, 4]; +const DISPLAY_RES = [dataViewEnd(LOOK_AHEAD_CFG), UINT_SIZE, 2]; +const DISPLAY_FOV = [dataViewEnd(DISPLAY_RES), FLOAT_SIZE, 1]; +const DISPLAY_ZOOM = [dataViewEnd(DISPLAY_FOV), FLOAT_SIZE, 1]; +const DISPLAY_NORTH_OFFSET = [dataViewEnd(DISPLAY_ZOOM), FLOAT_SIZE, 1]; +const LENS_DISTANCE_RATIO = [dataViewEnd(DISPLAY_NORTH_OFFSET), FLOAT_SIZE, 1]; +const SBS_ENABLED = [dataViewEnd(LENS_DISTANCE_RATIO), BOOL_SIZE, 1]; +const SBS_CONTENT = [dataViewEnd(SBS_ENABLED), BOOL_SIZE, 1]; +const SBS_MODE_STRETCHED = [dataViewEnd(SBS_CONTENT), BOOL_SIZE, 1]; +const CUSTOM_BANNER_ENABLED = [dataViewEnd(SBS_MODE_STRETCHED), BOOL_SIZE, 1]; +const IMU_QUAT_DATA = [dataViewEnd(CUSTOM_BANNER_ENABLED), FLOAT_SIZE, 16]; + +// cached after first retrieval +const shaderUniformLocations = { + 'enabled': null, + 'imu_quat_data': null, + 'look_ahead_cfg': null, + 'stage_aspect_ratio': null, + 'display_aspect_ratio': null, + 'display_zoom': null, + 'display_north_offset': null, + 'lens_distance_ratio': null, + 'sbs_enabled': null, + 'sbs_content': null, + 'sbs_mode_stretched': null, + 'custom_banner_enabled': null, + 'half_fov_z_rads': null, + 'half_fov_y_rads': null, + 'screen_distance': null +}; + +function dataViewUint8(dataView, dataViewInfo) { + return dataView.getUint8(dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX]); +} + +function dataViewUint(dataView, dataViewInfo) { + return dataView.getUint32(dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX], true); +} + +function dataViewUintArray(dataView, dataViewInfo) { + const uintArray = [] + const offset = dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX]; + for (let i = 0; i < dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX]; i++) { + uintArray.push(dataView.getUint32(offset, true)); + offset += UINT_SIZE; + } + return uintArray; +} + +function dataViewFloat(dataView, dataViewInfo) { + return dataView.getFloat32(dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX], true); +} + +function dataViewFloatArray(dataView, dataViewInfo) { + const floatArray = [] + const offset = dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX]; + for (let i = 0; i < dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX]; i++) { + floatArray.push(dataView.getFloat32(offset, true)); + offset += FLOAT_SIZE; + } + return floatArray; +} + + +function getShaderSource(path) { + const file = Gio.file_new_for_path(path); + const data = file.load_contents(null); + + // version string helps with linting, but GNOME extension doesn't like it, so remove it if it's there + return data[1].toString().replace(/^#version .*$/gm, '') + '\n'; +} + +function transferUniformBoolean(effect, locationName, dataView, dataViewInfo) { + // GLSL bool is a float under the hood, evaluates false if 0 or 0.0, true otherwise + effect.set_uniform_float(locationName, 1, [dataViewUint8(dataView, dataViewInfo)]); +} + +function setUniformFloat(effect, locationName, dataViewInfo, value) { + effect.set_uniform_float(shaderUniformLocations[locationName], dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX], value); +} + +function transferUniformFloat(effect, locationName, dataView, dataViewInfo) { + setUniformFloat(effect, locationName, dataViewInfo, dataViewFloatArray(dataView, dataViewInfo)); + +function setUniformMatrix(effect, locationName, components, dataView, dataViewInfo) { + const numValues = dataViewInfo[DATA_VIEW_INFO_COUNT_INDEX]; + if (numValues / componenents !== components) { + throw new Error('Invalid matrix size'); + } + + const floatArray = [].fill(0, 0, numValues); + const offset = dataViewInfo[DATA_VIEW_INFO_OFFSET_INDEX]; + for (let i = 0; i < numValues; i++) { + // GLSL uses column-major order, so we need to transpose the matrix + const row = i % components; + const column = Math.floor(i / components); + + floatArray[row * components + column] = dataView.getFloat32(offset, true); + offset += FLOAT_SIZE; + } + effect.set_uniform_matrix(shaderUniformLocations[locationName], true, components, floatArray); +} + +function getEpochSec() { + return Math.floor(Date.now() / 1000); +} + +function degreeToRadian(degree) { + return degree * Math.PI / 180; +} + +// config doesn't change frequently, so we'll set these all at once, periodically +function setConfigUniformVarables(effect, dataView) { + const version = dataViewUint8(dataView, VERSION); + const date = dataViewUint(dataView, EPOCH_SEC_OFFSET); + const validKeepalive = Math.abs(getEpochSec() - date) < 5; + const imuData = dataViewFloatArray(dataView, IMU_QUAT_DATA); + const imuResetState = imuData[0] === 0.0 && imuData[1] === 0.0 && imuData[2] === 0.0 && imuData[3] === 1.0; + const enabled = dataViewUint8(dataView, ENABLED) !== 0 && version === DATA_LAYOUT_VERSION && validKeepalive && !imuResetState; + if (enabled) { + const displayRes = dataViewUintArray(dataView, DISPLAY_RES); + const displayFov = dataViewFloat(dataView, DISPLAY_FOV); + const lensDistanceRatio = dataViewFloat(dataView, LENS_DISTANCE_RATIO); + + // compute these values once, they only change when the XR device changes + const displayAspectRatio = displayRes[0] / displayRes[1]; + const stageAspectRatio = global.stage.get_width() / global.stage.get_height(); + const diagToVertRatio = Math.sqrt(Math.pow(stageAspectRatio, 2) + 1); + const halfFovZRads = degreeToRadian(displayFov / diagToVertRatio) / 2; + const halfFovYRads = halfFovZRads * stageAspectRatio; + const screenDistance = 1.0 - lensDistanceRatio; + + // all these values are passed directly to the shader, unmodified + transferUniformFloat(effect, 'imu_quat_data', dataView, IMU_QUAT_DATA); + transferUniformFloat(effect, 'look_ahead_cfg', dataView, LOOK_AHEAD_CFG); + transferUniformFloat(effect, 'display_zoom', dataView, DISPLAY_ZOOM); + transferUniformFloat(effect, 'display_north_offset', dataView, DISPLAY_NORTH_OFFSET); + transferUniformFloat(effect, 'lens_distance_ratio', dataView, LENS_DISTANCE_RATIO); + transferUniformBoolean(effect, 'sbs_enabled', dataView, SBS_ENABLED); + transferUniformBoolean(effect, 'sbs_content', dataView, SBS_CONTENT); + transferUniformBoolean(effect, 'sbs_mode_stretched', dataView, SBS_MODE_STRETCHED); + transferUniformBoolean(effect, 'custom_banner_enabled', dataView, CUSTOM_BANNER_ENABLED); + + // no dataViewInfo, so we set these manually + effect.set_uniform_float(shaderUniformLocations['stage_aspect_ratio'], 1, [stageAspectRatio]); + effect.set_uniform_float(shaderUniformLocations['display_aspect_ratio'], 1, [displayAspectRatio]); + effect.set_uniform_float(shaderUniformLocations['half_fov_z_rads'], 1, [halfFovZRads]); + effect.set_uniform_float(shaderUniformLocations['half_fov_y_rads'], 1, [halfFovYRads]); + effect.set_uniform_float(shaderUniformLocations['screen_distance'], 1, [screenDistance]); + } + setUniformBoolean(effect, shaderUniformLocations['enabled'], ENABLED, enabled); +} class Extension { enable() { var XREffect = GObject.registerClass({}, class XREffect extends Shell.GLSLEffect { vfunc_build_pipeline() { - // TODO - replace this with the sombrero shader - const declares = ` - uniform sampler2D uDesktopTexture; - `; - const code = ` - cogl_color_out = texture2D(uDesktopTexture, cogl_tex_coord_in[0].xy); - `; - this.add_glsl_snippet(Shell.SnippetHook.FRAGMENT, declares, code, false); + // Shell.GLSLEffect requires the declarations and the main source code as separate + // strings. As it's more convenient to store the in one GLSL file, we use a regex + // here to split the source code in two parts. + const code = getShaderSource('/home/wayne/IdeaProjects/nreal/breezy-desktop/gnome/breezydesktop@org.xronlinux/IMUAdjust.frag'); + + const main = 'PS_IMU_Transform(vec4(0, 0, 0, 0), cogl_tex_coord_in[0].xy, cogl_color_out);'; + this.add_glsl_snippet(Shell.SnippetHook.FRAGMENT, code, main, false); } // TODO - read IMU data and update uniform variables vfunc_paint_target(node, paintContext) { - if (!this._initialized) { - this.set_uniform_float(this.get_uniform_location('uDesktopTexture'), 1, [0]); - this._initialized = true; - } - super.vfunc_paint_target(node, paintContext); + const data = this._shared_mem_file.load_contents(null); + if (data[0]) { + const buffer = new Uint8Array(data[1]).buffer; + var dataView = new DataView(buffer); + var repaintNeeded = false; + if (!this._initialized) { + this.set_uniform_float(this.get_uniform_location('uDesktopTexture'), 1, [0]); + this._shared_mem_file = Gio.file_new_for_path("/dev/shm/imu_data"); + + // iterate over shaderUniformLocations keys and set the uniform locations + for (let key in shaderUniformLocations) { + shaderUniformLocations[key] = this.get_uniform_location(key); + } + + setConfigUniformVarables(this, dataView); + + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 1000/50, () => { + repaintNeeded = true; + this.queue_repaint(); + return GLib.SOURCE_CONTINUE; + }); + + GLib.timeout_add(GLib.PRIORITY_DEFAULT, 250, () => { + setConfigUniformVarables(this, dataView); + return GLib.SOURCE_CONTINUE; + }); + Meta.CursorTracker.get_for_display(global.display).set_pointer_visible(true); + console.log(`is_rendering_hardware_accelerated: ${Meta.CursorTracker.get_for_display(global.display).backend.is_rendering_hardware_accelerated()}`); + + this._initialized = true; + } + + transferUniformFloat(this, 'imu_quat_data', dataView, IMU_QUAT_DATA); + + // if (repaintNeeded) { + super.vfunc_paint_target(node, paintContext); + // } + } } }); - Main.uiGroup.add_effect(new XREffect()); + global.stage.add_effect(new XREffect()); } disable() {