RomanNum3ral
/
wayvr
mirror of https://github.com/wayvr-org/wayvr.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

wgui: dav1d video playback works

This commit is contained in:
Aleksander 2026-06-05 12:08:25 +02:00
parent 5c929c4fee
commit 69803ec6cd
4 changed files with 233 additions and 45 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
uidev/assets/gui/video.xml
View File

@ -1,9 +1,9 @@
<layout>
<elements>
<rectangle position="absolute" width="10000" height="10000"/>
<rectangle position="absolute" width="10000" height="10000" color="#112233"/>
<div margin="16" gap="8" flex_direction="column">
<label text="aaa"/>
<Video src="test.ivf" width="320" height="240"/>
<label text="640x200"/>
<Video src="video.ivf" width="640" height="200"/>
</div>
</elements>
</layout>

BIN
uidev/assets/video.ivf Normal file
View File

Binary file not shown.

129
wgui/src/components/video.rs
View File

@ -1,14 +1,24 @@
use image::ImageBuffer;
use taffy::prelude::percent;
use crate::{
assets::AssetPath,
components::{Component, ComponentBase, ComponentTrait, RefreshData},
drawing::Color,
layout::{WidgetID, WidgetPair},
layout::{Layout, WidgetID, WidgetPair},
renderer_vk::text::custom_glyph::{CustomGlyphContent, CustomGlyphData},
video_dec::{self, Av1Decoder, IvfReader},
widget::{
ConstructEssentials,
image::WidgetImage,
rectangle::{WidgetRectangle, WidgetRectangleParams},
},
};
use std::{cell::RefCell, rc::Rc};
use std::{
cell::RefCell,
rc::{Rc, Weak},
sync::Arc,
};
#[derive(Default)]
pub struct Params<'a> {
@ -16,11 +26,18 @@ pub struct Params<'a> {
pub src: Option<AssetPath<'a>>,
}
struct State {}
struct State {
demuxer: IvfReader,
decoder: video_dec::Av1Decoder,
self_ref: Weak<ComponentVideo>,
glyph_id: usize,
}
struct Data {
#[allow(dead_code)]
id_container: WidgetID,
id_image: WidgetID,
}
#[allow(dead_code)]
@ -39,16 +56,88 @@ impl ComponentTrait for ComponentVideo {
&mut self.base
}
fn refresh(&self, _data: &mut RefreshData) {
// nothing to do
fn refresh(&self, data: &mut RefreshData) {
let mut state = self.state.borrow_mut();
match self.read_frame(&mut state, data.layout) {
Ok(playing) => {
if !playing {
if let Err(e) = state.play(data.layout) {
log::error!("{e}");
}
return;
}
}
Err(e) => {
log::error!("read_frame failed: {e}");
return;
}
}
if let Some(component) = state.self_ref.upgrade() {
data.layout.defer_component_refresh(Component(component));
}
}
}
impl ComponentVideo {}
impl ComponentVideo {
fn read_frame(&self, state: &mut State, layout: &mut Layout) -> anyhow::Result<bool> {
let Some(mut image) = layout.state.widgets.get_as::<WidgetImage>(self.data.id_image) else {
return Ok(false);
};
let rgbx_frame = match state.decoder.read_frame(&mut state.demuxer)? {
video_dec::ReadFrameResult::Ok(rgbx_frame) => rgbx_frame,
video_dec::ReadFrameResult::EndOfFile => {
log::info!("got EOF");
return Ok(false);
}
};
let Some(buffer) = ImageBuffer::from_raw(rgbx_frame.width.into(), rgbx_frame.height.into(), rgbx_frame.data) else {
anyhow::bail!("ImageBuffer failed");
};
let glyph_content = CustomGlyphContent::Image(buffer);
image.set_content(
&mut layout.alterables,
Some(CustomGlyphData {
id: state.glyph_id,
content: Arc::new(glyph_content),
}),
);
// force-update image content (FIXME: stream image data directly instead)
state.glyph_id += 1;
Ok(true)
}
}
impl State {
fn play(&mut self, layout: &mut Layout) -> anyhow::Result<()> {
self.decoder = Av1Decoder::new()?;
self.demuxer.rewind();
if let Some(component) = self.self_ref.upgrade() {
layout.defer_component_refresh(Component(component));
}
Ok(())
}
}
pub fn construct(ess: &mut ConstructEssentials, params: Params) -> anyhow::Result<(WidgetPair, Rc<ComponentVideo>)> {
let style = params.style;
let Some(src) = params.src else {
anyhow::bail!("missing src");
};
let video_data = ess.layout.state.globals.get_asset(src)?;
let demuxer = IvfReader::new(video_data)?;
let decoder = Av1Decoder::new()?;
let (root, _) = ess.layout.add_child(
ess.parent,
WidgetRectangle::create(WidgetRectangleParams {
@ -58,10 +147,30 @@ pub fn construct(ess: &mut ConstructEssentials, params: Params) -> anyhow::Resul
style,
)?;
let id_container = root.id;
let data = Rc::new(Data { id_container });
let (image, _) = ess.layout.add_child(
root.id,
WidgetImage::create(Default::default()),
taffy::Style {
size: taffy::Size {
width: percent(1.0),
height: percent(1.0),
},
..Default::default()
},
)?;
let state = Rc::new(RefCell::new(State {}));
let id_container = root.id;
let data = Rc::new(Data {
id_container,
id_image: image.id,
});
let state = Rc::new(RefCell::new(State {
demuxer,
decoder,
self_ref: Default::default(),
glyph_id: 0,
}));
let base = ComponentBase {
id: root.id,
@ -70,6 +179,8 @@ pub fn construct(ess: &mut ConstructEssentials, params: Params) -> anyhow::Resul
let video = Rc::new(ComponentVideo { base, data, state });
video.state.borrow_mut().self_ref = Rc::downgrade(&video);
ess.layout.defer_component_refresh(Component(video.clone()));
Ok((root, video))
}

143
wgui/src/video_dec.rs
View File

@ -2,13 +2,14 @@ use std::ptr::null_mut;
use bytes::{Buf, Bytes};
use dav1d_sys::{
Dav1dContext, Dav1dData, Dav1dPicture, Dav1dSettings, dav1d_close, dav1d_data_unref, dav1d_default_settings,
DAV1D_ERR_AGAIN, Dav1dContext, Dav1dData, Dav1dPicture, Dav1dSettings, dav1d_close, dav1d_default_settings,
dav1d_get_picture, dav1d_open, dav1d_picture_unref, dav1d_send_data,
};
pub struct IvfReader {
_file_data: Vec<u8>, // Used by `reader`, do not remove!
file_data: Vec<u8>, // Used by `reader`, do not remove!
reader: Bytes,
header_size: usize,
pub cur_frame: u32,
pub num_frames: u32,
pub framerate: f32,
@ -39,6 +40,11 @@ pub struct IvfReader {
const IVF_MAGIC: [u8; 4] = [0x44, 0x4B, 0x49, 0x46];
pub enum IvfReadFrameResult<'a> {
Ok((&'a [u8], u64 /* pts */)),
EndOfFile,
}
impl IvfReader {
pub fn new(file_data: Vec<u8>) -> anyhow::Result<IvfReader> {
// safety: both reader and file_data are located in the same struct
@ -80,8 +86,11 @@ impl IvfReader {
log::info!("IvfReader: width {header_width}, height {header_height}, framerate {framerate}");
let header_size = file_data.len() - reader.remaining();
Ok(IvfReader {
_file_data: file_data,
header_size,
file_data,
reader,
cur_frame: 0,
width: header_width,
@ -91,14 +100,24 @@ impl IvfReader {
})
}
pub fn rewind(&mut self) {
self.reader = Bytes::from_static(unsafe { std::mem::transmute::<&[u8], &'static [u8]>(&self.file_data) });
// do not read header again
self.reader.advance(self.header_size);
}
// Read demuxed video packet to a chunk
pub fn read_frame(&mut self) -> anyhow::Result<(&[u8], u64 /* pts */)> {
pub fn read_frame(&mut self) -> anyhow::Result<IvfReadFrameResult<'_>> {
if self.reader.remaining() == 0 {
return Ok(IvfReadFrameResult::EndOfFile);
}
let frame_size = self.reader.try_get_u32_le()? as usize;
let frame_pts = self.reader.try_get_u64_le()?;
if frame_size > 8 * 1024 * 1024 {
// something went really wrong
anyhow::bail!("Invalid frame size");
anyhow::bail!("Invalid frame size {frame_size}");
}
// SAFETY: reader.chunk() slice lifetime is the same as Self, no risk here.
@ -113,14 +132,15 @@ impl IvfReader {
self.cur_frame += 1;
Ok((chunk_a, frame_pts))
Ok(IvfReadFrameResult::Ok((chunk_a, frame_pts)))
}
}
pub struct RgbFrame {
width: u16,
height: u16,
data: Vec<u8>,
// RGB with 255 A
pub struct RgbxFrame {
pub width: u16,
pub height: u16,
pub data: Vec<u8>,
}
fn yuv_to_rgb(y: u8, u: u8, v: u8) -> (u8, u8, u8) {
@ -138,7 +158,7 @@ fn yuv_to_rgb(y: u8, u: u8, v: u8) -> (u8, u8, u8) {
)
}
impl RgbFrame {
impl RgbxFrame {
// Simple, temporary, best-effort yuv420->rgb converter. Not performant at all, but at least it works.
// Temporary till we get native YUV support in shaders (as 3 vulkan textures).
// SAFETY: this function expects YUV420 data only.
@ -151,15 +171,15 @@ impl RgbFrame {
data_v: *const u8,
stride_luma: usize,
stride_chroma: usize,
) -> RgbFrame {
) -> RgbxFrame {
unsafe {
let channels = 3; // rgb
let channels = 4; // rgbx
let rgb_size = width as usize * height as usize * channels;
let mut rgb = Vec::<u8>::with_capacity(rgb_size);
rgb.set_len(rgb_size);
let rgbx_size = width as usize * height as usize * channels;
let mut rgbx = Vec::<u8>::with_capacity(rgbx_size);
rgbx.set_len(rgbx_size);
let ptr_rgb = rgb.as_mut_ptr();
let ptr_rgbx = rgbx.as_mut_ptr();
for y in 0..height as usize {
for x in 0..width as usize {
@ -169,16 +189,17 @@ impl RgbFrame {
let (r, g, b) = yuv_to_rgb(val_y, val_u, val_v);
let pos = y * (width as usize * channels) + x * channels;
*ptr_rgb.add(pos) = r;
*ptr_rgb.add(pos + 1) = g;
*ptr_rgb.add(pos + 2) = b;
*ptr_rgbx.add(pos) = r;
*ptr_rgbx.add(pos + 1) = g;
*ptr_rgbx.add(pos + 2) = b;
*ptr_rgbx.add(pos + 3) = 255;
}
}
RgbFrame {
RgbxFrame {
width,
height,
data: rgb,
data: rgbx,
}
}
}
@ -196,6 +217,23 @@ impl Drop for Av1Decoder {
}
}
enum GetPictureResult {
Ok,
Again,
Failed(i32),
}
enum ReadFrameIterResult {
Ok(RgbxFrame),
Again,
EndOfFile,
}
pub enum ReadFrameResult {
Ok(RgbxFrame),
EndOfFile,
}
impl Av1Decoder {
pub fn new() -> anyhow::Result<Self> {
unsafe {
@ -211,12 +249,25 @@ impl Av1Decoder {
}
}
pub fn read_frame(&mut self, reader: &mut IvfReader) -> anyhow::Result<RgbFrame> {
let (packet, pts) = reader.read_frame()?;
fn get_picture(&mut self, picture: &mut Dav1dPicture) -> GetPictureResult {
let ret = unsafe { dav1d_get_picture(self.ctx, picture) };
if ret == DAV1D_ERR_AGAIN {
return GetPictureResult::Again;
}
let mut ret;
if ret < 0 {
return GetPictureResult::Failed(ret);
}
GetPictureResult::Ok
}
fn send_data(&mut self, reader: &mut IvfReader) -> anyhow::Result<bool> {
unsafe {
let (packet, pts) = match reader.read_frame()? {
IvfReadFrameResult::Ok((packet, pts)) => (packet, pts),
IvfReadFrameResult::EndOfFile => return Ok(false),
};
let mut data: Dav1dData = std::mem::zeroed();
data.data = packet.as_ptr();
data.sz = packet.len();
@ -225,17 +276,31 @@ impl Av1Decoder {
data.m.offset = -1;
data.m.duration = 0;
ret = dav1d_send_data(self.ctx, &raw mut data);
dav1d_data_unref(&raw mut data);
let ret = dav1d_send_data(self.ctx, &raw mut data);
if ret < 0 {
anyhow::bail!("dav1d_send_data failed: {ret}");
}
Ok(true)
}
}
fn read_frame_iter(&mut self, reader: &mut IvfReader) -> anyhow::Result<ReadFrameIterResult> {
unsafe {
let mut picture: Dav1dPicture = std::mem::zeroed();
ret = dav1d_get_picture(self.ctx, &raw mut picture);
if ret < 0 {
dav1d_picture_unref(&raw mut picture);
anyhow::bail!("dav1d_get_picture failure: {ret}")
match self.get_picture(&mut picture) {
GetPictureResult::Ok => {}
GetPictureResult::Again => {
dav1d_picture_unref(&raw mut picture);
if !self.send_data(reader)? {
return Ok(ReadFrameIterResult::EndOfFile);
}
return Ok(ReadFrameIterResult::Again);
}
GetPictureResult::Failed(code) => {
dav1d_picture_unref(&raw mut picture);
anyhow::bail!("dav1d_get_picture failed: {code}");
}
}
let pic_y = picture.data[0]; // luma Y
@ -244,7 +309,7 @@ impl Av1Decoder {
let stride_luma = picture.stride[0];
let stride_chroma = picture.stride[1];
let rgb = RgbFrame::from_yuv(
let rgbx = RgbxFrame::from_yuv(
reader.width,
reader.height,
pic_y as *const u8,
@ -256,7 +321,19 @@ impl Av1Decoder {
dav1d_picture_unref(&raw mut picture);
Ok(rgb)
Ok(ReadFrameIterResult::Ok(rgbx))
}
}
pub fn read_frame(&mut self, reader: &mut IvfReader) -> anyhow::Result<ReadFrameResult> {
loop {
match self.read_frame_iter(reader)? {
ReadFrameIterResult::Ok(rgbx_frame) => {
return Ok(ReadFrameResult::Ok(rgbx_frame));
}
ReadFrameIterResult::Again => { /* loop again */ }
ReadFrameIterResult::EndOfFile => return Ok(ReadFrameResult::EndOfFile),
}
}
}
}