diff --git a/uidev/assets/gui/video.xml b/uidev/assets/gui/video.xml
index 622a8b54..0a82b94c 100644
--- a/uidev/assets/gui/video.xml
+++ b/uidev/assets/gui/video.xml
@@ -1,9 +1,9 @@
-
+
-
-
+
+
diff --git a/uidev/assets/video.ivf b/uidev/assets/video.ivf
new file mode 100644
index 00000000..5f430163
Binary files /dev/null and b/uidev/assets/video.ivf differ
diff --git a/wgui/src/components/video.rs b/wgui/src/components/video.rs
index 5d994c41..c080b206 100644
--- a/wgui/src/components/video.rs
+++ b/wgui/src/components/video.rs
@@ -1,14 +1,24 @@
+use image::ImageBuffer;
+use taffy::prelude::percent;
+
use crate::{
assets::AssetPath,
components::{Component, ComponentBase, ComponentTrait, RefreshData},
drawing::Color,
- layout::{WidgetID, WidgetPair},
+ layout::{Layout, WidgetID, WidgetPair},
+ renderer_vk::text::custom_glyph::{CustomGlyphContent, CustomGlyphData},
+ video_dec::{self, Av1Decoder, IvfReader},
widget::{
ConstructEssentials,
+ image::WidgetImage,
rectangle::{WidgetRectangle, WidgetRectangleParams},
},
};
-use std::{cell::RefCell, rc::Rc};
+use std::{
+ cell::RefCell,
+ rc::{Rc, Weak},
+ sync::Arc,
+};
#[derive(Default)]
pub struct Params<'a> {
@@ -16,11 +26,18 @@ pub struct Params<'a> {
pub src: Option>,
}
-struct State {}
+struct State {
+ demuxer: IvfReader,
+ decoder: video_dec::Av1Decoder,
+ self_ref: Weak,
+ glyph_id: usize,
+}
struct Data {
#[allow(dead_code)]
id_container: WidgetID,
+
+ id_image: WidgetID,
}
#[allow(dead_code)]
@@ -39,16 +56,88 @@ impl ComponentTrait for ComponentVideo {
&mut self.base
}
- fn refresh(&self, _data: &mut RefreshData) {
- // nothing to do
+ fn refresh(&self, data: &mut RefreshData) {
+ let mut state = self.state.borrow_mut();
+
+ match self.read_frame(&mut state, data.layout) {
+ Ok(playing) => {
+ if !playing {
+ if let Err(e) = state.play(data.layout) {
+ log::error!("{e}");
+ }
+ return;
+ }
+ }
+ Err(e) => {
+ log::error!("read_frame failed: {e}");
+ return;
+ }
+ }
+
+ if let Some(component) = state.self_ref.upgrade() {
+ data.layout.defer_component_refresh(Component(component));
+ }
}
}
-impl ComponentVideo {}
+impl ComponentVideo {
+ fn read_frame(&self, state: &mut State, layout: &mut Layout) -> anyhow::Result {
+ let Some(mut image) = layout.state.widgets.get_as::(self.data.id_image) else {
+ return Ok(false);
+ };
+
+ let rgbx_frame = match state.decoder.read_frame(&mut state.demuxer)? {
+ video_dec::ReadFrameResult::Ok(rgbx_frame) => rgbx_frame,
+ video_dec::ReadFrameResult::EndOfFile => {
+ log::info!("got EOF");
+ return Ok(false);
+ }
+ };
+
+ let Some(buffer) = ImageBuffer::from_raw(rgbx_frame.width.into(), rgbx_frame.height.into(), rgbx_frame.data) else {
+ anyhow::bail!("ImageBuffer failed");
+ };
+
+ let glyph_content = CustomGlyphContent::Image(buffer);
+
+ image.set_content(
+ &mut layout.alterables,
+ Some(CustomGlyphData {
+ id: state.glyph_id,
+ content: Arc::new(glyph_content),
+ }),
+ );
+
+ // force-update image content (FIXME: stream image data directly instead)
+ state.glyph_id += 1;
+
+ Ok(true)
+ }
+}
+
+impl State {
+ fn play(&mut self, layout: &mut Layout) -> anyhow::Result<()> {
+ self.decoder = Av1Decoder::new()?;
+ self.demuxer.rewind();
+ if let Some(component) = self.self_ref.upgrade() {
+ layout.defer_component_refresh(Component(component));
+ }
+ Ok(())
+ }
+}
pub fn construct(ess: &mut ConstructEssentials, params: Params) -> anyhow::Result<(WidgetPair, Rc)> {
let style = params.style;
+ let Some(src) = params.src else {
+ anyhow::bail!("missing src");
+ };
+
+ let video_data = ess.layout.state.globals.get_asset(src)?;
+
+ let demuxer = IvfReader::new(video_data)?;
+ let decoder = Av1Decoder::new()?;
+
let (root, _) = ess.layout.add_child(
ess.parent,
WidgetRectangle::create(WidgetRectangleParams {
@@ -58,10 +147,30 @@ pub fn construct(ess: &mut ConstructEssentials, params: Params) -> anyhow::Resul
style,
)?;
- let id_container = root.id;
- let data = Rc::new(Data { id_container });
+ let (image, _) = ess.layout.add_child(
+ root.id,
+ WidgetImage::create(Default::default()),
+ taffy::Style {
+ size: taffy::Size {
+ width: percent(1.0),
+ height: percent(1.0),
+ },
+ ..Default::default()
+ },
+ )?;
- let state = Rc::new(RefCell::new(State {}));
+ let id_container = root.id;
+ let data = Rc::new(Data {
+ id_container,
+ id_image: image.id,
+ });
+
+ let state = Rc::new(RefCell::new(State {
+ demuxer,
+ decoder,
+ self_ref: Default::default(),
+ glyph_id: 0,
+ }));
let base = ComponentBase {
id: root.id,
@@ -70,6 +179,8 @@ pub fn construct(ess: &mut ConstructEssentials, params: Params) -> anyhow::Resul
let video = Rc::new(ComponentVideo { base, data, state });
+ video.state.borrow_mut().self_ref = Rc::downgrade(&video);
+
ess.layout.defer_component_refresh(Component(video.clone()));
Ok((root, video))
}
diff --git a/wgui/src/video_dec.rs b/wgui/src/video_dec.rs
index f55d1253..b12c241e 100644
--- a/wgui/src/video_dec.rs
+++ b/wgui/src/video_dec.rs
@@ -2,13 +2,14 @@ use std::ptr::null_mut;
use bytes::{Buf, Bytes};
use dav1d_sys::{
- Dav1dContext, Dav1dData, Dav1dPicture, Dav1dSettings, dav1d_close, dav1d_data_unref, dav1d_default_settings,
+ DAV1D_ERR_AGAIN, Dav1dContext, Dav1dData, Dav1dPicture, Dav1dSettings, dav1d_close, dav1d_default_settings,
dav1d_get_picture, dav1d_open, dav1d_picture_unref, dav1d_send_data,
};
pub struct IvfReader {
- _file_data: Vec, // Used by `reader`, do not remove!
+ file_data: Vec, // Used by `reader`, do not remove!
reader: Bytes,
+ header_size: usize,
pub cur_frame: u32,
pub num_frames: u32,
pub framerate: f32,
@@ -39,6 +40,11 @@ pub struct IvfReader {
const IVF_MAGIC: [u8; 4] = [0x44, 0x4B, 0x49, 0x46];
+pub enum IvfReadFrameResult<'a> {
+ Ok((&'a [u8], u64 /* pts */)),
+ EndOfFile,
+}
+
impl IvfReader {
pub fn new(file_data: Vec) -> anyhow::Result {
// safety: both reader and file_data are located in the same struct
@@ -80,8 +86,11 @@ impl IvfReader {
log::info!("IvfReader: width {header_width}, height {header_height}, framerate {framerate}");
+ let header_size = file_data.len() - reader.remaining();
+
Ok(IvfReader {
- _file_data: file_data,
+ header_size,
+ file_data,
reader,
cur_frame: 0,
width: header_width,
@@ -91,14 +100,24 @@ impl IvfReader {
})
}
+ pub fn rewind(&mut self) {
+ self.reader = Bytes::from_static(unsafe { std::mem::transmute::<&[u8], &'static [u8]>(&self.file_data) });
+ // do not read header again
+ self.reader.advance(self.header_size);
+ }
+
// Read demuxed video packet to a chunk
- pub fn read_frame(&mut self) -> anyhow::Result<(&[u8], u64 /* pts */)> {
+ pub fn read_frame(&mut self) -> anyhow::Result> {
+ if self.reader.remaining() == 0 {
+ return Ok(IvfReadFrameResult::EndOfFile);
+ }
+
let frame_size = self.reader.try_get_u32_le()? as usize;
let frame_pts = self.reader.try_get_u64_le()?;
if frame_size > 8 * 1024 * 1024 {
// something went really wrong
- anyhow::bail!("Invalid frame size");
+ anyhow::bail!("Invalid frame size {frame_size}");
}
// SAFETY: reader.chunk() slice lifetime is the same as Self, no risk here.
@@ -113,14 +132,15 @@ impl IvfReader {
self.cur_frame += 1;
- Ok((chunk_a, frame_pts))
+ Ok(IvfReadFrameResult::Ok((chunk_a, frame_pts)))
}
}
-pub struct RgbFrame {
- width: u16,
- height: u16,
- data: Vec,
+// RGB with 255 A
+pub struct RgbxFrame {
+ pub width: u16,
+ pub height: u16,
+ pub data: Vec,
}
fn yuv_to_rgb(y: u8, u: u8, v: u8) -> (u8, u8, u8) {
@@ -138,7 +158,7 @@ fn yuv_to_rgb(y: u8, u: u8, v: u8) -> (u8, u8, u8) {
)
}
-impl RgbFrame {
+impl RgbxFrame {
// Simple, temporary, best-effort yuv420->rgb converter. Not performant at all, but at least it works.
// Temporary till we get native YUV support in shaders (as 3 vulkan textures).
// SAFETY: this function expects YUV420 data only.
@@ -151,15 +171,15 @@ impl RgbFrame {
data_v: *const u8,
stride_luma: usize,
stride_chroma: usize,
- ) -> RgbFrame {
+ ) -> RgbxFrame {
unsafe {
- let channels = 3; // rgb
+ let channels = 4; // rgbx
- let rgb_size = width as usize * height as usize * channels;
- let mut rgb = Vec::::with_capacity(rgb_size);
- rgb.set_len(rgb_size);
+ let rgbx_size = width as usize * height as usize * channels;
+ let mut rgbx = Vec::::with_capacity(rgbx_size);
+ rgbx.set_len(rgbx_size);
- let ptr_rgb = rgb.as_mut_ptr();
+ let ptr_rgbx = rgbx.as_mut_ptr();
for y in 0..height as usize {
for x in 0..width as usize {
@@ -169,16 +189,17 @@ impl RgbFrame {
let (r, g, b) = yuv_to_rgb(val_y, val_u, val_v);
let pos = y * (width as usize * channels) + x * channels;
- *ptr_rgb.add(pos) = r;
- *ptr_rgb.add(pos + 1) = g;
- *ptr_rgb.add(pos + 2) = b;
+ *ptr_rgbx.add(pos) = r;
+ *ptr_rgbx.add(pos + 1) = g;
+ *ptr_rgbx.add(pos + 2) = b;
+ *ptr_rgbx.add(pos + 3) = 255;
}
}
- RgbFrame {
+ RgbxFrame {
width,
height,
- data: rgb,
+ data: rgbx,
}
}
}
@@ -196,6 +217,23 @@ impl Drop for Av1Decoder {
}
}
+enum GetPictureResult {
+ Ok,
+ Again,
+ Failed(i32),
+}
+
+enum ReadFrameIterResult {
+ Ok(RgbxFrame),
+ Again,
+ EndOfFile,
+}
+
+pub enum ReadFrameResult {
+ Ok(RgbxFrame),
+ EndOfFile,
+}
+
impl Av1Decoder {
pub fn new() -> anyhow::Result {
unsafe {
@@ -211,12 +249,25 @@ impl Av1Decoder {
}
}
- pub fn read_frame(&mut self, reader: &mut IvfReader) -> anyhow::Result {
- let (packet, pts) = reader.read_frame()?;
+ fn get_picture(&mut self, picture: &mut Dav1dPicture) -> GetPictureResult {
+ let ret = unsafe { dav1d_get_picture(self.ctx, picture) };
+ if ret == DAV1D_ERR_AGAIN {
+ return GetPictureResult::Again;
+ }
- let mut ret;
+ if ret < 0 {
+ return GetPictureResult::Failed(ret);
+ }
+ GetPictureResult::Ok
+ }
+
+ fn send_data(&mut self, reader: &mut IvfReader) -> anyhow::Result {
unsafe {
+ let (packet, pts) = match reader.read_frame()? {
+ IvfReadFrameResult::Ok((packet, pts)) => (packet, pts),
+ IvfReadFrameResult::EndOfFile => return Ok(false),
+ };
let mut data: Dav1dData = std::mem::zeroed();
data.data = packet.as_ptr();
data.sz = packet.len();
@@ -225,17 +276,31 @@ impl Av1Decoder {
data.m.offset = -1;
data.m.duration = 0;
- ret = dav1d_send_data(self.ctx, &raw mut data);
- dav1d_data_unref(&raw mut data);
+ let ret = dav1d_send_data(self.ctx, &raw mut data);
if ret < 0 {
anyhow::bail!("dav1d_send_data failed: {ret}");
}
+ Ok(true)
+ }
+ }
+
+ fn read_frame_iter(&mut self, reader: &mut IvfReader) -> anyhow::Result {
+ unsafe {
let mut picture: Dav1dPicture = std::mem::zeroed();
- ret = dav1d_get_picture(self.ctx, &raw mut picture);
- if ret < 0 {
- dav1d_picture_unref(&raw mut picture);
- anyhow::bail!("dav1d_get_picture failure: {ret}")
+ match self.get_picture(&mut picture) {
+ GetPictureResult::Ok => {}
+ GetPictureResult::Again => {
+ dav1d_picture_unref(&raw mut picture);
+ if !self.send_data(reader)? {
+ return Ok(ReadFrameIterResult::EndOfFile);
+ }
+ return Ok(ReadFrameIterResult::Again);
+ }
+ GetPictureResult::Failed(code) => {
+ dav1d_picture_unref(&raw mut picture);
+ anyhow::bail!("dav1d_get_picture failed: {code}");
+ }
}
let pic_y = picture.data[0]; // luma Y
@@ -244,7 +309,7 @@ impl Av1Decoder {
let stride_luma = picture.stride[0];
let stride_chroma = picture.stride[1];
- let rgb = RgbFrame::from_yuv(
+ let rgbx = RgbxFrame::from_yuv(
reader.width,
reader.height,
pic_y as *const u8,
@@ -256,7 +321,19 @@ impl Av1Decoder {
dav1d_picture_unref(&raw mut picture);
- Ok(rgb)
+ Ok(ReadFrameIterResult::Ok(rgbx))
+ }
+ }
+
+ pub fn read_frame(&mut self, reader: &mut IvfReader) -> anyhow::Result {
+ loop {
+ match self.read_frame_iter(reader)? {
+ ReadFrameIterResult::Ok(rgbx_frame) => {
+ return Ok(ReadFrameResult::Ok(rgbx_frame));
+ }
+ ReadFrameIterResult::Again => { /* loop again */ }
+ ReadFrameIterResult::EndOfFile => return Ok(ReadFrameResult::EndOfFile),
+ }
}
}
}