From 85541de504840c076026202a57760c9a3c01be66 Mon Sep 17 00:00:00 2001 From: rdb Date: Mon, 24 Feb 2025 22:45:48 +0100 Subject: [PATCH] grutil: Add support for playing video via HTML video+WebGL --- .../glstuff/glGraphicsStateGuardian_src.cxx | 28 + panda/src/grutil/config_grutil.cxx | 7 + panda/src/grutil/htmlVideoTexture.I | 12 + panda/src/grutil/htmlVideoTexture.cxx | 489 ++++++++++++++++++ panda/src/grutil/htmlVideoTexture.h | 115 ++++ panda/src/grutil/p3grutil_composite1.cxx | 1 + 6 files changed, 652 insertions(+) create mode 100644 panda/src/grutil/htmlVideoTexture.I create mode 100644 panda/src/grutil/htmlVideoTexture.cxx create mode 100644 panda/src/grutil/htmlVideoTexture.h diff --git a/panda/src/glstuff/glGraphicsStateGuardian_src.cxx b/panda/src/glstuff/glGraphicsStateGuardian_src.cxx index f95dcc7467..6ba7a5e01d 100644 --- a/panda/src/glstuff/glGraphicsStateGuardian_src.cxx +++ b/panda/src/glstuff/glGraphicsStateGuardian_src.cxx @@ -70,6 +70,10 @@ #include "displayInformation.h" #include "completionCounter.h" +#ifdef __EMSCRIPTEN__ +#include "htmlVideoTexture.h" +#endif + #if defined(HAVE_CG) && !defined(OPENGLES) #include #endif @@ -14776,6 +14780,30 @@ upload_texture_view(CLP(TextureContext) *gtc, int view, bool needs_reload, image_ptr = ptimage; } if (image_ptr == nullptr) { +#ifdef __EMSCRIPTEN__ + if (tex->is_of_type(HTMLVideoTexture::get_class_type())) { + // Load directly from the video element into WebGL. + int result = EM_ASM_INT({ + var video = window._htmlVideoData[$7].video; + if (video.readyState < video.HAVE_CURRENT_DATA) { + return 0; + } + GLctx.pixelStorei(GLctx.UNPACK_FLIP_Y_WEBGL, true); + if ($8) { + GLctx.texImage2D($0, $1, $2, $3, $4, 0, $5, $6, video); + } else { + GLctx.texSubImage2D($0, $1, 0, 0, $3, $4, $5, $6, video); + } + GLctx.pixelStorei(GLctx.UNPACK_FLIP_Y_WEBGL, false); + return 1; + }, target, 0, internal_format, width, height, external_format, component_type, tex, needs_reload); + + if (result) { + continue; + } + } +#endif + if (level < num_ram_mipmap_levels) { // We were told we'd have this many RAM mipmap images, but we // don't. Raise a warning. diff --git a/panda/src/grutil/config_grutil.cxx b/panda/src/grutil/config_grutil.cxx index 6f74fd5e1b..fea92c6673 100644 --- a/panda/src/grutil/config_grutil.cxx +++ b/panda/src/grutil/config_grutil.cxx @@ -17,6 +17,7 @@ #include "meshDrawer.h" #include "meshDrawer2D.h" #include "geoMipTerrain.h" +#include "htmlVideoTexture.h" #include "movieTexture.h" #include "pandaSystem.h" #include "texturePool.h" @@ -137,4 +138,10 @@ init_libgrutil() { TexturePool *ts = TexturePool::get_global_ptr(); ts->register_texture_type(MovieTexture::make_texture, "avi m2v mov mpg mpeg mp4 wmv asf flv nut ogm mkv ogv webm"); #endif + +#ifdef __EMSCRIPTEN__ + HTMLVideoTexture::init_type(); + + ts->register_texture_type(HTMLVideoTexture::make_texture, "avi m2v mov mpg mpeg mp4 wmv asf flv nut ogm mkv ogv webm"); +#endif } diff --git a/panda/src/grutil/htmlVideoTexture.I b/panda/src/grutil/htmlVideoTexture.I new file mode 100644 index 0000000000..6feecd77ab --- /dev/null +++ b/panda/src/grutil/htmlVideoTexture.I @@ -0,0 +1,12 @@ +/** + * PANDA 3D SOFTWARE + * Copyright (c) Carnegie Mellon University. All rights reserved. + * + * All use of this software is subject to the terms of the revised BSD + * license. You should have received a copy of this license along + * with this source code in a file named "LICENSE." + * + * @file htmlVideoTexture.I + * @author rdb + * @date 2025-02-24 + */ diff --git a/panda/src/grutil/htmlVideoTexture.cxx b/panda/src/grutil/htmlVideoTexture.cxx new file mode 100644 index 0000000000..cc083ea225 --- /dev/null +++ b/panda/src/grutil/htmlVideoTexture.cxx @@ -0,0 +1,489 @@ +/** + * PANDA 3D SOFTWARE + * Copyright (c) Carnegie Mellon University. All rights reserved. + * + * All use of this software is subject to the terms of the revised BSD + * license. You should have received a copy of this license along + * with this source code in a file named "LICENSE." + * + * @file htmlVideoTexture.cxx + * @author rdb + * @date 2025-02-24 + */ + +#include "htmlVideoTexture.h" + +#ifdef __EMSCRIPTEN__ + +#include "config_grutil.h" +#include "virtualFileSystem.h" +#include "virtualFileHTTP.h" + +#include + +#ifndef CPPPARSER +// If the browser supports video frame callback, we use that instead of relying +// on cull callbacks. +extern "C" void EMSCRIPTEN_KEEPALIVE +_video_frame_callback(HTMLVideoTexture *texture, int width, int height, double time) { + texture->video_frame_callback(width, height, time); +} + +static bool get_supports_video_frame_callback() { + static bool supported = EM_ASM_INT({ + return 'requestVideoFrameCallback' in HTMLVideoElement.prototype; + }); + return supported; +} +#endif + +TypeHandle HTMLVideoTexture::_type_handle; + +/** + * Creates a blank movie texture. Movies must be added using do_read_one. + */ +HTMLVideoTexture:: +HTMLVideoTexture(const std::string &name) : + Texture(name) +{ + EM_ASM_INT({ + if (!window._htmlVideoData) { + window._htmlVideoData = {}; + } + var video = document.createElement("video"); + video.style.display = 'none'; + video.defaultMuted = true; + document.body.appendChild(video); + window._htmlVideoData[$0] = {video: video}; + }, this); +} + +/** + * xxx + */ +HTMLVideoTexture:: +~HTMLVideoTexture() { + clear(); + + EM_ASM_INT({ + var data = window._htmlVideoData[$0]; + if (data) { + if (data.video) { + if (data.callback) { + video.cancelVideoFrameCallback(data.callback); + delete data.callback; + } + data.video.remove(); + delete data.video; + } + if (data.objectURL) { + URL.revokeObjectURL(data.objectURL); + delete data.objectURL; + } + delete window._htmlVideoData[$0]; + } + }, this); +} + +/** + * Returns the length of the video. + */ +double HTMLVideoTexture:: +get_video_length() const { + return EM_ASM_DOUBLE({ + return window._htmlVideoData[$0].video.duration; + }, this); +} + +/** + * Returns the width in texels of the source video stream. This is not + * necessarily the width of the actual texture, since the texture may have + * been expanded to raise it to a power of 2. + */ +int HTMLVideoTexture:: +get_video_width() const { + return EM_ASM_DOUBLE({ + return window._htmlVideoData[$0].video.videoWidth; + }, this); +} + +/** + * Returns the height in texels of the source video stream. This is not + * necessarily the height of the actual texture, since the texture may have + * been expanded to raise it to a power of 2. + */ +int HTMLVideoTexture:: +get_video_height() const { + return EM_ASM_DOUBLE({ + return window._htmlVideoData[$0].video.videoHeight; + }, this); +} + +/** + * Start playing the video from where it was last paused. Has no effect if + * the video is not paused, or if the video's cursor is already at the end. + */ +void HTMLVideoTexture:: +restart() { + EM_ASM({ + window._htmlVideoData[$0].video.play(); + }, this); +} + +/** + * Stops a currently playing or looping movie right where it is. the video's + * cursor remains frozen at the point where it was stopped. + */ +void HTMLVideoTexture:: +stop() { + EM_ASM({ + window._htmlVideoData[$0].video.pause(); + }, this); +} + +/** + * Plays the video from the beginning. + */ +void HTMLVideoTexture:: +play() { + EM_ASM({ + var video = window._htmlVideoData[$0].video; + if (video.playing) { + video.pause(); + } + video.currentTime = 0.0; + video.play(); + }, this); +} + +/** + * Sets the video's cursor. + */ +void HTMLVideoTexture:: +set_time(double t) { + EM_ASM({ + window._htmlVideoData[$0].video.currentTime = $1; + }, this, t); +} + +/** + * Returns the current value of the video's cursor. + */ +double HTMLVideoTexture:: +get_time() const { + return EM_ASM_DOUBLE({ + return window._htmlVideoData[$0].video.currentTime; + }, this); +} + +/** + * If true, sets the video's loop count to 1 billion. If false, sets the + * movie's loop count to one. + */ +void HTMLVideoTexture:: +set_loop(bool loop) { + EM_ASM({ + window._htmlVideoData[$0].video.loop = $1; + }, this, loop); +} + +/** + * Returns true if the video's is looping. + */ +bool HTMLVideoTexture:: +get_loop() const { + return EM_ASM_INT({ + return window._htmlVideoData[$0].video.loop; + }, this); +} + +/** + * Sets the video's play-rate. This is the speed at which the video's cursor + * advances. The default is to advance 1.0 movie-seconds per real-time + * second. + */ +void HTMLVideoTexture:: +set_play_rate(double rate) { + EM_ASM({ + window._htmlVideoData[$0].video.playbackRate = $1; + }, this, rate); +} + +/** + * Gets the video's play-rate. + */ +double HTMLVideoTexture:: +get_play_rate() const { + return EM_ASM_DOUBLE({ + return window._htmlVideoData[$0].video.playbackRate; + }, this); +} + +/** + * Returns true if the video's cursor is advancing. + */ +bool HTMLVideoTexture:: +is_playing() const { + return EM_ASM_INT({ + return !window._htmlVideoData[$0].video.paused; + }, this); +} + +/** + * A factory function to make a new HTMLVideoTexture, used to pass to the + * TexturePool. + */ +PT(Texture) HTMLVideoTexture:: +make_texture() { + return new HTMLVideoTexture(""); +} + +/** + * Should be overridden by derived classes to return true if cull_callback() + * has been defined. Otherwise, returns false to indicate cull_callback() + * does not need to be called for this node during the cull traversal. + */ +bool HTMLVideoTexture:: +has_cull_callback() const { + return !get_supports_video_frame_callback(); +} + +/** + * This function will be called during the cull traversal to update the + * HTMLVideoTexture. This will just check whether the video time changed since + * the last time it was checked. + */ +bool HTMLVideoTexture:: +cull_callback(CullTraverser *, const CullTraverserData &) const { + ((HTMLVideoTexture *)this)->check_update(); + return true; +} + +/** + * + */ +void HTMLVideoTexture:: +video_frame_callback(int width, int height, double time) { + CDWriter cdata(_cycler, true); + if (width != cdata->_x_size || height != cdata->_y_size) { + cdata->_x_size = width; + cdata->_y_size = height; + cdata->inc_properties_modified(); + } + cdata->inc_image_modified(); + _last_time = time; +} + +/** + * + */ +bool HTMLVideoTexture:: +check_update() { + double current_time = get_time(); + if (current_time != _last_time) { + int width = get_video_width(); + int height = get_video_height(); + if (width == 0 && height == 0) { + return false; + } + + video_frame_callback(width, height, current_time); + return true; + } + return false; +} + +/** + * The protected implementation of clear(). Assumes the lock is already held. + */ +void HTMLVideoTexture:: +do_clear(Texture::CData *cdata) { + EM_ASM_INT({ + var data = window._htmlVideoData[$0]; + data.video.pause(); + data.video.src = ""; + if (data.callback) { + data.video.cancelVideoFrameCallback(data.callback); + delete data.callback; + } + if (data.objectURL) { + URL.revokeObjectURL(data.objectURL); + delete data.objectURL; + } + }, this); + + Texture::do_clear(cdata); +} + +/** + * Returns true if we can safely call do_unlock_and_reload_ram_image() in + * order to make the image available, or false if we shouldn't do this + * (because we know from a priori knowledge that it wouldn't work anyway). + */ +bool HTMLVideoTexture:: +do_can_reload(const Texture::CData *cdata) const { + return false; +} + +/** + * Works like adjust_size, but also considers the texture class. Movie + * textures, for instance, always pad outwards, never scale down. + */ +bool HTMLVideoTexture:: +do_adjust_this_size(const Texture::CData *cdata_tex, + int &x_size, int &y_size, const std::string &name, + bool for_padding) const { + // We always scale, for now. May change in the future. + AutoTextureScale ats = do_get_auto_texture_scale(cdata_tex); + if (ats == ATS_pad) { + ats = ATS_up; + } + + return adjust_size(x_size, y_size, name, for_padding, ats); +} + +/** + * Combines a color and alpha video image from the two indicated filenames. + * Both must be the same kind of video with similar properties. + */ +bool HTMLVideoTexture:: +do_read_one(Texture::CData *cdata_tex, + const Filename &fullpath, const Filename &alpha_fullpath, + int z, int n, int primary_file_num_channels, int alpha_file_channel, + const LoaderOptions &options, + bool header_only, BamCacheRecord *record) { + nassertr(n == 0, false); + if (!do_reconsider_z_size(cdata_tex, z, options)) { + return false; + } + nassertr(z >= 0 && z < cdata_tex->_z_size * cdata_tex->_num_views, false); + + if (!alpha_fullpath.empty()) { + grutil_cat.error() + << "HTMLVideoTexture does not support loading separate alpha video.\n"; + return false; + } + + VirtualFileSystem *vfs = VirtualFileSystem::get_global_ptr(); + PT(VirtualFile) vfile = vfs->get_file(fullpath, true); + if (vfile == nullptr) { + return false; + } + + clear(); + + if (record != nullptr) { + record->add_dependent_file(vfile); + } + + // Pass in the URL directly if it's an HTTP file. + if (vfile->is_of_type(VirtualFileHTTP::get_class_type())) { + URLSpec url = ((VirtualFileHTTP *)vfile.p())->get_url(); + + _last_time = EM_ASM_DOUBLE({ + var data = window._htmlVideoData[$0]; + var video = data.video; + video.src = UTF8ToString($1); + video.loop = true; + video.muted = true; + + video.load(); + return video.currentTime; + }, this, url.c_str()); + } + else { + vector_uchar data; + if (!vfile->read_file(data, true)) { + return false; + } + + _last_time = EM_ASM_DOUBLE({ + var data = window._htmlVideoData[$0]; + + var blob = new Blob([HEAPU8.subarray($1, $1 + $2)]); + data.objectURL = URL.createObjectURL(blob); + + var video = data.video; + video.src = data.objectURL; + video.loop = true; + video.muted = true; + + video.load(); + return video.currentTime; + }, this, (void *)data.data(), data.size()); + } + + // If the browser supports requestVideoframeCallback, we set up a callback to + // know when the next video frame is available instead of having to poll in + // cull_callback(). + if (get_supports_video_frame_callback()) { + EM_ASM({ + var data = window._htmlVideoData[$0]; + + function callback(now, metadata) { + __video_frame_callback($0, metadata.width, metadata.height, metadata.mediaTime); + data.callback = data.video.requestVideoFrameCallback(callback); + } + data.callback = data.video.requestVideoFrameCallback(callback); + }, this); + } + + int width = get_video_width(); + int height = get_video_height(); + if (width == 0 || height == 0) { + // Not yet known. Don't set the size to 0 since that will cause + // CardMaker::set_uv_range() to generate invalid coordinates. + width = 1; + height = 1; + } + + if (!do_reconsider_image_properties(cdata_tex, width, height, 3, T_unsigned_byte, z, options)) { + return false; + } + + if (z == 0) { + if (!has_name()) { + set_name(fullpath.get_basename_wo_extension()); + } + // Don't use has_filename() here, it will cause a deadlock + if (cdata_tex->_filename.empty()) { + cdata_tex->_filename = fullpath; + cdata_tex->_alpha_filename = alpha_fullpath; + } + + cdata_tex->_fullpath = fullpath; + cdata_tex->_alpha_fullpath = alpha_fullpath; + } + + cdata_tex->_primary_file_num_channels = primary_file_num_channels; + cdata_tex->_alpha_file_channel = alpha_file_channel; + + cdata_tex->_loaded_from_image = true; + play(); + check_update(); + return true; +} + +/** + * Loading a static image into an HTMLVideoTexture is an error. + */ +bool HTMLVideoTexture:: +do_load_one(Texture::CData *cdata_tex, + const PNMImage &pnmimage, const std::string &name, int z, int n, + const LoaderOptions &options) { + grutil_cat.error() << "You cannot load a static image into an HTMLVideoTexture\n"; + return false; +} + +/** + * Loading a static image into an HTMLVideoTexture is an error. + */ +bool HTMLVideoTexture:: +do_load_one(Texture::CData *cdata_tex, + const PfmFile &pfm, const std::string &name, int z, int n, + const LoaderOptions &options) { + grutil_cat.error() << "You cannot load a static image into an HTMLVideoTexture\n"; + return false; +} + +#endif // __EMSCRIPTEN__ diff --git a/panda/src/grutil/htmlVideoTexture.h b/panda/src/grutil/htmlVideoTexture.h new file mode 100644 index 0000000000..2e0c03f561 --- /dev/null +++ b/panda/src/grutil/htmlVideoTexture.h @@ -0,0 +1,115 @@ +/** + * PANDA 3D SOFTWARE + * Copyright (c) Carnegie Mellon University. All rights reserved. + * + * All use of this software is subject to the terms of the revised BSD + * license. You should have received a copy of this license along + * with this source code in a file named "LICENSE." + * + * @file htmlVideoTexture.h + * @author rdb + * @date 2025-02-24 + */ + +#ifndef HTMLVIDEOTEXTURE_H +#define HTMLVIDEOTEXTURE_H + +#include "pandabase.h" + +#ifdef __EMSCRIPTEN__ + +/** + * A texture that renders video frames from an HTMLVideoElement. + * Interface aims to be (nearly) identical to MovieTexture, with some + * limitations: + * - No RAM access of video data is possible + * - No explicit synchronization with audio + * - No loop count + */ +class EXPCL_PANDA_GRUTIL HTMLVideoTexture : public Texture { +PUBLISHED: + explicit HTMLVideoTexture(const std::string &name); + HTMLVideoTexture(const HTMLVideoTexture ©) = delete; + virtual ~HTMLVideoTexture(); + + double get_video_length() const; + int get_video_width() const; + int get_video_height() const; + + void restart(); + void stop(); + void play(); + void set_time(double t); + double get_time() const; + void set_loop(bool enable); + bool get_loop() const; + void set_play_rate(double play_rate); + double get_play_rate() const; + bool is_playing() const; + +PUBLISHED: + MAKE_PROPERTY(video_length, get_video_length); + MAKE_PROPERTY(video_width, get_video_width); + MAKE_PROPERTY(video_height, get_video_height); + + MAKE_PROPERTY(time, get_time, set_time); + MAKE_PROPERTY(loop, get_loop, set_loop); + MAKE_PROPERTY(play_rate, get_play_rate, set_play_rate); + MAKE_PROPERTY(playing, is_playing); + +public: + static PT(Texture) make_texture(); + + virtual bool has_cull_callback() const; + virtual bool cull_callback(CullTraverser *trav, const CullTraverserData &data) const; + + void video_frame_callback(int width, int height, double time); + +protected: + bool check_update(); + + virtual void do_clear(Texture::CData *cdata); + virtual bool do_can_reload(const Texture::CData *cdata) const; + + virtual bool do_adjust_this_size(const Texture::CData *cdata, + int &x_size, int &y_size, const std::string &name, + bool for_padding) const; + + virtual bool do_read_one(Texture::CData *cdata, + const Filename &fullpath, const Filename &alpha_fullpath, + int z, int n, int primary_file_num_channels, int alpha_file_channel, + const LoaderOptions &options, + bool header_only, BamCacheRecord *record); + virtual bool do_load_one(Texture::CData *cdata, + const PNMImage &pnmimage, const std::string &name, + int z, int n, const LoaderOptions &options); + virtual bool do_load_one(Texture::CData *cdata, + const PfmFile &pfm, const std::string &name, + int z, int n, const LoaderOptions &options); + +private: + double _last_time = 0.0; + +public: + static TypeHandle get_class_type() { + return _type_handle; + } + static void init_type() { + Texture::init_type(); + register_type(_type_handle, "HTMLVideoTexture", + Texture::get_class_type()); + } + virtual TypeHandle get_type() const { + return get_class_type(); + } + virtual TypeHandle force_init_type() {init_type(); return get_class_type();} + +private: + static TypeHandle _type_handle; +}; + +#include "htmlVideoTexture.I" + +#endif // __EMSCRIPTEN__ + +#endif diff --git a/panda/src/grutil/p3grutil_composite1.cxx b/panda/src/grutil/p3grutil_composite1.cxx index 85d1c61e50..ec5ed74be9 100644 --- a/panda/src/grutil/p3grutil_composite1.cxx +++ b/panda/src/grutil/p3grutil_composite1.cxx @@ -1,5 +1,6 @@ #include "cardMaker.cxx" #include "heightfieldTesselator.cxx" +#include "htmlVideoTexture.cxx" #include "geoMipTerrain.cxx" #include "shaderTerrainMesh.cxx" #include "config_grutil.cxx"