diff --git a/.gitmodules b/.gitmodules index fdfb8a1..347a507 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,9 +1,6 @@ [submodule "cmake/clang"] path = cmake/clang url = https://github.com/Xaymar/cmake-clang.git -[submodule "third-party/nvidia-arsdk"] - path = third-party/nvidia-arsdk - url = https://github.com/NVIDIA/BROADCAST-AR-SDK.git [submodule "third-party/nlohmann-json"] path = third-party/nlohmann-json url = https://github.com/nlohmann/json.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 0538b14..e322810 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -303,7 +303,6 @@ set(${PREFIX}ENABLE_FILTER_DENOISING ON CACHE BOOL "Enable Denoising filter") set(${PREFIX}ENABLE_FILTER_DENOISING_NVIDIA ON CACHE BOOL "Enable NVIDIA provider(s) for Denoising Filter") set(${PREFIX}ENABLE_FILTER_DISPLACEMENT ON CACHE BOOL "Enable Displacement Filter") set(${PREFIX}ENABLE_FILTER_DYNAMIC_MASK ON CACHE BOOL "Enable Dynamic Mask Filter") -set(${PREFIX}ENABLE_FILTER_NVIDIA_FACE_TRACKING ON CACHE BOOL "Enable NVIDIA Face Tracking Filter") set(${PREFIX}ENABLE_FILTER_SDF_EFFECTS ON CACHE BOOL "Enable SDF Effects Filter") set(${PREFIX}ENABLE_FILTER_SHADER ON CACHE BOOL "Enable Shader Filter") set(${PREFIX}ENABLE_FILTER_TRANSFORM ON CACHE BOOL "Enable Transform Filter") @@ -689,25 +688,6 @@ function(feature_filter_dynamic_mask RESOLVE) is_feature_enabled(FILTER_DYNAMIC_MASK T_CHECK) endfunction() -function(feature_filter_nvidia_face_tracking RESOLVE) - is_feature_enabled(FILTER_NVIDIA_FACE_TRACKING T_CHECK) - if(RESOLVE AND T_CHECK) - if(NOT D_PLATFORM_WINDOWS) - message(WARNING "${LOGPREFIX}: NVIDIA Face Tracking requires Windows. Disabling...") - set_feature_disabled(FILTER_NVIDIA_FACE_TRACKING ON) - elseif(NOT HAVE_NVIDIA_ARSDK) - message(WARNING "${LOGPREFIX}: NVIDIA Face Tracking requires NVIDIA AR SDK. Disabling...") - set_feature_disabled(FILTER_NVIDIA_FACE_TRACKING ON) - elseif(NOT HAVE_NVIDIA_CUDA) - message(WARNING "${LOGPREFIX}: NVIDIA Face Tracking requires NVIDIA CUDA. Disabling...") - set_feature_disabled(FILTER_NVIDIA_FACE_TRACKING ON) - endif() - elseif(T_CHECK) - set(REQUIRE_NVIDIA_ARSDK ON PARENT_SCOPE) - set(REQUIRE_NVIDIA_CUDA ON PARENT_SCOPE) - endif() -endfunction() - function(feature_filter_sdf_effects RESOLVE) is_feature_enabled(FILTER_SDF_EFFECTS T_CHECK) endfunction() @@ -812,7 +792,6 @@ feature_filter_color_grade(OFF) feature_filter_denoising(OFF) feature_filter_displacement(OFF) feature_filter_dynamic_mask(OFF) -feature_filter_nvidia_face_tracking(OFF) feature_filter_sdf_effects(OFF) feature_filter_shader(OFF) feature_filter_transform(OFF) @@ -902,17 +881,6 @@ if(REQUIRE_JSON) endif() endif() -#- NVIDIA Augmented Reality SDK (Windows) -set(HAVE_NVIDIA_ARSDK OFF) -if(REQUIRE_NVIDIA_ARSDK AND D_PLATFORM_WINDOWS) - if(EXISTS "${PROJECT_SOURCE_DIR}/third-party/nvidia-arsdk/version.h") - set(NVAR_ROOT "${PROJECT_SOURCE_DIR}/third-party/nvidia-arsdk") - endif() - - find_package(NVAR) - - set(HAVE_NVIDIA_ARSDK ${NVAR_FOUND}) -endif() #- NVIDIA Video Effects SDK set(HAVE_NVIDIA_VFX_SDK OFF) @@ -971,7 +939,6 @@ feature_filter_color_grade(ON) feature_filter_denoising(ON) feature_filter_displacement(ON) feature_filter_dynamic_mask(ON) -feature_filter_nvidia_face_tracking(ON) feature_filter_sdf_effects(ON) feature_filter_shader(ON) feature_filter_transform(ON) @@ -1063,19 +1030,7 @@ if(HAVE_JSON) list(APPEND PROJECT_INCLUDE_DIRS ${JSON_INCLUDE_DIR}) endif() -if(HAVE_NVIDIA_ARSDK) - list(APPEND PROJECT_PRIVATE_SOURCE - "source/nvidia/ar/nvidia-ar.hpp" - "source/nvidia/ar/nvidia-ar.cpp" - "source/nvidia/ar/nvidia-ar-feature.hpp" - "source/nvidia/ar/nvidia-ar-feature.cpp" - ) - list(APPEND PROJECT_LIBRARIES - nvARProxy - ) -endif() - -if(HAVE_NVIDIA_ARSDK OR HAVE_NVIDIA_VFX_SDK) +if(HAVE_NVIDIA_VFX_SDK) list(APPEND PROJECT_PRIVATE_SOURCE "source/nvidia/cv/nvidia-cv.hpp" "source/nvidia/cv/nvidia-cv.cpp" @@ -1394,18 +1349,6 @@ if(T_CHECK) ) endif() -# Filter/NVIDIA Face Tracking -is_feature_enabled(FILTER_NVIDIA_FACE_TRACKING T_CHECK) -if(T_CHECK) - list(APPEND PROJECT_PRIVATE_SOURCE - "source/filters/filter-nv-face-tracking.hpp" - "source/filters/filter-nv-face-tracking.cpp" - ) - list(APPEND PROJECT_DEFINITIONS - ENABLE_FILTER_NVIDIA_FACE_TRACKING - ) -endif() - # Filter/SDF Effects is_feature_enabled(FILTER_SDF_EFFECTS T_CHECK) if(T_CHECK) diff --git a/cmake/modules/FindNVAR.cmake b/cmake/modules/FindNVAR.cmake deleted file mode 100644 index 3e0d097..0000000 --- a/cmake/modules/FindNVAR.cmake +++ /dev/null @@ -1,59 +0,0 @@ -# Nvidia AR SDK -# -# Sets -# - NVAR_FOUND -# - NVAR_INCLUDE_DIRS -# - NVAR_SOURCE_DIRS -# -# - -include(FindPackageHandleStandardArgs) -find_package(PkgConfig QUIET) - -# Variables -set(NVAR_ROOT "" CACHE PATH "Path to NVidia AR SDK") - -find_path(NVAR_INCLUDE_DIRS - NAMES - "nvAR.h" "nvAR_defs.h" - HINTS - ENV NVAR_ROOT - ${NVAR_ROOT} - PATHS - /usr/include - /usr/local/include - /opt/local/include - PATH_SUFFIXES - include - nvar/include -) -find_path(NVAR_SOURCE_DIRS - NAMES - "nvARProxy.cpp" - HINTS - ENV NVAR_ROOT - ${NVAR_ROOT} - PATHS - /usr/include - /usr/local/include - /opt/local/include - PATH_SUFFIXES - src - nvar/src -) - -find_package_handle_standard_args(NVAR - FOUND_VAR NVAR_FOUND - REQUIRED_VARS NVAR_INCLUDE_DIRS NVAR_SOURCE_DIRS - VERSION_VAR NVAR_VERSION - HANDLE_COMPONENTS -) - -if(NVAR_FOUND AND NOT TARGET nvARProxy) - add_library(nvARProxy INTERFACE) - target_include_directories(nvARProxy - INTERFACE - ${NVAR_SOURCE_DIRS} - ${NVAR_INCLUDE_DIRS} - ) -endif() diff --git a/data/locale/en-US.ini b/data/locale/en-US.ini index 0b53f49..163593c 100644 --- a/data/locale/en-US.ini +++ b/data/locale/en-US.ini @@ -350,15 +350,6 @@ Filter.DynamicMask.Channel.Value="Base Value" Filter.DynamicMask.Channel.Multiplier="Multiplier" Filter.DynamicMask.Channel.Input="%s Input Value" -# Filter - NVIDIA Face Tracking -Filter.NVIDIA.FaceTracking="NVIDIA Face Tracking" -Filter.NVIDIA.FaceTracking.ROI="Region of Interest" -Filter.NVIDIA.FaceTracking.ROI.Zoom="Zoom" -Filter.NVIDIA.FaceTracking.ROI.Offset="Offset" -Filter.NVIDIA.FaceTracking.ROI.Offset.X="X" -Filter.NVIDIA.FaceTracking.ROI.Offset.Y="Y" -Filter.NVIDIA.FaceTracking.ROI.Stability="Stability" - # Filter - SDF Effects Filter.SDFEffects="SDF Effects" Filter.SDFEffects.Shadow.Inner="Inner Shadow" diff --git a/source/filters/filter-nv-face-tracking.cpp b/source/filters/filter-nv-face-tracking.cpp deleted file mode 100644 index 8c3088e..0000000 --- a/source/filters/filter-nv-face-tracking.cpp +++ /dev/null @@ -1,741 +0,0 @@ -/* - * Modern effects for a modern Streamer - * Copyright (C) 2017 Michael Fabian Dirks - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - */ - -#include "filter-nv-face-tracking.hpp" -#include -#include -#include -#include "nvidia/cuda/nvidia-cuda-context.hpp" -#include "obs/gs/gs-helper.hpp" -#include "obs/obs-tools.hpp" -#include "util/util-logging.hpp" - -#ifdef _DEBUG -#define ST_PREFIX "<%s> " -#define D_LOG_ERROR(x, ...) P_LOG_ERROR(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) -#define D_LOG_WARNING(x, ...) P_LOG_WARN(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) -#define D_LOG_INFO(x, ...) P_LOG_INFO(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) -#define D_LOG_DEBUG(x, ...) P_LOG_DEBUG(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) -#else -#define ST_PREFIX " " -#define D_LOG_ERROR(...) P_LOG_ERROR(ST_PREFIX __VA_ARGS__) -#define D_LOG_WARNING(...) P_LOG_WARN(ST_PREFIX __VA_ARGS__) -#define D_LOG_INFO(...) P_LOG_INFO(ST_PREFIX __VA_ARGS__) -#define D_LOG_DEBUG(...) P_LOG_DEBUG(ST_PREFIX __VA_ARGS__) -#endif - -#define ST_I18N "Filter.NVIDIA.FaceTracking" -#define ST_I18N_ROI ST_I18N ".ROI" -#define ST_I18N_ROI_ZOOM ST_I18N_ROI ".Zoom" -#define ST_I18N_ROI_OFFSET ST_I18N_ROI ".ROI.Offset" -#define ST_I18N_ROI_OFFSET_X ST_I18N_ROI_OFFSET ".X" -#define ST_I18N_ROI_OFFSET_Y ST_I18N_ROI_OFFSET ".Y" -#define ST_I18N_ROI_STABILITY ST_I18N_ROI ".Stability" - -#define ST_KEY_ROI_ZOOM "ROI.Zoom" -#define ST_KEY_ROI_OFFSET_X "ROI.Offset.X" -#define ST_KEY_ROI_OFFSET_Y "ROI.Offset.Y" -#define ST_KEY_ROI_STABILITY "ROI.Stability" - -using namespace streamfx::filter::nvidia; - -void ar_feature_deleter(NvAR_FeatureHandle v) -{ - face_tracking_factory::get()->get_ar()->destroy(v); -} - -face_tracking_instance::face_tracking_instance(obs_data_t* settings, obs_source_t* self) - : obs::source_instance(settings, self), - - _rt_is_fresh(false), _rt(), - - _cfg_zoom(1.0), _cfg_offset({0., 0.}), _cfg_stability(1.0), - - _geometry(), _filters(), _values(), - - _cuda(::streamfx::nvidia::cuda::obs::get()), _cuda_stream(), - - _ar_library(face_tracking_factory::get()->get_ar()), _ar_loaded(false), _ar_feature(), _ar_is_tracking(false), - _ar_bboxes_confidence(), _ar_bboxes_data(), _ar_bboxes(), _ar_texture(), _ar_texture_cuda_fresh(false), - _ar_texture_cuda(), _ar_texture_cuda_mem(), _ar_image(), _ar_image_bgr(), _ar_image_temp() -{ -#ifdef ENABLE_PROFILING - // Profiling - _profile_capture = streamfx::util::profiler::create(); - _profile_capture_realloc = streamfx::util::profiler::create(); - _profile_capture_copy = streamfx::util::profiler::create(); - _profile_ar_realloc = streamfx::util::profiler::create(); - _profile_ar_copy = streamfx::util::profiler::create(); - _profile_ar_transfer = streamfx::util::profiler::create(); - _profile_ar_run = streamfx::util::profiler::create(); - _profile_ar_calc = streamfx::util::profiler::create(); -#endif - - { // Create render target, vertex buffer, and CUDA stream. - auto gctx = streamfx::obs::gs::context{}; - _rt = std::make_shared(GS_RGBA_UNORM, GS_ZS_NONE); - _geometry = std::make_shared(uint32_t(4), uint8_t(1)); - auto cctx = _cuda->get_context()->enter(); - _cuda_stream = - std::make_shared<::streamfx::nvidia::cuda::stream>(::streamfx::nvidia::cuda::stream_flags::NON_BLOCKING, 0); - } - - { // Asynchronously load Face Tracking. - async_initialize(); - } - - { // Set up initial tracking data. - _values.center[0] = _values.center[1] = .5; - _values.size[0] = _values.size[1] = 1.; - refresh_region_of_interest(); - } -} - -face_tracking_instance::~face_tracking_instance() -{ - // Kill pending tasks. - streamfx::threadpool()->pop(_async_initialize); - streamfx::threadpool()->pop(_async_track); - - _ar_loaded.store(false); - std::unique_lock alk{_ar_lock}; - _ar_library->image_dealloc(&_ar_image_temp); - _ar_library->image_dealloc(&_ar_image_bgr); -} - -void face_tracking_instance::async_initialize(std::shared_ptr ptr) -{ - struct async_data { - std::shared_ptr source; - std::string models_path; - }; - - if (!ptr) { - // Spawn the work for the threadpool. - std::shared_ptr data = std::make_shared(); - data->source = - std::shared_ptr(obs_source_get_weak_source(_self), obs::obs_weak_source_deleter); - - { - std::filesystem::path models_path = _ar_library->get_ar_sdk_path(); - models_path = models_path.append("models"); - models_path = std::filesystem::absolute(models_path); - models_path.concat("\\"); - data->models_path = models_path.string(); - } - - _async_initialize = streamfx::threadpool()->push( - std::bind(&face_tracking_instance::async_initialize, this, std::placeholders::_1), data); - } else { - std::shared_ptr data = std::static_pointer_cast(ptr); - - // Try and acquire a strong source reference. - std::shared_ptr remote_work = - std::shared_ptr(obs_weak_source_get_source(data->source.get()), obs::obs_source_deleter); - if (!remote_work) { // If that failed, the source we are working for was deleted - abort now. - return; - } - - // Update the current CUDA context for working. - streamfx::obs::gs::context gctx; - auto cctx = _cuda->get_context()->enter(); - - // Create Face Detection feature. - { - NvAR_FeatureHandle fd_inst; - if (NvCV_Status res = _ar_library->create(NvAR_Feature_FaceDetection, &fd_inst); res != NVCV_SUCCESS) { - throw std::runtime_error("Failed to create Face Detection feature."); - } - _ar_feature = std::shared_ptr{fd_inst, ar_feature_deleter}; - } - - // Set the correct CUDA stream for processing. - if (NvCV_Status res = _ar_library->set_cuda_stream(_ar_feature.get(), NvAR_Parameter_Config(CUDAStream), - reinterpret_cast(_cuda_stream->get())); - res != NVCV_SUCCESS) { - throw std::runtime_error("Failed to set CUDA stream."); - } - - // Set the correct models path. - if (NvCV_Status res = - _ar_library->set_string(_ar_feature.get(), NvAR_Parameter_Config(ModelDir), data->models_path.c_str()); - res != NVCV_SUCCESS) { - throw std::runtime_error("Unable to set model path."); - } - - // Finally enable Temporal tracking if possible. - if (NvCV_Status res = _ar_library->set_uint32(_ar_feature.get(), NvAR_Parameter_Config(Temporal), 1); - res != NVCV_SUCCESS) { - DLOG_WARNING("<%s> Unable to enable Temporal tracking mode.", obs_source_get_name(remote_work.get())); - } - - // Create Bounding Boxes Data - _ar_bboxes_data.assign(1, {0., 0., 0., 0.}); - _ar_bboxes.boxes = _ar_bboxes_data.data(); - _ar_bboxes.max_boxes = std::clamp(static_cast(_ar_bboxes_data.size()), 0, 255); - _ar_bboxes.num_boxes = 0; - _ar_bboxes_confidence.resize(_ar_bboxes_data.size()); - if (NvCV_Status res = _ar_library->set_object(_ar_feature.get(), NvAR_Parameter_Output(BoundingBoxes), - &_ar_bboxes, sizeof(NvAR_BBoxes)); - res != NVCV_SUCCESS) { - throw std::runtime_error("Failed to set BoundingBoxes for Face Tracking feature."); - } - if (NvCV_Status res = _ar_library->set_float32_array( - _ar_feature.get(), NvAR_Parameter_Output(BoundingBoxesConfidence), _ar_bboxes_confidence.data(), - static_cast(_ar_bboxes_confidence.size())); - res != NVCV_SUCCESS) { - throw std::runtime_error("Failed to set BoundingBoxesConfidence for Face Tracking feature."); - } - - // And finally, load the feature (takes long). - if (NvCV_Status res = _ar_library->load(_ar_feature.get()); res != NVCV_SUCCESS) { - DLOG_ERROR("<%s> Failed to load Face Tracking feature.", obs_source_get_name(_self)); - _ar_loaded = false; - return; - } else { - _ar_loaded = true; - } - - _async_initialize.reset(); - } -} - -void face_tracking_instance::async_track(std::shared_ptr ptr) -{ - struct async_data { - std::shared_ptr source; - }; - - if (!_ar_loaded) - return; - - if (!ptr) { - // Check if we can track. - if (_ar_is_tracking) - return; // Can't track a new frame right now. - -#ifdef ENABLE_PROFILING - streamfx::obs::gs::debug_marker gdm{streamfx::obs::gs::debug_color_convert, "Start Asynchronous Tracking"}; -#endif - - // Don't push additional tracking frames while processing one. - _ar_is_tracking = true; - - // Spawn the work for the threadpool. - std::shared_ptr data = std::make_shared(); - data->source = - std::shared_ptr(obs_source_get_weak_source(_self), obs::obs_weak_source_deleter); - - // Check if things exist as planned. - if (!_ar_texture || (_ar_texture->get_width() != _size.first) || (_ar_texture->get_height() != _size.second)) { -#ifdef ENABLE_PROFILING - auto prof = _profile_capture_realloc->track(); - streamfx::obs::gs::debug_marker marker{streamfx::obs::gs::debug_color_allocate, "Reallocate GPU Buffer"}; -#endif - _ar_texture = - std::make_shared(_size.first, _size.second, GS_RGBA_UNORM, uint32_t(1), - nullptr, streamfx::obs::gs::texture::flags::None); - _ar_texture_cuda_fresh = false; - } - - { // Copy texture -#ifdef ENABLE_PROFILING - auto prof = _profile_capture_copy->track(); - streamfx::obs::gs::debug_marker marker{streamfx::obs::gs::debug_color_copy, "Copy Capture", - obs_source_get_name(_self)}; -#endif - gs_copy_texture(_ar_texture->get_object(), _rt->get_texture()->get_object()); - } - - // Push work - _async_track = streamfx::threadpool()->push( - std::bind(&face_tracking_instance::async_track, this, std::placeholders::_1), data); - } else { - // Prevent conflicts. - std::unique_lock alk{_ar_lock}; - if (!_ar_loaded) - return; - - // Try and acquire a strong source reference. - std::shared_ptr data = std::static_pointer_cast(ptr); - std::shared_ptr remote_work = - std::shared_ptr(obs_weak_source_get_source(data->source.get()), obs::obs_source_deleter); - if (!remote_work) { // If that failed, the source we are working for was deleted - abort now. - return; - } - - // Acquire GS context. - streamfx::obs::gs::context gctx{}; - - // Update the current CUDA context for working. - auto cctx = _cuda->get_context()->enter(); - - // Refresh any now broken buffers. - if (!_ar_texture_cuda_fresh) { -#ifdef ENABLE_PROFILING - auto prof = _profile_ar_realloc->track(); - streamfx::obs::gs::debug_marker marker{streamfx::obs::gs::debug_color_allocate, - "%s: Reallocate CUDA Buffers", obs_source_get_name(_self)}; -#endif - // Assign new texture and allocate new memory. - std::size_t pitch = _ar_texture->get_width() * 4ul; - _ar_texture_cuda = std::make_shared<::streamfx::nvidia::cuda::gstexture>(_ar_texture); - _ar_texture_cuda_mem = - std::make_shared<::streamfx::nvidia::cuda::memory>(pitch * _ar_texture->get_height()); - if (auto res = _ar_library->image_init(&_ar_image, static_cast(_ar_texture->get_width()), - static_cast(_ar_texture->get_height()), - static_cast(pitch), - reinterpret_cast(_ar_texture_cuda_mem->get()), NVCV_RGBA, - NVCV_U8, NVCV_INTERLEAVED, NVCV_CUDA); - res != NVCV_SUCCESS) { - DLOG_ERROR("<%s> Failed to allocate image for tracking.", obs_source_get_name(_self)); - return; - } - - // Reallocate transposed buffer. - _ar_library->image_dealloc(&_ar_image_temp); - _ar_library->image_dealloc(&_ar_image_bgr); - if (auto res = _ar_library->image_alloc(&_ar_image_bgr, _ar_image.width, _ar_image.height, NVCV_BGR, - NVCV_U8, NVCV_INTERLEAVED, NVCV_CUDA, 0); - res != NVCV_SUCCESS) { - DLOG_ERROR("<%s> Failed to allocate image for color conversion.", obs_source_get_name(_self)); - return; - } - - // Synchronize Streams. - _cuda_stream->synchronize(); - - // Finally set the input object. - if (NvCV_Status res = _ar_library->set_object(_ar_feature.get(), NvAR_Parameter_Input(Image), - &_ar_image_bgr, sizeof(NvCVImage)); - res != NVCV_SUCCESS) { - DLOG_ERROR("<%s> Failed to update input image for tracking.", obs_source_get_name(_self)); - return; - } - - // And mark the new texture as fresh. - _ar_texture_cuda_fresh = true; - } - - { // Copy from CUDA array to CUDA device memory. -#ifdef ENABLE_PROFILING - auto prof = _profile_ar_copy->track(); -#endif - ::streamfx::nvidia::cuda::memcpy2d_v2_t mc; - mc.src_x_in_bytes = 0; - mc.src_y = 0; - mc.src_memory_type = ::streamfx::nvidia::cuda::memory_type::ARRAY; - mc.src_host = nullptr; - mc.src_device = 0; - mc.src_array = _ar_texture_cuda->map(_cuda_stream); - mc.src_pitch = static_cast(_ar_image.pitch); - mc.dst_x_in_bytes = 0; - mc.dst_y = 0; - mc.dst_memory_type = ::streamfx::nvidia::cuda::memory_type::DEVICE; - mc.dst_host = 0; - mc.dst_device = reinterpret_cast<::streamfx::nvidia::cuda::device_ptr_t>(_ar_image.pixels); - mc.dst_array = 0; - mc.dst_pitch = static_cast(_ar_image.pitch); - mc.width_in_bytes = static_cast(_ar_image.pitch); - mc.height = _ar_image.height; - - if (::streamfx::nvidia::cuda::result res = _cuda->get_cuda()->cuMemcpy2DAsync(&mc, _cuda_stream->get()); - res != ::streamfx::nvidia::cuda::result::SUCCESS) { - DLOG_ERROR("<%s> Failed to prepare buffers for tracking.", obs_source_get_name(_self)); - return; - } - } - - { // Convert from RGBA 32-bit to BGR 24-bit. -#ifdef ENABLE_PROFILING - auto prof = _profile_ar_transfer->track(); -#endif - if (NvCV_Status res = - _ar_library->image_transfer(&_ar_image, &_ar_image_bgr, 1.0, - reinterpret_cast(_cuda_stream->get()), &_ar_image_temp); - res != NVCV_SUCCESS) { - DLOG_ERROR("<%s> Failed to convert from RGBX 32-bit to BGR 24-bit.", obs_source_get_name(_self)); - return; - } - - // Synchronize Streams. - _cuda_stream->synchronize(); - _cuda->get_context()->synchronize(); - } - - { // Track any faces. -#ifdef ENABLE_PROFILING - auto prof = _profile_ar_run->track(); -#endif - if (NvCV_Status res = _ar_library->run(_ar_feature.get()); res != NVCV_SUCCESS) { - DLOG_ERROR("<%s> Failed to run tracking.", obs_source_get_name(_self)); - return; - } - } - - // Are we tracking anything, and confident enough in the tracking? - if ((_ar_bboxes.num_boxes == 0) || (_ar_bboxes_confidence.at(0) < 0.3333)) { - // If not, just return to full frame. - std::unique_lock tlk{_values.lock}; - _values.center[0] = .5; - _values.center[1] = .5; - _values.size[0] = 1.; - _values.size[1] = 1.; - _values.velocity[0] = 0; - _values.velocity[1] = 0; - } else { - // If yes, begin tracking. -#ifdef ENABLE_PROFILING - auto prof = _profile_ar_calc->track(); -#endif - - double_t sx = static_cast(_ar_image_bgr.width); - double_t sy = static_cast(_ar_image_bgr.height); - double_t aspect = double_t(sx) / double_t(sy); - double_t fps = 0.; - - { - obs_video_info ovi; - obs_get_video_info(&ovi); - fps = static_cast(ovi.fps_num) / static_cast(ovi.fps_den); - } - - // Store values and center. - double_t bsx = _ar_bboxes.boxes[0].width; - double_t bsy = _ar_bboxes.boxes[0].height; - double_t bcx = _ar_bboxes.boxes[0].x + bsx / 2.0; - double_t bcy = _ar_bboxes.boxes[0].y + bsy / 2.0; - - // Zoom, Aspect Ratio, Offset - bsy = streamfx::util::math::lerp(sy, bsy, _cfg_zoom); - bsy = std::clamp(bsy, 10 * aspect, static_cast(_size.second)); - bsx = bsy * aspect; - bcx += _ar_bboxes.boxes[0].width * _cfg_offset.first; - bcy += _ar_bboxes.boxes[0].height * _cfg_offset.second; - - // Fit back into the frame - // - Above code guarantees that height is never bigger than the height of the frame. - // - Which also guarantees that width is never bigger than the width of the frame. - // Only cx and cy need to be adjusted now to always be in the frame. - bcx = std::clamp(bcx, (bsx / 2.), sx - (bsx / 2.)); - bcy = std::clamp(bcy, (bsy / 2.), sy - (bsy / 2.)); - - { // Update target values. - std::unique_lock tlk{_values.lock}; - _values.velocity[0] = -_values.center[0]; - _values.velocity[1] = -_values.center[1]; - _values.center[0] = bcx / sx; - _values.center[1] = bcy / sy; - _values.velocity[0] += _values.center[0]; - _values.velocity[1] += _values.center[1]; - _values.velocity[0] *= fps; - _values.velocity[1] *= fps; - _values.size[0] = bsx / sx; - _values.size[1] = bsy / sy; - } - } - - _async_track.reset(); - - // Allow new frames to be queued again. - _ar_is_tracking = false; - } -} - -void face_tracking_instance::refresh_geometry() -{ // Update Region of Interest Geometry. - auto v0 = _geometry->at(0); - auto v1 = _geometry->at(1); - auto v2 = _geometry->at(2); - auto v3 = _geometry->at(3); - - vec3_set(v3.position, static_cast(_size.first), static_cast(_size.second), 0.); - vec3_set(v2.position, v3.position->x, 0., 0.); - vec3_set(v1.position, 0., v3.position->y, 0.); - vec3_set(v0.position, 0., 0., 0.); - - float_t hsx = static_cast(_filters.size[0].get() / 2.); - float_t hsy = static_cast(_filters.size[1].get() / 2.); - vec4_set(v0.uv[0], static_cast(_filters.center[0].get() - hsx), - static_cast(_filters.center[1].get() - hsy), 0., 0.); - vec4_set(v1.uv[0], static_cast(_filters.center[0].get() - hsx), - static_cast(_filters.center[1].get() + hsy), 0., 0.); - vec4_set(v2.uv[0], static_cast(_filters.center[0].get() + hsx), - static_cast(_filters.center[1].get() - hsy), 0., 0.); - vec4_set(v3.uv[0], static_cast(_filters.center[0].get() + hsx), - static_cast(_filters.center[1].get() + hsy), 0., 0.); - - _geometry->update(true); -} - -void face_tracking_instance::refresh_region_of_interest() -{ - std::unique_lock tlk(_values.lock); - - double_t kalman_q = streamfx::util::math::lerp(1.0, 1e-6, _cfg_stability); - double_t kalman_r = - streamfx::util::math::lerp(std::numeric_limits::epsilon(), 1e+2, _cfg_stability); - - _filters.center[0] = streamfx::util::math::kalman1D{kalman_q, kalman_r, 1., _values.center[0]}; - _filters.center[1] = streamfx::util::math::kalman1D{kalman_q, kalman_r, 1., _values.center[1]}; - _filters.size[0] = streamfx::util::math::kalman1D{kalman_q, kalman_r, 1., _values.size[0]}; - _filters.size[1] = streamfx::util::math::kalman1D{kalman_q, kalman_r, 1., _values.size[1]}; -} - -void face_tracking_instance::load(obs_data_t* data) -{ - update(data); -} - -void face_tracking_instance::migrate(obs_data_t* data, uint64_t version) {} - -void face_tracking_instance::update(obs_data_t* data) -{ - _cfg_zoom = obs_data_get_double(data, ST_KEY_ROI_ZOOM) / 100.0; - _cfg_offset.first = obs_data_get_double(data, ST_KEY_ROI_OFFSET_X) / 100.0; - _cfg_offset.second = obs_data_get_double(data, ST_KEY_ROI_OFFSET_Y) / 100.0; - _cfg_stability = obs_data_get_double(data, ST_KEY_ROI_STABILITY) / 100.0; - - // Refresh the Region Of Interest - refresh_region_of_interest(); -} - -void face_tracking_instance::video_tick(float_t seconds) -{ - // If we aren't yet ready to do work, abort for now. - if (!_ar_loaded) { - return; - } - - // Update the input size. - if (obs_source_t* src = obs_filter_get_target(_self); src != nullptr) { - _size.first = obs_source_get_base_width(src); - _size.second = obs_source_get_base_height(src); - } - - // Update filters and geometry - { - std::unique_lock tlk(_values.lock); - _filters.center[0].filter(_values.center[0]); - _filters.center[1].filter(_values.center[1]); - _filters.size[0].filter(_values.size[0]); - _filters.size[1].filter(_values.size[1]); - _values.center[0] += _values.velocity[0] * seconds; - _values.center[1] += _values.velocity[1] * seconds; - } - refresh_geometry(); - - _rt_is_fresh = false; -} - -void face_tracking_instance::video_render(gs_effect_t*) -{ - obs_source_t* filter_parent = obs_filter_get_parent(_self); - obs_source_t* filter_target = obs_filter_get_target(_self); - gs_effect_t* default_effect = obs_get_base_effect(OBS_EFFECT_DEFAULT); - - if (!filter_parent || !filter_target || !_size.first || !_size.second || !_ar_loaded) { - obs_source_skip_video_filter(_self); - return; - } - -#ifdef ENABLE_PROFILING - streamfx::obs::gs::debug_marker gdmp{streamfx::obs::gs::debug_color_source, "NVIDIA Face Tracking '%s'...", - obs_source_get_name(_self)}; - streamfx::obs::gs::debug_marker gdmp2{streamfx::obs::gs::debug_color_source, "... on '%s'", - obs_source_get_name(obs_filter_get_parent(_self))}; -#endif - - if (!_rt_is_fresh) { // Capture the filter stack "below" us. -#ifdef ENABLE_PROFILING - auto prof = _profile_capture->track(); -#endif - - { -#ifdef ENABLE_PROFILING - streamfx::obs::gs::debug_marker gdm{streamfx::obs::gs::debug_color_cache, "Cache"}; -#endif - - if (obs_source_process_filter_begin(_self, GS_RGBA, OBS_NO_DIRECT_RENDERING)) { - auto op = _rt->render(_size.first, _size.second); - vec4 clr = {0., 0., 0., 0.}; - - gs_ortho(0., 1., 0., 1., -1., 1.); - gs_clear(GS_CLEAR_COLOR, &clr, 0., 0); - gs_enable_color(true, true, true, true); - gs_enable_blending(false); - auto old_fbsrgb = gs_framebuffer_srgb_enabled(); - gs_enable_framebuffer_srgb(gs_get_linear_srgb()); - obs_source_process_filter_tech_end(_self, default_effect, 1, 1, "Draw"); - gs_enable_framebuffer_srgb(old_fbsrgb); - } else { - obs_source_skip_video_filter(_self); - return; - } - } - - // Probably spawn new work. - async_track(nullptr); - - _rt_is_fresh = true; - } - - { // Draw Texture -#ifdef ENABLE_PROFILING - streamfx::obs::gs::debug_marker gdm{streamfx::obs::gs::debug_color_render, "Render"}; -#endif - - gs_effect_set_texture(gs_effect_get_param_by_name(default_effect, "image"), _rt->get_texture()->get_object()); - gs_load_vertexbuffer(_geometry->update(false)); - while (gs_effect_loop(default_effect, "Draw")) { - gs_draw(gs_draw_mode::GS_TRISTRIP, 0, 0); - } - gs_load_vertexbuffer(nullptr); - } -} - -#ifdef ENABLE_PROFILING -bool face_tracking_instance::button_profile(obs_properties_t* props, obs_property_t* property) -{ - DLOG_INFO("%-22s: %-10s %-10s %-10s %-10s %-10s", "Task", "Total", "Count", "Average", "99.9%ile", "95.0%ile"); - - std::pair> profilers[]{ - {"Capture", _profile_capture}, {"Reallocate", _profile_capture_realloc}, - {"Copy", _profile_capture_copy}, {"AR Reallocate", _profile_ar_realloc}, - {"AR Copy", _profile_ar_copy}, {"AR Convert", _profile_ar_transfer}, - {"AR Run", _profile_ar_run}, {"AR Calculate", _profile_ar_calc}, - }; - for (auto& kv : profilers) { - DLOG_INFO(" %-20s: %8lldµs %10lld %8lldµs %8lldµs %8lldµs", kv.first.c_str(), - std::chrono::duration_cast(kv.second->total_duration()).count(), - kv.second->count(), static_cast(kv.second->average_duration() / 1000.0), - std::chrono::duration_cast(kv.second->percentile(0.999)).count(), - std::chrono::duration_cast(kv.second->percentile(0.95)).count()); - } - - return false; -} -#endif - -face_tracking_factory::face_tracking_factory() -{ - // Try and load CUDA. - _cuda = ::streamfx::nvidia::cuda::obs::get(); - - // Try and load AR. - _ar = std::make_shared<::streamfx::nvidia::ar::ar>(); - - // Info - _info.id = S_PREFIX "filter-nvidia-face-tracking"; - _info.type = OBS_SOURCE_TYPE_FILTER; - _info.output_flags = OBS_SOURCE_VIDEO | OBS_SOURCE_CUSTOM_DRAW; - - set_resolution_enabled(false); - finish_setup(); - register_proxy("streamfx-nvidia-face-tracking"); -} - -face_tracking_factory::~face_tracking_factory() {} - -const char* face_tracking_factory::get_name() -{ - return D_TRANSLATE(ST_I18N); -} - -void face_tracking_factory::get_defaults2(obs_data_t* data) -{ - obs_data_set_default_double(data, ST_KEY_ROI_ZOOM, 50.0); - obs_data_set_default_double(data, ST_KEY_ROI_OFFSET_X, 0.0); - obs_data_set_default_double(data, ST_KEY_ROI_OFFSET_Y, -15.0); - obs_data_set_default_double(data, ST_KEY_ROI_STABILITY, 50.0); -} - -obs_properties_t* face_tracking_factory::get_properties2(face_tracking_instance* data) -{ - obs_properties_t* pr = obs_properties_create(); - - { - auto grp = obs_properties_create(); - obs_properties_add_group(pr, ST_I18N_ROI, D_TRANSLATE(ST_I18N_ROI), OBS_GROUP_NORMAL, grp); - { - auto p = obs_properties_add_float_slider(grp, ST_KEY_ROI_STABILITY, D_TRANSLATE(ST_I18N_ROI_STABILITY), 0, - 100.0, 0.01); - obs_property_float_set_suffix(p, " %"); - } - { - auto p = - obs_properties_add_float_slider(grp, ST_KEY_ROI_ZOOM, D_TRANSLATE(ST_I18N_ROI_ZOOM), 0, 200.0, 0.01); - obs_property_float_set_suffix(p, " %"); - } - { - auto grp2 = obs_properties_create(); - obs_properties_add_group(grp, ST_I18N_ROI_OFFSET, D_TRANSLATE(ST_I18N_ROI_OFFSET), OBS_GROUP_NORMAL, grp2); - - { - auto p = obs_properties_add_float_slider(grp2, ST_KEY_ROI_OFFSET_X, D_TRANSLATE(ST_I18N_ROI_OFFSET_X), - -50.0, 50.0, 0.01); - obs_property_float_set_suffix(p, " %"); - } - { - auto p = obs_properties_add_float_slider(grp2, ST_KEY_ROI_OFFSET_Y, D_TRANSLATE(ST_I18N_ROI_OFFSET_Y), - -50.0, 50.0, 0.01); - obs_property_float_set_suffix(p, " %"); - } - } - } -#ifdef ENABLE_PROFILING - { - obs_properties_add_button2( - pr, "Profile", "Profile", - [](obs_properties_t* props, obs_property_t* property, void* data) { - return reinterpret_cast(data)->button_profile(props, property); - }, - data); - } -#endif - - return pr; -} - -std::shared_ptr<::streamfx::nvidia::ar::ar> face_tracking_factory::get_ar() -{ - return _ar; -} - -std::shared_ptr _filter_nvidia_face_tracking_factory_instance = nullptr; - -void streamfx::filter::nvidia::face_tracking_factory::initialize() -try { - if (!_filter_nvidia_face_tracking_factory_instance) - _filter_nvidia_face_tracking_factory_instance = std::make_shared(); -} catch (const std::exception& ex) { - D_LOG_ERROR("Failed to initialize due to error: %s", ex.what()); -} catch (...) { - D_LOG_ERROR("Failed to initialize due to unknown error.", ""); -} - -void streamfx::filter::nvidia::face_tracking_factory::finalize() -{ - _filter_nvidia_face_tracking_factory_instance.reset(); -} - -std::shared_ptr streamfx::filter::nvidia::face_tracking_factory::get() -{ - return _filter_nvidia_face_tracking_factory_instance; -} diff --git a/source/filters/filter-nv-face-tracking.hpp b/source/filters/filter-nv-face-tracking.hpp deleted file mode 100644 index 7e07228..0000000 --- a/source/filters/filter-nv-face-tracking.hpp +++ /dev/null @@ -1,154 +0,0 @@ -/* - * Modern effects for a modern Streamer - * Copyright (C) 2017 Michael Fabian Dirks - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - */ - -#pragma once -#include "common.hpp" -#include -#include -#include "obs/gs/gs-effect.hpp" -#include "obs/gs/gs-rendertarget.hpp" -#include "obs/gs/gs-vertexbuffer.hpp" -#include "obs/obs-source-factory.hpp" - -// Nvidia -#include "nvidia/ar/nvidia-ar.hpp" -#include "nvidia/cuda/nvidia-cuda-context.hpp" -#include "nvidia/cuda/nvidia-cuda-gs-texture.hpp" -#include "nvidia/cuda/nvidia-cuda-memory.hpp" -#include "nvidia/cuda/nvidia-cuda-obs.hpp" -#include "nvidia/cuda/nvidia-cuda-stream.hpp" -#include "nvidia/cuda/nvidia-cuda.hpp" - -namespace streamfx::filter::nvidia { - class face_tracking_instance : public obs::source_instance { - // Filter Cache - std::pair _size; - bool _rt_is_fresh; - std::shared_ptr _rt; - - std::mutex _delete_protection; - - // Settings - double_t _cfg_zoom; - std::pair _cfg_offset; - double_t _cfg_stability; - - // Operational Data - std::shared_ptr _geometry; - struct { - streamfx::util::math::kalman1D center[2]; - streamfx::util::math::kalman1D size[2]; - } _filters; - struct { - std::mutex lock; - double_t center[2]; - double_t size[2]; - double_t velocity[2]; - } _values; - - // Nvidia CUDA interop - std::shared_ptr<::streamfx::nvidia::cuda::obs> _cuda; - std::shared_ptr<::streamfx::nvidia::cuda::stream> _cuda_stream; - - // Nvidia AR interop - std::shared_ptr<::streamfx::nvidia::ar::ar> _ar_library; - std::atomic_bool _ar_loaded; - std::shared_ptr _ar_feature; - std::atomic_bool _ar_is_tracking; - std::mutex _ar_lock; - std::vector _ar_bboxes_confidence; - std::vector _ar_bboxes_data; - NvAR_BBoxes _ar_bboxes; - std::shared_ptr _ar_texture; - bool _ar_texture_cuda_fresh; - std::shared_ptr<::streamfx::nvidia::cuda::gstexture> _ar_texture_cuda; - std::shared_ptr<::streamfx::nvidia::cuda::memory> _ar_texture_cuda_mem; - NvCVImage _ar_image; - NvCVImage _ar_image_bgr; - NvCVImage _ar_image_temp; - - // Tasks - std::shared_ptr<::streamfx::util::threadpool::task> _async_initialize; - std::shared_ptr<::streamfx::util::threadpool::task> _async_track; - -#ifdef ENABLE_PROFILING - // Profiling - std::shared_ptr _profile_capture; - std::shared_ptr _profile_capture_realloc; - std::shared_ptr _profile_capture_copy; - std::shared_ptr _profile_ar_realloc; - std::shared_ptr _profile_ar_copy; - std::shared_ptr _profile_ar_transfer; - std::shared_ptr _profile_ar_run; - std::shared_ptr _profile_ar_calc; -#endif - - public: - face_tracking_instance(obs_data_t*, obs_source_t*); - virtual ~face_tracking_instance() override; - - // Tasks - void async_initialize(std::shared_ptr = nullptr); - - void async_track(std::shared_ptr = nullptr); - - void refresh_geometry(); - - void refresh_region_of_interest(); - - virtual void load(obs_data_t* data) override; - - virtual void migrate(obs_data_t* data, uint64_t version) override; - - virtual void update(obs_data_t* data) override; - - virtual void video_tick(float_t seconds) override; - - virtual void video_render(gs_effect_t* effect) override; - -#ifdef ENABLE_PROFILING - bool button_profile(obs_properties_t* props, obs_property_t* property); -#endif - }; - - class face_tracking_factory - : public obs::source_factory { - std::shared_ptr<::streamfx::nvidia::cuda::obs> _cuda; - std::shared_ptr<::streamfx::nvidia::ar::ar> _ar; - - public: - face_tracking_factory(); - virtual ~face_tracking_factory() override; - - virtual const char* get_name() override; - - virtual void get_defaults2(obs_data_t* data) override; - - virtual obs_properties_t* get_properties2(filter::nvidia::face_tracking_instance* data) override; - - std::shared_ptr<::streamfx::nvidia::ar::ar> get_ar(); - - public: // Singleton - static void initialize(); - - static void finalize(); - - static std::shared_ptr get(); - }; -} // namespace streamfx::filter::nvidia diff --git a/source/nvidia/ar/nvidia-ar-feature.cpp b/source/nvidia/ar/nvidia-ar-feature.cpp deleted file mode 100644 index ae33059..0000000 --- a/source/nvidia/ar/nvidia-ar-feature.cpp +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Modern effects for a modern Streamer - * Copyright (C) 2020 Michael Fabian Dirks - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - */ - -#include "nvidia-ar-feature.hpp" - -streamfx::nvidia::ar::feature::feature(std::shared_ptr<::streamfx::nvidia::ar::ar> ar, NvAR_FeatureID feature) : _ar(ar) -{ - NvAR_FeatureHandle feat; - if (NvCV_Status res = _ar->create(feature, &feat); res != NVCV_SUCCESS) { - throw std::runtime_error("Failed to create feature."); - } - - _feature = std::shared_ptr{feat, [this](NvAR_FeatureHandle v) { _ar->destroy(v); }}; -} - -streamfx::nvidia::ar::feature::~feature() -{ - _feature.reset(); -} diff --git a/source/nvidia/ar/nvidia-ar-feature.hpp b/source/nvidia/ar/nvidia-ar-feature.hpp deleted file mode 100644 index 5d2462c..0000000 --- a/source/nvidia/ar/nvidia-ar-feature.hpp +++ /dev/null @@ -1,156 +0,0 @@ -/* - * Modern effects for a modern Streamer - * Copyright (C) 2020 Michael Fabian Dirks - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - */ - -#pragma once -#include -#include "nvidia-ar.hpp" -#include "nvidia/cuda/nvidia-cuda-stream.hpp" - -namespace streamfx::nvidia::ar { - class feature { - std::shared_ptr<::streamfx::nvidia::ar::ar> _ar; - std::shared_ptr _feature; - - public: - feature(std::shared_ptr<::streamfx::nvidia::ar::ar> ar, NvAR_FeatureID feature); - ~feature(); - - public: - template - inline NvCV_Status set(std::string name, T value); - template - inline NvCV_Status get(std::string name, T& value); - - template<> - inline NvCV_Status set(std::string name, int32_t value) - { - return _ar->set_int32(_feature.get(), name.c_str(), value); - } - - template<> - inline NvCV_Status get(std::string name, int32_t& value) - { - return _ar->get_int32(_feature.get(), name.c_str(), &value); - } - - template<> - inline NvCV_Status set(std::string name, uint32_t value) - { - return _ar->set_uint32(_feature.get(), name.c_str(), value); - } - - template<> - inline NvCV_Status get(std::string name, uint32_t& value) - { - return _ar->get_uint32(_feature.get(), name.c_str(), &value); - } - - template<> - inline NvCV_Status set(std::string name, uint64_t value) - { - return _ar->set_uint64(_feature.get(), name.c_str(), value); - } - - template<> - inline NvCV_Status get(std::string name, uint64_t& value) - { - return _ar->get_uint64(_feature.get(), name.c_str(), &value); - } - - template<> - inline NvCV_Status set(std::string name, std::float_t value) - { - return _ar->set_float32(_feature.get(), name.c_str(), value); - } - - template<> - inline NvCV_Status get(std::string name, std::float_t& value) - { - return _ar->get_float32(_feature.get(), name.c_str(), &value); - } - - template<> - inline NvCV_Status set(std::string name, std::vector value) - { - return _ar->set_float32_array(_feature.get(), name.c_str(), value.data(), - static_cast(value.size())); - } - - template<> - inline NvCV_Status get(std::string name, std::vector& value) - { - // ToDo: Validate this. - const float* vals = nullptr; - int val_count = 0; - NvCV_Status res = _ar->get_float32_array(_feature.get(), name.c_str(), &vals, &val_count); - if (res != NVCV_SUCCESS) { - return res; - } else { - value.resize(static_cast(val_count)); - for (std::size_t idx = 0; idx < static_cast(val_count); idx++) { - value[idx] = *vals; - vals++; - } - return res; - } - } - - template<> - inline NvCV_Status set(std::string name, std::double_t value) - { - return _ar->set_float64(_feature.get(), name.c_str(), value); - } - - template<> - inline NvCV_Status get(std::string name, std::double_t& value) - { - return _ar->get_float64(_feature.get(), name.c_str(), &value); - } - - template<> - inline NvCV_Status set(std::string name, std::string value) - { - return _ar->set_string(_feature.get(), name.c_str(), value.c_str()); - } - - template<> - inline NvCV_Status get(std::string name, std::string& value) - { - // ToDo: Validate this. - const char* buf; - NvCV_Status res = _ar->get_string(_feature.get(), name.c_str(), &buf); - if (res == NVCV_SUCCESS) { - value = std::string(buf, buf + strlen(buf)); - return res; - } else { - return res; - } - } - - template<> - inline NvCV_Status set(std::string name, std::shared_ptr<::streamfx::nvidia::cuda::stream> value) - { - return _ar->set_cuda_stream(_feature.get(), name.c_str(), reinterpret_cast(value->get())); - } - - template<> - inline NvCV_Status get(std::string name, std::shared_ptr<::streamfx::nvidia::cuda::stream>& value) - {} - }; -} // namespace streamfx::nvidia::ar diff --git a/source/nvidia/ar/nvidia-ar.cpp b/source/nvidia/ar/nvidia-ar.cpp deleted file mode 100644 index 6e38330..0000000 --- a/source/nvidia/ar/nvidia-ar.cpp +++ /dev/null @@ -1,269 +0,0 @@ -/* - * Modern effects for a modern Streamer - * Copyright (C) 2020 Michael Fabian Dirks - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - */ - -#include "nvidia-ar.hpp" -#include -#include -#include - -#ifdef WIN32 -#include -#include -#endif - -#include -#define nvGetProcAddress nvGetProcAddressCV -#define nvFreeLibrary nvFreeLibraryCV -#include -#undef nvGetProcAddress -#undef nvFreeLibrary - -streamfx::nvidia::ar::ar::ar() -{ - if (!getNvARLib()) - throw std::runtime_error("Failed to load NVIDIA AR SDK runtime."); -} - -streamfx::nvidia::ar::ar::~ar() {} - -std::filesystem::path streamfx::nvidia::ar::ar::get_ar_sdk_path() -{ - char* arsdk_path = getenv("NV_AR_SDK_PATH"); - if (arsdk_path) { - return std::filesystem::path(std::string{arsdk_path}); - } else { - std::filesystem::path res; -#ifdef WIN32 - std::vector dll_path_w; - dll_path_w.resize(65535); - DWORD size_w = GetModuleFileNameW(getNvARLib(), dll_path_w.data(), static_cast(dll_path_w.size())); - - std::vector dll_path; - dll_path.resize(65535); - std::size_t size = os_wcs_to_utf8(dll_path_w.data(), size_w, dll_path.data(), dll_path.size()); - - std::filesystem::path dll = std::string{dll_path.data(), dll_path.data() + size}; - res = dll.remove_filename(); -#endif - return res; - } -} - -NvCV_Status streamfx::nvidia::ar::ar::image_init(NvCVImage* im, unsigned width, unsigned height, int pitch, - void* pixels, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU) -{ - return NvCVImage_Init(im, width, height, pitch, pixels, format, type, isPlanar, onGPU); -} - -void streamfx::nvidia::ar::ar::image_init_view(NvCVImage* subImg, NvCVImage* fullImg, int x, int y, unsigned width, - unsigned height) -{ - NvCVImage_InitView(subImg, fullImg, x, y, width, height); -} - -NvCV_Status streamfx::nvidia::ar::ar::image_alloc(NvCVImage* im, unsigned width, unsigned height, - NvCVImage_PixelFormat format, NvCVImage_ComponentType type, - unsigned isPlanar, unsigned onGPU, unsigned alignment) -{ - return NvCVImage_Alloc(im, width, height, format, type, isPlanar, onGPU, alignment); -} - -NvCV_Status streamfx::nvidia::ar::ar::image_realloc(NvCVImage* im, unsigned width, unsigned height, - NvCVImage_PixelFormat format, NvCVImage_ComponentType type, - unsigned isPlanar, unsigned onGPU, unsigned alignment) -{ - return NvCVImage_Realloc(im, width, height, format, type, isPlanar, onGPU, alignment); -} - -void streamfx::nvidia::ar::ar::image_dealloc(NvCVImage* im) -{ - NvCVImage_Dealloc(im); -} - -NvCV_Status streamfx::nvidia::ar::ar::image_create(unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, - unsigned alignment, NvCVImage** out) -{ - return NvCVImage_Create(width, height, format, type, isPlanar, onGPU, alignment, out); -} - -void streamfx::nvidia::ar::ar::image_destroy(NvCVImage* im) -{ - NvCVImage_Destroy(im); -} - -void streamfx::nvidia::ar::ar::image_component_offsets(NvCVImage_PixelFormat format, int* rOff, int* gOff, int* bOff, - int* aOff, int* yOff) -{ - NvCVImage_ComponentOffsets(format, rOff, gOff, bOff, aOff, yOff); -} - -NvCV_Status streamfx::nvidia::ar::ar::image_transfer(const NvCVImage* src, NvCVImage* dst, float scale, - CUstream_st* stream, NvCVImage* tmp) -{ - return NvCVImage_Transfer(src, dst, scale, stream, tmp); -} - -NvCV_Status streamfx::nvidia::ar::ar::image_composite(const NvCVImage* src, const NvCVImage* mat, NvCVImage* dst) -{ - //return NvCVImage_Composite(src, mat, dst); - throw std::runtime_error("Not implemented."); -} - -NvCV_Status streamfx::nvidia::ar::ar::image_composite_over_constant(const NvCVImage* src, const NvCVImage* mat, - const unsigned char bgColor[3], NvCVImage* dst) -{ - return NvCVImage_CompositeOverConstant(src, mat, bgColor, dst); -} - -NvCV_Status streamfx::nvidia::ar::ar::image_flipy(const NvCVImage* src, NvCVImage* dst) -{ - return NvCVImage_FlipY(src, dst); -} - -NvCV_Status streamfx::nvidia::ar::ar::create(NvAR_FeatureID featureID, NvAR_FeatureHandle* handle) -{ - return NvAR_Create(featureID, handle); -} - -NvCV_Status streamfx::nvidia::ar::ar::destroy(NvAR_FeatureHandle handle) -{ - return NvAR_Destroy(handle); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_uint32(NvAR_FeatureHandle handle, const char* name, unsigned int val) -{ - return NvAR_SetU32(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_int32(NvAR_FeatureHandle handle, const char* name, int val) -{ - return NvAR_SetS32(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_float32(NvAR_FeatureHandle handle, const char* name, float val) -{ - return NvAR_SetF32(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_float64(NvAR_FeatureHandle handle, const char* name, double val) -{ - return NvAR_SetF64(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_uint64(NvAR_FeatureHandle handle, const char* name, unsigned long long val) -{ - return NvAR_SetU64(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_object(NvAR_FeatureHandle handle, const char* name, void* ptr, - unsigned long typeSize) -{ - return NvAR_SetObject(handle, name, ptr, typeSize); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_string(NvAR_FeatureHandle handle, const char* name, const char* str) -{ - return NvAR_SetString(handle, name, str); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_cuda_stream(NvAR_FeatureHandle handle, const char* name, CUstream stream) -{ - return NvAR_SetCudaStream(handle, name, stream); -} - -NvCV_Status streamfx::nvidia::ar::ar::set_float32_array(NvAR_FeatureHandle handle, const char* name, float* val, - int count) -{ - return NvAR_SetF32Array(handle, name, val, count); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_uint32(NvAR_FeatureHandle handle, const char* name, unsigned int* val) -{ - return NvAR_GetU32(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_int32(NvAR_FeatureHandle handle, const char* name, int* val) -{ - return NvAR_GetS32(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_float32(NvAR_FeatureHandle handle, const char* name, float* val) -{ - return NvAR_GetF32(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_float64(NvAR_FeatureHandle handle, const char* name, double* val) -{ - return NvAR_GetF64(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_uint64(NvAR_FeatureHandle handle, const char* name, unsigned long long* val) -{ - return NvAR_GetU64(handle, name, val); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_object(NvAR_FeatureHandle handle, const char* name, const void** ptr, - unsigned long typeSize) -{ - return NvAR_GetObject(handle, name, ptr, typeSize); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_string(NvAR_FeatureHandle handle, const char* name, const char** str) -{ - return NvAR_GetString(handle, name, str); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_cuda_stream(NvAR_FeatureHandle handle, const char* name, - const CUstream* stream) -{ - return NvAR_GetCudaStream(handle, name, stream); -} - -NvCV_Status streamfx::nvidia::ar::ar::get_float32_array(NvAR_FeatureHandle handle, const char* name, const float** vals, - int* count) -{ - return NvAR_GetF32Array(handle, name, vals, count); -} - -NvCV_Status streamfx::nvidia::ar::ar::run(NvAR_FeatureHandle handle) -{ - return NvAR_Run(handle); -} - -NvCV_Status streamfx::nvidia::ar::ar::load(NvAR_FeatureHandle handle) -{ - return NvAR_Load(handle); -} - -NvCV_Status streamfx::nvidia::ar::ar::cuda_stream_create(CUstream* stream) -{ - return NvAR_CudaStreamCreate(stream); -} - -NvCV_Status streamfx::nvidia::ar::ar::cuda_stream_destroy(CUstream stream) -{ - return NvAR_CudaStreamDestroy(stream); -} - -const char* streamfx::nvidia::ar::ar::cv_get_error_string_from_code(NvCV_Status code) -{ - return NvCV_GetErrorStringFromCode(code); -} diff --git a/source/nvidia/ar/nvidia-ar.hpp b/source/nvidia/ar/nvidia-ar.hpp deleted file mode 100644 index e1ebf9a..0000000 --- a/source/nvidia/ar/nvidia-ar.hpp +++ /dev/null @@ -1,94 +0,0 @@ -/* - * Modern effects for a modern Streamer - * Copyright (C) 2020 Michael Fabian Dirks - * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation; either version 2 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program; if not, write to the Free Software - * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA - */ - -#pragma once -#include -#include -#include -#include - -#ifdef _MSC_VER -#pragma warning(push) -#pragma warning(disable : 4138) -#endif -#include -#include -#include -#include -#ifdef _MSC_VER -#pragma warning(pop) -#endif - -namespace streamfx::nvidia::ar { - class ar { - public: - ar(); - ~ar(); - - std::filesystem::path get_ar_sdk_path(); - - public: - NvCV_Status image_init(NvCVImage* im, unsigned width, unsigned height, int pitch, void* pixels, - NvCVImage_PixelFormat format, NvCVImage_ComponentType type, unsigned isPlanar, - unsigned onGPU); - void image_init_view(NvCVImage* subImg, NvCVImage* fullImg, int x, int y, unsigned width, unsigned height); - NvCV_Status image_alloc(NvCVImage* im, unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment); - NvCV_Status image_realloc(NvCVImage* im, unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment); - void image_dealloc(NvCVImage* im); - NvCV_Status image_create(unsigned width, unsigned height, NvCVImage_PixelFormat format, - NvCVImage_ComponentType type, unsigned isPlanar, unsigned onGPU, unsigned alignment, - NvCVImage** out); - void image_destroy(NvCVImage* im); - void image_component_offsets(NvCVImage_PixelFormat format, int* rOff, int* gOff, int* bOff, int* aOff, - int* yOff); - NvCV_Status image_transfer(const NvCVImage* src, NvCVImage* dst, float scale, CUstream_st* stream, - NvCVImage* tmp); - NvCV_Status image_composite(const NvCVImage* src, const NvCVImage* mat, NvCVImage* dst); - NvCV_Status image_composite_over_constant(const NvCVImage* src, const NvCVImage* mat, - const unsigned char bgColor[3], NvCVImage* dst); - NvCV_Status image_flipy(const NvCVImage* src, NvCVImage* dst); - NvCV_Status create(NvAR_FeatureID featureID, NvAR_FeatureHandle* handle); - NvCV_Status destroy(NvAR_FeatureHandle handle); - NvCV_Status set_uint32(NvAR_FeatureHandle handle, const char* name, unsigned int val); - NvCV_Status set_int32(NvAR_FeatureHandle handle, const char* name, int val); - NvCV_Status set_float32(NvAR_FeatureHandle handle, const char* name, float val); - NvCV_Status set_float64(NvAR_FeatureHandle handle, const char* name, double val); - NvCV_Status set_uint64(NvAR_FeatureHandle handle, const char* name, unsigned long long val); - NvCV_Status set_object(NvAR_FeatureHandle handle, const char* name, void* ptr, unsigned long typeSize); - NvCV_Status set_string(NvAR_FeatureHandle handle, const char* name, const char* str); - NvCV_Status set_cuda_stream(NvAR_FeatureHandle handle, const char* name, CUstream stream); - NvCV_Status set_float32_array(NvAR_FeatureHandle handle, const char* name, float* val, int count); - NvCV_Status get_uint32(NvAR_FeatureHandle handle, const char* name, unsigned int* val); - NvCV_Status get_int32(NvAR_FeatureHandle handle, const char* name, int* val); - NvCV_Status get_float32(NvAR_FeatureHandle handle, const char* name, float* val); - NvCV_Status get_float64(NvAR_FeatureHandle handle, const char* name, double* val); - NvCV_Status get_uint64(NvAR_FeatureHandle handle, const char* name, unsigned long long* val); - NvCV_Status get_object(NvAR_FeatureHandle handle, const char* name, const void** ptr, unsigned long typeSize); - NvCV_Status get_string(NvAR_FeatureHandle handle, const char* name, const char** str); - NvCV_Status get_cuda_stream(NvAR_FeatureHandle handle, const char* name, const CUstream* stream); - NvCV_Status get_float32_array(NvAR_FeatureHandle handle, const char* name, const float** vals, int* count); - NvCV_Status run(NvAR_FeatureHandle handle); - NvCV_Status load(NvAR_FeatureHandle handle); - NvCV_Status cuda_stream_create(CUstream* stream); - NvCV_Status cuda_stream_destroy(CUstream stream); - const char* cv_get_error_string_from_code(NvCV_Status code); - }; -} // namespace streamfx::nvidia::ar diff --git a/source/plugin.cpp b/source/plugin.cpp index a6d8289..4bfeec7 100644 --- a/source/plugin.cpp +++ b/source/plugin.cpp @@ -50,9 +50,6 @@ #ifdef ENABLE_FILTER_DYNAMIC_MASK #include "filters/filter-dynamic-mask.hpp" #endif -#ifdef ENABLE_FILTER_NVIDIA_FACE_TRACKING -#include "filters/filter-nv-face-tracking.hpp" -#endif #ifdef ENABLE_FILTER_SDF_EFFECTS #include "filters/filter-sdf-effects.hpp" #endif @@ -164,9 +161,6 @@ try { #ifdef ENABLE_FILTER_DYNAMIC_MASK streamfx::filter::dynamic_mask::dynamic_mask_factory::initialize(); #endif -#ifdef ENABLE_FILTER_NVIDIA_FACE_TRACKING - streamfx::filter::nvidia::face_tracking_factory::initialize(); -#endif #ifdef ENABLE_FILTER_SDF_EFFECTS streamfx::filter::sdf_effects::sdf_effects_factory::initialize(); #endif @@ -259,9 +253,6 @@ try { #ifdef ENABLE_FILTER_DYNAMIC_MASK streamfx::filter::dynamic_mask::dynamic_mask_factory::finalize(); #endif -#ifdef ENABLE_FILTER_NVIDIA_FACE_TRACKING - streamfx::filter::nvidia::face_tracking_factory::finalize(); -#endif #ifdef ENABLE_FILTER_SDF_EFFECTS streamfx::filter::sdf_effects::sdf_effects_factory::finalize(); #endif