From 913ac4b3099ab104ca83716e986685c3b9fd7e63 Mon Sep 17 00:00:00 2001 From: Michael Fabian 'Xaymar' Dirks Date: Mon, 25 Oct 2021 19:56:56 +0200 Subject: [PATCH] nvidia/ar: Add modern wrapper for Maxine AR SDK --- CMakeLists.txt | 28 +++- source/nvidia/ar/nvidia-ar.cpp | 232 +++++++++++++++++++++++++++++++++ source/nvidia/ar/nvidia-ar.hpp | 182 ++++++++++++++++++++++++++ 3 files changed, 441 insertions(+), 1 deletion(-) create mode 100644 source/nvidia/ar/nvidia-ar.cpp create mode 100644 source/nvidia/ar/nvidia-ar.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index e322810..8cfe5cc 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -881,6 +881,22 @@ if(REQUIRE_JSON) endif() endif() +#- NVIDIA Augmented Reality SDK +set(HAVE_NVIDIA_AR_SDK OFF) +if(REQUIRE_NVIDIA_AR_SDK AND D_PLATFORM_WINDOWS) + if(EXISTS "${PROJECT_SOURCE_DIR}/third-party/nvidia-maxine-ar-sdk/version.h") + set(HAVE_NVIDIA_AR_SDK ON) + endif() + + if(NOT TARGET NVIDIA::AR) + add_library(NVIDIA::AR IMPORTED INTERFACE) + target_include_directories(NVIDIA::AR + INTERFACE + "${PROJECT_SOURCE_DIR}/third-party/nvidia-maxine-ar-sdk/nvar/include/" + "${PROJECT_SOURCE_DIR}/third-party/nvidia-maxine-ar-sdk/nvar/src/" + ) + endif() +endif() #- NVIDIA Video Effects SDK set(HAVE_NVIDIA_VFX_SDK OFF) @@ -1030,7 +1046,7 @@ if(HAVE_JSON) list(APPEND PROJECT_INCLUDE_DIRS ${JSON_INCLUDE_DIR}) endif() -if(HAVE_NVIDIA_VFX_SDK) +if(HAVE_NVIDIA_VFX_SDK OR HAVE_NVIDIA_AR_SDK) list(APPEND PROJECT_PRIVATE_SOURCE "source/nvidia/cv/nvidia-cv.hpp" "source/nvidia/cv/nvidia-cv.cpp" @@ -1041,6 +1057,16 @@ if(HAVE_NVIDIA_VFX_SDK) ) endif() +if(HAVE_NVIDIA_AR_SDK) + list(APPEND PROJECT_PRIVATE_SOURCE + "source/nvidia/ar/nvidia-ar.hpp" + "source/nvidia/ar/nvidia-ar.cpp" + ) + list(APPEND PROJECT_LIBRARIES + NVIDIA::AR + ) +endif() + if(HAVE_NVIDIA_VFX_SDK) list(APPEND PROJECT_PRIVATE_SOURCE "source/nvidia/vfx/nvidia-vfx.hpp" diff --git a/source/nvidia/ar/nvidia-ar.cpp b/source/nvidia/ar/nvidia-ar.cpp new file mode 100644 index 0000000..123cec3 --- /dev/null +++ b/source/nvidia/ar/nvidia-ar.cpp @@ -0,0 +1,232 @@ +// Copyright (c) 2021 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#include "nvidia-ar.hpp" +#include +#include +#include "nvidia/cuda/nvidia-cuda-obs.hpp" +#include "obs/gs/gs-helper.hpp" +#include "util/util-logging.hpp" +#include "util/util-platform.hpp" + +#ifdef _DEBUG +#define ST_PREFIX "<%s> " +#define D_LOG_ERROR(x, ...) P_LOG_ERROR(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_WARNING(x, ...) P_LOG_WARN(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_INFO(x, ...) P_LOG_INFO(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#define D_LOG_DEBUG(x, ...) P_LOG_DEBUG(ST_PREFIX##x, __FUNCTION_SIG__, __VA_ARGS__) +#else +#define ST_PREFIX " " +#define D_LOG_ERROR(...) P_LOG_ERROR(ST_PREFIX __VA_ARGS__) +#define D_LOG_WARNING(...) P_LOG_WARN(ST_PREFIX __VA_ARGS__) +#define D_LOG_INFO(...) P_LOG_INFO(ST_PREFIX __VA_ARGS__) +#define D_LOG_DEBUG(...) P_LOG_DEBUG(ST_PREFIX __VA_ARGS__) +#endif + +#ifdef WIN32 +#include +#include +#include + +#define ST_LIBRARY_NAME "nvARPose.dll" +#else +#define ST_LIBRARY_NAME "libnvARPose.so" +#endif + +#define P_NVAR_LOAD_SYMBOL(NAME) \ + { \ + NAME = reinterpret_cast(_library->load_symbol(#NAME)); \ + if (!NAME) \ + throw std::runtime_error("Failed to load '" #NAME "' from '" ST_LIBRARY_NAME "'."); \ + } + +streamfx::nvidia::ar::ar::~ar() +{ + D_LOG_DEBUG("Finalizing... (Addr: 0x%" PRIuPTR ")", this); + +#ifdef WIN32 + // Remove the DLL directory from the library loader paths. + if (_extra != nullptr) { + RemoveDllDirectory(reinterpret_cast(_extra)); + } +#endif + + { // The library may need to release Graphics and CUDA resources. + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + _library.reset(); + } +} + +streamfx::nvidia::ar::ar::ar() : _library(), _model_path() +{ + std::filesystem::path sdk_path; + auto gctx = ::streamfx::obs::gs::context(); + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + + D_LOG_DEBUG("Initializating... (Addr: 0x%" PRIuPTR ")", this); + + // Figure out where the Augmented Reality SDK is, if it is installed. +#ifdef WIN32 + { + // NVAR SDK only defines NVAR_MODEL_PATH, so we'll use that as our baseline. + DWORD env_size = GetEnvironmentVariableW(L"NVAR_MODEL_PATH", nullptr, 0); + if (env_size > 0) { + std::vector buffer(static_cast(env_size) + 1, 0); + env_size = GetEnvironmentVariableW(L"NVAR_MODEL_PATH", buffer.data(), buffer.size()); + _model_path = std::wstring(buffer.data(), buffer.size()); + + // The SDK is location one directory "up" from the model path. + sdk_path = std::filesystem::path(_model_path) / ".."; + } + + // If the environment variable wasn't set and our model path is still undefined, guess! + if (sdk_path.empty()) { + PWSTR str = nullptr; + HRESULT res = SHGetKnownFolderPath(FOLDERID_ProgramFiles, KF_FLAG_DEFAULT, nullptr, &str); + if (res == S_OK) { + sdk_path = std::wstring(str); + sdk_path /= "NVIDIA Corporation"; + sdk_path /= "NVIDIA AR SDK"; + CoTaskMemFree(str); + + // Model path is in 'models' subdirectory. + _model_path = sdk_path; + _model_path /= "models"; + } + } + + // Figure out absolute paths to everything. + _model_path = streamfx::util::platform::native_to_utf8(std::filesystem::absolute(_model_path)); + sdk_path = streamfx::util::platform::native_to_utf8(std::filesystem::absolute(sdk_path)); + } +#else + throw std::runtime_error("Not yet implemented."); +#endif + + // Check if any of the found paths are valid. + if (!std::filesystem::exists(sdk_path)) { + D_LOG_ERROR("No supported NVIDIA SDK is installed to provide '%s'.", ST_LIBRARY_NAME); + throw std::runtime_error("Failed to load '" ST_LIBRARY_NAME "'."); + } + + // Try and load the library. + { +#ifdef WIN32 + // On platforms where it is possible, modify the linker directories. + DLL_DIRECTORY_COOKIE ck = AddDllDirectory(sdk_path.wstring().c_str()); + _extra = reinterpret_cast(ck); + if (ck == 0) { + DWORD ec = GetLastError(); + std::string error; + { + LPWSTR str; + FormatMessageW(FORMAT_MESSAGE_FROM_SYSTEM | FORMAT_MESSAGE_ALLOCATE_BUFFER + | FORMAT_MESSAGE_IGNORE_INSERTS, + nullptr, ec, MAKELANGID(LANG_ENGLISH, SUBLANG_ENGLISH_US), + reinterpret_cast(&str), 0, nullptr); + error = ::streamfx::util::platform::native_to_utf8(std::wstring(str)); + LocalFree(str); + } + D_LOG_WARNING("Failed to add '%'s to the library loader paths with error: %s (Code %" PRIu32 ")", + sdk_path.string().c_str(), error.c_str(), ec); + } +#endif + + std::filesystem::path paths[] = { + ST_LIBRARY_NAME, + std::filesystem::path(sdk_path) / ST_LIBRARY_NAME, + }; + + for (auto path : paths) { + try { + _library = ::streamfx::util::library::load(path); + } catch (std::exception const& ex) { + D_LOG_ERROR("Failed to load '%s' with error: %s", path.string().c_str(), ex.what()); + } catch (...) { + D_LOG_ERROR("Failed to load '%s'.", path.string().c_str()); + } + + if (_library) { + break; + } + } + + if (!_library) { +#ifdef WIN32 + // Remove the DLL directory from the library loader paths. + if (_extra != nullptr) { + RemoveDllDirectory(reinterpret_cast(_extra)); + } +#endif + throw std::runtime_error("Failed to load " ST_LIBRARY_NAME "."); + } + } + + { // Load Symbols + P_NVAR_LOAD_SYMBOL(NvAR_GetVersion); + P_NVAR_LOAD_SYMBOL(NvAR_Create); + P_NVAR_LOAD_SYMBOL(NvAR_Destroy); + P_NVAR_LOAD_SYMBOL(NvAR_Run); + P_NVAR_LOAD_SYMBOL(NvAR_Load); + P_NVAR_LOAD_SYMBOL(NvAR_GetS32); + P_NVAR_LOAD_SYMBOL(NvAR_SetS32); + P_NVAR_LOAD_SYMBOL(NvAR_GetU32); + P_NVAR_LOAD_SYMBOL(NvAR_SetU32); + P_NVAR_LOAD_SYMBOL(NvAR_GetU64); + P_NVAR_LOAD_SYMBOL(NvAR_SetU64); + P_NVAR_LOAD_SYMBOL(NvAR_GetF32); + P_NVAR_LOAD_SYMBOL(NvAR_SetF32); + P_NVAR_LOAD_SYMBOL(NvAR_GetF64); + P_NVAR_LOAD_SYMBOL(NvAR_SetF64); + P_NVAR_LOAD_SYMBOL(NvAR_GetString); + P_NVAR_LOAD_SYMBOL(NvAR_SetString); + P_NVAR_LOAD_SYMBOL(NvAR_GetCudaStream); + P_NVAR_LOAD_SYMBOL(NvAR_SetCudaStream); + P_NVAR_LOAD_SYMBOL(NvAR_GetObject); + P_NVAR_LOAD_SYMBOL(NvAR_SetObject); + P_NVAR_LOAD_SYMBOL(NvAR_GetF32Array); + P_NVAR_LOAD_SYMBOL(NvAR_SetF32Array); + } + + { // Assign proper GPU. + auto cctx = ::streamfx::nvidia::cuda::obs::get()->get_context()->enter(); + NvAR_SetU32(nullptr, P_NVAR_CONFIG "GPU", 0); + } +} + +std::filesystem::path const& streamfx::nvidia::ar::ar::get_model_path() +{ + return _model_path; +} + +std::shared_ptr streamfx::nvidia::ar::ar::get() +{ + static std::weak_ptr instance; + static std::mutex lock; + + std::unique_lock ul(lock); + if (instance.expired()) { + auto hard_instance = std::make_shared(); + instance = hard_instance; + return hard_instance; + } + return instance.lock(); +} diff --git a/source/nvidia/ar/nvidia-ar.hpp b/source/nvidia/ar/nvidia-ar.hpp new file mode 100644 index 0000000..9ec661a --- /dev/null +++ b/source/nvidia/ar/nvidia-ar.hpp @@ -0,0 +1,182 @@ +// Copyright (c) 2021 Michael Fabian Dirks +// +// Permission is hereby granted, free of charge, to any person obtaining a copy +// of this software and associated documentation files (the "Software"), to deal +// in the Software without restriction, including without limitation the rights +// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +// copies of the Software, and to permit persons to whom the Software is +// furnished to do so, subject to the following conditions: +// +// The above copyright notice and this permission notice shall be included in all +// copies or substantial portions of the Software. +// +// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +// SOFTWARE. + +#pragma once +#include +#include "nvidia/cv/nvidia-cv.hpp" + +#define P_NVAR_DEFINE_FUNCTION(name, ...) \ + private: \ + typedef ::streamfx::nvidia::cv::result (*t##name)(__VA_ARGS__); \ + \ + public: \ + t##name name = nullptr; + +#define P_NVAR_INPUT "NvAR_Parameter_Input_" +#define P_NVAR_OUTPUT "NvAR_Parameter_Output_" +#define P_NVAR_CONFIG "NvAR_Parameter_Config_" + +/* + * Config Parameters: + * P_NVAR_CONFIG "BatchSize" + * P_NVAR_CONFIG "UseCudaGraph" + * P_NVAR_CONFIG "CUDAStream" + * P_NVAR_CONFIG "ExpressionCount" + * P_NVAR_CONFIG "FeatureDescription" + * P_NVAR_CONFIG "FocalLength" + * P_NVAR_CONFIG "GPU" + * P_NVAR_CONFIG "Landmarks_Size" + * P_NVAR_CONFIG "LandmarksConfidence_Size" + * P_NVAR_CONFIG "Mode" + * P_NVAR_CONFIG "TRTModelDir" + * P_NVAR_CONFIG "ModelDir" + * P_NVAR_CONFIG "ModelName" + * P_NVAR_CONFIG "NumKeyPoints" + * P_NVAR_CONFIG "ReferencePose" + * P_NVAR_CONFIG "ShapeEigenValueCount" + * P_NVAR_CONFIG "Temporal" + * P_NVAR_CONFIG "TriangleCount" + * P_NVAR_CONFIG "VertexCount" + * + * Input Parameters: + * P_NVAR_INPUT "Image" + * P_NVAR_INPUT "Width" + * P_NVAR_INPUT "Height" + * P_NVAR_INPUT "BoundingBoxes" + * P_NVAR_INPUT "BoundingBoxesConfidence" + * P_NVAR_INPUT "Landmarks" + * + * Output Parameters + * P_NVAR_OUTPUT "BoundingBoxes" + * P_NVAR_OUTPUT "BoundingBoxesConfidence" + * P_NVAR_OUTPUT "ExpressionCoefficients" + * P_NVAR_OUTPUT "FaceMesh" + * P_NVAR_OUTPUT "JointAngles" + * P_NVAR_OUTPUT "KeyPoints" + * P_NVAR_OUTPUT "KeyPoints3D" + * P_NVAR_OUTPUT "KeyPointsConfidence" + * P_NVAR_OUTPUT "Landmarks" + * P_NVAR_OUTPUT "LandmarksConfidence" + * P_NVAR_OUTPUT "Pose" + * P_NVAR_OUTPUT "RenderingParams" + * P_NVAR_OUTPUT "ShapeEigenValues" + */ + +namespace streamfx::nvidia::ar { + typedef const char* feature_t; + typedef const char* parameter_t; + typedef void* object_t; + typedef void* handle_t; + + static constexpr feature_t FEATURE_BODY_DETECTION = "BodyDetection"; + static constexpr feature_t FEATURE_BODY_POSE_ESTIMATION = "BodyPoseEstimation"; + static constexpr feature_t FEATURE_FACE_DETECTION = "FaceDetection"; + static constexpr feature_t FEATURE_FACE_BOX_DETECTION = "FaceBoxDetection"; + static constexpr feature_t FEATURE_FACE_RECONSTRUCTION = "Face3DReconstruction"; + static constexpr feature_t FEATURE_LANDMARK_DETECTION = "LandMarkDetection"; + + template + struct vec2 { + T x; + T y; + }; + + template + struct vec3 : public vec2 { + T z; + }; + + template + struct vec4 : public vec3 { + T w; + }; + + typedef vec2 point_t; + typedef vec4 frustum_t; + typedef vec4 quaternion_t; + typedef vec4 rect_t; + + struct bounds_t { + rect_t* rects; + uint8_t current; + uint8_t maximum; + }; + + struct face_mesh_t { + vec3* vertices; + size_t num_vertices; + vec3 indices; + size_t num_indices; + }; + + struct rendering_params_t { + frustum_t frustum; + quaternion_t rotation; + vec3 translation; + }; + + class ar { + std::shared_ptr<::streamfx::util::library> _library; + std::filesystem::path _model_path; +#ifdef WIN32 + void* _extra; +#endif + + public: + ~ar(); + ar(); + + std::filesystem::path const& get_model_path(); + + public: + P_NVAR_DEFINE_FUNCTION(NvAR_GetVersion, uint32_t* version); + + P_NVAR_DEFINE_FUNCTION(NvAR_Create, feature_t feature_id, handle_t* ptr); + P_NVAR_DEFINE_FUNCTION(NvAR_Destroy, handle_t ptr); + P_NVAR_DEFINE_FUNCTION(NvAR_Run, handle_t ptr); + P_NVAR_DEFINE_FUNCTION(NvAR_Load, handle_t ptr); + + P_NVAR_DEFINE_FUNCTION(NvAR_GetS32, handle_t ptr, parameter_t parameter, int32_t* value); + P_NVAR_DEFINE_FUNCTION(NvAR_SetS32, handle_t ptr, parameter_t parameter, int32_t value); + P_NVAR_DEFINE_FUNCTION(NvAR_GetU32, handle_t ptr, parameter_t parameter, uint32_t* value); + P_NVAR_DEFINE_FUNCTION(NvAR_SetU32, handle_t ptr, parameter_t parameter, uint32_t value); + P_NVAR_DEFINE_FUNCTION(NvAR_GetU64, handle_t ptr, parameter_t parameter, uint64_t* value); + P_NVAR_DEFINE_FUNCTION(NvAR_SetU64, handle_t ptr, parameter_t parameter, uint64_t value); + P_NVAR_DEFINE_FUNCTION(NvAR_GetF32, handle_t ptr, parameter_t parameter, float* value); + P_NVAR_DEFINE_FUNCTION(NvAR_SetF32, handle_t ptr, parameter_t parameter, float value); + P_NVAR_DEFINE_FUNCTION(NvAR_GetF64, handle_t ptr, parameter_t parameter, double* value); + P_NVAR_DEFINE_FUNCTION(NvAR_SetF64, handle_t ptr, parameter_t parameter, double value); + P_NVAR_DEFINE_FUNCTION(NvAR_GetString, handle_t ptr, parameter_t parameter, const char** value); + P_NVAR_DEFINE_FUNCTION(NvAR_SetString, handle_t ptr, parameter_t parameter, const char* value); + P_NVAR_DEFINE_FUNCTION(NvAR_GetCudaStream, handle_t ptr, parameter_t parameter, + ::streamfx::nvidia::cuda::stream_t* value); + P_NVAR_DEFINE_FUNCTION(NvAR_SetCudaStream, handle_t ptr, parameter_t parameter, + ::streamfx::nvidia::cuda::stream_t value); + P_NVAR_DEFINE_FUNCTION(NvAR_GetObject, handle_t ptr, parameter_t parameter, object_t* value, uint32_t size); + P_NVAR_DEFINE_FUNCTION(NvAR_SetObject, handle_t ptr, parameter_t parameter, object_t value, uint32_t size); + P_NVAR_DEFINE_FUNCTION(NvAR_GetF32Array, handle_t ptr, parameter_t parameter, const float** values, + int32_t* size); + P_NVAR_DEFINE_FUNCTION(NvAR_SetF32Array, handle_t ptr, parameter_t parameter, const float* values, + int32_t size); + + public: + static std::shared_ptr<::streamfx::nvidia::ar::ar> get(); + }; +} // namespace streamfx::nvidia::ar