obs-tools: Add work around for property memory corruption

The current implementation of obs_properties_remove_by_name corrupts the obs_properties_t object whenever it is called on the first or last property in the list. This leads to rapid unscheduled disassembly, and therefore must be fixed in order for this function to be used.

Fixed by upstream PR https://github.com/obsproject/obs-studio/pull/2257.

source/obs/obs-tools.cpp

@@ -22,6 +22,16 @@
 #include <stdexcept>
 #include "plugin.hpp"
 
+// OBS
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4201)
+#endif
+#include <obs-properties.h>
+#ifdef _MSC_VER
+#pragma warning(pop)
+#endif
+
 struct scs_searchdata {
 	obs_source_t*                 source;
 	bool                          found = false;
@@ -81,3 +91,92 @@ bool obs::tools::scene_contains_source(obs_scene_t* scene, obs_source_t* source)
 	obs_scene_enum_items(scene, scs_enum_items_cb, &sd);
 	return sd.found;
 }
+
+extern "C" {
+struct _hack_obs_properties;
+
+struct _hack_obs_property {
+	char*                  name;
+	char*                  desc;
+	char*                  long_desc;
+	void*                  priv;
+	enum obs_property_type type;
+	bool                   visible;
+	bool                   enabled;
+
+	struct _hack_obs_properties* parent;
+
+	obs_property_modified_t  modified;
+	obs_property_modified2_t modified2;
+
+	struct _hack_obs_property* next;
+};
+
+struct _hack_obs_properties {
+	void* param;
+	void (*destroy)(void* param);
+	uint32_t flags;
+
+	struct _hack_obs_property*  first_property;
+	struct _hack_obs_property** last;
+	struct _hack_obs_property*  parent;
+};
+}
+
+bool obs::tools::obs_properties_remove_by_name(obs_properties_t* props, const char* name)
+{
+	// Due to a bug in obs_properties_remove_by_name, calling it on the first or last element of a group corrupts the
+	// obs_properties_t's first and last pointers, which now point at nonsense.
+	//
+	// There are two ways to work around this issue for now:
+	// 1. Add some invisible properties to the beginning and end of the list, ensuring that you never hit the first or
+	//    last element with a obs_properties_remove_by_name.
+	// 2. Manually adjust the pointers using a dirty hack like in gs::mipmapper.
+	// I've opted for the 2nd way, at it is way simpler to implement.
+
+	// Assume that this is fixed in libobs 24.0.7 or newer.
+	if (obs_get_version() >= MAKE_SEMANTIC_VERSION(24, 0, 7)) {
+		::obs_properties_remove_by_name(props, name);
+		return true;
+	}
+
+	auto rprops = reinterpret_cast<_hack_obs_properties*>(props);
+	auto rprop  = reinterpret_cast<_hack_obs_property*>(obs_properties_get(props, name));
+
+	for (_hack_obs_property *el_prev = rprops->first_property, *el_cur = el_prev; el_cur != nullptr;
+		 el_prev = el_cur, el_cur = el_cur->next) {
+		if (strcmp(el_cur->name, name) == 0) {
+			// Store some information.
+			_hack_obs_property* next     = el_cur->next;
+			bool                is_first = (rprops->first_property == el_cur);
+			bool                is_last  = (rprops->last == &el_cur->next);
+			bool                is_solo  = (el_cur == el_prev);
+
+			// Call the real one which fixes the element pointer and deallocates the element.
+			::obs_properties_remove_by_name(props, name);
+
+			// Fix up the memory pointers after the element was deleted.
+			if (is_last) {
+				if (is_solo) {
+					rprops->last = &rprops->first_property;
+				} else {
+					rprops->last = &el_prev->next;
+				}
+			}
+			if (is_first) {
+				rprops->first_property = next;
+			}
+
+			// Finally break out as we no longer have to process the properties list.
+			return true;
+		}
+
+		if (el_cur->type == OBS_PROPERTY_GROUP) {
+			if (obs::tools::obs_properties_remove_by_name(
+					obs_property_group_content(reinterpret_cast<obs_property_t*>(el_cur)), name))
+				return true;
+		}
+	}
+
+	return false;
+}

source/obs/obs-tools.hpp

@@ -37,7 +37,9 @@
 namespace obs {
 	namespace tools {
 		bool scene_contains_source(obs_scene_t* scene, obs_source_t* source);
-	}
+
+		bool obs_properties_remove_by_name(obs_properties_t* props, const char* name);
+	} // namespace tools
 } // namespace obs
 
 #endif