diff --git a/dtool/src/dtoolbase/dtoolbase_cc.h b/dtool/src/dtoolbase/dtoolbase_cc.h index de8baeedab..d19db7f5ef 100644 --- a/dtool/src/dtoolbase/dtoolbase_cc.h +++ b/dtool/src/dtoolbase/dtoolbase_cc.h @@ -49,8 +49,6 @@ // interrogate pass (CPPPARSER isn't defined), this maps to public. #define PUBLISHED __published -#define PHAVE_ATOMIC 1 - typedef int ios_openmode; typedef int ios_fmtflags; typedef int ios_iostate; @@ -112,9 +110,6 @@ typedef std::ios::seekdir ios_seekdir; #define INLINE inline #endif -// Expect that we have access to the header. -#define PHAVE_ATOMIC 1 - // Determine the availability of C++11 features. #if defined(_MSC_VER) && _MSC_VER < 1900 // Visual Studio 2015 #error Microsoft Visual C++ 2015 or later is required to compile Panda3D. diff --git a/dtool/src/dtoolbase/mutexSpinlockImpl.h b/dtool/src/dtoolbase/mutexSpinlockImpl.h index cd858f5551..ca84f58ea9 100644 --- a/dtool/src/dtoolbase/mutexSpinlockImpl.h +++ b/dtool/src/dtoolbase/mutexSpinlockImpl.h @@ -19,9 +19,7 @@ #ifdef MUTEX_SPINLOCK -#ifdef PHAVE_ATOMIC #include -#endif /** * Uses a simple user-space spinlock to implement a mutex. It is usually not diff --git a/dtool/src/dtoolbase/patomic.I b/dtool/src/dtoolbase/patomic.I new file mode 100644 index 0000000000..dbb3934da0 --- /dev/null +++ b/dtool/src/dtoolbase/patomic.I @@ -0,0 +1,267 @@ +/** + * PANDA 3D SOFTWARE + * Copyright (c) Carnegie Mellon University. All rights reserved. + * + * All use of this software is subject to the terms of the revised BSD + * license. You should have received a copy of this license along + * with this source code in a file named "LICENSE." + * + * @file patomic.I + * @author rdb + * @date 2022-01-28 + */ + +/** + * Value initializer. + */ +template +constexpr patomic:: +patomic(T desired) noexcept : _value(desired) { +} + +/** + * Returns true if this is a lock free type (which it always is). + */ +template +ALWAYS_INLINE bool patomic:: +is_lock_free() const noexcept { + return true; +} + +/** + * Returns the stored value. + */ +template +ALWAYS_INLINE T patomic:: +load(std::memory_order order) const noexcept { + return _value; +} + +/** + * Returns the stored value. + */ +template +ALWAYS_INLINE patomic:: +operator T() const noexcept { + return _value; +} + +/** + * Changes the stored value. + */ +template +ALWAYS_INLINE void patomic:: +store(T desired, std::memory_order order) noexcept { + _value = desired; +} + +/** + * Changes the stored value. + */ +template +ALWAYS_INLINE T patomic:: +operator=(T desired) noexcept { + _value = desired; +} + +/** + * Changes the stored value, returning the previous value. + */ +template +ALWAYS_INLINE T patomic:: +exchange(T desired, std::memory_order) noexcept { + T current = _value; + _value = desired; + return current; +} + +/** + * Sets the desired value if the current value is as the first argument. + * If it is not, the current value is written to expected. + */ +template +ALWAYS_INLINE bool patomic:: +compare_exchange_weak(T &expected, T desired, + std::memory_order, std::memory_order) noexcept { + T current = _value; + if (_value == expected) { + _value = desired; + return true; + } else { + expected = current; + return false; + } +} + +/** + * Sets the desired value if the current value is as the first argument. + * If it is not, the current value is written to expected. + */ +template +ALWAYS_INLINE bool patomic:: +compare_exchange_strong(T &expected, T desired, + std::memory_order, std::memory_order) noexcept { + T current = _value; + if (_value == expected) { + _value = desired; + return true; + } else { + expected = current; + return false; + } +} + +/** + * Adds to the stored value, returns the old value. + */ +template +ALWAYS_INLINE T patomic:: +fetch_add(T arg, std::memory_order) noexcept { + T old = _value; + _value += arg; + return old; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +fetch_sub(T arg, std::memory_order) noexcept { + T old = _value; + _value -= arg; + return old; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +fetch_and(T arg, std::memory_order) noexcept { + T old = _value; + _value &= arg; + return old; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +fetch_or(T arg, std::memory_order) noexcept { + T old = _value; + _value |= arg; + return old; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +fetch_xor(T arg, std::memory_order) noexcept { + T old = _value; + _value ^= arg; + return old; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator ++(int) noexcept { + return _value++; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator --(int) noexcept { + return _value--; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator ++() noexcept { + return ++_value; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator --() noexcept { + return --_value; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator +=(T arg) noexcept { + return _value += arg; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator -=(T arg) noexcept { + return _value -= arg; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator &=(T arg) noexcept { + return _value &= arg; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator |=(T arg) noexcept { + return _value |= arg; +} + +/** + * + */ +template +ALWAYS_INLINE T patomic:: +operator ^=(T arg) noexcept { + return _value ^= arg; +} + + +/** + * Sets the flag to true and returns the previous value. + */ +ALWAYS_INLINE bool patomic_flag:: +test_and_set(std::memory_order order) noexcept { + bool value = __internal_flag; + __internal_flag = true; + return value; +} + +/** + * Sets the flag to false. + */ +ALWAYS_INLINE void patomic_flag:: +clear(std::memory_order order) noexcept { + __internal_flag = false; +} diff --git a/dtool/src/dtoolbase/patomic.h b/dtool/src/dtoolbase/patomic.h new file mode 100644 index 0000000000..903391cf3e --- /dev/null +++ b/dtool/src/dtoolbase/patomic.h @@ -0,0 +1,108 @@ +/** + * PANDA 3D SOFTWARE + * Copyright (c) Carnegie Mellon University. All rights reserved. + * + * All use of this software is subject to the terms of the revised BSD + * license. You should have received a copy of this license along + * with this source code in a file named "LICENSE." + * + * @file patomic.h + * @author rdb + * @date 2022-01-28 + */ + +#ifndef PATOMIC_H +#define PATOMIC_H + +#include "dtoolbase.h" +#include "selectThreadImpl.h" + +#include + +#if defined(THREAD_DUMMY_IMPL) || defined(THREAD_SIMPLE_IMPL) + +/** + * Dummy implementation of std::atomic that does not do any atomic operations, + * used when compiling without HAVE_THREADS or with SIMPLE_THREADS. + */ +template +struct patomic { + using value_type = T; + + constexpr patomic() noexcept = default; + constexpr patomic(T desired) noexcept; + + ALWAYS_INLINE patomic(const patomic &) = delete; + ALWAYS_INLINE patomic &operator=(const patomic &) = delete; + + static constexpr bool is_always_lock_free = true; + ALWAYS_INLINE bool is_lock_free() const noexcept; + + ALWAYS_INLINE T load(std::memory_order order = std::memory_order_seq_cst) const noexcept; + ALWAYS_INLINE operator T() const noexcept; + + ALWAYS_INLINE void store(T desired, std::memory_order order = std::memory_order_seq_cst) noexcept; + ALWAYS_INLINE T operator=(T desired) noexcept; + + ALWAYS_INLINE T exchange(T desired, std::memory_order order = std::memory_order_seq_cst) noexcept; + + ALWAYS_INLINE bool compare_exchange_weak(T &expected, T desired, + std::memory_order success = std::memory_order_seq_cst, + std::memory_order failure = std::memory_order_seq_cst) noexcept; + ALWAYS_INLINE bool compare_exchange_strong(T &expected, T desired, + std::memory_order success = std::memory_order_seq_cst, + std::memory_order failure = std::memory_order_seq_cst) noexcept; + + ALWAYS_INLINE T fetch_add(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept; + ALWAYS_INLINE T fetch_sub(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept; + ALWAYS_INLINE T fetch_and(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept; + ALWAYS_INLINE T fetch_or(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept; + ALWAYS_INLINE T fetch_xor(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept; + + ALWAYS_INLINE T operator ++(int) noexcept; + ALWAYS_INLINE T operator --(int) noexcept; + ALWAYS_INLINE T operator ++() noexcept; + ALWAYS_INLINE T operator --() noexcept; + ALWAYS_INLINE T operator +=(T arg) noexcept; + ALWAYS_INLINE T operator -=(T arg) noexcept; + ALWAYS_INLINE T operator &=(T arg) noexcept; + ALWAYS_INLINE T operator |=(T arg) noexcept; + ALWAYS_INLINE T operator ^=(T arg) noexcept; + +private: + T _value; +}; + +/** + * Dummy implementation of std::atomic_flag that does not do any atomic + * operations. + */ +struct EXPCL_DTOOL_DTOOLBASE patomic_flag { + constexpr patomic_flag() noexcept = default; + + patomic_flag(const patomic_flag &) = delete; + patomic_flag &operator=(const patomic_flag &) = delete; + + ALWAYS_INLINE bool test_and_set(std::memory_order order = std::memory_order_seq_cst) noexcept; + ALWAYS_INLINE void clear(std::memory_order order = std::memory_order_seq_cst) noexcept; + + bool __internal_flag = false; +}; + +#define patomic_thread_fence(order) (std::atomic_signal_fence((order))) + +#include "patomic.I" + +#else + +// We're using real threading, so use the real implementation. +template +using patomic = std::atomic; + +typedef std::atomic_flag patomic_flag; + +#define patomic_thread_fence(order) (std::atomic_thread_fence((order))) + +#endif + +#endif diff --git a/dtool/src/dtoolbase/typeHandle.cxx b/dtool/src/dtoolbase/typeHandle.cxx index c72c3f9d99..6b7ee5aaf7 100644 --- a/dtool/src/dtoolbase/typeHandle.cxx +++ b/dtool/src/dtoolbase/typeHandle.cxx @@ -13,7 +13,6 @@ #include "typeHandle.h" #include "typeRegistryNode.h" -#include "atomicAdjust.h" /** * Returns the total allocated memory used by objects of this type, for the @@ -29,7 +28,7 @@ get_memory_usage(MemoryClass memory_class) const { } else { TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr); assert(rnode != nullptr); - return (size_t)AtomicAdjust::get(rnode->_memory_usage[memory_class]); + return rnode->_memory_usage[memory_class].load(std::memory_order_relaxed); } #endif // DO_MEMORY_USAGE return 0; @@ -48,10 +47,8 @@ inc_memory_usage(MemoryClass memory_class, size_t size) { if ((*this) != TypeHandle::none()) { TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr); assert(rnode != nullptr); - AtomicAdjust::add(rnode->_memory_usage[memory_class], (AtomicAdjust::Integer)size); - // cerr << *this << ".inc(" << memory_class << ", " << size << ") -> " << - // rnode->_memory_usage[memory_class] << "\n"; - if (rnode->_memory_usage[memory_class] < 0) { + size_t prev = rnode->_memory_usage[memory_class].fetch_add(size, std::memory_order_relaxed); + if (prev + size < prev) { std::cerr << "Memory usage overflow for type " << rnode->_name << ".\n"; abort(); } @@ -72,10 +69,8 @@ dec_memory_usage(MemoryClass memory_class, size_t size) { if ((*this) != TypeHandle::none()) { TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr); assert(rnode != nullptr); - AtomicAdjust::add(rnode->_memory_usage[memory_class], -(AtomicAdjust::Integer)size); - // cerr << *this << ".dec(" << memory_class << ", " << size << ") -> " << - // rnode->_memory_usage[memory_class] << "\n"; - assert(rnode->_memory_usage[memory_class] >= 0); + size_t prev = rnode->_memory_usage[memory_class].fetch_sub(size, std::memory_order_relaxed); + assert(prev - size <= prev); } #endif // DO_MEMORY_USAGE } @@ -97,8 +92,8 @@ allocate_array(size_t size) { #endif TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr); assert(rnode != nullptr); - AtomicAdjust::add(rnode->_memory_usage[MC_array], (AtomicAdjust::Integer)alloc_size); - if (rnode->_memory_usage[MC_array] < 0) { + size_t prev = rnode->_memory_usage[MC_array].fetch_add(alloc_size, std::memory_order_relaxed); + if (prev + size < prev) { std::cerr << "Memory usage overflow for type " << rnode->_name << ".\n"; abort(); } @@ -124,8 +119,11 @@ reallocate_array(void *old_ptr, size_t size) { TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr); assert(rnode != nullptr); - AtomicAdjust::add(rnode->_memory_usage[MC_array], (AtomicAdjust::Integer)new_size - (AtomicAdjust::Integer)old_size); - assert(rnode->_memory_usage[MC_array] >= 0); + if (new_size > old_size) { + rnode->_memory_usage[MC_array].fetch_add(new_size - old_size, std::memory_order_relaxed); + } else { + rnode->_memory_usage[MC_array].fetch_sub(old_size - new_size, std::memory_order_relaxed); + } } #else void *new_ptr = PANDA_REALLOC_ARRAY(old_ptr, size); @@ -146,8 +144,8 @@ deallocate_array(void *ptr) { if ((*this) != TypeHandle::none()) { TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr); assert(rnode != nullptr); - AtomicAdjust::add(rnode->_memory_usage[MC_array], -(AtomicAdjust::Integer)alloc_size); - assert(rnode->_memory_usage[MC_array] >= 0); + size_t prev = rnode->_memory_usage[MC_array].fetch_sub(alloc_size, std::memory_order_relaxed); + assert(prev - alloc_size <= prev); } #endif // DO_MEMORY_USAGE PANDA_FREE_ARRAY(ptr); diff --git a/dtool/src/dtoolbase/typeRegistryNode.cxx b/dtool/src/dtoolbase/typeRegistryNode.cxx index 19b4629236..291a6ec95a 100644 --- a/dtool/src/dtoolbase/typeRegistryNode.cxx +++ b/dtool/src/dtoolbase/typeRegistryNode.cxx @@ -23,10 +23,9 @@ bool TypeRegistryNode::_paranoid_inheritance = false; */ TypeRegistryNode:: TypeRegistryNode(TypeHandle handle, const std::string &name, TypeHandle &ref) : - _handle(handle), _name(name), _ref(ref) + _handle(handle), _name(name), _ref(ref), _memory_usage{} { clear_subtree(); - memset(_memory_usage, 0, sizeof(_memory_usage)); } /** diff --git a/dtool/src/dtoolbase/typeRegistryNode.h b/dtool/src/dtoolbase/typeRegistryNode.h index 7dd7f387cc..6ff2275103 100644 --- a/dtool/src/dtoolbase/typeRegistryNode.h +++ b/dtool/src/dtoolbase/typeRegistryNode.h @@ -18,6 +18,7 @@ #include "typeHandle.h" #include "numeric_types.h" +#include "patomic.h" #include #include @@ -50,7 +51,7 @@ public: Classes _child_classes; PyObject *_python_type = nullptr; - AtomicAdjust::Integer _memory_usage[TypeHandle::MC_limit]; + patomic _memory_usage[TypeHandle::MC_limit]; static bool _paranoid_inheritance; diff --git a/dtool/src/prc/notify.cxx b/dtool/src/prc/notify.cxx index 2c1b737ddb..2a82f2926d 100644 --- a/dtool/src/prc/notify.cxx +++ b/dtool/src/prc/notify.cxx @@ -18,13 +18,10 @@ #include "configVariableBool.h" #include "filename.h" #include "config_prc.h" +#include "patomic.h" #include -#ifdef PHAVE_ATOMIC -#include -#endif - #ifdef BUILD_IPHONE #include #endif @@ -439,7 +436,7 @@ config_initialized() { "The filename to which to write all the output of notify"); // We use this to ensure that only one thread can initialize the output. - static std::atomic_flag initialized = ATOMIC_FLAG_INIT; + static patomic_flag initialized = ATOMIC_FLAG_INIT; std::string value = notify_output.get_value(); if (!value.empty() && !initialized.test_and_set()) { diff --git a/panda/src/event/asyncFuture.cxx b/panda/src/event/asyncFuture.cxx index 696680bb1a..45bcc0e392 100644 --- a/panda/src/event/asyncFuture.cxx +++ b/panda/src/event/asyncFuture.cxx @@ -39,6 +39,7 @@ AsyncFuture:: if (result_ref != nullptr) { _result_ref.cheat() = nullptr; if (!result_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete _result; } _result = nullptr; diff --git a/panda/src/express/referenceCount.I b/panda/src/express/referenceCount.I index f4d71b1117..0344ed3839 100644 --- a/panda/src/express/referenceCount.I +++ b/panda/src/express/referenceCount.I @@ -27,9 +27,9 @@ TypeHandle RefCountObj::_type_handle; * inheritance. */ INLINE ReferenceCount:: -ReferenceCount() { - _weak_list = nullptr; - _ref_count = 0; +ReferenceCount() : + _weak_list(nullptr), + _ref_count(0) { #ifdef DO_MEMORY_USAGE MemoryUsage::record_pointer(this); #endif @@ -44,9 +44,9 @@ ReferenceCount() { * try. */ INLINE ReferenceCount:: -ReferenceCount(const ReferenceCount &) { - _weak_list = nullptr; - _ref_count = 0; +ReferenceCount(const ReferenceCount &) : + _weak_list(nullptr), + _ref_count(0) { #ifdef DO_MEMORY_USAGE MemoryUsage::record_pointer(this); #endif @@ -69,7 +69,7 @@ operator = (const ReferenceCount &) { // to create an automatic (local variable) instance of a class that derives // from ReferenceCount. Or maybe your headers are out of sync, and you need // to make clean in direct or some higher tree. - nassertv(_ref_count != deleted_ref_count); + nassertv(_ref_count.load(std::memory_order_relaxed) != deleted_ref_count); } /** @@ -78,23 +78,29 @@ operator = (const ReferenceCount &) { ReferenceCount:: ~ReferenceCount() { TAU_PROFILE("ReferenceCount::~ReferenceCount()", " ", TAU_USER); + + // We can safely use relaxed ordering for everything in this destructor, + // since (1) we already issued an acquire barrier before invoking delete, + // and (2) we are the only thread accessing this object at this point. + int ref_count = _ref_count.load(std::memory_order_relaxed); + // If this assertion fails, we're trying to delete an object that was just // deleted. Possibly you used a real pointer instead of a PointerTo at some // point, and the object was deleted when the PointerTo went out of scope. // Maybe you tried to create an automatic (local variable) instance of a // class that derives from ReferenceCount. Or maybe your headers are out of // sync, and you need to make clean in direct or some higher tree. - nassertv(_ref_count != deleted_ref_count); + nassertv(ref_count != deleted_ref_count); // If this assertion fails, we're trying to delete a static object that // still has an outstanding reference count. You should make sure that all // references to your static objects are gone by the time the object itself // destructs. - nassertv(_ref_count <= local_ref_count); + nassertv(ref_count <= local_ref_count); // If this assertion fails, the reference counts are all screwed up // altogether. Maybe some errant code stomped all over memory somewhere. - nassertv(_ref_count >= 0); + nassertv(ref_count >= 0); // If this assertion fails, someone tried to delete this object while its // reference count was still positive. Maybe you tried to point a PointerTo @@ -105,19 +111,20 @@ ReferenceCount:: // Another possibility is you inadvertently omitted a copy constructor for a // ReferenceCount object, and then bitwise copied a dynamically allocated // value--reference count and all--onto a locally allocated one. - nassertv(_ref_count == 0 || _ref_count == local_ref_count); + nassertv(ref_count == 0 || ref_count == local_ref_count); // Tell our weak reference holders that we're going away now. - if (_weak_list != nullptr) { + WeakReferenceList *weak_list = _weak_list.load(std::memory_order_relaxed); + if (weak_list != nullptr) { ((WeakReferenceList *)_weak_list)->mark_deleted(); - _weak_list = nullptr; + _weak_list.store(nullptr, std::memory_order_release); } #ifndef NDEBUG // Ok, all clear to delete. Now set the reference count to // deleted_ref_count, so we'll have a better chance of noticing if we happen // to have a stray pointer to it still out there. - _ref_count = deleted_ref_count; + _ref_count.store(deleted_ref_count, std::memory_order_relaxed); #endif #ifdef DO_MEMORY_USAGE @@ -133,7 +140,7 @@ get_ref_count() const { #ifdef _DEBUG test_ref_count_integrity(); #endif - return (int)AtomicAdjust::get(_ref_count); + return _ref_count.load(std::memory_order_acquire); } /** @@ -154,7 +161,7 @@ ref() const { nassertv(test_ref_count_integrity()); #endif - AtomicAdjust::inc(_ref_count); + _ref_count.fetch_add(1, std::memory_order_relaxed); } /** @@ -184,9 +191,9 @@ unref() const { // If this assertion fails, you tried to unref an object with a zero // reference count. Are you using ref() and unref() directly? Are you sure // you can't use PointerTo's? - nassertr(_ref_count > 0, 0); + nassertr(_ref_count.load(std::memory_order_relaxed) > 0, 0); #endif - return AtomicAdjust::dec(_ref_count); + return _ref_count.fetch_sub(1, std::memory_order_release) != 1; } /** @@ -227,11 +234,11 @@ test_ref_count_nonzero() const { */ INLINE void ReferenceCount:: local_object() { + int prev_count = _ref_count.exchange(local_ref_count, std::memory_order_relaxed); + // If this assertion fails, you didn't call this immediately after creating // a local object. - nassertv(_ref_count == 0); - - _ref_count = local_ref_count; + nassertv(prev_count == 0); } /** @@ -242,7 +249,7 @@ local_object() { */ INLINE bool ReferenceCount:: has_weak_list() const { - return _weak_list != nullptr; + return _weak_list.load(std::memory_order_relaxed) != nullptr; } /** @@ -255,10 +262,10 @@ has_weak_list() const { */ INLINE WeakReferenceList *ReferenceCount:: get_weak_list() const { - if (AtomicAdjust::get_ptr(_weak_list) == nullptr) { + if (_weak_list.load(std::memory_order_relaxed) == nullptr) { ((ReferenceCount *)this)->create_weak_list(); } - return (WeakReferenceList *)AtomicAdjust::get_ptr(_weak_list); + return _weak_list.load(std::memory_order_consume); } /** @@ -273,7 +280,7 @@ weak_ref() { #ifdef _DEBUG nassertr(test_ref_count_integrity(), nullptr); #else - nassertr(_ref_count != deleted_ref_count, nullptr); + nassertr(_ref_count.load(std::memory_order_relaxed) != deleted_ref_count, nullptr); #endif WeakReferenceList *weak_ref = get_weak_list(); weak_ref->ref(); @@ -290,7 +297,7 @@ weak_unref() { #ifdef _DEBUG nassertv(test_ref_count_integrity()); #endif - WeakReferenceList *weak_list = (WeakReferenceList *)_weak_list; + WeakReferenceList *weak_list = _weak_list.load(std::memory_order_consume); nassertv(weak_list != nullptr); bool nonzero = weak_list->unref(); nassertv(nonzero); @@ -307,13 +314,16 @@ ref_if_nonzero() const { #ifdef _DEBUG test_ref_count_integrity(); #endif - AtomicAdjust::Integer ref_count; + int ref_count = _ref_count.load(std::memory_order_relaxed); do { - ref_count = AtomicAdjust::get(_ref_count); if (ref_count <= 0) { return false; } - } while (ref_count != AtomicAdjust::compare_and_exchange(_ref_count, ref_count, ref_count + 1)); + } + while (!_ref_count.compare_exchange_weak(ref_count, ref_count + 1, + std::memory_order_seq_cst, + std::memory_order_relaxed)); + return true; } @@ -321,15 +331,21 @@ ref_if_nonzero() const { * Atomically decreases the reference count of this object if it is one. * Do not use this. This exists only to implement a special case with the * state cache. - * @return false if the reference count was decremented to zero. + * @return false on success, ie. if the reference count was decremented to 0. */ INLINE bool ReferenceCount:: unref_if_one() const { #ifdef _DEBUG nassertr(test_ref_count_integrity(), 0); - nassertr(_ref_count > 0, 0); + nassertr(_ref_count.load(std::memory_order_relaxed) > 0, 0); #endif - return (AtomicAdjust::compare_and_exchange(_ref_count, 1, 0) != 1); + + // Presumably if the ref count becomes 0, someone is about to delete the + // object or something like that, hence the acquire order on success. + int expected = 1; + return !_ref_count.compare_exchange_strong(expected, 0, + std::memory_order_acquire, + std::memory_order_relaxed); } /** @@ -350,6 +366,7 @@ unref_delete(RefCountType *ptr) { if (!ptr->unref()) { // If the reference count has gone to zero, delete the object. + patomic_thread_fence(std::memory_order_acquire); delete ptr; } } diff --git a/panda/src/express/referenceCount.cxx b/panda/src/express/referenceCount.cxx index f446b3be46..0d5f98ea84 100644 --- a/panda/src/express/referenceCount.cxx +++ b/panda/src/express/referenceCount.cxx @@ -23,17 +23,19 @@ TypeHandle ReferenceCount::_type_handle; */ bool ReferenceCount:: do_test_ref_count_integrity() const { + int ref_count = _ref_count.load(std::memory_order_relaxed); + // If this assertion fails, we're trying to delete an object that was just // deleted. Possibly you used a real pointer instead of a PointerTo at some // point, and the object was deleted when the PointerTo went out of scope. // Maybe you tried to create an automatic (local variable) instance of a // class that derives from ReferenceCount. Or maybe your headers are out of // sync, and you need to make clean in direct or some higher tree. - nassertr(_ref_count != deleted_ref_count, false); + nassertr(ref_count != deleted_ref_count, false); // If this assertion fails, the reference counts are all screwed up // altogether. Maybe some errant code stomped all over memory somewhere. - nassertr(_ref_count >= 0, false); + nassertr(ref_count >= 0, false); return true; } @@ -44,7 +46,7 @@ do_test_ref_count_integrity() const { bool ReferenceCount:: do_test_ref_count_nonzero() const { nassertr(do_test_ref_count_integrity(), false); - nassertr(_ref_count > 0, false); + nassertr(_ref_count.load(std::memory_order_relaxed) > 0, false); return true; } @@ -54,11 +56,12 @@ do_test_ref_count_nonzero() const { */ void ReferenceCount:: create_weak_list() { - WeakReferenceList *weak_list = new WeakReferenceList; - void *orig = - AtomicAdjust::compare_and_exchange_ptr(_weak_list, nullptr, weak_list); - if (orig != nullptr) { + WeakReferenceList *new_list = new WeakReferenceList; + WeakReferenceList *old_list = nullptr; + if (!_weak_list.compare_exchange_strong(old_list, new_list, + std::memory_order_release, + std::memory_order_relaxed)) { // Someone else created it first. - delete weak_list; + delete new_list; } } diff --git a/panda/src/express/referenceCount.h b/panda/src/express/referenceCount.h index 09ef7b4501..b57b85d4a3 100644 --- a/panda/src/express/referenceCount.h +++ b/panda/src/express/referenceCount.h @@ -23,6 +23,7 @@ #include "atomicAdjust.h" #include "numeric_types.h" #include "deletedChain.h" +#include "patomic.h" #include @@ -89,8 +90,8 @@ private: deleted_ref_count = -100, }; - mutable AtomicAdjust::Integer _ref_count; - AtomicAdjust::Pointer _weak_list; // WeakReferenceList * + mutable patomic _ref_count; + patomic _weak_list; public: static TypeHandle get_class_type() { diff --git a/panda/src/express/weakPointerToBase.I b/panda/src/express/weakPointerToBase.I index 3b27047da1..11b04dabfe 100644 --- a/panda/src/express/weakPointerToBase.I +++ b/panda/src/express/weakPointerToBase.I @@ -117,6 +117,7 @@ INLINE WeakPointerToBase:: ~WeakPointerToBase() { WeakReferenceList *old_ref = (WeakReferenceList *)_weak_ref; if (old_ref != nullptr && !old_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete old_ref; } } @@ -143,6 +144,7 @@ reassign(To *ptr) { // Now remove the old reference. if (old_ref != nullptr && !old_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete old_ref; } } @@ -183,6 +185,7 @@ reassign(const WeakPointerToBase ©) { // Now remove the old reference. if (old_ref != nullptr && !old_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete old_ref; } } @@ -205,6 +208,7 @@ reassign(WeakPointerToBase &&from) noexcept { // Now delete the old pointer. if (old_ref != nullptr && !old_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete old_ref; } } @@ -233,6 +237,7 @@ reassign(const WeakPointerToBase ©) { // Now remove the old reference. if (old_ref != nullptr && !old_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete old_ref; } } @@ -260,6 +265,7 @@ reassign(WeakPointerToBase &&from) noexcept { // Now delete the old pointer. if (old_ref != nullptr && !old_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete old_ref; } } @@ -627,6 +633,7 @@ clear() { // Now remove the old reference. if (old_ref != nullptr && !old_ref->unref()) { + patomic_thread_fence(std::memory_order_acquire); delete old_ref; } } diff --git a/panda/src/express/weakReferenceList.I b/panda/src/express/weakReferenceList.I index eb6e7b9976..c9368cc51b 100644 --- a/panda/src/express/weakReferenceList.I +++ b/panda/src/express/weakReferenceList.I @@ -18,7 +18,7 @@ */ INLINE void WeakReferenceList:: ref() const { - AtomicAdjust::inc(_count); + _count.fetch_add(1, std::memory_order_relaxed); } /** @@ -30,7 +30,7 @@ ref() const { */ INLINE bool WeakReferenceList:: unref() const { - return AtomicAdjust::dec(_count); + return _count.fetch_sub(1, std::memory_order_release) != 1; } /** @@ -41,5 +41,5 @@ unref() const { */ INLINE bool WeakReferenceList:: was_deleted() const { - return AtomicAdjust::get(_count) < _alive_offset; + return _count.load(std::memory_order_relaxed) < _alive_offset; } diff --git a/panda/src/express/weakReferenceList.cxx b/panda/src/express/weakReferenceList.cxx index ac1363c85a..438a10df1c 100644 --- a/panda/src/express/weakReferenceList.cxx +++ b/panda/src/express/weakReferenceList.cxx @@ -27,7 +27,7 @@ WeakReferenceList() : _count(_alive_offset) { */ WeakReferenceList:: ~WeakReferenceList() { - nassertv(_count == 0); + nassertv(_count.load(std::memory_order_relaxed) == 0); } /** @@ -91,7 +91,7 @@ mark_deleted() { // Decrement the special offset added to the weak pointer count to indicate // that it can be deleted when all the weak references have gone. - AtomicAdjust::Integer result = AtomicAdjust::add(_count, -_alive_offset); + int result = _count.fetch_sub(_alive_offset, std::memory_order_relaxed) - _alive_offset; _lock.unlock(); if (result == 0) { // There are no weak references remaining either, so delete this. diff --git a/panda/src/express/weakReferenceList.h b/panda/src/express/weakReferenceList.h index 30ab68edae..69874741c5 100644 --- a/panda/src/express/weakReferenceList.h +++ b/panda/src/express/weakReferenceList.h @@ -17,6 +17,7 @@ #include "pandabase.h" #include "pmap.h" #include "mutexImpl.h" +#include "patomic.h" class WeakPointerCallback; @@ -53,8 +54,8 @@ private: // This has a very large number added to it if the object is still alive. // It could be 1, but having it be a large number makes it easy to check // whether the object has been deleted or not. - static const AtomicAdjust::Integer _alive_offset = (1 << 30); - mutable AtomicAdjust::Integer _count; + static const int _alive_offset = (1 << 30); + mutable patomic _count; friend class ReferenceCount; }; diff --git a/panda/src/pgui/pgScrollFrame.h b/panda/src/pgui/pgScrollFrame.h index bb8b3a3af3..5a7c19386d 100644 --- a/panda/src/pgui/pgScrollFrame.h +++ b/panda/src/pgui/pgScrollFrame.h @@ -19,10 +19,7 @@ #include "pgVirtualFrame.h" #include "pgSliderBarNotify.h" #include "pgSliderBar.h" - -#ifdef PHAVE_ATOMIC -#include -#endif +#include "patomic.h" /** * This is a special kind of frame that pretends to be much larger than it @@ -96,7 +93,7 @@ private: private: bool _needs_remanage; bool _needs_recompute_clip; - std::atomic_flag _canvas_computed; + patomic_flag _canvas_computed; bool _has_virtual_frame; LVecBase4 _virtual_frame; diff --git a/panda/src/physics/physicalNode.cxx b/panda/src/physics/physicalNode.cxx index d6103101e6..dee169bfe5 100644 --- a/panda/src/physics/physicalNode.cxx +++ b/panda/src/physics/physicalNode.cxx @@ -13,13 +13,10 @@ #include "physicalNode.h" #include "physicsManager.h" - -#ifdef PHAVE_ATOMIC -#include -#endif +#include "patomic.h" // static stuff. -static std::atomic_flag warned_copy_physical_node = ATOMIC_FLAG_INIT; +static patomic_flag warned_copy_physical_node = ATOMIC_FLAG_INIT; TypeHandle PhysicalNode::_type_handle;