dtoolbase: Introduce patomic<> as replacement for AtomicAdjust

This typedefs to std::atomic<> when building with true threading, and uses a dummy implementation without.

This lets us use the full range of atomic operations offered by C++11, including explicit specification of memory fences.  Using barriers lets the compiler generate more optimal code since currently we are using the quite strict sequential-consistent memory ordering for all operations.  ReferenceCount has been changed to use the correct barriers (I hope).  This may especially make a difference on weak ordering systems such as ARM.

Over time we should gradually replace the use of AtomicAdjust with the new patomic file.

dtool/src/dtoolbase/dtoolbase_cc.h

@@ -49,8 +49,6 @@
 // interrogate pass (CPPPARSER isn't defined), this maps to public.
 #define PUBLISHED __published
 
-#define PHAVE_ATOMIC 1
-
 typedef int ios_openmode;
 typedef int ios_fmtflags;
 typedef int ios_iostate;
@@ -112,9 +110,6 @@ typedef std::ios::seekdir ios_seekdir;
 #define INLINE inline
 #endif
 
-// Expect that we have access to the <atomic> header.
-#define PHAVE_ATOMIC 1
-
 // Determine the availability of C++11 features.
 #if defined(_MSC_VER) && _MSC_VER < 1900 // Visual Studio 2015
 #error Microsoft Visual C++ 2015 or later is required to compile Panda3D.

dtool/src/dtoolbase/mutexSpinlockImpl.h

@@ -19,9 +19,7 @@
 
 #ifdef MUTEX_SPINLOCK
 
-#ifdef PHAVE_ATOMIC
 #include <atomic>
-#endif
 
 /**
  * Uses a simple user-space spinlock to implement a mutex.  It is usually not

dtool/src/dtoolbase/patomic.I

@@ -0,0 +1,267 @@
+/**
+ * PANDA 3D SOFTWARE
+ * Copyright (c) Carnegie Mellon University.  All rights reserved.
+ *
+ * All use of this software is subject to the terms of the revised BSD
+ * license.  You should have received a copy of this license along
+ * with this source code in a file named "LICENSE."
+ *
+ * @file patomic.I
+ * @author rdb
+ * @date 2022-01-28
+ */
+
+/**
+ * Value initializer.
+ */
+template<class T>
+constexpr patomic<T>::
+patomic(T desired) noexcept : _value(desired) {
+}
+
+/**
+ * Returns true if this is a lock free type (which it always is).
+ */
+template<class T>
+ALWAYS_INLINE bool patomic<T>::
+is_lock_free() const noexcept {
+  return true;
+}
+
+/**
+ * Returns the stored value.
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+load(std::memory_order order) const noexcept {
+  return _value;
+}
+
+/**
+ * Returns the stored value.
+ */
+template<class T>
+ALWAYS_INLINE patomic<T>::
+operator T() const noexcept {
+  return _value;
+}
+
+/**
+ * Changes the stored value.
+ */
+template<class T>
+ALWAYS_INLINE void patomic<T>::
+store(T desired, std::memory_order order) noexcept {
+  _value = desired;
+}
+
+/**
+ * Changes the stored value.
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator=(T desired) noexcept {
+  _value = desired;
+}
+
+/**
+ * Changes the stored value, returning the previous value.
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+exchange(T desired, std::memory_order) noexcept {
+  T current = _value;
+  _value = desired;
+  return current;
+}
+
+/**
+ * Sets the desired value if the current value is as the first argument.
+ * If it is not, the current value is written to expected.
+ */
+template<class T>
+ALWAYS_INLINE bool patomic<T>::
+compare_exchange_weak(T &expected, T desired,
+                      std::memory_order, std::memory_order) noexcept {
+  T current = _value;
+  if (_value == expected) {
+    _value = desired;
+    return true;
+  } else {
+    expected = current;
+    return false;
+  }
+}
+
+/**
+ * Sets the desired value if the current value is as the first argument.
+ * If it is not, the current value is written to expected.
+ */
+template<class T>
+ALWAYS_INLINE bool patomic<T>::
+compare_exchange_strong(T &expected, T desired,
+                        std::memory_order, std::memory_order) noexcept {
+  T current = _value;
+  if (_value == expected) {
+    _value = desired;
+    return true;
+  } else {
+    expected = current;
+    return false;
+  }
+}
+
+/**
+ * Adds to the stored value, returns the old value.
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+fetch_add(T arg, std::memory_order) noexcept {
+  T old = _value;
+  _value += arg;
+  return old;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+fetch_sub(T arg, std::memory_order) noexcept {
+  T old = _value;
+  _value -= arg;
+  return old;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+fetch_and(T arg, std::memory_order) noexcept {
+  T old = _value;
+  _value &= arg;
+  return old;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+fetch_or(T arg, std::memory_order) noexcept {
+  T old = _value;
+  _value |= arg;
+  return old;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+fetch_xor(T arg, std::memory_order) noexcept {
+  T old = _value;
+  _value ^= arg;
+  return old;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator ++(int) noexcept {
+  return _value++;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator --(int) noexcept {
+  return _value--;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator ++() noexcept {
+  return ++_value;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator --() noexcept {
+  return --_value;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator +=(T arg) noexcept {
+  return _value += arg;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator -=(T arg) noexcept {
+  return _value -= arg;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator &=(T arg) noexcept {
+  return _value &= arg;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator |=(T arg) noexcept {
+  return _value |= arg;
+}
+
+/**
+ *
+ */
+template<class T>
+ALWAYS_INLINE T patomic<T>::
+operator ^=(T arg) noexcept {
+  return _value ^= arg;
+}
+
+
+/**
+ * Sets the flag to true and returns the previous value.
+ */
+ALWAYS_INLINE bool patomic_flag::
+test_and_set(std::memory_order order) noexcept {
+  bool value = __internal_flag;
+  __internal_flag = true;
+  return value;
+}
+
+/**
+ * Sets the flag to false.
+ */
+ALWAYS_INLINE void patomic_flag::
+clear(std::memory_order order) noexcept {
+  __internal_flag = false;
+}

dtool/src/dtoolbase/patomic.h

@@ -0,0 +1,108 @@
+/**
+ * PANDA 3D SOFTWARE
+ * Copyright (c) Carnegie Mellon University.  All rights reserved.
+ *
+ * All use of this software is subject to the terms of the revised BSD
+ * license.  You should have received a copy of this license along
+ * with this source code in a file named "LICENSE."
+ *
+ * @file patomic.h
+ * @author rdb
+ * @date 2022-01-28
+ */
+
+#ifndef PATOMIC_H
+#define PATOMIC_H
+
+#include "dtoolbase.h"
+#include "selectThreadImpl.h"
+
+#include <atomic>
+
+#if defined(THREAD_DUMMY_IMPL) || defined(THREAD_SIMPLE_IMPL)
+
+/**
+ * Dummy implementation of std::atomic that does not do any atomic operations,
+ * used when compiling without HAVE_THREADS or with SIMPLE_THREADS.
+ */
+template<class T>
+struct patomic {
+  using value_type = T;
+
+  constexpr patomic() noexcept = default;
+  constexpr patomic(T desired) noexcept;
+
+  ALWAYS_INLINE patomic(const patomic &) = delete;
+  ALWAYS_INLINE patomic &operator=(const patomic &) = delete;
+
+  static constexpr bool is_always_lock_free = true;
+  ALWAYS_INLINE bool is_lock_free() const noexcept;
+
+  ALWAYS_INLINE T load(std::memory_order order = std::memory_order_seq_cst) const noexcept;
+  ALWAYS_INLINE operator T() const noexcept;
+
+  ALWAYS_INLINE void store(T desired, std::memory_order order = std::memory_order_seq_cst) noexcept;
+  ALWAYS_INLINE T operator=(T desired) noexcept;
+
+  ALWAYS_INLINE T exchange(T desired, std::memory_order order = std::memory_order_seq_cst) noexcept;
+
+  ALWAYS_INLINE bool compare_exchange_weak(T &expected, T desired,
+                                           std::memory_order success = std::memory_order_seq_cst,
+                                           std::memory_order failure = std::memory_order_seq_cst) noexcept;
+  ALWAYS_INLINE bool compare_exchange_strong(T &expected, T desired,
+                                             std::memory_order success = std::memory_order_seq_cst,
+                                             std::memory_order failure = std::memory_order_seq_cst) noexcept;
+
+  ALWAYS_INLINE T fetch_add(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept;
+  ALWAYS_INLINE T fetch_sub(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept;
+  ALWAYS_INLINE T fetch_and(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept;
+  ALWAYS_INLINE T fetch_or(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept;
+  ALWAYS_INLINE T fetch_xor(T arg, std::memory_order order = std::memory_order_seq_cst) noexcept;
+
+  ALWAYS_INLINE T operator ++(int) noexcept;
+  ALWAYS_INLINE T operator --(int) noexcept;
+  ALWAYS_INLINE T operator ++() noexcept;
+  ALWAYS_INLINE T operator --() noexcept;
+  ALWAYS_INLINE T operator +=(T arg) noexcept;
+  ALWAYS_INLINE T operator -=(T arg) noexcept;
+  ALWAYS_INLINE T operator &=(T arg) noexcept;
+  ALWAYS_INLINE T operator |=(T arg) noexcept;
+  ALWAYS_INLINE T operator ^=(T arg) noexcept;
+
+private:
+  T _value;
+};
+
+/**
+ * Dummy implementation of std::atomic_flag that does not do any atomic
+ * operations.
+ */
+struct EXPCL_DTOOL_DTOOLBASE patomic_flag {
+  constexpr patomic_flag() noexcept = default;
+
+  patomic_flag(const patomic_flag &) = delete;
+  patomic_flag &operator=(const patomic_flag &) = delete;
+
+  ALWAYS_INLINE bool test_and_set(std::memory_order order = std::memory_order_seq_cst) noexcept;
+  ALWAYS_INLINE void clear(std::memory_order order = std::memory_order_seq_cst) noexcept;
+
+  bool __internal_flag = false;
+};
+
+#define patomic_thread_fence(order) (std::atomic_signal_fence((order)))
+
+#include "patomic.I"
+
+#else
+
+// We're using real threading, so use the real implementation.
+template<class T>
+using patomic = std::atomic<T>;
+
+typedef std::atomic_flag patomic_flag;
+
+#define patomic_thread_fence(order) (std::atomic_thread_fence((order)))
+
+#endif
+
+#endif

dtool/src/dtoolbase/typeHandle.cxx

@@ -13,7 +13,6 @@
 
 #include "typeHandle.h"
 #include "typeRegistryNode.h"
-#include "atomicAdjust.h"
 
 /**
  * Returns the total allocated memory used by objects of this type, for the
@@ -29,7 +28,7 @@ get_memory_usage(MemoryClass memory_class) const {
   } else {
     TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr);
     assert(rnode != nullptr);
-    return (size_t)AtomicAdjust::get(rnode->_memory_usage[memory_class]);
+    return rnode->_memory_usage[memory_class].load(std::memory_order_relaxed);
   }
 #endif  // DO_MEMORY_USAGE
   return 0;
@@ -48,10 +47,8 @@ inc_memory_usage(MemoryClass memory_class, size_t size) {
   if ((*this) != TypeHandle::none()) {
     TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr);
     assert(rnode != nullptr);
-    AtomicAdjust::add(rnode->_memory_usage[memory_class], (AtomicAdjust::Integer)size);
-    // cerr << *this << ".inc(" << memory_class << ", " << size << ") -> " <<
-    // rnode->_memory_usage[memory_class] << "\n";
-    if (rnode->_memory_usage[memory_class] < 0) {
+    size_t prev = rnode->_memory_usage[memory_class].fetch_add(size, std::memory_order_relaxed);
+    if (prev + size < prev) {
       std::cerr << "Memory usage overflow for type " << rnode->_name << ".\n";
       abort();
     }
@@ -72,10 +69,8 @@ dec_memory_usage(MemoryClass memory_class, size_t size) {
   if ((*this) != TypeHandle::none()) {
     TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr);
     assert(rnode != nullptr);
-    AtomicAdjust::add(rnode->_memory_usage[memory_class], -(AtomicAdjust::Integer)size);
-    // cerr << *this << ".dec(" << memory_class << ", " << size << ") -> " <<
-    // rnode->_memory_usage[memory_class] << "\n";
-    assert(rnode->_memory_usage[memory_class] >= 0);
+    size_t prev = rnode->_memory_usage[memory_class].fetch_sub(size, std::memory_order_relaxed);
+    assert(prev - size <= prev);
   }
 #endif  // DO_MEMORY_USAGE
 }
@@ -97,8 +92,8 @@ allocate_array(size_t size) {
 #endif
     TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr);
     assert(rnode != nullptr);
-    AtomicAdjust::add(rnode->_memory_usage[MC_array], (AtomicAdjust::Integer)alloc_size);
-    if (rnode->_memory_usage[MC_array] < 0) {
+    size_t prev = rnode->_memory_usage[MC_array].fetch_add(alloc_size, std::memory_order_relaxed);
+    if (prev + size < prev) {
       std::cerr << "Memory usage overflow for type " << rnode->_name << ".\n";
       abort();
     }
@@ -124,8 +119,11 @@ reallocate_array(void *old_ptr, size_t size) {
 
     TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr);
     assert(rnode != nullptr);
-    AtomicAdjust::add(rnode->_memory_usage[MC_array], (AtomicAdjust::Integer)new_size - (AtomicAdjust::Integer)old_size);
-    assert(rnode->_memory_usage[MC_array] >= 0);
+    if (new_size > old_size) {
+      rnode->_memory_usage[MC_array].fetch_add(new_size - old_size, std::memory_order_relaxed);
+    } else {
+      rnode->_memory_usage[MC_array].fetch_sub(old_size - new_size, std::memory_order_relaxed);
+    }
   }
 #else
   void *new_ptr = PANDA_REALLOC_ARRAY(old_ptr, size);
@@ -146,8 +144,8 @@ deallocate_array(void *ptr) {
   if ((*this) != TypeHandle::none()) {
     TypeRegistryNode *rnode = TypeRegistry::ptr()->look_up(*this, nullptr);
     assert(rnode != nullptr);
-    AtomicAdjust::add(rnode->_memory_usage[MC_array], -(AtomicAdjust::Integer)alloc_size);
-    assert(rnode->_memory_usage[MC_array] >= 0);
+    size_t prev = rnode->_memory_usage[MC_array].fetch_sub(alloc_size, std::memory_order_relaxed);
+    assert(prev - alloc_size <= prev);
   }
 #endif  // DO_MEMORY_USAGE
   PANDA_FREE_ARRAY(ptr);

dtool/src/dtoolbase/typeRegistryNode.cxx

@@ -23,10 +23,9 @@ bool TypeRegistryNode::_paranoid_inheritance = false;
  */
 TypeRegistryNode::
 TypeRegistryNode(TypeHandle handle, const std::string &name, TypeHandle &ref) :
-  _handle(handle), _name(name), _ref(ref)
+  _handle(handle), _name(name), _ref(ref), _memory_usage{}
 {
   clear_subtree();
-  memset(_memory_usage, 0, sizeof(_memory_usage));
 }
 
 /**

dtool/src/dtoolbase/typeRegistryNode.h

@@ -18,6 +18,7 @@
 
 #include "typeHandle.h"
 #include "numeric_types.h"
+#include "patomic.h"
 
 #include <assert.h>
 #include <vector>
@@ -50,7 +51,7 @@ public:
   Classes _child_classes;
   PyObject *_python_type = nullptr;
 
-  AtomicAdjust::Integer _memory_usage[TypeHandle::MC_limit];
+  patomic<size_t> _memory_usage[TypeHandle::MC_limit];
 
   static bool _paranoid_inheritance;
 

dtool/src/prc/notify.cxx

@@ -18,13 +18,10 @@
 #include "configVariableBool.h"
 #include "filename.h"
 #include "config_prc.h"
+#include "patomic.h"
 
 #include <ctype.h>
 
-#ifdef PHAVE_ATOMIC
-#include <atomic>
-#endif
-
 #ifdef BUILD_IPHONE
 #include <fcntl.h>
 #endif
@@ -439,7 +436,7 @@ config_initialized() {
        "The filename to which to write all the output of notify");
 
     // We use this to ensure that only one thread can initialize the output.
-    static std::atomic_flag initialized = ATOMIC_FLAG_INIT;
+    static patomic_flag initialized = ATOMIC_FLAG_INIT;
 
     std::string value = notify_output.get_value();
     if (!value.empty() && !initialized.test_and_set()) {

panda/src/event/asyncFuture.cxx

@@ -39,6 +39,7 @@ AsyncFuture::
   if (result_ref != nullptr) {
     _result_ref.cheat() = nullptr;
     if (!result_ref->unref()) {
+      patomic_thread_fence(std::memory_order_acquire);
       delete _result;
     }
     _result = nullptr;

panda/src/express/referenceCount.I

@@ -27,9 +27,9 @@ TypeHandle RefCountObj<Base>::_type_handle;
  * inheritance.
  */
 INLINE ReferenceCount::
-ReferenceCount() {
-  _weak_list = nullptr;
-  _ref_count = 0;
+ReferenceCount() :
+  _weak_list(nullptr),
+  _ref_count(0) {
 #ifdef DO_MEMORY_USAGE
   MemoryUsage::record_pointer(this);
 #endif
@@ -44,9 +44,9 @@ ReferenceCount() {
  * try.
  */
 INLINE ReferenceCount::
-ReferenceCount(const ReferenceCount &) {
-  _weak_list = nullptr;
-  _ref_count = 0;
+ReferenceCount(const ReferenceCount &) :
+  _weak_list(nullptr),
+  _ref_count(0) {
 #ifdef DO_MEMORY_USAGE
   MemoryUsage::record_pointer(this);
 #endif
@@ -69,7 +69,7 @@ operator = (const ReferenceCount &) {
   // to create an automatic (local variable) instance of a class that derives
   // from ReferenceCount.  Or maybe your headers are out of sync, and you need
   // to make clean in direct or some higher tree.
-  nassertv(_ref_count != deleted_ref_count);
+  nassertv(_ref_count.load(std::memory_order_relaxed) != deleted_ref_count);
 }
 
 /**
@@ -78,23 +78,29 @@ operator = (const ReferenceCount &) {
 ReferenceCount::
 ~ReferenceCount() {
   TAU_PROFILE("ReferenceCount::~ReferenceCount()", " ", TAU_USER);
+
+  // We can safely use relaxed ordering for everything in this destructor,
+  // since (1) we already issued an acquire barrier before invoking delete,
+  // and (2) we are the only thread accessing this object at this point.
+  int ref_count = _ref_count.load(std::memory_order_relaxed);
+
   // If this assertion fails, we're trying to delete an object that was just
   // deleted.  Possibly you used a real pointer instead of a PointerTo at some
   // point, and the object was deleted when the PointerTo went out of scope.
   // Maybe you tried to create an automatic (local variable) instance of a
   // class that derives from ReferenceCount.  Or maybe your headers are out of
   // sync, and you need to make clean in direct or some higher tree.
-  nassertv(_ref_count != deleted_ref_count);
+  nassertv(ref_count != deleted_ref_count);
 
   // If this assertion fails, we're trying to delete a static object that
   // still has an outstanding reference count.  You should make sure that all
   // references to your static objects are gone by the time the object itself
   // destructs.
-  nassertv(_ref_count <= local_ref_count);
+  nassertv(ref_count <= local_ref_count);
 
   // If this assertion fails, the reference counts are all screwed up
   // altogether.  Maybe some errant code stomped all over memory somewhere.
-  nassertv(_ref_count >= 0);
+  nassertv(ref_count >= 0);
 
   // If this assertion fails, someone tried to delete this object while its
   // reference count was still positive.  Maybe you tried to point a PointerTo
@@ -105,19 +111,20 @@ ReferenceCount::
   // Another possibility is you inadvertently omitted a copy constructor for a
   // ReferenceCount object, and then bitwise copied a dynamically allocated
   // value--reference count and all--onto a locally allocated one.
-  nassertv(_ref_count == 0 || _ref_count == local_ref_count);
+  nassertv(ref_count == 0 || ref_count == local_ref_count);
 
   // Tell our weak reference holders that we're going away now.
-  if (_weak_list != nullptr) {
+  WeakReferenceList *weak_list = _weak_list.load(std::memory_order_relaxed);
+  if (weak_list != nullptr) {
     ((WeakReferenceList *)_weak_list)->mark_deleted();
-    _weak_list = nullptr;
+    _weak_list.store(nullptr, std::memory_order_release);
   }
 
 #ifndef NDEBUG
   // Ok, all clear to delete.  Now set the reference count to
   // deleted_ref_count, so we'll have a better chance of noticing if we happen
   // to have a stray pointer to it still out there.
-  _ref_count = deleted_ref_count;
+  _ref_count.store(deleted_ref_count, std::memory_order_relaxed);
 #endif
 
 #ifdef DO_MEMORY_USAGE
@@ -133,7 +140,7 @@ get_ref_count() const {
 #ifdef _DEBUG
   test_ref_count_integrity();
 #endif
-  return (int)AtomicAdjust::get(_ref_count);
+  return _ref_count.load(std::memory_order_acquire);
 }
 
 /**
@@ -154,7 +161,7 @@ ref() const {
   nassertv(test_ref_count_integrity());
 #endif
 
-  AtomicAdjust::inc(_ref_count);
+  _ref_count.fetch_add(1, std::memory_order_relaxed);
 }
 
 /**
@@ -184,9 +191,9 @@ unref() const {
   // If this assertion fails, you tried to unref an object with a zero
   // reference count.  Are you using ref() and unref() directly?  Are you sure
   // you can't use PointerTo's?
-  nassertr(_ref_count > 0, 0);
+  nassertr(_ref_count.load(std::memory_order_relaxed) > 0, 0);
 #endif
-  return AtomicAdjust::dec(_ref_count);
+  return _ref_count.fetch_sub(1, std::memory_order_release) != 1;
 }
 
 /**
@@ -227,11 +234,11 @@ test_ref_count_nonzero() const {
  */
 INLINE void ReferenceCount::
 local_object() {
+  int prev_count = _ref_count.exchange(local_ref_count, std::memory_order_relaxed);
+
   // If this assertion fails, you didn't call this immediately after creating
   // a local object.
-  nassertv(_ref_count == 0);
-
-  _ref_count = local_ref_count;
+  nassertv(prev_count == 0);
 }
 
 /**
@@ -242,7 +249,7 @@ local_object() {
  */
 INLINE bool ReferenceCount::
 has_weak_list() const {
-  return _weak_list != nullptr;
+  return _weak_list.load(std::memory_order_relaxed) != nullptr;
 }
 
 /**
@@ -255,10 +262,10 @@ has_weak_list() const {
  */
 INLINE WeakReferenceList *ReferenceCount::
 get_weak_list() const {
-  if (AtomicAdjust::get_ptr(_weak_list) == nullptr) {
+  if (_weak_list.load(std::memory_order_relaxed) == nullptr) {
     ((ReferenceCount *)this)->create_weak_list();
   }
-  return (WeakReferenceList *)AtomicAdjust::get_ptr(_weak_list);
+  return _weak_list.load(std::memory_order_consume);
 }
 
 /**
@@ -273,7 +280,7 @@ weak_ref() {
 #ifdef _DEBUG
   nassertr(test_ref_count_integrity(), nullptr);
 #else
-  nassertr(_ref_count != deleted_ref_count, nullptr);
+  nassertr(_ref_count.load(std::memory_order_relaxed) != deleted_ref_count, nullptr);
 #endif
   WeakReferenceList *weak_ref = get_weak_list();
   weak_ref->ref();
@@ -290,7 +297,7 @@ weak_unref() {
 #ifdef _DEBUG
   nassertv(test_ref_count_integrity());
 #endif
-  WeakReferenceList *weak_list = (WeakReferenceList *)_weak_list;
+  WeakReferenceList *weak_list = _weak_list.load(std::memory_order_consume);
   nassertv(weak_list != nullptr);
   bool nonzero = weak_list->unref();
   nassertv(nonzero);
@@ -307,13 +314,16 @@ ref_if_nonzero() const {
 #ifdef _DEBUG
   test_ref_count_integrity();
 #endif
-  AtomicAdjust::Integer ref_count;
+  int ref_count = _ref_count.load(std::memory_order_relaxed);
   do {
-    ref_count = AtomicAdjust::get(_ref_count);
     if (ref_count <= 0) {
       return false;
     }
-  } while (ref_count != AtomicAdjust::compare_and_exchange(_ref_count, ref_count, ref_count + 1));
+  }
+  while (!_ref_count.compare_exchange_weak(ref_count, ref_count + 1,
+                                           std::memory_order_seq_cst,
+                                           std::memory_order_relaxed));
+
   return true;
 }
 
@@ -321,15 +331,21 @@ ref_if_nonzero() const {
  * Atomically decreases the reference count of this object if it is one.
  * Do not use this.  This exists only to implement a special case with the
  * state cache.
- * @return false if the reference count was decremented to zero.
+ * @return false on success, ie. if the reference count was decremented to 0.
  */
 INLINE bool ReferenceCount::
 unref_if_one() const {
 #ifdef _DEBUG
   nassertr(test_ref_count_integrity(), 0);
-  nassertr(_ref_count > 0, 0);
+  nassertr(_ref_count.load(std::memory_order_relaxed) > 0, 0);
 #endif
-  return (AtomicAdjust::compare_and_exchange(_ref_count, 1, 0) != 1);
+
+  // Presumably if the ref count becomes 0, someone is about to delete the
+  // object or something like that, hence the acquire order on success.
+  int expected = 1;
+  return !_ref_count.compare_exchange_strong(expected, 0,
+                                             std::memory_order_acquire,
+                                             std::memory_order_relaxed);
 }
 
 /**
@@ -350,6 +366,7 @@ unref_delete(RefCountType *ptr) {
 
   if (!ptr->unref()) {
     // If the reference count has gone to zero, delete the object.
+    patomic_thread_fence(std::memory_order_acquire);
     delete ptr;
   }
 }

panda/src/express/referenceCount.cxx

@@ -23,17 +23,19 @@ TypeHandle ReferenceCount::_type_handle;
  */
 bool ReferenceCount::
 do_test_ref_count_integrity() const {
+  int ref_count = _ref_count.load(std::memory_order_relaxed);
+
   // If this assertion fails, we're trying to delete an object that was just
   // deleted.  Possibly you used a real pointer instead of a PointerTo at some
   // point, and the object was deleted when the PointerTo went out of scope.
   // Maybe you tried to create an automatic (local variable) instance of a
   // class that derives from ReferenceCount.  Or maybe your headers are out of
   // sync, and you need to make clean in direct or some higher tree.
-  nassertr(_ref_count != deleted_ref_count, false);
+  nassertr(ref_count != deleted_ref_count, false);
 
   // If this assertion fails, the reference counts are all screwed up
   // altogether.  Maybe some errant code stomped all over memory somewhere.
-  nassertr(_ref_count >= 0, false);
+  nassertr(ref_count >= 0, false);
 
   return true;
 }
@@ -44,7 +46,7 @@ do_test_ref_count_integrity() const {
 bool ReferenceCount::
 do_test_ref_count_nonzero() const {
   nassertr(do_test_ref_count_integrity(), false);
-  nassertr(_ref_count > 0, false);
+  nassertr(_ref_count.load(std::memory_order_relaxed) > 0, false);
 
   return true;
 }
@@ -54,11 +56,12 @@ do_test_ref_count_nonzero() const {
  */
 void ReferenceCount::
 create_weak_list() {
-  WeakReferenceList *weak_list = new WeakReferenceList;
-  void *orig =
-    AtomicAdjust::compare_and_exchange_ptr(_weak_list, nullptr, weak_list);
-  if (orig != nullptr) {
+  WeakReferenceList *new_list = new WeakReferenceList;
+  WeakReferenceList *old_list = nullptr;
+  if (!_weak_list.compare_exchange_strong(old_list, new_list,
+                                          std::memory_order_release,
+                                          std::memory_order_relaxed)) {
     // Someone else created it first.
-    delete weak_list;
+    delete new_list;
   }
 }

panda/src/express/referenceCount.h

@@ -23,6 +23,7 @@
 #include "atomicAdjust.h"
 #include "numeric_types.h"
 #include "deletedChain.h"
+#include "patomic.h"
 
 #include <stdlib.h>
 
@@ -89,8 +90,8 @@ private:
     deleted_ref_count = -100,
   };
 
-  mutable AtomicAdjust::Integer _ref_count;
-  AtomicAdjust::Pointer _weak_list;  // WeakReferenceList *
+  mutable patomic<int> _ref_count;
+  patomic<WeakReferenceList *> _weak_list;
 
 public:
   static TypeHandle get_class_type() {

panda/src/express/weakPointerToBase.I

@@ -117,6 +117,7 @@ INLINE WeakPointerToBase<T>::
 ~WeakPointerToBase() {
   WeakReferenceList *old_ref = (WeakReferenceList *)_weak_ref;
   if (old_ref != nullptr && !old_ref->unref()) {
+    patomic_thread_fence(std::memory_order_acquire);
     delete old_ref;
   }
 }
@@ -143,6 +144,7 @@ reassign(To *ptr) {
 
     // Now remove the old reference.
     if (old_ref != nullptr && !old_ref->unref()) {
+      patomic_thread_fence(std::memory_order_acquire);
       delete old_ref;
     }
   }
@@ -183,6 +185,7 @@ reassign(const WeakPointerToBase<To> &copy) {
 
     // Now remove the old reference.
     if (old_ref != nullptr && !old_ref->unref()) {
+      patomic_thread_fence(std::memory_order_acquire);
       delete old_ref;
     }
   }
@@ -205,6 +208,7 @@ reassign(WeakPointerToBase<To> &&from) noexcept {
 
     // Now delete the old pointer.
     if (old_ref != nullptr && !old_ref->unref()) {
+      patomic_thread_fence(std::memory_order_acquire);
       delete old_ref;
     }
   }
@@ -233,6 +237,7 @@ reassign(const WeakPointerToBase<Y> &copy) {
 
     // Now remove the old reference.
     if (old_ref != nullptr && !old_ref->unref()) {
+      patomic_thread_fence(std::memory_order_acquire);
       delete old_ref;
     }
   }
@@ -260,6 +265,7 @@ reassign(WeakPointerToBase<Y> &&from) noexcept {
 
     // Now delete the old pointer.
     if (old_ref != nullptr && !old_ref->unref()) {
+      patomic_thread_fence(std::memory_order_acquire);
       delete old_ref;
     }
   }
@@ -627,6 +633,7 @@ clear() {
 
   // Now remove the old reference.
   if (old_ref != nullptr && !old_ref->unref()) {
+    patomic_thread_fence(std::memory_order_acquire);
     delete old_ref;
   }
 }

panda/src/express/weakReferenceList.I

@@ -18,7 +18,7 @@
  */
 INLINE void WeakReferenceList::
 ref() const {
-  AtomicAdjust::inc(_count);
+  _count.fetch_add(1, std::memory_order_relaxed);
 }
 
 /**
@@ -30,7 +30,7 @@ ref() const {
  */
 INLINE bool WeakReferenceList::
 unref() const {
-  return AtomicAdjust::dec(_count);
+  return _count.fetch_sub(1, std::memory_order_release) != 1;
 }
 
 /**
@@ -41,5 +41,5 @@ unref() const {
  */
 INLINE bool WeakReferenceList::
 was_deleted() const {
-  return AtomicAdjust::get(_count) < _alive_offset;
+  return _count.load(std::memory_order_relaxed) < _alive_offset;
 }

panda/src/express/weakReferenceList.cxx

@@ -27,7 +27,7 @@ WeakReferenceList() : _count(_alive_offset) {
  */
 WeakReferenceList::
 ~WeakReferenceList() {
-  nassertv(_count == 0);
+  nassertv(_count.load(std::memory_order_relaxed) == 0);
 }
 
 /**
@@ -91,7 +91,7 @@ mark_deleted() {
 
   // Decrement the special offset added to the weak pointer count to indicate
   // that it can be deleted when all the weak references have gone.
-  AtomicAdjust::Integer result = AtomicAdjust::add(_count, -_alive_offset);
+  int result = _count.fetch_sub(_alive_offset, std::memory_order_relaxed) - _alive_offset;
   _lock.unlock();
   if (result == 0) {
     // There are no weak references remaining either, so delete this.

panda/src/express/weakReferenceList.h

@@ -17,6 +17,7 @@
 #include "pandabase.h"
 #include "pmap.h"
 #include "mutexImpl.h"
+#include "patomic.h"
 
 class WeakPointerCallback;
 
@@ -53,8 +54,8 @@ private:
   // This has a very large number added to it if the object is still alive.
   // It could be 1, but having it be a large number makes it easy to check
   // whether the object has been deleted or not.
-  static const AtomicAdjust::Integer _alive_offset = (1 << 30);
-  mutable AtomicAdjust::Integer _count;
+  static const int _alive_offset = (1 << 30);
+  mutable patomic<int> _count;
 
   friend class ReferenceCount;
 };

panda/src/pgui/pgScrollFrame.h

@@ -19,10 +19,7 @@
 #include "pgVirtualFrame.h"
 #include "pgSliderBarNotify.h"
 #include "pgSliderBar.h"
-
-#ifdef PHAVE_ATOMIC
-#include <atomic>
-#endif
+#include "patomic.h"
 
 /**
  * This is a special kind of frame that pretends to be much larger than it
@@ -96,7 +93,7 @@ private:
 private:
   bool _needs_remanage;
   bool _needs_recompute_clip;
-  std::atomic_flag _canvas_computed;
+  patomic_flag _canvas_computed;
 
   bool _has_virtual_frame;
   LVecBase4 _virtual_frame;

panda/src/physics/physicalNode.cxx

@@ -13,13 +13,10 @@
 
 #include "physicalNode.h"
 #include "physicsManager.h"
-
-#ifdef PHAVE_ATOMIC
-#include <atomic>
-#endif
+#include "patomic.h"
 
 // static stuff.
-static std::atomic_flag warned_copy_physical_node = ATOMIC_FLAG_INIT;
+static patomic_flag warned_copy_physical_node = ATOMIC_FLAG_INIT;
 
 TypeHandle PhysicalNode::_type_handle;