Fix implementation of atomic update max and move to a central location

While unlikely to have had any serious effects because of limited use, the
previous implementation was not actually atomic due to a data race and
incorrectly coded CAS loop. We also had duplicates of this code in a few
places, it's now been moved to a single location with all other atomic
operations.

intern/atomic/atomic_ops.h

@@ -100,6 +100,7 @@ ATOMIC_INLINE size_t atomic_sub_and_fetch_z(size_t *p, size_t x);
 ATOMIC_INLINE size_t atomic_fetch_and_add_z(size_t *p, size_t x);
 ATOMIC_INLINE size_t atomic_fetch_and_sub_z(size_t *p, size_t x);
 ATOMIC_INLINE size_t atomic_cas_z(size_t *v, size_t old, size_t _new);
+ATOMIC_INLINE size_t atomic_fetch_and_update_max_z(size_t *p, size_t x); /* Uses CAS loop, see warning below. */
 
 ATOMIC_INLINE unsigned int atomic_add_and_fetch_u(unsigned int *p, unsigned int x);
 ATOMIC_INLINE unsigned int atomic_sub_and_fetch_u(unsigned int *p, unsigned int x);

intern/atomic/intern/atomic_ops_ext.h

@@ -111,6 +111,17 @@ ATOMIC_INLINE size_t atomic_cas_z(size_t *v, size_t old, size_t _new)
 #endif
 }
 
+ATOMIC_INLINE size_t atomic_fetch_and_update_max_z(size_t *p, size_t x)
+{
+	size_t prev_value;
+	while((prev_value = *p) < x) {
+		if(atomic_cas_z(p, prev_value, x) == prev_value) {
+			break;
+		}
+	}
+	return prev_value;
+}
+
 /******************************************************************************/
 /* unsigned operations. */
 ATOMIC_INLINE unsigned int atomic_add_and_fetch_u(unsigned int *p, unsigned int x)

intern/cycles/util/util_atomic.h

@@ -22,16 +22,6 @@
 /* Using atomic ops header from Blender. */
 #include "atomic_ops.h"
 
-ATOMIC_INLINE void atomic_update_max_z(size_t *maximum_value, size_t value)
-{
-	size_t prev_value = *maximum_value;
-	while(prev_value < value) {
-		if(atomic_cas_z(maximum_value, prev_value, value) != prev_value) {
-			break;
-		}
-	}
-}
-
 #define atomic_add_and_fetch_float(p, x) atomic_add_and_fetch_fl((p), (x))
 
 #define atomic_fetch_and_inc_uint32(p) atomic_fetch_and_add_uint32((p), 1)

intern/cycles/util/util_stats.h

@@ -30,7 +30,7 @@ public:
 
 	void mem_alloc(size_t size) {
 		atomic_add_and_fetch_z(&mem_used, size);
-		atomic_update_max_z(&mem_peak, mem_used);
+		atomic_fetch_and_update_max_z(&mem_peak, mem_used);
 	}
 
 	void mem_free(size_t size) {

intern/guardedalloc/intern/mallocn_lockfree_impl.c

@@ -76,12 +76,7 @@ enum {
 MEM_INLINE void update_maximum(size_t *maximum_value, size_t value)
 {
 #ifdef USE_ATOMIC_MAX
-	size_t prev_value = *maximum_value;
+	atomic_fetch_and_update_max_z(maximum_value, value);
-	while (prev_value < value) {
-		if (atomic_cas_z(maximum_value, prev_value, value) != prev_value) {
-			break;
-		}
-	}
 #else
 	*maximum_value = value > *maximum_value ? value : *maximum_value;
 #endif