From 9a868cd846ac9759b96cd2cbbb99a9a30bb43972 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20J=2E=20Est=C3=A9banez?= Date: Thu, 22 Jun 2017 05:05:59 +0200 Subject: Add atomics for 64-bit integers The public API looks the same but with added overloads. The implementation uses templates or overloads depending on the context to avoid code duplication. --- core/safe_refcount.cpp | 119 ++++++++++++++++++++++++++++++++++++------------- core/safe_refcount.h | 4 ++ 2 files changed, 93 insertions(+), 30 deletions(-) diff --git a/core/safe_refcount.cpp b/core/safe_refcount.cpp index e4a5a994e6..4a57c09f1e 100644 --- a/core/safe_refcount.cpp +++ b/core/safe_refcount.cpp @@ -33,7 +33,10 @@ #ifdef NO_THREADS -uint32_t atomic_conditional_increment(register uint32_t *pw) { +/* Bogus implementation unaware of multiprocessing */ + +template +static _ALWAYS_INLINE_ T _atomic_conditional_increment_impl(register T *pw) { if (*pw == 0) return 0; @@ -43,74 +46,130 @@ uint32_t atomic_conditional_increment(register uint32_t *pw) { return *pw; } -uint32_t atomic_increment(register uint32_t *pw) { +template +static _ALWAYS_INLINE_ T _atomic_decrement_impl(register T *pw) { - (*pw)++; + (*pw)--; return *pw; } -uint32_t atomic_decrement(register uint32_t *pw) { +template +static _ALWAYS_INLINE_T _atomic_increment_impl(register T *pw) { - (*pw)--; + (*pw)++; return *pw; } -#else +#elif defined(__GNUC__) -#ifdef _MSC_VER +/* Implementation for GCC & Clang */ -// don't pollute my namespace! -#include -uint32_t atomic_conditional_increment(register uint32_t *pw) { +// GCC guarantees atomic intrinsics for sizes of 1, 2, 4 and 8 bytes. +// Clang states it supports GCC atomic builtins. - /* try to increment until it actually works */ - // taken from boost +template +static _ALWAYS_INLINE_ T _atomic_conditional_increment_impl(register T *pw) { while (true) { - uint32_t tmp = static_cast(*pw); + T tmp = static_cast(*pw); if (tmp == 0) return 0; // if zero, can't add to it anymore - if (InterlockedCompareExchange((LONG volatile *)pw, tmp + 1, tmp) == tmp) + if (__sync_val_compare_and_swap(pw, tmp, tmp + 1) == tmp) return tmp + 1; } } -uint32_t atomic_decrement(register uint32_t *pw) { +template +static _ALWAYS_INLINE_ T _atomic_decrement_impl(register T *pw) { + + return __sync_sub_and_fetch(pw, 1); +} + +template +static _ALWAYS_INLINE_ T _atomic_increment_impl(register T *pw) { + + return __sync_add_and_fetch(pw, 1); +} + +#elif defined(_MSC_VER) + +/* Implementation for MSVC-Windows */ + +// don't pollute my namespace! +#include + +#define ATOMIC_CONDITIONAL_INCREMENT_BODY(m_pw, m_win_type, m_win_cmpxchg, m_cpp_type) \ + /* try to increment until it actually works */ \ + /* taken from boost */ \ + while (true) { \ + m_cpp_type tmp = static_cast(*(m_pw)); \ + if (tmp == 0) \ + return 0; /* if zero, can't add to it anymore */ \ + if (m_win_cmpxchg((m_win_type volatile *)(m_pw), tmp + 1, tmp) == tmp) \ + return tmp + 1; \ + } + +static _ALWAYS_INLINE_ uint32_t _atomic_conditional_increment_impl(register uint32_t *pw) { + + ATOMIC_CONDITIONAL_INCREMENT_BODY(pw, LONG, InterlockedCompareExchange, uint32_t) +} + +static _ALWAYS_INLINE_ uint32_t _atomic_decrement_impl(register uint32_t *pw) { + return InterlockedDecrement((LONG volatile *)pw); } -uint32_t atomic_increment(register uint32_t *pw) { +static _ALWAYS_INLINE_ uint32_t _atomic_increment_impl(register uint32_t *pw) { + return InterlockedIncrement((LONG volatile *)pw); } -#elif defined(__GNUC__) -uint32_t atomic_conditional_increment(register uint32_t *pw) { +static _ALWAYS_INLINE_ uint64_t _atomic_conditional_increment_impl(register uint64_t *pw) { - while (true) { - uint32_t tmp = static_cast(*pw); - if (tmp == 0) - return 0; // if zero, can't add to it anymore - if (__sync_val_compare_and_swap(pw, tmp, tmp + 1) == tmp) - return tmp + 1; - } + ATOMIC_CONDITIONAL_INCREMENT_BODY(pw, LONGLONG, InterlockedCompareExchange64, uint64_t) } -uint32_t atomic_decrement(register uint32_t *pw) { +static _ALWAYS_INLINE_ uint64_t _atomic_decrement_impl(register uint64_t *pw) { - return __sync_sub_and_fetch(pw, 1); + return InterlockedDecrement64((LONGLONG volatile *)pw); } -uint32_t atomic_increment(register uint32_t *pw) { +static _ALWAYS_INLINE_ uint64_t _atomic_increment_impl(register uint64_t *pw) { - return __sync_add_and_fetch(pw, 1); + return InterlockedIncrement64((LONGLONG volatile *)pw); } #else + //no threads supported? #error Must provide atomic functions for this platform or compiler! #endif -#endif +// The actual advertised functions; they'll call the right implementation + +uint32_t atomic_conditional_increment(register uint32_t *counter) { + return _atomic_conditional_increment_impl(counter); +} + +uint32_t atomic_decrement(register uint32_t *pw) { + return _atomic_decrement_impl(pw); +} + +uint32_t atomic_increment(register uint32_t *pw) { + return _atomic_increment_impl(pw); +} + +uint64_t atomic_conditional_increment(register uint64_t *counter) { + return _atomic_conditional_increment_impl(counter); +} + +uint64_t atomic_decrement(register uint64_t *pw) { + return _atomic_decrement_impl(pw); +} + +uint64_t atomic_increment(register uint64_t *pw) { + return _atomic_increment_impl(pw); +} diff --git a/core/safe_refcount.h b/core/safe_refcount.h index d30f563b56..5c9fd996a0 100644 --- a/core/safe_refcount.h +++ b/core/safe_refcount.h @@ -40,6 +40,10 @@ uint32_t atomic_conditional_increment(register uint32_t *counter); uint32_t atomic_decrement(register uint32_t *pw); uint32_t atomic_increment(register uint32_t *pw); +uint64_t atomic_conditional_increment(register uint64_t *counter); +uint64_t atomic_decrement(register uint64_t *pw); +uint64_t atomic_increment(register uint64_t *pw); + struct SafeRefCount { uint32_t count; -- cgit v1.2.3 From 00d60125df8a853bafe76b970138dfb75083d628 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pedro=20J=2E=20Est=C3=A9banez?= Date: Tue, 11 Jul 2017 17:41:53 +0200 Subject: Implement add & subtract atomics --- core/safe_refcount.cpp | 72 ++++++++++++++++++++++++++++++++++++++++++++++++++ core/safe_refcount.h | 4 +++ 2 files changed, 76 insertions(+) diff --git a/core/safe_refcount.cpp b/core/safe_refcount.cpp index 4a57c09f1e..1bd16f9e4f 100644 --- a/core/safe_refcount.cpp +++ b/core/safe_refcount.cpp @@ -62,6 +62,22 @@ static _ALWAYS_INLINE_T _atomic_increment_impl(register T *pw) { return *pw; } +template +static _ALWAYS_INLINE_ T _atomic_sub_impl(register T *pw, register T val) { + + (*pw) -= val; + + return *pw; +} + +template +static _ALWAYS_INLINE_T _atomic_add_impl(register T *pw, register T val) { + + (*pw) += val; + + return *pw; +} + #elif defined(__GNUC__) /* Implementation for GCC & Clang */ @@ -93,6 +109,18 @@ static _ALWAYS_INLINE_ T _atomic_increment_impl(register T *pw) { return __sync_add_and_fetch(pw, 1); } +template +static _ALWAYS_INLINE_ T _atomic_sub_impl(register T *pw, register T val) { + + return __sync_sub_and_fetch(pw, val); +} + +template +static _ALWAYS_INLINE_ T _atomic_add_impl(register T *pw, register T val) { + + return __sync_add_and_fetch(pw, val); +} + #elif defined(_MSC_VER) /* Implementation for MSVC-Windows */ @@ -126,6 +154,20 @@ static _ALWAYS_INLINE_ uint32_t _atomic_increment_impl(register uint32_t *pw) { return InterlockedIncrement((LONG volatile *)pw); } +static _ALWAYS_INLINE_ uint32_t _atomic_sub_impl(register uint32_t *pw, register uint32_t val) { + +#if _WIN32_WINNT >= 0x0601 // Windows 7+ + return InterlockedExchangeSubtract(pw, val) - val; +#else + return InterlockedExchangeAdd((LONG volatile *)pw, -(int32_t)val) - val; +#endif +} + +static _ALWAYS_INLINE_ uint32_t _atomic_add_impl(register uint32_t *pw, register uint32_t val) { + + return InterlockedAdd((LONG volatile *)pw, val); +} + static _ALWAYS_INLINE_ uint64_t _atomic_conditional_increment_impl(register uint64_t *pw) { ATOMIC_CONDITIONAL_INCREMENT_BODY(pw, LONGLONG, InterlockedCompareExchange64, uint64_t) @@ -141,6 +183,20 @@ static _ALWAYS_INLINE_ uint64_t _atomic_increment_impl(register uint64_t *pw) { return InterlockedIncrement64((LONGLONG volatile *)pw); } +static _ALWAYS_INLINE_ uint64_t _atomic_sub_impl(register uint64_t *pw, register uint64_t val) { + +#if _WIN32_WINNT >= 0x0601 // Windows 7+ + return InterlockedExchangeSubtract64(pw, val) - val; +#else + return InterlockedExchangeAdd64((LONGLONG volatile *)pw, -(int64_t)val) - val; +#endif +} + +static _ALWAYS_INLINE_ uint64_t _atomic_add_impl(register uint64_t *pw, register uint64_t val) { + + return InterlockedAdd64((LONGLONG volatile *)pw, val); +} + #else //no threads supported? @@ -162,6 +218,14 @@ uint32_t atomic_increment(register uint32_t *pw) { return _atomic_increment_impl(pw); } +uint32_t atomic_sub(register uint32_t *pw, register uint32_t val) { + return _atomic_sub_impl(pw, val); +} + +uint32_t atomic_add(register uint32_t *pw, register uint32_t val) { + return _atomic_add_impl(pw, val); +} + uint64_t atomic_conditional_increment(register uint64_t *counter) { return _atomic_conditional_increment_impl(counter); } @@ -173,3 +237,11 @@ uint64_t atomic_decrement(register uint64_t *pw) { uint64_t atomic_increment(register uint64_t *pw) { return _atomic_increment_impl(pw); } + +uint64_t atomic_sub(register uint64_t *pw, register uint64_t val) { + return _atomic_sub_impl(pw, val); +} + +uint64_t atomic_add(register uint64_t *pw, register uint64_t val) { + return _atomic_add_impl(pw, val); +} diff --git a/core/safe_refcount.h b/core/safe_refcount.h index 5c9fd996a0..a2d2b5e127 100644 --- a/core/safe_refcount.h +++ b/core/safe_refcount.h @@ -39,10 +39,14 @@ uint32_t atomic_conditional_increment(register uint32_t *counter); uint32_t atomic_decrement(register uint32_t *pw); uint32_t atomic_increment(register uint32_t *pw); +uint32_t atomic_sub(register uint32_t *pw, register uint32_t val); +uint32_t atomic_add(register uint32_t *pw, register uint32_t val); uint64_t atomic_conditional_increment(register uint64_t *counter); uint64_t atomic_decrement(register uint64_t *pw); uint64_t atomic_increment(register uint64_t *pw); +uint64_t atomic_sub(register uint64_t *pw, register uint64_t val); +uint64_t atomic_add(register uint64_t *pw, register uint64_t val); struct SafeRefCount { -- cgit v1.2.3