Merge pull request #9014 from RandomShaper/improve-mem-stats

Improve reliability of memory stats
Memory block byte tagging

4 files changed, 122 insertions, 51 deletions

diff --git a/core/os/memory.cpp b/core/os/memory.cpp
index 069ee48fae..dfc62bf981 100644
--- a/core/os/memory.cpp
+++ b/core/os/memory.cpp
@@ -29,6 +29,7 @@
 /*************************************************************************/
 #include "memory.h"
 #include "copymem.h"
+#include "core/safe_refcount.h"
 #include "error_macros.h"
 #include <stdio.h>
 #include <stdlib.h>
@@ -43,42 +44,44 @@ void *operator new(size_t p_size, void *(*p_allocfunc)(size_t p_size)) {
 	return p_allocfunc(p_size);
 }
 
-#include <stdio.h>
-
 #ifdef DEBUG_ENABLED
 size_t Memory::mem_usage = 0;
 size_t Memory::max_usage = 0;
+#ifdef DEBUG_MEMORY_TAGGING
+#define MEM_UNINIT_TAG 0xEA
+#define MEM_RELEASED_TAG 0xAE
 #endif
-
-size_t Memory::alloc_count = 0;
-
-void *Memory::alloc_static(size_t p_bytes, bool p_pad_align) {
-
-#ifdef DEBUG_ENABLED
-	bool prepad = true;
+#define PREPAD true
 #else
-	bool prepad = p_pad_align;
+#define PREPAD p_pad_align
 #endif
 
-	void *mem = malloc(p_bytes + (prepad ? PAD_ALIGN : 0));
+uint64_t Memory::alloc_count = 0;
 
-	alloc_count++;
+void *Memory::alloc_static(size_t p_bytes, bool p_pad_align) {
 
-	ERR_FAIL_COND_V(!mem, NULL);
+	void *mem = malloc(p_bytes + (PREPAD ? PAD_ALIGN : 0));
 
-	if (prepad) {
-		uint64_t *s = (uint64_t *)mem;
-		*s = p_bytes;
+	ERR_FAIL_COND_V(!mem, NULL);
 
-		uint8_t *s8 = (uint8_t *)mem;
+	atomic_increment(&alloc_count);
 
+	if (PREPAD) {
 #ifdef DEBUG_ENABLED
 		mem_usage += p_bytes;
 		if (mem_usage > max_usage) {
 			max_usage = mem_usage;
 		}
 #endif
-		return s8 + PAD_ALIGN;
+
+		uint64_t *s = (uint64_t *)mem;
+		*s = p_bytes;
+
+		uint8_t *s8 = (uint8_t *)mem + PAD_ALIGN;
+#if defined DEBUG_ENABLED && defined DEBUG_MEMORY_TAGGING
+		memset(s8, MEM_UNINIT_TAG, p_bytes);
+#endif
+		return s8;
 	} else {
 		return mem;
 	}
@@ -92,38 +95,44 @@ void *Memory::realloc_static(void *p_memory, size_t p_bytes, bool p_pad_align) {
 
 	uint8_t *mem = (uint8_t *)p_memory;
 
-#ifdef DEBUG_ENABLED
-	bool prepad = true;
-#else
-	bool prepad = p_pad_align;
-#endif
-
-	if (prepad) {
+	if (PREPAD) {
 		mem -= PAD_ALIGN;
 		uint64_t *s = (uint64_t *)mem;
 
+		if (p_bytes == 0) {
 #ifdef DEBUG_ENABLED
-		mem_usage -= *s;
-		mem_usage += p_bytes;
+			mem_usage -= *s;
+#ifdef DEBUG_MEMORY_TAGGING
+			memset(mem, MEM_RELEASED_TAG, PAD_ALIGN + *s);
+#endif
 #endif
-
-		if (p_bytes == 0) {
 			free(mem);
 			return NULL;
 		} else {
-			*s = p_bytes;
+#if defined DEBUG_ENABLED && defined DEBUG_MEMORY_TAGGING
+			if (p_bytes < *s) {
+				memset(mem + PAD_ALIGN + p_bytes, MEM_RELEASED_TAG, *s - p_bytes);
+			}
+#endif
 
 			mem = (uint8_t *)realloc(mem, p_bytes + PAD_ALIGN);
 			ERR_FAIL_COND_V(!mem, NULL);
 
 			s = (uint64_t *)mem;
-
+#ifdef DEBUG_ENABLED
+			mem_usage -= *s;
+			mem_usage += p_bytes;
+#ifdef DEBUG_MEMORY_TAGGING
+			if (p_bytes > *s) {
+				memset(mem + PAD_ALIGN + *s, MEM_UNINIT_TAG, p_bytes - *s);
+			}
+#endif
+#endif
 			*s = p_bytes;
 
 			return mem + PAD_ALIGN;
 		}
 	} else {
-
 		mem = (uint8_t *)realloc(mem, p_bytes);
 
 		ERR_FAIL_COND_V(mem == NULL && p_bytes > 0, NULL);
@@ -138,27 +147,19 @@ void Memory::free_static(void *p_ptr, bool p_pad_align) {
 
 	uint8_t *mem = (uint8_t *)p_ptr;
 
-#ifdef DEBUG_ENABLED
-	bool prepad = true;
-#else
-	bool prepad = p_pad_align;
-#endif
-
-	alloc_count--;
-
-	if (prepad) {
+	if (PREPAD) {
 		mem -= PAD_ALIGN;
-		uint64_t *s = (uint64_t *)mem;
-
 #ifdef DEBUG_ENABLED
-		mem_usage -= *s;
+		const uint64_t s = *((uint64_t *)mem);
+		mem_usage -= s;
+#ifdef DEBUG_MEMORY_TAGGING
+		memset(mem, MEM_RELEASED_TAG, PAD_ALIGN + s);
+#endif
 #endif
-
-		free(mem);
-	} else {
-
-		free(mem);
 	}
+	atomic_decrement(&alloc_count);
+
+	free(mem);
 }
 
 size_t Memory::get_mem_available() {
diff --git a/core/os/memory.h b/core/os/memory.h
index b3eb599955..8c3e26219a 100644
--- a/core/os/memory.h
+++ b/core/os/memory.h
@@ -48,8 +48,7 @@ class Memory {
 	static size_t mem_usage;
 	static size_t max_usage;
 #endif
-
-	static size_t alloc_count;
+	static uint64_t alloc_count;
 
 public:
 	static void *alloc_static(size_t p_bytes, bool p_pad_align = false);
diff --git a/core/safe_refcount.cpp b/core/safe_refcount.cpp
index e4a5a994e6..95a274cda7 100644
--- a/core/safe_refcount.cpp
+++ b/core/safe_refcount.cpp
@@ -57,6 +57,30 @@ uint32_t atomic_decrement(register uint32_t *pw) {
 	return *pw;
 }
 
+uint64_t atomic_conditional_increment(register uint64_t *pw) {
+
+	if (*pw == 0)
+		return 0;
+
+	(*pw)++;
+
+	return *pw;
+}
+
+uint64_t atomic_increment(register uint64_t *pw) {
+
+	(*pw)++;
+
+	return *pw;
+}
+
+uint64_t atomic_decrement(register uint64_t *pw) {
+
+	(*pw)--;
+
+	return *pw;
+}
+
 #else
 
 #ifdef _MSC_VER
@@ -84,6 +108,28 @@ uint32_t atomic_decrement(register uint32_t *pw) {
 uint32_t atomic_increment(register uint32_t *pw) {
 	return InterlockedIncrement((LONG volatile *)pw);
 }
+
+uint64_t atomic_conditional_increment(register uint64_t *pw) {
+
+	/* try to increment until it actually works */
+	// taken from boost
+
+	while (true) {
+		uint64_t tmp = static_cast<uint64_t const volatile &>(*pw);
+		if (tmp == 0)
+			return 0; // if zero, can't add to it anymore
+		if (InterlockedCompareExchange64((LONGLONG volatile *)pw, tmp + 1, tmp) == tmp)
+			return tmp + 1;
+	}
+}
+
+uint64_t atomic_decrement(register uint64_t *pw) {
+	return InterlockedDecrement64((LONGLONG volatile *)pw);
+}
+
+uint64_t atomic_increment(register uint64_t *pw) {
+	return InterlockedIncrement64((LONGLONG volatile *)pw);
+}
 #elif defined(__GNUC__)
 
 uint32_t atomic_conditional_increment(register uint32_t *pw) {
@@ -107,6 +153,27 @@ uint32_t atomic_increment(register uint32_t *pw) {
 	return __sync_add_and_fetch(pw, 1);
 }
 
+uint64_t atomic_conditional_increment(register uint64_t *pw) {
+
+	while (true) {
+		uint64_t tmp = static_cast<uint64_t const volatile &>(*pw);
+		if (tmp == 0)
+			return 0; // if zero, can't add to it anymore
+		if (__sync_val_compare_and_swap(pw, tmp, tmp + 1) == tmp)
+			return tmp + 1;
+	}
+}
+
+uint64_t atomic_decrement(register uint64_t *pw) {
+
+	return __sync_sub_and_fetch(pw, 1);
+}
+
+uint64_t atomic_increment(register uint64_t *pw) {
+
+	return __sync_add_and_fetch(pw, 1);
+}
+
 #else
 //no threads supported?
 #error Must provide atomic functions for this platform or compiler!
diff --git a/core/safe_refcount.h b/core/safe_refcount.h
index d30f563b56..5c9fd996a0 100644
--- a/core/safe_refcount.h
+++ b/core/safe_refcount.h
@@ -40,6 +40,10 @@ uint32_t atomic_conditional_increment(register uint32_t *counter);
 uint32_t atomic_decrement(register uint32_t *pw);
 uint32_t atomic_increment(register uint32_t *pw);
 
+uint64_t atomic_conditional_increment(register uint64_t *counter);
+uint64_t atomic_decrement(register uint64_t *pw);
+uint64_t atomic_increment(register uint64_t *pw);
+
 struct SafeRefCount {
 
 	uint32_t count;