Fix allocation bug if compiled with modern clang or gcc

* Add overflow checked intrinsic abstractions that check on overflow.
* Use them for memory allocation code.
* Use size_t type for memory allocation code to support full platform dependent width.

Fixes #3756.

1 files changed, 22 insertions, 5 deletions

diff --git a/drivers/unix/memory_pool_static_malloc.cpp b/drivers/unix/memory_pool_static_malloc.cpp
index e75b682c19..f89b55de12 100644
--- a/drivers/unix/memory_pool_static_malloc.cpp
+++ b/drivers/unix/memory_pool_static_malloc.cpp
@@ -48,7 +48,12 @@ void* MemoryPoolStaticMalloc::alloc(size_t p_bytes,const char *p_description) {
 
 	#else
 
-		int total = p_bytes + DEFAULT_ALIGNMENT;
+		size_t total;
+		#if defined(_add_overflow)
+			if (_add_overflow(p_bytes, DEFAULT_ALIGNMENT, &total)) return NULL;
+		#else
+			total = p_bytes + DEFAULT_ALIGNMENT;
+		#endif
 		uint8_t* ptr = (uint8_t*)_alloc(total, p_description);
 		ERR_FAIL_COND_V( !ptr, ptr );
 		int ofs = (DEFAULT_ALIGNMENT - ((uintptr_t)ptr & (DEFAULT_ALIGNMENT - 1)));
@@ -64,11 +69,18 @@ void* MemoryPoolStaticMalloc::_alloc(size_t p_bytes,const char *p_description) {
 	MutexLock lock(mutex);
 	
 #ifdef DEBUG_MEMORY_ENABLED
-	void *mem=malloc(p_bytes+sizeof(RingPtr)); /// add for size and ringlist
+
+	size_t total;
+	#if defined(_add_overflow)
+		if (_add_overflow(p_bytes, sizeof(RingPtr), &total)) return NULL;
+	#else
+		total = p_bytes + sizeof(RingPtr);
+	#endif
+	void *mem=malloc(total); /// add for size and ringlist
 
 	if (!mem) {
-		printf("**ERROR: out of memory while allocating %i bytes by %s?\n",(int) p_bytes, p_description);
-		printf("**ERROR: memory usage is %i\n", (int)get_total_usage());
+		printf("**ERROR: out of memory while allocating %lu bytes by %s?\n", (unsigned long) p_bytes, p_description);
+		printf("**ERROR: memory usage is %lu\n", (unsigned long) get_total_usage());
 	};
 	
 	ERR_FAIL_COND_V(!mem,0); //out of memory, or unreasonable request
@@ -129,7 +141,12 @@ void* MemoryPoolStaticMalloc::realloc(void *p_memory,size_t p_bytes) {
 		if (!p_memory)
 			return alloc(p_bytes);
 
-		int total = p_bytes + DEFAULT_ALIGNMENT;
+		size_t total;
+		#if defined(_add_overflow)
+			if (_add_overflow(p_bytes, DEFAULT_ALIGNMENT, &total)) return NULL;
+		#else
+			total = p_bytes + DEFAULT_ALIGNMENT;
+		#endif
 		uint8_t* mem = (uint8_t*)p_memory;
 		int ofs = *(mem-1);
 		mem = mem - ofs;