Fix allocation bug if compiled with modern clang or gcc

* Add overflow checked intrinsic abstractions that check on overflow.
* Use them for memory allocation code.
* Use size_t type for memory allocation code to support full platform dependent width.

Fixes #3756.

2 files changed, 56 insertions, 9 deletions

diff --git a/core/typedefs.h b/core/typedefs.h
index 1ca7a4f66d..91d4b8f81d 100644
--- a/core/typedefs.h
+++ b/core/typedefs.h
@@ -154,6 +154,23 @@ inline void __swap_tmpl(T &x, T &y ) {
 	((m_hex>='A' && m_hex<='F')?(10+m_hex-'A'):\
 	((m_hex>='a' && m_hex<='f')?(10+m_hex-'a'):0)))
 
+// Macro to check whether we are compiled by clang
+// and we have a specific builtin
+#if defined(__llvm__) && defined(__has_builtin)
+	#define _llvm_has_builtin(x) __has_builtin(x)
+#else
+	#define _llvm_has_builtin(x) 0
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 5)) || _llvm_has_builtin(__builtin_mul_overflow)
+#    define _mul_overflow __builtin_mul_overflow
+#endif
+
+#if (defined(__GNUC__) && (__GNUC__ >= 5)) || _llvm_has_builtin(__builtin_add_overflow)
+#    define _add_overflow __builtin_add_overflow
+#endif
+
+
 
 
 
@@ -167,6 +184,19 @@ static _FORCE_INLINE_ unsigned int nearest_power_of_2(unsigned int x) {
 	x |= x >> 4;
 	x |= x >> 8;
 	x |= x >> 16;
+
+	return ++x;
+}
+
+template<class T>
+static _FORCE_INLINE_ T nearest_power_of_2_templated(T x) {
+
+	--x;
+	// If the compiler is smart, it unrolls this loop
+	// If its dumb, this is a bit slow.
+	for (size_t i = 0; i < sizeof(T); i++)
+		x |= x >> (1 << i);
+
 	return ++x;
 }
 
diff --git a/core/vector.h b/core/vector.h
index 641721f401..398d7f1bd5 100644
--- a/core/vector.h
+++ b/core/vector.h
@@ -68,11 +68,26 @@ class Vector {
 		return reinterpret_cast<T*>(_ptr);
  		
  	}
- 	
-	_FORCE_INLINE_ int _get_alloc_size(int p_elements) const {
- 	
- 		return  nearest_power_of_2(p_elements*sizeof(T)+sizeof(SafeRefCount)+sizeof(int));
- 	}
+
+	_FORCE_INLINE_ size_t _get_alloc_size(size_t p_elements) const {
+		return nearest_power_of_2_templated(p_elements*sizeof(T)+sizeof(SafeRefCount)+sizeof(int));
+	}
+
+	_FORCE_INLINE_ bool _get_alloc_size_checked(size_t p_elements, size_t *out) const {
+#if defined(_add_overflow) && defined(_mul_overflow)
+		size_t o;
+		size_t p;
+		if (_mul_overflow(p_elements, sizeof(T), &o)) return false;
+		if (_add_overflow(o, sizeof(SafeRefCount)+sizeof(int), &p)) return false;
+		*out = nearest_power_of_2_templated(p);
+		return true;
+#else
+		// Speed is more important than correctness here, do the operations unchecked
+		// and hope the best
+		*out = _get_alloc_size(p_elements);
+		return true;
+#endif
+	}
  	
 	void _unref(void *p_data);
 	
@@ -257,19 +272,21 @@ Error Vector<T>::resize(int p_size) {
 	// possibly changing size, copy on write
 	_copy_on_write();
 	
+	size_t alloc_size;
+	ERR_FAIL_COND_V(!_get_alloc_size_checked(p_size, &alloc_size), ERR_OUT_OF_MEMORY);
+
 	if (p_size>size()) {
 
 		if (size()==0) {
 			// alloc from scratch
-			void* ptr=memalloc(_get_alloc_size(p_size));
+			void* ptr=memalloc(alloc_size);
 			ERR_FAIL_COND_V( !ptr ,ERR_OUT_OF_MEMORY);
 			_ptr=(T*)((uint8_t*)ptr+sizeof(int)+sizeof(SafeRefCount));
 			_get_refcount()->init(); // init refcount
 			*_get_size()=0; // init size (currently, none)
 
 		} else {
-			
-			void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount),_get_alloc_size(p_size));
+			void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount), alloc_size);
 			ERR_FAIL_COND_V( !_ptrnew ,ERR_OUT_OF_MEMORY);
 			_ptr=(T*)((uint8_t*)_ptrnew+sizeof(int)+sizeof(SafeRefCount));
 		}
@@ -293,7 +310,7 @@ Error Vector<T>::resize(int p_size) {
 			t->~T();
 		}
 
-		void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount),_get_alloc_size(p_size));
+		void *_ptrnew = (T*)memrealloc((uint8_t*)_ptr-sizeof(int)-sizeof(SafeRefCount), alloc_size);
 		ERR_FAIL_COND_V( !_ptrnew ,ERR_OUT_OF_MEMORY);
 		
 		_ptr=(T*)((uint8_t*)_ptrnew+sizeof(int)+sizeof(SafeRefCount));