Implement native extension system

* Deprecates GDNative in favor of a simpler, lower level interface.
* New extension system allows registering core engine classes.
* Simple header interface in gdnative_interace.h

9 files changed, 245 insertions, 34 deletions

diff --git a/core/object/SCsub b/core/object/SCsub
index 5d429960e5..dc116aeb19 100644
--- a/core/object/SCsub
+++ b/core/object/SCsub
@@ -2,6 +2,11 @@
 
 Import("env")
 
+import make_virtuals
+from platform_methods import run_in_subprocess
+
+env.CommandNoCache(["gdvirtual.gen.inc"], "make_virtuals.py", run_in_subprocess(make_virtuals.run))
+
 env_object = env.Clone()
 
 env_object.add_source_files(env.core_sources, "*.cpp")
diff --git a/core/object/class_db.cpp b/core/object/class_db.cpp
index df36587662..a10405dfae 100644
--- a/core/object/class_db.cpp
+++ b/core/object/class_db.cpp
@@ -503,9 +503,9 @@ void ClassDB::add_compatibility_class(const StringName &p_class, const StringNam
 
 thread_local bool initializing_with_extension = false;
 thread_local ObjectNativeExtension *initializing_extension = nullptr;
-thread_local void *initializing_extension_instance = nullptr;
+thread_local GDExtensionClassInstancePtr initializing_extension_instance = nullptr;
 
-void ClassDB::instance_get_native_extension_data(ObjectNativeExtension **r_extension, void **r_extension_instance) {
+void ClassDB::instance_get_native_extension_data(ObjectNativeExtension **r_extension, GDExtensionClassInstancePtr *r_extension_instance) {
 	if (initializing_with_extension) {
 		*r_extension = initializing_extension;
 		*r_extension_instance = initializing_extension_instance;
@@ -539,7 +539,7 @@ Object *ClassDB::instantiate(const StringName &p_class) {
 	if (ti->native_extension) {
 		initializing_with_extension = true;
 		initializing_extension = ti->native_extension;
-		initializing_extension_instance = ti->native_extension->create_instance(ti->native_extension->create_instance_userdata);
+		initializing_extension_instance = ti->native_extension->create_instance(ti->native_extension->class_userdata);
 	}
 	return ti->creation_func();
 }
@@ -1603,6 +1603,11 @@ void ClassDB::register_extension_class(ObjectNativeExtension *p_extension) {
 	classes[p_extension->class_name] = c;
 }
 
+void ClassDB::unregister_extension_class(const StringName &p_class) {
+	ERR_FAIL_COND(!classes.has(p_class));
+	classes.erase(p_class);
+}
+
 RWLock ClassDB::lock;
 
 void ClassDB::cleanup_defaults() {
diff --git a/core/object/class_db.h b/core/object/class_db.h
index a4af535149..af528bfde7 100644
--- a/core/object/class_db.h
+++ b/core/object/class_db.h
@@ -204,6 +204,7 @@ public:
 	}
 
 	static void register_extension_class(ObjectNativeExtension *p_extension);
+	static void unregister_extension_class(const StringName &p_class);
 
 	template <class T>
 	static Object *_create_ptr_func() {
@@ -232,7 +233,8 @@ public:
 	static bool is_parent_class(const StringName &p_class, const StringName &p_inherits);
 	static bool can_instantiate(const StringName &p_class);
 	static Object *instantiate(const StringName &p_class);
-	static void instance_get_native_extension_data(ObjectNativeExtension **r_extension, void **r_extension_instance);
+	static void instance_get_native_extension_data(ObjectNativeExtension **r_extension, GDExtensionClassInstancePtr *r_extension_instance);
+
 	static APIType get_api_type(const StringName &p_class);
 
 	static uint64_t get_api_hash(APIType p_api);
diff --git a/core/object/make_virtuals.py b/core/object/make_virtuals.py
new file mode 100644
index 0000000000..2c6b8cddc9
--- /dev/null
+++ b/core/object/make_virtuals.py
@@ -0,0 +1,152 @@
+proto = """
+#define GDVIRTUAL$VER($RET m_name $ARG) \\
+GDNativeExtensionClassCallVirtual _gdvirtual_##m_name = (_get_extension() && _get_extension()->get_virtual) ? _get_extension()->get_virtual(_get_extension()->class_userdata, #m_name) : (GDNativeExtensionClassCallVirtual) nullptr;\\
+StringName _gdvirtual_##m_name##_sn = #m_name;\\
+bool _gdvirtual_##m_name##_call($CALLARGS) $CONST { \\
+	ScriptInstance *script_instance = ((Object*)(this))->get_script_instance();\\
+	if (script_instance) {\\
+		Callable::CallError ce; \\
+		$CALLSIARGS\\
+		$CALLSIBEGINscript_instance->call(_gdvirtual_##m_name##_sn, $CALLSIARGPASS, ce);\\
+		if (ce.error == Callable::CallError::CALL_OK) {\\
+			$CALLSIRET\\
+			return true;\\
+		}    \\
+	}\\
+	if (_gdvirtual_##m_name) {\\
+		$CALLPTRARGS\\
+		$CALLPTRRETDEF\\
+		_gdvirtual_##m_name(_get_extension_instance(),$CALLPTRARGPASS,$CALLPTRRETPASS);\\
+		$CALLPTRRET\\
+		return true;\\
+	}\\
+\\
+    return false;\\
+}\\
+\\
+_FORCE_INLINE_ static MethodInfo _gdvirtual_##m_name##_get_method_info() { \\
+    MethodInfo method_info;\\
+    method_info.name = #m_name;\\
+    method_info.flags = METHOD_FLAG_VIRTUAL;\\
+    $FILL_METHOD_INFO\\
+    return method_info;\\
+}
+
+
+"""
+
+
+def generate_version(argcount, const=False, returns=False):
+    s = proto
+    sproto = str(argcount)
+    method_info = ""
+    if returns:
+        sproto += "R"
+        s = s.replace("$RET", "m_ret, ")
+        s = s.replace("$CALLPTRRETDEF", "PtrToArg<m_ret>::EncodeT ret;")
+        method_info += "\tmethod_info.return_val = GetTypeInfo<m_ret>::get_class_info();\\\n"
+    else:
+        s = s.replace("$RET", "")
+        s = s.replace("$CALLPTRRETDEF", "")
+
+    if const:
+        sproto += "C"
+        s = s.replace("$CONST", "const")
+        method_info += "\tmethod_info.flags|=METHOD_FLAG_CONST;\\\n"
+    else:
+        s = s.replace("$CONST", "")
+
+    s = s.replace("$VER", sproto)
+    argtext = ""
+    callargtext = ""
+    callsiargs = ""
+    callsiargptrs = ""
+    callptrargsptr = ""
+    if argcount > 0:
+        argtext += ", "
+        callsiargs = "Variant vargs[" + str(argcount) + "]={"
+        callsiargptrs = "\t\tconst Variant *vargptrs[" + str(argcount) + "]={"
+        callptrargsptr = "\t\tconst GDNativeTypePtr argptrs[" + str(argcount) + "]={"
+    callptrargs = ""
+    for i in range(argcount):
+        if i > 0:
+            argtext += ", "
+            callargtext += ", "
+            callsiargs += ", "
+            callsiargptrs += ", "
+            callptrargs += "\t\t"
+            callptrargsptr += ", "
+        argtext += "m_type" + str(i + 1)
+        callargtext += "const m_type" + str(i + 1) + "& arg" + str(i + 1)
+        callsiargs += "Variant(arg" + str(i + 1) + ")"
+        callsiargptrs += "&vargs[" + str(i) + "]"
+        callptrargs += (
+            "PtrToArg<m_type" + str(i + 1) + ">::EncodeT argval" + str(i + 1) + " = arg" + str(i + 1) + ";\\\n"
+        )
+        callptrargsptr += "&argval" + str(i + 1)
+        method_info += "\tmethod_info.arguments.push_back(GetTypeInfo<m_type" + str(i + 1) + ">::get_class_info());\\\n"
+
+    if argcount:
+        callsiargs += "};\\\n"
+        callsiargptrs += "};\\\n"
+        s = s.replace("$CALLSIARGS", callsiargs + callsiargptrs)
+        s = s.replace("$CALLSIARGPASS", "(const Variant **)vargptrs," + str(argcount))
+        callptrargsptr += "};\\\n"
+        s = s.replace("$CALLPTRARGS", callptrargs + callptrargsptr)
+        s = s.replace("$CALLPTRARGPASS", "(const GDNativeTypePtr*)argptrs")
+    else:
+        s = s.replace("$CALLSIARGS", "")
+        s = s.replace("$CALLSIARGPASS", "nullptr, 0")
+        s = s.replace("$CALLPTRARGS", "")
+        s = s.replace("$CALLPTRARGPASS", "nullptr")
+
+    if returns:
+        if argcount > 0:
+            callargtext += ","
+            callargtext += " m_ret& r_ret"
+        s = s.replace("$CALLSIBEGIN", "Variant ret = ")
+        s = s.replace("$CALLSIRET", "r_ret = ret;")
+        s = s.replace("$CALLPTRRETPASS", "&ret")
+        s = s.replace("$CALLPTRRET", "r_ret = ret;")
+    else:
+        s = s.replace("$CALLSIBEGIN", "")
+        s = s.replace("$CALLSIRET", "")
+        s = s.replace("$CALLPTRRETPASS", "nullptr")
+        s = s.replace("$CALLPTRRET", "")
+
+    s = s.replace("$ARG", argtext)
+    s = s.replace("$CALLARGS", callargtext)
+    s = s.replace("$FILL_METHOD_INFO", method_info)
+
+    return s
+
+
+def run(target, source, env):
+
+    max_versions = 12
+
+    txt = """
+#ifndef GDVIRTUAL_GEN_H
+#define GDVIRTUAL_GEN_H
+
+
+"""
+
+    for i in range(max_versions + 1):
+
+        txt += "/* " + str(i) + " Arguments */\n\n"
+        txt += generate_version(i, False, False)
+        txt += generate_version(i, False, True)
+        txt += generate_version(i, True, False)
+        txt += generate_version(i, True, True)
+
+    txt += "#endif"
+
+    with open(target[0], "w") as f:
+        f.write(txt)
+
+
+if __name__ == "__main__":
+    from platform_methods import subprocess_main
+
+    subprocess_main(globals())
diff --git a/core/object/method_bind.cpp b/core/object/method_bind.cpp
index 9c5ed60708..c53104fe3f 100644
--- a/core/object/method_bind.cpp
+++ b/core/object/method_bind.cpp
@@ -34,6 +34,35 @@
 
 #include "method_bind.h"
 
+uint32_t MethodBind::get_hash() const {
+	uint32_t hash = hash_djb2_one_32(has_return() ? 1 : 0);
+	hash = hash_djb2_one_32(get_argument_count(), hash);
+
+#ifndef _MSC_VER
+#warning This needs proper class name and argument type for hashing
+#endif
+#if 0
+
+	for (int i = (has_return() ? -1 : 0); i < get_argument_count(); i++) {
+		PropertyInfo pi = i == -1 ? get_return_info() : get_argument_info(i);
+		hash = hash_djb2_one_32(get_argument_type(i), hash);
+		if (pi.class_name != StringName()) {
+			hash = hash_djb2_one_32(pi.class_name.operator String().hash(), hash);
+		}
+	}
+#endif
+	hash = hash_djb2_one_32(get_default_argument_count(), hash);
+	for (int i = 0; i < get_default_argument_count(); i++) {
+		Variant v = get_default_argument(i);
+		hash = hash_djb2_one_32(v.hash(), hash);
+	}
+
+	hash = hash_djb2_one_32(is_const(), hash);
+	hash = hash_djb2_one_32(is_vararg(), hash);
+
+	return hash;
+}
+
 #ifdef DEBUG_METHODS_ENABLED
 PropertyInfo MethodBind::get_argument_info(int p_argument) const {
 	ERR_FAIL_INDEX_V(p_argument, get_argument_count(), PropertyInfo());
diff --git a/core/object/method_bind.h b/core/object/method_bind.h
index 7030ae201b..92b964772a 100644
--- a/core/object/method_bind.h
+++ b/core/object/method_bind.h
@@ -135,6 +135,8 @@ public:
 
 	void set_default_arguments(const Vector<Variant> &p_defargs);
 
+	uint32_t get_hash() const;
+
 	MethodBind();
 	virtual ~MethodBind();
 };
diff --git a/core/object/object.cpp b/core/object/object.cpp
index 00b89ab398..1c8db89e5e 100644
--- a/core/object/object.cpp
+++ b/core/object/object.cpp
@@ -386,12 +386,20 @@ void Object::set(const StringName &p_name, const Variant &p_value, bool *r_valid
 	}
 
 	if (_extension && _extension->set) {
-		if (_extension->set(_extension_instance, &p_name, &p_value)) {
+// C style pointer casts should never trigger a compiler warning because the risk is assumed by the user, so GCC should keep quiet about it.
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wignored-qualifiers"
+#endif
+		if (_extension->set(_extension_instance, (const GDNativeStringNamePtr)&p_name, (const GDNativeVariantPtr)&p_value)) {
 			if (r_valid) {
 				*r_valid = true;
 			}
 			return;
 		}
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
 	}
 
 	//try built-in setgetter
@@ -459,14 +467,22 @@ Variant Object::get(const StringName &p_name, bool *r_valid) const {
 			return ret;
 		}
 	}
-
 	if (_extension && _extension->get) {
-		if (_extension->get(_extension_instance, &p_name, &ret)) {
+// C style pointer casts should never trigger a compiler warning because the risk is assumed by the user, so GCC should keep quiet about it.
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wignored-qualifiers"
+#endif
+
+		if (_extension->get(_extension_instance, (const GDNativeStringNamePtr)&p_name, (GDNativeVariantPtr)&ret)) {
 			if (r_valid) {
 				*r_valid = true;
 			}
 			return ret;
 		}
+#if defined(__GNUC__) && !defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
 	}
 
 	//try built-in setgetter
@@ -616,7 +632,7 @@ void Object::get_property_list(List<PropertyInfo> *p_list, bool p_reversed) cons
 
 	if (_extension && _extension->get_property_list) {
 		uint32_t pcount;
-		const ObjectNativeExtension::PInfo *pinfo = _extension->get_property_list(_extension_instance, &pcount);
+		const GDNativePropertyInfo *pinfo = _extension->get_property_list(_extension_instance, &pcount);
 		for (uint32_t i = 0; i < pcount; i++) {
 			p_list->push_back(PropertyInfo(Variant::Type(pinfo[i].type), pinfo[i].class_name, PropertyHint(pinfo[i].hint), pinfo[i].hint_string, pinfo[i].usage, pinfo[i].class_name));
 		}
@@ -1812,7 +1828,7 @@ Object::~Object() {
 	script_instance = nullptr;
 
 	if (_extension && _extension->free_instance) {
-		_extension->free_instance(_extension->create_instance_userdata, _extension_instance);
+		_extension->free_instance(_extension->class_userdata, _extension_instance);
 		_extension = nullptr;
 		_extension_instance = nullptr;
 	}
diff --git a/core/object/object.h b/core/object/object.h
index 65621a47ca..461ed482fe 100644
--- a/core/object/object.h
+++ b/core/object/object.h
@@ -31,6 +31,7 @@
 #ifndef OBJECT_H
 #define OBJECT_H
 
+#include "core/extension/gdnative_interface.h"
 #include "core/object/object_id.h"
 #include "core/os/rw_lock.h"
 #include "core/os/spin_lock.h"
@@ -244,29 +245,18 @@ class MethodBind;
 
 struct ObjectNativeExtension {
 	ObjectNativeExtension *parent = nullptr;
+	List<ObjectNativeExtension *> children;
 	StringName parent_class_name;
 	StringName class_name;
 	bool editor_class = false;
-	bool (*set)(void *instance, const void *name, const void *value) = nullptr;
-	bool (*get)(void *instance, const void *name, void *ret_variant) = nullptr;
-	struct PInfo {
-		uint32_t type;
-		const char *name;
-		const char *class_name;
-		uint32_t hint;
-		const char *hint_string;
-		uint32_t usage;
-	};
-	const PInfo *(*get_property_list)(void *instance, uint32_t *count) = nullptr;
-	void (*free_property_list)(void *instance, const PInfo *) = nullptr;
-
-	//call is not used, as all methods registered in ClassDB
-
-	void (*notification)(void *instance, int32_t what) = nullptr;
-	const char *(*to_string)(void *instance) = nullptr;
-
-	void (*reference)(void *instance) = nullptr;
-	void (*unreference)(void *instance) = nullptr;
+	GDNativeExtensionClassSet set;
+	GDNativeExtensionClassGet get;
+	GDNativeExtensionClassGetPropertyList get_property_list;
+	GDNativeExtensionClassFreePropertyList free_property_list;
+	GDNativeExtensionClassNotification notification;
+	GDNativeExtensionClassToString to_string;
+	GDNativeExtensionClassReference reference;
+	GDNativeExtensionClassReference unreference;
 
 	_FORCE_INLINE_ bool is_class(const String &p_class) const {
 		const ObjectNativeExtension *e = this;
@@ -278,11 +268,16 @@ struct ObjectNativeExtension {
 		}
 		return false;
 	}
-	void *create_instance_userdata = nullptr;
-	void *(*create_instance)(void *create_instance_userdata) = nullptr;
-	void (*free_instance)(void *create_instance_userdata, void *instance) = nullptr;
+	void *class_userdata = nullptr;
+
+	GDNativeExtensionClassCreateInstance create_instance;
+	GDNativeExtensionClassFreeInstance free_instance;
+	GDNativeExtensionClassGetVirtual get_virtual;
 };
 
+#define GDVIRTUAL_CALL(m_name, ...) _gdvirtual_##m_name##_call(__VA_ARGS__)
+#define GDVIRTUAL_BIND(m_name) ClassDB::add_virtual_method(get_class_static(), _gdvirtual_##m_name##_get_method_info());
+
 /*
    the following is an incomprehensible blob of hacks and workarounds to compensate for many of the fallencies in C++. As a plus, this macro pretty much alone defines the object model.
 */
@@ -497,7 +492,7 @@ private:
 	friend void postinitialize_handler(Object *);
 
 	ObjectNativeExtension *_extension = nullptr;
-	void *_extension_instance = nullptr;
+	GDExtensionClassInstancePtr _extension_instance = nullptr;
 
 	struct SignalData {
 		struct Slot {
@@ -554,8 +549,9 @@ private:
 	Object(bool p_reference);
 
 protected:
+	friend class NativeExtensionMethodBind;
 	_ALWAYS_INLINE_ const ObjectNativeExtension *_get_extension() const { return _extension; }
-	_ALWAYS_INLINE_ void *_get_extension_instance() const { return _extension_instance; }
+	_ALWAYS_INLINE_ GDExtensionClassInstancePtr _get_extension_instance() const { return _extension_instance; }
 	virtual void _initialize_classv() { initialize_class(); }
 	virtual bool _setv(const StringName &p_name, const Variant &p_property) { return false; };
 	virtual bool _getv(const StringName &p_name, Variant &r_property) const { return false; };
diff --git a/core/object/ref_counted.h b/core/object/ref_counted.h
index 61780eb061..e0af2c18bb 100644
--- a/core/object/ref_counted.h
+++ b/core/object/ref_counted.h
@@ -258,6 +258,8 @@ struct PtrToArg<Ref<T>> {
 		return Ref<T>(const_cast<T *>(reinterpret_cast<const T *>(p_ptr)));
 	}
 
+	typedef Ref<T> EncodeT;
+
 	_FORCE_INLINE_ static void encode(Ref<T> p_val, const void *p_ptr) {
 		*(Ref<RefCounted> *)p_ptr = p_val;
 	}
@@ -265,6 +267,8 @@ struct PtrToArg<Ref<T>> {
 
 template <class T>
 struct PtrToArg<const Ref<T> &> {
+	typedef Ref<T> EncodeT;
+
 	_FORCE_INLINE_ static Ref<T> convert(const void *p_ptr) {
 		return Ref<T>((T *)p_ptr);
 	}