Merge pull request #60313 from BastiaanOlij/xr_hand_tracking

Adding OpenXR hand tracking support

10 files changed, 823 insertions, 7 deletions

diff --git a/modules/openxr/SCsub b/modules/openxr/SCsub
index 298977f96d..11cc525484 100644
--- a/modules/openxr/SCsub
+++ b/modules/openxr/SCsub
@@ -83,6 +83,7 @@ module_obj = []
 
 env_openxr.add_source_files(module_obj, "*.cpp")
 env_openxr.add_source_files(module_obj, "action_map/*.cpp")
+env_openxr.add_source_files(module_obj, "scene/*.cpp")
 
 # We're a little more targeted with our extensions
 if env["platform"] == "android":
@@ -91,6 +92,7 @@ if env["vulkan"]:
     env_openxr.add_source_files(module_obj, "extensions/openxr_vulkan_extension.cpp")
 
 env_openxr.add_source_files(module_obj, "extensions/openxr_htc_vive_tracker_extension.cpp")
+env_openxr.add_source_files(module_obj, "extensions/openxr_hand_tracking_extension.cpp")
 
 env.modules_sources += module_obj
 
diff --git a/modules/openxr/config.py b/modules/openxr/config.py
index 650146fdd3..279168cc59 100644
--- a/modules/openxr/config.py
+++ b/modules/openxr/config.py
@@ -18,6 +18,7 @@ def get_doc_classes():
         "OpenXRActionMap",
         "OpenXRInteractionProfile",
         "OpenXRIPBinding",
+        "OpenXRHand",
     ]
 
 
diff --git a/modules/openxr/doc_classes/OpenXRHand.xml b/modules/openxr/doc_classes/OpenXRHand.xml
new file mode 100644
index 0000000000..5d5f8a6126
--- /dev/null
+++ b/modules/openxr/doc_classes/OpenXRHand.xml
@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="UTF-8" ?>
+<class name="OpenXRHand" inherits="Node3D" version="4.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:noNamespaceSchemaLocation="../../../doc/class.xsd">
+	<brief_description>
+		Node supporting finger tracking in OpenXR.
+	</brief_description>
+	<description>
+		This node enables OpenXR's hand tracking functionality. The node should be a child node of an [XROrigin3D] node, tracking will update its position to where the player's actual hand is positioned. This node also updates the skeleton of a properly skinned hand model. The hand mesh should be a child node of this node.
+	</description>
+	<tutorials>
+	</tutorials>
+	<members>
+		<member name="hand" type="int" setter="set_hand" getter="get_hand" enum="OpenXRHand.Hands" default="0">
+			Specifies whether this node tracks the left or right hand of the player.
+		</member>
+		<member name="hand_skeleton" type="NodePath" setter="set_hand_skeleton" getter="get_hand_skeleton" default="NodePath(&quot;&quot;)">
+			Set a [Skeleton3D] node for which the pose positions will be updated.
+		</member>
+		<member name="motion_range" type="int" setter="set_motion_range" getter="get_motion_range" enum="OpenXRHand.MotionRange" default="0">
+			Set the motion range (if supported) limiting the hand motion.
+		</member>
+	</members>
+	<constants>
+		<constant name="HAND_LEFT" value="0" enum="Hands">
+			Tracking the player's left hand.
+		</constant>
+		<constant name="HAND_RIGHT" value="1" enum="Hands">
+			Tracking the player's right hand.
+		</constant>
+		<constant name="HAND_MAX" value="2" enum="Hands">
+			Maximum supported hands.
+		</constant>
+		<constant name="MOTION_RANGE_UNOBSTRUCTED" value="0" enum="MotionRange">
+			When player grips, hand skeleton will form a full fist.
+		</constant>
+		<constant name="MOTION_RANGE_CONFORM_TO_CONTROLLER" value="1" enum="MotionRange">
+			When player grips, hand skeleton conforms to the controller the player is holding.
+		</constant>
+		<constant name="MOTION_RANGE_MAX" value="2" enum="MotionRange">
+			Maximum supported motion ranges.
+		</constant>
+	</constants>
+</class>
diff --git a/modules/openxr/extensions/openxr_hand_tracking_extension.cpp b/modules/openxr/extensions/openxr_hand_tracking_extension.cpp
new file mode 100644
index 0000000000..4b30965ce5
--- /dev/null
+++ b/modules/openxr/extensions/openxr_hand_tracking_extension.cpp
@@ -0,0 +1,268 @@
+/*************************************************************************/
+/*  openxr_hand_tracking_extension.cpp                                   */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "openxr_hand_tracking_extension.h"
+#include "../openxr_api.h"
+#include "core/string/print_string.h"
+#include "servers/xr_server.h"
+
+#include <openxr/openxr.h>
+
+OpenXRHandTrackingExtension *OpenXRHandTrackingExtension::singleton = nullptr;
+
+OpenXRHandTrackingExtension *OpenXRHandTrackingExtension::get_singleton() {
+	return singleton;
+}
+
+OpenXRHandTrackingExtension::OpenXRHandTrackingExtension(OpenXRAPI *p_openxr_api) :
+		OpenXRExtensionWrapper(p_openxr_api) {
+	singleton = this;
+
+	// Extensions we use for our hand tracking.
+	request_extensions[XR_EXT_HAND_TRACKING_EXTENSION_NAME] = &hand_tracking_ext;
+	request_extensions[XR_EXT_HAND_JOINTS_MOTION_RANGE_EXTENSION_NAME] = &hand_motion_range_ext;
+	request_extensions[XR_FB_HAND_TRACKING_AIM_EXTENSION_NAME] = &hand_tracking_aim_state_ext;
+
+	// Make sure this is cleared until we actually request it
+	handTrackingSystemProperties.supportsHandTracking = false;
+}
+
+OpenXRHandTrackingExtension::~OpenXRHandTrackingExtension() {
+	singleton = nullptr;
+}
+
+void OpenXRHandTrackingExtension::on_instance_created(const XrInstance p_instance) {
+	if (hand_tracking_ext) {
+		EXT_INIT_XR_FUNC(xrCreateHandTrackerEXT);
+		EXT_INIT_XR_FUNC(xrDestroyHandTrackerEXT);
+		EXT_INIT_XR_FUNC(xrLocateHandJointsEXT);
+
+		hand_tracking_ext = xrCreateHandTrackerEXT_ptr && xrDestroyHandTrackerEXT_ptr && xrLocateHandJointsEXT_ptr;
+	}
+}
+
+void OpenXRHandTrackingExtension::on_session_destroyed() {
+	cleanup_hand_tracking();
+}
+
+void OpenXRHandTrackingExtension::on_instance_destroyed() {
+	xrCreateHandTrackerEXT_ptr = nullptr;
+	xrDestroyHandTrackerEXT_ptr = nullptr;
+	xrLocateHandJointsEXT_ptr = nullptr;
+}
+
+void *OpenXRHandTrackingExtension::set_system_properties_and_get_next_pointer(void *p_next_pointer) {
+	if (!hand_tracking_ext) {
+		// not supported...
+		return p_next_pointer;
+	}
+
+	handTrackingSystemProperties = {
+		XR_TYPE_SYSTEM_HAND_TRACKING_PROPERTIES_EXT, // type
+		p_next_pointer, // next
+		false, // supportsHandTracking
+	};
+
+	return &handTrackingSystemProperties;
+}
+
+void OpenXRHandTrackingExtension::on_state_ready() {
+	if (!handTrackingSystemProperties.supportsHandTracking) {
+		// not supported...
+		return;
+	}
+
+	// Setup our hands and reset data
+	for (int i = 0; i < MAX_OPENXR_TRACKED_HANDS; i++) {
+		// we'll do this later
+		hand_trackers[i].is_initialised = false;
+		hand_trackers[i].hand_tracker = XR_NULL_HANDLE;
+
+		hand_trackers[i].aimState.aimPose = { { 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } };
+		hand_trackers[i].aimState.pinchStrengthIndex = 0.0;
+		hand_trackers[i].aimState.pinchStrengthMiddle = 0.0;
+		hand_trackers[i].aimState.pinchStrengthRing = 0.0;
+		hand_trackers[i].aimState.pinchStrengthLittle = 0.0;
+
+		hand_trackers[i].locations.isActive = false;
+
+		for (int j = 0; j < XR_HAND_JOINT_COUNT_EXT; j++) {
+			hand_trackers[i].joint_locations[j] = { 0, { { 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } }, 0.0 };
+			hand_trackers[i].joint_velocities[j] = { 0, { 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } };
+		}
+	}
+}
+
+void OpenXRHandTrackingExtension::on_process() {
+	if (!handTrackingSystemProperties.supportsHandTracking) {
+		// not supported...
+		return;
+	}
+
+	// process our hands
+	const XrTime time = openxr_api->get_next_frame_time(); // This data will be used for the next frame we render
+
+	XrResult result;
+
+	for (int i = 0; i < MAX_OPENXR_TRACKED_HANDS; i++) {
+		if (hand_trackers[i].hand_tracker == XR_NULL_HANDLE) {
+			XrHandTrackerCreateInfoEXT createInfo = {
+				XR_TYPE_HAND_TRACKER_CREATE_INFO_EXT, // type
+				nullptr, // next
+				i == 0 ? XR_HAND_LEFT_EXT : XR_HAND_RIGHT_EXT, // hand
+				XR_HAND_JOINT_SET_DEFAULT_EXT, // handJointSet
+			};
+
+			result = xrCreateHandTrackerEXT(openxr_api->get_session(), &createInfo, &hand_trackers[i].hand_tracker);
+			if (XR_FAILED(result)) {
+				// not successful? then we do nothing.
+				print_line("OpenXR: Failed to obtain hand tracking information [", openxr_api->get_error_string(result), "]");
+				hand_trackers[i].is_initialised = false;
+			} else {
+				void *next_pointer = nullptr;
+				if (hand_tracking_aim_state_ext) {
+					hand_trackers[i].aimState.type = XR_TYPE_HAND_TRACKING_AIM_STATE_FB;
+					hand_trackers[i].aimState.next = next_pointer;
+					hand_trackers[i].aimState.status = 0;
+					hand_trackers[i].aimState.aimPose = { { 0.0, 0.0, 0.0, 0.0 }, { 0.0, 0.0, 0.0 } };
+					hand_trackers[i].aimState.pinchStrengthIndex = 0.0;
+					hand_trackers[i].aimState.pinchStrengthMiddle = 0.0;
+					hand_trackers[i].aimState.pinchStrengthRing = 0.0;
+					hand_trackers[i].aimState.pinchStrengthLittle = 0.0;
+
+					next_pointer = &hand_trackers[i].aimState;
+				}
+
+				hand_trackers[i].velocities.type = XR_TYPE_HAND_JOINT_VELOCITIES_EXT;
+				hand_trackers[i].velocities.next = next_pointer;
+				hand_trackers[i].velocities.jointCount = XR_HAND_JOINT_COUNT_EXT;
+				hand_trackers[i].velocities.jointVelocities = hand_trackers[i].joint_velocities;
+				next_pointer = &hand_trackers[i].velocities;
+
+				hand_trackers[i].locations.type = XR_TYPE_HAND_JOINT_LOCATIONS_EXT;
+				hand_trackers[i].locations.next = next_pointer;
+				hand_trackers[i].locations.isActive = false;
+				hand_trackers[i].locations.jointCount = XR_HAND_JOINT_COUNT_EXT;
+				hand_trackers[i].locations.jointLocations = hand_trackers[i].joint_locations;
+
+				hand_trackers[i].is_initialised = true;
+			}
+		}
+
+		if (hand_trackers[i].is_initialised) {
+			void *next_pointer = nullptr;
+
+			XrHandJointsMotionRangeInfoEXT motionRangeInfo;
+
+			if (hand_motion_range_ext) {
+				motionRangeInfo.type = XR_TYPE_HAND_JOINTS_MOTION_RANGE_INFO_EXT;
+				motionRangeInfo.next = next_pointer;
+				motionRangeInfo.handJointsMotionRange = hand_trackers[i].motion_range;
+
+				next_pointer = &motionRangeInfo;
+			}
+
+			XrHandJointsLocateInfoEXT locateInfo = {
+				XR_TYPE_HAND_JOINTS_LOCATE_INFO_EXT, // type
+				next_pointer, // next
+				openxr_api->get_play_space(), // baseSpace
+				time, // time
+			};
+
+			result = xrLocateHandJointsEXT(hand_trackers[i].hand_tracker, &locateInfo, &hand_trackers[i].locations);
+			if (XR_FAILED(result)) {
+				// not successful? then we do nothing.
+				print_line("OpenXR: Failed to get tracking for hand", i, "[", openxr_api->get_error_string(result), "]");
+				continue;
+			}
+
+			// For some reason an inactive controller isn't coming back as inactive but has coordinates either as NAN or very large
+			const XrPosef &palm = hand_trackers[i].joint_locations[XR_HAND_JOINT_PALM_EXT].pose;
+			if (
+					!hand_trackers[i].locations.isActive || isnan(palm.position.x) || palm.position.x < -1000000.00 || palm.position.x > 1000000.00) {
+				hand_trackers[i].locations.isActive = false; // workaround, make sure its inactive
+			}
+
+			/* TODO change this to managing the controller from openxr_interface
+			if (hand_tracking_aim_state_ext && hand_trackers[i].locations.isActive && check_bit(XR_HAND_TRACKING_AIM_VALID_BIT_FB, hand_trackers[i].aimState.status)) {
+				// Controllers are updated based on the aim state's pose and pinches' strength
+				if (hand_trackers[i].aim_state_godot_controller == -1) {
+					hand_trackers[i].aim_state_godot_controller =
+							arvr_api->godot_arvr_add_controller(
+									const_cast<char *>(hand_controller_names[i]),
+									i + HAND_CONTROLLER_ID_OFFSET,
+									true,
+									true);
+				}
+			}
+			*/
+		}
+	}
+}
+
+void OpenXRHandTrackingExtension::on_state_stopping() {
+	// cleanup
+	cleanup_hand_tracking();
+}
+
+void OpenXRHandTrackingExtension::cleanup_hand_tracking() {
+	XRServer *xr_server = XRServer::get_singleton();
+	ERR_FAIL_NULL(xr_server);
+
+	for (int i = 0; i < MAX_OPENXR_TRACKED_HANDS; i++) {
+		if (hand_trackers[i].hand_tracker != XR_NULL_HANDLE) {
+			xrDestroyHandTrackerEXT(hand_trackers[i].hand_tracker);
+
+			hand_trackers[i].is_initialised = false;
+			hand_trackers[i].hand_tracker = XR_NULL_HANDLE;
+		}
+	}
+}
+
+bool OpenXRHandTrackingExtension::get_active() {
+	return handTrackingSystemProperties.supportsHandTracking;
+}
+
+const OpenXRHandTrackingExtension::HandTracker *OpenXRHandTrackingExtension::get_hand_tracker(uint32_t p_hand) const {
+	ERR_FAIL_UNSIGNED_INDEX_V(p_hand, MAX_OPENXR_TRACKED_HANDS, nullptr);
+
+	return &hand_trackers[p_hand];
+}
+
+XrHandJointsMotionRangeEXT OpenXRHandTrackingExtension::get_motion_range(uint32_t p_hand) const {
+	ERR_FAIL_UNSIGNED_INDEX_V(p_hand, MAX_OPENXR_TRACKED_HANDS, XR_HAND_JOINTS_MOTION_RANGE_MAX_ENUM_EXT);
+
+	return hand_trackers[p_hand].motion_range;
+}
+
+void OpenXRHandTrackingExtension::set_motion_range(uint32_t p_hand, XrHandJointsMotionRangeEXT p_motion_range) {
+	ERR_FAIL_UNSIGNED_INDEX(p_hand, MAX_OPENXR_TRACKED_HANDS);
+	hand_trackers[p_hand].motion_range = p_motion_range;
+}
diff --git a/modules/openxr/extensions/openxr_hand_tracking_extension.h b/modules/openxr/extensions/openxr_hand_tracking_extension.h
new file mode 100644
index 0000000000..f8c26339b0
--- /dev/null
+++ b/modules/openxr/extensions/openxr_hand_tracking_extension.h
@@ -0,0 +1,96 @@
+/*************************************************************************/
+/*  openxr_hand_tracking_extension.h                                     */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef OPENXR_HAND_TRACKING_EXTENSION_H
+#define OPENXR_HAND_TRACKING_EXTENSION_H
+
+#include "openxr_extension_wrapper.h"
+
+#include "../util.h"
+
+#define MAX_OPENXR_TRACKED_HANDS 2
+
+class OpenXRHandTrackingExtension : public OpenXRExtensionWrapper {
+public:
+	struct HandTracker {
+		bool is_initialised = false;
+		XrHandJointsMotionRangeEXT motion_range = XR_HAND_JOINTS_MOTION_RANGE_UNOBSTRUCTED_EXT;
+
+		XrHandTrackerEXT hand_tracker = XR_NULL_HANDLE;
+		XrHandJointLocationEXT joint_locations[XR_HAND_JOINT_COUNT_EXT];
+		XrHandJointVelocityEXT joint_velocities[XR_HAND_JOINT_COUNT_EXT];
+
+		XrHandTrackingAimStateFB aimState;
+		XrHandJointVelocitiesEXT velocities;
+		XrHandJointLocationsEXT locations;
+	};
+
+	static OpenXRHandTrackingExtension *get_singleton();
+
+	OpenXRHandTrackingExtension(OpenXRAPI *p_openxr_api);
+	virtual ~OpenXRHandTrackingExtension() override;
+
+	virtual void on_instance_created(const XrInstance p_instance) override;
+	virtual void on_instance_destroyed() override;
+	virtual void on_session_destroyed() override;
+
+	virtual void *set_system_properties_and_get_next_pointer(void *p_next_pointer) override;
+	virtual void on_state_ready() override;
+	virtual void on_process() override;
+	virtual void on_state_stopping() override;
+
+	bool get_active();
+	const HandTracker *get_hand_tracker(uint32_t p_hand) const;
+
+	XrHandJointsMotionRangeEXT get_motion_range(uint32_t p_hand) const;
+	void set_motion_range(uint32_t p_hand, XrHandJointsMotionRangeEXT p_motion_range);
+
+private:
+	static OpenXRHandTrackingExtension *singleton;
+
+	// state
+	XrSystemHandTrackingPropertiesEXT handTrackingSystemProperties;
+	HandTracker hand_trackers[MAX_OPENXR_TRACKED_HANDS]; // Fixed for left and right hand
+
+	// related extensions
+	bool hand_tracking_ext = false;
+	bool hand_motion_range_ext = false;
+	bool hand_tracking_aim_state_ext = false;
+
+	// functions
+	void cleanup_hand_tracking();
+
+	// OpenXR API call wrappers
+	EXT_PROTO_XRRESULT_FUNC3(xrCreateHandTrackerEXT, (XrSession), p_session, (const XrHandTrackerCreateInfoEXT *), p_createInfo, (XrHandTrackerEXT *), p_handTracker)
+	EXT_PROTO_XRRESULT_FUNC1(xrDestroyHandTrackerEXT, (XrHandTrackerEXT), p_handTracker)
+	EXT_PROTO_XRRESULT_FUNC3(xrLocateHandJointsEXT, (XrHandTrackerEXT), p_handTracker, (const XrHandJointsLocateInfoEXT *), p_locateInfo, (XrHandJointLocationsEXT *), p_locations)
+};
+
+#endif // OPENXR_HAND_TRACKING_EXTENSION_H
diff --git a/modules/openxr/openxr_api.cpp b/modules/openxr/openxr_api.cpp
index 76f7b2a60a..08b76ddc6b 100644
--- a/modules/openxr/openxr_api.cpp
+++ b/modules/openxr/openxr_api.cpp
@@ -49,6 +49,7 @@
 #include "extensions/openxr_vulkan_extension.h"
 #endif
 
+#include "extensions/openxr_hand_tracking_extension.h"
 #include "extensions/openxr_htc_vive_tracker_extension.h"
 
 #include "modules/openxr/openxr_interface.h"
@@ -1742,6 +1743,7 @@ OpenXRAPI::OpenXRAPI() {
 
 	// register our other extensions
 	register_extension_wrapper(memnew(OpenXRHTCViveTrackerExtension(this)));
+	register_extension_wrapper(memnew(OpenXRHandTrackingExtension(this)));
 }
 
 OpenXRAPI::~OpenXRAPI() {
diff --git a/modules/openxr/openxr_api.h b/modules/openxr/openxr_api.h
index 4f84cc3b5b..fd32c2633e 100644
--- a/modules/openxr/openxr_api.h
+++ b/modules/openxr/openxr_api.h
@@ -102,7 +102,7 @@ private:
 	XrFormFactor form_factor = XR_FORM_FACTOR_HEAD_MOUNTED_DISPLAY;
 	XrViewConfigurationType view_configuration = XR_VIEW_CONFIGURATION_TYPE_PRIMARY_STEREO;
 	XrReferenceSpaceType reference_space = XR_REFERENCE_SPACE_TYPE_STAGE;
-	XrEnvironmentBlendMode environment_blend_mode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE;
+	// XrEnvironmentBlendMode environment_blend_mode = XR_ENVIRONMENT_BLEND_MODE_OPAQUE;
 
 	// state
 	XrInstance instance = XR_NULL_HANDLE;
@@ -139,7 +139,10 @@ private:
 	bool openxr_loader_init();
 	bool resolve_instance_openxr_symbols();
 
+#ifdef ANDROID_ENABLED
+	// On Android we keep tracker of our external OpenXR loader
 	void *openxr_loader_library_handle = nullptr;
+#endif
 
 	// function pointers
 #ifdef ANDROID_ENABLED
@@ -269,9 +272,7 @@ private:
 	// convencience
 	void copy_string_to_char_buffer(const String p_string, char *p_buffer, int p_buffer_len);
 
-protected:
-	friend class OpenXRVulkanExtension;
-
+public:
 	XrInstance get_instance() const { return instance; };
 	XrSystemId get_system_id() const { return system_id; };
 	XrSession get_session() const { return session; };
@@ -284,7 +285,6 @@ protected:
 	XRPose::TrackingConfidence transform_from_location(const XrHandJointLocationEXT &p_location, Transform3D &r_transform);
 	void parse_velocities(const XrSpaceVelocity &p_velocity, Vector3 &r_linear_velocity, Vector3 &r_angular_velocity);
 
-public:
 	static bool openxr_is_enabled(bool p_check_run_in_editor = true);
 	static OpenXRAPI *get_singleton();
 
@@ -301,8 +301,9 @@ public:
 	bool initialize_session();
 	void finish();
 
-	XrTime get_next_frame_time() { return frame_state.predictedDisplayTime + frame_state.predictedDisplayPeriod; };
-	bool can_render() { return instance != XR_NULL_HANDLE && session != XR_NULL_HANDLE && running && view_pose_valid && frame_state.shouldRender; };
+	XrSpace get_play_space() const { return play_space; }
+	XrTime get_next_frame_time() { return frame_state.predictedDisplayTime + frame_state.predictedDisplayPeriod; }
+	bool can_render() { return instance != XR_NULL_HANDLE && session != XR_NULL_HANDLE && running && view_pose_valid && frame_state.shouldRender; }
 
 	Size2 get_recommended_target_size();
 	XRPose::TrackingConfidence get_head_center(Transform3D &r_transform, Vector3 &r_linear_velocity, Vector3 &r_angular_velocity);
diff --git a/modules/openxr/register_types.cpp b/modules/openxr/register_types.cpp
index c765f169dc..5694fffe39 100644
--- a/modules/openxr/register_types.cpp
+++ b/modules/openxr/register_types.cpp
@@ -38,6 +38,8 @@
 #include "action_map/openxr_action_set.h"
 #include "action_map/openxr_interaction_profile.h"
 
+#include "scene/openxr_hand.h"
+
 #ifdef TOOLS_ENABLED
 
 #include "editor/editor_node.h"
@@ -82,6 +84,8 @@ void initialize_openxr_module(ModuleInitializationLevel p_level) {
 		GDREGISTER_CLASS(OpenXRIPBinding);
 		GDREGISTER_CLASS(OpenXRInteractionProfile);
 
+		GDREGISTER_CLASS(OpenXRHand);
+
 		XRServer *xr_server = XRServer::get_singleton();
 		if (xr_server) {
 			openxr_interface.instantiate();
diff --git a/modules/openxr/scene/openxr_hand.cpp b/modules/openxr/scene/openxr_hand.cpp
new file mode 100644
index 0000000000..2ae13a1026
--- /dev/null
+++ b/modules/openxr/scene/openxr_hand.cpp
@@ -0,0 +1,307 @@
+/*************************************************************************/
+/*  openxr_hand.cpp                                                      */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#include "../extensions/openxr_hand_tracking_extension.h"
+#include "../openxr_api.h"
+
+#include "openxr_hand.h"
+#include "scene/3d/skeleton_3d.h"
+#include "servers/xr_server.h"
+
+void OpenXRHand::_bind_methods() {
+	ClassDB::bind_method(D_METHOD("set_hand", "hand"), &OpenXRHand::set_hand);
+	ClassDB::bind_method(D_METHOD("get_hand"), &OpenXRHand::get_hand);
+
+	ClassDB::bind_method(D_METHOD("set_hand_skeleton", "hand_skeleton"), &OpenXRHand::set_hand_skeleton);
+	ClassDB::bind_method(D_METHOD("get_hand_skeleton"), &OpenXRHand::get_hand_skeleton);
+
+	ClassDB::bind_method(D_METHOD("set_motion_range", "motion_range"), &OpenXRHand::set_motion_range);
+	ClassDB::bind_method(D_METHOD("get_motion_range"), &OpenXRHand::get_motion_range);
+
+	ADD_PROPERTY(PropertyInfo(Variant::INT, "hand", PROPERTY_HINT_ENUM, "Left,Right"), "set_hand", "get_hand");
+	ADD_PROPERTY(PropertyInfo(Variant::INT, "motion_range", PROPERTY_HINT_ENUM, "Unobstructed,Conform to controller"), "set_motion_range", "get_motion_range");
+	ADD_PROPERTY(PropertyInfo(Variant::NODE_PATH, "hand_skeleton", PROPERTY_HINT_NODE_PATH_VALID_TYPES, "Skeleton3D"), "set_hand_skeleton", "get_hand_skeleton");
+
+	BIND_ENUM_CONSTANT(HAND_LEFT);
+	BIND_ENUM_CONSTANT(HAND_RIGHT);
+	BIND_ENUM_CONSTANT(HAND_MAX);
+
+	BIND_ENUM_CONSTANT(MOTION_RANGE_UNOBSTRUCTED);
+	BIND_ENUM_CONSTANT(MOTION_RANGE_CONFORM_TO_CONTROLLER);
+	BIND_ENUM_CONSTANT(MOTION_RANGE_MAX);
+}
+
+OpenXRHand::OpenXRHand() {
+	openxr_api = OpenXRAPI::get_singleton();
+	hand_tracking_ext = OpenXRHandTrackingExtension::get_singleton();
+}
+
+void OpenXRHand::set_hand(const Hands p_hand) {
+	ERR_FAIL_INDEX(p_hand, HAND_MAX);
+
+	hand = p_hand;
+}
+
+OpenXRHand::Hands OpenXRHand::get_hand() const {
+	return hand;
+}
+
+void OpenXRHand::set_hand_skeleton(const NodePath &p_hand_skeleton) {
+	hand_skeleton = p_hand_skeleton;
+
+	// TODO if inside tree call _get_bones()
+}
+
+void OpenXRHand::set_motion_range(const MotionRange p_motion_range) {
+	ERR_FAIL_INDEX(p_motion_range, MOTION_RANGE_MAX);
+	motion_range = p_motion_range;
+
+	_set_motion_range();
+}
+
+OpenXRHand::MotionRange OpenXRHand::get_motion_range() const {
+	return motion_range;
+}
+
+NodePath OpenXRHand::get_hand_skeleton() const {
+	return hand_skeleton;
+}
+
+void OpenXRHand::_set_motion_range() {
+	if (!hand_tracking_ext) {
+		return;
+	}
+
+	XrHandJointsMotionRangeEXT xr_motion_range;
+	switch (motion_range) {
+		case MOTION_RANGE_UNOBSTRUCTED:
+			xr_motion_range = XR_HAND_JOINTS_MOTION_RANGE_UNOBSTRUCTED_EXT;
+			break;
+		case MOTION_RANGE_CONFORM_TO_CONTROLLER:
+			xr_motion_range = XR_HAND_JOINTS_MOTION_RANGE_CONFORMING_TO_CONTROLLER_EXT;
+			break;
+		default:
+			xr_motion_range = XR_HAND_JOINTS_MOTION_RANGE_CONFORMING_TO_CONTROLLER_EXT;
+			break;
+	}
+
+	hand_tracking_ext->set_motion_range(hand, xr_motion_range);
+}
+
+Skeleton3D *OpenXRHand::get_skeleton() {
+	if (!has_node(hand_skeleton)) {
+		return nullptr;
+	}
+
+	Node *node = get_node(hand_skeleton);
+	if (!node) {
+		return nullptr;
+	}
+
+	Skeleton3D *skeleton = Object::cast_to<Skeleton3D>(node);
+	return skeleton;
+}
+
+void OpenXRHand::_get_bones() {
+	const char *bone_names[XR_HAND_JOINT_COUNT_EXT] = {
+		"Palm",
+		"Wrist",
+		"Thumb_Metacarpal",
+		"Thumb_Proximal",
+		"Thumb_Distal",
+		"Thumb_Tip",
+		"Index_Metacarpal",
+		"Index_Proximal",
+		"Index_Intermediate",
+		"Index_Distal",
+		"Index_Tip",
+		"Middle_Metacarpal",
+		"Middle_Proximal",
+		"Middle_Intermediate",
+		"Middle_Distal",
+		"Middle_Tip",
+		"Ring_Metacarpal",
+		"Ring_Proximal",
+		"Ring_Intermediate",
+		"Ring_Distal",
+		"Ring_Tip",
+		"Little_Metacarpal",
+		"Little_Proximal",
+		"Little_Intermediate",
+		"Little_Distal",
+		"Little_Tip",
+	};
+
+	// reset JIC
+	for (int i = 0; i < XR_HAND_JOINT_COUNT_EXT; i++) {
+		bones[i] = -1;
+	}
+
+	Skeleton3D *skeleton = get_skeleton();
+	if (!skeleton) {
+		return;
+	}
+
+	// We cast to spatials which should allow us to use any subclass of that.
+	for (int i = 0; i < XR_HAND_JOINT_COUNT_EXT; i++) {
+		String bone_name = bone_names[i];
+		if (hand == 0) {
+			bone_name += String("_L");
+		} else {
+			bone_name += String("_R");
+		}
+
+		bones[i] = skeleton->find_bone(bone_name);
+		if (bones[i] == -1) {
+			print_line("Couldn't obtain bone for", bone_name);
+		}
+	}
+}
+
+void OpenXRHand::_update_skeleton() {
+	if (openxr_api == nullptr || !openxr_api->is_initialized()) {
+		return;
+	} else if (hand_tracking_ext == nullptr || !hand_tracking_ext->get_active()) {
+		return;
+	}
+
+	Skeleton3D *skeleton = get_skeleton();
+	if (!skeleton) {
+		return;
+	}
+
+	// we cache our transforms so we can quickly calculate local transforms
+	XRPose::TrackingConfidence confidences[XR_HAND_JOINT_COUNT_EXT];
+	Quaternion quaternions[XR_HAND_JOINT_COUNT_EXT];
+	Quaternion inv_quaternions[XR_HAND_JOINT_COUNT_EXT];
+	Vector3 positions[XR_HAND_JOINT_COUNT_EXT];
+
+	const OpenXRHandTrackingExtension::HandTracker *hand_tracker = hand_tracking_ext->get_hand_tracker(hand);
+	const float ws = XRServer::get_singleton()->get_world_scale();
+
+	if (hand_tracker->is_initialised && hand_tracker->locations.isActive) {
+		for (int i = 0; i < XR_HAND_JOINT_COUNT_EXT; i++) {
+			confidences[i] = XRPose::XR_TRACKING_CONFIDENCE_NONE;
+			quaternions[i] = Quaternion();
+			positions[i] = Vector3();
+
+			const auto &location = hand_tracker->joint_locations[i];
+			const auto &pose = location.pose;
+
+			if (location.locationFlags & XR_SPACE_LOCATION_ORIENTATION_VALID_BIT) {
+				if (pose.orientation.x != 0 || pose.orientation.y != 0 || pose.orientation.y != 0 || pose.orientation.w != 0) {
+					quaternions[i] = Quaternion(pose.orientation.x, pose.orientation.y, pose.orientation.z, pose.orientation.w);
+					inv_quaternions[i] = quaternions[i].inverse();
+
+					if (location.locationFlags & XR_SPACE_LOCATION_POSITION_VALID_BIT) {
+						confidences[i] = XRPose::XR_TRACKING_CONFIDENCE_HIGH;
+						positions[i] = Vector3(pose.position.x * ws, pose.position.y * ws, pose.position.z * ws);
+
+						// TODO get inverse of position, we'll do this later. For now we're ignoring bone positions which generally works better anyway
+					} else {
+						confidences[i] = XRPose::XR_TRACKING_CONFIDENCE_LOW;
+					}
+				}
+			}
+		}
+
+		if (confidences[XR_HAND_JOINT_PALM_EXT] != XRPose::XR_TRACKING_CONFIDENCE_NONE) {
+			// now update our skeleton
+			for (int i = 0; i < XR_HAND_JOINT_COUNT_EXT; i++) {
+				if (bones[i] != -1) {
+					int bone = bones[i];
+					int parent = skeleton->get_bone_parent(bone);
+
+					// Get our target quaternion
+					Quaternion q = quaternions[i];
+
+					// get local translation, parent should already be processed
+					if (parent == -1) {
+						// use our palm location here, that is what we are tracking
+						q = inv_quaternions[XR_HAND_JOINT_PALM_EXT] * q;
+					} else {
+						int found = false;
+						for (int b = 0; b < XR_HAND_JOINT_COUNT_EXT && !found; b++) {
+							if (bones[b] == parent) {
+								q = inv_quaternions[b] * q;
+								found = true;
+							}
+						}
+					}
+
+					// And get the movement from our rest position
+					// Transform3D rest = skeleton->get_bone_rest(bones[i]);
+					// q = rest.basis.get_quaternion().inverse() * q;
+
+					// and set our pose
+					// skeleton->set_bone_pose_position(bones[i], v);
+					skeleton->set_bone_pose_rotation(bones[i], q);
+				}
+			}
+
+			Transform3D t;
+			t.basis = Basis(quaternions[XR_HAND_JOINT_PALM_EXT]);
+			t.origin = positions[XR_HAND_JOINT_PALM_EXT];
+			set_transform(t);
+
+			// show it
+			set_visible(true);
+		} else {
+			// hide it
+			set_visible(false);
+		}
+	} else {
+		// hide it
+		set_visible(false);
+	}
+}
+
+void OpenXRHand::_notification(int p_what) {
+	switch (p_what) {
+		case NOTIFICATION_ENTER_TREE: {
+			_get_bones();
+
+			set_process_internal(true);
+		} break;
+		case NOTIFICATION_EXIT_TREE: {
+			set_process_internal(false);
+
+			// reset
+			for (int i = 0; i < XR_HAND_JOINT_COUNT_EXT; i++) {
+				bones[i] = -1;
+			}
+		} break;
+		case NOTIFICATION_INTERNAL_PROCESS: {
+			_update_skeleton();
+		} break;
+		default: {
+		} break;
+	}
+}
diff --git a/modules/openxr/scene/openxr_hand.h b/modules/openxr/scene/openxr_hand.h
new file mode 100644
index 0000000000..f8499cc172
--- /dev/null
+++ b/modules/openxr/scene/openxr_hand.h
@@ -0,0 +1,93 @@
+/*************************************************************************/
+/*  openxr_hand.h                                                        */
+/*************************************************************************/
+/*                       This file is part of:                           */
+/*                           GODOT ENGINE                                */
+/*                      https://godotengine.org                          */
+/*************************************************************************/
+/* Copyright (c) 2007-2022 Juan Linietsky, Ariel Manzur.                 */
+/* Copyright (c) 2014-2022 Godot Engine contributors (cf. AUTHORS.md).   */
+/*                                                                       */
+/* Permission is hereby granted, free of charge, to any person obtaining */
+/* a copy of this software and associated documentation files (the       */
+/* "Software"), to deal in the Software without restriction, including   */
+/* without limitation the rights to use, copy, modify, merge, publish,   */
+/* distribute, sublicense, and/or sell copies of the Software, and to    */
+/* permit persons to whom the Software is furnished to do so, subject to */
+/* the following conditions:                                             */
+/*                                                                       */
+/* The above copyright notice and this permission notice shall be        */
+/* included in all copies or substantial portions of the Software.       */
+/*                                                                       */
+/* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,       */
+/* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF    */
+/* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.*/
+/* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY  */
+/* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,  */
+/* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE     */
+/* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.                */
+/*************************************************************************/
+
+#ifndef OPENXR_HAND_H
+#define OPENXR_HAND_H
+
+#include "scene/3d/node_3d.h"
+#include "scene/3d/skeleton_3d.h"
+
+class OpenXRAPI;
+class OpenXRHandTrackingExtension;
+
+class OpenXRHand : public Node3D {
+	GDCLASS(OpenXRHand, Node3D);
+
+public:
+	enum Hands {
+		HAND_LEFT,
+		HAND_RIGHT,
+		HAND_MAX
+	};
+
+	enum MotionRange {
+		MOTION_RANGE_UNOBSTRUCTED,
+		MOTION_RANGE_CONFORM_TO_CONTROLLER,
+		MOTION_RANGE_MAX
+	};
+
+private:
+	OpenXRAPI *openxr_api = nullptr;
+	OpenXRHandTrackingExtension *hand_tracking_ext = nullptr;
+
+	Hands hand = HAND_LEFT;
+	MotionRange motion_range = MOTION_RANGE_UNOBSTRUCTED;
+	NodePath hand_skeleton;
+
+	int64_t bones[XR_HAND_JOINT_COUNT_EXT];
+
+	void _set_motion_range();
+
+	Skeleton3D *get_skeleton();
+	void _get_bones();
+	void _update_skeleton();
+
+protected:
+	static void _bind_methods();
+
+public:
+	OpenXRHand();
+
+	void set_hand(const Hands p_hand);
+	Hands get_hand() const;
+
+	void set_motion_range(const MotionRange p_motion_range);
+	MotionRange get_motion_range() const;
+
+	void set_hand_skeleton(const NodePath &p_hand_skeleton);
+	NodePath get_hand_skeleton() const;
+
+	void _notification(int p_what);
+};
+
+VARIANT_ENUM_CAST(OpenXRHand::Hands)
+VARIANT_ENUM_CAST(OpenXRHand::MotionRange)
+
+#endif // OPENXR_HAND_H