WebXR: Add support for hand tracking

modules/webxr/doc_classes/WebXRInterface.xml

@@ -153,6 +153,10 @@
 		</method>
 	</methods>
 	<members>
+		<member name="enabled_features" type="String" setter="" getter="get_enabled_features">
+			A comma-separated list of features that were successfully enabled by [method XRInterface.initialize] when setting up the WebXR session.
+			This may include features requested by setting [member required_features] and [member optional_features].
+		</member>
 		<member name="optional_features" type="String" setter="set_optional_features" getter="get_optional_features">
 			A comma-seperated list of optional features used by [method XRInterface.initialize] when setting up the WebXR session.
 			If a user's browser or device doesn't support one of the given features, initialization will continue, but you won't be able to use the requested feature.

modules/webxr/godot_webxr.h

@@ -45,7 +45,7 @@ enum WebXRInputEvent {
 };
 
 typedef void (*GodotWebXRSupportedCallback)(char *p_session_mode, int p_supported);
-typedef void (*GodotWebXRStartedCallback)(char *p_reference_space_type);
+typedef void (*GodotWebXRStartedCallback)(char *p_reference_space_type, char *p_enabled_features);
 typedef void (*GodotWebXREndedCallback)();
 typedef void (*GodotWebXRFailedCallback)(char *p_message);
 typedef void (*GodotWebXRInputEventCallback)(int p_event_type, int p_input_source_id);
@@ -85,7 +85,10 @@ extern bool godot_webxr_update_input_source(
 		int *r_button_count,
 		float *r_buttons,
 		int *r_axes_count,
-		float *r_axes);
+		float *r_axes,
+		int *r_has_hand_data,
+		float *r_hand_joints,
+		float *r_hand_radii);
 
 extern char *godot_webxr_get_visibility_state();
 extern int godot_webxr_get_bounds_geometry(float **r_points);

modules/webxr/native/library_godot_webxr.js

@@ -318,9 +318,11 @@ const GodotWebXR = {
 					// callback don't bubble up here and cause Godot to try the
 					// next reference space.
 					window.setTimeout(function () {
-						const c_str = GodotRuntime.allocString(reference_space_type);
-						onstarted(c_str);
-						GodotRuntime.free(c_str);
+						const reference_space_c_str = GodotRuntime.allocString(reference_space_type);
+						const enabled_features_c_str = GodotRuntime.allocString(Array.from(session.enabledFeatures).join(","));
+						onstarted(reference_space_c_str, enabled_features_c_str);
+						GodotRuntime.free(reference_space_c_str);
+						GodotRuntime.free(enabled_features_c_str);
 					}, 0);
 				}
 
@@ -479,8 +481,8 @@ const GodotWebXR = {
 	},
 
 	godot_webxr_update_input_source__proxy: 'sync',
-	godot_webxr_update_input_source__sig: 'iiiiiiiiiiii',
-	godot_webxr_update_input_source: function (p_input_source_id, r_target_pose, r_target_ray_mode, r_touch_index, r_has_grip_pose, r_grip_pose, r_has_standard_mapping, r_button_count, r_buttons, r_axes_count, r_axes) {
+	godot_webxr_update_input_source__sig: 'iiiiiiiiiiiiiii',
+	godot_webxr_update_input_source: function (p_input_source_id, r_target_pose, r_target_ray_mode, r_touch_index, r_has_grip_pose, r_grip_pose, r_has_standard_mapping, r_button_count, r_buttons, r_axes_count, r_axes, r_has_hand_data, r_hand_joints, r_hand_radii) {
 		if (!GodotWebXR.session || !GodotWebXR.frame) {
 			return 0;
 		}
@@ -563,6 +565,19 @@ const GodotWebXR = {
 		GodotRuntime.setHeapValue(r_button_count, button_count, 'i32');
 		GodotRuntime.setHeapValue(r_axes_count, axes_count, 'i32');
 
+		// Hand tracking data.
+		let has_hand_data = false;
+		if (input_source.hand && r_hand_joints != 0 && r_hand_radii != 0) {
+			const hand_joint_array = new Float32Array(25 * 16);
+			const hand_radii_array = new Float32Array(25);
+			if (frame.fillPoses(input_source.hand.values(), space, hand_joint_array) && frame.fillJointRadii(input_source.hand.values(), hand_radii_array)) {
+				GodotRuntime.heapCopy(HEAPF32, hand_joint_array, r_hand_joints);
+				GodotRuntime.heapCopy(HEAPF32, hand_radii_array, r_hand_radii);
+				has_hand_data = true;
+			}
+		}
+		GodotRuntime.setHeapValue(r_has_hand_data, has_hand_data ? 1 : 0, 'i32');
+
 		return true;
 	},
 

modules/webxr/native/webxr.externs.js

@@ -229,13 +229,27 @@ XRFrame.prototype.session;
 XRFrame.prototype.getViewerPose = function (referenceSpace) {};
 
 /**
- *
  * @param {XRSpace} space
  * @param {XRSpace} baseSpace
  * @return {XRPose}
  */
 XRFrame.prototype.getPose = function (space, baseSpace) {};
 
+/**
+ * @param {Array<XRSpace>} spaces
+ * @param {XRSpace} baseSpace
+ * @param {Float32Array} transforms
+ * @return {boolean}
+ */
+XRFrame.prototype.fillPoses = function (spaces, baseSpace, transforms) {};
+
+/**
+ * @param {Array<XRJointSpace>} jointSpaces
+ * @param {Float32Array} radii
+ * @return {boolean}
+ */
+XRFrame.prototype.fillJointRadii = function (jointSpaces, radii) {};
+
 /**
  * @constructor
  */
@@ -498,11 +512,51 @@ XRInputSource.prototype.targetRayMode;
  */
 XRInputSource.prototype.targetRaySpace;
 
+/**
+ * @type {?XRHand}
+ */
+XRInputSource.prototype.hand;
+
+/**
+ * @constructor
+ */
+function XRHand() {};
+
+/**
+ * Note: In fact, XRHand acts like a Map<string, XRJointSpace>, but I don't know
+ * how to represent that here. So, we're just giving the one method we call.
+ *
+ * @return {Array<XRJointSpace>}
+ */
+XRHand.prototype.values = function () {};
+
+/**
+ * @type {number}
+ */
+XRHand.prototype.size;
+
+/**
+ * @param {string} key
+ * @return {XRJointSpace}
+ */
+XRHand.prototype.get = function (key) {};
+
 /**
  * @constructor
  */
 function XRSpace() {};
 
+/**
+ * @constructor
+ * @extends {XRSpace}
+ */
+function XRJointSpace() {};
+
+/**
+ * @type {string}
+ */
+XRJointSpace.prototype.jointName;
+
 /**
  * @constructor
  */

modules/webxr/webxr_interface.cpp

@@ -41,6 +41,7 @@ void WebXRInterface::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_optional_features", "optional_features"), &WebXRInterface::set_optional_features);
 	ClassDB::bind_method(D_METHOD("get_optional_features"), &WebXRInterface::get_optional_features);
 	ClassDB::bind_method(D_METHOD("get_reference_space_type"), &WebXRInterface::get_reference_space_type);
+	ClassDB::bind_method(D_METHOD("get_enabled_features"), &WebXRInterface::get_enabled_features);
 	ClassDB::bind_method(D_METHOD("set_requested_reference_space_types", "requested_reference_space_types"), &WebXRInterface::set_requested_reference_space_types);
 	ClassDB::bind_method(D_METHOD("get_requested_reference_space_types"), &WebXRInterface::get_requested_reference_space_types);
 	ClassDB::bind_method(D_METHOD("is_input_source_active", "input_source_id"), &WebXRInterface::is_input_source_active);
@@ -56,6 +57,7 @@ void WebXRInterface::_bind_methods() {
 	ADD_PROPERTY(PropertyInfo(Variant::STRING, "optional_features", PROPERTY_HINT_NONE), "set_optional_features", "get_optional_features");
 	ADD_PROPERTY(PropertyInfo(Variant::STRING, "requested_reference_space_types", PROPERTY_HINT_NONE), "set_requested_reference_space_types", "get_requested_reference_space_types");
 	ADD_PROPERTY(PropertyInfo(Variant::STRING, "reference_space_type", PROPERTY_HINT_NONE), "", "get_reference_space_type");
+	ADD_PROPERTY(PropertyInfo(Variant::STRING, "enabled_features", PROPERTY_HINT_NONE), "", "get_enabled_features");
 	ADD_PROPERTY(PropertyInfo(Variant::STRING, "visibility_state", PROPERTY_HINT_NONE), "", "get_visibility_state");
 
 	ADD_SIGNAL(MethodInfo("session_supported", PropertyInfo(Variant::STRING, "session_mode"), PropertyInfo(Variant::BOOL, "supported")));

modules/webxr/webxr_interface.h

@@ -62,6 +62,7 @@ public:
 	virtual void set_requested_reference_space_types(String p_requested_reference_space_types) = 0;
 	virtual String get_requested_reference_space_types() const = 0;
 	virtual String get_reference_space_type() const = 0;
+	virtual String get_enabled_features() const = 0;
 	virtual bool is_input_source_active(int p_input_source_id) const = 0;
 	virtual Ref<XRPositionalTracker> get_input_source_tracker(int p_input_source_id) const = 0;
 	virtual TargetRayMode get_input_source_target_ray_mode(int p_input_source_id) const = 0;

modules/webxr/webxr_interface_js.cpp

@@ -41,6 +41,7 @@
 #include "scene/main/window.h"
 #include "servers/rendering/renderer_compositor.h"
 #include "servers/rendering/rendering_server_globals.h"
+#include "servers/xr/xr_hand_tracker.h"
 
 #include <emscripten.h>
 #include <stdlib.h>
@@ -49,22 +50,23 @@ void _emwebxr_on_session_supported(char *p_session_mode, int p_supported) {
 	XRServer *xr_server = XRServer::get_singleton();
 	ERR_FAIL_NULL(xr_server);
 
-	Ref<XRInterface> interface = xr_server->find_interface("WebXR");
+	Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
 	ERR_FAIL_COND(interface.is_null());
 
 	String session_mode = String(p_session_mode);
 	interface->emit_signal(SNAME("session_supported"), session_mode, p_supported ? true : false);
 }
 
-void _emwebxr_on_session_started(char *p_reference_space_type) {
+void _emwebxr_on_session_started(char *p_reference_space_type, char *p_enabled_features) {
 	XRServer *xr_server = XRServer::get_singleton();
 	ERR_FAIL_NULL(xr_server);
 
-	Ref<XRInterface> interface = xr_server->find_interface("WebXR");
+	Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
 	ERR_FAIL_COND(interface.is_null());
 
 	String reference_space_type = String(p_reference_space_type);
-	static_cast<WebXRInterfaceJS *>(interface.ptr())->_set_reference_space_type(reference_space_type);
+	interface->_set_reference_space_type(reference_space_type);
+	interface->_set_enabled_features(p_enabled_features);
 	interface->emit_signal(SNAME("session_started"));
 }
 
@@ -72,7 +74,7 @@ void _emwebxr_on_session_ended() {
 	XRServer *xr_server = XRServer::get_singleton();
 	ERR_FAIL_NULL(xr_server);
 
-	Ref<XRInterface> interface = xr_server->find_interface("WebXR");
+	Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
 	ERR_FAIL_COND(interface.is_null());
 
 	interface->uninitialize();
@@ -83,7 +85,7 @@ void _emwebxr_on_session_failed(char *p_message) {
 	XRServer *xr_server = XRServer::get_singleton();
 	ERR_FAIL_NULL(xr_server);
 
-	Ref<XRInterface> interface = xr_server->find_interface("WebXR");
+	Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
 	ERR_FAIL_COND(interface.is_null());
 
 	interface->uninitialize();
@@ -96,17 +98,17 @@ extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_input_event(int p_event_type, i
 	XRServer *xr_server = XRServer::get_singleton();
 	ERR_FAIL_NULL(xr_server);
 
-	Ref<XRInterface> interface = xr_server->find_interface("WebXR");
+	Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
 	ERR_FAIL_COND(interface.is_null());
 
-	((WebXRInterfaceJS *)interface.ptr())->_on_input_event(p_event_type, p_input_source_id);
+	interface->_on_input_event(p_event_type, p_input_source_id);
 }
 
 extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_simple_event(char *p_signal_name) {
 	XRServer *xr_server = XRServer::get_singleton();
 	ERR_FAIL_NULL(xr_server);
 
-	Ref<XRInterface> interface = xr_server->find_interface("WebXR");
+	Ref<WebXRInterfaceJS> interface = xr_server->find_interface("WebXR");
 	ERR_FAIL_COND(interface.is_null());
 
 	StringName signal_name = StringName(p_signal_name);
@@ -149,14 +151,14 @@ String WebXRInterfaceJS::get_requested_reference_space_types() const {
 	return requested_reference_space_types;
 }
 
-void WebXRInterfaceJS::_set_reference_space_type(String p_reference_space_type) {
-	reference_space_type = p_reference_space_type;
-}
-
 String WebXRInterfaceJS::get_reference_space_type() const {
 	return reference_space_type;
 }
 
+String WebXRInterfaceJS::get_enabled_features() const {
+	return enabled_features;
+}
+
 bool WebXRInterfaceJS::is_input_source_active(int p_input_source_id) const {
 	ERR_FAIL_INDEX_V(p_input_source_id, input_source_count, false);
 	return input_sources[p_input_source_id].active;
@@ -256,7 +258,9 @@ bool WebXRInterfaceJS::initialize() {
 			return false;
 		}
 
-		// we must create a tracker for our head
+		enabled_features.clear();
+
+		// We must create a tracker for our head.
 		head_transform.basis = Basis();
 		head_transform.origin = Vector3();
 		head_tracker.instantiate();
@@ -265,7 +269,7 @@ bool WebXRInterfaceJS::initialize() {
 		head_tracker->set_tracker_desc("Players head");
 		xr_server->add_tracker(head_tracker);
 
-		// make this our primary interface
+		// Make this our primary interface.
 		xr_server->set_primary_interface(this);
 
 		// Clear render_targetsize to make sure it gets reset to the new size.
@@ -301,6 +305,14 @@ void WebXRInterfaceJS::uninitialize() {
 				head_tracker.unref();
 			}
 
+			for (int i = 0; i < HAND_MAX; i++) {
+				if (hand_trackers[i].is_valid()) {
+					xr_server->remove_hand_tracker(i == 0 ? "/user/left" : "/user/right");
+
+					hand_trackers[i].unref();
+				}
+			}
+
 			if (xr_server->get_primary_interface() == this) {
 				// no longer our primary interface
 				xr_server->set_primary_interface(nullptr);
@@ -321,7 +333,8 @@ void WebXRInterfaceJS::uninitialize() {
 		}
 
 		texture_cache.clear();
-		reference_space_type = "";
+		reference_space_type.clear();
+		enabled_features.clear();
 		initialized = false;
 	};
 };
@@ -572,6 +585,9 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) {
 	float buttons[10];
 	int axes_count;
 	float axes[10];
+	int has_hand_data;
+	float hand_joints[WEBXR_HAND_JOINT_MAX * 16];
+	float hand_radii[WEBXR_HAND_JOINT_MAX];
 
 	input_source.active = godot_webxr_update_input_source(
 			p_input_source_id,
@@ -584,7 +600,10 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) {
 			&button_count,
 			buttons,
 			&axes_count,
-			axes);
+			axes,
+			&has_hand_data,
+			hand_joints,
+			hand_radii);
 
 	if (!input_source.active) {
 		if (input_source.tracker.is_valid()) {
@@ -683,6 +702,56 @@ void WebXRInterfaceJS::_update_input_source(int p_input_source_id) {
 			touches[touch_index].position = position;
 		}
 	}
+
+	if (p_input_source_id < 2) {
+		Ref<XRHandTracker> hand_tracker = hand_trackers[p_input_source_id];
+		if (has_hand_data) {
+			// Transform orientations to match Godot Humanoid skeleton.
+			const Basis bone_adjustment(
+					Vector3(-1.0, 0.0, 0.0),
+					Vector3(0.0, 0.0, -1.0),
+					Vector3(0.0, -1.0, 0.0));
+
+			if (unlikely(hand_tracker.is_null())) {
+				hand_tracker.instantiate();
+				hand_tracker->set_hand(p_input_source_id == 0 ? XRHandTracker::HAND_LEFT : XRHandTracker::HAND_RIGHT);
+
+				// These flags always apply, since WebXR doesn't give us enough insight to be more fine grained.
+				BitField<XRHandTracker::HandJointFlags> joint_flags(XRHandTracker::HAND_JOINT_FLAG_POSITION_VALID | XRHandTracker::HAND_JOINT_FLAG_ORIENTATION_VALID | XRHandTracker::HAND_JOINT_FLAG_POSITION_TRACKED | XRHandTracker::HAND_JOINT_FLAG_ORIENTATION_TRACKED);
+				for (int godot_joint = 0; godot_joint < XRHandTracker::HAND_JOINT_MAX; godot_joint++) {
+					hand_tracker->set_hand_joint_flags((XRHandTracker::HandJoint)godot_joint, joint_flags);
+				}
+
+				hand_trackers[p_input_source_id] = hand_tracker;
+				xr_server->add_hand_tracker(p_input_source_id == 0 ? "/user/left" : "/user/right", hand_tracker);
+			}
+
+			hand_tracker->set_has_tracking_data(true);
+			for (int webxr_joint = 0; webxr_joint < WEBXR_HAND_JOINT_MAX; webxr_joint++) {
+				XRHandTracker::HandJoint godot_joint = (XRHandTracker::HandJoint)(webxr_joint + 1);
+
+				Transform3D joint_transform = _js_matrix_to_transform(hand_joints + (16 * webxr_joint));
+				joint_transform.basis *= bone_adjustment;
+				hand_tracker->set_hand_joint_transform(godot_joint, joint_transform);
+
+				hand_tracker->set_hand_joint_radius(godot_joint, hand_radii[webxr_joint]);
+			}
+
+			// WebXR doesn't have a palm joint, so we calculate it by finding the middle of the middle finger metacarpal bone.
+			{
+				// 10 is the WebXR middle finger metacarpal joint, and 12 is the offset to the transform origin.
+				const float *start_pos = hand_joints + (10 * 16) + 12;
+				// 11 is the WebXR middle finger phalanx proximal joint, and 12 is the offset to the transform origin.
+				const float *end_pos = hand_joints + (11 * 16) + 12;
+				Transform3D palm_transform;
+				palm_transform.origin = (Vector3(start_pos[0], start_pos[1], start_pos[2]) + Vector3(end_pos[0], end_pos[1], end_pos[2])) / 2.0;
+				hand_tracker->set_hand_joint_transform(XRHandTracker::HAND_JOINT_PALM, palm_transform);
+			}
+
+		} else if (hand_tracker.is_valid()) {
+			hand_tracker->set_has_tracking_data(false);
+		}
+	}
 }
 
 void WebXRInterfaceJS::_on_input_event(int p_event_type, int p_input_source_id) {

modules/webxr/webxr_interface_js.h

@@ -56,6 +56,7 @@ private:
 	String optional_features;
 	String requested_reference_space_types;
 	String reference_space_type;
+	String enabled_features;
 
 	Size2 render_targetsize;
 	RBMap<unsigned int, RID> texture_cache;
@@ -73,6 +74,10 @@ private:
 		int touch_index = -1;
 	} input_sources[input_source_count];
 
+	static const int WEBXR_HAND_JOINT_MAX = 25;
+	static const int HAND_MAX = 2;
+	Ref<XRHandTracker> hand_trackers[HAND_MAX];
+
 	RID color_texture;
 	RID depth_texture;
 
@@ -94,8 +99,8 @@ public:
 	virtual String get_optional_features() const override;
 	virtual void set_requested_reference_space_types(String p_requested_reference_space_types) override;
 	virtual String get_requested_reference_space_types() const override;
-	void _set_reference_space_type(String p_reference_space_type);
 	virtual String get_reference_space_type() const override;
+	virtual String get_enabled_features() const override;
 	virtual bool is_input_source_active(int p_input_source_id) const override;
 	virtual Ref<XRPositionalTracker> get_input_source_tracker(int p_input_source_id) const override;
 	virtual TargetRayMode get_input_source_target_ray_mode(int p_input_source_id) const override;
@@ -129,6 +134,9 @@ public:
 
 	void _on_input_event(int p_event_type, int p_input_source_id);
 
+	inline void _set_reference_space_type(String p_reference_space_type) { reference_space_type = p_reference_space_type; }
+	inline void _set_enabled_features(String p_enabled_features) { enabled_features = p_enabled_features; }
+
 	WebXRInterfaceJS();
 	~WebXRInterfaceJS();