Fix touch events when using smartphone AR with WebXR

modules/webxr/doc_classes/WebXRInterface.xml

@@ -109,6 +109,14 @@
 				- [signal squeezestart]
 			</description>
 		</method>
+		<method name="get_controller_target_ray_mode" qualifiers="const">
+			<return type="int" enum="WebXRInterface.TargetRayMode" />
+			<argument index="0" name="controller_id" type="int" />
+			<description>
+				Returns the target ray mode for the given [code]controller_id[/code].
+				This can help interpret the input coming from that controller. See [url=https://developer.mozilla.org/en-US/docs/Web/API/XRInputSource/targetRayMode]XRInputSource.targetRayMode[/url] for more information.
+			</description>
+		</method>
 		<method name="is_session_supported">
 			<return type="void" />
 			<argument index="0" name="session_mode" type="String" />
@@ -240,5 +248,17 @@
 		</signal>
 	</signals>
 	<constants>
+		<constant name="TARGET_RAY_MODE_UNKNOWN" value="0" enum="TargetRayMode">
+			We don't know the the target ray mode.
+		</constant>
+		<constant name="TARGET_RAY_MODE_GAZE" value="1" enum="TargetRayMode">
+			Target ray originates at the viewer's eyes and points in the direction they are looking.
+		</constant>
+		<constant name="TARGET_RAY_MODE_TRACKED_POINTER" value="2" enum="TargetRayMode">
+			Target ray from a handheld pointer, most likely a VR touch controller.
+		</constant>
+		<constant name="TARGET_RAY_MODE_SCREEN" value="3" enum="TargetRayMode">
+			Target ray from touch screen, mouse or other tactile input device.
+		</constant>
 	</constants>
 </class>

modules/webxr/godot_webxr.h

@@ -37,12 +37,21 @@ extern "C" {
 
 #include "stddef.h"
 
+enum WebXRInputEvent {
+	WEBXR_INPUT_EVENT_SELECTSTART,
+	WEBXR_INPUT_EVENT_SELECTEND,
+	WEBXR_INPUT_EVENT_SELECT,
+	WEBXR_INPUT_EVENT_SQUEEZESTART,
+	WEBXR_INPUT_EVENT_SQUEEZEEND,
+	WEBXR_INPUT_EVENT_SQUEEZE,
+};
+
 typedef void (*GodotWebXRSupportedCallback)(char *p_session_mode, int p_supported);
 typedef void (*GodotWebXRStartedCallback)(char *p_reference_space_type);
 typedef void (*GodotWebXREndedCallback)();
 typedef void (*GodotWebXRFailedCallback)(char *p_message);
 typedef void (*GodotWebXRControllerCallback)();
-typedef void (*GodotWebXRInputEventCallback)(char *p_signal_name, int p_controller_id);
+typedef void (*GodotWebXRInputEventCallback)(int p_event_type, int p_controller_id);
 typedef void (*GodotWebXRSimpleEventCallback)(char *p_signal_name);
 
 extern int godot_webxr_is_supported();
@@ -73,6 +82,7 @@ extern int godot_webxr_is_controller_connected(int p_controller);
 extern float *godot_webxr_get_controller_transform(int p_controller);
 extern int *godot_webxr_get_controller_buttons(int p_controller);
 extern int *godot_webxr_get_controller_axes(int p_controller);
+extern int godot_webxr_get_controller_target_ray_mode(int p_controller);
 
 extern char *godot_webxr_get_visibility_state();
 extern int *godot_webxr_get_bounds_geometry();

modules/webxr/native/library_godot_webxr.js

@@ -186,7 +186,7 @@ const GodotWebXR = {
 		// the first element, and the right hand is the second element, and any
 		// others placed at the 3rd position and up.
 		sampleControllers: () => {
-			if (!GodotWebXR.session || !GodotWebXR.frame) {
+			if (!GodotWebXR.session) {
 				return;
 			}
 
@@ -279,11 +279,12 @@ const GodotWebXR = {
 				}
 			});
 
-			['selectstart', 'select', 'selectend', 'squeezestart', 'squeeze', 'squeezeend'].forEach((input_event) => {
+			['selectstart', 'selectend', 'select', 'squeezestart', 'squeezeend', 'squeeze'].forEach((input_event, index) => {
 				session.addEventListener(input_event, function (evt) {
-					const c_str = GodotRuntime.allocString(input_event);
-					oninputevent(c_str, GodotWebXR.getControllerId(evt.inputSource));
-					GodotRuntime.free(c_str);
+					// Some controllers won't exist until an event occurs,
+					// for example, with "screen" input sources (touch).
+					GodotWebXR.sampleControllers();
+					oninputevent(index, GodotWebXR.getControllerId(evt.inputSource));
 				});
 			});
 
@@ -572,6 +573,35 @@ const GodotWebXR = {
 		return buf;
 	},
 
+	godot_webxr_get_controller_target_ray_mode__proxy: 'sync',
+	godot_webxr_get_controller_target_ray_mode__sig: 'ii',
+	godot_webxr_get_controller_target_ray_mode: function (p_controller) {
+		if (p_controller < 0 || p_controller >= GodotWebXR.controllers.length) {
+			return 0;
+		}
+
+		const controller = GodotWebXR.controllers[p_controller];
+		if (!controller) {
+			return 0;
+		}
+
+		switch (controller.targetRayMode) {
+		case 'gaze':
+			return 1;
+
+		case 'tracked-pointer':
+			return 2;
+
+		case 'screen':
+			return 3;
+
+		default:
+			break;
+		}
+
+		return 0;
+	},
+
 	godot_webxr_get_visibility_state__proxy: 'sync',
 	godot_webxr_get_visibility_state__sig: 'i',
 	godot_webxr_get_visibility_state: function () {

modules/webxr/webxr_interface.cpp

@@ -43,6 +43,7 @@ void WebXRInterface::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_requested_reference_space_types", "requested_reference_space_types"), &WebXRInterface::set_requested_reference_space_types);
 	ClassDB::bind_method(D_METHOD("get_requested_reference_space_types"), &WebXRInterface::get_requested_reference_space_types);
 	ClassDB::bind_method(D_METHOD("get_controller", "controller_id"), &WebXRInterface::get_controller);
+	ClassDB::bind_method(D_METHOD("get_controller_target_ray_mode", "controller_id"), &WebXRInterface::get_controller_target_ray_mode);
 	ClassDB::bind_method(D_METHOD("get_visibility_state"), &WebXRInterface::get_visibility_state);
 	ClassDB::bind_method(D_METHOD("get_bounds_geometry"), &WebXRInterface::get_bounds_geometry);
 
@@ -68,4 +69,9 @@ void WebXRInterface::_bind_methods() {
 
 	ADD_SIGNAL(MethodInfo("visibility_state_changed"));
 	ADD_SIGNAL(MethodInfo("reference_space_reset"));
+
+	BIND_ENUM_CONSTANT(TARGET_RAY_MODE_UNKNOWN);
+	BIND_ENUM_CONSTANT(TARGET_RAY_MODE_GAZE);
+	BIND_ENUM_CONSTANT(TARGET_RAY_MODE_TRACKED_POINTER);
+	BIND_ENUM_CONSTANT(TARGET_RAY_MODE_SCREEN);
 }

modules/webxr/webxr_interface.h

@@ -47,6 +47,13 @@ protected:
 	static void _bind_methods();
 
 public:
+	enum TargetRayMode {
+		TARGET_RAY_MODE_UNKNOWN,
+		TARGET_RAY_MODE_GAZE,
+		TARGET_RAY_MODE_TRACKED_POINTER,
+		TARGET_RAY_MODE_SCREEN,
+	};
+
 	virtual void is_session_supported(const String &p_session_mode) = 0;
 	virtual void set_session_mode(String p_session_mode) = 0;
 	virtual String get_session_mode() const = 0;
@@ -58,8 +65,11 @@ public:
 	virtual String get_requested_reference_space_types() const = 0;
 	virtual String get_reference_space_type() const = 0;
 	virtual Ref<ARVRPositionalTracker> get_controller(int p_controller_id) const = 0;
+	virtual TargetRayMode get_controller_target_ray_mode(int p_controller_id) const = 0;
 	virtual String get_visibility_state() const = 0;
 	virtual PoolVector3Array get_bounds_geometry() const = 0;
 };
 
+VARIANT_ENUM_CAST(WebXRInterface::TargetRayMode);
+
 #endif // WEBXR_INTERFACE_H

modules/webxr/webxr_interface_js.cpp

@@ -96,15 +96,14 @@ void _emwebxr_on_controller_changed() {
 	((WebXRInterfaceJS *)interface.ptr())->_on_controller_changed();
 }
 
-extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_input_event(char *p_signal_name, int p_input_source) {
+extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_input_event(int p_event_type, int p_input_source) {
 	ARVRServer *arvr_server = ARVRServer::get_singleton();
 	ERR_FAIL_NULL(arvr_server);
 
 	Ref<ARVRInterface> interface = arvr_server->find_interface("WebXR");
 	ERR_FAIL_COND(interface.is_null());
 
-	StringName signal_name = StringName(p_signal_name);
-	interface->emit_signal(signal_name, p_input_source + 1);
+	((WebXRInterfaceJS *)interface.ptr())->_on_input_event(p_event_type, p_input_source);
 }
 
 extern "C" EMSCRIPTEN_KEEPALIVE void _emwebxr_on_simple_event(char *p_signal_name) {
@@ -169,6 +168,15 @@ Ref<ARVRPositionalTracker> WebXRInterfaceJS::get_controller(int p_controller_id)
 	return arvr_server->find_by_type_and_id(ARVRServer::TRACKER_CONTROLLER, p_controller_id);
 }
 
+WebXRInterface::TargetRayMode WebXRInterfaceJS::get_controller_target_ray_mode(int p_controller_id) const {
+	ERR_FAIL_COND_V(p_controller_id <= 0, WebXRInterface::TARGET_RAY_MODE_UNKNOWN);
+
+	ARVRServer *arvr_server = ARVRServer::get_singleton();
+	ERR_FAIL_NULL_V(arvr_server, WebXRInterface::TARGET_RAY_MODE_UNKNOWN);
+
+	return (WebXRInterface::TargetRayMode)godot_webxr_get_controller_target_ray_mode(p_controller_id - 1);
+}
+
 String WebXRInterfaceJS::get_visibility_state() const {
 	char *c_str = godot_webxr_get_visibility_state();
 	if (c_str) {
@@ -228,6 +236,10 @@ bool WebXRInterfaceJS::initialize() {
 		// make this our primary interface
 		arvr_server->set_primary_interface(this);
 
+		// Clear state variables.
+		memset(controllers_state, 0, sizeof controllers_state);
+		memset(touching, 0, sizeof touching);
+
 		// Clear render_targetsize to make sure it gets reset to the new size.
 		// Clearing in uninitialize() doesn't work because a frame can still be
 		// rendered after it's called, which will fill render_targetsize again.
@@ -395,6 +407,7 @@ void WebXRInterfaceJS::_update_tracker(int p_controller_id) {
 		}
 
 		InputDefault *input = (InputDefault *)Input::get_singleton();
+		int joy_id = p_controller_id + 100;
 
 		float *tracker_matrix = godot_webxr_get_controller_transform(p_controller_id);
 		if (tracker_matrix) {
@@ -407,17 +420,39 @@ void WebXRInterfaceJS::_update_tracker(int p_controller_id) {
 		int *buttons = godot_webxr_get_controller_buttons(p_controller_id);
 		if (buttons) {
 			for (int i = 0; i < buttons[0]; i++) {
-				input->joy_button(p_controller_id + 100, i, *((float *)buttons + (i + 1)));
+				input->joy_button(joy_id, i, *((float *)buttons + (i + 1)));
 			}
 			free(buttons);
 		}
 
 		int *axes = godot_webxr_get_controller_axes(p_controller_id);
 		if (axes) {
+			WebXRInterface::TargetRayMode target_ray_mode = (WebXRInterface::TargetRayMode)godot_webxr_get_controller_target_ray_mode(p_controller_id);
+			if (target_ray_mode == WebXRInterface::TARGET_RAY_MODE_SCREEN) {
+				int touch_index = _get_touch_index(p_controller_id);
+				if (touch_index < 5 && touching[touch_index]) {
+					Vector2 joy_vector = _get_joy_vector_from_axes(axes);
+					Vector2 previous_joy_vector(input->get_joy_axis(joy_id, 0), input->get_joy_axis(joy_id, 1));
+
+					if (!joy_vector.is_equal_approx(previous_joy_vector)) {
+						Vector2 position = _get_screen_position_from_joy_vector(joy_vector);
+						Vector2 previous_position = _get_screen_position_from_joy_vector(previous_joy_vector);
+
+						Ref<InputEventScreenDrag> event;
+						event.instance();
+						event->set_index(touch_index);
+						event->set_position(position);
+						event->set_relative(position - previous_position);
+						input->parse_input_event(event);
+					}
+				}
+			}
+
 			for (int i = 0; i < axes[0]; i++) {
 				float value = *((float *)axes + (i + 1));
-				input->joy_axis(p_controller_id + 100, i, value);
+				input->joy_axis(joy_id, i, value);
 			}
+
 			free(axes);
 		}
 	} else if (tracker.is_valid()) {
@@ -437,6 +472,102 @@ void WebXRInterfaceJS::_on_controller_changed() {
 	}
 }
 
+void WebXRInterfaceJS::_on_input_event(int p_event_type, int p_input_source) {
+	if (p_event_type == WEBXR_INPUT_EVENT_SELECTSTART || p_event_type == WEBXR_INPUT_EVENT_SELECTEND) {
+		int target_ray_mode = godot_webxr_get_controller_target_ray_mode(p_input_source);
+		if (target_ray_mode == WebXRInterface::TARGET_RAY_MODE_SCREEN) {
+			int touch_index = _get_touch_index(p_input_source);
+			if (touch_index < 5) {
+				touching[touch_index] = (p_event_type == WEBXR_INPUT_EVENT_SELECTSTART);
+			}
+
+			int *axes = godot_webxr_get_controller_axes(p_input_source);
+			if (axes) {
+				Vector2 joy_vector = _get_joy_vector_from_axes(axes);
+				Vector2 position = _get_screen_position_from_joy_vector(joy_vector);
+				free(axes);
+
+				InputDefault *input = (InputDefault *)Input::get_singleton();
+
+				int joy_id = p_input_source + 100;
+				input->set_joy_axis(joy_id, 0, joy_vector.x);
+				input->set_joy_axis(joy_id, 1, joy_vector.y);
+
+				Ref<InputEventScreenTouch> event;
+				event.instance();
+				event->set_index(touch_index);
+				event->set_position(position);
+				event->set_pressed(p_event_type == WEBXR_INPUT_EVENT_SELECTSTART);
+				Input::get_singleton()->parse_input_event(event);
+			}
+		}
+	}
+
+	int controller_id = p_input_source + 1;
+	switch (p_event_type) {
+		case WEBXR_INPUT_EVENT_SELECTSTART:
+			emit_signal("selectstart", controller_id);
+			break;
+
+		case WEBXR_INPUT_EVENT_SELECTEND:
+			emit_signal("selectend", controller_id);
+			break;
+
+		case WEBXR_INPUT_EVENT_SELECT:
+			emit_signal("select", controller_id);
+			break;
+
+		case WEBXR_INPUT_EVENT_SQUEEZESTART:
+			emit_signal("squeezestart", controller_id);
+			break;
+
+		case WEBXR_INPUT_EVENT_SQUEEZEEND:
+			emit_signal("squeezeend", controller_id);
+			break;
+
+		case WEBXR_INPUT_EVENT_SQUEEZE:
+			emit_signal("squeeze", controller_id);
+			break;
+	}
+}
+
+int WebXRInterfaceJS::_get_touch_index(int p_input_source) {
+	int index = 0;
+	for (int i = 0; i < p_input_source; i++) {
+		if (godot_webxr_get_controller_target_ray_mode(i) == WebXRInterface::TARGET_RAY_MODE_SCREEN) {
+			index++;
+		}
+	}
+	return index;
+}
+
+Vector2 WebXRInterfaceJS::_get_joy_vector_from_axes(int *p_axes) {
+	float x_axis = 0.0;
+	float y_axis = 0.0;
+
+	if (p_axes[0] >= 2) {
+		x_axis = *((float *)p_axes + 1);
+		y_axis = *((float *)p_axes + 2);
+	}
+
+	return Vector2(x_axis, y_axis);
+}
+
+Vector2 WebXRInterfaceJS::_get_screen_position_from_joy_vector(const Vector2 &p_joy_vector) {
+	SceneTree *scene_tree = Object::cast_to<SceneTree>(OS::get_singleton()->get_main_loop());
+	if (!scene_tree) {
+		return Vector2();
+	}
+
+	Viewport *viewport = scene_tree->get_root();
+
+	// Invert the y-axis.
+	Vector2 position_percentage((p_joy_vector.x + 1.0f) / 2.0f, ((-p_joy_vector.y) + 1.0f) / 2.0f);
+	Vector2 position = viewport->get_size() * position_percentage;
+
+	return position;
+}
+
 void WebXRInterfaceJS::notification(int p_what) {
 	// Nothing to do here.
 }

modules/webxr/webxr_interface_js.h

@@ -54,11 +54,16 @@ private:
 	String reference_space_type;
 
 	bool controllers_state[2];
+	bool touching[5];
 	Size2 render_targetsize;
 
 	Transform _js_matrix_to_transform(float *p_js_matrix);
 	void _update_tracker(int p_controller_id);
 
+	Vector2 _get_joy_vector_from_axes(int *p_axes);
+	int _get_touch_index(int p_input_source);
+	Vector2 _get_screen_position_from_joy_vector(const Vector2 &p_joy_vector);
+
 public:
 	virtual void is_session_supported(const String &p_session_mode);
 	virtual void set_session_mode(String p_session_mode);
@@ -72,6 +77,7 @@ public:
 	void _set_reference_space_type(String p_reference_space_type);
 	virtual String get_reference_space_type() const;
 	virtual Ref<ARVRPositionalTracker> get_controller(int p_controller_id) const;
+	virtual TargetRayMode get_controller_target_ray_mode(int p_controller_id) const;
 	virtual String get_visibility_state() const;
 	virtual PoolVector3Array get_bounds_geometry() const;
 
@@ -92,6 +98,7 @@ public:
 	virtual void notification(int p_what);
 
 	void _on_controller_changed();
+	void _on_input_event(int p_event_type, int p_input_source);
 
 	WebXRInterfaceJS();
 	~WebXRInterfaceJS();