Merge pull request #91485 from dsnopek/openxr-composition-layers-hole-punch

OpenXR: Allow composition layers to enable hole punching

modules/openxr/doc_classes/OpenXRCompositionLayer.xml

@@ -32,6 +32,10 @@
 			Enables the blending the layer using its alpha channel.
 			Can be combined with [member Viewport.transparent_bg] to give the layer a transparent background.
 		</member>
+		<member name="enable_hole_punch" type="bool" setter="set_enable_hole_punch" getter="get_enable_hole_punch" default="false">
+			Enables a technique called "hole punching", which allows putting the composition layer behind the main projection layer (i.e. setting [member sort_order] to a negative value) while "punching a hole" through everything rendered by Godot so that the layer is still visible.
+			This can be used to create the illusion that the composition layer exists in the same 3D space as everything rendered by Godot, allowing objects to appear to pass both behind or in front of the composition layer.
+		</member>
 		<member name="layer_viewport" type="SubViewport" setter="set_layer_viewport" getter="get_layer_viewport">
 			The [SubViewport] to render on the composition layer.
 		</member>

modules/openxr/scene/openxr_composition_layer.cpp

@@ -40,6 +40,13 @@
 
 HashSet<SubViewport *> OpenXRCompositionLayer::viewports_in_use;
 
+static const char *HOLE_PUNCH_SHADER_CODE =
+		"shader_type spatial;\n"
+		"render_mode blend_mix, depth_draw_opaque, cull_back, shadow_to_opacity, shadows_disabled;\n"
+		"void fragment() {\n"
+		"\tALBEDO = vec3(0.0, 0.0, 0.0);\n"
+		"}\n";
+
 OpenXRCompositionLayer::OpenXRCompositionLayer() {
 	openxr_api = OpenXRAPI::get_singleton();
 	composition_layer_extension = OpenXRCompositionLayerExtension::get_singleton();
@@ -80,6 +87,9 @@ void OpenXRCompositionLayer::_bind_methods() {
 	ClassDB::bind_method(D_METHOD("set_layer_viewport", "viewport"), &OpenXRCompositionLayer::set_layer_viewport);
 	ClassDB::bind_method(D_METHOD("get_layer_viewport"), &OpenXRCompositionLayer::get_layer_viewport);
 
+	ClassDB::bind_method(D_METHOD("set_enable_hole_punch", "enable"), &OpenXRCompositionLayer::set_enable_hole_punch);
+	ClassDB::bind_method(D_METHOD("get_enable_hole_punch"), &OpenXRCompositionLayer::get_enable_hole_punch);
+
 	ClassDB::bind_method(D_METHOD("set_sort_order", "order"), &OpenXRCompositionLayer::set_sort_order);
 	ClassDB::bind_method(D_METHOD("get_sort_order"), &OpenXRCompositionLayer::get_sort_order);
 
@@ -93,6 +103,16 @@ void OpenXRCompositionLayer::_bind_methods() {
 	ADD_PROPERTY(PropertyInfo(Variant::OBJECT, "layer_viewport", PROPERTY_HINT_NODE_TYPE, "SubViewport"), "set_layer_viewport", "get_layer_viewport");
 	ADD_PROPERTY(PropertyInfo(Variant::INT, "sort_order", PROPERTY_HINT_NONE, ""), "set_sort_order", "get_sort_order");
 	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "alpha_blend", PROPERTY_HINT_NONE, ""), "set_alpha_blend", "get_alpha_blend");
+	ADD_PROPERTY(PropertyInfo(Variant::BOOL, "enable_hole_punch", PROPERTY_HINT_NONE, ""), "set_enable_hole_punch", "get_enable_hole_punch");
+}
+
+bool OpenXRCompositionLayer::_should_use_fallback_node() {
+	if (Engine::get_singleton()->is_editor_hint()) {
+		return true;
+	} else if (openxr_session_running) {
+		return enable_hole_punch || !is_natively_supported();
+	}
+	return false;
 }
 
 void OpenXRCompositionLayer::_create_fallback_node() {
@@ -103,21 +123,27 @@ void OpenXRCompositionLayer::_create_fallback_node() {
 	should_update_fallback_mesh = true;
 }
 
+void OpenXRCompositionLayer::_remove_fallback_node() {
+	ERR_FAIL_COND(fallback != nullptr);
+	remove_child(fallback);
+	fallback->queue_free();
+	fallback = nullptr;
+}
+
 void OpenXRCompositionLayer::_on_openxr_session_begun() {
-	if (!is_natively_supported()) {
-		if (!fallback) {
-			_create_fallback_node();
-		}
-	} else if (layer_viewport && is_visible() && is_inside_tree()) {
+	openxr_session_running = true;
+	if (layer_viewport && is_natively_supported() && is_visible() && is_inside_tree()) {
 		openxr_layer_provider->set_viewport(layer_viewport->get_viewport_rid(), layer_viewport->get_size());
 	}
+	if (!fallback && _should_use_fallback_node()) {
+		_create_fallback_node();
+	}
 }
 
 void OpenXRCompositionLayer::_on_openxr_session_stopping() {
-	if (fallback && !Engine::get_singleton()->is_editor_hint()) {
-		fallback->queue_free();
-		remove_child(fallback);
-		fallback = nullptr;
+	openxr_session_running = false;
+	if (fallback && !_should_use_fallback_node()) {
+		_remove_fallback_node();
 	} else {
 		openxr_layer_provider->set_viewport(RID(), Size2i());
 	}
@@ -152,7 +178,7 @@ void OpenXRCompositionLayer::set_layer_viewport(SubViewport *p_viewport) {
 
 	if (fallback) {
 		_reset_fallback_material();
-	} else if (openxr_api && openxr_api->is_running() && is_visible() && is_inside_tree()) {
+	} else if (openxr_session_running && is_visible() && is_inside_tree()) {
 		if (layer_viewport) {
 			openxr_layer_provider->set_viewport(layer_viewport->get_viewport_rid(), layer_viewport->get_size());
 		} else {
@@ -165,9 +191,33 @@ SubViewport *OpenXRCompositionLayer::get_layer_viewport() const {
 	return layer_viewport;
 }
 
+void OpenXRCompositionLayer::set_enable_hole_punch(bool p_enable) {
+	if (enable_hole_punch == p_enable) {
+		return;
+	}
+
+	enable_hole_punch = p_enable;
+	if (_should_use_fallback_node()) {
+		if (fallback) {
+			_reset_fallback_material();
+		} else {
+			_create_fallback_node();
+		}
+	} else if (fallback) {
+		_remove_fallback_node();
+	}
+
+	update_configuration_warnings();
+}
+
+bool OpenXRCompositionLayer::get_enable_hole_punch() const {
+	return enable_hole_punch;
+}
+
 void OpenXRCompositionLayer::set_sort_order(int p_order) {
 	if (openxr_layer_provider) {
 		openxr_layer_provider->set_sort_order(p_order);
+		update_configuration_warnings();
 	}
 }
 
@@ -212,15 +262,28 @@ void OpenXRCompositionLayer::_reset_fallback_material() {
 		return;
 	}
 
-	if (layer_viewport) {
+	if (enable_hole_punch && !Engine::get_singleton()->is_editor_hint() && is_natively_supported()) {
+		Ref<ShaderMaterial> material = fallback->get_surface_override_material(0);
+		if (material.is_null()) {
+			Ref<Shader> shader;
+			shader.instantiate();
+			shader->set_code(HOLE_PUNCH_SHADER_CODE);
+
+			material.instantiate();
+			material->set_shader(shader);
+
+			fallback->set_surface_override_material(0, material);
+		}
+	} else if (layer_viewport) {
 		Ref<StandardMaterial3D> material = fallback->get_surface_override_material(0);
 		if (material.is_null()) {
 			material.instantiate();
 			material->set_shading_mode(StandardMaterial3D::SHADING_MODE_UNSHADED);
-			material->set_flag(StandardMaterial3D::FLAG_DISABLE_DEPTH_TEST, true);
 			material->set_local_to_scene(true);
 			fallback->set_surface_override_material(0, material);
 		}
+
+		material->set_flag(StandardMaterial3D::FLAG_DISABLE_DEPTH_TEST, !enable_hole_punch);
 		material->set_transparency(get_alpha_blend() ? StandardMaterial3D::TRANSPARENCY_ALPHA : StandardMaterial3D::TRANSPARENCY_DISABLED);
 
 		Ref<ViewportTexture> texture = material->get_texture(StandardMaterial3D::TEXTURE_ALBEDO);
@@ -260,7 +323,7 @@ void OpenXRCompositionLayer::_notification(int p_what) {
 			}
 		} break;
 		case NOTIFICATION_VISIBILITY_CHANGED: {
-			if (!fallback && openxr_api && openxr_api->is_running() && is_inside_tree()) {
+			if (!fallback && openxr_session_running && is_inside_tree()) {
 				if (layer_viewport && is_visible()) {
 					openxr_layer_provider->set_viewport(layer_viewport->get_viewport_rid(), layer_viewport->get_size());
 				} else {
@@ -277,7 +340,7 @@ void OpenXRCompositionLayer::_notification(int p_what) {
 				composition_layer_extension->register_viewport_composition_layer_provider(openxr_layer_provider);
 			}
 
-			if (!fallback && layer_viewport && openxr_api && openxr_api->is_running() && is_visible()) {
+			if (!fallback && layer_viewport && openxr_session_running && is_visible()) {
 				openxr_layer_provider->set_viewport(layer_viewport->get_viewport_rid(), layer_viewport->get_size());
 			}
 		} break;
@@ -347,5 +410,9 @@ PackedStringArray OpenXRCompositionLayer::get_configuration_warnings() const {
 		warnings.push_back(RTR("OpenXR composition layers must have orthonormalized transforms (ie. no scale or shearing)."));
 	}
 
+	if (enable_hole_punch && get_sort_order() >= 0) {
+		warnings.push_back(RTR("Hole punching won't work as expected unless the sort order is less than zero."));
+	}
+
 	return warnings;
 }

modules/openxr/scene/openxr_composition_layer.h

@@ -46,13 +46,17 @@ class OpenXRCompositionLayer : public Node3D {
 	GDCLASS(OpenXRCompositionLayer, Node3D);
 
 	SubViewport *layer_viewport = nullptr;
+	bool enable_hole_punch = false;
 	MeshInstance3D *fallback = nullptr;
 	bool should_update_fallback_mesh = false;
+	bool openxr_session_running = false;
 
 	Dictionary extension_property_values;
 
+	bool _should_use_fallback_node();
 	void _create_fallback_node();
 	void _reset_fallback_material();
+	void _remove_fallback_node();
 
 protected:
 	OpenXRAPI *openxr_api = nullptr;
@@ -79,6 +83,9 @@ public:
 	void set_layer_viewport(SubViewport *p_viewport);
 	SubViewport *get_layer_viewport() const;
 
+	void set_enable_hole_punch(bool p_enable);
+	bool get_enable_hole_punch() const;
+
 	void set_sort_order(int p_order);
 	int get_sort_order() const;