Merge pull request #109970 from allenwp/add-debanding-smaa-before-spatial-upscaler

Add debanding to SMAA and apply debanding before spatial upscalers.

servers/rendering/renderer_rd/effects/smaa.cpp

@@ -181,6 +181,11 @@ void SMAA::process(Ref<RenderSceneBuffersRD> p_render_buffers, RID p_source_colo
 
 	smaa.blend_push_constant.inv_size[0] = inv_size.x;
 	smaa.blend_push_constant.inv_size[1] = inv_size.y;
+	if (debanding_mode == DEBANDING_MODE_8_BIT) {
+		smaa.blend_push_constant.flags |= SMAA_BLEND_FLAG_USE_8_BIT_DEBANDING;
+	} else if (debanding_mode == DEBANDING_MODE_10_BIT) {
+		smaa.blend_push_constant.flags |= SMAA_BLEND_FLAG_USE_10_BIT_DEBANDING;
+	}
 
 	RID linear_sampler = material_storage->sampler_rd_get_default(RS::CANVAS_ITEM_TEXTURE_FILTER_LINEAR, RS::CANVAS_ITEM_TEXTURE_REPEAT_DISABLED);
 

servers/rendering/renderer_rd/effects/smaa.h

@@ -59,7 +59,7 @@ private:
 	struct SMAAEdgePushConstant {
 		float inv_size[2];
 		float threshold;
-		float reserved;
+		float pad;
 	};
 
 	struct SMAAWeightPushConstant {
@@ -71,7 +71,13 @@ private:
 
 	struct SMAABlendPushConstant {
 		float inv_size[2];
-		float reserved[2];
+		uint32_t flags;
+		float pad;
+	};
+
+	enum SMAABlendFlags {
+		SMAA_BLEND_FLAG_USE_8_BIT_DEBANDING = (1 << 0),
+		SMAA_BLEND_FLAG_USE_10_BIT_DEBANDING = (1 << 1),
 	};
 
 	struct SMAAEffect {
@@ -103,6 +109,13 @@ public:
 
 	void allocate_render_targets(Ref<RenderSceneBuffersRD> p_render_buffers);
 	void process(Ref<RenderSceneBuffersRD> p_render_buffers, RID p_source_color, RID p_dst_framebuffer);
+
+	enum DebandingMode {
+		DEBANDING_MODE_DISABLED,
+		DEBANDING_MODE_8_BIT,
+		DEBANDING_MODE_10_BIT,
+	};
+	DebandingMode debanding_mode = DEBANDING_MODE_DISABLED;
 };
 
 } // namespace RendererRD

servers/rendering/renderer_rd/renderer_scene_render_rd.cpp

@@ -685,36 +685,48 @@ void RendererSceneRenderRD::_render_buffers_post_process_and_tonemap(const Rende
 
 		RID dest_fb;
 		RD::DataFormat dest_fb_format;
+		RD::DataFormat format_for_debanding;
 		if (spatial_upscaler != nullptr || use_smaa) {
 			// If we use a spatial upscaler to upscale or SMAA to antialias we need to write our result into an intermediate buffer.
 			// Note that this is cached so we only create the texture the first time.
 			dest_fb_format = _render_buffers_get_color_format();
 			RID dest_texture = rb->create_texture(SNAME("Tonemapper"), SNAME("destination"), dest_fb_format, RD::TEXTURE_USAGE_SAMPLING_BIT | RD::TEXTURE_USAGE_STORAGE_BIT | RD::TEXTURE_USAGE_COLOR_ATTACHMENT_BIT, RD::TEXTURE_SAMPLES_1, Size2i(), 0, 1, true, true);
 			dest_fb = FramebufferCacheRD::get_singleton()->get_cache(dest_texture);
+			if (use_smaa) {
+				format_for_debanding = dest_fb_format;
+			} else {
+				// Debanding is currently not supported when using spatial upscaling, so apply it before scaling.
+				// This produces suboptimal results because the image will be modified by spatial upscaling after
+				// debanding has been applied. Ideally, debanding should be applied as the final step before quantization
+				// to integer values, but in the case of MetalFX, it may not be worth the performance cost of creating a new
+				// intermediate buffer. In the case of FSR 1.0, the work of adding debanding support hasn't been done yet.
+				// Assume that the DataFormat that will be used by spatial_upscaler is the same as render_target_get_color_format.
+				format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, tonemap.convert_to_srgb);
+			}
 		} else {
 			// If we do a bilinear upscale we just render into our render target and our shader will upscale automatically.
 			// Target size in this case is lying as we never get our real target size communicated.
 			// Bit nasty but...
 
 			if (dest_is_msaa_2d) {
-				// Assume that the DataFormat of render_target_get_rd_texture_msaa is the same as render_target_get_color_format.
-				dest_fb_format = texture_storage->render_target_get_color_format(using_hdr, tonemap.convert_to_srgb);
 				dest_fb = FramebufferCacheRD::get_singleton()->get_cache(texture_storage->render_target_get_rd_texture_msaa(render_target));
+				// Assume that the DataFormat of render_target_get_rd_texture_msaa is the same as render_target_get_color_format.
+				format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, tonemap.convert_to_srgb);
 				texture_storage->render_target_set_msaa_needs_resolve(render_target, true); // Make sure this gets resolved.
 			} else {
-				// Assume that the DataFormat of render_target_get_rd_framebuffer is the same as render_target_get_color_format.
-				dest_fb_format = texture_storage->render_target_get_color_format(using_hdr, tonemap.convert_to_srgb);
 				dest_fb = texture_storage->render_target_get_rd_framebuffer(render_target);
+				// Assume that the DataFormat of render_target_get_rd_framebuffer is the same as render_target_get_color_format.
+				format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, tonemap.convert_to_srgb);
 			}
 		}
 
 		if (rb->get_use_debanding()) {
-			if (dest_fb_format >= RD::DATA_FORMAT_R8_UNORM && dest_fb_format <= RD::DATA_FORMAT_A8B8G8R8_SRGB_PACK32) {
+			if (_is_8bit_data_format(format_for_debanding)) {
 				tonemap.debanding_mode = RendererRD::ToneMapper::TonemapSettings::DebandingMode::DEBANDING_MODE_8_BIT;
-			} else if (dest_fb_format >= RD::DATA_FORMAT_A2R10G10B10_UNORM_PACK32 && dest_fb_format <= RD::DATA_FORMAT_A2B10G10R10_SINT_PACK32) {
+			} else if (_is_10bit_data_format(format_for_debanding)) {
 				tonemap.debanding_mode = RendererRD::ToneMapper::TonemapSettings::DebandingMode::DEBANDING_MODE_10_BIT;
 			} else {
-				// In this case, debanding will be handled later when quantizing to an integer data format. (During blit, for example.)
+				// In this case, debanding will be handled later when quantizing to an integer data format. (During blit or SMAA, for example.)
 				tonemap.debanding_mode = RendererRD::ToneMapper::TonemapSettings::DebandingMode::DEBANDING_MODE_DISABLED;
 			}
 		} else {
@@ -730,6 +742,7 @@ void RendererSceneRenderRD::_render_buffers_post_process_and_tonemap(const Rende
 		RENDER_TIMESTAMP("SMAA");
 		RD::get_singleton()->draw_command_begin_label("SMAA");
 
+		bool using_hdr = texture_storage->render_target_is_using_hdr(render_target);
 		RID dest_fb;
 		if (spatial_upscaler) {
 			rb->create_texture(SNAME("SMAA"), SNAME("destination"), _render_buffers_get_color_format(), RD::TEXTURE_USAGE_SAMPLING_BIT | RD::TEXTURE_USAGE_STORAGE_BIT | RD::TEXTURE_USAGE_COLOR_ATTACHMENT_BIT, RD::TEXTURE_SAMPLES_1, Size2i(), 0, 1, true, true);
@@ -739,30 +752,78 @@ void RendererSceneRenderRD::_render_buffers_post_process_and_tonemap(const Rende
 				RID source_texture = rb->get_texture_slice(SNAME("Tonemapper"), SNAME("destination"), v, 0);
 
 				RID dest_texture;
+				RD::DataFormat format_for_debanding;
 				if (spatial_upscaler) {
 					dest_texture = rb->get_texture_slice(SNAME("SMAA"), SNAME("destination"), v, 0);
+					// Debanding is currently not supported when using spatial upscaling, so apply it before scaling.
+					// This produces suboptimal results because the image will be modified by spatial upscaling after
+					// debanding has been applied. Ideally, debanding should be applied as the final step before quantization
+					// to integer values, but in the case of MetalFX, it may not be worth the performance cost of creating a new
+					// intermediate buffer. In the case of FSR 1.0, the work of adding debanding support hasn't been done yet.
+					// Assume that the DataFormat that will be used by spatial_upscaler is the same as render_target_get_color_format.
+					format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, !using_hdr);
 				} else {
 					dest_texture = texture_storage->render_target_get_rd_texture_slice(render_target, v);
+					// Assume that the DataFormat is the same as render_target_get_color_format.
+					format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, !using_hdr);
 				}
 				dest_fb = FramebufferCacheRD::get_singleton()->get_cache(dest_texture);
 
+				if (rb->get_use_debanding()) {
+					if (_is_8bit_data_format(format_for_debanding)) {
+						smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_8_BIT;
+					} else if (_is_10bit_data_format(format_for_debanding)) {
+						smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_10_BIT;
+					} else {
+						// In this case, debanding will be handled later when quantizing to an integer data format. (During blit, for example.)
+						smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_DISABLED;
+					}
+				} else {
+					smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_DISABLED;
+				}
+
 				smaa->process(rb, source_texture, dest_fb);
 			}
 		} else {
 			RID source_texture = rb->get_texture(SNAME("Tonemapper"), SNAME("destination"));
+			RD::DataFormat format_for_debanding;
 
 			if (spatial_upscaler) {
 				RID dest_texture = rb->create_texture(SNAME("SMAA"), SNAME("destination"), _render_buffers_get_color_format(), RD::TEXTURE_USAGE_SAMPLING_BIT | RD::TEXTURE_USAGE_STORAGE_BIT | RD::TEXTURE_USAGE_COLOR_ATTACHMENT_BIT, RD::TEXTURE_SAMPLES_1, Size2i(), 0, 1, true, true);
 				dest_fb = FramebufferCacheRD::get_singleton()->get_cache(dest_texture);
+				// Debanding is currently not supported when using spatial upscaling, so apply it before scaling.
+				// This produces suboptimal results because the image will be modified by spatial upscaling after
+				// debanding has been applied. Ideally, debanding should be applied as the final step before quantization
+				// to integer values, but in the case of MetalFX, it may not be worth the performance cost of creating a new
+				// intermediate buffer. In the case of FSR 1.0, the work of adding debanding support hasn't been done yet.
+				// Assume that the DataFormat that will be used by spatial_upscaler is the same as render_target_get_color_format.
+				format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, !using_hdr);
 			} else {
 				if (dest_is_msaa_2d) {
 					dest_fb = FramebufferCacheRD::get_singleton()->get_cache(texture_storage->render_target_get_rd_texture_msaa(render_target));
+					// Assume that the DataFormat of render_target_get_rd_texture_msaa is the same as render_target_get_color_format.
+					format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, !using_hdr);
 					texture_storage->render_target_set_msaa_needs_resolve(render_target, true); // Make sure this gets resolved.
 				} else {
 					dest_fb = texture_storage->render_target_get_rd_framebuffer(render_target);
+					// Assume that the DataFormat of render_target_get_rd_framebuffer is the same as render_target_get_color_format.
+					format_for_debanding = texture_storage->render_target_get_color_format(using_hdr, !using_hdr);
 				}
 			}
 
+			if (rb->get_use_debanding()) {
+				if (_is_8bit_data_format(format_for_debanding)) {
+					smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_8_BIT;
+				} else if (_is_10bit_data_format(format_for_debanding)) {
+					smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_10_BIT;
+				} else {
+					// In this case, debanding will be handled later when quantizing to an integer data format. (During blit, for example.)
+					smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_DISABLED;
+				}
+			} else {
+				smaa->debanding_mode = RendererRD::SMAA::DebandingMode::DEBANDING_MODE_DISABLED;
+			}
+
 			smaa->process(rb, source_texture, dest_fb);
 		}
 

servers/rendering/renderer_rd/renderer_scene_render_rd.h

@@ -113,6 +113,14 @@ protected:
 	void _post_process_subpass(RID p_source_texture, RID p_framebuffer, const RenderDataRD *p_render_data);
 	void _disable_clear_request(const RenderDataRD *p_render_data);
 
+	_FORCE_INLINE_ bool _is_8bit_data_format(RD::DataFormat p_data_format) {
+		return p_data_format >= RD::DATA_FORMAT_R8_UNORM && p_data_format <= RD::DATA_FORMAT_A8B8G8R8_SRGB_PACK32;
+	}
+
+	_FORCE_INLINE_ bool _is_10bit_data_format(RD::DataFormat p_data_format) {
+		return p_data_format >= RD::DATA_FORMAT_A2R10G10B10_UNORM_PACK32 && p_data_format <= RD::DATA_FORMAT_A2B10G10R10_SINT_PACK32;
+	}
+
 	// needed for a single argument calls (material and uv2)
 	PagedArrayPool<RenderGeometryInstance *> cull_argument_pool;
 	PagedArray<RenderGeometryInstance *> cull_argument; //need this to exist

servers/rendering/renderer_rd/shaders/effects/smaa_blending.glsl

@@ -34,7 +34,7 @@ layout(location = 1) out vec4 offset;
 
 layout(push_constant, std430) uniform Params {
 	vec2 inv_size;
-	vec2 reserved;
+	vec2 pad;
 }
 params;
 
@@ -62,9 +62,13 @@ layout(set = 1, binding = 0) uniform sampler2D blend_tex;
 
 layout(location = 0) out vec4 out_color;
 
+#define FLAG_USE_8_BIT_DEBANDING (1 << 0)
+#define FLAG_USE_10_BIT_DEBANDING (1 << 1)
+
 layout(push_constant, std430) uniform Params {
 	vec2 inv_size;
-	vec2 reserved;
+	uint flags;
+	float pad;
 }
 params;
 
@@ -95,6 +99,22 @@ void SMAAMovc(bvec4 cond, inout vec4 variable, vec4 value) {
 	SMAAMovc(cond.zw, variable.zw, value.zw);
 }
 
+// From https://alex.vlachos.com/graphics/Alex_Vlachos_Advanced_VR_Rendering_GDC2015.pdf
+// and https://www.shadertoy.com/view/MslGR8 (5th one starting from the bottom)
+// NOTE: `frag_coord` is in pixels (i.e. not normalized UV).
+// This dithering must be applied after encoding changes (linear/nonlinear) have been applied
+// as the final step before quantization from floating point to integer values.
+vec3 screen_space_dither(vec2 frag_coord, float bit_alignment_diviser) {
+	// Iestyn's RGB dither (7 asm instructions) from Portal 2 X360, slightly modified for VR.
+	// Removed the time component to avoid passing time into this shader.
+	vec3 dither = vec3(dot(vec2(171.0, 231.0), frag_coord));
+	dither.rgb = fract(dither.rgb / vec3(103.0, 71.0, 97.0));
+
+	// Subtract 0.5 to avoid slightly brightening the whole viewport.
+	// Use a dither strength of 100% rather than the 37.5% suggested by the original source.
+	return (dither.rgb - 0.5) / bit_alignment_diviser;
+}
+
 void main() {
 	vec4 a;
 	a.x = texture(blend_tex, offset.xy).a;
@@ -120,4 +140,11 @@ void main() {
 		out_color.rgb = linear_to_srgb(out_color.rgb);
 		out_color.a = texture(color_tex, tex_coord).a;
 	}
+	if (bool(params.flags & FLAG_USE_8_BIT_DEBANDING)) {
+		// Divide by 255 to align to 8-bit quantization.
+		out_color.rgb += screen_space_dither(gl_FragCoord.xy, 255.0);
+	} else if (bool(params.flags & FLAG_USE_10_BIT_DEBANDING)) {
+		// Divide by 1023 to align to 10-bit quantization.
+		out_color.rgb += screen_space_dither(gl_FragCoord.xy, 1023.0);
+	}
 }