GPULightmapper exclude back-face triangles while calculating bounces

Edges that are at the edge of a plane, may get behind the scene and will hit
back-face triangles which where included in the lighting calculations. This
caused leaking of light at the edge of planes.

In case a ray hits back-face triangle, it is skipped in the bounce calculations.

modules/lightmapper_rd/lm_compute.glsl

@@ -115,7 +115,12 @@ bool ray_hits_triangle(vec3 from, vec3 dir, float max_dist, vec3 p0, vec3 p1, ve
 	return (r_distance > params.bias) && (r_distance < max_dist) && all(greaterThanEqual(r_barycentric, vec3(0.0)));
 }
 
-bool trace_ray(vec3 p_from, vec3 p_to
+const uint RAY_MISS = 0;
+const uint RAY_FRONT = 1;
+const uint RAY_BACK = 2;
+const uint RAY_ANY = 3;
+
+uint trace_ray(vec3 p_from, vec3 p_to
 #if defined(MODE_BOUNCE_LIGHT) || defined(MODE_LIGHT_PROBES)
 		,
 		out uint r_triangle, out vec3 r_barycentric
@@ -125,6 +130,7 @@ bool trace_ray(vec3 p_from, vec3 p_to
 		out float r_distance, out vec3 r_normal
 #endif
 ) {
+
 	/* world coords */
 
 	vec3 rel = p_to - p_from;
@@ -150,10 +156,7 @@ bool trace_ray(vec3 p_from, vec3 p_to
 	while (all(greaterThanEqual(icell, ivec3(0))) && all(lessThan(icell, ivec3(params.grid_size))) && iters < 1000) {
 		uvec2 cell_data = texelFetch(usampler3D(grid, linear_sampler), icell, 0).xy;
 		if (cell_data.x > 0) { //triangles here
-			bool hit = false;
-#if defined(MODE_UNOCCLUDE)
-			bool hit_backface = false;
-#endif
+			uint hit = RAY_MISS;
 			float best_distance = 1e20;
 
 			for (uint i = 0; i < cell_data.x; i++) {
@@ -173,57 +176,46 @@ bool trace_ray(vec3 p_from, vec3 p_to
 				vec3 vtx0 = vertices.data[triangle.indices.x].position;
 				vec3 vtx1 = vertices.data[triangle.indices.y].position;
 				vec3 vtx2 = vertices.data[triangle.indices.z].position;
-#if defined(MODE_UNOCCLUDE)
+#if defined(MODE_UNOCCLUDE) || defined(MODE_BOUNCE_LIGHT) || defined(MODE_LIGHT_PROBES)
 				vec3 normal = -normalize(cross((vtx0 - vtx1), (vtx0 - vtx2)));
 
 				bool backface = dot(normal, dir) >= 0.0;
 #endif
+
 				float distance;
 				vec3 barycentric;
 
 				if (ray_hits_triangle(p_from, dir, rel_len, vtx0, vtx1, vtx2, distance, barycentric)) {
 #ifdef MODE_DIRECT_LIGHT
-					return true; //any hit good
+					return RAY_ANY; //any hit good
 #endif
 
-#if defined(MODE_UNOCCLUDE)
+#if defined(MODE_UNOCCLUDE) || defined(MODE_BOUNCE_LIGHT) || defined(MODE_LIGHT_PROBES)
 					if (!backface) {
 						// the case of meshes having both a front and back face in the same plane is more common than
 						// expected, so if this is a front-face, bias it closer to the ray origin, so it always wins over the back-face
 						distance = max(params.bias, distance - params.bias);
 					}
 
-					hit = true;
-
 					if (distance < best_distance) {
-						hit_backface = backface;
+						hit = backface ? RAY_BACK : RAY_FRONT;
 						best_distance = distance;
+#if defined(MODE_UNOCCLUDE)
 						r_distance = distance;
 						r_normal = normal;
-					}
-
 #endif
-
 #if defined(MODE_BOUNCE_LIGHT) || defined(MODE_LIGHT_PROBES)
-
-					hit = true;
-					if (distance < best_distance) {
-						best_distance = distance;
 						r_triangle = tidx;
 						r_barycentric = barycentric;
+#endif
 					}
 #endif
 				}
 			}
-#if defined(MODE_UNOCCLUDE)
+#if defined(MODE_UNOCCLUDE) || defined(MODE_BOUNCE_LIGHT) || defined(MODE_LIGHT_PROBES)
 
-			if (hit) {
-				return hit_backface;
-			}
-#endif
-#if defined(MODE_BOUNCE_LIGHT) || defined(MODE_LIGHT_PROBES)
-			if (hit) {
-				return true;
+			if (hit != RAY_MISS) {
+				return hit;
 			}
 #endif
 		}
@@ -239,7 +231,7 @@ bool trace_ray(vec3 p_from, vec3 p_to
 		iters++;
 	}
 
-	return false;
+	return RAY_MISS;
 }
 
 const float PI = 3.14159265f;
@@ -339,7 +331,7 @@ void main() {
 			continue; //no need to do anything
 		}
 
-		if (!trace_ray(position + light_dir * params.bias, light_pos)) {
+		if (trace_ray(position + light_dir * params.bias, light_pos) == RAY_MISS) {
 			vec3 light = lights.data[i].color * lights.data[i].energy * attenuation;
 			if (lights.data[i].static_bake) {
 				static_light += light;
@@ -410,6 +402,7 @@ void main() {
 			vec4(0.0, 0.0, 0.0, 1.0));
 #endif
 	vec3 light_average = vec3(0.0);
+	float active_rays = 0.0;
 	for (uint i = params.ray_from; i < params.ray_to; i++) {
 		vec3 ray_dir = normal_mat * vogel_hemisphere(i, params.ray_count, quick_hash(vec2(atlas_pos)));
 
@@ -417,7 +410,8 @@ void main() {
 		vec3 barycentric;
 
 		vec3 light = vec3(0.0);
-		if (trace_ray(position + ray_dir * params.bias, position + ray_dir * length(params.world_size), tidx, barycentric)) {
+		uint trace_result = trace_ray(position + ray_dir * params.bias, position + ray_dir * length(params.world_size), tidx, barycentric);
+		if (trace_result == RAY_FRONT) {
 			//hit a triangle
 			vec2 uv0 = vertices.data[triangles.data[tidx].indices.x].uv;
 			vec2 uv1 = vertices.data[triangles.data[tidx].indices.y].uv;
@@ -425,7 +419,8 @@ void main() {
 			vec3 uvw = vec3(barycentric.x * uv0 + barycentric.y * uv1 + barycentric.z * uv2, float(triangles.data[tidx].slice));
 
 			light = textureLod(sampler2DArray(source_light, linear_sampler), uvw, 0.0).rgb;
-		} else if (params.env_transform[0][3] == 0.0) { // Use env_transform[0][3] to indicate when we are computing the first bounce
+			active_rays += 1.0;
+		} else if (trace_result == RAY_MISS && params.env_transform[0][3] == 0.0) { // Use env_transform[0][3] to indicate when we are computing the first bounce
 			// Did not hit a triangle, reach out for the sky
 			vec3 sky_dir = normalize(mat3(params.env_transform) * ray_dir);
 
@@ -439,6 +434,7 @@ void main() {
 			st /= vec2(PI * 2.0, PI);
 
 			light = textureLod(sampler2D(environment, linear_sampler), st, 0.0).rgb;
+			active_rays += 1.0;
 		}
 
 		light_average += light;
@@ -462,7 +458,9 @@ void main() {
 	if (params.ray_from == 0) {
 		light_total = vec3(0.0);
 	} else {
-		light_total = imageLoad(bounce_accum, ivec3(atlas_pos, params.atlas_slice)).rgb;
+		vec4 accum = imageLoad(bounce_accum, ivec3(atlas_pos, params.atlas_slice));
+		light_total = accum.rgb;
+		active_rays += accum.a;
 	}
 
 	light_total += light_average;
@@ -477,7 +475,9 @@ void main() {
 
 #endif
 	if (params.ray_to == params.ray_count) {
-		light_total /= float(params.ray_count);
+		if (active_rays > 0) {
+			light_total /= active_rays;
+		}
 		imageStore(dest_light, ivec3(atlas_pos, params.atlas_slice), vec4(light_total, 1.0));
 #ifndef USE_SH_LIGHTMAPS
 		vec4 accum = imageLoad(accum_light, ivec3(atlas_pos, params.atlas_slice));
@@ -485,7 +485,7 @@ void main() {
 		imageStore(accum_light, ivec3(atlas_pos, params.atlas_slice), accum);
 #endif
 	} else {
-		imageStore(bounce_accum, ivec3(atlas_pos, params.atlas_slice), vec4(light_total, 1.0));
+		imageStore(bounce_accum, ivec3(atlas_pos, params.atlas_slice), vec4(light_total, active_rays));
 	}
 
 #endif
@@ -518,7 +518,7 @@ void main() {
 		float d;
 		vec3 norm;
 
-		if (trace_ray(base_pos, ray_to, d, norm)) {
+		if (trace_ray(base_pos, ray_to, d, norm) == RAY_BACK) {
 			if (d < min_d) {
 				vertex_pos = base_pos + rays[i] * d + norm * params.bias * 10.0; //this bias needs to be greater than the regular bias, because otherwise later, rays will go the other side when pointing back.
 				min_d = d;
@@ -558,7 +558,8 @@ void main() {
 		vec3 barycentric;
 		vec3 light;
 
-		if (trace_ray(position + ray_dir * params.bias, position + ray_dir * length(params.world_size), tidx, barycentric)) {
+		uint trace_result = trace_ray(position + ray_dir * params.bias, position + ray_dir * length(params.world_size), tidx, barycentric);
+		if (trace_result == RAY_FRONT) {
 			vec2 uv0 = vertices.data[triangles.data[tidx].indices.x].uv;
 			vec2 uv1 = vertices.data[triangles.data[tidx].indices.y].uv;
 			vec2 uv2 = vertices.data[triangles.data[tidx].indices.z].uv;
@@ -566,7 +567,7 @@ void main() {
 
 			light = textureLod(sampler2DArray(source_light, linear_sampler), uvw, 0.0).rgb;
 			light += textureLod(sampler2DArray(source_direct_light, linear_sampler), uvw, 0.0).rgb;
-		} else {
+		} else if (trace_result == RAY_MISS) {
 			//did not hit a triangle, reach out for the sky
 			vec3 sky_dir = normalize(mat3(params.env_transform) * ray_dir);