Tighter shadow culling - fix light colinear to frustum edge

In rare situations if a light is placed near colinear to a frustum edge, the extra culling plane derived can have an inaccurate normal due to floating point error.
This PR detects colinear triangles, and prevents adding a culling plane in this situation.

servers/visual/visual_server_light_culler.cpp

@@ -376,15 +376,19 @@ bool VisualServerLightCuller::_add_light_camera_planes(const LightSource &p_ligh
 	uint8_t *entry = &data.LUT_entries[lookup][0];
 	int n_edges = data.LUT_entry_sizes[lookup] - 1;
 
+	const Vector3 &pt2 = p_light_source.pos;
+
 	for (int e = 0; e < n_edges; e++) {
 		int i0 = entry[e];
 		int i1 = entry[e + 1];
 		const Vector3 &pt0 = data.frustum_points[i0];
 		const Vector3 &pt1 = data.frustum_points[i1];
 
-		// Create plane from 3 points.
-		Plane p(pt0, pt1, p_light_source.pos);
-		add_cull_plane(p);
+		if (!_is_colinear_tri(pt0, pt1, pt2)) {
+			// Create plane from 3 points.
+			Plane p(pt0, pt1, pt2);
+			add_cull_plane(p);
+		}
 	}
 
 	// Last to 0 edge.
@@ -395,9 +399,11 @@ bool VisualServerLightCuller::_add_light_camera_planes(const LightSource &p_ligh
 		const Vector3 &pt0 = data.frustum_points[i0];
 		const Vector3 &pt1 = data.frustum_points[i1];
 
-		// Create plane from 3 points.
-		Plane p(pt0, pt1, p_light_source.pos);
-		add_cull_plane(p);
+		if (!_is_colinear_tri(pt0, pt1, pt2)) {
+			// Create plane from 3 points.
+			Plane p(pt0, pt1, pt2);
+			add_cull_plane(p);
+		}
 	}
 
 #ifdef LIGHT_CULLER_DEBUG_LOGGING

servers/visual/visual_server_light_culler.h

@@ -146,6 +146,39 @@ private:
 	// Directional light gives parallel culling planes (as opposed to point lights).
 	bool add_light_camera_planes_directional(const LightSource &p_light_source);
 
+	// Avoid adding extra culling planes derived from near colinear triangles.
+	// The normals derived from these will be inaccurate, and can lead to false
+	// culling of objects that should be within the light volume.
+	bool _is_colinear_tri(const Vector3 &p_a, const Vector3 &p_b, const Vector3 &p_c) const {
+		// Lengths of sides a, b and c.
+		float la = (p_b - p_a).length();
+		float lb = (p_c - p_b).length();
+		float lc = (p_c - p_a).length();
+
+		// Get longest side into lc.
+		if (lb < la) {
+			SWAP(la, lb);
+		}
+		if (lc < lb) {
+			SWAP(lb, lc);
+		}
+
+		// Prevent divide by zero.
+		if (lc > 0.00001f) {
+			// If the summed length of the smaller two
+			// sides is close to the length of the longest side,
+			// the points are colinear, and the triangle is near degenerate.
+			float ld = ((la + lb) - lc) / lc;
+
+			// ld will be close to zero for colinear tris.
+			return ld < 0.00001f;
+		}
+
+		// Don't create planes from tiny triangles,
+		// they won't be accurate.
+		return true;
+	}
+
 	// Is the light culler active? maybe not in the editor...
 	bool is_caster_culling_active() const { return data.caster_culling_active; }
 	bool is_light_culling_active() const { return data.light_culling_active; }