Denominator in GetBaryCentricCoordinates should be positive (#1024)

Negative numbers can only occur when there is a floating point error because the triangle is degenerate.

Fixes #1008

Jolt/Geometry/ClosestPoint.h

@@ -59,16 +59,18 @@ namespace ClosestPoint
 		Vec3 v2 = inC - inB;
 
 		// Make sure that the shortest edge is included in the calculation to keep the products a * b - c * d as small as possible to preserve accuracy
-		float d00 = v0.Dot(v0);
-		float d11 = v1.Dot(v1);
-		float d22 = v2.Dot(v2);
+		float d00 = v0.LengthSq();
+		float d11 = v1.LengthSq();
+		float d22 = v2.LengthSq();
 		if (d00 <= d22)
 		{
 			// Use v0 and v1 to calculate barycentric coordinates
 			float d01 = v0.Dot(v1);
 
+			// Denominator must be positive:
+			// |v0|^2 * |v1|^2 - (v0 . v1)^2 = |v0|^2 * |v1|^2 * (1 - cos(angle)^2) >= 0
 			float denominator = d00 * d11 - d01 * d01;
-			if (abs(denominator) < 1.0e-12f)
+			if (denominator < 1.0e-12f)
 			{
 				// Degenerate triangle, return coordinates along longest edge
 				if (d00 > d11)
@@ -98,7 +100,7 @@ namespace ClosestPoint
 			float d12 = v1.Dot(v2);
 
 			float denominator = d11 * d22 - d12 * d12;
-			if (abs(denominator) < 1.0e-12f)
+			if (denominator < 1.0e-12f)
 			{
 				// Degenerate triangle, return coordinates along longest edge
 				if (d11 > d22)

UnitTests/Physics/CollideShapeTests.cpp

@@ -11,9 +11,11 @@
 #include <Jolt/Physics/Collision/Shape/CapsuleShape.h>
 #include <Jolt/Physics/Collision/Shape/ConvexHullShape.h>
 #include <Jolt/Physics/Collision/Shape/TriangleShape.h>
+#include <Jolt/Physics/Collision/Shape/CylinderShape.h>
 #include <Jolt/Physics/Collision/CollideShape.h>
 #include <Jolt/Physics/Collision/CollisionCollectorImpl.h>
 #include <Jolt/Physics/Collision/CollisionDispatch.h>
+#include <Jolt/Physics/Collision/CollideConvexVsTriangles.h>
 #include <Jolt/Geometry/EPAPenetrationDepth.h>
 #include "Layers.h"
 
@@ -405,6 +407,30 @@ TEST_SUITE("CollideShapeTests")
 		CHECK_APPROX_EQUAL(collector.mHit.mPenetrationDepth, expected_penetration_depth, 1.0e-6f);
 	}
 
+	// A test case of a triangle that's nearly parallel to a cylinder and is just penetrating it. This one was causing numerical issues. See issue #1008.
+	TEST_CASE("TestCollideParallelTriangleVsCylinder")
+	{
+		CylinderShape cylinder(0.85f, 0.25f, 0.02f);
+		cylinder.SetEmbedded();
+
+		Mat44 cylinder_transform = Mat44::sTranslation(Vec3(-42.8155518f, -4.32299995f, 12.1734285f));
+
+		CollideShapeSettings settings;
+		settings.mMaxSeparationDistance = 0.001f;
+		ClosestHitCollisionCollector<CollideShapeCollector> collector;
+		CollideConvexVsTriangles c(&cylinder, Vec3::sReplicate(1.0f), Vec3::sReplicate(1.0f), cylinder_transform, Mat44::sIdentity(), SubShapeID(), settings, collector);
+
+		Vec3 v0(-42.7954292f, -0.647318780f, 12.4227943f);
+		Vec3 v1(-29.9111290f, -0.647318780f, 12.4227943f);
+		Vec3 v2(-42.7954292f, -4.86970234f, 12.4227943f);
+		c.Collide(v0, v1, v2, 0, SubShapeID());
+
+		// Check there was a hit
+		CHECK(collector.HadHit());
+		CHECK(collector.mHit.mPenetrationDepth < 1.0e-4f);
+		CHECK(collector.mHit.mPenetrationAxis.Normalized().IsClose(Vec3::sAxisZ()));
+	}
+
 	// A test case of a box and a convex hull that are nearly touching and that should return a contact with correct normal because the collision settings specify a max separation distance. This was producing the wrong normal.
 	TEST_CASE("BoxVsConvexHullNoConvexRadius")
 	{