Fixed bug in MeshShape active edge calculation (#1758)

This could mark edges of non-manifold meshes as inactive instead of active.

Fixes #1757

Docs/ReleaseNotes.md

@@ -19,6 +19,7 @@ For breaking API changes see [this document](https://github.com/jrouwe/JoltPhysi
 * Fixed bug in `ManifoldBetweenTwoFaces` which would not find the correct manifold in case face 1 had 3 or more vertices and face 2 only 2. E.g. for a box resting the long edge of a cylinder this would mean that only a single contact point was found instead of 2 (the other way around would work fine).
 * Fixed bug in `ConvexHullShape::CollideSoftBodyVertices` where the wrong edge could be reported as the closest edge.
 * Fixed bug in `PhysicsSystem::OptimizeBroadPhase`. When calling this function after removing all bodies from the `PhysicsSystem`, the internal nodes would not be freed until bodies are added again. This could lead to running out of internal nodes in rare cases.
+* Fixed bug in `MeshShape` active edge calculation which could mark edges of non-manifold meshes as inactive instead of active.
 * Fixed passing underestimate of penetration depth in `ContactListener::OnContactPersisted` when the contact comes from the contact cache.
 * `QuadTree` will now fall back to the heap when running out of stack space during collision queries. Previously this would trigger an assert and some collisions would not be detected.
 * Fixed `BodyInterface::MoveKinematic`, `SetLinearVelocity`, `SetAngularVelocity`, `SetLinearAndAngularVelocity`, `AddLinearVelocity`, `AddLinearAndAngularVelocity`, `SetPositionRotationAndVelocity` and `SetMotionType` when body not added to the physics system yet.

Jolt/Physics/Collision/Shape/MeshShape.cpp

@@ -319,6 +319,7 @@ void MeshShape::sFindActiveEdges(const MeshShapeSettings &inSettings, IndexedTri
 				uint32 mask = 1 << (edge_idx + FLAGS_ACTIVE_EGDE_SHIFT);
 				JPH_ASSERT((triangle.mMaterialIndex & mask) == 0);
 				triangle.mMaterialIndex |= mask;
+				indices.mNumTriangles = 3; // Indicate that we have 3 or more triangles
 			}
 		}
 	}

UnitTests/Physics/ActiveEdgesTests.cpp

@@ -7,6 +7,7 @@
 #include "Layers.h"
 #include <Jolt/Physics/Collision/Shape/BoxShape.h>
 #include <Jolt/Physics/Collision/Shape/CapsuleShape.h>
+#include <Jolt/Physics/Collision/Shape/SphereShape.h>
 #include <Jolt/Physics/Collision/Shape/RotatedTranslatedShape.h>
 #include <Jolt/Physics/Collision/Shape/MeshShape.h>
 #include <Jolt/Physics/Collision/Shape/HeightFieldShape.h>
@@ -339,4 +340,67 @@ TEST_SUITE("ActiveEdgesTest")
 
 		sLinearCastCubeSlide(scaled_shape, true);
 	}
+
+	TEST_CASE("TestNonManifoldMesh")
+	{
+		// Test 3 triangles in a plane that all share the same edge
+		// Normally the shared edge would not be active, but since the mesh is non-manifold we expect all of them to be active
+		TriangleList triangles;
+		triangles.push_back(Triangle(Float3(0, 0, -1), Float3(0, 0, 1), Float3(1, 0, 0), 0));
+		triangles.push_back(Triangle(Float3(0, 0, 1), Float3(0, 0, -1), Float3(-1, 0, 0), 0));
+		triangles.push_back(Triangle(Float3(0, 0, 1), Float3(0, 0, -1), Float3(-0.5f, 0, 0), 0));
+
+		ShapeRefC shape = MeshShapeSettings(triangles).Create().Get();
+
+		ShapeRefC sphere = new SphereShape(0.1f);
+
+		CollideShapeSettings settings;
+		settings.mActiveEdgeMode = EActiveEdgeMode::CollideOnlyWithActive;
+
+		// Collide a sphere on both sides of the active edge so that a 45 degree normal will be found then the edge is active.
+		// An inactive edge will return a normal that is perpendicular to the plane.
+		{
+			AllHitCollisionCollector<CollideShapeCollector> collector;
+			CollisionDispatch::sCollideShapeVsShape(sphere, shape, Vec3::sOne(), Vec3::sOne(), Mat44::sTranslation(Vec3(0.05f, 0.05f, 0)), Mat44::sIdentity(), SubShapeIDCreator(), SubShapeIDCreator(), settings, collector);
+			CHECK(collector.mHits.size() == 3);
+
+			// We expect one interior hit because the sphere is above the triangle and 2 active edge hits that provide a normal pointing towards the sphere
+			int num_interior = 0, num_on_shared_edge = 0;
+			for (const CollideShapeResult &r : collector.mHits)
+				if (r.mContactPointOn2.IsClose(Vec3(0.05f, 0.0f, 0.0f)))
+				{
+					CHECK_APPROX_EQUAL(r.mPenetrationAxis.Normalized(), Vec3(0, -1, 0));
+					++num_interior;
+				}
+				else if (r.mContactPointOn2.IsNearZero())
+				{
+					CHECK_APPROX_EQUAL(r.mPenetrationAxis.Normalized(), Vec3(-1, -1, 0).Normalized(), 1.0e-5f);
+					++num_on_shared_edge;
+				}
+			CHECK(num_interior == 1);
+			CHECK(num_on_shared_edge == 2);
+		}
+
+		{
+			AllHitCollisionCollector<CollideShapeCollector> collector;
+			CollisionDispatch::sCollideShapeVsShape(sphere, shape, Vec3::sOne(), Vec3::sOne(), Mat44::sTranslation(Vec3(-0.05f, 0.05f, 0)), Mat44::sIdentity(), SubShapeIDCreator(), SubShapeIDCreator(), settings, collector);
+			CHECK(collector.mHits.size() == 3);
+
+			// We expect 2 interior hits because the sphere is above the triangle and 1 active edge hit that provide a normal pointing towards the sphere
+			int num_interior = 0, num_on_shared_edge = 0;
+			for (const CollideShapeResult &r : collector.mHits)
+				if (r.mContactPointOn2.IsClose(Vec3(-0.05f, 0.0f, 0.0f)))
+				{
+					CHECK_APPROX_EQUAL(r.mPenetrationAxis.Normalized(), Vec3(0, -1, 0));
+					++num_interior;
+				}
+				else if (r.mContactPointOn2.IsNearZero())
+				{
+					CHECK_APPROX_EQUAL(r.mPenetrationAxis.Normalized(), Vec3(1, -1, 0).Normalized(), 1.0e-5f);
+					++num_on_shared_edge;
+				}
+			CHECK(num_interior == 2);
+			CHECK(num_on_shared_edge == 1);
+		}
+	}
 }