Split back face mode between convex/triangles for ray casts (#1228)

See: https://github.com/godot-jolt/godot-jolt/issues/957

Jolt/Physics/Collision/RayCast.h

@@ -71,8 +71,14 @@ class RayCastSettings
 public:
 	JPH_OVERRIDE_NEW_DELETE
 
-	/// How backfacing triangles should be treated
-	EBackFaceMode				mBackFaceMode				= EBackFaceMode::IgnoreBackFaces;
+	/// Set the backfacing mode for all shapes
+	void						SetBackFaceMode(EBackFaceMode inMode) { mBackFaceModeTriangles = mBackFaceModeConvex = inMode; }
+
+	/// How backfacing triangles should be treated (should we report back facing hits for triangle based shapes, e.g. MeshShape/HeightFieldShape?)
+	EBackFaceMode				mBackFaceModeTriangles		= EBackFaceMode::IgnoreBackFaces;
+
+	/// How backfacing convex objects should be treated (should we report back facing hits for convex shapes?)
+	EBackFaceMode				mBackFaceModeConvex			= EBackFaceMode::IgnoreBackFaces;
 
 	/// If convex shapes should be treated as solid. When true, a ray starting inside a convex shape will generate a hit at fraction 0.
 	bool						mTreatConvexAsSolid			= true;

Jolt/Physics/Collision/Shape/BoxShape.cpp

@@ -206,7 +206,7 @@ void BoxShape::CastRay(const RayCast &inRay, const RayCastSettings &inRayCastSet
 		}
 
 		// Check back side hit
-		if (inRayCastSettings.mBackFaceMode == EBackFaceMode::CollideWithBackFaces
+		if (inRayCastSettings.mBackFaceModeConvex == EBackFaceMode::CollideWithBackFaces
 			&& max_fraction < ioCollector.GetEarlyOutFraction())
 		{
 			hit.mFraction = max_fraction;

Jolt/Physics/Collision/Shape/ConvexHullShape.cpp

@@ -1033,7 +1033,7 @@ void ConvexHullShape::CastRay(const RayCast &inRay, const RayCastSettings &inRay
 		}
 
 		// Check back side hit
-		if (inRayCastSettings.mBackFaceMode == EBackFaceMode::CollideWithBackFaces
+		if (inRayCastSettings.mBackFaceModeConvex == EBackFaceMode::CollideWithBackFaces
 			&& max_fraction < ioCollector.GetEarlyOutFraction())
 		{
 			hit.mFraction = max_fraction;

Jolt/Physics/Collision/Shape/ConvexShape.cpp

@@ -198,7 +198,7 @@ void ConvexShape::CastRay(const RayCast &inRay, const RayCastSettings &inRayCast
 		}
 
 		// Check if we want back facing hits and the collector still accepts additional hits
-		if (inRayCastSettings.mBackFaceMode == EBackFaceMode::CollideWithBackFaces && !ioCollector.ShouldEarlyOut())
+		if (inRayCastSettings.mBackFaceModeConvex == EBackFaceMode::CollideWithBackFaces && !ioCollector.ShouldEarlyOut())
 		{
 			// Invert the ray, going from the early out fraction back to the fraction where we found our forward hit
 			float start_fraction = min(1.0f, ioCollector.GetEarlyOutFraction());

Jolt/Physics/Collision/Shape/HeightFieldShape.cpp

@@ -2135,7 +2135,7 @@ void HeightFieldShape::CastRay(const RayCast &inRay, const RayCastSettings &inRa
 			mRayOrigin(inRay.mOrigin),
 			mRayDirection(inRay.mDirection),
 			mRayInvDirection(inRay.mDirection),
-			mBackFaceMode(inRayCastSettings.mBackFaceMode),
+			mBackFaceMode(inRayCastSettings.mBackFaceModeTriangles),
 			mShape(inShape),
 			mSubShapeIDCreator(inSubShapeIDCreator)
 		{

Jolt/Physics/Collision/Shape/MeshShape.cpp

@@ -773,7 +773,7 @@ void MeshShape::CastRay(const RayCast &inRay, const RayCastSettings &inRayCastSe
 	};
 
 	Visitor visitor(ioCollector);
-	visitor.mBackFaceMode = inRayCastSettings.mBackFaceMode;
+	visitor.mBackFaceMode = inRayCastSettings.mBackFaceModeTriangles;
 	visitor.mRayOrigin = inRay.mOrigin;
 	visitor.mRayDirection = inRay.mDirection;
 	visitor.mRayInvDirection.Set(inRay.mDirection);

Jolt/Physics/Collision/Shape/PlaneShape.cpp

@@ -218,7 +218,7 @@ void PlaneShape::CastRay(const RayCast &inRay, const RayCastSettings &inRayCastS
 
 	float dot = inRay.mDirection.Dot(mPlane.GetNormal());
 	if (dot != 0.0f // Parallel ray will not hit plane
-		&& (inRayCastSettings.mBackFaceMode == EBackFaceMode::CollideWithBackFaces || dot < 0.0f)) // Back face culling
+		&& (inRayCastSettings.mBackFaceModeConvex == EBackFaceMode::CollideWithBackFaces || dot < 0.0f)) // Back face culling
 	{
 		// Calculate hit with plane
 		float fraction = -distance / dot;

Jolt/Physics/Collision/Shape/Shape.cpp

@@ -311,7 +311,7 @@ void Shape::sCollidePointUsingRayCast(const Shape &inShape, Vec3Arg inPoint, con
 
 		// Configure the raycast
 		RayCastSettings settings;
-		settings.mBackFaceMode = EBackFaceMode::CollideWithBackFaces;
+		settings.SetBackFaceMode(EBackFaceMode::CollideWithBackFaces);
 
 		// Cast a ray that's 10% longer than the height of our bounding box
 		inShape.CastRay(RayCast { inPoint, 1.1f * bounds.GetSize().GetY() * Vec3::sAxisY() }, settings, inSubShapeIDCreator, collector, inShapeFilter);

Jolt/Physics/Collision/Shape/SphereShape.cpp

@@ -256,7 +256,7 @@ void SphereShape::CastRay(const RayCast &inRay, const RayCastSettings &inRayCast
 		}
 
 		// Check back side hit
-		if (inRayCastSettings.mBackFaceMode == EBackFaceMode::CollideWithBackFaces
+		if (inRayCastSettings.mBackFaceModeConvex == EBackFaceMode::CollideWithBackFaces
 			&& num_results > 1 // Ray should have 2 intersections
 			&& max_fraction < ioCollector.GetEarlyOutFraction()) // End of ray should be before early out fraction
 		{

Jolt/Physics/Collision/Shape/TriangleShape.cpp

@@ -239,7 +239,7 @@ void TriangleShape::CastRay(const RayCast &inRay, const RayCastSettings &inRayCa
 		return;
 
 	// Back facing check
-	if (inRayCastSettings.mBackFaceMode == EBackFaceMode::IgnoreBackFaces && (mV2 - mV1).Cross(mV3 - mV1).Dot(inRay.mDirection) > 0.0f)
+	if (inRayCastSettings.mBackFaceModeTriangles == EBackFaceMode::IgnoreBackFaces && (mV2 - mV1).Cross(mV3 - mV1).Dot(inRay.mDirection) > 0.0f)
 		return;
 
 	// Test ray against triangle

Jolt/Physics/Collision/ShapeCast.h

@@ -93,11 +93,14 @@ class ShapeCastSettings : public CollideSettingsBase
 public:
 	JPH_OVERRIDE_NEW_DELETE
 
-	/// How backfacing triangles should be treated (should we report moving out of a triangle?)
-	EBackFaceMode				mBackFaceModeTriangles				= EBackFaceMode::IgnoreBackFaces;
+	/// Set the backfacing mode for all shapes
+	void						SetBackFaceMode(EBackFaceMode inMode) { mBackFaceModeTriangles = mBackFaceModeConvex = inMode; }
+
+	/// How backfacing triangles should be treated (should we report moving from back to front for triangle based shapes, e.g. for MeshShape/HeightFieldShape?)
+	EBackFaceMode				mBackFaceModeTriangles = EBackFaceMode::IgnoreBackFaces;
 
 	/// How backfacing convex objects should be treated (should we report starting inside an object and moving out?)
-	EBackFaceMode				mBackFaceModeConvex					= EBackFaceMode::IgnoreBackFaces;
+	EBackFaceMode				mBackFaceModeConvex = EBackFaceMode::IgnoreBackFaces;
 
 	/// Indicates if we want to shrink the shape by the convex radius and then expand it again. This speeds up collision detection and gives a more accurate normal at the cost of a more 'rounded' shape.
 	bool						mUseShrunkenShapeAndConvexRadius = false;

Jolt/Physics/SoftBody/SoftBodyShape.cpp

@@ -92,7 +92,7 @@ void SoftBodyShape::CastRay(const RayCast &inRay, const RayCastSettings &inRayCa
 		Vec3 x3 = vertices[f.mVertex[2]].mPosition;
 
 		// Back facing check
-		if (inRayCastSettings.mBackFaceMode == EBackFaceMode::IgnoreBackFaces && (x2 - x1).Cross(x3 - x1).Dot(inRay.mDirection) > 0.0f)
+		if (inRayCastSettings.mBackFaceModeTriangles == EBackFaceMode::IgnoreBackFaces && (x2 - x1).Cross(x3 - x1).Dot(inRay.mDirection) > 0.0f)
 			continue;
 
 		// Test ray against triangle

Samples/SamplesApp.cpp

@@ -564,7 +564,8 @@ SamplesApp::SamplesApp()
 			mDebugUI->CreateSlider(probe_options, "Scale X", mShapeScale.GetX(), -5.0f, 5.0f, 0.1f, [this](float inValue) { mShapeScale.SetX(inValue); });
 			mDebugUI->CreateSlider(probe_options, "Scale Y", mShapeScale.GetY(), -5.0f, 5.0f, 0.1f, [this](float inValue) { mShapeScale.SetY(inValue); });
 			mDebugUI->CreateSlider(probe_options, "Scale Z", mShapeScale.GetZ(), -5.0f, 5.0f, 0.1f, [this](float inValue) { mShapeScale.SetZ(inValue); });
-			mDebugUI->CreateComboBox(probe_options, "Back Face Cull", { "On", "Off" }, (int)mBackFaceMode, [this](int inItem) { mBackFaceMode = (EBackFaceMode)inItem; });
+			mDebugUI->CreateComboBox(probe_options, "Back Face Cull Triangles", { "On", "Off" }, (int)mBackFaceModeTriangles, [this](int inItem) { mBackFaceModeTriangles = (EBackFaceMode)inItem; });
+			mDebugUI->CreateComboBox(probe_options, "Back Face Cull Convex", { "On", "Off" }, (int)mBackFaceModeConvex, [this](int inItem) { mBackFaceModeConvex = (EBackFaceMode)inItem; });
 			mDebugUI->CreateComboBox(probe_options, "Active Edge Mode", { "Only Active", "All" }, (int)mActiveEdgeMode, [this](int inItem) { mActiveEdgeMode = (EActiveEdgeMode)inItem; });
 			mDebugUI->CreateComboBox(probe_options, "Collect Faces Mode", { "Collect Faces", "No Faces" }, (int)mCollectFacesMode, [this](int inItem) { mCollectFacesMode = (ECollectFacesMode)inItem; });
 			mDebugUI->CreateSlider(probe_options, "Max Separation Distance", mMaxSeparationDistance, 0.0f, 5.0f, 0.1f, [this](float inValue) { mMaxSeparationDistance = inValue; });
@@ -1129,7 +1130,8 @@ bool SamplesApp::CastProbe(float inProbeLength, float &outFraction, RVec3 &outPo
 
 			// Create settings
 			RayCastSettings settings;
-			settings.mBackFaceMode = mBackFaceMode;
+			settings.mBackFaceModeTriangles = mBackFaceModeTriangles;
+			settings.mBackFaceModeConvex = mBackFaceModeConvex;
 			settings.mTreatConvexAsSolid = mTreatConvexAsSolid;
 
 			// Cast ray
@@ -1264,7 +1266,7 @@ bool SamplesApp::CastProbe(float inProbeLength, float &outFraction, RVec3 &outPo
 			// Create settings
 			CollideShapeSettings settings;
 			settings.mActiveEdgeMode = mActiveEdgeMode;
-			settings.mBackFaceMode = mBackFaceMode;
+			settings.mBackFaceMode = mBackFaceModeTriangles;
 			settings.mCollectFacesMode = mCollectFacesMode;
 			settings.mMaxSeparationDistance = mMaxSeparationDistance;
 
@@ -1353,8 +1355,8 @@ bool SamplesApp::CastProbe(float inProbeLength, float &outFraction, RVec3 &outPo
 			ShapeCastSettings settings;
 			settings.mUseShrunkenShapeAndConvexRadius = mUseShrunkenShapeAndConvexRadius;
 			settings.mActiveEdgeMode = mActiveEdgeMode;
-			settings.mBackFaceModeTriangles = mBackFaceMode;
-			settings.mBackFaceModeConvex = mBackFaceMode;
+			settings.mBackFaceModeTriangles = mBackFaceModeTriangles;
+			settings.mBackFaceModeConvex = mBackFaceModeConvex;
 			settings.mReturnDeepestPoint = mReturnDeepestPoint;
 			settings.mCollectFacesMode = mCollectFacesMode;
 

Samples/SamplesApp.h

@@ -191,7 +191,8 @@ private:
 	EProbeShape				mProbeShape = EProbeShape::Sphere;							// Shape to use for the mouse probe.
 	bool					mScaleShape = false;										// If the shape is scaled or not. When true mShapeScale is taken into account.
 	Vec3					mShapeScale = Vec3::sReplicate(1.0f);						// Scale in local space for the probe shape.
-	EBackFaceMode			mBackFaceMode = EBackFaceMode::CollideWithBackFaces;		// How to handle back facing triangles when doing a collision probe check.
+	EBackFaceMode			mBackFaceModeTriangles = EBackFaceMode::CollideWithBackFaces; // How to handle back facing triangles when doing a collision probe check.
+	EBackFaceMode			mBackFaceModeConvex = EBackFaceMode::CollideWithBackFaces; // How to handle back facing convex shapes when doing a collision probe check.
 	EActiveEdgeMode			mActiveEdgeMode = EActiveEdgeMode::CollideOnlyWithActive;	// How to handle active edges when doing a collision probe check.
 	ECollectFacesMode		mCollectFacesMode = ECollectFacesMode::NoFaces;				// If we should collect colliding faces
 	float					mMaxSeparationDistance = 0.0f;								// Max separation distance for collide shape test

UnitTests/Physics/RayShapeTests.cpp

@@ -100,7 +100,7 @@ TEST_SUITE("RayShapeTests")
 
 			// Ray cast settings
 			RayCastSettings settings;
-			settings.mBackFaceMode = EBackFaceMode::IgnoreBackFaces;
+			settings.SetBackFaceMode(EBackFaceMode::IgnoreBackFaces);
 			settings.mTreatConvexAsSolid = true;
 
 			AllHitCollisionCollector<CastRayCollector> collector;
@@ -136,7 +136,7 @@ TEST_SUITE("RayShapeTests")
 
 			// Ray cast settings
 			RayCastSettings settings;
-			settings.mBackFaceMode = EBackFaceMode::CollideWithBackFaces;
+			settings.SetBackFaceMode(EBackFaceMode::CollideWithBackFaces);
 			settings.mTreatConvexAsSolid = true;
 
 			AllHitCollisionCollector<CastRayCollector> collector;
@@ -183,7 +183,7 @@ TEST_SUITE("RayShapeTests")
 
 			// Ray cast settings
 			RayCastSettings settings;
-			settings.mBackFaceMode = EBackFaceMode::IgnoreBackFaces;
+			settings.SetBackFaceMode(EBackFaceMode::IgnoreBackFaces);
 			settings.mTreatConvexAsSolid = false;
 
 			AllHitCollisionCollector<CastRayCollector> collector;
@@ -220,7 +220,7 @@ TEST_SUITE("RayShapeTests")
 
 			// Ray cast settings
 			RayCastSettings settings;
-			settings.mBackFaceMode = EBackFaceMode::CollideWithBackFaces;
+			settings.SetBackFaceMode(EBackFaceMode::CollideWithBackFaces);
 			settings.mTreatConvexAsSolid = false;
 
 			AllHitCollisionCollector<CastRayCollector> collector;
@@ -318,7 +318,7 @@ TEST_SUITE("RayShapeTests")
 
 			// Ray cast settings
 			RayCastSettings settings;
-			settings.mBackFaceMode = EBackFaceMode::IgnoreBackFaces;
+			settings.SetBackFaceMode(EBackFaceMode::IgnoreBackFaces);
 			settings.mTreatConvexAsSolid = true;
 
 			AllHitCollisionCollector<CastRayCollector> collector;
@@ -352,7 +352,7 @@ TEST_SUITE("RayShapeTests")
 
 			// Ray cast settings
 			RayCastSettings settings;
-			settings.mBackFaceMode = EBackFaceMode::CollideWithBackFaces;
+			settings.SetBackFaceMode(EBackFaceMode::CollideWithBackFaces);
 			settings.mTreatConvexAsSolid = true;
 
 			AllHitCollisionCollector<CastRayCollector> collector;