Merge pull request #4892 from lsnoel/fixLHNormals

Correctly consider aiProcess_FlipWindingOrder and aiProcess_MakeLeftHanded when generating normals

code/PostProcessing/GenFaceNormalsProcess.cpp

@@ -67,6 +67,7 @@ GenFaceNormalsProcess::~GenFaceNormalsProcess() = default;
 bool GenFaceNormalsProcess::IsActive(unsigned int pFlags) const {
     force_ = (pFlags & aiProcess_ForceGenNormals) != 0;
     flippedWindingOrder_ = (pFlags & aiProcess_FlipWindingOrder) != 0;
+    leftHanded_ = (pFlags & aiProcess_MakeLeftHanded) != 0;
     return (pFlags & aiProcess_GenNormals) != 0;
 }
 
@@ -131,8 +132,10 @@ bool GenFaceNormalsProcess::GenMeshFaceNormals(aiMesh *pMesh) {
         const aiVector3D *pV1 = &pMesh->mVertices[face.mIndices[0]];
         const aiVector3D *pV2 = &pMesh->mVertices[face.mIndices[1]];
         const aiVector3D *pV3 = &pMesh->mVertices[face.mIndices[face.mNumIndices - 1]];
-        if (flippedWindingOrder_)
-            std::swap( pV2, pV3 );
+        // Boolean XOR - if either but not both of these flags is set, then the winding order has
+        // changed and the cross product to calculate the normal needs to be reversed
+        if (flippedWindingOrder_ != leftHanded_) 
+            std::swap(pV2, pV3);
         const aiVector3D vNor = ((*pV2 - *pV1) ^ (*pV3 - *pV1)).NormalizeSafe();
 
         for (unsigned int i = 0; i < face.mNumIndices; ++i) {

code/PostProcessing/GenFaceNormalsProcess.h

@@ -81,6 +81,7 @@ private:
     bool GenMeshFaceNormals(aiMesh* pcMesh);
     mutable bool force_ = false;
     mutable bool flippedWindingOrder_ = false;
+    mutable bool leftHanded_ = false;
 };
 
 } // end of namespace Assimp

code/PostProcessing/GenVertexNormalsProcess.cpp

@@ -69,6 +69,7 @@ GenVertexNormalsProcess::~GenVertexNormalsProcess() = default;
 bool GenVertexNormalsProcess::IsActive(unsigned int pFlags) const {
     force_ = (pFlags & aiProcess_ForceGenNormals) != 0;
     flippedWindingOrder_ = (pFlags & aiProcess_FlipWindingOrder) != 0;
+    leftHanded_ = (pFlags & aiProcess_MakeLeftHanded) != 0;
     return (pFlags & aiProcess_GenSmoothNormals) != 0;
 }
 
@@ -141,8 +142,10 @@ bool GenVertexNormalsProcess::GenMeshVertexNormals(aiMesh *pMesh, unsigned int m
         const aiVector3D *pV1 = &pMesh->mVertices[face.mIndices[0]];
         const aiVector3D *pV2 = &pMesh->mVertices[face.mIndices[1]];
         const aiVector3D *pV3 = &pMesh->mVertices[face.mIndices[face.mNumIndices - 1]];
-        if (flippedWindingOrder_)
-            std::swap( pV2, pV3 );
+        // Boolean XOR - if either but not both of these flags is set, then the winding order has
+        // changed and the cross product to calculate the normal needs to be reversed
+        if (flippedWindingOrder_ != leftHanded_)
+            std::swap(pV2, pV3);
         const aiVector3D vNor = ((*pV2 - *pV1) ^ (*pV3 - *pV1)).NormalizeSafe();
 
         for (unsigned int i = 0; i < face.mNumIndices; ++i) {

code/PostProcessing/GenVertexNormalsProcess.h

@@ -105,6 +105,7 @@ private:
     ai_real configMaxAngle;
     mutable bool force_ = false;
     mutable bool flippedWindingOrder_ = false;
+    mutable bool leftHanded_ = false;
 };
 
 } // end of namespace Assimp