Fixes possible out-of-bound read in findDegenerate (#5679)

This is possible, if the option "aiProcess_ValidateDataStructure" is deactivated as postprocessing step, but the option "aiProcess_FindDegenerates" is activated and the mesh contains invalid data structures

Co-authored-by: Kim Kulling <[email protected]>

code/PostProcessing/FindDegenerates.cpp

@@ -143,9 +143,13 @@ bool FindDegeneratesProcess::ExecuteOnMesh(aiMesh *mesh) {
     for (unsigned int a = 0; a < mesh->mNumFaces; ++a) {
         aiFace &face = mesh->mFaces[a];
         bool first = true;
+        auto vertex_in_range = [numVertices = mesh->mNumVertices](unsigned int vertex_idx) { return vertex_idx < numVertices; };
 
         // check whether the face contains degenerated entries
         for (unsigned int i = 0; i < face.mNumIndices; ++i) {
+            if (!std::all_of(face.mIndices, face.mIndices + face.mNumIndices, vertex_in_range))
+                continue;
+
             // Polygons with more than 4 points are allowed to have double points, that is
             // simulating polygons with holes just with concave polygons. However,
             // double points may not come directly after another.

test/unit/utFindDegenerates.cpp

@@ -206,3 +206,38 @@ TEST_F(FindDegeneratesProcessTest, meshRemoval) {
     EXPECT_EQ(scene->mRootNode->mNumMeshes, 1u);
     EXPECT_EQ(scene->mRootNode->mMeshes[0], 0u);
 }
+
+TEST_F(FindDegeneratesProcessTest, invalidVertexIndex) {
+    mProcess->EnableAreaCheck(true);
+    mProcess->EnableInstantRemoval(true);
+    mProcess->ExecuteOnMesh(mMesh);
+
+    std::unique_ptr<aiScene> scene(new aiScene);
+    scene->mNumMeshes = 1;
+    scene->mMeshes = new aiMesh *[1];
+
+    std::unique_ptr<aiMesh> mesh(new aiMesh);
+    mesh->mNumVertices = 1;
+    mesh->mVertices = new aiVector3D[1];
+    mesh->mVertices[0] = aiVector3D{ 0.0f, 0.0f, 0.0f };
+    mesh->mNumFaces = 1;
+    mesh->mFaces = new aiFace[1];
+    mesh->mFaces[0].mNumIndices = 3;
+    mesh->mFaces[0].mIndices = new unsigned int[3];
+    mesh->mFaces[0].mIndices[0] = 0;
+    mesh->mFaces[0].mIndices[1] = 1;
+    mesh->mFaces[0].mIndices[2] = 99999;
+
+    scene->mMeshes[0] = mesh.release();
+
+    scene->mRootNode = new aiNode;
+    scene->mRootNode->mNumMeshes = 1;
+    scene->mRootNode->mMeshes = new unsigned int[1];
+    scene->mRootNode->mMeshes[0] = 0;
+
+    mProcess->Execute(scene.get());
+
+    EXPECT_EQ(scene->mNumMeshes, 1u);
+    EXPECT_EQ(scene->mRootNode->mNumMeshes, 1u);
+    EXPECT_EQ(scene->mRootNode->mMeshes[0], 0u);
+}