Fix TSMesh::castRay() and small collision mesh

This is the fix outlined in
https://github.com/GarageGames/Torque3D/issues/485

Engine/source/ts/tsMesh.cpp

@@ -876,18 +876,33 @@ bool TSMesh::castRayRendered( S32 frame, const Point3F & start, const Point3F &
 
 bool TSMesh::addToHull( U32 idx0, U32 idx1, U32 idx2 )
 {
-   Point3F normal;
-   mCross(mVertexData[idx2].vert()-mVertexData[idx0].vert(),mVertexData[idx1].vert()-mVertexData[idx0].vert(),&normal);
-   if ( mDot( normal, normal ) < 0.001f )
-   {
-      mCross( mVertexData[idx0].vert() - mVertexData[idx1].vert(), mVertexData[idx2].vert() - mVertexData[idx1].vert(), &normal );
-      if ( mDot( normal, normal ) < 0.001f )
-      {
-         mCross( mVertexData[idx1].vert() - mVertexData[idx2].vert(), mVertexData[idx0].vert() - mVertexData[idx2].vert(), &normal );
-         if ( mDot( normal, normal ) < 0.001f )
-            return false;
-      }
-   }
+   // calculate the normal of this triangle... remember, we lose precision
+   // when we subtract two large numbers that are very close to each other,
+   // so depending on how we calculate the normal, we could get a 
+   // different result. so, we will calculate the normal three different
+   // ways and take the one that gives us the largest vector before we
+   // normalize.
+   Point3F normal1, normal2, normal3;
+   mCross(mVertexData[idx2].vert()-mVertexData[idx0].vert(),mVertexData[idx1].vert()-mVertexData[idx0].vert(),&normal1);
+   mCross(mVertexData[idx0].vert()-mVertexData[idx1].vert(),mVertexData[idx2].vert()-mVertexData[idx1].vert(),&normal2);
+   mCross(mVertexData[idx1].vert()-mVertexData[idx2].vert(),mVertexData[idx0].vert()-mVertexData[idx2].vert(),&normal3);
+   Point3F normal = normal1;
+   F32 greatestMagSquared = mDot(normal1, normal1);
+   F32 magSquared = mDot(normal2, normal2);
+   if (magSquared > greatestMagSquared)
+   {
+      normal = normal2;
+      greatestMagSquared = magSquared;
+   }
+   magSquared = mDot(normal3, normal3);
+   if (magSquared > greatestMagSquared)
+   {
+      normal = normal3;
+      greatestMagSquared = magSquared;
+   }
+   if (mDot(normal, normal) < 0.00000001f)
+       return false;
+
    normal.normalize();
    F32 k = mDot( normal, mVertexData[idx0].vert() );
    for ( S32 i = 0; i < planeNormals.size(); i++ )