Compute the 4 more weighted bone index for skin animation when there are several Joint buffers for a mesh.

jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfLoader.java

@@ -11,6 +11,8 @@ import com.jme3.renderer.queue.RenderQueue;
 import com.jme3.scene.*;
 import com.jme3.texture.Texture;
 import com.jme3.texture.Texture2D;
+import com.jme3.util.BufferUtils;
+import com.jme3.util.IntMap;
 import com.jme3.util.mikktspace.MikktspaceTangentGenerator;
 
 import java.io.*;
@@ -50,13 +52,12 @@ public class GltfLoader implements AssetLoader {
     private FloatArrayPopulator floatArrayPopulator = new FloatArrayPopulator();
     private Vector3fArrayPopulator vector3fArrayPopulator = new Vector3fArrayPopulator();
     private QuaternionArrayPopulator quaternionArrayPopulator = new QuaternionArrayPopulator();
-    private Matrix4fArrayPopulator matrix4fArrayPopulator = new Matrix4fArrayPopulator();
     private static Map<String, MaterialAdapter> defaultMaterialAdapters = new HashMap<>();
     private boolean useNormalsFlag = false;
-    private Transform tmpTransforms = new Transform();
     private Quaternion tmpQuat = new Quaternion();
 
     Map<SkinData, List<Spatial>> skinnedSpatials = new HashMap<>();
+    IntMap<SkinBuffers> skinBuffers = new IntMap<>();
 
     static {
         defaultMaterialAdapters.put("pbrMetallicRoughness", new PBRMaterialAdapter());
@@ -323,9 +324,29 @@ public class GltfLoader implements AssetLoader {
             }
             JsonObject attributes = meshObject.getAsJsonObject("attributes");
             assertNotNull(attributes, "No attributes defined for mesh " + mesh);
+
+            skinBuffers.clear();
+
             for (Map.Entry<String, JsonElement> entry : attributes.entrySet()) {
-                mesh.setBuffer(readAccessorData(entry.getValue().getAsInt(), new VertexBufferPopulator(getVertexBufferType(entry.getKey()))));
+                //special case for joints and weights buffer. If there are more than 4 bones per vertex, there might be several of them
+                //we need to read them all and to keep only the 4 that have the most weight on the vertex.
+                String bufferType = entry.getKey();
+                if (bufferType.startsWith("JOINTS")) {
+                    SkinBuffers buffs = getSkinBuffers(bufferType);
+                    SkinBuffers buffer = readAccessorData(entry.getValue().getAsInt(), new JointArrayPopulator());
+                    buffs.joints = buffer.joints;
+                    buffs.componentSize = buffer.componentSize;
+                } else if (bufferType.startsWith("WEIGHTS")) {
+                    SkinBuffers buffs = getSkinBuffers(bufferType);
+                    buffs.weights = readAccessorData(entry.getValue().getAsInt(), new FloatArrayPopulator());
+                } else {
+                    VertexBuffer vb = readAccessorData(entry.getValue().getAsInt(), new VertexBufferPopulator(getVertexBufferType(bufferType)));
+                    if (vb != null) {
+                        mesh.setBuffer(vb);
+                    }
+                }
             }
+            handleSkinningBuffers(mesh, skinBuffers);
 
             if (mesh.getBuffer(VertexBuffer.Type.BoneIndex) != null) {
                 //the mesh has some skinning let's create needed buffers for HW skinning
@@ -374,6 +395,28 @@ public class GltfLoader implements AssetLoader {
         return geomArray;
     }
 
+    public static class WeightData {
+        float value;
+        short index;
+        int componentSize;
+
+        public WeightData(float value, short index, int componentSize) {
+            this.value = value;
+            this.index = index;
+            this.componentSize = componentSize;
+        }
+    }
+
+    private SkinBuffers getSkinBuffers(String bufferType) {
+        int bufIndex = getIndex(bufferType);
+        SkinBuffers buffs = skinBuffers.get(bufIndex);
+        if (buffs == null) {
+            buffs = new SkinBuffers();
+            skinBuffers.put(bufIndex, buffs);
+        }
+        return buffs;
+    }
+
     private <R> R readAccessorData(int accessorIndex, Populator<R> populator) throws IOException {
 
         assertNotNull(accessors, "No accessor attribute in the gltf file");
@@ -399,8 +442,7 @@ public class GltfLoader implements AssetLoader {
         return populator.populate(bufferViewIndex, componentType, type, count, byteOffset);
     }
 
-    private void readBuffer(Integer bufferViewIndex, int byteOffset, int bufferSize, Object store, int numComponents) throws IOException {
-
+    private void readBuffer(Integer bufferViewIndex, int byteOffset, int bufferSize, Object store, int numComponents, int componentSize) throws IOException {
 
         JsonObject bufferView = bufferViews.get(bufferViewIndex).getAsJsonObject();
         Integer bufferIndex = getAsInteger(bufferView, "buffer");
@@ -415,7 +457,7 @@ public class GltfLoader implements AssetLoader {
         //int target = getAsInteger(bufferView, "target", 0);
 
         byte[] data = readData(bufferIndex);
-        populateBuffer(store, data, bufferSize, byteOffset + bvByteOffset, byteStride, numComponents);
+        populateBuffer(store, data, bufferSize, byteOffset + bvByteOffset, byteStride, numComponents, componentSize);
 
         //TODO extensions?
         //TODO extras?
@@ -606,7 +648,15 @@ public class GltfLoader implements AssetLoader {
                 //check if we are loading the same time array
                 //TODO specs actually don't forbid this...maybe remove this check and handle it.
                 if (animData.times != times) {
-                    throw new AssetLoadException("Channel has different input accessors for samplers");
+                    logger.log(Level.WARNING, "Channel has different input accessors for samplers");
+//                    for (float time : animData.times) {
+//                        System.err.print(time + ", ");
+//                    }
+//                    System.err.println("");
+//                    for (float time : times) {
+//                        System.err.print(time + ", ");
+//                    }
+//                    System.err.println("");
                 }
             }
             if (animData.length == null) {
@@ -623,7 +673,6 @@ public class GltfLoader implements AssetLoader {
                 Quaternion[] rotations = readAccessorData(dataIndex, quaternionArrayPopulator);
                 animData.rotations = rotations;
             }
-
         }
 
         if (name == null) {
@@ -911,6 +960,20 @@ public class GltfLoader implements AssetLoader {
         Transform armatureTransforms;
     }
 
+    public static class SkinBuffers {
+        short[] joints;
+        float[] weights;
+        int componentSize;
+
+        public SkinBuffers(short[] joints, int componentSize) {
+            this.joints = joints;
+            this.componentSize = componentSize;
+        }
+
+        public SkinBuffers() {
+        }
+    }
+
     private interface Populator<T> {
         T populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException;
     }
@@ -925,6 +988,11 @@ public class GltfLoader implements AssetLoader {
         @Override
         public VertexBuffer populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException {
 
+            if (bufferType == null) {
+                logger.log(Level.WARNING, "could not assign data to any VertexBuffer type for buffer view " + bufferViewIndex);
+                return null;
+            }
+
             VertexBuffer vb = new VertexBuffer(bufferType);
             VertexBuffer.Format format = getVertexBufferFormat(componentType);
             int numComponents = getNumberOfComponents(type);
@@ -935,7 +1003,7 @@ public class GltfLoader implements AssetLoader {
                 //no referenced buffer, specs says to pad the buffer with zeros.
                 padBuffer(buff, bufferSize);
             } else {
-                readBuffer(bufferViewIndex, byteOffset, bufferSize, buff, numComponents);
+                readBuffer(bufferViewIndex, byteOffset, bufferSize, buff, numComponents, format.getComponentSize());
             }
 
             if (bufferType == VertexBuffer.Type.Index) {
@@ -955,13 +1023,13 @@ public class GltfLoader implements AssetLoader {
 
             int numComponents = getNumberOfComponents(type);
             int dataSize = numComponents * count;
-            float[] data = new float[count];
+            float[] data = new float[dataSize];
 
             if (bufferViewIndex == null) {
                 //no referenced buffer, specs says to pad the data with zeros.
                 padBuffer(data, dataSize);
             } else {
-                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents);
+                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, 4);
             }
 
             return data;
@@ -982,9 +1050,8 @@ public class GltfLoader implements AssetLoader {
                 //no referenced buffer, specs says to pad the data with zeros.
                 padBuffer(data, dataSize);
             } else {
-                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents);
+                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, 4);
             }
-
             return data;
         }
     }
@@ -1002,30 +1069,47 @@ public class GltfLoader implements AssetLoader {
                 //no referenced buffer, specs says to pad the data with zeros.
                 padBuffer(data, dataSize);
             } else {
-                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents);
+                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, 4);
             }
 
             return data;
         }
     }
 
-    private class Matrix4fArrayPopulator implements Populator<Matrix4f[]> {
+    private class JointData {
+        short[] joints;
+        int componentSize;
+
+        public JointData(short[] joints, int componentSize) {
+            this.joints = joints;
+            this.componentSize = componentSize;
+        }
+    }
+
+    private class JointArrayPopulator implements Populator<SkinBuffers> {
 
         @Override
-        public Matrix4f[] populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException {
+        public SkinBuffers populate(Integer bufferViewIndex, int componentType, String type, int count, int byteOffset) throws IOException {
 
             int numComponents = getNumberOfComponents(type);
+
+            //can be bytes or shorts.
+            int componentSize = 1;
+            if (componentType == 5123) {
+                componentSize = 2;
+            }
+
             int dataSize = numComponents * count;
-            Matrix4f[] data = new Matrix4f[count];
+            short[] data = new short[dataSize];
 
             if (bufferViewIndex == null) {
                 //no referenced buffer, specs says to pad the data with zeros.
                 padBuffer(data, dataSize);
             } else {
-                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents);
+                readBuffer(bufferViewIndex, byteOffset, dataSize, data, numComponents, componentSize);
             }
 
-            return data;
+            return new SkinBuffers(data, componentSize);
         }
     }
 }

jme3-plugins/src/gltf/java/com/jme3/scene/plugins/gltf/GltfUtils.java

@@ -10,20 +10,21 @@ import com.jme3.scene.Mesh;
 import com.jme3.scene.Spatial;
 import com.jme3.scene.VertexBuffer;
 import com.jme3.texture.Texture;
-import com.jme3.util.LittleEndien;
+import com.jme3.util.*;
 
 import java.io.*;
 import java.nio.*;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
 
 /**
  * Created by Nehon on 07/08/2017.
  */
 public class GltfUtils {
 
+    private static final Logger logger = Logger.getLogger(GltfUtils.class.getName());
+
     public static Mesh.Mode getMeshMode(Integer mode) {
         if (mode == null) {
             return Mesh.Mode.Triangles;
@@ -120,11 +121,17 @@ public class GltfUtils {
             case "WEIGHTS_0":
                 return VertexBuffer.Type.BoneWeight;
             default:
-                throw new AssetLoadException("Unsupported buffer attribute: " + attribute);
+                logger.log(Level.WARNING, "Unsupported Vertex Buffer type " + attribute);
+                return null;
 
         }
     }
 
+    public static int getIndex(String name) {
+        String num = name.substring(name.lastIndexOf("_") + 1);
+        return Integer.parseInt(num);
+    }
+
     public static Texture.MagFilter getMagFilter(Integer value) {
         if (value == null) {
             return null;
@@ -201,7 +208,12 @@ public class GltfUtils {
             }
             buffer.rewind();
         }
-        if (store instanceof float[]) {
+        if (store instanceof short[]) {
+            short[] array = (short[]) store;
+            for (int i = 0; i < array.length; i++) {
+                array[i] = 0;
+            }
+        } else if (store instanceof float[]) {
             float[] array = (float[]) store;
             for (int i = 0; i < array.length; i++) {
                 array[i] = 0;
@@ -224,41 +236,43 @@ public class GltfUtils {
         }
     }
 
-    public static void populateBuffer(Object store, byte[] source, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    public static void populateBuffer(Object store, byte[] source, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
 
         if (store instanceof Buffer) {
             Buffer buffer = (Buffer) store;
             buffer.clear();
             if (buffer instanceof ByteBuffer) {
-                populateByteBuffer((ByteBuffer) buffer, source, length, byteOffset, byteStride, numComponents);
+                populateByteBuffer((ByteBuffer) buffer, source, length, byteOffset, byteStride, numComponents, componentSize);
                 return;
             }
             LittleEndien stream = getStream(source);
             if (buffer instanceof ShortBuffer) {
-                populateShortBuffer((ShortBuffer) buffer, stream, length, byteOffset, byteStride, numComponents);
+                populateShortBuffer((ShortBuffer) buffer, stream, length, byteOffset, byteStride, numComponents, componentSize);
             } else if (buffer instanceof IntBuffer) {
-                populateIntBuffer((IntBuffer) buffer, stream, length, byteOffset, byteStride, numComponents);
+                populateIntBuffer((IntBuffer) buffer, stream, length, byteOffset, byteStride, numComponents, componentSize);
             } else if (buffer instanceof FloatBuffer) {
-                populateFloatBuffer((FloatBuffer) buffer, stream, length, byteOffset, byteStride, numComponents);
+                populateFloatBuffer((FloatBuffer) buffer, stream, length, byteOffset, byteStride, numComponents, componentSize);
             }
             buffer.rewind();
             return;
         }
         LittleEndien stream = getStream(source);
+        if (store instanceof short[]) {
+            populateShortArray((short[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize);
+        } else
         if (store instanceof float[]) {
-            populateFloatArray((float[]) store, stream, length, byteOffset, byteStride, numComponents);
+            populateFloatArray((float[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize);
         } else if (store instanceof Vector3f[]) {
-            populateVector3fArray((Vector3f[]) store, stream, length, byteOffset, byteStride, numComponents);
+            populateVector3fArray((Vector3f[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize);
         } else if (store instanceof Quaternion[]) {
-            populateQuaternionArray((Quaternion[]) store, stream, length, byteOffset, byteStride, numComponents);
+            populateQuaternionArray((Quaternion[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize);
         } else if (store instanceof Matrix4f[]) {
-            populateMatrix4fArray((Matrix4f[]) store, stream, length, byteOffset, byteStride, numComponents);
+            populateMatrix4fArray((Matrix4f[]) store, stream, length, byteOffset, byteStride, numComponents, componentSize);
         }
     }
 
-    private static void populateByteBuffer(ByteBuffer buffer, byte[] source, int length, int byteOffset, int byteStride, int numComponents) {
+    private static void populateByteBuffer(ByteBuffer buffer, byte[] source, int length, int byteOffset, int byteStride, int numComponents, int componentSize) {
         int index = byteOffset;
-        int componentSize = 1;
         while (index < length + byteOffset) {
             for (int i = 0; i < numComponents; i++) {
                 buffer.put(source[index + i]);
@@ -267,9 +281,8 @@ public class GltfUtils {
         }
     }
 
-    private static void populateShortBuffer(ShortBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    private static void populateShortBuffer(ShortBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
         int index = byteOffset;
-        int componentSize = 2;
         int end = length * componentSize + byteOffset;
         stream.skipBytes(byteOffset);
         while (index < end) {
@@ -280,9 +293,8 @@ public class GltfUtils {
         }
     }
 
-    private static void populateIntBuffer(IntBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    private static void populateIntBuffer(IntBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
         int index = byteOffset;
-        int componentSize = 4;
         int end = length * componentSize + byteOffset;
         stream.skipBytes(byteOffset);
         while (index < end) {
@@ -293,9 +305,8 @@ public class GltfUtils {
         }
     }
 
-    private static void populateFloatBuffer(FloatBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    private static void populateFloatBuffer(FloatBuffer buffer, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
         int index = byteOffset;
-        int componentSize = 4;
         int end = length * componentSize + byteOffset;
         stream.skipBytes(byteOffset);
         while (index < end) {
@@ -306,9 +317,94 @@ public class GltfUtils {
         }
     }
 
-    private static void populateFloatArray(float[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    private static void populateShortArray(short[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
+        int index = byteOffset;
+        int end = length * componentSize + byteOffset;
+        stream.skipBytes(byteOffset);
+        int arrayIndex = 0;
+        while (index < end) {
+            for (int i = 0; i < numComponents; i++) {
+                if (componentSize == 2) {
+                    array[arrayIndex] = stream.readShort();
+                } else {
+                    array[arrayIndex] = stream.readByte();
+                }
+                arrayIndex++;
+            }
+
+            index += Math.max(componentSize * numComponents, byteStride);
+        }
+    }
+
+    public static byte[] toByteArray(short[] shortArray) {
+        byte[] bytes = new byte[shortArray.length];
+        for (int i = 0; i < shortArray.length; i++) {
+            bytes[i] = (byte) shortArray[i];
+        }
+        return bytes;
+    }
+
+
+    public static void handleSkinningBuffers(Mesh mesh, IntMap<GltfLoader.SkinBuffers> skinBuffers) {
+        if (skinBuffers.size() > 0) {
+            if (skinBuffers.size() == 1) {
+                GltfLoader.SkinBuffers buffs = skinBuffers.get(0);
+                setSkinBuffers(mesh, buffs.joints, skinBuffers.get(0).weights, buffs.componentSize);
+            } else {
+
+                int length = skinBuffers.get(0).joints.length;
+                short[] jointsArray = new short[length];
+                float[] weightsArray = new float[length];
+                List<GltfLoader.WeightData> weightData = new ArrayList<>();
+                int componentSize = 1;
+
+                for (int i = 0; i < weightsArray.length; i += 4) {
+                    weightData.clear();
+                    for (int j = 0; j < skinBuffers.size(); j++) {
+                        GltfLoader.SkinBuffers buffs = skinBuffers.get(j);
+                        for (int k = 0; k < 4; k++) {
+                            weightData.add(new GltfLoader.WeightData(buffs.weights[i + k], buffs.joints[i + k], buffs.componentSize));
+                        }
+
+                    }
+                    Collections.sort(weightData, new Comparator<GltfLoader.WeightData>() {
+                        @Override
+                        public int compare(GltfLoader.WeightData o1, GltfLoader.WeightData o2) {
+                            if (o1.value > o2.value) {
+                                return -1;
+                            } else if (o1.value < o2.value) {
+                                return 1;
+                            } else {
+                                return 0;
+                            }
+                        }
+                    });
+                    for (int j = 0; j < 4; j++) {
+                        GltfLoader.WeightData data = weightData.get(j);
+                        jointsArray[i + j] = data.index;
+                        weightsArray[i + j] = data.value;
+                        if (data.componentSize > componentSize) {
+                            componentSize = data.componentSize;
+                        }
+                    }
+                }
+                setSkinBuffers(mesh, jointsArray, weightsArray, componentSize);
+            }
+        }
+    }
+
+
+    public static void setSkinBuffers(Mesh mesh, short[] jointsArray, float[] weightsArray, int componentSize) {
+        if (componentSize == 1) {
+            mesh.setBuffer(VertexBuffer.Type.BoneIndex, 4, BufferUtils.createByteBuffer(toByteArray(jointsArray)));
+        } else {
+            mesh.setBuffer(VertexBuffer.Type.BoneIndex, 4, BufferUtils.createShortBuffer(jointsArray));
+        }
+        mesh.setBuffer(VertexBuffer.Type.BoneWeight, 4, BufferUtils.createFloatBuffer(weightsArray));
+    }
+
+    private static void populateFloatArray(float[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
         int index = byteOffset;
-        int componentSize = 4;
         int end = length * componentSize + byteOffset;
         stream.skipBytes(byteOffset);
         int arrayIndex = 0;
@@ -322,9 +418,8 @@ public class GltfUtils {
         }
     }
 
-    private static void populateVector3fArray(Vector3f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    private static void populateVector3fArray(Vector3f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
         int index = byteOffset;
-        int componentSize = 4;
         int end = length * componentSize + byteOffset;
         stream.skipBytes(byteOffset);
         int arrayIndex = 0;
@@ -341,9 +436,8 @@ public class GltfUtils {
         }
     }
 
-    private static void populateQuaternionArray(Quaternion[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    private static void populateQuaternionArray(Quaternion[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
         int index = byteOffset;
-        int componentSize = 4;
         int end = length * componentSize + byteOffset;
         stream.skipBytes(byteOffset);
         int arrayIndex = 0;
@@ -361,9 +455,8 @@ public class GltfUtils {
         }
     }
 
-    private static void populateMatrix4fArray(Matrix4f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents) throws IOException {
+    private static void populateMatrix4fArray(Matrix4f[] array, LittleEndien stream, int length, int byteOffset, int byteStride, int numComponents, int componentSize) throws IOException {
         int index = byteOffset;
-        int componentSize = 4;
         int end = length * componentSize + byteOffset;
         stream.skipBytes(byteOffset);
         int arrayIndex = 0;