Add FOR macro to GLSL preprocessor and J3MD (#1758)

* Add FOR macro to GLSL preprocessor

* Code cleanup

* fix  comment

jme3-core/src/plugins/java/com/jme3/material/plugins/J3MLoader.java

@@ -50,6 +50,9 @@ import com.jme3.util.PlaceholderAssets;
 import com.jme3.util.blockparser.BlockLanguageParser;
 import com.jme3.util.blockparser.Statement;
 import com.jme3.util.clone.Cloner;
+import jme3tools.shader.Preprocessor;
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
 import java.util.*;
@@ -798,6 +801,7 @@ public class J3MLoader implements AssetLoader {
             } else if (key.getExtension().equals("j3md") && key instanceof MaterialKey) {
                 throw new IOException("Material definitions must be loaded via AssetKey");
             }
+            in = Preprocessor.apply(in);
             loadFromRoot(BlockLanguageParser.parse(in));
         } finally {
             if (in != null){

jme3-core/src/plugins/java/com/jme3/shader/plugins/GLSLLoader.java

@@ -33,7 +33,7 @@ package com.jme3.shader.plugins;
 
 import com.jme3.asset.*;
 import com.jme3.asset.cache.AssetCache;
-
+import jme3tools.shader.Preprocessor;
 import java.io.*;
 import java.util.*;
 
@@ -85,8 +85,9 @@ public class GLSLLoader implements AssetLoader {
             }
 
             while ((ln = bufferedReader.readLine()) != null) {
-                if (ln.trim().startsWith("#import ")) {
-                    ln = ln.trim().substring(8).trim();
+                String tln = ln.trim();
+                if (tln.startsWith("#import ")) {
+                    ln = tln.substring(8).trim();
                     if (ln.startsWith("\"") && ln.endsWith("\"") && ln.length() > 3) {
                         // import user code
                         // remove quotes to get filename
@@ -105,7 +106,7 @@ public class GLSLLoader implements AssetLoader {
 
                         node.addDependency(sb.length(), dependNode);
                     }
-                } else if (ln.trim().startsWith("#extension ")) {
+                } else if (tln.startsWith("#extension ")) {
                     sbExt.append(ln).append('\n');
                 } else {
                     sb.append(ln).append('\n');
@@ -165,7 +166,9 @@ public class GLSLLoader implements AssetLoader {
         // The input stream provided is for the vertex shader,
         // to retrieve the fragment shader, use the content manager
         this.assetManager = info.getManager();
-        Reader reader = new InputStreamReader(info.openStream());
+        InputStream in = info.openStream();
+        in = Preprocessor.apply(in);
+        Reader reader = new InputStreamReader(in);
         boolean injectDependencies = true;
         if (info.getKey() instanceof ShaderAssetKey) {
             injectDependencies = ((ShaderAssetKey) info.getKey()).isInjectDependencies();

jme3-core/src/test/java/com/jme3/shader/GLSLPreprocessorTest.java

@@ -0,0 +1,78 @@
+/*
+ * Copyright (c) 2009-2022 jMonkeyEngine
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of 'jMonkeyEngine' nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package com.jme3.shader;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
+import java.io.BufferedReader;
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+
+import com.jme3.asset.AssetInfo;
+import com.jme3.asset.AssetKey;
+import com.jme3.system.TestUtil;
+
+import org.junit.Test;
+
+import jme3tools.shader.Preprocessor;
+
+
+public class GLSLPreprocessorTest {
+
+    String readAllAsString(InputStream is) throws Exception{
+        String output = "";
+        BufferedReader reader = new BufferedReader(new InputStreamReader(is));
+        while (true) {
+            String l = reader.readLine();
+            if (l == null) break;
+            if (output != "") output += "\n";
+            output += l;
+        }
+        reader.close();
+        return output;
+    }
+    
+    @Test
+    public void testFOR() throws Exception{
+        String source = "#for i=0..2 (#ifdef IS_SET$i $0 #endif)\n" +
+                "  uniform float m_Something$i;\n" +
+                "#endfor";
+        String processedSource= readAllAsString(Preprocessor.apply(new ByteArrayInputStream(source.getBytes("UTF-8"))));
+
+        AssetInfo testData = TestUtil.createAssetManager().locateAsset(new AssetKey("GLSLPreprocessorTest.testFOR.validOutput"));
+        assertNotNull(testData);
+        String sourceCheck=readAllAsString(testData.openStream());
+        assertEquals(sourceCheck, processedSource);                  
+    }
+}

jme3-core/src/test/resources/GLSLPreprocessorTest.testFOR.validOutput

@@ -0,0 +1,8 @@
+
+#ifdef IS_SET0 
+  uniform float m_Something0;
+ #endif
+
+#ifdef IS_SET1 
+  uniform float m_Something1;
+ #endif

jme3-core/src/tools/java/jme3tools/shader/Preprocessor.java

@@ -0,0 +1,121 @@
+/*
+ * Copyright (c) 2009-2022 jMonkeyEngine
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions are
+ * met:
+ *
+ * * Redistributions of source code must retain the above copyright
+ *   notice, this list of conditions and the following disclaimer.
+ *
+ * * Redistributions in binary form must reproduce the above copyright
+ *   notice, this list of conditions and the following disclaimer in the
+ *   documentation and/or other materials provided with the distribution.
+ *
+ * * Neither the name of 'jMonkeyEngine' nor the names of its contributors
+ *   may be used to endorse or promote products derived from this software
+ *   without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
+ * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
+ * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+package jme3tools.shader;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * GLSL Preprocessor
+ * 
+ * @author Riccardo Balbo
+ */
+public class Preprocessor {
+
+    public static InputStream apply(InputStream in) throws IOException {
+        ByteArrayOutputStream bos = new ByteArrayOutputStream();
+        byte chunk[] = new byte[1024];
+        int read;
+        while ((read = in.read(chunk)) != -1) {
+            bos.write(chunk, 0, read);
+        }
+        bos.close();
+        in.close();
+
+        String code = bos.toString("UTF-8");
+
+        code = Preprocessor.forMacro(code);
+
+        return new ByteArrayInputStream(code.getBytes("UTF-8"));
+    }
+
+    /**
+     * #for i=0..100 ( #ifdef ENABLE_INPUT_$i $0 #endif ) 
+     *      do something with $i
+     * #endfor
+     */
+    private static final Pattern FOR_REGEX = Pattern.compile("([^=]+)=\\s*([0-9]+)\\s*\\.\\.\\s*([0-9]+)\\s*\\((.+)\\)");
+
+    public static String forMacro(String code) {
+        StringBuilder expandedCode = new StringBuilder();
+        StringBuilder currentFor = null;
+        String forDec = null;
+        int skip = 0;
+        String codel[] = code.split("\n");
+        boolean captured = false;
+        for (String l : codel) {
+            if (!captured) {
+                String ln = l.trim();
+                if (ln.startsWith("#for")) {
+                    if (skip == 0) {
+                        forDec = ln;
+                        currentFor = new StringBuilder();
+                        skip++;
+                        continue;
+                    }
+                    skip++;
+                } else if (ln.startsWith("#endfor")) {
+                    skip--;
+                    if (skip == 0) {
+                        forDec = forDec.substring("#for ".length()).trim();
+
+                        Matcher matcher = FOR_REGEX.matcher(forDec);
+                        if (matcher.matches()) {
+                            String varN = "$" + matcher.group(1);
+                            int start = Integer.parseInt(matcher.group(2));
+                            int end = Integer.parseInt(matcher.group(3));
+                            String inj = matcher.group(4);
+                            if (inj.trim().isEmpty()) inj = "$0";
+                            String inCode = currentFor.toString();
+                            currentFor = null;
+                            for (int i = start; i < end; i++) {
+                                expandedCode.append("\n").append(inj.replace("$0", "\n" + inCode ).replace(varN, "" + i)).append("\n");
+                            }
+                            captured = true;
+                            continue;
+                        }
+                    }
+                }
+            }
+            if (currentFor != null) currentFor.append(l).append("\n");
+            else expandedCode.append(l).append("\n");
+        }
+        code = expandedCode.toString();
+        if (captured) code = forMacro(code);
+        return code;
+    }
+
+}