Added some more utilities methods to improve test output.

Corrrected format in TestTimer overloads to only show 2 decimal places.

Improved formatting of TestBuiltins test output.

src/ShaderGen.Tests/BuiltinsTests.cs

@@ -140,6 +140,9 @@ namespace ShaderGen.Tests
 
                 generationResult = sg.GenerateShaders();
             }
+            _output.WriteLine(string.Empty);
+            _output.WriteLine(TestUtil.Spacer1);
+            _output.WriteLine(string.Empty);
 
             /*
              * Loop through each backend to run tests.
@@ -148,9 +151,20 @@ namespace ShaderGen.Tests
             // Allocate enough space to store the result sets for each backend!
             Dictionary<ToolChain, byte[]> gpuResults =
                 toolChains.ToDictionary(t => t, t => new byte[mappings.ResultSetSize * TestLoops]);
-            
+
+            bool first = true;
             foreach (LanguageBackend backend in backends)
             {
+                if (first)
+                {
+                    first = false;
+                }
+                else
+                {
+                    _output.WriteLine(string.Empty);
+                    _output.WriteLine(TestUtil.Spacer2);
+                    _output.WriteLine(string.Empty);
+                }
                 ToolChain toolChain = ToolChain.Get(backend);
                 GeneratedShaderSet set;
                 CompileResult compilationResult;
@@ -185,9 +199,9 @@ namespace ShaderGen.Tests
                     ResourceFactory factory = graphicsDevice.ResourceFactory;
                     using (DeviceBuffer inOutBuffer = factory.CreateBuffer(
                         new BufferDescription(
-                            (uint) mappings.BufferSize,
+                            (uint)mappings.BufferSize,
                             BufferUsage.StructuredBufferReadWrite,
-                            (uint) mappings.StructSize)))
+                            (uint)mappings.StructSize)))
 
                     using (Shader computeShader = factory.CreateShader(
                         new ShaderDescription(
@@ -202,7 +216,7 @@ namespace ShaderGen.Tests
 
                     using (Pipeline computePipeline = factory.CreateComputePipeline(new ComputePipelineDescription(
                         computeShader,
-                        new[] {inOutStorageLayout},
+                        new[] { inOutStorageLayout },
                         1, 1, 1)))
 
 
@@ -233,7 +247,7 @@ namespace ShaderGen.Tests
                                     0,
                                     // Get the portion of test data for the current test loop
                                     Marshal.UnsafeAddrOfPinnedArrayElement(testData, mappings.BufferSize * test),
-                                    (uint) mappings.BufferSize);
+                                    (uint)mappings.BufferSize);
                                 graphicsDevice.WaitForIdle();
 
                                 // Execute compute shaders
@@ -268,60 +282,84 @@ namespace ShaderGen.Tests
                 }
             }
 
+            _output.WriteLine(string.Empty);
+            _output.WriteLine(TestUtil.Spacer1);
+            _output.WriteLine(string.Empty);
+
             Assert.True(gpuResults.Count > 0);
 
-            /*
-             * Finally, evaluate differences between results
-             */
-            // Get pointer array
-            int offset = 0;
-            byte[][] rArray = gpuResults.Values.ToArray();
-            foreach (MethodMap method in mappings.Methods)
+            string lastMethodName = null;
+            using (new TestTimer(_output, "Analysing results"))
             {
-                if (method.Return == null)
+                /*
+                 * Finally, evaluate differences between results
+                 */
+                // Get pointer array
+                int offset = 0;
+                byte[][] rArray = gpuResults.Values.ToArray();
+                foreach (MethodMap method in mappings.Methods)
                 {
-                    // This method has no results, so just skip it
-                    _output.WriteLine($"The {method.Method.Name} does not have any results to compare.");
-                    continue;
-                }
-
-                // Get the result field
-                PaddedStructCreator.Field resultField = mappings.BufferFields[method.Return];
-                int resultSize = resultField.AlignmentInfo.ShaderSize;
+                    if (method.Return == null)
+                    {
+                        // This method has no results, so just skip it
+                        _output.WriteLine($"The {method.Method.Name} does not have any results to compare.");
+                        continue;
+                    }
 
-                int failures = 0;
-                for (int test = 0; test < TestLoops; test++)
-                {
-                    // Perform simple byte scan to detect differences first.
-                    int s = test * mappings.ResultSetSize + offset;
-                    int e = s + resultSize;
+                    // Get the result field
+                    PaddedStructCreator.Field resultField = mappings.BufferFields[method.Return];
+                    int resultSize = resultField.AlignmentInfo.ShaderSize;
 
-                    while (s < e)
+                    int failures = 0;
+                    for (int test = 0; test < TestLoops; test++)
                     {
-                        byte check = cpuResults[s];
-                        for (int i = 0; i < rArray.Length; i++)
+                        // Perform simple byte scan to detect differences first.
+                        int s = test * mappings.ResultSetSize + offset;
+                        int e = s + resultSize;
+
+                        while (s < e)
                         {
-                            if (rArray[i][s] != check)
+                            byte check = cpuResults[s];
+                            for (int i = 0; i < rArray.Length; i++)
                             {
-                                goto failed;
+                                if (rArray[i][s] != check)
+                                {
+                                    goto failed;
+                                }
                             }
+
+                            s++;
                         }
 
-                        s++;
+                        continue;
+
+                        failed:
+                        failures++;
                     }
 
-                    continue;
+                    if (lastMethodName != method.Method.Name)
+                    {
+                        if (lastMethodName != null)
+                        {
+                            _output.WriteLine(string.Empty);
+                            _output.WriteLine(TestUtil.Spacer2);
+                            _output.WriteLine(string.Empty);
+                        }
 
-                    failed:
-                    failures++;
-                }
+                        lastMethodName = method.Method.Name;
+                    }
 
-                _output.WriteLine(
-                    failures > 0
-                        ? $"{method.Signature} had inconsistent results {failures} times out of {TestLoops} ({failures * 100.0 / TestLoops:#.##}%)."
-                        : $"{method.Signature} was always consistent.");
+                    _output.WriteLine(
+                        failures > 0
+                            ? $"{TestUtil.GetUnicodePieChart((double)failures / TestLoops)} {method.Signature} failed {failures}/{TestLoops} ({failures * 100.0 / TestLoops:#.##}%)."
+                            : $"✓ {method.Signature} consistent {TestLoops} times.");
 
-                offset += resultSize;
+                    offset += resultSize;
+                }
+
+                _output.WriteLine(string.Empty);
+                _output.WriteLine(TestUtil.Spacer2);
+                _output.WriteLine(string.Empty);
             }
         }
 
@@ -424,7 +462,7 @@ namespace ShaderGen.Tests
         /// <summary>
         /// Holds information about the mappings of tested methods to the buffer.
         /// </summary>
-        internal class Mappings
+        private class Mappings
         {
             /// <summary>
             /// The buffer size required.
@@ -534,7 +572,7 @@ namespace ShaderGen.Tests
         /// <summary>
         /// Holds information about the mapping of a tested method parameters and return to a buffer.
         /// </summary>
-        internal class MethodMap
+        private class MethodMap
         {
             /// <summary>
             /// The index of the method.
@@ -574,14 +612,14 @@ namespace ShaderGen.Tests
                 Signature =
                     $"{method.ReturnType.Name} {method.DeclaringType.FullName}.{method.Name}({string.Join(", ", Parameters.Select(p => $"{p.Key.ParameterType.Name} {p.Key.Name}"))})";
             }
-            
+
             /// <summary>
             /// Gets the signature.
             /// </summary>
             /// <value>
             /// The signature.
             /// </value>
-            public string Signature { get; private  set; }
+            public string Signature { get; private set; }
 
             /// <summary>
             /// Generates test data for this method, executes it and stores the result.

src/ShaderGen.Tests/ShaderBuiltinsTests.cs

@@ -349,9 +349,6 @@ namespace ShaderGen.Tests
             {
                 _output.WriteLine($"{failures.Count} methods experienced failures out of {ShaderBuiltinsComputeTest.Methods} ({100f * failures.Count / ShaderBuiltinsComputeTest.Methods:##.##}%).  Details follow...");
 
-                string spacer1 = new string('=', 80);
-                string spacer2 = new string('-', 80);
-
                 int failed = 0;
 
                 // Output failures
@@ -361,7 +358,7 @@ namespace ShaderGen.Tests
                     notIdential++;
                     int methodFailureCount = method.Value.Count;
                     _output.WriteLine(string.Empty);
-                    _output.WriteLine(spacer1);
+                    _output.WriteLine(TestUtil.Spacer1);
                     float failureRate = 100f * methodFailureCount / loops;
                     if (failureRate > MaximumFailureRate)
                     {
@@ -374,7 +371,7 @@ namespace ShaderGen.Tests
 
                     foreach (var group in method.Value.SelectMany(t => t.differences).ToLookup(f => f.fieldName).OrderByDescending(g => g.Count()))
                     {
-                        _output.WriteLine(spacer2);
+                        _output.WriteLine(TestUtil.Spacer2);
                         _output.WriteLine(string.Empty);
 
                         int fieldFailureCount = group.Count();

src/ShaderGen.Tests/TestTimer.cs

@@ -18,12 +18,12 @@ namespace ShaderGen.Tests
         public TestTimer(ITestOutputHelper output, string message)
         {
             _timeStamp = Stopwatch.GetTimestamp();
-            _action = t => output.WriteLine($"{message} took {t * 1000}ms");
+            _action = t => output.WriteLine($"{message} took {t * 1000:#.##}ms");
         }
         public TestTimer(ITestOutputHelper output, Func<string> getMessage)
         {
             _timeStamp = Stopwatch.GetTimestamp();
-            _action = t => output.WriteLine($"{getMessage()} took {t * 1000}ms");
+            _action = t => output.WriteLine($"{getMessage()} took {t * 1000:#.##}ms");
         }
         public TestTimer(ITestOutputHelper output, Func<double, string> getMessage)
         {
@@ -33,7 +33,7 @@ namespace ShaderGen.Tests
 
         public void Dispose()
         {
-            _action(((double) Stopwatch.GetTimestamp() - _timeStamp) / Stopwatch.Frequency);
+            _action(((double)Stopwatch.GetTimestamp() - _timeStamp) / Stopwatch.Frequency);
         }
     }
 }

src/ShaderGen.Tests/TestUtil.cs

@@ -20,6 +20,16 @@ namespace ShaderGen.Tests
 {
     internal static class TestUtil
     {
+        /// <summary>
+        /// A string of '═' symbols
+        /// </summary>
+        public static readonly string Spacer1 = new string('═', 80);
+
+        /// <summary>
+        /// A string of '━' symbols
+        /// </summary>
+        public static readonly string Spacer2 = new string('━', 80);
+
         private static readonly string ProjectBasePath = Path.Combine(AppContext.BaseDirectory, "TestAssets");
 
         public static Compilation GetCompilation()
@@ -427,5 +437,42 @@ namespace ShaderGen.Tests
                     : _memoryUnitsShort[unit]);
         }
         #endregion
+
+        public static bool ApproximatelyEqual(this float a, float b, float epsilon = float.Epsilon)
+        {
+            const float floatNormal = (1 << 23) * float.Epsilon;
+            float absA = Math.Abs(a);
+            float absB = Math.Abs(b);
+            float diff = Math.Abs(a - b);
+
+            if (a == b)
+            {
+                // Shortcut, handles infinities
+                return true;
+            }
+
+            if (a == 0.0f || b == 0.0f || diff < floatNormal)
+            {
+                // a or b is zero, or both are extremely close to it.
+                // relative error is less meaningful here
+                return diff < (epsilon * floatNormal);
+            }
+
+            // use relative error
+            return diff / Math.Min((absA + absB), float.MaxValue) < epsilon;
+        }
+
+        /// <summary>
+        /// The unicode characters to represent a pie chart.
+        /// </summary>
+        private static readonly char[] UnicodePieChars = { '○', '◔', '◑', '◕', '●' };
+
+        /// <summary>
+        /// Gets the unicode symbol to represent the <paramref name="ratio"/> as a pie chart.
+        /// </summary>
+        /// <param name="ratio">The ratio.</param>
+        /// <returns></returns>
+        public static char GetUnicodePieChart(double ratio) =>
+            UnicodePieChars[(int)Math.Round(Math.Max(Math.Min(ratio, 1.0), 0.0) * (UnicodePieChars.Length - 1))];
     }
 }