A failure to compile a backend will not cause the test set to be skipped from analysis (before it would cause it to show as different), allowing the tests to continue if at least two backends can be compared.

src/ShaderGen.Tests/AutoGenerated/BuiltinsTests.cs

@@ -189,6 +189,9 @@ namespace ShaderGen.Tests.AutoGenerated
             _output.WriteLine(TestUtil.Spacer1);
             _output.WriteLine(string.Empty);
 
+            Assert.True(testSets.Count(t => t.Executed) > 1,
+                "At least 2 test sets are required for comparison.");
+
             /*
              * Finally, evaluate differences between results
              */

src/ShaderGen.Tests/AutoGenerated/Failure.cs

@@ -73,6 +73,7 @@ namespace ShaderGen.Tests.AutoGenerated
                     IEqualityComparer<object> comparer)
         {
             IReadOnlyList<(object Result, IReadOnlyList<TestSet> TestSets)> results = testSets
+                .Where(t => t.Results != null)
                 .Select(t => (TestSet: t, Result: methodMap.GetResult(t.Results, test)))
                 .GroupBy(r => r.Result, r => r.TestSet, comparer)
                 .Select(g => (Result: g.Key, TestSets: (IReadOnlyList<TestSet>)g.ToArray()))

src/ShaderGen.Tests/AutoGenerated/TestSet.cs

@@ -39,14 +39,13 @@ namespace ShaderGen.Tests.AutoGenerated
         /// </summary>
         public byte[] Results { get; private set; }
 
-        private int _executed = 0;
         /// <summary>
         /// Gets a value indicating whether this <see cref="TestSet"/> has been executed yet.
         /// </summary>
         /// <value>
         ///   <c>true</c> if executed; otherwise, <c>false</c>.
         /// </value>
-        public bool Executed => _executed > 0;
+        public bool Executed => Results != null;
 
         /// <summary>
         /// Initializes a new instance of the <see cref="TestSet"/> class.
@@ -59,7 +58,20 @@ namespace ShaderGen.Tests.AutoGenerated
             Name = toolChain?.GraphicsBackend.ToString() ?? "CPU";
             ToolChain = toolChain;
             Backend = toolChain?.CreateBackend(testSets.Compilation);
-            Results = Array.Empty<byte>();
+        }
+
+        /// <summary>
+        /// Allocates the results buffer.
+        /// </summary>
+        /// <param name="output">The output.</param>
+        private void AllocateResults(ITestOutputHelper output)
+        {
+            // Create results data structure.
+            using (new TestTimer(output,
+                $"Creating result buffer ({(TestSets.Mappings.ResultSetSize * TestSets.TestLoops).ToMemorySize()})"))
+            {
+                Results = new byte[TestSets.Mappings.ResultSetSize * TestSets.TestLoops];
+            }
         }
 
         /// <summary>
@@ -73,20 +85,13 @@ namespace ShaderGen.Tests.AutoGenerated
             string csFunctionName,
             ITestOutputHelper output)
         {
-            if (Interlocked.CompareExchange(ref _executed, 1, 0) > 0)
+            if (Executed)
             {
                 output.WriteLine(
                     $"The {Name} tests have already been executed!");
                 return;
             }
 
-            // Create results data structure.
-            using (new TestTimer(output,
-                $"Creating result buffer ({(TestSets.Mappings.ResultSetSize * TestSets.TestLoops).ToMemorySize()})"))
-            {
-                Results = new byte[TestSets.Mappings.ResultSetSize * TestSets.TestLoops];
-            }
-
             TestSets testSets = TestSets;
             Mappings mappings = testSets.Mappings;
 
@@ -95,6 +100,7 @@ namespace ShaderGen.Tests.AutoGenerated
                 /*
                  * Generate the test data and the result set data for the CPU.
                  */
+                AllocateResults(output);
                 using (new TestTimer(output,
                     $"Running {testSets.TestLoops} iterations on the {Name} backend"))
                 {
@@ -172,6 +178,9 @@ namespace ShaderGen.Tests.AutoGenerated
 
                     output.WriteLine($"Created compute pipeline for {Name} backend.");
 
+                    // Allocate the results buffer
+                    AllocateResults(output);
+
                     using (new TestTimer(output,
                         $"Running {testSets.TestLoops} iterations on the {Name} backend"))
                     {
@@ -210,6 +219,7 @@ namespace ShaderGen.Tests.AutoGenerated
 
                                 // Read back test results
                                 MappedResource map = graphicsDevice.Map(stagingBuffer, MapMode.Read);
+
                                 mappings.SetResults(map.Data, Results, test);
                                 graphicsDevice.Unmap(stagingBuffer);
                             }