Fix IsTypeNumeric hang with incomplete array (#760)

* Fix IsTypeNumeric hang with incomplete array
* Fix IsTypeNumeric for empty nested struct.

tools/clang/lib/Sema/SemaHLSL.cpp

@@ -6406,15 +6406,25 @@ bool HLSLExternalSource::IsTypeNumeric(QualType type, UINT* count)
     return false;
   case AR_TOBJ_COMPOUND:
     {
+      UINT maxCount = 0;
+      { // Determine maximum count to prevent infinite loop on incomplete array
+        FlattenedTypeIterator itCount(SourceLocation(), type, *this);
+        maxCount = itCount.countRemaining();
+        if (!maxCount) {
+          return false; // empty struct.
+        }
+      }
       FlattenedTypeIterator it(SourceLocation(), type, *this);
-      // Return false for empty struct.
-      if (!it.hasCurrentElement())
-        return false;
       while (it.hasCurrentElement()) {
         bool isFieldNumeric = IsTypeNumeric(it.getCurrentElement(), &subCount);
         if (!isFieldNumeric) {
           return false;
         }
+        if (*count >= maxCount) {
+          // this element is an incomplete array at the end; iterator will not advance past this element.
+          // don't add to *count either, so *count will represent minimum size of the structure.
+          break;
+        }
         *count += (subCount * it.getCurrentElementSize());
         it.advanceCurrentElement(it.getCurrentElementSize());
       }

tools/clang/test/CodeGenHLSL/quick-test/empty_struct2.hlsl

@@ -0,0 +1,20 @@
+// RUN: %dxc -E main -T vs_6_0 %s | FileCheck %s
+
+// Make sure nest empty struct works.
+// CHECK: main
+
+struct KillerStruct {};
+
+struct InnerStruct {
+  KillerStruct s;
+};
+
+struct OuterStruct {
+  InnerStruct s;
+};
+
+cbuffer Params_cbuffer : register(b0) {
+  OuterStruct constants;
+};
+
+float4 main(float4 pos : POSITION) : SV_POSITION { return float4(0, 0, 0, 0); }

tools/clang/test/CodeGenHLSL/quick-test/incomp_array.hlsl

@@ -0,0 +1,77 @@
+// RUN: %dxc -T lib_6_1 %s | FileCheck %s
+
+// Verify no hang on incomplete array
+
+// CHECK: %struct.Special = type { <4 x float>, [0 x i32] }
+// CHECK: %"$Globals" = type { i32, %struct.Special }
+
+typedef const int inta[];
+
+// CHECK: @s_testa = internal unnamed_addr constant [3 x i32] [i32 1, i32 2, i32 3], align 4
+static inta s_testa = {1, 2, 3};
+
+int i;
+
+struct Special {
+  float4 member;
+  inta a;
+};
+
+Special c_special;
+
+static const Special s_special = { { 1, 2, 3, 4}, { 1, 2, 3 } };
+
+// CHECK: define void
+// CHECK: fn1
+// @"\01?fn1@@YA?AV?$vector@M$03@@USpecial@@@Z"
+float4 fn1(in Special in1: SEMANTIC_IN) : SEMANTIC_OUT {
+  // CHECK: call %dx.types.CBufRet.i32 @dx.op.cbufferLoadLegacy.i32(
+  // CHECK: i32 0)
+  // CHECK: extractvalue
+  // CHECK: , 0
+  // CHECK: getelementptr
+  // CHECK: load i32, i32*
+  // CHECK: sitofp i32
+  // CHECK: fadd float
+  return in1.member + (float)s_testa[i];
+}
+
+// CHECK: define void
+// CHECK: fn2
+// @"\01?fn2@@YA?AV?$vector@M$03@@USpecial@@@Z"
+float4 fn2(in Special in1: SEMANTIC_IN) : SEMANTIC_OUT {
+  // s_special.a[i] is broken: it just assumes 0.
+  return in1.member + (float)s_special.a[i];
+}
+
+// CHECK: define void
+// CHECK: fn3
+// @"\01?fn3@@YA?AV?$vector@M$03@@USpecial@@@Z"
+float4 fn3(in Special in1: SEMANTIC_IN) : SEMANTIC_OUT {
+  // CHECK: call %dx.types.CBufRet.i32 @dx.op.cbufferLoadLegacy.i32(
+  // CHECK: i32 0)
+  // CHECK: extractvalue
+  // CHECK: , 0
+  // CHECK: getelementptr
+  // CHECK: load i32, i32*
+  // CHECK: sitofp i32
+  // CHECK: fadd float
+  return in1.member + (float)in1.a[i];
+}
+
+// CHECK: define void
+// CHECK: fn4
+// @"\01?fn4@@YA?AV?$vector@M$03@@USpecial@@@Z"
+float4 fn4(in Special in1: SEMANTIC_IN) : SEMANTIC_OUT {
+  // CHECK: call %dx.types.CBufRet.i32 @dx.op.cbufferLoadLegacy.i32(
+  // CHECK: i32 0)
+  // CHECK: extractvalue
+  // CHECK: , 0
+  // CHECK: add
+  // CHECK: call %dx.types.CBufRet.i32 @dx.op.cbufferLoadLegacy.i32(
+  // CHECK: extractvalue
+  // CHECK: , 0
+  // CHECK: sitofp i32
+  // CHECK: fadd float
+  return in1.member + c_special.a[i];
+}