Improve recursion detection to handle member functions and add tests. (#2371)

Recursion detection did not consider member functions.

tools/clang/lib/Sema/SemaHLSL.cpp

@@ -2709,22 +2709,32 @@ namespace hlsl {
 
     bool VisitDeclRefExpr(DeclRefExpr *ref) {
       ValueDecl *valueDecl = ref->getDecl();
-      FunctionDecl *fnDecl = dyn_cast_or_null<FunctionDecl>(valueDecl);
-      fnDecl = getFunctionWithBody(fnDecl);
-      if (fnDecl) {
+      RecordFunctionDecl(dyn_cast_or_null<FunctionDecl>(valueDecl));
+      return true;
+    }
+
+    bool VisitCXXMemberCallExpr(CXXMemberCallExpr* callExpr)
+    {
+      RecordFunctionDecl(callExpr->getMethodDecl());
+      return true;
+    }
+
+    void RecordFunctionDecl(FunctionDecl* funcDecl)
+    {
+      funcDecl = getFunctionWithBody(funcDecl);
+      if (funcDecl) {
         if (m_sourceIt == m_callNodes.end()) {
           auto result = m_callNodes.insert(
-            std::pair<FunctionDecl *, CallNode>(m_source, CallNode{ m_source }));
+            std::pair<FunctionDecl*, CallNode>(m_source, CallNode{ m_source }));
           DXASSERT(result.second == true,
             "else setSourceFn didn't assign m_sourceIt");
           m_sourceIt = result.first;
         }
-        m_sourceIt->second.CalleeFns.insert(fnDecl);
-        if (!m_visitedFunctions.count(fnDecl)) {
-          m_pendingFunctions.push_back(fnDecl);
+        m_sourceIt->second.CalleeFns.insert(funcDecl);
+        if (!m_visitedFunctions.count(funcDecl)) {
+          m_pendingFunctions.push_back(funcDecl);
         }
       }
-      return true;
     }
   };
 

tools/clang/test/CodeGenHLSL/batch/declarations/functions/recursion/free_function_error.hlsl

@@ -0,0 +1,15 @@
+// RUN: %dxc -T vs_6_2 -E main %s | FileCheck %s
+
+// Test for recursion detection. Note that this cannot be a syntax test
+// because we detect from the entry point and syntax tests have none.
+
+// CHECK: error: recursive functions not allowed
+
+void B();
+void A() { B(); }
+void B() { A(); }
+
+void main()
+{
+  A();
+} 

tools/clang/test/CodeGenHLSL/batch/declarations/functions/recursion/instance_function_error.hlsl

@@ -0,0 +1,18 @@
+// RUN: %dxc -T vs_6_2 -E main %s | FileCheck %s
+
+// Test for recursion detection. Note that this cannot be a syntax test
+// because we detect from the entry point and syntax tests have none.
+
+// CHECK: error: recursive functions not allowed
+
+struct MyClass
+{
+  void A() { B(); }
+  void B() { A(); }
+};
+
+void main()
+{
+  MyClass c;
+  c.A();
+} 

tools/clang/test/CodeGenHLSL/batch/declarations/functions/recursion/static_function_error.hlsl

@@ -0,0 +1,17 @@
+// RUN: %dxc -T vs_6_2 -E main %s | FileCheck %s
+
+// Test for recursion detection. Note that this cannot be a syntax test
+// because we detect from the entry point and syntax tests have none.
+
+// CHECK: error: recursive functions not allowed
+
+struct MyClass
+{
+  static void A() { B(); }
+  static void B() { A(); }
+};
+
+void main()
+{
+  MyClass::A();
+}