Support dumping implicit top-level decls (#5007)

This change reworks how we were filtering top-level implicit decls from
the `-ast-dump` output. The change has a couple of key differences, but
the behavior of the `-ast-dump` flag remains unchanged.

The differences in this change are:
* By default Decl::dump _will_ dump implicit top-level decls which
allows attached debuggers to dump top-level decls.
* Top-level implicit decls are determined by the decl's `Implicit` bit
rather than an invalid source location.
* Disabling dumping implicit decls is now a language option, so it can
be toggled.
* A new `-ast-dump-implicit` flag is added to dump the AST including
implicit top-level decls.

The attached test case verifies that the `-ast-dump` flag does not dump
implicit top-level decls, and it is intended to be strict enough that
if we add new top-level decls in the future and fail to mark them as
`implicit` the test will fail.

The attached test also trys to fill some gaps in testing around the
lazy decl initialization by verifying that incomplete implicit decls
are generated, and that a used decl gets completed as expected.

include/dxc/Support/HLSLOptions.h

@@ -141,6 +141,7 @@ public:
   bool AllResourcesBound = false; // OPT_all_resources_bound
   bool IgnoreOptSemDefs = false; // OPT_ignore_opt_semdefs
   bool AstDump = false; // OPT_ast_dump
+  bool AstDumpImplicit = false; // OPT_ast_dump_implicit
   bool ColorCodeAssembly = false; // OPT_Cc
   bool CodeGenHighLevel = false; // OPT_fcgl
   bool AllowPreserveValues = false; // OPT_preserve_intermediate_values

include/dxc/Support/HLSLOptions.td

@@ -248,6 +248,8 @@ def _version : Flag<["--"], "version">, Group<hlslcore_Group>, Flags<[DriverOpti
 
 def ast_dump : Flag<["-", "/"], "ast-dump">, Flags<[CoreOption, DriverOption, HelpHidden]>,
   HelpText<"Dumps the parsed Abstract Syntax Tree.">; // should not be core, but handy workaround until explicit API written
+def ast_dump_implicit : Flag<["-", "/"], "ast-dump-implicit">, Flags<[CoreOption, DriverOption, HelpHidden]>,
+  HelpText<"Dumps the parsed Abstract Syntax Tree, including implicit top-level decls.">; 
 def dump_dependencies : Flag<["-", "/"], "M">, Flags<[CoreOption, DriverOption]>,
   HelpText<"Dumps the list of the compilation target dependencies.">;
 def write_dependencies : Flag<["-", "/"], "MD">, Flags<[CoreOption, DriverOption]>,

lib/DxcSupport/HLSLOptions.cpp

@@ -573,7 +573,10 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
       }
     }
   }
-  opts.AstDump = Args.hasFlag(OPT_ast_dump, OPT_INVALID, false);
+  opts.AstDumpImplicit = Args.hasFlag(OPT_ast_dump_implicit, OPT_INVALID, false);
+  // -ast-dump-implicit should imply -ast-dump.
+  opts.AstDump =
+      Args.hasFlag(OPT_ast_dump, OPT_INVALID, false) || opts.AstDumpImplicit;
   opts.WriteDependencies =
       Args.hasFlag(OPT_write_dependencies, OPT_INVALID, false);
   opts.OutputFileForDependencies =

tools/clang/include/clang/Basic/LangOptions.h

@@ -160,6 +160,7 @@ public:
   bool EnableDX9CompatMode = false;
   bool EnableFXCCompatMode = false;
   bool EnablePayloadAccessQualifiers = false;
+  bool DumpImplicitTopLevelDecls = true;
   // HLSL Change Ends
 
   bool SPIRV = false;  // SPIRV Change

tools/clang/lib/AST/ASTDumper.cpp

@@ -118,6 +118,8 @@ namespace  {
 
     bool ShowColors;
 
+    bool DumpImplicitTopLevelDecls = true; // HLSL Change
+
     /// Dump a child of the current node.
     template<typename Fn> void dumpChild(Fn doDumpChild) {
       // If we're at the top level, there's nothing interesting to do; just
@@ -210,6 +212,10 @@ namespace  {
         LastLocFilename(""), LastLocLine(~0U),
         ShowColors(ShowColors) { }
 
+    // HLSL Change Start - filter implicit decls.
+    void setDumpImplicitTopLevelDecls(bool B) { DumpImplicitTopLevelDecls = B; }
+    // HLSL Change End - filter implicit decls.
+
     void dumpDecl(const Decl *D);
     void dumpStmt(const Stmt *S);
     void dumpFullComment(const FullComment *C);
@@ -734,8 +740,12 @@ void ASTDumper::dumpDeclContext(const DeclContext *DC) {
   if (!DC)
     return;
 
+  // HLSL Change Starts: Don't dump implicit decls unless requested.
+  bool ShouldDumpImplicit = DumpImplicitTopLevelDecls || !DC->isTranslationUnit();
   for (auto *D : DC->noload_decls())
-    dumpDecl(D);
+    if (ShouldDumpImplicit || !D->isImplicit())
+      dumpDecl(D);
+  // HLSL Change Ends
 
   if (DC->hasExternalLexicalStorage()) {
     dumpChild([=]{
@@ -1071,15 +1081,6 @@ void ASTDumper::dumpHLSLUnusualAnnotations(const ArrayRef<hlsl::UnusualAnnotatio
 // HLSL Change Ends
 
 void ASTDumper::dumpDecl(const Decl *D) {
-  // HLSL Change Starts: Don't display decls with invalid SourceLocations.
-  if (D && D->getDeclContext() &&
-      D->getDeclContext()->getDeclKind() == Decl::Kind::TranslationUnit &&
-      D->getSourceRange().isInvalid())
-  {
-    return;
-  }
-  // HLSL Change Ends
-
   dumpChild([=] {
     if (!D) {
       ColorScope Color(*this, NullColor);
@@ -2447,6 +2448,8 @@ LLVM_DUMP_METHOD void Decl::dump() const { dump(llvm::errs()); }
 LLVM_DUMP_METHOD void Decl::dump(raw_ostream &OS) const {
   ASTDumper P(OS, &getASTContext().getCommentCommandTraits(),
               &getASTContext().getSourceManager());
+  // HLSL Change - Support suppressing dumping implicit decls.
+  P.setDumpImplicitTopLevelDecls(getASTContext().getLangOpts().DumpImplicitTopLevelDecls);
   P.dumpDecl(this);
 }
 

tools/clang/test/HLSLFileCheck/hlsl/template/FloatLiteralParameter.hlsl

@@ -1,4 +1,4 @@
-// RUN: %dxc -T cs_6_5 -HV 2021 -ast-dump %s | FileCheck -check-prefix=AST %s
+// RUN: %dxc -T cs_6_5 -HV 2021 -ast-dump-implicit %s | FileCheck -check-prefix=AST %s
 // RUN: %dxc -T cs_6_5 -HV 2021 %s | FileCheck %s
 RWStructuredBuffer<float> Output;
 
@@ -35,9 +35,9 @@ void main() {
 // literal types in, because we can constant fold them as with higher precision.
 
 // Verify that the abs call is still `literal float`
-// AST: DeclRefExpr 0x{{[0-9a-zA-Z]+}} <col:15> 'literal float (literal float)' lvalue Function 0x{{[0-9a-zA-Z]+}} 'abs'
+// AST: DeclRefExpr 0x{{[0-9a-zA-Z]+}} <col:15> 'literal float (literal float)' lvalue Function [[abs:0x[0-9a-zA-Z]+]] 'abs'
 
-// AST:      FunctionDecl 0x{{[0-9a-zA-Z]+}} <<invalid sloc>> <invalid sloc> implicit used abs 'literal float (literal float)' extern
+// AST:      FunctionDecl [[abs]] <<invalid sloc>> <invalid sloc> implicit used abs 'literal float (literal float)' extern
 // AST-NEXT: ParmVarDecl 0x{{[0-9a-zA-Z]+}} <<invalid sloc>> <invalid sloc> x 'literal float'
 
 

tools/clang/test/HLSLFileCheck/hlsl/template/IntLiteralParameter.hlsl

@@ -1,4 +1,4 @@
-// RUN: %dxc -T cs_6_5 -HV 2021 -ast-dump %s | FileCheck -check-prefix=AST %s
+// RUN: %dxc -T cs_6_5 -HV 2021 -ast-dump-implicit %s | FileCheck -check-prefix=AST %s
 // RUN: %dxc -T cs_6_5 -HV 2021 %s | FileCheck %s
 RWStructuredBuffer<int> Output;
 
@@ -35,9 +35,9 @@ void main() {
 // literal types in, because we can constant fold them as with higher precision.
 
 // Verify that the abs call is still `literal float`
-// AST: DeclRefExpr 0x{{[0-9a-zA-Z]+}} <col:15> 'literal int (literal int)' lvalue Function 0x{{[0-9a-zA-Z]+}} 'abs'
+// AST: DeclRefExpr 0x{{[0-9a-zA-Z]+}} <col:15> 'literal int (literal int)' lvalue Function [[abs:0x[0-9a-zA-Z]+]] 'abs'
 
-// AST:      FunctionDecl 0x{{[0-9a-zA-Z]+}} <<invalid sloc>> <invalid sloc> implicit used abs 'literal int (literal int)' extern
+// AST:      FunctionDecl [[abs]] <<invalid sloc>> <invalid sloc> implicit used abs 'literal int (literal int)' extern
 // AST-NEXT: ParmVarDecl 0x{{[0-9a-zA-Z]+}} <<invalid sloc>> <invalid sloc> x 'literal int'
 
 // Since template expansion can also fail in IRGen, we should make sure we have

tools/clang/test/HLSLFileCheck/infra/ast-dumping.hlsl

@@ -0,0 +1,51 @@
+// RUN: %dxc -T cs_6_0 -ast-dump %s | FileCheck %s -check-prefixes=CHECK,NO-IMPLICIT
+// RUN: %dxc -T cs_6_0 -ast-dump-implicit %s | FileCheck %s -check-prefixes=CHECK,IMPLICIT
+RWBuffer<float3> In;
+RWBuffer<float3> Out;
+
+[numthreads(1,1,1)]
+void main() {
+  Out[0] = In[0];
+}
+
+// CHECK: TranslationUnitDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc>
+
+// If we're not dumping implicit decls, there should be no additional decls.
+// NO-IMPLICIT-NOT: Decl
+
+// This test doesn't seek to be exhaustive, just checking a few common implicit
+// decls to ensure they got generated into the AST.
+// IMPLICIT: CXXRecordDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit class vector definition
+// IMPLICIT: CXXRecordDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit class matrix definition
+
+// This set of checks verifies that the implicit decls for buffers are
+// incomplete by looking at the `Buffer` type which is unused in this code and
+// verifying it doesn't have any methods.
+
+// IMPLICIT: CXXRecordDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit class Buffer
+// IMPLICIT-NEXT: FinalAttr {{0x[0-9a-fA-F]+}} <<invalid sloc>> Implicit final
+// IMPLICIT-NEXT: FieldDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit h 'element'
+// IMPLICIT-NOT: CXXMethodDecl
+// IMPLICIT-NEXT: ClassTemplateDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit Buffer
+// IMPLICIT-NEXT: TemplateTypeParmDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> class element
+// IMPLICIT-NEXT: TemplateArgument type 'vector<float, 4>':'vector<float, 4>'
+// IMPLICIT-NEXT: CXXRecordDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit class Buffer
+// IMPLICIT-NEXT: FinalAttr {{0x[0-9a-fA-F]+}} <<invalid sloc>> Implicit final
+// IMPLICIT-NEXT: FieldDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit h 'element'
+// IMPLICIT-NOT: CXXMethodDecl
+// IMPLICIT: CXXRecordDecl
+
+
+// This tests to verfy that the RWBuffer _is_ completed with method definitions.
+
+// IMPLICIT: ClassTemplateDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit RWBuffer
+// IMPLICIT-NEXT: TemplateTypeParmDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> class element
+// IMPLICIT-NEXT: CXXRecordDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit class RWBuffer definition
+// IMPLICIT-NEXT: FinalAttr {{0x[0-9a-fA-F]+}} <<invalid sloc>> Implicit final
+// IMPLICIT-NEXT: FieldDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> implicit h 'element'
+// IMPLICIT-NEXT: CXXMethodDecl {{0x[0-9a-fA-F]+}} <<invalid sloc>> <invalid sloc> operator[] 'element &(unsigned int) const'
+
+// CHECK: VarDecl {{0x[0-9a-fA-F]+}} <{{.*}}ast-dumping.hlsl:3:1, col:18> col:18 used In 'RWBuffer<float3>':'RWBuffer<vector<float, 3> >'
+// CHECK-NEXT: VarDecl {{0x[0-9a-fA-F]+}} <line:4:1, col:18> col:18 used Out 'RWBuffer<float3>':'RWBuffer<vector<float, 3> >'
+// CHECK-NEXT: FunctionDecl {{0x[0-9a-fA-F]+}} <line:7:1, line:9:1> line:7:6 main 'void ()'
+

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -1481,6 +1481,11 @@ public:
     compiler.getLangOpts().EnablePayloadAccessQualifiers = Opts.EnablePayloadQualifiers;
     compiler.getLangOpts().HLSLProfile =
           compiler.getCodeGenOpts().HLSLProfile = Opts.TargetProfile;
+    // Enable dumping implicit top level decls either if it was specifically
+    // requested or if we are not dumping the ast from the command line. That
+    // allows us to dump implicit AST nodes in the debugger.
+    compiler.getLangOpts().DumpImplicitTopLevelDecls =
+        Opts.AstDumpImplicit || !Opts.AstDump;
 
 // SPIRV change starts
 #ifdef ENABLE_SPIRV_CODEGEN