Make remove-unused-globals remove non-static globals (#2809)

- Add support for rewriter (dxr) to FileCheckerTest
- Add test for removing globals

include/dxc/Test/DxcTestUtils.h

@@ -100,7 +100,7 @@ public:
   FileRunCommandResult Run(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior, PluginToolsPaths *pPluginToolsPaths = nullptr );
   FileRunCommandResult RunHashTests(dxc::DxcDllSupport &DllSupport);
   
-  FileRunCommandResult ReadOptsForDxc(hlsl::options::MainArgs &argStrings, hlsl::options::DxcOpts &Opts);
+  FileRunCommandResult ReadOptsForDxc(hlsl::options::MainArgs &argStrings, hlsl::options::DxcOpts &Opts, unsigned flagsToInclude = 0);
 
   std::string Command;      // Command to run, eg %dxc
   std::string Arguments;    // Arguments to command
@@ -112,6 +112,7 @@ private:
   FileRunCommandResult RunDxv(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior);
   FileRunCommandResult RunOpt(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior);
   FileRunCommandResult RunD3DReflect(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior);
+  FileRunCommandResult RunDxr(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior);
   FileRunCommandResult RunTee(const FileRunCommandResult *Prior);
   FileRunCommandResult RunXFail(const FileRunCommandResult *Prior);
   FileRunCommandResult RunDxilVer(dxc::DxcDllSupport& DllSupport, const FileRunCommandResult* Prior);

tools/clang/test/HLSLFileCheck/rewriter/remove-unused-globals.hlsl

@@ -0,0 +1,71 @@
+// RUN: %dxr -E main -remove-unused-globals %s | FileCheck %s
+
+// CHECK: cbuffer TestCBuffer
+cbuffer TestCBuffer
+{
+// CHECK: int TestInt;
+  int TestInt;
+// UnusedFloat is not removed if inside a cbuffer declaration with a used global
+// CHECK: float UnusedFloat;
+  float UnusedFloat;
+}
+// CHECK: }
+
+// Unused cbuffers are not removed at this time
+// CHECK: cbuffer UnusedCBuffer
+cbuffer UnusedCBuffer
+{
+// CHECK: float FloatInUnusedCBuffer;
+  float FloatInUnusedCBuffer;
+}
+// CHECK: }
+
+// CHECK-NOT: RandomGlobal
+float4 RandomGlobal[2];
+
+// CHECK: LookupTable
+float4 LookupTable[1];
+
+// CHECK-NOT: UnusedBool
+bool UnusedBool;
+
+// CHECK: TestBool
+bool TestBool;
+
+// CHECK-NOT: UnusedTex
+Texture2D UnusedTex;
+
+// CHECK: TestTex
+Texture2D TestTex;
+
+// CHECK-NOT: UnusedSampler
+SamplerState UnusedSampler;
+
+// CHECK: TestSampler
+SamplerState TestSampler;
+
+// CHECK: float3 ReferencedButDead;
+float3 ReferencedButDead;
+
+// CHECK: void main(
+void main(in float4 SvPosition : SV_Position,
+          out float4 OutTarget0 : SV_Target0)
+{
+  float3 Color = 0;
+  uint Index = 0;
+
+  if (TestBool)
+  {
+    Index = TestInt;
+  }
+
+  // Should keep any referenced global, even if it is unused.
+  (void)ReferencedButDead;
+
+  float2 UV = LookupTable[Index].xy + LookupTable[Index].zw;
+  float4 Value = TestTex.SampleLevel(TestSampler, UV, 0);
+
+  Color += Value.rgb + (1 - Value.a);
+
+  OutTarget0 = float4 (Color, 1.0f);
+}

tools/clang/tools/libclang/dxcrewriteunused.cpp

@@ -422,7 +422,7 @@ static HRESULT DoRewriteUnused( TranslationUnitDecl *tu,
     if (tuDecl->isImplicit()) continue;
 
     VarDecl* varDecl = dyn_cast_or_null<VarDecl>(tuDecl);
-    if (varDecl != nullptr && varDecl->getFormalLinkage() == clang::Linkage::InternalLinkage) {
+    if (varDecl != nullptr) {
       unusedGlobals.insert(varDecl);
       if (const RecordType *recordType = varDecl->getType()->getAs<RecordType>()) {
         RecordDecl *recordDecl = recordType->getDecl();

tools/clang/unittests/HLSLTestLib/FileCheckerTest.cpp

@@ -33,6 +33,7 @@
 #include "llvm/Support/MD5.h"
 #include "dxc/Support/Global.h"
 #include "dxc/Support/dxcapi.use.h"
+#include "dxc/dxctools.h"
 #include "dxc/Support/HLSLOptions.h"
 #include "dxc/Support/Unicode.h"
 #include "dxc/DxilContainer/DxilContainer.h"
@@ -98,6 +99,9 @@ FileRunCommandResult FileRunCommandPart::Run(dxc::DxcDllSupport &DllSupport, con
   else if (0 == _stricmp(Command.c_str(), "%D3DReflect")) {
     return RunD3DReflect(DllSupport, Prior);
   }
+  else if (0 == _stricmp(Command.c_str(), "%dxr")) {
+    return RunDxr(DllSupport, Prior);
+  }
   else if (pPluginToolsPaths != nullptr) {
     auto it = pPluginToolsPaths->find(Command.c_str());
     if (it != pPluginToolsPaths->end()) {
@@ -151,7 +155,8 @@ FileRunCommandResult FileRunCommandPart::RunFileChecker(const FileRunCommandResu
 }
 
 FileRunCommandResult FileRunCommandPart::ReadOptsForDxc(
-    hlsl::options::MainArgs &argStrings, hlsl::options::DxcOpts &Opts) {
+    hlsl::options::MainArgs &argStrings, hlsl::options::DxcOpts &Opts,
+    unsigned flagsToInclude) {
   std::string args(strtrim(Arguments));
   const char *inputPos = strstr(args.c_str(), "%s");
   if (inputPos == nullptr)
@@ -164,7 +169,7 @@ FileRunCommandResult FileRunCommandPart::ReadOptsForDxc(
   argStrings = hlsl::options::MainArgs(splitArgs);
   std::string errorString;
   llvm::raw_string_ostream errorStream(errorString);
-  int RunResult = ReadDxcOpts(hlsl::options::getHlslOptTable(), /*flagsToInclude*/ 0,
+  int RunResult = ReadDxcOpts(hlsl::options::getHlslOptTable(), flagsToInclude,
                           argStrings, Opts, errorStream);
   errorStream.flush();
   if (RunResult)
@@ -646,6 +651,56 @@ FileRunCommandResult FileRunCommandPart::RunD3DReflect(dxc::DxcDllSupport &DllSu
   return FileRunCommandResult::Success(ss.str());
 }
 
+FileRunCommandResult FileRunCommandPart::RunDxr(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior) {
+  // Support piping stdin from prior if needed.
+  UNREFERENCED_PARAMETER(Prior);
+  hlsl::options::MainArgs args;
+  hlsl::options::DxcOpts opts;
+  FileRunCommandResult readOptsResult = ReadOptsForDxc(args, opts,
+    hlsl::options::HlslFlags::RewriteOption);
+  if (readOptsResult.ExitCode) return readOptsResult;
+
+  std::wstring entry =
+    Unicode::UTF8ToUTF16StringOrThrow(opts.EntryPoint.str().c_str());
+
+  std::vector<LPCWSTR> flags;
+  std::vector<std::wstring> argWStrings;
+  CopyArgsToWStrings(opts.Args, hlsl::options::RewriteOption, argWStrings);
+  for (const std::wstring &a : argWStrings)
+    flags.push_back(a.data());
+
+  CComPtr<IDxcLibrary> pLibrary;
+  CComPtr<IDxcRewriter2> pRewriter;
+  CComPtr<IDxcOperationResult> pResult;
+  CComPtr<IDxcBlobEncoding> pSource;
+  CComPtr<IDxcBlob> pResultBlob;
+  CComPtr<IDxcIncludeHandler> pIncludeHandler;
+
+  IFT(DllSupport.CreateInstance(CLSID_DxcLibrary, &pLibrary));
+  IFT(pLibrary->CreateBlobFromFile(CommandFileName, nullptr, &pSource));
+  IFT(pLibrary->CreateIncludeHandler(&pIncludeHandler));
+  IFT(DllSupport.CreateInstance(CLSID_DxcRewriter, &pRewriter));
+  IFT(pRewriter->RewriteWithOptions(pSource, CommandFileName,
+                                    flags.data(), flags.size(), nullptr, 0,
+                                    pIncludeHandler, &pResult));
+
+  HRESULT resultStatus;
+  IFT(pResult->GetStatus(&resultStatus));
+
+  FileRunCommandResult result = {};
+  CComPtr<IDxcBlobEncoding> pStdErr;
+  IFT(pResult->GetErrorBuffer(&pStdErr));
+  result.StdErr = BlobToUtf8(pStdErr);
+  result.ExitCode = resultStatus;
+  if (SUCCEEDED(resultStatus)) {
+    IFT(pResult->GetResult(&pResultBlob));
+    result.StdOut = BlobToUtf8(pResultBlob);
+  }
+
+  result.OpResult = pResult;
+  return result;
+}
+
 FileRunCommandResult FileRunCommandPart::RunTee(const FileRunCommandResult *Prior) {
   if (Prior == nullptr) {
     return FileRunCommandResult::Error("tee requires a prior command");