Add option to dump FileCheck input to directory for diffing (#2949)

include/dxc/Test/DxcTestUtils.h

@@ -97,7 +97,7 @@ public:
   FileRunCommandPart(const FileRunCommandPart&) = default;
   FileRunCommandPart(FileRunCommandPart&&) = default;
   
-  FileRunCommandResult Run(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior, PluginToolsPaths *pPluginToolsPaths = nullptr );
+  FileRunCommandResult Run(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior, PluginToolsPaths *pPluginToolsPaths = nullptr, LPCWSTR dumpName = nullptr);
   FileRunCommandResult RunHashTests(dxc::DxcDllSupport &DllSupport);
   
   FileRunCommandResult ReadOptsForDxc(hlsl::options::MainArgs &argStrings, hlsl::options::DxcOpts &Opts, unsigned flagsToInclude = 0);
@@ -107,7 +107,7 @@ public:
   LPCWSTR CommandFileName;  // File name replacement for %s
 
 private:
-  FileRunCommandResult RunFileChecker(const FileRunCommandResult *Prior);
+  FileRunCommandResult RunFileChecker(const FileRunCommandResult *Prior, LPCWSTR dumpName = nullptr);
   FileRunCommandResult RunDxc(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior);
   FileRunCommandResult RunDxv(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior);
   FileRunCommandResult RunOpt(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior);
@@ -137,8 +137,13 @@ public:
   std::string ErrorMessage;
   int RunResult;
   static FileRunTestResult RunHashTestFromFileCommands(LPCWSTR fileName);
-  static FileRunTestResult RunFromFileCommands(LPCWSTR fileName, PluginToolsPaths *pPluginToolsPaths = nullptr);
-  static FileRunTestResult RunFromFileCommands(LPCWSTR fileName, dxc::DxcDllSupport &dllSupport, PluginToolsPaths *pPluginToolsPaths = nullptr);
+  static FileRunTestResult RunFromFileCommands(LPCWSTR fileName,
+                                               PluginToolsPaths *pPluginToolsPaths = nullptr,
+                                               LPCWSTR dumpName = nullptr);
+  static FileRunTestResult RunFromFileCommands(LPCWSTR fileName,
+                                               dxc::DxcDllSupport &dllSupport,
+                                               PluginToolsPaths *pPluginToolsPaths = nullptr,
+                                               LPCWSTR dumpName = nullptr);
 };
 
 void AssembleToContainer(dxc::DxcDllSupport &dllSupport, IDxcBlob *pModule, IDxcBlob **pContainer);

include/dxc/Test/HlslTestUtils.h

@@ -28,6 +28,7 @@ using namespace std;
 
 #ifndef HLSLDATAFILEPARAM
 #define HLSLDATAFILEPARAM L"HlslDataDir"
+#define FILECHECKDUMPDIRPARAM L"FileCheckDumpDir"
 #endif
 
 // If TAEF verify macros are available, use them to alias other legacy
@@ -158,10 +159,16 @@ inline void LogErrorFmt(_In_z_ _Printf_format_string_ const wchar_t *fmt, ...) {
     WEX::Logging::Log::Error(buf.data());
 }
 
-inline std::wstring GetPathToHlslDataFile(const wchar_t* relative) {
+inline std::wstring GetPathToHlslDataFile(const wchar_t* relative, LPCWSTR paramName = HLSLDATAFILEPARAM) {
   WEX::TestExecution::SetVerifyOutput verifySettings(WEX::TestExecution::VerifyOutputSettings::LogOnlyFailures);
   WEX::Common::String HlslDataDirValue;
-  ASSERT_HRESULT_SUCCEEDED(WEX::TestExecution::RuntimeParameters::TryGetValue(HLSLDATAFILEPARAM, HlslDataDirValue));
+  if (std::wstring(paramName).compare(HLSLDATAFILEPARAM) != 0) {
+    // Not fatal, for instance, FILECHECKDUMPDIRPARAM will dump files before running FileCheck, so they can be compared run to run
+    if (FAILED(WEX::TestExecution::RuntimeParameters::TryGetValue(paramName, HlslDataDirValue)))
+      return std::wstring();
+  } else {
+    ASSERT_HRESULT_SUCCEEDED(WEX::TestExecution::RuntimeParameters::TryGetValue(HLSLDATAFILEPARAM, HlslDataDirValue));
+  }
 
   wchar_t envPath[MAX_PATH];
   wchar_t expanded[MAX_PATH];

tools/clang/unittests/HLSL/CompilerTest.cpp

@@ -384,8 +384,9 @@ public:
     VERIFY_IS_GREATER_THAN(numTestsRun, (unsigned)0, L"No test files found in batch directory.");
   }
 
-  void CodeGenTestCheckFullPath(LPCWSTR fullPath) {
-    FileRunTestResult t = FileRunTestResult::RunFromFileCommands(fullPath);
+  void CodeGenTestCheckFullPath(LPCWSTR fullPath, LPCWSTR dumpPath = nullptr) {
+    FileRunTestResult t = FileRunTestResult::RunFromFileCommands(fullPath,
+      /*pPluginToolsPaths*/nullptr, dumpPath);
     if (t.RunResult != 0) {
       CA2W commentWide(t.ErrorMessage.c_str(), CP_UTF8);
       WEX::Logging::Log::Comment(commentWide);
@@ -393,13 +394,18 @@ public:
     }
   }
 
-  void CodeGenTestCheck(LPCWSTR name, bool implicitDir = true) {
+  void CodeGenTestCheck(LPCWSTR name, bool implicitDir = true, LPCWSTR dumpPath = nullptr) {
     std::wstring path = name;
+    std::wstring dumpStr;
     if (implicitDir) {
       path.insert(0, L"..\\CodeGenHLSL\\");
       path = hlsl_test::GetPathToHlslDataFile(path.c_str());
+      if (!dumpPath) {
+        dumpStr = hlsl_test::GetPathToHlslDataFile(path.c_str(), FILECHECKDUMPDIRPARAM);
+        dumpPath = dumpStr.empty() ? nullptr : dumpStr.c_str();
+      }
     }
-    CodeGenTestCheckFullPath(path.c_str());
+    CodeGenTestCheckFullPath(path.c_str(), dumpPath);
   }
 
   void CodeGenTestCheckBatchDir(std::wstring suitePath, bool implicitDir = true) {
@@ -414,8 +420,10 @@ public:
     ::llvm::sys::fs::AutoPerThreadSystem pts(msf.get());
     IFTLLVM(pts.error_code());
 
+    std::wstring dumpPath;
     CW2A pUtf8Filename(suitePath.c_str());
     if (!llvm::sys::path::is_absolute(pUtf8Filename.m_psz)) {
+      dumpPath = hlsl_test::GetPathToHlslDataFile(suitePath.c_str(), FILECHECKDUMPDIRPARAM);
       suitePath = hlsl_test::GetPathToHlslDataFile(suitePath.c_str());
     }
 
@@ -435,9 +443,14 @@ public:
         continue;
       StringRef filename = Dir->path();
       CA2W wRelPath(filename.data());
+      std::wstring dumpStr;
+      if (!dumpPath.empty() && suitePath.compare(0, suitePath.size(), wRelPath.m_psz, suitePath.size()) == 0) {
+        dumpStr = dumpPath + (wRelPath.m_psz + suitePath.size());
+      }
 
       WEX::Logging::Log::StartGroup(wRelPath);
-      CodeGenTestCheck(wRelPath, /*implicitDir*/ false);
+      CodeGenTestCheck(wRelPath, /*implicitDir*/ false,
+        dumpStr.empty() ? nullptr : dumpStr.c_str());
       WEX::Logging::Log::EndGroup(wRelPath);
 
       numTestsRun++;

tools/clang/unittests/HLSLTestLib/FileCheckerTest.cpp

@@ -31,6 +31,9 @@
 
 #include "llvm/Support/raw_os_ostream.h"
 #include "llvm/Support/MD5.h"
+#include "llvm/Support/FileSystem.h"
+#include "llvm/Support/Path.h"
+#include "llvm/Support/raw_ostream.h"
 #include "dxc/Support/Global.h"
 #include "dxc/Support/dxcapi.use.h"
 #include "dxc/dxctools.h"
@@ -58,7 +61,8 @@ FileRunCommandResult FileRunCommandPart::RunHashTests(dxc::DxcDllSupport &DllSup
 }
 
 FileRunCommandResult FileRunCommandPart::Run(dxc::DxcDllSupport &DllSupport, const FileRunCommandResult *Prior, 
-                                             PluginToolsPaths *pPluginToolsPaths /*=nullptr*/) {
+                                             PluginToolsPaths *pPluginToolsPaths /*=nullptr*/,
+                                             LPCWSTR dumpName /*=nullptr*/) {
   bool isFileCheck =
     0 == _stricmp(Command.c_str(), "FileCheck") ||
     0 == _stricmp(Command.c_str(), "%FileCheck");
@@ -74,7 +78,7 @@ FileRunCommandResult FileRunCommandPart::Run(dxc::DxcDllSupport &DllSupport, con
 
   // We would add support for 'not' and 'llc' here.
   if (isFileCheck) {
-    return RunFileChecker(Prior);
+    return RunFileChecker(Prior, dumpName);
   }
   else if (isXFail) {
     return RunXFail(Prior);
@@ -116,7 +120,7 @@ FileRunCommandResult FileRunCommandPart::Run(dxc::DxcDllSupport &DllSupport, con
   return result;
 }
 
-FileRunCommandResult FileRunCommandPart::RunFileChecker(const FileRunCommandResult *Prior) {
+FileRunCommandResult FileRunCommandPart::RunFileChecker(const FileRunCommandResult *Prior, LPCWSTR dumpName /*=nullptr*/) {
   if (!Prior) return FileRunCommandResult::Error("Prior command required to generate stdin");
 
   FileCheckForTest t;
@@ -141,6 +145,18 @@ FileRunCommandResult FileRunCommandPart::RunFileChecker(const FileRunCommandResu
   }
   if (!hasInputFilename) return FileRunCommandResult::Error("Missing input filename");
 
+  if (dumpName) {
+    // Dump t.InputForStdin to file for comparison purposes
+    CW2A dumpNameUtf8(dumpName, CP_UTF8);
+    llvm::StringRef dumpPath(dumpNameUtf8.m_psz);
+    llvm::sys::fs::create_directories(llvm::sys::path::parent_path(dumpPath), /*IgnoreExisting*/true);
+    std::error_code ec;
+    llvm::raw_fd_ostream os(dumpPath, ec, llvm::sys::fs::OpenFlags::F_Text);
+    if (!ec) {
+      os << t.InputForStdin;
+    }
+  }
+
   FileRunCommandResult result {};
   // Run
   result.ExitCode = t.Run();
@@ -914,6 +930,7 @@ FileRunCommandResult FileRunCommandPart::RunFromPath(const std::string &toolPath
 class FileRunTestResultImpl : public FileRunTestResult {
   dxc::DxcDllSupport &m_support;
   PluginToolsPaths *m_pPluginToolsPaths;
+  LPCWSTR m_dumpName = nullptr;
 
   void RunHashTestFromCommands(LPCSTR commands, LPCWSTR fileName) {
     std::vector<FileRunCommandPart> parts;
@@ -934,7 +951,7 @@ class FileRunTestResultImpl : public FileRunTestResult {
     }
   }
 
-  void RunFileCheckFromCommands(LPCSTR commands, LPCWSTR fileName) {
+  void RunFileCheckFromCommands(LPCSTR commands, LPCWSTR fileName, LPCWSTR dumpName = nullptr) {
     std::vector<FileRunCommandPart> parts;
     ParseCommandParts(commands, fileName, parts);
 
@@ -947,7 +964,7 @@ class FileRunTestResultImpl : public FileRunTestResult {
     FileRunCommandResult result;
     FileRunCommandResult* previousResult = nullptr;
     for (FileRunCommandPart & part : parts) {
-      result = part.Run(m_support, previousResult, m_pPluginToolsPaths);
+      result = part.Run(m_support, previousResult, m_pPluginToolsPaths, dumpName);
       previousResult = &result;
       if (result.AbortPipeline) break;
     }
@@ -957,17 +974,28 @@ class FileRunTestResultImpl : public FileRunTestResult {
   }
 
 public:
-  FileRunTestResultImpl(dxc::DxcDllSupport &support, PluginToolsPaths *pPluginToolsPaths = nullptr)
-    : m_support(support), m_pPluginToolsPaths(pPluginToolsPaths) {}
+  FileRunTestResultImpl(dxc::DxcDllSupport &support, PluginToolsPaths *pPluginToolsPaths = nullptr,
+                        LPCWSTR dumpName = nullptr)
+    : m_support(support), m_pPluginToolsPaths(pPluginToolsPaths), m_dumpName(dumpName) {}
   void RunFileCheckFromFileCommands(LPCWSTR fileName) {
     // Assume UTF-8 files.
     auto cmds = GetRunLines(fileName);
     // Iterate over all RUN lines
+    unsigned runIdx = 0;
     for (auto &cmd : cmds) {
-      RunFileCheckFromCommands(cmd.c_str(), fileName);
+      std::wstring dumpStr;
+      std::wstringstream os;
+      LPCWSTR dumpName = nullptr;
+      if (m_dumpName) {
+        os << m_dumpName << L"." << runIdx << L".txt";
+        dumpStr = os.str();
+        dumpName = dumpStr.c_str();
+      }
+      RunFileCheckFromCommands(cmd.c_str(), fileName, dumpName);
       // If any of the RUN cmd fails then skip executing remaining cmds
       // and report the error
       if (this->RunResult != 0) break;
+      runIdx += 1;
     }
   }
 
@@ -987,17 +1015,19 @@ FileRunTestResult FileRunTestResult::RunHashTestFromFileCommands(LPCWSTR fileNam
 }
 
 FileRunTestResult FileRunTestResult::RunFromFileCommands(LPCWSTR fileName, 
-                                                         PluginToolsPaths *pPluginToolsPaths /*=nullptr*/) {
+                                                         PluginToolsPaths *pPluginToolsPaths /*=nullptr*/,
+                                                         LPCWSTR dumpName /*=nullptr*/) {
   dxc::DxcDllSupport dllSupport;
   IFT(dllSupport.Initialize());
-  FileRunTestResultImpl result(dllSupport, pPluginToolsPaths);
+  FileRunTestResultImpl result(dllSupport, pPluginToolsPaths, dumpName);
   result.RunFileCheckFromFileCommands(fileName);
   return result;
 }
 
 FileRunTestResult FileRunTestResult::RunFromFileCommands(LPCWSTR fileName, dxc::DxcDllSupport &dllSupport,
-                                      PluginToolsPaths *pPluginToolsPaths /*=nullptr*/) {
-  FileRunTestResultImpl result(dllSupport, pPluginToolsPaths);
+                                                         PluginToolsPaths *pPluginToolsPaths /*=nullptr*/,
+                                                         LPCWSTR dumpName /*=nullptr*/) {
+  FileRunTestResultImpl result(dllSupport, pPluginToolsPaths, dumpName);
   result.RunFileCheckFromFileCommands(fileName);
   return result;
 }

utils/hct/hcttest.cmd

@@ -155,7 +155,10 @@ if "%1"=="-clean" (
   shift /1
 ) else if "%1"=="-custom-bin-set" (
   set CUSTOM_BIN_SET=%~2
- shift /1
+  shift /1
+) else if "%1"=="-file-check-dump" (
+  set ADDITIONAL_OPTS=%ADDITIONAL_OPTS% /p:"FileCheckDumpDir=%~2\HLSL"
+  shift /1
 ) else if "%1"=="--" (
   shift /1
   goto :done_opt
@@ -372,6 +375,7 @@ echo   -adapter "adapter name" - overrides Adapter for execution tests
 echo   -verbose - for TAEF: turns off /parallel and removes logging filter
 echo   -custom-bin-set "file [file]..." - custom set of binaries to copy into test directory
 echo   -dxilconv-loc "dxilconv.dll location" - fetch dxilconv.dll from custom location
+echo   -file-check-dump "dump-path" - dump file-check inputs to files under dump-path
 echo.
 echo current BUILD_ARCH=%BUILD_ARCH%.  Override with:
 echo   -x86 targets an x86 build (aka. Win32)