Disable tests requiring name preservation on release builds. (#229)

* Disable tests requiring name preservation on release builds.
* Fix reversal of DxcVersionInfoFlags_Debug
* Enable parallel execution for unit tests
* Add -verbose to preserve logging and remove /parallel

tools/clang/tools/dxcompiler/dxcvalidator.cpp

@@ -186,7 +186,7 @@ HRESULT STDMETHODCALLTYPE DxcValidator::GetFlags(_Out_ UINT32 *pFlags) {
   if (pFlags == nullptr)
     return E_INVALIDARG;
   *pFlags = DxcVersionInfoFlags_None;
-#ifdef _NDEBUG
+#ifdef _DEBUG
   *pFlags |= DxcVersionInfoFlags_Debug;
 #endif
   return S_OK;

tools/clang/unittests/HLSL/CompilerTest.cpp

@@ -849,6 +849,14 @@ public:
   dxc::DxcDllSupport m_dllSupport;
   bool m_CompilerPreservesBBNames;
 
+  bool SkipIRSensitiveTest() {
+    if (!m_CompilerPreservesBBNames) {
+      WEX::Logging::Log::Comment(L"Test skipped due to name preservation requirment.");
+      return true;
+    }
+    return false;
+  }
+
   void CreateBlobPinned(_In_bytecount_(size) LPCVOID data, SIZE_T size,
                         UINT32 codePage, _Outptr_ IDxcBlobEncoding **ppBlob) {
     CComPtr<IDxcLibrary> library;
@@ -1250,7 +1258,7 @@ bool CompilerTest::InitSupport() {
     VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcValidator, &pValidator));
     VERIFY_SUCCEEDED(pValidator.QueryInterface(&pVersionInfo));
     VERIFY_SUCCEEDED(pVersionInfo->GetFlags(&VersionFlags));
-    m_CompilerPreservesBBNames = VersionFlags & DxcVersionInfoFlags_Debug;
+    m_CompilerPreservesBBNames = (VersionFlags & DxcVersionInfoFlags_Debug) ? true : false;
   }
   return true;
 }
@@ -3948,7 +3956,7 @@ TEST_F(CompilerTest, CodeGenDx12MiniEngineFxaapass2Vdebugcs){
 }
 
 TEST_F(CompilerTest, CodeGenDx12MiniEngineFxaaresolveworkqueuecs){
-  if (!m_CompilerPreservesBBNames) return;
+  if (SkipIRSensitiveTest()) return;
   CodeGenTestCheck(L"..\\CodeGenHLSL\\Samples\\MiniEngine\\FXAAResolveWorkQueueCS.hlsl");
 }
 

tools/clang/unittests/HLSL/DXIsenseTest.cpp

@@ -21,6 +21,7 @@ class DXIntellisenseTest
 {
 public:
   BEGIN_TEST_CLASS(DXIntellisenseTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
 

tools/clang/unittests/HLSL/DxilContainerTest.cpp

@@ -61,6 +61,7 @@ static uint8_t MaskCount(uint8_t V) {
 class DxilContainerTest {
 public:
   BEGIN_TEST_CLASS(DxilContainerTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
 

tools/clang/unittests/HLSL/ExecutionTest.cpp

@@ -188,6 +188,7 @@ class ExecutionTest {
 public:
   // By default, ignore these tests, which require a recent build to run properly.
   BEGIN_TEST_CLASS(ExecutionTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_CLASS_PROPERTY(L"Ignore", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()

tools/clang/unittests/HLSL/ExtensionTest.cpp

@@ -416,6 +416,7 @@ public:
 class ExtensionTest {
 public:
   BEGIN_TEST_CLASS(ExtensionTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
 

tools/clang/unittests/HLSL/FunctionTest.cpp

@@ -28,6 +28,7 @@ class FunctionTest
 {
 public:
   BEGIN_TEST_CLASS(FunctionTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
   TEST_METHOD(AllowedStorageClass);

tools/clang/unittests/HLSL/MSFileSysTest.cpp

@@ -33,6 +33,7 @@ class MSFileSysTest
 {
 public:
   BEGIN_TEST_CLASS(MSFileSysTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
 

tools/clang/unittests/HLSL/Objects.cpp

@@ -487,6 +487,7 @@ private:
   HlslIntellisenseSupport m_isenseSupport;
 public:
   BEGIN_TEST_CLASS(ObjectTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
 

tools/clang/unittests/HLSL/RewriterTest.cpp

@@ -44,6 +44,7 @@ using namespace hlsl_test;
 class RewriterTest {
 public:
   BEGIN_TEST_CLASS(RewriterTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
 

tools/clang/unittests/HLSL/ValidationTest.cpp

@@ -104,9 +104,12 @@ class ValidationTest
 {
 public:
   BEGIN_TEST_CLASS(ValidationTest)
+    TEST_CLASS_PROPERTY(L"Parallel", L"true")
     TEST_METHOD_PROPERTY(L"Priority", L"0")
   END_TEST_CLASS()
 
+  TEST_CLASS_SETUP(InitSupport);
+
   TEST_METHOD(WhenCorrectThenOK);
   TEST_METHOD(WhenMisalignedThenFail);
   TEST_METHOD(WhenEmptyFileThenFail);
@@ -288,6 +291,15 @@ public:
   TEST_METHOD(WhenFeatureInfoMismatchThenFail);
 
   dxc::DxcDllSupport m_dllSupport;
+  bool m_CompilerPreservesBBNames;
+
+  bool SkipIRSensitiveTest() {
+    if (!m_CompilerPreservesBBNames) {
+      WEX::Logging::Log::Comment(L"Test skipped due to name preservation requirment.");
+      return true;
+    }
+    return false;
+  }
 
   void TestCheck(LPCWSTR name) {
     std::wstring fullPath = hlsl_test::GetPathToHlslDataFile(name);
@@ -303,10 +315,6 @@ public:
     CComPtr<IDxcValidator> pValidator;
     CComPtr<IDxcOperationResult> pResult;
 
-    if (!m_dllSupport.IsEnabled()) {
-      VERIFY_SUCCEEDED(m_dllSupport.Initialize());
-    }
-
     VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcValidator, &pValidator));
     VERIFY_SUCCEEDED(pValidator->Validate(pBlob, DxcValidatorFlags_Default, &pResult));
 
@@ -314,9 +322,6 @@ public:
   }
 
   void CheckValidationMsgs(const char *pBlob, size_t blobSize, llvm::ArrayRef<LPCSTR> pErrorMsgs, bool bRegex = false) {
-    if (!m_dllSupport.IsEnabled()) {
-      VERIFY_SUCCEEDED(m_dllSupport.Initialize());
-    }
     CComPtr<IDxcLibrary> pLibrary;
     CComPtr<IDxcBlobEncoding> pBlobEncoding; // Encoding doesn't actually matter, it's binary.
     VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcLibrary, &pLibrary));
@@ -330,10 +335,6 @@ public:
     CComPtr<IDxcOperationResult> pResult;
     CComPtr<IDxcBlob> pProgram;
 
-    if (!m_dllSupport.IsEnabled()) {
-      VERIFY_SUCCEEDED(m_dllSupport.Initialize());
-    }
-
     CA2W shWide(pShaderModel, CP_UTF8);
     VERIFY_SUCCEEDED(
         m_dllSupport.CreateInstance(CLSID_DxcCompiler, &pCompiler));
@@ -346,9 +347,6 @@ public:
 
   void CompileSource(LPCSTR pSource, LPCSTR pShaderModel,
                      IDxcBlob **pResultBlob) {
-    if (!m_dllSupport.IsEnabled()) {
-      VERIFY_SUCCEEDED(m_dllSupport.Initialize());
-    }
     CComPtr<IDxcBlobEncoding> pSourceBlob;
     Utf8ToBlob(m_dllSupport, pSource, &pSourceBlob);
     CompileSource(pSourceBlob, pShaderModel, pResultBlob);
@@ -364,10 +362,6 @@ public:
     CComPtr<IDxcBlob> pText;
     CComPtr<IDxcBlobEncoding> pSourceBlob;
     
-    if (!m_dllSupport.IsEnabled()) {
-      VERIFY_SUCCEEDED(m_dllSupport.Initialize());
-    }
-
     Utf8ToBlob(m_dllSupport, pSource, &pSourceBlob);
 
     RewriteAssemblyToText(pSourceBlob, pShaderModel, pLookFors, pReplacements, &pText, bRegex);
@@ -431,9 +425,6 @@ public:
     std::wstring fullPath = hlsl_test::GetPathToHlslDataFile(name);
     CComPtr<IDxcLibrary> pLibrary;
     CComPtr<IDxcBlobEncoding> pSource;
-    if (!m_dllSupport.IsEnabled()) {
-      VERIFY_SUCCEEDED(m_dllSupport.Initialize());
-    }
     VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcLibrary, &pLibrary));
     VERIFY_SUCCEEDED(
         pLibrary->CreateBlobFromFile(fullPath.c_str(), nullptr, &pSource));
@@ -518,6 +509,22 @@ public:
   }
 };
 
+bool ValidationTest::InitSupport() {
+  if (!m_dllSupport.IsEnabled()) {
+    VERIFY_SUCCEEDED(m_dllSupport.Initialize());
+
+    // This is a very indirect way of testing this. Consider improving support.
+    CComPtr<IDxcValidator> pValidator;
+    CComPtr<IDxcVersionInfo> pVersionInfo;
+    UINT32 VersionFlags = 0;
+    VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcValidator, &pValidator));
+    VERIFY_SUCCEEDED(pValidator.QueryInterface(&pVersionInfo));
+    VERIFY_SUCCEEDED(pVersionInfo->GetFlags(&VersionFlags));
+    m_CompilerPreservesBBNames = (VersionFlags & DxcVersionInfoFlags_Debug) ? true : false;
+  }
+  return true;
+}
+
 TEST_F(ValidationTest, WhenCorrectThenOK) {
   CComPtr<IDxcBlob> pProgram;
   CompileSource("float4 main() : SV_Target { return 1; }", "ps_6_0", &pProgram);
@@ -637,6 +644,7 @@ TEST_F(ValidationTest, WhenDepthNotFloatThenFail) {
 }
 
 TEST_F(ValidationTest, BarrierFail) {
+  if (SkipIRSensitiveTest()) return;
     RewriteAssemblyCheckMsg(
       L"..\\CodeGenHLSL\\barrier.hlsl", "cs_6_0",
       {"dx.op.barrier(i32 80, i32 8)",
@@ -685,6 +693,7 @@ TEST_F(ValidationTest, CsThreadSizeFail) {
       });
 }
 TEST_F(ValidationTest, DeadLoopFail) {
+  if (SkipIRSensitiveTest()) return;
   RewriteAssemblyCheckMsg(
       L"..\\CodeGenHLSL\\loop1.hlsl", "ps_6_0",
       {"br i1 %exitcond, label %for.end.loopexit, label %for.body, !llvm.loop !([0-9]+)",
@@ -780,6 +789,7 @@ TEST_F(ValidationTest, MultiStream2Fail) {
       "Multiple GS output streams are used but 'XXX' is not pointlist");
 }
 TEST_F(ValidationTest, PhiTGSMFail) {
+  if (SkipIRSensitiveTest()) return;
   RewriteAssemblyCheckMsg(
       L"..\\CodeGenHLSL\\phiTGSM.hlsl", "cs_6_0",
       "ret void",
@@ -789,6 +799,7 @@ TEST_F(ValidationTest, PhiTGSMFail) {
       "TGSM pointers must originate from an unambiguous TGSM global variable");
 }
 TEST_F(ValidationTest, ReducibleFail) {
+  if (SkipIRSensitiveTest()) return;
   RewriteAssemblyCheckMsg(
       L"..\\CodeGenHLSL\\reducible.hlsl", "ps_6_0",
       {"%conv\n"
@@ -934,6 +945,7 @@ TEST_F(ValidationTest, SimpleGs1Fail) {
        "Stream index (5) must between 0 and 3"});
 }
 TEST_F(ValidationTest, UavBarrierFail) {
+  if (SkipIRSensitiveTest()) return;
   RewriteAssemblyCheckMsg(
       L"..\\CodeGenHLSL\\uavBarrier.hlsl", "ps_6_0",
       {"dx.op.barrier(i32 80, i32 2)",
@@ -955,6 +967,7 @@ TEST_F(ValidationTest, UndefValueFail) {
   TestCheck(L"..\\CodeGenHLSL\\UndefValue.hlsl");
 }
 TEST_F(ValidationTest, UpdateCounterFail) {
+  if (SkipIRSensitiveTest()) return;
   RewriteAssemblyCheckMsg(
       L"..\\CodeGenHLSL\\UpdateCounter2.hlsl", "ps_6_0",
       {"%2 = call i32 @dx.op.bufferUpdateCounter(i32 70, %dx.types.Handle %buf2_UAV_structbuf, i8 1)",

utils/hct/hcttest.cmd

@@ -10,13 +10,31 @@ rem Whether we built the project using ninja as the generator.
 set GENERATOR_NINJA=0
 
 set TEST_ALL=1
+set TEST_CLANG=0
+set TEST_CMD=0
+set TEST_EXEC=0
+set TEST_EXTRAS=0
+set TEST_EXEC_REQUIRED=0
 set TEST_CLANG_FILTER= /select: "@Priority<1"
 set TEST_EXEC_FILTER=ExecutionTest::*
+set LOG_FILTER=/logOutput:LowWithConsoleBuffering
 if "%BUILD_CONFIG%"=="" (
   set BUILD_CONFIG=Debug
 )
 set HCT_DIR=%~dp0
 
+if "%NUMBER_OF_PROCESSORS%"=="" (
+  set PARALLEL_OPTION=
+) else if %NUMBER_OF_PROCESSORS% LEQ 1 (
+  set PARALLEL_OPTION=
+) else if %NUMBER_OF_PROCESSORS% LEQ 4 (
+  set PARALLEL_OPTION=/parallel:%NUMBER_OF_PROCESSORS%
+) else (
+  rem Sweet spot for /parallel seems to be NUMBER_OF_PROCESSORS - 1
+  set /a PARALLEL_COUNT=%NUMBER_OF_PROCESSORS%-1
+  set PARALLEL_OPTION=/parallel:!PARALLEL_COUNT!
+)
+
 :opt_loop
 if "%1"=="" (goto :done_opt)
 
@@ -75,7 +93,10 @@ if "%1"=="-clean" (
 ) else if "%1"=="-adapter" (
   set TEST_ADAPTER= /p:"Adapter=%~2"
   shift /1
-) else IF "%1"=="--" (
+) else if "%1"=="-verbose" (
+  set LOG_FILTER=
+  set PARALLEL_OPTION=
+) else if "%1"=="--" (
   shift /1
   goto :done_opt
 ) else (
@@ -198,7 +219,7 @@ set TESTS_PASSED=0
 set TESTS_FAILED=0
 call :check_result "clang tests" %RES_CLANG%
 call :check_result "command line tests" %RES_CMD%
-if "%TEST_EXEC_REQUIRED%"=="1" (
+if "%TEST_EXEC%"=="1" (
   call :check_result "execution tests" %RES_EXEC%
 )
 call :check_result "hcttest-extras tests" %RES_EXTRAS%
@@ -232,6 +253,7 @@ echo   -clean - deletes test directory before copying binaries and testing
 echo   -ninja - artifacts were built using the Ninja generator
 echo   -rel   - builds release rather than debug
 echo   -adapter "adapter name" - overrides Adapter for execution tests
+echo   -verbose - for TAEF: turns off /parallel and removes logging filter
 echo.
 echo current BUILD_ARCH=%BUILD_ARCH%.  Override with:
 echo   -x86 targets an x86 build (aka. Win32)
@@ -266,8 +288,8 @@ rem %2 - first argument to te
 rem %3 - second argument to te
 rem %4 - third argument to te
 
-echo te /labMode /miniDumpOnCrash %TEST_DIR%\%*
-call te /labMode /miniDumpOnCrash %TEST_DIR%\%*
+echo te /labMode /miniDumpOnCrash %LOG_FILTER% %PARALLEL_OPTION% %TEST_DIR%\%*
+call te /labMode /miniDumpOnCrash %LOG_FILTER% %PARALLEL_OPTION% %TEST_DIR%\%*
 if errorlevel 1 (
   call :showtesample %*
   exit /b 1