Refactor ExecutionTests into a separate dll (#4905)

ExecutionTests are currently compiled into clang-hlsl-tests.dll together with all of the other DXC unit tests. Later in the code pipeline the ExecutionTests are getting transformed into HLK tests which have different dependency constraints than the clang HLSL tests.

Refactoring ExecutionTests into a separate dll allows us to set its dependencies and build settings accordingly and closely matching those of the HLK test dll. This will help HLK authors to implement the tests with those constraints in mind.

include/dxc/CMakeLists.txt

@@ -3,6 +3,7 @@ include(HCT)
 add_hlsl_hctgen(HLSLIntrinsicOp OUTPUT HlslIntrinsicOp.h)
 
 set(HLSL_TEST_DATA_DIR ${LLVM_SOURCE_DIR}/tools/clang/test/HLSL/)
+set(EXEC_TEST_DATA_DIR ${LLVM_SOURCE_DIR}/tools/clang/unittests/HLSLExec/)
 
 configure_file(
   ${LLVM_MAIN_INCLUDE_DIR}/dxc/Test/TestConfig.h.in

include/dxc/Test/HlslTestUtils.h

@@ -25,11 +25,11 @@
 #include "dxc/Support/Global.h" // DXASSERT_LOCALVAR
 #include "WEXAdapter.h"
 #endif
-#include "dxc/Support/Unicode.h"
 #include "dxc/DXIL/DxilConstants.h" // DenormMode
 
 #ifdef _HLK_CONF
-#define DEFAULT_TEST_DIR ""
+#define DEFAULT_TEST_DIR L""
+#define DEFAULT_EXEC_TEST_DIR DEFAULT_TEST_DIR
 #else
 #include "dxc/Test/TestConfig.h"
 #endif
@@ -210,7 +210,7 @@ inline void LogErrorFmt(_In_z_ _Printf_format_string_ const wchar_t *fmt, ...) {
     WEX::Logging::Log::Error(buf.data());
 }
 
-inline std::wstring GetPathToHlslDataFile(const wchar_t* relative, LPCWSTR paramName = HLSLDATAFILEPARAM) {
+inline std::wstring GetPathToHlslDataFile(const wchar_t* relative, LPCWSTR paramName = HLSLDATAFILEPARAM, LPCWSTR defaultDataDir = DEFAULT_TEST_DIR) {
   WEX::TestExecution::SetVerifyOutput verifySettings(WEX::TestExecution::VerifyOutputSettings::LogOnlyFailures);
   WEX::Common::String HlslDataDirValue;
   if (std::wstring(paramName).compare(HLSLDATAFILEPARAM) != 0) {
@@ -219,7 +219,7 @@ inline std::wstring GetPathToHlslDataFile(const wchar_t* relative, LPCWSTR param
       return std::wstring();
   } else {
     if (FAILED(WEX::TestExecution::RuntimeParameters::TryGetValue(HLSLDATAFILEPARAM, HlslDataDirValue)))
-      HlslDataDirValue = DEFAULT_TEST_DIR;
+      HlslDataDirValue = defaultDataDir;
   }
 
   wchar_t envPath[MAX_PATH];
@@ -318,6 +318,42 @@ inline HANDLE CreateNewFileForReadWrite(LPCWSTR path) {
   return sourceHandle;
 }
 
+// Copy of Unicode::IsStarMatchT/IsStarMatchWide is included here to avoid the dependency on
+// DXC support libraries.
+template<typename TChar>
+inline static
+bool IsStarMatchT(const TChar *pMask, size_t maskLen, const TChar *pName, size_t nameLen, TChar star) {
+  if (maskLen == 0 && nameLen == 0) {
+    return true;
+  }
+  if (maskLen == 0 || nameLen == 0) {
+    return false;
+  }
+
+  if (pMask[maskLen - 1] == star) {
+    // Prefix match.
+    if (maskLen == 1) { // For just '*', everything is a match.
+      return true;
+    }
+    --maskLen;
+    if (maskLen > nameLen) { // Mask is longer than name, can't be a match.
+      return false;
+    }
+    return 0 == memcmp(pMask, pName, sizeof(TChar) * maskLen);
+  }
+  else {
+    // Exact match.
+    if (nameLen != maskLen) {
+      return false;
+    }
+    return 0 == memcmp(pMask, pName, sizeof(TChar) * nameLen);
+  }
+}
+
+inline bool IsStarMatchWide(const wchar_t *pMask, size_t maskLen, const wchar_t *pName, size_t nameLen) {
+  return IsStarMatchT<wchar_t>(pMask, maskLen, pName, nameLen, L'*');
+}
+
 inline bool GetTestParamBool(LPCWSTR name) {
   WEX::Common::String ParamValue;
   WEX::Common::String NameValue;
@@ -336,8 +372,9 @@ inline bool GetTestParamBool(LPCWSTR name) {
   if (NameValue.IsEmpty()) {
     return false;
   }
-  return Unicode::IsStarMatchWide(ParamValue, ParamValue.GetLength(),
-                                  NameValue, NameValue.GetLength());
+
+  return hlsl_test::IsStarMatchWide(ParamValue, ParamValue.GetLength(),
+                                    NameValue, NameValue.GetLength());
 }
 
 inline bool GetTestParamUseWARP(bool defaultVal) {

include/dxc/Test/TestConfig.h.in

@@ -1 +1,2 @@
 #define DEFAULT_TEST_DIR L"@HLSL_TEST_DATA_DIR@"
+#define DEFAULT_EXEC_TEST_DIR L"@EXEC_TEST_DATA_DIR@"

tools/clang/unittests/CMakeLists.txt

@@ -47,6 +47,7 @@ if (HLSL_INCLUDE_TESTS)
   add_subdirectory(HLSLTestLib)
   if (WIN32) # These tests require MS specific TAEF and DIA SDK
     add_subdirectory(HLSLErrors)
+    add_subdirectory(HLSLExec)
     add_subdirectory(HLSLHost)
     add_subdirectory(dxc_batch)
     add_subdirectory(DxrFallback)

tools/clang/unittests/HLSL/CMakeLists.txt

@@ -3,7 +3,7 @@
 if(WIN32)
 find_package(TAEF REQUIRED)
 find_package(DiaSDK REQUIRED) # Used for constants and declarations.
-find_package(D3D12 REQUIRED) # Used for ExecutionTest.cpp.
+find_package(D3D12 REQUIRED)  # Used in DxilContainerTest for DXBC compilation/reflection
 endif(WIN32)
 
 set( LLVM_LINK_COMPONENTS
@@ -35,7 +35,6 @@ add_clang_library(ClangHLSLTests SHARED
   DxilModuleTest.cpp
   DxilResourceTests.cpp
   DXIsenseTest.cpp
-  ExecutionTest.cpp
   ExtensionTest.cpp
   FunctionTest.cpp
   LinkerTest.cpp
@@ -45,17 +44,14 @@ add_clang_library(ClangHLSLTests SHARED
   OptionsTest.cpp
   PixTest.cpp
   RewriterTest.cpp
-  ShaderOpTest.cpp
   SystemValueTest.cpp
   ValidationTest.cpp
   VerifierTest.cpp
-  clang-hlsl-tests.rc
   )
 
   add_dependencies(ClangUnitTests ClangHLSLTests)
 else (WIN32)
 set(HLSL_IGNORE_SOURCES
-  ExecutionTest.cpp
   LinkerTest.cpp
   MSFileSysTest.cpp
   PixTest.cpp

tools/clang/unittests/HLSLExec/CMakeLists.txt

@@ -0,0 +1,39 @@
+# Copyright (C) Microsoft Corporation. All rights reserved.
+# This file is distributed under the University of Illinois Open Source License. See LICENSE.TXT for details.
+find_package(TAEF REQUIRED)
+find_package(D3D12 REQUIRED) # Used for ExecutionTest.cpp.
+
+add_clang_library(ExecHLSLTests SHARED
+  ExecutionTest.cpp
+  ShaderOpTest.cpp
+  exec-hlsl-tests.rc
+  )
+
+add_dependencies(ClangUnitTests ExecHLSLTests)
+
+set_target_properties(ExecHLSLTests PROPERTIES FOLDER "Execution tests")
+
+target_link_libraries(ExecHLSLTests PRIVATE
+  dxcompiler
+  ${TAEF_LIBRARIES}
+  ${D3D12_LIBRARIES}
+  )
+
+# Add includes for platform helpers and dxc API.
+include_directories(${TAEF_INCLUDE_DIRS})
+include_directories(${DIASDK_INCLUDE_DIRS})
+include_directories(${D3D12_INCLUDE_DIRS})
+include_directories(${LLVM_MAIN_INCLUDE_DIR}/dxc/Test)
+
+add_dependencies(ExecHLSLTests dxcompiler)
+
+if (NOT CLANG_INCLUDE_TESTS)
+  set_target_properties(ExecHLSLTests PROPERTIES OUTPUT_NAME exec-hlsl-tests)
+  set_output_directory(ExecHLSLTests
+    ${LLVM_RUNTIME_OUTPUT_INTDIR} ${LLVM_LIBRARY_OUTPUT_INTDIR})
+endif()
+
+# Add a .user file with settings for te.exe.
+file(TO_NATIVE_PATH "${CMAKE_CURRENT_SOURCE_DIR}" DOS_STYLE_SOURCE_DIR)
+file(TO_NATIVE_PATH "${TAEF_BIN_DIR}" DOS_TAEF_BIN_DIR)
+configure_file(ExecHLSLTests.vcxproj.user.txt ExecHLSLTests.vcxproj.user)

tools/clang/unittests/HLSLExec/ExecHLSLTests.vcxproj.user.txt

@@ -0,0 +1,9 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project ToolsVersion="14.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+  <PropertyGroup>
+    <LocalDebuggerCommand>${DOS_TAEF_BIN_DIR}\$(PlatformTarget)\Te.exe</LocalDebuggerCommand>
+    <LocalDebuggerCommand Condition="'$(PlatformTarget)' == 'x64' And Exists('${DOS_TAEF_BIN_DIR}\amd64\Te.exe')">${DOS_TAEF_BIN_DIR}\amd64\Te.exe</LocalDebuggerCommand>
+    <LocalDebuggerCommandArguments>$(TargetPath) /inproc /runIgnoredTests /p:"ExperimentalShaders=*" /name:*</LocalDebuggerCommandArguments>
+    <DebuggerFlavor>WindowsLocalDebugger</DebuggerFlavor>
+  </PropertyGroup>
+</Project>

tools/clang/unittests/HLSL/ExecutionTest.cpp → tools/clang/unittests/HLSLExec/ExecutionTest.cpp

@@ -164,7 +164,7 @@ static void SavePixelsToFile(LPCVOID pPixels, DXGI_FORMAT format, UINT32 m_width
   CComPtr<IWICBitmap> pBitmap;
   CComPtr<IWICBitmapEncoder> pEncoder;
   CComPtr<IWICBitmapFrameEncode> pFrameEncode;
-  CComPtr<hlsl::AbstractMemoryStream> pStream;
+  CComPtr<IStream> pStream;
   CComPtr<IMalloc> pMalloc;
 
   struct PF {
@@ -183,17 +183,17 @@ static void SavePixelsToFile(LPCVOID pPixels, DXGI_FORMAT format, UINT32 m_width
   VERIFY_SUCCEEDED(ctx.Init());
   VERIFY_SUCCEEDED(CoCreateInstance(CLSID_WICImagingFactory, NULL, CLSCTX_INPROC_SERVER, IID_IWICImagingFactory, (LPVOID*)&pFactory));
   VERIFY_SUCCEEDED(CoGetMalloc(1, &pMalloc));
-  VERIFY_SUCCEEDED(hlsl::CreateMemoryStream(pMalloc, &pStream));
   VERIFY_ARE_NOT_EQUAL(pFormat, Vals + _countof(Vals));
   VERIFY_SUCCEEDED(pFactory->CreateBitmapFromMemory(m_width, m_height, pFormat->PixelFormat, m_width * pFormat->PixelSize, m_width * m_height * pFormat->PixelSize, (BYTE *)pPixels, &pBitmap));
   VERIFY_SUCCEEDED(pFactory->CreateEncoder(GUID_ContainerFormatBmp, nullptr, &pEncoder));
+  VERIFY_SUCCEEDED(SHCreateStreamOnFileEx(pFileName, STGM_WRITE, STGM_CREATE, 0, nullptr, &pStream));
   VERIFY_SUCCEEDED(pEncoder->Initialize(pStream, WICBitmapEncoderNoCache));
   VERIFY_SUCCEEDED(pEncoder->CreateNewFrame(&pFrameEncode, nullptr));
   VERIFY_SUCCEEDED(pFrameEncode->Initialize(nullptr));
   VERIFY_SUCCEEDED(pFrameEncode->WriteSource(pBitmap, nullptr));
   VERIFY_SUCCEEDED(pFrameEncode->Commit());
   VERIFY_SUCCEEDED(pEncoder->Commit());
-  IFT(hlsl::WriteBinaryFile(pFileName, pStream->GetPtr(), pStream->GetPtrSize()));
+  VERIFY_SUCCEEDED(pStream->Commit(STGC_DEFAULT));
 }
 
 // Checks if the given warp version supports the given operation.
@@ -470,7 +470,6 @@ public:
   END_TEST_METHOD()
 
   dxc::DxcDllSupport m_support;
-  VersionSupportInfo m_ver;
 
   bool m_D3DInitCompleted = false;
   bool m_ExperimentalModeEnabled = false;
@@ -522,6 +521,48 @@ public:
     return true;
   }
 
+  std::wstring DxcBlobToWide(_In_ IDxcBlob *pBlob) {
+    if (!pBlob)
+      return std::wstring();
+
+    CComPtr<IDxcBlobWide> pBlobWide;
+    if (SUCCEEDED(pBlob->QueryInterface(&pBlobWide)))
+      return std::wstring(pBlobWide->GetStringPointer(), pBlobWide->GetStringLength());
+
+    CComPtr<IDxcBlobEncoding> pBlobEncoding;
+    IFT(pBlob->QueryInterface(&pBlobEncoding));
+    BOOL known;
+    UINT32 codePage;
+    IFT(pBlobEncoding->GetEncoding(&known, &codePage));
+    if (!known) {
+      throw std::runtime_error("unknown codepage for blob.");
+    }
+
+    std::wstring result;
+    if (codePage == DXC_CP_WIDE) {
+      const wchar_t* text = (const wchar_t *)pBlob->GetBufferPointer();
+      size_t length = pBlob->GetBufferSize() / 2;
+      if (length >= 1 && text[length-1] == L'\0')
+        length -= 1;  // Exclude null-terminator
+      result.resize(length);
+      memcpy(&result[0], text, length);
+      return result;
+    }
+    if (codePage == CP_UTF8) {
+      const char* text = (const char *)pBlob->GetBufferPointer();
+      size_t length = pBlob->GetBufferSize();
+      if (length >= 1 && text[length-1] == '\0')
+        length -= 1;  // Exclude null-terminator
+      if (length == 0)
+        return std::wstring();
+      int wideLength = ::MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, text, (int)length, nullptr, 0);
+      result.resize(wideLength);
+      ::MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS, text, (int)length, &result[0], wideLength);
+      return result;
+    }
+    throw std::runtime_error("Unsupported codepage.");
+  }
+
 // Do not remove the following line - it is used by TranslateExecutionTest.py
 // MARKER: ExecutionTest/DxilConf Shared Implementation Start
 
@@ -645,10 +686,10 @@ public:
     VERIFY_SUCCEEDED(pCompiler->Compile(pTextBlob, L"hlsl.hlsl", pEntryPoint, pTargetProfile, pOptions, numOptions, nullptr, 0, nullptr, &pResult));
     VERIFY_SUCCEEDED(pResult->GetStatus(&resultCode));
     if (FAILED(resultCode)) {
+#ifndef _HLK_CONF
       CComPtr<IDxcBlobEncoding> errors;
       VERIFY_SUCCEEDED(pResult->GetErrorBuffer(&errors));
-#ifndef _HLK_CONF
-      LogCommentFmt(L"Failed to compile shader: %s", BlobToWide(errors).data());
+      LogCommentFmt(L"Failed to compile shader: %s", DxcBlobToWide(errors).data());
 #endif
     }
     VERIFY_SUCCEEDED(resultCode);
@@ -1643,7 +1684,7 @@ public:
     CComPtr<IDxcLibrary> pLibrary;
     CComPtr<IDxcBlobEncoding> pBlob;
     CComPtr<IStream> pStream;
-    std::wstring path = GetPathToHlslDataFile(relativePath);
+    std::wstring path = GetPathToHlslDataFile(relativePath, HLSLDATAFILEPARAM, DEFAULT_EXEC_TEST_DIR);
     VERIFY_SUCCEEDED(m_support.CreateInstance(CLSID_DxcLibrary, &pLibrary));
     VERIFY_SUCCEEDED(pLibrary->CreateBlobFromFile(path.c_str(), nullptr, &pBlob));
     VERIFY_SUCCEEDED(pLibrary->CreateStreamFromBlobReadOnly(pBlob, &pStream));

tools/clang/unittests/HLSL/ShaderOpTest.cpp → tools/clang/unittests/HLSLExec/ShaderOpTest.cpp

@@ -24,7 +24,6 @@
 
 #include "dxc/dxcapi.h"             // IDxcCompiler
 #include "dxc/Support/Global.h"     // OutputDebugBytes
-#include "dxc/Support/Unicode.h"    // IsStarMatchWide
 #include "dxc/Support/dxcapi.use.h" // DxcDllSupport
 #include "dxc/DXIL/DxilConstants.h" // ComponentType
 #include "WexTestClass.h"           // TAEF
@@ -112,8 +111,8 @@ bool UseHardwareDevice(const DXGI_ADAPTER_DESC1 &desc, LPCWSTR AdapterName) {
 
   if (!AdapterName)
     return true;
-  return Unicode::IsStarMatchWide(AdapterName, wcslen(AdapterName),
-                                   desc.Description, wcslen(desc.Description));
+  return hlsl_test::IsStarMatchWide(AdapterName, wcslen(AdapterName),
+                                    desc.Description, wcslen(desc.Description));
 }
 
 void GetHardwareAdapter(IDXGIFactory2 *pFactory, LPCWSTR AdapterName,

tools/clang/unittests/HLSLTestLib/DxcTestUtils.cpp

@@ -14,6 +14,7 @@
 #include "dxc/Test/HlslTestUtils.h"
 #include "dxc/Support/HLSLOptions.h"
 #include "dxc/Support/Global.h"
+#include "dxc/Support/Unicode.h"
 #include "llvm/ADT/StringRef.h"
 #include "llvm/ADT/APInt.h"
 #include "llvm/Support/ManagedStatic.h"

utils/hct/hcttest.cmd

@@ -291,7 +291,7 @@ if not exist %TEST_DIR% (mkdir %TEST_DIR%)
 echo Copying binaries to test to %TEST_DIR%:
 if "%CUSTOM_BIN_SET%"=="" (
   if not "%TEST_USE_LIT%"=="1" (
-    call %HCT_DIR%\hctcopy.cmd %BIN_DIR% %TEST_DIR% clang-hlsl-tests.dll
+    call %HCT_DIR%\hctcopy.cmd %BIN_DIR% %TEST_DIR% clang-hlsl-tests.dll exec-hlsl-tests.dll
   )
   call %HCT_DIR%\hctcopy.cmd %BIN_DIR% %TEST_DIR% dxa.exe dxc.exe dxexp.exe dxopt.exe dxr.exe dxv.exe dxcompiler.dll d3dcompiler_dxc_bridge.dll dxl.exe dxc_batch.exe dxlib_sample.dll
   if errorlevel 1 exit /b 1
@@ -369,10 +369,10 @@ if "%TEST_EXEC%"=="1" (
   call :copyagility
 )
 
-set EXEC_COMMON_ARGS= /p:"HlslDataDir=%HLSL_SRC_DIR%\tools\clang\test\HLSL" /runIgnoredTests /p:"ExperimentalShaders=*" %TEST_ADAPTER% %USE_AGILITY_SDK%
+set EXEC_COMMON_ARGS= /runIgnoredTests /p:"ExperimentalShaders=*" %TEST_ADAPTER% %USE_AGILITY_SDK%
 if "%TEST_EXEC%"=="1" (
   echo Sniffing for D3D12 configuration ...
-  call :runte clang-hlsl-tests.dll /select:"@Name='ExecutionTest::BasicTriangleTest' AND @Architecture='%TEST_ARCH%'" %EXEC_COMMON_ARGS% 
+  call :runte exec-hlsl-tests.dll /select:"@Name='ExecutionTest::BasicTriangleTest' AND @Architecture='%TEST_ARCH%'" %EXEC_COMMON_ARGS% 
   set RES_EXEC=!ERRORLEVEL!
   if errorlevel 1 (
     if not "%TEST_EXEC_REQUIRED%"=="1" (
@@ -391,7 +391,7 @@ if "%TEST_EXEC%"=="1" (
   ) else (
     set SELECT_FILTER= /select:"@Name='%TEST_EXEC_FILTER%' AND @Priority<2 AND @Architecture='%TEST_ARCH%'"
   )
-  call :runte clang-hlsl-tests.dll !SELECT_FILTER! %EXEC_COMMON_ARGS% %ADDITIONAL_OPTS%
+  call :runte exec-hlsl-tests.dll !SELECT_FILTER! %EXEC_COMMON_ARGS% %ADDITIONAL_OPTS%
   set RES_EXEC=!ERRORLEVEL!
 )