Fix lit test fails (#4973)

1. added dxc_batch as dependent for clang tests.
2. Not run exec test by default.
3. Strip directory which has dxil.dll from %PATH% for taef test which not need it.
4. Add select_filter and use @Name for TestName.
    * Enable mix select filter by use shell=True when open subprocess.
5. Only run execution test when dxil.dll exist or experimental mode enabled.

cmake/modules/FindTAEF.cmake

@@ -41,10 +41,27 @@ set(TAEF_INCLUDE_DIRS ${TAEF_INCLUDE_DIR})
 set(TAEF_NUGET_BIN ${TAEF_INCLUDE_DIR}/../Binaries/Release)
 set(TAEF_SDK_BIN ${TAEF_INCLUDE_DIR}/../../Runtimes/TAEF)
 
+if ((CMAKE_GENERATOR_PLATFORM STREQUAL "x64") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" STREQUAL "x64"))
+  set(TAEF_BIN_ARCH "amd64")
+  set(TAEF_ARCH "x64")
+elseif ((CMAKE_GENERATOR_PLATFORM STREQUAL "x86") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" STREQUAL "x86"))
+  set(TAEF_BIN_ARCH "x86")
+  set(TAEF_ARCH "x86")
+elseif ((CMAKE_GENERATOR_PLATFORM MATCHES "ARM64.*") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "ARM64.*"))
+  set(TAEF_BIN_ARCH "arm64")
+  set(TAEF_ARCH "arm64")
+elseif ((CMAKE_GENERATOR_PLATFORM MATCHES "ARM.*") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "ARM.*"))
+  set(TAEF_BIN_ARCH "arm")
+  set(TAEF_ARCH "arm")
+endif((CMAKE_GENERATOR_PLATFORM STREQUAL "x64") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" STREQUAL "x64"))
+
+set (TAEF_ARCH ${TAEF_ARCH} CACHE INTERNAL "arch for taef test")
+
 find_program(TAEF_EXECUTABLE te.exe PATHS
-  $ENV{HLSL_TAEF_DIR}/${CMAKE_C_COMPILER_ARCHITECTURE_ID}
-  ${TAEF_NUGET_BIN}/${CMAKE_C_COMPILER_ARCHITECTURE_ID}
-  ${TAEF_SDK_BIN}/${CMAKE_C_COMPILER_ARCHITECTURE_ID}
+  ${CMAKE_SOURCE_DIR}/external/taef/build/Binaries/${TAEF_BIN_ARCH}
+  $ENV{HLSL_TAEF_DIR}/${TAEF_BIN_ARCH}
+  ${TAEF_NUGET_BIN}/${TAEF_BIN_ARCH}
+  ${TAEF_SDK_BIN}/${TAEF_BIN_ARCH}
   ${WINDOWS_KIT_10_PATH}
   ${WINDOWS_KIT_81_PATH}
   )
@@ -55,16 +72,6 @@ else()
   message(ERROR "Unable to find TAEF binaries.")
 endif()
 
-if ((CMAKE_GENERATOR_PLATFORM STREQUAL "x64") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" STREQUAL "x64"))
-  set(TAEF_ARCH "x64")
-elseif ((CMAKE_GENERATOR_PLATFORM STREQUAL "x86") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" STREQUAL "x86"))
-  set(TAEF_ARCH "x86")
-elseif ((CMAKE_GENERATOR_PLATFORM MATCHES "ARM64.*") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "ARM64.*"))
-  set(TAEF_ARCH "arm64")
-elseif ((CMAKE_GENERATOR_PLATFORM MATCHES "ARM.*") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" MATCHES "ARM.*"))
-  set(TAEF_ARCH "arm")
-endif((CMAKE_GENERATOR_PLATFORM STREQUAL "x64") OR ("${CMAKE_C_COMPILER_ARCHITECTURE_ID}" STREQUAL "x64"))
-
 
 include(FindPackageHandleStandardArgs)
 # handle the QUIETLY and REQUIRED arguments and set TAEF_FOUND to TRUE

projects/dxilconv/test/taef/lit.cfg

@@ -30,17 +30,15 @@ hlsl_data_dir = os.path.join(config.llvm_src_root, 'projects', 'dxilconv', 'test
 
 test_dir = os.path.join(config.llvm_obj_root, config.llvm_build_mode, 'test')
 
-param_hlsl_data_dir = str.format('/p:HlslDataDir={}', hlsl_data_dir)
+param_hlsl_data_dir = str.format('/p:"HlslDataDir={}"', hlsl_data_dir)
 extra_params = [param_hlsl_data_dir]
 
 verbose = lit_config.params.get('verbose', None)
 if verbose == None:
-    extra_params.append('/logOutput:LowWithConsoleBuffering')
+    extra_params.append('/logOutput:"LowWithConsoleBuffering"')
 
 arch = getattr(config, 'taef_arch', None)
-arch_filter = str.format("@Architecture='{}'", arch)
-select_filter = str.format("/select:\"{}\"", arch_filter)
-extra_params.append(select_filter)
+select_filter = str.format("@Architecture='{}'", arch)
 
-config.test_format = lit.formats.TaefTest(config.te, test_dll, test_dir, extra_params)
+config.test_format = lit.formats.TaefTest(config.te, test_dll, test_dir, select_filter,extra_params)
 

tools/clang/test/CMakeLists.txt

@@ -77,7 +77,7 @@ endif()
 set(CLANG_TEST_DEPS dxc dxa dxopt dxl dxv dxr dxcompiler clang-tblgen llvm-config opt FileCheck count not ClangUnitTests)
 if (WIN32)
 list(APPEND CLANG_TEST_DEPS
-     ExecHLSLTests
+     dxc_batch ExecHLSLTests
     )
 endif()
 add_custom_target(clang-test-depends DEPENDS ${CLANG_TEST_DEPS})
@@ -89,6 +89,7 @@ add_lit_testsuite(check-clang "Running the Clang regression tests"
   ${CMAKE_CURRENT_BINARY_DIR}
   #LIT ${LLVM_LIT}
   PARAMS ${CLANG_TEST_PARAMS}
+    skip_taef_exec=True
   DEPENDS ${CLANG_TEST_DEPS}
   ARGS ${CLANG_TEST_EXTRA_ARGS}
   )
@@ -123,7 +124,7 @@ if (WIN32)
             ${CMAKE_CURRENT_SOURCE_DIR}/taef_exec
             PARAMS ${CLANG_TEST_PARAMS}
                   clang_taef_exec_site_config=${CMAKE_CURRENT_BINARY_DIR}/taef_exec/lit.site.cfg
-            DEPENDS ExecHLSLTests
+            DEPENDS ExecHLSLTests dxexp
             ARGS ${CLANG_TEST_EXTRA_ARGS}
           )
 endif()

tools/clang/test/lit.cfg

@@ -495,8 +495,6 @@ if config.enable_backtrace == "1":
 
 if config.spirv:
     config.available_features.add("spirv")
-else:
-    config.limit_to_features.add("spirv")
 
 # Check if we should run long running tests.
 if lit_config.params.get("run_long_tests", None) == "true":

tools/clang/test/taef/lit.cfg

@@ -35,21 +35,20 @@ else:
 
   test_dir = os.path.join(config.llvm_obj_root, config.llvm_build_mode, 'test')
 
-  param_hlsl_data_dir = str.format('/p:HlslDataDir={}', hlsl_data_dir)
+  param_hlsl_data_dir = str.format('/p:"HlslDataDir={}"', hlsl_data_dir)
   extra_params = [param_hlsl_data_dir]
 
   verbose = lit_config.params.get('verbose', None)
   if verbose == None:
-      extra_params.append('/logOutput:LowWithConsoleBuffering')
+      extra_params.append('/logOutput:"LowWithConsoleBuffering"')
 
   arch = getattr(config, 'taef_arch', None)
   arch_filter = str.format("@Architecture='{}'", arch)
   no_priority = lit_config.params.get('no_priority', None)
   if no_priority:
-      select_filter = str.format("/select:\"{}\"", arch_filter)
+      select_filter = arch_filter
   else:
       priority_filter = "@Priority<1"
-      select_filter = str.format("/select:\"{} AND {}\"", priority_filter, arch_filter)
-  extra_params.append(select_filter)
+      select_filter = str.format("{} AND {}", priority_filter, arch_filter)
 
-  config.test_format = lit.formats.TaefTest(config.te, test_dll, test_dir, extra_params)
+  config.test_format = lit.formats.TaefTest(config.te, test_dll, test_dir, select_filter, extra_params)

tools/clang/test/taef_exec/lit.cfg

@@ -4,6 +4,7 @@
 
 import os
 import platform
+import subprocess
 
 import lit.formats
 import lit.util
@@ -24,45 +25,59 @@ if te is None:
     if site_cfg and os.path.exists(site_cfg):
         lit_config.load_config(config, site_cfg)
         raise SystemExit
-
-# TAEF only runs on Windows
-if platform.system() != 'Windows':
+bin_dir = os.path.join(config.llvm_obj_root, config.llvm_build_mode, 'bin')
+skip_taef_exec = lit_config.params.get('skip_taef_exec', None)
+if skip_taef_exec or platform.system() != 'Windows':
   config.unsupported = True
 else:
-  test_dll = os.path.join(config.llvm_obj_root, config.llvm_build_mode, 'bin', 'ExecHLSLTests.dll')
+  dxil_dll = os.path.join(bin_dir, 'dxil.dll')
+  if os.path.isfile(dxil_dll):
+    print(str.format("\nUsed {} as validator\n", dxil_dll))
+  else:
+    print(str.format("\nCannot find validator in {}\n", bin_dir))
+    dxexp = os.path.join(bin_dir, 'dxexp.exe')
+    result = subprocess.run([dxexp], capture_output=True)
+    no_experimental = "Experimental shader model feature failed"
+    out = lit.util.convert_string(result.stdout)
+    if no_experimental in out:
+        print("""Developer mode is not enabled in Windows settings, preventing the experimental shader models feature.
+Execution tests require DXIL.dll to sign shaders, or the experimental shader models feature to run unsigned shaders.""")
+        config.unsupported = True
+
+if config.unsupported == False:
+  test_dll = os.path.join(bin_dir, 'ExecHLSLTests.dll')
 
   hlsl_data_dir = os.path.join(config.llvm_src_root, 'tools', 'clang', 'unittests', 'HLSLExec')
 
   test_dir = os.path.join(config.llvm_obj_root, config.llvm_build_mode, 'test')
-  param_hlsl_data_dir = str.format('/p:HlslDataDir={}', hlsl_data_dir)
+  param_hlsl_data_dir = str.format('/p:"HlslDataDir={}"', hlsl_data_dir)
 
-  extra_params = ["/p:\"ExperimentalShaders=*\"", param_hlsl_data_dir]
+  extra_params = ['/p:"ExperimentalShaders=*\"', param_hlsl_data_dir]
 
   verbose = lit_config.params.get('verbose', None)
   if verbose == None:
-      extra_params.append('/logOutput:LowWithConsoleBuffering')
-
-  arch = getattr(config, 'taef_arch', None)
-  arch_filter = str.format("@Architecture='{}'", arch)
-  priority_filter = "@Priority<2"
-  exec_future = lit_config.params.get('exec_future', None)
-  if exec_future:
-      priority_filter = "@Priority=2"
-
-  select_filter = str.format("/select:\"{} AND {}\"", priority_filter, arch_filter)
-  extra_params.append(select_filter)
+      extra_params.append('/logOutput:"LowWithConsoleBuffering"')
 
   adapter = lit_config.params.get('adapter', None)
   if adapter:
-    extra_params.append(str.format("/p:Adapter={}", adapter))
+      extra_params.append(str.format('/p:"Adapter={}"', adapter))
 
   agility_sdk = lit_config.params.get('agility_sdk', None)
   if agility_sdk:
-    extra_params.append("/p:D3D12SDKVersion=1")
+    extra_params.append('/p:"D3D12SDKVersion=1"')
 
   # use ';' to split multiple params.
   taef_extra_params = lit_config.params.get('taef_exec_extra_params', None)
   if taef_extra_params:
-    extra_params.extend(taef_extra_params.split(';'))
+      extra_params.extend(taef_extra_params.split(';'))
+
+  arch = getattr(config, 'taef_arch', None)
+  arch_filter = str.format("@Architecture='{}'", arch)
+  priority_filter = "@Priority<2"
+  exec_future = lit_config.params.get('exec_future', None)
+  if exec_future:
+      priority_filter = "@Priority=2"
+
+  select_filter = str.format("{} AND {}", priority_filter, arch_filter)
 
-  config.test_format = lit.formats.TaefTest(config.te, test_dll, test_dir, extra_params)
+  config.test_format = lit.formats.TaefTest(config.te, test_dll, test_dir, select_filter, extra_params)

utils/hct/hctbuild.cmd

@@ -435,7 +435,8 @@ if "%DO_SETUP%"=="1" (
     "%CMAKE_PATH%" -DCMAKE_BUILD_TYPE:STRING=%1 %CMAKE_OPTS% -G %4 %HLSL_SRC_DIR% >> %3\cmake-log.txt 2>&1
   ) else (
     rem -DCMAKE_BUILD_TYPE:STRING=%1 is not necessary for multi-config generators like VS
-    echo Running "%CMAKE_PATH%" %CMAKE_OPTS% -G %4 %5 %HLSL_SRC_DIR% > %3\cmake-log.txt
+    rem but need CMAKE_BUILD_TYPE to generate lit cfg.
+    echo Running "%CMAKE_PATH%" -DCMAKE_BUILD_TYPE:STRING=%1  %CMAKE_OPTS% -G %4 %5 %HLSL_SRC_DIR% > %3\cmake-log.txt
     "%CMAKE_PATH%" %CMAKE_OPTS% -G %4 %5 %HLSL_SRC_DIR% >> %3\cmake-log.txt 2>&1
   )
   if %SHOW_CMAKE_LOG%==1 (

utils/lit/lit/TestingConfig.py

@@ -1,13 +1,23 @@
 import os
 import sys
+import platform
 
 OldPy = sys.version_info[0] == 2 and sys.version_info[1] < 7
 
+def strip_dxil_validator_path(env_path):
+    dxil_name, separator = ('dxil.dll', ';') if platform.system() == 'Windows' else ('dxil.so', ';')
+    return separator.join([
+        p for p in env_path.split(separator)
+        if not os.path.isfile(os.path.join(p, dxil_name))
+        ])
+
+
 class TestingConfig:
     """"
     TestingConfig - Information on the tests inside a suite.
     """
 
+
     @staticmethod
     def fromdefaults(litConfig):
         """
@@ -16,9 +26,15 @@ class TestingConfig:
         Create a TestingConfig object with default values.
         """
         # Set the environment based on the command line arguments.
+
+        # strip dxil validator dir if not need it.
+        all_path = os.pathsep.join(litConfig.path +
+                                     [os.environ.get('PATH','')])
+
+        all_path = strip_dxil_validator_path(all_path)
+
         environment = {
-            'PATH' : os.pathsep.join(litConfig.path +
-                                     [os.environ.get('PATH','')]),
+            'PATH' : all_path,
             'LLVM_DISABLE_CRASH_REPORT' : '1',
             }
 

utils/lit/lit/discovery.py

@@ -74,6 +74,7 @@ def getTestSuite(item, litConfig, cache):
     return ts, tuple(relative + tuple(components))
 
 def getLocalConfig(ts, path_in_suite, litConfig, cache):
+
     def search1(path_in_suite):
         # Get the parent config.
         if not path_in_suite:
@@ -95,6 +96,7 @@ def getLocalConfig(ts, path_in_suite, litConfig, cache):
         if litConfig.debug:
             litConfig.note('loading local config %r' % cfgpath)
         config.load_from_path(cfgpath, litConfig)
+
         return config
 
     def search(path_in_suite):

utils/lit/lit/formats/taef.py

@@ -1,17 +1,47 @@
 from __future__ import absolute_import
 import os
 import sys
+import signal
+import subprocess
 
 import lit.Test
 import lit.TestRunner
 import lit.util
 from .base import TestFormat
 
+# TAEF must be run with custom command line string and shell=True
+# because of the way it manually processes quoted arguments in a
+# non-standard way.
+def executeCommandForTaef(command, cwd=None, env=None):
+    command = ' '.join(command)
+    p = subprocess.Popen(command, cwd=cwd,
+                        shell=True,
+                        stdin=subprocess.PIPE,
+                        stdout=subprocess.PIPE,
+                        stderr=subprocess.PIPE,
+                        env=env,
+                        # Close extra file handles on UNIX (on Windows this cannot be done while
+                        # also redirecting input). Taef only run on windows.
+                        close_fds=False)
+    out,err = p.communicate()
+    exitCode = p.wait()
+
+    # Detect Ctrl-C in subprocess.
+    if exitCode == -signal.SIGINT:
+        raise KeyboardInterrupt
+
+    # Ensure the resulting output is always of string type.
+    out = lit.util.convert_string(out)
+    err = lit.util.convert_string(err)
+
+    return out, err, exitCode
+
 class TaefTest(TestFormat):
-    def __init__(self, te_path, test_dll, test_path, extra_params):
+    def __init__(self, te_path, test_dll, test_path, select_filter, extra_params):
         self.te = te_path
         self.test_dll = test_dll
         self.test_path = test_path
+        self.select_filter = select_filter
         self.extra_params = extra_params
         # NOTE: when search test, always running on test_dll,
         #       use test_searched to make sure only add test once.
@@ -41,6 +71,7 @@ class TaefTest(TestFormat):
             # this is for windows
             lines = lines.replace('\r', '')
             lines = lines.split('\n')
+
         except:
             litConfig.error("unable to discover taef tests in %r, using %s" % (dll_path, self.te))
             raise StopIteration
@@ -85,10 +116,17 @@ class TaefTest(TestFormat):
 
         testPath,testName = os.path.split(test.getSourcePath())
 
+        select_filter = str.format("@Name='{}'", testName)
+
+        if self.select_filter != "":
+            select_filter = str.format("{} AND {}", select_filter, self.select_filter)
+
+        select_filter = str.format('/select:"{}"', select_filter)
+
         cmd = [self.te, test_dll, '/inproc',
+                select_filter,
                 '/miniDumpOnCrash', '/unicodeOutput:false',
-                str.format('/outputFolder:{}', self.test_path),
-                str.format('/name:{}', testName)]
+                str.format('/outputFolder:{}', self.test_path)]
         cmd.extend(self.extra_params)
 
         if litConfig.useValgrind:
@@ -97,13 +135,18 @@ class TaefTest(TestFormat):
         if litConfig.noExecute:
             return lit.Test.PASS, ''
 
-        out, err, exitCode = lit.util.executeCommand(
-            cmd, env=test.config.environment)
+        out, err, exitCode = executeCommandForTaef(
+            cmd, env = test.config.environment)
 
         if exitCode:
             skipped = 'Failed=0, Blocked=0, Not Run=0, Skipped=1'
             if skipped in out:
                 return lit.Test.UNSUPPORTED, ''
+
+            unselected = 'The selection criteria did not match any tests.'
+            if unselected in out:
+                return lit.Test.UNSUPPORTED, ''
+
             return lit.Test.FAIL, out + err
 
         summary = 'Summary: Total='