Support taef run in lit. (#4762)

* Support taef run in lib.

lit.formats.taef is added to run taef under lit.

New lit target check-clang-hlsl is added to run clang-hlsl-test taef tests.

Add lit-target for dxil conv to set different hlsl data dir.

projects/dxilconv/CMakeLists.txt

@@ -24,5 +24,6 @@ if(WIN32)
   add_subdirectory(tools)
   if (HLSL_INCLUDE_TESTS)
     add_subdirectory(unittests)
+    add_subdirectory(test)
   endif (HLSL_INCLUDE_TESTS)
 endif()

projects/dxilconv/test/CMakeLists.txt

@@ -0,0 +1,18 @@
+# Test runner infrastructure for dxilconv. This configures the dxilconv test trees
+# for use by Lit, and delegates to LLVM's lit test handlers.
+
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/taef/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/taef/lit.site.cfg
+  )
+
+
+# HLSL Change Begin - Add taef tests
+add_lit_target("check-dxilconv" "Running lit suite dxilconv"
+          ${CMAKE_CURRENT_SOURCE_DIR}/taef
+          PARAMS
+                 dxilconv_site_config=${CMAKE_CURRENT_BINARY_DIR}/taef/lit.site.cfg
+          DEPENDS dxilconv-tests
+          ARGS ${CLANG_TEST_EXTRA_ARGS}
+        )
+# HLSL Change End

projects/dxilconv/test/taef/lit.cfg

@@ -0,0 +1,37 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+import lit.util
+
+# name: The name of this test suite.
+config.name = 'dxilconv'
+
+# Check that the object root is known.
+llvm_obj_root = getattr(config, 'llvm_obj_root', None)
+if llvm_obj_root is None:
+    # Otherwise, we haven't loaded the site specific configuration (the user is
+    # probably trying to run on a test file directly, and either the site
+    # configuration hasn't been created by the build system, or we are in an
+    # out-of-tree build situation).
+
+    # Check for 'clang_unit_site_config' user parameter, and use that if available.
+    site_cfg = lit_config.params.get('dxilconv_site_config', None)
+    if site_cfg and os.path.exists(site_cfg):
+        lit_config.load_config(config, site_cfg)
+        raise SystemExit
+
+
+te = os.path.join(config.llvm_obj_root, "TAEF", config.arch, 'te.exe')
+
+test_dll = os.path.join(config.llvm_obj_root, config.build_mode, 'bin', 'dxilconv-tests.dll')
+
+hlsl_data_dir = os.path.join(config.llvm_src_root, 'projects', 'dxilconv', 'test')
+
+test_dir = os.path.join(config.llvm_obj_root, config.build_mode, 'test')
+
+config.test_format = lit.formats.TaefTest(te, test_dll, hlsl_data_dir, test_dir)
+

projects/dxilconv/test/taef/lit.site.cfg.in

@@ -0,0 +1,11 @@
+import sys
+
+## Autogenerated by LLVM/Clang configuration.
+# Do not edit!
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.build_mode = lit_config.params["build_mode"]
+config.arch = "@DXC_BUILD_ARCH@"
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@LLVM_SOURCE_DIR@/projects/dxilconv/test/taef/lit.cfg")

tools/clang/test/CMakeLists.txt

@@ -19,6 +19,11 @@ configure_lit_site_cfg(
   ${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg
   )
 
+configure_lit_site_cfg(
+  ${CMAKE_CURRENT_SOURCE_DIR}/clang-hlsl/lit.site.cfg.in
+  ${CMAKE_CURRENT_BINARY_DIR}/clang-hlsl/lit.site.cfg
+  )
+
 option(CLANG_TEST_USE_VG "Run Clang tests under Valgrind" OFF)
 if(CLANG_TEST_USE_VG)
   set(CLANG_TEST_EXTRA_ARGS ${CLANG_TEST_EXTRA_ARGS} "--vg")
@@ -92,3 +97,13 @@ add_lit_target("check-clang-unit" "Running lit suite clang-unit"
           ARGS ${CLANG_TEST_EXTRA_ARGS}
         )
 # HLSL Change End - Add a separate target for clang unit tests
+
+# HLSL Change Begin - Add taef tests
+add_lit_target("check-hlsl" "Running lit suite hlsl"
+          ${CMAKE_CURRENT_SOURCE_DIR}/clang-hlsl
+          PARAMS ${CLANG_TEST_PARAMS}
+                 clang_hlsl_site_config=${CMAKE_CURRENT_BINARY_DIR}/clang-hlsl/lit.site.cfg
+          DEPENDS clang-hlsl-tests
+          ARGS ${CLANG_TEST_EXTRA_ARGS}
+        )
+# HLSL Change End

tools/clang/test/clang-hlsl/lit.cfg

@@ -0,0 +1,37 @@
+# -*- Python -*-
+
+# Configuration file for the 'lit' test runner.
+
+import os
+
+import lit.formats
+import lit.util
+
+# name: The name of this test suite.
+config.name = 'clang-hlsl'
+
+# Check that the object root is known.
+llvm_obj_root = getattr(config, 'llvm_obj_root', None)
+if llvm_obj_root is None:
+    # Otherwise, we haven't loaded the site specific configuration (the user is
+    # probably trying to run on a test file directly, and either the site
+    # configuration hasn't been created by the build system, or we are in an
+    # out-of-tree build situation).
+
+    # Check for 'clang_unit_site_config' user parameter, and use that if available.
+    site_cfg = lit_config.params.get('clang_hlsl_site_config', None)
+    if site_cfg and os.path.exists(site_cfg):
+        lit_config.load_config(config, site_cfg)
+        raise SystemExit
+
+
+te = os.path.join(config.llvm_obj_root, "TAEF", config.arch, 'te.exe')
+
+test_dll = os.path.join(config.llvm_obj_root, config.build_mode, 'bin', 'clang-hlsl-tests.dll')
+
+hlsl_data_dir = os.path.join(config.llvm_src_root, 'tools', 'clang', 'test', 'hlsl')
+
+test_dir = os.path.join(config.llvm_obj_root, config.build_mode, 'test')
+#projects\dxilconv\test
+config.test_format = lit.formats.TaefTest(te, test_dll, hlsl_data_dir, test_dir)
+

tools/clang/test/clang-hlsl/lit.site.cfg.in

@@ -0,0 +1,11 @@
+import sys
+
+## Autogenerated by LLVM/Clang configuration.
+# Do not edit!
+config.llvm_src_root = "@LLVM_SOURCE_DIR@"
+config.llvm_obj_root = "@LLVM_BINARY_DIR@"
+config.build_mode = lit_config.params["build_mode"]
+config.arch = "@DXC_BUILD_ARCH@"
+
+# Let the main config do the real work.
+lit_config.load_config(config, "@CLANG_SOURCE_DIR@/test/clang-hlsl/lit.cfg")

utils/lit/lit/formats/__init__.py

@@ -1,4 +1,5 @@
 from __future__ import absolute_import
 from lit.formats.base import TestFormat, FileBasedTest, OneCommandPerFileTest
 from lit.formats.googletest import GoogleTest
+from lit.formats.taef import TaefTest
 from lit.formats.shtest import ShTest

utils/lit/lit/formats/taef.py

@@ -0,0 +1,120 @@
+from __future__ import absolute_import
+import os
+import sys
+
+import lit.Test
+import lit.TestRunner
+import lit.util
+from .base import TestFormat
+
+class TaefTest(TestFormat):
+    def __init__(self, te_path, test_dll, hlsl_data_dir, test_path):
+        self.te = te_path
+        self.test_dll = test_dll
+        self.hlsl_data_dir = hlsl_data_dir
+        self.test_path = test_path
+        # NOTE: when search test, always running on test_dll,
+        #       use test_searched to make sure only add test once.
+        #       If TaeftTest is created in directory with sub directory,
+        #       getTestsInDirectory will be called more than once.
+        self.test_searched = False
+
+    def getTaefTests(self, dll_path, litConfig, localConfig):
+        """getTaefTests()
+
+        Return the tests available in taef test dll.
+
+        Args:
+          litConfig: LitConfig instance
+          localConfig: TestingConfig instance"""
+
+        # TE:F:\repos\DxcGitHub\hlsl.bin\TAEF\x64\te.exe
+        # test dll : F:\repos\DxcGitHub\hlsl.bin\Debug\test\clang-hlsl-tests.dll
+        # /list
+
+        if litConfig.debug:
+            litConfig.note('searching taef test in %r' % dll_path)
+
+        try:
+            lines = lit.util.capture([self.te, dll_path, '/list'],
+                                     env=localConfig.environment)
+            # this is for windows
+            lines = lines.replace('\r', '')
+            lines = lines.split('\n')
+        except:
+            litConfig.error("unable to discover taef in %r" % path)
+            raise StopIteration
+
+        for ln in lines:
+            # The test name is like VerifierTest::RunUnboundedResourceArrays.
+            if ln.find('::') == -1:
+                continue
+
+            yield ln.strip()
+
+    # Note: path_in_suite should not include the executable name.
+    def getTestsInExecutable(self, testSuite, path_in_suite, execpath,
+                             litConfig, localConfig):
+
+        # taef test should be dll.
+        if not execpath.endswith('dll'):
+            return
+
+        (dirname, basename) = os.path.split(execpath)
+        # Discover the tests in this executable.
+        for testname in self.getTaefTests(execpath, litConfig, localConfig):
+            testPath = path_in_suite + (basename, testname)
+            yield lit.Test.Test(testSuite, testPath, localConfig, file_path=execpath)
+
+    def getTestsInDirectory(self, testSuite, path_in_suite,
+                            litConfig, localConfig):
+        # Make sure self.test_dll only search once.
+        if self.test_searched:
+            return
+
+        self.test_searched = True
+
+        filepath = self.test_dll
+        for test in self.getTestsInExecutable(
+                testSuite, path_in_suite, filepath,
+                litConfig, localConfig):
+            yield test
+
+    def execute(self, test, litConfig):
+        test_dll = test.getFilePath()
+
+        testPath,testName = os.path.split(test.getSourcePath())
+
+        param_hlsl_data_dir = str.format('/p:HlslDataDir={}', self.hlsl_data_dir)
+        cmd = [self.te, test_dll, '/inproc',
+                param_hlsl_data_dir,
+                '/miniDumpOnCrash', '/unicodeOutput:false',
+                '/logOutput:LowWithConsoleBuffering',
+                str.format('/outputFolder:{}', self.test_path),
+                str.format('/name:{}', testName)]
+
+        if litConfig.useValgrind:
+            cmd = litConfig.valgrindArgs + cmd
+
+        if litConfig.noExecute:
+            return lit.Test.PASS, ''
+
+        out, err, exitCode = lit.util.executeCommand(
+            cmd, env=test.config.environment)
+
+        if exitCode:
+            return lit.Test.FAIL, out + err
+
+        summary = 'Summary: Total='
+        if summary not in out:
+            msg = ('Unable to find %r in taef output:\n\n%s%s' %
+                   (summary, out, err))
+            return lit.Test.UNRESOLVED, msg
+        no_fail = 'Failed=0, Blocked=0, Not Run=0, Skipped=0'
+        if no_fail not in out == -1:
+            msg = ('Unable to find %r in taef output:\n\n%s%s' %
+                   (no_fail, out, err))
+            return lit.Test.UNRESOLVED, msg
+
+        return lit.Test.PASS,''
+