Make it possible to enable building LLVM & Clang unit tests (#4020)

* Changes to get LLVM unit tests building

This change gets the LLVM unit tests building again and running through
LIT via the `check-llvm-unit` target.

This change does not have the tests passing! Subsequent changes will
get the unit tests passing. This change also disables some tests where
the LLVM code is no longer used and making those tests work will
require substantial effort.

* Changes to get Clang unit tests building

This change gets the Clang unit tests building again and running
through LIT via the `check-clang-unit` target.

This change does not have the tests passing! Subsequent changes will
get the unit tests passing. This change also disables some tests where
the Clang code is no longer used and making those tests work will
require substantial effort.

* A few extra Windows fixes

This adds some missing APIs to the Windows Filesystem code and adds an
option to hctbuild to enable building the LLVM & Clang unit tests.

* Disable libClangFormat tests

These tests are a bit gnarly to repair... unless we really start using
the format library we probably just want to disable these tests.

CMakeLists.txt

@@ -621,6 +621,10 @@ if( MINGW )
   llvm_replace_compiler_option(CMAKE_CXX_FLAGS_RELEASE "-O3" "-O2")
 endif()
 
+if(LLVM_INCLUDE_TESTS AND WIN32)
+  add_definitions(/DMSFT_SUPPORTS_CHILD_PROCESSES)
+endif()
+
 # Put this before tblgen. Else we have a circular dependence.
 add_subdirectory(lib/Support)
 add_subdirectory(lib/MSSupport) # HLSL Change
@@ -683,7 +687,7 @@ if( LLVM_INCLUDE_EXAMPLES AND 0 ) # HLSL Change - never build examples
   add_subdirectory(examples)
 endif()
 
-if( LLVM_INCLUDE_TESTS AND 0 ) # HLSL Change - never build tests/unittests
+if( LLVM_INCLUDE_TESTS )
   add_subdirectory(test)
   add_subdirectory(unittests)
   if (MSVC)

include/llvm/Option/OptTable.h

@@ -170,7 +170,7 @@ public:
                  unsigned FlagsToInclude, unsigned FlagsToExclude) const;
 
   void PrintHelp(raw_ostream &OS, const char *Name, const char *Title,
-                 /* HLSL Change - version info */ const char *VersionInfo,
+                 /* HLSL Change - version info */ const char *VersionInfo = "",
                  bool ShowHidden = false) const;
 };
 } // end namespace opt

lib/Support/Windows/MSFileSystem.inc.cpp

@@ -307,6 +307,12 @@ namespace path {
     // Just use the caller's original path.
     return UTF8ToUTF16(Path8Str, Path16);
   }
+
+bool home_directory(SmallVectorImpl<char> &result) {
+  assert("HLSL Unimplemented!");
+  return false;
+}
+
 } // end namespace path
 
 namespace fs {
@@ -1040,6 +1046,15 @@ error_code openFileForWrite(const Twine &Name, int &ResultFD,
   return error_code();
 }
 
+std::error_code resize_file(int FD, uint64_t Size) {
+#ifdef HAVE__CHSIZE_S
+  errno_t error = ::_chsize_s(FD, Size);
+#else
+  errno_t error = ::_chsize(FD, Size);
+#endif
+  return std::error_code(error, std::generic_category());
+}
+
 } // end namespace fs
 
 namespace path {

test/CMakeLists.txt

@@ -109,6 +109,17 @@ add_lit_testsuites(LLVM ${CMAKE_CURRENT_SOURCE_DIR}
   DEPENDS ${LLVM_TEST_DEPENDS}
   )
 
+# HLSL Change Begin - When building IDE configurations, generate the unit test config
+if (CMAKE_CONFIGURATION_TYPES)
+add_lit_target(check-llvm-unit "Running lit suite llvm-unit"
+          ${CMAKE_CURRENT_BINARY_DIR}/Unit
+            PARAMS llvm_site_config=${CMAKE_CURRENT_BINARY_DIR}/lit.site.cfg
+                   llvm_unit_site_config=${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg
+         DEPENDS UnitTests
+       )
+endif()
+# HLSL Change End - Add a separate target for clang unit tests
+
 # Setup a legacy alias for 'check-llvm'. This will likely change to be an
 # alias for 'check-all' at some point in the future.
 add_custom_target(check)

tools/clang/CMakeLists.txt

@@ -499,7 +499,7 @@ if( CLANG_INCLUDE_TESTS OR HLSL_INCLUDE_TESTS ) # HLSL Change
       clang_unit_site_config=${CMAKE_CURRENT_BINARY_DIR}/test/Unit/lit.site.cfg
       )
   endif()
-#  add_subdirectory(test) # HLSL Change - disable pure .lit tests
+  add_subdirectory(test)
 
   if(CLANG_BUILT_STANDALONE)
     # Add a global check rule now that all subdirectories have been traversed

tools/clang/test/CMakeLists.txt

@@ -75,3 +75,14 @@ set_target_properties(check-clang PROPERTIES FOLDER "Clang tests")
 add_custom_target(clang-test)
 add_dependencies(clang-test check-clang)
 set_target_properties(clang-test PROPERTIES FOLDER "Clang tests")
+
+
+# HLSL Change Begin - Add a separate target for clang unit tests
+add_lit_target("check-clang-unit" "Running lit suite clang-unit"
+          ${CMAKE_CURRENT_SOURCE_DIR}/Unit
+          PARAMS ${CLANG_TEST_PARAMS}
+                 clang_unit_site_config=${CMAKE_CURRENT_BINARY_DIR}/Unit/lit.site.cfg
+          DEPENDS ClangUnitTests
+          ARGS ${CLANG_TEST_EXTRA_ARGS}
+        )
+# HLSL Change End - Add a separate target for clang unit tests

tools/clang/unittests/CMakeLists.txt

@@ -13,22 +13,22 @@ if ( CLANG_INCLUDE_TESTS ) # HLSL Change
 
 add_subdirectory(Basic)
 add_subdirectory(Lex)
-add_subdirectory(Driver)
+#add_subdirectory(Driver) # HLSL Disabled :(
 if(CLANG_ENABLE_STATIC_ANALYZER)
   add_subdirectory(StaticAnalyzer)
   add_subdirectory(Frontend)
 endif()
-add_subdirectory(ASTMatchers)
-add_subdirectory(AST)
-add_subdirectory(Tooling)
-add_subdirectory(Format)
+#add_subdirectory(ASTMatchers) # HLSL Disabled :(
+#add_subdirectory(AST) # HLSL Disabled :(
+#add_subdirectory(Tooling) # HLSL Disabled :(
+#add_subdirectory(Format) # HLSL Disabled :(
 add_subdirectory(Rewrite)
-add_subdirectory(Sema)
+#add_subdirectory(Sema) # HLSL Disabled :(
 add_subdirectory(CodeGen)
 # FIXME: libclang unit tests are disabled on Windows due
 # to failures, mostly in libclang.VirtualFileOverlay_*.
 if(NOT WIN32)
-  add_subdirectory(libclang)
+  #add_subdirectory(libclang) # HLSL Disabled :(
 endif()
 
 endif (CLANG_INCLUDE_TESTS) # HLSL Change

tools/clang/unittests/CodeGen/BufferSourceTest.cpp

@@ -38,12 +38,12 @@ const char TestProgram[] =
     "};                             "
     "EmitCXXGlobalInitFunc test;    ";
 
-TEST(BufferSourceTest, EmitCXXGlobalInitFunc) {
+TEST(BufferSourceTest, DISABLED_EmitCXXGlobalInitFunc) { // HLSL - Disabled
     CompilerInstance compiler;
 
     compiler.createDiagnostics();
-    compiler.getLangOpts().CPlusPlus = 1;
-    compiler.getLangOpts().CPlusPlus11 = 1;
+    //compiler.getLangOpts().CPlusPlus = 1;   // HLSL Change - hardcoded
+    //compiler.getLangOpts().CPlusPlus11 = 1; // HLSL Change - hardcoded
 
     compiler.getTargetOpts().Triple = llvm::Triple::normalize(
         llvm::sys::getProcessTriple());

tools/clang/unittests/Lex/PPCallbacksTest.cpp

@@ -190,7 +190,7 @@ protected:
   PragmaOpenCLExtensionCallbacks::CallbackParameters 
   PragmaOpenCLExtensionCall(const char* SourceText) {
     LangOptions OpenCLLangOpts;
-    OpenCLLangOpts.OpenCL = 1;
+    // OpenCLLangOpts.OpenCL = 1; HLSL Change - This is hard set
 
     std::unique_ptr<llvm::MemoryBuffer> SourceBuf =
         llvm::MemoryBuffer::getMemBuffer(SourceText, "test.cl");
@@ -322,7 +322,7 @@ TEST_F(PPCallbacksTest, TrigraphInMacro) {
   ASSERT_EQ("\"tri\?\?-graph.h\"", GetSourceString(Range));
 }
 
-TEST_F(PPCallbacksTest, OpenCLExtensionPragmaEnabled) {
+TEST_F(PPCallbacksTest, DISABLED_OpenCLExtensionPragmaEnabled) { // HLSL Change
   const char* Source =
     "#pragma OPENCL EXTENSION cl_khr_fp64 : enable\n";
 
@@ -334,7 +334,7 @@ TEST_F(PPCallbacksTest, OpenCLExtensionPragmaEnabled) {
   ASSERT_EQ(ExpectedState, Parameters.State);
 }
 
-TEST_F(PPCallbacksTest, OpenCLExtensionPragmaDisabled) {
+TEST_F(PPCallbacksTest, DISABLED_OpenCLExtensionPragmaDisabled) { // HLSL Change
   const char* Source =
     "#pragma OPENCL EXTENSION cl_khr_fp16 : disable\n";
 

unittests/Analysis/CMakeLists.txt

@@ -1,4 +1,5 @@
 set(LLVM_LINK_COMPONENTS
+  DXIL # HLSL Change
   IPA
   Analysis
   AsmParser

unittests/Analysis/CallGraphTest.cpp

@@ -24,11 +24,12 @@ template <typename Ty> void canSpecializeGraphTraitsIterators(Ty *G) {
   auto X = ++I;
 
   // Should be able to iterate over all nodes of the graph.
-  static_assert(std::is_same<decltype(*I), NodeTy &>::value,
+  // HLSL - Changed iterators to unique_tr, requiring extra deference
+  static_assert(std::is_same<decltype(**I), NodeTy &>::value,
                 "Node type does not match");
-  static_assert(std::is_same<decltype(*X), NodeTy &>::value,
+  static_assert(std::is_same<decltype(**X), NodeTy &>::value,
                 "Node type does not match");
-  static_assert(std::is_same<decltype(*E), NodeTy &>::value,
+  static_assert(std::is_same<decltype(**E), NodeTy &>::value,
                 "Node type does not match");
 
   NodeTy *N = GraphTraits<Ty *>::getEntryNode(G);

unittests/CMakeLists.txt

@@ -15,13 +15,13 @@ add_subdirectory(ADT)
 add_subdirectory(Analysis)
 add_subdirectory(AsmParser)
 add_subdirectory(Bitcode)
-add_subdirectory(CodeGen)
-add_subdirectory(DebugInfo)
+# add_subdirectory(CodeGen) - HLSL doesn't codegen...
+# add_subdirectory(DebugInfo) - HLSL doesn't generate dwarf
 # add_subdirectory(ExecutionEngine) - HLSL Change - removed
 add_subdirectory(IR)
-add_subdirectory(LineEditor)
+# add_subdirectory(LineEditor) - HLSL Change - removed
 add_subdirectory(Linker)
-add_subdirectory(MC)
+# add_subdirectory(MC) - HLSL doesn't codegen...
 add_subdirectory(Option)
 add_subdirectory(ProfileData)
 add_subdirectory(Support)

unittests/IR/CMakeLists.txt

@@ -2,6 +2,7 @@ set(LLVM_LINK_COMPONENTS
   Analysis
   AsmParser
   Core
+  DXIL  # HLSL Change
   IPA
   Support
   )

unittests/Support/CommandLineTest.cpp

@@ -86,8 +86,11 @@ TEST(CommandLineTest, ModifyExisitingOption) {
   cl::Option *Retrieved = Map["test-option"];
   ASSERT_EQ(&TestOption, Retrieved) << "Retrieved wrong option.";
 
-  ASSERT_EQ(&cl::GeneralCategory,Retrieved->Category) <<
+  // HLSL - Change begin
+  // No context here but HLSL changed GeneralCategory to a pointer.
+  ASSERT_EQ(cl::GeneralCategory,Retrieved->Category) << 
     "Incorrect default option category.";
+  // HLSL - Change end
 
   Retrieved->setCategory(TestCategory);
   ASSERT_EQ(&TestCategory,Retrieved->Category) <<
@@ -224,7 +227,7 @@ TEST(CommandLineTest, AliasRequired) {
   testAliasRequired(array_lengthof(opts1), opts1);
   testAliasRequired(array_lengthof(opts2), opts2);
 }
-
+/* HLSL Change begin
 TEST(CommandLineTest, HideUnrelatedOptions) {
   StackOption<int> TestOption1("hide-option-1");
   StackOption<int> TestOption2("hide-option-2", cl::cat(TestCategory));
@@ -264,5 +267,6 @@ TEST(CommandLineTest, HideUnrelatedOptionsMulti) {
   ASSERT_EQ(cl::NotHidden, Map["help"]->getOptionHiddenFlag())
       << "Hid default option that should be visable.";
 }
+HLSL Change End */
 
 }  // anonymous namespace

unittests/Support/MathExtrasTest.cpp

@@ -163,7 +163,8 @@ TEST(MathExtras, FloatBits) {
 
 TEST(MathExtras, DoubleBits) {
   static const double kValue = 87987234.983498;
-  EXPECT_FLOAT_EQ(kValue, BitsToDouble(DoubleToBits(kValue)));
+  // HLSL Change -fix implicit cast
+  EXPECT_DOUBLE_EQ(kValue, BitsToDouble(DoubleToBits(kValue)));
 }
 
 TEST(MathExtras, MinAlign) {

unittests/Support/raw_ostream_test.cpp

@@ -71,7 +71,8 @@ TEST(raw_ostreamTest, Types_Buffered) {
   // void*
   EXPECT_EQ("0x0", printToString((void*) nullptr));
   EXPECT_EQ("0xbeef", printToString((void*) 0xbeef));
-  EXPECT_EQ("0xdeadbeef", printToString((void*) 0xdeadbeef));
+  // HLSL Change - fix conversion
+  EXPECT_EQ("0xdeadbeef", printToString((void*) 0xdeadbeefull));
 
   // Min and max.
   EXPECT_EQ("18446744073709551615", printToString(UINT64_MAX));
@@ -102,7 +103,8 @@ TEST(raw_ostreamTest, Types_Unbuffered) {
   // void*
   EXPECT_EQ("0x0", printToStringUnbuffered((void*) nullptr));
   EXPECT_EQ("0xbeef", printToStringUnbuffered((void*) 0xbeef));
-  EXPECT_EQ("0xdeadbeef", printToStringUnbuffered((void*) 0xdeadbeef));
+  // HLSL Change - fix conversion
+  EXPECT_EQ("0xdeadbeef", printToStringUnbuffered((void*) 0xdeadbeefull));
 
   // Min and max.
   EXPECT_EQ("18446744073709551615", printToStringUnbuffered(UINT64_MAX));

unittests/Transforms/IPO/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   Core
+  DXIL   # HLSL Change
   Support
   IPO
   )

unittests/Transforms/Utils/CMakeLists.txt

@@ -1,5 +1,6 @@
 set(LLVM_LINK_COMPONENTS
   Core
+  DXIL # HLSL Change
   Support
   TransformUtils
   )

utils/hct/hctbuild.cmd

@@ -186,6 +186,11 @@ if "%1"=="-show-cmake-log" (
   set SHOW_CMAKE_LOG=1
   shift /1 & goto :parse_args
 )  
+if "%1"=="-enable-lit" (
+  echo Enable LIT testing
+  set ENABLE_LIT=On
+  shift /1 & goto :parse_args
+)
 rem Begin SPIRV change
 if "%1"=="-spirv" (
   echo SPIR-V codegen is enabled.
@@ -283,6 +288,10 @@ if "%DXC_CMAKE_SYSTEM_VERSION%"=="" (
   set DXC_CMAKE_SYSTEM_VERSION=10.0.17763.0
 )
 
+if "%ENABLE_LIT%"=="" (
+  set ENABLE_LIT=Off
+)
+
 set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_OPTIONAL_PROJS_IN_DEFAULT:BOOL=%ALL_DEFS%
 set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_ENABLE_ANALYZE:BOOL=%ANALYZE%
 set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_OFFICIAL_BUILD:BOOL=%OFFICIAL%
@@ -294,7 +303,7 @@ set CMAKE_OPTS=%CMAKE_OPTS% -DENABLE_SPIRV_CODEGEN:BOOL=%SPIRV%
 set CMAKE_OPTS=%CMAKE_OPTS% -DSPIRV_BUILD_TESTS:BOOL=%SPV_TEST%
 set CMAKE_OPTS=%CMAKE_OPTS% -DCLANG_ENABLE_ARCMT:BOOL=OFF
 set CMAKE_OPTS=%CMAKE_OPTS% -DCLANG_ENABLE_STATIC_ANALYZER:BOOL=OFF
-set CMAKE_OPTS=%CMAKE_OPTS% -DCLANG_INCLUDE_TESTS:BOOL=OFF -DLLVM_INCLUDE_TESTS:BOOL=OFF
+set CMAKE_OPTS=%CMAKE_OPTS% -DCLANG_INCLUDE_TESTS:BOOL=%ENABLE_LIT% -DLLVM_INCLUDE_TESTS:BOOL=%ENABLE_LIT%
 set CMAKE_OPTS=%CMAKE_OPTS% -DHLSL_INCLUDE_TESTS:BOOL=ON
 set CMAKE_OPTS=%CMAKE_OPTS% -DLLVM_TARGETS_TO_BUILD:STRING=None
 set CMAKE_OPTS=%CMAKE_OPTS% -DLLVM_INCLUDE_DOCS:BOOL=OFF -DLLVM_INCLUDE_EXAMPLES:BOOL=OFF