Merged PR 13: Support debug info for linking.

Support debug info for linking.

include/dxc/HLSL/DxilMetadataHelper.h

@@ -383,7 +383,7 @@ public:
                        float &MaxTessFactor);
 
   // Utility functions.
-  static bool IsKnownNamedMetaData(llvm::NamedMDNode &Node);
+  static bool IsKnownNamedMetaData(const llvm::NamedMDNode &Node);
   static void combineDxilMetadata(llvm::Instruction *K, const llvm::Instruction *J);
   static llvm::ConstantAsMetadata *Int32ToConstMD(int32_t v, llvm::LLVMContext &Ctx);
   llvm::ConstantAsMetadata *Int32ToConstMD(int32_t v);

lib/HLSL/DxilLinker.cpp

@@ -23,6 +23,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "llvm/ADT/StringMap.h"
+#include "llvm/ADT/SetVector.h"
 #include <memory>
 #include <vector>
 
@@ -323,6 +324,7 @@ struct DxilLinkJob {
   void AddFunction(llvm::Function *F);
 
 private:
+  void LinkNamedMDNodes(Module *pM, ValueToValueMapTy &vmap);
   void AddDxilOperations(Module *pM);
   bool AddGlobals(DxilModule &DM, ValueToValueMapTy &vmap);
   void CloneFunctions(ValueToValueMapTy &vmap);
@@ -497,6 +499,49 @@ void DxilLinkJob::AddResourceToDM(DxilModule &DM) {
   }
 }
 
+void DxilLinkJob::LinkNamedMDNodes(Module *pM, ValueToValueMapTy &vmap) {
+  SetVector<Module *> moduleSet;
+  for (auto &it : m_functionDefs) {
+    DxilLib *pLib = it.second;
+    moduleSet.insert(pLib->GetDxilModule().GetModule());
+  }
+  // Link normal NamedMDNode.
+  // TODO: skip duplicate operands.
+  for (Module *pSrcM : moduleSet) {
+    const NamedMDNode *pSrcModFlags = pSrcM->getModuleFlagsMetadata();
+    for (const NamedMDNode &NMD : pSrcM->named_metadata()) {
+      // Don't link module flags here. Do them separately.
+      if (&NMD == pSrcModFlags)
+        continue;
+      // Skip dxil metadata which will be regenerated.
+      if (DxilMDHelper::IsKnownNamedMetaData(NMD))
+        continue;
+      NamedMDNode *DestNMD = pM->getOrInsertNamedMetadata(NMD.getName());
+      // Add Src elements into Dest node.
+      for (const MDNode *op : NMD.operands())
+        DestNMD->addOperand(MapMetadata(op, vmap, RF_None, /*TypeMap*/ nullptr,
+                                        /*ValMaterializer*/ nullptr));
+    }
+  }
+  // Link mod flags.
+  SetVector<MDNode *> flagSet;
+  for (Module *pSrcM : moduleSet) {
+    NamedMDNode *pSrcModFlags = pSrcM->getModuleFlagsMetadata();
+    if (pSrcModFlags) {
+      for (MDNode *flag : pSrcModFlags->operands()) {
+        flagSet.insert(flag);
+      }
+    }
+  }
+  // TODO: check conflict in flags.
+  if (!flagSet.empty()) {
+    NamedMDNode *ModFlags = pM->getOrInsertModuleFlagsMetadata();
+    for (MDNode *flag : flagSet) {
+      ModFlags->addOperand(flag);
+    }
+  }
+}
+
 void DxilLinkJob::AddDxilOperations(Module *pM) {
   for (auto &it : m_dxilFunctions) {
     Function *F = it.second;
@@ -686,8 +731,6 @@ DxilLinkJob::Link(std::pair<DxilFunctionLinkInfo *, DxilLib *> &entryLinkPair,
   // Set EntryProps
   DM.SetShaderProperties(&props);
 
-  // Debug info.
-
   // Add global
   bool bSuccess = AddGlobals(DM, vmap);
   if (!bSuccess)
@@ -716,6 +759,9 @@ DxilLinkJob::Link(std::pair<DxilFunctionLinkInfo *, DxilLib *> &entryLinkPair,
   // Add resource to DM.
   // This should be after functions cloned.
   AddResourceToDM(DM);
+  
+  // Link metadata like debug info.
+  LinkNamedMDNodes(pM.get(), vmap);
 
   RunPreparePass(*pM);
 
@@ -775,8 +821,6 @@ DxilLinkJob::LinkToLib(const ShaderModel *pSM) {
   }
   DM.ResetEntrySignatureMap(std::move(DxilEntrySignatureMap));
 
-  // Debug info.
-
   // Add global
   bool bSuccess = AddGlobals(DM, vmap);
   if (!bSuccess)
@@ -792,6 +836,9 @@ DxilLinkJob::LinkToLib(const ShaderModel *pSM) {
   // This should be after functions cloned.
   AddResourceToDM(DM);
 
+  // Link metadata like debug info.
+  LinkNamedMDNodes(pM.get(), vmap);
+
   RunPreparePass(*pM);
 
   return pM;
@@ -978,7 +1025,7 @@ bool DxilLinkerImpl::AddFunctions(SmallVector<StringRef, 4> &workList,
       pLib->LazyLoadFunction(F);
     }
     for (Function *F : linkPair.first->usedFunctions) {
-      if (hlsl::OP::IsDxilOpFunc(F)) {
+      if (hlsl::OP::IsDxilOpFunc(F) || F->isIntrinsic()) {
         // Add dxil operations directly.
         linkJob.AddFunction(F);
       } else {
@@ -1047,13 +1094,13 @@ std::unique_ptr<llvm::Module> DxilLinkerImpl::Link(StringRef entry,
 
       addedFunctionSet.insert(name);
     }
-    // Add every dxil functions.
+    // Add every dxil functions and llvm intrinsic.
     for (auto *pLib : libSet) {
       auto &DM = pLib->GetDxilModule();
       DM.GetOP();
       auto *pM = DM.GetModule();
       for (Function &F : pM->functions()) {
-        if (hlsl::OP::IsDxilOpFunc(&F)) {
+        if (hlsl::OP::IsDxilOpFunc(&F) || F.isIntrinsic()) {
           linkJob.AddFunction(&F);
         }
       }

lib/HLSL/DxilMetadataHelper.cpp

@@ -1530,7 +1530,7 @@ void DxilExtraPropertyHelper::LoadSignatureElementProperties(const MDOperand &MD
 //
 // Utilities.
 //
-bool DxilMDHelper::IsKnownNamedMetaData(llvm::NamedMDNode &Node) {
+bool DxilMDHelper::IsKnownNamedMetaData(const llvm::NamedMDNode &Node) {
   StringRef name = Node.getName();
   for (unsigned i = 0; i < DxilMDNames.size(); i++) {
     if (name == DxilMDNames[i]) {

tools/clang/tools/dxcompiler/dxclinker.cpp

@@ -35,6 +35,7 @@
 #include "llvm/IR/Module.h"
 #include "llvm/Support/raw_ostream.h"
 #include "clang/Frontend/TextDiagnosticPrinter.h"
+#include "dxc/Support/HLSLOptions.h"
 
 using namespace hlsl;
 using namespace llvm;
@@ -166,13 +167,7 @@ HRESULT STDMETHODCALLTYPE DxcLinker::Link(
   // Prepare UTF8-encoded versions of API values.
   CW2A pUtf8TargetProfile(pTargetProfile, CP_UTF8);
   CW2A pUtf8EntryPoint(pEntryName, CP_UTF8);
-  // TODO: read and validate options.
-  bool bIsLib = StringRef(pUtf8TargetProfile.m_psz).startswith_lower("lib");
-  if (bIsLib) {
-    if (StringRef(pUtf8EntryPoint.m_psz).size() > 0) {
-      return E_INVALIDARG;
-    }
-  }
+
   CComPtr<AbstractMemoryStream> pOutputStream;
 
   // Detach previous libraries.
@@ -187,6 +182,22 @@ HRESULT STDMETHODCALLTYPE DxcLinker::Link(
     IFT(CoGetMalloc(1, &pMalloc));
     IFT(CreateMemoryStream(pMalloc, &pOutputStream));
 
+    // Read and validate options.
+    int argCountInt;
+    IFT(UIntToInt(argCount, &argCountInt));
+    hlsl::options::MainArgs mainArgs(argCountInt,
+                                     const_cast<LPCWSTR *>(pArguments), 0);
+    hlsl::options::DxcOpts opts;
+    CW2A pUtf8TargetProfile(pTargetProfile, CP_UTF8);
+    // Set target profile before reading options and validate
+    opts.TargetProfile = pUtf8TargetProfile.m_psz;
+    bool finished;
+    dxcutil::ReadOptsAndValidate(mainArgs, opts, pOutputStream, ppResult,
+                                 finished);
+    if (finished) {
+      return S_OK;
+    }
+
     std::string warnings;
     llvm::raw_string_ostream w(warnings);
     IFT(CreateMemoryStream(pMalloc, &pDiagStream));
@@ -222,17 +233,28 @@ HRESULT STDMETHODCALLTYPE DxcLinker::Link(
         WriteBitcodeToFile(pM.get(), outStream);
         outStream.flush();
 
+        // Always save debug info. If lib has debug info, the link result will
+        // have debug info.
+        SerializeDxilFlags SerializeFlags =
+            SerializeDxilFlags::IncludeDebugNamePart;
+        // Unless we want to strip it right away, include it in the container.
+        if (!opts.StripDebug) {
+          SerializeFlags |= SerializeDxilFlags::IncludeDebugInfoPart;
+        }
+        if (opts.DebugNameForSource) {
+          SerializeFlags |= SerializeDxilFlags::DebugNameDependOnSource;
+        }
         // Validation.
         HRESULT valHR = S_OK;
         // Skip validation on lib for now.
-        if (!bIsLib) {
+        if (!opts.TargetProfile.startswith("lib_")) {
           valHR = dxcutil::ValidateAndAssembleToContainer(
-              std::move(pM), pOutputBlob, pMalloc, SerializeDxilFlags::None,
+              std::move(pM), pOutputBlob, pMalloc, SerializeFlags,
               pOutputStream,
               /*bDebugInfo*/ false, Diag);
         } else {
           dxcutil::AssembleToContainer(std::move(pM), pOutputBlob, m_pMalloc,
-                                       SerializeDxilFlags::None, pOutputStream);
+                                       SerializeFlags, pOutputStream);
         }
         // Callback after valid DXIL is produced
         if (SUCCEEDED(valHR)) {

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -214,28 +214,6 @@ private:
     }
   }
 
-  void ReadOptsAndValidate(hlsl::options::MainArgs &mainArgs,
-                           hlsl::options::DxcOpts &opts,
-                           AbstractMemoryStream *pOutputStream,
-                           _COM_Outptr_ IDxcOperationResult **ppResult,
-                           bool &finished) {
-    const llvm::opt::OptTable *table = ::options::getHlslOptTable();
-    raw_stream_ostream outStream(pOutputStream);
-    if (0 != hlsl::options::ReadDxcOpts(table, hlsl::options::CompilerFlags,
-                                        mainArgs, opts, outStream)) {
-      CComPtr<IDxcBlob> pErrorBlob;
-      IFT(pOutputStream->QueryInterface(&pErrorBlob));
-      CComPtr<IDxcBlobEncoding> pErrorBlobWithEncoding;
-      outStream.flush();
-      IFT(DxcCreateBlobWithEncodingSet(pErrorBlob.p, CP_UTF8,
-                                       &pErrorBlobWithEncoding));
-      IFT(DxcOperationResult::CreateFromResultErrorStatus(nullptr, pErrorBlobWithEncoding.p, E_INVALIDARG, ppResult));
-      finished = true;
-      return;
-    }
-    DXASSERT(opts.HLSLVersion > 2015, "else ReadDxcOpts didn't fail for non-isense");
-    finished = false;
-  }
 public:
   DXC_MICROCOM_TM_ADDREF_RELEASE_IMPL()
   DXC_MICROCOM_TM_CTOR(DxcCompiler)
@@ -333,7 +311,7 @@ public:
       // Set target profile before reading options and validate
       opts.TargetProfile = pUtf8TargetProfile.m_psz;
       bool finished;
-      ReadOptsAndValidate(mainArgs, opts, pOutputStream, ppResult, finished);
+      dxcutil::ReadOptsAndValidate(mainArgs, opts, pOutputStream, ppResult, finished);
       if (finished) {
         hr = S_OK;
         goto Cleanup;
@@ -632,7 +610,7 @@ public:
       hlsl::options::MainArgs mainArgs(argCountInt, pArguments, 0);
       hlsl::options::DxcOpts opts;
       bool finished;
-      ReadOptsAndValidate(mainArgs, opts, pOutputStream, ppResult, finished);
+      dxcutil::ReadOptsAndValidate(mainArgs, opts, pOutputStream, ppResult, finished);
       if (finished) {
         hr = S_OK;
         goto Cleanup;

tools/clang/tools/dxcompiler/dxcutil.cpp

@@ -28,6 +28,7 @@
 #include "llvm/Support/raw_ostream.h"
 #include "llvm/Transforms/Utils/Cloning.h"
 #include "dxc/Support/dxcapi.impl.h"
+#include "dxc/Support/HLSLOptions.h"
 
 #include "llvm/Support/Path.h"
 
@@ -125,6 +126,31 @@ void AssembleToContainer(std::unique_ptr<llvm::Module> pM,
                                        SerializeFlags);
 }
 
+void ReadOptsAndValidate(hlsl::options::MainArgs &mainArgs,
+                         hlsl::options::DxcOpts &opts,
+                         AbstractMemoryStream *pOutputStream,
+                         _COM_Outptr_ IDxcOperationResult **ppResult,
+                         bool &finished) {
+  const llvm::opt::OptTable *table = ::options::getHlslOptTable();
+  raw_stream_ostream outStream(pOutputStream);
+  if (0 != hlsl::options::ReadDxcOpts(table, hlsl::options::CompilerFlags,
+                                      mainArgs, opts, outStream)) {
+    CComPtr<IDxcBlob> pErrorBlob;
+    IFT(pOutputStream->QueryInterface(&pErrorBlob));
+    CComPtr<IDxcBlobEncoding> pErrorBlobWithEncoding;
+    outStream.flush();
+    IFT(DxcCreateBlobWithEncodingSet(pErrorBlob.p, CP_UTF8,
+                                     &pErrorBlobWithEncoding));
+    IFT(DxcOperationResult::CreateFromResultErrorStatus(
+        nullptr, pErrorBlobWithEncoding.p, E_INVALIDARG, ppResult));
+    finished = true;
+    return;
+  }
+  DXASSERT(opts.HLSLVersion > 2015,
+           "else ReadDxcOpts didn't fail for non-isense");
+  finished = false;
+}
+
 HRESULT ValidateAndAssembleToContainer(
     std::unique_ptr<llvm::Module> pM, CComPtr<IDxcBlob> &pOutputBlob,
     IMalloc *pMalloc, SerializeDxilFlags SerializeFlags,

tools/clang/tools/dxcompiler/dxcutil.h

@@ -31,8 +31,11 @@ class Twine;
 namespace hlsl {
 enum class SerializeDxilFlags : uint32_t;
 class AbstractMemoryStream;
-}
-
+namespace options {
+class MainArgs;
+class DxcOpts;
+} // namespace options
+} // namespace hlsl
 
 namespace dxcutil {
 HRESULT ValidateAndAssembleToContainer(
@@ -47,7 +50,11 @@ void AssembleToContainer(std::unique_ptr<llvm::Module> pM,
                          hlsl::SerializeDxilFlags SerializeFlags,
                          CComPtr<hlsl::AbstractMemoryStream> &pModuleBitcode);
 HRESULT Disassemble(IDxcBlob *pProgram, llvm::raw_string_ostream &Stream);
-
+void ReadOptsAndValidate(hlsl::options::MainArgs &mainArgs,
+                         hlsl::options::DxcOpts &opts,
+                         hlsl::AbstractMemoryStream *pOutputStream,
+                         _COM_Outptr_ IDxcOperationResult **ppResult,
+                         bool &finished);
 void CreateOperationResultFromOutputs(
     IDxcBlob *pResultBlob, CComPtr<IStream> &pErrorStream,
     const std::string &warnings, bool hasErrorOccurred,

tools/clang/unittests/HLSL/LinkerTest.cpp

@@ -60,7 +60,8 @@ public:
         m_dllSupport.CreateInstance(CLSID_DxcLinker, pResultLinker));
   }
 
-  void CompileLib(LPCWSTR filename, IDxcBlob **pResultBlob) {
+  void CompileLib(LPCWSTR filename, IDxcBlob **pResultBlob, LPCWSTR *pArguments,
+                  UINT32 argCount) {
     std::wstring fullPath = hlsl_test::GetPathToHlslDataFile(filename);
     CComPtr<IDxcBlobEncoding> pSource;
     CComPtr<IDxcLibrary> pLibrary;
@@ -77,11 +78,15 @@ public:
     VERIFY_SUCCEEDED(
         m_dllSupport.CreateInstance(CLSID_DxcCompiler, &pCompiler));
     VERIFY_SUCCEEDED(pCompiler->Compile(pSource, L"hlsl.hlsl", L"", shWide,
-                                        nullptr, 0, nullptr, 0, nullptr,
-                                        &pResult));
+                                        pArguments, argCount, nullptr, 0,
+                                        nullptr, &pResult));
     VERIFY_SUCCEEDED(pResult->GetResult(pResultBlob));
   }
 
+  void CompileLib(LPCWSTR filename, IDxcBlob **pResourceBlob) {
+    CompileLib(filename, pResourceBlob, nullptr, 0);
+  }
+
   void RegisterDxcModule(LPCWSTR pLibName, IDxcBlob *pBlob,
                          IDxcLinker *pLinker) {
     VERIFY_SUCCEEDED(pLinker->RegisterLibrary(pLibName, pBlob));
@@ -301,10 +306,15 @@ TEST_F(LinkerTest, RunLinkResRet) {
 }
 
 TEST_F(LinkerTest, RunLinkToLib) {
+  LPCWSTR option[] = {L"-Zi"};
+
   CComPtr<IDxcBlob> pEntryLib;
-  CompileLib(L"..\\CodeGenHLSL\\shader-compat-suite\\lib_out_param_res.hlsl", &pEntryLib);
+  CompileLib(L"..\\CodeGenHLSL\\shader-compat-suite\\lib_out_param_res.hlsl",
+             &pEntryLib, option, 1);
   CComPtr<IDxcBlob> pLib;
-  CompileLib(L"..\\CodeGenHLSL\\shader-compat-suite\\lib_out_param_res_imp.hlsl", &pLib);
+  CompileLib(
+      L"..\\CodeGenHLSL\\shader-compat-suite\\lib_out_param_res_imp.hlsl",
+      &pLib, option, 1);
 
   CComPtr<IDxcLinker> pLinker;
   CreateLinker(&pLinker);
@@ -315,7 +325,7 @@ TEST_F(LinkerTest, RunLinkToLib) {
   LPCWSTR libName2 = L"test";
   RegisterDxcModule(libName2, pLib, pLinker);
 
-  Link(L"", L"lib_6_2", pLinker, {libName, libName2}, {}, {"alloca"});
+  Link(L"", L"lib_6_2", pLinker, {libName, libName2}, {"!llvm.dbg.cu"}, {});
 }
 
 TEST_F(LinkerTest, RunLinkFailSelectRes) {