Added new DxilPDBInfo definition (#4669)

- Added new PDBI part for PDBs to eventually replace SRCI
- Added new IDxcPdbUtils2 that can read PDBI
- Removed all functionalities to do with GetFullPDB (and its tests)

include/dxc/DxilContainer/DxilContainer.h

@@ -96,6 +96,7 @@ enum DxilFourCC {
   DFCC_RuntimeData              = DXIL_FOURCC('R', 'D', 'A', 'T'),
   DFCC_ShaderHash               = DXIL_FOURCC('H', 'A', 'S', 'H'),
   DFCC_ShaderSourceInfo         = DXIL_FOURCC('S', 'R', 'C', 'I'),
+  DFCC_ShaderPDBInfo            = DXIL_FOURCC('P', 'D', 'B', 'I'),
   DFCC_CompilerVersion          = DXIL_FOURCC('V', 'E', 'R', 'S'),
 };
 
@@ -380,6 +381,32 @@ struct DxilSourceInfo_SourceContentsEntry {
 
 #pragma pack(pop)
 
+enum class DxilShaderPDBInfoVersion : uint16_t {
+  Version_0 = 0, // At this point, the data is still subject to change.
+  LatestPlus1,
+  Latest = LatestPlus1-1
+};
+
+enum class DxilShaderPDBInfoCompressionType : uint16_t {
+  Uncompressed,
+  Zlib,
+};
+
+// Header for generic PDB info. It's only found in the shader PDB and contains
+// all the information about shader's creation, such as compilation args,
+// shader sources, shader libraries, etc.
+//
+// This data part replaces DxilSourceInfo completely. Instead of using a custom
+// format, it uses the existing RDAT shader reflection format. See
+// include\dxc\DxilContainer\RDAT_PdbInfoTypes.inl for more information.
+//
+struct DxilShaderPDBInfo {
+  DxilShaderPDBInfoVersion Version;
+  DxilShaderPDBInfoCompressionType CompressionType; 
+  uint32_t SizeInBytes;
+  uint32_t UncompressedSizeInBytes;
+};
+
 /// Gets a part header by index.
 inline const DxilPartHeader *
 GetDxilContainerPart(const DxilContainerHeader *pHeader, uint32_t index) {

include/dxc/DxilContainer/DxilRuntimeReflection.h

@@ -39,6 +39,15 @@ enum RuntimeDataVersion {
   RDAT_Version_10 = 0x10,
 };
 
+enum class RuntimeDataGroup : uint32_t {
+  Core    = 0,
+  PdbInfo = 1,
+};
+
+constexpr uint32_t RDAT_PART_ID_WITH_GROUP(RuntimeDataGroup group, uint32_t id) {
+  return (((uint32_t)(group) << 16) | ((id) & 0xFFFF));
+}
+
 enum class RuntimeDataPartType : uint32_t {
   Invalid             = 0,
   StringBuffer        = 1,
@@ -49,8 +58,13 @@ enum class RuntimeDataPartType : uint32_t {
   RawBytes            = 5,
   SubobjectTable      = 6,
   Last_1_4 = SubobjectTable,
+
   LastPlus1,
   LastExperimental = LastPlus1 - 1,
+
+  DxilPdbInfoTable        = RDAT_PART_ID_WITH_GROUP(RuntimeDataGroup::PdbInfo, 1),
+  DxilPdbInfoSourceTable  = RDAT_PART_ID_WITH_GROUP(RuntimeDataGroup::PdbInfo, 2),
+  DxilPdbInfoLibraryTable = RDAT_PART_ID_WITH_GROUP(RuntimeDataGroup::PdbInfo, 3),
 };
 
 inline
@@ -68,6 +82,11 @@ enum class RecordTableIndex : unsigned {
   ResourceTable,
   FunctionTable,
   SubobjectTable,
+
+  DxilPdbInfoTable,
+  DxilPdbInfoSourceTable,
+  DxilPdbInfoLibraryTable,
+
   RecordTableCount
 };
 

include/dxc/DxilContainer/RDAT_Macros.inl

@@ -357,6 +357,7 @@
 
 #include "RDAT_SubobjectTypes.inl"
 #include "RDAT_LibraryTypes.inl"
+#include "RDAT_PdbInfoTypes.inl"
 
 #undef DEF_RDAT_TYPES
 #undef DEF_RDAT_ENUMS

include/dxc/DxilContainer/RDAT_PdbInfoTypes.inl

@@ -0,0 +1,43 @@
+///////////////////////////////////////////////////////////////////////////////
+//                                                                           //
+// RDAT_PdbInfoTypes.inl                                                     //
+// Copyright (C) Microsoft Corporation. All rights reserved.                 //
+// This file is distributed under the University of Illinois Open Source     //
+// License. See LICENSE.TXT for details.                                     //
+//                                                                           //
+// Defines macros use to define types Dxil PDBInfo data.                     //
+//                                                                           //
+///////////////////////////////////////////////////////////////////////////////
+
+#ifdef DEF_RDAT_TYPES
+
+#define RECORD_TYPE DxilPdbInfoLibrary
+RDAT_STRUCT_TABLE(DxilPdbInfoLibrary, DxilPdbInfoLibraryTable)
+  RDAT_STRING(Name)
+  RDAT_BYTES(Data)
+RDAT_STRUCT_END()
+#undef RECORD_TYPE
+
+#define RECORD_TYPE DxilPdbInfoSource
+RDAT_STRUCT_TABLE(DxilPdbInfoSource, DxilPdbInfoSourceTable)
+  RDAT_STRING(Name)
+  RDAT_STRING(Content)
+RDAT_STRUCT_END()
+#undef RECORD_TYPE
+
+#define RECORD_TYPE DxilPdbInfo
+RDAT_STRUCT_TABLE(DxilPdbInfo, DxilPdbInfoTable)
+  RDAT_RECORD_ARRAY_REF(DxilPdbInfoSource, Sources)
+  RDAT_RECORD_ARRAY_REF(DxilPdbInfoLibrary, Libraries)
+  RDAT_STRING_ARRAY_REF(ArgPairs)
+  RDAT_BYTES(Hash)
+  RDAT_STRING(PdbName)
+  RDAT_VALUE(uint32_t, CustomToolchainId)
+  RDAT_BYTES(CustomToolchainData)
+  RDAT_BYTES(WholeDxil)
+RDAT_STRUCT_END()
+#undef RECORD_TYPE
+
+#endif // DEF_RDAT_TYPES
+
+

include/dxc/DxilPdbInfo/DxilPdbInfoWriter.h

@@ -0,0 +1,24 @@
+///////////////////////////////////////////////////////////////////////////////
+//                                                                           //
+// DxilPdbInfoWriter.h                                                       //
+// Copyright (C) Microsoft Corporation. All rights reserved.                 //
+// This file is distributed under the University of Illinois Open Source     //
+// License. See LICENSE.TXT for details.                                     //
+//                                                                           //
+///////////////////////////////////////////////////////////////////////////////
+//
+// Helper for writing a valid hlsl::DxilShaderPDBInfo part
+//
+#pragma once
+
+#include "dxc/Support/Global.h"
+#include <vector>
+
+struct IMalloc;
+
+namespace hlsl {
+
+HRESULT WritePdbInfoPart(IMalloc *pMalloc, const void *pUncompressedPdbInfoData, size_t size, std::vector<char> *outBuffer);
+
+}
+

include/dxc/dxcapi.h

@@ -669,6 +669,47 @@ struct IDxcPdbUtils : public IUnknown {
   virtual HRESULT STDMETHODCALLTYPE OverrideRootSignature(_In_ const WCHAR *pRootSignature) = 0;
 };
 
+CROSS_PLATFORM_UUIDOF(IDxcPdbUtils2, "4315D938-F369-4F93-95A2-252017CC3807")
+struct IDxcPdbUtils2 : public IUnknown {
+  virtual HRESULT STDMETHODCALLTYPE Load(_In_ IDxcBlob *pPdbOrDxil) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetSourceCount(_Out_ UINT32 *pCount) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetSource(_In_ UINT32 uIndex, _COM_Outptr_ IDxcBlobEncoding **ppResult) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetSourceName(_In_ UINT32 uIndex, _COM_Outptr_ IDxcBlobWide **ppResult) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetLibraryPDBCount(UINT32 *pCount) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetLibraryPDB(_In_ UINT32 uIndex, _COM_Outptr_ IDxcPdbUtils2 **ppOutPdbUtils, _COM_Outptr_opt_result_maybenull_ IDxcBlobWide **ppLibraryName) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetFlagCount(_Out_ UINT32 *pCount) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetFlag(_In_ UINT32 uIndex, _COM_Outptr_ IDxcBlobWide **ppResult) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetArgCount(_Out_ UINT32 *pCount) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetArg(_In_ UINT32 uIndex, _COM_Outptr_ IDxcBlobWide **ppResult) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetArgPairCount(_Out_ UINT32 *pCount) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetArgPair(_In_ UINT32 uIndex, _COM_Outptr_result_maybenull_ IDxcBlobWide **ppName, _COM_Outptr_result_maybenull_ IDxcBlobWide **ppValue) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetDefineCount(_Out_ UINT32 *pCount) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetDefine(_In_ UINT32 uIndex, _COM_Outptr_ IDxcBlobWide **ppResult) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetTargetProfile(_COM_Outptr_result_maybenull_ IDxcBlobWide **ppResult) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetEntryPoint(_COM_Outptr_result_maybenull_ IDxcBlobWide **ppResult) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetMainFileName(_COM_Outptr_result_maybenull_ IDxcBlobWide **ppResult) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetHash(_COM_Outptr_result_maybenull_ IDxcBlob **ppResult) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetName(_COM_Outptr_result_maybenull_ IDxcBlobWide **ppResult) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetVersionInfo(_COM_Outptr_result_maybenull_ IDxcVersionInfo **ppVersionInfo) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetCustomToolchainID(_Out_ UINT32 *pID) = 0;
+  virtual HRESULT STDMETHODCALLTYPE GetCustomToolchainData(_COM_Outptr_result_maybenull_ IDxcBlob **ppBlob) = 0;
+
+  virtual HRESULT STDMETHODCALLTYPE GetWholeDxil(_COM_Outptr_result_maybenull_ IDxcBlob **ppResult) = 0;
+
+  virtual BOOL STDMETHODCALLTYPE IsFullPDB() = 0;
+  virtual BOOL STDMETHODCALLTYPE IsPDBRef() = 0;
+};
+
 // Note: __declspec(selectany) requires 'extern'
 // On Linux __declspec(selectany) is removed and using 'extern' results in link error.
 #ifdef _MSC_VER

lib/CMakeLists.txt

@@ -26,6 +26,7 @@ add_subdirectory(HLSL) # HLSL Change
 add_subdirectory(DXIL) # HLSL Change
 add_subdirectory(DxilContainer) # HLSL Change
 add_subdirectory(DxilRDATBuilder) # HLSL Change
+add_subdirectory(DxilPdbInfo) # HLSL Change
 add_subdirectory(DxilPIXPasses) # HLSL Change
 if(WIN32) # HLSL Change
   add_subdirectory(DxilDia) # HLSL Change

lib/DxilPdbInfo/CMakeLists.txt

@@ -0,0 +1,9 @@
+
+add_llvm_library(LLVMDxilPdbInfo
+  DxilPdbInfoWriter.cpp
+
+  ADDITIONAL_HEADER_DIRS
+)
+
+add_dependencies(LLVMDxilPdbInfo intrinsics_gen)
+

lib/DxilPdbInfo/DxilPdbInfoWriter.cpp

@@ -0,0 +1,39 @@
+#include "dxc/DxilPdbInfo/DxilPdbInfoWriter.h"
+
+#include "dxc/Support/WinIncludes.h"
+#include "dxc/Support/Global.h"
+
+#include "dxc/DxilContainer/DxilRuntimeReflection.h"
+#include "dxc/DxilContainer/DxilContainer.h"
+#include "dxc/DxilCompression/DxilCompressionHelpers.h"
+
+using namespace hlsl;
+
+HRESULT hlsl::WritePdbInfoPart(IMalloc *pMalloc, const void *pUncompressedPdbInfoData, size_t size, std::vector<char> *outBuffer) {
+  // Write to the output buffer.
+  outBuffer->clear();
+
+  hlsl::DxilShaderPDBInfo header = {};
+  header.CompressionType = hlsl::DxilShaderPDBInfoCompressionType::Zlib; // TODO: Add option to do uncompressed version.
+  header.UncompressedSizeInBytes = size;
+  header.Version = hlsl::DxilShaderPDBInfoVersion::Latest;
+  {
+    const size_t lastSize = outBuffer->size();
+    outBuffer->resize(outBuffer->size() + sizeof(header));
+    memcpy(outBuffer->data()+lastSize, &header, sizeof(header));
+  }
+
+  // Then write the compressed RDAT data.
+  hlsl::ZlibResult result = hlsl::ZlibCompressAppend(pMalloc, pUncompressedPdbInfoData, size, *outBuffer);
+
+  if (result == hlsl::ZlibResult::OutOfMemory)
+    IFTBOOL(false, E_OUTOFMEMORY);
+  IFTBOOL(result == hlsl::ZlibResult::Success, E_FAIL);
+
+  IFTBOOL(outBuffer->size() >= sizeof(header), E_FAIL);
+  header.SizeInBytes = outBuffer->size() - sizeof(header);
+  memcpy(outBuffer->data(), &header, sizeof(header));
+
+  return S_OK;
+}
+

lib/DxilPdbInfo/LLVMBuild.txt

@@ -0,0 +1,17 @@
+; Copyright (C) Microsoft Corporation. All rights reserved.
+; This file is distributed under the University of Illinois Open Source License. See LICENSE.TXT for details.
+;
+; This is an LLVMBuild description file for the components in this subdirectory.
+;
+; For more information on the LLVMBuild system, please see:
+;
+;   http://llvm.org/docs/LLVMBuild.html
+;
+;===------------------------------------------------------------------------===;
+
+[component_0]
+type = Library
+name = DxilPdbInfo
+parent = Libraries
+required_libraries = DxilRDATBuilder Core Support
+

lib/LLVMBuild.txt

@@ -42,6 +42,7 @@ subdirectories =
  DXIL
  DxilContainer
  DxilRDATBuilder
+ DxilPdbInfo
  DxilDia
  DxrFallback
  DxilRootSignature

tools/clang/unittests/HLSL/CompilerTest.cpp

@@ -1328,106 +1328,9 @@ static void VerifyPdbUtil(dxc::DxcDllSupport &dllSupport,
   // Make the pix debug info
   if (IsFullPDB) {
     VERIFY_IS_TRUE(pPdbUtils->IsFullPDB());
-    CComPtr<IDxcBlob> pPDBBlob;
-    VERIFY_SUCCEEDED(pPdbUtils->GetFullPDB(&pPDBBlob));
-
-    CComPtr<IDxcPixDxilDebugInfoFactory> pFactory;
-    VERIFY_SUCCEEDED(pPdbUtils->QueryInterface(&pFactory));
-    CComPtr<IDxcPixCompilationInfo> pCompInfo;
-    VERIFY_ARE_EQUAL(E_NOTIMPL, pFactory->NewDxcPixCompilationInfo(&pCompInfo));
-    CComPtr<IDxcPixDxilDebugInfo> pDebugInfo;
-    VERIFY_SUCCEEDED(pFactory->NewDxcPixDxilDebugInfo(&pDebugInfo));
-    VERIFY_ARE_NOT_EQUAL(pDebugInfo, nullptr);
-
-    // Recompile when it's a full PDB anyway.
-    {
-      CComPtr<IDxcResult> pResult;
-      VERIFY_SUCCEEDED(pPdbUtils->CompileForFullPDB(&pResult));
-
-      HRESULT compileStatus = S_OK;
-      VERIFY_SUCCEEDED(pResult->GetStatus(&compileStatus));
-      VERIFY_SUCCEEDED(compileStatus);
-
-      CComPtr<IDxcBlob> pRecompiledPdbBlob;
-      VERIFY_SUCCEEDED(pResult->GetOutput(DXC_OUT_PDB, IID_PPV_ARGS(&pRecompiledPdbBlob), nullptr));
-    }
-
   }
   else {
     VERIFY_IS_FALSE(pPdbUtils->IsFullPDB());
-    CComPtr<IDxcBlob> pFullPdb;
-    VERIFY_SUCCEEDED(pPdbUtils->GetFullPDB(&pFullPdb));
-
-    // Save a copy of the arg pairs
-    std::vector<std::pair< std::wstring, std::wstring> > pairsStorage;
-    UINT32 uNumArgsPairs = 0;
-    VERIFY_SUCCEEDED(pPdbUtils->GetArgPairCount(&uNumArgsPairs));
-    for (UINT32 i = 0; i < uNumArgsPairs; i++) {
-      CComBSTR pName, pValue;
-      VERIFY_SUCCEEDED(pPdbUtils->GetArgPair(i, &pName, &pValue));
-      std::pair< std::wstring, std::wstring> pairStorage;
-      pairStorage.first  = pName  ? pName  : L"";
-      pairStorage.second = pValue ? pValue : L"";
-      pairsStorage.push_back(pairStorage);
-    }
-
-    // Set an obviously wrong RS and verify compilation fails
-    {
-      VERIFY_SUCCEEDED(pPdbUtils->OverrideRootSignature(L""));
-      CComPtr<IDxcResult> pResult;
-      VERIFY_SUCCEEDED(pPdbUtils->CompileForFullPDB(&pResult));
-
-      HRESULT result = S_OK;
-      VERIFY_SUCCEEDED(pResult->GetStatus(&result));
-      VERIFY_FAILED(result);
-
-      CComPtr<IDxcBlobEncoding> pErr;
-      VERIFY_SUCCEEDED(pResult->GetErrorBuffer(&pErr));
-    }
-
-    // Set an obviously wrong set of args and verify compilation fails
-    {
-
-      std::vector<DxcArgPair> pairs;
-      for (auto &p : pairsStorage) {
-        DxcArgPair pair = {};
-        pair.pName = p.first.c_str();
-        pair.pValue = p.second.c_str();
-        pairs.push_back(pair);
-      }
-
-      VERIFY_SUCCEEDED(pPdbUtils->OverrideArgs(pairs.data(), pairs.size()));
-
-      CComPtr<IDxcResult> pResult;
-      VERIFY_SUCCEEDED(pPdbUtils->CompileForFullPDB(&pResult));
-
-      HRESULT result = S_OK;
-      VERIFY_SUCCEEDED(pResult->GetStatus(&result));
-      VERIFY_SUCCEEDED(result);
-    }
-
-    auto ReplaceDebugFlagPair = [](const std::vector<std::pair<const WCHAR *, const WCHAR *> > &List) -> std::vector<std::pair<const WCHAR *, const WCHAR *> > {
-      std::vector<std::pair<const WCHAR *, const WCHAR *> > ret;
-      for (unsigned i = 0; i < List.size(); i++) {
-        if (!wcscmp(List[i].first, L"/Zs") || !wcscmp(List[i].first, L"-Zs"))
-          ret.push_back(std::pair<const WCHAR *, const WCHAR *>(L"-Zi", nullptr));
-        else
-          ret.push_back(List[i]);
-      }
-      return ret;
-    };
-
-    auto NewExpectedFlags = ReplaceDebugFlagPair(ExpectedFlags);
-    auto NewExpectedArgs  = ReplaceDebugFlagPair(ExpectedArgs);
-
-    VerifyPdbUtil(dllSupport, pFullPdb, pPdbUtils,
-      pMainFileName,
-      NewExpectedArgs, NewExpectedFlags, ExpectedDefines,
-      pCompiler, HasVersion, /*IsFullPDB*/true,
-      /*TestReflection*/true,      
-      HasHashAndPdbName, 
-      /*TestEntryPoint*/false,
-      MainSource, IncludedFile);
   }
 
   // Now, test that dia interface doesn't crash (even if it fails).
@@ -1671,6 +1574,33 @@ void CompilerTest::TestPdbUtils(bool bSlim, bool bSourceInDebugModule, bool bStr
       /*TestEntryPoint*/bTestEntryPoint,
       main_source, included_File);
   }
+
+  {
+    CComPtr<IDxcPdbUtils2> pPdbUtils2;
+    VERIFY_SUCCEEDED(pPdbUtils.QueryInterface(&pPdbUtils2));
+    {
+      CComPtr<IDxcPdbUtils> pPdbUtils_Again;
+      VERIFY_SUCCEEDED(pPdbUtils2.QueryInterface(&pPdbUtils_Again));
+      {
+        CComPtr<IDxcPdbUtils2> pPdbUtils2_Again;
+        VERIFY_SUCCEEDED(pPdbUtils_Again.QueryInterface(&pPdbUtils2_Again));
+        VERIFY_ARE_EQUAL(pPdbUtils2_Again, pPdbUtils2);
+
+        VERIFY_ARE_EQUAL(pPdbUtils2.p->AddRef(), 5);
+        VERIFY_ARE_EQUAL(pPdbUtils2.p->Release(), 4);
+      }
+      VERIFY_ARE_EQUAL(pPdbUtils_Again, pPdbUtils);
+
+      VERIFY_ARE_EQUAL(pPdbUtils2.p->AddRef(), 4);
+      VERIFY_ARE_EQUAL(pPdbUtils2.p->Release(), 3);
+    }
+
+    VERIFY_ARE_EQUAL(pPdbUtils2.p->AddRef(), 3);
+    VERIFY_ARE_EQUAL(pPdbUtils2.p->Release(), 2);
+  }
+
+  VERIFY_ARE_EQUAL(pPdbUtils.p->AddRef(), 2);
+  VERIFY_ARE_EQUAL(pPdbUtils.p->Release(), 1);
 }
 
 TEST_F(CompilerTest, CompileThenTestReflectionWithProgramHeader) {
@@ -1845,11 +1775,6 @@ TEST_F(CompilerTest, CompileThenTestPdbUtilsWarningOpt) {
 
   VERIFY_SUCCEEDED(pPdbUtils->Load(pPdb));
   TestPdb(pPdbUtils);
-
-  CComPtr<IDxcBlob> pFullPdb;
-  VERIFY_SUCCEEDED(pPdbUtils->GetFullPDB(&pFullPdb));
-  VERIFY_SUCCEEDED(pPdbUtils->Load(pFullPdb));
-  TestPdb(pPdbUtils);
 }
 
 TEST_F(CompilerTest, CompileThenTestPdbInPrivate) {
@@ -1946,12 +1871,6 @@ TEST_F(CompilerTest, CompileThenTestPdbUtilsRelativePath) {
   VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcPdbUtils, &pPdbUtils));
 
   VERIFY_SUCCEEDED(pPdbUtils->Load(pPdb));
-
-  CComPtr<IDxcBlob> pFullPdb;
-  VERIFY_SUCCEEDED(pPdbUtils->GetFullPDB(&pFullPdb));
-
-  VERIFY_SUCCEEDED(pPdbUtils->Load(pFullPdb));
-  VERIFY_IS_TRUE(pPdbUtils->IsFullPDB());
 }