Putting debug info in PDB container (#2215)

include/dxc/DXIL/DxilPDB.h

@@ -0,0 +1,24 @@
+///////////////////////////////////////////////////////////////////////////////
+//                                                                           //
+// DxilPDB.h                                                                 //
+// Copyright (C) Microsoft Corporation. All rights reserved.                 //
+// This file is distributed under the University of Illinois Open Source     //
+// License. See LICENSE.TXT for details.                                     //
+//                                                                           //
+// Helpers to wrap debug information in a PDB container.                     //
+//                                                                           //
+///////////////////////////////////////////////////////////////////////////////
+
+#include "dxc/Support/WinIncludes.h"
+
+struct IDxcBlob;
+struct IStream;
+struct IMalloc;
+
+namespace hlsl {
+namespace pdb {
+
+  HRESULT LoadDataFromStream(IMalloc *pMalloc, IStream *pIStream, IDxcBlob **pOutContainer);
+  HRESULT WriteDxilPDB(IMalloc *pMalloc, IDxcBlob *pContainer, IDxcBlob **ppOutBlob); 
+}
+}

lib/DXIL/CMakeLists.txt

@@ -18,6 +18,7 @@ add_llvm_library(LLVMDXIL
   DxilSubobject.cpp
   DxilTypeSystem.cpp
   DxilUtil.cpp
+  DxilPDB.cpp
 
   ADDITIONAL_HEADER_DIRS
   ${LLVM_MAIN_INCLUDE_DIR}/llvm/IR

lib/DXIL/DxilPDB.cpp

@@ -0,0 +1,497 @@
+///////////////////////////////////////////////////////////////////////////////
+//                                                                           //
+// DxilPDB.cpp                                                               //
+// Copyright (C) Microsoft Corporation. All rights reserved.                 //
+// This file is distributed under the University of Illinois Open Source     //
+// License. See LICENSE.TXT for details.                                     //
+//                                                                           //
+// Helpers to wrap debug information in a PDB container.                     //
+//                                                                           //
+///////////////////////////////////////////////////////////////////////////////
+
+//
+// This file contains code that helps creating our special PDB format. PDB
+// format contains streams at fixed locations. Outside of those fixed
+// locations, unless they are listed in the stream hash table, there is be no
+// way to know what the stream is. As far as normal PDB's are concerned, they
+// dont' really exist.
+// 
+// For our purposes, we always put our data in one stream at a fixed index
+// defined below. The data is an ordinary DXIL container format, with parts
+// that are relevant for debugging.
+//
+
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/ADT/ArrayRef.h"
+#include "llvm/ADT/SmallString.h"
+#include "llvm/Support/Endian.h"
+
+#include "dxc/DXIL/DxilPDB.h"
+#include "dxc/Support/WinIncludes.h"
+#include "dxc/Support/Global.h"
+#include "dxc/Support/FileIOHelper.h"
+#include "dxc/DxilContainer/DxilContainer.h"
+#include "dxc/dxcapi.h"
+#include "dxc/Support/dxcapi.impl.h"
+
+using namespace llvm;
+
+// MSF header
+static const char kMsfMagic[] = {'M',  'i',  'c',    'r', 'o', 's',  'o',  'f',
+                                 't',  ' ',  'C',    '/', 'C', '+',  '+',  ' ',
+                                 'M',  'S',  'F',    ' ', '7', '.',  '0',  '0',
+                                 '\r', '\n', '\x1a', 'D', 'S', '\0', '\0', '\0'};
+
+static const uint32_t kDataStreamIndex = 5; // This is the fixed stream index where we will store our custom data.
+static const uint32_t kMsfBlockSize = 512;
+
+// The superblock is overlaid at the beginning of the file (offset 0).
+// It starts with a magic header and is followed by information which
+// describes the layout of the file system.
+struct MSF_SuperBlock {
+  char MagicBytes[sizeof(kMsfMagic)];
+  // The file system is split into a variable number of fixed size elements.
+  // These elements are referred to as blocks.  The size of a block may vary
+  // from system to system.
+  support::ulittle32_t BlockSize;
+  // The index of the free block map.
+  support::ulittle32_t FreeBlockMapBlock;
+  // This contains the number of blocks resident in the file system.  In
+  // practice, NumBlocks * BlockSize is equivalent to the size of the MSF
+  // file.
+  support::ulittle32_t NumBlocks;
+  // This contains the number of bytes which make up the directory.
+  support::ulittle32_t NumDirectoryBytes;
+  // This field's purpose is not yet known.
+  support::ulittle32_t Unknown1;
+  // This contains the block # of the block map.
+  support::ulittle32_t BlockMapAddr;
+};
+static_assert(sizeof(MSF_SuperBlock) <= kMsfBlockSize, "MSF Block too small.");
+
+// Calculate how many blocks are needed
+static uint32_t CalculateNumBlocks(uint32_t BlockSize, uint32_t Size) {
+  return (Size / BlockSize) + 
+      ((Size % BlockSize) ? 1 : 0);
+}
+
+static HRESULT ReadAllBytes(IStream *pStream, void *pDst, size_t uSize) {
+  ULONG uBytesRead = 0;
+  IFR(pStream->Read(pDst, uSize, &uBytesRead));
+  if (uBytesRead != uSize)
+    return E_FAIL;
+  return S_OK;
+}
+
+struct MSFWriter {
+
+  struct Stream {
+    ArrayRef<char> Data;
+    unsigned NumBlocks = 0;
+  };
+  struct StreamLayout {
+    MSF_SuperBlock SB;
+  };
+
+  int m_NumBlocks = 0;
+  SmallVector<Stream, 8> m_Streams;
+
+  static uint32_t GetNumBlocks(uint32_t Size) {
+    return CalculateNumBlocks(kMsfBlockSize, Size);
+  }
+
+  uint32_t AddStream(ArrayRef<char> Data) {
+    uint32_t ID = m_Streams.size();
+    Stream S;
+    S.Data = Data;
+    S.NumBlocks = GetNumBlocks(Data.size());
+    m_NumBlocks += S.NumBlocks;
+    m_Streams.push_back(S);
+    return ID;
+  }
+
+  uint32_t AddEmptyStream() {
+    return AddStream({});
+  }
+
+  uint32_t CalculateDirectorySize() {
+    uint32_t DirectorySizeInBytes = 0;
+    DirectorySizeInBytes += sizeof(uint32_t);
+    DirectorySizeInBytes += m_Streams.size() * 4;
+    for (int i = 0; i < m_Streams.size(); i++) {
+      DirectorySizeInBytes += m_Streams[i].NumBlocks * 4;
+    }
+    return DirectorySizeInBytes;
+  }
+
+  MSF_SuperBlock CalculateSuperblock() {
+    MSF_SuperBlock SB = {};
+    memcpy(SB.MagicBytes, kMsfMagic, sizeof(kMsfMagic));
+    SB.BlockSize = kMsfBlockSize;
+    SB.NumDirectoryBytes = CalculateDirectorySize();
+    SB.NumBlocks = 3 + m_NumBlocks + GetNumBlocks(SB.NumDirectoryBytes);
+    SB.FreeBlockMapBlock = 1;
+    SB.BlockMapAddr = 3;
+    return SB;
+  }
+
+  struct BlockWriter {
+    uint32_t BlocksWritten = 0;
+    raw_ostream &OS;
+
+    BlockWriter(raw_ostream &OS) : OS(OS) {}
+
+    void WriteZeroPads(uint32_t Count) {
+      for (unsigned i = 0; i < Count; i++)
+        OS.write(0);
+    }
+
+    void WriteEmptyBlock() {
+      BlocksWritten++;
+      WriteZeroPads(kMsfBlockSize);
+    }
+
+    void WriteBlocks(uint32_t NumBlocks, const void *Data, uint32_t Size) {
+      assert(NumBlocks >= GetNumBlocks(Size) && "Cannot fit data into the requested number of blocks!");
+      uint32_t TotalSize = NumBlocks * kMsfBlockSize;
+      OS.write((char *)Data, Size);
+      WriteZeroPads(TotalSize - Size);
+      BlocksWritten += NumBlocks;
+    }
+
+    void WriteUint32(uint32_t Value) {
+      support::ulittle32_t ValueLE;
+      ValueLE = Value;
+      OS.write((char *)&ValueLE, sizeof(ValueLE));
+    }
+  };
+
+  void WriteBlocks(raw_ostream &OS, ArrayRef<char> Data, uint32_t NumBlocks) {
+    assert(NumBlocks >= GetNumBlocks(Data.size()) && "Cannot fit data into the requested number of blocks!");
+    uint32_t TotalSize = NumBlocks * kMsfBlockSize;
+    OS.write(Data.data(), Data.size());
+    WriteZeroPadding(OS, TotalSize - Data.size());
+  }
+  void WriteZeroPadding(raw_ostream &OS, int Count) {
+    for (int i = 0; i < Count; i++)
+      OS.write(0);
+  }
+
+  static support::ulittle32_t MakeUint32LE(uint32_t Value) {
+    support::ulittle32_t ValueLE;
+    ValueLE = Value;
+    return ValueLE;
+  }
+
+  void WriteToStream(raw_ostream &OS) {
+    MSF_SuperBlock SB = CalculateSuperblock();
+    const uint32_t NumDirectoryBlocks = GetNumBlocks(SB.NumDirectoryBytes);
+    const uint32_t StreamDirectoryAddr = SB.BlockMapAddr;
+    const uint32_t StreamDirectoryStart = StreamDirectoryAddr + 1;
+    const uint32_t StreamStart = StreamDirectoryStart + NumDirectoryBlocks;
+
+    BlockWriter Writer(OS);
+    Writer.WriteBlocks(1, &SB, sizeof(SB)); // Super Block
+    Writer.WriteEmptyBlock();               // FPM 1
+    Writer.WriteEmptyBlock();               // FPM 2
+
+    // BlockAddr
+    // This block contains a list of uint32's that point to the blocks that
+    // make up the stream directory.
+    {
+      SmallVector<support::ulittle32_t, 4> BlockAddr;
+      uint32_t Start = StreamDirectoryStart;
+      for (unsigned i = 0; i < NumDirectoryBlocks; i++) {
+        support::ulittle32_t V;
+        V = Start++;
+        BlockAddr.push_back(V);
+      }
+      Writer.WriteBlocks(1, BlockAddr.data(), sizeof(BlockAddr[0])*BlockAddr.size());
+    }
+
+    // Stream Directory. Describes where all the streams are
+    // Looks like this:
+    // 
+    {
+      SmallVector<support::ulittle32_t, 32> StreamDirectoryData;
+      StreamDirectoryData.push_back(MakeUint32LE(m_Streams.size()));
+      for (unsigned i = 0; i < m_Streams.size(); i++) {
+        StreamDirectoryData.push_back(MakeUint32LE(m_Streams[i].Data.size()));
+      }
+      uint32_t Start = StreamStart;
+      for (unsigned i = 0; i < m_Streams.size(); i++) {
+        auto &Stream = m_Streams[i];
+        for (unsigned j = 0; j < Stream.NumBlocks; j++) {
+          StreamDirectoryData.push_back(MakeUint32LE(Start++));
+        }
+      }
+      Writer.WriteBlocks(NumDirectoryBlocks, StreamDirectoryData.data(), StreamDirectoryData.size()*sizeof(StreamDirectoryData[0]));
+    }
+
+    // Write the streams.
+    {
+      for (unsigned i = 0; i < m_Streams.size(); i++) {
+        auto &Stream = m_Streams[i];
+        Writer.WriteBlocks(Stream.NumBlocks, Stream.Data.data(), Stream.Data.size());
+      }
+    }
+
+  }
+};
+
+enum class PdbStreamVersion : uint32_t {
+  VC2 = 19941610,
+  VC4 = 19950623,
+  VC41 = 19950814,
+  VC50 = 19960307,
+  VC98 = 19970604,
+  VC70Dep = 19990604,
+  VC70 = 20000404,
+  VC80 = 20030901,
+  VC110 = 20091201,
+  VC140 = 20140508,
+};
+
+struct PdbStreamHeader {
+  support::ulittle32_t Version;
+  support::ulittle32_t Signature;
+  support::ulittle32_t Age;
+  uint8_t UniqueId[16];
+};
+static_assert(sizeof(PdbStreamHeader) == 28, "PDB Header incorrect.");
+
+static
+SmallVector<char, 0> WritePdbStream(ArrayRef<BYTE> Hash) {
+  PdbStreamHeader Header = {};
+  Header.Version = (uint32_t)PdbStreamVersion::VC70;
+  Header.Age = 1;
+  Header.Signature = 0;
+  memcpy(Header.UniqueId, Hash.data(), std::min(Hash.size(), sizeof(Header.UniqueId)));
+
+  SmallVector<char, 0> Result;
+  raw_svector_ostream OS(Result);
+
+  auto WriteU32 = [&](uint32_t val) {
+    support::ulittle32_t valLE;
+    valLE = val;
+    OS.write((char *)&valLE, sizeof(valLE));
+  };
+
+  OS.write((char *)&Header, 28);
+  WriteU32(0); // String buffer size
+  WriteU32(0); // Size
+  WriteU32(1); // Capacity // Capacity is required to be 1.
+  WriteU32(0); // Present count
+  WriteU32(0); // Deleted count
+
+  WriteU32(0); // Key
+  WriteU32(0); // Value
+
+  OS.flush();
+  return Result;
+}
+
+static HRESULT FindShaderHash(IDxcBlob *pContainer, BYTE *pDigest, UINT32 uCapacity, UINT32 *pBytesWritten) {
+  if (!hlsl::IsValidDxilContainer((hlsl::DxilContainerHeader *)pContainer->GetBufferPointer(), pContainer->GetBufferSize()))
+    return E_FAIL;
+
+  hlsl::DxilContainerHeader *DxilHeader = (hlsl::DxilContainerHeader *)pContainer->GetBufferPointer();
+  for (unsigned i = 0; i < DxilHeader->PartCount; i++) {
+    hlsl::DxilPartHeader *PartHeader = GetDxilContainerPart(DxilHeader, i);
+    if (PartHeader->PartFourCC == hlsl::DFCC_ShaderHash) {
+      hlsl::DxilShaderHash *HashHeader = (hlsl::DxilShaderHash *)(PartHeader+1);
+      if (sizeof(HashHeader->Digest) > uCapacity)
+        return E_FAIL;
+      memcpy(pDigest, HashHeader->Digest, sizeof(HashHeader->Digest));
+      *pBytesWritten = sizeof(HashHeader->Digest);
+      return S_OK;
+    }
+  }
+  return E_FAIL;
+}
+
+HRESULT hlsl::pdb::WriteDxilPDB(IMalloc *pMalloc, IDxcBlob *pContainer, IDxcBlob **ppOutBlob) {
+  if (!hlsl::IsValidDxilContainer((hlsl::DxilContainerHeader *)pContainer->GetBufferPointer(), pContainer->GetBufferSize()))
+    return E_FAIL;
+
+  BYTE HashData[16];
+  UINT32 uHashSize = 0;
+  IFR(FindShaderHash(pContainer, HashData, sizeof(HashData), &uHashSize));
+  SmallVector<char, 0> PdbStream = WritePdbStream({ HashData, HashData+uHashSize });
+
+  MSFWriter Writer;
+  Writer.AddEmptyStream();     // Old Directory
+  Writer.AddStream(PdbStream); // PDB Header
+
+  // Fixed streams
+  Writer.AddEmptyStream(); // TPI
+  Writer.AddEmptyStream(); // DBI
+  Writer.AddEmptyStream(); // IPI
+  
+  Writer.AddStream({ (char *)pContainer->GetBufferPointer(), pContainer->GetBufferSize() }); // Actual data block
+  
+  CComPtr<hlsl::AbstractMemoryStream> pStream;
+  IFR(hlsl::CreateMemoryStream(pMalloc, &pStream));
+
+  raw_stream_ostream OS(pStream);
+  Writer.WriteToStream(OS);
+  OS.flush();
+
+  IFR(pStream.QueryInterface(ppOutBlob));
+
+  return S_OK;
+}
+
+
+struct PDBReader {
+  IStream *m_pStream = nullptr;
+  IMalloc *m_pMalloc = nullptr;
+  UINT32 m_uOriginalOffset = 0;
+  MSF_SuperBlock m_SB = {};
+  HRESULT m_Status = S_OK;
+
+  HRESULT SetPosition(INT32 sOffset) {
+    LARGE_INTEGER Distance = {};
+    Distance.QuadPart = m_uOriginalOffset + sOffset;
+    ULARGE_INTEGER NewLocation = {};
+    return m_pStream->Seek(Distance, STREAM_SEEK_SET, &NewLocation);
+  }
+
+  PDBReader(IMalloc *pMalloc, IStream *pStream) : m_pStream(pStream), m_pMalloc(pMalloc) {
+    m_Status = ReadSuperblock(&m_SB);
+  }
+
+  // Reset the stream back to its original position, regardless of
+  // we succeeded or failed.
+  ~PDBReader() {
+    SetPosition(0);
+  }
+
+  HRESULT GetStatus() { return m_Status; }
+
+  HRESULT ReadSuperblock(MSF_SuperBlock *pSB) {
+    IFR(ReadAllBytes(m_pStream, pSB, sizeof(*pSB)));
+    if (memcmp(pSB->MagicBytes, kMsfMagic, sizeof(kMsfMagic)) != 0)
+      return E_FAIL;
+
+    return S_OK;
+  }
+
+  HRESULT ReadU32(UINT32 *pValue) {
+    support::ulittle32_t ValueLE;
+    IFR(ReadAllBytes(m_pStream, &ValueLE, sizeof(ValueLE)));
+    *pValue = ValueLE;
+    return S_OK;
+  }
+
+  HRESULT GoToBeginningOfBlock(UINT32 uBlock) {
+    return SetPosition(uBlock * m_SB.BlockSize);
+  }
+
+  HRESULT OffsetByU32(int sCount) {
+    LARGE_INTEGER Offset = {};
+    ULARGE_INTEGER BytesMoved = {};
+    Offset.QuadPart = sCount * sizeof(UINT32);
+
+    return m_pStream->Seek(Offset, STREAM_SEEK_CUR, &BytesMoved);
+  }
+
+  HRESULT ReadU32ListFromBlocks(ArrayRef<uint32_t> Blocks, UINT32 uOffsetByU32, UINT32 uNumU32, SmallVectorImpl<uint32_t> &Output) {
+    if (Blocks.size() == 0) return E_FAIL;
+    Output.clear();
+
+    for (unsigned i = 0; i < uNumU32; i++) {
+      UINT32 uOffsetInBytes = (uOffsetByU32+i) * sizeof(UINT32);
+      UINT32 BlockIndex = uOffsetInBytes / m_SB.BlockSize;
+      UINT32 ByteOffset = uOffsetInBytes % m_SB.BlockSize;
+
+      UINT32 uBlock = Blocks[BlockIndex];
+      IFR(GoToBeginningOfBlock(uBlock));
+      IFR(OffsetByU32(ByteOffset / sizeof(UINT32)));
+
+      UINT32 uData = 0;
+      IFR(ReadU32(&uData));
+
+      Output.push_back(uData);
+    }
+
+    return S_OK;
+  }
+
+  HRESULT ReadContainedData(IDxcBlob **ppData) {
+    if (FAILED(m_Status)) return m_Status;
+
+    UINT32 uNumDirectoryBlocks =
+      CalculateNumBlocks(m_SB.BlockSize, m_SB.NumDirectoryBytes);
+
+    // Load in the directory blocks
+    llvm::SmallVector<uint32_t, 32> DirectoryBlocks;
+    IFR(GoToBeginningOfBlock(m_SB.BlockMapAddr))
+    for (unsigned i = 0; i < uNumDirectoryBlocks; i++) {
+      UINT32 uBlock = 0;
+      IFR(ReadU32(&uBlock));
+      DirectoryBlocks.push_back(uBlock);
+    }
+
+    // Load Num streams
+    UINT32 uNumStreams = 0;
+    IFR(GoToBeginningOfBlock(DirectoryBlocks[0]));
+    IFR(ReadU32(&uNumStreams));
+
+    // If we don't have enough streams, then give up.
+    if (uNumStreams <= kDataStreamIndex)
+      return E_FAIL;
+
+    llvm::SmallVector<uint32_t, 6> StreamSizes;
+    IFR(ReadU32ListFromBlocks(DirectoryBlocks, 1, uNumStreams, StreamSizes));
+
+    UINT32 uOffsets = 0;
+    for (unsigned i = 0; i <= kDataStreamIndex-1; i++) {
+      UINT32 uNumBlocks = CalculateNumBlocks(m_SB.BlockSize, StreamSizes[i]);
+      uOffsets += uNumBlocks;
+    }
+
+    llvm::SmallVector<uint32_t, 12> DataBlocks;
+    IFR(ReadU32ListFromBlocks(DirectoryBlocks, 1 + uNumStreams + uOffsets, 
+      CalculateNumBlocks(m_SB.BlockSize, StreamSizes[kDataStreamIndex]), DataBlocks));
+
+    if (DataBlocks.size() == 0)
+      return E_FAIL;
+
+    IFR(GoToBeginningOfBlock(DataBlocks[0]));
+
+    CComPtr<hlsl::AbstractMemoryStream> pResult;
+    IFR(CreateMemoryStream(m_pMalloc, &pResult));
+
+    std::vector<char> CopyBuffer;
+    CopyBuffer.resize(m_SB.BlockSize);
+    for (unsigned i = 0; i < DataBlocks.size(); i++) {
+      IFR(GoToBeginningOfBlock(DataBlocks[i]));
+      IFR(ReadAllBytes(m_pStream, CopyBuffer.data(), m_SB.BlockSize));
+      ULONG uSizeWritten = 0;
+      IFR(pResult->Write(CopyBuffer.data(), m_SB.BlockSize, &uSizeWritten));
+      if (uSizeWritten != m_SB.BlockSize)
+        return E_FAIL;
+    }
+
+    IFR(pResult.QueryInterface(ppData));
+
+    return S_OK;
+  }
+};
+
+
+HRESULT hlsl::pdb::LoadDataFromStream(IMalloc *pMalloc, IStream *pIStream, IDxcBlob **ppContainer) {
+  PDBReader Reader(pMalloc, pIStream);
+
+  CComPtr<IDxcBlob> pDataBlob;
+  IFR(Reader.ReadContainedData(&pDataBlob));
+
+  if (!hlsl::IsValidDxilContainer((hlsl::DxilContainerHeader *)pDataBlob->GetBufferPointer(), pDataBlob->GetBufferSize()))
+    return E_FAIL;
+
+  *ppContainer = pDataBlob.Detach();
+
+  return S_OK;
+}
+

lib/DxilContainer/DxilContainerAssembler.cpp

@@ -1628,7 +1628,7 @@ void hlsl::SerializeDxilContainerForModule(DxilModule *pModule,
     if (DebugName.empty()) {
       md5.stringifyResult(HashContent.Digest, Hash);
       DebugNameStr += Hash;
-      DebugNameStr += ".lld";
+      DebugNameStr += ".pdb";
       DebugName = DebugNameStr;
     }
 

lib/DxilDia/DxilDiaDataSource.cpp

@@ -13,6 +13,7 @@
 
 #include "dxc/DxilContainer/DxilContainer.h"
 #include "dxc/DXIL/DxilUtil.h"
+#include "dxc/DXIL/DxilPDB.h"
 #include "dxc/Support/FileIOHelper.h"
 #include "dxc/Support/dxcapi.impl.h"
 
@@ -65,14 +66,29 @@ std::unique_ptr<llvm::MemoryBuffer> getMemBufferFromStream(_In_ IStream *pStream
 }
 }  // namespace dxil_dia
 
-STDMETHODIMP dxil_dia::DataSource::loadDataFromIStream(_In_ IStream *pIStream) {
-  DxcThreadMalloc TM(m_pMalloc);
-  if (m_module.get() != nullptr) {
-    return E_FAIL;
-  }
-  m_context.reset();
-  m_finder.reset();
+STDMETHODIMP dxil_dia::DataSource::loadDataFromIStream(_In_ IStream *pInputIStream) {
   try {
+    DxcThreadMalloc TM(m_pMalloc);
+    if (m_module.get() != nullptr) {
+      return E_FAIL;
+    }
+
+    CComPtr<IStream> pIStream = pInputIStream;
+    CComPtr<IDxcBlob> pContainer;
+    if (SUCCEEDED(hlsl::pdb::LoadDataFromStream(m_pMalloc, pInputIStream, &pContainer))) {
+      hlsl::DxilPartHeader *PartHeader =
+        hlsl::GetDxilPartByType((hlsl::DxilContainerHeader *)pContainer->GetBufferPointer(), hlsl::DFCC_ShaderDebugInfoDXIL);
+      if (!PartHeader)
+        return E_FAIL;
+      CComPtr<IDxcBlobEncoding> pPinnedBlob;
+      IFR(hlsl::DxcCreateBlobWithEncodingFromPinned(PartHeader+1, PartHeader->PartSize, CP_ACP, &pPinnedBlob));
+      pIStream.Release();
+      IFR(hlsl::CreateReadOnlyBlobStream(pPinnedBlob, &pIStream));
+    }
+
+    m_context.reset();
+    m_finder.reset();
+
     m_context = std::make_shared<llvm::LLVMContext>();
     llvm::MemoryBuffer *pBitcodeBuffer;
     std::unique_ptr<llvm::MemoryBuffer> pEmbeddedBuffer;
@@ -142,4 +158,4 @@ HRESULT CreateDxcDiaDataSource(_In_ REFIID riid, _Out_ LPVOID* ppv) {
   }
 
   return result.p->QueryInterface(riid, ppv);
-}
+}

lib/HLSL/DxilContainerReflection.cpp

@@ -25,6 +25,7 @@
 #include "dxc/Support/FileIOHelper.h"
 #include "dxc/Support/dxcapi.impl.h"
 #include "dxc/DXIL/DxilFunctionProps.h"
+#include "dxc/DXIL/DxilPDB.h"
 
 #include <unordered_set>
 #include "llvm/ADT/SetVector.h"
@@ -223,6 +224,17 @@ public:
 
 _Use_decl_annotations_
 HRESULT DxilContainerReflection::Load(IDxcBlob *pContainer) {
+
+  CComPtr<IDxcBlob> pPDBContainer;
+  {
+    DxcThreadMalloc DxcMalloc(m_pMalloc);
+    CComPtr<IStream> pStream;
+    IFR(hlsl::CreateReadOnlyBlobStream(pContainer, &pStream));
+    if (SUCCEEDED(hlsl::pdb::LoadDataFromStream(m_pMalloc, pStream, &pPDBContainer))) {
+      pContainer = pPDBContainer;
+    }
+  }
+
   if (pContainer == nullptr) {
     m_container.Release();
     m_pHeader = nullptr;

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -32,6 +32,7 @@
 #include "dxc/Support/WinIncludes.h"
 #include "dxc/DxilContainer/DxilContainerAssembler.h"
 #include "dxc/dxcapi.internal.h"
+#include "dxc/DXIL/DxilPDB.h"
 
 #include "dxc/Support/dxcapi.use.h"
 #include "dxc/Support/Global.h"
@@ -101,6 +102,143 @@ static void CreateOperationResultFromOutputs(
                                    ppResult);
 }
 
+static bool ShouldPartBeIncludedInPDB(UINT32 FourCC) {
+  switch (FourCC) {
+  case hlsl::DFCC_ShaderDebugName:
+  case hlsl::DFCC_ShaderHash:
+    return true;
+  }
+  return false;
+}
+
+static HRESULT CreateContainerForPDB(IMalloc *pMalloc, IDxcBlob *pOldContainer, IDxcBlob *pDebugBlob, IDxcBlob **ppNewContaner) {
+  // If the pContainer is not a valid container, give up.
+  if (!hlsl::IsValidDxilContainer((hlsl::DxilContainerHeader *)pOldContainer->GetBufferPointer(), pOldContainer->GetBufferSize()))
+    return E_FAIL;
+
+  hlsl::DxilContainerHeader *DxilHeader = (hlsl::DxilContainerHeader *)pOldContainer->GetBufferPointer();
+  hlsl::DxilProgramHeader *ProgramHeader = nullptr;
+
+  struct Part {
+    typedef std::function<HRESULT(IStream *)> WriteProc;
+    UINT32 uFourCC = 0;
+    UINT32 uSize = 0;
+    WriteProc Writer;
+
+    Part(UINT32 uFourCC, UINT32 uSize, WriteProc Writer) :
+      uFourCC(uFourCC),
+      uSize(uSize),
+      Writer(Writer)
+    {}
+  };
+
+  // Compute offset table.
+  SmallVector<UINT32, 4> OffsetTable;
+  SmallVector<Part, 4> PartWriters;
+  UINT32 uTotalPartsSize = 0;
+  for (unsigned i = 0; i < DxilHeader->PartCount; i++) {
+    hlsl::DxilPartHeader *PartHeader = GetDxilContainerPart(DxilHeader, i);
+    if (ShouldPartBeIncludedInPDB(PartHeader->PartFourCC)) {
+      OffsetTable.push_back(uTotalPartsSize);
+      uTotalPartsSize += PartHeader->PartSize + sizeof(*PartHeader);
+
+      UINT32 uSize = PartHeader->PartSize;
+      const void *pPartData = PartHeader+1;
+      Part NewPart(
+        PartHeader->PartFourCC,
+        uSize,
+        [pPartData, uSize](IStream *pStream) {
+          ULONG uBytesWritten = 0;
+          IFR(pStream->Write(pPartData, uSize, &uBytesWritten));
+          return S_OK;
+        }
+      );
+      PartWriters.push_back(NewPart);
+    }
+
+    // Could use any of these. We're mostly after the header version and all that.
+    if (PartHeader->PartFourCC == hlsl::DFCC_DXIL ||
+      PartHeader->PartFourCC == hlsl::DFCC_ShaderDebugInfoDXIL)
+    {
+      ProgramHeader = (hlsl::DxilProgramHeader *)(PartHeader+1);
+    }
+  }
+
+  if (!ProgramHeader)
+    return E_FAIL;
+
+  {
+    static auto AlignByDword = [](UINT32 uSize, UINT32 *pPaddingBytes) {
+      UINT32 uRem = uSize % sizeof(UINT32);
+      UINT32 uResult = (uSize/sizeof(UINT32) + (uRem ? 1 : 0)) * sizeof(UINT32);
+      *pPaddingBytes = uRem ? (sizeof(UINT32)-uRem) : 0;
+      return uResult;
+    };
+
+    UINT32 uPaddingSize = 0;
+    UINT32 uPartSize = AlignByDword(sizeof(hlsl::DxilProgramHeader) + pDebugBlob->GetBufferSize(), &uPaddingSize);
+
+    OffsetTable.push_back(uTotalPartsSize);
+    uTotalPartsSize += uPartSize + sizeof(hlsl::DxilPartHeader);
+
+    Part NewPart(
+      hlsl::DFCC_ShaderDebugInfoDXIL,
+      uPartSize,
+      [uPartSize, ProgramHeader, pDebugBlob, uPaddingSize](IStream *pStream) {
+        hlsl::DxilProgramHeader Header = *ProgramHeader;
+        Header.BitcodeHeader.BitcodeSize = pDebugBlob->GetBufferSize();
+        Header.BitcodeHeader.BitcodeOffset = sizeof(hlsl::DxilBitcodeHeader);
+        Header.SizeInUint32 = uPartSize / sizeof(UINT32);
+
+        ULONG uBytesWritten = 0;
+        IFR(pStream->Write(&Header, sizeof(Header), &uBytesWritten));
+        IFR(pStream->Write(pDebugBlob->GetBufferPointer(), pDebugBlob->GetBufferSize(), &uBytesWritten));
+        if(uPaddingSize) {
+          UINT32 uPadding = 0;
+          assert(uPaddingSize <= sizeof(uPadding) && "Padding size calculation is wrong.");
+          IFR(pStream->Write(&uPadding, uPaddingSize, &uBytesWritten));
+        }
+        return S_OK;
+      }
+    );
+    PartWriters.push_back(NewPart);
+  }
+
+  // Offset the offset table by the offset table itself
+  for (unsigned i = 0; i < OffsetTable.size(); i++)
+    OffsetTable[i] += sizeof(hlsl::DxilContainerHeader) + OffsetTable.size() * sizeof(UINT32);
+
+  // Create the new header
+  hlsl::DxilContainerHeader NewDxilHeader = *DxilHeader;
+  NewDxilHeader.PartCount = OffsetTable.size();
+  NewDxilHeader.ContainerSizeInBytes =
+    sizeof(NewDxilHeader) +
+    OffsetTable.size() * sizeof(UINT32) +
+    uTotalPartsSize;
+
+  // Write it to the result stream
+  ULONG uSizeWritten = 0;
+  CComPtr<hlsl::AbstractMemoryStream> pStrippedContainerStream;
+  IFR(hlsl::CreateMemoryStream(pMalloc, &pStrippedContainerStream));
+  IFR(pStrippedContainerStream->Write(&NewDxilHeader, sizeof(NewDxilHeader), &uSizeWritten));
+
+  // Write offset table
+  IFR(pStrippedContainerStream->Write(OffsetTable.data(), OffsetTable.size() * sizeof(OffsetTable.data()[0]), &uSizeWritten));
+
+  for (unsigned i = 0; i < PartWriters.size(); i++) {
+    auto &Writer = PartWriters[i];
+    hlsl::DxilPartHeader PartHeader = {};
+    PartHeader.PartFourCC = Writer.uFourCC;
+    PartHeader.PartSize = Writer.uSize;
+    IFR(pStrippedContainerStream->Write(&PartHeader, sizeof(PartHeader), &uSizeWritten));
+    IFR(Writer.Writer(pStrippedContainerStream));
+  }
+
+  IFR(pStrippedContainerStream.QueryInterface(ppNewContaner));
+
+  return S_OK;
+}
+
 #ifdef _WIN32
 
 #pragma fenv_access(on)
@@ -649,7 +787,11 @@ public:
       DXVERIFY_NOMSG(SUCCEEDED((*ppResult)->GetStatus(&status)));
       if (SUCCEEDED(status)) {
         if (opts.IsDebugInfoEnabled() && ppDebugBlob) {
-          DXVERIFY_NOMSG(SUCCEEDED(pOutputStream.QueryInterface(ppDebugBlob)));
+          CComPtr<IDxcBlob> pStrippedContainer;
+          CComPtr<IDxcBlob> pDebugBitcodeBlob;
+          DXVERIFY_NOMSG(SUCCEEDED(pOutputStream.QueryInterface(&pDebugBitcodeBlob)));
+          DXVERIFY_NOMSG(SUCCEEDED(CreateContainerForPDB(m_pMalloc, pOutputBlob, pDebugBitcodeBlob, &pStrippedContainer)));
+          DXVERIFY_NOMSG(SUCCEEDED((hlsl::pdb::WriteDxilPDB(m_pMalloc, pStrippedContainer, ppDebugBlob)))); 
         }
         if (ppDebugBlobName) {
           *ppDebugBlobName = DebugBlobName.Detach();

tools/clang/unittests/HLSL/CompilerTest.cpp

@@ -205,6 +205,7 @@ public:
 
   TEST_METHOD(CompileWhenDebugThenDIPresent)
   TEST_METHOD(CompileDebugLines)
+  TEST_METHOD(CompileDebugPDB)
 
   TEST_METHOD(CompileWhenDefinesThenApplied)
   TEST_METHOD(CompileWhenDefinesManyThenApplied)
@@ -1021,6 +1022,53 @@ TEST_F(CompilerTest, CompileWhenDebugThenDIPresent) {
 #endif
 }
 
+// Test that the new PDB format still works with Dia
+TEST_F(CompilerTest, CompileDebugPDB) {
+  const char *hlsl = R"(
+    [RootSignature("")]
+    float main(float pos : A) : SV_Target {
+      float x = abs(pos);
+      float y = sin(pos);
+      float z = x + y;
+      return z;
+    }
+  )";
+  CComPtr<IDxcLibrary> pLib;
+  VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcLibrary, &pLib));
+
+  CComPtr<IDxcCompiler> pCompiler;
+  CComPtr<IDxcCompiler2> pCompiler2;
+
+  CComPtr<IDxcOperationResult> pResult;
+  CComPtr<IDxcBlobEncoding> pSource;
+  CComPtr<IDxcBlob> pProgram;
+  CComPtr<IDxcBlob> pPdbBlob;
+  WCHAR *pDebugName = nullptr;
+
+  VERIFY_SUCCEEDED(CreateCompiler(&pCompiler));
+  VERIFY_SUCCEEDED(pCompiler.QueryInterface(&pCompiler2));
+  CreateBlobFromText(hlsl, &pSource);
+  LPCWSTR args[] = { L"/Zi", L"/Qembed_debug" };
+  VERIFY_SUCCEEDED(pCompiler2->CompileWithDebug(pSource, L"source.hlsl", L"main",
+    L"ps_6_0", args, _countof(args), nullptr, 0, nullptr, &pResult, &pDebugName, &pPdbBlob));
+  VERIFY_SUCCEEDED(pResult->GetResult(&pProgram));
+
+  CComPtr<IDiaDataSource> pDiaSource;
+  CComPtr<IStream> pProgramStream;
+
+  VERIFY_SUCCEEDED(pLib->CreateStreamFromBlobReadOnly(pPdbBlob, &pProgramStream));
+  VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcDiaDataSource, &pDiaSource));
+  VERIFY_SUCCEEDED(pDiaSource->loadDataFromIStream(pProgramStream));
+
+  // Test that IDxcContainerReflection can consume a PDB container
+  CComPtr<IDxcContainerReflection> pReflection;
+  VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcContainerReflection, &pReflection));
+  VERIFY_SUCCEEDED(pReflection->Load(pPdbBlob));
+
+  UINT32 uDebugInfoIndex = 0;
+  VERIFY_SUCCEEDED(pReflection->FindFirstPartKind(hlsl::DFCC_ShaderDebugInfoDXIL, &uDebugInfoIndex));
+}
+
 TEST_F(CompilerTest, CompileDebugLines) {
   CComPtr<IDiaDataSource> pDiaSource;
   VERIFY_SUCCEEDED(CreateDiaSourceForCompile(

tools/clang/unittests/HLSL/FileCheckerTest.cpp

@@ -245,10 +245,14 @@ static HRESULT GetDxilBitcode(dxc::DxcDllSupport &DllSupport, IDxcBlob *pCompile
 static HRESULT CompileForHash(hlsl::options::DxcOpts &opts, LPCWSTR CommandFileName, dxc::DxcDllSupport &DllSupport, std::vector<LPCWSTR> &flags, llvm::SmallString<32> &Hash, std::string &output) {
   CComPtr<IDxcLibrary> pLibrary;
   CComPtr<IDxcCompiler> pCompiler;
+  CComPtr<IDxcCompiler2> pCompiler2;
   CComPtr<IDxcOperationResult> pResult;
   CComPtr<IDxcBlobEncoding> pSource;
   CComPtr<IDxcBlob> pCompiledBlob;
+  CComPtr<IDxcBlob> pCompiledName;
   CComPtr<IDxcIncludeHandler> pIncludeHandler;
+  WCHAR *pDebugName = nullptr;
+  CComPtr<IDxcBlob> pPDBBlob;
 
   std::wstring entry =
       Unicode::UTF8ToUTF16StringOrThrow(opts.EntryPoint.str().c_str());
@@ -259,8 +263,9 @@ static HRESULT CompileForHash(hlsl::options::DxcOpts &opts, LPCWSTR CommandFileN
   IFT(pLibrary->CreateBlobFromFile(CommandFileName, nullptr, &pSource));
   IFT(pLibrary->CreateIncludeHandler(&pIncludeHandler));
   IFT(DllSupport.CreateInstance(CLSID_DxcCompiler, &pCompiler));
-  IFT(pCompiler->Compile(pSource, CommandFileName, entry.c_str(), profile.c_str(),
-    flags.data(), flags.size(), nullptr, 0, pIncludeHandler, &pResult));
+  IFT(pCompiler.QueryInterface(&pCompiler2));
+  IFT(pCompiler2->CompileWithDebug(pSource, CommandFileName, entry.c_str(), profile.c_str(),
+    flags.data(), flags.size(), nullptr, 0, pIncludeHandler, &pResult, &pDebugName, &pPDBBlob));
 
   HRESULT resultStatus = 0;
   IFT(pResult->GetStatus(&resultStatus));
@@ -293,6 +298,15 @@ static HRESULT CompileForHash(hlsl::options::DxcOpts &opts, LPCWSTR CommandFileN
     md5.final(md5Result);
     md5.stringifyResult(md5Result, Hash);
 
+    // Test that PDB is generated correctly.
+    // This test needs to be done elsewhere later, ideally a fully
+    // customizable test on all our test set with different compile options.
+    if (pPDBBlob) {
+      IFT(pReflection->Load(pPDBBlob));
+      UINT32 uDebugInfoIndex = 0;
+      IFT(pReflection->FindFirstPartKind(hlsl::DFCC_ShaderDebugInfoDXIL, &uDebugInfoIndex));
+    }
+
     return S_OK;
   }
   else {

utils/hct/hcttestcmds.cmd

@@ -48,27 +48,27 @@ if %errorlevel% equ 0 (
   call :cleanup 2>nul
   exit /b 1
 )
-rem del .lld file if exists
-dir %CD%\*.lld 1>nul
+rem del .pdb file if exists
+dir %CD%\*.pdb 1>nul
 if %errorlevel% equ 0 (
-  del %CD%\*.lld
+  del %CD%\*.pdb
 )
-rem /Fd implies /Qstrip_debug ; path with \ produces auto hash-named .lld file
+rem /Fd implies /Qstrip_debug ; path with \ produces auto hash-named .pdb file
 dxc.exe /T ps_6_0 "%testfiles%\smoke.hlsl" /Zi /Fd .\ /Fo smoke.hlsl.strip 1>nul
 if %errorlevel% neq 0 (
   echo Failed - %CD%\dxc.exe /T ps_6_0 "%testfiles%\smoke.hlsl" /Fd %CD%\
   call :cleanup 2>nul
   exit /b 1
 )
-rem .lld file should be produced
-dir %CD%\*.lld 1>nul
+rem .pdb file should be produced
+dir %CD%\*.pdb 1>nul
 if %errorlevel% neq 0 (
-  echo Failed to find some .lld file at %CD%
+  echo Failed to find some .pdb file at %CD%
   call :cleanup 2>nul
   exit /b 1
 )
-rem auto debug name is hex digest + .lld
-dxc.exe -dumpbin smoke.hlsl.strip | findstr -r -c:"shader debug name: [0-9a-f]*.lld" 1>nul
+rem auto debug name is hex digest + .pdb
+dxc.exe -dumpbin smoke.hlsl.strip | findstr -r -c:"shader debug name: [0-9a-f]*.pdb" 1>nul
 if %errorlevel% neq 0 (
   echo Failed to find shader debug name.
   call :cleanup 2>nul
@@ -82,10 +82,10 @@ if %errorlevel% equ 0 (
 )
 
 rem Embed debug info
-rem first delete .lld file if exists
-dir %CD%\*.lld 1>nul
+rem first delete .pdb file if exists
+dir %CD%\*.pdb 1>nul
 if %errorlevel% equ 0 (
-  del %CD%\*.lld
+  del %CD%\*.pdb
 )
 dxc.exe /T ps_6_0 "%testfiles%\smoke.hlsl" /Zi /Qembed_debug /Fo smoke.hlsl.embedpdb 1>nul
 if %errorlevel% neq 0 (
@@ -93,15 +93,15 @@ if %errorlevel% neq 0 (
   call :cleanup 2>nul
   exit /b 1
 )
-rem .lld file should NOT be produced
-dir %CD%\*.lld 1>nul
+rem .pdb file should NOT be produced
+dir %CD%\*.pdb 1>nul
 if %errorlevel% equ 0 (
-  echo Found unexpected .lld file at %CD%
+  echo Found unexpected .pdb file at %CD%
   call :cleanup 2>nul
   exit /b 1
 )
-rem should have auto debug name, which is hex digest + .lld
-dxc.exe -dumpbin smoke.hlsl.embedpdb | findstr -r -c:"shader debug name: [0-9a-f]*.lld" 1>nul
+rem should have auto debug name, which is hex digest + .pdb
+dxc.exe -dumpbin smoke.hlsl.embedpdb | findstr -r -c:"shader debug name: [0-9a-f]*.pdb" 1>nul
 if %errorlevel% neq 0 (
   echo Failed to find shader debug name.
   call :cleanup 2>nul
@@ -116,10 +116,10 @@ if %errorlevel% neq 0 (
 
 del smoke.hlsl.embedpdb
 rem Auto-embed debug info when no debug output, and expect warning signifying that this is the case.
-rem first delete .lld file if exists
-dir %CD%\*.lld 1>nul
+rem first delete .pdb file if exists
+dir %CD%\*.pdb 1>nul
 if %errorlevel% equ 0 (
-  del %CD%\*.lld
+  del %CD%\*.pdb
 )
 dxc.exe /T ps_6_0 "%testfiles%\smoke.hlsl" /Zi /Fo smoke.hlsl.embedpdb /Fe smoke.err.embedpdb 1>nul
 if %errorlevel% neq 0 (
@@ -134,15 +134,15 @@ if %errorlevel% neq 0 (
   call :cleanup 2>nul
   exit /b 1
 )
-rem .lld file should NOT be produced
-dir %CD%\*.lld 1>nul
+rem .pdb file should NOT be produced
+dir %CD%\*.pdb 1>nul
 if %errorlevel% equ 0 (
-  echo Found unexpected .lld file at %CD%
+  echo Found unexpected .pdb file at %CD%
   call :cleanup 2>nul
   exit /b 1
 )
-rem should have auto debug name, which is hex digest + .lld
-dxc.exe -dumpbin smoke.hlsl.embedpdb | findstr -r -c:"shader debug name: [0-9a-f]*.lld" 1>nul
+rem should have auto debug name, which is hex digest + .pdb
+dxc.exe -dumpbin smoke.hlsl.embedpdb | findstr -r -c:"shader debug name: [0-9a-f]*.pdb" 1>nul
 if %errorlevel% neq 0 (
   echo Failed to find shader debug name.
   call :cleanup 2>nul
@@ -162,8 +162,8 @@ if %errorlevel% neq 0 (
   call :cleanup 2>nul
   exit /b 1
 )
-rem auto debug name is hex digest + .lld
-dxc.exe -dumpbin smoke.hlsl.strip | findstr -r -c:"shader debug name: [0-9a-f]*.lld" 1>nul
+rem auto debug name is hex digest + .pdb
+dxc.exe -dumpbin smoke.hlsl.strip | findstr -r -c:"shader debug name: [0-9a-f]*.pdb" 1>nul
 if %errorlevel% neq 0 (
   echo Failed to find shader debug name.
   call :cleanup 2>nul
@@ -767,10 +767,10 @@ exit /b 0
 
 :cleanup
 exit /b 0
-rem del .lld file if exists
-dir %CD%\*.lld 1>nul
+rem del .pdb file if exists
+dir %CD%\*.pdb 1>nul
 if %errorlevel% equ 0 (
-  del %CD%\*.lld
+  del %CD%\*.pdb
 )
 del %CD%\NonUniform.cso
 del %CD%\NonUniformNoRootSig.cso