Add shader hash blob part (#2212)

- Change default to hash/name based on binary, not source
- Allow inclusion of hash/name without debug info (/Zi)

include/dxc/DxilContainer/DxilContainer.h

@@ -40,6 +40,17 @@ struct DxilContainerHash {
   uint8_t Digest[DxilContainerHashSize];
 };
 
+enum class DxilShaderHashFlags : uint32_t {
+  None = 0,           // No flags defined.
+  IncludesSource = 1, // This flag indicates that the shader hash was computed
+                      // taking into account source information (-Zss)
+};
+
+typedef struct DxilShaderHash {
+  uint32_t Flags; // DxilShaderHashFlags
+  uint8_t Digest[DxilContainerHashSize];
+} DxcShaderHash;
+
 struct DxilContainerVersion {
   uint16_t Major;
   uint16_t Minor;
@@ -83,6 +94,7 @@ enum DxilFourCC {
   DFCC_DXIL                     = DXIL_FOURCC('D', 'X', 'I', 'L'),
   DFCC_PipelineStateValidation  = DXIL_FOURCC('P', 'S', 'V', '0'),
   DFCC_RuntimeData              = DXIL_FOURCC('R', 'D', 'A', 'T'),
+  DFCC_ShaderHash               = DXIL_FOURCC('H', 'A', 'S', 'H'),
 };
 
 #undef DXIL_FOURCC

lib/DxcSupport/HLSLOptions.cpp

@@ -638,17 +638,18 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
     return 1;
   }
 
-  if (!opts.DebugNameForBinary && !opts.DebugNameForSource) {
-    if (opts.DebugInfo)
-      opts.DebugNameForSource = true;
-    else
-      opts.DebugNameForBinary = true;
-  }
-  else if (opts.DebugNameForBinary && opts.DebugNameForSource) {
+  if (opts.DebugInfo && !opts.DebugNameForBinary && !opts.DebugNameForSource) {
+    opts.DebugNameForBinary = true;
+  } else if (opts.DebugNameForBinary && opts.DebugNameForSource) {
     errors << "Cannot specify both /Zss and /Zsb";
     return 1;
   }
 
+  if (opts.DebugNameForSource && !opts.DebugInfo) {
+    errors << "/Zss requires debug info (/Zi)";
+    return 1;
+  }
+
   if (opts.IsLibraryProfile() && Minor == 0xF) {
     // Disable validation for offline link only target
     opts.DisableValidation = true;

lib/DxilContainer/DxilContainerAssembler.cpp

@@ -1476,6 +1476,9 @@ void hlsl::SerializeDxilContainerForModule(DxilModule *pModule,
   pModule->GetValidatorVersion(ValMajor, ValMinor);
   if (ValMajor == 1 && ValMinor == 0)
     Flags &= ~SerializeDxilFlags::IncludeDebugNamePart;
+  bool bSupportsShaderHash = true;
+  if (ValMajor == 1 && ValMinor < 5)
+    bSupportsShaderHash = false;
 
   DxilContainerWriter_impl writer;
 
@@ -1602,22 +1605,28 @@ void hlsl::SerializeDxilContainerForModule(DxilModule *pModule,
   // Serialize debug name if requested.
   CComPtr<AbstractMemoryStream> pHashStream;
   std::string DebugNameStr; // Used if constructing name based on hash
+  DxilShaderHash HashContent;
   if (Flags & SerializeDxilFlags::IncludeDebugNamePart) {
+    // If the debug name should be specific to the sources, base the name on the debug
+    // bitcode, which will include the source references, line numbers, etc. Otherwise,
+    // do it exclusively on the target shader bitcode.
+    if (Flags & SerializeDxilFlags::DebugNameDependOnSource) {
+      pHashStream = CComPtr<AbstractMemoryStream>(pModuleBitcode);
+      HashContent.Flags = (uint32_t)DxilShaderHashFlags::IncludesSource;
+    } else {
+      pHashStream = CComPtr<AbstractMemoryStream>(pProgramStream);
+      HashContent.Flags = (uint32_t)DxilShaderHashFlags::None;
+    }
+
+    ArrayRef<uint8_t> Data((uint8_t *)pHashStream->GetPtr(),
+                           pHashStream->GetPtrSize());
+    llvm::MD5 md5;
+    SmallString<32> Hash;
+    md5.update(Data);
+    md5.final(HashContent.Digest);
+
     if (DebugName.empty()) {
-      // If the debug name should be specific to the sources, base the name on the debug
-      // bitcode, which will include the source references, line numbers, etc. Otherwise,
-      // do it exclusively on the target shader bitcode.
-      pHashStream = (int)(Flags & SerializeDxilFlags::DebugNameDependOnSource)
-                      ? CComPtr<AbstractMemoryStream>(pModuleBitcode)
-                      : CComPtr<AbstractMemoryStream>(pProgramStream);
-
-      ArrayRef<uint8_t> Data((uint8_t *)pHashStream->GetPtr(), pHashStream->GetPtrSize());
-      llvm::MD5 md5;
-      llvm::MD5::MD5Result md5Result;
-      SmallString<32> Hash;
-      md5.update(Data);
-      md5.final(md5Result);
-      md5.stringifyResult(md5Result, Hash);
+      md5.stringifyResult(HashContent.Digest, Hash);
       DebugNameStr += Hash;
       DebugNameStr += ".lld";
       DebugName = DebugNameStr;
@@ -1643,6 +1652,15 @@ void hlsl::SerializeDxilContainerForModule(DxilModule *pModule,
       unsigned padLen = (4 - ((sizeof(DxilShaderDebugName) + cbWritten) & 0x3));
       IFT(pStream->Write(Pad, padLen, &cbWritten));
     });
+
+    if (bSupportsShaderHash) {
+      writer.AddPart(DFCC_ShaderHash, sizeof(HashContent),
+        [HashContent]
+        (AbstractMemoryStream *pStream)
+      {
+        IFT(WriteStreamValue(pStream, HashContent));
+      });
+    }
   }
 
   // Compute padded bitcode size.

lib/HLSL/DxilValidation.cpp

@@ -5091,7 +5091,8 @@ void GetValidationVersion(_Out_ unsigned *pMajor, _Out_ unsigned *pMinor) {
   // - packed u8x4/i8x4 dot with accumulate to i32
   // - half dot2 with accumulate to float
   // 1.5 adds:
-  // TODO: Fill this in.
+  // - WaveMatch, WaveMultiPrefixOp, WaveMultiPrefixBitCount
+  // - HASH container part support
   *pMajor = 1;
   *pMinor = 5;
 }
@@ -5410,6 +5411,12 @@ HRESULT ValidateDxilContainerParts(llvm::Module *pModule,
     case DFCC_ShaderDebugName:
       continue;
 
+    case DFCC_ShaderHash:
+      if (pPart->PartSize != sizeof(DxilShaderHash)) {
+        ValCtx.EmitFormatError(ValidationRule::ContainerPartInvalid, { szFourCC });
+      }
+      break;
+
     // Runtime Data (RDAT) for libraries
     case DFCC_RuntimeData:
       if (ValCtx.isLibProfile) {

tools/clang/tools/dxcompiler/dxcdisassembler.cpp

@@ -1510,6 +1510,19 @@ HRESULT Disassemble(IDxcBlob *pProgram, raw_string_ostream &Stream) {
       }
     }
 
+    it = std::find_if(begin(pContainer), end(pContainer),
+      DxilPartIsType(DFCC_ShaderHash));
+    if (it != end(pContainer)) {
+      const DxilShaderHash *pHashContent =
+        reinterpret_cast<const DxilShaderHash *>(GetDxilPartData(*it));
+      Stream << "; shader hash: ";
+      for (int i = 0; i < 16; ++i)
+        Stream << format("%.2x", pHashContent->Digest[i]);
+      if (pHashContent->Flags & (uint32_t)DxilShaderHashFlags::IncludesSource)
+        Stream << " (includes source)";
+      Stream << "\n";
+    }
+
     it = std::find_if(begin(pContainer), end(pContainer),
                       DxilPartIsType(DFCC_DXIL));
     if (it == end(pContainer)) {

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -586,7 +586,12 @@ public:
           SerializeFlags |= SerializeDxilFlags::IncludeDebugInfoPart;
         }
         if (opts.DebugNameForSource) {
+          // Implies name part
+          SerializeFlags |= SerializeDxilFlags::IncludeDebugNamePart;
           SerializeFlags |= SerializeDxilFlags::DebugNameDependOnSource;
+        } else if (opts.DebugNameForBinary) {
+          // Implies name part
+          SerializeFlags |= SerializeDxilFlags::IncludeDebugNamePart;
         }
         if (opts.StripReflection) {
           SerializeFlags |= SerializeDxilFlags::StripReflectionFromDxilPart;

tools/clang/unittests/HLSL/DxilContainerTest.cpp

@@ -35,6 +35,9 @@
 #include <filesystem>
 #endif
 
+#include "llvm/Support/Format.h"
+#include "llvm/Support/raw_ostream.h"
+
 #include "HLSLTestData.h"
 #include "HlslTestUtils.h"
 #include "DxcTestUtils.h"
@@ -419,7 +422,17 @@ public:
     if (hrVer == E_NOINTERFACE) return false;
     VERIFY_SUCCEEDED(hrVer);
     VERIFY_SUCCEEDED(pVersionInfo->GetVersion(&Major, &Minor));
-    return Major == 1 && (Minor >= 1);
+    return !(Major == 1 && Minor < 1);
+  }
+
+  bool DoesValidatorSupportShaderHash() {
+    CComPtr<IDxcVersionInfo> pVersionInfo;
+    UINT Major, Minor;
+    HRESULT hrVer = m_dllSupport.CreateInstance(CLSID_DxcValidator, &pVersionInfo);
+    if (hrVer == E_NOINTERFACE) return false;
+    VERIFY_SUCCEEDED(hrVer);
+    VERIFY_SUCCEEDED(pVersionInfo->GetVersion(&Major, &Minor));
+    return !(Major == 1 && Minor < 5);
   }
 
   std::string CompileToDebugName(LPCSTR program, LPCWSTR entryPoint,
@@ -440,6 +453,28 @@ public:
     return std::string((const char *)(pDebugName + 1));
   }
 
+  std::string CompileToShaderHash(LPCSTR program, LPCWSTR entryPoint,
+    LPCWSTR target, LPCWSTR *pArguments, UINT32 argCount) {
+    CComPtr<IDxcBlob> pProgram;
+    CComPtr<IDxcBlob> pHashBlob;
+    CComPtr<IDxcContainerReflection> pContainer;
+    UINT32 index;
+
+    CompileToProgram(program, entryPoint, target, pArguments, argCount, &pProgram);
+    VERIFY_SUCCEEDED(m_dllSupport.CreateInstance(CLSID_DxcContainerReflection, &pContainer));
+    VERIFY_SUCCEEDED(pContainer->Load(pProgram));
+    if (FAILED(pContainer->FindFirstPartKind(hlsl::DFCC_ShaderHash, &index))) {
+      return std::string();
+    }
+    VERIFY_SUCCEEDED(pContainer->GetPartContent(index, &pHashBlob));
+    const hlsl::DxilShaderHash *pShaderHash = (hlsl::DxilShaderHash *)pHashBlob->GetBufferPointer();
+    std::string result;
+    llvm::raw_string_ostream os(result);
+    for (int i = 0; i < 16; ++i)
+      os << llvm::format("%.2x", pShaderHash->Digest[i]);
+    return os.str();
+  }
+
   std::string DisassembleProgram(LPCSTR program, LPCWSTR entryPoint,
                                  LPCWSTR target) {
     CComPtr<IDxcCompiler> pCompiler;
@@ -525,6 +560,7 @@ TEST_F(DxilContainerTest, CompileWhenDebugSourceThenSourceMatters) {
   LPCWSTR Zi[] = { L"/Zi", L"/Qembed_debug" };
   LPCWSTR ZiZss[] = { L"/Zi", L"/Qembed_debug", L"/Zss" };
   LPCWSTR ZiZsb[] = { L"/Zi", L"/Qembed_debug", L"/Zsb" };
+  LPCWSTR Zsb[] = { L"/Zsb" };
   
   // No debug info, no debug name...
   std::string noName = CompileToDebugName(program1, L"main", L"ps_6_0", nullptr, 0);
@@ -541,9 +577,9 @@ TEST_F(DxilContainerTest, CompileWhenDebugSourceThenSourceMatters) {
   std::string sourceName1Again = CompileToDebugName(program1, L"main", L"ps_6_0", Zi, _countof(Zi));
   VERIFY_ARE_EQUAL_STR(sourceName1.c_str(), sourceName1Again.c_str());
 
-  // Changes in source become changes in name.
+  // Use program binary by default, so name should be the same.
   std::string sourceName2 = CompileToDebugName(program2, L"main", L"ps_6_0", Zi, _countof(Zi));
-  VERIFY_IS_FALSE(0 == strcmp(sourceName2.c_str(), sourceName1.c_str()));
+  VERIFY_IS_TRUE(0 == strcmp(sourceName2.c_str(), sourceName1.c_str()));
 
   // Source again, different because different switches are specified.
   std::string sourceName1Zss = CompileToDebugName(program1, L"main", L"ps_6_0", ZiZss, _countof(ZiZss));
@@ -554,6 +590,25 @@ TEST_F(DxilContainerTest, CompileWhenDebugSourceThenSourceMatters) {
   std::string binName2 = CompileToDebugName(program2, L"main", L"ps_6_0", ZiZsb, _countof(ZiZsb));
   VERIFY_ARE_EQUAL_STR(binName1.c_str(), binName2.c_str());
   VERIFY_IS_FALSE(0 == strcmp(sourceName1Zss.c_str(), binName1.c_str()));
+
+  if (!DoesValidatorSupportShaderHash())
+    return;
+
+  // Verify source hash
+  std::string binHash1Zss = CompileToShaderHash(program1, L"main", L"ps_6_0", ZiZss, _countof(ZiZss));
+  VERIFY_IS_FALSE(binHash1Zss.empty());
+
+  // bin hash when compiling with /Zi
+  std::string binHash1 = CompileToShaderHash(program1, L"main", L"ps_6_0", ZiZsb, _countof(ZiZsb));
+  VERIFY_IS_FALSE(binHash1.empty());
+
+  // Without /Zi hash for /Zsb should be the same
+  std::string binHash2 = CompileToShaderHash(program2, L"main", L"ps_6_0", Zsb, _countof(Zsb));
+  VERIFY_IS_FALSE(binHash2.empty());
+  VERIFY_ARE_EQUAL_STR(binHash1.c_str(), binHash2.c_str());
+
+  // Source hash and bin hash should be different
+  VERIFY_IS_FALSE(0 == strcmp(binHash1Zss.c_str(), binHash1.c_str()));
 }
 #endif // _WIN32