[spirv] Embed preprocessed source code when -Zi (#1453)

Preprocessed source code could be quite useful for developers
to have a functionality-equivalent shader that they want to
debug.

tools/clang/include/clang/SPIRV/InstBuilder.h

@@ -23,6 +23,7 @@
 #include "spirv/unified1/spirv.hpp11"
 #include "llvm/ADT/ArrayRef.h"
 #include "llvm/ADT/Optional.h"
+#include "llvm/ADT/StringRef.h"
 
 namespace clang {
 namespace spirv {
@@ -93,10 +94,10 @@ public:
   // Instruction building methods.
   InstBuilder &opNop();
   InstBuilder &opUndef(uint32_t result_type, uint32_t result_id);
-  InstBuilder &opSourceContinued(std::string continued_source);
+  InstBuilder &opSourceContinued(llvm::StringRef continued_source);
   InstBuilder &opSource(spv::SourceLanguage source_language, uint32_t version,
                         llvm::Optional<uint32_t> file,
-                        llvm::Optional<std::string> source);
+                        llvm::Optional<llvm::StringRef> source);
   InstBuilder &opSourceExtension(std::string extension);
   InstBuilder &opName(uint32_t target, std::string name);
   InstBuilder &opMemberName(uint32_t type, uint32_t member, std::string name);
@@ -1092,7 +1093,7 @@ private:
   void encodeMemoryAccess(spv::MemoryAccessMask value);
   void encodeExecutionMode(spv::ExecutionMode value);
   void encodeDecoration(spv::Decoration value);
-  void encodeString(std::string value);
+  void encodeString(llvm::StringRef value);
 
   WordConsumer TheConsumer;
   std::vector<uint32_t> TheInst;       ///< The instruction under construction.

tools/clang/include/clang/SPIRV/ModuleBuilder.h

@@ -341,6 +341,8 @@ public:
   /// \brief Sets the source file name and the <result-id> for the OpString for
   /// the file name.
   inline void setSourceFileName(uint32_t id, std::string name);
+  /// \brief Sets the main source file content.
+  inline void setSourceFileContent(llvm::StringRef content);
 
   /// \brief Adds an execution mode to the module under construction.
   void addExecutionMode(uint32_t entryPointId, spv::ExecutionMode em,
@@ -553,6 +555,10 @@ void ModuleBuilder::setSourceFileName(uint32_t id, std::string name) {
   theModule.setSourceFileName(id, std::move(name));
 }
 
+void ModuleBuilder::setSourceFileContent(llvm::StringRef content) {
+  theModule.setSourceFileContent(content);
+}
+
 } // end namespace spirv
 } // end namespace clang
 

tools/clang/include/clang/SPIRV/Structure.h

@@ -308,6 +308,7 @@ public:
   inline void addExecutionMode(Instruction &&);
   inline void setShaderModelVersion(uint32_t);
   inline void setSourceFileName(uint32_t id, std::string name);
+  inline void setSourceFileContent(llvm::StringRef content);
   // TODO: source code debug information
   inline void addDebugName(uint32_t targetId, llvm::StringRef name,
                            llvm::Optional<uint32_t> memberIndex = llvm::None);
@@ -341,6 +342,7 @@ private:
   uint32_t shaderModelVersion;
   uint32_t sourceFileNameId; // The <result-id> for the OpString for file name
   std::string sourceFileName;
+  llvm::StringRef sourceFileContent;
   // TODO: source code debug information
   std::set<DebugName> debugNames;
   llvm::SetVector<std::pair<uint32_t, const Decoration *>> decorations;
@@ -503,6 +505,10 @@ void SPIRVModule::setSourceFileName(uint32_t id, std::string name) {
   sourceFileName = std::move(name);
 }
 
+void SPIRVModule::setSourceFileContent(llvm::StringRef content) {
+  sourceFileContent = content;
+}
+
 void SPIRVModule::addDebugName(uint32_t targetId, llvm::StringRef name,
                                llvm::Optional<uint32_t> memberIndex) {
 

tools/clang/lib/SPIRV/InstBuilderAuto.cpp

@@ -116,7 +116,7 @@ InstBuilder &InstBuilder::opUndef(uint32_t result_type, uint32_t result_id) {
   return *this;
 }
 
-InstBuilder &InstBuilder::opSourceContinued(std::string continued_source) {
+InstBuilder &InstBuilder::opSourceContinued(llvm::StringRef continued_source) {
   if (!TheInst.empty()) {
     TheStatus = Status::NestedInst;
     return *this;
@@ -132,7 +132,7 @@ InstBuilder &InstBuilder::opSourceContinued(std::string continued_source) {
 InstBuilder &InstBuilder::opSource(spv::SourceLanguage source_language,
                                    uint32_t version,
                                    llvm::Optional<uint32_t> file,
-                                   llvm::Optional<std::string> source) {
+                                   llvm::Optional<llvm::StringRef> source) {
   if (!TheInst.empty()) {
     TheStatus = Status::NestedInst;
     return *this;

tools/clang/lib/SPIRV/InstBuilderManual.cpp

@@ -234,7 +234,7 @@ InstBuilder &InstBuilder::opSpecConstant(uint32_t resultType, uint32_t resultId,
   return *this;
 }
 
-void InstBuilder::encodeString(std::string value) {
+void InstBuilder::encodeString(llvm::StringRef value) {
   const auto &words = string::encodeSPIRVString(value);
   TheInst.insert(TheInst.end(), words.begin(), words.end());
 }

tools/clang/lib/SPIRV/SPIRVEmitter.cpp

@@ -610,9 +610,18 @@ SPIRVEmitter::SPIRVEmitter(CompilerInstance &ci, EmitSPIRVOptions &options)
 
   // Set debug info
   const auto &inputFiles = ci.getFrontendOpts().Inputs;
-  if (options.enableDebugInfo && !inputFiles.empty())
+  if (options.enableDebugInfo && !inputFiles.empty()) {
+    // File name
     theBuilder.setSourceFileName(theContext.takeNextId(),
                                  inputFiles.front().getFile().str());
+
+    // Source code
+    const auto &sm = ci.getSourceManager();
+    const llvm::MemoryBuffer *mainFile =
+        sm.getBuffer(sm.getMainFileID(), SourceLocation());
+    theBuilder.setSourceFileContent(
+        StringRef(mainFile->getBufferStart(), mainFile->getBufferSize()));
+  }
 }
 
 void SPIRVEmitter::HandleTranslationUnit(ASTContext &context) {

tools/clang/lib/SPIRV/Structure.cpp

@@ -17,6 +17,32 @@ namespace spirv {
 namespace {
 constexpr uint32_t kGeneratorNumber = 14;
 constexpr uint32_t kToolVersion = 0;
+
+/// Chops the given original string into multiple smaller ones to make sure they
+/// can be encoded in a sequence of OpSourceContinued instructions following an
+/// OpSource instruction.
+void chopString(llvm::StringRef original,
+                llvm::SmallVectorImpl<llvm::StringRef> *chopped) {
+  const uint32_t maxCharInOpSource = 0xFFFFu - 5u; // Minus operands and nul
+  const uint32_t maxCharInContinue = 0xFFFFu - 2u; // Minus opcode and nul
+
+  chopped->clear();
+  if (original.size() > maxCharInOpSource) {
+    chopped->push_back(llvm::StringRef(original.data(), maxCharInOpSource));
+    original = llvm::StringRef(original.data() + maxCharInOpSource,
+                               original.size() - maxCharInOpSource);
+    while (original.size() > maxCharInContinue) {
+      chopped->push_back(llvm::StringRef(original.data(), maxCharInContinue));
+      original = llvm::StringRef(original.data() + maxCharInContinue,
+                                 original.size() - maxCharInContinue);
+    }
+    if (!original.empty()) {
+      chopped->push_back(original);
+    }
+  } else if (!original.empty()) {
+    chopped->push_back(original);
+  }
+}
 } // namespace
 
 // === Instruction implementations ===
@@ -289,10 +315,21 @@ void SPIRVModule::take(InstBuilder *builder) {
       fileName = sourceFileNameId;
     }
 
+    llvm::SmallVector<llvm::StringRef, 2> choppedSrcCode;
+    llvm::Optional<llvm::StringRef> firstSnippet;
+    chopString(sourceFileContent, &choppedSrcCode);
+    if (!choppedSrcCode.empty()) {
+      firstSnippet = llvm::Optional<llvm::StringRef>(choppedSrcCode.front());
+    }
+
     builder
         ->opSource(spv::SourceLanguage::HLSL, shaderModelVersion, fileName,
-                   llvm::None)
+                   firstSnippet)
         .x();
+
+    for (uint32_t i = 1; i < choppedSrcCode.size(); ++i) {
+      builder->opSourceContinued(choppedSrcCode[i]).x();
+    }
   }
 
   // BasicBlock debug names should be emitted only for blocks that are

tools/clang/test/CodeGenSPIRV/spirv.debug.opsource.hlsl

@@ -4,6 +4,13 @@
 // CHECK-SAME: spirv.debug.opsource.hlsl
 // CHECK:      OpSource HLSL 610 [[str]]
 
+// Make sure we have #line directive emitted by the preprocessor
+// CHECK-SAME: #line 1
+
+// Make sure we have the original source code
+// CHECK:      numthreads(8, 1, 1)
+// CHECK-NEXT: void main()
+
 [numthreads(8, 1, 1)]
 void main() {
 }

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -316,6 +316,7 @@ public:
         (argCount > 0 && pArguments == nullptr) || pEntryPoint == nullptr ||
         pTargetProfile == nullptr)
       return E_INVALIDARG;
+
     *ppResult = nullptr;
     AssignToOutOpt(nullptr, ppDebugBlobName);
     AssignToOutOpt(nullptr, ppDebugBlob);
@@ -324,23 +325,15 @@ public:
     CComPtr<IDxcBlobEncoding> utf8Source;
     CComPtr<AbstractMemoryStream> pOutputStream;
     CHeapPtr<wchar_t> DebugBlobName;
+
     DxcEtw_DXCompilerCompile_Start();
     pSourceName = (pSourceName && *pSourceName) ? pSourceName : L"hlsl.hlsl"; // declared optional, so pick a default
     DxcThreadMalloc TM(m_pMalloc);
-    IFC(hlsl::DxcGetBlobAsUtf8(pSource, &utf8Source));
 
     try {
-      CComPtr<IDxcBlob> pOutputBlob;
-      dxcutil::DxcArgsFileSystem *msfPtr =
-        dxcutil::CreateDxcArgsFileSystem(utf8Source, pSourceName, pIncludeHandler);
-      std::unique_ptr<::llvm::sys::fs::MSFileSystem> msf(msfPtr);
-
-      ::llvm::sys::fs::AutoPerThreadSystem pts(msf.get());
-      IFTLLVM(pts.error_code());
-
       IFT(CreateMemoryStream(m_pMalloc, &pOutputStream));
-      IFT(pOutputStream.QueryInterface(&pOutputBlob));
 
+      // Parse command-line options into DxcOpts
       int argCountInt;
       IFT(UIntToInt(argCount, &argCountInt));
       hlsl::options::MainArgs mainArgs(argCountInt, pArguments, 0);
@@ -348,12 +341,46 @@ public:
       CW2A pUtf8TargetProfile(pTargetProfile, CP_UTF8);
       // Set target profile before reading options and validate
       opts.TargetProfile = pUtf8TargetProfile.m_psz;
-      bool finished;
+      bool finished = false;
       ReadOptsAndValidate(mainArgs, opts, pOutputStream, ppResult, finished);
       if (finished) {
         hr = S_OK;
         goto Cleanup;
       }
+
+#ifdef ENABLE_SPIRV_CODEGEN
+      // We want to embed the preprocessed source code in the final SPIR-V if
+      // debug information is enabled. Therefore, we invoke Preprocess() here
+      // first for such case. Then we invoke the compilation process over the
+      // preprocessed source code, so that line numbers are consistent with the
+      // embedded source code.
+      CComPtr<IDxcBlob> ppSrcCode;
+      if (opts.GenSPIRV && opts.DebugInfo) {
+        CComPtr<IDxcOperationResult> ppSrcCodeResult;
+        IFT(Preprocess(pSource, pSourceName, pArguments, argCount, pDefines,
+                       defineCount, pIncludeHandler, &ppSrcCodeResult));
+        HRESULT status;
+        IFT(ppSrcCodeResult->GetStatus(&status));
+        if (SUCCEEDED(status)) {
+          IFT(ppSrcCodeResult->GetResult(&ppSrcCode));
+        }
+        pSource = ppSrcCode;
+      }
+#endif // ENABLE_SPIRV_CODEGEN
+
+      // Convert source code encoding
+      IFC(hlsl::DxcGetBlobAsUtf8(pSource, &utf8Source));
+
+      CComPtr<IDxcBlob> pOutputBlob;
+      dxcutil::DxcArgsFileSystem *msfPtr =
+        dxcutil::CreateDxcArgsFileSystem(utf8Source, pSourceName, pIncludeHandler);
+      std::unique_ptr<::llvm::sys::fs::MSFileSystem> msf(msfPtr);
+
+      ::llvm::sys::fs::AutoPerThreadSystem pts(msf.get());
+      IFTLLVM(pts.error_code());
+
+      IFT(pOutputStream.QueryInterface(&pOutputBlob));
+
       if (opts.DisplayIncludeProcess)
         msfPtr->EnableDisplayIncludeProcess();
 
@@ -500,6 +527,7 @@ public:
           spirvOpts.targetEnv = opts.SpvTargetEnv;
           spirvOpts.enable16BitTypes = opts.Enable16BitTypes;
           spirvOpts.enableDebugInfo = opts.DebugInfo;
+
           clang::EmitSPIRVAction action(spirvOpts);
           FrontendInputFile file(utf8SourceName.m_psz, IK_HLSL);
           action.BeginSourceFile(compiler, file);