Preprocess shader first, then split it into snippet when lib share compile.

tools/clang/lib/Frontend/Rewrite/CMakeLists.txt

@@ -9,7 +9,6 @@ add_clang_library(clangRewriteFrontend
   FrontendActions_rewrite.cpp
   HTMLPrint.cpp
   InclusionRewriter.cpp
-  InclusionToSnippetRewriter.cpp  # HLSL Change
   RewriteMacros.cpp
   # RewriteModernObjC.cpp # HLSL Change
   RewriteObjC.cpp

tools/clang/lib/Frontend/Rewrite/InclusionToSnippetRewriter.cpp

@@ -1,669 +0,0 @@
-//===--- InclusionToSnippetRewriter.cpp - Rewrite includes into snippets---===//
-//
-//                     The LLVM Compiler Infrastructure
-//
-// This file is distributed under the University of Illinois Open Source
-// License. See LICENSE.TXT for details.
-//
-//===----------------------------------------------------------------------===//
-//
-// This code rewrites include invocations into snippets.  This gives you
-// an ArrayRef with included files as element.
-// Clone of InclusionRewriter.
-//
-//===----------------------------------------------------------------------===//
-
-#include "clang/Rewrite/Frontend/Rewriters.h"
-#include "clang/Basic/SourceManager.h"
-#include "clang/Frontend/PreprocessorOutputOptions.h"
-#include "clang/Lex/HeaderSearch.h"
-#include "clang/Lex/Pragma.h"
-#include "clang/Lex/Preprocessor.h"
-#include "llvm/ADT/SmallString.h"
-#include "llvm/Support/raw_ostream.h"
-
-using namespace clang;
-using namespace llvm;
-
-namespace {
-
-class InclusionToSnippetRewriter : public PPCallbacks {
-  /// Information about which #includes were actually performed,
-  /// created by preprocessor callbacks.
-  struct IncludedFile {
-    FileID Id;
-    SrcMgr::CharacteristicKind FileType;
-    IncludedFile(FileID Id, SrcMgr::CharacteristicKind FileType)
-        : Id(Id), FileType(FileType) {}
-  };
-  Preprocessor &PP; ///< Used to find inclusion directives.
-  SourceManager &SM; ///< Used to read and manage source files.
-  std::string OSStr;  ///< The string for string_ostream.
-  raw_string_ostream OS; ///< The destination stream for rewritten contents.
-
-  int IfLevel; ///< Count of if which don't have endif in same snippet.
-  int EndIfLevel; ///<  Count of endif which don't have if in same snippet.
-  StringRef MainEOL; ///< The line ending marker to use.
-  const llvm::MemoryBuffer *PredefinesBuffer; ///< The preprocessor predefines.
-  bool ShowLineMarkers; ///< Show #line markers.
-  bool UseLineDirectives; ///< Use of line directives or line markers.
-  /// Tracks where inclusions that change the file are found.
-  std::map<unsigned, IncludedFile> FileIncludes;
-  /// Tracks where inclusions that import modules are found.
-  std::map<unsigned, const Module *> ModuleIncludes;
-  /// Used transitively for building up the FileIncludes mapping over the
-  /// various \c PPCallbacks callbacks.
-  SourceLocation LastInclusionLocation;
-  std::vector<std::string> &Snippets;
-  uint64_t                  SnippetOffset;
-public:
-  InclusionToSnippetRewriter(Preprocessor &PP, bool ShowLineMarkers,
-                    bool UseLineDirectives,
-                    std::vector<std::string> &Snippets);
-  bool Process(FileID FileId, SrcMgr::CharacteristicKind FileType);
-  void setPredefinesBuffer(const llvm::MemoryBuffer *Buf) {
-    PredefinesBuffer = Buf;
-  }
-  void detectMainFileEOL();
-private:
-  void FileChanged(SourceLocation Loc, FileChangeReason Reason,
-                   SrcMgr::CharacteristicKind FileType,
-                   FileID PrevFID) override;
-  void FileSkipped(const FileEntry &SkippedFile, const Token &FilenameTok,
-                   SrcMgr::CharacteristicKind FileType) override;
-  void InclusionDirective(SourceLocation HashLoc, const Token &IncludeTok,
-                          StringRef FileName, bool IsAngled,
-                          CharSourceRange FilenameRange, const FileEntry *File,
-                          StringRef SearchPath, StringRef RelativePath,
-                          const Module *Imported) override;
-  void WriteLineInfo(const char *Filename, int Line,
-                     SrcMgr::CharacteristicKind FileType,
-                     StringRef Extra = StringRef());
-  void WriteImplicitModuleImport(const Module *Mod);
-  void OutputContentUpTo(const MemoryBuffer &FromFile,
-                         unsigned &WriteFrom, unsigned WriteTo,
-                         StringRef EOL, int &lines,
-                         bool EnsureNewline);
-  void CommentOutDirective(Lexer &DirectivesLex, const Token &StartToken,
-                           const MemoryBuffer &FromFile, StringRef EOL,
-                           unsigned &NextToWrite, int &Lines);
-  bool HandleHasInclude(FileID FileId, Lexer &RawLex,
-                        const DirectoryLookup *Lookup, Token &Tok,
-                        bool &FileExists);
-  const IncludedFile *FindIncludeAtLocation(SourceLocation Loc) const;
-  const Module *FindModuleAtLocation(SourceLocation Loc) const;
-  StringRef NextIdentifierName(Lexer &RawLex, Token &RawToken);
-  void WriteSnippet(bool bStartFile, StringRef filename);
-};
-
-}  // end anonymous namespace
-
-/// Initializes an InclusionRewriter with a \p PP source and \p OS destination.
-InclusionToSnippetRewriter::InclusionToSnippetRewriter(
-    Preprocessor &PP, bool ShowLineMarkers, bool UseLineDirectives,
-    std::vector<std::string> &Snippets)
-    : PP(PP), SM(PP.getSourceManager()), OS(OSStr), IfLevel(0), EndIfLevel(0),
-      MainEOL("\n"), PredefinesBuffer(nullptr),
-      ShowLineMarkers(ShowLineMarkers), UseLineDirectives(UseLineDirectives),
-      LastInclusionLocation(SourceLocation()), Snippets(Snippets),
-      SnippetOffset(0) {}
-
-/// Write appropriate line information as either #line directives or GNU line
-/// markers depending on what mode we're in, including the \p Filename and
-/// \p Line we are located at, using the specified \p EOL line separator, and
-/// any \p Extra context specifiers in GNU line directives.
-void InclusionToSnippetRewriter::WriteLineInfo(const char *Filename, int Line,
-                                      SrcMgr::CharacteristicKind FileType,
-                                      StringRef Extra) {
-  if (!ShowLineMarkers)
-    return;
-  if (UseLineDirectives) {
-    OS << "#line" << ' ' << Line << ' ' << '"';
-    OS.write_escaped(Filename);
-    OS << '"';
-  } else {
-    // Use GNU linemarkers as described here:
-    // http://gcc.gnu.org/onlinedocs/cpp/Preprocessor-Output.html
-    OS << '#' << ' ' << Line << ' ' << '"';
-    OS.write_escaped(Filename);
-    OS << '"';
-    if (!Extra.empty())
-      OS << Extra;
-    if (FileType == SrcMgr::C_System)
-      // "`3' This indicates that the following text comes from a system header
-      // file, so certain warnings should be suppressed."
-      OS << " 3";
-    else if (FileType == SrcMgr::C_ExternCSystem)
-      // as above for `3', plus "`4' This indicates that the following text
-      // should be treated as being wrapped in an implicit extern "C" block."
-      OS << " 3 4";
-  }
-  OS << MainEOL;
-}
-
-void InclusionToSnippetRewriter::WriteImplicitModuleImport(const Module *Mod) {
-  OS << "@import " << Mod->getFullModuleName() << ";"
-     << " /* clang -frewrite-includes: implicit import */" << MainEOL;
-}
-
-/// FileChanged - Whenever the preprocessor enters or exits a #include file
-/// it invokes this handler.
-void InclusionToSnippetRewriter::FileChanged(SourceLocation Loc,
-                                    FileChangeReason Reason,
-                                    SrcMgr::CharacteristicKind NewFileType,
-                                    FileID) {
-  if (Reason != EnterFile)
-    return;
-  if (LastInclusionLocation.isInvalid())
-    // we didn't reach this file (eg: the main file) via an inclusion directive
-    return;
-  FileID Id = FullSourceLoc(Loc, SM).getFileID();
-  auto P = FileIncludes.insert(std::make_pair(
-      LastInclusionLocation.getRawEncoding(), IncludedFile(Id, NewFileType)));
-  (void)P;
-  assert(P.second && "Unexpected revisitation of the same include directive");
-  LastInclusionLocation = SourceLocation();
-}
-
-/// Called whenever an inclusion is skipped due to canonical header protection
-/// macros.
-void InclusionToSnippetRewriter::FileSkipped(const FileEntry &/*SkippedFile*/,
-                                    const Token &/*FilenameTok*/,
-                                    SrcMgr::CharacteristicKind /*FileType*/) {
-  assert(!LastInclusionLocation.isInvalid() &&
-         "A file, that wasn't found via an inclusion directive, was skipped");
-  LastInclusionLocation = SourceLocation();
-}
-
-/// This should be called whenever the preprocessor encounters include
-/// directives. It does not say whether the file has been included, but it
-/// provides more information about the directive (hash location instead
-/// of location inside the included file). It is assumed that the matching
-/// FileChanged() or FileSkipped() is called after this.
-void InclusionToSnippetRewriter::InclusionDirective(SourceLocation HashLoc,
-                                           const Token &/*IncludeTok*/,
-                                           StringRef /*FileName*/,
-                                           bool /*IsAngled*/,
-                                           CharSourceRange /*FilenameRange*/,
-                                           const FileEntry * /*File*/,
-                                           StringRef /*SearchPath*/,
-                                           StringRef /*RelativePath*/,
-                                           const Module *Imported) {
-  assert(LastInclusionLocation.isInvalid() &&
-         "Another inclusion directive was found before the previous one "
-         "was processed");
-  if (Imported) {
-    auto P = ModuleIncludes.insert(
-        std::make_pair(HashLoc.getRawEncoding(), Imported));
-    (void)P;
-    assert(P.second && "Unexpected revisitation of the same include directive");
-  } else
-    LastInclusionLocation = HashLoc;
-}
-
-/// Simple lookup for a SourceLocation (specifically one denoting the hash in
-/// an inclusion directive) in the map of inclusion information, FileChanges.
-const InclusionToSnippetRewriter::IncludedFile *
-InclusionToSnippetRewriter::FindIncludeAtLocation(SourceLocation Loc) const {
-  const auto I = FileIncludes.find(Loc.getRawEncoding());
-  if (I != FileIncludes.end())
-    return &I->second;
-  return nullptr;
-}
-
-/// Simple lookup for a SourceLocation (specifically one denoting the hash in
-/// an inclusion directive) in the map of module inclusion information.
-const Module *
-InclusionToSnippetRewriter::FindModuleAtLocation(SourceLocation Loc) const {
-  const auto I = ModuleIncludes.find(Loc.getRawEncoding());
-  if (I != ModuleIncludes.end())
-    return I->second;
-  return nullptr;
-}
-
-/// Detect the likely line ending style of \p FromFile by examining the first
-/// newline found within it.
-static StringRef DetectEOL(const MemoryBuffer &FromFile) {
-  // Detect what line endings the file uses, so that added content does not mix
-  // the style. We need to check for "\r\n" first because "\n\r" will match
-  // "\r\n\r\n".
-  const char *Pos = strchr(FromFile.getBufferStart(), '\n');
-  if (!Pos)
-    return "\n";
-  if (Pos - 1 >= FromFile.getBufferStart() && Pos[-1] == '\r')
-    return "\r\n";
-  if (Pos + 1 < FromFile.getBufferEnd() && Pos[1] == '\r')
-    return "\n\r";
-  return "\n";
-}
-
-void InclusionToSnippetRewriter::detectMainFileEOL() {
-  bool Invalid;
-  const MemoryBuffer &FromFile = *SM.getBuffer(SM.getMainFileID(), &Invalid);
-  assert(!Invalid);
-  if (Invalid)
-    return; // Should never happen, but whatever.
-  MainEOL = DetectEOL(FromFile);
-}
-
-/// Writes out bytes from \p FromFile, starting at \p NextToWrite and ending at
-/// \p WriteTo - 1.
-void InclusionToSnippetRewriter::OutputContentUpTo(const MemoryBuffer &FromFile,
-                                          unsigned &WriteFrom, unsigned WriteTo,
-                                          StringRef LocalEOL, int &Line,
-                                          bool EnsureNewline) {
-  if (WriteTo <= WriteFrom)
-    return;
-  if (&FromFile == PredefinesBuffer) {
-    // Ignore the #defines of the predefines buffer.
-    WriteFrom = WriteTo;
-    return;
-  }
-
-  // If we would output half of a line ending, advance one character to output
-  // the whole line ending.  All buffers are null terminated, so looking ahead
-  // one byte is safe.
-  if (LocalEOL.size() == 2 &&
-      LocalEOL[0] == (FromFile.getBufferStart() + WriteTo)[-1] &&
-      LocalEOL[1] == (FromFile.getBufferStart() + WriteTo)[0])
-    WriteTo++;
-
-  StringRef TextToWrite(FromFile.getBufferStart() + WriteFrom,
-                        WriteTo - WriteFrom);
-
-  if (MainEOL == LocalEOL) {
-    OS << TextToWrite;
-    // count lines manually, it's faster than getPresumedLoc()
-    Line += TextToWrite.count(LocalEOL);
-    if (EnsureNewline && !TextToWrite.endswith(LocalEOL))
-      OS << MainEOL;
-  } else {
-    // Output the file one line at a time, rewriting the line endings as we go.
-    StringRef Rest = TextToWrite;
-    while (!Rest.empty()) {
-      StringRef LineText;
-      std::tie(LineText, Rest) = Rest.split(LocalEOL);
-      OS << LineText;
-      Line++;
-      if (!Rest.empty())
-        OS << MainEOL;
-    }
-    if (TextToWrite.endswith(LocalEOL) || EnsureNewline)
-      OS << MainEOL;
-  }
-  WriteFrom = WriteTo;
-}
-
-/// Print characters from \p FromFile starting at \p NextToWrite up until the
-/// inclusion directive at \p StartToken, then print out the inclusion
-/// inclusion directive disabled by a #if directive, updating \p NextToWrite
-/// and \p Line to track the number of source lines visited and the progress
-/// through the \p FromFile buffer.
-void InclusionToSnippetRewriter::CommentOutDirective(Lexer &DirectiveLex,
-                                            const Token &StartToken,
-                                            const MemoryBuffer &FromFile,
-                                            StringRef LocalEOL,
-                                            unsigned &NextToWrite, int &Line) {
-  OutputContentUpTo(FromFile, NextToWrite,
-                    SM.getFileOffset(StartToken.getLocation()), LocalEOL, Line,
-                    false);
-  Token DirectiveToken;
-  do {
-    DirectiveLex.LexFromRawLexer(DirectiveToken);
-  } while (!DirectiveToken.is(tok::eod) && DirectiveToken.isNot(tok::eof));
-  if (&FromFile == PredefinesBuffer) {
-    // OutputContentUpTo() would not output anything anyway.
-    return;
-  }
-  OS << "#if 0 /* expanded by -frewrite-includes */" << MainEOL;
-  OutputContentUpTo(FromFile, NextToWrite,
-                    SM.getFileOffset(DirectiveToken.getLocation()) +
-                        DirectiveToken.getLength(),
-                    LocalEOL, Line, true);
-  OS << "#endif /* expanded by -frewrite-includes */" << MainEOL;
-}
-
-/// Find the next identifier in the pragma directive specified by \p RawToken.
-StringRef InclusionToSnippetRewriter::NextIdentifierName(Lexer &RawLex,
-                                                Token &RawToken) {
-  RawLex.LexFromRawLexer(RawToken);
-  if (RawToken.is(tok::raw_identifier))
-    PP.LookUpIdentifierInfo(RawToken);
-  if (RawToken.is(tok::identifier))
-    return RawToken.getIdentifierInfo()->getName();
-  return StringRef();
-}
-
-// Expand __has_include and __has_include_next if possible. If there's no
-// definitive answer return false.
-bool InclusionToSnippetRewriter::HandleHasInclude(
-    FileID FileId, Lexer &RawLex, const DirectoryLookup *Lookup, Token &Tok,
-    bool &FileExists) {
-  // Lex the opening paren.
-  RawLex.LexFromRawLexer(Tok);
-  if (Tok.isNot(tok::l_paren))
-    return false;
-
-  RawLex.LexFromRawLexer(Tok);
-
-  SmallString<128> FilenameBuffer;
-  StringRef Filename;
-  // Since the raw lexer doesn't give us angle_literals we have to parse them
-  // ourselves.
-  // FIXME: What to do if the file name is a macro?
-  if (Tok.is(tok::less)) {
-    RawLex.LexFromRawLexer(Tok);
-
-    FilenameBuffer += '<';
-    do {
-      if (Tok.is(tok::eod)) // Sanity check.
-        return false;
-
-      if (Tok.is(tok::raw_identifier))
-        PP.LookUpIdentifierInfo(Tok);
-
-      // Get the string piece.
-      SmallVector<char, 128> TmpBuffer;
-      bool Invalid = false;
-      StringRef TmpName = PP.getSpelling(Tok, TmpBuffer, &Invalid);
-      if (Invalid)
-        return false;
-
-      FilenameBuffer += TmpName;
-
-      RawLex.LexFromRawLexer(Tok);
-    } while (Tok.isNot(tok::greater));
-
-    FilenameBuffer += '>';
-    Filename = FilenameBuffer;
-  } else {
-    if (Tok.isNot(tok::string_literal))
-      return false;
-
-    bool Invalid = false;
-    Filename = PP.getSpelling(Tok, FilenameBuffer, &Invalid);
-    if (Invalid)
-      return false;
-  }
-
-  // Lex the closing paren.
-  RawLex.LexFromRawLexer(Tok);
-  if (Tok.isNot(tok::r_paren))
-    return false;
-
-  // Now ask HeaderInfo if it knows about the header.
-  // FIXME: Subframeworks aren't handled here. Do we care?
-  bool isAngled = PP.GetIncludeFilenameSpelling(Tok.getLocation(), Filename);
-  const DirectoryLookup *CurDir;
-  const FileEntry *FileEnt = PP.getSourceManager().getFileEntryForID(FileId);
-  SmallVector<std::pair<const FileEntry *, const DirectoryEntry *>, 1>
-      Includers;
-  Includers.push_back(std::make_pair(FileEnt, FileEnt->getDir()));
-  const FileEntry *File = PP.getHeaderSearchInfo().LookupFile(
-      Filename, SourceLocation(), isAngled, nullptr, CurDir, Includers, nullptr,
-      nullptr, nullptr, false);
-
-  FileExists = File != nullptr;
-  return true;
-}
-
-void InclusionToSnippetRewriter::WriteSnippet(bool bStartFile, StringRef filename) {
-  if (OS.GetNumBytesInBuffer()) {
-    if (IfLevel == 0 && EndIfLevel == 0) {
-      Snippets.emplace_back(OS.str().substr(SnippetOffset));
-    } else {
-      std::string endIfPatch;
-      if (IfLevel > 0) {
-        // #if without #endif.
-        for (int i = 0; i < IfLevel; i++) {
-          endIfPatch = endIfPatch + "\n#endif";
-        }
-      }
-
-      std::string ifPatch;
-      if (EndIfLevel > 0) {
-        // #endif without #if.
-        // Need add #if 1 at beginning.
-        // Use #if 1 because new file only open when inside true condition.
-        IfLevel = -IfLevel;
-        for (int i = 0; i < EndIfLevel; i++) {
-          ifPatch = ifPatch + "#if 1\n";
-        }
-      }
-      Snippets.emplace_back(ifPatch + OS.str().substr(SnippetOffset) + endIfPatch);
-    }
-
-    // Clear level.
-    IfLevel = 0;
-    EndIfLevel = 0;
-    SnippetOffset = OSStr.size();
-  }
-}
-
-/// Use a raw lexer to analyze \p FileId, incrementally copying parts of it
-/// and including content of included files recursively.
-bool InclusionToSnippetRewriter::Process(FileID FileId,
-                                SrcMgr::CharacteristicKind FileType)
-{
-
-  bool Invalid;
-  const MemoryBuffer &FromFile = *SM.getBuffer(FileId, &Invalid);
-  assert(!Invalid && "Attempting to process invalid inclusion");
-  const char *FileName = FromFile.getBufferIdentifier();
-
-
-  // Write Snippet before Process a file.
-  WriteSnippet(/*bStartFile*/true, FileName);
-
-
-  Lexer RawLex(FileId, &FromFile, PP.getSourceManager(), PP.getLangOpts());
-  RawLex.SetCommentRetentionState(false);
-
-  StringRef LocalEOL = DetectEOL(FromFile);
-
-  // Per the GNU docs: "1" indicates entering a new file.
-  if (FileId == SM.getMainFileID() || FileId == PP.getPredefinesFileID())
-    WriteLineInfo(FileName, 1, FileType, "");
-  else
-    WriteLineInfo(FileName, 1, FileType, " 1");
-
-  if (SM.getFileIDSize(FileId) == 0)
-    return false;
-
-  // The next byte to be copied from the source file, which may be non-zero if
-  // the lexer handled a BOM.
-  unsigned NextToWrite = SM.getFileOffset(RawLex.getSourceLocation());
-  assert(SM.getLineNumber(FileId, NextToWrite) == 1);
-  int Line = 1; // The current input file line number.
-
-  Token RawToken;
-  RawLex.LexFromRawLexer(RawToken);
-
-  // TODO: Consider adding a switch that strips possibly unimportant content,
-  // such as comments, to reduce the size of repro files.
-  while (RawToken.isNot(tok::eof)) {
-    if (RawToken.is(tok::hash) && RawToken.isAtStartOfLine()) {
-      RawLex.setParsingPreprocessorDirective(true);
-      Token HashToken = RawToken;
-      RawLex.LexFromRawLexer(RawToken);
-      if (RawToken.is(tok::raw_identifier))
-        PP.LookUpIdentifierInfo(RawToken);
-      if (RawToken.getIdentifierInfo() != nullptr) {
-        switch (RawToken.getIdentifierInfo()->getPPKeywordID()) {
-          case tok::pp_include:
-          case tok::pp_include_next:
-          case tok::pp_import: {
-            CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL, NextToWrite,
-              Line);
-            if (FileId != PP.getPredefinesFileID())
-              WriteLineInfo(FileName, Line - 1, FileType, "");
-            StringRef LineInfoExtra;
-            SourceLocation Loc = HashToken.getLocation();
-            if (const Module *Mod = FindModuleAtLocation(Loc))
-              WriteImplicitModuleImport(Mod);
-            else if (const IncludedFile *Inc = FindIncludeAtLocation(Loc)) {
-              // include and recursively process the file
-              if (Process(Inc->Id, Inc->FileType)) {
-                // and set lineinfo back to this file, if the nested one was
-                // actually included
-                // `2' indicates returning to a file (after having included
-                // another file.
-                LineInfoExtra = " 2";
-              }
-            }
-            // fix up lineinfo (since commented out directive changed line
-            // numbers) for inclusions that were skipped due to header guards
-            WriteLineInfo(FileName, Line, FileType, LineInfoExtra);
-            break;
-          }
-          case tok::pp_pragma: {
-            StringRef Identifier = NextIdentifierName(RawLex, RawToken);
-            if (Identifier == "clang" || Identifier == "GCC") {
-              if (NextIdentifierName(RawLex, RawToken) == "system_header") {
-                // keep the directive in, commented out
-                CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
-                  NextToWrite, Line);
-                // update our own type
-                FileType = SM.getFileCharacteristic(RawToken.getLocation());
-                WriteLineInfo(FileName, Line, FileType);
-              }
-            } else if (Identifier == "once") {
-              // keep the directive in, commented out
-              CommentOutDirective(RawLex, HashToken, FromFile, LocalEOL,
-                NextToWrite, Line);
-              WriteLineInfo(FileName, Line, FileType);
-            }
-            break;
-          }
-          // Update IfLevel
-          case tok::pp_ifdef:
-          case tok::pp_ifndef:
-            IfLevel++;
-            break;
-          case tok::pp_if: {
-            IfLevel++;
-          }
-          case tok::pp_elif: {
-            bool elif = (RawToken.getIdentifierInfo()->getPPKeywordID() ==
-                         tok::pp_elif);
-            // Rewrite special builtin macros to avoid pulling in host details.
-            do {
-              // Walk over the directive.
-              RawLex.LexFromRawLexer(RawToken);
-              if (RawToken.is(tok::raw_identifier))
-                PP.LookUpIdentifierInfo(RawToken);
-
-              if (RawToken.is(tok::identifier)) {
-                bool HasFile;
-                SourceLocation Loc = RawToken.getLocation();
-
-                // Rewrite __has_include(x)
-                if (RawToken.getIdentifierInfo()->isStr("__has_include")) {
-                  if (!HandleHasInclude(FileId, RawLex, nullptr, RawToken,
-                                        HasFile))
-                    continue;
-                  // Rewrite __has_include_next(x)
-                } else if (RawToken.getIdentifierInfo()->isStr(
-                               "__has_include_next")) {
-                  const DirectoryLookup *Lookup = PP.GetCurDirLookup();
-                  if (Lookup)
-                    ++Lookup;
-
-                  if (!HandleHasInclude(FileId, RawLex, Lookup, RawToken,
-                                        HasFile))
-                    continue;
-                } else {
-                  continue;
-                }
-                // Replace the macro with (0) or (1), followed by the commented
-                // out macro for reference.
-                OutputContentUpTo(FromFile, NextToWrite, SM.getFileOffset(Loc),
-                                  LocalEOL, Line, false);
-                OS << '(' << (int) HasFile << ")/*";
-                OutputContentUpTo(FromFile, NextToWrite,
-                                  SM.getFileOffset(RawToken.getLocation()) +
-                                      RawToken.getLength(),
-                                  LocalEOL, Line, false);
-                OS << "*/";
-              }
-            } while (RawToken.isNot(tok::eod));
-            if (elif) {
-              OutputContentUpTo(FromFile, NextToWrite,
-                                SM.getFileOffset(RawToken.getLocation()) +
-                                    RawToken.getLength(),
-                                LocalEOL, Line, /*EnsureNewline=*/ true);
-              WriteLineInfo(FileName, Line, FileType);
-            }
-            break;
-          }
-          // Update IfLevel
-          case tok::pp_endif: {
-            IfLevel--;
-            if (IfLevel<0) {
-              IfLevel = 0;
-              EndIfLevel++;
-            }
-          }
-          case tok::pp_else: {
-            // We surround every #include by #if 0 to comment it out, but that
-            // changes line numbers. These are fixed up right after that, but
-            // the whole #include could be inside a preprocessor conditional
-            // that is not processed. So it is necessary to fix the line
-            // numbers one the next line after each #else/#endif as well.
-            RawLex.SetKeepWhitespaceMode(true);
-            do {
-              RawLex.LexFromRawLexer(RawToken);
-            } while (RawToken.isNot(tok::eod) && RawToken.isNot(tok::eof));
-            OutputContentUpTo(FromFile, NextToWrite,
-                              SM.getFileOffset(RawToken.getLocation()) +
-                                  RawToken.getLength(),
-                              LocalEOL, Line, /*EnsureNewline=*/ true);
-            WriteLineInfo(FileName, Line, FileType);
-            RawLex.SetKeepWhitespaceMode(false);
-          }
-          default:
-            break;
-        }
-      }
-      RawLex.setParsingPreprocessorDirective(false);
-    }
-    RawLex.LexFromRawLexer(RawToken);
-  }
-  OutputContentUpTo(FromFile, NextToWrite,
-                    SM.getFileOffset(SM.getLocForEndOfFile(FileId)), LocalEOL,
-                    Line, /*EnsureNewline=*/true);
-
-  // Write Snippet after Process a file.
-  WriteSnippet(/*bStartFile*/false, FileName);
-  return true;
-}
-
-/// RewriteIncludesToSnippet - Write include files into snippets.
-void clang::RewriteIncludesToSnippet(Preprocessor &PP,
-                                     const PreprocessorOutputOptions &Opts,
-                                     std::vector<std::string> &Snippets) {
-  SourceManager &SM = PP.getSourceManager();
-  InclusionToSnippetRewriter *Rewrite = new InclusionToSnippetRewriter(
-      PP, Opts.ShowLineMarkers, Opts.UseLineDirectives, Snippets);
-  Rewrite->detectMainFileEOL();
-
-  PP.addPPCallbacks(std::unique_ptr<PPCallbacks>(Rewrite));
-  PP.IgnorePragmas();
-
-  // First let the preprocessor process the entire file and call callbacks.
-  // Callbacks will record which #include's were actually performed.
-  PP.EnterMainSourceFile();
-  Token Tok;
-  // Only preprocessor directives matter here, so disable macro expansion
-  // everywhere else as an optimization.
-  // TODO: It would be even faster if the preprocessor could be switched
-  // to a mode where it would parse only preprocessor directives and comments,
-  // nothing else matters for parsing or processing.
-  PP.SetMacroExpansionOnlyInDirectives();
-  do {
-    PP.Lex(Tok);
-  } while (Tok.isNot(tok::eof));
-  Rewrite->setPredefinesBuffer(SM.getBuffer(PP.getPredefinesFileID()));
-  Rewrite->Process(PP.getPredefinesFileID(), SrcMgr::C_User);
-  Rewrite->Process(SM.getMainFileID(), SrcMgr::C_User);
-}

tools/clang/tools/dxlib-sample/CMakeLists.txt

@@ -25,10 +25,6 @@ add_clang_library(dxlib_sample SHARED ${SOURCES})
 
 target_link_libraries(dxlib_sample
   PRIVATE
-  clangAST
-  clangCodeGen
-  clangSema
-  clangRewriteFrontend
   dxcompiler
   )
 

tools/clang/tools/dxlib-sample/lib_cache_manager.cpp

@@ -122,12 +122,7 @@ hash_code LibCacheManagerImpl::GetHash(const std::string &header, const std::str
   for (auto &Arg : compiler.arguments) {
     libHash = CombineWStr(libHash, Arg);
   }
-  for (auto &Define : compiler.defines) {
-    libHash = CombineWStr(libHash, Define.Name);
-    if (Define.Value) {
-      libHash = CombineWStr(libHash, Define.Value);
-    }
-  }
+  // snippet has been processed so don't add define to hash.
   return libHash;
 }
 

tools/clang/tools/dxlib-sample/lib_share_compile.cpp

@@ -118,7 +118,7 @@ HRESULT CompileFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName,
 
     // Preprocess.
     std::unique_ptr<IncludeToLibPreprocessor> preprocessor = IncludeToLibPreprocessor::CreateIncludeToLibPreprocessor(pInclude);
-    preprocessor->SetupDefines(defines.data(), defines.size());
+
     if (arguments.size()) {
       CComPtr<AbstractMemoryStream> pOutputStream;
       IFT(CreateMemoryStream(GetGlobalHeapMalloc(), &pOutputStream));
@@ -145,7 +145,8 @@ HRESULT CompileFromBlob(IDxcBlobEncoding *pSource, LPCWSTR pSourceName,
       }
     }
 
-    preprocessor->Preprocess(pSource, pSourceName);
+    preprocessor->Preprocess(pSource, pSourceName, arguments.data(),
+                             arguments.size(), defines.data(), defines.size());
 
     CompileInput compilerInput{defines, arguments};
 

tools/clang/tools/dxlib-sample/lib_share_helper.h

@@ -50,9 +50,11 @@ class IncludeToLibPreprocessor {
 public:
   virtual ~IncludeToLibPreprocessor() {}
 
-  virtual void SetupDefines(const DxcDefine *pDefines, unsigned defineCount) = 0;
   virtual void AddIncPath(llvm::StringRef path) = 0;
-  virtual HRESULT Preprocess(IDxcBlob *pSource, LPCWSTR pFilename) = 0;
+  virtual HRESULT Preprocess(IDxcBlob *pSource, LPCWSTR pFilename,
+                             LPCWSTR *pArguments, UINT32 argCount,
+                             const DxcDefine *pDefines,
+                             unsigned defineCount) = 0;
 
   virtual const std::vector<std::string> &GetSnippets() const = 0;
   static std::unique_ptr<IncludeToLibPreprocessor>

tools/clang/tools/dxlib-sample/lib_share_preprocessor.cpp

@@ -12,19 +12,14 @@
 #include "dxc/Support/WinIncludes.h"
 #include "dxc/Support/Global.h"
 #include "dxc/Support/microcom.h"
-#include "clang/Rewrite/Frontend/Rewriters.h"
 #include "dxc/Support/FileIOHelper.h"
 
-#include "clang/Frontend/TextDiagnosticPrinter.h"
-
-#include "clang/Frontend/ASTUnit.h"
-#include "clang/Basic/TargetInfo.h"
-#include "clang/Frontend/CompilerInstance.h"
-#include "clang/Lex/Preprocessor.h"
-#include "clang/Sema/SemaConsumer.h"
 #include "dxc/Support/dxcfilesystem.h"
 #include "lib_share_helper.h"
 #include "llvm/ADT/STLExtras.h"
+#include "llvm/ADT/StringSet.h"
+#include "llvm/Support/raw_ostream.h"
+#include "llvm/Support/Path.h"
 
 using namespace libshare;
 
@@ -49,6 +44,9 @@ bool IsAbsoluteOrCurDirRelative(const Twine &T) {
 }
 
 namespace {
+
+const StringRef file_region = "\n#pragma region\n";
+
 class IncPathIncludeHandler : public IDxcIncludeHandler {
 public:
   DXC_MICROCOM_ADDREF_RELEASE_IMPL(m_dwRef)
@@ -81,7 +79,6 @@ public:
     CComPtr<IDxcBlob> pIncludeSource;
     if (m_includePathList.empty()) {
       IFT(m_pIncludeHandler->LoadSource(pFilename, ppIncludeSource));
-      return S_OK;
     } else {
       bool bLoaded = false;
       // Not support same filename in different directory.
@@ -96,11 +93,22 @@ public:
       }
       if (!bLoaded) {
         IFT(m_pIncludeHandler->LoadSource(pFilename, ppIncludeSource));
-        return S_OK;
-      } else {
-        return S_OK;
       }
     }
+
+    CComPtr<IDxcBlobEncoding> utf8Source;
+    IFT(hlsl::DxcGetBlobAsUtf8(*ppIncludeSource, &utf8Source));
+
+    StringRef Data((LPSTR)utf8Source->GetBufferPointer());
+    Twine regionData = file_region + Data + file_region;
+    std::string strRegionData = regionData.str();
+
+    CComPtr<IDxcBlobEncoding> pEncodingIncludeSource;
+    IFT(DxcCreateBlobWithEncodingOnMallocCopy(
+        GetGlobalHeapMalloc(), strRegionData.c_str(), strRegionData.size(),
+        CP_UTF8, &pEncodingIncludeSource));
+    *ppIncludeSource = pEncodingIncludeSource.Detach();
+    return S_OK;
   }
 
 private:
@@ -112,6 +120,8 @@ private:
 
 } // namespace
 
+HRESULT CreateCompiler(IDxcCompiler **ppCompiler);
+
 namespace {
 
 class IncludeToLibPreprocessorImpl : public IncludeToLibPreprocessor {
@@ -122,155 +132,42 @@ public:
       : m_pIncludeHandler(handler) {
   }
 
-  void SetupDefines(const DxcDefine *pDefines, unsigned defineCount) override;
   void AddIncPath(StringRef path) override;
-  HRESULT Preprocess(IDxcBlob *pSource, LPCWSTR pFilename) override;
+  HRESULT Preprocess(IDxcBlob *pSource, LPCWSTR pFilename, LPCWSTR *pArguments,
+                     UINT32 argCount, const DxcDefine *pDefines,
+                     unsigned defineCount) override;
 
   const std::vector<std::string> &GetSnippets() const override { return m_snippets; }
 
 private:
+  HRESULT SplitShaderIntoSnippets(IDxcBlob *pSource);
   IDxcIncludeHandler *m_pIncludeHandler;
   // Snippets split by #include.
   std::vector<std::string> m_snippets;
-  // Defines.
-  std::vector<std::wstring> m_defineStrs;
-  std::vector<DxcDefine> m_defines;
   std::vector<std::string> m_includePathList;
 };
 
-void IncludeToLibPreprocessorImpl::SetupDefines(const DxcDefine *pDefines,
-                                            unsigned defineCount) {
-  // Make sure no resize.
-  m_defineStrs.reserve(defineCount * 2);
-  m_defines.reserve(defineCount);
-  for (unsigned i = 0; i < defineCount; i++) {
-    const DxcDefine &define = pDefines[i];
+HRESULT IncludeToLibPreprocessorImpl::SplitShaderIntoSnippets(IDxcBlob *pSource) {
+  CComPtr<IDxcBlobEncoding> utf8Source;
+  IFT(hlsl::DxcGetBlobAsUtf8(pSource, &utf8Source));
 
-    m_defineStrs.emplace_back(define.Name);
-    DxcDefine tmpDefine;
-    tmpDefine.Name = m_defineStrs.back().c_str();
-    tmpDefine.Value = nullptr;
-    if (define.Value) {
-      m_defineStrs.emplace_back(define.Value);
-      tmpDefine.Value = m_defineStrs.back().c_str();
-    }
-    m_defines.emplace_back(tmpDefine);
+  StringRef Data((LPSTR)utf8Source->GetBufferPointer(),
+                 utf8Source->GetBufferSize());
+  SmallVector<StringRef,8> splitResult;
+  Data.split(splitResult, file_region, /*maxSplit*/-1, /*keepEmpty*/false);
+  for (StringRef snippet : splitResult) {
+    m_snippets.emplace_back(snippet);
   }
+  return S_OK;
 }
 
 void IncludeToLibPreprocessorImpl::AddIncPath(StringRef path) {
   m_includePathList.emplace_back(path);
 }
 
-using namespace clang;
-static
-void SetupCompilerForRewrite(CompilerInstance &compiler,
-                             _In_ DxcLangExtensionsHelperApply *helper,
-                             _In_ LPCSTR pMainFile,
-                             _In_ TextDiagnosticPrinter *diagPrinter,
-                             _In_opt_ clang::ASTUnit::RemappedFile *rewrite,
-                             _In_opt_ LPCSTR pDefines) {
-  // Setup a compiler instance.
-  std::shared_ptr<TargetOptions> targetOptions(new TargetOptions);
-  targetOptions->Triple = llvm::sys::getDefaultTargetTriple();
-  compiler.HlslLangExtensions = helper;
-  compiler.createDiagnostics(diagPrinter, false);
-  compiler.createFileManager();
-  compiler.createSourceManager(compiler.getFileManager());
-  compiler.setTarget(TargetInfo::CreateTargetInfo(compiler.getDiagnostics(), targetOptions));
-  // Not use builtin includes.
-  compiler.getHeaderSearchOpts().UseBuiltinIncludes = false;
-
-  PreprocessorOptions &PPOpts = compiler.getPreprocessorOpts();
-  if (rewrite != nullptr) {
-    if (llvm::MemoryBuffer *pMemBuf = rewrite->second) {
-      compiler.getPreprocessorOpts().addRemappedFile(StringRef(pMainFile), pMemBuf);
-    }
-
-    PPOpts.RemappedFilesKeepOriginalName = true;
-  }
-
-  PreprocessorOutputOptions &PPOutOpts = compiler.getPreprocessorOutputOpts();
-  PPOutOpts.ShowLineMarkers = false;
-
-  compiler.createPreprocessor(TU_Complete);
-
-  if (pDefines) {
-    std::string newDefines = compiler.getPreprocessor().getPredefines();
-    newDefines += pDefines;
-    compiler.getPreprocessor().setPredefines(newDefines);
-  }
-
-  compiler.createASTContext();
-  compiler.setASTConsumer(std::unique_ptr<ASTConsumer>(new SemaConsumer()));
-  compiler.createSema(TU_Complete, nullptr);
-
-  const FileEntry *mainFileEntry = compiler.getFileManager().getFile(StringRef(pMainFile));
-  if (mainFileEntry == nullptr) {
-    throw ::hlsl::Exception(HRESULT_FROM_WIN32(ERROR_FILE_NOT_FOUND));
-  }
-  compiler.getSourceManager().setMainFileID(
-    compiler.getSourceManager().createFileID(mainFileEntry, SourceLocation(), SrcMgr::C_User));
-}
-
-static std::string DefinesToString(_In_count_(defineCount) DxcDefine *pDefines,
-                                   _In_ UINT32 defineCount) {
-  std::string defineStr;
-  for (UINT32 i = 0; i < defineCount; i++) {
-    CW2A utf8Name(pDefines[i].Name, CP_UTF8);
-    CW2A utf8Value(pDefines[i].Value, CP_UTF8);
-    defineStr += "#define ";
-    defineStr += utf8Name;
-    defineStr += " ";
-    defineStr += utf8Value ? utf8Value : "1";
-    defineStr += "\n";
-  }
-
-  return defineStr;
-}
-
-static void RewriteToSnippets(IDxcBlob *pSource, LPCWSTR pFilename,
-    std::vector<DxcDefine> &defines, IDxcIncludeHandler *pIncHandler,
-    std::vector<std::string> &Snippets) {
-  std::string warnings;
-  raw_string_ostream w(warnings);
-
-  CW2A pUtf8Name(pFilename);
-  dxcutil::DxcArgsFileSystem *msfPtr =
-      dxcutil::CreateDxcArgsFileSystem(pSource, pFilename, pIncHandler);
-  std::unique_ptr<::llvm::sys::fs::MSFileSystem> msf(msfPtr);
-
-  ::llvm::sys::fs::AutoPerThreadSystem pts(msf.get());
-  IFTLLVM(pts.error_code());
-
-  StringRef Data((LPSTR)pSource->GetBufferPointer(), pSource->GetBufferSize());
-  std::unique_ptr<llvm::MemoryBuffer> pBuffer(
-      llvm::MemoryBuffer::getMemBufferCopy(Data, pUtf8Name.m_psz));
-  std::unique_ptr<clang::ASTUnit::RemappedFile> pRemap(
-      new clang::ASTUnit::RemappedFile(pUtf8Name.m_psz, pBuffer.release()));
-
-  std::string definesStr = DefinesToString(defines.data(), defines.size());
-  // Not support langExt yet.
-  DxcLangExtensionsHelperApply *pLangExtensionsHelper = nullptr;
-  // Setup a compiler instance.
-  clang::CompilerInstance compiler;
-  std::unique_ptr<clang::TextDiagnosticPrinter> diagPrinter =
-      llvm::make_unique<clang::TextDiagnosticPrinter>(
-          w, &compiler.getDiagnosticOpts());
-  SetupCompilerForRewrite(compiler, pLangExtensionsHelper, pUtf8Name.m_psz,
-                          diagPrinter.get(), pRemap.get(), definesStr.c_str());
-
-  // Parse the source file.
-  compiler.getDiagnosticClient().BeginSourceFile(compiler.getLangOpts(),
-                                                 &compiler.getPreprocessor());
-
-  clang::RewriteIncludesToSnippet(compiler.getPreprocessor(),
-                                  compiler.getPreprocessorOutputOpts(),
-                                  Snippets);
-}
-
-HRESULT IncludeToLibPreprocessorImpl::Preprocess(IDxcBlob *pSource,
-                                             LPCWSTR pFilename) {
+HRESULT IncludeToLibPreprocessorImpl::Preprocess(
+    IDxcBlob *pSource, LPCWSTR pFilename, LPCWSTR *pArguments, UINT32 argCount,
+    const DxcDefine *pDefines, unsigned defineCount) {
   std::string s, warnings;
   raw_string_ostream o(s);
   raw_string_ostream w(warnings);
@@ -281,7 +178,23 @@ HRESULT IncludeToLibPreprocessorImpl::Preprocess(IDxcBlob *pSource,
   // AddRef to hold incPathIncludeHandler.
   // If not, DxcArgsFileSystem will kill it.
   incPathIncludeHandler.AddRef();
-  RewriteToSnippets(pSource, pFilename, m_defines, &incPathIncludeHandler, m_snippets);
+
+  CComPtr<IDxcCompiler> compiler;
+  CComPtr<IDxcOperationResult> pPreprocessResult;
+
+  IFR(CreateCompiler(&compiler));
+
+  IFR(compiler->Preprocess(pSource, L"input.hlsl", pArguments, argCount,
+                           pDefines, defineCount, &incPathIncludeHandler,
+                           &pPreprocessResult));
+
+  HRESULT status;
+  IFT(pPreprocessResult->GetStatus(&status));
+  if (SUCCEEDED(status)) {
+    CComPtr<IDxcBlob> pProgram;
+    IFT(pPreprocessResult->GetResult(&pProgram));
+    IFT(SplitShaderIntoSnippets(pProgram));
+  }
   return S_OK;
 }
 } // namespace