Add an option to allow pre-expand token-pasting (##) operands (#1065)

The fxc compiler evaluates both operands before performing the
token-pasting operation. But the default Clang way is to follow
C standard, which token-pastes operands as-is, without any
pre-expanding.

This commit adds support for the fxc behavior via the command
line option: -flegacy-macro-expansion.

Fixes https://github.com/Microsoft/DirectXShaderCompiler/issues/1005

include/dxc/Support/HLSLOptions.h

@@ -152,6 +152,8 @@ public:
   bool DisassembleInstNumbers; //OPT_Ni
   bool DisassembleByteOffset; //OPT_No
   bool DisaseembleHex; //OPT_Lx
+  bool LegacyMacroExpansion; // OPT_flegacy_macro_expansion
+
   bool IsRootSignatureProfile();
   bool IsLibraryProfile();
 

include/dxc/Support/HLSLOptions.td

@@ -349,3 +349,6 @@ def nologo : Flag<["-", "/"], "nologo">, Group<hlslcore_Group>, Flags<[DriverOpt
 
 // Also removed: compress, decompress, /Gch (child effect), /Gec (back compat), /Gpp (partial precision)
 // /Op - no support for preshaders.
+
+def flegacy_macro_expansion : Flag<["-"], "flegacy-macro-expansion">, Group<hlslcomp_Group>, Flags<[CoreOption, DriverOption]>,
+    HelpText<"Expand the operands before performing token-pasting operation (fxc behavior)">;

lib/DxcSupport/HLSLOptions.cpp

@@ -406,6 +406,7 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
   opts.DisassembleInstNumbers = Args.hasFlag(OPT_Ni, OPT_INVALID, false);
   opts.DisassembleByteOffset = Args.hasFlag(OPT_No, OPT_INVALID, false);
   opts.DisaseembleHex = Args.hasFlag(OPT_Lx, OPT_INVALID, false);
+  opts.LegacyMacroExpansion = Args.hasFlag(OPT_flegacy_macro_expansion, OPT_INVALID, false);
 
   if (opts.DefaultColMajor && opts.DefaultRowMajor) {
     errors << "Cannot specify /Zpr and /Zpc together, use /? to get usage information";

tools/clang/include/clang/Lex/PreprocessorOptions.h

@@ -58,6 +58,8 @@ public:
   // HLSL Change Begin - ignore line directives.
   /// \brief Whether we should ignore #line directives.
   unsigned IgnoreLineDirectives : 1;
+  /// \brief Expand the operands before performing token-pasting (fxc behavior)
+  unsigned ExpandTokPastingArg : 1;
   // HLSL Change End
 
   /// The implicit PCH included at the start of the translation unit, or empty.

tools/clang/lib/Lex/TokenLexer.cpp

@@ -17,6 +17,7 @@
 #include "clang/Lex/MacroArgs.h"
 #include "clang/Lex/MacroInfo.h"
 #include "clang/Lex/Preprocessor.h"
+#include "clang/Lex/PreprocessorOptions.h" // HLSL Change
 #include "llvm/ADT/SmallString.h"
 using namespace clang;
 
@@ -261,7 +262,7 @@ void TokenLexer::ExpandFunctionArguments() {
     // If it is not the LHS/RHS of a ## operator, we must pre-expand the
     // argument and substitute the expanded tokens into the result.  This is
     // C99 6.10.3.1p1.
-    if (!PasteBefore && !PasteAfter) {
+    if (PP.PPOpts.get()->ExpandTokPastingArg || !PasteBefore && !PasteAfter) { // HLSL Change
       const Token *ResultArgToks;
 
       // Only preexpand the argument if it could possibly need it.  This

tools/clang/tools/dxc/dxc.cpp

@@ -852,6 +852,10 @@ void DxcContext::Preprocess() {
   IFT(CreateInstance(CLSID_DxcLibrary, &pLibrary));
   IFT(pLibrary->CreateIncludeHandler(&pIncludeHandler));
 
+  // Carry forward the options that control preprocessor
+  if (m_Opts.LegacyMacroExpansion)
+    args.push_back(L"-flegacy-macro-expansion");
+
   ReadFileIntoBlob(m_dxcSupport, StringRefUtf16(m_Opts.InputFile), &pSource);
   IFT(CreateInstance(CLSID_DxcCompiler, &pCompiler));
   IFT(pCompiler->Preprocess(pSource, StringRefUtf16(m_Opts.InputFile), args.data(), args.size(), m_Opts.Defines.data(), m_Opts.Defines.size(), pIncludeHandler, &pPreprocessResult));

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -785,6 +785,8 @@ public:
     }
 
     PPOpts.IgnoreLineDirectives = Opts.IgnoreLineDirectives;
+    // fxc compatibility: pre-expand operands before performing token-pasting
+    PPOpts.ExpandTokPastingArg = Opts.LegacyMacroExpansion;
 
     // Pick additional arguments.
     clang::HeaderSearchOptions &HSOpts = compiler.getHeaderSearchOpts();

tools/clang/unittests/HLSL/CompilerTest.cpp

@@ -938,6 +938,7 @@ public:
   TEST_METHOD(CodeGenCBufferStructArray)
   TEST_METHOD(CodeGenPatchLength)
   TEST_METHOD(PreprocessWhenValidThenOK)
+  TEST_METHOD(PreprocessWhenExpandTokenPastingOperandThenAccept)
   TEST_METHOD(WhenSigMismatchPCFunctionThenFail)
 
   // Dx11 Sample
@@ -5840,6 +5841,56 @@ TEST_F(CompilerTest, PreprocessWhenValidThenOK) {
     "int BAR;\n", text.c_str());
 }
 
+TEST_F(CompilerTest, PreprocessWhenExpandTokenPastingOperandThenAccept) {
+  // Tests that we can turn on fxc's behavior (pre-expanding operands before
+  // performing token-pasting) using -flegacy-macro-expansion
+
+  CComPtr<IDxcCompiler> pCompiler;
+  CComPtr<IDxcOperationResult> pResult;
+  CComPtr<IDxcBlobEncoding> pSource;
+
+  LPCWSTR expandOption = L"-flegacy-macro-expansion";
+
+  VERIFY_SUCCEEDED(CreateCompiler(&pCompiler));
+
+  CreateBlobFromText(R"(
+#define SET_INDEX0                10
+#define BINDING_INDEX0            5
+
+#define SET(INDEX)                SET_INDEX##INDEX
+#define BINDING(INDEX)            BINDING_INDEX##INDEX
+
+#define SET_BIND(NAME,SET,BIND)   resource_set_##SET##_bind_##BIND##_##NAME
+
+#define RESOURCE(NAME,INDEX)      SET_BIND(NAME, SET(INDEX), BINDING(INDEX))
+
+    Texture2D<float4> resource_set_10_bind_5_tex;
+
+  float4 main() : SV_Target{
+    return RESOURCE(tex, 0)[uint2(1, 2)];
+  }
+)",
+                     &pSource);
+  VERIFY_SUCCEEDED(pCompiler->Preprocess(pSource, L"file.hlsl", &expandOption,
+                                         1, nullptr, 0, nullptr, &pResult));
+  HRESULT hrOp;
+  VERIFY_SUCCEEDED(pResult->GetStatus(&hrOp));
+  VERIFY_SUCCEEDED(hrOp);
+
+  CComPtr<IDxcBlob> pOutText;
+  VERIFY_SUCCEEDED(pResult->GetResult(&pOutText));
+  std::string text(BlobToUtf8(pOutText));
+  VERIFY_ARE_EQUAL_STR(R"(#line 1 "file.hlsl"
+#line 12 "file.hlsl"
+    Texture2D<float4> resource_set_10_bind_5_tex;
+
+  float4 main() : SV_Target{
+    return resource_set_10_bind_5_tex[uint2(1, 2)];
+  }
+)",
+                       text.c_str());
+}
+
 TEST_F(CompilerTest, WhenSigMismatchPCFunctionThenFail) {
   CComPtr<IDxcCompiler> pCompiler;
   CComPtr<IDxcOperationResult> pResult;