[spirv] Expose -Oconfig for running custom optimizer recipes (#1466)

docs/SPIR-V.rst

@@ -347,7 +347,61 @@ Optimization
 
 Optimization is also delegated to SPIRV-Tools. Right now there are no difference
 between optimization levels greater than zero; they will all invoke the same
-optimization recipe. This may change in the future.
+optimization recipe. That is, the recipe behind ``spirv-opt -O``.  If you want to
+run a custom optimization recipe, you can do so using the command line option ``-Oconfig=``
+and specifying a comma-separated list of your desired passes. The passes are invoked in the 
+specified order.
+
+For example, you can specify ``-Oconfig=--loop-unroll,--scalar-replacement=300,--eliminate-dead-code-aggressive``
+to firstly invoke loop unrolling, then invoke scalar replacement of aggregates, lastly invoke aggressive dead code elimination.
+All valid options to ``spirv-opt`` are accepted as components to the comma-separated list.
+
+Here are the typical passes in alphabetical order:
+
+* ``--ccp``
+* ``--cfg-cleanup``
+* ``--convert-local-access-chains``
+* ``--copy-propagate-arrays``
+* ``--eliminate-dead-branches``
+* ``--eliminate-dead-code-aggressive``
+* ``--eliminate-dead-functions``
+* ``--eliminate-local-multi-store``
+* ``--eliminate-local-single-block``
+* ``--eliminate-local-single-store``
+* ``--flatten-decorations``
+* ``--if-conversion``
+* ``--inline-entry-points-exhaustive``
+* ``--local-redundancy-elimination``
+* ``--loop-fission``
+* ``--loop-fusion``
+* ``--loop-unroll``
+* ``--loop-unroll-partial=[<n>]``
+* ``--loop-peeling`` (requires ``--loop-peeling-threshold``)
+* ``--merge-blocks``
+* ``--merge-return``
+* ``--loop-unswitch``
+* ``--private-to-local``
+* ``--reduce-load-size``
+* ``--redundancy-elimination``
+* ``--remove-duplicates``
+* ``--replace-invalid-opcode``
+* ``--ssa-rewrite``
+* ``--scalar-replacement[=<n>]``
+* ``--simplify-instructions``
+* ``--vector-dce``
+
+
+Besides, there are two special batch options; each stands for a recommended recipe by itself:
+
+* ``-O``: A bunch of passes in an appropriate order that attempt to improve performance of generated code. Same as ``spirv-opt -O``. Also same as SPIR-V CodeGen's default recipe.
+
+* ``-Os``: A bunch of passes in an appropriate order that attempt to reduce the size of the generated code. Same as ``spirv-opt -Os``.
+
+So if you want to run loop unrolling additionally after the default optimization recipe, you can specify
+``-Oconfig=-O,--loop-unroll``.
+
+For the whole list of accepted passes and details about each one, please see ``spirv-opt``'s help manual (``spirv-opt --help``),
+or the SPIRV-Tools `optimizer header file <https://github.com/KhronosGroup/SPIRV-Tools/blob/master/include/spirv-tools/optimizer.hpp>`_.
 
 Validation
 ~~~~~~~~~~

external/SPIRV-Tools

@@ -1 +1 @@
-Subproject commit 208921efe813b457ee909693a270233fbb012987
+Subproject commit 8a0ec22f1303c158a10b0b6604dd8473d63f6131

include/dxc/Support/HLSLOptions.h

@@ -174,6 +174,7 @@ public:
   llvm::SmallVector<int32_t, 4> VkUShift;  // OPT_fvk_u_shift
   llvm::SmallVector<llvm::StringRef, 4> SpvExtensions; // OPT_fspv_extension
   llvm::StringRef SpvTargetEnv;                        // OPT_fspv_target_env
+  llvm::SmallVector<llvm::StringRef, 4> SpvOconfig;    // OPT_Oconfig
 #endif
   // SPIRV Change Ends
 };

include/dxc/Support/HLSLOptions.td

@@ -262,6 +262,8 @@ def fspv_target_env_EQ : Joined<["-"], "fspv-target-env=">, Group<spirv_Group>,
   HelpText<"Specify the target environment: vulkan1.0 (default) or vulkan1.1">;
 def Wno_vk_ignored_features : Joined<["-"], "Wno-vk-ignored-features">, Group<spirv_Group>, Flags<[CoreOption, DriverOption, HelpHidden]>,
   HelpText<"Do not emit warnings for ingored features resulting from no Vulkan support">;
+def Oconfig : CommaJoined<["-"], "Oconfig=">, Group<spirv_Group>, Flags<[CoreOption]>,
+  HelpText<"Specify a comma-separated list of SPIRV-Tools passes to customize optimization configuration">;
 // SPIRV Change Ends
 
 //////////////////////////////////////////////////////////////////////////////

lib/DxcSupport/HLSLOptions.cpp

@@ -568,6 +568,24 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
   }
 
   opts.SpvTargetEnv = Args.getLastArgValue(OPT_fspv_target_env_EQ, "vulkan1.0");
+
+  // Handle -Oconfig=<comma-separated-list> option.
+  uint32_t numOconfigs = 0;
+  for (const Arg *A : Args.filtered(OPT_Oconfig)) {
+    ++numOconfigs;
+    if (numOconfigs > 1) {
+      errors << "-Oconfig should not be specified more than once";
+      return 1;
+    }
+    if (Args.getLastArg(OPT_O0, OPT_O1, OPT_O2, OPT_O3, OPT_O4)) {
+      errors << "-Oconfig should not be used together with -O";
+      return 1;
+    }
+    for (const auto v : A->getValues()) {
+      opts.SpvOconfig.push_back(v);
+    }
+  }
+
 #else
   if (Args.hasFlag(OPT_spirv, OPT_INVALID, false) ||
       Args.hasFlag(OPT_fvk_invert_y, OPT_INVALID, false) ||
@@ -579,6 +597,7 @@ int ReadDxcOpts(const OptTable *optionTable, unsigned flagsToInclude,
       !Args.getLastArgValue(OPT_fvk_stage_io_order_EQ).empty() ||
       !Args.getLastArgValue(OPT_fspv_extension_EQ).empty() ||
       !Args.getLastArgValue(OPT_fspv_target_env_EQ).empty() ||
+      !Args.getLastArgValue(OPT_Oconfig).empty() ||
       !Args.getLastArgValue(OPT_fvk_b_shift).empty() ||
       !Args.getLastArgValue(OPT_fvk_t_shift).empty() ||
       !Args.getLastArgValue(OPT_fvk_s_shift).empty() ||

tools/clang/include/clang/SPIRV/EmitSPIRVOptions.h

@@ -50,6 +50,7 @@ struct EmitSPIRVOptions {
   spirv::LayoutRule cBufferLayoutRule;
   spirv::LayoutRule tBufferLayoutRule;
   spirv::LayoutRule sBufferLayoutRule;
+  llvm::SmallVector<llvm::StringRef, 4> optConfig;
 
   // Initializes dependent fields appropriately
   void Initialize();

tools/clang/lib/SPIRV/SPIRVEmitter.cpp

@@ -214,6 +214,7 @@ bool spirvToolsLegalize(spv_target_env env, std::vector<uint32_t> *module,
 }
 
 bool spirvToolsOptimize(spv_target_env env, std::vector<uint32_t> *module,
+                        const llvm::SmallVector<llvm::StringRef, 4> &flags,
                         std::string *messages) {
   spvtools::Optimizer optimizer(env);
 
@@ -222,9 +223,18 @@ bool spirvToolsOptimize(spv_target_env env, std::vector<uint32_t> *module,
                  const spv_position_t & /*position*/,
                  const char *message) { *messages += message; });
 
-  optimizer.RegisterPerformancePasses();
-
-  optimizer.RegisterPass(spvtools::CreateCompactIdsPass());
+  if (flags.empty()) {
+    optimizer.RegisterPerformancePasses();
+    optimizer.RegisterPass(spvtools::CreateCompactIdsPass());
+  } else {
+    // Command line options use llvm::SmallVector and llvm::StringRef, whereas
+    // SPIR-V optimizer uses std::vector and std::string.
+    std::vector<std::string> stdFlags;
+    for (const auto &f : flags)
+      stdFlags.push_back(f.str());
+    if (!optimizer.RegisterPassesFromFlags(stdFlags))
+      return false;
+  }
 
   return optimizer.Run(module->data(), module->size(), module);
 }
@@ -695,7 +705,8 @@ void SPIRVEmitter::HandleTranslationUnit(ASTContext &context) {
     // Run optimization passes
     if (theCompilerInstance.getCodeGenOpts().OptimizationLevel > 0) {
       std::string messages;
-      if (!spirvToolsOptimize(targetEnv, &m, &messages)) {
+      if (!spirvToolsOptimize(targetEnv, &m, spirvOptions.optConfig,
+                              &messages)) {
         emitFatalError("failed to optimize SPIR-V: %0", {}) << messages;
         emitNote("please file a bug report on "
                  "https://github.com/Microsoft/DirectXShaderCompiler/issues "

tools/clang/test/CodeGenSPIRV/spirv.opt.cl.oconfig.hlsl

@@ -0,0 +1,36 @@
+// Run: %dxc -T vs_6_0 -E main -Oconfig=-O,--loop-unroll
+
+// Note: The above recipe should unroll the loop, but should not
+// reduce the whole shader to "return (3,3,3,3)"
+
+float4 main() : SV_POSITION {
+  int j = 0;
+  int i = 0;
+
+  [unroll(3)] for (i = 0; i < 3; ++i) {
+    j++;
+  }
+
+  return float4(j, j, j, j);
+}
+
+// CHECK: OpSLessThan %bool %int_0 %int_3
+// CHECK: OpBranch
+// CHECK: OpLabel
+// CHECK: OpIAdd %int %int_0 %int_1
+// CHECK: OpIAdd %int %int_0 %int_1
+// CHECK: OpBranch
+// CHECK: OpLabel
+// CHECK: OpSLessThan %bool {{%\d+}} %int_3
+// CHECK: OpBranch
+// CHECK: OpLabel
+// CHECK: OpIAdd %int {{%\d+}} %int_1
+// CHECK: OpIAdd %int {{%\d+}} %int_1
+// CHECK: OpBranch
+// CHECK: OpLabel
+// CHECK: OpSLessThan %bool {{%\d+}} %int_3
+// CHECK: OpBranch
+// CHECK: OpLabel
+// CHECK: OpIAdd %int {{%\d+}} %int_1
+// CHECK: OpIAdd %int {{%\d+}} %int_1
+// CHECK: OpBranch

tools/clang/test/CodeGenSPIRV/spirv.opt.invalid-flag.cl.oconfig.hlsl

@@ -0,0 +1,5 @@
+// Run: %dxc -T ps_6_0 -E main -Oconfig=--test-unknown-flag
+
+void main() {}
+
+// CHECK: failed to optimize SPIR-V: Unknown flag '--test-unknown-flag'

tools/clang/test/CodeGenSPIRV/spirv.opt.multiple.cl.oconfig.hlsl

@@ -0,0 +1,5 @@
+// Run: %dxc -T ps_6_0 -E main -Oconfig=-O -Oconfig=-Os
+
+void main() {}
+
+// CHECK: -Oconfig should not be specified more than once

tools/clang/test/CodeGenSPIRV/spirv.opt.with-O0.cl.oconfig.hlsl

@@ -0,0 +1,5 @@
+// Run: %dxc -T ps_6_0 -E main -O0 -Oconfig=--loop-unroll
+
+void main() {}
+
+// CHECK: -Oconfig should not be used together with -O

tools/clang/test/CodeGenSPIRV/spirv.opt.with-O1.cl.oconfig.hlsl

@@ -0,0 +1,5 @@
+// Run: %dxc -T ps_6_0 -E main -O1 -Oconfig=--loop-unroll
+
+void main() {}
+
+// CHECK: -Oconfig should not be used together with -O

tools/clang/test/CodeGenSPIRV/spirv.opt.with-O2.cl.oconfig.hlsl

@@ -0,0 +1,5 @@
+// Run: %dxc -T ps_6_0 -E main -O2 -Oconfig=--loop-unroll
+
+void main() {}
+
+// CHECK: -Oconfig should not be used together with -O

tools/clang/test/CodeGenSPIRV/spirv.opt.with-O3.cl.oconfig.hlsl

@@ -0,0 +1,5 @@
+// Run: %dxc -T ps_6_0 -E main -O3 -Oconfig=--loop-unroll
+
+void main() {}
+
+// CHECK: -Oconfig should not be used together with -O

tools/clang/test/CodeGenSPIRV/spirv.opt.with-O4.cl.oconfig.hlsl

@@ -0,0 +1,5 @@
+// Run: %dxc -T ps_6_0 -E main -O4 -Oconfig=--loop-unroll
+
+void main() {}
+
+// CHECK: -Oconfig should not be used together with -O

tools/clang/tools/dxcompiler/dxcompilerobj.cpp

@@ -527,6 +527,7 @@ public:
           spirvOpts.targetEnv = opts.SpvTargetEnv;
           spirvOpts.enable16BitTypes = opts.Enable16BitTypes;
           spirvOpts.enableDebugInfo = opts.DebugInfo;
+          spirvOpts.optConfig = opts.SpvOconfig;
 
           clang::EmitSPIRVAction action(spirvOpts);
           FrontendInputFile file(utf8SourceName.m_psz, IK_HLSL);

tools/clang/unittests/SPIRV/CodeGenSPIRVTest.cpp

@@ -1245,6 +1245,29 @@ TEST_F(FileTest, SpirvExtensionCLUnknown) {
 TEST_F(FileTest, SpirvExtensionAllowAllKHR) {
   runFileTest("spirv.ext.allow-all-khr.hlsl");
 }
+// Test -Oconfig command line option.
+TEST_F(FileTest, SpirvOptOconfigMultipleUses) {
+  runFileTest("spirv.opt.multiple.cl.oconfig.hlsl", Expect::Failure);
+}
+TEST_F(FileTest, SpirvOptOconfigAndO0) {
+  runFileTest("spirv.opt.with-O0.cl.oconfig.hlsl", Expect::Failure);
+}
+TEST_F(FileTest, SpirvOptOconfigAndO1) {
+  runFileTest("spirv.opt.with-O1.cl.oconfig.hlsl", Expect::Failure);
+}
+TEST_F(FileTest, SpirvOptOconfigAndO2) {
+  runFileTest("spirv.opt.with-O2.cl.oconfig.hlsl", Expect::Failure);
+}
+TEST_F(FileTest, SpirvOptOconfigAndO3) {
+  runFileTest("spirv.opt.with-O3.cl.oconfig.hlsl", Expect::Failure);
+}
+TEST_F(FileTest, SpirvOptOconfigAndO4) {
+  runFileTest("spirv.opt.with-O4.cl.oconfig.hlsl", Expect::Failure);
+}
+TEST_F(FileTest, SpirvOptOconfigInvalidFlag) {
+  runFileTest("spirv.opt.invalid-flag.cl.oconfig.hlsl", Expect::Failure);
+}
+TEST_F(FileTest, SpirvOptOconfig) { runFileTest("spirv.opt.cl.oconfig.hlsl"); }
 
 // For shader stage input/output interface
 // For semantic SV_Position, SV_ClipDistance, SV_CullDistance

tools/clang/unittests/SPIRV/FileTestUtils.cpp

@@ -143,14 +143,21 @@ bool runCompilerWithSpirvGeneration(const llvm::StringRef inputFilePath,
     CComPtr<IDxcIncludeHandler> pIncludeHandler;
     HRESULT resultStatus;
 
+    bool running_specific_opt_recipe = false;
+    for (const auto &arg : rest)
+      if (arg.substr(0, 8) == L"-Oconfig")
+        running_specific_opt_recipe = true;
+
     std::vector<LPCWSTR> flags;
     flags.push_back(L"-E");
     flags.push_back(entry.c_str());
     flags.push_back(L"-T");
     flags.push_back(profile.c_str());
     flags.push_back(L"-spirv");
-    // Disable legalization and optimization for testing
-    flags.push_back(L"-fcgl");
+    // Disable legalization and optimization for testing, unless the caller
+    // wants to run a specific optimization recipe (with -Oconfig).
+    if (!running_specific_opt_recipe)
+      flags.push_back(L"-fcgl");
     // Disable validation. We'll run it manually.
     flags.push_back(L"-Vd");
     for (const auto &arg : rest)