vulkan: Assorted improvements, including working alpha testing

panda/src/shaderpipeline/spirVTransformer.cxx

@@ -394,3 +394,61 @@ bind_descriptor_set(uint32_t set, const pvector<uint32_t> &ids) {
     }
   }
 }
+
+/**
+ * Change OpenGL conventions to Vulkan conventions, including stripping uniform
+ * locations.
+ */
+void SpirVTransformer::
+change_to_vulkan_conventions() {
+  auto it = _preamble.begin() + 5;
+  while (it != _preamble.end()) {
+    spv::Op opcode = (spv::Op)(*it & spv::OpCodeMask);
+    uint32_t wcount = *it >> spv::WordCountShift;
+    nassertd(wcount > 0) break;
+
+    if (opcode == spv::OpExecutionMode && wcount >= 3 &&
+        (spv::ExecutionMode)*(it + 2) == spv::ExecutionModeOriginLowerLeft) {
+      *(it + 2) = spv::ExecutionModeOriginUpperLeft;
+      break;
+    }
+
+    std::advance(it, wcount);
+  }
+
+  it = _annotations.begin();
+  while (it != _annotations.end()) {
+    spv::Op opcode = (spv::Op)(*it & spv::OpCodeMask);
+    uint32_t wcount = *it >> spv::WordCountShift;
+    nassertd(wcount > 0) break;
+
+    if (opcode == spv::OpDecorate && wcount >= 3) {
+      spv::Decoration decoration = (spv::Decoration)*(it + 2);
+
+      if (decoration == spv::DecorationLocation) {
+        Definition &def = _db.modify_definition(*(it + 1));
+        if (def._storage_class == spv::StorageClassUniformConstant) {
+          it = _annotations.erase(it, it + wcount);
+          def._location = -1;
+          continue;
+        }
+      }
+      else if (decoration == spv::DecorationBuiltIn) {
+        spv::BuiltIn &builtin = (spv::BuiltIn &)*(it + 3);
+
+        switch (builtin) {
+        case spv::BuiltInVertexId:
+          builtin = spv::BuiltInVertexIndex;
+          break;
+        case spv::BuiltInInstanceId:
+          builtin = spv::BuiltInInstanceIndex;
+          break;
+        default:
+          break;
+        }
+      }
+    }
+
+    std::advance(it, wcount);
+  }
+}

panda/src/shaderpipeline/spirVTransformer.h

@@ -49,6 +49,8 @@ public:
   void strip_bindings();
   void bind_descriptor_set(uint32_t set, const pvector<uint32_t> &ids);
 
+  void change_to_vulkan_conventions();
+
 private:
   // Stores the module split into the different sections for easier
   // concurrent modification of the various sections.

panda/src/vulkandisplay/vulkanGraphicsStateGuardian.cxx

@@ -2068,12 +2068,7 @@ release_shader(ShaderContext *context) {
 
   // According to the Vulkan spec, it is safe to delete a shader module even
   // if pipelines using it are still in use, so let's do it now.
-  for (size_t i = 0; i <= (size_t)Shader::Stage::COMPUTE; ++i) {
-    if (sc->_modules[i] != VK_NULL_HANDLE) {
-      vkDestroyShaderModule(_device, sc->_modules[i], nullptr);
-      sc->_modules[i] = VK_NULL_HANDLE;
-    }
-  }
+  sc->destroy_modules(_device);
 
   // Destroy the pipeline states that use these modules.
   //TODO: is this safe?
@@ -3891,6 +3886,7 @@ make_pipeline(VulkanShaderContext *sc,
       << " color_write_mask=" << key._color_write_mask
       << " logic_op=" << key._logic_op
       << " transparency_mode=" << key._transparency_mode
+      << " alpha_test_attrib=" << key._alpha_test_attrib
       << "\n";
   }
 
@@ -3907,6 +3903,31 @@ make_pipeline(VulkanShaderContext *sc,
   };
   uint32_t num_stages = 0;
 
+  VkSpecializationInfo fragment_spec_info;
+  VkSpecializationMapEntry fragment_spec_map_entry;
+  float alpha_test_ref;
+
+  RenderAttrib::PandaCompareFunc alpha_test_mode = RenderAttrib::M_none;
+  if (key._alpha_test_attrib != nullptr) {
+    alpha_test_mode = key._alpha_test_attrib->get_mode();
+    if (alpha_test_mode != RenderAttrib::M_never) {
+      alpha_test_ref = key._alpha_test_attrib->get_reference_alpha();
+    } else {
+      // Rather than create special case code for the rare case of M_never, we
+      // instead turn it into a equal test with NaN.
+      alpha_test_ref = make_nan((float)0);
+    }
+
+    fragment_spec_info.mapEntryCount = 1;
+    fragment_spec_info.pMapEntries = &fragment_spec_map_entry;
+    fragment_spec_info.dataSize = 4;
+    fragment_spec_info.pData = &alpha_test_ref;
+
+    fragment_spec_map_entry.constantID = 0;
+    fragment_spec_map_entry.offset = 0;
+    fragment_spec_map_entry.size = 4;
+  }
+
   for (size_t i = 0; i <= (size_t)Shader::Stage::COMPUTE; ++i) {
     if (sc->_modules[i] != VK_NULL_HANDLE) {
       VkPipelineShaderStageCreateInfo &stage = stages[num_stages++];
@@ -3914,9 +3935,14 @@ make_pipeline(VulkanShaderContext *sc,
       stage.pNext = nullptr;
       stage.flags = 0;
       stage.stage = stage_flags[i];
-      stage.module = sc->_modules[i];
-      stage.pName = "main";
       stage.pSpecializationInfo = nullptr;
+      if (i == (size_t)Shader::Stage::FRAGMENT && alpha_test_mode != RenderAttrib::M_none) {
+        stage.module = sc->get_fragment_module(_device, alpha_test_mode);
+        stage.pSpecializationInfo = &fragment_spec_info;
+      } else {
+        stage.module = sc->get_module((Shader::Stage)i);
+      }
+      stage.pName = "main";
     }
   }
 

panda/src/vulkandisplay/vulkanShaderContext.I

@@ -32,6 +32,14 @@ INLINE VulkanShaderContext::
 ~VulkanShaderContext()  {
 }
 
+/**
+ * Returns the module for the given stage, or VK_NULL_HANDLE.
+ */
+INLINE VkShaderModule VulkanShaderContext::
+get_module(Shader::Stage stage) const {
+  return _modules[(size_t)stage];
+}
+
 /**
  * Returns true if these two PipelineKey objects are identical.
  */
@@ -49,7 +57,8 @@ operator ==(const PipelineKey &other) const {
       && _color_write_mask == other._color_write_mask
       && _logic_op == other._logic_op
       && _color_blend_attrib == other._color_blend_attrib
-      && _transparency_mode == other._transparency_mode;
+      && _transparency_mode == other._transparency_mode
+      && _alpha_test_attrib == other._alpha_test_attrib;
 }
 
 /**
@@ -96,5 +105,8 @@ operator < (const PipelineKey &other) const {
   if (_transparency_mode != other._transparency_mode) {
     return _transparency_mode < other._transparency_mode;
   }
+  if (_alpha_test_attrib != other._alpha_test_attrib) {
+    return _alpha_test_attrib < other._alpha_test_attrib;
+  }
   return 0;
 }

panda/src/vulkandisplay/vulkanShaderContext.cxx

@@ -17,6 +17,7 @@
 #include "spirVTransformer.h"
 #include "spirVConvertBoolToIntPass.h"
 #include "spirVHoistStructResourcesPass.h"
+#include "spirVInjectAlphaTestPass.h"
 #include "spirVMakeBlockPass.h"
 #include "spirVRemoveUnusedVariablesPass.h"
 
@@ -25,6 +26,26 @@ static PStatCollector _update_sattr_descriptor_set_pcollector("Draw:Update Descr
 
 TypeHandle VulkanShaderContext::_type_handle;
 
+/**
+ *
+ */
+void VulkanShaderContext::
+destroy_modules(VkDevice device) {
+  for (size_t i = 0; i <= (size_t)Shader::Stage::COMPUTE; ++i) {
+    if (_modules[i] != VK_NULL_HANDLE) {
+      vkDestroyShaderModule(device, _modules[i], nullptr);
+      _modules[i] = VK_NULL_HANDLE;
+    }
+  }
+
+  for (size_t i = 0; i < sizeof(_alpha_test_modules) / sizeof(VkShaderModule); ++i) {
+    if (_alpha_test_modules[i] != VK_NULL_HANDLE) {
+      vkDestroyShaderModule(device, _alpha_test_modules[i], nullptr);
+      _alpha_test_modules[i] = VK_NULL_HANDLE;
+    }
+  }
+}
+
 /**
  * Creates the shader modules.
  */
@@ -156,9 +177,7 @@ create_modules(VkDevice device, const ShaderType::Struct *push_constant_block_ty
 
     const ShaderModuleSpirV *spv_module = (const ShaderModuleSpirV *)module.p();
     SpirVTransformer transformer(spv_module->_instructions);
-
-    // These are not used in Vulkan, and the validation layers trip over them.
-    transformer.strip_uniform_locations();
+    transformer.change_to_vulkan_conventions();
 
     // Determine the ids making up the inputs for the descriptor sets.
     pvector<uint32_t> tattr_set_ids(tattr_set_params.size(), 0u);
@@ -206,7 +225,10 @@ create_modules(VkDevice device, const ShaderType::Struct *push_constant_block_ty
             chain._var_id = spv_module->get_parameter(index).id;
             auto it = hoist_pass._hoisted_vars.find(chain);
             if (it != hoist_pass._hoisted_vars.end()) {
-              tattr_set_ids[i] = it->second;
+              // Check if it hasn't been removed due to being unused.
+              if (transformer.get_db().has_definition(it->second)) {
+                tattr_set_ids[i] = it->second;
+              }
             }
           }
         }
@@ -219,7 +241,10 @@ create_modules(VkDevice device, const ShaderType::Struct *push_constant_block_ty
             chain._var_id = spv_module->get_parameter(index).id;
             auto it = hoist_pass._hoisted_vars.find(chain);
             if (it != hoist_pass._hoisted_vars.end()) {
-              sattr_set_ids[i] = it->second;
+              // Check if it hasn't been removed due to being unused.
+              if (transformer.get_db().has_definition(it->second)) {
+                sattr_set_ids[i] = it->second;
+              }
             }
           }
         }
@@ -263,30 +288,7 @@ create_modules(VkDevice device, const ShaderType::Struct *push_constant_block_ty
       transformer.bind_descriptor_set(VulkanGraphicsStateGuardian::DS_shader_attrib, sattr_set_ids);
     }
 
-    // Change OpenGL conventions to Vulkan conventions.
     ShaderModuleSpirV::InstructionStream instructions = transformer.get_result();
-    for (ShaderModuleSpirV::Instruction op : instructions) {
-      if (op.opcode == spv::OpExecutionMode) {
-        if (op.nargs >= 2 && (spv::ExecutionMode)op.args[1] == spv::ExecutionModeOriginLowerLeft) {
-          op.args[1] = spv::ExecutionModeOriginUpperLeft;
-        }
-      }
-      else if (op.opcode == spv::OpDecorate) {
-        if (op.nargs >= 3 && op.args[1] == spv::DecorationBuiltIn) {
-          switch ((spv::BuiltIn)op.args[2]) {
-          case spv::BuiltInVertexId:
-            op.args[2] = spv::BuiltInVertexIndex;
-            break;
-          case spv::BuiltInInstanceId:
-            op.args[2] = spv::BuiltInInstanceIndex;
-            break;
-          default:
-            break;
-          }
-        }
-      }
-    }
-
 #ifndef NDEBUG
     if (!instructions.validate(SPV_ENV_VULKAN_1_0)) {
       success = false;
@@ -304,9 +306,20 @@ create_modules(VkDevice device, const ShaderType::Struct *push_constant_block_ty
     VkResult err;
     err = vkCreateShaderModule(device, &module_info, nullptr, &_modules[(size_t)spv_module->get_stage()]);
     if (err) {
-      vulkan_error(err, "Failed to create shader modules");
+      vulkan_error(err, "Failed to create shader module");
       success = false;
     }
+
+    if (spv_module->get_stage() == Shader::Stage::FRAGMENT) {
+      // Set us up to easily create versions of the shader for various alpha
+      // testing modes.
+      SpirVInjectAlphaTestPass pass(SpirVInjectAlphaTestPass::M_greater, 0, true);
+      transformer.run(pass);
+      if (pass._compare_op_offset != 0) {
+        _alpha_test_code = transformer.get_result();
+        _alpha_test_compare_op_offset = pass._compare_op_offset;
+      }
+    }
   }
 
   if (!success) {
@@ -321,6 +334,64 @@ create_modules(VkDevice device, const ShaderType::Struct *push_constant_block_ty
   return success;
 }
 
+/**
+ * Returns the module for the given stage, or VK_NULL_HANDLE.
+ */
+VkShaderModule VulkanShaderContext::
+get_fragment_module(VkDevice device, RenderAttrib::PandaCompareFunc alpha_test_mode) {
+  if (alpha_test_mode == RenderAttrib::M_none ||
+      alpha_test_mode == RenderAttrib::M_always ||
+      _alpha_test_compare_op_offset == 0) {
+    return _modules[(size_t)Shader::Stage::FRAGMENT];
+  }
+
+  size_t i = alpha_test_mode - RenderAttrib::M_never;
+  VkShaderModule module = _alpha_test_modules[i];
+  if (module != VK_NULL_HANDLE) {
+    return module;
+  }
+
+  if (vulkandisplay_cat.is_debug()) {
+    vulkandisplay_cat.debug()
+      << "Modifying module for shader " << _shader->get_filename() << " for "
+         "alpha test mode " << alpha_test_mode << "\n";
+  }
+
+  // Creating a special case for handling M_never isn't worth it, because it's
+  // pretty much a useless mode, so we instead only pass it if it compares as
+  // equal to NaN, which nothing ever will.
+  static const uint32_t opcodes[] {
+    (5u << spv::WordCountShift) | spv::OpFUnordNotEqual, // M_never
+    (5u << spv::WordCountShift) | spv::OpFOrdGreaterThanEqual, // M_less
+    (5u << spv::WordCountShift) | spv::OpFUnordNotEqual, // M_equal
+    (5u << spv::WordCountShift) | spv::OpFOrdGreaterThan, // M_less_equal
+    (5u << spv::WordCountShift) | spv::OpFOrdLessThanEqual, // M_greater
+    (5u << spv::WordCountShift) | spv::OpFOrdEqual, // M_not_equal
+    (5u << spv::WordCountShift) | spv::OpFOrdLessThan, // M_greater_equal
+  };
+
+  uint32_t *code = (uint32_t *)_alpha_test_code.get_data();
+  nassertr(code[_alpha_test_compare_op_offset] == ((5u << spv::WordCountShift) | spv::OpFOrdLessThanEqual), VK_NULL_HANDLE);
+  code[_alpha_test_compare_op_offset] = opcodes[i];
+
+  VkShaderModuleCreateInfo module_info;
+  module_info.sType = VK_STRUCTURE_TYPE_SHADER_MODULE_CREATE_INFO;
+  module_info.pNext = nullptr;
+  module_info.flags = 0;
+  module_info.codeSize = _alpha_test_code.get_data_size() * 4;
+  module_info.pCode = code;
+
+  VkResult err;
+  err = vkCreateShaderModule(device, &module_info, nullptr, &module);
+  if (err) {
+    vulkan_error(err, "Failed to create shader module with injected alpha test");
+    return VK_NULL_HANDLE;
+  }
+
+  _alpha_test_modules[i] = module;
+  return module;
+}
+
 /**
  * Collects the resources from the given parameter and adds them to the
  * descriptor table.
@@ -868,6 +939,13 @@ get_pipeline(VulkanGraphicsStateGuardian *gsg, const RenderState *state,
     key._transparency_mode = transp->get_mode();
   }
 
+  if (state->get_alpha_test_mode() != RenderAttrib::M_none &&
+      state->get_alpha_test_mode() != RenderAttrib::M_always) {
+    const AlphaTestAttrib *alpha_test;
+    state->get_attrib_def(alpha_test);
+    key._alpha_test_attrib = alpha_test;
+  }
+
   PipelineMap::const_iterator it;
   it = _pipeline_map.find(key);
   if (it == _pipeline_map.end()) {

panda/src/vulkandisplay/vulkanShaderContext.h

@@ -24,6 +24,7 @@
 #include "depthWriteAttrib.h"
 #include "logicOpAttrib.h"
 #include "colorBlendAttrib.h"
+#include "alphaTestAttrib.h"
 #include "transparencyAttrib.h"
 
 #include "small_vector.h"
@@ -45,8 +46,12 @@ public:
 
   ALLOC_DELETED_CHAIN(VulkanShaderContext);
 
+  void destroy_modules(VkDevice device);
   bool create_modules(VkDevice device, const ShaderType::Struct *push_constant_block_type);
 
+  INLINE VkShaderModule get_module(Shader::Stage module) const;
+  VkShaderModule get_fragment_module(VkDevice device, RenderAttrib::PandaCompareFunc alpha_test_mode);
+
   const ShaderType *r_extract_resources(const Shader::Parameter &param, const AccessChain &chain,
                                         pmap<AccessChain, Descriptor> &descriptors,
                                         const ShaderType *type, int &resource_index);
@@ -95,11 +100,17 @@ public:
     LogicOpAttrib::Operation _logic_op;
     CPT(ColorBlendAttrib) _color_blend_attrib;
     TransparencyAttrib::Mode _transparency_mode;
+    CPT(AlphaTestAttrib) _alpha_test_attrib;
   };
 
 private:
   VkPipelineBindPoint _bind_point = VK_PIPELINE_BIND_POINT_GRAPHICS;
   VkShaderModule _modules[(size_t)Shader::Stage::COMPUTE + 1];
+
+  VkShaderModule _alpha_test_modules[RenderAttrib::M_always - RenderAttrib::M_never] = {};
+  ShaderModuleSpirV::InstructionStream _alpha_test_code;
+  uint32_t _alpha_test_compare_op_offset = 0;
+
   VkDescriptorSetLayout _tattr_descriptor_set_layout = VK_NULL_HANDLE;
   VkDescriptorSetLayout _sattr_descriptor_set_layout = VK_NULL_HANDLE;
   VkDescriptorSetLayout _dynamic_uniform_descriptor_set_layout = VK_NULL_HANDLE;