vulkan: improve depth/stencil texture support.

src/modules/graphics/vulkan/Graphics.cpp

@@ -1884,15 +1884,15 @@ VkSurfaceFormatKHR Graphics::chooseSwapSurfaceFormat(const std::vector<VkSurface
 	if (isGammaCorrect())
 	{
 		formatOrder = {
-				VK_FORMAT_B8G8R8A8_SRGB,
-				VK_FORMAT_R8G8B8A8_SRGB,
+			VK_FORMAT_B8G8R8A8_SRGB,
+			VK_FORMAT_R8G8B8A8_SRGB,
 		};
 	}
 	else
 	{
 		formatOrder = {
-				VK_FORMAT_B8G8R8A8_UNORM,
-				VK_FORMAT_R8G8B8A8_SNORM,
+			VK_FORMAT_B8G8R8A8_UNORM,
+			VK_FORMAT_R8G8B8A8_SNORM,
 		};
 	}
 
@@ -2392,15 +2392,24 @@ void Graphics::setRenderPass(const RenderTargets &rts, int pixelw, int pixelh, b
 
 	FramebufferConfiguration configuration{};
 
-	std::vector<std::tuple<VkImage, PixelFormat>> transitionImages;
+	std::vector<std::tuple<VkImage, PixelFormat, VkImageLayout, VkImageLayout>> transitionImages;
 
 	for (const auto &color : rts.colors)
 	{
-		configuration.colorViews.push_back(dynamic_cast<Texture*>(color.texture)->getRenderTargetView(color.mipmap, color.slice));
-		transitionImages.push_back({ (VkImage)color.texture->getHandle(), color.texture->getPixelFormat() });
+		auto tex = (Texture*)color.texture;
+		configuration.colorViews.push_back(tex->getRenderTargetView(color.mipmap, color.slice));
+		VkImageLayout imagelayout = tex->getImageLayout();
+		if (imagelayout != VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL)
+			transitionImages.push_back({ (VkImage)tex->getHandle(), tex->getPixelFormat(), imagelayout, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL });
 	}
 	if (rts.depthStencil.texture != nullptr)
-		configuration.staticData.depthView = dynamic_cast<Texture*>(rts.depthStencil.texture)->getRenderTargetView(rts.depthStencil.mipmap, rts.depthStencil.slice);
+	{
+		auto tex = (Texture*)rts.depthStencil.texture;
+		configuration.staticData.depthView = tex->getRenderTargetView(rts.depthStencil.mipmap, rts.depthStencil.slice);
+		VkImageLayout imagelayout = tex->getImageLayout();
+		if (imagelayout != VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL)
+			transitionImages.push_back({ (VkImage)tex->getHandle(), tex->getPixelFormat(), imagelayout, VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL });
+	}
 
 	configuration.staticData.width = static_cast<uint32_t>(pixelw);
 	configuration.staticData.height = static_cast<uint32_t>(pixelh);
@@ -2450,8 +2459,8 @@ void Graphics::startRenderPass()
 	renderPassState.framebufferConfiguration.staticData.renderPass = renderPassState.beginInfo.renderPass;
 	renderPassState.beginInfo.framebuffer = getFramebuffer(renderPassState.framebufferConfiguration);
 
-	for (const auto &[image, format] : renderPassState.transitionImages)
-		Vulkan::cmdTransitionImageLayout(commandBuffers.at(currentFrame), image, format, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL);
+	for (const auto &[image, format, imageLayout, renderLayout] : renderPassState.transitionImages)
+		Vulkan::cmdTransitionImageLayout(commandBuffers.at(currentFrame), image, format, imageLayout, renderLayout);
 
 	vkCmdBeginRenderPass(commandBuffers.at(currentFrame), &renderPassState.beginInfo, VK_SUBPASS_CONTENTS_INLINE);
 }
@@ -2462,8 +2471,8 @@ void Graphics::endRenderPass()
 
 	vkCmdEndRenderPass(commandBuffers.at(currentFrame));
 
-	for (const auto &[image, format] : renderPassState.transitionImages)
-		Vulkan::cmdTransitionImageLayout(commandBuffers.at(currentFrame), image, format, VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL);
+	for (const auto &[image, format, imageLayout, renderLayout] : renderPassState.transitionImages)
+		Vulkan::cmdTransitionImageLayout(commandBuffers.at(currentFrame), image, format, renderLayout, imageLayout);
 
 	for (auto &colorAttachment : renderPassState.renderPassConfiguration.colorAttachments)
 		colorAttachment.loadOp = VK_ATTACHMENT_LOAD_OP_LOAD;

src/modules/graphics/vulkan/Graphics.h

@@ -236,7 +236,7 @@ struct RenderpassState
 	RenderPassConfiguration renderPassConfiguration{};
 	FramebufferConfiguration framebufferConfiguration{};
 	VkPipeline pipeline = VK_NULL_HANDLE;
-	std::vector<std::tuple<VkImage, PixelFormat>> transitionImages;
+	std::vector<std::tuple<VkImage, PixelFormat, VkImageLayout, VkImageLayout>> transitionImages;
 	uint32_t numColorAttachments = 0;
 	float width = 0.0f;
 	float height = 0.0f;

src/modules/graphics/vulkan/Texture.cpp

@@ -151,12 +151,16 @@ bool Texture::loadVolatile()
 
 		auto commandBuffer = vgfx->getCommandBufferForDataTransfer();
 
-		if (isPixelFormatDepthStencil(format))
-			imageLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
-		else if (computeWrite)
+		if (computeWrite)
 			imageLayout = VK_IMAGE_LAYOUT_GENERAL;
-		else
+		else if (readable)
 			imageLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+		else if (renderTarget && isPixelFormatDepthStencil(format))
+			imageLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+		else if (renderTarget)
+			imageLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+		else // TODO: is there a better layout for this situation?
+			imageLayout = VK_IMAGE_LAYOUT_TRANSFER_SRC_OPTIMAL;
 
 		Vulkan::cmdTransitionImageLayout(commandBuffer, textureImage, format,
 			VK_IMAGE_LAYOUT_UNDEFINED, imageLayout,
@@ -383,40 +387,32 @@ void Texture::clear()
 	range.baseArrayLayer = 0;
 	range.layerCount = VK_REMAINING_ARRAY_LAYERS;
 
-	if (imageLayout == VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL)
-	{
-		Vulkan::cmdTransitionImageLayout(commandBuffer, textureImage, format,
-			VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-			0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS);
-
-		auto clearColor = getClearColor(this, ColorD(0, 0, 0, 0));
-
-		vkCmdClearColorImage(commandBuffer, textureImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &clearColor, 1, &range);
+	VkImageLayout clearLayout = imageLayout == VK_IMAGE_LAYOUT_GENERAL ? imageLayout : VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL;
 
+	if (clearLayout != imageLayout)
+	{
 		Vulkan::cmdTransitionImageLayout(commandBuffer, textureImage, format,
-			VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL,
+			imageLayout, clearLayout,
 			0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS);
 	}
-	else if (imageLayout == VK_IMAGE_LAYOUT_GENERAL)
-	{
-		auto clearColor = getClearColor(this, ColorD(0, 0, 0, 0));
 
-		vkCmdClearColorImage(commandBuffer, textureImage, VK_IMAGE_LAYOUT_GENERAL, &clearColor, 1, &range);
-	}
-	else
+	if (isPixelFormatDepthStencil(format))
 	{
-		Vulkan::cmdTransitionImageLayout(commandBuffer, textureImage, format,
-			imageLayout, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL,
-			0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS);
-
 		VkClearDepthStencilValue depthStencilColor{};
 		depthStencilColor.depth = 0.0f;
 		depthStencilColor.stencil = 0;
+		vkCmdClearDepthStencilImage(commandBuffer, textureImage, clearLayout, &depthStencilColor, 1, &range);
+	}
+	else
+	{
+		auto clearColor = getClearColor(this, ColorD(0, 0, 0, 0));
+		vkCmdClearColorImage(commandBuffer, textureImage, clearLayout, &clearColor, 1, &range);
+	}
 
-		vkCmdClearDepthStencilImage(commandBuffer, textureImage, VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, &depthStencilColor, 1, &range);
-
+	if (clearLayout != imageLayout)
+	{
 		Vulkan::cmdTransitionImageLayout(commandBuffer, textureImage, format,
-			VK_IMAGE_LAYOUT_TRANSFER_DST_OPTIMAL, imageLayout,
+			clearLayout, imageLayout,
 			0, VK_REMAINING_MIP_LEVELS, 0, VK_REMAINING_ARRAY_LAYERS);
 	}
 }