WIP: Getting Vulkan up to date

 - When writing image descriptor sets make sure to set the valid image layout

Source/BansheeVulkanRenderAPI/BsVulkanCommandBuffer.cpp

@@ -336,39 +336,7 @@ namespace bs { namespace ct
 
 		executeLayoutTransitions();
 
-		// Check if any frame-buffer attachments are also used as shader inputs, in which case we make them read-only
-		RenderSurfaceMask readMask = RT_NONE;
-
-		UINT32 numColorAttachments = mFramebuffer->getNumColorAttachments();
-		for(UINT32 i = 0; i < numColorAttachments; i++)
-		{
-			const VulkanFramebufferAttachment& fbAttachment = mFramebuffer->getColorAttachment(i);
-			ImageSubresourceInfo& subresourceInfo = findSubresourceInfo(fbAttachment.image, fbAttachment.surface.arraySlice,
-				fbAttachment.surface.mipLevel);
-
-			bool readOnly = subresourceInfo.isShaderInput;
-
-			if(readOnly)
-				readMask.set((RenderSurfaceMaskBits)(1 << i));
-		}
-
-		if (mFramebuffer->hasDepthAttachment())
-		{
-			const VulkanFramebufferAttachment& fbAttachment = mFramebuffer->getDepthStencilAttachment();
-			ImageSubresourceInfo& subresourceInfo = findSubresourceInfo(fbAttachment.image, fbAttachment.surface.arraySlice,
-				fbAttachment.surface.mipLevel);
-
-			bool readOnly = subresourceInfo.isShaderInput;
-
-			if (readOnly)
-				readMask.set(RT_DEPTH);
-
-			if ((mRenderTargetReadOnlyFlags & FBT_DEPTH) != 0)
-				readMask.set(RT_DEPTH);
-
-			if ((mRenderTargetReadOnlyFlags & FBT_STENCIL) != 0)
-				readMask.set(RT_STENCIL);
-		}
+		RenderSurfaceMask readMask = getFBReadMask();
 
 		VkRenderPassBeginInfo renderPassBeginInfo;
 		renderPassBeginInfo.sType = VK_STRUCTURE_TYPE_RENDER_PASS_BEGIN_INFO;
@@ -1725,6 +1693,73 @@ namespace bs { namespace ct
 							 1, &barrier);
 	}
 
+	VkImageLayout VulkanCmdBuffer::getCurrentLayout(VulkanImage* image, const VkImageSubresourceRange& range, 
+		bool inRenderPass)
+	{
+		UINT32 face = range.baseArrayLayer;
+		UINT32 mip = range.baseMipLevel;
+
+		// The assumption is that all the subresources in the range will have the same layout, as this should be handled
+		// by registerResource(), or by external code (in the case of transfers). So we only check the first subresource.
+		VulkanImageSubresource* subresource = image->getSubresource(face, mip);
+
+		auto iterFind = mImages.find(image);
+		if (iterFind == mImages.end())
+			return subresource->getLayout();
+
+		UINT32 imageInfoIdx = iterFind->second;
+		ImageInfo& imageInfo = mImageInfos[imageInfoIdx];
+
+		ImageSubresourceInfo* subresourceInfos = &mSubresourceInfoStorage[imageInfo.subresourceInfoIdx];
+		for(UINT32 i = 0; i < imageInfo.numSubresourceInfos; i++)
+		{
+			ImageSubresourceInfo& entry = subresourceInfos[i];
+			if(face >= entry.range.baseArrayLayer && face < (entry.range.baseArrayLayer + entry.range.layerCount) &&
+			   mip >= entry.range.baseMipLevel && mip < (entry.range.baseMipLevel + entry.range.levelCount))
+			{
+				// If it's a FB attachment, retrieve its layout after the render pass begins
+				if(entry.isFBAttachment && inRenderPass && mFramebuffer)
+				{
+					RenderSurfaceMask readMask = getFBReadMask();
+
+					// Is it a depth-stencil attachment?
+					if(mFramebuffer->hasDepthAttachment() && mFramebuffer->getDepthStencilAttachment().image == image)
+					{
+						if (readMask.isSet(RT_DEPTH))
+						{
+							if (readMask.isSet(RT_STENCIL))
+								return VK_IMAGE_LAYOUT_DEPTH_STENCIL_READ_ONLY_OPTIMAL;
+							else // Depth readable but stencil isn't
+								return VK_IMAGE_LAYOUT_GENERAL;
+						}
+
+						return VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+					}
+					else // It is a color attachment
+					{
+						UINT32 numColorAttachments = mFramebuffer->getNumColorAttachments();
+						for (UINT32 j = 0; j < numColorAttachments; j++)
+						{
+							const VulkanFramebufferAttachment& attachment = mFramebuffer->getColorAttachment(j);
+
+							if (attachment.image == image)
+							{
+								if (readMask.isSet((RenderSurfaceMaskBits)(1 << attachment.index)))
+									return VK_IMAGE_LAYOUT_GENERAL;
+								else
+									return VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+							}
+						}
+					}
+				}
+
+				return entry.requiredLayout;
+			}
+		}
+
+		return image->getOptimalLayout();
+	}
+
 	void VulkanCmdBuffer::registerResource(VulkanResource* res, VulkanUseFlags flags)
 	{
 		auto insertResult = mResources.insert(std::make_pair(res, ResourceUseHandle()));
@@ -1753,8 +1788,8 @@ namespace bs { namespace ct
 		registerResource(res, range, layout, layout, flags, usage);
 	}
 
-	void VulkanCmdBuffer::registerResource(VulkanImage* res, const VkImageSubresourceRange& range, VkImageLayout newLayout,
-		VkImageLayout finalLayout, VulkanUseFlags flags, ResourceUsage usage)
+	void VulkanCmdBuffer::registerResource(VulkanImage* res, const VkImageSubresourceRange& range, 
+		VkImageLayout newLayout, VkImageLayout finalLayout, VulkanUseFlags flags, ResourceUsage usage)
 	{
 		bool isFBAttachment = usage == ResourceUsage::Framebuffer;
 		bool isTransfer = usage == ResourceUsage::Transfer;
@@ -2271,6 +2306,46 @@ namespace bs { namespace ct
 		acquireNewBuffer();
 	}
 
+	RenderSurfaceMask VulkanCmdBuffer::getFBReadMask()
+	{
+		// Check if any frame-buffer attachments are also used as shader inputs, in which case we make them read-only
+		RenderSurfaceMask readMask = RT_NONE;
+
+		UINT32 numColorAttachments = mFramebuffer->getNumColorAttachments();
+		for(UINT32 i = 0; i < numColorAttachments; i++)
+		{
+			const VulkanFramebufferAttachment& fbAttachment = mFramebuffer->getColorAttachment(i);
+			ImageSubresourceInfo& subresourceInfo = findSubresourceInfo(fbAttachment.image, fbAttachment.surface.arraySlice,
+				fbAttachment.surface.mipLevel);
+
+			bool readOnly = subresourceInfo.isShaderInput;
+
+			if(readOnly)
+				readMask.set((RenderSurfaceMaskBits)(1 << i));
+		}
+
+		if (mFramebuffer->hasDepthAttachment())
+		{
+			const VulkanFramebufferAttachment& fbAttachment = mFramebuffer->getDepthStencilAttachment();
+			ImageSubresourceInfo& subresourceInfo = findSubresourceInfo(fbAttachment.image, fbAttachment.surface.arraySlice,
+				fbAttachment.surface.mipLevel);
+
+			bool readOnly = subresourceInfo.isShaderInput;
+
+			if (readOnly)
+				readMask.set(RT_DEPTH);
+
+			if ((mRenderTargetReadOnlyFlags & FBT_DEPTH) != 0)
+				readMask.set(RT_DEPTH);
+
+			if ((mRenderTargetReadOnlyFlags & FBT_STENCIL) != 0)
+				readMask.set(RT_STENCIL);
+		}
+
+		return readMask;
+	}
+
+
 	VulkanCommandBuffer::~VulkanCommandBuffer()
 	{
 		mBuffer->reset();

Source/BansheeVulkanRenderAPI/BsVulkanCommandBuffer.h

@@ -343,6 +343,19 @@ namespace bs { namespace ct
 		void setLayout(VkImage image, VkAccessFlags srcAccessFlags, VkAccessFlags dstAccessFlags, 
 			VkImageLayout oldLayout, VkImageLayout newLayout, const VkImageSubresourceRange& range);
 
+		/**
+		 * Returns the current layout of the specified image, as seen by this command buffer. This is different from the
+		 * global layout stored in VulkanImage itself, as it includes any transitions performed by the command buffer
+		 * (at the current point in time), while the global layout is only updated after a command buffer as been submitted.
+		 *
+		 * @param[in]	image			Image to lookup the layout for.
+		 * @param[in]	range			Subresource range of the image to lookup the layout for.
+		 * @param[in]	inRenderPass	If true this will return the layout of the image after the render pass begins.
+		 *								If false it will return the current layout of the image. These may be different
+		 *								in the case the image is used in the framebuffer, in which case the render pass
+		 *								may perform an automated layout transition when it begins.
+		 */
+		VkImageLayout getCurrentLayout(VulkanImage* image, const VkImageSubresourceRange& range, bool inRenderPass);
 
 	private:
 		friend class VulkanCmdBufferPool;
@@ -450,6 +463,9 @@ namespace bs { namespace ct
 		/** Gets all queries registered on this command buffer that haven't been ended. */
 		void getInProgressQueries(Vector<VulkanTimerQuery*>& timer, Vector<VulkanOcclusionQuery*>& occlusion) const;
 
+		/** Returns the read mask for the current framebuffer. */
+		RenderSurfaceMask getFBReadMask();
+
 		UINT32 mId;
 		UINT32 mQueueFamily;
 		State mState;

Source/BansheeVulkanRenderAPI/BsVulkanGpuParams.cpp

@@ -714,33 +714,42 @@ namespace bs { namespace ct
 			// Register with command buffer
 			const TextureProperties& props = element->getProperties();
 
-			VkImageLayout layout;
+			const TextureSurface& surface = mSampledTextureData[i].surface;
+			VkImageSubresourceRange range = resource->getRange(surface);
 
 			// Keep dynamic textures in general layout, so they can be easily mapped by CPU
+			VkImageLayout layout;
 			if (props.getUsage() & TU_DYNAMIC)
 				layout = VK_IMAGE_LAYOUT_GENERAL;
 			else
 				layout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
 
-			const TextureSurface& surface = mSampledTextureData[i].surface;
-			VkImageSubresourceRange range = resource->getRange(surface);
-
 			buffer.registerResource(resource, range, layout, layout, VulkanUseFlag::Read, ResourceUsage::ShaderBind);
 
+			// Actual layout might be different than requested if the image is also used as a FB attachment
+			layout = buffer.getCurrentLayout(resource, range, true);
+
 			// Check if internal resource changed from what was previously bound in the descriptor set
 			assert(perDeviceData.sampledImages[i] != VK_NULL_HANDLE);
 
+			UINT32 set, slot;
+			mParamInfo->getSetSlot(GpuPipelineParamInfo::ParamType::Texture, i, set, slot);
+
+			UINT32 bindingIdx = vkParamInfo.getBindingIdx(set, slot);
+			VkDescriptorImageInfo& imgInfo = perDeviceData.perSetData[set].writeInfos[bindingIdx].image;
+
 			VkImage vkImage = resource->getHandle();
 			if (perDeviceData.sampledImages[i] != vkImage)
 			{
 				perDeviceData.sampledImages[i] = vkImage;
+				imgInfo.imageView = resource->getView(surface, false);
 
-				UINT32 set, slot;
-				mParamInfo->getSetSlot(GpuPipelineParamInfo::ParamType::Texture, i, set, slot);
-
-				UINT32 bindingIdx = vkParamInfo.getBindingIdx(set, slot);
-				perDeviceData.perSetData[set].writeInfos[bindingIdx].image.imageView = resource->getView(surface, false);
+				mSetsDirty[set] = true;
+			}
 
+			if(imgInfo.imageLayout != layout)
+			{
+				imgInfo.imageLayout = layout;
 				mSetsDirty[set] = true;
 			}
 		}