VulkanResource now tracks exactly on which queue is it being used on

Source/BansheeUtility/Include/BsUtil.h

@@ -26,5 +26,28 @@ namespace BansheeEngine
 	/**	Generates an MD5 hash string for the provided source string. */
 	String BS_UTILITY_EXPORT md5(const String& source);
 
+	/** Sets contents of a struct to zero. */
+	template<class T>
+	void bs_zero_out(T& s)
+	{
+		std::memset(&s, 0, sizeof(T));
+	}
+
+	/** Sets contents of a static array to zero. */
+	template<class T, size_t N>
+	void bs_zero_out(T(&arr)[N])
+	{
+		std::memset(arr, 0, sizeof(T) * N);
+	}
+
+	/** Sets contents of a block of memory to zero. */
+	template<class T>
+	void bs_zero_out(T * arr, size_t count)
+	{
+		assert(arr != nullptr);
+		assert(count != 0);
+		std::memset(arr, 0, sizeof(T) * count);
+	}
+
 	/** @} */
 }

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBuffer.h

@@ -207,6 +207,7 @@ namespace BansheeEngine
 		UnorderedMap<VulkanResource*, ResourceUseHandle> mResources;
 		UnorderedMap<VulkanResource*, ImageInfo> mImages;
 		UnorderedMap<VulkanResource*, BufferInfo> mBuffers;
+		UINT32 mGlobalQueueIdx;
 
 		VkSemaphore mSemaphoresTemp[BS_MAX_COMMAND_BUFFERS + 1]; // +1 for present semaphore
 		UnorderedMap<UINT32, TransitionInfo> mTransitionInfoTemp;

Source/BansheeVulkanRenderAPI/Include/BsVulkanResource.h

@@ -51,16 +51,23 @@ namespace BansheeEngine
 		 * A resource can only be used by a single command buffer at a time unless resource concurrency is enabled.
 		 * 
 		 * Must follow a notifyBound(). Must eventually be followed by a notifyDone().
+		 * 
+		 * @param[in]	globalQueueIdx	Global index of the queue the resource is being used in.
+		 * @param[in]	queueFamily		Family of the queue the resource is being used in.
+		 * @param[in]	useFlags		Flags that determine in what way is the resource being used.
 		 */
-		void notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags useFlags);
+		void notifyUsed(UINT32 globalQueueIdx, UINT32 queueFamily, VulkanUseFlags useFlags);
 
 		/** 
 		 * Notifies the resource that it is no longer used by on the GPU. This makes the resource usable on other command
 		 * buffers again.
 		 * 
 		 * Must follow a notifyUsed().
+		 * 
+		 * @param[in]	globalQueueIdx	Global index of the queue that finished using the resource.
+		 * @param[in]	useFlags		Use flags that specify how was the resource being used.
 		 */
-		void notifyDone();
+		void notifyDone(UINT32 globalQueueIdx, VulkanUseFlags useFlags);
 
 		/** 
 		 * Notifies the resource that it is no longer queued on the command buffer. This is similar to notifyDone(), but
@@ -97,6 +104,15 @@ namespace BansheeEngine
 		 */
 		UINT32 getQueueFamily() const { Lock(mMutex); return mQueueFamily; }
 
+		/** 
+		 * Returns a mask that has bits set for every queue that the resource is currently used by.
+		 *
+		 * @param[out]	useFlags	Output parameter that notifies the caller in what way is the resource being used.
+		 * @return					Bitmask of which queues is the resource used on. This has the same format as sync mask
+		 *							created by CommandSyncMask.
+		 */
+		UINT32 getUseInfo(VulkanUseFlags& useFlags) const;
+
 		/** Returns true if the resource is only allowed to be used by a single queue family at once. */
 		bool isExclusive() const { Lock(mMutex); return mState != State::Shared; }
 
@@ -115,10 +131,14 @@ namespace BansheeEngine
 			Destroyed
 		};
 
+		static const UINT32 MAX_UNIQUE_QUEUES = BS_MAX_QUEUES_PER_TYPE * GQT_COUNT;
+
 		VulkanResourceManager* mOwner;
 		UINT32 mQueueFamily;
 		State mState;
-		VulkanUseFlags mUseFlags;
+
+		UINT8 mReadUses[MAX_UNIQUE_QUEUES];
+		UINT8 mWriteUses[MAX_UNIQUE_QUEUES];
 		
 		UINT32 mNumUsedHandles;
 		UINT32 mNumBoundHandles;

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBuffer.cpp

@@ -100,7 +100,7 @@ namespace BansheeEngine
 	VulkanCmdBuffer::VulkanCmdBuffer(VulkanDevice& device, UINT32 id, VkCommandPool pool, UINT32 queueFamily, bool secondary)
 		: mId(id), mQueueFamily(queueFamily), mState(State::Ready), mDevice(device), mPool(pool), mFenceCounter(0)
 		, mFramebuffer(VK_NULL_HANDLE), mRenderPass(VK_NULL_HANDLE), mPresentSemaphore(VK_NULL_HANDLE)
-		, mRenderTargetWidth(0), mRenderTargetHeight(0)
+		, mRenderTargetWidth(0), mRenderTargetHeight(0), mGlobalQueueIdx(-1)
 	{
 		VkCommandBufferAllocateInfo cmdBufferAllocInfo;
 		cmdBufferAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
@@ -250,167 +250,12 @@ namespace BansheeEngine
 		mState = State::Recording;
 	}
 
-	void VulkanCmdBuffer::refreshFenceStatus()
-	{
-		VkResult result = vkGetFenceStatus(mDevice.getLogical(), mFence);
-		assert(result == VK_SUCCESS || result == VK_NOT_READY);
-
-		bool signaled = result == VK_SUCCESS;
-
-		if (mState == State::Submitted)
-		{
-			if(signaled)
-			{
-				mState = State::Ready;
-				vkResetCommandBuffer(mCmdBuffer, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); // Note: Maybe better not to release resources?
-
-				result = vkResetFences(mDevice.getLogical(), 1, &mFence);
-				assert(result == VK_SUCCESS);
-
-				mFenceCounter++;
-
-				for (auto& entry : mResources)
-				{
-					ResourceUseHandle& useHandle = entry.second;
-					assert(useHandle.used);
-
-					entry.first->notifyDone();
-				}
-
-				for (auto& entry : mImages)
-				{
-					ResourceUseHandle& useHandle = entry.second.useHandle;
-					assert(useHandle.used);
-
-					entry.first->notifyDone();
-				}
-
-				for (auto& entry : mBuffers)
-				{
-					ResourceUseHandle& useHandle = entry.second.useHandle;
-					assert(useHandle.used);
-
-					entry.first->notifyDone();
-				}
-
-				mResources.clear();
-				mImages.clear();
-				mBuffers.clear();
-			}
-		}
-		else
-			assert(!signaled); // We reset the fence along with mState so this shouldn't be possible
-	}
-
-	void VulkanCmdBuffer::setRenderTarget(const SPtr<RenderTargetCore>& rt)
-	{
-		assert(mState != State::RecordingRenderPass && mState != State::Submitted);
-
-		if(rt == nullptr)
-		{
-			mFramebuffer = VK_NULL_HANDLE;
-			mRenderPass = VK_NULL_HANDLE;
-			mPresentSemaphore = VK_NULL_HANDLE;
-			mRenderTargetWidth = 0;
-			mRenderTargetHeight = 0;
-		}
-		else
-		{
-			rt->getCustomAttribute("FB", &mFramebuffer);
-			rt->getCustomAttribute("RP", &mRenderPass);
-			
-			if (rt->getProperties().isWindow())
-				rt->getCustomAttribute("PS", &mPresentSemaphore);
-			else
-				mPresentSemaphore = VK_NULL_HANDLE;
-
-			mRenderTargetWidth = rt->getProperties().getWidth();
-			mRenderTargetHeight = rt->getProperties().getHeight();
-		}
-
-
-
-		// TODO
-	}
-
-	void VulkanCmdBuffer::registerResource(VulkanResource* res, VulkanUseFlags flags)
-	{
-		auto insertResult = mResources.insert(std::make_pair(res, ResourceUseHandle()));
-		if(insertResult.second) // New element
-		{
-			ResourceUseHandle& useHandle = insertResult.first->second;
-			useHandle.used = false;
-			useHandle.flags = flags;
-
-			res->notifyBound();
-		}
-		else // Existing element
-		{
-			ResourceUseHandle& useHandle = insertResult.first->second;
-
-			assert(!useHandle.used);
-			useHandle.flags |= flags;
-		}
-	}
-
-	void VulkanCmdBuffer::registerResource(VulkanImage* res, VkAccessFlags accessFlags, VkImageLayout layout, 
-		const VkImageSubresourceRange& range, VulkanUseFlags flags)
-	{
-		auto insertResult = mImages.insert(std::make_pair(res, ImageInfo()));
-		if (insertResult.second) // New element
-		{
-			ImageInfo& imageInfo = insertResult.first->second;
-			imageInfo.accessFlags = accessFlags;
-			imageInfo.layout = layout;
-			imageInfo.range = range;
-
-			imageInfo.useHandle.used = false;
-			imageInfo.useHandle.flags = flags;
-
-			res->notifyBound();
-		}
-		else // Existing element
-		{
-			ImageInfo& imageInfo = insertResult.first->second;
-
-			assert(!imageInfo.useHandle.used);
-			imageInfo.useHandle.flags |= flags;
-
-			assert(imageInfo.layout == layout && "Cannot bind the same image with two different layouts on the same command buffer.");
-			imageInfo.accessFlags |= accessFlags;
-			imageInfo.range = range;
-		}
-	}
-
-	void VulkanCmdBuffer::registerResource(VulkanBuffer* res, VkAccessFlags accessFlags, VulkanUseFlags flags)
-	{
-		auto insertResult = mBuffers.insert(std::make_pair(res, BufferInfo()));
-		if (insertResult.second) // New element
-		{
-			BufferInfo& bufferInfo = insertResult.first->second;
-			bufferInfo.accessFlags = accessFlags;
-
-			bufferInfo.useHandle.used = false;
-			bufferInfo.useHandle.flags = flags;
-
-			res->notifyBound();
-		}
-		else // Existing element
-		{
-			BufferInfo& bufferInfo = insertResult.first->second;
-
-			assert(!bufferInfo.useHandle.used);
-			bufferInfo.useHandle.flags |= flags;
-			bufferInfo.accessFlags |= accessFlags;
-		}
-	}
-
 	void VulkanCmdBuffer::submit(VulkanQueue* queue, UINT32 queueIdx, UINT32 syncMask)
 	{
 		assert(isReadyForSubmit());
 
 		// Issue pipeline barriers for queue transitions (need to happen on original queue first, then on new queue)
-		for(auto& entry : mBuffers)
+		for (auto& entry : mBuffers)
 		{
 			VulkanBuffer* resource = static_cast<VulkanBuffer*>(entry.first);
 
@@ -436,7 +281,7 @@ namespace BansheeEngine
 			}
 		}
 
-		for(auto& entry : mImages)
+		for (auto& entry : mImages)
 		{
 			VulkanImage* resource = static_cast<VulkanImage*>(entry.first);
 
@@ -501,11 +346,11 @@ namespace BansheeEngine
 					continue;
 
 				UINT32 numQueues = device.getNumQueues(otherQueueType);
-				for(UINT32 j = 0; j < numQueues; j++)
+				for (UINT32 j = 0; j < numQueues; j++)
 				{
 					// Try to find a queue not currently executing
 					VulkanQueue* curQueue = device.getQueue(otherQueueType, j);
-					if(!curQueue->isExecuting())
+					if (!curQueue->isExecuting())
 					{
 						otherQueue = curQueue;
 						otherQueueIdx = j;
@@ -513,7 +358,7 @@ namespace BansheeEngine
 				}
 
 				// Can't find empty one, use the first one then
-				if(otherQueue == nullptr)
+				if (otherQueue == nullptr)
 				{
 					otherQueue = device.getQueue(otherQueueType, 0);
 					otherQueueIdx = 0;
@@ -576,13 +421,14 @@ namespace BansheeEngine
 
 		queue->submit(this, mSemaphoresTemp, numSemaphores);
 
+		mGlobalQueueIdx = CommandSyncMask::getGlobalQueueIdx(queue->getType(), queueIdx);
 		for (auto& entry : mResources)
 		{
 			ResourceUseHandle& useHandle = entry.second;
 			assert(!useHandle.used);
 
 			useHandle.used = true;
-			entry.first->notifyUsed(this, useHandle.flags);
+			entry.first->notifyUsed(mGlobalQueueIdx, mQueueFamily, useHandle.flags);
 		}
 
 		for (auto& entry : mImages)
@@ -591,7 +437,7 @@ namespace BansheeEngine
 			assert(!useHandle.used);
 
 			useHandle.used = true;
-			entry.first->notifyUsed(this, useHandle.flags);
+			entry.first->notifyUsed(mGlobalQueueIdx, mQueueFamily, useHandle.flags);
 		}
 
 		for (auto& entry : mBuffers)
@@ -600,7 +446,7 @@ namespace BansheeEngine
 			assert(!useHandle.used);
 
 			useHandle.used = true;
-			entry.first->notifyUsed(this, useHandle.flags);
+			entry.first->notifyUsed(mGlobalQueueIdx, mQueueFamily, useHandle.flags);
 		}
 
 		cbm.refreshStates(deviceIdx);
@@ -620,6 +466,161 @@ namespace BansheeEngine
 		}
 	}
 
+	void VulkanCmdBuffer::refreshFenceStatus()
+	{
+		VkResult result = vkGetFenceStatus(mDevice.getLogical(), mFence);
+		assert(result == VK_SUCCESS || result == VK_NOT_READY);
+
+		bool signaled = result == VK_SUCCESS;
+
+		if (mState == State::Submitted)
+		{
+			if(signaled)
+			{
+				mState = State::Ready;
+				vkResetCommandBuffer(mCmdBuffer, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); // Note: Maybe better not to release resources?
+
+				result = vkResetFences(mDevice.getLogical(), 1, &mFence);
+				assert(result == VK_SUCCESS);
+
+				mFenceCounter++;
+
+				for (auto& entry : mResources)
+				{
+					ResourceUseHandle& useHandle = entry.second;
+					assert(useHandle.used);
+
+					entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
+				}
+
+				for (auto& entry : mImages)
+				{
+					ResourceUseHandle& useHandle = entry.second.useHandle;
+					assert(useHandle.used);
+
+					entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
+				}
+
+				for (auto& entry : mBuffers)
+				{
+					ResourceUseHandle& useHandle = entry.second.useHandle;
+					assert(useHandle.used);
+
+					entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
+				}
+
+				mResources.clear();
+				mImages.clear();
+				mBuffers.clear();
+			}
+		}
+		else
+			assert(!signaled); // We reset the fence along with mState so this shouldn't be possible
+	}
+
+	void VulkanCmdBuffer::setRenderTarget(const SPtr<RenderTargetCore>& rt)
+	{
+		assert(mState != State::RecordingRenderPass && mState != State::Submitted);
+
+		if(rt == nullptr)
+		{
+			mFramebuffer = VK_NULL_HANDLE;
+			mRenderPass = VK_NULL_HANDLE;
+			mPresentSemaphore = VK_NULL_HANDLE;
+			mRenderTargetWidth = 0;
+			mRenderTargetHeight = 0;
+		}
+		else
+		{
+			rt->getCustomAttribute("FB", &mFramebuffer);
+			rt->getCustomAttribute("RP", &mRenderPass);
+			
+			if (rt->getProperties().isWindow())
+				rt->getCustomAttribute("PS", &mPresentSemaphore);
+			else
+				mPresentSemaphore = VK_NULL_HANDLE;
+
+			mRenderTargetWidth = rt->getProperties().getWidth();
+			mRenderTargetHeight = rt->getProperties().getHeight();
+		}
+
+
+
+		// TODO
+	}
+
+	void VulkanCmdBuffer::registerResource(VulkanResource* res, VulkanUseFlags flags)
+	{
+		auto insertResult = mResources.insert(std::make_pair(res, ResourceUseHandle()));
+		if(insertResult.second) // New element
+		{
+			ResourceUseHandle& useHandle = insertResult.first->second;
+			useHandle.used = false;
+			useHandle.flags = flags;
+
+			res->notifyBound();
+		}
+		else // Existing element
+		{
+			ResourceUseHandle& useHandle = insertResult.first->second;
+
+			assert(!useHandle.used);
+			useHandle.flags |= flags;
+		}
+	}
+
+	void VulkanCmdBuffer::registerResource(VulkanImage* res, VkAccessFlags accessFlags, VkImageLayout layout, 
+		const VkImageSubresourceRange& range, VulkanUseFlags flags)
+	{
+		auto insertResult = mImages.insert(std::make_pair(res, ImageInfo()));
+		if (insertResult.second) // New element
+		{
+			ImageInfo& imageInfo = insertResult.first->second;
+			imageInfo.accessFlags = accessFlags;
+			imageInfo.layout = layout;
+			imageInfo.range = range;
+
+			imageInfo.useHandle.used = false;
+			imageInfo.useHandle.flags = flags;
+
+			res->notifyBound();
+		}
+		else // Existing element
+		{
+			ImageInfo& imageInfo = insertResult.first->second;
+
+			assert(!imageInfo.useHandle.used);
+			imageInfo.useHandle.flags |= flags;
+
+			assert(imageInfo.layout == layout && "Cannot bind the same image with two different layouts on the same command buffer.");
+			imageInfo.accessFlags |= accessFlags;
+			imageInfo.range = range;
+		}
+	}
+
+	void VulkanCmdBuffer::registerResource(VulkanBuffer* res, VkAccessFlags accessFlags, VulkanUseFlags flags)
+	{
+		auto insertResult = mBuffers.insert(std::make_pair(res, BufferInfo()));
+		if (insertResult.second) // New element
+		{
+			BufferInfo& bufferInfo = insertResult.first->second;
+			bufferInfo.accessFlags = accessFlags;
+
+			bufferInfo.useHandle.used = false;
+			bufferInfo.useHandle.flags = flags;
+
+			res->notifyBound();
+		}
+		else // Existing element
+		{
+			BufferInfo& bufferInfo = insertResult.first->second;
+
+			assert(!bufferInfo.useHandle.used);
+			bufferInfo.useHandle.flags |= flags;
+			bufferInfo.accessFlags |= accessFlags;
+		}
+	}
+
 	VulkanCommandBuffer::VulkanCommandBuffer(VulkanDevice& device, GpuQueueType type, UINT32 deviceIdx,
 		UINT32 queueIdx, bool secondary)
 		: CommandBuffer(type, deviceIdx, queueIdx, secondary), mBuffer(nullptr)

Source/BansheeVulkanRenderAPI/Source/BsVulkanResource.cpp

@@ -15,6 +15,9 @@ namespace BansheeEngine
 		mState = concurrency ? State::Shared : State::Normal;
 		mNumUsedHandles = 0;
 		mNumBoundHandles = 0;
+
+		bs_zero_out(mReadUses);
+		bs_zero_out(mWriteUses);
 	}
 
 	VulkanResource::~VulkanResource()
@@ -33,33 +36,53 @@ namespace BansheeEngine
 		mNumBoundHandles++;
 	}
 
-	void VulkanResource::notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags useFlags)
+	void VulkanResource::notifyUsed(UINT32 globalQueueIdx, UINT32 queueFamily, VulkanUseFlags useFlags)
 	{
 		Lock lock(mMutex);
 		assert(mState != State::Destroyed);
+		assert(useFlags != VulkanUseFlag::None);
 
 		if(isUsed() && mState == State::Normal) // Used without support for concurrency
 		{
-			assert(mQueueFamily == buffer->getQueueFamily() && 
+			assert(mQueueFamily == queueFamily &&
 				"Vulkan resource without concurrency support can only be used by one queue family at once.");
 		}
 
 		mNumUsedHandles++;
-		mQueueFamily = buffer->getQueueFamily();
-		mUseFlags |= useFlags;
+		mQueueFamily = queueFamily;
+
+		assert(globalQueueIdx < MAX_UNIQUE_QUEUES);
+		
+		if (useFlags.isSet(VulkanUseFlag::Read))
+		{
+			assert(mReadUses[globalQueueIdx] < 255 && "Resource used in too many command buffers at once.");
+			mReadUses[globalQueueIdx]++;
+		}
+		
+		if(useFlags.isSet(VulkanUseFlag::Write))
+		{
+			assert(mWriteUses[globalQueueIdx] < 255 && "Resource used in too many command buffers at once.");
+			mWriteUses[globalQueueIdx]++;
+		}
 	}
 
-	void VulkanResource::notifyDone()
+	void VulkanResource::notifyDone(UINT32 globalQueueIdx, VulkanUseFlags useFlags)
 	{
 		Lock lock(mMutex);
 		mNumUsedHandles--;
 		mNumBoundHandles--;
 
-		// Note: If resource is used on different command buffers with different use flags, we should clear individual flags
-		// depending on which command buffer finished. But this requires extra per-command buffer state tracking, so we
-		// instead just clear all flags at once when all command buffers finish.
-		if (!isUsed())
-			mUseFlags = VulkanUseFlag::None;
+		if (useFlags.isSet(VulkanUseFlag::Read))
+		{
+			assert(mReadUses[globalQueueIdx] > 0);
+			mReadUses[globalQueueIdx]--;
+		}
+
+		if (useFlags.isSet(VulkanUseFlag::Write))
+		{
+			assert(mWriteUses[globalQueueIdx] > 0);
+			mWriteUses[globalQueueIdx]--;
+		}
 
 		if (!isBound() && mState == State::Destroyed) // Queued for destruction
 			mOwner->destroy(this);
@@ -74,6 +97,29 @@ namespace BansheeEngine
 			mOwner->destroy(this);
 	}
 
+	UINT32 VulkanResource::getUseInfo(VulkanUseFlags& useFlags) const
+	{
+		useFlags = VulkanUseFlag::None;
+
+		UINT32 mask = 0;
+		for(UINT32 i = 0; i < MAX_UNIQUE_QUEUES; i++)
+		{
+			if (mReadUses[i] > 0)
+			{
+				mask |= 1 << i;
+				useFlags |= VulkanUseFlag::Read;
+			}
+
+			if (mWriteUses[i] > 0)
+			{
+				mask |= 1 << i;
+				useFlags |= VulkanUseFlag::Write;
+			}
+		}
+
+		return mask;
+	}
+
 	void VulkanResource::destroy()
 	{
 		Lock lock(mMutex);