Vulkan resource tracking now works across multiple command buffers

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBuffer.h

@@ -76,6 +76,9 @@ namespace BansheeEngine
 		/** Returns an unique identifier of this command buffer. */
 		UINT32 getId() const { return mId; }
 
+		/** Returns the index of the queue family this command buffer is executing on. */
+		UINT32 getQueueFamily() const { return mQueueFamily; }
+
 		/** Makes the command buffer ready to start recording commands. */
 		void begin();
 

Source/BansheeVulkanRenderAPI/Include/BsVulkanResource.h

@@ -3,6 +3,7 @@
 #pragma once
 
 #include "BsVulkanPrerequisites.h"
+#include "BsStaticAlloc.h"
 
 namespace BansheeEngine
 {
@@ -30,7 +31,7 @@ namespace BansheeEngine
 	class VulkanResource
 	{
 	public:
-		VulkanResource(VulkanResourceManager* owner);
+		VulkanResource(VulkanResourceManager* owner, bool concurrency);
 		virtual ~VulkanResource();
 
 		/** 
@@ -39,13 +40,13 @@ namespace BansheeEngine
 		 * 
 		 * A resource can only be used by a single command buffer at a time.
 		 */
-		virtual void notifyUsed(VulkanCmdBuffer& buffer, VulkanUseFlags flags);
+		virtual void notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags flags);
 
 		/** 
 		 * Notifies the resource that it is no longer used by on the GPU. This makes the resource usable on other command
 		 * buffers again.
 		 */
-		virtual void notifyDone();
+		virtual void notifyDone(VulkanCmdBuffer* buffer);
 
 		/** 
 		 * Checks is the resource currently used on a device. 
@@ -54,7 +55,7 @@ namespace BansheeEngine
 		 *			done on the device but this method may still report true. If you need to know the latest state
 		 *			call VulkanCommandBufferManager::refreshStates() before checking for usage.
 		 */
-		bool isUsed() const { return mCmdBufferId != -1; }
+		bool isUsed() const { return mNumHandles > 0; }
 
 		/** 
 		 * Destroys the resource and frees its memory. If the resource is currently being used on a device, the
@@ -63,10 +64,31 @@ namespace BansheeEngine
 		void destroy();
 
 	protected:
+		/** Possible states of this object. */
+		enum class State
+		{
+			Normal,
+			Shared,
+			Destroyed
+		};
+
+		/** Information about use of this resource on a specific command buffer. */
+		struct UseHandle
+		{
+			VulkanCmdBuffer* buffer;
+			VulkanUseFlags flags;
+		};
+
 		VulkanResourceManager* mOwner;
-		VulkanUseFlags mFlags;
-		UINT32 mCmdBufferId = -1;
-		bool mIsDestroyed = false;
+		UINT32 mQueueFamily;
+		State mState;
+		
+		UseHandle* mHandles;
+		UINT32 mNumHandles;
+		UINT32 mHandleCapacity;
+		
+		static const UINT32 INITIAL_HANDLE_CAPACITY = 2;
+		StaticAlloc<sizeof(UseHandle) * INITIAL_HANDLE_CAPACITY> mAlloc;
 	};
 
 	/** Creates and destroys annd VulkanResource%s on a single device. */
@@ -78,11 +100,13 @@ namespace BansheeEngine
 		/** 
 		 * Creates a new Vulkan resource of the specified type. User must call VulkanResource::destroy() when done using
 		 * the resource. 
+		 * 
+		 * @param[in]	concurrency		If true, the resource is allowed to be used on multiple queue types at once.
 		 */
 		template<class Type, class... Args>
-		VulkanResource* create(Args &&...args)
+		VulkanResource* create(bool concurrency, Args &&...args)
 		{
-			VulkanResource* resource = new (bs_alloc(sizeof(Type))) Type(std::forward<Args>(args)...);
+			VulkanResource* resource = new (bs_alloc(sizeof(Type))) Type(this, concurrency, std::forward<Args>(args)...);
 
 #if BS_DEBUG_MODE
 			mResources.insert(resource);

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBuffer.cpp

@@ -214,7 +214,7 @@ namespace BansheeEngine
 				mFenceCounter++;
 
 				for (auto& entry : mResources)
-					entry.first->notifyDone();
+					entry.first->notifyDone(this);
 
 				mResources.clear();
 			}
@@ -233,7 +233,7 @@ namespace BansheeEngine
 		// TODO - Issue pipeline barrier for resources transitioning to a new queue family
 
 		for (auto& entry : mResources)
-			entry.first->notifyUsed(*this, entry.second.flags);
+			entry.first->notifyUsed(this, entry.second.flags);
 	}
 
 	VulkanCommandBuffer::VulkanCommandBuffer(VulkanDevice& device, UINT32 id, GpuQueueType type, UINT32 deviceIdx,

Source/BansheeVulkanRenderAPI/Source/BsVulkanResource.cpp

@@ -5,42 +5,93 @@
 
 namespace BansheeEngine
 {
-	VulkanResource::VulkanResource(VulkanResourceManager* owner)
-		:mOwner(owner)
+	VulkanResource::VulkanResource(VulkanResourceManager* owner, bool concurrency)
+		: mOwner(owner), mState(concurrency ? State::Shared : State::Normal), mNumHandles(0)
+		, mHandleCapacity(INITIAL_HANDLE_CAPACITY)
 	{
-		
+		UINT32 bytesCapacity = sizeof(UseHandle) * mHandleCapacity;
+		mHandles = (UseHandle*)mAlloc.alloc(sizeof(UseHandle) * INITIAL_HANDLE_CAPACITY);
+		memset(mHandles, 0, bytesCapacity);
 	}
 
 	VulkanResource::~VulkanResource()
 	{
-		assert(mIsDestroyed && "Vulkan resource getting destructed without destroy() called first.");
+		assert(mState == State::Destroyed && "Vulkan resource getting destructed without destroy() called first.");
+
+		mAlloc.free(mHandles);
+		mAlloc.clear();
 	}
 
-	void VulkanResource::notifyUsed(VulkanCmdBuffer& buffer, VulkanUseFlags flags)
+	void VulkanResource::notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags flags)
 	{
-		assert(!isUsed() && !mIsDestroyed);
+		assert(mState != State::Destroyed);
+
+		if(isUsed() && mState == State::Normal) // Used without support for concurrency
+		{
+			assert(mQueueFamily == buffer->getQueueFamily() && 
+				"Vulkan resource without concurrency support can only be used by one queue family at once.");
+		}
+
+		mNumHandles++;
+
+		if(mNumHandles > mHandleCapacity)
+		{
+			UINT32 bytesCapacity = sizeof(UseHandle) * mHandleCapacity;
+			void* tempData = bs_stack_alloc(bytesCapacity);
+			memcpy(tempData, mHandles, bytesCapacity);
+
+			mAlloc.free(mHandles);
+			mHandles = (UseHandle*)mAlloc.alloc(bytesCapacity * 2);
+			memcpy(mHandles, tempData, bytesCapacity);
+
+			bs_stack_free(tempData);
 
-		// Note: I could allow resource usage on multiple command buffers at once by keeping a list of separate usage
-		// flags and ID's, per buffer.
+			memset(mHandles + mHandleCapacity, 0, bytesCapacity);
+			mHandleCapacity *= 2;
+		}
 
-		mCmdBufferId = buffer.getId();
-		mFlags |= flags;
+		for(UINT32 i = 0; i < mHandleCapacity; i++)
+		{
+			if (mHandles[i].buffer == nullptr)
+				continue;
+
+			mHandles[i].buffer = buffer;
+			mHandles[i].flags |= flags;
+
+			break;
+		}
+
+		mQueueFamily = buffer->getQueueFamily();
 	}
 
-	void VulkanResource::notifyDone()
+	void VulkanResource::notifyDone(VulkanCmdBuffer* buffer)
 	{
-		mCmdBufferId = -1;
-		mFlags = VulkanUseFlag::None;
+		bool foundBuffer = false;
+		for(UINT32 i = 0; i < mNumHandles; i++)
+		{
+			if(mHandles[i].buffer == buffer)
+			{
+				mHandles[i].buffer = nullptr;
+				mHandles[i].flags = VulkanUseFlag::None;
+
+				foundBuffer = true;
+				break;
+			}
+		}
+
+		assert(foundBuffer);
+
+		mNumHandles--;
 
-		if (mIsDestroyed) // Queued for destruction
+		if (!isUsed() && mState == State::Destroyed) // Queued for destruction
 			mOwner->destroy(this);
 	}
 
 	void VulkanResource::destroy()
 	{
-		assert(!mIsDestroyed && "Vulkan resource destroy() called more than once.");
+		assert(mState != State::Destroyed && "Vulkan resource destroy() called more than once.");
 
-		mIsDestroyed = true;
+		mState = State::Destroyed;
 
 		// If not used, destroy right away, otherwise check when it is reported as finished on the device
 		if (!isUsed())