VulkanResource changes:

 - Separate flags to check if its bound or used on a command buffer
 - It is now thread safe

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBuffer.h

@@ -145,9 +145,10 @@ namespace BansheeEngine
 		friend class VulkanCmdBufferPool;
 		friend class VulkanCommandBuffer;
 
-		/** Information about a resource currently queued for use on the command buffer. */
-		struct ResourceInfo
+		/** Contains information about a single Vulkan resource bound/used on this command buffer. */
+		struct ResourceUseHandle
 		{
+			bool used;
 			VulkanUseFlags flags;
 		};
 
@@ -161,7 +162,7 @@ namespace BansheeEngine
 		VkSemaphore mSemaphore;
 		UINT32 mFenceCounter;
 
-		UnorderedMap<VulkanResource*, ResourceInfo> mResources;
+		UnorderedMap<VulkanResource*, ResourceUseHandle> mResources;
 		UnorderedSet<SPtr<VulkanGpuParams>> mBoundParams;
 
 		VkSemaphore mSemaphoresTemp[BS_MAX_COMMAND_BUFFERS];

Source/BansheeVulkanRenderAPI/Include/BsVulkanResource.h

@@ -35,6 +35,8 @@ namespace BansheeEngine
 	/** 
 	 * Wraps one or multiple native Vulkan objects. Allows the object usage to be tracked in command buffers, handles
 	 * ownership transitions between different queues, and handles delayed object destruction.
+	 * 
+	 * @note Thread safe
 	 */
 	class VulkanResource
 	{
@@ -42,19 +44,25 @@ namespace BansheeEngine
 		VulkanResource(VulkanResourceManager* owner, bool concurrency, VulkanResourceType type = VulkanResourceType::Generic);
 		virtual ~VulkanResource();
 
+		/** 
+		 * Notifies the resource that it is currently bound to a command buffer. Buffer hasn't yet been submitted so the
+		 * resource isn't being used on the GPU yet.
+		 */
+		void notifyBound();
+
 		/** 
 		 * Notifies the resource that it is currently being used on the provided command buffer. This means the command
 		 * buffer has actually been submitted to the queue and the resource is used by the GPU.
 		 * 
-		 * A resource can only be used by a single command buffer at a time.
+		 * A resource can only be used by a single command buffer at a time unless resource concurrency is enabled.
 		 */
-		virtual void notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags flags);
+		void notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags useFlags);
 
 		/** 
 		 * Notifies the resource that it is no longer used by on the GPU. This makes the resource usable on other command
 		 * buffers again.
 		 */
-		virtual void notifyDone(VulkanCmdBuffer* buffer);
+		void notifyDone();
 
 		/** 
 		 * Checks is the resource currently used on a device. 
@@ -63,10 +71,19 @@ namespace BansheeEngine
 		 *			done on the device but this method may still report true. If you need to know the latest state
 		 *			call VulkanCommandBufferManager::refreshStates() before checking for usage.
 		 */
-		bool isUsed() const { return mNumHandles > 0; }
+		bool isUsed() const { Lock(mMutex); return mNumUsedHandles > 0; }
+
+		/** 
+		 * Checks is the resource currently bound to any command buffer.
+		 *
+		 * @note	Resource usage is only checked at certain points of the program. This means the resource could be
+		 *			done on the device but this method may still report true. If you need to know the latest state
+		 *			call VulkanCommandBufferManager::refreshStates() before checking for usage.
+		 */
+		bool isBound() const { Lock(mMutex); return mNumBoundHandles > 0; }
 
 		/** Returns the type of the object wrapped by the resource. */
-		VulkanResourceType getType() const { return mType; }
+		VulkanResourceType getType() const { Lock(mMutex); return mType; }
 
 		/** 
 		 * Returns the queue family the resource is currently owned by. Returns -1 if owned by no queue.
@@ -74,10 +91,10 @@ namespace BansheeEngine
 		 * @note	If resource concurrency is enabled, then this value has no meaning as the resource can be used on
 		 *			multiple queue families at once.
 		 */
-		UINT32 getQueueFamily() const { return mQueueFamily; }
+		UINT32 getQueueFamily() const { Lock(mMutex); return mQueueFamily; }
 
 		/** Returns true if the resource is only allowed to be used by a single queue family at once. */
-		bool isExclusive() const { return mState != State::Shared; }
+		bool isExclusive() const { Lock(mMutex); return mState != State::Shared; }
 
 		/** 
 		 * Destroys the resource and frees its memory. If the resource is currently being used on a device, the
@@ -94,27 +111,23 @@ namespace BansheeEngine
 			Destroyed
 		};
 
-		/** Information about use of this resource on a specific command buffer. */
-		struct UseHandle
-		{
-			VulkanCmdBuffer* buffer;
-			VulkanUseFlags flags;
-		};
-
 		VulkanResourceManager* mOwner;
 		UINT32 mQueueFamily;
 		State mState;
 		VulkanResourceType mType;
+		VulkanUseFlags mUseFlags;
 		
-		UseHandle* mHandles;
-		UINT32 mNumHandles;
-		UINT32 mHandleCapacity;
-		
-		static const UINT32 INITIAL_HANDLE_CAPACITY = 2;
-		StaticAlloc<sizeof(UseHandle) * INITIAL_HANDLE_CAPACITY> mAlloc;
+		UINT32 mNumUsedHandles;
+		UINT32 mNumBoundHandles;
+
+		Mutex mMutex;
 	};
 
-	/** Creates and destroys annd VulkanResource%s on a single device. */
+	/** 
+	 * Creates and destroys annd VulkanResource%s on a single device. 
+	 * 
+	 * @note Thread safe
+	 */
 	class VulkanResourceManager
 	{
 	public:
@@ -131,6 +144,7 @@ namespace BansheeEngine
 			Type* resource = new (bs_alloc(sizeof(Type))) Type(this, std::forward<Args>(args)...);
 
 #if BS_DEBUG_MODE
+			Lock lock(mMutex);
 			mResources.insert(resource);
 #endif
 
@@ -153,6 +167,7 @@ namespace BansheeEngine
 
 #if BS_DEBUG_MODE
 		UnorderedSet<VulkanResource*> mResources;
+		Mutex mMutex;
 #endif
 	};
 

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBuffer.cpp

@@ -215,7 +215,12 @@ namespace BansheeEngine
 				mFenceCounter++;
 
 				for (auto& entry : mResources)
-					entry.first->notifyDone(this);
+				{
+					ResourceUseHandle& useHandle = entry.second;
+					assert(useHandle.used);
+
+					entry.first->notifyDone();
+				}
 
 				mResources.clear();
 				mBoundParams.clear();
@@ -227,7 +232,22 @@ namespace BansheeEngine
 
 	void VulkanCmdBuffer::registerResource(VulkanResource* res, VulkanUseFlags flags)
 	{
-		mResources[res].flags |= flags;
+		auto insertResult = mResources.insert(std::make_pair(res, ResourceUseHandle()));
+		if(insertResult.second) // New element
+		{
+			ResourceUseHandle& useHandle = insertResult.first->second;
+			useHandle.used = false;
+			useHandle.flags = flags;
+
+			res->notifyBound();
+		}
+		else // Existing element
+		{
+			ResourceUseHandle& useHandle = insertResult.first->second;
+
+			assert(!useHandle.used);
+			useHandle.flags |= flags;
+		}
 	}
 
 	void VulkanCmdBuffer::registerGpuParams(const SPtr<VulkanGpuParams>& params)
@@ -341,7 +361,13 @@ namespace BansheeEngine
 		queue->submit(this, mSemaphoresTemp, numSemaphores);
 
 		for (auto& entry : mResources)
-			entry.first->notifyUsed(this, entry.second.flags);
+		{
+			ResourceUseHandle& useHandle = entry.second;
+			assert(!useHandle.used);
+
+			useHandle.used = true;
+			entry.first->notifyUsed();
+		}
 
 		cbm.refreshStates(deviceIdx);
 

Source/BansheeVulkanRenderAPI/Source/BsVulkanResource.cpp

@@ -2,28 +2,41 @@
 //**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
 #include "BsVulkanResource.h"
 #include "BsVulkanCommandBuffer.h"
+#include "BsCoreThread.h"
 
 namespace BansheeEngine
 {
 	VulkanResource::VulkanResource(VulkanResourceManager* owner, bool concurrency, VulkanResourceType type)
-		: mOwner(owner), mQueueFamily(-1), mState(concurrency ? State::Shared : State::Normal), mType(type), mNumHandles(0)
-		, mHandleCapacity(INITIAL_HANDLE_CAPACITY)
 	{
-		UINT32 bytesCapacity = sizeof(UseHandle) * mHandleCapacity;
-		mHandles = (UseHandle*)mAlloc.alloc(sizeof(UseHandle) * INITIAL_HANDLE_CAPACITY);
-		memset(mHandles, 0, bytesCapacity);
+		Lock lock(mMutex);
+
+		mOwner = owner;
+		mQueueFamily = -1;
+		mState = concurrency ? State::Shared : State::Normal;
+		mType = type;
+		mNumUsedHandles = 0;
+		mNumBoundHandles = 0;
 	}
 
 	VulkanResource::~VulkanResource()
 	{
+		THROW_IF_NOT_CORE_THREAD
+
+		Lock lock(mMutex);
 		assert(mState == State::Destroyed && "Vulkan resource getting destructed without destroy() called first.");
+	}
+
+	void VulkanResource::notifyBound()
+	{
+		Lock lock(mMutex);
+		assert(mState != State::Destroyed);
 
-		mAlloc.free(mHandles);
-		mAlloc.clear();
+		mNumBoundHandles++;
 	}
 
-	void VulkanResource::notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags flags)
+	void VulkanResource::notifyUsed(VulkanCmdBuffer* buffer, VulkanUseFlags useFlags)
 	{
+		Lock lock(mMutex);
 		assert(mState != State::Destroyed);
 
 		if(isUsed() && mState == State::Normal) // Used without support for concurrency
@@ -32,69 +45,36 @@ namespace BansheeEngine
 				"Vulkan resource without concurrency support can only be used by one queue family at once.");
 		}
 
-		mNumHandles++;
-
-		if(mNumHandles > mHandleCapacity)
-		{
-			UINT32 bytesCapacity = sizeof(UseHandle) * mHandleCapacity;
-			void* tempData = bs_stack_alloc(bytesCapacity);
-			memcpy(tempData, mHandles, bytesCapacity);
-
-			mAlloc.free(mHandles);
-			mHandles = (UseHandle*)mAlloc.alloc(bytesCapacity * 2);
-			memcpy(mHandles, tempData, bytesCapacity);
-
-			bs_stack_free(tempData);
-
-			memset(mHandles + mHandleCapacity, 0, bytesCapacity);
-			mHandleCapacity *= 2;
-		}
-
-		for(UINT32 i = 0; i < mHandleCapacity; i++)
-		{
-			if (mHandles[i].buffer == nullptr)
-				continue;
-
-			mHandles[i].buffer = buffer;
-			mHandles[i].flags |= flags;
-
-			break;
-		}
-
+		mNumUsedHandles++;
 		mQueueFamily = buffer->getQueueFamily();
+		mUseFlags |= useFlags;
 	}
 
-	void VulkanResource::notifyDone(VulkanCmdBuffer* buffer)
+	void VulkanResource::notifyDone()
 	{
-		bool foundBuffer = false;
-		for(UINT32 i = 0; i < mNumHandles; i++)
-		{
-			if(mHandles[i].buffer == buffer)
-			{
-				mHandles[i].buffer = nullptr;
-				mHandles[i].flags = VulkanUseFlag::None;
-
-				foundBuffer = true;
-				break;
-			}
-		}
-
-		assert(foundBuffer);
+		Lock lock(mMutex);
+		mNumUsedHandles--;
+		mNumBoundHandles--;
 
-		mNumHandles--;
+		// Note: If resource is used on different command buffers with different use flags, we should clear individual flags
+		// depending on which command buffer finished. But this requires extra per-command buffer state tracking, so we
+		// instead just clear all flags at once when all command buffers finish.
+		if (!isUsed())
+			mUseFlags = VulkanUseFlag::None;
 
-		if (!isUsed() && mState == State::Destroyed) // Queued for destruction
+		if (!isBound() && mState == State::Destroyed) // Queued for destruction
 			mOwner->destroy(this);
 	}
 
 	void VulkanResource::destroy()
 	{
+		Lock lock(mMutex);
 		assert(mState != State::Destroyed && "Vulkan resource destroy() called more than once.");
 
 		mState = State::Destroyed;
 
-		// If not used, destroy right away, otherwise check when it is reported as finished on the device
-		if (!isUsed())
+		// If not bound anyhwere, destroy right away, otherwise check when it is reported as finished on the device
+		if (!isBound())
 			mOwner->destroy(this);
 	}
 
@@ -105,6 +85,7 @@ namespace BansheeEngine
 	VulkanResourceManager::~VulkanResourceManager()
 	{
 #if BS_DEBUG_MODE
+		Lock lock(mMutex);
 		assert(mResources.empty() && "Resource manager shutting down but not all resources were released.");
 #endif
 	}
@@ -112,9 +93,10 @@ namespace BansheeEngine
 	void VulkanResourceManager::destroy(VulkanResource* resource)
 	{
 #if BS_DEBUG_MODE
+		Lock lock(mMutex);
 		mResources.erase(resource);
 #endif
 
 		bs_delete(resource);
 	}
-}
+}