Added an automated way to managed Vulkan object lifetime and usage across multiple queues

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBuffer.h

@@ -5,6 +5,7 @@
 #include "BsVulkanPrerequisites.h"
 #include "BsCommandBuffer.h"
 #include "BsVulkanRenderAPI.h"
+#include "BsVulkanResource.h"
 
 namespace BansheeEngine
 {
@@ -37,6 +38,7 @@ namespace BansheeEngine
 		VkCommandPool mPools[VQT_COUNT];
 
 		VulkanCmdBuffer* mBuffers[VQT_COUNT][BS_MAX_QUEUES_PER_TYPE][BS_MAX_VULKAN_COMMAND_BUFFERS_PER_QUEUE];
+		UINT32 mNextId;
 	};
 
 	/** 
@@ -61,9 +63,12 @@ namespace BansheeEngine
 		};
 
 	public:
-		VulkanCmdBuffer(VulkanDevice& device, VkCommandPool pool, bool secondary);
+		VulkanCmdBuffer(VulkanDevice& device, UINT32 id, VkCommandPool pool, bool secondary);
 		~VulkanCmdBuffer();
 
+		/** Returns an unique identifier of this command buffer. */
+		UINT32 getId() const { return mId; }
+
 		/** Makes the command buffer ready to start recording commands. */
 		void begin();
 
@@ -100,10 +105,26 @@ namespace BansheeEngine
 		/** Checks the internal fence and changes command buffer state if done executing. */
 		void refreshFenceStatus();
 
+		/** 
+		 * Lets the command buffer know that the provided resource has been queued on it, and will be used by the
+		 * device when the command buffer is submitted.
+		 */
+		void registerResource(VulkanResource* res, VulkanUseFlags flags);
+
 	private:
 		friend class VulkanCmdBufferPool;
 		friend class VulkanCommandBuffer;
 
+		/** Information about a resource currently queued for use on the command buffer. */
+		struct ResourceInfo
+		{
+			VulkanUseFlags flags;
+		};
+
+		/** Called after the buffer has been submitted to the queue. */
+		void notifySubmit();
+
+		UINT32 mId;
 		State mState;
 		VulkanDevice& mDevice;
 		VkCommandPool mPool;
@@ -111,6 +132,8 @@ namespace BansheeEngine
 		VkFence mFence;
 		VkSemaphore mSemaphore;
 		UINT32 mFenceCounter;
+
+		UnorderedMap<VulkanResource*, ResourceInfo> mResources;
 	};
 
 	/** CommandBuffer implementation for Vulkan. */
@@ -149,4 +172,4 @@ namespace BansheeEngine
 	};
 
 	/** @} */
-}
+}

Source/BansheeVulkanRenderAPI/Include/BsVulkanDevice.h

@@ -52,6 +52,9 @@ namespace BansheeEngine
 		/** Returns a manager that can be used for allocating descriptor layouts and sets. */
 		VulkanDescriptorManager& getDescriptorManager() const { return *mDescriptorManager; }
 
+		/** Returns a manager that can be used for allocating Vulkan objects wrapped as managed resources. */
+		VulkanResourceManager& getResourceManager() const { return *mResourceManager; }
+
 		/** 
 		 * Allocates memory for the provided image, and binds it to the image. Returns null if it cannot find memory
 		 * with the specified flags.
@@ -82,6 +85,7 @@ namespace BansheeEngine
 
 		VulkanCmdBufferPool* mCommandBufferPool;
 		VulkanDescriptorManager* mDescriptorManager;
+		VulkanResourceManager* mResourceManager;
 
 		VkPhysicalDeviceProperties mDeviceProperties;
 		VkPhysicalDeviceFeatures mDeviceFeatures;

Source/BansheeVulkanRenderAPI/Include/BsVulkanPrerequisites.h

@@ -43,6 +43,7 @@ namespace BansheeEngine
 	class VulkanCmdBuffer;
 	class VulkanCommandBuffer;
 	class VulkanQueue;
+	class VulkanResourceManager;
 
 	VkAllocationCallbacks* gVulkanAllocator = nullptr;
 

Source/BansheeVulkanRenderAPI/Include/BsVulkanResource.h

@@ -1 +1,109 @@
+//********************************** Banshee Engine (www.banshee3d.com) **************************************************//
+//**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
 #pragma once
+
+#include "BsVulkanPrerequisites.h"
+
+namespace BansheeEngine
+{
+	/** @addtogroup Vulkan
+	 *  @{
+	 */
+
+	/** Flags that determine how is a resource being used by the GPU. */
+	enum class VulkanUseFlag
+	{
+		None = 0,
+		Read = 0x1,
+		Write = 0x2
+	};
+
+	class VulkanResourceManager;
+
+	typedef Flags<VulkanUseFlag> VulkanUseFlags;
+	BS_FLAGS_OPERATORS(VulkanUseFlag);
+
+	/** 
+	 * Wraps one or multiple native Vulkan objects. Allows the object usage to be tracked in command buffers, handles
+	 * ownership transitions between different queues, and handles delayed object destruction.
+	 */
+	class VulkanResource
+	{
+	public:
+		VulkanResource(VulkanResourceManager* owner);
+		virtual ~VulkanResource();
+
+		/** 
+		 * Notifies the resource that it is currently being used on the provided command buffer. This means the command
+		 * buffer has actually been submitted to the queue and the resource is used by the GPU.
+		 * 
+		 * A resource can only be used by a single command buffer at a time.
+		 */
+		virtual void notifyUsed(VulkanCmdBuffer& buffer, VulkanUseFlags flags);
+
+		/** 
+		 * Notifies the resource that it is no longer used by on the GPU. This makes the resource usable on other command
+		 * buffers again.
+		 */
+		virtual void notifyDone();
+
+		/** 
+		 * Checks is the resource currently used on a device. 
+		 *
+		 * @note	Resource usage is only checked at certain points of the program. This means the resource could be
+		 *			done on the device but this method may still report true. If you need to know the latest state
+		 *			call VulkanCommandBufferManager::refreshStates() before checking for usage.
+		 */
+		bool isUsed() const { return mCmdBufferId != -1; }
+
+		/** 
+		 * Destroys the resource and frees its memory. If the resource is currently being used on a device, the
+		 * destruction is delayed until the device is done with it.
+		 */
+		void destroy();
+
+	protected:
+		VulkanResourceManager* mOwner;
+		VulkanUseFlags mFlags;
+		UINT32 mCmdBufferId = -1;
+		bool mIsDestroyed = false;
+	};
+
+	/** Creates and destroys annd VulkanResource%s on a single device. */
+	class VulkanResourceManager
+	{
+	public:
+		~VulkanResourceManager();
+
+		/** 
+		 * Creates a new Vulkan resource of the specified type. User must call VulkanResource::destroy() when done using
+		 * the resource. 
+		 */
+		template<class Type, class... Args>
+		VulkanResource* create(Args &&...args)
+		{
+			VulkanResource* resource = new (bs_alloc(sizeof(Type))) Type(std::forward<Args>(args)...);
+
+#if BS_DEBUG_MODE
+			mResources.insert(resource);
+#endif
+
+			return resource;
+		}
+
+	private:
+		friend VulkanResource;
+
+		/** 
+		 * Destroys a previously created Vulkan resource. Caller must ensure the resource is not currently being used
+		 * on the device.
+		 */
+		void destroy(VulkanResource* resource);
+
+#if BS_DEBUG_MODE
+		UnorderedSet<VulkanResource*> mResources;
+#endif
+	};
+
+	/** @} */
+}

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBuffer.cpp

@@ -9,7 +9,7 @@
 namespace BansheeEngine
 {
 	VulkanCmdBufferPool::VulkanCmdBufferPool(VulkanDevice& device)
-		:mDevice(device), mPools{}, mBuffers {}
+		:mDevice(device), mPools{}, mBuffers {}, mNextId(1)
 	{
 		for (UINT32 i = 0; i < VQT_COUNT; i++)
 		{
@@ -87,7 +87,7 @@ namespace BansheeEngine
 	{
 		VkCommandPool pool = getPool(type);
 
-		return bs_new<VulkanCmdBuffer>(mDevice, pool, secondary);
+		return bs_new<VulkanCmdBuffer>(mDevice, mNextId++, pool, secondary);
 	}
 
 	VkCommandPool VulkanCmdBufferPool::getPool(VulkanQueueType type)
@@ -99,8 +99,8 @@ namespace BansheeEngine
 		return pool;
 	}
 
-	VulkanCmdBuffer::VulkanCmdBuffer(VulkanDevice& device, VkCommandPool pool, bool secondary)
-		:mState(State::Ready), mDevice(device), mPool(pool)
+	VulkanCmdBuffer::VulkanCmdBuffer(VulkanDevice& device, UINT32 id, VkCommandPool pool, bool secondary)
+		:mId(id), mState(State::Ready), mDevice(device), mPool(pool), mFenceCounter(0)
 	{
 		VkCommandBufferAllocateInfo cmdBufferAllocInfo;
 		cmdBufferAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
@@ -212,11 +212,28 @@ namespace BansheeEngine
 				assert(result == VK_SUCCESS);
 
 				mFenceCounter++;
+
+				for (auto& entry : mResources)
+					entry.first->notifyDone();
+
+				mResources.clear();
 			}
 		}
 		else
 			assert(!signaled); // We reset the fence along with mState so this shouldn't be possible
+	}
+
+	void VulkanCmdBuffer::registerResource(VulkanResource* res, VulkanUseFlags flags)
+	{
+		mResources[res].flags |= flags;
+	}
+
+	void VulkanCmdBuffer::notifySubmit()
+	{
+		// TODO - Issue pipeline barrier for resources transitioning to a new queue family
 
+		for (auto& entry : mResources)
+			entry.first->notifyUsed(*this, entry.second.flags);
 	}
 
 	VulkanCommandBuffer::VulkanCommandBuffer(VulkanDevice& device, UINT32 id, CommandBufferType type, UINT32 deviceIdx,
@@ -292,6 +309,7 @@ namespace BansheeEngine
 
 		cmdBufManager.refreshStates(mDeviceIdx);
 
+		mBuffer->notifySubmit();
 		mQueue->notifySubmit(*this, mBuffer->getFenceCounter());
 
 		// Note: Uncommented for debugging only, prevents any device concurrency issues.

Source/BansheeVulkanRenderAPI/Source/BsVulkanDevice.cpp

@@ -111,6 +111,7 @@ namespace BansheeEngine
 		// Create pools/managers
 		mCommandBufferPool = bs_new<VulkanCmdBufferPool>(*this);
 		mDescriptorManager = bs_new<VulkanDescriptorManager>(*this);
+		mResourceManager = bs_new<VulkanResourceManager>();
 	}
 
 	VulkanDevice::~VulkanDevice()
@@ -124,8 +125,10 @@ namespace BansheeEngine
 				bs_delete(mQueueInfos[i].queues[j]);
 		}
 
+		bs_delete(mResourceManager);
 		bs_delete(mDescriptorManager);
 		bs_delete(mCommandBufferPool);
+		
 		vkDestroyDevice(mLogicalDevice, gVulkanAllocator);
 	}
 

Source/BansheeVulkanRenderAPI/Source/BsVulkanHardwareBuffer.cpp

@@ -41,7 +41,7 @@ namespace BansheeEngine
 			mAllocations[i].memory = mAllocations[i].device->allocateMemory(reqs, flags);
 		}
 
-		mSizeInBytes = reqs.size;
+		mSizeInBytes = (UINT32)reqs.size;
 	}
 
 	VulkanHardwareBuffer::~VulkanHardwareBuffer()

Source/BansheeVulkanRenderAPI/Source/BsVulkanResource.cpp

@@ -0,0 +1,65 @@
+//********************************** Banshee Engine (www.banshee3d.com) **************************************************//
+//**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
+#include "BsVulkanResource.h"
+#include "BsVulkanCommandBuffer.h"
+
+namespace BansheeEngine
+{
+	VulkanResource::VulkanResource(VulkanResourceManager* owner)
+		:mOwner(owner)
+	{
+		
+	}
+
+	VulkanResource::~VulkanResource()
+	{
+		assert(mIsDestroyed && "Vulkan resource getting destructed without destroy() called first.");
+	}
+
+	void VulkanResource::notifyUsed(VulkanCmdBuffer& buffer, VulkanUseFlags flags)
+	{
+		assert(!isUsed() && !mIsDestroyed);
+
+		// Note: I could allow resource usage on multiple command buffers at once by keeping a list of separate usage
+		// flags and ID's, per buffer.
+
+		mCmdBufferId = buffer.getId();
+		mFlags |= flags;
+	}
+
+	void VulkanResource::notifyDone()
+	{
+		mCmdBufferId = -1;
+		mFlags = VulkanUseFlag::None;
+
+		if (mIsDestroyed) // Queued for destruction
+			mOwner->destroy(this);
+	}
+
+	void VulkanResource::destroy()
+	{
+		assert(!mIsDestroyed && "Vulkan resource destroy() called more than once.");
+
+		mIsDestroyed = true;
+
+		// If not used, destroy right away, otherwise check when it is reported as finished on the device
+		if (!isUsed())
+			mOwner->destroy(this);
+	}
+
+	VulkanResourceManager::~VulkanResourceManager()
+	{
+#if BS_DEBUG_MODE
+		assert(mResources.empty() && "Resource manager shutting down but not all resources were released.");
+#endif
+	}
+
+	void VulkanResourceManager::destroy(VulkanResource* resource)
+	{
+#if BS_DEBUG_MODE
+		mResources.erase(resource);
+#endif
+
+		bs_delete(resource);
+	}
+}