Refactoring Vulkan command buffers:

 - They can now choose on which queue to execute
 - More intuitive design to synchronize between command buffers on different queues
 - Sync primitives required for synchronization + counter to track when command buffer is done executing
 - Command buffers can now be re-used immediately by using an internal pool for low-level command buffers

Source/BansheeCore/CMakeSources.cmake

@@ -152,7 +152,7 @@ set(BS_BANSHEECORE_INC_RENDERAPI
 	"Include/BsGpuPipelineState.h"
 )
 
-set(BS_BANSHEECORE_INC_RENDERAPI_FACTORIES
+set(BS_BANSHEECORE_INC_RENDERAPI_MANAGERS
 	"Include/BsRenderWindowManager.h"
 	"Include/BsRenderStateManager.h"
 	"Include/BsQueryManager.h"
@@ -444,7 +444,7 @@ set(BS_BANSHEECORE_SRC_RENDERAPI
 	"Source/BsGpuPipelineState.cpp"
 )
 
-set(BS_BANSHEECORE_SRC_RENDERAPI_FACTORIES
+set(BS_BANSHEECORE_SRC_RENDERAPI_MANAGERS
 	"Source/BsGpuProgramManager.cpp"
 	"Source/BsHardwareBufferManager.cpp"
 	"Source/BsMeshManager.cpp"
@@ -452,6 +452,7 @@ set(BS_BANSHEECORE_SRC_RENDERAPI_FACTORIES
 	"Source/BsRenderStateManager.cpp"
 	"Source/BsRenderWindowManager.cpp"
 	"Source/BsRenderAPIManager.cpp"
+	"Source/BsCommandBufferManager.cpp"
 )
 
 set(BS_BANSHEECORE_SRC_NOFILTER
@@ -549,7 +550,7 @@ source_group("Source Files\\Localization" FILES ${BS_BANSHEECORE_SRC_LOCALIZATIO
 source_group("Source Files\\RTTI" FILES ${BS_BANSHEECORE_SRC_RTTI})
 source_group("Header Files\\Profiling" FILES ${BS_BANSHEECORE_INC_PROFILING})
 source_group("Header Files\\RenderAPI" FILES ${BS_BANSHEECORE_INC_RENDERAPI})
-source_group("Header Files\\RenderAPI\\Factories" FILES ${BS_BANSHEECORE_INC_RENDERAPI_FACTORIES})
+source_group("Header Files\\RenderAPI\\Managers" FILES ${BS_BANSHEECORE_INC_RENDERAPI_MANAGERS})
 source_group("Source Files\\CoreThread" FILES ${BS_BANSHEECORE_SRC_CORETHREAD})
 source_group("Header Files" FILES ${BS_BANSHEECORE_INC_NOFILTER})
 source_group("Header Files\\Material" FILES ${BS_BANSHEECORE_INC_MATERIAL})
@@ -569,7 +570,7 @@ source_group("Source Files\\Input" FILES ${BS_BANSHEECORE_SRC_INPUT})
 source_group("Header Files\\Localization" FILES ${BS_BANSHEECORE_INC_LOCALIZATION})
 source_group("Source Files\\Text" FILES ${BS_BANSHEECORE_SRC_TEXT})
 source_group("Source Files\\RenderAPI" FILES ${BS_BANSHEECORE_SRC_RENDERAPI})
-source_group("Source Files\\RenderAPI\\Factories" FILES ${BS_BANSHEECORE_SRC_RENDERAPI_FACTORIES})
+source_group("Source Files\\RenderAPI\\Managers" FILES ${BS_BANSHEECORE_SRC_RENDERAPI_MANAGERS})
 source_group("Source Files" FILES ${BS_BANSHEECORE_SRC_NOFILTER})
 source_group("Source Files\\Physics" FILES ${BS_BANSHEECORE_SRC_PHYSICS})
 source_group("Source Files\\Scene" FILES ${BS_BANSHEECORE_SRC_SCENE})
@@ -617,6 +618,6 @@ set(BS_BANSHEECORE_SRC
 	${BS_BANSHEECORE_SRC_AUDIO}
 	${BS_BANSHEECORE_INC_ANIMATION}
 	${BS_BANSHEECORE_SRC_ANIMATION}
-	${BS_BANSHEECORE_INC_RENDERAPI_FACTORIES}
-	${BS_BANSHEECORE_SRC_RENDERAPI_FACTORIES}
+	${BS_BANSHEECORE_INC_RENDERAPI_MANAGERS}
+	${BS_BANSHEECORE_SRC_RENDERAPI_MANAGERS}
 )

Source/BansheeCore/Include/BsCommandBuffer.h

@@ -10,6 +10,23 @@ namespace BansheeEngine
 	 *  @{
 	 */
 
+	/** Mask that determines synchronization between command buffers executing on different hardware queues. */
+	class BS_CORE_EXPORT CommandSyncMask
+	{
+	public:
+		/** 
+		 * Registers a dependency on a command buffer. Use getMask() to get the new mask value after registering all 
+		 * dependencies.
+		 */
+		void addDependency(const SPtr<CommandBuffer>& buffer);
+
+		/** Returns a combined mask that contains all the required dependencies. */
+		UINT32 getMask() const { return mMask; }
+
+	private:
+		UINT32 mMask = 0;
+	};
+
 	/** 
 	 * Contains a list of render API commands that can be queued for execution on the GPU. User is allowed to populate the
 	 * command buffer from any thread, ensuring render API command generation can be multi-threaded. Command buffers
@@ -19,7 +36,7 @@ namespace BansheeEngine
 	class BS_CORE_EXPORT CommandBuffer
 	{
 	public:
-		virtual ~CommandBuffer() {}
+		virtual ~CommandBuffer();
 
 		/**
 		 * Creates a new CommandBuffer.
@@ -27,23 +44,35 @@ namespace BansheeEngine
 		 * @param[in]	type		Determines what type of commands can be added to the command buffer.
 		 * @param[in]	deviceIdx	Index of the GPU the command buffer will be used to queue commands on. 0 is always
 		 *							the primary available GPU.
-		 * @param[in]	syncMask	Determines how is concurrency handled between multiple command buffers on the same 
-		 *							device. If buffers don't share the same bits in the sync mask they are allowed to
-		 *							execute concurrently on the GPU, however it is up to the user to ensure that they
-		 *							do not contain any resources depended on each other. 
+		 * @param[in]	queueIdx	Index of the hardware queue the command buffer will be used on. Command buffers with
+		 *							the same index will execute sequentially, but command buffers with different queue
+		 *							indices may execute in parallel, for a potential performance improvement. Queue indices
+		 *							are unique per buffer type (e.g. upload index 0 and graphics index 0 may map to 
+		 *							different queues internally). Must be in range [0, 7].
 		 * @param[in]	secondary	If true the command buffer will not be allowed to execute on its own, but it can
 		 *							be appended to a primary command buffer. 
 		 * @return					New CommandBuffer instance.
 		 * 
+		 * @note The parallelism provided by @p queueIdx is parallelism on the GPU itself, it has nothing to do with CPU
+		 *		 parallelism or threads.
 		 */
-		static SPtr<CommandBuffer> create(CommandBufferType type, UINT32 deviceIdx = 0, UINT32 syncMask = 0x00000001, 
+		static SPtr<CommandBuffer> create(CommandBufferType type, UINT32 deviceIdx = 0, UINT32 queueIdx = 0, 
 			bool secondary = false);
 
+		/** @name Internal
+		 *  @{
+		 */
+
+		/** Returns a unique ID of this command buffer. */
+		UINT32 _getId() const { return mId; }
+
+		/** @} */
 	protected:
-		CommandBuffer(CommandBufferType type, UINT32 syncMask, bool secondary);
+		CommandBuffer(UINT32 id, CommandBufferType type, UINT32 queueIdx, bool secondary);
 
+		UINT32 mId;
 		CommandBufferType mType;
-		UINT32 mSyncMask;
+		UINT32 mQueueIdx;
 		bool mIsSecondary;
 	};
 

Source/BansheeCore/Include/BsCommandBufferManager.h

@@ -22,8 +22,21 @@ namespace BansheeEngine
 		virtual ~CommandBufferManager() {}
 
 		/** @copydoc CommandBuffer::create */
-		virtual SPtr<CommandBuffer> create(CommandBufferType type, UINT32 deviceIdx = 0, UINT32 syncMask = 0xFFFFFFFF, 
-			bool secondary = false) = 0;
+		SPtr<CommandBuffer> create(CommandBufferType type, UINT32 deviceIdx = 0, UINT32 queueIdx = 0,
+			bool secondary = false);
+
+	protected:
+		friend CommandBuffer;
+
+		/** Creates a command buffer with the specified ID. See create(). */
+		virtual SPtr<CommandBuffer> createInternal(UINT32 id, CommandBufferType type, UINT32 deviceIdx = 0, 
+			UINT32 queueIdx = 0, bool secondary = false) = 0;
+
+		/** Called by a command buffer just before it is destroyed. */
+		void notifyCommandBufferDestroyed(UINT32 id);
+
+	private:
+		bool mActiveCommandBuffers[BS_MAX_COMMAND_BUFFERS];
 	};
 
 	/** @} */

Source/BansheeCore/Include/BsCorePrerequisites.h

@@ -170,6 +170,12 @@
 #define BS_MAX_MULTIPLE_RENDER_TARGETS 8
 #define BS_FORCE_SINGLETHREADED_RENDERING 0
 
+/** Maximum number of CommandBuffer%s that may exist at once. */
+#define BS_MAX_COMMAND_BUFFERS 32
+
+/** Maximum number of individual GPU queues, per type. */
+#define BS_MAX_QUEUES_PER_TYPE 8
+
 // Windows Settings
 #if BS_PLATFORM == BS_PLATFORM_WIN32
 

Source/BansheeCore/Include/BsRenderAPI.h

@@ -446,9 +446,19 @@ namespace BansheeEngine
 		/** 
 		 * Executes all commands in the provided command buffer. Command buffer cannot be secondary.
 		 *
+		 * @param[in]	commandBuffer	Command buffer whose commands to execute.
+		 * @param[in]	syncMask		Optional synchronization mask that determines if the submitted command buffer
+		 *								depends on any other command buffers. Use the CommandSyncMask class to generate
+		 *								a mask using existing command buffers.
+		 *								
+		 *								This mask is only relevant if your command buffers are executing on different
+		 *								hardware queues, and are somehow dependant. If they are executing on the same queue
+		 *								(default) then they will execute sequentially in the order they are submitted.
+		 *								Otherwise, if there is a dependency, you must make state it explicitly here.
+		 *
 		 * @note	Core thread only.
 		 */
-		virtual void executeCommands(const SPtr<CommandBuffer>& commandBuffer) = 0;
+		virtual void executeCommands(const SPtr<CommandBuffer>& commandBuffer, UINT32 syncMask = 0xFFFFFFFF) = 0;
 
 		/**
 		 * Gets the capabilities of a specific GPU.

Source/BansheeCore/Source/BsCommandBuffer.cpp

@@ -5,15 +5,28 @@
 
 namespace BansheeEngine
 {
-	CommandBuffer::CommandBuffer(CommandBufferType type, UINT32 syncMask, bool secondary)
-		:mType(type), mSyncMask(syncMask), mIsSecondary(secondary)
+	void CommandSyncMask::addDependency(const SPtr<CommandBuffer>& buffer)
 	{
+		if (buffer == nullptr)
+			return;
 
+		mMask |= 1 << buffer->_getId();
 	}
 
-	SPtr<CommandBuffer> CommandBuffer::create(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask,
+	CommandBuffer::CommandBuffer(UINT32 id, CommandBufferType type, UINT32 queueIdx, bool secondary)
+		:mId(id), mType(type), mQueueIdx(queueIdx), mIsSecondary(secondary)
+	{
+
+	}
+
+	CommandBuffer::~CommandBuffer()
+	{
+		CommandBufferManager::instance().notifyCommandBufferDestroyed(mId);
+	}
+
+	SPtr<CommandBuffer> CommandBuffer::create(CommandBufferType type, UINT32 deviceIdx, UINT32 queueIdx,
 		bool secondary)
 	{
-		return CommandBufferManager::instance().create(type, deviceIdx, syncMask, secondary);
+		return CommandBufferManager::instance().create(type, deviceIdx, queueIdx, secondary);
 	}
 }

Source/BansheeCore/Source/BsCommandBufferManager.cpp

@@ -0,0 +1,34 @@
+//********************************** Banshee Engine (www.banshee3d.com) **************************************************//
+//**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
+#include "BsCommandBufferManager.h"
+
+namespace BansheeEngine
+{
+	SPtr<CommandBuffer> CommandBufferManager::create(CommandBufferType type, UINT32 deviceIdx, UINT32 queueIdx,
+		bool secondary)
+	{
+		UINT32 id = -1;
+		for(UINT32 i = 0; i < BS_MAX_COMMAND_BUFFERS; i++)
+		{
+			if (!mActiveCommandBuffers[i])
+			{
+				id = i;
+				break;
+			}
+		}
+
+		if(id == -1)
+		{
+			LOGERR("Attempting to allocate more than 32 command buffers. This is not supported. ");
+			return nullptr;
+		}
+
+		mActiveCommandBuffers[id] = true;
+		return createInternal(id, type, deviceIdx, queueIdx, secondary);
+	}
+
+	void CommandBufferManager::notifyCommandBufferDestroyed(UINT32 id)
+	{
+		mActiveCommandBuffers[id] = false;
+	}
+}

Source/BansheeD3D11RenderAPI/Include/BsD3D11CommandBuffer.h

@@ -35,7 +35,7 @@ namespace BansheeEngine
 		friend class D3D11CommandBufferManager;
 		friend class D3D11RenderAPI;
 
-		D3D11CommandBuffer(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask, bool secondary);
+		D3D11CommandBuffer(UINT32 id, CommandBufferType type, UINT32 deviceIdx, UINT32 queueIdx, bool secondary);
 
 		UINT32 mDeviceIdx;
 		Vector<std::function<void()>> mCommands;

Source/BansheeD3D11RenderAPI/Include/BsD3D11CommandBufferManager.h

@@ -19,8 +19,8 @@ namespace BansheeEngine
 	class D3D11CommandBufferManager : public CommandBufferManager
 	{
 	public:
-		/** @copydoc CommandBufferManager::create() */
-		SPtr<CommandBuffer> create(CommandBufferType type, UINT32 deviceIdx = 0, UINT32 syncMask = 0xFFFFFFFF, 
+		/** @copydoc CommandBufferManager::createInternal() */
+		SPtr<CommandBuffer> createInternal(UINT32 id, CommandBufferType type, UINT32 deviceIdx = 0, UINT32 queueIdx = 0, 
 			bool secondary = false) override;
 	};
 

Source/BansheeD3D11RenderAPI/Include/BsD3D11RenderAPI.h

@@ -100,7 +100,7 @@ namespace BansheeEngine
 		void addCommands(const SPtr<CommandBuffer>& commandBuffer, const SPtr<CommandBuffer>& secondary) override;
 
 		/** @copydoc RenderAPICore::executeCommands() */
-		void executeCommands(const SPtr<CommandBuffer>& commandBuffer) override;
+		void executeCommands(const SPtr<CommandBuffer>& commandBuffer, UINT32 syncMask = 0xFFFFFFFF) override;
 
 		/** @copydoc RenderAPICore::convertProjectionMatrix */
 		void convertProjectionMatrix(const Matrix4& matrix, Matrix4& dest) override;

Source/BansheeD3D11RenderAPI/Source/BsD3D11CommandBuffer.cpp

@@ -4,8 +4,8 @@
 
 namespace BansheeEngine
 {
-	D3D11CommandBuffer::D3D11CommandBuffer(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask, bool secondary)
-		: CommandBuffer(type, syncMask, secondary), mDeviceIdx(deviceIdx), mActiveDrawOp(DOT_TRIANGLE_LIST)
+	D3D11CommandBuffer::D3D11CommandBuffer(UINT32 id, CommandBufferType type, UINT32 deviceIdx, UINT32 queueIdx, bool secondary)
+		: CommandBuffer(id, type, queueIdx, secondary), mDeviceIdx(deviceIdx), mActiveDrawOp(DOT_TRIANGLE_LIST)
 	{
 		if (deviceIdx != 0)
 			BS_EXCEPT(InvalidParametersException, "Only a single device supported on DX11.");

Source/BansheeD3D11RenderAPI/Source/BsD3D11CommandBufferManager.cpp

@@ -5,10 +5,10 @@
 
 namespace BansheeEngine
 {
-	SPtr<CommandBuffer> D3D11CommandBufferManager::create(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask,
-		bool secondary)
+	SPtr<CommandBuffer> D3D11CommandBufferManager::createInternal(UINT32 id, CommandBufferType type, UINT32 deviceIdx, 
+		UINT32 queueIdx, bool secondary)
 	{
-		CommandBuffer* buffer = new (bs_alloc<D3D11CommandBuffer>()) D3D11CommandBuffer(type, deviceIdx, syncMask, secondary);
+		CommandBuffer* buffer = new (bs_alloc<D3D11CommandBuffer>()) D3D11CommandBuffer(id, type, deviceIdx, queueIdx, secondary);
 		return bs_shared_ptr(buffer);
 	}
 }

Source/BansheeD3D11RenderAPI/Source/BsD3D11RenderAPI.cpp

@@ -1095,7 +1095,7 @@ namespace BansheeEngine
 		cb->appendSecondary(secondaryCb);
 	}
 
-	void D3D11RenderAPI::executeCommands(const SPtr<CommandBuffer>& commandBuffer)
+	void D3D11RenderAPI::executeCommands(const SPtr<CommandBuffer>& commandBuffer, UINT32 syncMask)
 	{
 		SPtr<D3D11CommandBuffer> cb = std::static_pointer_cast<D3D11CommandBuffer>(commandBuffer);
 		cb->executeCommands();

Source/BansheeGLRenderAPI/Include/BsGLCommandBuffer.h

@@ -4,7 +4,6 @@
 
 #include "BsGLPrerequisites.h"
 #include "BsCommandBuffer.h"
-#include "BsDrawOps.h"
 #include "BsGLRenderAPI.h"
 
 namespace BansheeEngine
@@ -36,7 +35,7 @@ namespace BansheeEngine
 		friend class GLCommandBufferManager;
 		friend class GLRenderAPI;
 
-		GLCommandBuffer(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask, bool secondary);
+		GLCommandBuffer(UINT32 id, CommandBufferType type, UINT32 deviceIdx, UINT32 queueIdx, bool secondary);
 
 		UINT32 mDeviceIdx;
 		Vector<std::function<void()>> mCommands;

Source/BansheeGLRenderAPI/Include/BsGLCommandBufferManager.h

@@ -19,8 +19,8 @@ namespace BansheeEngine
 	class GLCommandBufferManager : public CommandBufferManager
 	{
 	public:
-		/** @copydoc CommandBufferManager::create() */
-		SPtr<CommandBuffer> create(CommandBufferType type, UINT32 deviceIdx = 0, UINT32 syncMask = 0xFFFFFFFF, 
+		/** @copydoc CommandBufferManager::createInternal() */
+		SPtr<CommandBuffer> createInternal(UINT32 id, CommandBufferType type, UINT32 deviceIdx = 0, UINT32 queueIdx = 0, 
 			bool secondary = false) override;
 	};
 

Source/BansheeGLRenderAPI/Include/BsGLRenderAPI.h

@@ -102,7 +102,7 @@ namespace BansheeEngine
 		void addCommands(const SPtr<CommandBuffer>& commandBuffer, const SPtr<CommandBuffer>& secondary) override;
 
 		/** @copydoc RenderAPICore::executeCommands() */
-		void executeCommands(const SPtr<CommandBuffer>& commandBuffer) override;
+		void executeCommands(const SPtr<CommandBuffer>& commandBuffer, UINT32 syncMask = 0xFFFFFFFF) override;
 
 		/** @copydoc RenderAPICore::convertProjectionMatrix() */
 		void convertProjectionMatrix(const Matrix4& matrix, Matrix4& dest) override;

Source/BansheeGLRenderAPI/Source/BsGLCommandBuffer.cpp

@@ -4,8 +4,8 @@
 
 namespace BansheeEngine
 {
-	GLCommandBuffer::GLCommandBuffer(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask, bool secondary)
-		: CommandBuffer(type, syncMask, secondary), mDeviceIdx(deviceIdx), mCurrentDrawOperation(DOT_TRIANGLE_LIST)
+	GLCommandBuffer::GLCommandBuffer(UINT32 id, CommandBufferType type, UINT32 deviceIdx, UINT32 queueIdx, bool secondary)
+		: CommandBuffer(id, type, queueIdx, secondary), mDeviceIdx(deviceIdx), mCurrentDrawOperation(DOT_TRIANGLE_LIST)
 	{
 		if (deviceIdx != 0)
 			BS_EXCEPT(InvalidParametersException, "Only a single device supported on DX11.");

Source/BansheeGLRenderAPI/Source/BsGLCommandBufferManager.cpp

@@ -5,10 +5,10 @@
 
 namespace BansheeEngine
 {
-	SPtr<CommandBuffer> GLCommandBufferManager::create(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask,
-		bool secondary)
+	SPtr<CommandBuffer> GLCommandBufferManager::createInternal(UINT32 id, CommandBufferType type, UINT32 deviceIdx, 
+		UINT32 queueIdx, bool secondary)
 	{
-		CommandBuffer* buffer = new (bs_alloc<GLCommandBuffer>()) GLCommandBuffer(type, deviceIdx, syncMask, secondary);
+		CommandBuffer* buffer = new (bs_alloc<GLCommandBuffer>()) GLCommandBuffer(id, type, deviceIdx, queueIdx, secondary);
 		return bs_shared_ptr(buffer);
 	}
 }

Source/BansheeGLRenderAPI/Source/BsGLRenderAPI.cpp

@@ -1160,7 +1160,7 @@ namespace BansheeEngine
 		cb->appendSecondary(secondaryCb);
 	}
 
-	void GLRenderAPI::executeCommands(const SPtr<CommandBuffer>& commandBuffer)
+	void GLRenderAPI::executeCommands(const SPtr<CommandBuffer>& commandBuffer, UINT32 syncMask)
 	{
 		SPtr<GLCommandBuffer> cb = std::static_pointer_cast<GLCommandBuffer>(commandBuffer);
 		cb->executeCommands();

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBuffer.h

@@ -12,23 +12,124 @@ namespace BansheeEngine
 	 *  @{
 	 */
 
+	class LVulkanCommandBuffer;
+
+#define BS_MAX_VULKAN_COMMAND_BUFFERS_PER_QUEUE 32
+
+	/** Pool that allocates and distributes Vulkan command buffers. */
+	class CommandBufferPool
+	{
+	public:
+		CommandBufferPool(VulkanDevice& device);
+		~CommandBufferPool();
+
+		/** Attempts to find a free command buffer, or creates a new one if not found. */
+		LVulkanCommandBuffer* getBuffer(CommandBufferType type, UINT32 queueIdx, bool secondary);
+
+	private:
+		/** Creates a new command buffer. */
+		LVulkanCommandBuffer* createBuffer(VulkanQueueType type, bool secondary);
+
+		/** Returns a Vulkan command pool for the specified queue type. */
+		VkCommandPool getPool(VulkanQueueType type);
+
+		VulkanDevice& mDevice;
+		VkCommandPool mPools[VQT_COUNT];
+
+		LVulkanCommandBuffer* mBuffers[VQT_COUNT][BS_MAX_QUEUES_PER_TYPE][BS_MAX_VULKAN_COMMAND_BUFFERS_PER_QUEUE];
+	};
+
+	/** 
+	 * Represents a direct wrapper over an internal Vulkan command buffer. This is unlike VulkanCommandBuffer which is a
+	 * higher level class, and it allows for re-use by internally using multiple low-level command buffers.
+	 */
+	class LVulkanCommandBuffer
+	{
+		/** Possible states a command buffer can be in. */
+		enum class State
+		{
+			/** Buffer is ready to be re-used. */
+			Ready,
+			/** Buffer is currently recording commands, but isn't recording a render pass. */
+			Recording,
+			/** Buffer is currently recording render pass commands. */
+			RecordingRenderPass,
+			/** Buffer is done recording but hasn't been submitted. */
+			RecordingDone,
+			/** Buffer is done recording and is currently submitted on a queue. */
+			Submitted
+		};
+
+	public:
+		LVulkanCommandBuffer(VulkanDevice& device, VkCommandPool pool, bool secondary);
+		~LVulkanCommandBuffer();
+
+		/** Makes the command buffer ready to start recording commands. */
+		void begin();
+
+		/** Ends command buffer command recording (as started with begin()). */
+		void end();
+
+		/** Begins render pass recording. Must be called within begin()/end() calls. */
+		void beginRenderPass();
+
+		/** Ends render pass recording (as started with beginRenderPass(). */
+		void endRenderPass();
+
+		/** Returns the handle to the internal Vulkan command buffer wrapped by this object. */
+		VkCommandBuffer getHandle() const { return mCmdBuffer; }
+
+		/** Returns a fence that can be used for tracking when the command buffer is done executing. */
+		VkFence getFence() const { return mFence; }
+
+		/** 
+		 * Returns a semaphore that may be used for synchronizing execution between command buffers executing on different 
+		 * queues. 
+		 */
+		VkSemaphore getSemaphore() const { return mSemaphore; }
+
+		/** Returns a counter that gets incremented whenever the command buffer is done executing. */
+		UINT32 getFenceCounter() const { return mFenceCounter; }
+
+		/** Checks the internal fence and changes command buffer state if done executing. */
+		void refreshFenceStatus();
+
+	private:
+		friend class CommandBufferPool;
+
+		State mState;
+		VulkanDevice& mDevice;
+		VkCommandPool mPool;
+		VkCommandBuffer mCmdBuffer;
+		VkFence mFence;
+		VkSemaphore mSemaphore;
+		UINT32 mFenceCounter;
+	};
+
 	/** CommandBuffer implementation for Vulkan. */
 	class VulkanCommandBuffer : public CommandBuffer
 	{
 	public:
-		~VulkanCommandBuffer();
+		/** 
+		 * Returns the handle to the internal command buffer. This is a lower-level command buffer that more directly
+		 * maps to Vulkan's command buffers.
+		 */
+		LVulkanCommandBuffer& getBuffer() const { return *mBuffer; }
 
-		/** Returns the internal Vulkan command buffer wrapped by this object. */
-		VkCommandBuffer getInternal() const { return mBuffer; }
+		/** 
+		 * Tasks the command buffer to find a new internal command buffer. Call this after the command buffer has been
+		 * submitted to a queue (it's not allowed to be used until the queue is done with it).
+		 */
+		void acquireNewBuffer();
 
 	private:
 		friend class VulkanCommandBufferManager;
 
-		VulkanCommandBuffer(const SPtr<VulkanDevice>& device, CommandBufferType type, UINT32 syncMask, bool secondary);
+		VulkanCommandBuffer(VulkanDevice& device, UINT32 id, CommandBufferType type, UINT32 queueIdx, 
+			bool secondary);
 
-		VkCommandBuffer mBuffer;
-		VkCommandPool mPool;
-		VkDevice mDevice;
+		LVulkanCommandBuffer* mBuffer;
+		VulkanDevice& mDevice;
 	};
 
 	/** @} */

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBufferManager.h

@@ -22,8 +22,8 @@ namespace BansheeEngine
 		VulkanCommandBufferManager(const VulkanRenderAPI& rapi);
 		~VulkanCommandBufferManager();
 
-		/** @copydoc CommandBufferManager::create() */
-		SPtr<CommandBuffer> create(CommandBufferType type, UINT32 deviceIdx = 0, UINT32 syncMask = 0xFFFFFFFF, 
+		/** @copydoc CommandBufferManager::createInternal() */
+		SPtr<CommandBuffer> createInternal(UINT32 id, CommandBufferType type, UINT32 deviceIdx = 0, UINT32 queueIdx = 0,
 			bool secondary = false) override;
 
 	private:

Source/BansheeVulkanRenderAPI/Include/BsVulkanDevice.h

@@ -11,8 +11,6 @@ namespace BansheeEngine
 	 *  @{
 	 */
 
-#define BS_MAX_VULKAN_QUEUES_PER_TYPE 4
-
 	/** Represents a single GPU device usable by Vulkan. */
 	class VulkanDevice
 	{
@@ -47,11 +45,8 @@ namespace BansheeEngine
 		 */
 		UINT32 getQueueFamily(VulkanQueueType type) const { return mQueueInfos[(int)type].familyIdx; }
 
-		/** 
-		 * Returns a command pool that may be used for allocating command buffers for this queue family. Only the graphics 
-		 * command pool is guaranteed to exist, others may return null.
-		 */
-		VkCommandPool getCommandPool(VulkanQueueType type) const { return mQueueInfos[(int)type].commandPool; }
+		/** Returns a pool that can be used for allocating command buffers for all queues on this device. */
+		CommandBufferPool& getCmdBufferPool() const { return *mCommandBufferPool; }
 
 		/** 
 		 * Allocates memory for the provided image, and binds it to the image. Returns null if it cannot find memory
@@ -81,6 +76,8 @@ namespace BansheeEngine
 		VkPhysicalDevice mPhysicalDevice;
 		VkDevice mLogicalDevice;
 
+		CommandBufferPool* mCommandBufferPool;
+
 		VkPhysicalDeviceProperties mDeviceProperties;
 		VkPhysicalDeviceFeatures mDeviceFeatures;
 		VkPhysicalDeviceMemoryProperties mMemoryProperties;
@@ -90,10 +87,10 @@ namespace BansheeEngine
 		{
 			UINT32 familyIdx;
 			Vector<VkQueue> queues;
-			VkCommandPool commandPool;
 		};
 
 		QueueInfo mQueueInfos[VQT_COUNT];
+
 	};
 
 	/** @} */

Source/BansheeVulkanRenderAPI/Include/BsVulkanPrerequisites.h

@@ -37,6 +37,7 @@ namespace BansheeEngine
 	class VulkanDevice;
 	class VulkanGLSLProgramFactory;
 	class VulkanSwapChain;
+	class CommandBufferPool;
 
 	VkAllocationCallbacks* gVulkanAllocator = nullptr;
 

Source/BansheeVulkanRenderAPI/Include/BsVulkanRenderAPI.h

@@ -100,7 +100,7 @@ namespace BansheeEngine
 		void addCommands(const SPtr<CommandBuffer>& commandBuffer, const SPtr<CommandBuffer>& secondary) override;
 
 		/** @copydoc RenderAPICore::executeCommands() */
-		void executeCommands(const SPtr<CommandBuffer>& commandBuffer) override;
+		void executeCommands(const SPtr<CommandBuffer>& commandBuffer, UINT32 syncMask = 0xFFFFFFFF) override;
 
 		/** @copydoc RenderAPICore::convertProjectionMatrix */
 		void convertProjectionMatrix(const Matrix4& matrix, Matrix4& dest) override;

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBuffer.cpp

@@ -6,29 +6,228 @@
 
 namespace BansheeEngine
 {
-	VulkanCommandBuffer::VulkanCommandBuffer(const SPtr<VulkanDevice>& device, CommandBufferType type, UINT32 syncMask, 
-		bool secondary)
-		: CommandBuffer(type, syncMask, secondary)
+	CommandBufferPool::CommandBufferPool(VulkanDevice& device)
+		:mDevice(device), mPools{}, mBuffers {}
 	{
+		for (UINT32 i = 0; i < VQT_COUNT; i++)
+		{
+			UINT32 familyIdx = device.getQueueFamily((VulkanQueueType)i);
+
+			if (familyIdx == (UINT32)-1)
+				continue;
+
+			VkCommandPoolCreateInfo poolInfo;
+			poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
+			poolInfo.pNext = nullptr;
+			poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
+			poolInfo.queueFamilyIndex = familyIdx;
+
+			vkCreateCommandPool(device.getLogical(), &poolInfo, gVulkanAllocator, &mPools[i]);
+		}
+	}
+
+	CommandBufferPool::~CommandBufferPool()
+	{
+		for (UINT32 i = 0; i < VQT_COUNT; i++)
+		{
+			for(UINT32 j = 0; j < BS_MAX_QUEUES_PER_TYPE; j++)
+			{
+				LVulkanCommandBuffer** buffers = mBuffers[i][j];
+				for(UINT32 k = 0; k < BS_MAX_VULKAN_COMMAND_BUFFERS_PER_QUEUE; k++)
+				{
+					if (buffers[k] == nullptr)
+						break;
+
+					bs_delete(buffers[k]);
+				}
+			}
+		}
+
+		for (UINT32 i = 0; i < VQT_COUNT; i++)
+		{
+			if (mPools[i] == VK_NULL_HANDLE)
+				continue;
+
+			vkDestroyCommandPool(mDevice.getLogical(), mPools[i], gVulkanAllocator);
+		}
+	}
+
+	LVulkanCommandBuffer* CommandBufferPool::getBuffer(CommandBufferType type, UINT32 queueIdx, bool secondary)
+	{
+		assert(queueIdx < BS_MAX_QUEUES_PER_TYPE);
+
 		VulkanQueueType queueType = VulkanUtility::getQueueType(type);
-		mDevice = device->getLogical();
-		mPool = device->getCommandPool(queueType);
+		LVulkanCommandBuffer** buffers = mBuffers[queueType][queueIdx];
+
+		UINT32 i = 0;
+		for(; i < BS_MAX_VULKAN_COMMAND_BUFFERS_PER_QUEUE; i++)
+		{
+			if (buffers[i] == nullptr)
+				break;
+
+			if(buffers[i]->mState == LVulkanCommandBuffer::State::Ready)
+			{
+				buffers[i]->begin();
+				return buffers[i];
+			}
+		}
 
-		if(mPool == VK_NULL_HANDLE)
-			mPool = device->getCommandPool(VQT_GRAPHICS); // Graphics queue is guaranteed to exist
+		assert(i < BS_MAX_VULKAN_COMMAND_BUFFERS_PER_QUEUE && 
+			"Too many command buffers allocated. Increment BS_MAX_VULKAN_COMMAND_BUFFERS_PER_QUEUE to a higher value. ");
 
-		VkCommandBufferAllocateInfo bufferInfo = {};
-		bufferInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
-		bufferInfo.commandPool = mPool;
-		bufferInfo.level = secondary ? VK_COMMAND_BUFFER_LEVEL_SECONDARY : VK_COMMAND_BUFFER_LEVEL_PRIMARY;
-		bufferInfo.commandBufferCount = 1;
+		buffers[i] = createBuffer(queueType, secondary);
+		buffers[i]->begin();
+
+		return buffers[i];
+	}
+
+	LVulkanCommandBuffer* CommandBufferPool::createBuffer(VulkanQueueType type, bool secondary)
+	{
+		VkCommandPool pool = getPool(type);
 
-		VkResult result = vkAllocateCommandBuffers(mDevice, &bufferInfo, &mBuffer);
+		return bs_new<LVulkanCommandBuffer>(mDevice, pool, secondary);
+	}
+
+	VkCommandPool CommandBufferPool::getPool(VulkanQueueType type)
+	{
+		VkCommandPool pool = mPools[type];
+		if (pool == VK_NULL_HANDLE)
+			pool = mPools[VQT_GRAPHICS]; // Graphics queue is guaranteed to exist
+
+		return pool;
+	}
+
+	LVulkanCommandBuffer::LVulkanCommandBuffer(VulkanDevice& device, VkCommandPool pool, bool secondary)
+		:mState(State::Ready), mDevice(device), mPool(pool)
+	{
+		VkCommandBufferAllocateInfo cmdBufferAllocInfo;
+		cmdBufferAllocInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_ALLOCATE_INFO;
+		cmdBufferAllocInfo.pNext = nullptr;
+		cmdBufferAllocInfo.commandPool = pool;
+		cmdBufferAllocInfo.level = secondary ? VK_COMMAND_BUFFER_LEVEL_SECONDARY : VK_COMMAND_BUFFER_LEVEL_PRIMARY;
+		cmdBufferAllocInfo.commandBufferCount = 1;
+
+		VkResult result = vkAllocateCommandBuffers(mDevice.getLogical(), &cmdBufferAllocInfo, &mCmdBuffer);
+		assert(result == VK_SUCCESS);
+
+		VkFenceCreateInfo fenceCI;
+		fenceCI.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+		fenceCI.pNext = nullptr;
+		fenceCI.flags = 0;
+
+		result = vkCreateFence(mDevice.getLogical(), &fenceCI, gVulkanAllocator, &mFence);
+		assert(result == VK_SUCCESS);
+
+		VkSemaphoreCreateInfo semaphoreCI;
+		semaphoreCI.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
+		semaphoreCI.pNext = nullptr;
+		semaphoreCI.flags = 0;
+
+		result = vkCreateSemaphore(mDevice.getLogical(), &semaphoreCI, gVulkanAllocator, &mSemaphore);
 		assert(result == VK_SUCCESS);
 	}
 
-	VulkanCommandBuffer::~VulkanCommandBuffer()
+	LVulkanCommandBuffer::~LVulkanCommandBuffer()
 	{
-		vkFreeCommandBuffers(mDevice, mPool, 1, &mBuffer);
+		VkDevice device = mDevice.getLogical();
+
+		if(mState == State::Submitted)
+		{
+			// Wait 1s
+			UINT64 waitTime = 1000 * 1000 * 1000;
+			VkResult result = vkWaitForFences(device, 1, &mFence, true, waitTime);
+			assert(result == VK_SUCCESS || result == VK_TIMEOUT);
+
+			if (result == VK_TIMEOUT)
+				LOGWRN("Freeing a command buffer before done executing because fence wait expired!");
+		}
+		
+		vkDestroyFence(device, mFence, gVulkanAllocator);
+		vkDestroySemaphore(device, mSemaphore, gVulkanAllocator);
+		vkFreeCommandBuffers(device, mPool, 1, &mCmdBuffer);
+	}
+
+	void LVulkanCommandBuffer::begin()
+	{
+		assert(mState == State::Ready);
+
+		VkCommandBufferBeginInfo beginInfo;
+		beginInfo.sType = VK_STRUCTURE_TYPE_COMMAND_BUFFER_BEGIN_INFO;
+		beginInfo.pNext = nullptr;
+		beginInfo.flags = VK_COMMAND_BUFFER_USAGE_ONE_TIME_SUBMIT_BIT;
+		beginInfo.pInheritanceInfo = nullptr;
+
+		VkResult result = vkBeginCommandBuffer(mCmdBuffer, &beginInfo);
+		assert(result == VK_SUCCESS);
+
+		mState = State::Recording;
+	}
+
+	void LVulkanCommandBuffer::end()
+	{
+		assert(mState == State::Recording);
+
+		VkResult result = vkEndCommandBuffer(mCmdBuffer);
+		assert(result == VK_SUCCESS);
+
+		mState = State::RecordingDone;
+	}
+
+	void LVulkanCommandBuffer::beginRenderPass()
+	{
+		assert(mState == State::Recording);
+
+		// TODO
+		BS_EXCEPT(NotImplementedException, "Not implemented");
+
+		mState = State::RecordingRenderPass;
+	}
+
+	void LVulkanCommandBuffer::endRenderPass()
+	{
+		assert(mState == State::RecordingRenderPass);
+
+		vkCmdEndRenderPass(mCmdBuffer);
+
+		mState = State::Recording;
+	}
+
+	void LVulkanCommandBuffer::refreshFenceStatus()
+	{
+		VkResult result = vkGetFenceStatus(mDevice.getLogical(), mFence);
+		assert(result == VK_SUCCESS || result == VK_NOT_READY);
+
+		bool signaled = result == VK_SUCCESS;
+
+		if (mState == State::Submitted)
+		{
+			if(signaled)
+			{
+				mState = State::Ready;
+				vkResetCommandBuffer(mCmdBuffer, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); // Note: Maybe better not to release resources?
+
+				result = vkResetFences(mDevice.getLogical(), 1, &mFence);
+				assert(result == VK_SUCCESS);
+
+				mFenceCounter++;
+			}
+		}
+		else
+			assert(!signaled); // We reset the fence along with mState so this shouldn't be possible
+
+	}
+
+	VulkanCommandBuffer::VulkanCommandBuffer(VulkanDevice& device, UINT32 id, CommandBufferType type, 
+		UINT32 queueIdx, bool secondary)
+		: CommandBuffer(id, type, queueIdx, secondary), mBuffer(nullptr), mDevice(device)
+	{
+		acquireNewBuffer();
+	}
+
+	void VulkanCommandBuffer::acquireNewBuffer()
+	{
+		CommandBufferPool& pool = mDevice.getCmdBufferPool();
+
+		mBuffer = pool.getBuffer(mType, mQueueIdx, mIsSecondary);
 	}
 }

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBufferManager.cpp

@@ -15,8 +15,8 @@ namespace BansheeEngine
 		
 	}
 
-	SPtr<CommandBuffer> VulkanCommandBufferManager::create(CommandBufferType type, UINT32 deviceIdx, UINT32 syncMask,
-		bool secondary)
+	SPtr<CommandBuffer> VulkanCommandBufferManager::createInternal(UINT32 id, CommandBufferType type, UINT32 deviceIdx, 
+		UINT32 queueIdx, bool secondary)
 	{
 		UINT32 numDevices = mRapi._getNumDevices();
 		if(deviceIdx >= numDevices)
@@ -30,7 +30,7 @@ namespace BansheeEngine
 		SPtr<VulkanDevice> device = mRapi._getDevice(deviceIdx);
 
 		CommandBuffer* buffer = 
-			new (bs_alloc<VulkanCommandBuffer>()) VulkanCommandBuffer(device, type, syncMask, secondary);
+			new (bs_alloc<VulkanCommandBuffer>()) VulkanCommandBuffer(*device, id, type, queueIdx, secondary);
 
 		return bs_shared_ptr(buffer);
 	}

Source/BansheeVulkanRenderAPI/Source/BsVulkanDevice.cpp

@@ -1,6 +1,7 @@
 //********************************** Banshee Engine (www.banshee3d.com) **************************************************//
 //**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
 #include "BsVulkanDevice.h"
+#include "BsVulkanCommandBuffer.h"
 
 namespace BansheeEngine
 {
@@ -22,7 +23,7 @@ namespace BansheeEngine
 		vkGetPhysicalDeviceQueueFamilyProperties(device, &numQueueFamilies, queueFamilyProperties.data());
 
 		// Create queues
-		const float defaultQueuePriorities[BS_MAX_VULKAN_QUEUES_PER_TYPE] = { 0.0f };
+		const float defaultQueuePriorities[BS_MAX_QUEUES_PER_TYPE] = { 0.0f };
 		Vector<VkDeviceQueueCreateInfo> queueCreateInfos;
 
 		auto populateQueueInfo = [&](VulkanQueueType type, uint32_t familyIdx)
@@ -34,7 +35,7 @@ namespace BansheeEngine
 			createInfo.pNext = nullptr;
 			createInfo.flags = 0;
 			createInfo.queueFamilyIndex = familyIdx;
-			createInfo.queueCount = std::min(queueFamilyProperties[familyIdx].queueCount, (uint32_t)BS_MAX_VULKAN_QUEUES_PER_TYPE);
+			createInfo.queueCount = std::min(queueFamilyProperties[familyIdx].queueCount, (uint32_t)BS_MAX_QUEUES_PER_TYPE);
 			createInfo.pQueuePriorities = defaultQueuePriorities;
 
 			mQueueInfos[type].familyIdx = familyIdx;
@@ -101,33 +102,14 @@ namespace BansheeEngine
 		}
 
 		// Create command buffer pools
-		for (UINT32 i = 0; i < VQT_COUNT; i++)
-		{
-			if (mQueueInfos[i].familyIdx == (UINT32)-1)
-				continue;
-
-			VkCommandPoolCreateInfo poolInfo;
-			poolInfo.sType = VK_STRUCTURE_TYPE_COMMAND_POOL_CREATE_INFO;
-			poolInfo.pNext = nullptr;
-			poolInfo.flags = VK_COMMAND_POOL_CREATE_RESET_COMMAND_BUFFER_BIT;
-			poolInfo.queueFamilyIndex = mQueueInfos[i].familyIdx;
-
-			vkCreateCommandPool(mLogicalDevice, &poolInfo, gVulkanAllocator, &mQueueInfos[i].commandPool);
-		}
+		mCommandBufferPool = bs_new<CommandBufferPool>(*this);
 	}
 
 	VulkanDevice::~VulkanDevice()
 	{
 		vkDeviceWaitIdle(mLogicalDevice);
 
-		for (UINT32 i = 0; i < VQT_COUNT; i++)
-		{
-			if (mQueueInfos[i].commandPool == VK_NULL_HANDLE)
-				continue;
-
-			vkDestroyCommandPool(mLogicalDevice, mQueueInfos[i].commandPool, gVulkanAllocator);
-		}
-
+		bs_delete(mCommandBufferPool);
 		vkDestroyDevice(mLogicalDevice, gVulkanAllocator);
 	}
 

Source/BansheeVulkanRenderAPI/Source/BsVulkanGpuParams.cpp

@@ -122,7 +122,11 @@ namespace BansheeEngine
 
 	VulkanGpuParams::~VulkanGpuParams()
 	{
+		// TODO - Need to wait to ensure it isn't used on the GPU anymore
+
 		// TODO - CLean up mSets
+		// - Queue for destroy, remember fence counters for all available queues, only destroy after all queues execute?
+		// - Or ensure the object knows which queue it was used on?
 	}
 
 	void VulkanGpuParams::setParamBlockBuffer(UINT32 set, UINT32 slot, const ParamsBufferType& paramBlockBuffer)

Source/BansheeVulkanRenderAPI/Source/BsVulkanRenderAPI.cpp

@@ -399,7 +399,7 @@ namespace BansheeEngine
 
 	}
 
-	void VulkanRenderAPI::executeCommands(const SPtr<CommandBuffer>& commandBuffer)
+	void VulkanRenderAPI::executeCommands(const SPtr<CommandBuffer>& commandBuffer, UINT32 syncMask)
 	{
 
 	}