Vulkan: Refactoring inter-queue dependencies due to new revelation that a semaphore can only be waited on by one queue submit, no more

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBuffer.h

@@ -17,6 +17,9 @@ namespace bs
 
 #define BS_MAX_VULKAN_CB_PER_QUEUE_FAMILY BS_MAX_QUEUES_PER_TYPE * 32
 
+// Maximum number of command buffers that another command buffer can be dependant on (via a sync mask)
+#define BS_MAX_VULKAN_CB_DEPENDENCIES 2
+
 	/** Wrapper around a Vulkan semaphore object that manages its usage and lifetime. */
 	class VulkanSemaphore : public VulkanResource
 	{
@@ -139,17 +142,25 @@ namespace bs
 		VkFence getFence() const { return mFence; }
 
 		/** 
-		 * Returns a semaphore that may be used for synchronizing execution between command buffers executing on different 
-		 * queues. 
+		 * Returns a semaphore that may be used for synchronizing execution between command buffers executing on the same
+		 * queue.
 		 */
-		VulkanSemaphore* getSemaphore() const { return mSemaphore; }
+		VulkanSemaphore* getIntraQueueSemaphore() const { return mIntraQueueSemaphore; }
 
 		/** 
-		 * Allocates a new semaphore that may be used for synchronizing execution between command buffers executing on different 
-		 * queues. Releases the previously allocated semaphore, if one exist. Use getSemaphore() to retrieve latest
-		 * allocated semaphore.
+		 * Returns a semaphore that may be used for synchronizing execution between command buffers executing on different
+		 * queues. Note that these semaphores get used each time they are requested, and there is only a fixed number
+		 * available. If all are used up, null will be returned. New semaphores are generated when allocateSemaphores()
+		 * is called.
 		 */
-		VulkanSemaphore* allocateSemaphore();
+		VulkanSemaphore* requestInterQueueSemaphore() const;
+
+		/** 
+		 * Allocates a new set of semaphores that may be used for synchronizing execution between different command buffers.
+		 * Releases the previously allocated semaphores, if they exist. Use getIntraQueueSemaphore() & 
+		 * requestInterQueueSemaphore() to retrieve latest allocated semaphores.
+		 */
+		void allocateSemaphores();
 
 		/** Returns true if the command buffer is currently being processed by the device. */
 		bool isSubmitted() const { return mState == State::Submitted; }
@@ -332,9 +343,12 @@ namespace bs
 		VkCommandPool mPool;
 		VkCommandBuffer mCmdBuffer;
 		VkFence mFence;
-		VulkanSemaphore* mSemaphore;
 		UINT32 mFenceCounter;
 
+		VulkanSemaphore* mIntraQueueSemaphore;
+		VulkanSemaphore* mInterQueueSemaphores[BS_MAX_VULKAN_CB_DEPENDENCIES];
+		mutable UINT32 mNumUsedInterQueueSemaphores;
+
 		VulkanFramebuffer* mFramebuffer;
 		UINT32 mRenderTargetWidth;
 		UINT32 mRenderTargetHeight;

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBuffer.cpp

@@ -127,7 +127,8 @@ namespace bs
 	}
 
 	VulkanCmdBuffer::VulkanCmdBuffer(VulkanDevice& device, UINT32 id, VkCommandPool pool, UINT32 queueFamily, bool secondary)
-		: mId(id), mQueueFamily(queueFamily), mState(State::Ready), mDevice(device), mPool(pool), mSemaphore(nullptr)
+		: mId(id), mQueueFamily(queueFamily), mState(State::Ready), mDevice(device), mPool(pool)
+		, mIntraQueueSemaphore(nullptr), mInterQueueSemaphores(), mNumUsedInterQueueSemaphores(0)
 		, mFenceCounter(0), mFramebuffer(nullptr), mRenderTargetWidth(0)
 		, mRenderTargetHeight(0), mRenderTargetDepthReadOnly(false), mRenderTargetLoadMask(RT_NONE), mGlobalQueueIdx(-1)
 		, mViewport(0.0f, 0.0f, 1.0f, 1.0f), mScissor(0, 0, 0, 0), mStencilRef(0), mDrawOp(DOT_TRIANGLE_LIST)
@@ -205,9 +206,15 @@ namespace bs
 			}
 		}
 
-		if (mSemaphore != nullptr)
-			mSemaphore->destroy();
+		if (mIntraQueueSemaphore != nullptr)
+			mIntraQueueSemaphore->destroy();
 		
+		for(UINT32 i = 0; i < BS_MAX_VULKAN_CB_DEPENDENCIES; i++)
+		{
+			if (mInterQueueSemaphores[i] != nullptr)
+				mInterQueueSemaphores[i]->destroy();
+		}
+
 		vkDestroyFence(device, mFence, gVulkanAllocator);
 		vkFreeCommandBuffers(device, mPool, 1, &mCmdBuffer);
 
@@ -333,13 +340,30 @@ namespace bs
 		mState = State::Recording;
 	}
 
-	VulkanSemaphore* VulkanCmdBuffer::allocateSemaphore()
+	void VulkanCmdBuffer::allocateSemaphores()
 	{
-		if (mSemaphore != nullptr)
-			mSemaphore->destroy();
+		if (mIntraQueueSemaphore != nullptr)
+			mIntraQueueSemaphore->destroy();
+
+		mIntraQueueSemaphore = mDevice.getResourceManager().create<VulkanSemaphore>();
+		
+		for (UINT32 i = 0; i < BS_MAX_VULKAN_CB_DEPENDENCIES; i++)
+		{
+			if (mInterQueueSemaphores[i] != nullptr)
+				mInterQueueSemaphores[i]->destroy();
+
+			mInterQueueSemaphores[i] = mDevice.getResourceManager().create<VulkanSemaphore>();
+		}
+
+		mNumUsedInterQueueSemaphores = 0;
+	}
+
+	VulkanSemaphore* VulkanCmdBuffer::requestInterQueueSemaphore() const
+	{
+		if (mNumUsedInterQueueSemaphores >= BS_MAX_VULKAN_CB_DEPENDENCIES)
+			return nullptr;
 
-		mSemaphore = mDevice.getResourceManager().create<VulkanSemaphore>();
-		return mSemaphore;
+		return mInterQueueSemaphores[mNumUsedInterQueueSemaphores++];
 	}
 
 	void VulkanCmdBuffer::submit(VulkanQueue* queue, UINT32 queueIdx, UINT32 syncMask)

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBufferManager.cpp

@@ -150,17 +150,18 @@ namespace bs
 	}
 
 	void VulkanCommandBufferManager::getSyncSemaphores(UINT32 deviceIdx, UINT32 syncMask, VulkanSemaphore** semaphores,
-		UINT32& count)
+													   UINT32& count)
 	{
+		bool semaphoreRequestFailed = false;
 		SPtr<VulkanDevice> device = mRapi._getDevice(deviceIdx);
 
 		UINT32 semaphoreIdx = 0;
-		for(UINT32 i = 0; i < GQT_COUNT; i++)
+		for (UINT32 i = 0; i < GQT_COUNT; i++)
 		{
 			GpuQueueType queueType = (GpuQueueType)i;
 
 			UINT32 numQueues = device->getNumQueues(queueType);
-			for(UINT32 j = 0; j < numQueues; j++)
+			for (UINT32 j = 0; j < numQueues; j++)
 			{
 				VulkanQueue* queue = device->getQueue(queueType, j);
 				VulkanCmdBuffer* lastCB = queue->getLastCommandBuffer();
@@ -174,11 +175,26 @@ namespace bs
 				if ((syncMask & queueMask) == 0)
 					continue;
 
-				semaphores[semaphoreIdx++] = lastCB->getSemaphore();
+				VulkanSemaphore* semaphore = lastCB->requestInterQueueSemaphore();
+				if (semaphore == nullptr)
+				{
+					semaphoreRequestFailed = true;
+					continue;
+				}
+
+				semaphores[semaphoreIdx++] = semaphore;
 			}
 		}
 
 		count = semaphoreIdx;
+
+		if (semaphoreRequestFailed)
+		{
+			LOGERR("Failed to allocate semaphores for a command buffer sync. This means some of the dependency requests "
+				"will not be fulfilled. This happened because a command buffer has too many dependant command "
+				"buffers. The maximum allowed number is " + toString(BS_MAX_VULKAN_CB_DEPENDENCIES) + " but can be "
+				"increased by incrementing the value of BS_MAX_VULKAN_CB_DEPENDENCIES.");
+		}
 	}
 
 	void VulkanCommandBufferManager::refreshStates(UINT32 deviceIdx)

Source/BansheeVulkanRenderAPI/Source/BsVulkanQueue.cpp

@@ -24,7 +24,8 @@ namespace bs
 
 	void VulkanQueue::submit(VulkanCmdBuffer* cmdBuffer, VulkanSemaphore** waitSemaphores, UINT32 semaphoresCount)
 	{
-		VulkanSemaphore* signalSemaphore = cmdBuffer->allocateSemaphore();
+		cmdBuffer->allocateSemaphores();
+		VulkanSemaphore* signalSemaphore = cmdBuffer->getIntraQueueSemaphore();
 
 		VkCommandBuffer vkCmdBuffer = cmdBuffer->getHandle();
 		VkSemaphore vkSemaphore = signalSemaphore->getHandle();
@@ -81,7 +82,9 @@ namespace bs
 		{
 			const SubmitInfo& entry = mQueuedBuffers[i];
 
-			VulkanSemaphore* signalSemaphore = entry.cmdBuffer->allocateSemaphore();
+			entry.cmdBuffer->allocateSemaphores();
+			VulkanSemaphore* signalSemaphore = entry.cmdBuffer->getIntraQueueSemaphore();
+
 			commandBuffers[i] = entry.cmdBuffer->getHandle();
 			signalSemaphores[i] = signalSemaphore->getHandle();
 
@@ -235,7 +238,7 @@ namespace bs
 		// Wait on previous CB, as we want execution to proceed in order
 		if (mLastCommandBuffer != nullptr && mLastCommandBuffer->isSubmitted() && !mLastCBSemaphoreUsed)
 		{
-			VulkanSemaphore* prevSemaphore = mLastCommandBuffer->getSemaphore();
+			VulkanSemaphore* prevSemaphore = mLastCommandBuffer->getIntraQueueSemaphore();
 
 			prevSemaphore->notifyBound();
 			prevSemaphore->notifyUsed(0, 0, VulkanUseFlag::Read | VulkanUseFlag::Write);