Vulkan: Changing how command buffer resets work, so they handle the new multi-CB submit more gracefully

Source/BansheeVulkanRenderAPI/Include/BsVulkanCommandBuffer.h

@@ -174,11 +174,14 @@ namespace bs
 		/** Returns true if the command buffer is currently recording a render pass. */
 		bool isInRenderPass() const { return mState == State::RecordingRenderPass; }
 
-		/** Returns a counter that gets incremented whenever the command buffer is done executing. */
-		UINT32 getFenceCounter() const { return mFenceCounter; }
+		/** Checks the internal fence if done executing. */
+		bool checkFenceStatus() const;
 
-		/** Checks the internal fence and changes command buffer state if done executing. */
-		void refreshFenceStatus();
+		/** 
+		 * Resets the command buffer back in Ready state. Should be called when command buffer is done executing on a 
+		 * queue. 
+		 */
+		void reset();
 
 		/** 
 		 * Lets the command buffer know that the provided resource has been queued on it, and will be used by the
@@ -343,7 +346,6 @@ namespace bs
 		VkCommandPool mPool;
 		VkCommandBuffer mCmdBuffer;
 		VkFence mFence;
-		UINT32 mFenceCounter;
 
 		VulkanSemaphore* mIntraQueueSemaphore;
 		VulkanSemaphore* mInterQueueSemaphores[BS_MAX_VULKAN_CB_DEPENDENCIES];

Source/BansheeVulkanRenderAPI/Include/BsVulkanQueue.h

@@ -93,13 +93,15 @@ namespace bs
 		/** Information about a single submitted command buffer. */
 		struct SubmitInfo
 		{
-			SubmitInfo(VulkanCmdBuffer* cmdBuffer, UINT32 submitIdx, UINT32 numSemaphores)
-				:cmdBuffer(cmdBuffer), submitIdx(submitIdx), numSemaphores(numSemaphores)
+			SubmitInfo(VulkanCmdBuffer* cmdBuffer, UINT32 submitIdx, UINT32 numSemaphores, UINT32 numCommandBuffers)
+				: cmdBuffer(cmdBuffer), submitIdx(submitIdx), numSemaphores(numSemaphores)
+				, numCommandBuffers(numCommandBuffers)
 			{ }
 
 			VulkanCmdBuffer* cmdBuffer;
 			UINT32 submitIdx;
 			UINT32 numSemaphores;
+			UINT32 numCommandBuffers;
 		};
 
 		VulkanDevice& mDevice;
@@ -111,7 +113,8 @@ namespace bs
 		Vector<SubmitInfo> mQueuedBuffers;
 		Vector<VulkanSemaphore*> mQueuedSemaphores;
 
-		List<SubmitInfo> mActiveBuffers;
+		List<SubmitInfo> mActiveSubmissions;
+		Queue<VulkanCmdBuffer*> mActiveBuffers;
 		Queue<VulkanSemaphore*> mActiveSemaphores;
 		VulkanCmdBuffer* mLastCommandBuffer;
 		bool mLastCBSemaphoreUsed;

Source/BansheeVulkanRenderAPI/Source/BsVulkanCommandBuffer.cpp

@@ -129,7 +129,7 @@ namespace bs
 	VulkanCmdBuffer::VulkanCmdBuffer(VulkanDevice& device, UINT32 id, VkCommandPool pool, UINT32 queueFamily, bool secondary)
 		: mId(id), mQueueFamily(queueFamily), mState(State::Ready), mDevice(device), mPool(pool)
 		, mIntraQueueSemaphore(nullptr), mInterQueueSemaphores(), mNumUsedInterQueueSemaphores(0)
-		, mFenceCounter(0), mFramebuffer(nullptr), mRenderTargetWidth(0)
+		, mFramebuffer(nullptr), mRenderTargetWidth(0)
 		, mRenderTargetHeight(0), mRenderTargetDepthReadOnly(false), mRenderTargetLoadMask(RT_NONE), mGlobalQueueIdx(-1)
 		, mViewport(0.0f, 0.0f, 1.0f, 1.0f), mScissor(0, 0, 0, 0), mStencilRef(0), mDrawOp(DOT_TRIANGLE_LIST)
 		, mNumBoundDescriptorSets(0), mGfxPipelineRequiresBind(true), mCmpPipelineRequiresBind(true)
@@ -173,7 +173,7 @@ namespace bs
 				LOGWRN("Freeing a command buffer before done executing because fence wait expired!");
 
 			// Resources have been marked as used, make sure to notify them we're done with them
-			refreshFenceStatus();
+			reset();
 		}
 		else if(mState != State::Ready) 
 		{
@@ -613,57 +613,53 @@ namespace bs
 		mQueuedLayoutTransitions.clear();
 	}
 
-	void VulkanCmdBuffer::refreshFenceStatus()
+	bool VulkanCmdBuffer::checkFenceStatus() const
 	{
 		VkResult result = vkGetFenceStatus(mDevice.getLogical(), mFence);
 		assert(result == VK_SUCCESS || result == VK_NOT_READY);
 
-		bool signaled = result == VK_SUCCESS;
-
-		if (mState == State::Submitted)
-		{
-			if(signaled)
-			{
-				mState = State::Ready;
-				vkResetCommandBuffer(mCmdBuffer, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); // Note: Maybe better not to release resources?
+		return result == VK_SUCCESS;
+	}
 
-				mFenceCounter++;
+	void VulkanCmdBuffer::reset()
+	{
+		if (mState != State::Submitted)
+			return;
 
-				for (auto& entry : mResources)
-				{
-					ResourceUseHandle& useHandle = entry.second;
-					assert(useHandle.used);
+		mState = State::Ready;
+		vkResetCommandBuffer(mCmdBuffer, VK_COMMAND_BUFFER_RESET_RELEASE_RESOURCES_BIT); // Note: Maybe better not to release resources?
 
-					entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
-				}
+		for (auto& entry : mResources)
+		{
+			ResourceUseHandle& useHandle = entry.second;
+			assert(useHandle.used);
 
-				for (auto& entry : mImages)
-				{
-					UINT32 imageInfoIdx = entry.second;
-					ImageInfo& imageInfo = mImageInfos[imageInfoIdx];
+			entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
+		}
 
-					ResourceUseHandle& useHandle = imageInfo.useHandle;
-					assert(useHandle.used);
+		for (auto& entry : mImages)
+		{
+			UINT32 imageInfoIdx = entry.second;
+			ImageInfo& imageInfo = mImageInfos[imageInfoIdx];
 
-					entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
-				}
+			ResourceUseHandle& useHandle = imageInfo.useHandle;
+			assert(useHandle.used);
 
-				for (auto& entry : mBuffers)
-				{
-					ResourceUseHandle& useHandle = entry.second.useHandle;
-					assert(useHandle.used);
+			entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
+		}
 
-					entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
-				}
+		for (auto& entry : mBuffers)
+		{
+			ResourceUseHandle& useHandle = entry.second.useHandle;
+			assert(useHandle.used);
 
-				mResources.clear();
-				mImages.clear();
-				mBuffers.clear();
-				mImageInfos.clear();
-			}
+			entry.first->notifyDone(mGlobalQueueIdx, useHandle.flags);
 		}
-		else
-			assert(!signaled); // We reset the fence along with mState so this shouldn't be possible
+
+		mResources.clear();
+		mImages.clear();
+		mBuffers.clear();
+		mImageInfos.clear();
 	}
 
 	void VulkanCmdBuffer::setRenderTarget(const SPtr<RenderTargetCore>& rt, bool readOnlyDepthStencil, 

Source/BansheeVulkanRenderAPI/Source/BsVulkanQueue.cpp

@@ -42,12 +42,13 @@ namespace bs
 		mLastCommandBuffer = cmdBuffer;
 		mLastCBSemaphoreUsed = false;
 
-		mActiveBuffers.push_back(SubmitInfo(cmdBuffer, mNextSubmitIdx++, semaphoresCount));
+		mActiveSubmissions.push_back(SubmitInfo(cmdBuffer, mNextSubmitIdx++, semaphoresCount, 1));
+		mActiveBuffers.push(cmdBuffer);
 	}
 
 	void VulkanQueue::queueSubmit(VulkanCmdBuffer* cmdBuffer, VulkanSemaphore** waitSemaphores, UINT32 semaphoresCount)
 	{
-		mQueuedBuffers.push_back(SubmitInfo(cmdBuffer, 0, semaphoresCount));
+		mQueuedBuffers.push_back(SubmitInfo(cmdBuffer, 0, semaphoresCount, 1));
 
 		for (UINT32 i = 0; i < semaphoresCount; i++)
 			mQueuedSemaphores.push_back(waitSemaphores[i]);
@@ -55,30 +56,30 @@ namespace bs
 
 	void VulkanQueue::submitQueued()
 	{
-		UINT32 numSubmits = (UINT32)mQueuedBuffers.size();
-		if (numSubmits == 0)
+		UINT32 numCBs = (UINT32)mQueuedBuffers.size();
+		if (numCBs == 0)
 			return;
 
-		UINT32 totalNumWaitSemaphores = (UINT32)mQueuedSemaphores.size() + numSubmits;
+		UINT32 totalNumWaitSemaphores = (UINT32)mQueuedSemaphores.size() + numCBs;
 
 		UINT8* data = (UINT8*)bs_stack_alloc((sizeof(VkSubmitInfo) + sizeof(VkCommandBuffer) + sizeof(VkSemaphore)) * 
-			numSubmits + sizeof(VkSemaphore) * totalNumWaitSemaphores);
+			numCBs + sizeof(VkSemaphore) * totalNumWaitSemaphores);
 		UINT8* dataPtr = data;
 
 		VkSubmitInfo* submitInfos = (VkSubmitInfo*)dataPtr;
-		dataPtr += sizeof(VkSubmitInfo) * numSubmits;
+		dataPtr += sizeof(VkSubmitInfo) * numCBs;
 
 		VkCommandBuffer* commandBuffers = (VkCommandBuffer*)dataPtr;
-		dataPtr += sizeof(VkCommandBuffer) * numSubmits;
+		dataPtr += sizeof(VkCommandBuffer) * numCBs;
 
 		VkSemaphore* signalSemaphores = (VkSemaphore*)dataPtr;
-		dataPtr += sizeof(VkSemaphore) * numSubmits;
+		dataPtr += sizeof(VkSemaphore) * numCBs;
 
 		VkSemaphore* waitSemaphores = (VkSemaphore*)dataPtr;
 		dataPtr += sizeof(VkSemaphore) * totalNumWaitSemaphores;
 
 		UINT32 semaphoreIdx = 0;
-		for(UINT32 i = 0; i < numSubmits; i++)
+		for(UINT32 i = 0; i < numCBs; i++)
 		{
 			const SubmitInfo& entry = mQueuedBuffers[i];
 
@@ -98,11 +99,15 @@ namespace bs
 			mLastCommandBuffer = entry.cmdBuffer; // Needs to be set because getSubmitInfo depends on it
 			mLastCBSemaphoreUsed = false;
 
-			mActiveBuffers.push_back(SubmitInfo(entry.cmdBuffer, mNextSubmitIdx++, semaphoresCount));
+			mActiveBuffers.push(entry.cmdBuffer);
 			semaphoreIdx += semaphoresCount;
 		}
 
-		VkResult result = vkQueueSubmit(mQueue, 1, submitInfos, mLastCommandBuffer->getFence());
+		VulkanCmdBuffer* lastCB = mQueuedBuffers[numCBs - 1].cmdBuffer;
+		UINT32 totalNumSemaphores = semaphoreIdx;
+		mActiveSubmissions.push_back(SubmitInfo(lastCB, mNextSubmitIdx++, totalNumSemaphores, numCBs));
+
+		VkResult result = vkQueueSubmit(mQueue, numCBs, submitInfos, mLastCommandBuffer->getFence());
 		assert(result == VK_SUCCESS);
 
 		mQueuedBuffers.clear();
@@ -166,7 +171,7 @@ namespace bs
 		VkResult result = vkQueuePresentKHR(mQueue, &presentInfo);
 		assert(result == VK_SUCCESS || result == VK_SUBOPTIMAL_KHR);
 
-		mActiveBuffers.push_back(SubmitInfo(nullptr, mNextSubmitIdx++, semaphoresCount));
+		mActiveSubmissions.push_back(SubmitInfo(nullptr, mNextSubmitIdx++, semaphoresCount, 0));
 	}
 
 	void VulkanQueue::waitIdle() const
@@ -179,8 +184,8 @@ namespace bs
 	{
 		UINT32 lastFinishedSubmission = 0;
 
-		auto iter = mActiveBuffers.begin();
-		while (iter != mActiveBuffers.end())
+		auto iter = mActiveSubmissions.begin();
+		while (iter != mActiveSubmissions.end())
 		{
 			VulkanCmdBuffer* cmdBuffer = iter->cmdBuffer;
 			if (cmdBuffer == nullptr)
@@ -189,8 +194,7 @@ namespace bs
 				continue;
 			}
 
-			cmdBuffer->refreshFenceStatus();
-			if (cmdBuffer->isSubmitted())
+			if(!cmdBuffer->checkFenceStatus())
 				break; // No chance of any later CBs of being done either
 				
 			lastFinishedSubmission = iter->submitIdx;
@@ -203,8 +207,8 @@ namespace bs
 		if (queueEmpty)
 			lastFinishedSubmission = mNextSubmitIdx - 1;
 
-		iter = mActiveBuffers.begin();
-		while (iter != mActiveBuffers.end())
+		iter = mActiveSubmissions.begin();
+		while (iter != mActiveSubmissions.end())
 		{
 			if (iter->submitIdx > lastFinishedSubmission)
 				break;
@@ -217,7 +221,15 @@ namespace bs
 				semaphore->notifyDone(0, VulkanUseFlag::Read | VulkanUseFlag::Write);
 			}
 
-			iter = mActiveBuffers.erase(iter);
+			for(UINT32 i = 0; i < iter->numCommandBuffers; i++)
+			{
+				VulkanCmdBuffer* cb = mActiveBuffers.front();
+				mActiveBuffers.pop();
+
+				cb->reset();
+			}
+
+			iter = mActiveSubmissions.erase(iter);
 		}
 	}