Vulkan: More fixes

Source/BansheeVulkanRenderAPI/Include/BsVulkanQueue.h

@@ -53,8 +53,11 @@ namespace bs
 		/** 
 		 * Checks if any of the active command buffers finished executing on the queue and updates their states 
 		 * accordingly. 
+		 * 
+		 * @param[in]	queueEmpty	Set to true if the caller guarantees the queue will be empty (e.g. on shutdown). This
+		 *							allows the system to free all needed resources.
 		 */
-		void refreshStates();
+		void refreshStates(bool queueEmpty = false);
 
 		/** Returns the last command buffer that was submitted on this queue. */
 		VulkanCmdBuffer* getLastCommandBuffer() const { return mLastCommandBuffer; }

Source/BansheeVulkanRenderAPI/Source/BsVulkanDevice.cpp

@@ -124,13 +124,18 @@ namespace bs
 		{
 			UINT32 numQueues = (UINT32)mQueueInfos[i].queues.size();
 			for (UINT32 j = 0; j < numQueues; j++)
+			{
+				mQueueInfos[i].queues[j]->refreshStates(true);
 				bs_delete(mQueueInfos[i].queues[j]);
+			}
 		}
 
-		bs_delete(mResourceManager);
 		bs_delete(mDescriptorManager);
 		bs_delete(mQueryPool);
 		bs_delete(mCommandBufferPool);
+
+		// Needs to happen after query pool & command buffer pool shutdown, to ensure their resources are destroyed
+		bs_delete(mResourceManager);
 		
 		vkDestroyDevice(mLogicalDevice, gVulkanAllocator);
 	}

Source/BansheeVulkanRenderAPI/Source/BsVulkanGpuParams.cpp

@@ -27,19 +27,20 @@ namespace bs
 
 	VulkanGpuParams::~VulkanGpuParams()
 	{
+		Lock lock(mMutex);
+
+		UINT32 numSets = mParamInfo->getNumSets();
+		for (UINT32 i = 0; i < BS_MAX_DEVICES; i++)
 		{
-			Lock lock(mMutex);
+			if (mPerDeviceData[i].perSetData == nullptr)
+				continue;
 
-			UINT32 numSets = mParamInfo->getNumSets();
-			for (UINT32 i = 0; i < BS_MAX_DEVICES; i++)
+			for (UINT32 j = 0; j < numSets; j++)
 			{
-				for (UINT32 j = 0; j < numSets; j++)
-				{
-					for (auto& entry : mPerDeviceData[i].perSetData[j].sets)
-						entry->destroy();
+				for (auto& entry : mPerDeviceData[i].perSetData[j].sets)
+					entry->destroy();
 
-					mPerDeviceData[i].perSetData[j].sets.~Vector<VulkanDescriptorSet*>();
-				}
+				mPerDeviceData[i].perSetData[j].sets.~Vector<VulkanDescriptorSet*>();
 			}
 		}
 	}

Source/BansheeVulkanRenderAPI/Source/BsVulkanGpuPipelineState.cpp

@@ -80,7 +80,11 @@ namespace bs
 																	 GpuDeviceFlags deviceMask)
 		:GraphicsPipelineStateCore(desc, deviceMask), mScissorEnabled(false), mDeviceMask(deviceMask)
 	{
-		
+		for (UINT32 i = 0; i < BS_MAX_DEVICES; i++)
+		{
+			mPerDeviceData[i].device = nullptr;
+			mPerDeviceData[i].pipelineLayout = VK_NULL_HANDLE;
+		}
 	}
 
 	VulkanGraphicsPipelineStateCore::~VulkanGraphicsPipelineStateCore()
@@ -298,27 +302,23 @@ namespace bs
 
 		for (UINT32 i = 0; i < BS_MAX_DEVICES; i++)
 		{
+			if (devices[i] == nullptr)
+				continue;
+
 			mPerDeviceData[i].device = devices[i];
 
-			if (devices[i] != nullptr)
-			{
-				VulkanDescriptorManager& descManager = mPerDeviceData[i].device->getDescriptorManager();
-				VulkanGpuPipelineParamInfo& vkParamInfo = static_cast<VulkanGpuPipelineParamInfo&>(*mParamInfo);
+			VulkanDescriptorManager& descManager = mPerDeviceData[i].device->getDescriptorManager();
+			VulkanGpuPipelineParamInfo& vkParamInfo = static_cast<VulkanGpuPipelineParamInfo&>(*mParamInfo);
 
-				UINT32 numLayouts = vkParamInfo.getNumSets();
-				VulkanDescriptorLayout** layouts = (VulkanDescriptorLayout**)bs_stack_alloc(sizeof(VulkanDescriptorLayout*) * numLayouts);
+			UINT32 numLayouts = vkParamInfo.getNumSets();
+			VulkanDescriptorLayout** layouts = (VulkanDescriptorLayout**)bs_stack_alloc(sizeof(VulkanDescriptorLayout*) * numLayouts);
 
-				for (UINT32 j = 0; j < numLayouts; j++)
-					layouts[j] = vkParamInfo.getLayout(i, j);
+			for (UINT32 j = 0; j < numLayouts; j++)
+				layouts[j] = vkParamInfo.getLayout(i, j);
 
-				mPerDeviceData[i].pipelineLayout = descManager.getPipelineLayout(layouts, numLayouts);
+			mPerDeviceData[i].pipelineLayout = descManager.getPipelineLayout(layouts, numLayouts);
 
-				bs_stack_free(layouts);
-			}
-			else
-			{
-				mPerDeviceData[i].pipelineLayout = VK_NULL_HANDLE;
-			}
+			bs_stack_free(layouts);
 		}
 
 		BS_INC_RENDER_STAT_CAT(ResCreated, RenderStatObject_PipelineState);
@@ -501,7 +501,9 @@ namespace bs
 	VulkanComputePipelineStateCore::VulkanComputePipelineStateCore(const SPtr<GpuProgramCore>& program, 
 		GpuDeviceFlags deviceMask)
 		:ComputePipelineStateCore(program, deviceMask), mDeviceMask(deviceMask)
-	{ }
+	{
+		bs_zero_out(mPerDeviceData);
+	}
 
 	VulkanComputePipelineStateCore::~VulkanComputePipelineStateCore()
 	{
@@ -549,44 +551,39 @@ namespace bs
 
 		for (UINT32 i = 0; i < BS_MAX_DEVICES; i++)
 		{
+			if (devices[i] == nullptr)
+				continue;
+
 			mPerDeviceData[i].device = devices[i];
 
-			if (devices[i] != nullptr)
-			{
-				VulkanDescriptorManager& descManager = devices[i]->getDescriptorManager();
-				VulkanResourceManager& rescManager = devices[i]->getResourceManager();
-				VulkanGpuPipelineParamInfo& vkParamInfo = static_cast<VulkanGpuPipelineParamInfo&>(*mParamInfo);
+			VulkanDescriptorManager& descManager = devices[i]->getDescriptorManager();
+			VulkanResourceManager& rescManager = devices[i]->getResourceManager();
+			VulkanGpuPipelineParamInfo& vkParamInfo = static_cast<VulkanGpuPipelineParamInfo&>(*mParamInfo);
 
-				UINT32 numLayouts = vkParamInfo.getNumSets();
-				VulkanDescriptorLayout** layouts = (VulkanDescriptorLayout**)bs_stack_alloc(sizeof(VulkanDescriptorLayout*) * numLayouts);
+			UINT32 numLayouts = vkParamInfo.getNumSets();
+			VulkanDescriptorLayout** layouts = (VulkanDescriptorLayout**)bs_stack_alloc(sizeof(VulkanDescriptorLayout*) * numLayouts);
 
-				for (UINT32 j = 0; j < numLayouts; j++)
-					layouts[j] = vkParamInfo.getLayout(i, j);
+			for (UINT32 j = 0; j < numLayouts; j++)
+				layouts[j] = vkParamInfo.getLayout(i, j);
 
-				VulkanShaderModule* module = vkProgram->getShaderModule(i);
+			VulkanShaderModule* module = vkProgram->getShaderModule(i);
 
-				if (module != nullptr)
-					pipelineCI.stage.module = module->getHandle();
-				else
-					pipelineCI.stage.module = VK_NULL_HANDLE;
+			if (module != nullptr)
+				pipelineCI.stage.module = module->getHandle();
+			else
+				pipelineCI.stage.module = VK_NULL_HANDLE;
 
-				pipelineCI.layout = descManager.getPipelineLayout(layouts, numLayouts);
+			pipelineCI.layout = descManager.getPipelineLayout(layouts, numLayouts);
 
-				VkPipeline pipeline;
-				VkResult result = vkCreateComputePipelines(devices[i]->getLogical(), VK_NULL_HANDLE, 1, &pipelineCI,
-														   gVulkanAllocator, &pipeline);
-				assert(result == VK_SUCCESS);
+			VkPipeline pipeline;
+			VkResult result = vkCreateComputePipelines(devices[i]->getLogical(), VK_NULL_HANDLE, 1, &pipelineCI,
+														gVulkanAllocator, &pipeline);
+			assert(result == VK_SUCCESS);
 
 
-				mPerDeviceData[i].pipeline = rescManager.create<VulkanPipeline>(pipeline);
-				mPerDeviceData[i].pipelineLayout = pipelineCI.layout;
-				bs_stack_free(layouts);
-			}
-			else
-			{
-				mPerDeviceData[i].pipeline = nullptr;
-				mPerDeviceData[i].pipelineLayout = VK_NULL_HANDLE;
-			}
+			mPerDeviceData[i].pipeline = rescManager.create<VulkanPipeline>(pipeline);
+			mPerDeviceData[i].pipelineLayout = pipelineCI.layout;
+			bs_stack_free(layouts);
 		}
 
 		BS_INC_RENDER_STAT_CAT(ResCreated, RenderStatObject_PipelineState);

Source/BansheeVulkanRenderAPI/Source/BsVulkanQueryManager.cpp

@@ -24,10 +24,16 @@ namespace bs
 		Lock(mMutex);
 
 		for (auto& entry : mTimerQueries)
-			entry->destroy();
+		{
+			if(entry != nullptr)
+				entry->destroy();
+		}
 
 		for (auto& entry : mOcclusionQueries)
-			entry->destroy();
+		{
+			if(entry != nullptr)
+				entry->destroy();
+		}
 
 		for (auto& entry : mTimerPools)
 			vkDestroyQueryPool(mDevice.getLogical(), entry.pool, gVulkanAllocator);

Source/BansheeVulkanRenderAPI/Source/BsVulkanQueue.cpp

@@ -116,7 +116,7 @@ namespace bs
 		assert(result == VK_SUCCESS);
 	}
 
-	void VulkanQueue::refreshStates()
+	void VulkanQueue::refreshStates(bool queueEmpty)
 	{
 		UINT32 lastFinishedSubmission = 0;
 
@@ -138,6 +138,12 @@ namespace bs
 			++iter;
 		}
 
+		// If last submission was a present() call, it won't be freed until a command buffer after it is done. However on
+		// shutdown there might not be a CB following it. So we instead check this special flag and free everything when its
+		// true.
+		if (queueEmpty)
+			lastFinishedSubmission = mNextSubmitIdx - 1;
+
 		iter = mActiveBuffers.begin();
 		while (iter != mActiveBuffers.end())
 		{

Source/BansheeVulkanRenderAPI/Source/BsVulkanUtility.cpp

@@ -342,9 +342,9 @@ namespace bs
 		case CULL_NONE:
 			return VK_CULL_MODE_NONE;
 		case CULL_CLOCKWISE:
-			return VK_CULL_MODE_BACK_BIT;
-		case CULL_COUNTERCLOCKWISE:
 			return VK_CULL_MODE_FRONT_BIT;
+		case CULL_COUNTERCLOCKWISE:
+			return VK_CULL_MODE_BACK_BIT;
 		}
 
 		// Unsupported type