Command buffer submit refactoring

AnKi/Gr/CommandBuffer.h

@@ -93,10 +93,13 @@ class CommandBuffer : public GrObject
 public:
 	static constexpr GrObjectType kClassType = GrObjectType::kCommandBuffer;
 
-	/// Finalize and submit if it's primary command buffer and just finalize if it's second level.
-	/// @param[in]  waitFences Optionally wait for some fences.
-	/// @param[out] signalFence Optionaly create fence that will be signaled when the submission is done.
-	void flush(ConstWeakArray<FencePtr> waitFences = {}, FencePtr* signalFence = nullptr);
+	CommandBufferFlag getFlags() const
+	{
+		return m_flags;
+	}
+
+	/// Finalize the command buffer.
+	void endRecording();
 
 	/// @name State manipulation
 	/// @{
@@ -467,6 +470,8 @@ public:
 	/// @}
 
 protected:
+	CommandBufferFlag m_flags = CommandBufferFlag::kNone;
+
 	/// Construct.
 	CommandBuffer(CString name)
 		: GrObject(kClassType, name)

AnKi/Gr/GrManager.h

@@ -8,6 +8,7 @@
 #include <AnKi/Gr/Common.h>
 #include <AnKi/Gr/GrObject.h>
 #include <AnKi/Util/String.h>
+#include <AnKi/Util/WeakArray.h>
 #include <AnKi/Core/CVarSet.h>
 
 namespace anki {
@@ -56,6 +57,16 @@ public:
 	/// Wait for all work to finish.
 	void finish();
 
+	/// Finalize and submit if it's primary command buffer and just finalize if it's second level.
+	/// @param[in]  waitFences Optionally wait for some fences.
+	/// @param[out] signalFence Optionaly create fence that will be signaled when the submission is done.
+	void submit(WeakArray<CommandBuffer*> cmdbs, WeakArray<Fence*> waitFences = {}, FencePtr* signalFence = nullptr);
+
+	void submit(CommandBuffer* cmdb, WeakArray<Fence*> waitFences = {}, FencePtr* signalFence = nullptr)
+	{
+		submit(WeakArray<CommandBuffer*>(&cmdb, 1), waitFences, signalFence);
+	}
+
 	/// @name Object creation methods. They are thread-safe.
 	/// @{
 	[[nodiscard]] BufferPtr newBuffer(const BufferInitInfo& init);

AnKi/Gr/RenderGraph.cpp

@@ -1366,7 +1366,11 @@ void RenderGraph::runSecondLevel()
 					pass.m_callback(*ctx);
 				}
 
-				ctx->m_commandBuffer->flush();
+				ctx->m_commandBuffer->endRecording();
+				if(!(ctx->m_commandBuffer->getFlags() & CommandBufferFlag::kSecondLevel))
+				{
+					GrManager::getSingleton().submit(ctx->m_commandBuffer);
+				}
 			});
 		}
 	}
@@ -1497,7 +1501,8 @@ void RenderGraph::flush(FencePtr* optionalFence)
 		}
 
 		// Flush
-		m_ctx->m_graphicsCmdbs[i]->flush({}, (i == m_ctx->m_graphicsCmdbs.getSize() - 1) ? optionalFence : nullptr);
+		m_ctx->m_graphicsCmdbs[i]->endRecording();
+		GrManager::getSingleton().submit(m_ctx->m_graphicsCmdbs[i].get(), {}, (i == m_ctx->m_graphicsCmdbs.getSize() - 1) ? optionalFence : nullptr);
 	}
 }
 

AnKi/Gr/Utils/SegregatedListsGpuMemoryPool.cpp

@@ -171,7 +171,8 @@ Error SegregatedListsGpuMemoryPool::allocateChunk(Chunk*& newChunk, PtrSize& chu
 		barriers[1].m_nextUsage = m_bufferUsage;
 		cmdb->setPipelineBarrier({}, ConstWeakArray<BufferBarrierInfo>{&barriers[1], 1}, {});
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		// Create the new chunk
 		newChunk = newInstance<Chunk>(GrMemoryPool::getSingleton());

AnKi/Gr/Vulkan/VkCommandBuffer.cpp

@@ -33,36 +33,10 @@ CommandBuffer* CommandBuffer::newInstance(const CommandBufferInitInfo& init)
 	return impl;
 }
 
-void CommandBuffer::flush(ConstWeakArray<FencePtr> waitFences, FencePtr* signalFence)
+void CommandBuffer::endRecording()
 {
 	ANKI_VK_SELF(CommandBufferImpl);
 	self.endRecording();
-
-	if(!self.isSecondLevel())
-	{
-		Array<MicroSemaphorePtr, 8> waitSemaphores;
-		for(U32 i = 0; i < waitFences.getSize(); ++i)
-		{
-			waitSemaphores[i] = static_cast<const FenceImpl&>(*waitFences[i]).m_semaphore;
-		}
-
-		MicroSemaphorePtr signalSemaphore;
-		getGrManagerImpl().flushCommandBuffer(self.getMicroCommandBuffer(), self.renderedToDefaultFramebuffer(),
-											  WeakArray<MicroSemaphorePtr>(waitSemaphores.getBegin(), waitFences.getSize()),
-											  (signalFence) ? &signalSemaphore : nullptr);
-
-		if(signalFence)
-		{
-			FenceImpl* fenceImpl = anki::newInstance<FenceImpl>(GrMemoryPool::getSingleton(), "SignalFence");
-			fenceImpl->m_semaphore = signalSemaphore;
-			signalFence->reset(fenceImpl);
-		}
-	}
-	else
-	{
-		ANKI_ASSERT(signalFence == nullptr);
-		ANKI_ASSERT(waitFences.getSize() == 0);
-	}
 }
 
 void CommandBuffer::bindVertexBuffer(U32 binding, Buffer* buff, PtrSize offset, PtrSize stride, VertexStepRate stepRate)
@@ -1369,6 +1343,11 @@ CommandBufferImpl::~CommandBufferImpl()
 
 #if ANKI_EXTRA_CHECKS
 	ANKI_ASSERT(m_debugMarkersPushed == 0);
+
+	if(!m_submitted && !(m_flags & CommandBufferFlag::kSecondLevel))
+	{
+		ANKI_VK_LOGW("Command buffer not submitted");
+	}
 #endif
 }
 

AnKi/Gr/Vulkan/VkCommandBuffer.h

@@ -85,13 +85,25 @@ public:
 
 	void endRecording();
 
+	Bool isFinalized() const
+	{
+		return m_finalized;
+	}
+
+#if ANKI_EXTRA_CHECKS
+	void setSubmitted()
+	{
+		ANKI_ASSERT(!m_submitted);
+		m_submitted = true;
+	}
+#endif
+
 private:
 	StackMemoryPool* m_pool = nullptr;
 
 	MicroCommandBufferPtr m_microCmdb;
 	VkCommandBuffer m_handle = VK_NULL_HANDLE;
 	ThreadId m_tid = ~ThreadId(0);
-	CommandBufferFlag m_flags = CommandBufferFlag::kNone;
 	Bool m_renderedToDefaultFb : 1 = false;
 	Bool m_finalized : 1 = false;
 	Bool m_empty : 1 = true;
@@ -101,6 +113,7 @@ private:
 	U32 m_commandCount = 0;
 	U32 m_setPushConstantsSize = 0;
 	U32 m_debugMarkersPushed = 0;
+	Bool m_submitted = false;
 #endif
 
 	Framebuffer* m_activeFb = nullptr;

AnKi/Gr/Vulkan/VkGrManager.cpp

@@ -21,6 +21,7 @@
 #include <AnKi/Gr/RenderGraph.h>
 #include <AnKi/Gr/Vulkan/VkAccelerationStructure.h>
 #include <AnKi/Gr/Vulkan/VkGrUpscaler.h>
+#include <AnKi/Gr/Vulkan/VkFence.h>
 
 #include <AnKi/Window/NativeWindow.h>
 #if ANKI_WINDOWING_SYSTEM_SDL
@@ -155,6 +156,41 @@ ANKI_NEW_GR_OBJECT(GrUpscaler)
 #undef ANKI_NEW_GR_OBJECT
 #undef ANKI_NEW_GR_OBJECT_NO_INIT_INFO
 
+void GrManager::submit(WeakArray<CommandBuffer*> cmdbs, WeakArray<Fence*> waitFences, FencePtr* signalFence)
+{
+	Bool renderedToDefaultFb = false;
+	Array<MicroCommandBuffer*, 16> mcmdbs;
+	for(U32 i = 0; i < cmdbs.getSize(); ++i)
+	{
+		CommandBufferImpl& cmdb = *static_cast<CommandBufferImpl*>(cmdbs[i]);
+		ANKI_ASSERT(cmdb.isFinalized());
+
+		mcmdbs[i] = cmdb.getMicroCommandBuffer().get();
+		renderedToDefaultFb = renderedToDefaultFb || cmdb.renderedToDefaultFramebuffer();
+
+#if ANKI_ASSERTIONS_ENABLED
+		cmdb.setSubmitted();
+#endif
+	}
+
+	Array<MicroSemaphore*, 8> waitSemaphores;
+	for(U32 i = 0; i < waitFences.getSize(); ++i)
+	{
+		waitSemaphores[i] = static_cast<const FenceImpl&>(*waitFences[i]).m_semaphore.get();
+	}
+
+	MicroSemaphorePtr signalSemaphore;
+	getGrManagerImpl().flushCommandBuffers({mcmdbs.getBegin(), cmdbs.getSize()}, renderedToDefaultFb,
+										   {waitSemaphores.getBegin(), waitFences.getSize()}, (signalFence) ? &signalSemaphore : nullptr, false);
+
+	if(signalFence)
+	{
+		FenceImpl* fenceImpl = anki::newInstance<FenceImpl>(GrMemoryPool::getSingleton(), "SignalFence");
+		fenceImpl->m_semaphore = signalSemaphore;
+		signalFence->reset(fenceImpl);
+	}
+}
+
 GrManagerImpl::~GrManagerImpl()
 {
 	ANKI_VK_LOGI("Destroying Vulkan backend");
@@ -1547,8 +1583,8 @@ void GrManagerImpl::resetFrame(PerFrame& frame)
 	frame.m_renderSemaphore.reset(nullptr);
 }
 
-void GrManagerImpl::flushCommandBuffer(MicroCommandBufferPtr cmdb, Bool cmdbRenderedToSwapchain, WeakArray<MicroSemaphorePtr> userWaitSemaphores,
-									   MicroSemaphorePtr* userSignalSemaphore, Bool wait)
+void GrManagerImpl::flushCommandBuffers(WeakArray<MicroCommandBuffer*> cmdbs, Bool cmdbRenderedToSwapchain,
+										WeakArray<MicroSemaphore*> userWaitSemaphores, MicroSemaphorePtr* userSignalSemaphore, Bool wait)
 {
 	constexpr U32 maxSemaphores = 8;
 
@@ -1564,11 +1600,18 @@ void GrManagerImpl::flushCommandBuffer(MicroCommandBufferPtr cmdb, Bool cmdbRend
 	// First thing, create a fence
 	MicroFencePtr fence = m_fenceFactory.newInstance();
 
-	// Command buffer
-	const VkCommandBuffer handle = cmdb->getHandle();
-	cmdb->setFence(fence.get());
-	submit.commandBufferCount = 1;
-	submit.pCommandBuffers = &handle;
+	// Command buffers
+	Array<VkCommandBuffer, 16> handles;
+	submit.pCommandBuffers = handles.getBegin();
+	VulkanQueueType queueType = cmdbs[0]->getVulkanQueueType();
+	for(MicroCommandBuffer* cmdb : cmdbs)
+	{
+		handles[submit.commandBufferCount] = cmdb->getHandle();
+		cmdb->setFence(fence.get());
+		++submit.commandBufferCount;
+		ANKI_ASSERT(cmdb->getVulkanQueueType() == queueType);
+		queueType = cmdb->getVulkanQueueType();
+	}
 
 	// Handle user semaphores
 	Array<U64, maxSemaphores> waitTimelineValues;
@@ -1581,9 +1624,8 @@ void GrManagerImpl::flushCommandBuffer(MicroCommandBufferPtr cmdb, Bool cmdbRend
 	timelineInfo.pSignalSemaphoreValues = &signalTimelineValues[0];
 	submit.pNext = &timelineInfo;
 
-	for(MicroSemaphorePtr& userWaitSemaphore : userWaitSemaphores)
+	for(MicroSemaphore* userWaitSemaphore : userWaitSemaphores)
 	{
-		ANKI_ASSERT(userWaitSemaphore.isCreated());
 		ANKI_ASSERT(userWaitSemaphore->isTimeline());
 		waitSemaphores[submit.waitSemaphoreCount] = userWaitSemaphore->getHandle();
 
@@ -1640,16 +1682,16 @@ void GrManagerImpl::flushCommandBuffer(MicroCommandBufferPtr cmdb, Bool cmdbRend
 			// Update the swapchain's fence
 			m_crntSwapchain->setFence(fence.get());
 
-			frame.m_queueWroteToSwapchainImage = cmdb->getVulkanQueueType();
+			frame.m_queueWroteToSwapchainImage = queueType;
 		}
 
 		// Submit
 		ANKI_TRACE_SCOPED_EVENT(VkQueueSubmit);
-		ANKI_VK_CHECKF(vkQueueSubmit(m_queues[cmdb->getVulkanQueueType()], 1, &submit, fence->getHandle()));
+		ANKI_VK_CHECKF(vkQueueSubmit(m_queues[queueType], 1, &submit, fence->getHandle()));
 
 		if(wait)
 		{
-			vkQueueWaitIdle(m_queues[cmdb->getVulkanQueueType()]);
+			vkQueueWaitIdle(m_queues[queueType]);
 		}
 	}
 

AnKi/Gr/Vulkan/VkGrManager.h

@@ -103,8 +103,8 @@ public:
 	}
 	/// @}
 
-	void flushCommandBuffer(MicroCommandBufferPtr cmdb, Bool cmdbRenderedToSwapchain, WeakArray<MicroSemaphorePtr> waitSemaphores,
-							MicroSemaphorePtr* signalSemaphore, Bool wait = false);
+	void flushCommandBuffers(WeakArray<MicroCommandBuffer*> cmdbs, Bool cmdbRenderedToSwapchain, WeakArray<MicroSemaphore*> waitSemaphores,
+							 MicroSemaphorePtr* signalSemaphore, Bool wait);
 
 	/// @name Memory
 	/// @{

AnKi/Gr/Vulkan/VkGrUpscaler.cpp

@@ -154,7 +154,8 @@ Error GrUpscalerImpl::createDlssFeature(const UVec2& srcRes, const UVec2& dstRes
 	ANKI_NGX_CHECK(
 		NGX_VULKAN_CREATE_DLSS_EXT(cmdbImpl.getHandle(), creationNodeMask, visibilityNodeMask, &m_dlssFeature, m_ngxParameters, &dlssCreateParams));
 	FencePtr fence;
-	cmdb->flush({}, &fence);
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 	fence->clientWait(60.0_sec);
 
 	return Error::kNone;

AnKi/Renderer/DepthDownscale.cpp

@@ -71,7 +71,8 @@ Error DepthDownscale::initInternal()
 		cmdb->fillBuffer(m_counterBuffer.get(), 0, kMaxPtrSize, 0);
 
 		FencePtr fence;
-		cmdb->flush({}, &fence);
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 
 		fence->clientWait(6.0_sec);
 	}

AnKi/Renderer/Renderer.cpp

@@ -592,7 +592,8 @@ TexturePtr Renderer::createAndClearRenderTarget(const TextureInitInfo& inf, Text
 		}
 	}
 
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 
 	return tex;
 }

AnKi/Renderer/RendererObject.cpp

@@ -104,7 +104,8 @@ void RendererObject::zeroBuffer(Buffer* buff)
 	cmdb->fillBuffer(buff, 0, kMaxPtrSize, 0);
 
 	FencePtr fence;
-	cmdb->flush({}, &fence);
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 
 	fence->clientWait(16.0_sec);
 }

AnKi/Renderer/Utils/HzbGenerator.cpp

@@ -65,7 +65,8 @@ Error HzbGenerator::init()
 		cmdb->fillBuffer(m_counterBuffer.get(), 0, kMaxPtrSize, 0);
 
 		FencePtr fence;
-		cmdb->flush({}, &fence);
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 
 		fence->clientWait(6.0_sec);
 	}

AnKi/Resource/ImageResource.cpp

@@ -211,7 +211,8 @@ Error ImageResource::load(const ResourceFilename& filename, Bool async)
 		cmdb->setPipelineBarrier({&barrier, 1}, {}, {});
 
 		FencePtr outFence;
-		cmdb->flush({}, &outFence);
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get(), {}, &outFence);
 		outFence->clientWait(60.0_sec);
 	}
 
@@ -355,7 +356,8 @@ Error ImageResource::load(LoadingContext& ctx)
 
 		// Flush batch
 		FencePtr fence;
-		cmdb->flush({}, &fence);
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 
 		for(U i = 0; i < handleCount; ++i)
 		{

AnKi/Resource/MeshResource.cpp

@@ -212,7 +212,8 @@ Error MeshResource::load(const ResourceFilename& filename, Bool async)
 
 		cmdb->setPipelineBarrier({}, {&barrier, 1}, {});
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 	}
 
 	// Submit the loading task
@@ -422,7 +423,8 @@ Error MeshResource::loadAsync(MeshBinaryLoader& loader) const
 
 	// Finalize
 	FencePtr fence;
-	cmdb->flush({}, &fence);
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 
 	for(U32 i = 0; i < handleCount; ++i)
 	{

AnKi/Scene/Components/GlobalIlluminationProbeComponent.cpp

@@ -92,7 +92,8 @@ Error GlobalIlluminationProbeComponent::update(SceneComponentUpdateInfo& info, B
 		texBarrier.m_nextUsage = m_volTex->getTextureUsage();
 		cmdb->setPipelineBarrier({&texBarrier, 1}, {}, {});
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 	}
 
 	// Any update

AnKi/Scene/Components/ParticleEmitterComponent.cpp

@@ -244,7 +244,9 @@ ParticleEmitterComponent::ParticleEmitterComponent(SceneNode* node)
 	barrier.m_previousUsage = BufferUsageBit::kTransferDestination;
 	barrier.m_nextUsage = dstBuff->getBufferUsage();
 	cmdb->setPipelineBarrier({}, {&barrier, 1}, {});
-	cmdb->flush();
+	cmdb->endRecording();
+
+	GrManager::getSingleton().submit(cmdb.get());
 }
 
 ParticleEmitterComponent::~ParticleEmitterComponent()

AnKi/Ui/Font.cpp

@@ -107,7 +107,8 @@ void Font::createTexture(const void* data, U32 width, U32 height)
 	barrier.m_nextUsage = TextureUsageBit::kSampledFragment;
 	cmdb->setPipelineBarrier({&barrier, 1}, {}, {});
 
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 }
 
 } // end namespace anki

Tests/Gr/Gr.cpp

@@ -15,6 +15,7 @@
 #include <AnKi/Resource/TransferGpuAllocator.h>
 #include <AnKi/ShaderCompiler/ShaderProgramParser.h>
 #include <AnKi/Collision/Aabb.h>
+#include <AnKi/Util/WeakArray.h>
 #include <ctime>
 
 using namespace anki;
@@ -489,7 +490,8 @@ ANKI_TEST(Gr, ClearScreen)
 		cmdb->beginRenderPass(fb.get(), {TextureUsageBit::kFramebufferWrite}, {});
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -551,7 +553,8 @@ void main()
 		cmdb->draw(PrimitiveTopology::kTriangles, 3);
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -720,7 +723,8 @@ void main()
 			cmdb->beginRenderPass(fb[0].get(), {{TextureUsageBit::kFramebufferWrite}}, {});
 			cmdb->endRenderPass();
 			setTextureSurfaceBarrier(cmdb, rt, TextureUsageBit::kFramebufferWrite, TextureUsageBit::kSampledFragment, TextureSurfaceInfo(0, 0, 0, 0));
-			cmdb->flush();
+			cmdb->endRecording();
+			GrManager::getSingleton().submit(cmdb.get());
 		}
 
 		CommandBufferInitInfo cinit;
@@ -750,7 +754,8 @@ void main()
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -891,7 +896,8 @@ float4 main(VertOut i) : SV_TARGET0
 		cmdb->draw(PrimitiveTopology::kTriangles, 3);
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -971,7 +977,8 @@ ANKI_TEST(Gr, DrawWithVertex)
 		cmdb->draw(PrimitiveTopology::kTriangles, 3);
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -1109,7 +1116,8 @@ ANKI_TEST(Gr, DrawWithTexture)
 	}
 
 	FencePtr fence;
-	cmdb->flush({}, &fence);
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 	transfAlloc->release(handle0, fence);
 	transfAlloc->release(handle1, fence);
 	transfAlloc->release(handle2, fence);
@@ -1190,7 +1198,8 @@ void main()
 		cmdb->draw(PrimitiveTopology::kTriangles, 6);
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -1336,7 +1345,8 @@ static void drawOffscreen(GrManager& gr, Bool useSecondLevel)
 
 			drawOffscreenDrawcalls(gr, prog, cmdb2, TEX_SIZE, indices, verts);
 
-			cmdb2->flush();
+			cmdb->endRecording();
+			GrManager::getSingleton().submit(cmdb.get());
 
 			CommandBuffer* pCmdb = cmdb2.get();
 			cmdb->pushSecondLevelCommandBuffers({&pCmdb, 1});
@@ -1362,7 +1372,8 @@ static void drawOffscreen(GrManager& gr, Bool useSecondLevel)
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		// End
 		gr.swapBuffers();
@@ -1447,7 +1458,8 @@ ANKI_TEST(Gr, ImageLoadStore)
 
 	setTextureSurfaceBarrier(cmdb, tex, TextureUsageBit::kTransferDestination, TextureUsageBit::kUavComputeWrite, TextureSurfaceInfo(1, 0, 0, 0));
 
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 
 	const U ITERATION_COUNT = 100;
 	U iterations = ITERATION_COUNT;
@@ -1483,7 +1495,8 @@ ANKI_TEST(Gr, ImageLoadStore)
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		// End
 		g_gr->swapBuffers();
@@ -1552,7 +1565,8 @@ ANKI_TEST(Gr, 3DTextures)
 	setTextureVolumeBarrier(cmdb, a, TextureUsageBit::kTransferDestination, TextureUsageBit::kSampledFragment, TextureVolumeInfo(1));
 
 	FencePtr fence;
-	cmdb->flush({}, &fence);
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get(), {}, &fence);
 	transfAlloc->release(handle0, fence);
 	transfAlloc->release(handle1, fence);
 
@@ -1594,7 +1608,8 @@ ANKI_TEST(Gr, 3DTextures)
 		cmdb->endRenderPass();
 		presentBarrierB(cmdb, presentTex);
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		// End
 		g_gr->swapBuffers();
@@ -1922,7 +1937,8 @@ void main()
 
 	setBufferBarrier(cmdb, resultBuff, BufferUsageBit::kUavComputeWrite, BufferUsageBit::kUavComputeWrite, 0, resultBuff->getSize());
 
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 	g_gr->finish();
 
 	// Get the result
@@ -2027,7 +2043,8 @@ void main()
 	cmdb->draw(PrimitiveTopology::kTriangles, 3);
 	cmdb->endRenderPass();
 	presentBarrierB(cmdb, presentTex);
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 
 	g_gr->swapBuffers();
 	g_gr->finish();
@@ -2153,7 +2170,8 @@ void main()
 	cmdb->draw(PrimitiveTopology::kTriangles, 3);
 	cmdb->endRenderPass();
 	presentBarrierB(cmdb, presentTex);
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 
 	g_gr->swapBuffers();
 	g_gr->finish();
@@ -2227,7 +2245,8 @@ void main()
 	cmdb->bindShaderProgram(prog.get());
 	cmdb->dispatchCompute(1, 1, 1);
 
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 	g_gr->finish();
 
 	// Check result
@@ -2438,7 +2457,8 @@ void main()
 
 	cmdb->dispatchCompute(1, 1, 1);
 
-	cmdb->flush();
+	cmdb->endRecording();
+	GrManager::getSingleton().submit(cmdb.get());
 	g_gr->finish();
 
 	// Check
@@ -2659,7 +2679,8 @@ void main()
 		cmdb->buildAccelerationStructure(tlas.get(), scratchBuff.get(), 0);
 		setAccelerationStructureBarrier(cmdb, tlas, AccelerationStructureUsageBit::kBuild, AccelerationStructureUsageBit::kFragmentRead);
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 	}
 
 	// Draw
@@ -2705,7 +2726,8 @@ void main()
 
 		setTextureBarrier(cmdb, presentTex, TextureUsageBit::kFramebufferWrite, TextureUsageBit::kPresent, TextureSubresourceInfo{});
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -3553,7 +3575,8 @@ void main()
 
 		setTextureBarrier(cmdb, presentTex, TextureUsageBit::kUavComputeWrite, TextureUsageBit::kPresent, TextureSubresourceInfo());
 
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 
 		g_gr->swapBuffers();
 
@@ -3674,9 +3697,14 @@ void main()
 	// Submit
 #if 1
 	FencePtr fence;
-	incrementCmdb->flush({}, &fence);
-	checkCmdb->flush(Array<FencePtr, 1>{fence}, &fence);
-	incrementCmdb2->flush(Array<FencePtr, 1>{fence}, &fence);
+	incrementCmdb->endRecording();
+	GrManager::getSingleton().submit(incrementCmdb.get(), {}, &fence);
+	checkCmdb->endRecording();
+	Fence* pFence = fence.get();
+	GrManager::getSingleton().submit(checkCmdb.get(), {&pFence, 1}, &fence);
+	incrementCmdb2->endRecording();
+	pFence = fence.get();
+	GrManager::getSingleton().submit(incrementCmdb2.get(), {&pFence, 1}, &fence);
 	fence->clientWait(kMaxSecond);
 #else
 	incrementCmdb->flush();

Tests/Gr/GrMeshShaders.cpp

@@ -260,7 +260,8 @@ float3 main(VertOut input) : SV_TARGET0
 			barrier.m_nextUsage = TextureUsageBit::kPresent;
 			cmdb->setPipelineBarrier({&barrier, 1}, {}, {});
 
-			cmdb->flush();
+			cmdb->endRecording();
+			GrManager::getSingleton().submit(cmdb.get());
 
 			gr->swapBuffers();
 

Tests/Gr/GrTextureBuffer.cpp

@@ -62,7 +62,8 @@ void main()
 		cmdb->bindUavBuffer(0, 1, storageBuff.get(), 0, kMaxPtrSize);
 		cmdb->bindShaderProgram(prog.get());
 		cmdb->dispatchCompute(1, 1, 1);
-		cmdb->flush();
+		cmdb->endRecording();
+		GrManager::getSingleton().submit(cmdb.get());
 		gr->finish();
 
 		const Vec4* inData = static_cast<const Vec4*>(storageBuff->map(0, kMaxPtrSize, BufferMapAccessBit::kRead));

Tests/Ui/Ui.cpp

@@ -131,7 +131,8 @@ ANKI_TEST(Ui, Ui)
 			barrier.m_nextUsage = TextureUsageBit::kPresent;
 			cmdb->setPipelineBarrier({&barrier, 1}, {}, {});
 
-			cmdb->flush();
+			cmdb->endRecording();
+			GrManager::getSingleton().submit(cmdb.get());
 
 			gr->swapBuffers();
 			RebarTransientMemoryPool::getSingleton().endFrame();