Vulkan: Some work on the fences and semaphores

include/anki/gr/CommandBuffer.h

@@ -78,6 +78,22 @@ private:
 	PtrSize m_chunkSize = 1024 * 64;
 };
 
+/// Command buffer initialization flags.
+enum class CommandBufferFlag
+{
+	SECOND_LEVEL = 1 << 0,
+
+	/// The command buffer is the frame's first. Or one of the first.
+	FRAME_FIRST = 1 << 1,
+
+	/// The command buffer is the frame's last. Or one of the last.
+	FRAME_LAST = 1 << 2,
+
+	/// It will contain a handfull of commands.
+	SMALL_BATCH = 1 << 3
+};
+ANKI_ENUM_ALLOW_NUMERIC_OPERATIONS(CommandBufferFlag, inline)
+
 /// Command buffer init info.
 class CommandBufferInitInfo
 {

include/anki/gr/vulkan/CommandBufferImpl.h

@@ -83,45 +83,7 @@ public:
 		vkCmdDraw(m_handle, count, instanceCount, first, baseInstance);
 	}
 
-	void flush(CommandBuffer* cmdb);
-
-private:
-	StackAllocator<U8> m_alloc;
-
-	VkCommandBuffer m_handle = VK_NULL_HANDLE;
-	Bool8 m_secondLevel = false;
-	Bool8 m_frameFirst = false;
-	Bool8 m_frameLast = false;
-	Bool8 m_renderedToDefaultFb = false;
-	Bool8 m_flushed = false;
-	Bool8 m_empty = true;
-	Thread::Id m_tid = 0;
-
-	Bool m_firstRpassDrawcall = true; ///< First drawcall in a renderpass.
-	FramebufferPtr m_activeFb;
-
-	/// @name cleanup_references
-	/// @{
-	List<PipelinePtr> m_pplineList;
-	List<FramebufferPtr> m_fbList;
-	List<ResourceGroupPtr> m_rcList;
-/// @}
-
-#if ANKI_ASSERTIONS
-	// Debug stuff
-	Bool8 m_insideRenderPass = false;
-	VkSubpassContents m_subpassContents = VK_SUBPASS_CONTENTS_MAX_ENUM;
-#endif
-
-	/// Some common operations per command.
-	void commandCommon();
-
-	void drawcallCommon();
-
-	Bool insideRenderPass() const
-	{
-		return m_activeFb.isCreated();
-	}
+	void endRecording();
 
 	void setImageBarrier(VkPipelineStageFlags srcStage,
 		VkAccessFlags srcAccess,
@@ -162,6 +124,69 @@ private:
 		vkCmdPipelineBarrier(
 			m_handle, srcStage, dstStage, 0, 0, nullptr, 0, nullptr, 1, &inf);
 	}
+
+	void setImageBarrier(VkPipelineStageFlags srcStage,
+		VkAccessFlags srcAccess,
+		VkImageLayout prevLayout,
+		VkPipelineStageFlags dstStage,
+		VkAccessFlags dstAccess,
+		VkImageLayout newLayout,
+		VkImage img,
+		const VkImageSubresourceRange& range)
+	{
+		ANKI_ASSERT(img);
+		VkImageMemoryBarrier inf = {};
+		inf.sType = VK_STRUCTURE_TYPE_IMAGE_MEMORY_BARRIER;
+		inf.srcAccessMask = srcAccess;
+		inf.dstAccessMask = dstAccess;
+		inf.oldLayout = prevLayout;
+		inf.newLayout = newLayout;
+		inf.srcQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+		inf.dstQueueFamilyIndex = VK_QUEUE_FAMILY_IGNORED;
+		inf.image = img;
+		inf.subresourceRange = range;
+
+		vkCmdPipelineBarrier(
+			m_handle, srcStage, dstStage, 0, 0, nullptr, 0, nullptr, 1, &inf);
+	}
+
+private:
+	StackAllocator<U8> m_alloc;
+
+	VkCommandBuffer m_handle = VK_NULL_HANDLE;
+	Bool8 m_secondLevel = false;
+	Bool8 m_frameFirst = false;
+	Bool8 m_frameLast = false;
+	Bool8 m_renderedToDefaultFb = false;
+	Bool8 m_finalized = false;
+	Bool8 m_empty = true;
+	Thread::Id m_tid = 0;
+
+	Bool m_firstRpassDrawcall = true; ///< First drawcall in a renderpass.
+	FramebufferPtr m_activeFb;
+
+	/// @name cleanup_references
+	/// @{
+	List<PipelinePtr> m_pplineList;
+	List<FramebufferPtr> m_fbList;
+	List<ResourceGroupPtr> m_rcList;
+/// @}
+
+#if ANKI_ASSERTIONS
+	// Debug stuff
+	Bool8 m_insideRenderPass = false;
+	VkSubpassContents m_subpassContents = VK_SUBPASS_CONTENTS_MAX_ENUM;
+#endif
+
+	/// Some common operations per command.
+	void commandCommon();
+
+	void drawcallCommon();
+
+	Bool insideRenderPass() const
+	{
+		return m_activeFb.isCreated();
+	}
 };
 /// @}
 

include/anki/gr/vulkan/ObjectRecycler.h → include/anki/gr/vulkan/CommandBufferInternal.h

@@ -13,14 +13,15 @@ namespace anki
 /// @addtogroup vulkan
 /// @{
 
-class CommandBufferObjectRecycler
+/// Command bufffer object recycler.
+class CommandBufferFactory
 {
 public:
-	CommandBufferObjectRecycler()
+	CommandBufferFactory()
 	{
 	}
 
-	~CommandBufferObjectRecycler();
+	~CommandBufferFactory();
 
 	ANKI_USE_RESULT Error init(GenericMemoryPoolAllocator<U8> alloc,
 		VkDevice dev,
@@ -32,7 +33,7 @@ public:
 	/// Free a command buffer.
 	void deleteCommandBuffer(VkCommandBuffer cmdb, Bool secondLevel);
 
-	void collectGarbage();
+	void collect();
 
 	Bool isCreated() const
 	{
@@ -56,4 +57,4 @@ private:
 };
 /// @}
 
-} // end namespace anki
+} // end namespace anki

include/anki/gr/vulkan/Common.h

@@ -44,12 +44,98 @@ class GrManagerImpl;
 		}                                                                      \
 	} while(0)
 
-/// Debug initialize a vulkan structure
-#if ANKI_DEBUG
-#define ANKI_VK_MEMSET_DBG(struct_) memset(&struct_, 0xCC, sizeof(struct_))
-#else
-#define ANKI_VK_MEMSET_DBG(struct_) ((void)0)
-#endif
+/// A smart wrapper over some internal vulkan objects.
+template<typename TVkHandle, typename TControlBlock>
+class VulkanSmartWrapper
+{
+public:
+	VulkanSmartWrapper()
+	{
+	}
+
+	VulkanSmartWrapper(const VulkanSmartWrapper& b)
+	{
+		if(b.m_cb)
+		{
+			m_handle = b.m_handle;
+			m_cb = b.m_cb;
+			m_cb->m_refcount.fetchAdd(1);
+		}
+	}
+
+	VulkanSmartWrapper(TVkHandle handle, TControlBlock* cb)
+		: m_handle(handle)
+		, m_cb(cb)
+	{
+	}
+
+	~VulkanSmartWrapper()
+	{
+		destroy();
+	}
+
+	VulkanSmartWrapper& operator=(const VulkanSmartWrapper& b)
+	{
+		destroy();
+
+		if(b.m_cb)
+		{
+			m_handle = b.m_handle;
+			m_cb = b.m_cb;
+			m_cb->m_refcount.fetchAdd(1);
+		}
+
+		return *this;
+	}
+
+	const TVkHandle& get() const
+	{
+		ANKI_ASSERT(m_handle);
+		return m_handle;
+	}
+
+	operator const TVkHandle&() const
+	{
+		return get();
+	}
+
+	const TVkHandle* operator&() const
+	{
+		return &get();
+	}
+
+	operator Bool() const
+	{
+		return m_cb != nullptr;
+	}
+
+	void reset()
+	{
+		destroy();
+	}
+
+protected:
+	TVkHandle m_handle = VK_NULL_HANDLE;
+	TControlBlock* m_cb = nullptr;
+
+	void destroy()
+	{
+		if(m_cb)
+		{
+			U count = m_cb->m_refcount.fetchSub(1);
+			if(count == 1)
+			{
+				ANKI_ASSERT(m_handle);
+				m_cb->deleteHandle(m_handle);
+				auto alloc = m_cb->getAllocator();
+				alloc.deleteInstance(m_cb);
+
+				m_handle = VK_NULL_HANDLE;
+				m_cb = nullptr;
+			}
+		}
+	}
+};
 
 ANKI_USE_RESULT inline Bool formatIsDepthStencil(PixelFormat fmt)
 {

include/anki/gr/vulkan/Fence.h

@@ -0,0 +1,107 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#pragma once
+
+#include <anki/gr/vulkan/Common.h>
+
+namespace anki
+{
+
+// Forward
+class FenceFactory;
+
+/// @addtogroup vulkan
+/// @{
+
+/// Fence wrapper over VkFence.
+class Fence : public NonCopyable
+{
+	friend class FenceFactory;
+	friend class FencePtrDeleter;
+
+public:
+	Fence(FenceFactory* f);
+
+	~Fence();
+
+	const VkFence& getHandle() const
+	{
+		ANKI_ASSERT(m_handle);
+		return m_handle;
+	}
+
+	Atomic<U32>& getRefcount()
+	{
+		return m_refcount;
+	}
+
+	GrAllocator<U8> getAllocator() const;
+
+	void wait();
+
+	Bool done() const;
+
+private:
+	VkFence m_handle = VK_NULL_HANDLE;
+	Atomic<U32> m_refcount = {0};
+	FenceFactory* m_factory = nullptr;
+};
+
+/// Deleter for FencePtr.
+class FencePtrDeleter
+{
+public:
+	void operator()(Fence* f);
+};
+
+/// Fence smart pointer.
+using FencePtr = IntrusivePtr<Fence, FencePtrDeleter>;
+
+/// A factory of fences.
+class FenceFactory
+{
+	friend class Fence;
+	friend class FencePtrDeleter;
+
+public:
+	FenceFactory()
+	{
+	}
+
+	~FenceFactory()
+	{
+	}
+
+	void init(GrAllocator<U8> alloc, VkDevice dev)
+	{
+		ANKI_ASSERT(dev);
+		m_alloc = alloc;
+		m_dev = dev;
+	}
+
+	void destroy();
+
+	/// Create a new fence pointer.
+	FencePtr newInstance()
+	{
+		return FencePtr(newFence());
+	}
+
+private:
+	GrAllocator<U8> m_alloc;
+	VkDevice m_dev = VK_NULL_HANDLE;
+	DynamicArray<Fence*> m_fences;
+	U32 m_fenceCount = 0;
+	Mutex m_mtx;
+
+	Fence* newFence();
+	void deleteFence(Fence* fence);
+};
+/// @}
+
+} // end namespace anki
+
+#include <anki/gr/vulkan/Fence.inl.h>

include/anki/gr/vulkan/Fence.inl.h

@@ -0,0 +1,76 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#include <anki/gr/vulkan/Fence.h>
+
+namespace anki
+{
+
+//==============================================================================
+// Fence                                                                       =
+//==============================================================================
+
+//==============================================================================
+inline Fence::Fence(FenceFactory* f)
+	: m_factory(f)
+{
+	ANKI_ASSERT(f);
+	VkFenceCreateInfo ci = {};
+	ci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
+
+	ANKI_VK_CHECKF(vkCreateFence(m_factory->m_dev, &ci, nullptr, &m_handle));
+}
+
+//==============================================================================
+inline Fence::~Fence()
+{
+	if(m_handle)
+	{
+		vkDestroyFence(m_factory->m_dev, m_handle, nullptr);
+	}
+}
+
+//==============================================================================
+inline GrAllocator<U8> Fence::getAllocator() const
+{
+	return m_factory->m_alloc;
+}
+
+//==============================================================================
+inline void Fence::wait()
+{
+	ANKI_ASSERT(m_handle);
+	ANKI_VK_CHECKF(vkWaitForFences(m_factory->m_dev, 1, &m_handle, true, ~0U));
+}
+
+//==============================================================================
+inline Bool Fence::done() const
+{
+	ANKI_ASSERT(m_handle);
+	VkResult status = vkGetFenceStatus(m_factory->m_dev, m_handle);
+	if(status == VK_SUCCESS)
+	{
+		return true;
+	}
+	else if(status != VK_NOT_READY)
+	{
+		ANKI_LOGF("vkGetFenceStatus() failed");
+	}
+
+	return false;
+}
+
+//==============================================================================
+// FencePtrDeleter                                                             =
+//==============================================================================
+
+//==============================================================================
+inline void FencePtrDeleter::operator()(Fence* f)
+{
+	ANKI_ASSERT(f);
+	f->m_factory->deleteFence(f);
+}
+
+} // end namespace anki

include/anki/gr/vulkan/GrManagerImpl.h

@@ -7,7 +7,9 @@
 
 #include <anki/gr/vulkan/Common.h>
 #include <anki/gr/vulkan/GpuMemoryAllocator.h>
-#include <anki/gr/vulkan/ObjectRecycler.h>
+#include <anki/gr/vulkan/CommandBufferInternal.h>
+#include <anki/gr/vulkan/Semaphore.h>
+#include <anki/gr/vulkan/Fence.h>
 #include <anki/util/HashMap.h>
 
 namespace anki
@@ -67,6 +69,9 @@ public:
 		return m_globalPipelineLayout;
 	}
 
+	/// @name object_creation
+	/// @{
+
 	ANKI_USE_RESULT Error allocateDescriptorSet(VkDescriptorSet& out)
 	{
 		out = VK_NULL_HANDLE;
@@ -100,6 +105,17 @@ public:
 	void deleteCommandBuffer(
 		VkCommandBuffer cmdb, Bool secondLevel, Thread::Id tid);
 
+	SemaphorePtr newSemaphore()
+	{
+		return m_semaphores.newInstance();
+	}
+
+	FencePtr newFence()
+	{
+		return m_fences.newInstance();
+	}
+	/// @}
+
 	VkFormat getDefaultSurfaceFormat() const
 	{
 		return m_surfaceFormat;
@@ -134,7 +150,10 @@ public:
 		return m_frame;
 	}
 
-	void flushCommandBuffer(CommandBufferImpl& impl, CommandBufferPtr ptr);
+	void flushCommandBuffer(CommandBufferPtr ptr,
+		SemaphorePtr signalSemaphore = {},
+		WeakArray<SemaphorePtr> waitSemaphores = {},
+		WeakArray<VkPipelineStageFlags> waitPplineStages = {});
 
 	/// @name Memory
 	/// @{
@@ -185,11 +204,11 @@ private:
 		VkImage m_image = VK_NULL_HANDLE;
 		VkImageView m_imageView = VK_NULL_HANDLE;
 
-		VkFence m_presentFence = VK_NULL_HANDLE;
-		VkSemaphore m_acquireSemaphore = VK_NULL_HANDLE;
+		FencePtr m_presentFence;
+		SemaphorePtr m_acquireSemaphore;
 
 		/// The semaphore that the submit that renders to the default FB.
-		VkSemaphore m_renderSemaphore = VK_NULL_HANDLE;
+		SemaphorePtr m_renderSemaphore;
 
 		/// Keep it here for deferred cleanup.
 		List<CommandBufferPtr> m_cmdbsSubmitted;
@@ -245,12 +264,15 @@ private:
 	class PerThread
 	{
 	public:
-		CommandBufferObjectRecycler m_cmdbs;
+		CommandBufferFactory m_cmdbs;
 	};
 
 	HashMap<Thread::Id, PerThread, PerThreadHasher, PerThreadCompare>
 		m_perThread;
 	SpinLock m_perThreadMtx;
+
+	FenceFactory m_fences;
+	SemaphoreFactory m_semaphores;
 /// @}
 
 #if ANKI_ASSERTIONS

include/anki/gr/vulkan/Semaphore.h

@@ -0,0 +1,102 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#pragma once
+
+#include <anki/gr/vulkan/Fence.h>
+
+namespace anki
+{
+
+// Forward
+class SemaphoreFactory;
+
+/// @addtogroup vulkan
+/// @{
+
+/// Simple semaphore wrapper.
+class Semaphore : public NonCopyable
+{
+	friend class SemaphoreFactory;
+	friend class SemaphorePtrDeleter;
+
+public:
+	Semaphore(SemaphoreFactory* f);
+
+	~Semaphore();
+
+	const VkSemaphore& getHandle() const
+	{
+		ANKI_ASSERT(m_handle);
+		return m_handle;
+	}
+
+	GrAllocator<U8> getAllocator() const;
+
+	Atomic<U32>& getRefcount()
+	{
+		return m_refcount;
+	}
+
+	/// Set the associated fence.
+	void setFence(FencePtr fence)
+	{
+		m_fence = fence;
+	}
+
+private:
+	VkSemaphore m_handle = VK_NULL_HANDLE;
+	Atomic<U32> m_refcount = {0};
+	SemaphoreFactory* m_factory = nullptr;
+
+	/// Fence to find out when it's safe to reuse this semaphore.
+	FencePtr m_fence;
+};
+
+/// SemaphorePtr deleter.
+class SemaphorePtrDeleter
+{
+public:
+	void operator()(Semaphore* s);
+};
+
+/// Semaphore smart pointer.
+using SemaphorePtr = IntrusivePtr<Semaphore, SemaphorePtrDeleter>;
+
+/// Factory of semaphores.
+class SemaphoreFactory
+{
+	friend class Semaphore;
+	friend class SemaphorePtrDeleter;
+
+public:
+	void init(GrAllocator<U8> alloc, VkDevice dev)
+	{
+		ANKI_ASSERT(dev);
+		m_alloc = alloc;
+		m_dev = dev;
+	}
+
+	void destroy();
+
+	SemaphorePtr newInstance();
+
+private:
+	GrAllocator<U8> m_alloc;
+	DynamicArray<Semaphore*> m_sems;
+	U32 m_semCount = 0;
+	VkDevice m_dev = VK_NULL_HANDLE;
+	Mutex m_mtx;
+
+	void destroySemaphore(Semaphore* s);
+
+	/// Iterate the semaphores and release the fences.
+	void releaseFences();
+};
+/// @}
+
+} // end namespace anki
+
+#include <anki/gr/vulkan/Semaphore.inl.h>

include/anki/gr/vulkan/Semaphore.inl.h

@@ -0,0 +1,46 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#include <anki/gr/vulkan/Semaphore.h>
+
+namespace anki
+{
+
+//==============================================================================
+// Semaphore                                                                   =
+//==============================================================================
+
+//==============================================================================
+inline Semaphore::Semaphore(SemaphoreFactory* f)
+	: m_factory(f)
+{
+	ANKI_ASSERT(f);
+	VkSemaphoreCreateInfo ci = {};
+	ci.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
+	ANKI_VK_CHECKF(
+		vkCreateSemaphore(m_factory->m_dev, &ci, nullptr, &m_handle));
+}
+
+//==============================================================================
+inline Semaphore::~Semaphore()
+{
+	if(m_handle)
+	{
+		vkDestroySemaphore(m_factory->m_dev, m_handle, nullptr);
+	}
+}
+
+//==============================================================================
+// SemaphorePtrDeleter                                                         =
+//==============================================================================
+
+//==============================================================================
+inline void SemaphorePtrDeleter::operator()(Semaphore* s)
+{
+	ANKI_ASSERT(s);
+	s->m_factory->destroySemaphore(s);
+}
+
+} // end namespace anki

include/anki/gr/vulkan/TextureImpl.h

@@ -7,6 +7,7 @@
 
 #include <anki/gr/vulkan/VulkanObject.h>
 #include <anki/gr/vulkan/GpuMemoryAllocator.h>
+#include <anki/gr/vulkan/Semaphore.h>
 
 namespace anki
 {
@@ -43,9 +44,11 @@ private:
 	SamplerPtr m_sampler;
 	U32 m_memIdx = MAX_U32;
 	GpuMemoryAllocationHandle m_memHandle;
+	SemaphorePtr m_initLayoutSem;
 
-	U8 m_mipCount = 0;
 	TextureType m_type = TextureType::CUBE;
+	U8 m_mipCount = 0;
+	U32 m_layerCount = 0;
 	VkImageAspectFlags m_aspect = 0;
 
 	ANKI_USE_RESULT static VkFormatFeatureFlags calcFeatures(

src/gr/vulkan/CommandBuffer.cpp

@@ -5,6 +5,7 @@
 
 #include <anki/gr/CommandBuffer.h>
 #include <anki/gr/vulkan/CommandBufferImpl.h>
+#include <anki/gr/vulkan/GrManagerImpl.h>
 
 #include <anki/gr/Pipeline.h>
 #include <anki/gr/ResourceGroup.h>
@@ -45,7 +46,11 @@ CommandBufferInitHints CommandBuffer::computeInitHints() const
 //==============================================================================
 void CommandBuffer::flush()
 {
-	m_impl->flush(this);
+	m_impl->endRecording();
+	m_impl->getGrManagerImpl().flushCommandBuffer(CommandBufferPtr(this),
+		SemaphorePtr(),
+		WeakArray<SemaphorePtr>(),
+		WeakArray<VkPipelineStageFlags>());
 }
 
 //==============================================================================
@@ -197,4 +202,4 @@ Bool CommandBuffer::isEmpty() const
 	return false;
 }
 
-} // end namespace anki
+} // end namespace anki

src/gr/vulkan/CommandBufferImpl.cpp

@@ -31,7 +31,7 @@ CommandBufferImpl::~CommandBufferImpl()
 		ANKI_LOGW("Command buffer was empty");
 	}
 
-	if(!m_flushed)
+	if(!m_finalized)
 	{
 		ANKI_LOGW("Command buffer was not flushed");
 	}
@@ -109,7 +109,7 @@ void CommandBufferImpl::commandCommon()
 		&& "Commands must be recorder by the thread this command buffer was "
 		   "created");
 
-	ANKI_ASSERT(!m_flushed);
+	ANKI_ASSERT(!m_finalized);
 	ANKI_ASSERT(m_handle);
 }
 
@@ -201,10 +201,9 @@ void CommandBufferImpl::drawcallCommon()
 }
 
 //==============================================================================
-void CommandBufferImpl::flush(CommandBuffer* cmdb)
+void CommandBufferImpl::endRecording()
 {
-	ANKI_ASSERT(cmdb);
-	ANKI_ASSERT(!m_flushed);
+	ANKI_ASSERT(!m_finalized);
 	ANKI_ASSERT(!m_empty);
 
 	if(m_frameLast)
@@ -224,8 +223,7 @@ void CommandBufferImpl::flush(CommandBuffer* cmdb)
 	}
 
 	ANKI_VK_CHECKF(vkEndCommandBuffer(m_handle));
-	getGrManagerImpl().flushCommandBuffer(*this, CommandBufferPtr(cmdb));
-	m_flushed = true;
+	m_finalized = true;
 }
 
 //==============================================================================
@@ -270,4 +268,4 @@ void CommandBufferImpl::bindResourceGroup(
 	m_rcList.pushBack(m_alloc, rc);
 }
 
-} // end namespace anki
+} // end namespace anki

src/gr/vulkan/ObjectRecycler.cpp → src/gr/vulkan/CommandBufferInternal.cpp

@@ -3,13 +3,13 @@
 // Code licensed under the BSD License.
 // http://www.anki3d.org/LICENSE
 
-#include <anki/gr/vulkan/ObjectRecycler.h>
+#include <anki/gr/vulkan/CommandBufferInternal.h>
 
 namespace anki
 {
 
 //==============================================================================
-CommandBufferObjectRecycler::~CommandBufferObjectRecycler()
+CommandBufferFactory::~CommandBufferFactory()
 {
 	for(CmdbType& type : m_types)
 	{
@@ -28,7 +28,7 @@ CommandBufferObjectRecycler::~CommandBufferObjectRecycler()
 }
 
 //==============================================================================
-Error CommandBufferObjectRecycler::init(
+Error CommandBufferFactory::init(
 	GenericMemoryPoolAllocator<U8> alloc, VkDevice dev, uint32_t queueFamily)
 {
 	m_alloc = alloc;
@@ -45,7 +45,7 @@ Error CommandBufferObjectRecycler::init(
 }
 
 //==============================================================================
-VkCommandBuffer CommandBufferObjectRecycler::newCommandBuffer(Bool secondLevel)
+VkCommandBuffer CommandBufferFactory::newCommandBuffer(Bool secondLevel)
 {
 	ANKI_ASSERT(isCreated());
 
@@ -80,7 +80,7 @@ VkCommandBuffer CommandBufferObjectRecycler::newCommandBuffer(Bool secondLevel)
 }
 
 //==============================================================================
-void CommandBufferObjectRecycler::deleteCommandBuffer(
+void CommandBufferFactory::deleteCommandBuffer(
 	VkCommandBuffer cmdb, Bool secondLevel)
 {
 	ANKI_ASSERT(isCreated());
@@ -100,4 +100,4 @@ void CommandBufferObjectRecycler::deleteCommandBuffer(
 	++type.m_count;
 }
 
-} // end namespace anki
+} // end namespace anki

src/gr/vulkan/Fence.cpp

@@ -0,0 +1,86 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#include <anki/gr/vulkan/Fence.h>
+
+namespace anki
+{
+
+//==============================================================================
+void FenceFactory::destroy()
+{
+	for(U i = 0; i < m_fenceCount; ++i)
+	{
+		m_alloc.deleteInstance(m_fences[i]);
+	}
+
+	m_fences.destroy(m_alloc);
+}
+
+//==============================================================================
+Fence* FenceFactory::newFence()
+{
+	LockGuard<Mutex> lock(m_mtx);
+
+	Fence* out = nullptr;
+	
+	if(m_fenceCount > 0)
+	{
+		U count = m_fenceCount;
+		while(count--)
+		{
+			VkResult status = 
+				vkGetFenceStatus(m_dev, m_fences[count]->getHandle());
+			if(status == VK_SUCCESS)
+			{
+				out = m_fences[count];
+				ANKI_VK_CHECKF(
+					vkResetFences(m_dev, 1, &m_fences[count]->getHandle()));
+			
+				// Pop it	
+				for(U i = count; i < m_fenceCount - 1; ++i)
+				{
+					m_fences[i] = m_fences[i + 1];
+				}
+		
+				--m_fenceCount;
+				
+				break;
+			}
+			else if(status != VK_NOT_READY)
+			{
+				ANKI_LOGF("vkGetFenceStatus() failed");
+			}
+		}
+	}
+	
+	if(out == nullptr)
+	{
+		// Create a new one
+		out = m_alloc.newInstance<Fence>(this);
+	}
+
+	ANKI_ASSERT(out->m_refcount.get() == 0);
+	return out;
+}
+
+//==============================================================================
+void FenceFactory::deleteFence(Fence* fence)
+{
+	ANKI_ASSERT(fence);
+
+	LockGuard<Mutex> lock(m_mtx);
+
+	if(m_fences.getSize() <= m_fenceCount)
+	{
+		// Grow storage
+		m_fences.resize(m_alloc, max<U>(1, m_fences.getSize() * 2));
+	}
+
+	m_fences[m_fenceCount] = fence;
+	++m_fenceCount;
+}
+
+} // end namespace anki

src/gr/vulkan/GrManagerImpl.cpp

@@ -80,7 +80,7 @@ public:
 //==============================================================================
 GrManagerImpl::~GrManagerImpl()
 {
-	// First thing: wait for the GPU
+	// FIRST THING: wait for the GPU
 	if(m_queue)
 	{
 		LockGuard<Mutex> lock(m_queueSubmitMtx);
@@ -127,23 +127,9 @@ GrManagerImpl::~GrManagerImpl()
 			x.m_imageView = VK_NULL_HANDLE;
 		}
 
-		if(x.m_presentFence)
-		{
-			vkDestroyFence(m_device, x.m_presentFence, nullptr);
-			x.m_presentFence = VK_NULL_HANDLE;
-		}
-
-		if(x.m_acquireSemaphore)
-		{
-			vkDestroySemaphore(m_device, x.m_acquireSemaphore, nullptr);
-			x.m_acquireSemaphore = VK_NULL_HANDLE;
-		}
-
-		if(x.m_renderSemaphore)
-		{
-			vkDestroySemaphore(m_device, x.m_renderSemaphore, nullptr);
-			x.m_renderSemaphore = VK_NULL_HANDLE;
-		}
+		x.m_presentFence.reset(nullptr);
+		x.m_acquireSemaphore.reset(nullptr);
+		x.m_renderSemaphore.reset(nullptr);
 
 		x.m_cmdbsSubmitted.destroy(getAllocator());
 	}
@@ -152,6 +138,9 @@ GrManagerImpl::~GrManagerImpl()
 
 	m_gpuMemAllocs.destroy(getAllocator());
 
+	m_semaphores.destroy(); // Destroy before fences
+	m_fences.destroy();
+
 	if(m_swapchain)
 	{
 		vkDestroySwapchainKHR(m_device, m_swapchain, nullptr);
@@ -215,6 +204,8 @@ Error GrManagerImpl::initInternal(const GrManagerInitInfo& init)
 	}
 
 	glslang::InitializeProcess();
+	m_fences.init(getAllocator(), m_device);
+	m_semaphores.init(getAllocator(), m_device);
 
 	return ErrorCode::NONE;
 }
@@ -766,19 +757,14 @@ void GrManagerImpl::beginFrame()
 	PerFrame& frame = m_perFrame[m_frame % MAX_FRAMES_IN_FLIGHT];
 
 	// Create a semaphore
-	VkSemaphoreCreateInfo semaphoreCi = {};
-	semaphoreCi.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
-	ANKI_ASSERT(frame.m_acquireSemaphore == VK_NULL_HANDLE);
-
-	ANKI_VK_CHECKF(vkCreateSemaphore(
-		m_device, &semaphoreCi, nullptr, &frame.m_acquireSemaphore));
+	frame.m_acquireSemaphore = newSemaphore();
 
 	// Get new image
 	uint32_t imageIdx;
 	ANKI_VK_CHECKF(vkAcquireNextImageKHR(m_device,
 		m_swapchain,
 		UINT64_MAX,
-		frame.m_acquireSemaphore,
+		frame.m_acquireSemaphore->getHandle(),
 		0,
 		&imageIdx));
 	ANKI_ASSERT(
@@ -795,14 +781,12 @@ void GrManagerImpl::endFrame()
 	PerFrame& waitFrame = m_perFrame[waitFrameIdx];
 	if(waitFrame.m_presentFence)
 	{
-		// Wait
-		ANKI_VK_CHECKF(vkWaitForFences(
-			m_device, 1, &waitFrame.m_presentFence, true, MAX_U64));
+		waitFrame.m_presentFence->wait();
 	}
 
 	resetFrame(waitFrame);
 
-	if(frame.m_renderSemaphore == VK_NULL_HANDLE)
+	if(!frame.m_renderSemaphore)
 	{
 		ANKI_LOGW("Nobody draw to the default framebuffer");
 	}
@@ -812,8 +796,9 @@ void GrManagerImpl::endFrame()
 	VkPresentInfoKHR present = {};
 	present.sType = VK_STRUCTURE_TYPE_PRESENT_INFO_KHR;
 	present.waitSemaphoreCount = (frame.m_renderSemaphore) ? 1 : 0;
-	present.pWaitSemaphores =
-		(frame.m_renderSemaphore) ? &frame.m_renderSemaphore : nullptr;
+	present.pWaitSemaphores = (frame.m_renderSemaphore)
+		? &frame.m_renderSemaphore->getHandle()
+		: nullptr;
 	present.swapchainCount = 1;
 	present.pSwapchains = &m_swapchain;
 	present.pImageIndices = &imageIdx;
@@ -837,30 +822,10 @@ void GrManagerImpl::endFrame()
 
 //==============================================================================
 void GrManagerImpl::resetFrame(PerFrame& frame)
-{
-	if(frame.m_presentFence)
-	{
-		// Recycle fence
-		ANKI_VK_CHECKF(vkResetFences(m_device, 1, &frame.m_presentFence));
-	}
-	else
-	{
-		VkFenceCreateInfo ci = {};
-		ci.sType = VK_STRUCTURE_TYPE_FENCE_CREATE_INFO;
-		vkCreateFence(m_device, &ci, nullptr, &frame.m_presentFence);
-	}
-
-	if(frame.m_acquireSemaphore)
-	{
-		vkDestroySemaphore(m_device, frame.m_acquireSemaphore, nullptr);
-		frame.m_acquireSemaphore = VK_NULL_HANDLE;
-	}
-
-	if(frame.m_renderSemaphore)
-	{
-		vkDestroySemaphore(m_device, frame.m_renderSemaphore, nullptr);
-		frame.m_renderSemaphore = VK_NULL_HANDLE;
-	}
+{	
+	frame.m_presentFence.reset(nullptr);
+	frame.m_acquireSemaphore.reset(nullptr);
+	frame.m_renderSemaphore.reset(nullptr);
 
 	frame.m_cmdbsSubmitted.destroy(getAllocator());
 }
@@ -918,49 +883,68 @@ void GrManagerImpl::deleteCommandBuffer(
 }
 
 //==============================================================================
-void GrManagerImpl::flushCommandBuffer(
-	CommandBufferImpl& impl, CommandBufferPtr ptr)
+void GrManagerImpl::flushCommandBuffer(CommandBufferPtr cmdb,
+	SemaphorePtr signalSemaphore,
+	WeakArray<SemaphorePtr> waitSemaphores,
+	WeakArray<VkPipelineStageFlags> waitPplineStages)
 {
-	VkPipelineStageFlags waitStage =
-		VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
-
+	CommandBufferImpl& impl = cmdb->getImplementation();
 	VkCommandBuffer handle = impl.getHandle();
 
 	VkSubmitInfo submit = {};
 	submit.sType = VK_STRUCTURE_TYPE_SUBMIT_INFO;
 
-	VkFence fenceToTrigger = VK_NULL_HANDLE;
+	FencePtr fence = newFence();
 	PerFrame& frame = m_perFrame[m_frame % MAX_FRAMES_IN_FLIGHT];
 
+	if(signalSemaphore)
+	{
+		submit.pSignalSemaphores = &signalSemaphore->getHandle();
+		submit.signalSemaphoreCount = 1;
+		signalSemaphore->setFence(fence);
+	}
+
+	Array<VkSemaphore, 16> allWaitSemaphores;
+	Array<VkPipelineStageFlags, 16> allWaitPplineStages;
+	for(U i = 0; i < waitSemaphores.getSize(); ++i)
+	{
+		ANKI_ASSERT(waitSemaphores[i]);
+		allWaitSemaphores[submit.waitSemaphoreCount] =
+			waitSemaphores[i]->getHandle();
+		allWaitPplineStages[submit.waitSemaphoreCount] = waitPplineStages[i];
+		++submit.waitSemaphoreCount;
+	}
+
 	// Do some special stuff for the last command buffer
 	if(impl.renderedToDefaultFramebuffer())
 	{
-		submit.waitSemaphoreCount = 1;
-		submit.pWaitDstStageMask = &waitStage;
-		submit.pWaitSemaphores = &frame.m_acquireSemaphore;
+		allWaitSemaphores[submit.waitSemaphoreCount] =
+			frame.m_acquireSemaphore->getHandle();
+		allWaitPplineStages[submit.waitSemaphoreCount] =
+			VK_PIPELINE_STAGE_COLOR_ATTACHMENT_OUTPUT_BIT;
+		++submit.waitSemaphoreCount;
 
 		// Create the semaphore to signal
-		ANKI_ASSERT(frame.m_renderSemaphore == VK_NULL_HANDLE
+		ANKI_ASSERT(!frame.m_renderSemaphore
 			&& "Only one begin/end render pass is allowed with the default fb");
-		VkSemaphoreCreateInfo ci = {};
-		ci.sType = VK_STRUCTURE_TYPE_SEMAPHORE_CREATE_INFO;
-		ANKI_VK_CHECKF(vkCreateSemaphore(
-			m_device, &ci, nullptr, &frame.m_renderSemaphore));
+		frame.m_renderSemaphore = newSemaphore();
 
 		submit.signalSemaphoreCount = 1;
-		submit.pSignalSemaphores = &frame.m_renderSemaphore;
+		submit.pSignalSemaphores = &frame.m_renderSemaphore->getHandle();
 
-		fenceToTrigger = frame.m_presentFence;
-		ANKI_ASSERT(fenceToTrigger);
+		frame.m_presentFence = fence;
 	}
 
+	submit.pWaitSemaphores = &allWaitSemaphores[0];
+	submit.pWaitDstStageMask = &allWaitPplineStages[0];
+
 	submit.commandBufferCount = 1;
 	submit.pCommandBuffers = &handle;
 
 	// Lock to submit
 	LockGuard<Mutex> lock(m_queueSubmitMtx);
 
-	frame.m_cmdbsSubmitted.pushBack(getAllocator(), ptr);
+	frame.m_cmdbsSubmitted.pushBack(getAllocator(), cmdb);
 
 #if ANKI_ASSERTIONS
 	if(impl.isTheFirstFramebufferOfTheFrame())
@@ -976,7 +960,7 @@ void GrManagerImpl::flushCommandBuffer(
 	}
 #endif
 
-	ANKI_VK_CHECKF(vkQueueSubmit(m_queue, 1, &submit, fenceToTrigger));
+	ANKI_VK_CHECKF(vkQueueSubmit(m_queue, 1, &submit, fence->getHandle()));
 }
 
 } // end namespace anki

src/gr/vulkan/Semaphore.cpp

@@ -0,0 +1,97 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#include <anki/gr/vulkan/Semaphore.h>
+
+namespace anki
+{
+
+//==============================================================================
+void SemaphoreFactory::destroy()
+{
+	for(U i = 0; i < m_semCount; ++i)
+	{
+		m_alloc.deleteInstance(m_sems[i]);
+	}
+
+	m_sems.destroy(m_alloc);
+}
+
+//==============================================================================
+void SemaphoreFactory::releaseFences()
+{
+	U count = m_semCount;
+	while(count--)
+	{
+		Semaphore& sem = *m_sems[count];
+		if(sem.m_fence && sem.m_fence->done())
+		{
+			sem.m_fence.reset(nullptr);
+		}
+	}
+}
+
+//==============================================================================
+SemaphorePtr SemaphoreFactory::newInstance()
+{
+	LockGuard<Mutex> lock(m_mtx);
+
+	Semaphore* out = nullptr;
+
+	if(m_semCount > 0)
+	{
+		releaseFences();
+
+		U count = m_semCount;
+		while(count--)
+		{
+			if(!m_sems[count]->m_fence)
+			{
+				out = m_sems[count];
+		
+				// Pop it		
+				for(U i = count; i < m_semCount - 1; ++i)
+				{
+					m_sems[i] = m_sems[i + 1];
+				}
+				
+				--m_semCount;
+				
+				break;
+			}
+		}
+	}
+
+	if(out == nullptr)
+	{
+		// Create a new one
+		out = m_alloc.newInstance<Semaphore>(this);
+	}
+
+	ANKI_ASSERT(out->m_refcount.get() == 0);
+	return SemaphorePtr(out);
+}
+
+//==============================================================================
+void SemaphoreFactory::destroySemaphore(Semaphore* s)
+{
+	ANKI_ASSERT(s);
+	ANKI_ASSERT(s->m_refcount.get() == 0);
+
+	LockGuard<Mutex> lock(m_mtx);
+
+	if(m_sems.getSize() <= m_semCount)
+	{
+		// Grow storage
+		m_sems.resize(m_alloc, max<U>(1, m_sems.getSize() * 2));
+	}
+
+	m_sems[m_semCount] = s;
+	++m_semCount;
+	
+	releaseFences();
+}
+
+} // end namespace anki

src/gr/vulkan/TextureImpl.cpp

@@ -7,6 +7,8 @@
 #include <anki/gr/Sampler.h>
 #include <anki/gr/GrManager.h>
 #include <anki/gr/vulkan/GrManagerImpl.h>
+#include <anki/gr/CommandBuffer.h>
+#include <anki/gr/vulkan/CommandBufferImpl.h>
 
 namespace anki
 {
@@ -35,6 +37,11 @@ TextureImpl::TextureImpl(GrManager* manager)
 //==============================================================================
 TextureImpl::~TextureImpl()
 {
+	if(m_viewHandle)
+	{
+		vkDestroyImageView(getDevice(), m_viewHandle, nullptr);
+	}
+
 	if(m_imageHandle)
 	{
 		vkDestroyImage(getDevice(), m_imageHandle, nullptr);
@@ -151,6 +158,41 @@ Error TextureImpl::init(const TextureInitInfo& init)
 	ANKI_CHECK(initImage(ctx));
 	ANKI_CHECK(initView(ctx));
 
+	// Change the image layout
+	VkImageLayout newLayout;
+	CommandBufferInitInfo cmdbinit;
+	CommandBufferPtr cmdb = getGrManager().newInstance<CommandBuffer>(cmdbinit);
+	if(init.m_framebufferAttachment)
+	{
+		if(formatIsDepthStencil(init.m_format))
+		{
+			newLayout = VK_IMAGE_LAYOUT_DEPTH_STENCIL_ATTACHMENT_OPTIMAL;
+		}
+		else
+		{
+			newLayout = VK_IMAGE_LAYOUT_COLOR_ATTACHMENT_OPTIMAL;
+		}
+	}
+	else
+	{
+		newLayout = VK_IMAGE_LAYOUT_SHADER_READ_ONLY_OPTIMAL;
+	}
+
+	VkImageSubresourceRange range;
+	range.aspectMask = m_aspect;
+	range.baseArrayLayer = 0;
+	range.baseMipLevel = 0;
+	range.layerCount = m_layerCount;
+	range.levelCount = m_mipCount;
+
+	cmdb->getImplementation().setImageBarrier(
+		0, 0, VK_IMAGE_LAYOUT_UNDEFINED, 0, 0, newLayout, m_imageHandle, range);
+
+	m_initLayoutSem = getGrManagerImpl().newSemaphore();
+
+	cmdb->getImplementation().endRecording();
+	getGrManagerImpl().flushCommandBuffer(cmdb, m_initLayoutSem);
+
 	return ErrorCode::NONE;
 }
 
@@ -228,6 +270,8 @@ Error TextureImpl::initImage(CreateContext& ctx)
 		ANKI_ASSERT(0);
 	}
 
+	m_layerCount = ci.arrayLayers;
+
 	ci.mipLevels = init.m_mipmapsCount;
 	ANKI_ASSERT(ci.mipLevels > 0);
 	m_mipCount = init.m_mipmapsCount;
@@ -308,4 +352,4 @@ Error TextureImpl::initView(CreateContext& ctx)
 	return ErrorCode::NONE;
 }
 
-} // end namespace anki
+} // end namespace anki