Vulkan: Replace the GPU mem manager with the new one

src/anki/gr/vulkan/BufferImpl.cpp

@@ -20,7 +20,7 @@ BufferImpl::~BufferImpl()
 
 	if(m_memHandle)
 	{
-		getGrManagerImpl().getGpuMemoryAllocator().freeMemory(m_memHandle);
+		getGrManagerImpl().getGpuMemoryManager().freeMemory(m_memHandle);
 	}
 }
 
@@ -54,14 +54,14 @@ Error BufferImpl::init(PtrSize size, BufferUsageBit usage, BufferMapAccessBit ac
 		// Only write
 
 		// Device & host but not coherent
-		memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(req.memoryTypeBits,
+		memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(req.memoryTypeBits,
 			VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT,
 			VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
 
 		// Fallback: host and not coherent
 		if(memIdx == MAX_U32)
 		{
-			memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+			memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 				req.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, VK_MEMORY_PROPERTY_HOST_COHERENT_BIT);
 		}
 
@@ -69,7 +69,7 @@ Error BufferImpl::init(PtrSize size, BufferUsageBit usage, BufferMapAccessBit ac
 		if(memIdx == MAX_U32)
 		{
 			ANKI_LOGW("Vulkan: Using a fallback mode for write-only buffer");
-			memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+			memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 				req.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, 0);
 		}
 	}
@@ -78,7 +78,7 @@ Error BufferImpl::init(PtrSize size, BufferUsageBit usage, BufferMapAccessBit ac
 		// Read or read/write
 
 		// Cached & coherent
-		memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(req.memoryTypeBits,
+		memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(req.memoryTypeBits,
 			VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT
 				| VK_MEMORY_PROPERTY_HOST_COHERENT_BIT,
 			0);
@@ -86,7 +86,7 @@ Error BufferImpl::init(PtrSize size, BufferUsageBit usage, BufferMapAccessBit ac
 		// Fallback: Just cached
 		if(memIdx == MAX_U32)
 		{
-			memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+			memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 				req.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT | VK_MEMORY_PROPERTY_HOST_CACHED_BIT, 0);
 		}
 
@@ -94,7 +94,7 @@ Error BufferImpl::init(PtrSize size, BufferUsageBit usage, BufferMapAccessBit ac
 		if(memIdx == MAX_U32)
 		{
 			ANKI_LOGW("Vulkan: Using a fallback mode for read/write buffer");
-			memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+			memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 				req.memoryTypeBits, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT, 0);
 		}
 	}
@@ -105,13 +105,13 @@ Error BufferImpl::init(PtrSize size, BufferUsageBit usage, BufferMapAccessBit ac
 		ANKI_ASSERT(access == BufferMapAccessBit::NONE);
 
 		// Device only
-		memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+		memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 			req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
 
 		// Fallback: Device with anything else
 		if(memIdx == MAX_U32)
 		{
-			memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+			memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 				req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0);
 		}
 	}
@@ -122,7 +122,7 @@ Error BufferImpl::init(PtrSize size, BufferUsageBit usage, BufferMapAccessBit ac
 	m_memoryFlags = props.memoryTypes[memIdx].propertyFlags;
 
 	// Allocate
-	getGrManagerImpl().getGpuMemoryAllocator().allocateMemory(memIdx, req.size, req.alignment, true, m_memHandle);
+	getGrManagerImpl().getGpuMemoryManager().allocateMemory(memIdx, req.size, req.alignment, true, m_memHandle);
 
 	// Bind mem to buffer
 	ANKI_VK_CHECK(vkBindBufferMemory(getDevice(), m_handle, m_memHandle.m_memory, m_memHandle.m_offset));
@@ -141,7 +141,7 @@ void* BufferImpl::map(PtrSize offset, PtrSize range, BufferMapAccessBit access)
 	ANKI_ASSERT(!m_mapped);
 	ANKI_ASSERT(offset + range <= m_size);
 
-	void* ptr = getGrManagerImpl().getGpuMemoryAllocator().getMappedAddress(m_memHandle);
+	void* ptr = getGrManagerImpl().getGpuMemoryManager().getMappedAddress(m_memHandle);
 	ANKI_ASSERT(ptr);
 
 #if ANKI_ASSERTIONS

src/anki/gr/vulkan/BufferImpl.h

@@ -6,7 +6,7 @@
 #pragma once
 
 #include <anki/gr/vulkan/VulkanObject.h>
-#include <anki/gr/vulkan/GpuMemoryAllocator.h>
+#include <anki/gr/vulkan/GpuMemoryManager.h>
 
 namespace anki
 {
@@ -66,7 +66,7 @@ public:
 
 private:
 	VkBuffer m_handle = VK_NULL_HANDLE;
-	GpuMemoryAllocationHandle m_memHandle;
+	GpuMemoryHandle m_memHandle;
 	BufferMapAccessBit m_access = BufferMapAccessBit::NONE;
 	U32 m_size = 0;
 	VkMemoryPropertyFlags m_memoryFlags = 0;

src/anki/gr/vulkan/CommandBufferImpl.cpp

@@ -765,12 +765,12 @@ void CommandBufferImpl::flushWriteQueryResults()
 				return a.m_pool < b.m_pool;
 			}
 
-			if(a.m_buffer != a.m_buffer)
+			if(a.m_buffer != b.m_buffer)
 			{
 				return a.m_buffer < b.m_buffer;
 			}
 
-			if(a.m_offset != a.m_offset)
+			if(a.m_offset != b.m_offset)
 			{
 				return a.m_offset < b.m_offset;
 			}

src/anki/gr/vulkan/GpuMemoryAllocator.cpp

@@ -1,445 +0,0 @@
-// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
-// All rights reserved.
-// Code licensed under the BSD License.
-// http://www.anki3d.org/LICENSE
-
-#include <anki/gr/vulkan/GpuMemoryAllocator.h>
-#include <anki/util/BitSet.h>
-
-namespace anki
-{
-
-/// Define this type just to show what is what.
-using Page = U32;
-
-/// The size of the page. This is the minumum allocation size as well.
-const U PAGE_SIZE = 8 * 1024;
-
-/// Max number of sub allocations (aka slots) per chunk.
-const U MAX_SLOTS_PER_CHUNK = 128;
-
-#define ANKI_CHECK_HANDLE(handle_) ANKI_ASSERT(handle_.m_memory&& handle_.m_chunk&& handle_.m_chunk->m_class)
-
-class GpuMemoryAllocatorChunk : public IntrusiveListEnabled<GpuMemoryAllocatorChunk>
-{
-public:
-	/// GPU mem.
-	VkDeviceMemory m_mem = VK_NULL_HANDLE;
-
-	/// The in use slots mask.
-	BitSet<MAX_SLOTS_PER_CHUNK, U8> m_inUseSlots = {false};
-
-	/// The number of in-use slots.
-	U32 m_inUseSlotCount = 0;
-
-	/// The owner.
-	GpuMemoryAllocatorClass* m_class = nullptr;
-
-	/// It points to a CPU address if mapped.
-	U8* m_mappedAddress = nullptr;
-
-	/// Protect the m_mappedAddress. It's a SpinLock because we don't want a whole mutex for every
-	/// GpuMemoryAllocatorChunk.
-	SpinLock m_mtx;
-
-	/// If true it contains linear resources.
-	Bool8 m_linearResources = false;
-};
-
-class GpuMemoryAllocatorClass
-{
-public:
-	/// The active chunks.
-	IntrusiveList<GpuMemoryAllocatorChunk> m_inUseChunks;
-
-	/// The empty chunks.
-	IntrusiveList<GpuMemoryAllocatorChunk> m_unusedChunks;
-
-	/// The size of each chunk.
-	Page m_chunkPages = 0;
-
-	/// The max slot size for this class.
-	U32 m_maxSlotSize = 0;
-
-	/// The number of slots for a single chunk.
-	U32 m_slotsPerChunkCount = 0;
-
-	Mutex m_mtx;
-};
-
-GpuMemoryAllocatorMemType::~GpuMemoryAllocatorMemType()
-{
-	for(Class& cl : m_classes)
-	{
-		if(!cl.m_inUseChunks.isEmpty())
-		{
-			ANKI_LOGW("Forgot to deallocate GPU memory");
-
-			while(!cl.m_inUseChunks.isEmpty())
-			{
-				Chunk* chunk = &cl.m_inUseChunks.getBack();
-				cl.m_inUseChunks.popBack();
-
-				// Unmap
-				if(chunk->m_mappedAddress)
-				{
-					vkUnmapMemory(m_dev, chunk->m_mem);
-				}
-
-				vkFreeMemory(m_dev, chunk->m_mem, nullptr);
-				m_alloc.deleteInstance(chunk);
-			}
-		}
-
-		while(!cl.m_unusedChunks.isEmpty())
-		{
-			Chunk* chunk = &cl.m_unusedChunks.getBack();
-			cl.m_unusedChunks.popBack();
-
-			// Unmap
-			if(chunk->m_mappedAddress)
-			{
-				vkUnmapMemory(m_dev, chunk->m_mem);
-			}
-
-			vkFreeMemory(m_dev, chunk->m_mem, nullptr);
-			m_alloc.deleteInstance(chunk);
-		}
-	}
-
-	m_classes.destroy(m_alloc);
-}
-
-void GpuMemoryAllocatorMemType::init(GenericMemoryPoolAllocator<U8> alloc, VkDevice dev, U memoryTypeIdx)
-{
-	m_alloc = alloc;
-	m_dev = dev;
-	m_memIdx = memoryTypeIdx;
-
-	//
-	// Initialize the classes
-	//
-	const U CLASS_COUNT = 6;
-	m_classes.create(m_alloc, CLASS_COUNT);
-	static const CString MESSAGE = "VK: Creating memory class. "
-								   "Chunk size: %u, maxSlotSize: %u, allocsPerChunk: %u";
-
-#define ANKI_PRINT_MSG()                                                                                               \
-	if(memoryTypeIdx == 0)                                                                                             \
-	{                                                                                                                  \
-		ANKI_LOGI(                                                                                                     \
-			&MESSAGE[0], c.m_chunkPages* PAGE_SIZE, c.m_maxSlotSize, c.m_chunkPages* PAGE_SIZE / c.m_maxSlotSize)      \
-	}
-
-	// 1st class. From (0, 256] bytes
-	{
-		Class& c = m_classes[0];
-		c.m_chunkPages = 2;
-		c.m_maxSlotSize = 256;
-		ANKI_PRINT_MSG();
-	}
-
-	// 2st class. From (256, 4K] bytes
-	{
-		Class& c = m_classes[1];
-		c.m_chunkPages = 32;
-		c.m_maxSlotSize = 4 * 1024;
-		ANKI_PRINT_MSG();
-	}
-
-	// 3rd class. From (4K, 128K] bytes
-	{
-		Class& c = m_classes[2];
-		c.m_chunkPages = 1024;
-		c.m_maxSlotSize = 128 * 1024;
-		ANKI_PRINT_MSG();
-	}
-
-	// 4rth class. From (128K, 1M] bytes
-	{
-		Class& c = m_classes[3];
-		c.m_chunkPages = 4 * 1024;
-		c.m_maxSlotSize = 1 * 1024 * 1024;
-		ANKI_PRINT_MSG();
-	}
-
-	// 5th class. From (1M, 10M] bytes
-	{
-		Class& c = m_classes[4];
-		c.m_chunkPages = 10 * 1024;
-		c.m_maxSlotSize = 10 * 1024 * 1024;
-		ANKI_PRINT_MSG();
-	}
-
-	// 6th class. From (10M, 80M] bytes
-	{
-		Class& c = m_classes[5];
-		c.m_chunkPages = 20 * 1024;
-		c.m_maxSlotSize = 80 * 1024 * 1024;
-		ANKI_PRINT_MSG();
-	}
-
-	for(Class& c : m_classes)
-	{
-		ANKI_ASSERT(((c.m_chunkPages * PAGE_SIZE) % c.m_maxSlotSize) == 0);
-		c.m_slotsPerChunkCount = (c.m_chunkPages * PAGE_SIZE) / c.m_maxSlotSize;
-		ANKI_ASSERT(c.m_slotsPerChunkCount <= MAX_SLOTS_PER_CHUNK);
-	}
-}
-
-GpuMemoryAllocatorMemType::Class* GpuMemoryAllocatorMemType::findClass(PtrSize size, U32 alignment)
-{
-	ANKI_ASSERT(size > 0 && alignment > 0);
-
-	PtrSize lowLimit = 0;
-	Class* it = m_classes.getBegin();
-	const Class* end = m_classes.getEnd();
-
-	while(it != end)
-	{
-		PtrSize highLimit = it->m_maxSlotSize;
-
-		if(size > lowLimit && size <= highLimit)
-		{
-			if(alignment <= highLimit)
-			{
-				// Found the class
-				return it;
-			}
-			else
-			{
-				// The class found doesn't have the proper alignment. Need to
-				// go higher
-
-				while(++it != end)
-				{
-					if(alignment <= it->m_maxSlotSize)
-					{
-						// Now found something
-						return it;
-					}
-				}
-			}
-		}
-
-		lowLimit = highLimit;
-		++it;
-	}
-
-	return nullptr;
-}
-
-GpuMemoryAllocatorMemType::Chunk* GpuMemoryAllocatorMemType::findChunkWithUnusedSlot(Class& cl, Bool linearResource)
-{
-	auto it = cl.m_inUseChunks.getBegin();
-	const auto end = cl.m_inUseChunks.getEnd();
-	while(it != end)
-	{
-		if(it->m_inUseSlotCount < cl.m_slotsPerChunkCount && it->m_linearResources == linearResource)
-		{
-			return &(*it);
-		}
-
-		++it;
-	}
-
-	return nullptr;
-}
-
-GpuMemoryAllocatorMemType::Chunk& GpuMemoryAllocatorMemType::createChunk(Class& cl, Bool linearResources)
-{
-	Chunk* chunk = nullptr;
-
-	if(cl.m_unusedChunks.isEmpty())
-	{
-		// Create new
-
-		chunk = m_alloc.newInstance<Chunk>();
-		chunk->m_class = &cl;
-
-		VkMemoryAllocateInfo ci = {};
-		ci.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
-		ci.allocationSize = cl.m_chunkPages * PAGE_SIZE;
-		ci.memoryTypeIndex = m_memIdx;
-		ANKI_VK_CHECKF(vkAllocateMemory(m_dev, &ci, nullptr, &chunk->m_mem));
-	}
-	else
-	{
-		// Recycle
-
-		chunk = &cl.m_unusedChunks.getFront();
-		cl.m_unusedChunks.popFront();
-	}
-
-	chunk->m_linearResources = linearResources;
-	cl.m_inUseChunks.pushBack(chunk);
-
-	ANKI_ASSERT(chunk->m_mem && chunk->m_class == &cl && chunk->m_inUseSlotCount == 0 && !chunk->m_inUseSlots.getAny());
-
-	return *chunk;
-}
-
-void GpuMemoryAllocatorMemType::allocate(
-	PtrSize size, U alignment, Bool linearResource, GpuMemoryAllocationHandle& handle)
-{
-	handle.m_memory = VK_NULL_HANDLE;
-
-	// Find the class for the given size
-	Class* cl = findClass(size, alignment);
-	ANKI_ASSERT(cl && "Didn't found a suitable class");
-
-	LockGuard<Mutex> lock(cl->m_mtx);
-	Chunk* chunk = findChunkWithUnusedSlot(*cl, linearResource);
-
-	// Create a new chunk if needed
-	if(chunk == nullptr)
-	{
-		chunk = &createChunk(*cl, linearResource);
-	}
-
-	// Allocate from chunk
-	U bitCount = cl->m_slotsPerChunkCount;
-	for(U i = 0; i < bitCount; ++i)
-	{
-		if(!chunk->m_inUseSlots.get(i))
-		{
-			// Found an empty slot, allocate from it
-			chunk->m_inUseSlots.set(i);
-			++chunk->m_inUseSlotCount;
-
-			handle.m_memory = chunk->m_mem;
-			handle.m_offset = i * cl->m_maxSlotSize;
-			handle.m_chunk = chunk;
-
-			break;
-		}
-	}
-
-	ANKI_ASSERT(handle.m_memory && handle.m_chunk);
-	ANKI_ASSERT(isAligned(alignment, handle.m_offset));
-}
-
-void GpuMemoryAllocatorMemType::destroyChunk(Class& cl, Chunk& chunk)
-{
-	// Push the chunk to unused area
-	cl.m_inUseChunks.erase(&chunk);
-	cl.m_unusedChunks.pushBack(&chunk);
-
-	// Unmap. This may free some VA
-	if(chunk.m_mappedAddress)
-	{
-		vkUnmapMemory(m_dev, chunk.m_mem);
-		chunk.m_mappedAddress = nullptr;
-	}
-}
-
-void GpuMemoryAllocatorMemType::free(GpuMemoryAllocationHandle& handle)
-{
-	ANKI_CHECK_HANDLE(handle);
-
-	Chunk& chunk = *handle.m_chunk;
-	Class& cl = *chunk.m_class;
-
-	LockGuard<Mutex> lock(cl.m_mtx);
-	U slotIdx = handle.m_offset / cl.m_maxSlotSize;
-
-	ANKI_ASSERT(chunk.m_inUseSlots.get(slotIdx));
-	ANKI_ASSERT(chunk.m_inUseSlotCount > 0);
-	chunk.m_inUseSlots.unset(slotIdx);
-	--chunk.m_inUseSlotCount;
-
-	if(chunk.m_inUseSlotCount == 0)
-	{
-		destroyChunk(cl, chunk);
-	}
-
-	handle = {};
-}
-
-void* GpuMemoryAllocatorMemType::getMappedAddress(GpuMemoryAllocationHandle& handle)
-{
-	ANKI_CHECK_HANDLE(handle);
-
-	Chunk& chunk = *handle.m_chunk;
-	U8* out = nullptr;
-
-	{
-		LockGuard<SpinLock> lock(chunk.m_mtx);
-		if(chunk.m_mappedAddress)
-		{
-			out = chunk.m_mappedAddress;
-		}
-		else
-		{
-			ANKI_VK_CHECKF(vkMapMemory(
-				m_dev, chunk.m_mem, 0, chunk.m_class->m_chunkPages * PAGE_SIZE, 0, reinterpret_cast<void**>(&out)));
-
-			chunk.m_mappedAddress = out;
-		}
-	}
-
-	ANKI_ASSERT(out);
-	return static_cast<void*>(out + handle.m_offset);
-}
-
-GpuMemoryAllocator::~GpuMemoryAllocator()
-{
-}
-
-void GpuMemoryAllocator::init(VkPhysicalDevice pdev, VkDevice dev, GrAllocator<U8> alloc)
-{
-	vkGetPhysicalDeviceMemoryProperties(pdev, &m_memoryProperties);
-
-	// Create the high level allocators
-	m_gpuAllocs.create(alloc, m_memoryProperties.memoryTypeCount);
-	U idx = 0;
-	for(GpuMemoryAllocatorMemType& a : m_gpuAllocs)
-	{
-		a.init(alloc, dev, idx++);
-	}
-
-	m_dev = dev;
-	m_alloc = alloc;
-}
-
-void GpuMemoryAllocator::destroy()
-{
-	m_gpuAllocs.destroy(m_alloc);
-}
-
-U GpuMemoryAllocator::findMemoryType(
-	U resourceMemTypeBits, VkMemoryPropertyFlags preferFlags, VkMemoryPropertyFlags avoidFlags) const
-{
-	U preferedHigh = MAX_U32;
-	U preferedMed = MAX_U32;
-
-	// Iterate all mem types
-	for(U i = 0; i < m_memoryProperties.memoryTypeCount; i++)
-	{
-		if(resourceMemTypeBits & (1u << i))
-		{
-			VkMemoryPropertyFlags flags = m_memoryProperties.memoryTypes[i].propertyFlags;
-
-			if((flags & preferFlags) == preferFlags)
-			{
-				preferedMed = i;
-
-				if((flags & avoidFlags) != avoidFlags)
-				{
-					preferedHigh = i;
-				}
-			}
-		}
-	}
-
-	if(preferedHigh < MAX_U32)
-	{
-		return preferedHigh;
-	}
-	else
-	{
-		return preferedMed;
-	}
-}
-
-} // end namespace anki

src/anki/gr/vulkan/GpuMemoryAllocator.h

@@ -1,134 +0,0 @@
-// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
-// All rights reserved.
-// Code licensed under the BSD License.
-// http://www.anki3d.org/LICENSE
-
-#pragma once
-
-#include <anki/gr/vulkan/Common.h>
-#include <anki/util/List.h>
-
-namespace anki
-{
-
-// Forward
-class GpuMemoryAllocatorChunk;
-class GpuMemoryAllocatorClass;
-
-/// @addtorgoup vulkan
-/// @{
-
-/// The handle that is returned from GpuMemoryAllocator's allocations.
-class GpuMemoryAllocationHandle
-{
-	friend class GpuMemoryAllocatorMemType;
-
-public:
-	VkDeviceMemory m_memory = VK_NULL_HANDLE;
-	PtrSize m_offset = MAX_PTR_SIZE;
-	U32 m_memoryTypeIndex = MAX_U32;
-
-	Bool isEmpty() const
-	{
-		return m_memory == VK_NULL_HANDLE;
-	}
-
-	operator bool() const
-	{
-		return m_memory != VK_NULL_HANDLE;
-	}
-
-private:
-	GpuMemoryAllocatorChunk* m_chunk = nullptr;
-};
-
-/// Dynamic GPU memory allocator for a specific type.
-class GpuMemoryAllocatorMemType
-{
-public:
-	GpuMemoryAllocatorMemType()
-	{
-	}
-
-	~GpuMemoryAllocatorMemType();
-
-	void init(GenericMemoryPoolAllocator<U8> alloc, VkDevice dev, U memoryTypeIdx);
-
-	/// Allocate GPU memory.
-	void allocate(PtrSize size, U alignment, Bool linearResource, GpuMemoryAllocationHandle& handle);
-
-	/// Free allocated memory.
-	void free(GpuMemoryAllocationHandle& handle);
-
-	/// Get CPU visible address.
-	void* getMappedAddress(GpuMemoryAllocationHandle& handle);
-
-private:
-	using Class = GpuMemoryAllocatorClass;
-	using Chunk = GpuMemoryAllocatorChunk;
-
-	GenericMemoryPoolAllocator<U8> m_alloc;
-
-	VkDevice m_dev = VK_NULL_HANDLE;
-	U32 m_memIdx;
-
-	/// The memory classes.
-	DynamicArray<Class> m_classes;
-
-	Class* findClass(PtrSize size, U32 alignment);
-
-	Chunk* findChunkWithUnusedSlot(Class& cl, Bool linearResource);
-
-	/// Create or recycle chunk.
-	Chunk& createChunk(Class& cl, Bool linearResources);
-
-	/// Park the chunk.
-	void destroyChunk(Class& cl, Chunk& chunk);
-};
-
-/// Dynamic GPU memory allocator for all types.
-class GpuMemoryAllocator : public NonCopyable
-{
-public:
-	GpuMemoryAllocator()
-	{
-	}
-
-	~GpuMemoryAllocator();
-
-	void init(VkPhysicalDevice pdev, VkDevice dev, GrAllocator<U8> alloc);
-
-	void destroy();
-
-	/// Allocate memory.
-	void allocateMemory(U memTypeIdx, PtrSize size, U alignment, Bool linearResource, GpuMemoryAllocationHandle& handle)
-	{
-		m_gpuAllocs[memTypeIdx].allocate(size, alignment, linearResource, handle);
-		handle.m_memoryTypeIndex = memTypeIdx;
-	}
-
-	/// Free memory.
-	void freeMemory(GpuMemoryAllocationHandle& handle)
-	{
-		m_gpuAllocs[handle.m_memoryTypeIndex].free(handle);
-	}
-
-	/// Map memory.
-	ANKI_USE_RESULT void* getMappedAddress(GpuMemoryAllocationHandle& handle)
-	{
-		return m_gpuAllocs[handle.m_memoryTypeIndex].getMappedAddress(handle);
-	}
-
-	/// Find a suitable memory type.
-	U findMemoryType(U resourceMemTypeBits, VkMemoryPropertyFlags preferFlags, VkMemoryPropertyFlags avoidFlags) const;
-
-private:
-	/// One for each mem type.
-	DynamicArray<GpuMemoryAllocatorMemType> m_gpuAllocs;
-	VkPhysicalDeviceMemoryProperties m_memoryProperties;
-	VkDevice m_dev;
-	GrAllocator<U8> m_alloc;
-};
-/// @}
-
-} // end namespace anki

src/anki/gr/vulkan/GpuMemoryManager.cpp

@@ -18,17 +18,17 @@ public:
 	PtrSize m_chunkSize;
 };
 
-static unsigned long long int operator"" _B(unsigned long long int x)
+static constexpr unsigned long long int operator""_B(unsigned long long int x)
 {
 	return x;
 }
 
-static unsigned long long int operator"" _KB(unsigned long long int x)
+static constexpr unsigned long long int operator""_KB(unsigned long long int x)
 {
 	return x * 1024;
 }
 
-static unsigned long long int operator"" _MB(unsigned long long int x)
+static constexpr unsigned long long int operator""_MB(unsigned long long int x)
 {
 	return x * 1024 * 1024;
 }
@@ -41,26 +41,79 @@ static const Array<ClassInf, CLASS_COUNT> CLASSES = {{{256_B, 16_KB},
 	{64_MB, 256_MB},
 	{128_MB, 256_MB}}};
 
-class GpuMemoryManager::Memory : public ClassAllocatorMemory, public IntrusiveListEnabled<GpuMemoryManager::Memory>
+class GpuMemoryManager::Memory final : public ClassAllocatorMemory,
+									   public IntrusiveListEnabled<GpuMemoryManager::Memory>
 {
 public:
 	VkDeviceMemory m_handle = VK_NULL_HANDLE;
+
+	void* m_mappedAddress = nullptr;
+	SpinLock m_mtx;
+
 	U8 m_classIdx = MAX_U8;
 };
 
-class GpuMemoryManager::Interface : public ClassAllocatorInterface
+class GpuMemoryManager::Interface final : public ClassAllocatorInterface
 {
 public:
-	uint32_t m_memTypeIdx = MAX_U32;
-	MemoryTypeCommon* m_memTypeCommon = nullptr;
 	GrAllocator<U8> m_alloc;
+	Array<IntrusiveList<Memory>, CLASS_COUNT> m_vacantMemory;
+	Mutex m_mtx;
 	VkDevice m_dev = VK_NULL_HANDLE;
+	U8 m_memTypeIdx = MAX_U8;
+
+	Error allocate(U classIdx, ClassAllocatorMemory*& cmem)
+	{
+		Memory* mem;
+
+		LockGuard<Mutex> lock(m_mtx);
+
+		if(!m_vacantMemory[classIdx].isEmpty())
+		{
+			// Recycle
+			mem = &m_vacantMemory[classIdx].getFront();
+			m_vacantMemory[classIdx].popFront();
+		}
+		else
+		{
+			// Create new
+			mem = m_alloc.newInstance<Memory>();
 
-	Interface() = default;
+			VkMemoryAllocateInfo ci = {};
+			ci.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
+			ci.allocationSize = CLASSES[classIdx].m_chunkSize;
+			ci.memoryTypeIndex = m_memTypeIdx;
+			ANKI_VK_CHECKF(vkAllocateMemory(m_dev, &ci, nullptr, &mem->m_handle));
 
-	Error allocate(U classIdx, ClassAllocatorMemory*& mem);
+			mem->m_classIdx = classIdx;
+		}
 
-	void free(ClassAllocatorMemory* mem);
+		ANKI_ASSERT(mem);
+		ANKI_ASSERT(mem->m_handle);
+		ANKI_ASSERT(mem->m_classIdx == classIdx);
+		ANKI_ASSERT(mem->m_mappedAddress == nullptr);
+		cmem = mem;
+
+		return ErrorCode::NONE;
+	}
+
+	void free(ClassAllocatorMemory* cmem)
+	{
+		ANKI_ASSERT(cmem);
+
+		Memory* mem = static_cast<Memory*>(cmem);
+		ANKI_ASSERT(mem->m_handle);
+
+		LockGuard<Mutex> lock(m_mtx);
+		m_vacantMemory[mem->m_classIdx].pushBack(mem);
+
+		// Unmap
+		if(mem->m_mappedAddress)
+		{
+			vkUnmapMemory(m_dev, mem->m_handle);
+			mem->m_mappedAddress = nullptr;
+		}
+	}
 
 	U getClassCount() const
 	{
@@ -72,58 +125,169 @@ public:
 		slotSize = CLASSES[classIdx].m_slotSize;
 		chunkSize = CLASSES[classIdx].m_chunkSize;
 	}
+
+	void collectGarbage()
+	{
+		LockGuard<Mutex> lock(m_mtx);
+
+		for(U classIdx = 0; classIdx < CLASS_COUNT; ++classIdx)
+		{
+			while(!m_vacantMemory[classIdx].isEmpty())
+			{
+				Memory* mem = &m_vacantMemory[classIdx].getFront();
+				m_vacantMemory[classIdx].popFront();
+
+				if(mem->m_mappedAddress)
+				{
+					vkUnmapMemory(m_dev, mem->m_handle);
+				}
+
+				vkFreeMemory(m_dev, mem->m_handle, nullptr);
+
+				m_alloc.deleteInstance(mem);
+			}
+		}
+	}
+
+	// Mapp memory
+	void* mapMemory(ClassAllocatorMemory* cmem)
+	{
+		ANKI_ASSERT(cmem);
+		Memory* mem = static_cast<Memory*>(cmem);
+		void* out;
+
+		LockGuard<SpinLock> lock(mem->m_mtx);
+		if(mem->m_mappedAddress)
+		{
+			out = mem->m_mappedAddress;
+		}
+		else
+		{
+			ANKI_VK_CHECKF(vkMapMemory(m_dev, mem->m_handle, 0, CLASSES[mem->m_classIdx].m_chunkSize, 0, &out));
+			mem->m_mappedAddress = out;
+		}
+
+		ANKI_ASSERT(out);
+		return out;
+	}
 };
 
-class GpuMemoryManager::MemoryTypeCommon
+GpuMemoryManager::~GpuMemoryManager()
 {
-public:
-	Array<IntrusiveList<Memory>, CLASS_COUNT> m_vacantMemory;
-	Mutex m_mtx;
-};
+}
 
-Error GpuMemoryManager::Interface::allocate(U classIdx, ClassAllocatorMemory*& mem)
+void GpuMemoryManager::destroy()
 {
-	const PtrSize chunkSize = CLASSES[classIdx].m_chunkSize;
+	for(Interface& iface : m_ifaces)
+	{
+		iface.collectGarbage();
+	}
 
-	LockGuard<Mutex> lock(m_memTypeCommon->m_mtx);
+	m_ifaces.destroy(m_alloc);
+	m_callocs.destroy(m_alloc);
+}
 
-	Memory* pmem = nullptr;
+void GpuMemoryManager::init(VkPhysicalDevice pdev, VkDevice dev, GrAllocator<U8> alloc)
+{
+	ANKI_ASSERT(pdev);
+	ANKI_ASSERT(dev);
 
-	if(!m_memTypeCommon->m_vacantMemory[classIdx].isEmpty())
+	// Print some info
+	for(const ClassInf& c : CLASSES)
 	{
-		pmem = &m_memTypeCommon->m_vacantMemory[classIdx].getFront();
-		m_memTypeCommon->m_vacantMemory[classIdx].popFront();
+		ANKI_LOGI("VK: GPU mem class. Chunk size: %u, slotSize: %u, allocsPerChunk %u",
+			c.m_chunkSize,
+			c.m_slotSize,
+			c.m_chunkSize / c.m_slotSize);
 	}
-	else
+
+	vkGetPhysicalDeviceMemoryProperties(pdev, &m_memoryProperties);
+
+	m_alloc = alloc;
+
+	m_ifaces.create(alloc, m_memoryProperties.memoryTypeCount);
+	for(U i = 0; i < m_ifaces.getSize(); ++i)
 	{
-		pmem = m_alloc.newInstance<Memory>();
+		Interface& iface = m_ifaces[i];
 
-		VkMemoryAllocateInfo ci = {};
-		ci.sType = VK_STRUCTURE_TYPE_MEMORY_ALLOCATE_INFO;
-		ci.allocationSize = chunkSize;
-		ci.memoryTypeIndex = m_memTypeIdx;
-		ANKI_VK_CHECKF(vkAllocateMemory(m_dev, &ci, nullptr, &pmem->m_handle));
+		iface.m_alloc = alloc;
+		iface.m_dev = dev;
+		iface.m_memTypeIdx = i;
+	}
 
-		pmem->m_classIdx = classIdx;
+	// One allocator per type per linear/non-linear resources
+	m_callocs.create(alloc, m_memoryProperties.memoryTypeCount * 2);
+	for(U i = 0; i < m_callocs.getSize(); ++i)
+	{
+		m_callocs[i].init(m_alloc, &m_ifaces[i / 2]);
 	}
+}
+
+void GpuMemoryManager::allocateMemory(
+	U memTypeIdx, PtrSize size, U alignment, Bool linearResource, GpuMemoryHandle& handle)
+{
+	ClassAllocator& calloc = m_callocs[memTypeIdx * 2 + ((linearResource) ? 0 : 1)];
+	Error err = calloc.allocate(size, alignment, handle.m_classHandle);
+	(void)err;
+
+	handle.m_memory = static_cast<Memory*>(handle.m_classHandle.m_memory)->m_handle;
+	handle.m_offset = handle.m_classHandle.m_offset;
+	handle.m_linear = linearResource;
+	handle.m_memTypeIdx = memTypeIdx;
+}
+
+void GpuMemoryManager::freeMemory(GpuMemoryHandle& handle)
+{
+	ANKI_ASSERT(handle);
+
+	ClassAllocator& calloc = m_callocs[handle.m_memTypeIdx * 2 + ((handle.m_linear) ? 0 : 1)];
+	calloc.free(handle.m_classHandle);
 
-	ANKI_ASSERT(pmem);
-	mem = pmem;
-	return ErrorCode::NONE;
+	handle = {};
 }
 
-void GpuMemoryManager::Interface::free(ClassAllocatorMemory* mem)
+void* GpuMemoryManager::getMappedAddress(GpuMemoryHandle& handle)
 {
-	ANKI_ASSERT(mem);
-	Memory* pmem = static_cast<Memory*>(mem);
-	ANKI_ASSERT(pmem->m_handle);
+	ANKI_ASSERT(handle);
 
-	LockGuard<Mutex> lock(m_memTypeCommon->m_mtx);
-	m_memTypeCommon->m_vacantMemory[pmem->m_classIdx].pushBack(pmem);
+	Interface& iface = m_ifaces[handle.m_memTypeIdx];
+	U8* out = static_cast<U8*>(iface.mapMemory(handle.m_classHandle.m_memory));
+	return static_cast<void*>(out + handle.m_offset);
 }
 
-void GpuMemoryManager::init(VkPhysicalDevice pdev, VkDevice dev, GrAllocator<U8> alloc)
+U GpuMemoryManager::findMemoryType(
+	U resourceMemTypeBits, VkMemoryPropertyFlags preferFlags, VkMemoryPropertyFlags avoidFlags) const
 {
+	U preferedHigh = MAX_U32;
+	U preferedMed = MAX_U32;
+
+	// Iterate all mem types
+	for(U i = 0; i < m_memoryProperties.memoryTypeCount; i++)
+	{
+		if(resourceMemTypeBits & (1u << i))
+		{
+			VkMemoryPropertyFlags flags = m_memoryProperties.memoryTypes[i].propertyFlags;
+
+			if((flags & preferFlags) == preferFlags)
+			{
+				preferedMed = i;
+
+				if((flags & avoidFlags) != avoidFlags)
+				{
+					preferedHigh = i;
+				}
+			}
+		}
+	}
+
+	if(preferedHigh < MAX_U32)
+	{
+		return preferedHigh;
+	}
+	else
+	{
+		return preferedMed;
+	}
 }
 
 } // end namespace anki

src/anki/gr/vulkan/GpuMemoryManager.h

@@ -15,10 +15,23 @@ namespace anki
 /// @{
 
 /// The handle that is returned from GpuMemoryManager's allocations.
-class GpuMemoryHandle : public ClassAllocatorHandle
+class GpuMemoryHandle
 {
+	friend class GpuMemoryManager;
+
 public:
 	VkDeviceMemory m_memory = VK_NULL_HANDLE;
+	PtrSize m_offset = MAX_PTR_SIZE;
+
+	operator Bool() const
+	{
+		return m_memory != VK_NULL_HANDLE && m_offset < MAX_PTR_SIZE && m_memTypeIdx < MAX_U8;
+	}
+
+private:
+	ClassAllocatorHandle m_classHandle;
+	U8 m_memTypeIdx = MAX_U8;
+	Bool8 m_linear = false;
 };
 
 /// Dynamic GPU memory allocator for all types.
@@ -47,12 +60,12 @@ public:
 
 private:
 	class Memory;
-	class MemoryTypeCommon;
 	class Interface;
 
 	GrAllocator<U8> m_alloc;
 	VkDevice m_dev;
-	DynamicArray<MemoryTypeCommon> m_memType;
+	DynamicArray<Interface> m_ifaces;
+	DynamicArray<ClassAllocator> m_callocs;
 	VkPhysicalDeviceMemoryProperties m_memoryProperties;
 };
 /// @}

src/anki/gr/vulkan/GrManagerImpl.cpp

@@ -59,7 +59,7 @@ GrManagerImpl::~GrManagerImpl()
 
 	m_dsetAlloc.destroy();
 	m_transientMem.destroy();
-	m_gpuAlloc.destroy();
+	m_gpuMemManager.destroy();
 
 	m_semaphores.destroy(); // Destroy before fences
 	m_fences.destroy();
@@ -475,7 +475,7 @@ Error GrManagerImpl::initMemory(const ConfigSet& cfg)
 {
 	vkGetPhysicalDeviceMemoryProperties(m_physicalDevice, &m_memoryProperties);
 
-	m_gpuAlloc.init(m_physicalDevice, m_device, getAllocator());
+	m_gpuMemManager.init(m_physicalDevice, m_device, getAllocator());
 
 	// Transient mem
 	ANKI_CHECK(m_transientMem.init(cfg));

src/anki/gr/vulkan/GrManagerImpl.h

@@ -6,7 +6,7 @@
 #pragma once
 
 #include <anki/gr/vulkan/Common.h>
-#include <anki/gr/vulkan/GpuMemoryAllocator.h>
+#include <anki/gr/vulkan/GpuMemoryManager.h>
 #include <anki/gr/vulkan/Semaphore.h>
 #include <anki/gr/vulkan/Fence.h>
 #include <anki/gr/vulkan/TransientMemoryManager.h>
@@ -131,9 +131,9 @@ public:
 
 	/// @name Memory
 	/// @{
-	GpuMemoryAllocator& getGpuMemoryAllocator()
+	GpuMemoryManager& getGpuMemoryManager()
 	{
-		return m_gpuAlloc;
+		return m_gpuMemManager;
 	}
 
 	TransientMemoryManager& getTransientMemoryManager()
@@ -236,7 +236,7 @@ private:
 	VkPhysicalDeviceMemoryProperties m_memoryProperties;
 
 	/// The main allocator.
-	GpuMemoryAllocator m_gpuAlloc;
+	GpuMemoryManager m_gpuMemManager;
 
 	TransientMemoryManager m_transientMem;
 	/// @}

src/anki/gr/vulkan/TextureImpl.cpp

@@ -38,7 +38,7 @@ TextureImpl::~TextureImpl()
 
 	if(m_memHandle)
 	{
-		getGrManagerImpl().getGpuMemoryAllocator().freeMemory(m_memHandle);
+		getGrManagerImpl().getGpuMemoryManager().freeMemory(m_memHandle);
 	}
 }
 
@@ -272,20 +272,20 @@ Error TextureImpl::initImage(const TextureInitInfo& init_)
 	VkMemoryRequirements req = {};
 	vkGetImageMemoryRequirements(getDevice(), m_imageHandle, &req);
 
-	U memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+	U memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 		req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, VK_MEMORY_PROPERTY_HOST_VISIBLE_BIT);
 
 	// Fallback
 	if(memIdx == MAX_U32)
 	{
-		memIdx = getGrManagerImpl().getGpuMemoryAllocator().findMemoryType(
+		memIdx = getGrManagerImpl().getGpuMemoryManager().findMemoryType(
 			req.memoryTypeBits, VK_MEMORY_PROPERTY_DEVICE_LOCAL_BIT, 0);
 	}
 
 	ANKI_ASSERT(memIdx != MAX_U32);
 
 	// Allocate
-	getGrManagerImpl().getGpuMemoryAllocator().allocateMemory(memIdx, req.size, req.alignment, false, m_memHandle);
+	getGrManagerImpl().getGpuMemoryManager().allocateMemory(memIdx, req.size, req.alignment, false, m_memHandle);
 
 	// Bind mem to image
 	ANKI_VK_CHECK(vkBindImageMemory(getDevice(), m_imageHandle, m_memHandle.m_memory, m_memHandle.m_offset));

src/anki/gr/vulkan/TextureImpl.h

@@ -6,7 +6,7 @@
 #pragma once
 
 #include <anki/gr/vulkan/VulkanObject.h>
-#include <anki/gr/vulkan/GpuMemoryAllocator.h>
+#include <anki/gr/vulkan/GpuMemoryManager.h>
 #include <anki/gr/vulkan/Semaphore.h>
 #include <anki/gr/common/Misc.h>
 #include <anki/util/HashMap.h>
@@ -32,7 +32,7 @@ public:
 
 	VkImage m_imageHandle = VK_NULL_HANDLE;
 
-	GpuMemoryAllocationHandle m_memHandle;
+	GpuMemoryHandle m_memHandle;
 
 	U32 m_width = 0;
 	U32 m_height = 0;

src/anki/gr/vulkan/TransientMemoryManager.h

@@ -5,7 +5,7 @@
 
 #pragma once
 
-#include <anki/gr/vulkan/GpuMemoryAllocator.h>
+#include <anki/gr/vulkan/Common.h>
 #include <anki/gr/common/GpuFrameRingAllocator.h>
 #include <anki/gr/common/Misc.h>