Move the dynamic memory allocator logic in a common place. Also create a new block allocator for the transfer memory

include/anki/gr/common/GpuBlockAllocator.h

@@ -0,0 +1,60 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#pragma once
+
+#include <anki/gr/Common.h>
+#include <anki/util/List.h>
+
+namespace anki
+{
+
+/// @addtogroup graphics
+/// @{
+
+/// Manages pre-allocated, always mapped GPU memory in blocks.
+class GpuBlockAllocator : public NonCopyable
+{
+public:
+	/// Default constructor.
+	GpuBlockAllocator()
+	{
+	}
+
+	/// Destructor.
+	~GpuBlockAllocator();
+
+	/// Initialize the allocator using pre-allocated CPU mapped memory.
+	void init(GenericMemoryPoolAllocator<U8> alloc,
+		void* cpuMappedMem,
+		PtrSize cpuMappedMemSize,
+		PtrSize blockSize);
+
+	/// Allocate GPU memory.
+	ANKI_USE_RESULT void* allocate(
+		PtrSize size, U alignment, DynamicBufferToken& handle);
+
+	/// Free GPU memory.
+	void free(void* ptr);
+
+private:
+	class Block;
+
+	GenericMemoryPoolAllocator<U8> m_alloc;
+	U8* m_mem = nullptr;
+	PtrSize m_size = 0;
+	PtrSize m_blockSize = 0;
+	DynamicArray<Block> m_blocks;
+
+	DynamicArray<U32> m_freeBlocksStack;
+	U32 m_freeBlockCount = 0;
+	U32 m_currentBlock = MAX_U32;
+	Mutex m_mtx;
+
+	Bool blockHasEnoughSpace(U blockIdx, PtrSize size, U alignment) const;
+};
+/// @}
+
+} // end namespace anki

include/anki/gr/common/GpuFrameRingAllocator.h

@@ -0,0 +1,59 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#pragma once
+
+#include <anki/gr/Common.h>
+
+namespace anki
+{
+
+/// @addtogroup graphics
+/// @{
+
+/// Manages pre-allocated, always mapped GPU memory for per frame usage.
+class GpuFrameRingAllocator : public NonCopyable
+{
+	friend class DynamicMemorySerializeCommand;
+
+public:
+	GpuFrameRingAllocator()
+	{
+	}
+
+	~GpuFrameRingAllocator()
+	{
+	}
+
+	/// Initialize with pre-allocated always mapped memory.
+	/// @param[in] cpuMappedMem Pre-allocated always mapped GPU memory.
+	/// @param size The size of the cpuMappedMem.
+	/// @param alignment The working alignment.
+	/// @param maxAllocationSize The size in @a allocate cannot exceed
+	///        maxAllocationSize.
+	void init(void* cpuMappedMem,
+		PtrSize size,
+		U32 alignment,
+		PtrSize maxAllocationSize = MAX_PTR_SIZE);
+
+	/// Allocate memory for a dynamic buffer.
+	ANKI_USE_RESULT void* allocate(PtrSize size, DynamicBufferToken& token);
+
+	/// Call this at the end of the frame.
+	/// @return The bytes that were not used. Used for statistics.
+	PtrSize endFrame();
+
+private:
+	U8* m_cpuAddress = nullptr; ///< Host address of the buffer.
+	PtrSize m_size = 0; ///< The full size of the buffer.
+	U32 m_alignment = 0; ///< Always work in that alignment.
+	PtrSize m_maxAllocationSize = 0; ///< For debugging.
+
+	Atomic<PtrSize> m_offset = {0};
+	U64 m_frame = 0;
+};
+/// @}
+
+} // end namespace anki

src/gr/common/GpuBlockAllocator.cpp

@@ -0,0 +1,152 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#include <anki/gr/common/GpuBlockAllocator.h>
+
+namespace anki
+{
+
+//==============================================================================
+// GpuBlockAllocator::Block                                                    =
+//==============================================================================
+class GpuBlockAllocator::Block
+	: public IntrusiveListEnabled<GpuBlockAllocator::Block>
+{
+public:
+	PtrSize m_offset = 0;
+	U32 m_allocationCount = 0;
+};
+
+//==============================================================================
+// GpuBlockAllocator                                                           =
+//==============================================================================
+
+//==============================================================================
+GpuBlockAllocator::~GpuBlockAllocator()
+{
+	if(m_freeBlockCount != m_blocks.getSize())
+	{
+		ANKI_LOGW("Forgot to free memory");
+	}
+
+	m_blocks.destroy(m_alloc);
+	m_freeBlocksStack.destroy(m_alloc);
+}
+
+//==============================================================================
+void GpuBlockAllocator::init(GenericMemoryPoolAllocator<U8> alloc,
+	void* cpuMappedMem,
+	PtrSize cpuMappedMemSize,
+	PtrSize blockSize)
+{
+	ANKI_ASSERT(cpuMappedMem && cpuMappedMemSize > 0 && blockSize > 0);
+	ANKI_ASSERT((cpuMappedMemSize % blockSize) == 0);
+
+	m_alloc = alloc;
+	m_mem = static_cast<U8*>(cpuMappedMem);
+	m_size = cpuMappedMemSize;
+	m_blockSize = blockSize;
+	m_blocks.create(alloc, cpuMappedMemSize / blockSize);
+
+	m_freeBlocksStack.create(alloc, m_blocks.getSize());
+	m_freeBlockCount = m_blocks.getSize();
+	m_currentBlock = MAX_U32;
+
+	U count = m_freeBlocksStack.getSize();
+	for(U32& i : m_freeBlocksStack)
+	{
+		i = --count;
+	}
+}
+
+//==============================================================================
+Bool GpuBlockAllocator::blockHasEnoughSpace(
+	U blockIdx, PtrSize size, U alignment) const
+{
+	ANKI_ASSERT(size > 0);
+
+	const Block& block = m_blocks[blockIdx];
+
+	U8* allocEnd = getAlignedRoundUp(alignment, m_mem + block.m_offset) + size;
+	U8* blockEnd = m_mem + blockIdx * m_blockSize + m_blockSize;
+
+	return allocEnd <= blockEnd;
+}
+
+//==============================================================================
+void* GpuBlockAllocator::allocate(
+	PtrSize size, U alignment, DynamicBufferToken& handle)
+{
+	ANKI_ASSERT(size < m_blockSize);
+
+	Block* block = nullptr;
+	U8* ptr = nullptr;
+
+	LockGuard<Mutex> lock(m_mtx);
+
+	if(m_currentBlock == MAX_U32
+		|| !blockHasEnoughSpace(m_currentBlock, size, alignment))
+	{
+		// Need new block
+		if(m_freeBlockCount > 0)
+		{
+			// Pop block from free
+			U blockIdx = --m_freeBlockCount;
+
+			block = &m_blocks[blockIdx];
+			block->m_offset = blockIdx * m_blockSize;
+			ANKI_ASSERT(block->m_allocationCount == 0);
+
+			// Make it in-use
+			m_currentBlock = blockIdx;
+		}
+	}
+
+	if(block)
+	{
+		ptr = getAlignedRoundUp(alignment, m_mem + block->m_offset);
+
+		PtrSize outOffset = ptr - m_mem;
+		block->m_offset = outOffset + size;
+		ANKI_ASSERT(
+			block->m_offset <= (block - &m_blocks[0] + 1) * m_blockSize);
+
+		++block->m_allocationCount;
+
+		// Update the handle
+		handle.m_offset = outOffset;
+		handle.m_range = size;
+	}
+
+	return static_cast<void*>(ptr);
+}
+
+//==============================================================================
+void GpuBlockAllocator::free(void* vptr)
+{
+	U8* ptr = static_cast<U8*>(vptr);
+	ANKI_ASSERT(ptr);
+	ANKI_ASSERT(ptr >= m_mem && ptr < m_mem + m_size);
+
+	PtrSize offset = static_cast<PtrSize>(ptr - m_mem);
+	U blockIdx = offset / m_blockSize;
+
+	LockGuard<Mutex> lock(m_mtx);
+
+	ANKI_ASSERT(m_blocks[blockIdx].m_allocationCount > 0);
+	if((--m_blocks[blockIdx].m_allocationCount) == 0)
+	{
+		// Block no longer in use
+
+		if(m_currentBlock == blockIdx)
+		{
+			m_currentBlock = MAX_U32;
+		}
+
+		m_freeBlocksStack[m_freeBlockCount++] = blockIdx;
+	}
+}
+
+} // end namespace anki

src/gr/common/GpuFrameRingAllocator.cpp

@@ -0,0 +1,84 @@
+// Copyright (C) 2009-2016, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#include <anki/gr/common/GpuFrameRingAllocator.h>
+
+namespace anki
+{
+
+//==============================================================================
+void GpuFrameRingAllocator::init(
+	void* cpuMappedMem, PtrSize size, U32 alignment, PtrSize maxAllocationSize)
+{
+	ANKI_ASSERT(
+		cpuMappedMem && size > 0 && alignment > 0 && maxAllocationSize > 0);
+
+	PtrSize perFrameSize = size / MAX_FRAMES_IN_FLIGHT;
+	alignRoundDown(alignment, perFrameSize);
+	m_size = perFrameSize * MAX_FRAMES_IN_FLIGHT;
+
+	m_cpuAddress = static_cast<U8*>(cpuMappedMem);
+	m_alignment = alignment;
+	m_maxAllocationSize = maxAllocationSize;
+}
+
+//==============================================================================
+PtrSize GpuFrameRingAllocator::endFrame()
+{
+	PtrSize perFrameSize = m_size / MAX_FRAMES_IN_FLIGHT;
+
+	PtrSize crntFrameStartOffset =
+		perFrameSize * (m_frame % MAX_FRAMES_IN_FLIGHT);
+
+	PtrSize nextFrameStartOffset =
+		perFrameSize * ((m_frame + 1) % MAX_FRAMES_IN_FLIGHT);
+
+	PtrSize crntOffset = m_offset.exchange(nextFrameStartOffset);
+	ANKI_ASSERT(crntOffset >= crntFrameStartOffset);
+
+	PtrSize bytesUsed = crntOffset - crntFrameStartOffset;
+	PtrSize bytesNotUsed =
+		(bytesUsed > perFrameSize) ? 0 : perFrameSize - bytesUsed;
+
+	++m_frame;
+	return bytesNotUsed;
+}
+
+//==============================================================================
+void* GpuFrameRingAllocator::allocate(
+	PtrSize originalSize, DynamicBufferToken& token)
+{
+	ANKI_ASSERT(originalSize > 0);
+	ANKI_ASSERT(m_cpuAddress);
+
+	// Align size
+	PtrSize size = getAlignedRoundUp(m_alignment, originalSize);
+	ANKI_ASSERT(size <= m_maxAllocationSize && "Too high!");
+
+	PtrSize offset = m_offset.fetchAdd(size);
+	PtrSize perFrameSize = m_size / MAX_FRAMES_IN_FLIGHT;
+	PtrSize crntFrameStartOffset =
+		perFrameSize * (m_frame % MAX_FRAMES_IN_FLIGHT);
+
+	if(offset - crntFrameStartOffset + size <= perFrameSize)
+	{
+		ANKI_ASSERT(isAligned(m_alignment, m_cpuAddress + offset));
+		ANKI_ASSERT((offset + size) <= m_size);
+
+		// Encode token
+		token.m_offset = offset;
+		token.m_range = originalSize;
+
+		return static_cast<void*>(m_cpuAddress + offset);
+	}
+	else
+	{
+		ANKI_LOGF("Out of GPU dynamic memory");
+	}
+
+	return nullptr;
+}
+
+} // end namespace anki