Optimizing memory

include/anki/util/Allocator.h

@@ -94,7 +94,7 @@ public:
 	{
 		m_pool = reinterpret_cast<TPool*>(
 			allocCb(allocCbUserData, nullptr, sizeof(TPool), alignof(TPool)));
-		if(!m_pool)
+		if(ANKI_UNLIKELY(!m_pool))
 		{
 			ANKI_LOGF("Out of memory");
 		}
@@ -155,6 +155,11 @@ public:
 			: alignof(value_type);
 
 		void* out = m_pool->allocate(size, alignment);
+		if(ANKI_UNLIKELY(out == nullptr))
+		{
+			ANKI_LOGF("Out of memory");
+		}
+
 		return reinterpret_cast<pointer>(out);
 	}
 

include/anki/util/Memory.h

@@ -263,40 +263,26 @@ private:
 class ChainMemoryPool: public BaseMemoryPool
 {
 public:
-	/// Chunk allocation method. Defines the size a newely created chunk should
-	/// have compared to the last created. Used to grow chunks over the number 
-	/// of allocations that happen.
-	enum class ChunkGrowMethod: U8
-	{
-		NONE,
-		FIXED, ///< All chunks have the same size
-		MULTIPLY, ///< Next chuck's size will be old_chuck_size * a_value
-		ADD ///< Next chuck's size will be old_chuck_size + a_value
-	};
-
 	/// Default constructor
 	ChainMemoryPool();
 
 	/// Destroy
 	~ChainMemoryPool() final;
 
-	/// Constructor with parameters
-	/// @param allocCb The allocation function callback
-	/// @param allocCbUserData The user data to pass to the allocation function
-	/// @param initialChunkSize The size of the first chunk
-	/// @param maxChunkSize The size of the chunks cannot exceed that number
-	/// @param chunkAllocStepMethod How new chunks grow compared to the old ones
-	/// @param chunkAllocStep Used along with chunkAllocStepMethod and defines
-	///                       the ammount of chunk size increase 
+	/// Creates the pool.
+	/// @param allocCb The allocation function callback.
+	/// @param allocCbUserData The user data to pass to the allocation function.
+	/// @param initialChunkSize The size of the first chunk.
+	/// @param nextChunkScale Value that controls the next chunk.
+	/// @param nextChunkBias Value that controls the next chunk. 
 	/// @param alignmentBytes The maximum supported alignment for returned
-	///                       memory
+	///                       memory.
 	void create(
 		AllocAlignedCallback allocCb, 
 		void* allocCbUserData,
 		PtrSize initialChunkSize,
-		PtrSize maxChunkSize,
-		ChunkGrowMethod chunkAllocStepMethod = ChunkGrowMethod::MULTIPLY, 
-		PtrSize chunkAllocStep = 2, 
+		F32 nextChunkScale = 2.0,
+		PtrSize nextChunkBias = 0,
 		PtrSize alignmentBytes = ANKI_SAFE_ALIGNMENT);
 
 	/// Allocate memory. This operation is thread safe
@@ -333,6 +319,9 @@ private:
 		/// Used to identify if the chunk can be deleted
 		PtrSize m_allocationsCount = 0;
 
+		/// Previous chunk in the list
+		Chunk* m_prev = nullptr;
+
 		/// Next chunk in the list
 		Chunk* m_next = nullptr;
 	};
@@ -355,11 +344,14 @@ private:
 	/// Chunk max size.
 	PtrSize m_maxSize = 0;
 
-	/// Chunk allocation method value.
-	U32 m_step = 0;
+	/// Chunk scale.
+	F32 m_scale = 2.0;
+
+	/// Chunk bias.
+	PtrSize m_bias = 0;
 
-	/// Chunk allocation method.
-	ChunkGrowMethod m_method = ChunkGrowMethod::NONE;
+	/// Cache a value.
+	PtrSize m_headerSize;
 
 	/// Compute the size for the next chunk.
 	/// @param size The current allocation size.

src/gr/gl/CommandBufferImpl.cpp

@@ -21,8 +21,7 @@ void CommandBufferImpl::create(const InitHints& hints)
 		pool.getAllocationCallback(),
 		pool.getAllocationCallbackUserData(),
 		hints.m_chunkSize, 
-		InitHints::MAX_CHUNK_SIZE, 
-		ChainMemoryPool::ChunkGrowMethod::ADD,
+		1.0,
 		hints.m_chunkSize);
 }
 

src/physics/PhysicsWorld.cpp

@@ -44,9 +44,8 @@ Error PhysicsWorld::create(AllocAlignedCallback allocCb, void* allocCbData)
 	m_alloc = ChainAllocator<U8>(
 		allocCb, allocCbData, 
 		1024 * 10,
-		1024 * 1024 * 10,
-		ChainMemoryPool::ChunkGrowMethod::MULTIPLY,
-		2);
+		2.0,
+		0);
 	
 	// Set allocators
 	gAlloc = &m_alloc;

src/scene/SceneGraph.cpp

@@ -120,8 +120,7 @@ Error SceneGraph::create(
 	m_alloc = SceneAllocator<U8>(
 		allocCb, allocCbData, 
 		1024 * 10,
-		1024 * 10,
-		ChainMemoryPool::ChunkGrowMethod::FIXED,
+		1.0,
 		0);
 	m_frameAlloc = SceneFrameAllocator<U8>(
 		allocCb, allocCbData, frameAllocatorSize);

src/script/ScriptManager.cpp

@@ -30,7 +30,7 @@ Error ScriptManager::create(
 	ANKI_LOGI("Initializing scripting engine...");
 
 	m_scene = scene;
-	m_alloc = ChainAllocator<U8>(allocCb, allocCbData, 1024, 1024 * 1024);
+	m_alloc = ChainAllocator<U8>(allocCb, allocCbData, 1024, 1.0, 0);
 
 	Error err = m_lua.create(m_alloc, this);
 	if(err) return err;

src/util/Memory.cpp

@@ -506,9 +506,8 @@ void ChainMemoryPool::create(
 	AllocAlignedCallback allocCb, 
 	void* allocCbUserData,
 	PtrSize initialChunkSize,
-	PtrSize maxChunkSize,
-	ChunkGrowMethod chunkAllocStepMethod, 
-	PtrSize chunkAllocStep, 
+	F32 nextChunkScale,
+	PtrSize nextChunkBias, 
 	PtrSize alignmentBytes)
 {
 	ANKI_ASSERT(!isCreated());
@@ -519,9 +518,9 @@ void ChainMemoryPool::create(
 	m_allocCbUserData = allocCbUserData;
 	m_alignmentBytes = alignmentBytes;
 	m_initSize = initialChunkSize;
-	m_maxSize = maxChunkSize;
-	m_step = chunkAllocStep;
-	m_method = chunkAllocStepMethod;
+	m_scale = nextChunkScale;
+	m_bias = nextChunkBias;
+	m_headerSize = max(m_alignmentBytes, sizeof(Chunk*));
 	
 	m_lock = reinterpret_cast<SpinLock*>(m_allocCb(
 		m_allocCbUserData, nullptr, sizeof(SpinLock), alignof(SpinLock)));
@@ -534,23 +533,10 @@ void ChainMemoryPool::create(
 	// Initial size should be > 0
 	ANKI_ASSERT(m_initSize > 0 && "Wrong arg");
 
-	// On fixed step should be 0
-	if(m_method == ChunkGrowMethod::FIXED)
-	{
-		ANKI_ASSERT(m_step == 0 && "Wrong arg");
-	}
-
 	// On fixed initial size is the same as the max
-	if(m_method == ChunkGrowMethod::FIXED)
+	if(m_scale == 0.0 && m_bias == 0)
 	{
-		ANKI_ASSERT(m_initSize == m_maxSize && "Wrong arg");
-	}
-
-	// On add and mul the max size should be greater than initial
-	if(m_method == ChunkGrowMethod::ADD 
-		|| m_method == ChunkGrowMethod::MULTIPLY)
-	{
-		ANKI_ASSERT(m_initSize < m_maxSize && "Wrong arg");
+		ANKI_ASSERT(0 && "Wrong arg");
 	}
 }
 
@@ -558,7 +544,6 @@ void ChainMemoryPool::create(
 void* ChainMemoryPool::allocate(PtrSize size, PtrSize alignment)
 {
 	ANKI_ASSERT(isCreated());
-	ANKI_ASSERT(size <= m_maxSize);
 
 	Chunk* ch;
 	void* mem = nullptr;
@@ -603,52 +588,23 @@ void ChainMemoryPool::free(void* ptr)
 		return;
 	}
 
-	LockGuard<SpinLock> lock(*m_lock);
+	// Get the chunk
+	U8* mem = static_cast<U8*>(ptr);
+	mem -= m_headerSize;
+	Chunk* chunk = *reinterpret_cast<Chunk**>(mem);
 
-	// Find the chunk the ptr belongs to
-	Chunk* chunk = m_headChunk;
-	Chunk* prevChunk = nullptr;
-	while(chunk)
-	{
-		const U8* from = chunk->m_memory;
-		const U8* to = from + chunk->m_memsize;
-		const U8* cptr = reinterpret_cast<const U8*>(ptr);
-		if(cptr >= from && cptr < to)
-		{
-			break;
-		}
+	ANKI_ASSERT(chunk != nullptr);
+	ANKI_ASSERT((mem >= chunk->m_memory 
+		&& mem < (chunk->m_memory + chunk->m_memsize))
+		&& "Wrong chunk");
 
-		prevChunk = chunk;
-		chunk = chunk->m_next;
-	}
-
-	ANKI_ASSERT(chunk != nullptr 
-		&& "Not initialized or ptr is incorrect");
+	LockGuard<SpinLock> lock(*m_lock);
 
 	// Decrease the deallocation refcount and if it's zero delete the chunk
 	ANKI_ASSERT(chunk->m_allocationsCount > 0);
 	if(--chunk->m_allocationsCount == 0)
 	{
 		// Chunk is empty. Delete it
-
-		if(prevChunk != nullptr)
-		{
-			ANKI_ASSERT(m_headChunk != chunk);
-			prevChunk->m_next = chunk->m_next;
-		}
-
-		if(chunk == m_headChunk)
-		{
-			ANKI_ASSERT(prevChunk == nullptr);
-			m_headChunk = chunk->m_next;
-		}
-
-		if(chunk == m_tailChunk)
-		{
-			m_tailChunk = prevChunk;
-		}
-
-		// Finaly delete it
 		destroyChunk(chunk);
 	}
 
@@ -690,52 +646,29 @@ PtrSize ChainMemoryPool::getAllocatedSize() const
 //==============================================================================
 PtrSize ChainMemoryPool::computeNewChunkSize(PtrSize size) const
 {
-	ANKI_ASSERT(size <= m_maxSize);
+	size += m_headerSize;
 
-	// Get the size of the next chunk
 	PtrSize crntMaxSize;
-	if(m_method == ChunkGrowMethod::FIXED)
+	if(m_tailChunk != nullptr)
 	{
-		crntMaxSize = m_initSize;
+		// Get the size of previous
+		crntMaxSize = m_tailChunk->m_memsize;
+
+		// Compute new size
+		crntMaxSize = F32(crntMaxSize) * m_scale + m_bias;
 	}
 	else
 	{
-		// Get the size of the previous max chunk
-		if(m_tailChunk != nullptr)
-		{
-			// Get the size of previous
-			crntMaxSize = m_tailChunk->m_memsize;
-			ANKI_ASSERT(crntMaxSize > 0);
-
-			// Increase it
-			if(m_method == ChainMemoryPool::ChunkGrowMethod::MULTIPLY)
-			{
-				crntMaxSize *= m_step;
-			}
-			else
-			{
-				ANKI_ASSERT(m_method 
-					== ChainMemoryPool::ChunkGrowMethod::ADD);
-				crntMaxSize += m_step;
-			}
-		}
-		else
-		{
-			// No chunks. Choose initial size
-
-			ANKI_ASSERT(m_headChunk == nullptr);
-			crntMaxSize = m_initSize;
-		}
-
-		ANKI_ASSERT(crntMaxSize > 0);
-
-		// Fix the size
-		crntMaxSize = min(crntMaxSize, m_maxSize);
+		// No chunks. Choose initial size
+		ANKI_ASSERT(m_headChunk == nullptr);
+		crntMaxSize = m_initSize;
 	}
 
-	size = max(crntMaxSize, size);
+	crntMaxSize = max(crntMaxSize, size);
+
+	ANKI_ASSERT(crntMaxSize > 0);
 
-	return size;
+	return crntMaxSize;
 }
 
 //==============================================================================
@@ -769,6 +702,7 @@ ChainMemoryPool::Chunk* ChainMemoryPool::createNewChunk(PtrSize size)
 		if(m_tailChunk)
 		{
 			m_tailChunk->m_next = chunk;
+			chunk->m_prev = m_tailChunk;
 			m_tailChunk = chunk;
 		}
 		else
@@ -790,15 +724,17 @@ void* ChainMemoryPool::allocateFromChunk(
 	Chunk* ch, PtrSize size, PtrSize alignment)
 {
 	ANKI_ASSERT(ch);
-	ANKI_ASSERT(size <= m_maxSize);
 	ANKI_ASSERT(ch->m_top <= ch->m_memory + ch->m_memsize);
 
 	U8* mem = ch->m_top;
 	alignRoundUp(m_alignmentBytes, mem);
-	U8* newTop = mem + size;
+	U8* newTop = mem + m_headerSize + size;
 
 	if(newTop <= ch->m_memory + ch->m_memsize)
 	{
+		*reinterpret_cast<Chunk**>(mem) = ch;
+		mem += m_headerSize;
+
 		ch->m_top = newTop;
 		++ch->m_allocationsCount;
 	}
@@ -816,6 +752,28 @@ void ChainMemoryPool::destroyChunk(Chunk* ch)
 {
 	ANKI_ASSERT(ch);
 
+	if(ch == m_tailChunk)
+	{
+		m_tailChunk = ch->m_prev;
+	}
+
+	if(ch == m_headChunk)
+	{
+		m_headChunk = ch->m_next;
+	}
+
+	if(ch->m_prev)
+	{
+		ANKI_ASSERT(ch->m_prev->m_next == ch);
+		ch->m_prev->m_next = ch->m_next;
+	}
+
+	if(ch->m_next)
+	{
+		ANKI_ASSERT(ch->m_next->m_prev == ch);
+		ch->m_next->m_prev = ch->m_prev;
+	}
+
 #if ANKI_DEBUG
 	memset(ch, 0xCC, 
 		getAlignedRoundUp(m_alignmentBytes, sizeof(Chunk)) + ch->m_memsize);

tests/util/Memory.cpp

@@ -99,8 +99,7 @@ ANKI_TEST(Util, ChainMemoryPool)
 
 		pool.create(
 			allocAligned, nullptr,
-			size, size + 1, 
-			ChainMemoryPool::ChunkGrowMethod::MULTIPLY, 2, 1);
+			size, 2.0, 0, 1);
 
 		void* mem = pool.allocate(5, 1);
 		ANKI_TEST_EXPECT_NEQ(mem, nullptr);
@@ -120,8 +119,7 @@ ANKI_TEST(Util, ChainMemoryPool)
 		
 		pool.create(
 			allocAligned, nullptr,
-			size, size * 2, 
-			ChainMemoryPool::ChunkGrowMethod::MULTIPLY, 2, 1);
+			size, 2.0, 0, 1);
 
 		void* mem = pool.allocate(size, 1);
 		ANKI_TEST_EXPECT_NEQ(mem, nullptr);