Browse Source

Support WeakArray in serialization

Panagiotis Christopoulos Charitos 6 years ago
parent
commit
a51f3f4be5

+ 22 - 53
src/anki/shader_compiler/ShaderProgramBinary.h

@@ -53,8 +53,7 @@ class ShaderProgramBinaryMutator
 {
 public:
 	Array<char, MAX_SHADER_BINARY_NAME_LENGTH + 1> m_name;
-	MutatorValue* m_values;
-	U32 m_valueCount;
+	WeakArray<MutatorValue> m_values;
 	Bool m_instanceCount;
 
 	template<typename TSerializer, typename TClass>
@@ -62,9 +61,8 @@ public:
 	{
 		s.doArray(
 			"m_name", offsetof(ShaderProgramBinaryMutator, m_name), &self.m_name[0], MAX_SHADER_BINARY_NAME_LENGTH + 1);
-		s.doValue("m_valueCount", offsetof(ShaderProgramBinaryMutator, m_valueCount), self.m_valueCount);
+		s.doValue("m_values", offsetof(ShaderProgramBinaryMutator, m_values), self.m_values);
 		s.doValue("m_instanceCount", offsetof(ShaderProgramBinaryMutator, m_instanceCount), self.m_instanceCount);
-		s.doDynamicArray("m_values", offsetof(ShaderProgramBinaryMutator, m_values), self.m_values, self.m_valueCount);
 	}
 
 	template<typename TDeserializer>
@@ -85,41 +83,27 @@ class ShaderProgramBinaryVariant
 {
 public:
 	ActiveProgramInputVariableMask m_activeVariables = {false};
-	MutatorValue* m_mutatorValues;
-	ShaderVariableBlockInfo* m_blockInfos;
-	I16* m_bindings;
-	U32 m_mutatorValueCount;
-	U32 m_inputVariableCount;
-	Array<U32, U32(ShaderType::COUNT)> m_binaryIndices; ///< Index in ShaderProgramBinary::m_codeBlocks.
-	U32 m_blockSize;
+	WeakArray<MutatorValue> m_mutation;
 	Bool m_usesPushConstants;
+	WeakArray<ShaderVariableBlockInfo> m_blockInfos;
+	U32 m_blockSize;
+	WeakArray<I16> m_bindings;
+	Array<U32, U32(ShaderType::COUNT)> m_binaryIndices; ///< Index in ShaderProgramBinary::m_codeBlocks.
 
 	template<typename TSerializer, typename TClass>
 	static void serializeCommon(TSerializer& s, TClass self)
 	{
 		s.doValue("m_activeVariables", offsetof(ShaderProgramBinaryVariant, m_activeVariables), self.m_activeVariables);
+		s.doValue("m_mutation", offsetof(ShaderProgramBinaryVariant, m_mutation), self.m_mutation);
 		s.doValue(
-			"m_mutatorValueCount", offsetof(ShaderProgramBinaryVariant, m_mutatorValueCount), self.m_mutatorValueCount);
-		s.doValue("m_inputVariableCount",
-			offsetof(ShaderProgramBinaryVariant, m_inputVariableCount),
-			self.m_inputVariableCount);
+			"m_usesPushConstants", offsetof(ShaderProgramBinaryVariant, m_usesPushConstants), self.m_usesPushConstants);
+		s.doValue("m_blockInfos", offsetof(ShaderProgramBinaryVariant, m_blockInfos), self.m_blockInfos);
+		s.doValue("m_blockSize", offsetof(ShaderProgramBinaryVariant, m_blockSize), self.m_blockSize);
+		s.doValue("m_bindings", offsetof(ShaderProgramBinaryVariant, m_bindings), self.m_bindings);
 		s.doArray("m_binaryIndices",
 			offsetof(ShaderProgramBinaryVariant, m_binaryIndices),
 			&self.m_binaryIndices[0],
 			U32(ShaderType::COUNT));
-		s.doValue("m_blockSize", offsetof(ShaderProgramBinaryVariant, m_blockSize), self.m_blockSize);
-		s.doValue(
-			"m_usesPushConstants", offsetof(ShaderProgramBinaryVariant, m_usesPushConstants), self.m_usesPushConstants);
-		s.doDynamicArray("m_mutatorValues",
-			offsetof(ShaderProgramBinaryVariant, m_mutatorValues),
-			self.m_mutatorValues,
-			self.m_mutatorValueCount);
-		s.doDynamicArray("m_blockInfos",
-			offsetof(ShaderProgramBinaryVariant, m_blockInfos),
-			self.m_blockInfos,
-			self.m_inputVariableCount);
-		s.doDynamicArray(
-			"m_bindings", offsetof(ShaderProgramBinaryVariant, m_bindings), self.m_bindings, self.m_inputVariableCount);
 	}
 
 	template<typename TDeserializer>
@@ -139,14 +123,12 @@ public:
 class ShaderProgramBinaryCode
 {
 public:
-	U8* m_binary;
-	PtrSize m_binarySize;
+	WeakArray<U8, PtrSize> m_binary;
 
 	template<typename TSerializer, typename TClass>
 	static void serializeCommon(TSerializer& s, TClass self)
 	{
-		s.doValue("m_binarySize", offsetof(ShaderProgramBinaryCode, m_binarySize), self.m_binarySize);
-		s.doDynamicArray("m_binary", offsetof(ShaderProgramBinaryCode, m_binary), self.m_binary, self.m_binarySize);
+		s.doValue("m_binary", offsetof(ShaderProgramBinaryCode, m_binary), self.m_binary);
 	}
 
 	template<typename TDeserializer>
@@ -167,14 +149,10 @@ class ShaderProgramBinary
 {
 public:
 	Array<U8, 8> m_magic;
-	ShaderProgramBinaryMutator* m_mutators;
-	ShaderProgramBinaryInput* m_inputVariables;
-	ShaderProgramBinaryCode* m_codeBlocks;
-	ShaderProgramBinaryVariant* m_variants;
-	U32 m_mutatorCount;
-	U32 m_inputVariableCount;
-	U32 m_codeBlockCount;
-	U32 m_variantCount;
+	WeakArray<ShaderProgramBinaryMutator> m_mutators;
+	WeakArray<ShaderProgramBinaryInput> m_inputVariables;
+	WeakArray<ShaderProgramBinaryCode> m_codeBlocks;
+	WeakArray<ShaderProgramBinaryVariant> m_variants;
 	U32 m_descriptorSet;
 	ShaderTypeBit m_presentShaderTypes;
 
@@ -182,22 +160,13 @@ public:
 	static void serializeCommon(TSerializer& s, TClass self)
 	{
 		s.doArray("m_magic", offsetof(ShaderProgramBinary, m_magic), &self.m_magic[0], 8);
-		s.doValue("m_mutatorCount", offsetof(ShaderProgramBinary, m_mutatorCount), self.m_mutatorCount);
-		s.doValue(
-			"m_inputVariableCount", offsetof(ShaderProgramBinary, m_inputVariableCount), self.m_inputVariableCount);
-		s.doValue("m_codeBlockCount", offsetof(ShaderProgramBinary, m_codeBlockCount), self.m_codeBlockCount);
-		s.doValue("m_variantCount", offsetof(ShaderProgramBinary, m_variantCount), self.m_variantCount);
+		s.doValue("m_mutators", offsetof(ShaderProgramBinary, m_mutators), self.m_mutators);
+		s.doValue("m_inputVariables", offsetof(ShaderProgramBinary, m_inputVariables), self.m_inputVariables);
+		s.doValue("m_codeBlocks", offsetof(ShaderProgramBinary, m_codeBlocks), self.m_codeBlocks);
+		s.doValue("m_variants", offsetof(ShaderProgramBinary, m_variants), self.m_variants);
 		s.doValue("m_descriptorSet", offsetof(ShaderProgramBinary, m_descriptorSet), self.m_descriptorSet);
 		s.doValue(
 			"m_presentShaderTypes", offsetof(ShaderProgramBinary, m_presentShaderTypes), self.m_presentShaderTypes);
-		s.doDynamicArray("m_mutators", offsetof(ShaderProgramBinary, m_mutators), self.m_mutators, self.m_mutatorCount);
-		s.doDynamicArray("m_inputVariables",
-			offsetof(ShaderProgramBinary, m_inputVariables),
-			self.m_inputVariables,
-			self.m_inputVariableCount);
-		s.doDynamicArray(
-			"m_codeBlocks", offsetof(ShaderProgramBinary, m_codeBlocks), self.m_codeBlocks, self.m_codeBlockCount);
-		s.doDynamicArray("m_variants", offsetof(ShaderProgramBinary, m_variants), self.m_variants, self.m_variantCount);
 	}
 
 	template<typename TDeserializer>

+ 11 - 19
src/anki/shader_compiler/ShaderProgramBinary.xml

@@ -18,8 +18,7 @@
 		<class name="ShaderProgramBinaryMutator" comment="Shader program mutator">
 			<members>
 				<member name="m_name" type="char" array_size="MAX_SHADER_BINARY_NAME_LENGTH + 1" />
-				<member name="m_values" type="MutatorValue" pointer="true" array_size="m_valueCount" />
-				<member name="m_valueCount" type="U32" />
+				<member name="m_values" type="WeakArray&lt;MutatorValue&gt;" />
 				<member name="m_instanceCount" type="Bool" />
 			</members>
 		</class>
@@ -27,35 +26,28 @@
 		<class name="ShaderProgramBinaryVariant">
 			<members>
 				<member name="m_activeVariables" type="ActiveProgramInputVariableMask" constructor="false" />
-				<member name="m_mutatorValues" type="MutatorValue" pointer="true" array_size="m_mutatorValueCount" />
-				<member name="m_blockInfos" type="ShaderVariableBlockInfo" pointer="true" array_size="m_inputVariableCount" />
-				<member name="m_bindings" type="I16" pointer="true" array_size="m_inputVariableCount" />
-				<member name="m_mutatorValueCount" type="U32" />
-				<member name="m_inputVariableCount" type="U32" />
-				<member name="m_binaryIndices" type="U32" array_size="U32(ShaderType::COUNT)" comment="Index in ShaderProgramBinary::m_codeBlocks" />
-				<member name="m_blockSize" type="U32" />
+				<member name="m_mutation" type="WeakArray&lt;MutatorValue&gt;" />
 				<member name="m_usesPushConstants" type="Bool" />
+				<member name="m_blockInfos" type="WeakArray&lt;ShaderVariableBlockInfo&gt;" />
+				<member name="m_blockSize" type="U32" />
+				<member name="m_bindings" type="WeakArray&lt;I16&gt;" />
+				<member name="m_binaryIndices" type="U32" array_size="U32(ShaderType::COUNT)" comment="Index in ShaderProgramBinary::m_codeBlocks" />
 			</members>
 		</class>
 
 		<class name="ShaderProgramBinaryCode">
 			<members>
-				<member name="m_binary" type="U8" pointer="true" array_size="m_binarySize" />
-				<member name="m_binarySize" type="PtrSize" />
+				<member name="m_binary" type="WeakArray&lt;U8, PtrSize&gt;" />
 			</members>
 		</class>
 
 		<class name="ShaderProgramBinary">
 			<members>
 				<member name="m_magic" type="U8" array_size="8" />
-				<member name="m_mutators" type="ShaderProgramBinaryMutator" pointer="true" array_size="m_mutatorCount" />
-				<member name="m_inputVariables" type="ShaderProgramBinaryInput" pointer="true" array_size="m_inputVariableCount" />
-				<member name="m_codeBlocks" type="ShaderProgramBinaryCode" pointer="true" array_size="m_codeBlockCount" />
-				<member name="m_variants" type="ShaderProgramBinaryVariant" pointer="true" array_size="m_variantCount" />
-				<member name="m_mutatorCount" type="U32" />
-				<member name="m_inputVariableCount" type="U32" />
-				<member name="m_codeBlockCount" type="U32" />
-				<member name="m_variantCount" type="U32" />
+				<member name="m_mutators" type="WeakArray&lt;ShaderProgramBinaryMutator&gt;" />
+				<member name="m_inputVariables" type="WeakArray&lt;ShaderProgramBinaryInput&gt;" />
+				<member name="m_codeBlocks" type="WeakArray&lt;ShaderProgramBinaryCode&gt;" />
+				<member name="m_variants" type="WeakArray&lt;ShaderProgramBinaryVariant&gt;" />
 				<member name="m_descriptorSet" type="U32" />
 				<member name="m_presentShaderTypes" type="ShaderTypeBit" />
 			</members>

+ 103 - 88
src/anki/shader_compiler/ShaderProgramCompiler.cpp

@@ -23,7 +23,9 @@ Error ShaderProgramBinaryWrapper::serializeToFile(CString fname) const
 	ANKI_CHECK(file.open(fname, FileOpenFlag::WRITE | FileOpenFlag::BINARY));
 
 	BinarySerializer serializer;
-	ANKI_CHECK(serializer.serialize(*m_binary, m_alloc, file));
+	StackAllocator<U8> tmpAlloc(
+		m_alloc.getMemoryPool().getAllocationCallback(), m_alloc.getMemoryPool().getAllocationCallbackUserData(), 4_KB);
+	ANKI_CHECK(serializer.serialize(*m_binary, tmpAlloc, file));
 
 	if(memcmp(SHADER_BINARY_MAGIC, &m_binary->m_magic[0], 0) != 0)
 	{
@@ -58,29 +60,32 @@ void ShaderProgramBinaryWrapper::cleanup()
 
 	if(!m_singleAllocation)
 	{
-		for(PtrSize i = 0; i < m_binary->m_mutatorCount; ++i)
+		for(ShaderProgramBinaryMutator& mutator : m_binary->m_mutators)
 		{
-			m_alloc.getMemoryPool().free(m_binary->m_mutators[i].m_values);
+			m_alloc.getMemoryPool().free(mutator.m_values.getBegin());
 		}
 
-		m_alloc.getMemoryPool().free(m_binary->m_inputVariables);
+		m_alloc.getMemoryPool().free(m_binary->m_mutators.getBegin());
+		m_alloc.getMemoryPool().free(m_binary->m_inputVariables.getBegin());
 
-		for(PtrSize i = 0; i < m_binary->m_codeBlockCount; ++i)
+		for(ShaderProgramBinaryCode& code : m_binary->m_codeBlocks)
 		{
-			m_alloc.getMemoryPool().free(m_binary->m_codeBlocks[i].m_binary);
+			m_alloc.getMemoryPool().free(code.m_binary.getBegin());
 		}
-		m_alloc.getMemoryPool().free(m_binary->m_codeBlocks);
+		m_alloc.getMemoryPool().free(m_binary->m_codeBlocks.getBegin());
 
-		for(PtrSize i = 0; i < m_binary->m_variantCount; ++i)
+		for(ShaderProgramBinaryVariant& variant : m_binary->m_variants)
 		{
-			m_alloc.getMemoryPool().free(m_binary->m_variants[i].m_mutatorValues);
-			m_alloc.getMemoryPool().free(m_binary->m_variants[i].m_blockInfos);
-			m_alloc.getMemoryPool().free(m_binary->m_variants[i].m_bindings);
+			m_alloc.getMemoryPool().free(variant.m_mutation.getBegin());
+			m_alloc.getMemoryPool().free(variant.m_blockInfos.getBegin());
+			m_alloc.getMemoryPool().free(variant.m_bindings.getBegin());
 		}
-		m_alloc.getMemoryPool().free(m_binary->m_variants);
+		m_alloc.getMemoryPool().free(m_binary->m_variants.getBegin());
 	}
 
 	m_alloc.getMemoryPool().free(m_binary);
+	m_binary = nullptr;
+	m_singleAllocation = false;
 }
 
 /// Spin the dials. Used to compute all mutator combinations.
@@ -180,8 +185,7 @@ static Error compileVariant(ConstWeakArray<MutatorValue> mutation,
 			memcpy(code, &spirv[0], spirv.getSizeInBytes());
 
 			ShaderProgramBinaryCode block;
-			block.m_binary = code;
-			block.m_binarySize = spirv.getSizeInBytes();
+			block.m_binary.setArray(code, spirv.getSizeInBytes());
 			codeBlocks.emplaceBack(block);
 
 			codeBlockHashes.emplaceBack(newHash);
@@ -191,10 +195,11 @@ static Error compileVariant(ConstWeakArray<MutatorValue> mutation,
 	}
 
 	// Mutator values
-	variant.m_mutatorValues = binaryAlloc.newArray<I32>(parser.getMutators().getSize());
+	variant.m_mutation.setArray(
+		binaryAlloc.newArray<MutatorValue>(parser.getMutators().getSize()), parser.getMutators().getSize());
 	for(U32 i = 0; i < parser.getMutators().getSize(); ++i)
 	{
-		variant.m_mutatorValues[i] = mutation[i];
+		variant.m_mutation[i] = mutation[i];
 	}
 
 	// Input vars
@@ -204,9 +209,12 @@ static Error compileVariant(ConstWeakArray<MutatorValue> mutation,
 		defaultInfo.m_arrayStride = -1;
 		defaultInfo.m_matrixStride = -1;
 		defaultInfo.m_offset = -1;
-		variant.m_blockInfos = binaryAlloc.newArray<ShaderVariableBlockInfo>(parser.getInputs().getSize(), defaultInfo);
+		variant.m_blockInfos.setArray(
+			binaryAlloc.newArray<ShaderVariableBlockInfo>(parser.getInputs().getSize(), defaultInfo),
+			parser.getInputs().getSize());
 
-		variant.m_bindings = binaryAlloc.newArray<I16>(parser.getInputs().getSize(), -1);
+		variant.m_bindings.setArray(
+			binaryAlloc.newArray<I16>(parser.getInputs().getSize(), -1), parser.getInputs().getSize());
 
 		for(U32 i = 0; i < parser.getInputs().getSize(); ++i)
 		{
@@ -229,8 +237,6 @@ static Error compileVariant(ConstWeakArray<MutatorValue> mutation,
 	}
 
 	// Misc
-	variant.m_mutatorValueCount = parser.getMutators().getSize();
-	variant.m_inputVariableCount = parser.getInputs().getSize();
 	variant.m_blockSize = parserVariant.getBlockSize();
 	variant.m_usesPushConstants = parserVariant.usesPushConstants();
 
@@ -263,10 +269,11 @@ Error compileShaderProgram(CString fname,
 	// Inputs
 	if(parser.getInputs().getSize() > 0)
 	{
-		binary.m_inputVariableCount = parser.getInputs().getSize();
-		binary.m_inputVariables = binaryAllocator.newArray<ShaderProgramBinaryInput>(binary.m_inputVariableCount);
+		binary.m_inputVariables.setArray(
+			binaryAllocator.newArray<ShaderProgramBinaryInput>(parser.getInputs().getSize()),
+			parser.getInputs().getSize());
 
-		for(U32 i = 0; i < binary.m_inputVariableCount; ++i)
+		for(U32 i = 0; i < binary.m_inputVariables.getSize(); ++i)
 		{
 			ShaderProgramBinaryInput& out = binary.m_inputVariables[i];
 			const ShaderProgramParserInput& in = parser.getInputs()[i];
@@ -283,16 +290,17 @@ Error compileShaderProgram(CString fname,
 	}
 	else
 	{
-		ANKI_ASSERT(binary.m_inputVariableCount == 0 && binary.m_inputVariables == nullptr);
+		ANKI_ASSERT(binary.m_inputVariables.getSize() == 0);
 	}
 
 	// Mutators
+	U32 variantCount = 0;
 	if(parser.getMutators().getSize() > 0)
 	{
-		binary.m_mutatorCount = parser.getMutators().getSize();
-		binary.m_mutators = binaryAllocator.newArray<ShaderProgramBinaryMutator>(binary.m_mutatorCount);
+		binary.m_mutators.setArray(binaryAllocator.newArray<ShaderProgramBinaryMutator>(parser.getMutators().getSize()),
+			parser.getMutators().getSize());
 
-		for(U32 i = 0; i < binary.m_mutatorCount; ++i)
+		for(U32 i = 0; i < binary.m_mutators.getSize(); ++i)
 		{
 			ShaderProgramBinaryMutator& out = binary.m_mutators[i];
 			const ShaderProgramParserMutator& in = parser.getMutators()[i];
@@ -300,16 +308,18 @@ Error compileShaderProgram(CString fname,
 			ANKI_ASSERT(in.getName().getLength() < out.m_name.getSize());
 			memcpy(&out.m_name[0], in.getName().cstr(), in.getName().getLength() + 1);
 
-			out.m_valueCount = in.getValues().getSize();
-			out.m_values = binaryAllocator.newArray<I32>(out.m_valueCount);
-			memcpy(out.m_values, &in.getValues()[0], in.getValues().getSizeInBytes());
+			out.m_values.setArray(binaryAllocator.newArray<I32>(in.getValues().getSize()), in.getValues().getSize());
+			memcpy(out.m_values.getBegin(), in.getValues().getBegin(), in.getValues().getSizeInBytes());
 
 			out.m_instanceCount = in.isInstanceCount();
+
+			// Update the count
+			variantCount = (i == 0) ? out.m_values.getSize() : variantCount * out.m_values.getSize();
 		}
 	}
 	else
 	{
-		ANKI_ASSERT(binary.m_mutatorCount == 0 && binary.m_mutators == nullptr);
+		ANKI_ASSERT(binary.m_mutators.getSize() == 0);
 	}
 
 	// Create all variants
@@ -319,11 +329,13 @@ Error compileShaderProgram(CString fname,
 		DynamicArrayAuto<MutatorValue> mutation(tempAllocator, parser.getMutators().getSize());
 		DynamicArrayAuto<MutatorValue> mutation2(tempAllocator, parser.getMutators().getSize());
 		DynamicArrayAuto<U32> dials(tempAllocator, parser.getMutators().getSize(), 0);
-		DynamicArrayAuto<ShaderProgramBinaryVariant> variants(binaryAllocator);
+		DynamicArrayAuto<ShaderProgramBinaryVariant> variants(binaryAllocator, variantCount);
 		DynamicArrayAuto<ShaderProgramBinaryCode> codeBlocks(binaryAllocator);
 		DynamicArrayAuto<U64> codeBlockHashes(tempAllocator);
 		HashMapAuto<U64, U32> mutationToVariantIdx(tempAllocator);
 
+		variantCount = 0;
+
 		// Spin for all possible combinations of mutators and
 		// - Create the spirv
 		// - Populate the binary variant
@@ -339,15 +351,14 @@ Error compileShaderProgram(CString fname,
 				parser.rewriteMutation(WeakArray<MutatorValue>(mutation.getBegin(), mutation.getSize()));
 
 			// Create the variant
-			ShaderProgramBinaryVariant& variant = *variants.emplaceBack();
+			ShaderProgramBinaryVariant& variant = variants[variantCount++];
 			if(!rewritten)
 			{
 				// New and unique variant, add it
 				ANKI_CHECK(compileVariant(
 					mutation, parser, variant, codeBlocks, codeBlockHashes, tempAllocator, binaryAllocator));
 
-				mutationToVariantIdx.emplace(
-					computeHash(&mutation[0], mutation.getSizeInBytes()), variants.getSize() - 1);
+				mutationToVariantIdx.emplace(computeHash(&mutation[0], mutation.getSizeInBytes()), variantCount - 1);
 			}
 			else
 			{
@@ -359,8 +370,8 @@ Error compileShaderProgram(CString fname,
 				{
 					// Original variant not found, create it
 
-					ShaderProgramBinaryVariant& other = *variants.emplaceBack();
-					originalVariantIdx = variants.getSize() - 1;
+					ShaderProgramBinaryVariant& other = variants[variantCount++];
+					originalVariantIdx = variantCount - 1;
 
 					ANKI_CHECK(compileVariant(
 						mutation, parser, other, codeBlocks, codeBlockHashes, tempAllocator, binaryAllocator));
@@ -371,27 +382,30 @@ Error compileShaderProgram(CString fname,
 
 				// Copy the original variant to the current variant
 				{
+					ANKI_ASSERT(originalVariantIdx < variantCount);
 					const ShaderProgramBinaryVariant& other = variants[originalVariantIdx];
 
 					variant = other;
 
-					variant.m_mutatorValues = binaryAllocator.newArray<MutatorValue>(variant.m_mutatorValueCount);
-					memcpy(variant.m_mutatorValues,
-						mutation2.getBegin(),
-						sizeof(variant.m_mutatorValues[0]) * variant.m_mutatorValueCount);
+					variant.m_mutation.setArray(
+						binaryAllocator.newArray<MutatorValue>(other.m_mutation.getSize()), other.m_mutation.getSize());
+					memcpy(
+						variant.m_mutation.getBegin(), other.m_mutation.getBegin(), other.m_mutation.getSizeInBytes());
 
-					if(variant.m_inputVariableCount > 0)
+					if(other.m_blockInfos.getSize())
 					{
-						variant.m_blockInfos =
-							binaryAllocator.newArray<ShaderVariableBlockInfo>(variant.m_inputVariableCount);
-						memcpy(variant.m_blockInfos,
-							other.m_blockInfos,
-							sizeof(variant.m_blockInfos[0]) * variant.m_inputVariableCount);
-
-						variant.m_bindings = binaryAllocator.newArray<I16>(variant.m_inputVariableCount);
-						memcpy(variant.m_bindings,
-							other.m_bindings,
-							sizeof(variant.m_bindings[0]) * variant.m_inputVariableCount);
+						variant.m_blockInfos.setArray(
+							binaryAllocator.newArray<ShaderVariableBlockInfo>(other.m_blockInfos.getSize()),
+							other.m_blockInfos.getSize());
+						memcpy(variant.m_blockInfos.getBegin(),
+							other.m_blockInfos.getBegin(),
+							other.m_blockInfos.getSizeInBytes());
+
+						variant.m_bindings.setArray(
+							binaryAllocator.newArray<I16>(other.m_bindings.getSize()), other.m_bindings.getSize());
+						memcpy(variant.m_bindings.getBegin(),
+							other.m_bindings.getBegin(),
+							other.m_bindings.getSizeInBytes());
 					}
 
 					mutationToVariantIdx.emplace(
@@ -400,13 +414,17 @@ Error compileShaderProgram(CString fname,
 			}
 		} while(!spinDials(dials, parser.getMutators()));
 
+		ANKI_ASSERT(variantCount == variants.getSize());
+
 		// Store to binary
-		binary.m_variantCount = variants.getSize();
 		U32 size, storage;
-		variants.moveAndReset(binary.m_variants, size, storage);
+		ShaderProgramBinaryVariant* firstVariant;
+		variants.moveAndReset(firstVariant, size, storage);
+		binary.m_variants.setArray(firstVariant, size);
 
-		binary.m_codeBlockCount = codeBlocks.getSize();
-		codeBlocks.moveAndReset(binary.m_codeBlocks, size, storage);
+		ShaderProgramBinaryCode* firstCodeBlock;
+		codeBlocks.moveAndReset(firstCodeBlock, size, storage);
+		binary.m_codeBlocks.setArray(firstCodeBlock, size);
 	}
 	else
 	{
@@ -414,16 +432,16 @@ Error compileShaderProgram(CString fname,
 		DynamicArrayAuto<ShaderProgramBinaryCode> codeBlocks(binaryAllocator);
 		DynamicArrayAuto<U64> codeBlockHashes(tempAllocator);
 
-		binary.m_variantCount = 1;
-		binary.m_variants = binaryAllocator.newInstance<ShaderProgramBinaryVariant>();
+		binary.m_variants.setArray(binaryAllocator.newInstance<ShaderProgramBinaryVariant>(), 1);
 
 		ANKI_CHECK(compileVariant(
 			mutation, parser, binary.m_variants[0], codeBlocks, codeBlockHashes, tempAllocator, binaryAllocator));
 		ANKI_ASSERT(codeBlocks.getSize() == U32(__builtin_popcount(U32(parser.getShaderTypes()))));
 
-		binary.m_codeBlockCount = codeBlocks.getSize();
+		ShaderProgramBinaryCode* firstCodeBlock;
 		U32 size, storage;
-		codeBlocks.moveAndReset(binary.m_codeBlocks, size, storage);
+		codeBlocks.moveAndReset(firstCodeBlock, size, storage);
+		binary.m_codeBlocks.setArray(firstCodeBlock, size);
 	}
 
 	// Misc
@@ -441,14 +459,14 @@ void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAut
 	StringListAuto lines(alloc);
 
 	lines.pushBack("**MUTATORS**\n");
-	if(binary.m_mutatorCount > 0)
+	if(binary.m_mutators.getSize() > 0)
 	{
-		for(U i = 0; i < binary.m_mutatorCount; ++i)
+		for(const ShaderProgramBinaryMutator& mutator : binary.m_mutators)
 		{
-			lines.pushBackSprintf(ANKI_TAB "\"%s\"", &binary.m_mutators[i].m_name[0]);
-			for(U j = 0; j < binary.m_mutators[i].m_valueCount; ++j)
+			lines.pushBackSprintf(ANKI_TAB "\"%s\"", &mutator.m_name[0]);
+			for(MutatorValue value : mutator.m_values)
 			{
-				lines.pushBackSprintf(" %d", binary.m_mutators[i].m_values[j]);
+				lines.pushBackSprintf(" %d", value);
 			}
 			lines.pushBack("\n");
 		}
@@ -459,11 +477,10 @@ void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAut
 	}
 
 	lines.pushBack("\n**INPUT VARIABLES**\n");
-	if(binary.m_inputVariableCount > 0)
+	if(binary.m_inputVariables.getSize() > 0)
 	{
-		for(U i = 0; i < binary.m_inputVariableCount; ++i)
+		for(const ShaderProgramBinaryInput& input : binary.m_inputVariables)
 		{
-			const ShaderProgramBinaryInput& input = binary.m_inputVariables[i];
 			lines.pushBackSprintf(ANKI_TAB "\"%s\" ", &input.m_name[0]);
 			if(input.m_firstSpecializationConstantIndex < MAX_U32)
 			{
@@ -480,14 +497,15 @@ void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAut
 	}
 
 	lines.pushBack("\n**BINARIES**\n");
-	for(U i = 0; i < binary.m_codeBlockCount; ++i)
+	U32 count = 0;
+	for(const ShaderProgramBinaryCode& code : binary.m_codeBlocks)
 	{
 		spirv_cross::CompilerGLSL::Options options;
 		options.vulkan_semantics = true;
 
-		const unsigned int* spvb = reinterpret_cast<const unsigned int*>(binary.m_codeBlocks[i].m_binary);
-		ANKI_ASSERT((binary.m_codeBlocks[i].m_binarySize % (sizeof(unsigned int))) == 0);
-		std::vector<unsigned int> spv(spvb, spvb + binary.m_codeBlocks[i].m_binarySize / sizeof(unsigned int));
+		const unsigned int* spvb = reinterpret_cast<const unsigned int*>(code.m_binary.getBegin());
+		ANKI_ASSERT((code.m_binary.getSize() % (sizeof(unsigned int))) == 0);
+		std::vector<unsigned int> spv(spvb, spvb + code.m_binary.getSize() / sizeof(unsigned int));
 		spirv_cross::CompilerGLSL compiler(spv);
 		compiler.set_common_options(options);
 
@@ -497,15 +515,14 @@ void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAut
 		StringAuto newGlsl(alloc);
 		sourceLines.join("\n" ANKI_TAB ANKI_TAB, newGlsl);
 
-		lines.pushBackSprintf(ANKI_TAB "%" PRIuFAST32 " \n" ANKI_TAB ANKI_TAB "%s\n", i, newGlsl.cstr());
+		lines.pushBackSprintf(ANKI_TAB "%" PRIuFAST32 " \n" ANKI_TAB ANKI_TAB "%s\n", count++, newGlsl.cstr());
 	}
 
 	lines.pushBack("\n**SHADER VARIANTS**\n");
-	for(U i = 0; i < binary.m_variantCount; ++i)
+	count = 0;
+	for(const ShaderProgramBinaryVariant& variant : binary.m_variants)
 	{
-		const ShaderProgramBinaryVariant& variant = binary.m_variants[i];
-
-		lines.pushBackSprintf(ANKI_TAB "%" PRIuFAST32 "\n", i);
+		lines.pushBackSprintf(ANKI_TAB "%" PRIuFAST32 "\n", count++);
 
 		// Misc
 		ANKI_ASSERT(variant.m_activeVariables.getData().getSize() == 2);
@@ -517,12 +534,11 @@ void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAut
 
 		// Mutator values
 		lines.pushBack(ANKI_TAB ANKI_TAB "mutatorValues ");
-		if(variant.m_mutatorValueCount > 0)
+		if(variant.m_mutation.getSize() > 0)
 		{
-			for(U j = 0; j < variant.m_mutatorValueCount; ++j)
+			for(U32 j = 0; j < variant.m_mutation.getSize(); ++j)
 			{
-				lines.pushBackSprintf(
-					"\"%s\" %" PRId32 " ", &binary.m_mutators[j].m_name[0], variant.m_mutatorValues[j]);
+				lines.pushBackSprintf("\"%s\" %" PRId32 " ", &binary.m_mutators[j].m_name[0], variant.m_mutation[j]);
 			}
 		}
 		else
@@ -533,11 +549,10 @@ void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAut
 
 		// Block infos
 		lines.pushBack(ANKI_TAB ANKI_TAB "blockInfos ");
-		if(variant.m_inputVariableCount > 0)
+		if(variant.m_blockInfos.getSize() > 0)
 		{
-			for(U j = 0; j < variant.m_inputVariableCount; ++j)
+			for(const ShaderVariableBlockInfo& inf : variant.m_blockInfos)
 			{
-				const ShaderVariableBlockInfo& inf = variant.m_blockInfos[j];
 				lines.pushBackSprintf("%" PRIi16 "|%" PRIi16 "|%" PRIi16 "|%" PRIi16 " ",
 					inf.m_offset,
 					inf.m_arraySize,
@@ -553,17 +568,17 @@ void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAut
 
 		// Bindings
 		lines.pushBack(ANKI_TAB ANKI_TAB "bindings ");
-		if(variant.m_inputVariableCount > 0)
+		if(variant.m_bindings.getSize() > 0)
 		{
-			for(U j = 0; j < variant.m_inputVariableCount; ++j)
+			for(I32 binding : variant.m_bindings)
 			{
-				if(variant.m_bindings[j] < 0)
+				if(binding < 0)
 				{
 					lines.pushBack("N/A ");
 				}
 				else
 				{
-					lines.pushBackSprintf("%" PRIi16 " ", variant.m_bindings[j]);
+					lines.pushBackSprintf("%" PRIi32 " ", binding);
 				}
 			}
 		}

+ 34 - 0
src/anki/util/Serializer.h

@@ -4,6 +4,7 @@
 // http://www.anki3d.org/LICENSE
 
 #include <anki/util/File.h>
+#include <anki/util/WeakArray.h>
 
 #pragma once
 
@@ -39,6 +40,39 @@ public:
 	}
 };
 
+/// Specialization for WeakArray.
+template<typename T, typename TSize>
+class SerializeFunctor<WeakArray<T, TSize>>
+{
+public:
+	template<typename TSerializer>
+	void operator()(const WeakArray<T, TSize>& x, TSerializer& serializer)
+	{
+		const TSize size = x.getSize();
+		serializer.doDynamicArray("m_array", 0, (x.getSize()) ? &x[0] : nullptr, size);
+		serializer.doValue("m_size", sizeof(void*), size);
+	}
+};
+
+/// Specialization for WeakArray.
+template<typename T, typename TSize>
+class DeserializeFunctor<WeakArray<T, TSize>>
+{
+public:
+	template<typename TDeserializer>
+	void operator()(WeakArray<T, TSize>& x, TDeserializer& deserializer)
+	{
+		TSize size;
+		deserializer.doValue("m_size", sizeof(void*), size);
+		T* arr = nullptr;
+		if(size > 0)
+		{
+			deserializer.doDynamicArray("m_array", 0, arr, size);
+		}
+		x = WeakArray<T, TSize>(arr, size);
+	}
+};
+
 /// Serializes to binary files.
 class BinarySerializer : public NonCopyable
 {

+ 16 - 0
src/anki/util/WeakArray.h

@@ -172,6 +172,14 @@ public:
 		return m_data[m_size - 1];
 	}
 
+	/// Set the array pointer and its size.
+	void setArray(Value* array, Size size)
+	{
+		ANKI_ASSERT((array && size > 0) || (array == nullptr && size == 0));
+		m_data = array;
+		m_size = size;
+	}
+
 	Size getSize() const
 	{
 		return m_size;
@@ -314,6 +322,14 @@ public:
 		return m_data[m_size - 1];
 	}
 
+	/// Set the array pointer and its size.
+	void setArray(Value* array, Size size)
+	{
+		ANKI_ASSERT(array && size > 0 || array == nullptr && size == 0);
+		m_data = array;
+		m_size = size;
+	}
+
 	Size getSize() const
 	{
 		return m_size;

+ 7 - 9
tests/util/Serializer.cpp

@@ -15,22 +15,20 @@ ANKI_TEST(Util, BinarySerializer)
 	b[0].m_array[1] = 3;
 	b[0].m_array[2] = 4;
 	Array<U32, 3> bDarr = {{0xFF12EE34, 0xAA12BB34, 0xCC12DD34}};
-	b[0].m_darraySize = bDarr.getSize();
-	b[0].m_darray = &bDarr[0];
+	b[0].m_darray = bDarr;
 
 	b[1].m_array[0] = 255;
 	b[1].m_array[1] = 127;
 	b[1].m_array[2] = 55;
 	Array<U32, 1> bDarr2 = {{0x12345678}};
-	b[1].m_darraySize = bDarr2.getSize();
-	b[1].m_darray = &bDarr2[0];
+	b[1].m_darray = bDarr2;
 
 	ClassA a = {};
 	a.m_array[0] = 123;
 	a.m_array[1] = 56;
-	a.m_u32 = b.getSize();
+	a.m_u32 = 321;
 	a.m_u64 = 0x123456789ABCDEFF;
-	a.m_darray = &b[0];
+	a.m_darray = b;
 
 	HeapAllocator<U8> alloc(allocAligned, nullptr);
 
@@ -54,13 +52,13 @@ ANKI_TEST(Util, BinarySerializer)
 
 		ANKI_TEST_EXPECT_EQ(pa->m_array[0], a.m_array[0]);
 		ANKI_TEST_EXPECT_EQ(pa->m_u32, a.m_u32);
+		ANKI_TEST_EXPECT_EQ(pa->m_darray.getSize(), a.m_darray.getSize());
 
-		for(U32 i = 0; i < pa->m_u32; ++i)
+		for(U32 i = 0; i < pa->m_darray.getSize(); ++i)
 		{
 			ANKI_TEST_EXPECT_EQ(pa->m_darray[i].m_array[1], b[i].m_array[1]);
-			ANKI_TEST_EXPECT_EQ(pa->m_darray[i].m_darraySize, b[i].m_darraySize);
 
-			for(U32 j = 0; j < pa->m_darray[i].m_darraySize; ++j)
+			for(U32 j = 0; j < pa->m_darray[i].m_darray.getSize(); ++j)
 			{
 				ANKI_TEST_EXPECT_EQ(pa->m_darray[i].m_darray[j], b[i].m_darray[j]);
 			}

+ 10 - 12
tests/util/SerializerTest.h

@@ -17,15 +17,13 @@ class ClassB
 {
 public:
 	Array<U8, 3> m_array;
-	U32* m_darray;
-	U8 m_darraySize;
+	WeakArray<U32> m_darray;
 
 	template<typename TSerializer, typename TClass>
-	static void serializeCommon(TSerializer& serializer, TClass self)
+	static void serializeCommon(TSerializer& s, TClass self)
 	{
-		serializer.doArray("m_array", offsetof(ClassB, m_array), &self.m_array[0], 3);
-		serializer.doValue("m_darraySize", offsetof(ClassB, m_darraySize), self.m_darraySize);
-		serializer.doDynamicArray("m_darray", offsetof(ClassB, m_darray), self.m_darray, self.m_darraySize);
+		s.doArray("m_array", offsetof(ClassB, m_array), &self.m_array[0], 3);
+		s.doValue("m_darray", offsetof(ClassB, m_darray), self.m_darray);
 	}
 
 	template<typename TDeserializer>
@@ -48,15 +46,15 @@ public:
 	Array<U8, 2> m_array;
 	U32 m_u32;
 	U64 m_u64;
-	ClassB* m_darray;
+	WeakArray<ClassB> m_darray;
 
 	template<typename TSerializer, typename TClass>
-	static void serializeCommon(TSerializer& serializer, TClass self)
+	static void serializeCommon(TSerializer& s, TClass self)
 	{
-		serializer.doArray("m_array", offsetof(ClassA, m_array), &self.m_array[0], 2);
-		serializer.doValue("m_u32", offsetof(ClassA, m_u32), self.m_u32);
-		serializer.doValue("m_u64", offsetof(ClassA, m_u64), self.m_u64);
-		serializer.doDynamicArray("m_darray", offsetof(ClassA, m_darray), self.m_darray, self.m_u32);
+		s.doArray("m_array", offsetof(ClassA, m_array), &self.m_array[0], 2);
+		s.doValue("m_u32", offsetof(ClassA, m_u32), self.m_u32);
+		s.doValue("m_u64", offsetof(ClassA, m_u64), self.m_u64);
+		s.doValue("m_darray", offsetof(ClassA, m_darray), self.m_darray);
 	}
 
 	template<typename TDeserializer>

+ 2 - 3
tests/util/SerializerTest.xml

@@ -7,8 +7,7 @@
 		<class name="ClassB">
 			<members>
 				<member name="m_array" type="U8" array_size="3" />
-				<member name="m_darray" type="U32" pointer="true" array_size="m_darraySize" />
-				<member name="m_darraySize" type="U8" />
+				<member name="m_darray" type="WeakArray&lt;U32&gt;" />
 			</members>
 		</class>
 
@@ -17,7 +16,7 @@
 				<member name="m_array" type="U8" array_size="2" />
 				<member name="m_u32" type="U32" />
 				<member name="m_u64" type="U64" />
-				<member name="m_darray" type="ClassB" pointer="true" array_size="m_u32" />
+				<member name="m_darray" type="WeakArray&lt;ClassB&gt;" />
 			</members>
 		</class>
 	</classes>