Shader compiler is almost complete. Needs more testing

src/anki/resource/ShaderProgramPreProcessor.h

@@ -148,7 +148,7 @@ public:
 		return m_inputs;
 	}
 
-	ShaderTypeBit getShaderStages() const
+	ShaderTypeBit getShaderTypes() const
 	{
 		return m_shaderTypes;
 	}

src/anki/resource/ShaderProgramResource.cpp

@@ -230,7 +230,7 @@ Error ShaderProgramResource::load(const ResourceFilename& filename, Bool async)
 
 	// Set some other vars
 	m_descriptorSet = U8(pp.getDescritproSet());
-	m_shaderStages = pp.getShaderStages();
+	m_shaderStages = pp.getShaderTypes();
 	if(instancedMutatorIdx != MAX_U)
 	{
 		m_instancingMutator = &m_mutators[instancedMutatorIdx];

src/anki/shader_compiler/ShaderProgramCompiler.cpp

@@ -7,11 +7,12 @@
 #include <anki/shader_compiler/ShaderProgramParser.h>
 #include <anki/shader_compiler/Glslang.h>
 #include <anki/util/Serializer.h>
+#include <SPIRV-Cross/spirv_glsl.hpp>
 
 namespace anki
 {
 
-static const char* SHADER_BINARY_MAGIC = "ANKISHB1";
+static const char* SHADER_BINARY_MAGIC = "ANKISDR1";
 
 Error ShaderProgramBinaryWrapper::serializeToFile(CString fname) const
 {
@@ -23,6 +24,12 @@ Error ShaderProgramBinaryWrapper::serializeToFile(CString fname) const
 	BinarySerializer serializer;
 	ANKI_CHECK(serializer.serialize(*m_binary, m_alloc, file));
 
+	if(memcmp(SHADER_BINARY_MAGIC, &m_binary->m_magic[0], 0) != 0)
+	{
+		ANKI_SHADER_COMPILER_LOGE("Corrupted or wrong version of shader binary: %s", fname.cstr());
+		return Error::USER_DATA;
+	}
+
 	return Error::NONE;
 }
 
@@ -41,11 +48,73 @@ Error ShaderProgramBinaryWrapper::deserializeFromFile(CString fname)
 	return Error::NONE;
 }
 
-/// XXX
+void ShaderProgramBinaryWrapper::cleanup()
+{
+	if(m_binary == nullptr)
+	{
+		return;
+	}
+
+	if(!m_singleAllocation)
+	{
+		for(PtrSize i = 0; i < m_binary->m_mutatorCount; ++i)
+		{
+			m_alloc.getMemoryPool().free(m_binary->m_mutators[i].m_values);
+		}
+
+		m_alloc.getMemoryPool().free(m_binary->m_inputVariables);
+
+		for(PtrSize i = 0; i < m_binary->m_codeBlockCount; ++i)
+		{
+			m_alloc.getMemoryPool().free(m_binary->m_codeBlocks[i].m_binary);
+		}
+		m_alloc.getMemoryPool().free(m_binary->m_codeBlocks);
+
+		for(PtrSize i = 0; i < m_binary->m_variantCount; ++i)
+		{
+			m_alloc.getMemoryPool().free(m_binary->m_variants[i].m_mutatorValues);
+			m_alloc.getMemoryPool().free(m_binary->m_variants[i].m_blockInfos);
+			m_alloc.getMemoryPool().free(m_binary->m_variants[i].m_bindings);
+		}
+		m_alloc.getMemoryPool().free(m_binary->m_variants);
+	}
+
+	m_alloc.getMemoryPool().free(m_binary);
+}
+
+/// Spin the dials. Used to compute all mutator combinations.
 static Bool spinDials(DynamicArrayAuto<U32> dials, ConstWeakArray<ShaderProgramParserMutator> mutators)
 {
-	// XXX
-	return false;
+	ANKI_ASSERT(dials.getSize() == mutators.getSize() && dials.getSize() > 0);
+	constexpr Bool done = true;
+	constexpr Bool notDone = false;
+
+	U crntDial = dials.getSize() - 1;
+	while(true)
+	{
+		// Turn dial
+		++dials[crntDial];
+
+		if(dials[crntDial] >= mutators[crntDial].getValues().getSize())
+		{
+			if(crntDial == 0)
+			{
+				// Reached the 1st dial, stop spinning
+				return done;
+			}
+			else
+			{
+				dials[crntDial] = 0;
+				--crntDial;
+			}
+		}
+		else
+		{
+			return notDone;
+		}
+	}
+
+	return notDone;
 }
 
 static Error compileVariant(ConstWeakArray<ShaderProgramParserMutatorState> mutatorStates,
@@ -78,7 +147,7 @@ static Error compileVariant(ConstWeakArray<ShaderProgramParserMutatorState> muta
 	// Compile stages
 	for(ShaderType shaderType = ShaderType::FIRST; shaderType < ShaderType::COUNT; ++shaderType)
 	{
-		if(!(shaderTypeToBit(shaderType) & parser.getShaderStages()))
+		if(!(shaderTypeToBit(shaderType) & parser.getShaderTypes()))
 		{
 			variant.m_binaryIndices[shaderType] = MAX_U32;
 			continue;
@@ -167,14 +236,14 @@ static Error compileVariant(ConstWeakArray<ShaderProgramParserMutatorState> muta
 	return Error::NONE;
 }
 
-Error ShaderProgramCompiler::compile(CString fname,
+Error compileShaderProgram(CString fname,
 	ShaderProgramFilesystemInterface& fsystem,
 	GenericMemoryPoolAllocator<U8> tempAllocator,
 	U32 pushConstantsSize,
 	U32 backendMinor,
 	U32 backendMajor,
 	GpuVendor gpuVendor,
-	ShaderProgramBinaryWrapper& binaryW) const
+	ShaderProgramBinaryWrapper& binaryW)
 {
 	// Initialize the binary
 	binaryW.cleanup();
@@ -267,7 +336,7 @@ Error ShaderProgramCompiler::compile(CString fname,
 			ShaderProgramBinaryVariant& variant = *variants.emplaceBack();
 			ANKI_CHECK(
 				compileVariant(states, parser, variant, codeBlocks, codeBlockHashes, tempAllocator, binaryAllocator));
-		} while(spinDials(dials, parser.getMutators()));
+		} while(!spinDials(dials, parser.getMutators()));
 
 		// Store to binary
 		binary.m_variantCount = U32(variants.getSizeInBytes());
@@ -285,18 +354,82 @@ Error ShaderProgramCompiler::compile(CString fname,
 
 		binary.m_variantCount = 1;
 		binary.m_variants = binaryAllocator.newInstance<ShaderProgramBinaryVariant>();
-		binary.m_variants = {};
 
 		ANKI_CHECK(compileVariant(
 			states, parser, binary.m_variants[0], codeBlocks, codeBlockHashes, tempAllocator, binaryAllocator));
-		ANKI_ASSERT(codeBlocks.getSize() == 1);
+		ANKI_ASSERT(codeBlocks.getSize() == U32(__builtin_popcount(U32(parser.getShaderTypes()))));
 
 		binary.m_codeBlockCount = 1;
 		PtrSize size, storage;
 		codeBlocks.moveAndReset(binary.m_codeBlocks, size, storage);
 	}
 
+	// Misc
+	binary.m_descriptorSet = parser.getDescritproSet();
+	binary.m_presentShaderTypes = parser.getShaderTypes();
+
 	return Error::NONE;
 }
 
+void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAuto& humanReadable)
+{
+	GenericMemoryPoolAllocator<U8> alloc = humanReadable.getAllocator();
+	StringListAuto lines(alloc);
+
+	lines.pushBack("MUTATORS\n");
+	if(binary.m_mutatorCount > 0)
+	{
+		for(U i = 0; i < binary.m_mutatorCount; ++i)
+		{
+			lines.pushBackSprintf("\"%s\"", &binary.m_mutators[i].m_name[0]);
+			for(U j = 0; j < binary.m_mutators[i].m_valueCount; ++j)
+			{
+				lines.pushBackSprintf(" %d", binary.m_mutators[i].m_values[j]);
+			}
+			lines.pushBack("\n");
+		}
+	}
+	else
+	{
+		lines.pushBack("N/A\n");
+	}
+
+	lines.pushBack("\nINPUT VARIABLES\n");
+	if(binary.m_inputVariableCount > 0)
+	{
+		for(U i = 0; i < binary.m_inputVariableCount; ++i)
+		{
+			const ShaderProgramBinaryInput& input = binary.m_inputVariables[i];
+			lines.pushBackSprintf("\"%s\" ", &input.m_name[0]);
+			lines.pushBackSprintf("firstSpecializationConstant %" PRIu32 " ", input.m_firstSpecializationConstantIndex);
+			lines.pushBackSprintf("instanced  %" PRIu32 " ", U32(input.m_instanced));
+			lines.pushBackSprintf("dataType %" PRIu8 "\n", U8(input.m_dataType));
+		}
+	}
+	else
+	{
+		lines.pushBack("N/A\n");
+	}
+
+	lines.pushBack("\nBINARIES\n");
+	for(U i = 0; i < binary.m_codeBlockCount; ++i)
+	{
+		spirv_cross::CompilerGLSL::Options options;
+		options.vulkan_semantics = true;
+
+		const unsigned int* spvb = reinterpret_cast<const unsigned int*>(binary.m_codeBlocks[i].m_binary);
+		ANKI_ASSERT((binary.m_codeBlocks[i].m_binarySize % (sizeof(unsigned int))) == 0);
+		std::vector<unsigned int> spv(spvb, spvb + binary.m_codeBlocks[i].m_binarySize / sizeof(unsigned int));
+		spirv_cross::CompilerGLSL compiler(spv);
+		compiler.set_common_options(options);
+
+		std::string glsl = compiler.compile();
+		lines.pushBackSprintf("idx %" PRIuFAST32 ":\n%s\n--------\n", i, glsl.c_str());
+	}
+
+	// TODO variants
+
+	lines.join("", humanReadable);
+}
+
 } // end namespace anki

src/anki/shader_compiler/ShaderProgramCompiler.h

@@ -19,7 +19,14 @@ namespace anki
 /// @memberof ShaderProgramCompiler
 class ShaderProgramBinaryWrapper : public NonCopyable
 {
-	friend class ShaderProgramCompiler;
+	friend Error compileShaderProgram(CString fname,
+		ShaderProgramFilesystemInterface& fsystem,
+		GenericMemoryPoolAllocator<U8> tempAllocator,
+		U32 pushConstantsSize,
+		U32 backendMinor,
+		U32 backendMajor,
+		GpuVendor gpuVendor,
+		ShaderProgramBinaryWrapper& binary);
 
 public:
 	ShaderProgramBinaryWrapper(GenericMemoryPoolAllocator<U8> alloc)
@@ -45,18 +52,17 @@ private:
 };
 
 /// Takes an AnKi special shader program and spits a binary.
-class ShaderProgramCompiler : public NonCopyable
-{
-public:
-	ANKI_USE_RESULT Error compile(CString fname,
-		ShaderProgramFilesystemInterface& fsystem,
-		GenericMemoryPoolAllocator<U8> tempAllocator,
-		U32 pushConstantsSize,
-		U32 backendMinor,
-		U32 backendMajor,
-		GpuVendor gpuVendor,
-		ShaderProgramBinaryWrapper& binary) const;
-};
+ANKI_USE_RESULT Error compileShaderProgram(CString fname,
+	ShaderProgramFilesystemInterface& fsystem,
+	GenericMemoryPoolAllocator<U8> tempAllocator,
+	U32 pushConstantsSize,
+	U32 backendMinor,
+	U32 backendMajor,
+	GpuVendor gpuVendor,
+	ShaderProgramBinaryWrapper& binary);
+
+/// XXX
+void disassembleShaderProgramBinary(const ShaderProgramBinary& binary, StringAuto& humanReadable);
 /// @}
 
 } // end namespace anki

src/anki/shader_compiler/ShaderProgramParser.h

@@ -238,7 +238,7 @@ public:
 		return m_inputs;
 	}
 
-	ShaderTypeBit getShaderStages() const
+	ShaderTypeBit getShaderTypes() const
 	{
 		return m_shaderTypes;
 	}

src/anki/util/Memory.cpp

@@ -147,45 +147,6 @@ Bool BaseMemoryPool::isCreated() const
 	return m_allocCb != nullptr;
 }
 
-void* BaseMemoryPool::allocate(PtrSize size, PtrSize alignmentBytes)
-{
-	void* out = nullptr;
-	switch(m_type)
-	{
-	case Type::HEAP:
-		out = static_cast<HeapMemoryPool*>(this)->allocate(size, alignmentBytes);
-		break;
-	case Type::STACK:
-		out = static_cast<StackMemoryPool*>(this)->allocate(size, alignmentBytes);
-		break;
-	case Type::CHAIN:
-		out = static_cast<ChainMemoryPool*>(this)->allocate(size, alignmentBytes);
-		break;
-	default:
-		ANKI_ASSERT(0);
-	}
-
-	return out;
-}
-
-void BaseMemoryPool::free(void* ptr)
-{
-	switch(m_type)
-	{
-	case Type::HEAP:
-		static_cast<HeapMemoryPool*>(this)->free(ptr);
-		break;
-	case Type::STACK:
-		static_cast<StackMemoryPool*>(this)->free(ptr);
-		break;
-	case Type::CHAIN:
-		static_cast<ChainMemoryPool*>(this)->free(ptr);
-		break;
-	default:
-		ANKI_ASSERT(0);
-	}
-}
-
 HeapMemoryPool::HeapMemoryPool()
 	: BaseMemoryPool(Type::HEAP)
 {
@@ -250,6 +211,11 @@ void HeapMemoryPool::free(void* ptr)
 {
 	ANKI_ASSERT(isCreated());
 
+	if(ANKI_UNLIKELY(ptr == nullptr))
+	{
+		return;
+	}
+
 #if ANKI_MEM_SIGNATURES
 	U8* memU8 = static_cast<U8*>(ptr);
 	memU8 -= m_headerSize;
@@ -433,6 +399,11 @@ void StackMemoryPool::free(void* ptr)
 {
 	ANKI_ASSERT(isCreated());
 
+	if(ANKI_UNLIKELY(ptr == nullptr))
+	{
+		return;
+	}
+
 	// ptr shouldn't be null or not aligned. If not aligned it was not
 	// allocated by this class
 	ANKI_ASSERT(ptr != nullptr && isAligned(m_alignmentBytes, ptr));
@@ -592,6 +563,7 @@ void* ChainMemoryPool::allocate(PtrSize size, PtrSize alignment)
 void ChainMemoryPool::free(void* ptr)
 {
 	ANKI_ASSERT(isCreated());
+
 	if(ANKI_UNLIKELY(ptr == nullptr))
 	{
 		return;

src/anki/util/Memory.h

@@ -53,20 +53,6 @@ void* allocAligned(void* userData, void* ptr, PtrSize size, PtrSize alignment);
 class BaseMemoryPool : public NonCopyable
 {
 public:
-	/// Pool type.
-	enum class Type : U8
-	{
-		NONE,
-		HEAP,
-		STACK,
-		CHAIN
-	};
-
-	BaseMemoryPool(Type type)
-		: m_type(type)
-	{
-	}
-
 	virtual ~BaseMemoryPool();
 
 	/// Allocate memory. This operation MAY be thread safe
@@ -110,6 +96,15 @@ public:
 	}
 
 protected:
+	/// Pool type.
+	enum class Type : U8
+	{
+		NONE,
+		HEAP,
+		STACK,
+		CHAIN
+	};
+
 	/// User allocation function.
 	AllocAlignedCallback m_allocCb = nullptr;
 
@@ -119,15 +114,20 @@ protected:
 	/// Allocations count.
 	Atomic<U32> m_allocationsCount = {0};
 
+	BaseMemoryPool(Type type)
+		: m_type(type)
+	{
+	}
+
 	/// Check if already created.
 	Bool isCreated() const;
 
 private:
-	/// Type.
-	Type m_type = Type::NONE;
-
 	/// Refcount.
 	Atomic<U32> m_refcount = {0};
+
+	/// Type.
+	Type m_type = Type::NONE;
 };
 
 /// A dummy interface to match the StackMemoryPool and ChainMemoryPool interfaces in order to be used by the same
@@ -370,6 +370,41 @@ private:
 	/// Destroy a chunk.
 	void destroyChunk(Chunk* ch);
 };
+
+inline void* BaseMemoryPool::allocate(PtrSize size, PtrSize alignmentBytes)
+{
+	void* out = nullptr;
+	switch(m_type)
+	{
+	case Type::HEAP:
+		out = static_cast<HeapMemoryPool*>(this)->allocate(size, alignmentBytes);
+		break;
+	case Type::STACK:
+		out = static_cast<StackMemoryPool*>(this)->allocate(size, alignmentBytes);
+		break;
+	default:
+		ANKI_ASSERT(m_type == Type::CHAIN);
+		out = static_cast<ChainMemoryPool*>(this)->allocate(size, alignmentBytes);
+	}
+
+	return out;
+}
+
+inline void BaseMemoryPool::free(void* ptr)
+{
+	switch(m_type)
+	{
+	case Type::HEAP:
+		static_cast<HeapMemoryPool*>(this)->free(ptr);
+		break;
+	case Type::STACK:
+		static_cast<StackMemoryPool*>(this)->free(ptr);
+		break;
+	default:
+		ANKI_ASSERT(m_type == Type::CHAIN);
+		static_cast<ChainMemoryPool*>(this)->free(ptr);
+	}
+}
 /// @}
 
 } // end namespace anki

src/anki/util/String.h

@@ -706,6 +706,11 @@ public:
 		return *this;
 	}
 
+	GenericMemoryPoolAllocator<Char> getAllocator() const
+	{
+		return m_alloc;
+	}
+
 	/// Initialize using a const string.
 	void create(const CStringType& cstr)
 	{

tests/shader_compiler/ShaderProgramCompiler.cpp

@@ -0,0 +1,56 @@
+// Copyright (C) 2009-2019, Panagiotis Christopoulos Charitos and contributors.
+// All rights reserved.
+// Code licensed under the BSD License.
+// http://www.anki3d.org/LICENSE
+
+#include <tests/framework/Framework.h>
+#include <anki/shader_compiler/ShaderProgramCompiler.h>
+
+ANKI_TEST(ShaderCompiler, ShaderProgramCompiler)
+{
+	const CString sourceCode = R"(
+#pragma anki start vert
+out gl_PerVertex
+{
+	Vec4 gl_Position;
+};
+
+void main()
+{
+	gl_Position = Vec4(gl_Position);
+}
+#pragma anki end
+
+#pragma anki start frag
+layout(location = 0) out Vec3 out_color;
+
+void main()
+{
+	out_color = Vec3(0.0);
+}
+#pragma anki end
+	)";
+
+	// Write the file
+	{
+		File file;
+		ANKI_TEST_EXPECT_NO_ERR(file.open("test.glslp", FileOpenFlag::WRITE));
+		ANKI_TEST_EXPECT_NO_ERR(file.writeText(sourceCode));
+	}
+
+	class Fsystem : public ShaderProgramFilesystemInterface
+	{
+	public:
+		Error readAllText(CString filename, StringAuto& txt) final
+		{
+			File file;
+			ANKI_CHECK(file.open(filename, FileOpenFlag::READ));
+			ANKI_CHECK(file.readAllText(txt));
+			return Error::NONE;
+		}
+	} fsystem;
+
+	HeapAllocator<U8> alloc(allocAligned, nullptr);
+	ShaderProgramBinaryWrapper binary(alloc);
+	ANKI_TEST_EXPECT_NO_ERR(compileShaderProgram("test.glslp", fsystem, alloc, 128, 1, 1, GpuVendor::AMD, binary));
+}