More work on the compiler

src/anki/resource/ShaderProgramResource2.cpp

@@ -4,22 +4,138 @@
 // http://www.anki3d.org/LICENSE
 
 #include <anki/resource/ShaderProgramResource2.h>
+#include <anki/resource/ResourceManager.h>
+#include <anki/gr/ShaderProgram.h>
+#include <anki/util/Filesystem.h>
 
 namespace anki
 {
 
+ShaderProgramResourceVariant2::ShaderProgramResourceVariant2()
+{
+}
+
+ShaderProgramResourceVariant2::~ShaderProgramResourceVariant2()
+{
+}
+
 ShaderProgramResource2::ShaderProgramResource2(ResourceManager* manager)
 	: ResourceObject(manager)
+	, m_binary(getAllocator())
 {
 }
 
 ShaderProgramResource2::~ShaderProgramResource2()
 {
+	m_mutators.destroy(getAllocator());
 }
 
 Error ShaderProgramResource2::load(const ResourceFilename& filename, Bool async)
 {
+	// Load the binary from the cache. It should have been compiled there
+	StringAuto baseFilename(getTempAllocator());
+	getFilepathFilename(filename, baseFilename);
+	StringAuto binaryFilename(getTempAllocator());
+	binaryFilename.sprintf("%s/%s.bin", getManager().getCacheDirectory().cstr(), binaryFilename.cstr());
+	ANKI_CHECK(m_binary.deserializeFromFile(binaryFilename));
+	const ShaderProgramBinary& binary = m_binary.getBinary();
+
+	// Create the mutators
+	if(binary.m_mutators.getSize() > 0)
+	{
+		m_mutators.create(getAllocator(), binary.m_mutators.getSize());
+
+		for(U32 i = 0; i < binary.m_mutators.getSize(); ++i)
+		{
+			m_mutators[i].m_name = binary.m_mutators[i].m_name.getBegin();
+			ANKI_ASSERT(m_mutators[i].m_name.getLength() > 0);
+			m_mutators[i].m_values = binary.m_mutators[i].m_values;
+		}
+	}
+
+	// TODO create the inputs
+
 	return Error::NONE;
 }
 
+void ShaderProgramResource2::getOrCreateVariant(
+	const ShaderProgramResourceVariantInitInfo2& info, const ShaderProgramResourceVariant2*& variant) const
+{
+	// Sanity checks
+	ANKI_ASSERT(info.m_setMutators.getEnabledBitCount() == m_mutators.getSize());
+
+	// Compute variant hash
+	U64 hash = 0;
+	if(info.m_mutationCount)
+	{
+		hash = computeHash(info.m_mutation.getBegin(), info.m_mutationCount * sizeof(info.m_mutation[0]));
+	}
+
+	if(info.m_constantValueCount)
+	{
+		hash = appendHash(
+			info.m_constantValues.getBegin(), info.m_constantValueCount * sizeof(info.m_constantValues[0]), hash);
+	}
+
+	// Check if the variant is in the cache
+	{
+		RLockGuard<RWMutex> lock(m_mtx);
+
+		auto it = m_variants.find(hash);
+		variant = (it != m_variants.getEnd()) ? *it : nullptr;
+	}
+
+	if(variant != nullptr)
+	{
+		// Done
+		return;
+	}
+
+	// Create the variant
+	WLockGuard<RWMutex> lock(m_mtx);
+
+	ShaderProgramResourceVariant2* v = getAllocator().newInstance<ShaderProgramResourceVariant2>();
+	initVariant(info, *v);
+	m_variants.emplace(getAllocator(), hash, v);
+	variant = v;
+}
+
+void ShaderProgramResource2::initVariant(
+	const ShaderProgramResourceVariantInitInfo2& info, ShaderProgramResourceVariant2& variant) const
+{
+	const ShaderProgramBinary& binary = m_binary.getBinary();
+
+	// Get the binary program variant
+	const ShaderProgramBinaryVariant* binaryVariant = nullptr;
+	if(m_mutators.getSize())
+	{
+		// Create the mutation hash
+		const U64 hash = computeHash(info.m_mutation.getBegin(), info.m_mutationCount * sizeof(info.m_mutation[0]));
+
+		// Search for the mutation in the binary
+		// TODO optimize the search
+		for(const ShaderProgramBinaryMutation& mutation : binary.m_mutations)
+		{
+			if(mutation.m_hash == hash)
+			{
+				binaryVariant = &binary.m_variants[mutation.m_variantIndex];
+				break;
+			}
+		}
+	}
+	else
+	{
+		ANKI_ASSERT(binary.m_variants.getSize() == 1);
+		binaryVariant = &binary.m_variants[0];
+	}
+
+	ANKI_ASSERT(binaryVariant);
+
+	// XXX
+	for(const Input& in : m_inputVars)
+	{
+		(void)in;
+	}
+}
+
 } // end namespace anki

src/anki/resource/ShaderProgramResource2.h

@@ -6,7 +6,7 @@
 #pragma once
 
 #include <anki/resource/ResourceObject.h>
-#include <anki/shader_compiler/Common.h>
+#include <anki/shader_compiler/ShaderProgramCompiler.h>
 #include <anki/gr/utils/Functions.h>
 #include <anki/util/BitSet.h>
 #include <anki/util/String.h>
@@ -17,6 +17,9 @@
 namespace anki
 {
 
+// Forward
+class ShaderProgramResourceVariantInitInfo2;
+
 /// @addtogroup resource
 /// @{
 
@@ -25,17 +28,8 @@ namespace anki
 class ShaderProgramResourceMutator2 : public NonCopyable
 {
 public:
-	String m_name;
-	DynamicArray<MutatorValue> m_values;
-};
-
-/// @memberof ShaderProgramResource2
-class ShaderProgramResourcePartialMutation2 : public NonCopyable
-{
-public:
-	const ShaderProgramResourceMutator2* m_mutator = nullptr;
-	MutatorValue m_value = 0;
-	U64 m_hash = 0;
+	CString m_name;
+	ConstWeakArray<MutatorValue> m_values;
 };
 
 /// Shader program resource variant.
@@ -45,6 +39,7 @@ public:
 	String m_name;
 	Bool m_constant = false;
 	ShaderVariableDataType m_dataType = ShaderVariableDataType::NONE;
+	U32 m_index = MAX_U32;
 
 	Bool isTexture() const
 	{
@@ -66,42 +61,29 @@ public:
 /// Shader program resource variant.
 class ShaderProgramResourceVariant2
 {
-public:
-	// TODO
-};
+	friend class ShaderProgramResource2;
 
-/// The value of a constant.
-class ShaderProgramResourceConstantValue2
-{
 public:
-	union
-	{
-		I32 m_int;
-		IVec2 m_ivec2;
-		IVec3 m_ivec3;
-		IVec4 m_ivec4;
-
-		U32 m_uint;
-		UVec2 m_uvec2;
-		UVec3 m_uvec3;
-		UVec4 m_uvec4;
+	ShaderProgramResourceVariant2();
 
-		F32 m_float;
-		Vec2 m_vec2;
-		Vec3 m_vec3;
-		Vec4 m_vec4;
-	};
+	~ShaderProgramResourceVariant2();
 
-	U32 m_inputVariableIndex;
-	U8 _m_padding[sizeof(Vec4) - sizeof(m_inputVariableIndex)];
+	const ShaderProgramPtr& getProgram() const
+	{
+		return m_prog;
+	}
 
-	ShaderProgramResourceConstantValue2()
+	/// Return true of the the variable is active.
+	Bool variableActive(const ShaderProgramResourceInputVariable2& var) const
 	{
-		zeroMemory(*this);
+		return m_activeInputVars.get(var.m_index);
 	}
-};
 
-static_assert(sizeof(ShaderProgramResourceConstantValue2) == sizeof(Vec4) * 2, "Need it to be packed");
+private:
+	ShaderProgramPtr m_prog;
+
+	BitSet<128, U64> m_activeInputVars = {false};
+};
 
 /// Shader program resource. It loads special AnKi programs.
 class ShaderProgramResource2 : public ResourceObject
@@ -158,91 +140,105 @@ public:
 		return !!(m_shaderStages & ShaderTypeBit::TESSELLATION_EVALUATION);
 	}
 
+	/// Get or create a graphics shader program variant.
+	/// @note It's thread-safe.
+	void getOrCreateVariant(
+		const ShaderProgramResourceVariantInitInfo2& info, const ShaderProgramResourceVariant2*& variant) const;
+
 private:
 	using Mutator = ShaderProgramResourceMutator2;
 	using Input = ShaderProgramResourceInputVariable2;
 
+	ShaderProgramBinaryWrapper m_binary;
+
 	DynamicArray<Input> m_inputVars;
 	DynamicArray<Mutator> m_mutators;
 
 	mutable HashMap<U64, ShaderProgramResourceVariant2*> m_variants;
-	mutable Mutex m_mtx;
+	mutable RWMutex m_mtx;
 
 	ShaderTypeBit m_shaderStages = ShaderTypeBit::NONE;
+
+	void initVariant(const ShaderProgramResourceVariantInitInfo2& info, ShaderProgramResourceVariant2& variant) const;
 };
 
-/// Smart initializer of multiple ShaderProgramResourceConstantValue.
-class ShaderProgramResourceConstantValueInitList2
+/// The value of a constant.
+class ShaderProgramResourceConstantValue2
 {
 public:
-	ShaderProgramResourceConstantValueInitList2(ShaderProgramResource2Ptr ptr)
-		: m_ptr(ptr)
-	{
-	}
-
-	~ShaderProgramResourceConstantValueInitList2()
+	union
 	{
-	}
+		I32 m_int;
+		IVec2 m_ivec2;
+		IVec3 m_ivec3;
+		IVec4 m_ivec4;
 
-	template<typename T>
-	ShaderProgramResourceConstantValueInitList2& add(CString name, const T& t)
-	{
-		const ShaderProgramResourceInputVariable2* in = m_ptr->tryFindInputVariable(name);
-		ANKI_ASSERT(in);
-		ANKI_ASSERT(in->m_constant);
-		ANKI_ASSERT(in->m_dataType == getShaderVariableTypeFromTypename<T>());
-		m_constantValues[m_count].m_inputVariableIndex = U32(in - m_ptr->getInputVariables().getBegin());
-		memcpy(&m_constantValues[m_count].m_int, &t, sizeof(T));
-		++m_count;
-		return *this;
-	}
+		F32 m_float;
+		Vec2 m_vec2;
+		Vec3 m_vec3;
+		Vec4 m_vec4;
+	};
 
-	ConstWeakArray<ShaderProgramResourceConstantValue2> get() const
-	{
-		// TODO check if all are set
-		return ConstWeakArray<ShaderProgramResourceConstantValue2>(&m_constantValues[0], m_count);
-	}
+	U32 m_inputVariableIndex;
+	U8 _m_padding[sizeof(Vec4) - sizeof(m_inputVariableIndex)];
 
-	ShaderProgramResourceConstantValue2& operator[](U32 idx)
+	ShaderProgramResourceConstantValue2()
 	{
-		return m_constantValues[idx];
+		zeroMemory(*this);
 	}
-
-private:
-	ShaderProgramResource2Ptr m_ptr;
-	U32 m_count = 0;
-	Array<ShaderProgramResourceConstantValue2, 64> m_constantValues;
 };
+static_assert(sizeof(ShaderProgramResourceConstantValue2) == sizeof(Vec4) * 2, "Need it to be packed");
 
-/// Smart initializer of multiple ShaderProgramResourceMutation2.
-class ShaderProgramResourceMutationInitList2
+/// Smart initializer of multiple ShaderProgramResourceConstantValue.
+class ShaderProgramResourceVariantInitInfo2
 {
+	friend class ShaderProgramResource2;
+
 public:
-	ShaderProgramResourceMutationInitList2(ShaderProgramResource2Ptr ptr)
+	ShaderProgramResourceVariantInitInfo2(const ShaderProgramResource2Ptr& ptr)
 		: m_ptr(ptr)
 	{
 	}
 
-	~ShaderProgramResourceMutationInitList2()
+	~ShaderProgramResourceVariantInitInfo2()
 	{
 	}
 
-	ShaderProgramResourceMutationInitList2& add(CString name, MutatorValue t)
+	template<typename T>
+	ShaderProgramResourceVariantInitInfo2& addConstant(CString name, const T& value)
+	{
+		const ShaderProgramResourceInputVariable2* in = m_ptr->tryFindInputVariable(name);
+		ANKI_ASSERT(in);
+		ANKI_ASSERT(in->m_constant);
+		ANKI_ASSERT(in->m_dataType == getShaderVariableTypeFromTypename<T>());
+		m_constantValues[m_constantValueCount].m_inputVariableIndex = U32(in - m_ptr->getInputVariables().getBegin());
+		memcpy(&m_constantValues[m_constantValueCount].m_int, &value, sizeof(T));
+		++m_constantValueCount;
+		return *this;
+	}
+
+	ShaderProgramResourceVariantInitInfo2& addMutation(CString name, MutatorValue t)
 	{
 		const ShaderProgramResourceMutator2* m = m_ptr->tryFindMutator(name);
+		const PtrSize idx = m - m_ptr->getMutators().getBegin();
 		ANKI_ASSERT(m);
-		m_mutation[m_count] = t;
-		m_setMutators.set(m - m_ptr->getMutators().getBegin());
-		++m_count;
+		m_mutation[idx] = t;
+		m_setMutators.set(idx);
+		++m_mutationCount;
 		return *this;
 	}
 
 private:
-	static constexpr U32 MAX_MUTATORS = 64;
+	static constexpr U32 MAX_CONSTANTS = 32;
+	static constexpr U32 MAX_MUTATORS = 32;
 
 	ShaderProgramResource2Ptr m_ptr;
-	U32 m_count = 0;
-	Array<MutatorValue, MAX_MUTATORS> m_mutation;
+
+	U32 m_constantValueCount = 0;
+	Array<ShaderProgramResourceConstantValue2, MAX_CONSTANTS> m_constantValues;
+
+	U32 m_mutationCount = 0;
+	Array<MutatorValue, MAX_MUTATORS> m_mutation; ///< The order of storing the values is important. It will be hashed.
 	BitSet<MAX_MUTATORS> m_setMutators = {false};
 };
 /// @}

src/anki/shader_compiler/ShaderProgramBinary.h

@@ -298,7 +298,7 @@ public:
 	WeakArray<ShaderProgramBinaryMutator> m_mutators;
 	WeakArray<ShaderProgramBinaryCodeBlock> m_codeBlocks;
 	WeakArray<ShaderProgramBinaryVariant> m_variants;
-	WeakArray<ShaderProgramBinaryMutation> m_mutations;
+	WeakArray<ShaderProgramBinaryMutation> m_mutations; ///< I'ts sorted using the mutation's hash.
 	ShaderTypeBit m_presentShaderTypes = ShaderTypeBit::NONE;
 
 	template<typename TSerializer, typename TClass>

src/anki/shader_compiler/ShaderProgramBinary.xml

@@ -88,7 +88,7 @@
 				<member name="m_mutators" type="WeakArray&lt;ShaderProgramBinaryMutator&gt;" />
 				<member name="m_codeBlocks" type="WeakArray&lt;ShaderProgramBinaryCodeBlock&gt;" />
 				<member name="m_variants" type="WeakArray&lt;ShaderProgramBinaryVariant&gt;" />
-				<member name="m_mutations" type="WeakArray&lt;ShaderProgramBinaryMutation&gt;" />
+				<member name="m_mutations" type="WeakArray&lt;ShaderProgramBinaryMutation&gt;" comment="I'ts sorted using the mutation's hash" />
 				<member name="m_presentShaderTypes" type="ShaderTypeBit" constructor="= ShaderTypeBit::NONE" />
 			</members>
 		</class>

src/anki/shader_compiler/ShaderProgramCompiler.cpp

@@ -401,6 +401,11 @@ Error compileShaderProgram(CString fname,
 		binary.m_mutations[0].m_variantIndex = 0;
 	}
 
+	// Sort the mutations
+	std::sort(binary.m_mutations.getBegin(),
+		binary.m_mutations.getEnd(),
+		[](const ShaderProgramBinaryMutation& a, const ShaderProgramBinaryMutation& b) { return a.m_hash < b.m_hash; });
+
 	// Misc
 	binary.m_presentShaderTypes = parser.getShaderTypes();
 

src/anki/shader_compiler/ShaderProgramParser.cpp

@@ -69,27 +69,44 @@ static const char* SHADER_HEADER = R"(#version 450 core
 #define _ANKI_CONCATENATE(a, b) a##b
 #define ANKI_CONCATENATE(a, b) _ANKI_CONCATENATE(a, b)
 
-#define ANKI_SPECIALIZATION_CONSTANT_I32(x, id, defltVal) layout(constant_id = id) const I32 x = defltVal
-
-#define ANKI_SPECIALIZATION_CONSTANT_IVEC2(x, id, defltVal) \
-	layout(constant_id = id) const I32 ANKI_CONCATENATE(x, _0) = defltVal[0]; \
-	layout(constant_id = id + 1) const I32 ANKI_CONCATENATE(x, _1) = defltVal[1]
-
-#define ANKI_SPECIALIZATION_CONSTANT_IVEC3(x, id, defltVal) \
-	layout(constant_id = id) const I32 ANKI_CONCATENATE(x, _0) = defltVal[0]; \
-	layout(constant_id = id + 1) const I32 ANKI_CONCATENATE(x, _1) = defltVal[1]; \
-	layout(constant_id = id + 2) const I32 ANKI_CONCATENATE(x, _2) = defltVal[2]
-
-#define ANKI_SPECIALIZATION_CONSTANT_F32(x, id, defltVal) layout(constant_id = id) const F32 x = defltVal
-
-#define ANKI_SPECIALIZATION_CONSTANT_VEC2(x, id, defltVal) \
-	layout(constant_id = id) const F32 ANKI_CONCATENATE(x, _0) = defltVal[0]; \
-	layout(constant_id = id + 1) const F32 ANKI_CONCATENATE(x, _1) = defltVal[1]
-
-#define ANKI_SPECIALIZATION_CONSTANT_VEC3(x, id, defltVal) \
-	layout(constant_id = id) const F32 ANKI_CONCATENATE(x, _0) = defltVal[0]; \
-	layout(constant_id = id + 1) const F32 ANKI_CONCATENATE(x, _1) = defltVal[1]; \
-	layout(constant_id = id + 2) const F32 ANKI_CONCATENATE(x, _2) = defltVal[2]
+#define ANKI_SPECIALIZATION_CONSTANT_X(type, x, id, defltVal) layout(constant_id = id) const type x = defltVal;
+
+#define ANKI_SPECIALIZATION_CONSTANT_X2(type, componentType, x, id, defltVal) \
+	layout(constant_id = id + 0) const componentType ANKI_CONCATENATE(_anki_const_0_2_, x) = defltVal[0]; \
+	layout(constant_id = id + 1) const componentType ANKI_CONCATENATE(_anki_const_1_2_, x) = defltVal[1]; \
+	const type x = type( \
+		ANKI_CONCATENATE(_anki_const_0_2_, x), \
+		ANKI_CONCATENATE(_anki_const_1_2_, x));
+
+#define ANKI_SPECIALIZATION_CONSTANT_X3(type, componentType, x, id, defltVal) \
+	layout(constant_id = id + 0) const componentType ANKI_CONCATENATE(_anki_const_0_3_, x) = defltVal[0]; \
+	layout(constant_id = id + 1) const componentType ANKI_CONCATENATE(_anki_const_1_3_, x) = defltVal[1]; \
+	layout(constant_id = id + 2) const componentType ANKI_CONCATENATE(_anki_const_2_3_, x) = defltVal[2]; \
+	const type x = type( \
+		ANKI_CONCATENATE(_anki_const_0_3_, x), \
+		ANKI_CONCATENATE(_anki_const_1_3_, x), \
+		ANKI_CONCATENATE(_anki_const_2_3_, x));
+
+#define ANKI_SPECIALIZATION_CONSTANT_X4(type, componentType, x, id, defltVal) \
+	layout(constant_id = id + 0) const componentType ANKI_CONCATENATE(_anki_const_0_4_, x) = defltVal[0]; \
+	layout(constant_id = id + 1) const componentType ANKI_CONCATENATE(_anki_const_1_4_, x) = defltVal[1]; \
+	layout(constant_id = id + 2) const componentType ANKI_CONCATENATE(_anki_const_2_4_, x) = defltVal[2]; \
+	layout(constant_id = id + 3) const componentType ANKI_CONCATENATE(_anki_const_3_4_, x) = defltVal[3]; \
+	const type x = type( \
+		ANKI_CONCATENATE(_anki_const_0_4_, x), \
+		ANKI_CONCATENATE(_anki_const_1_4_, x), \
+		ANKI_CONCATENATE(_anki_const_2_4_, x)); \
+		ANKI_CONCATENATE(_anki_const_3_4_, x));
+
+#define ANKI_SPECIALIZATION_CONSTANT_I32(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X(I32, x, id, defltVal)
+#define ANKI_SPECIALIZATION_CONSTANT_IVEC2(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X2(IVec2, I32, x, id, defltVal)
+#define ANKI_SPECIALIZATION_CONSTANT_IVEC3(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X3(IVec3, I32, x, id, defltVal)
+#define ANKI_SPECIALIZATION_CONSTANT_IVEC4(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X4(IVec4, I32, x, id, defltVal)
+
+#define ANKI_SPECIALIZATION_CONSTANT_F32(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X(F32, x, id, defltVal)
+#define ANKI_SPECIALIZATION_CONSTANT_VEC2(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X2(Vec2, F32, x, id, defltVal)
+#define ANKI_SPECIALIZATION_CONSTANT_VEC3(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X3(Vec3, F32, x, id, defltVal)
+#define ANKI_SPECIALIZATION_CONSTANT_VEC4(x, id, defltVal) ANKI_SPECIALIZATION_CONSTANT_X4(Vec4, F32, x, id, defltVal)
 )";
 
 ShaderProgramParser::ShaderProgramParser(CString fname,