Add functionality to decrease shader compilation times

AnKi/Importer/GltfImporterMaterial.cpp

@@ -51,7 +51,7 @@ static ImporterString getTextureUri(const cgltf_texture_view& view)
 	return out;
 }
 
-/// Read the texture and find out if
+/// Read the texture and find out if it has constant color. If a component is not constant return -1
 static Error findConstantColorsInImage(CString fname, Vec4& constantColor)
 {
 	ImageLoader iloader(&ImporterMemoryPool::getSingleton());

AnKi/ShaderCompiler/ShaderProgramCompiler.cpp

@@ -95,8 +95,9 @@ static Bool spinDials(ShaderCompilerDynamicArray<U32>& dials, ConstWeakArray<Sha
 
 static void compileVariantAsync(const ShaderProgramParser& parser, ShaderProgramBinaryMutation& mutation,
 								ShaderCompilerDynamicArray<ShaderProgramBinaryVariant>& variants,
-								ShaderCompilerDynamicArray<ShaderProgramBinaryCodeBlock>& codeBlocks, ShaderProgramAsyncTaskInterface& taskManager,
-								Mutex& mtx, Atomic<I32>& error)
+								ShaderCompilerDynamicArray<ShaderProgramBinaryCodeBlock>& codeBlocks,
+								ShaderCompilerDynamicArray<U64>& sourceCodeHashes, ShaderProgramAsyncTaskInterface& taskManager, Mutex& mtx,
+								Atomic<I32>& error)
 {
 	class Ctx
 	{
@@ -105,6 +106,7 @@ static void compileVariantAsync(const ShaderProgramParser& parser, ShaderProgram
 		ShaderProgramBinaryMutation* m_mutation;
 		ShaderCompilerDynamicArray<ShaderProgramBinaryVariant>* m_variants;
 		ShaderCompilerDynamicArray<ShaderProgramBinaryCodeBlock>* m_codeBlocks;
+		ShaderCompilerDynamicArray<U64>* m_sourceCodeHashes;
 		Mutex* m_mtx;
 		Atomic<I32>* m_err;
 	};
@@ -114,6 +116,7 @@ static void compileVariantAsync(const ShaderProgramParser& parser, ShaderProgram
 	ctx->m_mutation = &mutation;
 	ctx->m_variants = &variants;
 	ctx->m_codeBlocks = &codeBlocks;
+	ctx->m_sourceCodeHashes = &sourceCodeHashes;
 	ctx->m_mtx = &mtx;
 	ctx->m_err = &error;
 
@@ -157,6 +160,32 @@ static void compileVariantAsync(const ShaderProgramParser& parser, ShaderProgram
 				ShaderCompilerString source;
 				ctx.m_parser->generateVariant(ctx.m_mutation->m_values, technique, shaderType, source);
 
+				// Check if the source code was found before
+				const U64 sourceCodeHash = source.computeHash();
+
+				if(technique.m_activeMutators[shaderType] != kMaxU64)
+				{
+					LockGuard lock(*ctx.m_mtx);
+
+					ANKI_ASSERT(ctx.m_sourceCodeHashes->getSize() == ctx.m_codeBlocks->getSize());
+
+					Bool found = false;
+					for(U32 i = 0; i < ctx.m_sourceCodeHashes->getSize(); ++i)
+					{
+						if((*ctx.m_sourceCodeHashes)[i] == sourceCodeHash)
+						{
+							codeBlockIndices[t].m_codeBlockIndices[shaderType] = i;
+							found = true;
+							break;
+						}
+					}
+
+					if(found)
+					{
+						continue;
+					}
+				}
+
 				ShaderCompilerDynamicArray<U8> spirv;
 				err = compileHlslToSpirv(source, shaderType, ctx.m_parser->compileWith16bitTypes(), spirv, compilerErrorLog);
 				if(err)
@@ -189,6 +218,9 @@ static void compileVariantAsync(const ShaderProgramParser& parser, ShaderProgram
 						spirv.moveAndReset(codeBlock.m_binary);
 						codeBlock.m_hash = newHash;
 
+						ctx.m_sourceCodeHashes->emplaceBack(sourceCodeHash);
+						ANKI_ASSERT(ctx.m_sourceCodeHashes->getSize() == ctx.m_codeBlocks->getSize());
+
 						++newCodeBlockCount;
 					}
 				}
@@ -330,6 +362,7 @@ Error compileShaderProgramInternal(CString fname, ShaderProgramFilesystemInterfa
 		dials.resize(parser.getMutators().getSize(), 0);
 		ShaderCompilerDynamicArray<ShaderProgramBinaryVariant> variants;
 		ShaderCompilerDynamicArray<ShaderProgramBinaryCodeBlock> codeBlocks;
+		ShaderCompilerDynamicArray<U64> sourceCodeHashes;
 		ShaderCompilerDynamicArray<ShaderProgramBinaryMutation> mutations;
 		mutations.resize(mutationCount);
 		ShaderCompilerHashMap<U64, U32> mutationHashToIdx;
@@ -365,7 +398,7 @@ Error compileShaderProgramInternal(CString fname, ShaderProgramFilesystemInterfa
 			{
 				// New and unique mutation and thus variant, add it
 
-				compileVariantAsync(parser, mutation, variants, codeBlocks, taskManager, mtx, errorAtomic);
+				compileVariantAsync(parser, mutation, variants, codeBlocks, sourceCodeHashes, taskManager, mtx, errorAtomic);
 
 				ANKI_ASSERT(mutationHashToIdx.find(mutation.m_hash) == mutationHashToIdx.getEnd());
 				mutationHashToIdx.emplace(mutation.m_hash, mutationCount - 1);
@@ -390,8 +423,9 @@ Error compileShaderProgramInternal(CString fname, ShaderProgramFilesystemInterfa
 		newArray(memPool, 1, binary->m_mutations);
 		ShaderCompilerDynamicArray<ShaderProgramBinaryVariant> variants;
 		ShaderCompilerDynamicArray<ShaderProgramBinaryCodeBlock> codeBlocks;
+		ShaderCompilerDynamicArray<U64> sourceCodeHashes;
 
-		compileVariantAsync(parser, binary->m_mutations[0], variants, codeBlocks, taskManager, mtx, errorAtomic);
+		compileVariantAsync(parser, binary->m_mutations[0], variants, codeBlocks, sourceCodeHashes, taskManager, mtx, errorAtomic);
 
 		ANKI_CHECK(taskManager.joinTasks());
 		ANKI_CHECK(Error(errorAtomic.getNonAtomically()));

AnKi/ShaderCompiler/ShaderProgramParser.cpp

@@ -137,7 +137,7 @@ Error ShaderProgramParser::parsePragmaTechniqueStart(const ShaderCompilerString*
 	ANKI_ASSERT(begin && end);
 
 	const PtrSize tokenCount = end - begin;
-	if(tokenCount != 1 && tokenCount != 2)
+	if(tokenCount == 0)
 	{
 		ANKI_PP_ERROR_MALFORMED();
 	}
@@ -152,18 +152,48 @@ Error ShaderProgramParser::parsePragmaTechniqueStart(const ShaderCompilerString*
 	++begin;
 	if(begin == end)
 	{
-		// Last token
 		techniqueName = "Unnamed";
 	}
-	else
+	else if(*begin == "uses_mutators")
+	{
+		techniqueName = "Unnamed";
+	}
+	else if(*begin != "uses_mutators")
 	{
 		techniqueName = *begin;
-
 		++begin;
-		if(begin != end)
+	}
+
+	// Mutators
+	U64 activeMutators = kMaxU64;
+	if(begin != end)
+	{
+		if(*begin != "uses_mutators")
 		{
 			ANKI_PP_ERROR_MALFORMED();
 		}
+		++begin;
+
+		activeMutators = 0;
+		for(; begin != end; ++begin)
+		{
+			// Find mutator
+			U32 count = 0;
+			for(const Mutator& mutator : m_mutators)
+			{
+				if(mutator.m_name == *begin)
+				{
+					activeMutators |= 1_U64 << U64(count);
+					break;
+				}
+				++count;
+			}
+
+			if(count == m_mutators.getSize())
+			{
+				ANKI_PP_ERROR_MALFORMED_MSG("Mutator not found");
+			}
+		}
 	}
 
 	// Checks
@@ -204,6 +234,7 @@ Error ShaderProgramParser::parsePragmaTechniqueStart(const ShaderCompilerString*
 	}
 
 	technique->m_shaderTypes |= ShaderTypeBit(1 << shaderType);
+	technique->m_activeMutators[shaderType] = activeMutators;
 
 	ANKI_ASSERT(extra->m_sourceLines[shaderType].getSize() == 0);
 	extra->m_sourceLines[shaderType] = m_commonSourceLines;
@@ -825,9 +856,16 @@ void ShaderProgramParser::generateVariant(ConstWeakArray<MutatorValue> mutation,
 	ANKI_ASSERT(!!(technique.m_shaderTypes & ShaderTypeBit(1 << shaderType)));
 
 	source.destroy();
+
+	ANKI_ASSERT(&technique >= m_techniques.getBegin() && &technique < m_techniques.getEnd());
+	const U32 tIdx = U32(&technique - m_techniques.getBegin());
+
 	for(U32 i = 0; i < mutation.getSize(); ++i)
 	{
-		source += ShaderCompilerString().sprintf("#define %s %d\n", m_mutators[i].m_name.cstr(), mutation[i]);
+		if(!!(technique.m_activeMutators[shaderType] & (1_U64 << U64(i))))
+		{
+			source += ShaderCompilerString().sprintf("#define %s %d\n", m_mutators[i].m_name.cstr(), mutation[i]);
+		}
 	}
 
 	source += ShaderCompilerString().sprintf("#define ANKI_TECHNIQUE_%s 1\n", technique.m_name.cstr());
@@ -845,9 +883,6 @@ void ShaderProgramParser::generateVariant(ConstWeakArray<MutatorValue> mutation,
 		source += "#define ANKI_SUPPORTS_16BIT_TYPES 0\n";
 	}
 
-	ANKI_ASSERT(&technique >= m_techniques.getBegin() && &technique < m_techniques.getEnd());
-	const U32 tIdx = U32(&technique - m_techniques.getBegin());
-
 	ANKI_ASSERT(m_techniqueExtras[tIdx].m_sources[shaderType].getLength() > 0);
 	source += m_techniqueExtras[tIdx].m_sources[shaderType];
 }

AnKi/ShaderCompiler/ShaderProgramParser.h

@@ -50,6 +50,7 @@ class ShaderProgramParserTechnique
 public:
 	ShaderCompilerString m_name;
 	ShaderTypeBit m_shaderTypes = ShaderTypeBit::kNone;
+	Array<U64, U32(ShaderType::kCount)> m_activeMutators = {};
 };
 
 /// This is a special preprocessor that run before the usual preprocessor. Its purpose is to add some meta information
@@ -58,11 +59,11 @@ public:
 /// It supports the following expressions:
 /// #include {<> | ""}
 /// #pragma once
-/// #pragma anki mutator NAME VALUE0 [VALUE1 [VALUE2] ...]
-/// #pragma anki skip_mutation MUTATOR0 VALUE0 [MUTATOR1 VALUE1 [MUTATOR2 VALUE2] ...]
+/// #pragma anki mutator NAME VALUE0 [VALUE1 [VALUE2 ...]]
+/// #pragma anki skip_mutation MUTATOR0 VALUE0 [MUTATOR1 VALUE1 [MUTATOR2 VALUE2 ...]]
 /// #pragma anki 16bit // Works only in HLSL. Gain 16bit types but loose min16xxx types
-/// #pragma anki technique_start {vert | tessc | tesse | geom | task | mesh | frag | comp | rgen | ahit | chit | miss | int | call} [NAME]
-/// #pragma anki technique_end {vert | tessc | tesse | geom | task | mesh | frag | comp | rgen | ahit | chit | miss | int | call} [NAME]
+/// #pragma anki technique_start STAGE [NAME] [uses_mutators [USES_MUTATOR1 [USES_MUTATOR2 ...]]]
+/// #pragma anki technique_end STAGE [NAME]
 ///
 /// #pragma anki struct NAME
 /// #	pragma anki member TYPE NAME

AnKi/Shaders/GBufferGeneric.ankiprog

@@ -16,7 +16,37 @@
 
 #pragma anki skip_mutation ALPHA_TEST 1 DIFFUSE_TEX 0
 
-// Some defines the clear up things
+#include <AnKi/Shaders/Include/MaterialTypes.h>
+#include <AnKi/Shaders/Include/GpuSceneFunctions.h>
+#include <AnKi/Shaders/PackFunctions.hlsl>
+#include <AnKi/Shaders/Functions.hlsl>
+#include <AnKi/Shaders/MaterialShadersCommon.hlsl>
+#include <AnKi/Shaders/RtShadows.hlsl>
+
+// Define a few things to avoid compilation errors
+#if defined(ANKI_TECHNIQUE_RtShadows) && defined(ANKI_CLOSEST_HIT_SHADER)
+#	define ALPHA_TEST 0
+#	define DIFFUSE_TEX 0
+#	define ANKI_VELOCITY 0
+#	define ANKI_BONES 0
+#	define PARALLAX 0
+#endif
+
+#if defined(ANKI_TECHNIQUE_RtShadows) && defined(ANKI_ANY_HIT_SHADER)
+#	define ANKI_VELOCITY 0
+#	define ANKI_BONES 0
+#	define PARALLAX 0
+#endif
+
+#if defined(ANKI_TASK_SHADER)
+#	define ALPHA_TEST 0
+#	define DIFFUSE_TEX 0
+#	define ANKI_VELOCITY 0
+#	define ANKI_BONES 0
+#	define PARALLAX 0
+#endif
+
+// General defines
 #define GBUFFER defined(ANKI_TECHNIQUE_GBuffer)
 #define REALLY_ALPHA_TEST (ALPHA_TEST && DIFFUSE_TEX)
 #define UVS (GBUFFER || REALLY_ALPHA_TEST)
@@ -32,13 +62,6 @@
 #define PRIMITIVE_NO_SAMPLING_POINTS_CULLING 1
 #define PRIMITIVE_ANY_CULLING (PRIMITIVE_BACKFACE_CULLING || PRIMITIVE_NO_SAMPLING_POINTS_CULLING)
 
-#include <AnKi/Shaders/Include/MaterialTypes.h>
-#include <AnKi/Shaders/Include/GpuSceneFunctions.h>
-#include <AnKi/Shaders/PackFunctions.hlsl>
-#include <AnKi/Shaders/Functions.hlsl>
-#include <AnKi/Shaders/MaterialShadersCommon.hlsl>
-#include <AnKi/Shaders/RtShadows.hlsl>
-
 #pragma anki struct AnKiLocalConstants
 
 #pragma anki member U32 m_diffTex
@@ -120,6 +143,7 @@ struct Mat3x4_2
 	Mat3x4 m_b;
 };
 
+#if ANKI_BONES
 Mat3x4_2 loadBoneTransforms(UnpackedMeshVertex vert, U32 boneTransformsOffset, U32 index)
 {
 	const U32 boneIdx = vert.m_boneIndices[index];
@@ -128,7 +152,6 @@ Mat3x4_2 loadBoneTransforms(UnpackedMeshVertex vert, U32 boneTransformsOffset, U
 	return g_gpuScene.Load<Mat3x4_2>(byteOffset);
 }
 
-#if ANKI_BONES
 void skinning(UnpackedMeshVertex vert, U32 boneTransformsOffset, inout Vec3 pos, inout Vec3 prevPos, inout RVec3 normal)
 {
 	Mat3x4_2 mats = loadBoneTransforms(vert, boneTransformsOffset, 0);
@@ -155,7 +178,7 @@ void skinning(UnpackedMeshVertex vert, U32 boneTransformsOffset, inout Vec3 pos,
 }
 #endif
 
-#if(ANKI_VELOCITY || ANKI_BONES) && GBUFFER
+#if REALLY_VELOCITY
 void velocity(Mat3x4 worldTransform, Mat3x4 prevWorldTransform, Vec3 prevLocalPos, inout VertOut output)
 {
 	ANKI_MAYBE_UNUSED(prevWorldTransform);
@@ -638,32 +661,32 @@ FragOut main(VertOut input)
 // ===========================================================================
 // Define the techniques                                                     =
 // ===========================================================================
-#pragma anki technique_start vert GBuffer
+#pragma anki technique_start vert GBuffer uses_mutators ANKI_VELOCITY ANKI_BONES DIFFUSE_TEX PARALLAX ALPHA_TEST
 #pragma anki technique_end vert GBuffer
 
-#pragma anki technique_start task GBuffer
+#pragma anki technique_start task GBuffer uses_mutators
 #pragma anki technique_end task GBuffer
 
-#pragma anki technique_start mesh GBuffer
+#pragma anki technique_start mesh GBuffer uses_mutators ANKI_VELOCITY ANKI_BONES DIFFUSE_TEX PARALLAX ALPHA_TEST
 #pragma anki technique_end mesh GBuffer
 
 #pragma anki technique_start frag GBuffer
 #pragma anki technique_end frag GBuffer
 
-#pragma anki technique_start vert Shadows
+#pragma anki technique_start vert Shadows uses_mutators ANKI_VELOCITY ANKI_BONES DIFFUSE_TEX PARALLAX ALPHA_TEST
 #pragma anki technique_end vert Shadows
 
-#pragma anki technique_start task Shadows
+#pragma anki technique_start task Shadows uses_mutators
 #pragma anki technique_end task Shadows
 
-#pragma anki technique_start mesh Shadows
+#pragma anki technique_start mesh Shadows uses_mutators ANKI_VELOCITY ANKI_BONES DIFFUSE_TEX PARALLAX ALPHA_TEST
 #pragma anki technique_end mesh Shadows
 
 #pragma anki technique_start frag Shadows
 #pragma anki technique_end frag Shadows
 
-#pragma anki technique_start ahit RtShadows
+#pragma anki technique_start ahit RtShadows uses_mutators ALPHA_TEST DIFFUSE_TEX
 #pragma anki technique_end ahit RtShadows
 
-#pragma anki technique_start chit RtShadows
+#pragma anki technique_start chit RtShadows uses_mutators
 #pragma anki technique_end chit RtShadows

AnKi/Shaders/Intellisense.hlsl

@@ -35,6 +35,7 @@
 #define ANKI_FRAGMENT_SHADER
 #define ANKI_MESH_SHADER
 #define ANKI_COMPUTE_SHADER
+#define ANKI_CLOSEST_HIT_SHADER
 
 using I8 = int;
 using I16 = int;