Start adding some bits on the shader compilation

src/anki/core/App.cpp

@@ -26,6 +26,7 @@
 #include <anki/core/StagingGpuMemoryManager.h>
 #include <anki/ui/UiManager.h>
 #include <anki/ui/Canvas.h>
+#include <anki/shader_compiler/ShaderProgramCompiler.h>
 
 #if ANKI_OS_ANDROID
 #	include <android_native_app_glue.h>
@@ -294,11 +295,11 @@ void App::cleanup()
 
 Error App::init(const ConfigSet& config, AllocAlignedCallback allocCb, void* allocCbUserData)
 {
-	Error err = initInternal(config, allocCb, allocCbUserData);
+	const Error err = initInternal(config, allocCb, allocCbUserData);
 	if(err)
 	{
+		ANKI_CORE_LOGE("App initialization failed. Shutting down");
 		cleanup();
-		ANKI_CORE_LOGE("App initialization failed");
 	}
 
 	return err;
@@ -435,6 +436,7 @@ Error App::initInternal(const ConfigSet& config_, AllocAlignedCallback allocCb,
 	m_resources = m_heapAlloc.newInstance<ResourceManager>();
 
 	ANKI_CHECK(m_resources->init(rinit));
+	ANKI_CHECK(compileAllShaders());
 
 	//
 	// UI
@@ -706,4 +708,51 @@ void App::initMemoryCallbacks(AllocAlignedCallback allocCb, void* allocCbUserDat
 	}
 }
 
+Error App::compileAllShaders()
+{
+	ANKI_CORE_LOGI("Compiling shaders");
+
+	ANKI_CHECK(m_resourceFs->iterateAllFilenames([&](CString fname) -> Error {
+		// Check file extension
+		StringAuto extension(m_heapAlloc);
+		getFilepathExtension(fname, extension);
+		if(extension.getLength() == 0 || extension != "ankiprog")
+		{
+			return Error::NONE;
+		}
+
+		// TODO check if it's in the cache
+
+		// Load interface
+		class Interface : public ShaderProgramFilesystemInterface
+		{
+		public:
+			ResourceFilesystem* m_fsystem = nullptr;
+
+			Error readAllText(CString filename, StringAuto& txt) final
+			{
+				ResourceFilePtr file;
+				ANKI_CHECK(m_fsystem->openFile(filename, file));
+				ANKI_CHECK(file->readAllText(txt));
+				return Error::NONE;
+			}
+		} iface;
+		iface.m_fsystem = m_resourceFs;
+
+		// Compile
+		ShaderProgramBinaryWrapper binary(m_heapAlloc);
+		ANKI_CHECK(compileShaderProgram(
+			fname, iface, m_heapAlloc, m_gr->getDeviceCapabilities(), m_gr->getBindlessLimits(), binary));
+
+		// Save the binary to the cache
+		StringAuto storeFname(m_heapAlloc);
+		storeFname.sprintf("%s/%s.ankiprogbin", m_cacheDir.cstr(), fname.cstr());
+		ANKI_CHECK(binary.serializeToFile(storeFname));
+
+		return Error::NONE;
+	}));
+
+	return Error::NONE;
+}
+
 } // end namespace anki

src/anki/core/App.h

@@ -216,6 +216,8 @@ private:
 
 	/// Inject a new UI element in the render queue for displaying various stuff.
 	void injectUiElements(DynamicArrayAuto<UiQueueElement>& elements, RenderQueue& rqueue);
+
+	ANKI_USE_RESULT Error compileAllShaders();
 };
 
 } // end namespace anki

src/anki/resource/ResourceFilesystem.cpp

@@ -29,10 +29,10 @@ public:
 		return m_file.read(buff, size);
 	}
 
-	ANKI_USE_RESULT Error readAllText(GenericMemoryPoolAllocator<U8> alloc, String& out) override
+	ANKI_USE_RESULT Error readAllText(StringAuto& out) override
 	{
 		ANKI_TRACE_SCOPED_EVENT(RSRC_FILE_READ);
-		return m_file.readAllText(alloc, out);
+		return m_file.readAllText(out);
 	}
 
 	ANKI_USE_RESULT Error readU32(U32& u) override
@@ -141,10 +141,10 @@ public:
 		return Error::NONE;
 	}
 
-	ANKI_USE_RESULT Error readAllText(GenericMemoryPoolAllocator<U8> alloc, String& out) override
+	ANKI_USE_RESULT Error readAllText(StringAuto& out) override
 	{
 		ANKI_ASSERT(m_size);
-		out.create(alloc, '?', m_size);
+		out.create('?', m_size);
 		return read(&out[0], m_size);
 	}
 

src/anki/resource/ResourceFilesystem.h

@@ -37,7 +37,7 @@ public:
 	virtual ANKI_USE_RESULT Error read(void* buff, PtrSize size) = 0;
 
 	/// Read all the contents of a text file. If the file is not rewined it will probably fail
-	virtual ANKI_USE_RESULT Error readAllText(GenericMemoryPoolAllocator<U8> alloc, String& out) = 0;
+	virtual ANKI_USE_RESULT Error readAllText(StringAuto& out) = 0;
 
 	/// Read 32bit unsigned integer. Set the endianness if the file's endianness is different from the machine's
 	virtual ANKI_USE_RESULT Error readU32(U32& u) = 0;
@@ -87,6 +87,20 @@ public:
 	/// Search the path list to find the file. Then open the file for reading. It's thread-safe.
 	ANKI_USE_RESULT Error openFile(const ResourceFilename& filename, ResourceFilePtr& file);
 
+	/// Iterate all the filenames from all paths provided.
+	template<typename TFunc>
+	ANKI_USE_RESULT Error iterateAllFilenames(TFunc func) const
+	{
+		for(const Path& path : m_paths)
+		{
+			for(const String& fname : path.m_files)
+			{
+				ANKI_CHECK(func(fname.toCString()));
+			}
+		}
+		return Error::NONE;
+	}
+
 #if !ANKI_TESTS
 private:
 #endif

src/anki/resource/ResourceObject.cpp

@@ -44,7 +44,7 @@ Error ResourceObject::openFileReadAllText(const CString& filename, StringAuto& t
 
 	// Read string
 	text = StringAuto(getTempAllocator());
-	ANKI_CHECK(file->readAllText(getTempAllocator(), text));
+	ANKI_CHECK(file->readAllText(text));
 
 	return Error::NONE;
 }

src/anki/resource/ScriptResource.cpp

@@ -19,7 +19,9 @@ Error ScriptResource::load(const ResourceFilename& filename, Bool async)
 	ResourceFilePtr file;
 	ANKI_CHECK(openFile(filename, file));
 
-	ANKI_CHECK(file->readAllText(getAllocator(), m_source));
+	StringAuto src(getAllocator());
+	ANKI_CHECK(file->readAllText(src));
+	m_source.create(getAllocator(), src);
 
 	return Error::NONE;
 }

src/anki/resource/ShaderProgramPreProcessor.cpp

@@ -222,7 +222,7 @@ Error ShaderProgramPreprocessor::parseFile(CString fname, U32 depth)
 	ResourceFilePtr file;
 	ANKI_CHECK(m_fsystem->openFile(fname, file));
 	StringAuto txt(m_alloc);
-	ANKI_CHECK(file->readAllText(m_alloc, txt));
+	ANKI_CHECK(file->readAllText(txt));
 
 	StringListAuto lines(m_alloc);
 	lines.splitString(txt.toCString(), '\n');

src/anki/shader_compiler/ShaderProgramBinary.h

@@ -123,6 +123,7 @@ public:
 	Array<char, MAX_SHADER_BINARY_NAME_LENGTH + 1> m_name;
 	ShaderVariableDataType m_type = ShaderVariableDataType::NONE;
 	U32 m_constantId = MAX_U32;
+	ShaderTypeBit m_shaderStages = ShaderTypeBit::NONE;
 
 	template<typename TSerializer, typename TClass>
 	static void serializeCommon(TSerializer& s, TClass self)
@@ -133,6 +134,7 @@ public:
 			MAX_SHADER_BINARY_NAME_LENGTH + 1);
 		s.doValue("m_type", offsetof(ShaderProgramBinaryConstant, m_type), self.m_type);
 		s.doValue("m_constantId", offsetof(ShaderProgramBinaryConstant, m_constantId), self.m_constantId);
+		s.doValue("m_shaderStages", offsetof(ShaderProgramBinaryConstant, m_shaderStages), self.m_shaderStages);
 	}
 
 	template<typename TDeserializer>

src/anki/shader_compiler/ShaderProgramBinary.xml

@@ -40,6 +40,7 @@
 				<member name="m_name" type="char" array_size="MAX_SHADER_BINARY_NAME_LENGTH + 1" />
 				<member name="m_type" type="ShaderVariableDataType" constructor="= ShaderVariableDataType::NONE" />
 				<member name="m_constantId" type="U32" constructor="= MAX_U32"/>
+				<member name="m_shaderStages" type="ShaderTypeBit" constructor="= ShaderTypeBit::NONE"/>
 			</members>
 		</class>
 

src/anki/shader_compiler/ShaderProgramReflection.cpp

@@ -36,7 +36,8 @@ private:
 	ANKI_USE_RESULT Error opaqueReflection(
 		const spirv_cross::Resource& res, DynamicArrayAuto<ShaderProgramBinaryOpaque>& opaques) const;
 
-	ANKI_USE_RESULT Error constsReflection(DynamicArrayAuto<ShaderProgramBinaryConstant>& consts) const;
+	ANKI_USE_RESULT Error constsReflection(
+		DynamicArrayAuto<ShaderProgramBinaryConstant>& consts, ShaderType stage) const;
 
 	ANKI_USE_RESULT Error blockVariablesReflection(
 		spirv_cross::TypeID resourceId, DynamicArrayAuto<ShaderProgramBinaryVariable>& vars) const;
@@ -454,7 +455,7 @@ Error SpirvReflector::opaqueReflection(
 	return Error::NONE;
 }
 
-Error SpirvReflector::constsReflection(DynamicArrayAuto<ShaderProgramBinaryConstant>& consts) const
+Error SpirvReflector::constsReflection(DynamicArrayAuto<ShaderProgramBinaryConstant>& consts, ShaderType stage) const
 {
 	spirv_cross::SmallVector<spirv_cross::SpecializationConstant> specConsts = get_specialization_constants();
 	for(const spirv_cross::SpecializationConstant& c : specConsts)
@@ -488,9 +489,9 @@ Error SpirvReflector::constsReflection(DynamicArrayAuto<ShaderProgramBinaryConst
 			return Error::USER_DATA;
 		}
 
-		// Add it
-		Bool found = false;
-		for(const ShaderProgramBinaryConstant& other : consts)
+		// Search for it
+		ShaderProgramBinaryConstant* foundConst = nullptr;
+		for(ShaderProgramBinaryConstant& other : consts)
 		{
 			const Bool nameSame = strcmp(other.m_name.getBegin(), newConst.m_name.getBegin()) == 0;
 			const Bool typeSame = other.m_type == newConst.m_type;
@@ -506,15 +507,21 @@ Error SpirvReflector::constsReflection(DynamicArrayAuto<ShaderProgramBinaryConst
 
 			if(idSame)
 			{
-				found = true;
+				foundConst = &other;
 				break;
 			}
 		}
 
-		if(!found)
+		// Add it or update it
+		if(foundConst == nullptr)
 		{
 			consts.emplaceBack(newConst);
 		}
+		else
+		{
+			ANKI_ASSERT(foundConst->m_shaderStages != ShaderTypeBit::NONE);
+			foundConst->m_shaderStages |= shaderTypeToBit(stage);
+		}
 	}
 
 	return Error::NONE;
@@ -582,7 +589,7 @@ Error SpirvReflector::performSpirvReflection(ShaderProgramBinaryReflection& refl
 		}
 
 		// Spec consts
-		ANKI_CHECK(compiler.constsReflection(specializationConstants));
+		ANKI_CHECK(compiler.constsReflection(specializationConstants, type));
 	}
 
 	ShaderProgramBinaryBlock* firstBlock;

src/anki/util/Win32Minimal.cpp

@@ -3,7 +3,7 @@
 // Code licensed under the BSD License.
 // http://www.anki3d.org/LICENSE
 
-/// @file 
+/// @file
 /// This file is mainly used to test that Windows.h works with our own minimal version of it.
 
 #include "Win32Minimal.h"

tests/resource/ResourceFilesystem.cpp

@@ -21,7 +21,7 @@ ANKI_TEST(Resource, ResourceFilesystem)
 		ResourceFilePtr file;
 		ANKI_TEST_EXPECT_NO_ERR(fs.openFile("subdir0/hello.txt", file));
 		StringAuto txt(alloc);
-		ANKI_TEST_EXPECT_NO_ERR(file->readAllText(alloc, txt));
+		ANKI_TEST_EXPECT_NO_ERR(file->readAllText(txt));
 		ANKI_TEST_EXPECT_EQ(txt, "hello\n");
 	}
 
@@ -30,7 +30,7 @@ ANKI_TEST(Resource, ResourceFilesystem)
 		ResourceFilePtr file;
 		ANKI_TEST_EXPECT_NO_ERR(fs.openFile("subdir0/hello.txt", file));
 		StringAuto txt(alloc);
-		ANKI_TEST_EXPECT_NO_ERR(file->readAllText(alloc, txt));
+		ANKI_TEST_EXPECT_NO_ERR(file->readAllText(txt));
 		ANKI_TEST_EXPECT_EQ(txt, "hell\n");
 	}
 }