The shader compiler can pre-process source

CMakeLists.txt

@@ -108,7 +108,7 @@ else()
 	message(FATAL_ERROR "Couldn't determine the window backend. You need to specify it manually")
 endif()
 
-set(ANKI_GR_BACKEND "GL" CACHE STRING "The graphics API (GL or VULKAN)")
+set(ANKI_GR_BACKEND "VULKAN" CACHE STRING "The graphics API (VULKAN or GL)")
 
 if(${ANKI_GR_BACKEND} STREQUAL "GL")
 	set(GL TRUE)

README.md

@@ -1,13 +1,13 @@
 [![AnKi logo](http://anki3d.org/wp-content/uploads/2015/11/logo_248.png)](http://anki3d.org)
 
-AnKi 3D engine is a Linux and Windows opensource game engine that runs on OpenGL 4.5 and Vulkan 1.0 (Experimental).
+AnKi 3D engine is a Linux and Windows opensource game engine that runs on Vulkan 1.1 and OpenGL 4.5 (now deprecated).
 
 [![Video](http://img.youtube.com/vi/va7nZ2EFR4c/0.jpg)](http://www.youtube.com/watch?v=va7nZ2EFR4c)
 
 1 License
 =========
 
-AnKi's license is BSD. This practically means that you can use the source or parts of the source on proprietary and non 
+AnKi's license is BSD. This practically means that you can use the source or parts of the source on proprietary and non
 proprietary products as long as you follow the conditions of the license.
 
 See `LICENSE` file for more info.
@@ -17,14 +17,14 @@ See `LICENSE` file for more info.
 
 | OS      | Master Branch Build Status                                                                                                                                    |
 | ------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| Linux   | [![Build Status Linux](https://travis-ci.org/godlikepanos/anki-3d-engine.svg?branch=master)](https://travis-ci.org/godlikepanos/anki-3d-engine)               | 
+| Linux   | [![Build Status Linux](https://travis-ci.org/godlikepanos/anki-3d-engine.svg?branch=master)](https://travis-ci.org/godlikepanos/anki-3d-engine)               |
 | Windows | [![Build status Windows](https://ci.appveyor.com/api/projects/status/waij29m7o8ajjoqh?svg=true)](https://ci.appveyor.com/project/godlikepanos/anki-3d-engine) |
 
 To checkout the source including the submodules type:
 
 	git clone --recurse-submodules https://github.com/godlikepanos/anki-3d-engine.git anki
 
-AnKi's build system is using `CMake`. A great effort was made to ease the building process that's why the number of 
+AnKi's build system is using `CMake`. A great effort was made to ease the building process that's why the number of
 external dependencies are almost none.
 
 2.1 On Linux
@@ -79,8 +79,8 @@ To build the release version open `PowerShell` and type:
 3 Next steps
 ============
 
-Try to build with `samples` enabled (search for the `ANKI_BUILD_SAMPLES=ON` option in your CMake GUI) and try running 
-the sponza executable. Then you will be able to see sponza running in AnKi. All samples must run from within their 
+Try to build with `samples` enabled (search for the `ANKI_BUILD_SAMPLES=ON` option in your CMake GUI) and try running
+the sponza executable. Then you will be able to see sponza running in AnKi. All samples must run from within their
 directory.
 
 	$cd path/to/anki/samples/sponza

src/anki/gr/CommandBuffer.h

@@ -83,15 +83,6 @@ public:
 	}
 };
 
-/// Command buffer initialization hints. They are used to optimize the allocators of a command buffer.
-class CommandBufferInitHints
-{
-	friend class CommandBufferImpl;
-
-private:
-	PtrSize m_chunkSize = 1024 * 1024;
-};
-
 /// Command buffer initialization flags.
 enum class CommandBufferFlag : U8
 {
@@ -120,7 +111,6 @@ public:
 	FramebufferPtr m_framebuffer; ///< For second level command buffers.
 	Array<TextureUsageBit, MAX_COLOR_ATTACHMENTS> m_colorAttachmentUsages = {};
 	TextureUsageBit m_depthStencilAttachmentUsage = TextureUsageBit::NONE;
-	CommandBufferInitHints m_hints;
 
 	CommandBufferFlag m_flags = CommandBufferFlag::NONE;
 
@@ -138,9 +128,6 @@ class CommandBuffer : public GrObject
 public:
 	static const GrObjectType CLASS_TYPE = GrObjectType::COMMAND_BUFFER;
 
-	/// Compute initialization hints.
-	CommandBufferInitHints computeInitHints() const;
-
 	/// Finalize and submit if it's primary command buffer and just finalize if it's second level.
 	/// @param[out] fence Optionaly create fence.
 	void flush(FencePtr* fence = nullptr);

src/anki/gr/RenderGraph.cpp

@@ -82,8 +82,19 @@ class RenderGraph::Pass
 public:
 	// WARNING!!!!!: Whatever you put here needs manual destruction in RenderGraph::reset()
 
+	/// WARNING: Should be the same as RenderPassDependency::TextureInfo
+	class ConsumedTextureInfo
+	{
+	public:
+		RenderTargetHandle m_handle;
+		TextureUsageBit m_usage;
+		TextureSubresourceInfo m_subresource;
+	};
+
 	DynamicArray<U32> m_dependsOn;
 
+	DynamicArray<ConsumedTextureInfo> m_consumedTextures;
+
 	RenderPassWorkCallback m_callback;
 	void* m_userData;
 
@@ -96,16 +107,6 @@ public:
 
 	Bool m_drawsToPresentable = false;
 
-	/// WARNING: Should be the same as RenderPassDependency::TextureInfo
-	class ConsumedTextureInfo
-	{
-	public:
-		RenderTargetHandle m_handle;
-		TextureUsageBit m_usage;
-		TextureSubresourceInfo m_subresource;
-	};
-	DynamicArray<ConsumedTextureInfo> m_consumedTextures;
-
 	FramebufferPtr& fb()
 	{
 		return m_secondLevelCmdbInitInfo.m_framebuffer;

src/anki/gr/ShaderCompiler.cpp

@@ -9,10 +9,18 @@
 #include <anki/util/StringList.h>
 #include <anki/util/Filesystem.h>
 #include <anki/core/Trace.h>
+
+#if defined(__GNUC__)
+#	pragma GCC diagnostic push
+#	pragma GCC diagnostic ignored "-Wundef"
+#endif
 #include <glslang/Public/ShaderLang.h>
 #include <glslang/SPIRV/GlslangToSpv.h>
 #include <glslang/StandAlone/DirStackFileIncluder.h>
 #include <SPIRV-Cross/spirv_glsl.hpp>
+#if defined(__GNUC__)
+#	pragma GCC diagnostic pop
+#endif
 
 namespace anki
 {
@@ -231,28 +239,94 @@ static TBuiltInResource setLimits()
 
 static const TBuiltInResource GLSLANG_LIMITS = setLimits();
 
-class ShaderCompiler::BuildContext
+static void preappendAnkiSpecific(CString source, const ShaderCompilerOptions& options, StringAuto& out)
+{
+	// Gen the new source
+	out.sprintf(SHADER_HEADER,
+		(options.m_outLanguage == ShaderLanguage::GLSL) ? "GL" : "VULKAN",
+		options.m_gpuCapabilities.m_minorApiVersion,
+		options.m_gpuCapabilities.m_majorApiVersion,
+		&GPU_VENDOR_STR[options.m_gpuCapabilities.m_gpuVendor][0],
+		SHADER_NAME[options.m_shaderType],
+		// GL bindings
+		MAX_UNIFORM_BUFFER_BINDINGS,
+		MAX_STORAGE_BUFFER_BINDINGS,
+		MAX_TEXTURE_BINDINGS,
+		MAX_IMAGE_BINDINGS, // Images
+		(MAX_TEXTURE_BINDINGS + MAX_IMAGE_BINDINGS) * MAX_DESCRIPTOR_SETS, // Push constant location
+		// VK bindings
+		0,
+		MAX_TEXTURE_BINDINGS,
+		MAX_TEXTURE_BINDINGS + MAX_UNIFORM_BUFFER_BINDINGS,
+		MAX_TEXTURE_BINDINGS + MAX_UNIFORM_BUFFER_BINDINGS + MAX_STORAGE_BUFFER_BINDINGS,
+		&source[0]);
+}
+
+I32 ShaderCompiler::m_refcount = {0};
+Mutex ShaderCompiler::m_refcountMtx;
+
+ShaderCompiler::ShaderCompiler(GenericMemoryPoolAllocator<U8> alloc)
+	: m_alloc(alloc)
+{
+	LockGuard<Mutex> lock(m_refcountMtx);
+
+	const I32 refcount = m_refcount++;
+	ANKI_ASSERT(refcount >= 0);
+
+	if(refcount == 0)
+	{
+		glslang::InitializeProcess();
+	}
+}
+
+ShaderCompiler::~ShaderCompiler()
 {
-public:
-	CString m_originalSrc;
-	CString m_src;
-	ShaderCompilerOptions m_options;
-	GenericMemoryPoolAllocator<U8> m_alloc;
-};
-
-static ANKI_USE_RESULT Error genSpirv(const ShaderCompiler::BuildContext& ctx, std::vector<unsigned int>& spirv)
+	LockGuard<Mutex> lock(m_refcountMtx);
+
+	const I32 refcount = m_refcount--;
+	ANKI_ASSERT(refcount >= 0);
+
+	if(refcount == 1)
+	{
+		glslang::FinalizeProcess();
+	}
+}
+
+Error ShaderCompiler::preprocessCommon(CString in, const ShaderCompilerOptions& options, StringAuto& out) const
+{
+	const EShLanguage stage = ankiToGlslangShaderType(options.m_shaderType);
+
+	glslang::TShader shader(stage);
+	Array<const char*, 1> csrc = {{&in[0]}};
+	shader.setStrings(&csrc[0], 1);
+
+	DirStackFileIncluder includer;
+	EShMessages messages = EShMsgDefault;
+	std::string glslangOut;
+	if(!shader.preprocess(&GLSLANG_LIMITS, 450, ENoProfile, false, false, messages, &glslangOut, includer))
+	{
+		ShaderCompiler::logShaderErrorCode(shader.getInfoLog(), in, m_alloc);
+		return Error::USER_DATA;
+	}
+
+	out.append(glslangOut.c_str());
+
+	return Error::NONE;
+}
+
+Error ShaderCompiler::genSpirv(CString src, const ShaderCompilerOptions& options, DynamicArrayAuto<U8>& spirv) const
 {
-	const EShLanguage stage = ankiToGlslangShaderType(ctx.m_options.m_shaderType);
+	const EShLanguage stage = ankiToGlslangShaderType(options.m_shaderType);
 
 	EShMessages messages = EShMsgSpvRules;
-	if(ctx.m_options.m_outLanguage == ShaderLanguage::SPIRV)
+	if(options.m_outLanguage == ShaderLanguage::SPIRV)
 	{
 		messages = static_cast<EShMessages>(messages | EShMsgVulkanRules);
 	}
 
 	// Setup the shader
 	glslang::EShTargetLanguageVersion langVersion;
-	if(ctx.m_options.m_outLanguage == ShaderLanguage::SPIRV && ctx.m_options.m_gpuCapabilities.m_minorApiVersion > 0)
+	if(options.m_outLanguage == ShaderLanguage::SPIRV && options.m_gpuCapabilities.m_minorApiVersion > 0)
 	{
 		langVersion = glslang::EShTargetSpv_1_3;
 	}
@@ -262,12 +336,12 @@ static ANKI_USE_RESULT Error genSpirv(const ShaderCompiler::BuildContext& ctx, s
 	}
 
 	glslang::TShader shader(stage);
-	Array<const char*, 1> csrc = {{&ctx.m_src[0]}};
+	Array<const char*, 1> csrc = {{&src[0]}};
 	shader.setStrings(&csrc[0], 1);
 	shader.setEnvTarget(glslang::EShTargetSpv, langVersion);
 	if(!shader.parse(&GLSLANG_LIMITS, 100, false, messages))
 	{
-		ShaderCompiler::logShaderErrorCode(shader.getInfoLog(), ctx.m_src, ctx.m_alloc);
+		ShaderCompiler::logShaderErrorCode(shader.getInfoLog(), src, m_alloc);
 		return Error::USER_DATA;
 	}
 
@@ -285,59 +359,47 @@ static ANKI_USE_RESULT Error genSpirv(const ShaderCompiler::BuildContext& ctx, s
 	glslang::SpvOptions spvOptions;
 	spvOptions.optimizeSize = true;
 	spvOptions.disableOptimizer = false;
-	glslang::GlslangToSpv(*program.getIntermediate(stage), spirv, &spvOptions);
-
-	return Error::NONE;
-}
+	std::vector<unsigned int> glslangSpirv;
+	glslang::GlslangToSpv(*program.getIntermediate(stage), glslangSpirv, &spvOptions);
 
-/// Just run the preprocessor.
-static ANKI_USE_RESULT Error preprocess(const ShaderCompiler::BuildContext& ctx, std::string& src)
-{
-	const EShLanguage stage = ankiToGlslangShaderType(ctx.m_options.m_shaderType);
-
-	glslang::TShader shader(stage);
-	Array<const char*, 1> csrc = {{&ctx.m_src[0]}};
-	shader.setStrings(&csrc[0], 1);
-
-	DirStackFileIncluder includer;
-	EShMessages messages = EShMsgDefault;
-	if(!shader.preprocess(&GLSLANG_LIMITS, 450, ENoProfile, false, false, messages, &src, includer))
-	{
-		ShaderCompiler::logShaderErrorCode(shader.getInfoLog(), ctx.m_src, ctx.m_alloc);
-		return Error::USER_DATA;
-	}
+	// Store
+	spirv.resize(glslangSpirv.size() * sizeof(unsigned int));
+	memcpy(&spirv[0], &glslangSpirv[0], spirv.getSizeInBytes());
 
 	return Error::NONE;
 }
 
-I32 ShaderCompiler::m_refcount = {0};
-Mutex ShaderCompiler::m_refcountMtx;
-
-ShaderCompiler::ShaderCompiler(GenericMemoryPoolAllocator<U8> alloc)
-	: m_alloc(alloc)
+Error ShaderCompiler::preprocess(
+	CString source, const ShaderCompilerOptions& options, const StringList& defines, StringAuto& out) const
 {
-	LockGuard<Mutex> lock(m_refcountMtx);
+	ANKI_ASSERT(!source.isEmpty() && source.getLength() > 0);
 
-	const I32 refcount = m_refcount++;
-	ANKI_ASSERT(refcount >= 0);
+	ANKI_TRACE_SCOPED_EVENT(GR_SHADER_COMPILE);
 
-	if(refcount == 0)
+	// Append defines
+	StringAuto newSource(m_alloc);
+	auto it = defines.getBegin();
+	auto end = defines.getEnd();
+	while(it != end)
 	{
-		glslang::InitializeProcess();
+		newSource.append("#define ");
+		newSource.append(it->toCString());
+		newSource.append(" = (");
+		++it;
+		ANKI_ASSERT(it != end);
+		newSource.append(it->toCString());
+		newSource.append(")\n");
 	}
-}
+	newSource.append(source);
 
-ShaderCompiler::~ShaderCompiler()
-{
-	LockGuard<Mutex> lock(m_refcountMtx);
+	// Add the extra code
+	StringAuto fullSrc(m_alloc);
+	preappendAnkiSpecific(newSource.toCString(), options, fullSrc);
 
-	const I32 refcount = m_refcount--;
-	ANKI_ASSERT(refcount >= 0);
+	// Do the work
+	ANKI_CHECK(preprocessCommon(fullSrc.toCString(), options, out));
 
-	if(refcount == 1)
-	{
-		glslang::FinalizeProcess();
-	}
+	return Error::NONE;
 }
 
 Error ShaderCompiler::compile(CString source, const ShaderCompilerOptions& options, DynamicArrayAuto<U8>& bin) const
@@ -348,34 +410,8 @@ Error ShaderCompiler::compile(CString source, const ShaderCompilerOptions& optio
 	ANKI_TRACE_SCOPED_EVENT(GR_SHADER_COMPILE);
 
 	// Create the context
-	BuildContext ctx;
-	ctx.m_originalSrc = source;
-	ctx.m_options = options;
-	ctx.m_alloc = m_alloc;
-
-	// Gen the new source
-	StringAuto fullSrc(m_alloc);
-
-	fullSrc.sprintf(SHADER_HEADER,
-		(options.m_outLanguage == ShaderLanguage::GLSL) ? "GL" : "VULKAN",
-		options.m_gpuCapabilities.m_minorApiVersion,
-		options.m_gpuCapabilities.m_majorApiVersion,
-		&GPU_VENDOR_STR[options.m_gpuCapabilities.m_gpuVendor][0],
-		SHADER_NAME[options.m_shaderType],
-		// GL bindings
-		MAX_UNIFORM_BUFFER_BINDINGS,
-		MAX_STORAGE_BUFFER_BINDINGS,
-		MAX_TEXTURE_BINDINGS,
-		MAX_IMAGE_BINDINGS, // Images
-		(MAX_TEXTURE_BINDINGS + MAX_IMAGE_BINDINGS) * MAX_DESCRIPTOR_SETS, // Push constant location
-		// VK bindings
-		0,
-		MAX_TEXTURE_BINDINGS,
-		MAX_TEXTURE_BINDINGS + MAX_UNIFORM_BUFFER_BINDINGS,
-		MAX_TEXTURE_BINDINGS + MAX_UNIFORM_BUFFER_BINDINGS + MAX_STORAGE_BUFFER_BINDINGS,
-		&source[0]);
-
-	ctx.m_src = fullSrc.toCString();
+	StringAuto finalSrc(m_alloc);
+	preappendAnkiSpecific(source, options, finalSrc);
 
 	// Compile
 	if(options.m_outLanguage == ShaderLanguage::GLSL)
@@ -393,23 +429,16 @@ Error ShaderCompiler::compile(CString source, const ShaderCompilerOptions& optio
 		}
 #else
 		// Preprocess the source because MESA sucks and can't do it
-		std::string preprocessedSrc;
-		ANKI_CHECK(preprocess(ctx, preprocessedSrc));
-		ANKI_ASSERT(preprocessedSrc.length() > 0);
+		StringAuto out(m_alloc);
+		ANKI_CHECK(preprocessCommon(finalSrc.toCString(), options, out));
 
-		bin.resize(preprocessedSrc.length() + 1);
-		memcpy(&bin[0], &preprocessedSrc[0], bin.getSize());
+		bin.resize(out.getLength() + 1);
+		memcpy(&bin[0], &out[0], bin.getSizeInBytes());
 #endif
 	}
 	else
 	{
-		std::vector<unsigned int> spv;
-		err = genSpirv(ctx, spv);
-		if(!err)
-		{
-			bin.resize(spv.size() * sizeof(unsigned int));
-			memcpy(&bin[0], &spv[0], bin.getSize());
-		}
+		ANKI_CHECK(genSpirv(finalSrc.toCString(), options, bin));
 	}
 
 #if 0

src/anki/gr/ShaderCompiler.h

@@ -11,6 +11,9 @@
 namespace anki
 {
 
+// Forward
+class StringList;
+
 /// @addtogroup graphics
 /// @{
 
@@ -45,8 +48,6 @@ public:
 class ShaderCompiler
 {
 public:
-	class BuildContext;
-
 	ShaderCompiler(GenericMemoryPoolAllocator<U8> alloc);
 
 	~ShaderCompiler();
@@ -58,6 +59,9 @@ public:
 	ANKI_USE_RESULT Error compile(
 		CString source, const ShaderCompilerOptions& options, DynamicArrayAuto<U8>& bin) const;
 
+	ANKI_USE_RESULT Error preprocess(
+		CString source, const ShaderCompilerOptions& options, const StringList& defines, StringAuto& out) const;
+
 anki_internal:
 	static void logShaderErrorCode(CString error, CString source, GenericMemoryPoolAllocator<U8> alloc);
 
@@ -65,6 +69,11 @@ private:
 	GenericMemoryPoolAllocator<U8> m_alloc;
 	static I32 m_refcount;
 	static Mutex m_refcountMtx;
+
+	ANKI_USE_RESULT Error genSpirv(
+		CString src, const ShaderCompilerOptions& options, DynamicArrayAuto<U8>& spirv) const;
+
+	ANKI_USE_RESULT Error preprocessCommon(CString in, const ShaderCompilerOptions& options, StringAuto& out) const;
 };
 
 /// Like ShaderCompiler but on steroids. It uses a cache to avoid compiling shaders else it calls

src/anki/gr/gl/CommandBuffer.cpp

@@ -1268,12 +1268,6 @@ void CommandBuffer::generateMipmaps3d(TextureViewPtr tex)
 	ANKI_ASSERT(!!"TODO");
 }
 
-CommandBufferInitHints CommandBuffer::computeInitHints() const
-{
-	ANKI_GL_SELF_CONST(CommandBufferImpl);
-	return self.computeInitHints();
-}
-
 void CommandBuffer::pushSecondLevelCommandBuffer(CommandBufferPtr cmdb)
 {
 	class ExecCmdbCommand final : public GlCommand

src/anki/gr/gl/CommandBufferImpl.cpp

@@ -90,14 +90,6 @@ Error CommandBufferImpl::executeAllCommands()
 	return err;
 }
 
-CommandBufferImpl::InitHints CommandBufferImpl::computeInitHints() const
-{
-	InitHints out;
-	out.m_chunkSize = m_alloc.getMemoryPool().getMemoryCapacity();
-
-	return out;
-}
-
 void CommandBufferImpl::flushDrawcall(CommandBuffer& cmdb)
 {
 	ANKI_ASSERT(!!(m_flags & CommandBufferFlag::GRAPHICS_WORK));

src/anki/gr/gl/CommandBufferImpl.h

@@ -40,8 +40,6 @@ public:
 class CommandBufferImpl final : public CommandBuffer
 {
 public:
-	using InitHints = CommandBufferInitHints;
-
 	GlCommand* m_firstCommand = nullptr;
 	GlCommand* m_lastCommand = nullptr;
 	CommandBufferAllocator<U8> m_alloc;
@@ -74,9 +72,6 @@ public:
 		return m_alloc;
 	}
 
-	/// Compute initialization hints.
-	InitHints computeInitHints() const;
-
 	/// Create a new command and add it to the chain.
 	template<typename TCommand, typename... TArgs>
 	void pushBackNewCommand(TArgs&&... args);

src/anki/gr/vulkan/CommandBuffer.cpp

@@ -21,13 +21,6 @@ CommandBuffer* CommandBuffer::newInstance(GrManager* manager, const CommandBuffe
 	return impl;
 }
 
-CommandBufferInitHints CommandBuffer::computeInitHints() const
-{
-	// TODO
-	CommandBufferInitHints hints;
-	return hints;
-}
-
 void CommandBuffer::flush(FencePtr* fence)
 {
 	ANKI_VK_SELF(CommandBufferImpl);