Shader precaching mechanism. Call Graphics::BeginDumpShaders() or use the engine command line parameter -ds <file> to create a shader dump file of the variations that are loaded during program run, and Graphics::PrecacheShaders() to load the dump file.

Moved GetShader() from Renderer subsystem to Graphics, which is more logical as loading shaders is a low-level operation, while Renderer should be concerned with high-level rendering of 3D scenes. Now eg. UI does not need to depend on Renderer at all.

Simplified internal state of ShaderVariation & ShaderProgram objects, the success of compile/link can be deduced from the presence of a non-null GPU object.

Fixed FILE_READWRITE mode to not clear the file contents, and to handle alternating reads and writes correctly.

Bin/CoreData/Shaders/HLSL/Lighting.hlsl

@@ -151,7 +151,7 @@ float GetShadow(float4 shadowPos)
             tex2Dproj(sShadowMap, float4(shadowPos.x, shadowPos.y + offsets.y, shadowPos.zw)).r,
             tex2Dproj(sShadowMap, float4(shadowPos.xy + offsets.xy, shadowPos.zw)).r
         );
-        #ifdef HWSHADOW
+        #ifndef SHADOWCMP
             return cShadowIntensity.y + dot(inLight, cShadowIntensity.x);
         #else
             #ifndef POINTLIGHT
@@ -163,7 +163,7 @@ float GetShadow(float4 shadowPos)
     #else
         // Take one sample
         float inLight = tex2Dproj(sShadowMap, shadowPos).r;
-        #ifdef HWSHADOW
+        #ifndef SHADOWCMP
             return cShadowIntensity.y + cShadowIntensity.x * inLight;
         #else
             #ifndef POINTLIGHT

Bin/Data/NinjaSnowWarShaders.txt

@@ -0,0 +1,33 @@
+Shadow_SKINNED Shadow
+LitSolid_DIRLIGHT_PERPIXEL_SHADOW LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_PERPIXEL_SHADOW
+LitSolid_DIRLIGHT_PERPIXEL_SHADOW_SKINNED LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_PERPIXEL_SHADOW
+Unlit Unlit_DIFFMAP
+Unlit_BILLBOARD_VERTEXCOLOR Unlit_DIFFMAP_VERTEXCOLOR
+Shadow Shadow
+Shadow_INSTANCED Shadow
+LitSolid_DIRLIGHT_INSTANCED_PERPIXEL_SHADOW LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_PERPIXEL_SHADOW
+LitSolid_PERPIXEL_POINTLIGHT_SHADOW LitSolid_DIFFMAP_PERPIXEL_POINTLIGHT_SHADOW
+LitSolid_PERPIXEL_POINTLIGHT_SHADOW_SKINNED LitSolid_DIFFMAP_PERPIXEL_POINTLIGHT_SHADOW
+Stencil Stencil
+LitSolid_DIRLIGHT_PERPIXEL_SHADOW LitSolid_AMBIENT_DIRLIGHT_PERPIXEL_SHADOW
+LitSolid_PERPIXEL_POINTLIGHT_SHADOW LitSolid_PERPIXEL_POINTLIGHT_SHADOW
+LitSolid_DIRLIGHT_PERPIXEL_SHADOW_SKINNED LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_LQSHADOW_PERPIXEL_SHADOW
+LitSolid_DIRLIGHT_INSTANCED_PERPIXEL_SHADOW LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_LQSHADOW_PERPIXEL_SHADOW
+LitSolid_PERPIXEL_POINTLIGHT_SHADOW LitSolid_DIFFMAP_LQSHADOW_PERPIXEL_POINTLIGHT_SHADOW
+LitSolid_PERPIXEL_POINTLIGHT_SHADOW_SKINNED LitSolid_DIFFMAP_LQSHADOW_PERPIXEL_POINTLIGHT_SHADOW
+LitSolid_INSTANCED_PERPIXEL_POINTLIGHT_SHADOW LitSolid_DIFFMAP_LQSHADOW_PERPIXEL_POINTLIGHT_SHADOW
+LitSolid_PERPIXEL_POINTLIGHT LitSolid_DIFFMAP_PERPIXEL_POINTLIGHT
+LitSolid_DIRLIGHT_PERPIXEL_SHADOW LitSolid_AMBIENT_DIRLIGHT_LQSHADOW_PERPIXEL_SHADOW
+LitSolid_PERPIXEL_POINTLIGHT_SHADOW LitSolid_LQSHADOW_PERPIXEL_POINTLIGHT_SHADOWBasic_DIFFMAP_VERTEXCOLOR Basic_ALPHAMASK_DIFFMAP_VERTEXCOLOR
+Basic_DIFFMAP_VERTEXCOLOR Basic_ALPHAMAP_VERTEXCOLOR
+Basic_VERTEXCOLOR Basic_VERTEXCOLOR
+LitSolid_INSTANCED_PERPIXEL_POINTLIGHT_SHADOW LitSolid_DIFFMAP_PERPIXEL_POINTLIGHT_SHADOW
+LitSolid_DIRLIGHT_PERPIXEL_SHADOW LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_LQSHADOW_PERPIXEL_SHADOW
+Basic_DIFFMAP_VERTEXCOLOR Basic_ALPHAMASK_DIFFMAP_VERTEXCOLOR
+LitSolid_PERPIXEL_POINTLIGHT_SHADOW LitSolid_LQSHADOW_PERPIXEL_POINTLIGHT_SHADOW
+LitSolid_DIRLIGHT_PERPIXEL LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_PERPIXEL
+LitSolid_DIRLIGHT_PERPIXEL_SKINNED LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_PERPIXEL
+LitSolid_DIRLIGHT_INSTANCED_PERPIXEL LitSolid_AMBIENT_DIFFMAP_DIRLIGHT_PERPIXEL
+LitSolid_PERPIXEL_POINTLIGHT_SKINNED LitSolid_DIFFMAP_PERPIXEL_POINTLIGHT
+LitSolid_DIRLIGHT_PERPIXEL LitSolid_AMBIENT_DIRLIGHT_PERPIXEL
+LitSolid_PERPIXEL_POINTLIGHT LitSolid_PERPIXEL_POINTLIGHT

Bin/Data/Scripts/NinjaSnowWar.as

@@ -165,6 +165,10 @@ void InitScene()
             dirLight.shadowIntensity = 0.333f;
         }
     }
+    
+    // Precache shaders if possible
+    if (cache.Exists("NinjaSnowWarShaders.txt"))
+        graphics.PrecacheShaders(cache.GetFile("NinjaSnowWarShaders.txt"));
 }
 
 void InitNetworking()

Docs/GettingStarted.dox

@@ -192,6 +192,7 @@ The engine can be configured using the following command line options.
 -r <freq>    Sound mixing frequency in Hz
 -p <paths>   Resource path(s) to use, separated by semicolons
 -log <level> Change the log level, valid 'level' values are 'debug', 'info', 'warning', 'error'
+-ds <file>   Dump used shader variations to a file for precaching
 -borderless  Borderless window mode
 -headless    Headless mode. No application window will be created
 -prepass     Use light pre-pass rendering

Docs/Reference.dox

@@ -143,6 +143,7 @@ The full list of supported parameters, their datatypes and default values:
 - TripleBuffer (bool) Whether to use triple-buffering. Default false.
 - VSync (bool) Whether to wait for vertical sync when presenting rendering window contents. Default false.
 - Multisample (int) Hardware multisampling level. Default 1 (no multisampling.)
+- DumpShaders (string) Filename to dump used shader variations to for precaching.
 - %RenderPath (string) Default renderpath resource name. Default empty, which causes forward rendering (Bin/CoreData/RenderPaths/Forward.xml) to be used.
 - Shadows (bool) Shadow rendering enable. Default true.
 - LowQualityShadows (bool) Low-quality (1 sample) shadow mode. Default false.
@@ -671,6 +672,7 @@ Graphics implements the low-level functionality:
 - Setting the screen mode
 - Keeping track of GPU resources
 - Keeping track of rendering context state (current rendertarget, vertex and index buffers, textures, shaders and renderstates)
+- Loading shaders
 - Performing primitive rendering operations
 - Handling lost device
 
@@ -1040,6 +1042,14 @@ current system, look up the defines from the description files and add them dire
 
 GLSL shaders in 1.3 and before used separate .vert and .frag files for the vertex and pixel shader source code. Now these are combined into .glsl files that include both shaders. Include files are likewise merged into .glsl files instead of separate vertex and pixel shader includes. To merge your shader code, append the pixel shader source into the vertex shader source file without the varyings definition (which should already be in the vertex shader code). Merge and fix include statements: change the file extension to .glsl. Note that some include files depend on each other so it is safest to always include Uniforms.glsl and Samplers.glsl first. Finally change the file extension of the vertex shader source file from .vert to .glsl and delete the .frag file. If you have pieces of pixel shader code (for example additional functions or variables) that produce errors when included in the vertex shader compile, wrap these with #ifdef COMPILEPS, and vice versa for vertex shader code.
 
+\section Shaders_Precaching
+
+The shader variations that are possibly used by a material technique in different lighting conditions and rendering passes are enumerated at material load time, but because of their large amount, are not actually compiled or loaded from bytecode before being used in rendering. Especially on OpenGL the compiling of shaders just before rendering can cause hitches in the framerate. To avoid this, used shader combinations can be dumped out to a file, then preloaded. See \ref Graphics::BeginDumpShaders "BeginDumpShaders()", \ref Graphics::EndDumpShaders "EndDumpShaders()" and \ref Graphics::PrecacheShaders "PrecacheShaders()" in the Graphics subsystem. The command line parameters -ds <file> can be used to instruct the Engine to begin dumping shaders automatically on startup.
+
+Note that the shader variations that actually end up being loaded will vary due to graphics settings, for example shadow quality or instancing on/off.
+
+When you create your own compilation defines, do not use the underscore _ in them, as that will interfere with the shader precaching: compilation defines will be separated by underscores in the shader variation names and it is impossible to distinguish between underscores separating the defines, and underscores inside a define.
+
 \page RenderPaths Render path
 
 %Scene rendering and any post-processing on a Viewport is defined by its RenderPath object, which can either be read from an XML file or be created programmatically.

Source/Engine/Core/Object.h

@@ -23,7 +23,6 @@
 #pragma once
 
 #include "LinkedList.h"
-#include "Ptr.h"
 #include "Variant.h"
 
 namespace Urho3D

Source/Engine/Engine/Engine.cpp

@@ -275,6 +275,8 @@ bool Engine::Initialize(const VariantMap& parameters)
             GetParameter(parameters, "MultiSample", 1).GetInt()
         ))
             return false;
+        if (HasParameter(parameters, "DumpShaders"))
+            graphics->BeginDumpShaders(GetParameter(parameters, "DumpShaders", String::EMPTY).GetString());
 
         if (HasParameter(parameters, "RenderPath"))
             renderer->SetDefaultRenderPath(cache->GetResource<XMLFile>(GetParameter(parameters, "RenderPath").GetString()));
@@ -702,6 +704,11 @@ VariantMap Engine::ParseParameters(const Vector<String>& arguments)
                 ret["ResourcePaths"] = value;
                 ++i;
             }
+            else if (argument == "ds" && !value.Empty())
+            {
+                ret["DumpShaders"] = value;
+                ++i;
+            }
             #ifdef ENABLE_TESTING
             else if (argument == "timeout" && !value.Empty())
             {

Source/Engine/Graphics/DebugRenderer.cpp

@@ -31,7 +31,6 @@
 #include "Light.h"
 #include "Polyhedron.h"
 #include "Profiler.h"
-#include "Renderer.h"
 #include "ResourceCache.h"
 #include "ShaderVariation.h"
 #include "VertexBuffer.h"
@@ -310,15 +309,14 @@ void DebugRenderer::Render()
         return;
 
     Graphics* graphics = GetSubsystem<Graphics>();
-    Renderer* renderer = GetSubsystem<Renderer>();
 
     if (!graphics || graphics->IsDeviceLost())
         return;
 
     PROFILE(RenderDebugGeometry);
 
-    ShaderVariation* vs = renderer->GetShader(VS, "Basic", "VERTEXCOLOR");
-    ShaderVariation* ps = renderer->GetShader(PS, "Basic", "VERTEXCOLOR");
+    ShaderVariation* vs = graphics->GetShader(VS, "Basic", "VERTEXCOLOR");
+    ShaderVariation* ps = graphics->GetShader(PS, "Basic", "VERTEXCOLOR");
     
     unsigned numVertices = (lines_.Size() + noDepthLines_.Size()) * 2;
     // Resize the vertex buffer if too small or much too large

Source/Engine/Graphics/Direct3D9/D3D9Graphics.cpp

@@ -29,6 +29,7 @@
 #include "CustomGeometry.h"
 #include "DebugRenderer.h"
 #include "DecalSet.h"
+#include "File.h"
 #include "Graphics.h"
 #include "GraphicsEvents.h"
 #include "GraphicsImpl.h"
@@ -39,6 +40,7 @@
 #include "ParticleEmitter.h"
 #include "ProcessUtils.h"
 #include "Profiler.h"
+#include "ResourceCache.h"
 #include "Shader.h"
 #include "ShaderVariation.h"
 #include "Skybox.h"
@@ -205,7 +207,9 @@ Graphics::Graphics(Context* context) :
     numPrimitives_(0),
     numBatches_(0),
     maxScratchBufferRequest_(0),
-    defaultTextureFilterMode_(FILTER_BILINEAR)
+    defaultTextureFilterMode_(FILTER_BILINEAR),
+    shaderPath_("Shaders/HLSL/"),
+    shaderExtension_(".hlsl")
 {
     SetTextureUnitMappings();
     
@@ -1026,7 +1030,7 @@ void Graphics::SetShaders(ShaderVariation* vs, ShaderVariation* ps)
         }
         
         // Create the shader now if not yet created. If already attempted, do not retry
-        if (vs && !vs->IsCompiled())
+        if (vs && !vs->GetGPUObject())
         {
             if (vs->GetCompilerOutput().Empty())
             {
@@ -1069,7 +1073,7 @@ void Graphics::SetShaders(ShaderVariation* vs, ShaderVariation* ps)
                 i->second_.register_ = M_MAX_UNSIGNED;
         }
         
-        if (ps && !ps->IsCompiled())
+        if (ps && !ps->GetGPUObject())
         {
             if (ps->GetCompilerOutput().Empty())
             {
@@ -1102,6 +1106,17 @@ void Graphics::SetShaders(ShaderVariation* vs, ShaderVariation* ps)
         
         pixelShader_ = ps;
     }
+    
+    // If used shaders are being dumped, check now for unique combination and write if necessary
+    if (shaderDumpFile_ && vertexShader_ && pixelShader_)
+    {
+        String shaderNames = vertexShader_->GetName() + " " + pixelShader_->GetName();
+        if (!usedShaders_.Contains(shaderNames))
+        {
+            shaderDumpFile_->WriteLine(shaderNames);
+            usedShaders_.Insert(shaderNames);
+        }
+    }
 }
 
 void Graphics::SetShaderParameter(StringHash param, const float* data, unsigned count)
@@ -1879,6 +1894,59 @@ void Graphics::SetForceSM2(bool enable)
         LOGERROR("Force Shader Model 2 can not be changed after setting the initial screen mode");
 }
 
+void Graphics::BeginDumpShaders(const String& fileName)
+{
+    shaderDumpFile_ = new File(context_, fileName, FILE_READWRITE);
+    if (!shaderDumpFile_->IsOpen())
+    {
+        shaderDumpFile_.Reset();
+        return;
+    }
+    
+    LOGDEBUG("Begin dumping shaders to file " + fileName);
+    
+    // Read existing combinations from the file (if any) to avoid duplicates
+    usedShaders_.Clear();
+    while (!shaderDumpFile_->IsEof())
+        usedShaders_.Insert(shaderDumpFile_->ReadLine());
+}
+
+void Graphics::EndDumpShaders()
+{
+    if (shaderDumpFile_)
+    {
+        LOGDEBUG("End dumping shaders");
+        shaderDumpFile_.Reset();
+    }
+}
+
+void Graphics::PrecacheShaders(Deserializer& source)
+{
+    PROFILE(PrecacheShaders);
+    
+    LOGDEBUG("Begin precaching shaders");
+    
+    while (!source.IsEof())
+    {
+        Vector<String> shaders = source.ReadLine().Split(' ');
+        if (shaders.Size() == 2)
+        {
+            // Shader names are stored as Name_DEFINE1_DEFINE2 etc.
+            unsigned vsNameEnd = shaders[0].Find('_');
+            unsigned psNameEnd = shaders[1].Find('_');
+            ShaderVariation* vs = GetShader(VS, shaders[0].Substring(0, vsNameEnd), vsNameEnd != String::NPOS ?
+                shaders[0].Substring(vsNameEnd + 1).Replaced('_', ' ') : String::EMPTY);
+            ShaderVariation* ps = GetShader(PS, shaders[1].Substring(0, psNameEnd), psNameEnd != String::NPOS ?
+                shaders[1].Substring(psNameEnd + 1).Replaced('_', ' ') : String::EMPTY);
+            // Set the shaders in use to fully compile them
+            if (vs && ps)
+                SetShaders(vs, ps);
+        }
+    }
+    
+    LOGDEBUG("End precaching shaders");
+}
+
 bool Graphics::IsInitialized() const
 {
     return impl_->window_ != 0 && impl_->GetDevice() != 0;
@@ -1971,6 +2039,29 @@ unsigned Graphics::GetFormat(CompressedFormat format) const
     return 0;
 }
 
+ShaderVariation* Graphics::GetShader(ShaderType type, const String& name, const String& defines) const
+{
+    return GetShader(type, name.CString(), defines.CString());
+}
+
+ShaderVariation* Graphics::GetShader(ShaderType type, const char* name, const char* defines) const
+{
+    if (lastShaderName_ != name || !lastShader_)
+    {
+        ResourceCache* cache = GetSubsystem<ResourceCache>();
+        
+        String fullShaderName = shaderPath_ + name + shaderExtension_;
+        // Try to reduce repeated error log prints because of missing shaders
+        if (lastShaderName_ == name && !cache->Exists(fullShaderName))
+            return 0;
+        
+        lastShader_ = cache->GetResource<Shader>(fullShaderName);
+        lastShaderName_ = name;
+    }
+    
+    return lastShader_ ? lastShader_->GetVariation(type, defines) : (ShaderVariation*)0;
+}
+
 VertexBuffer* Graphics::GetVertexBuffer(unsigned index) const
 {
     return index < MAX_VERTEX_STREAMS ? vertexBuffers_[index] : 0;

Source/Engine/Graphics/Direct3D9/D3D9Graphics.h

@@ -24,6 +24,7 @@
 
 #include "ArrayPtr.h"
 #include "Color.h"
+#include "HashSet.h"
 #include "Image.h"
 #include "Object.h"
 #include "Plane.h"
@@ -33,11 +34,13 @@
 namespace Urho3D
 {
 
+class File;
 class Image;
 class IndexBuffer;
 class GPUObject;
 class GraphicsImpl;
 class RenderSurface;
+class Shader;
 class ShaderVariation;
 class Texture;
 class Texture2D;
@@ -211,6 +214,12 @@ public:
     void ResetStreamFrequencies();
     /// Set force Shader Model 2 flag. Only effective before setting the initial screen mode.
     void SetForceSM2(bool enable);
+    /// Begin dumping shader variation names to a file for precaching.
+    void BeginDumpShaders(const String& fileName);
+    /// End dumping shader variations names.
+    void EndDumpShaders();
+    /// Precache shader variations from a file containing rows of shader names, generated with BeginDumpShaders().
+    void PrecacheShaders(Deserializer& source);
     
     /// Return whether rendering initialized.
     bool IsInitialized() const;
@@ -278,7 +287,11 @@ public:
     IntVector2 GetDesktopResolution() const;
     /// Return hardware format for a compressed image format, or 0 if unsupported.
     unsigned GetFormat(CompressedFormat format) const;
-    /// Return vertex buffer by index.
+    /// Return a shader variation by name and defines.
+    ShaderVariation* GetShader(ShaderType type, const String& name, const String& defines = String::EMPTY) const;
+    /// Return a shader variation by name and defines.
+    ShaderVariation* GetShader(ShaderType type, const char* name, const char* defines) const;
+    /// Return current vertex buffer by index.
     VertexBuffer* GetVertexBuffer(unsigned index) const;
     /// Return current index buffer.
     IndexBuffer* GetIndexBuffer() const { return indexBuffer_; }
@@ -559,6 +572,18 @@ private:
     TextureFilterMode defaultTextureFilterMode_;
     /// Remembered shader parameter sources.
     const void* shaderParameterSources_[MAX_SHADER_PARAMETER_GROUPS];
+    /// Base directory for shaders.
+    String shaderPath_;
+    /// File extension for shaders.
+    String shaderExtension_;
+    /// Last used shader in shader variation query.
+    mutable WeakPtr<Shader> lastShader_;
+    /// Last used shader name in shader variation query.
+    mutable String lastShaderName_;
+    /// File shader variations are dumped into. Null if dumping not active.
+    SharedPtr<File> shaderDumpFile_;
+    /// Hash set for detecting already dumped variations.
+    HashSet<String> usedShaders_;
 };
 
 /// Register Graphics library objects.

Source/Engine/Graphics/Direct3D9/D3D9ShaderVariation.cpp

@@ -42,8 +42,7 @@ namespace Urho3D
 ShaderVariation::ShaderVariation(Shader* owner, ShaderType type) :
     GPUObject(owner->GetSubsystem<Graphics>()),
     owner_(owner),
-    type_(type),
-    compiled_(false)
+    type_(type)
 {
     for (unsigned i = 0; i < MAX_TEXTURE_UNITS; ++i)
         useTextureUnit_[i] = false;
@@ -97,8 +96,6 @@ bool ShaderVariation::Create()
             (const DWORD*)&byteCode[0],
             (IDirect3DVertexShader9**)&object_)))
             compilerOutput_ = "Could not create vertex shader";
-        else
-            compiled_ = true;
     }
     else
     {
@@ -106,11 +103,9 @@ bool ShaderVariation::Create()
             (const DWORD*)&byteCode[0],
             (IDirect3DPixelShader9**)&object_)))
             compilerOutput_ = "Could not create pixel shader";
-        else
-            compiled_ = true;
     }
     
-    return compiled_;
+    return object_ != 0;
 }
 
 void ShaderVariation::Release()
@@ -136,7 +131,6 @@ void ShaderVariation::Release()
         }
         
         object_ = 0;
-        compiled_ = false;
         compilerOutput_.Clear();
         
         for (unsigned i = 0; i < MAX_TEXTURE_UNITS; ++i)
@@ -395,8 +389,6 @@ void ShaderVariation::SaveByteCode(const PODVector<unsigned>& byteCode, const St
 {
     ResourceCache* cache = owner_->GetSubsystem<ResourceCache>();
     FileSystem* fileSystem = owner_->GetSubsystem<FileSystem>();
-    if (!cache || !fileSystem)
-        return;
     
     String path = GetPath(cache->GetResourceFileName(owner_->GetName())) + "Cache/";
     String fullName = path + GetFileNameAndExtension(binaryShaderName);

Source/Engine/Graphics/Direct3D9/D3D9ShaderVariation.h

@@ -88,8 +88,6 @@ public:
     const String& GetName() const { return name_; }
     /// Return defines.
     const String& GetDefines() const { return defines_; }
-    /// Return whether successfully compiled.
-    bool IsCompiled() const { return compiled_; }
     /// Return compile error/warning string.
     const String& GetCompilerOutput() const { return compilerOutput_; }
     /// Return whether uses a parameter.
@@ -121,8 +119,6 @@ private:
     String defines_;
     /// Shader compile error string.
     String compilerOutput_;
-    /// Compiled flag.
-    bool compiled_;
     /// Shader parameters.
     HashMap<StringHash, ShaderParameter> parameters_;
     /// Texture unit use flags.

Source/Engine/Graphics/OpenGL/OGLGraphics.cpp

@@ -30,6 +30,7 @@
 #include "CustomGeometry.h"
 #include "DebugRenderer.h"
 #include "DecalSet.h"
+#include "File.h"
 #include "Graphics.h"
 #include "GraphicsEvents.h"
 #include "GraphicsImpl.h"
@@ -42,6 +43,7 @@
 #include "ProcessUtils.h"
 #include "Profiler.h"
 #include "RenderSurface.h"
+#include "ResourceCache.h"
 #include "Shader.h"
 #include "ShaderProgram.h"
 #include "ShaderVariation.h"
@@ -188,7 +190,9 @@ Graphics::Graphics(Context* context_) :
     shadowMapFormat_(GL_DEPTH_COMPONENT16),
     hiresShadowMapFormat_(GL_DEPTH_COMPONENT24),
     defaultTextureFilterMode_(FILTER_BILINEAR),
-    releasingGPUObjects_(false)
+    releasingGPUObjects_(false),
+    shaderPath_("Shaders/GLSL/"),
+    shaderExtension_(".glsl")
 {
     SetTextureUnitMappings();
     ResetCachedState();
@@ -1014,7 +1018,7 @@ void Graphics::SetShaders(ShaderVariation* vs, ShaderVariation* ps)
     ClearParameterSources();
     
     // Compile the shaders now if not yet compiled. If already attempted, do not retry
-    if (vs && !vs->IsCompiled())
+    if (vs && !vs->GetGPUObject())
     {
         if (vs->GetCompilerOutput().Empty())
         {
@@ -1033,7 +1037,7 @@ void Graphics::SetShaders(ShaderVariation* vs, ShaderVariation* ps)
             vs = 0;
     }
     
-    if (ps && !ps->IsCompiled())
+    if (ps && !ps->GetGPUObject())
     {
         if (ps->GetCompilerOutput().Empty())
         {
@@ -1070,7 +1074,7 @@ void Graphics::SetShaders(ShaderVariation* vs, ShaderVariation* ps)
         if (i != shaderPrograms_.End())
         {
             // Use the existing linked program
-            if (i->second_->IsLinked())
+            if (i->second_->GetGPUObject())
             {
                 glUseProgram(i->second_->GetGPUObject());
                 shaderProgram_ = i->second_;
@@ -1105,6 +1109,18 @@ void Graphics::SetShaders(ShaderVariation* vs, ShaderVariation* ps)
             shaderPrograms_[combination] = newProgram;
         }
     }
+    
+
+    // If used shaders are being dumped, check now for unique combination and write if necessary
+    if (shaderDumpFile_ && vertexShader_ && pixelShader_)
+    {
+        String shaderNames = vertexShader_->GetName() + " " + pixelShader_->GetName();
+        if (!usedShaders_.Contains(shaderNames))
+        {
+            shaderDumpFile_->WriteLine(shaderNames);
+            usedShaders_.Insert(shaderNames);
+        }
+    }
 }
 
 void Graphics::SetShaderParameter(StringHash param, const float* data, unsigned count)
@@ -1868,6 +1884,59 @@ void Graphics::SetForceSM2(bool enable)
 {
 }
 
+void Graphics::BeginDumpShaders(const String& fileName)
+{
+    shaderDumpFile_ = new File(context_, fileName, FILE_READWRITE);
+    if (!shaderDumpFile_->IsOpen())
+    {
+        shaderDumpFile_.Reset();
+        return;
+    }
+    
+    LOGDEBUG("Begin dumping shaders to file " + fileName);
+    
+    // Read existing combinations from the file (if any) to avoid duplicates
+    usedShaders_.Clear();
+    while (!shaderDumpFile_->IsEof())
+        usedShaders_.Insert(shaderDumpFile_->ReadLine());
+}
+
+void Graphics::EndDumpShaders()
+{
+    if (shaderDumpFile_)
+    {
+        LOGDEBUG("End dumping shaders");
+        shaderDumpFile_.Reset();
+    }
+}
+
+void Graphics::PrecacheShaders(Deserializer& source)
+{
+    PROFILE(PrecacheShaders);
+    
+    LOGDEBUG("Begin precaching shaders");
+    
+    while (!source.IsEof())
+    {
+        Vector<String> shaders = source.ReadLine().Split(' ');
+        if (shaders.Size() == 2)
+        {
+            // Shader names are stored as Name_DEFINE1_DEFINE2 etc.
+            unsigned vsNameEnd = shaders[0].Find('_');
+            unsigned psNameEnd = shaders[1].Find('_');
+            ShaderVariation* vs = GetShader(VS, shaders[0].Substring(0, vsNameEnd), vsNameEnd != String::NPOS ?
+                shaders[0].Substring(vsNameEnd + 1).Replaced('_', ' ') : String::EMPTY);
+            ShaderVariation* ps = GetShader(PS, shaders[1].Substring(0, psNameEnd), psNameEnd != String::NPOS ?
+                shaders[1].Substring(psNameEnd + 1).Replaced('_', ' ') : String::EMPTY);
+            // Set the shaders in use to fully compile them
+            if (vs && ps)
+                SetShaders(vs, ps);
+        }
+    }
+    
+    LOGDEBUG("End precaching shaders");
+}
+
 bool Graphics::IsInitialized() const
 {
     return impl_->window_ != 0;
@@ -1981,6 +2050,28 @@ unsigned Graphics::GetFormat(CompressedFormat format) const
     }
 }
 
+ShaderVariation* Graphics::GetShader(ShaderType type, const String& name, const String& defines) const
+{
+    return GetShader(type, name.CString(), defines.CString());
+}
+
+ShaderVariation* Graphics::GetShader(ShaderType type, const char* name, const char* defines) const
+{
+    if (lastShaderName_ != name || !lastShader_)
+    {
+        ResourceCache* cache = GetSubsystem<ResourceCache>();
+        
+        String fullShaderName = shaderPath_ + name + shaderExtension_;
+        // Try to reduce repeated error log prints because of missing shaders
+        if (lastShaderName_ == name && !cache->Exists(fullShaderName))
+            return 0;
+        
+        lastShader_ = cache->GetResource<Shader>(fullShaderName);
+        lastShaderName_ = name;
+    }
+    
+    return lastShader_ ? lastShader_->GetVariation(type, defines) : (ShaderVariation*)0;
+}
 
 VertexBuffer* Graphics::GetVertexBuffer(unsigned index) const
 {

Source/Engine/Graphics/OpenGL/OGLGraphics.h

@@ -25,7 +25,7 @@
 #include "ArrayPtr.h"
 #include "Color.h"
 #include "GraphicsDefs.h"
-#include "HashMap.h"
+#include "HashSet.h"
 #include "Image.h"
 #include "Object.h"
 #include "Plane.h"
@@ -34,11 +34,13 @@
 namespace Urho3D
 {
 
+class File;
 class Image;
 class IndexBuffer;
 class GPUObject;
 class GraphicsImpl;
 class RenderSurface;
+class Shader;
 class ShaderProgram;
 class ShaderVariation;
 class Texture;
@@ -217,6 +219,12 @@ public:
     void ResetStreamFrequencies();
     /// Set force Shader Model 2 flag. No-op on OpenGL.
     void SetForceSM2(bool enable);
+    /// Begin dumping shader variation names to a file for precaching.
+    void BeginDumpShaders(const String& fileName);
+    /// End dumping shader variations names.
+    void EndDumpShaders();
+    /// Precache shader variations from a file containing rows of shader names, generated with BeginDumpShaders().
+    void PrecacheShaders(Deserializer& source);
 
     /// Return whether rendering initialized.
     bool IsInitialized() const;
@@ -286,7 +294,11 @@ public:
     IntVector2 GetDesktopResolution() const;
     /// Return hardware format for a compressed image format, or 0 if unsupported.
     unsigned GetFormat(CompressedFormat format) const;
-    /// Return vertex buffer by index.
+    /// Return a shader variation by name and defines.
+    ShaderVariation* GetShader(ShaderType type, const String& name, const String& defines = String::EMPTY) const;
+    /// Return a shader variation by name and defines.
+    ShaderVariation* GetShader(ShaderType type, const char* name, const char* defines) const;
+    /// Return current vertex buffer by index.
     VertexBuffer* GetVertexBuffer(unsigned index) const;
     /// Return index buffer.
     IndexBuffer* GetIndexBuffer() const { return indexBuffer_; }
@@ -571,6 +583,18 @@ private:
     bool useClipPlane_;
     /// Releasing GPU objects flag.
     bool releasingGPUObjects_;
+    /// Base directory for shaders.
+    String shaderPath_;
+    /// File extension for shaders.
+    String shaderExtension_;
+    /// Last used shader in shader variation query.
+    mutable WeakPtr<Shader> lastShader_;
+    /// Last used shader name in shader variation query.
+    mutable String lastShaderName_;
+    /// File shader variations are dumped into. Null if dumping not active.
+    SharedPtr<File> shaderDumpFile_;
+    /// Hash set for detecting already dumped variations.
+    HashSet<String> usedShaders_;
 };
 
 /// Register Graphics library objects.

Source/Engine/Graphics/OpenGL/OGLShaderProgram.cpp

@@ -34,8 +34,7 @@ namespace Urho3D
 ShaderProgram::ShaderProgram(Graphics* graphics, ShaderVariation* vertexShader, ShaderVariation* pixelShader) :
     GPUObject(graphics),
     vertexShader_(vertexShader),
-    pixelShader_(pixelShader),
-    linked_(false)
+    pixelShader_(pixelShader)
 {
     for (unsigned i = 0; i < MAX_TEXTURE_UNITS; ++i)
         useTextureUnit_[i] = false;
@@ -53,7 +52,7 @@ void ShaderProgram::OnDeviceLost()
     if (graphics_ && graphics_->GetShaderProgram() == this)
         graphics_->SetShaders(0, 0);
     
-    linked_ = false;
+
     linkerOutput_.Clear();
 }
 
@@ -73,7 +72,6 @@ void ShaderProgram::Release()
         }
         
         object_ = 0;
-        linked_ = false;
         linkerOutput_.Clear();
         shaderParameters_.Clear();
         
@@ -121,18 +119,19 @@ bool ShaderProgram::Link()
     
     int linked, length;
     glGetProgramiv(object_, GL_LINK_STATUS, &linked);
-    linked_ = linked != 0;
-    if (!linked_)
+    if (!linked)
     {
         glGetProgramiv(object_, GL_INFO_LOG_LENGTH, &length);
         linkerOutput_.Resize(length);
         int outLength;
         glGetProgramInfoLog(object_, length, &outLength, &linkerOutput_[0]);
+        glDeleteProgram(object_);
+        object_ = 0;
     }
     else
         linkerOutput_.Clear();
     
-    if (!linked_)
+    if (!object_)
         return false;
     
     const int MAX_PARAMETER_NAME_LENGTH = 256;

Source/Engine/Graphics/OpenGL/OGLShaderProgram.h

@@ -69,8 +69,6 @@ public:
     bool HasTextureUnit(TextureUnit unit) const { return useTextureUnit_[unit]; }
     /// Return the info for a shader parameter, or null if does not exist.
     const ShaderParameter* GetParameter(StringHash param) const;
-    /// Return whether successfully linked.
-    bool IsLinked() const { return linked_; }
     /// Return linker output.
     const String& GetLinkerOutput() const { return linkerOutput_; }
     
@@ -85,8 +83,6 @@ private:
     bool useTextureUnit_[MAX_TEXTURE_UNITS];
     /// Shader link error string.
     String linkerOutput_;
-    /// Linked flag.
-    bool linked_;
 };
 
 }

Source/Engine/Graphics/OpenGL/OGLShaderVariation.cpp

@@ -36,8 +36,7 @@ namespace Urho3D
 ShaderVariation::ShaderVariation(Shader* owner, ShaderType type) :
     GPUObject(owner->GetSubsystem<Graphics>()),
     owner_(owner),
-    type_(type),
-    compiled_(false)
+    type_(type)
 {
 }
 
@@ -49,8 +48,7 @@ ShaderVariation::~ShaderVariation()
 void ShaderVariation::OnDeviceLost()
 {
     GPUObject::OnDeviceLost();
-    
-    compiled_ = false;
+
     compilerOutput_.Clear();
     
     if (graphics_)
@@ -81,7 +79,6 @@ void ShaderVariation::Release()
         }
         
         object_ = 0;
-        compiled_ = false;
         compilerOutput_.Clear();
         
         graphics_->CleanupShaderPrograms();
@@ -91,7 +88,7 @@ void ShaderVariation::Release()
 bool ShaderVariation::Create()
 {
     Release();
-    
+
     if (!owner_)
     {
         compilerOutput_ = "Owner shader has expired";
@@ -139,18 +136,19 @@ bool ShaderVariation::Create()
     
     int compiled, length;
     glGetShaderiv(object_, GL_COMPILE_STATUS, &compiled);
-    compiled_ = compiled != 0;
-    if (!compiled_)
+    if (!compiled)
     {
         glGetShaderiv(object_, GL_INFO_LOG_LENGTH, &length);
         compilerOutput_.Resize(length);
         int outLength;
         glGetShaderInfoLog(object_, length, &outLength, &compilerOutput_[0]);
+        glDeleteShader(object_);
+        object_ = 0;
     }
     else
         compilerOutput_.Clear();
     
-    return compiled_;
+    return object_ != 0;
 }
 
 void ShaderVariation::SetName(const String& name)

Source/Engine/Graphics/OpenGL/OGLShaderVariation.h

@@ -60,8 +60,6 @@ public:
     const String& GetName() const { return name_; }
     /// Return defines.
     const String& GetDefines() const { return defines_; }
-    /// Return whether successfully compiled.
-    bool IsCompiled() const { return compiled_; }
     /// Return compile error/warning string.
     const String& GetCompilerOutput() const { return compilerOutput_; }
     
@@ -76,8 +74,6 @@ private:
     String defines_;
     /// Shader compile error string.
     String compilerOutput_;
-    /// Compiled flag.
-    bool compiled_;
 };
 
 }

Source/Engine/Graphics/Renderer.cpp

@@ -38,7 +38,6 @@
 #include "RenderPath.h"
 #include "ResourceCache.h"
 #include "Scene.h"
-#include "Shader.h"
 #include "ShaderVariation.h"
 #include "Technique.h"
 #include "Texture2D.h"
@@ -181,10 +180,10 @@ static const char* shadowVariations[] =
     ""
     #else
     // On Direct3D the quality is always "low" if not using hardware shadow compare
-    "",
-    "LQSHADOW HWSHADOW ",
-    "",
-    "HWSHADOW "
+    "SHADOWCMP",
+    "LQSHADOW",
+    "SHADOWCMP",
+    ""
     #endif
 };
 
@@ -288,14 +287,6 @@ Renderer::Renderer(Context* context) :
     shadersDirty_(true),
     initialized_(false)
 {
-    #ifndef USE_OPENGL
-    shaderPath_ = "Shaders/HLSL/";
-    shaderExtension_ = ".hlsl";
-    #else
-    shaderPath_ = "Shaders/GLSL/";
-    shaderExtension_ = ".glsl";
-    #endif
-
     SubscribeToEvent(E_SCREENMODE, HANDLER(Renderer, HandleScreenMode));
     SubscribeToEvent(E_GRAPHICSFEATURES, HANDLER(Renderer, HandleGraphicsFeatures));
     
@@ -560,29 +551,6 @@ unsigned Renderer::GetNumOccluders(bool allViews) const
     return numOccluders;
 }
 
-
-ShaderVariation* Renderer::GetShader(ShaderType type, const String& name, const String& defines) const
-{
-    return GetShader(type, name.CString(), defines.CString());
-}
-
-ShaderVariation* Renderer::GetShader(ShaderType type, const char* name, const char* defines) const
-{
-    if (lastShaderName_ != name || !lastShader_)
-    {
-        ResourceCache* cache = GetSubsystem<ResourceCache>();
-        String fullShaderName = shaderPath_ + name + shaderExtension_;
-        // Try to reduce repeated error log prints because of missing shaders
-        if (lastShaderName_ == name && !cache->Exists(fullShaderName))
-            return 0;
-        
-        lastShader_ = cache->GetResource<Shader>(fullShaderName);
-        lastShaderName_ = name;
-    }
-    
-    return lastShader_ ? lastShader_->GetVariation(type, defines) : (ShaderVariation*)0;
-}
-
 void Renderer::Update(float timeStep)
 {
     PROFILE(UpdateViews);
@@ -1237,8 +1205,8 @@ void Renderer::SetLightVolumeBatchShaders(Batch& batch, const String& vsName, co
         psi += DLPS_ORTHO;
     }
     
-    batch.vertexShader_ = GetShader(VS, vsName, deferredLightVSVariations[vsi]);
-    batch.pixelShader_ = GetShader(PS, psName, deferredLightPSVariations_[psi]);
+    batch.vertexShader_ = graphics_->GetShader(VS, vsName, deferredLightVSVariations[vsi]);
+    batch.pixelShader_ = graphics_->GetShader(PS, psName, deferredLightPSVariations_[psi]);
 }
 
 void Renderer::SetCullMode(CullMode mode, Camera* camera)
@@ -1352,7 +1320,7 @@ void Renderer::OptimizeLightByStencil(Light* light, Camera* camera)
         graphics_->SetColorWrite(false);
         graphics_->SetDepthWrite(false);
         graphics_->SetStencilTest(true, CMP_ALWAYS, OP_REF, OP_KEEP, OP_KEEP, lightStencilValue_);
-        graphics_->SetShaders(GetShader(VS, "Stencil"), GetShader(PS, "Stencil"));
+        graphics_->SetShaders(graphics_->GetShader(VS, "Stencil"), graphics_->GetShader(PS, "Stencil"));
         graphics_->SetShaderParameter(VSP_VIEWPROJ, projection * view);
         graphics_->SetShaderParameter(VSP_MODEL, light->GetVolumeTransform(camera));
         
@@ -1546,7 +1514,7 @@ void Renderer::LoadPassShaders(Technique* tech, StringHash type)
             unsigned g = k / MAX_LIGHT_VS_VARIATIONS;
             unsigned l = k % MAX_LIGHT_VS_VARIATIONS;
             
-            vertexShaders[j] = GetShader(VS, pass->GetVertexShader(), pass->GetVertexShaderDefines() + " " +
+            vertexShaders[j] = graphics_->GetShader(VS, pass->GetVertexShader(), pass->GetVertexShaderDefines() + " " +
                 lightVSVariations[l] + geometryVSVariations[g] + heightFogVariations[h]);
         }
         for (unsigned j = 0; j < MAX_LIGHT_PS_VARIATIONS * 2; ++j)
@@ -1555,11 +1523,11 @@ void Renderer::LoadPassShaders(Technique* tech, StringHash type)
             unsigned h = j / MAX_LIGHT_PS_VARIATIONS;
             if (k & LPS_SHADOW)
             {
-                pixelShaders[j] = GetShader(PS, pass->GetPixelShader(), pass->GetPixelShaderDefines() + " " +
+                pixelShaders[j] = graphics_->GetShader(PS, pass->GetPixelShader(), pass->GetPixelShaderDefines() + " " +
                     lightPSVariations[k] + shadowVariations[shadows] + heightFogVariations[h]);
             }
             else
-                pixelShaders[j] = GetShader(PS, pass->GetPixelShader(), pass->GetPixelShaderDefines() + " " +
+                pixelShaders[j] = graphics_->GetShader(PS, pass->GetPixelShader(), pass->GetPixelShaderDefines() + " " +
                     lightPSVariations[k] + heightFogVariations[h]);
         }
     }
@@ -1575,7 +1543,7 @@ void Renderer::LoadPassShaders(Technique* tech, StringHash type)
                 unsigned h = j / (MAX_GEOMETRYTYPES * MAX_VERTEXLIGHT_VS_VARIATIONS);
                 unsigned g = k / MAX_VERTEXLIGHT_VS_VARIATIONS;
                 unsigned l = k % MAX_VERTEXLIGHT_VS_VARIATIONS;
-                vertexShaders[j] = GetShader(VS, pass->GetVertexShader(), pass->GetVertexShaderDefines() + " " +
+                vertexShaders[j] = graphics_->GetShader(VS, pass->GetVertexShader(), pass->GetVertexShaderDefines() + " " +
                     vertexLightVSVariations[l] + geometryVSVariations[g] + heightFogVariations[h]);
             }
         }
@@ -1586,7 +1554,7 @@ void Renderer::LoadPassShaders(Technique* tech, StringHash type)
             {
                 unsigned k = j % MAX_GEOMETRYTYPES;
                 unsigned h = j / MAX_GEOMETRYTYPES;
-                vertexShaders[j] = GetShader(VS, pass->GetVertexShader(), pass->GetVertexShaderDefines() + " " +
+                vertexShaders[j] = graphics_->GetShader(VS, pass->GetVertexShader(), pass->GetVertexShaderDefines() + " " +
                     geometryVSVariations[k] + heightFogVariations[h]);
             }
         }
@@ -1594,7 +1562,7 @@ void Renderer::LoadPassShaders(Technique* tech, StringHash type)
         pixelShaders.Resize(2);
         for (unsigned j = 0; j < 2; ++j)
         {
-            pixelShaders[j] = GetShader(PS, pass->GetPixelShader(), pass->GetPixelShaderDefines() + " " +
+            pixelShaders[j] = graphics_->GetShader(PS, pass->GetPixelShader(), pass->GetPixelShaderDefines() + " " +
                 heightFogVariations[j]);
         }
     }

Source/Engine/Graphics/Renderer.h

@@ -43,7 +43,6 @@ class Graphics;
 class RenderPath;
 class RenderSurface;
 class ResourceCache;
-class Shader;
 class Skeleton;
 class OcclusionBuffer;
 class Texture2D;
@@ -287,10 +286,6 @@ public:
     TextureCube* GetIndirectionCubeMap() const { return indirectionCubeMap_; }
     /// Return the instancing vertex buffer
     VertexBuffer* GetInstancingBuffer() const { return dynamicInstancing_ ? instancingBuffer_ : (VertexBuffer*)0; }
-    /// Return a shader variation by name and defines.
-    ShaderVariation* GetShader(ShaderType type, const String& name, const String& defines = String::EMPTY) const;
-    /// Return a shader variation by name and defines.
-    ShaderVariation* GetShader(ShaderType type, const char* name, const char* defines) const;
     /// Return the frame update parameters.
     const FrameInfo& GetFrameInfo() const { return frame_; }
     
@@ -430,16 +425,8 @@ private:
     HashSet<Technique*> shaderErrorDisplayed_;
     /// Mutex for shadow camera allocation.
     Mutex rendererMutex_;
-    /// Base directory for shaders.
-    String shaderPath_;
-    /// File extension for shaders.
-    String shaderExtension_;
     /// Current variation names for deferred light volume shaders.
     Vector<String> deferredLightPSVariations_;
-    /// Last used shader in shader variation query.
-    mutable WeakPtr<Shader> lastShader_;
-    /// Last used shader name in shader variation query.
-    mutable String lastShaderName_;
     /// Frame info for rendering.
     FrameInfo frame_;
     /// Texture anisotropy level.

Source/Engine/Graphics/View.cpp

@@ -1549,10 +1549,10 @@ void View::RenderQuad(RenderPathCommand& command)
         return;
     
     // If shader can not be found, clear it from the command to prevent redundant attempts
-    ShaderVariation* vs = renderer_->GetShader(VS, command.vertexShaderName_, command.vertexShaderDefines_);
+    ShaderVariation* vs = graphics_->GetShader(VS, command.vertexShaderName_, command.vertexShaderDefines_);
     if (!vs)
         command.vertexShaderName_ = String::EMPTY;
-    ShaderVariation* ps = renderer_->GetShader(PS, command.pixelShaderName_, command.pixelShaderDefines_);
+    ShaderVariation* ps = graphics_->GetShader(PS, command.pixelShaderName_, command.pixelShaderDefines_);
     if (!ps)
         command.pixelShaderName_ = String::EMPTY;
     
@@ -1807,7 +1807,7 @@ void View::BlitFramebuffer(Texture2D* source, RenderSurface* destination, bool d
     graphics_->SetViewport(viewRect_);
     
     String shaderName = "CopyFramebuffer";
-    graphics_->SetShaders(renderer_->GetShader(VS, shaderName), renderer_->GetShader(PS, shaderName));
+    graphics_->SetShaders(graphics_->GetShader(VS, shaderName), graphics_->GetShader(PS, shaderName));
     
     float rtWidth = (float)rtSize_.x_;
     float rtHeight = (float)rtSize_.y_;

Source/Engine/IO/File.cpp

@@ -41,6 +41,7 @@ static const wchar_t* openMode[] =
 {
     L"rb",
     L"wb",
+    L"r+b",
     L"w+b"
 };
 #else
@@ -48,6 +49,7 @@ static const char* openMode[] =
 {
     "rb",
     "wb",
+    "r+b",
     "w+b"
 };
 #endif
@@ -66,7 +68,9 @@ File::File(Context* context) :
     readBufferSize_(0),
     offset_(0),
     checksum_(0),
-    compressed_(false)
+    compressed_(false),
+    readSyncNeeded_(false),
+    writeSyncNeeded_(false)
 {
 }
 
@@ -81,7 +85,9 @@ File::File(Context* context, const String& fileName, FileMode mode) :
     readBufferSize_(0),
     offset_(0),
     checksum_(0),
-    compressed_(false)
+    compressed_(false),
+    readSyncNeeded_(false),
+    writeSyncNeeded_(false)
 {
     Open(fileName, mode);
 }
@@ -97,7 +103,9 @@ File::File(Context* context, PackageFile* package, const String& fileName) :
     readBufferSize_(0),
     offset_(0),
     checksum_(0),
-    compressed_(false)
+    compressed_(false),
+    readSyncNeeded_(false),
+    writeSyncNeeded_(false)
 {
     Open(package, fileName);
 }
@@ -161,6 +169,16 @@ bool File::Open(const String& fileName, FileMode mode)
     handle_ = fopen(GetNativePath(fileName).CString(), openMode[mode]);
     #endif
 
+    // If file did not exist in readwrite mode, retry with write-update mode
+    if (mode == FILE_READWRITE && !handle_)
+    {
+        #ifdef WIN32
+        handle_ = _wfopen(GetWideNativePath(fileName).CString(), openMode[mode + 1]);
+        #else
+        handle_ = fopen(GetNativePath(fileName).CString(), openMode[mode + 1]);
+        #endif
+    }
+    
     if (!handle_)
     {
         LOGERROR("Could not open file " + fileName);
@@ -173,6 +191,8 @@ bool File::Open(const String& fileName, FileMode mode)
     offset_ = 0;
     checksum_ = 0;
     compressed_ = false;
+    readSyncNeeded_ = false;
+    writeSyncNeeded_ = false;
 
     fseek((FILE*)handle_, 0, SEEK_END);
     size_ = ftell((FILE*)handle_);
@@ -209,7 +229,9 @@ bool File::Open(PackageFile* package, const String& fileName)
     position_ = 0;
     size_ = entry->size_;
     compressed_ = package->IsCompressed();
-
+    readSyncNeeded_ = false;
+    writeSyncNeeded_ = false;
+    
     fseek((FILE*)handle_, offset_, SEEK_SET);
     return true;
 }
@@ -304,6 +326,13 @@ unsigned File::Read(void* dest, unsigned size)
         return size;
     }
 
+    // Need to reassign the position due to internal buffering when transitioning from writing to reading
+    if (readSyncNeeded_)
+    {
+        fseek((FILE*)handle_, position_ + offset_, SEEK_SET);
+        readSyncNeeded_ = false;
+    }
+    
     size_t ret = fread(dest, size, 1, (FILE*)handle_);
     if (ret != 1)
     {
@@ -313,6 +342,7 @@ unsigned File::Read(void* dest, unsigned size)
         return 0;
     }
 
+    writeSyncNeeded_ = true;
     position_ += size;
     return size;
 }
@@ -368,6 +398,8 @@ unsigned File::Seek(unsigned position)
 
     fseek((FILE*)handle_, position + offset_, SEEK_SET);
     position_ = position;
+    readSyncNeeded_ = false;
+    writeSyncNeeded_ = false;
     return position_;
 }
 
@@ -388,6 +420,13 @@ unsigned File::Write(const void* data, unsigned size)
     if (!size)
         return 0;
 
+    // Need to reassign the position due to internal buffering when transitioning from reading to writing
+    if (writeSyncNeeded_)
+    {
+        fseek((FILE*)handle_, position_ + offset_, SEEK_SET);
+        writeSyncNeeded_ = false;
+    }
+    
     if (fwrite(data, size, 1, (FILE*)handle_) != 1)
     {
         // Return to the position where the write began
@@ -396,6 +435,7 @@ unsigned File::Write(const void* data, unsigned size)
         return 0;
     }
 
+    readSyncNeeded_ = true;
     position_ += size;
     if (position_ > size_)
         size_ = position_;

Source/Engine/IO/File.h

@@ -115,6 +115,10 @@ private:
     unsigned checksum_;
     /// Compression flag.
     bool compressed_;
+    /// Synchronization needed before read -flag.
+    bool readSyncNeeded_;
+    /// Synchronization needed before write -flag.
+    bool writeSyncNeeded_;
 };
 
 }

Source/Engine/LuaScript/pkgs/Graphics/Graphics.pkg

@@ -1,3 +1,4 @@
+$#include "File.h"
 $#include "Graphics.h"
 
 class Graphics : public Object
@@ -17,7 +18,11 @@ class Graphics : public Object
     void Minimize();
     void Close();
     bool TakeScreenShot(Image& destImage);
-    
+    void BeginDumpShaders(const String fileName);
+    void EndDumpShaders();
+    void PrecacheShaders(Deserializer& source);
+    tolua_outside void GraphicsPrecacheShaders @ PrecacheShaders(const String fileName);
+
     bool IsInitialized() const;
     void* GetExternalWindow() const;
     const String GetWindowTitle() const;
@@ -84,6 +89,16 @@ Graphics* GetGraphics();
 tolua_readonly tolua_property__get_set Graphics* graphics;
 
 ${
+static void GraphicsPrecacheShaders(Graphics* graphics, const String& fileName)
+{
+    if (!graphics)
+        return;
+
+    File file(graphics->GetContext());
+    if (file.Open(fileName, FILE_READ))
+        graphics->PrecacheShaders(file);
+}
+
 #define TOLUA_DISABLE_tolua_GraphicsLuaAPI_GetGraphics00
 static int tolua_GraphicsLuaAPI_GetGraphics00(lua_State* tolua_S)
 {

Source/Engine/Script/GraphicsAPI.cpp

@@ -1210,6 +1210,12 @@ static CScriptArray* GraphicsGetMultiSampleLevels(Graphics* ptr)
     return VectorToArray<int>(ptr->GetMultiSampleLevels(), "Array<int>");
 }
 
+static void GraphicsPrecacheShaders(File* file, Graphics* ptr)
+{
+    if (file)
+        ptr->PrecacheShaders(*file);
+}
+
 static Graphics* GetGraphics()
 {
     return GetScriptContext()->GetSubsystem<Graphics>();
@@ -1226,6 +1232,9 @@ static void RegisterGraphics(asIScriptEngine* engine)
     engine->RegisterObjectMethod("Graphics", "void Minimize()", asMETHOD(Graphics, Minimize), asCALL_THISCALL);
     engine->RegisterObjectMethod("Graphics", "void Close()", asMETHOD(Graphics, Close), asCALL_THISCALL);
     engine->RegisterObjectMethod("Graphics", "bool TakeScreenShot(Image@+)", asMETHOD(Graphics, TakeScreenShot), asCALL_THISCALL);
+    engine->RegisterObjectMethod("Graphics", "void BeginDumpShaders(const String&in)", asMETHOD(Graphics, BeginDumpShaders), asCALL_THISCALL);
+    engine->RegisterObjectMethod("Graphics", "void EndDumpShaders()", asMETHOD(Graphics, EndDumpShaders), asCALL_THISCALL);
+    engine->RegisterObjectMethod("Graphics", "void PrecacheShaders(File@+)", asFUNCTION(GraphicsPrecacheShaders), asCALL_CDECL_OBJLAST);
     engine->RegisterObjectMethod("Graphics", "void set_windowTitle(const String&in)", asMETHOD(Graphics, SetWindowTitle), asCALL_THISCALL);
     engine->RegisterObjectMethod("Graphics", "const String& get_windowTitle() const", asMETHOD(Graphics, GetWindowTitle), asCALL_THISCALL);
     engine->RegisterObjectMethod("Graphics", "void set_windowIcon(Image@+)", asMETHOD(Graphics, SetWindowIcon), asCALL_THISCALL);

Source/Engine/UI/UI.cpp

@@ -38,7 +38,6 @@
 #include "Matrix3x4.h"
 #include "MessageBox.h"
 #include "Profiler.h"
-#include "Renderer.h"
 #include "ResourceCache.h"
 #include "ScrollBar.h"
 #include "Shader.h"
@@ -654,8 +653,7 @@ void UI::Render(VertexBuffer* buffer, const PODVector<UIBatch>& batches, unsigne
     // Engine does not render when window is closed or device is lost
     assert(graphics_ && graphics_->IsInitialized() && !graphics_->IsDeviceLost());
 
-    Renderer* renderer = GetSubsystem<Renderer>();
-    if (!renderer || batches.Empty())
+    if (batches.Empty())
         return;
 
     Vector2 invScreenSize(1.0f / (float)graphics_->GetWidth(), 1.0f / (float)graphics_->GetHeight());
@@ -679,12 +677,12 @@ void UI::Render(VertexBuffer* buffer, const PODVector<UIBatch>& batches, unsigne
     graphics_->ResetRenderTargets();
     graphics_->SetVertexBuffer(buffer);
 
-    ShaderVariation* noTextureVS = renderer->GetShader(VS, "Basic", "VERTEXCOLOR");
-    ShaderVariation* diffTextureVS = renderer->GetShader(VS, "Basic", "DIFFMAP VERTEXCOLOR");
-    ShaderVariation* noTexturePS = renderer->GetShader(PS, "Basic", "VERTEXCOLOR");
-    ShaderVariation* diffTexturePS = renderer->GetShader(PS, "Basic", "DIFFMAP VERTEXCOLOR");
-    ShaderVariation* diffMaskTexturePS = renderer->GetShader(PS, "Basic", "DIFFMAP ALPHAMASK VERTEXCOLOR");
-    ShaderVariation* alphaTexturePS = renderer->GetShader(PS, "Basic", "ALPHAMAP VERTEXCOLOR");
+    ShaderVariation* noTextureVS = graphics_->GetShader(VS, "Basic", "VERTEXCOLOR");
+    ShaderVariation* diffTextureVS = graphics_->GetShader(VS, "Basic", "DIFFMAP VERTEXCOLOR");
+    ShaderVariation* noTexturePS = graphics_->GetShader(PS, "Basic", "VERTEXCOLOR");
+    ShaderVariation* diffTexturePS = graphics_->GetShader(PS, "Basic", "DIFFMAP VERTEXCOLOR");
+    ShaderVariation* diffMaskTexturePS = graphics_->GetShader(PS, "Basic", "DIFFMAP ALPHAMASK VERTEXCOLOR");
+    ShaderVariation* alphaTexturePS = graphics_->GetShader(PS, "Basic", "ALPHAMAP VERTEXCOLOR");
 
     unsigned alphaFormat = Graphics::GetAlphaFormat();
 

Source/Tools/Urho3DPlayer/Urho3DPlayer.cpp

@@ -93,6 +93,7 @@ void Urho3DPlayer::Setup()
             "-r <freq>    Sound mixing frequency in Hz\n"
             "-p <paths>   Resource path(s) to use, separated by semicolons\n"
             "-log <level> Change the log level, valid 'level' values are 'debug', 'info', 'warning', 'error'\n"
+            "-ds <file>   Dump used shader variations to a file for precaching\n"
             "-borderless  Borderless window mode\n"
             "-headless    Headless mode. No application window will be created\n"
             "-prepass     Use light pre-pass rendering\n"