Background loading of Materials.

Store dependencies of 3D & cube textures so that they will be properly live reloaded if the individual images change.

Source/Engine/Graphics/Direct3D9/D3D9Texture3D.cpp

@@ -55,6 +55,8 @@ void Texture3D::RegisterObject(Context* context)
 
 bool Texture3D::BeginLoad(Deserializer& source)
 {
+    ResourceCache* cache = GetSubsystem<ResourceCache>();
+
     // In headless mode, do not actually load the texture, just return success
     if (!graphics_)
         return true;
@@ -70,6 +72,8 @@ bool Texture3D::BeginLoad(Deserializer& source)
     String texPath, texName, texExt;
     SplitPath(GetName(), texPath, texName, texExt);
     
+    cache->ResetDependencies(this);
+
     loadParameters_ = new XMLFile(context_);
     if (!loadParameters_->Load(source))
     {
@@ -91,7 +95,8 @@ bool Texture3D::BeginLoad(Deserializer& source)
         if (volumeTexPath.Empty())
             name = texPath + name;
 
-        loadImage_ = GetSubsystem<ResourceCache>()->GetTempResource<Image>(name);
+        loadImage_ = cache->GetTempResource<Image>(name);
+        cache->StoreResourceDependency(this, name);
     }
     else if (colorlutElem)
     {
@@ -111,6 +116,7 @@ bool Texture3D::BeginLoad(Deserializer& source)
             loadImage_.Reset();
             return false;
         }
+        cache->StoreResourceDependency(this, name);
     }
 
     return false;

Source/Engine/Graphics/Direct3D9/D3D9TextureCube.cpp

@@ -298,6 +298,8 @@ bool TextureCube::BeginLoad(Deserializer& source)
         return true;
     }
     
+    cache->ResetDependencies(this);
+
     String texPath, texName, texExt;
     SplitPath(GetName(), texPath, texName, texExt);
     
@@ -309,7 +311,7 @@ bool TextureCube::BeginLoad(Deserializer& source)
     }
     
     loadImages_.Clear();
-    
+
     XMLElement textureElem = loadParameters_->GetRoot();
     XMLElement faceElem = textureElem.GetChild("face");
     while (faceElem)
@@ -323,6 +325,8 @@ bool TextureCube::BeginLoad(Deserializer& source)
             name = texPath + name;
         
         loadImages_.Push(cache->GetTempResource<Image>(name));
+        cache->StoreResourceDependency(this, name);
+        
         faceElem = faceElem.GetNext("face");
     }
     

Source/Engine/Graphics/Material.cpp

@@ -172,17 +172,66 @@ bool Material::BeginLoad(Deserializer& source)
     if (!graphics)
         return true;
 
-    SharedPtr<XMLFile> xml(new XMLFile(context_));
-    if (!xml->Load(source))
+    loadXML_ = new XMLFile(context_);
+    if (loadXML_->Load(source))
+    {
+        // If async loading, scan the XML content beforehand for technique & texture resources
+        // and request them to also be loaded. Can not do anything else at this point
+        if (GetAsyncLoadState() == ASYNC_LOADING)
+        {
+            ResourceCache* cache = GetSubsystem<ResourceCache>();
+            XMLElement rootElem = loadXML_->GetRoot();
+            XMLElement techniqueElem = rootElem.GetChild("technique");
+            while (techniqueElem)
+            {
+                cache->BackgroundLoadResource<Technique>(techniqueElem.GetAttribute("name"), true, this);
+                techniqueElem = techniqueElem.GetNext("technique");
+            }
+
+            XMLElement textureElem = rootElem.GetChild("texture");
+            while (textureElem)
+            {
+                String name = textureElem.GetAttribute("name");
+                // Detect cube maps by file extension: they are defined by an XML file
+                /// \todo Differentiate with 3D textures by actually reading the XML content
+                if (GetExtension(name) == ".xml")
+                    cache->BackgroundLoadResource<TextureCube>(name, true, this);
+                else
+                    cache->BackgroundLoadResource<Texture2D>(name, true, this);
+                textureElem = textureElem.GetNext("texture");
+            }
+        }
+
+        return true;
+    }
+    else
     {
         ResetToDefaults();
+        loadXML_.Reset();
         return false;
     }
+}
 
-    XMLElement rootElem = xml->GetRoot();
-    return Load(rootElem);
+bool Material::EndLoad()
+{
+    // In headless mode, do not actually load the material, just return success
+    Graphics* graphics = GetSubsystem<Graphics>();
+    if (!graphics)
+        return true;
+
+    bool success = false;
+    if (loadXML_)
+    {
+        // If async loading, get the techniques / textures which should be ready now
+        XMLElement rootElem = loadXML_->GetRoot();
+        success = Load(rootElem);
+    }
+
+    loadXML_.Reset();
+    return success;
 }
 
+
 bool Material::Save(Serializer& dest) const
 {
     SharedPtr<XMLFile> xml(new XMLFile(context_));
@@ -206,6 +255,7 @@ bool Material::Load(const XMLElement& source)
 
     XMLElement techniqueElem = source.GetChild("technique");
     techniques_.Clear();
+    
     while (techniqueElem)
     {
         Technique* tech = cache->GetResource<Technique>(techniqueElem.GetAttribute("name"));
@@ -219,6 +269,7 @@ bool Material::Load(const XMLElement& source)
                 newTechnique.lodDistance_ = techniqueElem.GetFloat("loddistance");
             techniques_.Push(newTechnique);
         }
+
         techniqueElem = techniqueElem.GetNext("technique");
     }
 
@@ -234,6 +285,7 @@ bool Material::Load(const XMLElement& source)
         {
             String name = textureElem.GetAttribute("name");
             // Detect cube maps by file extension: they are defined by an XML file
+            /// \todo Differentiate with 3D textures by actually reading the XML content
             if (GetExtension(name) == ".xml")
                 SetTexture(unit, cache->GetResource<TextureCube>(name));
             else
@@ -660,6 +712,10 @@ void Material::CheckOcclusion()
 
 void Material::ResetToDefaults()
 {
+    // Needs to be a no-op when async loading, as this does a GetResource() which is not allowed from worker threads
+    if (!Thread::IsMainThread())
+        return;
+
     SetNumTechniques(1);
     SetTechnique(0, GetSubsystem<ResourceCache>()->GetResource<Technique>("Techniques/NoTexture.xml"));
 

Source/Engine/Graphics/Material.h

@@ -104,6 +104,8 @@ public:
 
     /// Load resource from stream. May be called from a worker thread. Return true if successful.
     virtual bool BeginLoad(Deserializer& source);
+    /// Finish resource loading. Always called from the main thread. Return true if successful.
+    virtual bool EndLoad();
     /// Save resource. Return true if successful.
     virtual bool Save(Serializer& dest) const;
 
@@ -222,6 +224,8 @@ private:
     bool specular_;
     /// Last animation update frame number.
     unsigned animationFrameNumber_;
+    /// XML file used while loading.
+    SharedPtr<XMLFile> loadXML_;
 };
 
 }

Source/Engine/Graphics/OpenGL/OGLTexture3D.cpp

@@ -60,6 +60,8 @@ void Texture3D::RegisterObject(Context* context)
 
 bool Texture3D::BeginLoad(Deserializer& source)
 {
+    ResourceCache* cache = GetSubsystem<ResourceCache>();
+
     // In headless mode, do not actually load the texture, just return success
     if (!graphics_)
         return true;
@@ -75,6 +77,8 @@ bool Texture3D::BeginLoad(Deserializer& source)
     String texPath, texName, texExt;
     SplitPath(GetName(), texPath, texName, texExt);
     
+    cache->ResetDependencies(this);
+
     loadParameters_ = new XMLFile(context_);
     if (!loadParameters_->Load(source))
     {
@@ -96,7 +100,8 @@ bool Texture3D::BeginLoad(Deserializer& source)
         if (volumeTexPath.Empty())
             name = texPath + name;
 
-        loadImage_ = GetSubsystem<ResourceCache>()->GetTempResource<Image>(name);
+        loadImage_ = cache->GetTempResource<Image>(name);
+        cache->StoreResourceDependency(this, name);
     }
     else if (colorlutElem)
     {
@@ -116,6 +121,7 @@ bool Texture3D::BeginLoad(Deserializer& source)
             loadImage_.Reset();
             return false;
         }
+        cache->StoreResourceDependency(this, name);
     }
 
     return false;

Source/Engine/Graphics/OpenGL/OGLTextureCube.cpp

@@ -258,6 +258,8 @@ bool TextureCube::BeginLoad(Deserializer& source)
         return true;
     }
     
+    cache->ResetDependencies(this);
+    
     String texPath, texName, texExt;
     SplitPath(GetName(), texPath, texName, texExt);
     
@@ -283,6 +285,8 @@ bool TextureCube::BeginLoad(Deserializer& source)
             name = texPath + name;
         
         loadImages_.Push(cache->GetTempResource<Image>(name));
+        cache->StoreResourceDependency(this, name);
+        
         faceElem = faceElem.GetNext("face");
     }
     

Source/Engine/Resource/ResourceCache.cpp

@@ -124,12 +124,13 @@ void ResourceCache::ThreadFunction()
                 // Need to lock the queue again when manipulating other entries
                 backgroundLoadMutex_.Acquire();
                 
-                for (unsigned i = 0; i < item.dependents_.Size(); ++i)
+                for (HashSet<Pair<StringHash, StringHash> >::Iterator i = item.dependents_.Begin(); i != item.dependents_.End();
+                    ++i)
                 {
                     HashMap<Pair<StringHash, StringHash>, BackgroundLoadItem>::Iterator j =
-                        backgroundLoadQueue_.Find(item.dependents_[i]);
+                        backgroundLoadQueue_.Find(*i);
                     if (j != backgroundLoadQueue_.End())
-                        j->second_.dependencies_.Remove(key);
+                        j->second_.dependencies_.Erase(key);
                 }
                 
                 item.dependents_.Clear();
@@ -649,6 +650,8 @@ bool ResourceCache::BackgroundLoadResource(StringHash type, const char* nameIn,
         return false;
     }
     
+    LOGDEBUG("Background loading resource " + name);
+
     item.resource_->SetName(name);
     item.resource_->SetAsyncLoadState(ASYNC_QUEUED);
     
@@ -659,8 +662,8 @@ bool ResourceCache::BackgroundLoadResource(StringHash type, const char* nameIn,
         HashMap<Pair<StringHash, StringHash>, BackgroundLoadItem>::Iterator i = backgroundLoadQueue_.Find(callerKey);
         if (i != backgroundLoadQueue_.End())
         {
-            i->second_.dependents_.Push(callerKey);
-            item.dependencies_.Push(key);
+            i->second_.dependents_.Insert(callerKey);
+            item.dependencies_.Insert(key);
         }
         else
             LOGWARNING("Resource " + caller->GetName() + " requested for a background loaded resource but was not in the background load queue");
@@ -1137,7 +1140,7 @@ void ResourceCache::FinishBackgroundLoading(HashMap<Pair<StringHash, StringHash>
     if (success)
     {
 #ifdef URHO3D_PROFILING
-        String profileBlockName("EndLoad" + GetTypeName());
+        String profileBlockName("EndLoad" + resource->GetTypeName());
     
         Profiler* profiler = GetSubsystem<Profiler>();
         if (profiler)

Source/Engine/Resource/ResourceCache.h

@@ -62,9 +62,9 @@ struct BackgroundLoadItem
     /// Resource.
     SharedPtr<Resource> resource_;
     /// Resources depended on for loading.
-    Vector<Pair<StringHash, StringHash> > dependencies_;
+    HashSet<Pair<StringHash, StringHash> > dependencies_;
     /// Resources that depend on this resource's loading.
-    Vector<Pair<StringHash, StringHash> > dependents_;
+    HashSet<Pair<StringHash, StringHash> > dependents_;
     /// Whether to send failure event.
     bool sendEventOnFailure_;
 };