Improvements to resource handling

Source/AtomicEditorWork/Editors/SceneEditor3D/SceneView3D.cpp

@@ -375,7 +375,7 @@ void SceneView3D::HandleDragEnded(StringHash eventType, VariantMap& eventData)
 
         ResourceCache* cache = GetSubsystem<ResourceCache>();
 
-        const String& importer = asset->GetImporterName();
+        const String& importer = asset->GetImporterTypeName();
 
         if (importer == "ModelImporter")
         {

Source/ToolCore/Assets/Asset.cpp

@@ -1,6 +1,8 @@
 
+#include <Atomic/IO/File.h>
 #include <Atomic/IO/FileSystem.h>
 
+#include "AssetDatabase.h"
 #include "ModelImporter.h"
 #include "FolderImporter.h"
 #include "SceneImporter.h"
@@ -10,9 +12,8 @@
 namespace ToolCore
 {
 
-Asset::Asset(Context* context, const String& guid, unsigned timestamp) :
+Asset::Asset(Context* context) :
     Object(context),
-    guid_(guid),
     dirty_(false),
     isFolder_(false)
 {
@@ -24,6 +25,20 @@ Asset::~Asset()
 
 }
 
+bool Asset::CheckCacheFile()
+{
+    FileSystem* fs = GetSubsystem<FileSystem>();
+    AssetDatabase* db = GetSubsystem<AssetDatabase>();
+    String cachePath = db->GetCachePath();
+
+    String cacheFile = cachePath + guid_;
+
+    if (!fs->FileExists(cacheFile))
+        return false;
+
+    return true;
+}
+
 bool Asset::Import()
 {
     if (importer_.Null())
@@ -32,61 +47,170 @@ bool Asset::Import()
     return importer_->Import(guid_);
 }
 
-bool Asset::SetPath(const String& path)
+// load .asset
+bool Asset::Load()
 {
     FileSystem* fs = GetSubsystem<FileSystem>();
+    AssetDatabase* db = GetSubsystem<AssetDatabase>();
 
-    path_ = path;
+    String assetFilename = GetDotAssetFilename();
 
-    if (importer_.Null())
+    SharedPtr<File> file(new File(context_, assetFilename));
+    json_ = new JSONFile(context_);
+    json_->Load(*file);
+    file->Close();
+
+    JSONValue root = json_->GetRoot();
+
+    assert(root.GetInt("version") == ASSET_VERSION);
+
+    guid_ = root.GetString("guid");
+
+    db->RegisterGUID(guid_);
+
+    timestamp_ = (unsigned) root.GetDouble("timestamp");
+
+    dirty_ = false;
+    if (!CheckCacheFile() || timestamp_ < fs->GetLastModifiedTime(path_))
+        dirty_ = true;
+
+    // handle import
+
+    if (importer_.NotNull())
+        importer_->LoadSettings(root);
+
+    json_ = 0;
+
+    return true;
+
+}
+
+// save .asset
+bool Asset::Save()
+{
+    String assetFilename = GetDotAssetFilename();
+
+    json_ = new JSONFile(context_);
+
+    JSONValue root = json_->CreateRoot();
+
+    root.SetInt("version", ASSET_VERSION);
+    root.SetString("guid", guid_);
+    root.SetDouble("timestamp", (double) timestamp_);
+
+    // handle import
+
+    if (importer_.NotNull())
     {
-        if (fs->DirExists(path))
-        {
-            name_ = GetFileName(RemoveTrailingSlash(path));
-            isFolder_ = true;
-            importer_ = new FolderImporter(context_);
-        }
-        else
-        {
-            String ext = GetExtension(path);
-
-            // todo, externalize recognizers
-            if (ext == ".fbx")
-            {
-                name_ = GetFileName(path);
-                importer_ = new ModelImporter(context_);
-            }
-            else if (ext == ".scene")
-            {
-                name_ = GetFileName(path);
-                importer_ = new SceneImporter(context_);
-            }
-            else if (ext == ".material")
-            {
-                name_ = GetFileName(path);
-                importer_ = new MaterialImporter(context_);
-            }
+        importer_->SaveSettings(root);
 
-        }
+        SharedPtr<File> file(new File(context_, assetFilename, FILE_WRITE));
+        json_->Save(*file);
+        file->Close();
+    }
+
+    json_ = 0;
+
+    return true;
+
+}
+
+String Asset::GetDotAssetFilename()
+{
+    assert(path_.Length());
+
+    FileSystem* fs = GetSubsystem<FileSystem>();
 
-        if (importer_.Null())
-            return false;
+    String assetFilename = path_ + ".asset";
 
-        String assetPath = path + ".asset";
+    if (fs->DirExists(path_)) {
 
-        if (fs->FileExists(assetPath))
+        assetFilename = RemoveTrailingSlash(path_) + ".asset";
+
+    }
+
+    return assetFilename;
+
+}
+
+bool Asset::CreateImporter()
+{
+    assert(importer_.Null());
+
+    FileSystem* fs = GetSubsystem<FileSystem>();
+
+    if (fs->DirExists(path_))
+    {
+        name_ = GetFileName(RemoveTrailingSlash(path_));
+        isFolder_ = true;
+        importer_ = new FolderImporter(context_);
+    }
+    else
+    {
+        String ext = GetExtension(path_);
+
+        name_ = GetFileName(path_);
+
+        // todo, externalize recognizers
+        if (ext == ".fbx")
+        {
+            importer_ = new ModelImporter(context_);
+        }
+        else if (ext == ".scene")
         {
-            importer_->Load(guid_);
+            importer_ = new SceneImporter(context_);
         }
-        else
+        else if (ext == ".material")
         {
-            importer_->Save(guid_);
+            importer_ = new MaterialImporter(context_);
         }
 
-        SetDirty(importer_->IsDirty());
+    }
+
+    if (importer_.Null())
+        return false;
+
+    return true;
 
+}
+
+
+bool Asset::SetPath(const String& path)
+{
+
+    assert(!guid_.Length());
+    assert(!path_.Length());
+
+    // need to update path, not set, which should only be done on first import
+    assert(importer_.Null());
+
+    FileSystem* fs = GetSubsystem<FileSystem>();
+    AssetDatabase* db = GetSubsystem<AssetDatabase>();
+
+    path_ = path;
+
+    // create importer based on path
+    CreateImporter();
+
+    String assetFilename = GetDotAssetFilename();
+
+    if (fs->FileExists(assetFilename))
+    {
+        // load the json, todo: handle fail
+        Load();
+    }
+    else
+    {
+        guid_ = db->GenerateAssetGUID();
+        timestamp_ = fs->GetLastModifiedTime(path);
+
+        Save();
     }
 
+    // TODO: handle failed
+
+
+
 }
 
 }

Source/ToolCore/Assets/Asset.h

@@ -10,40 +10,59 @@ using namespace Atomic;
 namespace ToolCore
 {
 
+#define ASSET_VERSION 1
+
 class Asset : public Object
 {
     OBJECT(Asset);
 
 public:
     /// Construct.
-    Asset(Context* context, const String& guid, unsigned timestamp);
+    Asset(Context* context);
     virtual ~Asset();
 
     bool Import();
 
-    const String& GetGUID() const { return guid_; }
-
-    const String& GetImporterName() { return importer_.Null() ? String::EMPTY : importer_->GetTypeName(); }
-
-    const String& GetName() { return name_; }
+    // the .fbx, .png, etc path, attempts to load .asset, creates missing .asset
+    bool SetPath(const String& path);
 
+    const String& GetGUID() const { return guid_; }
+    const String& GetName() const { return name_; }
     const String& GetPath() const { return path_; }
-    bool SetPath(const String& path);
+    unsigned GetTimestamp() const { return timestamp_; }
+
+    const String& GetImporterTypeName() { return importer_.Null() ? String::EMPTY : importer_->GetTypeName(); }
 
     void SetDirty(bool dirty) { dirty_ = dirty; }
     bool IsDirty() const { return dirty_; }
 
+    // get the .asset filename
+    String GetDotAssetFilename();
+
     bool IsFolder() const { return isFolder_; }
 
 private:
 
+    // load .asset
+    bool Load();
+    // save .asset
+    bool Save();
+
+    bool CreateImporter();
+
+    bool CheckCacheFile();
+
     String guid_;
+
+    // can change
     String path_;
     String name_;
+    unsigned timestamp_;
 
     bool dirty_;
     bool isFolder_;
 
+    SharedPtr<JSONFile> json_;
     SharedPtr<AssetImporter> importer_;
 };
 

Source/ToolCore/Assets/AssetDatabase.cpp

@@ -35,37 +35,44 @@ String AssetDatabase::GetCachePath()
 
 }
 
-String AssetDatabase::GeneratePathGUID(const String &path)
+String AssetDatabase::GenerateAssetGUID()
 {
-    FileSystem* fs = GetSubsystem<FileSystem>();
 
-    Poco::MD5Engine md5;
+    Time* time = GetSubsystem<Time>();
 
-    if (fs->DirExists(path))
+    while (true)
     {
-        md5.update(path.CString(), path.Length());
-    }
-    else
-    {
-        SharedPtr<File> file(new File(context_, path));
-        PODVector<unsigned char> data;
+        Poco::MD5Engine md5;
+        PODVector<unsigned> data;
 
-        if (!data.Size())
+        for (unsigned i = 0; i < 16; i++)
         {
-            // zero length file uses path name instead of data
-            data.Resize(path.Length());
-            memcpy(&data[0], path.CString(), path.Length());
+            data.Push(time->GetTimeSinceEpoch() + random() % 65535);
         }
-        else
+
+        md5.update(&data[0], data.Size() * sizeof(unsigned));
+
+        String guid = Poco::MD5Engine::digestToHex(md5.digest()).c_str();
+
+        if (!usedGUID_.Contains(guid))
         {
-            data.Resize(file->GetSize());
-            file->Read(&data[0], data.Size());
+            RegisterGUID(guid);
+            return guid;
         }
+    }
 
-        md5.update(&data[0], data.Size());
+    assert(0);
+    return "";
+}
+
+void AssetDatabase::RegisterGUID(const String& guid)
+{
+    if (usedGUID_.Contains(guid))
+    {
+        assert(0);
     }
 
-    return Poco::MD5Engine::digestToHex(md5.digest()).c_str();
+    usedGUID_.Push(guid);
 }
 
 void AssetDatabase::Import(const String& path)
@@ -109,9 +116,68 @@ Asset* AssetDatabase::GetAssetByPath(const String& path)
 
 }
 
+void AssetDatabase::PruneOrphanedDotAssetFiles()
+{
+    FileSystem* fs = GetSubsystem<FileSystem>();
+
+    const String& resourcePath = project_->GetResourcePath();
+
+    Vector<String> allResults;
+
+    fs->ScanDir(allResults, resourcePath, "*.asset", SCAN_FILES, true);
+
+    for (unsigned i = 0; i < allResults.Size(); i++)
+    {
+        String dotAssetFilename = resourcePath + allResults[i];
+        String assetFilename = ReplaceExtension(dotAssetFilename, "");
+
+        // remove orphaned asset files
+        if (!fs->FileExists(assetFilename) && !fs->DirExists(assetFilename))
+        {
+
+            LOGINFOF("Removing orphaned asset file: %s", dotAssetFilename.CString());
+            fs->Delete(dotAssetFilename);
+        }
+
+    }
+}
+
+String AssetDatabase::GetDotAssetFilename(const String& path)
+{
+    FileSystem* fs = GetSubsystem<FileSystem>();
+
+    String assetFilename = path + ".asset";
+
+    if (fs->DirExists(path)) {
+
+        assetFilename = RemoveTrailingSlash(path) + ".asset";
+    }
+
+    return assetFilename;
+
+}
+
+void AssetDatabase::AddAsset(SharedPtr<Asset>& asset)
+{
+    assets_.Push(asset);
+}
+
+void AssetDatabase::ImportDirtyAssets()
+{
+
+    PODVector<Asset*> assets;
+    GetDirtyAssets(assets);
+
+    for (unsigned i = 0; i < assets.Size(); i++)
+    {
+        assets[i]->Import();
+    }
+
+}
 
 void AssetDatabase::Scan()
 {
+    PruneOrphanedDotAssetFiles();
 
     FileSystem* fs = GetSubsystem<FileSystem>();
     const String& resourcePath = project_->GetResourcePath();
@@ -139,38 +205,45 @@ void AssetDatabase::Scan()
         filteredResults.Push(path);
     }
 
-    Vector<String> importResults;
-
     for (unsigned i = 0; i < filteredResults.Size(); i++)
     {
         const String& path = filteredResults[i];
-        importResults.Push(path);
-    }
-
-    for (unsigned i = 0; i < importResults.Size(); i++)
-    {
-        const String& path = importResults[i];
 
-        String md5 = GeneratePathGUID(path);
+        String dotAssetFilename = GetDotAssetFilename(path);
 
-        // get the current time stamp
-        unsigned ctimestamp = fs->GetLastModifiedTime(path);
-
-        if (!GetAssetByPath(path))
+        if (!fs->FileExists(dotAssetFilename))
         {
-            SharedPtr<Asset> asset(new Asset(context_, md5, ctimestamp));
-            assets_.Push(asset);
+            // new asset
+            SharedPtr<Asset> asset(new Asset(context_));
             asset->SetPath(path);
+            AddAsset(asset);
         }
-    }
+        else
+        {
+            SharedPtr<File> file(new File(context_, dotAssetFilename));
+            SharedPtr<JSONFile> json(new JSONFile(context_));
+            json->Load(*file);
+            file->Close();
 
-    PODVector<Asset*> dirty;
-    GetDirtyAssets(dirty);
+            JSONValue root = json->GetRoot();
+
+            assert(root.GetInt("version") == ASSET_VERSION);
+
+            String guid = root.GetString("guid");
+
+            if (!GetAssetByGUID(guid))
+            {
+                SharedPtr<Asset> asset(new Asset(context_));
+                asset->SetPath(path);
+                AddAsset(asset);
+            }
+
+        }
 
-    for (unsigned i = 0; i < dirty.Size(); i++)
-    {
-        dirty[i]->Import();
     }
+
+    ImportDirtyAssets();
+
 }
 
 void AssetDatabase::GetFolderAssets(String folder, PODVector<Asset*>& assets) const

Source/ToolCore/Assets/AssetDatabase.h

@@ -22,7 +22,10 @@ public:
     virtual ~AssetDatabase();
 
     Asset* GetAssetByGUID(const String& guid);
-    Asset* GetAssetByPath(const String& guid);
+    Asset* GetAssetByPath(const String& path);
+
+    String GenerateAssetGUID();
+    void RegisterGUID(const String& guid);
 
     String GetCachePath();
 
@@ -32,16 +35,25 @@ public:
 
     void GetDirtyAssets(PODVector<Asset*>& assets);
 
+    String GetDotAssetFilename(const String& path);
+
 private:
 
     void HandleProjectLoaded(StringHash eventType, VariantMap& eventData);
 
-    String GeneratePathGUID(const String& path);
+    void AddAsset(SharedPtr<Asset>& asset);
+
+    void PruneOrphanedDotAssetFiles();
+
     void Import(const String& path);
 
+    void ImportDirtyAssets();
+
     SharedPtr<Project> project_;
     List<SharedPtr<Asset>> assets_;
 
+    Vector<String> usedGUID_;
+
 };
 
 }

Source/ToolCore/Assets/AssetImporter.cpp

@@ -7,9 +7,7 @@
 namespace ToolCore
 {
 
-AssetImporter::AssetImporter(Context* context) : Object(context),
-    dirty_(true),
-    timestamp_(0)
+AssetImporter::AssetImporter(Context* context) : Object(context)
 {
     SetDefaults();
 }
@@ -24,75 +22,27 @@ void AssetImporter::SetDefaults()
 
 }
 
-bool AssetImporter::Load(const String& guid)
+bool AssetImporter::LoadSettings(JSONValue& root)
 {
-    AssetDatabase* db = GetSubsystem<AssetDatabase>();
-    Asset* asset = db->GetAssetByGUID(guid);
-    assert(asset);
-
-    const String& path = asset->GetPath();
-    String assetPath = path + ".asset";
-
-    SharedPtr<File> file(new File(context_, assetPath));
-
-    json_ = new JSONFile(context_);
-    json_->Load(*file);
-    file->Close();
-
-    LoadInternal();
-
-    json_ = 0;
-
-
+    jsonRoot_ = root;
+    LoadSettingsInternal();
     return true;
 }
 
-bool AssetImporter::LoadInternal()
+bool AssetImporter::LoadSettingsInternal()
 {
-    JSONValue root = json_->GetRoot();
-
-    assert(root.GetInt("version") == 1);
-    guid_ = root.GetString("guid");
-    timestamp_ = (unsigned) root.GetDouble("timestamp");
-
     return true;
 }
 
-bool AssetImporter::Save(const String& guid)
+bool AssetImporter::SaveSettings(JSONValue& root)
 {
-
-    FileSystem* fs = GetSubsystem<FileSystem>();
-    AssetDatabase* db = GetSubsystem<AssetDatabase>();
-    Asset* asset = db->GetAssetByGUID(guid);
-    assert(asset);
-
-    const String& path = asset->GetPath();
-    String assetPath = path + ".asset";
-
-    timestamp_ = fs->GetLastModifiedTime(path);
-    guid_ = guid;
-
-    json_ = new JSONFile(context_);
-    json_->CreateRoot();
-    SaveInternal();
-
-    SharedPtr<File> file(new File(context_, assetPath, FILE_WRITE));
-    json_->Save(*file, String("   "));
-    file->Close();
-
-    json_ = 0;
-
+    jsonRoot_ = root;
+    SaveSettingsInternal();
     return true;
 }
 
-bool AssetImporter::SaveInternal()
+bool AssetImporter::SaveSettingsInternal()
 {
-    JSONValue root = json_->GetRoot();
-
-    root.SetInt("version", ASSET_VERSION);
-    root.SetString("guid", guid_);
-    root.SetDouble("timestamp", (double) timestamp_);
-
     return true;
 }
 

Source/ToolCore/Assets/AssetImporter.h

@@ -9,8 +9,6 @@ using namespace Atomic;
 namespace ToolCore
 {
 
-#define ASSET_VERSION 1
-
 /// deals with .asset files
 class AssetImporter : public Object
 {
@@ -21,12 +19,10 @@ public:
     AssetImporter(Context* context);
     virtual ~AssetImporter();
 
-    bool IsDirty() { return dirty_; }
-
     // load .asset
-    bool Load(const String& guid);
+    bool LoadSettings(JSONValue& root);
     // save .asset
-    bool Save(const String& guid);
+    bool SaveSettings(JSONValue& root);
 
     virtual void SetDefaults();
 
@@ -35,14 +31,10 @@ public:
 
 protected:
 
-    SharedPtr<JSONFile> json_;
-    bool dirty_;
-
-    String guid_;
-    unsigned timestamp_;
+    JSONValue jsonRoot_;
 
-    virtual bool LoadInternal();
-    virtual bool SaveInternal();
+    virtual bool LoadSettingsInternal();
+    virtual bool SaveSettingsInternal();
 
 };
 

Source/ToolCore/Assets/FolderImporter.cpp

@@ -32,26 +32,22 @@ bool FolderImporter::Import(const String& guid)
     return true;
 }
 
-bool FolderImporter::LoadInternal()
+bool FolderImporter::LoadSettingsInternal()
 {
-    if (!AssetImporter::LoadInternal())
+    if (!AssetImporter::LoadSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.GetChild("FolderImporter", JSON_OBJECT);
+    JSONValue import = jsonRoot_.GetChild("FolderImporter", JSON_OBJECT);
 
     return true;
 }
 
-bool FolderImporter::SaveInternal()
+bool FolderImporter::SaveSettingsInternal()
 {
-    if (!AssetImporter::SaveInternal())
+    if (!AssetImporter::SaveSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.CreateChild("FolderImporter");
+    JSONValue import = jsonRoot_.CreateChild("FolderImporter");
 
     return true;
 }

Source/ToolCore/Assets/FolderImporter.h

@@ -21,8 +21,8 @@ public:
 
 protected:
 
-    virtual bool LoadInternal();
-    virtual bool SaveInternal();
+    virtual bool LoadSettingsInternal();
+    virtual bool SaveSettingsInternal();
 
 };
 

Source/ToolCore/Assets/MaterialImporter.cpp

@@ -32,26 +32,22 @@ bool MaterialImporter::Import(const String& guid)
     return true;
 }
 
-bool MaterialImporter::LoadInternal()
+bool MaterialImporter::LoadSettingsInternal()
 {
-    if (!AssetImporter::LoadInternal())
+    if (!AssetImporter::LoadSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.GetChild("MaterialImporter", JSON_OBJECT);
+    JSONValue import = jsonRoot_.GetChild("MaterialImporter", JSON_OBJECT);
 
     return true;
 }
 
-bool MaterialImporter::SaveInternal()
+bool MaterialImporter::SaveSettingsInternal()
 {
-    if (!AssetImporter::SaveInternal())
+    if (!AssetImporter::SaveSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.CreateChild("MaterialImporter");
+    JSONValue import = jsonRoot_.CreateChild("MaterialImporter");
 
     return true;
 }

Source/ToolCore/Assets/MaterialImporter.h

@@ -21,8 +21,8 @@ public:
 
 protected:
 
-    virtual bool LoadInternal();
-    virtual bool SaveInternal();
+    virtual bool LoadSettingsInternal();
+    virtual bool SaveSettingsInternal();
 
 };
 

Source/ToolCore/Assets/ModelImporter.cpp

@@ -52,26 +52,22 @@ bool ModelImporter::Import(const String& guid)
     return true;
 }
 
-bool ModelImporter::LoadInternal()
+bool ModelImporter::LoadSettingsInternal()
 {
-    if (!AssetImporter::LoadInternal())
+    if (!AssetImporter::LoadSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.GetChild("ModelImporter", JSON_OBJECT);
+    JSONValue import = jsonRoot_.GetChild("ModelImporter", JSON_OBJECT);
 
     return true;
 }
 
-bool ModelImporter::SaveInternal()
+bool ModelImporter::SaveSettingsInternal()
 {
-    if (!AssetImporter::SaveInternal())
+    if (!AssetImporter::SaveSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.CreateChild("ModelImporter");
+    JSONValue import = jsonRoot_.CreateChild("ModelImporter");
 
     return true;
 }

Source/ToolCore/Assets/ModelImporter.h

@@ -21,8 +21,8 @@ public:
 
 protected:
 
-    virtual bool LoadInternal();
-    virtual bool SaveInternal();
+    virtual bool LoadSettingsInternal();
+    virtual bool SaveSettingsInternal();
 
 };
 

Source/ToolCore/Assets/SceneImporter.cpp

@@ -32,26 +32,22 @@ bool SceneImporter::Import(const String& guid)
     return true;
 }
 
-bool SceneImporter::LoadInternal()
+bool SceneImporter::LoadSettingsInternal()
 {
-    if (!AssetImporter::LoadInternal())
+    if (!AssetImporter::LoadSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.GetChild("SceneImporter", JSON_OBJECT);
+    JSONValue import = jsonRoot_.GetChild("SceneImporter", JSON_OBJECT);
 
     return true;
 }
 
-bool SceneImporter::SaveInternal()
+bool SceneImporter::SaveSettingsInternal()
 {
-    if (!AssetImporter::SaveInternal())
+    if (!AssetImporter::SaveSettingsInternal())
         return false;
 
-    JSONValue root = json_->GetRoot();
-
-    JSONValue import = root.CreateChild("SceneImporter");
+    JSONValue import = jsonRoot_.CreateChild("SceneImporter");
 
     return true;
 }

Source/ToolCore/Assets/SceneImporter.h

@@ -21,8 +21,8 @@ public:
 
 protected:
 
-    virtual bool LoadInternal();
-    virtual bool SaveInternal();
+    virtual bool LoadSettingsInternal();
+    virtual bool SaveSettingsInternal();
 
 };