Automatically update builtin resource data lists when resources are added/removed from their folders

Source/BansheeEditor/Source/BsBuiltinEditorResources.cpp

@@ -395,8 +395,6 @@ namespace bs
 	{
 		Resources::instance().unloadAllUnused();
 
-		// TODO - Update DataList.json if needed
-
 		Path dataListsFilePath = BuiltinRawDataFolder + DataListFile;
 		SPtr<DataStream> dataListStream = FileSystem::openFile(dataListsFilePath);
 
@@ -407,10 +405,34 @@ namespace bs
 		json includesJSON = dataListJSON["Includes"];
 		json shadersJSON = dataListJSON["Shaders"];
 
-		BuiltinResourcesHelper::importAssets(iconsJSON, EditorRawIconsFolder, EditorIconFolder, mResourceManifest, BuiltinResourcesHelper::ImportMode::Sprite);
+		// Update DataList.json if needed
+		bool updatedDataLists = false;
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(EditorRawIconsFolder, BuiltinResourcesHelper::AssetType::Sprite, iconsJSON);
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(EditorRawShaderIncludeFolder, BuiltinResourcesHelper::AssetType::Normal, includesJSON);
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(EditorRawShaderFolder, BuiltinResourcesHelper::AssetType::Normal, shadersJSON);
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(EditorRawSkinFolder, BuiltinResourcesHelper::AssetType::Sprite, skinJSON);
+
+		dataListStream->close();
+
+		if (updatedDataLists)
+		{
+			FileSystem::remove(dataListsFilePath);
+
+			dataListJSON["Skin"] = skinJSON;
+			dataListJSON["Icons"] = iconsJSON;
+			dataListJSON["Includes"] = includesJSON;
+			dataListJSON["Shaders"] = shadersJSON;
+
+			String jsonString = dataListJSON.dump(4).c_str();
+			dataListStream = FileSystem::createAndOpenFile(dataListsFilePath);
+			dataListStream->writeString(jsonString);
+			dataListStream->close();
+		}
+
+		BuiltinResourcesHelper::importAssets(iconsJSON, EditorRawIconsFolder, EditorIconFolder, mResourceManifest, BuiltinResourcesHelper::AssetType::Sprite);
 		BuiltinResourcesHelper::importAssets(includesJSON, EditorRawShaderIncludeFolder, EditorShaderIncludeFolder, mResourceManifest); // Hidden dependency: Includes must be imported before shaders
 		BuiltinResourcesHelper::importAssets(shadersJSON, EditorRawShaderFolder, EditorShaderFolder, mResourceManifest);
-		BuiltinResourcesHelper::importAssets(skinJSON, EditorRawSkinFolder, EditorSkinFolder, mResourceManifest, BuiltinResourcesHelper::ImportMode::Sprite);
+		BuiltinResourcesHelper::importAssets(skinJSON, EditorRawSkinFolder, EditorSkinFolder, mResourceManifest, BuiltinResourcesHelper::AssetType::Sprite);
 
 		// Import fonts
 		BuiltinResourcesHelper::importFont(BuiltinRawDataFolder + DefaultFontFilename, DefaultFontFilename, 

Source/BansheeEngine/Include/BsBuiltinResourcesHelper.h

@@ -11,21 +11,18 @@ namespace bs
 	 *  @{
 	 */
 
-	//using Json = nlohmann::basic_json<Map, Vector, String, bool, INT64, UINT64, float, StdAlloc>;
-	//using WJson = nlohmann::basic_json<Map, Vector, WString, bool, INT64, UINT64, float, StdAlloc>;
-
 	/**	Provides various methods commonly used for managing builtin resources. */
 	class BS_EXPORT BuiltinResourcesHelper
 	{
 	public:
 		/** Flags that can control asset import. */
-		enum class ImportMode
+		enum class AssetType
 		{
 			/** No flags, just import asset as normal. Each entry is expected to have an "UUID" and a "Path" field. */
-			None,
+			Normal,
 			/** 
 			 * Assumes imported assets are textures. Will generate sprite assets for each imported texture. Expects 
-			 * "UUID", "Path" and "SpriteUUID" fields present in per-entry JSON.
+			 * "TextureUUID", "Path" and "SpriteUUID" fields present in per-entry JSON.
 			 */
 			Sprite,
 		};
@@ -42,7 +39,7 @@ namespace bs
 		 * @param[in]	mode			Mode that controls how are files imported.
 		 */
 		static void importAssets(const nlohmann::json& entries, const Path& inputFolder, const Path& outputFolder, 
-			const SPtr<ResourceManifest>& manifest, ImportMode mode = ImportMode::None);
+			const SPtr<ResourceManifest>& manifest, AssetType mode = AssetType::Normal);
 
 		/**
 		 * Imports a font from the specified file. Imported font assets are saved in the output folder. All saved resources
@@ -51,6 +48,16 @@ namespace bs
 		static void importFont(const Path& inputFile, const WString& outputName, const Path& outputFolder, 
 			const Vector<UINT32>& fontSizes, bool antialiasing, const String& UUID, const SPtr<ResourceManifest>& manifest);
 
+		/** 
+		 * Scans the provided folder for any files that are currently not part of the provided JSON entries. If some are
+		 * found they are appended to the JSON entry array. Returns true if any new files were found, false otherwise.
+		 * 
+		 * @param[in]		folder		Folder to check for new entries.
+		 * @param[in]		type		Type of entries in the folder. Determines the type of JSON data generated.
+		 * @param[in, out]	entries		Current data file entries.
+		 */
+		static bool updateJSON(const Path& folder, AssetType type, nlohmann::json& entries);
+
 		/** Writes a timestamp with the current date and time in the specified file. */
 		static void writeTimestamp(const Path& file);
 

Source/BansheeEngine/Source/BsBuiltinResources.cpp

@@ -360,7 +360,15 @@ namespace bs
 		// Hidden dependency: Textures need to be generated before shaders as they may use the default textures
 		generateTextures();
 
-		// TODO - Update DataList.json if needed
+		Path dataListsFilePath = mBuiltinRawDataFolder + DataListFile;
+		SPtr<DataStream> dataListStream = FileSystem::openFile(dataListsFilePath);
+		json dataListJSON = json::parse(dataListStream->getAsString().c_str());
+
+		json skinJSON = dataListJSON["Skin"];
+		json cursorsJSON = dataListJSON["Cursors"];
+		json iconsJSON = dataListJSON["Icons"];
+		json includesJSON = dataListJSON["Includes"];
+		json shadersJSON = dataListJSON["Shaders"];
 
 		Path rawSkinFolder = mBuiltinRawDataFolder + SkinFolder;
 		Path rawCursorFolder = mBuiltinRawDataFolder + CursorFolder;
@@ -368,26 +376,41 @@ namespace bs
 		Path rawShaderFolder = mBuiltinRawDataFolder + ShaderFolder;
 		Path rawShaderIncludeFolder = mBuiltinRawDataFolder + ShaderIncludeFolder;
 
+		// Update DataList.json if needed
+		bool updatedDataLists = false;
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(rawCursorFolder, BuiltinResourcesHelper::AssetType::Normal, cursorsJSON);
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(rawIconFolder, BuiltinResourcesHelper::AssetType::Normal, iconsJSON);
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(rawShaderIncludeFolder, BuiltinResourcesHelper::AssetType::Normal, includesJSON);
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(rawShaderFolder, BuiltinResourcesHelper::AssetType::Normal, shadersJSON);
+		updatedDataLists |= BuiltinResourcesHelper::updateJSON(rawSkinFolder, BuiltinResourcesHelper::AssetType::Sprite, skinJSON);
+
+		dataListStream->close();
+
+		if(updatedDataLists)
+		{
+			FileSystem::remove(dataListsFilePath);
+
+			dataListJSON["Skin"] = skinJSON;
+			dataListJSON["Cursors"] = cursorsJSON;
+			dataListJSON["Icons"] = iconsJSON;
+			dataListJSON["Includes"] = includesJSON;
+			dataListJSON["Shaders"] = shadersJSON;
+
+			String jsonString = dataListJSON.dump(4).c_str();
+			dataListStream = FileSystem::createAndOpenFile(dataListsFilePath);
+			dataListStream->writeString(jsonString);
+			dataListStream->close();
+		}
+		
 		Path skinFolder = mBuiltinDataFolder + SkinFolder;
 		Path iconFolder = mBuiltinDataFolder + IconFolder;
 		Path shaderIncludeFolder = mBuiltinDataFolder + ShaderIncludeFolder;
 
-		Path dataListsFilePath = mBuiltinRawDataFolder + DataListFile;
-		SPtr<DataStream> dataListStream = FileSystem::openFile(dataListsFilePath);
-
-		json dataListJSON = json::parse(dataListStream->getAsString().c_str());
-
-		json skinJSON = dataListJSON["Skin"];
-		json cursorsJSON = dataListJSON["Cursors"];
-		json iconsJSON = dataListJSON["Icons"];
-		json includesJSON = dataListJSON["Includes"];
-		json shadersJSON = dataListJSON["Shaders"];
-
 		BuiltinResourcesHelper::importAssets(cursorsJSON, rawCursorFolder, mEngineCursorFolder, mResourceManifest);
 		BuiltinResourcesHelper::importAssets(iconsJSON, rawIconFolder, iconFolder, mResourceManifest);
 		BuiltinResourcesHelper::importAssets(includesJSON, rawShaderIncludeFolder, shaderIncludeFolder, mResourceManifest); // Hidden dependency: Includes must be imported before shaders
 		BuiltinResourcesHelper::importAssets(shadersJSON, rawShaderFolder, mEngineShaderFolder, mResourceManifest);
-		BuiltinResourcesHelper::importAssets(skinJSON, rawSkinFolder, skinFolder, mResourceManifest, BuiltinResourcesHelper::ImportMode::Sprite);
+		BuiltinResourcesHelper::importAssets(skinJSON, rawSkinFolder, skinFolder, mResourceManifest, BuiltinResourcesHelper::AssetType::Sprite);
 
 		// Import font
 		BuiltinResourcesHelper::importFont(mBuiltinRawDataFolder + DefaultFontFilename, DefaultFontFilename, 

Source/BansheeEngine/Source/BsBuiltinResourcesHelper.cpp

@@ -15,13 +15,14 @@
 #include "BsResourceManifest.h"
 #include "BsFileSystem.h"
 #include "BsCoreThread.h"
+#include "BsUUID.h"
 
 using json = nlohmann::json;
 
 namespace bs
 {
 	void BuiltinResourcesHelper::importAssets(const nlohmann::json& entries, const Path& inputFolder, 
-		const Path& outputFolder, const SPtr<ResourceManifest>& manifest, ImportMode mode)
+		const Path& outputFolder, const SPtr<ResourceManifest>& manifest, AssetType mode)
 	{
 		if (!FileSystem::exists(inputFolder))
 			return;
@@ -32,7 +33,7 @@ namespace bs
 		FileSystem::createDir(outputFolder);
 
 		Path spriteOutputFolder = outputFolder + "/Sprites/";
-		if(mode == ImportMode::Sprite)
+		if(mode == AssetType::Sprite)
 			FileSystem::createDir(spriteOutputFolder);
 
 		auto importResource = [&](const String& fileName, const String& UUID)
@@ -158,12 +159,12 @@ namespace bs
 			std::string uuid;
 
 			bool isIcon = false;
-			if (mode == ImportMode::None)
+			if (mode == AssetType::Normal)
 			{
 				uuid = entry["UUID"];
 				isIcon = entry.find("UUID16") != entry.end();
 			}
-			else if (mode == ImportMode::Sprite)
+			else if (mode == AssetType::Sprite)
 			{
 				uuid = entry["TextureUUID"];
 				isIcon = entry.find("TextureUUID16") != entry.end();
@@ -173,7 +174,7 @@ namespace bs
 			if (outputRes == nullptr)
 				continue;
 
-			if (mode == ImportMode::Sprite)
+			if (mode == AssetType::Sprite)
 			{
 				std::string spriteUUID = entry["SpriteUUID"];
 
@@ -187,13 +188,13 @@ namespace bs
 				iconData.source = static_resource_cast<Texture>(outputRes);
 				iconData.name = name.c_str();
 
-				if (mode == ImportMode::None)
+				if (mode == AssetType::Normal)
 				{
 					iconData.TextureUUIDs[0] = entry["UUID48"];
 					iconData.TextureUUIDs[1] = entry["UUID32"];
 					iconData.TextureUUIDs[2] = entry["UUID16"];
 				}
-				else if (mode == ImportMode::Sprite)
+				else if (mode == AssetType::Sprite)
 				{
 					iconData.TextureUUIDs[0] = entry["TextureUUID48"];
 					iconData.TextureUUIDs[1] = entry["TextureUUID32"];
@@ -250,7 +251,7 @@ namespace bs
 			HTexture tex32 = saveTexture(scaled32, outputPath32, iconsToGenerate[i].TextureUUIDs[1]);
 			HTexture tex16 = saveTexture(scaled16, outputPath16, iconsToGenerate[i].TextureUUIDs[2]);
 
-			if (mode == ImportMode::Sprite)
+			if (mode == AssetType::Sprite)
 			{
 				generateSprite(tex48, iconsToGenerate[i].name + "48", iconsToGenerate[i].SpriteUUIDs[0].c_str());
 				generateSprite(tex32, iconsToGenerate[i].name + "32", iconsToGenerate[i].SpriteUUIDs[1].c_str());
@@ -301,6 +302,78 @@ namespace bs
 		}
 	}
 
+	bool BuiltinResourcesHelper::updateJSON(const Path& folder, AssetType type, nlohmann::json& entries)
+	{
+		UnorderedSet<Path> existingEntries;
+		for(auto& entry : entries)
+		{
+			std::string strPath = entry["Path"];
+			Path path = strPath.c_str();
+
+			existingEntries.insert(path);
+		}
+
+		bool foundChanges = false;
+		auto checkForChanges = [&](const Path& filePath)
+		{
+			Path relativePath = filePath.getRelative(folder);
+
+			auto iterFind = existingEntries.find(relativePath);
+			if(iterFind == existingEntries.end())
+			{
+				if(type == AssetType::Normal)
+				{
+					String uuid = UUIDGenerator::generateRandom();
+					nlohmann::json newEntry =
+					{ 
+						{ "Path", relativePath.toString().c_str() },
+						{ "UUID", uuid.c_str() } 
+					};
+
+					entries.push_back(newEntry);
+				}
+				else // Sprite
+				{
+					String texUuid = UUIDGenerator::generateRandom();
+					String spriteUuid = UUIDGenerator::generateRandom();
+					nlohmann::json newEntry = 
+					{ 
+						{ "Path", relativePath.toString().c_str() },
+						{ "SpriteUUID", spriteUuid.c_str() },
+						{ "TextureUUID", texUuid.c_str() }
+					};
+
+					entries.push_back(newEntry);
+				}
+
+				foundChanges = true;
+			}
+
+			return true;
+		};
+
+		FileSystem::iterate(folder, checkForChanges, nullptr);
+
+		// Prune deleted entries
+		auto iter = entries.begin();
+		while(iter != entries.end())
+		{
+			std::string strPath = (*iter)["Path"];
+			Path path = strPath.c_str();
+			path = path.getAbsolute(folder);
+
+			if (!FileSystem::exists(path))
+			{
+				iter = entries.erase(iter);
+				foundChanges = true;
+			}
+			else
+				++iter;
+		}
+
+		return foundChanges;
+	}
+
 	void BuiltinResourcesHelper::writeTimestamp(const Path& file)
 	{
 		SPtr<DataStream> fileStream = FileSystem::createAndOpenFile(file);