resource: do not put duplicate entries in packages

src/core/containers/hash_map.h

@@ -53,7 +53,7 @@ namespace hash_map
 	/// Returns a pointer to the first item in the map, can be used to
 	/// efficiently iterate over the elements (in random order).
 	/// @note
-	/// You should check whether the item is valid with hash_map::is_hole().
+	/// You should skip invalid items with HASH_MAP_SKIP_HOLE().
 	template <typename TKey, typename TValue, typename Hash, typename KeyEqual> const typename HashMap<TKey, TValue, Hash, KeyEqual>::Entry* begin(const HashMap<TKey, TValue, Hash, KeyEqual>& m);
 	template <typename TKey, typename TValue, typename Hash, typename KeyEqual> const typename HashMap<TKey, TValue, Hash, KeyEqual>::Entry* end(const HashMap<TKey, TValue, Hash, KeyEqual>& m);
 
@@ -302,13 +302,13 @@ namespace hash_map
 	}
 
 	template <typename TKey, typename TValue, typename Hash, typename KeyEqual>
-	const typename HashMap<TKey, TValue, Hash, KeyEqual>::Entry* begin(const HashMap<TKey, TValue, Hash, KeyEqual>& m)
+	inline const typename HashMap<TKey, TValue, Hash, KeyEqual>::Entry* begin(const HashMap<TKey, TValue, Hash, KeyEqual>& m)
 	{
 		return m._data;
 	}
 
 	template <typename TKey, typename TValue, typename Hash, typename KeyEqual>
-	const typename HashMap<TKey, TValue, Hash, KeyEqual>::Entry* end(const HashMap<TKey, TValue, Hash, KeyEqual>& m)
+	inline const typename HashMap<TKey, TValue, Hash, KeyEqual>::Entry* end(const HashMap<TKey, TValue, Hash, KeyEqual>& m)
 	{
 		return m._data + m._capacity;
 	}

src/core/containers/hash_set.h

@@ -36,6 +36,17 @@ namespace hash_set
 	/// Calls destructor on the items.
 	template <typename TKey, typename Hash, typename KeyEqual> void clear(HashSet<TKey, Hash, KeyEqual>& m);
 
+	/// Returns whether the @a entry in the set @a m contains data or is a hole.
+	/// If the entry is a hole you should not touch data in the entry.
+	template <typename TKey, typename Hash, typename KeyEqual> bool is_hole(const HashSet<TKey, Hash, KeyEqual>& m, const TKey* entry);
+
+	/// Returns a pointer to the first item in the set, can be used to
+	/// efficiently iterate over the elements (in random order).
+	/// @note
+	/// You should skip invalid items with HASH_SET_SKIP_HOLE().
+	template <typename TKey, typename Hash, typename KeyEqual> const TKey* begin(const HashSet<TKey, Hash, KeyEqual>& m);
+	template <typename TKey, typename Hash, typename KeyEqual> const TKey* end(const HashSet<TKey, Hash, KeyEqual>& m);
+
 } // namespace hash_set
 
 namespace hash_set_internal
@@ -259,6 +270,27 @@ namespace hash_set
 		m._size = 0;
 	}
 
+	template <typename TKey, typename Hash, typename KeyEqual>
+	bool is_hole(const HashSet<TKey, Hash, KeyEqual>& m, const TKey* entry)
+	{
+		const u32 ii = u32(entry - m._data);
+		const u32 index = m._index[ii].index;
+
+		return index == hash_set_internal::FREE || hash_set_internal::is_deleted(index);
+	}
+
+	template <typename TKey, typename Hash, typename KeyEqual>
+	inline const TKey* begin(const HashSet<TKey, Hash, KeyEqual>& m)
+	{
+		return m._data;
+	}
+
+	template <typename TKey, typename Hash, typename KeyEqual>
+	inline const TKey* end(const HashSet<TKey, Hash, KeyEqual>& m)
+	{
+		return m._data + m._capacity;
+	}
+
 } // namespace hash_set
 
 template <typename TKey, typename Hash, typename KeyEqual>
@@ -343,4 +375,8 @@ HashSet<TKey, Hash, KeyEqual>& HashSet<TKey, Hash, KeyEqual>::operator=(const Ha
 	return *this;
 }
 
+#define HASH_SET_SKIP_HOLE(m, cur) \
+	if (hash_set::is_hole(m, cur)) \
+		continue
+
 } // namespace crown

src/resource/package_resource.cpp

@@ -22,9 +22,18 @@
 
 namespace crown
 {
+template<>
+struct hash<PackageResource::Resource>
+{
+	u32 operator()(const PackageResource::Resource& val) const
+	{
+		return (u32)resource_id(val.type, val.name)._id;
+	}
+};
+
 namespace package_resource_internal
 {
-	s32 bring_in_requirements(Array<PackageResource::Resource>& output, CompileOptions& opts, ResourceId res_id)
+	s32 bring_in_requirements(HashSet<PackageResource::Resource>& output, CompileOptions& opts, ResourceId res_id)
 	{
 		HashMap<DynamicString, u32> reqs_deffault(default_allocator());
 		HashMap<DynamicString, u32>& reqs = hash_map::get(opts._data_compiler._data_requirements, res_id, reqs_deffault);
@@ -41,18 +50,15 @@ namespace package_resource_internal
 
 			const StringId64 req_type_hash(req_type);
 			const StringId64 req_name_hash(req_filename, req_name_len);
-			array::push_back(output, PackageResource::Resource(req_type_hash, req_name_hash));
+			hash_set::insert(output, PackageResource::Resource(req_type_hash, req_name_hash));
 
-			bring_in_requirements(output
-				, opts
-				, resource_id(req_type_hash, req_name_hash)
-				);
+			bring_in_requirements(output, opts, resource_id(req_type_hash, req_name_hash));
 		}
 
 		return 0;
 	}
 
-	s32 compile_resources(const char* type, const JsonArray& names, Array<PackageResource::Resource>& output, CompileOptions& opts)
+	s32 compile_resources(HashSet<PackageResource::Resource>& output, CompileOptions& opts, const char* type, const JsonArray& names)
 	{
 		const StringId64 type_hash = StringId64(type);
 
@@ -64,13 +70,10 @@ namespace package_resource_internal
 			DATA_COMPILER_ASSERT_RESOURCE_EXISTS(type, name.c_str(), opts);
 
 			const StringId64 name_hash = sjson::parse_resource_name(names[i]);
-			array::push_back(output, PackageResource::Resource(type_hash, name_hash));
+			hash_set::insert(output, PackageResource::Resource(type_hash, name_hash));
 
 			// Bring in requirements
-			bring_in_requirements(output
-				, opts
-				, resource_id(type_hash, name_hash)
-				);
+			bring_in_requirements(output, opts, resource_id(type_hash, name_hash));
 		}
 
 		return 0;
@@ -78,12 +81,8 @@ namespace package_resource_internal
 
 	s32 compile(CompileOptions& opts)
 	{
-		Buffer buf = opts.read();
-
 		TempAllocator4096 ta;
 		JsonObject object(ta);
-		sjson::parse(buf, object);
-
 		JsonArray texture(ta);
 		JsonArray script(ta);
 		JsonArray sound(ta);
@@ -97,6 +96,12 @@ namespace package_resource_internal
 		JsonArray shader(ta);
 		JsonArray sprite_animation(ta);
 
+		Array<PackageResource::Resource> resources(default_allocator());
+		HashSet<PackageResource::Resource> resources_set(default_allocator());
+
+		Buffer buf = opts.read();
+		sjson::parse(buf, object);
+
 		if (json_object::has(object, "texture"))          sjson::parse_array(object["texture"], texture);
 		if (json_object::has(object, "lua"))              sjson::parse_array(object["lua"], script);
 		if (json_object::has(object, "sound"))            sjson::parse_array(object["sound"], sound);
@@ -110,36 +115,27 @@ namespace package_resource_internal
 		if (json_object::has(object, "shader"))           sjson::parse_array(object["shader"], shader);
 		if (json_object::has(object, "sprite_animation")) sjson::parse_array(object["sprite_animation"], sprite_animation);
 
-		Array<PackageResource::Resource> resources(default_allocator());
-
-		if (compile_resources("texture", texture, resources, opts) != 0) return -1;
-		if (compile_resources("lua", script, resources, opts) != 0) return -1;
-		if (compile_resources("sound", sound, resources, opts) != 0) return -1;
-		if (compile_resources("mesh", mesh, resources, opts) != 0) return -1;
-		if (compile_resources("unit", unit, resources, opts) != 0) return -1;
-		if (compile_resources("sprite", sprite, resources, opts) != 0) return -1;
-		if (compile_resources("material", material, resources, opts) != 0) return -1;
-		if (compile_resources("font", font, resources, opts) != 0) return -1;
-		if (compile_resources("level", level, resources, opts) != 0) return -1;
-		if (compile_resources("physics_config", phyconf, resources, opts) != 0) return -1;
-		if (compile_resources("shader", shader, resources, opts) != 0) return -1;
-		if (compile_resources("sprite_animation", sprite_animation, resources, opts) != 0) return -1;
-
-		// Remove duplicated entries
-		HashSet<ResourceId> duplicates(default_allocator());
-		for (u32 i = 0; i < array::size(resources); ++i)
+		if (compile_resources(resources_set, opts, "texture", texture) != 0) return -1;
+		if (compile_resources(resources_set, opts, "lua", script) != 0) return -1;
+		if (compile_resources(resources_set, opts, "sound", sound) != 0) return -1;
+		if (compile_resources(resources_set, opts, "mesh", mesh) != 0) return -1;
+		if (compile_resources(resources_set, opts, "unit", unit) != 0) return -1;
+		if (compile_resources(resources_set, opts, "sprite", sprite) != 0) return -1;
+		if (compile_resources(resources_set, opts, "material", material) != 0) return -1;
+		if (compile_resources(resources_set, opts, "font", font) != 0) return -1;
+		if (compile_resources(resources_set, opts, "level", level) != 0) return -1;
+		if (compile_resources(resources_set, opts, "physics_config", phyconf) != 0) return -1;
+		if (compile_resources(resources_set, opts, "shader", shader) != 0) return -1;
+		if (compile_resources(resources_set, opts, "sprite_animation", sprite_animation) != 0) return -1;
+
+		// Generate resource list
+		auto cur = hash_set::begin(resources_set);
+		auto end = hash_set::end(resources_set);
+		for (; cur != end; ++cur)
 		{
-			ResourceId res_id = resource_id(resources[i].type, resources[i].name);
-
-			if (hash_set::has(duplicates, res_id))
-			{
-				resources[i] = resources[array::size(resources) - 1];
-				array::pop_back(resources);
-			}
-			else
-			{
-				hash_set::insert(duplicates, res_id);
-			}
+			HASH_SET_SKIP_HOLE(resources_set, cur);
+
+			array::push_back(resources, *cur);
 		}
 
 		// Write

src/resource/package_resource.h

@@ -22,6 +22,10 @@ struct PackageResource
 		StringId64 type;
 		StringId64 name;
 
+		Resource()
+		{
+		}
+
 		Resource(StringId64 t, StringId64 n)
 			: type(t)
 			, name(n)

src/resource/types.h

@@ -61,7 +61,7 @@ struct UnitResource;
 #define RESOURCE_VERSION_LEVEL            RESOURCE_VERSION(5)
 #define RESOURCE_VERSION_MATERIAL         RESOURCE_VERSION(2)
 #define RESOURCE_VERSION_MESH             RESOURCE_VERSION(1)
-#define RESOURCE_VERSION_PACKAGE          RESOURCE_VERSION(2)
+#define RESOURCE_VERSION_PACKAGE          RESOURCE_VERSION(3)
 #define RESOURCE_VERSION_PHYSICS_CONFIG   RESOURCE_VERSION(1)
 #define RESOURCE_VERSION_SCRIPT           RESOURCE_VERSION(1)
 #define RESOURCE_VERSION_SHADER           RESOURCE_VERSION(1)