Selaa lähdekoodia

Refactor serialization/RTTI system so it supports streaming

BearishSun 9 vuotta sitten
vanhempi
sitoutus
63c61046cd
30 muutettua tiedostoa jossa 487 lisäystä ja 515 poistoa
  1. 0 2
      Build/VS2015/BansheeUtility.vcxproj
  2. 0 6
      Build/VS2015/BansheeUtility.vcxproj.filters
  3. 7 14
      Source/BansheeCore/Include/BsAudioClipRTTI.h
  4. 17 24
      Source/BansheeCore/Include/BsMaterialParamsRTTI.h
  5. 9 15
      Source/BansheeCore/Include/BsMeshDataRTTI.h
  6. 10 17
      Source/BansheeCore/Include/BsPixelDataRTTI.h
  7. 2 2
      Source/BansheeCore/Include/BsPrefabDiffRTTI.h
  8. 0 2
      Source/BansheeCore/Include/BsTextureRTTI.h
  9. 6 6
      Source/BansheeCore/Source/BsPrefabDiff.cpp
  10. 2 12
      Source/BansheeEditor/Source/BsEditorTestSuite.cpp
  11. 15 19
      Source/BansheePhysX/Include/BsPhysXMeshRTTI.h
  12. 1 0
      Source/BansheeUtility/Include/BsBinaryDiff.h
  13. 12 10
      Source/BansheeUtility/Include/BsBinarySerializer.h
  14. 8 2
      Source/BansheeUtility/Include/BsDataStream.h
  15. 1 2
      Source/BansheeUtility/Include/BsFileSerializer.h
  16. 1 0
      Source/BansheeUtility/Include/BsFwdDeclUtil.h
  17. 0 54
      Source/BansheeUtility/Include/BsManagedDataBlock.h
  18. 0 1
      Source/BansheeUtility/Include/BsRTTIField.h
  19. 15 36
      Source/BansheeUtility/Include/BsRTTIManagedDataBlockField.h
  20. 17 32
      Source/BansheeUtility/Include/BsRTTIType.h
  21. 32 7
      Source/BansheeUtility/Include/BsSerializedObject.h
  22. 63 22
      Source/BansheeUtility/Include/BsSerializedObjectRTTI.h
  23. 55 9
      Source/BansheeUtility/Source/BsBinaryDiff.cpp
  24. 153 153
      Source/BansheeUtility/Source/BsBinarySerializer.cpp
  25. 7 3
      Source/BansheeUtility/Source/BsDataStream.cpp
  26. 11 27
      Source/BansheeUtility/Source/BsFileSerializer.cpp
  27. 0 33
      Source/BansheeUtility/Source/BsManagedDataBlock.cpp
  28. 4 1
      Source/BansheeUtility/Source/BsMemorySerializer.cpp
  29. 35 0
      Source/BansheeUtility/Source/BsSerializedObject.cpp
  30. 4 4
      Source/SBansheeEngine/Source/BsManagedDiff.cpp

+ 0 - 2
Build/VS2015/BansheeUtility.vcxproj

@@ -304,7 +304,6 @@
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsTorus.cpp" />
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsUtil.cpp" />
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsVector2I.cpp" />
-    <ClCompile Include="..\..\Source\BansheeUtility\Source\BsManagedDataBlock.cpp" />
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsMemoryAllocator.cpp" />
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsMemStack.cpp" />
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsRadian.cpp" />
@@ -361,7 +360,6 @@
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsTorus.h" />
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsVector2I.h" />
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsIReflectable.h" />
-    <ClInclude Include="..\..\Source\BansheeUtility\Include\BsManagedDataBlock.h" />
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsMemoryAllocator.h" />
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsMemAllocProfiler.h" />
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsModule.h" />

+ 0 - 6
Build/VS2015/BansheeUtility.vcxproj.filters

@@ -266,9 +266,6 @@
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsTaskScheduler.h">
       <Filter>Header Files\Threading</Filter>
     </ClInclude>
-    <ClInclude Include="..\..\Source\BansheeUtility\Include\BsManagedDataBlock.h">
-      <Filter>Header Files\Serialization</Filter>
-    </ClInclude>
     <ClInclude Include="..\..\Source\BansheeUtility\Include\BsTestSuite.h">
       <Filter>Header Files\Testing</Filter>
     </ClInclude>
@@ -484,9 +481,6 @@
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsGlobalFrameAlloc.cpp">
       <Filter>Source Files\Allocators</Filter>
     </ClCompile>
-    <ClCompile Include="..\..\Source\BansheeUtility\Source\BsManagedDataBlock.cpp">
-      <Filter>Source Files\Serialization</Filter>
-    </ClCompile>
     <ClCompile Include="..\..\Source\BansheeUtility\Source\BsLog.cpp">
       <Filter>Source Files\Debug</Filter>
     </ClCompile>

+ 7 - 14
Source/BansheeCore/Include/BsAudioClipRTTI.h

@@ -5,6 +5,7 @@
 #include "BsCorePrerequisites.h"
 #include "BsRTTIType.h"
 #include "BsAudioClip.h"
+#include "BsDataStream.h"
 
 namespace BansheeEngine
 {
@@ -25,30 +26,22 @@ namespace BansheeEngine
 			BS_RTTI_MEMBER_PLAIN(mNumSamples, 5)
 		BS_END_RTTI_MEMBERS
 
-		ManagedDataBlock getData(AudioClip* obj)
+		SPtr<DataStream> getData(AudioClip* obj, UINT32& size)
 		{
-			ManagedDataBlock dataBlock((UINT8*)obj->getData(), obj->getDataSize());
-			return dataBlock;
+			size = obj->getDataSize();
+			return bs_shared_ptr_new<MemoryDataStream>(obj->getData(), size, false);
 		}
 
-		void setData(AudioClip* obj, ManagedDataBlock val)
+		void setData(AudioClip* obj, const SPtr<DataStream>& val, UINT32 size)
 		{
-			// TODO: This method isn't needed as this data will be loaded from the data stream
-			obj->setData(val.getData(), obj->mNumSamples);
-		}
-
-		static UINT8* allocateData(AudioClip* obj, UINT32 numBytes)
-		{
-			// TODO: This method isn't needed as this data will be loaded from the data stream
-			return (UINT8*)bs_alloc(numBytes);
+			// TODO - Load from steam
 		}
 
 	public:
 		AudioClipRTTI()
 			:mInitMembers(this)
 		{
-			addDataBlockField("mData", 6, &AudioClipRTTI::getData, &AudioClipRTTI::setData, 
-				0, &AudioClipRTTI::allocateData);
+			addDataBlockField("mData", 6, &AudioClipRTTI::getData, &AudioClipRTTI::setData, 0);
 		}
 
 		const String& getRTTIName() override

+ 17 - 24
Source/BansheeCore/Include/BsMaterialParamsRTTI.h

@@ -6,6 +6,7 @@
 #include "BsRTTIType.h"
 #include "BsMaterialParams.h"
 #include "BsSamplerState.h"
+#include "BsDataStream.h"
 
 namespace BansheeEngine
 {
@@ -53,28 +54,24 @@ namespace BansheeEngine
 	class BS_CORE_EXPORT StructParamDataRTTI : public RTTIType<MaterialParams::StructParamData, IReflectable, StructParamDataRTTI>
 	{
 	public:
-		ManagedDataBlock getDataBuffer(MaterialParams::StructParamData* obj)
+		SPtr<DataStream> getDataBuffer(MaterialParams::StructParamData* obj, UINT32& size)
 		{
-			return ManagedDataBlock(obj->data, obj->dataSize);
-		}
+			size = obj->dataSize;
 
-		void setDataBuffer(MaterialParams::StructParamData* obj, ManagedDataBlock value)
-		{
-			// Do nothing as the buffer was already assigned when it was allocated
+			return bs_shared_ptr_new<MemoryDataStream>(obj->data, obj->dataSize, false);
 		}
 
-		static UINT8* allocateDataBuffer(MaterialParams::StructParamData* obj, UINT32 numBytes)
+		void setDataBuffer(MaterialParams::StructParamData* obj, const SPtr<DataStream>& value, UINT32 size)
 		{
-			obj->data = (UINT8*)bs_alloc(numBytes);
-			obj->dataSize = numBytes;
+			obj->data = (UINT8*)bs_alloc(size);
+			value->read(obj->data, size);
 
-			return obj->data;
+			obj->dataSize = size;
 		}
 
 		StructParamDataRTTI()
 		{
-			addDataBlockField("dataBuffer", 0, &StructParamDataRTTI::getDataBuffer,
-				&StructParamDataRTTI::setDataBuffer, 0, &StructParamDataRTTI::allocateDataBuffer);
+			addDataBlockField("dataBuffer", 0, &StructParamDataRTTI::getDataBuffer, &StructParamDataRTTI::setDataBuffer, 0);
 		}
 
 		const String& getRTTIName() override
@@ -125,22 +122,19 @@ namespace BansheeEngine
 			// Do nothing
 		}
 
-		ManagedDataBlock getDataBuffer(MaterialParams* obj)
+		SPtr<DataStream> getDataBuffer(MaterialParams* obj, UINT32& size)
 		{
-			return ManagedDataBlock(obj->mDataParamsBuffer, obj->mDataSize);
-		}
+			size = obj->mDataSize;
 
-		void setDataBuffer(MaterialParams* obj, ManagedDataBlock value)
-		{
-			// Do nothing as the buffer was already assigned when it was allocated
+			return bs_shared_ptr_new<MemoryDataStream>(obj->mDataParamsBuffer, obj->mDataSize, false);
 		}
 
-		static UINT8* allocateDataBuffer(MaterialParams* obj, UINT32 numBytes)
+		void setDataBuffer(MaterialParams* obj, const SPtr<DataStream>& value, UINT32 size)
 		{
-			obj->mDataParamsBuffer = obj->mAlloc.alloc(numBytes);
-			obj->mDataSize = numBytes;
+			obj->mDataParamsBuffer = obj->mAlloc.alloc(size);
+			value->read(obj->mDataParamsBuffer, size);
 
-			return obj->mDataParamsBuffer;
+			obj->mDataSize = size;
 		}
 
 		MaterialParams::StructParamData& getStructParam(MaterialParams* obj, UINT32 idx) { return obj->mStructParams[idx]; }
@@ -184,8 +178,7 @@ namespace BansheeEngine
 			addPlainArrayField("paramData", 0, &MaterialParamsRTTI::getParamData, &MaterialParamsRTTI::getParamDataArraySize, 
 				&MaterialParamsRTTI::setParamData, &MaterialParamsRTTI::setParamDataArraySize);
 
-			addDataBlockField("dataBuffer", 1, &MaterialParamsRTTI::getDataBuffer,
-				&MaterialParamsRTTI::setDataBuffer, 0, &MaterialParamsRTTI::allocateDataBuffer);
+			addDataBlockField("dataBuffer", 1, &MaterialParamsRTTI::getDataBuffer, &MaterialParamsRTTI::setDataBuffer, 0);
 
 			addReflectableArrayField("structParams", 2, &MaterialParamsRTTI::getStructParam,
 				&MaterialParamsRTTI::getStructArraySize, &MaterialParamsRTTI::setStructParam, &MaterialParamsRTTI::setStructArraySize);

+ 9 - 15
Source/BansheeCore/Include/BsMeshDataRTTI.h

@@ -5,8 +5,8 @@
 #include "BsCorePrerequisites.h"
 #include "BsRTTIType.h"
 #include "BsMeshData.h"
-#include "BsManagedDataBlock.h"
 #include "BsVertexDeclaration.h"
+#include "BsDataStream.h"
 
 namespace BansheeEngine
 {
@@ -32,23 +32,17 @@ namespace BansheeEngine
 		UINT32& getNumIndices(MeshData* obj) { return obj->mNumIndices; }
 		void setNumIndices(MeshData* obj, UINT32& value) { obj->mNumIndices = value; }
 
-		ManagedDataBlock getData(MeshData* obj) 
-		{ 
-			ManagedDataBlock dataBlock((UINT8*)obj->getData(), obj->getInternalBufferSize());
-			return dataBlock; 
-		}
+		SPtr<DataStream> getData(MeshData* obj, UINT32& size)
+		{
+			size = obj->getInternalBufferSize();
 
-		void setData(MeshData* obj, ManagedDataBlock val) 
-		{ 
-			// Nothing to do here, the pointer we provided already belongs to PixelData
-			// so the data is already written
+			return bs_shared_ptr_new<MemoryDataStream>(obj->getData(), size, false);
 		}
 
-		static UINT8* allocateData(MeshData* obj, UINT32 numBytes)
+		void setData(MeshData* obj, const SPtr<DataStream>& value, UINT32 size)
 		{
-			obj->allocateInternalBuffer(numBytes);
-
-			return obj->getData();
+			obj->allocateInternalBuffer(size);
+			value->read(obj->getData(), size);
 		}
 
 	public:
@@ -60,7 +54,7 @@ namespace BansheeEngine
 			addPlainField("mNumVertices", 2, &MeshDataRTTI::getNumVertices, &MeshDataRTTI::setNumVertices);
 			addPlainField("mNumIndices", 3, &MeshDataRTTI::getNumIndices, &MeshDataRTTI::setNumIndices);
 
-			addDataBlockField("data", 4, &MeshDataRTTI::getData, &MeshDataRTTI::setData, 0, &MeshDataRTTI::allocateData);
+			addDataBlockField("data", 4, &MeshDataRTTI::getData, &MeshDataRTTI::setData, 0);
 		}
 
 		SPtr<IReflectable> newRTTIObject() override

+ 10 - 17
Source/BansheeCore/Include/BsPixelDataRTTI.h

@@ -5,7 +5,7 @@
 #include "BsCorePrerequisites.h"
 #include "BsPixelData.h"
 #include "BsRTTIType.h"
-#include "BsManagedDataBlock.h"
+#include "BsDataStream.h"
 
 namespace BansheeEngine
 {
@@ -43,25 +43,19 @@ namespace BansheeEngine
 		PixelFormat& getFormat(PixelData* obj) { return obj->mFormat; }
 		void setFormat(PixelData* obj, PixelFormat& val) { obj->mFormat = val; }
 
-		ManagedDataBlock getData(PixelData* obj) 
-		{ 
-			ManagedDataBlock dataBlock((UINT8*)obj->getData(), obj->getConsecutiveSize());
-			return dataBlock; 
-		}
+		SPtr<DataStream> getData(PixelData* obj, UINT32& size)
+		{
+			size = obj->getConsecutiveSize();
 
-		void setData(PixelData* obj, ManagedDataBlock val) 
-		{ 
-			// Nothing to do here, the pointer we provided already belongs to PixelData
-			// so the data is already written
+			return bs_shared_ptr_new<MemoryDataStream>(obj->getData(), size, false);
 		}
 
-		static UINT8* allocateData(PixelData* obj, UINT32 numBytes)
+		void setData(PixelData* obj, const SPtr<DataStream>& value, UINT32 size)
 		{
-			obj->allocateInternalBuffer(numBytes);
-
-			return obj->getData();
+			obj->allocateInternalBuffer(size);
+			value->read(obj->getData(), size);
 		}
-
+		
 	public:
 		PixelDataRTTI()
 		{
@@ -74,8 +68,7 @@ namespace BansheeEngine
 			addPlainField("rowPitch", 6, &PixelDataRTTI::getRowPitch, &PixelDataRTTI::setRowPitch);
 			addPlainField("slicePitch", 7, &PixelDataRTTI::getSlicePitch, &PixelDataRTTI::setSlicePitch);
 			addPlainField("format", 8, &PixelDataRTTI::getFormat, &PixelDataRTTI::setFormat);
-
-			addDataBlockField("data", 9, &PixelDataRTTI::getData, &PixelDataRTTI::setData, 0, &PixelDataRTTI::allocateData);
+			addDataBlockField("data", 9, &PixelDataRTTI::getData, &PixelDataRTTI::setData, 0);
 		}
 
 		virtual const String& getRTTIName() override

+ 2 - 2
Source/BansheeCore/Include/BsPrefabDiffRTTI.h

@@ -157,7 +157,7 @@ namespace BansheeEngine
 				SerializedHandle& handle = handleData[idx];
 
 				handle.object = handleObject;
-				handle.handle = std::static_pointer_cast<GameObjectHandleBase>(bs._decodeIntermediate(handleObject));
+				handle.handle = std::static_pointer_cast<GameObjectHandleBase>(bs._decodeFromIntermediate(handleObject));
 
 				idx++;
 			}
@@ -178,7 +178,7 @@ namespace BansheeEngine
 			for (auto& serializedHandle : handleData)
 			{
 				if (serializedHandle.handle != nullptr)
-					*serializedHandle.object = *bs._encodeIntermediate(serializedHandle.handle.get());
+					*serializedHandle.object = *bs._encodeToIntermediate(serializedHandle.handle.get());
 			}
 
 			prefabDiff->mRTTIData = nullptr;

+ 0 - 2
Source/BansheeCore/Include/BsTextureRTTI.h

@@ -5,9 +5,7 @@
 #include "BsCorePrerequisites.h"
 #include "BsRTTIType.h"
 #include "BsTexture.h"
-#include "BsManagedDataBlock.h"
 #include "BsMath.h"
-#include "BsCoreApplication.h"
 #include "BsCoreThread.h"
 #include "BsRenderAPI.h"
 #include "BsTextureManager.h"

+ 6 - 6
Source/BansheeCore/Source/BsPrefabDiff.cpp

@@ -112,7 +112,7 @@ namespace BansheeEngine
 		for (auto& addedComponentData : diff->addedComponents)
 		{
 			BinarySerializer bs;
-			SPtr<Component> component = std::static_pointer_cast<Component>(bs._decodeIntermediate(addedComponentData));
+			SPtr<Component> component = std::static_pointer_cast<Component>(bs._decodeFromIntermediate(addedComponentData));
 
 			object->addComponentInternal(component);
 		}
@@ -120,7 +120,7 @@ namespace BansheeEngine
 		for (auto& addedChildData : diff->addedChildren)
 		{
 			BinarySerializer bs;
-			SPtr<SceneObject> sceneObject = std::static_pointer_cast<SceneObject>(bs._decodeIntermediate(addedChildData));
+			SPtr<SceneObject> sceneObject = std::static_pointer_cast<SceneObject>(bs._decodeFromIntermediate(addedChildData));
 			sceneObject->setParent(object);
 			sceneObject->_instantiate();
 		}
@@ -271,7 +271,7 @@ namespace BansheeEngine
 			if (!foundMatching)
 			{
 				BinarySerializer bs;
-				SPtr<SerializedObject> obj = bs._encodeIntermediate(instanceChild.get());
+				SPtr<SerializedObject> obj = bs._encodeToIntermediate(instanceChild.get());
 
 				if (output == nullptr)
 					output = bs_shared_ptr_new<PrefabObjectDiff>();
@@ -300,8 +300,8 @@ namespace BansheeEngine
 				if (prefabComponent->getLinkId() == instanceComponent->getLinkId())
 				{
 					BinarySerializer bs;
-					SPtr<SerializedObject> encodedPrefab = bs._encodeIntermediate(prefabComponent.get());
-					SPtr<SerializedObject> encodedInstance = bs._encodeIntermediate(instanceComponent.get());
+					SPtr<SerializedObject> encodedPrefab = bs._encodeToIntermediate(prefabComponent.get());
+					SPtr<SerializedObject> encodedInstance = bs._encodeToIntermediate(instanceComponent.get());
 
 					IDiff& diffHandler = prefabComponent->getRTTI()->getDiffHandler();
 					SPtr<SerializedObject> diff = diffHandler.generateDiff(encodedPrefab, encodedInstance);
@@ -360,7 +360,7 @@ namespace BansheeEngine
 			if (!foundMatching)
 			{
 				BinarySerializer bs;
-				SPtr<SerializedObject> obj = bs._encodeIntermediate(instanceComponent.get());
+				SPtr<SerializedObject> obj = bs._encodeToIntermediate(instanceComponent.get());
 
 				if (output == nullptr)
 					output = bs_shared_ptr_new<PrefabObjectDiff>();

+ 2 - 12
Source/BansheeEditor/Source/BsEditorTestSuite.cpp

@@ -521,24 +521,14 @@ namespace BansheeEngine
 		newObj->arrObjB[1].strA = "strawberry";
 		newObj->arrObjPtrB[0]->intA = 99100;
 
-		MemorySerializer ms;
-		UINT32 orgDataLength = 0;
-		UINT8* orgData = ms.encode(orgObj.get(), orgDataLength);
-
-		UINT32 newDataLength = 0;
-		UINT8* newData = ms.encode(newObj.get(), newDataLength);
-
 		BinarySerializer bs;
-		SPtr<SerializedObject> orgSerialized = bs._decodeIntermediate(orgData, orgDataLength);
-		SPtr<SerializedObject> newSerialized = bs._decodeIntermediate(newData, newDataLength);
+		SPtr<SerializedObject> orgSerialized = bs._encodeToIntermediate(orgObj.get());
+		SPtr<SerializedObject> newSerialized = bs._encodeToIntermediate(newObj.get());
 
 		IDiff& diffHandler = orgObj->getRTTI()->getDiffHandler();
 		SPtr<SerializedObject> objDiff = diffHandler.generateDiff(orgSerialized, newSerialized);
 		diffHandler.applyDiff(orgObj, objDiff);
 
-		bs_free(orgData);
-		bs_free(newData);
-
 		BS_TEST_ASSERT(orgObj->intA == newObj->intA);
 		BS_TEST_ASSERT(orgObj->strA == newObj->strA);
 		BS_TEST_ASSERT(orgObj->strB == newObj->strB);

+ 15 - 19
Source/BansheePhysX/Include/BsPhysXMeshRTTI.h

@@ -5,6 +5,7 @@
 #include "BsPhysXPrerequisites.h"
 #include "BsRTTIType.h"
 #include "BsPhysXMesh.h"
+#include "BsDataStream.h"
 
 namespace BansheeEngine
 {
@@ -16,30 +17,25 @@ namespace BansheeEngine
 	class BS_PHYSX_EXPORT FPhysXMeshRTTI : public RTTIType<FPhysXMesh, FPhysicsMesh, FPhysXMeshRTTI>
 	{
 	private:
-		ManagedDataBlock getCookedData(FPhysXMesh* obj)
-		{
-			ManagedDataBlock dataBlock((UINT8*)obj->mCookedData, obj->mCookedDataSize);
-			return dataBlock;
-		}
-
-		void setCookedData(FPhysXMesh* obj, ManagedDataBlock val)
-		{
-			// Nothing to do here, the pointer we provided in allocateCookedData() already belongs to PhysXMesh
-			// so the data is already written
+		SPtr<DataStream> getCookedData(FPhysXMesh* obj, UINT32& size)
+		{
+			size = obj->mCookedDataSize;
+
+			return bs_shared_ptr_new<MemoryDataStream>(obj->mCookedData, obj->mCookedDataSize, false);
+		}
+
+		void setCookedData(FPhysXMesh* obj, const SPtr<DataStream>& value, UINT32 size)
+		{
+			obj->mCookedData = (UINT8*)bs_alloc(size);
+			obj->mCookedDataSize = size;
+
+			value->read(obj->mCookedData, size);
 		}
 
-		static UINT8* allocateCookedData(FPhysXMesh* obj, UINT32 numBytes)
-		{
-			obj->mCookedData = (UINT8*)bs_alloc(numBytes);
-			obj->mCookedDataSize = numBytes;
-
-			return obj->mCookedData;
-		}
 	public:
 		FPhysXMeshRTTI()
 		{
-			addDataBlockField("mCookedData", 0, 
-				&FPhysXMeshRTTI::getCookedData, &FPhysXMeshRTTI::setCookedData, 0, &FPhysXMeshRTTI::allocateCookedData);
+			addDataBlockField("mCookedData", 0, &FPhysXMeshRTTI::getCookedData, &FPhysXMeshRTTI::setCookedData, 0);
 		}
 
 		/** @copydoc IReflectable::onDeserializationEnded */

+ 1 - 0
Source/BansheeUtility/Include/BsBinaryDiff.h

@@ -64,6 +64,7 @@ namespace BansheeEngine
 			UINT32 type;
 			SPtr<IReflectable> object;
 			UINT8* value;
+			SPtr<DataStream> streamValue;
 			UINT32 size;
 
 			union

+ 12 - 10
Source/BansheeUtility/Include/BsBinarySerializer.h

@@ -70,7 +70,7 @@ namespace BansheeEngine
 		 * @param[in]	data  		Binary data to decode.
 		 * @param[in]	dataLength	Length of the data in bytes.
 		 */
-		SPtr<IReflectable> decode(UINT8* data, UINT32 dataLength);
+		SPtr<IReflectable> decode(const SPtr<DataStream>& data, UINT32 dataLength);
 
 		/** @name Internal 
 		 *  @{
@@ -84,24 +84,25 @@ namespace BansheeEngine
 		 *							and will be set to null. If false then references will be encoded as well and restored
 		 *							upon decoding.
 		 */
-		SPtr<SerializedObject> _encodeIntermediate(IReflectable* object, bool shallow = false);
+		SPtr<SerializedObject> _encodeToIntermediate(IReflectable* object, bool shallow = false);
 
 		/**
-		 * Decodes an object in memory into an intermediate representation for easier parsing.
+		 * Decodes a serialized object into an intermediate representation for easier parsing.
 		 *			
 		 * @param[in] 	data  		Binary data to decode.
 		 * @param[in]	dataLength	Length of the data in bytes.
 		 * @param[in]	copyData	Determines should the data be copied or just referenced. If referenced then the returned
-		 *							serialized object will be invalid as soon as the original data buffer is destroyed. 
-		 *							Referencing is faster than copying.
+		 *							serialized object will be invalid as soon as the original data buffer is destroyed.
+		 *							Referencing is faster than copying. If the source data stream is a file stream the data
+		 *							will always be copied.
 		 *
 		 * @note
 		 * References to field data will point to the original buffer and will become invalid when it is destroyed.
 		 */
-		SPtr<SerializedObject> _decodeIntermediate(UINT8* data, UINT32 dataLength, bool copyData = false);
+		SPtr<SerializedObject> _decodeToIntermediate(const SPtr<DataStream>& data, UINT32 dataLength, bool copyData = false);
 
 		/** Decodes an intermediate representation of a serialized object into the actual object. */
-		SPtr<IReflectable> _decodeIntermediate(const SPtr<SerializedObject>& serializedObject);
+		SPtr<IReflectable> _decodeFromIntermediate(const SPtr<SerializedObject>& serializedObject);
 
 		/** @} */
 
@@ -135,14 +136,15 @@ namespace BansheeEngine
 		};
 
 		/** Encodes a single IReflectable object. */
-		UINT8* encodeInternal(IReflectable* object, UINT32 objectId, UINT8* buffer, UINT32& bufferLength, UINT32* bytesWritten,
+		UINT8* encodeEntry(IReflectable* object, UINT32 objectId, UINT8* buffer, UINT32& bufferLength, UINT32* bytesWritten,
 			std::function<UINT8*(UINT8* buffer, UINT32 bytesWritten, UINT32& newBufferSize)> flushBufferCallback, bool shallow);
 
 		/**	Decodes a single IReflectable object. */
-		void decodeInternal(const SPtr<IReflectable>& object, const SPtr<SerializedObject>& serializableObject);
+		void decodeEntry(const SPtr<IReflectable>& object, const SPtr<SerializedObject>& serializableObject);
 
 		/**	Decodes an object in memory into an intermediate representation for easier parsing. */
-		bool decodeIntermediateInternal(UINT8* data, UINT32 dataLength, UINT32& bytesRead, SPtr<SerializedObject>& output, bool copyData);
+		bool decodeEntry(const SPtr<DataStream>& data, UINT32 dataLength, UINT32& bytesRead, SPtr<SerializedObject>& output, 
+			bool copyData, bool streamDataBlock);
 
 		/**	Helper method for encoding a complex object and copying its data to a buffer. */
 		UINT8* complexTypeToBuffer(IReflectable* object, UINT8* buffer, UINT32& bufferLength, UINT32* bytesWritten,

+ 8 - 2
Source/BansheeUtility/Include/BsDataStream.h

@@ -48,7 +48,8 @@ namespace BansheeEngine
 
 		virtual bool isReadable() const { return (mAccess & READ) != 0; }
 		virtual bool isWriteable() const { return (mAccess & WRITE) != 0; }
-       
+		virtual bool isFile() const = 0;
+
         /** Reads data from the buffer and copies it to the specified value. */
         template<typename T> DataStream& operator>>(T& val);
 
@@ -149,8 +150,9 @@ namespace BansheeEngine
 		 *
 		 * @param[in] 	memory		Memory to wrap the data stream around.
 		 * @param[in]	size		Size of the memory chunk in bytes.
+		 * @param[in]	freeOnClose	Should the memory buffer be freed when the data stream goes out of scope.
 		 */
-		MemoryDataStream(void* memory, size_t size);
+		MemoryDataStream(void* memory, size_t size, bool freeOnClose = true);
 		
 		/**
 		 * Create a stream which pre-buffers the contents of another stream. Data from the other buffer will be entirely 
@@ -170,6 +172,8 @@ namespace BansheeEngine
 
 		~MemoryDataStream();
 
+		bool isFile() const override { return false; }
+
 		/** Get a pointer to the start of the memory block this stream holds. */
 		UINT8* getPtr() const { return mData; }
 		
@@ -245,6 +249,8 @@ namespace BansheeEngine
 
 		~FileDataStream();
 
+		bool isFile() const override { return true; }
+
         /** @copydoc DataStream::read */
 		size_t read(void* buf, size_t count) override;
 

+ 1 - 2
Source/BansheeUtility/Include/BsFileSerializer.h

@@ -41,7 +41,6 @@ namespace BansheeEngine
 	{
 	public:
 		FileDecoder(const Path& fileLocation);
-		~FileDecoder();
 
 		/**	Deserializes an IReflectable object by reading the binary data at the provided file location. */
 		SPtr<IReflectable> decode();
@@ -50,7 +49,7 @@ namespace BansheeEngine
 		void skip();
 
 	private:
-		std::ifstream mInputStream;
+		SPtr<DataStream> mInputStream;
 	};
 
 	/** @} */

+ 1 - 0
Source/BansheeUtility/Include/BsFwdDeclUtil.h

@@ -111,5 +111,6 @@ namespace BansheeEngine
 		TID_SerializedArrayEntry = 64,
 		TID_SerializedSubObject = 65,
 		TID_UnorderedSet = 66,
+		TID_SerializedDataBlock = 67
 	};
 }

+ 0 - 54
Source/BansheeUtility/Include/BsManagedDataBlock.h

@@ -1,54 +0,0 @@
-//********************************** Banshee Engine (www.banshee3d.com) **************************************************//
-//**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
-#pragma once
-
-#include "BsPrerequisitesUtil.h"
-
-namespace BansheeEngine
-{
-	/** @addtogroup Serialization
-	 *  @{
-	 */
-
-	/**
-	 * Data block holding an array of bytes, usually used in serialization.
-	 * 			
-	 * Ownership of the data blocked is passed to the latest copy of the ManagedDataBlock. Data will be automatically 
-	 * freed once the last copy is destroyed.
-	 */
-	class BS_UTILITY_EXPORT ManagedDataBlock
-	{
-	public:
-		/**
-		 * Constructor
-		 *
-		 * @param[in] 	data			Array of bytes to store. Direct pointer to the provided array will be stored, 
-		 *								no copying will be done. 
-		 * @param[in]	size			Size of the array, in bytes.
-		 */
-		ManagedDataBlock(UINT8* data, UINT32 size); 
-
-		/**
-		 * Constructor that will automatically allocate an internal buffer of the specified size. Copying ManagedDataBlock 
-		 * transfers ownership of the buffer to the copy of the buffer. Buffer is deleted when the latest copy is deleted.
-		 *
-		 * @param[in]	size	The size of the data in bytes.
-		 */
-		ManagedDataBlock(UINT32 size);
-
-		ManagedDataBlock(const ManagedDataBlock& source);
-
-		~ManagedDataBlock();
-
-		UINT8* getData() { return mData; }
-		UINT32 getSize() { return mData ? mSize : 0; }
-
-	private:
-		UINT8* mData;
-		UINT32 mSize;
-		bool mManaged;
-		mutable bool mIsDataOwner;
-	};
-
-	/** @} */
-}

+ 0 - 1
Source/BansheeUtility/Include/BsRTTIField.h

@@ -8,7 +8,6 @@
 
 #include "BsPrerequisitesUtil.h"
 #include "BsIReflectable.h"
-#include "BsManagedDataBlock.h"
 #include "BsException.h"
 #include "BsAny.h"
 

+ 15 - 36
Source/BansheeUtility/Include/BsRTTIManagedDataBlockField.h

@@ -4,7 +4,6 @@
 
 #include "BsPrerequisitesUtil.h"
 #include "BsRTTIField.h"
-#include "BsManagedDataBlock.h"
 
 namespace BansheeEngine
 {
@@ -25,19 +24,11 @@ namespace BansheeEngine
 	 */
 	struct RTTIManagedDataBlockFieldBase : public RTTIField
 	{
-		Any mCustomAllocator;
-
 		/** Retrieves a managed data block from the specified instance. */
-		virtual ManagedDataBlock getValue(void* object) = 0;
+		virtual SPtr<DataStream> getValue(void* object, UINT32& size) = 0;
 
 		/** Sets a managed data block on the specified instance. */
-		virtual void setValue(void* object, ManagedDataBlock value) = 0;
-
-		/**
-		 * Allocate memory for the managed data block. Used primarily to allocate memory before sending it to 
-		 * setValue() method.
-		 */
-		virtual UINT8* allocate(void* object, UINT32 bytes) = 0;
+		virtual void setValue(void* object, const SPtr<DataStream>& data, UINT32 size) = 0;
 	};
 
 	/** Class containing a managed data block field containing a specific type. */
@@ -54,28 +45,26 @@ namespace BansheeEngine
 		 * @param[in]	getter  		The getter method for the field. Must be a specific signature: SerializableDataBlock(ObjectType*)
 		 * @param[in]	setter  		The setter method for the field. Must be a specific signature: void(ObjectType*, SerializableDataBlock)	
 		 * @param[in]	flags			Various flags you can use to specialize how systems handle this field. See RTTIFieldFlag.
-		 * @param[in]	customAllocator (optional) Custom allocator that will be used when de-serializing DataBlock memory.
 		 */
-		void initSingle(const String& name, UINT16 uniqueId, Any getter, Any setter, UINT64 flags, Any customAllocator = Any())
+		void initSingle(const String& name, UINT16 uniqueId, Any getter, Any setter, UINT64 flags)
 		{
 			initAll(getter, setter, nullptr, nullptr, name, uniqueId, false, SerializableFT_DataBlock, flags);
-			mCustomAllocator = customAllocator;
 		}
 
 		/** @copydoc RTTIField::getTypeSize */
-		virtual UINT32 getTypeSize() override
+		UINT32 getTypeSize() override
 		{
 			return 0; // Data block types don't store size the conventional way
 		}
 
 		/** @copydoc RTTIField::hasDynamicSize */
-		virtual bool hasDynamicSize() override
+		bool hasDynamicSize() override
 		{
 			return true;
 		}
 
 		/** @copydoc RTTIField::getArraySize */
-		virtual UINT32 getArraySize(void* object) override
+		UINT32 getArraySize(void* object) override
 		{
 			BS_EXCEPT(InternalErrorException, 
 				"Data block types don't support arrays.");
@@ -84,39 +73,29 @@ namespace BansheeEngine
 		}
 
 		/** @copydoc RTTIField::setArraySize */
-		virtual void setArraySize(void* object, UINT32 size) override
+		void setArraySize(void* object, UINT32 size) override
 		{
 			BS_EXCEPT(InternalErrorException, 
 				"Data block types don't support arrays.");
 		}
 
 		/** @copydoc RTTIManagedDataBlockFieldBase::getValue */
-		virtual ManagedDataBlock getValue(void* object) override
+		SPtr<DataStream> getValue(void* object, UINT32& size) override
 		{
 			ObjectType* castObj = static_cast<ObjectType*>(object);
-			std::function<ManagedDataBlock(ObjectType*)> f = any_cast<std::function<ManagedDataBlock(ObjectType*)>>(valueGetter);
-			return f(castObj);
+			std::function<SPtr<DataStream>(ObjectType*, UINT32&)> f = any_cast<std::function<SPtr<DataStream>(ObjectType*, UINT32&)>>(valueGetter);
+
+			return f(castObj, size);
 		}
 
 		/** @copydoc RTTIManagedDataBlockFieldBase::setValue */
-		virtual void setValue(void* object, ManagedDataBlock value) override
+		void setValue(void* object, const SPtr<DataStream>& value, UINT32 size) override
 		{
 			ObjectType* castObj = static_cast<ObjectType*>(object);
-			std::function<void(ObjectType*, ManagedDataBlock)> f = any_cast<std::function<void(ObjectType*, ManagedDataBlock)>>(valueSetter);
-			f(castObj, value);
-		}
+			std::function<void(ObjectType*, const SPtr<DataStream>&, UINT32)> f = 
+				any_cast<std::function<void(ObjectType*, const SPtr<DataStream>&, UINT32)>>(valueSetter);
 
-		/** @copydoc RTTIManagedDataBlockFieldBase::allocate */
-		virtual UINT8* allocate(void* object, UINT32 bytes) override
-		{
-			if(mCustomAllocator.empty())
-				return (UINT8*)bs_alloc(bytes);
-			else
-			{
-				ObjectType* castObj = static_cast<ObjectType*>(object);
-				std::function<UINT8*(ObjectType*, UINT32)> f = any_cast<std::function<UINT8*(ObjectType*, UINT32)>>(mCustomAllocator);
-				return f(castObj, bytes);
-			}
+			f(castObj, value, size);
 		}
 	};
 

+ 17 - 32
Source/BansheeUtility/Include/BsRTTIType.h

@@ -7,7 +7,6 @@
 #include <unordered_map>
 
 #include "BsPrerequisitesUtil.h"
-#include "BsManagedDataBlock.h"
 #include "BsRTTIField.h"
 #include "BsRTTIPlainField.h"
 #include "BsRTTIReflectableField.h"
@@ -397,13 +396,13 @@ namespace BansheeEngine
 		 * @note	Caller must ensure instance type is valid for this field.
 		 */
 		template <class ObjectType>
-		void setDataBlockValue(ObjectType* object, const String& name, ManagedDataBlock value)
+		void setDataBlockValue(ObjectType* object, const String& name, const SPtr<DataStream>& value, UINT32 size)
 		{
 			RTTIField* genericField = findField(name);
 			genericField->checkIsDataBlock();
 
 			RTTIManagedDataBlockFieldBase* field = static_cast<RTTIManagedDataBlockFieldBase*>(genericField);
-			field->setValue(object, value);
+			field->setValue(object, value, size);
 		}
 
 		/**
@@ -533,13 +532,13 @@ namespace BansheeEngine
 		 * @note	Caller must ensure instance type is valid for this field.
 		 */
 		template <class ObjectType>
-		ManagedDataBlock getDataBlockValue(ObjectType* object, const String& name)
+		SPtr<DataStream> getDataBlockValue(ObjectType* object, const String& name, UINT32& size)
 		{
 			RTTIField* genericField = findField(name);
 			genericField->checkIsDataBlock();
 
 			RTTIManagedDataBlockFieldBase* field = static_cast<RTTIManagedDataBlockFieldBase*>(genericField);
-			return field->getValue(object);
+			return field->getValue(object, size);
 		}
 
 		/**
@@ -935,12 +934,12 @@ namespace BansheeEngine
 		 * @param[in]	flags		Various flags you can use to specialize how systems handle this field. See RTTIFieldFlag.
 		 */
 		template<class ObjectType>
-		void addDataBlockField(const String& name, UINT32 uniqueId, ManagedDataBlock (ObjectType::*getter)(), 
-			void (ObjectType::*setter)(ManagedDataBlock) = nullptr, UINT64 flags = 0, UINT8* (customAllocator)(ObjectType*, UINT32) = 0)
+		void addDataBlockField(const String& name, UINT32 uniqueId, SPtr<DataStream> (ObjectType::*getter)(UINT32&), 
+			void (ObjectType::*setter)(const SPtr<DataStream>&, UINT32) = nullptr, UINT64 flags = 0)
 		{
 			addDataBlockField<ObjectType>(name, uniqueId, 
-				std::function<ManagedDataBlock(ObjectType*)>(getter),  
-				std::function<void(ObjectType*, ManagedDataBlock)>(setter), flags, customAllocator);
+				std::function<SPtr<DataStream>(ObjectType*, UINT32&)>(getter),
+				std::function<void(ObjectType*, const SPtr<DataStream>&, UINT32)>(setter), flags);
 		}	
 
 	protected:
@@ -1050,27 +1049,14 @@ namespace BansheeEngine
 		}
 
 		template<class InterfaceType, class ObjectType>
-		void addDataBlockField(const String& name, UINT32 uniqueId, ManagedDataBlock (InterfaceType::*getter)(ObjectType*), 
-			void (InterfaceType::*setter)(ObjectType*, ManagedDataBlock), UINT64 flags = 0, 
-			UINT8* (customAllocator)(ObjectType*, UINT32) = 0)
+		void addDataBlockField(const String& name, UINT32 uniqueId, SPtr<DataStream> (InterfaceType::*getter)(ObjectType*, UINT32&), 
+			void (InterfaceType::*setter)(ObjectType*, const SPtr<DataStream>&, UINT32), UINT64 flags = 0)
 		{
 			using namespace std::placeholders;
 
-			if(customAllocator != 0)
-			{
-				std::function<UINT8*(ObjectType*, UINT32)> customAllocFunc = std::bind(customAllocator, _1, _2);
-
-				addDataBlockField<ObjectType>(name, uniqueId, 
-					std::function<ManagedDataBlock(ObjectType*)>(std::bind(getter, static_cast<InterfaceType*>(this), _1)),  
-					std::function<void(ObjectType*, ManagedDataBlock)>(std::bind(setter, static_cast<InterfaceType*>(this), _1, _2)), flags, 
-					customAllocFunc);
-			}
-			else
-			{
-				addDataBlockField<ObjectType>(name, uniqueId, 
-					std::function<ManagedDataBlock(ObjectType*)>(std::bind(getter, static_cast<InterfaceType*>(this), _1)),  
-					std::function<void(ObjectType*, ManagedDataBlock)>(std::bind(setter, static_cast<InterfaceType*>(this), _1, _2)), flags);
-			}
+			addDataBlockField<ObjectType>(name, uniqueId, 
+				std::function<SPtr<DataStream>(ObjectType*, UINT32&)>(std::bind(getter, static_cast<InterfaceType*>(this), _1, _2)),
+				std::function<void(ObjectType*, const SPtr<DataStream>&, UINT32)>(std::bind(setter, static_cast<InterfaceType*>(this), _1, _2, _3)), flags);
 		}	
 
 	private:
@@ -1144,12 +1130,11 @@ namespace BansheeEngine
 		}
 
 		template<class ObjectType>
-		void addDataBlockField(const String& name, UINT32 uniqueId, Any getter, Any setter, UINT64 flags, 
-			Any customAllocator = Any())
+		void addDataBlockField(const String& name, UINT32 uniqueId, Any getter, Any setter, UINT64 flags)
 		{
-			RTTIManagedDataBlockField<ManagedDataBlock, ObjectType>* newField = 
-				bs_new<RTTIManagedDataBlockField<ManagedDataBlock, ObjectType>>();
-			newField->initSingle(name, uniqueId, getter,  setter, flags, customAllocator);
+			RTTIManagedDataBlockField<UINT8*, ObjectType>* newField = 
+				bs_new<RTTIManagedDataBlockField<UINT8*, ObjectType>>();
+			newField->initSingle(name, uniqueId, getter, setter, flags);
 			addNewField(newField);
 		}	
 	};

+ 32 - 7
Source/BansheeUtility/Include/BsSerializedObject.h

@@ -32,7 +32,7 @@ namespace BansheeEngine
 	public:
 		friend class SerializedInstanceRTTI;
 		static RTTITypeBase* getRTTIStatic();
-		virtual RTTITypeBase* getRTTI() const override;
+		RTTITypeBase* getRTTI() const override;
 	};
 
 	/** An intermediate serialized data for a single field in an object. */
@@ -51,7 +51,7 @@ namespace BansheeEngine
 	public:
 		friend class SerializedEntryRTTI;
 		static RTTITypeBase* getRTTIStatic();
-		virtual RTTITypeBase* getRTTI() const override;
+		RTTITypeBase* getRTTI() const override;
 	};
 
 	/** A serialized value representing a single entry in an array. */
@@ -70,7 +70,7 @@ namespace BansheeEngine
 	public:
 		friend class SerializedArrayEntryRTTI;
 		static RTTITypeBase* getRTTIStatic();
-		virtual RTTITypeBase* getRTTI() const override;
+		RTTITypeBase* getRTTI() const override;
 	};
 
 	/**
@@ -92,7 +92,7 @@ namespace BansheeEngine
 	public:
 		friend class SerializedSubObjectRTTI;
 		static RTTITypeBase* getRTTIStatic();
-		virtual RTTITypeBase* getRTTI() const override;
+		RTTITypeBase* getRTTI() const override;
 	};
 
 	/** A serialized object consisting of multiple sub-objects, one for each inherited class. */
@@ -112,7 +112,7 @@ namespace BansheeEngine
 	public:
 		friend class SerializedObjectRTTI;
 		static RTTITypeBase* getRTTIStatic();
-		virtual RTTITypeBase* getRTTI() const override;
+		RTTITypeBase* getRTTI() const override;
 	};
 
 	/** Contains data for a serialized value of a specific field or array entry. */
@@ -143,7 +143,32 @@ namespace BansheeEngine
 	public:
 		friend class SerializedFieldRTTI;
 		static RTTITypeBase* getRTTIStatic();
-		virtual RTTITypeBase* getRTTI() const override;
+		RTTITypeBase* getRTTI() const override;
+	};
+
+	/** Contains data for a serialized value of a data block field. */
+	struct BS_UTILITY_EXPORT SerializedDataBlock : SerializedInstance
+	{
+		SerializedDataBlock()
+			:offset(0), size(0)
+		{
+
+		}
+
+		/** @copydoc SerializedInstance::clone */
+		SPtr<SerializedInstance> clone(bool cloneData = true) override;
+
+		SPtr<DataStream> stream;
+		UINT32 offset;
+		UINT32 size;
+
+		/************************************************************************/
+		/* 								RTTI		                     		*/
+		/************************************************************************/
+	public:
+		friend class SerializedDataBlockRTTI;
+		static RTTITypeBase* getRTTIStatic();
+		RTTITypeBase* getRTTI() const override;
 	};
 
 	/** A serialized array containing a list of all its entries. */
@@ -165,7 +190,7 @@ namespace BansheeEngine
 	public:
 		friend class SerializedArrayRTTI;
 		static RTTITypeBase* getRTTIStatic();
-		virtual RTTITypeBase* getRTTI() const override;
+		RTTITypeBase* getRTTI() const override;
 	};
 
 	/** @} */

+ 63 - 22
Source/BansheeUtility/Include/BsSerializedObjectRTTI.h

@@ -5,6 +5,7 @@
 #include "BsPrerequisitesUtil.h"
 #include "BsRTTIType.h"
 #include "BsSerializedObject.h"
+#include "BsDataStream.h"
 
 namespace BansheeEngine
 {
@@ -19,18 +20,18 @@ namespace BansheeEngine
 		SerializedInstanceRTTI()
 		{ }
 
-		virtual const String& getRTTIName() override
+		const String& getRTTIName() override
 		{
 			static String name = "SerializedInstance";
 			return name;
 		}
 
-		virtual UINT32 getRTTIId() override
+		UINT32 getRTTIId() override
 		{
 			return TID_SerializedInstance;
 		}
 
-		virtual SPtr<IReflectable> newRTTIObject() override
+		SPtr<IReflectable> newRTTIObject() override
 		{
 			return nullptr;
 		}
@@ -39,49 +40,89 @@ namespace BansheeEngine
 	class BS_UTILITY_EXPORT SerializedFieldRTTI : public RTTIType <SerializedField, SerializedInstance, SerializedFieldRTTI>
 	{
 	private:
-		ManagedDataBlock getData(SerializedField* obj)
+		SPtr<DataStream> getData(SerializedField* obj, UINT32& size)
 		{
-			ManagedDataBlock dataBlock((UINT8*)obj->value, obj->size);
-			return dataBlock;
-		}
+			size = obj->size;
 
-		void setData(SerializedField* obj, ManagedDataBlock val)
-		{
-			// Nothing to do here, the pointer we provided already belongs to SerializedField
-			// so the data is already written
+			return bs_shared_ptr_new<MemoryDataStream>(obj->value, obj->size, false);
 		}
 
-		static UINT8* allocateData(SerializedField* obj, UINT32 numBytes)
+		void setData(SerializedField* obj, const SPtr<DataStream>& value, UINT32 size)
 		{
-			obj->value = (UINT8*)bs_alloc(numBytes);
-			obj->size = numBytes;
+			obj->value = (UINT8*)bs_alloc(size);
+			obj->size = size;
 			obj->ownsMemory = true;
 
-			return obj->value;
+			value->read(obj->value, size);
 		}
+
 	public:
 		SerializedFieldRTTI()
 		{
-			addDataBlockField("data", 0, &SerializedFieldRTTI::getData, &SerializedFieldRTTI::setData, 0, &SerializedFieldRTTI::allocateData);
+			addDataBlockField("data", 0, &SerializedFieldRTTI::getData, &SerializedFieldRTTI::setData, 0);
 		}
 
-		virtual const String& getRTTIName() override
+		const String& getRTTIName() override
 		{
 			static String name = "SerializedField";
 			return name;
 		}
 
-		virtual UINT32 getRTTIId() override
+		UINT32 getRTTIId() override
 		{
 			return TID_SerializedField;
 		}
 
-		virtual SPtr<IReflectable> newRTTIObject() override
+		SPtr<IReflectable> newRTTIObject() override
 		{
 			return bs_shared_ptr_new<SerializedField>();
 		}
 	};
 
+	class BS_UTILITY_EXPORT SerializedDataBlockRTTI : public RTTIType <SerializedDataBlock, SerializedInstance, SerializedDataBlockRTTI>
+	{
+	private:
+		SPtr<DataStream> getData(SerializedDataBlock* obj, UINT32& size)
+		{
+			size = obj->size;
+			obj->stream->seek(obj->offset);
+
+			return obj->stream;
+		}
+
+		void setData(SerializedDataBlock* obj, const SPtr<DataStream>& value, UINT32 size)
+		{
+			UINT8* data = (UINT8*)bs_alloc(size);
+			SPtr<MemoryDataStream> memStream = bs_shared_ptr_new<MemoryDataStream>(data, size);
+			value->read(data, size);
+
+			obj->stream = memStream;
+			obj->size = size;
+			obj->offset = 0;
+		}
+	public:
+		SerializedDataBlockRTTI()
+		{
+			addDataBlockField("data", 0, &SerializedDataBlockRTTI::getData, &SerializedDataBlockRTTI::setData, 0);
+		}
+
+		const String& getRTTIName() override
+		{
+			static String name = "SerializedDataBlock";
+			return name;
+		}
+
+		UINT32 getRTTIId() override
+		{
+			return TID_SerializedDataBlock;
+		}
+
+		SPtr<IReflectable> newRTTIObject() override
+		{
+			return bs_shared_ptr_new<SerializedDataBlock>();
+		}
+	};
+
 	class BS_UTILITY_EXPORT SerializedObjectRTTI : public RTTIType <SerializedObject, SerializedInstance, SerializedObjectRTTI>
 	{
 	private:
@@ -111,18 +152,18 @@ namespace BansheeEngine
 				&SerializedObjectRTTI::setEntry, &SerializedObjectRTTI::setNumEntries);
 		}
 
-		virtual const String& getRTTIName() override
+		const String& getRTTIName() override
 		{
 			static String name = "SerializedObject";
 			return name;
 		}
 
-		virtual UINT32 getRTTIId() override
+		UINT32 getRTTIId() override
 		{
 			return TID_SerializedObject;
 		}
 
-		virtual SPtr<IReflectable> newRTTIObject() override
+		SPtr<IReflectable> newRTTIObject() override
 		{
 			return bs_shared_ptr_new<SerializedObject>();
 		}

+ 55 - 9
Source/BansheeUtility/Source/BsBinaryDiff.cpp

@@ -5,6 +5,7 @@
 #include "BsBinarySerializer.h"
 #include "BsBinaryCloner.h"
 #include "BsRTTIType.h"
+#include "BsDataStream.h"
 
 namespace BansheeEngine
 {
@@ -51,7 +52,6 @@ namespace BansheeEngine
 		}
 			break;
 		case SerializableFT_Plain:
-		case SerializableFT_DataBlock:
 		{
 			SPtr<SerializedField> orgFieldData = std::static_pointer_cast<SerializedField>(orgData);
 			SPtr<SerializedField> newFieldData = std::static_pointer_cast<SerializedField>(newData);
@@ -64,6 +64,53 @@ namespace BansheeEngine
 				modification = newFieldData->clone();
 		}
 			break;
+		case SerializableFT_DataBlock:
+		{
+			SPtr<SerializedDataBlock> orgFieldData = std::static_pointer_cast<SerializedDataBlock>(orgData);
+			SPtr<SerializedDataBlock> newFieldData = std::static_pointer_cast<SerializedDataBlock>(newData);
+
+			bool isModified = orgFieldData->size != newFieldData->size;
+			if (!isModified)
+			{
+				UINT8* orgStreamData = nullptr;
+				if(orgFieldData->stream->isFile())
+				{
+					orgStreamData = (UINT8*)bs_stack_alloc(orgFieldData->size);
+					orgFieldData->stream->seek(orgFieldData->offset);
+					orgFieldData->stream->read(orgStreamData, orgFieldData->size);
+				}
+				else
+				{
+					SPtr<MemoryDataStream> orgMemStream = std::static_pointer_cast<MemoryDataStream>(orgFieldData->stream);
+					orgStreamData = orgMemStream->getCurrentPtr();
+				}
+
+				UINT8* newStreamData = nullptr;
+				if (newFieldData->stream->isFile())
+				{
+					newStreamData = (UINT8*)bs_stack_alloc(newFieldData->size);
+					newFieldData->stream->seek(newFieldData->offset);
+					newFieldData->stream->read(newStreamData, newFieldData->size);
+				}
+				else
+				{
+					SPtr<MemoryDataStream> newMemStream = std::static_pointer_cast<MemoryDataStream>(newFieldData->stream);
+					newStreamData = newMemStream->getCurrentPtr();
+				}
+
+				isModified = memcmp(orgStreamData, newStreamData, newFieldData->size) != 0;
+
+				if (newFieldData->stream->isFile())
+					bs_stack_free(newStreamData);
+
+				if (orgFieldData->stream->isFile())
+					bs_stack_free(orgStreamData);
+			}
+
+			if (isModified)
+				modification = newFieldData->clone();
+		}
+		break;
 		}
 
 		return modification;
@@ -181,11 +228,7 @@ namespace BansheeEngine
 				case Diff_DataBlock:
 				{
 					RTTIManagedDataBlockFieldBase* field = static_cast<RTTIManagedDataBlockFieldBase*>(command.field);
-					UINT8* dataCopy = field->allocate(destObject, command.size);
-					memcpy(dataCopy, command.value, command.size);
-
-					ManagedDataBlock value(dataCopy, command.size); // Not managed because I assume the owner class will decide whether to delete the data or keep it
-					field->setValue(destObject, value);
+					field->setValue(destObject, command.streamValue, command.size);
 				}
 					break;
 				default:
@@ -302,9 +345,11 @@ namespace BansheeEngine
 						switch (genericField->mType)
 						{
 						case SerializableFT_Plain:
-						case SerializableFT_DataBlock:
 							modification = bs_shared_ptr_new<SerializedField>();
 							break;
+						case SerializableFT_DataBlock:
+							modification = bs_shared_ptr_new<SerializedDataBlock>();
+							break;
 						default:
 							break;
 						}
@@ -606,12 +651,13 @@ namespace BansheeEngine
 						break;
 					case SerializableFT_DataBlock:
 					{
-						SPtr<SerializedField> diffFieldData = std::static_pointer_cast<SerializedField>(diffData);
+						SPtr<SerializedDataBlock> diffFieldData = std::static_pointer_cast<SerializedDataBlock>(diffData);
 
 						DiffCommand command;
 						command.field = genericField;
 						command.type = Diff_DataBlock;
-						command.value = diffFieldData->value;
+						command.streamValue = diffFieldData->stream;
+						command.value = nullptr;
 						command.size = diffFieldData->size;
 
 						diffCommands.push_back(command);

+ 153 - 153
Source/BansheeUtility/Source/BsBinarySerializer.cpp

@@ -12,6 +12,7 @@
 #include "BsRTTIReflectablePtrField.h"
 #include "BsRTTIManagedDataBlockField.h"
 #include "BsMemorySerializer.h"
+#include "BsDataStream.h"
 
 #include <unordered_set>
 
@@ -57,7 +58,7 @@ namespace BansheeEngine
 		UINT32 objectId = findOrCreatePersistentId(object);
 		
 		// Encode primary object and its value types
-		buffer = encodeInternal(object, objectId, buffer, bufferLength, bytesWritten, flushBufferCallback, shallow);
+		buffer = encodeEntry(object, objectId, buffer, bufferLength, bytesWritten, flushBufferCallback, shallow);
 		if(buffer == nullptr)
 		{
 			BS_EXCEPT(InternalErrorException, 
@@ -81,7 +82,7 @@ namespace BansheeEngine
 				serializedObjects.insert(curObjectid);
 				mObjectsToEncode.erase(iter);
 
-				buffer = encodeInternal(curObject.get(), curObjectid, buffer, 
+				buffer = encodeEntry(curObject.get(), curObjectid, buffer, 
 					bufferLength, bytesWritten, flushBufferCallback, shallow);
 				if(buffer == nullptr)
 				{
@@ -118,19 +119,19 @@ namespace BansheeEngine
 		mObjectAddrToId.clear();
 	}
 
-	SPtr<IReflectable> BinarySerializer::decode(UINT8* data, UINT32 dataLength)
+	SPtr<IReflectable> BinarySerializer::decode(const SPtr<DataStream>& data, UINT32 dataLength)
 	{
 		if (dataLength == 0)
 			return nullptr;
 
-		SPtr<SerializedObject> intermediateObject = _decodeIntermediate(data, dataLength);
+		SPtr<SerializedObject> intermediateObject = _decodeToIntermediate(data, dataLength);
 		if (intermediateObject == nullptr)
 			return nullptr;
 
-		return _decodeIntermediate(intermediateObject);
+		return _decodeFromIntermediate(intermediateObject);
 	}
 
-	SPtr<IReflectable> BinarySerializer::_decodeIntermediate(const SPtr<SerializedObject>& serializedObject)
+	SPtr<IReflectable> BinarySerializer::_decodeFromIntermediate(const SPtr<SerializedObject>& serializedObject)
 	{
 		mObjectMap.clear();
 
@@ -142,7 +143,7 @@ namespace BansheeEngine
 			auto iterNewObj = mObjectMap.insert(std::make_pair(serializedObject, ObjectToDecode(output, serializedObject)));
 
 			iterNewObj.first->second.decodeInProgress = true;
-			decodeInternal(output, serializedObject);
+			decodeEntry(output, serializedObject);
 			iterNewObj.first->second.decodeInProgress = false;
 			iterNewObj.first->second.isDecoded = true;
 		}
@@ -156,7 +157,7 @@ namespace BansheeEngine
 				continue;
 
 			objToDecode.decodeInProgress = true;
-			decodeInternal(objToDecode.object, objToDecode.serializedObject);
+			decodeEntry(objToDecode.object, objToDecode.serializedObject);
 			objToDecode.decodeInProgress = false;
 			objToDecode.isDecoded = true;
 		}
@@ -165,7 +166,53 @@ namespace BansheeEngine
 		return output;
 	}
 
-	UINT8* BinarySerializer::encodeInternal(IReflectable* object, UINT32 objectId, UINT8* buffer, UINT32& bufferLength, 
+	SPtr<SerializedObject> BinarySerializer::_encodeToIntermediate(IReflectable* object, bool shallow)
+	{
+		// TODO: This is a hacky way of generating an intermediate format to save development time and complexity.
+		// It is hacky because it requires a full on encode to binary and then decode into intermediate. It should 
+		// be better to modify encoding process so it outputs the intermediate format directly (similar to how decoding works). 
+		// This also means that once you have an intermediate format you cannot use it to encode to binary. 
+
+		std::function<void*(UINT32)> allocator = &MemoryAllocator<GenAlloc>::allocate;
+
+		MemorySerializer ms;
+		UINT32 dataLength = 0;
+		UINT8* data = ms.encode(object, dataLength, allocator, shallow);
+
+		SPtr<MemoryDataStream> stream = bs_shared_ptr_new<MemoryDataStream>(data, dataLength);
+
+		BinarySerializer bs;
+		SPtr<SerializedObject> obj = bs._decodeToIntermediate(stream, dataLength, true);
+
+		return obj;
+	}
+
+	SPtr<SerializedObject> BinarySerializer::_decodeToIntermediate(const SPtr<DataStream>& data, UINT32 dataLength, bool copyData)
+	{
+		bool streamDataBlock = false;
+		if (!copyData && data->isFile())
+		{
+			LOGWRN("Deserializing without data copy but the source stream is a file stream. Enabling data copy.");
+
+			copyData = true;
+			streamDataBlock = true;
+		}
+
+		UINT32 bytesRead = 0;
+		mInterimObjectMap.clear();
+
+		SPtr<SerializedObject> rootObj;
+		bool hasMore = decodeEntry(data, dataLength, bytesRead, rootObj, copyData, streamDataBlock);
+		while (hasMore)
+		{
+			SPtr<SerializedObject> dummyObj;
+			hasMore = decodeEntry(data, dataLength, bytesRead, dummyObj, copyData, streamDataBlock);
+		}
+
+		return rootObj;
+	}
+
+	UINT8* BinarySerializer::encodeEntry(IReflectable* object, UINT32 objectId, UINT8* buffer, UINT32& bufferLength, 
 		UINT32* bytesWritten, std::function<UINT8*(UINT8*, UINT32, UINT32&)> flushBufferCallback, bool shallow)
 	{
 		RTTITypeBase* si = object->getRTTI();
@@ -253,14 +300,13 @@ namespace BansheeEngine
 									curField->arrayElemToBuffer(object, arrIdx, tempBuffer);
 
 									buffer = dataBlockToBuffer(tempBuffer, typeSize, buffer, bufferLength, bytesWritten, flushBufferCallback);
+									bs_stack_free(tempBuffer);
+
 									if (buffer == nullptr || bufferLength == 0)
 									{
-										bs_stack_free(tempBuffer);
 										si->onSerializationEnded(object);
 										return nullptr;
 									}
-
-									bs_stack_free(tempBuffer);
 								}
 								else
 								{
@@ -326,14 +372,13 @@ namespace BansheeEngine
 								curField->toBuffer(object, tempBuffer);
 								
 								buffer = dataBlockToBuffer(tempBuffer, typeSize, buffer, bufferLength, bytesWritten, flushBufferCallback);
+								bs_stack_free(tempBuffer);
+
 								if (buffer == nullptr || bufferLength == 0)
 								{
-									bs_stack_free(tempBuffer);
 									si->onSerializationEnded(object);
 									return nullptr;
 								}
-
-								bs_stack_free(tempBuffer);
 							}
 							else
 							{
@@ -347,16 +392,20 @@ namespace BansheeEngine
 					case SerializableFT_DataBlock:
 						{
 							RTTIManagedDataBlockFieldBase* curField = static_cast<RTTIManagedDataBlockFieldBase*>(curGenericField);
-							ManagedDataBlock value = curField->getValue(object);
+
+							UINT32 dataBlockSize = 0;
+							SPtr<DataStream> blockStream = curField->getValue(object, dataBlockSize);
 
 							// Data block size
-							UINT32 dataBlockSize = value.getSize();
 							COPY_TO_BUFFER(&dataBlockSize, sizeof(UINT32))
 
 							// Data block data
-							UINT8* dataToStore = value.getData();
+							UINT8* dataToStore = (UINT8*)bs_stack_alloc(dataBlockSize);
+							blockStream->read(dataToStore, dataBlockSize);
 
 							buffer = dataBlockToBuffer(dataToStore, dataBlockSize, buffer, bufferLength, bytesWritten, flushBufferCallback);
+							bs_stack_free(dataToStore);
+
 							if (buffer == nullptr || bufferLength == 0)
 							{
 								si->onSerializationEnded(object);
@@ -383,57 +432,18 @@ namespace BansheeEngine
 		return buffer;
 	}
 
-	SPtr<SerializedObject> BinarySerializer::_encodeIntermediate(IReflectable* object, bool shallow)
-	{
-		// TODO: This is a hacky way of generating an intermediate format to save development time and complexity.
-		// It is hacky because it requires a full on encode to binary and then decode into intermediate. It should 
-		// be better to modify encoding process so it outputs the intermediate format directly (similar to how decoding works). 
-		// This also means that once you have an intermediate format you cannot use it to encode to binary. 
-
-		std::function<void*(UINT32)> allocator = &MemoryAllocator<GenAlloc>::allocate;
-
-		MemorySerializer ms;
-		UINT32 dataLength = 0;
-		UINT8* data = ms.encode(object, dataLength, allocator, shallow);
-
-		BinarySerializer bs;
-		SPtr<SerializedObject> obj = bs._decodeIntermediate(data, dataLength, true);
-
-		bs_free(data);
-		return obj;
-	}
-
-	SPtr<SerializedObject> BinarySerializer::_decodeIntermediate(UINT8* data, UINT32 dataLength, bool copyData)
+	bool BinarySerializer::decodeEntry(const SPtr<DataStream>& data, UINT32 dataLength, UINT32& bytesRead,
+		SPtr<SerializedObject>& output, bool copyData, bool streamDataBlock)
 	{
-		UINT32 bytesRead = 0;
-		mInterimObjectMap.clear();
-
-		SPtr<SerializedObject> rootObj;
-		bool hasMore = decodeIntermediateInternal(data, dataLength, bytesRead, rootObj, copyData);
-		while (hasMore)
-		{
-			UINT8* dataPtr = data + bytesRead;
-
-			SPtr<SerializedObject> dummyObj;
-			hasMore = decodeIntermediateInternal(dataPtr, dataLength, bytesRead, dummyObj, copyData);
-		}
-
-		return rootObj;
-	}
+		ObjectMetaData objectMetaData;
+		objectMetaData.objectMeta = 0;
+		objectMetaData.typeId = 0;
 
-	bool BinarySerializer::decodeIntermediateInternal(UINT8* data, UINT32 dataLength, UINT32& bytesRead, SPtr<SerializedObject>& output, bool copyData)
-	{
-		if ((bytesRead + sizeof(ObjectMetaData)) > dataLength)
+		if(data->read(&objectMetaData, sizeof(ObjectMetaData)) != sizeof(ObjectMetaData))
 		{
-			BS_EXCEPT(InternalErrorException,
-				"Error decoding data.");
+			BS_EXCEPT(InternalErrorException, "Error decoding data.");
 		}
 
-		ObjectMetaData objectMetaData;
-		objectMetaData.objectMeta = 0;
-		objectMetaData.typeId = 0;
-		memcpy(&objectMetaData, data, sizeof(ObjectMetaData));
-		data += sizeof(ObjectMetaData);
 		bytesRead += sizeof(ObjectMetaData);
 
 		UINT32 objectId = 0;
@@ -475,27 +485,22 @@ namespace BansheeEngine
 		while (bytesRead < dataLength)
 		{
 			int metaData = -1;
-
-			if ((bytesRead + META_SIZE) > dataLength)
+			if(data->read(&metaData, META_SIZE) != META_SIZE)
 			{
-				BS_EXCEPT(InternalErrorException,
-					"Error decoding data.");
+				BS_EXCEPT(InternalErrorException, "Error decoding data.");
 			}
 
-			memcpy((void*)&metaData, data, META_SIZE);
-
 			if (isObjectMetaData(metaData)) // We've reached a new object or a base class of the current one
 			{
-				if ((bytesRead + sizeof(ObjectMetaData)) > dataLength)
-				{
-					BS_EXCEPT(InternalErrorException,
-						"Error decoding data.");
-				}
-
 				ObjectMetaData objMetaData;
 				objMetaData.objectMeta = 0;
 				objMetaData.typeId = 0;
-				memcpy(&objMetaData, data, sizeof(ObjectMetaData));
+
+				data->seek(data->tell() - META_SIZE);
+				if (data->read(&objMetaData, sizeof(ObjectMetaData)) != sizeof(ObjectMetaData))
+				{
+					BS_EXCEPT(InternalErrorException, "Error decoding data.");
+				}
 
 				UINT32 objId = 0;
 				UINT32 objTypeId = 0;
@@ -522,18 +527,17 @@ namespace BansheeEngine
 						serializedSubObject->typeId = objTypeId;
 					}
 
-					data += sizeof(ObjectMetaData);
 					bytesRead += sizeof(ObjectMetaData);
 					continue;
 				}
 				else
 				{
 					// Found new object, we're done
+					data->seek(data->tell() - sizeof(ObjectMetaData));
 					return true;
 				}
 			}
 
-			data += META_SIZE;
 			bytesRead += META_SIZE;
 
 			bool isArray;
@@ -585,14 +589,11 @@ namespace BansheeEngine
 			int arrayNumElems = 1;
 			if (isArray)
 			{
-				if ((bytesRead + NUM_ELEM_FIELD_SIZE) > dataLength)
+				if(data->read(&arrayNumElems, NUM_ELEM_FIELD_SIZE) != NUM_ELEM_FIELD_SIZE)
 				{
-					BS_EXCEPT(InternalErrorException,
-						"Error decoding data.");
+					BS_EXCEPT(InternalErrorException, "Error decoding data.");
 				}
 
-				memcpy((void*)&arrayNumElems, data, NUM_ELEM_FIELD_SIZE);
-				data += NUM_ELEM_FIELD_SIZE;
 				bytesRead += NUM_ELEM_FIELD_SIZE;
 
 				SPtr<SerializedArray> serializedArray;
@@ -613,15 +614,12 @@ namespace BansheeEngine
 
 					for (int i = 0; i < arrayNumElems; i++)
 					{
-						if ((bytesRead + COMPLEX_TYPE_FIELD_SIZE) > dataLength)
+						int childObjectId = 0;
+						if(data->read(&childObjectId, COMPLEX_TYPE_FIELD_SIZE) != COMPLEX_TYPE_FIELD_SIZE)
 						{
-							BS_EXCEPT(InternalErrorException,
-								"Error decoding data.");
+							BS_EXCEPT(InternalErrorException, "Error decoding data.");
 						}
 
-						int childObjectId = 0;
-						memcpy(&childObjectId, data, COMPLEX_TYPE_FIELD_SIZE);
-						data += COMPLEX_TYPE_FIELD_SIZE;
 						bytesRead += COMPLEX_TYPE_FIELD_SIZE;
 
 						if (curField != nullptr)
@@ -658,18 +656,14 @@ namespace BansheeEngine
 					{
 						if (curField != nullptr)
 						{
-							UINT32 bytesReadStart = bytesRead;
 							SPtr<SerializedObject> serializedArrayEntry;
-							decodeIntermediateInternal(data, dataLength, bytesRead, serializedArrayEntry, copyData);
+							decodeEntry(data, dataLength, bytesRead, serializedArrayEntry, copyData, streamDataBlock);
 
 							SerializedArrayEntry arrayEntry;
 							arrayEntry.serialized = serializedArrayEntry;
 							arrayEntry.index = i;
 
 							serializedArray->entries[i] = arrayEntry;
-
-							UINT32 complexTypeSize = bytesRead - bytesReadStart;
-							data += complexTypeSize;
 						}
 					}
 					break;
@@ -682,7 +676,10 @@ namespace BansheeEngine
 					{
 						UINT32 typeSize = fieldSize;
 						if (hasDynamicSize)
-							memcpy(&typeSize, data, sizeof(UINT32));
+						{
+							data->read(&typeSize, sizeof(UINT32));
+							data->seek(data->tell() - sizeof(UINT32));
+						}
 
 						if (curField != nullptr)
 						{
@@ -691,11 +688,17 @@ namespace BansheeEngine
 							if (copyData)
 							{
 								serializedField->value = (UINT8*)bs_alloc(typeSize);
-								memcpy(serializedField->value, data, typeSize);
+								data->read(serializedField->value, typeSize);
+
 								serializedField->ownsMemory = true;
 							}
-							else
-								serializedField->value = data;
+							else // Guaranteed not to be a file stream, as we check earlier
+							{
+								SPtr<MemoryDataStream> memStream = std::static_pointer_cast<MemoryDataStream>(data);
+								serializedField->value = memStream->getCurrentPtr();
+
+								data->skip(typeSize);
+							}
 
 							serializedField->size = typeSize;
 
@@ -705,8 +708,9 @@ namespace BansheeEngine
 
 							serializedArray->entries[i] = arrayEntry;
 						}
+						else
+							data->skip(typeSize);
 
-						data += typeSize;
 						bytesRead += typeSize;
 					}
 					break;
@@ -725,15 +729,12 @@ namespace BansheeEngine
 				{
 					RTTIReflectablePtrFieldBase* curField = static_cast<RTTIReflectablePtrFieldBase*>(curGenericField);
 
-					if ((bytesRead + COMPLEX_TYPE_FIELD_SIZE) > dataLength)
+					int childObjectId = 0;
+					if(data->read(&childObjectId, COMPLEX_TYPE_FIELD_SIZE) != COMPLEX_TYPE_FIELD_SIZE)
 					{
-						BS_EXCEPT(InternalErrorException,
-							"Error decoding data.");
+						BS_EXCEPT(InternalErrorException, "Error decoding data.");
 					}
 
-					int childObjectId = 0;
-					memcpy(&childObjectId, data, COMPLEX_TYPE_FIELD_SIZE);
-					data += COMPLEX_TYPE_FIELD_SIZE;
 					bytesRead += COMPLEX_TYPE_FIELD_SIZE;
 
 					if (curField != nullptr)
@@ -764,15 +765,11 @@ namespace BansheeEngine
 
 					if (curField != nullptr)
 					{
-						UINT32 bytesReadStart = bytesRead;
 						SPtr<SerializedObject> serializedChildObj;
-						decodeIntermediateInternal(data, dataLength, bytesRead, serializedChildObj, copyData);
+						decodeEntry(data, dataLength, bytesRead, serializedChildObj, copyData, streamDataBlock);
 
 						serializedEntry = serializedChildObj;
 						hasModification = true;
-
-						UINT32 complexTypeSize = bytesRead - bytesReadStart;
-						data += complexTypeSize;
 					}
 
 					break;
@@ -783,28 +780,35 @@ namespace BansheeEngine
 
 					UINT32 typeSize = fieldSize;
 					if (hasDynamicSize)
-						memcpy(&typeSize, data, sizeof(UINT32));
+					{
+						data->read(&typeSize, sizeof(UINT32));
+						data->seek(data->tell() - sizeof(UINT32));
+					}
 
 					if (curField != nullptr)
 					{
 						SPtr<SerializedField> serializedField = bs_shared_ptr_new<SerializedField>();
-
 						if (copyData)
 						{
 							serializedField->value = (UINT8*)bs_alloc(typeSize);
-							memcpy(serializedField->value, data, typeSize);
+							data->read(serializedField->value, typeSize);
+
 							serializedField->ownsMemory = true;
 						}
-						else
-							serializedField->value = data;
+						else // Guaranteed not to be a file stream, as we check earlier
+						{
+							SPtr<MemoryDataStream> memStream = std::static_pointer_cast<MemoryDataStream>(data);
+							serializedField->value = memStream->getCurrentPtr();
 
-						serializedField->size = typeSize;
+							data->skip(typeSize);
+						}
 
 						serializedEntry = serializedField;
 						hasModification = true;
 					}
+					else
+						data->skip(typeSize);
 
-					data += typeSize;
 					bytesRead += typeSize;
 					break;
 				}
@@ -812,45 +816,44 @@ namespace BansheeEngine
 				{
 					RTTIManagedDataBlockFieldBase* curField = static_cast<RTTIManagedDataBlockFieldBase*>(curGenericField);
 
-					if ((bytesRead + DATA_BLOCK_TYPE_FIELD_SIZE) > dataLength)
-					{
-						BS_EXCEPT(InternalErrorException,
-							"Error decoding data.");
-					}
-
 					// Data block size
 					UINT32 dataBlockSize = 0;
-					memcpy(&dataBlockSize, data, DATA_BLOCK_TYPE_FIELD_SIZE);
-					data += DATA_BLOCK_TYPE_FIELD_SIZE;
-					bytesRead += DATA_BLOCK_TYPE_FIELD_SIZE;
-
-					if ((bytesRead + dataBlockSize) > dataLength)
+					if(data->read(&dataBlockSize, DATA_BLOCK_TYPE_FIELD_SIZE) != DATA_BLOCK_TYPE_FIELD_SIZE)
 					{
-						BS_EXCEPT(InternalErrorException,
-							"Error decoding data.");
+						BS_EXCEPT(InternalErrorException, "Error decoding data.");
 					}
 
+					bytesRead += DATA_BLOCK_TYPE_FIELD_SIZE;
+
 					// Data block data
 					if (curField != nullptr)
 					{
-						SPtr<SerializedField> serializedField = bs_shared_ptr_new<SerializedField>();
+						SPtr<SerializedDataBlock> serializedDataBlock = bs_shared_ptr_new<SerializedDataBlock>();
 
-						if (copyData)
+						if (streamDataBlock || !copyData)
 						{
-							serializedField->value = (UINT8*)bs_alloc(dataBlockSize);
-							memcpy(serializedField->value, data, dataBlockSize);
-							serializedField->ownsMemory = true;
+							serializedDataBlock->stream = data;
+							serializedDataBlock->offset = (UINT32)data->tell();
+
+							data->skip(dataBlockSize);
 						}
 						else
-							serializedField->value = data;
-
-						serializedField->size = dataBlockSize;
+						{
+							UINT8* dataBlockBuffer = (UINT8*)bs_alloc(dataBlockSize);
+							data->read(dataBlockBuffer, dataBlockSize);
 
-						serializedEntry = serializedField;
+							SPtr<DataStream> stream = bs_shared_ptr_new<MemoryDataStream>(dataBlockBuffer, dataBlockSize);
+							serializedDataBlock->stream = stream;
+							serializedDataBlock->offset = 0;
+						}
+						serializedDataBlock->size = dataBlockSize;
+						
+						serializedEntry = serializedDataBlock;
 						hasModification = true;
 					}
+					else
+						data->skip(dataBlockSize);
 
-					data += dataBlockSize;
 					bytesRead += dataBlockSize;
 
 					break;
@@ -875,7 +878,7 @@ namespace BansheeEngine
 		return false;
 	}
 
-	void BinarySerializer::decodeInternal(const SPtr<IReflectable>& object, const SPtr<SerializedObject>& serializableObject)
+	void BinarySerializer::decodeEntry(const SPtr<IReflectable>& object, const SPtr<SerializedObject>& serializableObject)
 	{
 		UINT32 numSubObjects = (UINT32)serializableObject->subObjects.size();
 
@@ -947,7 +950,7 @@ namespace BansheeEngine
 									else
 									{
 										objToDecode.decodeInProgress = true;
-										decodeInternal(objToDecode.object, objToDecode.serializedObject);
+										decodeEntry(objToDecode.object, objToDecode.serializedObject);
 										objToDecode.decodeInProgress = false;
 										objToDecode.isDecoded = true;
 									}
@@ -977,7 +980,7 @@ namespace BansheeEngine
 							if (childRtti != nullptr)
 							{
 								SPtr<IReflectable> newObject = childRtti->newRTTIObject();
-								decodeInternal(newObject, arrayElemData);
+								decodeEntry(newObject, arrayElemData);
 								curField->setArrayValue(object.get(), arrayElem.first, *newObject);
 							}
 						}
@@ -1040,7 +1043,7 @@ namespace BansheeEngine
 								else
 								{
 									objToDecode.decodeInProgress = true;
-									decodeInternal(objToDecode.object, objToDecode.serializedObject);
+									decodeEntry(objToDecode.object, objToDecode.serializedObject);
 									objToDecode.decodeInProgress = false;
 									objToDecode.isDecoded = true;
 								}
@@ -1067,7 +1070,7 @@ namespace BansheeEngine
 						if (childRtti != nullptr)
 						{
 							SPtr<IReflectable> newObject = childRtti->newRTTIObject();
-							decodeInternal(newObject, fieldObjectData);
+							decodeEntry(newObject, fieldObjectData);
 							curField->setValue(object.get(), *newObject);
 						}
 						break;
@@ -1087,14 +1090,11 @@ namespace BansheeEngine
 					{
 						RTTIManagedDataBlockFieldBase* curField = static_cast<RTTIManagedDataBlockFieldBase*>(curGenericField);
 
-						SPtr<SerializedField> fieldData = std::static_pointer_cast<SerializedField>(entryData);
+						SPtr<SerializedDataBlock> fieldData = std::static_pointer_cast<SerializedDataBlock>(entryData);
 						if (fieldData != nullptr)
 						{
-							UINT8* dataCopy = curField->allocate(object.get(), fieldData->size); // TODO - Low priority. I need to read files better, so I
-							memcpy(dataCopy, fieldData->value, fieldData->size);		//    can just pass the buffer pointer directly without copying (possibly large amounts of data)
-
-							ManagedDataBlock value(dataCopy, fieldData->size); // Not managed because I assume the owner class will decide whether to delete the data or keep it
-							curField->setValue(object.get(), value);
+							fieldData->stream->seek(fieldData->offset);
+							curField->setValue(object.get(), fieldData->stream, fieldData->size);
 						}
 
 						break;
@@ -1202,7 +1202,7 @@ namespace BansheeEngine
 	{
 		if (object != nullptr)
 		{
-			buffer = encodeInternal(object, 0, buffer, bufferLength, bytesWritten, flushBufferCallback, shallow);
+			buffer = encodeEntry(object, 0, buffer, bufferLength, bytesWritten, flushBufferCallback, shallow);
 
 			// Encode terminator field
 			// Complex types require terminator fields because they can be embedded within other complex types and we need

+ 7 - 3
Source/BansheeUtility/Source/BsDataStream.cpp

@@ -245,8 +245,8 @@ namespace BansheeEngine
 		}
 	}
 
-    MemoryDataStream::MemoryDataStream(void* memory, size_t inSize)
-		: DataStream(READ | WRITE), mData(nullptr)
+    MemoryDataStream::MemoryDataStream(void* memory, size_t inSize, bool freeOnClose)
+		: DataStream(READ | WRITE), mData(nullptr), mFreeOnClose(freeOnClose)
     {
         mData = mPos = static_cast<UINT8*>(memory);
         mSize = inSize;
@@ -264,6 +264,7 @@ namespace BansheeEngine
 		mData = (UINT8*)bs_alloc((UINT32)mSize);
 		mPos = mData;
 		mEnd = mData + sourceStream.read(mData, mSize);
+		mFreeOnClose = true;
 
         assert(mEnd >= mPos);
     }
@@ -277,6 +278,7 @@ namespace BansheeEngine
 		mData = (UINT8*)bs_alloc((UINT32)mSize);
 		mPos = mData;
 		mEnd = mData + sourceStream->read(mData, mSize);
+		mFreeOnClose = true;
 
         assert(mEnd >= mPos);
     }
@@ -350,7 +352,9 @@ namespace BansheeEngine
     {
         if (mData != nullptr)
         {
-            bs_free(mData);
+			if(mFreeOnClose)
+				bs_free(mData);
+
             mData = nullptr;
         }
     }

+ 11 - 27
Source/BansheeUtility/Source/BsFileSerializer.cpp

@@ -6,6 +6,7 @@
 #include "BsIReflectable.h"
 #include "BsBinarySerializer.h"
 #include "BsFileSystem.h"
+#include "BsDataStream.h"
 #include "BsDebug.h"
 #include <numeric>
 
@@ -63,56 +64,39 @@ namespace BansheeEngine
 
 	FileDecoder::FileDecoder(const Path& fileLocation)
 	{
-		mInputStream.open(fileLocation.toWString().c_str(), std::ios::in | std::ios::ate | std::ios::binary);
+		mInputStream = FileSystem::openFile(fileLocation, true);
 
-		if (mInputStream.fail())
-		{
-			LOGWRN("Failed to open file: \"" + fileLocation.toString() + "\". Error: " + strerror(errno) + ".");
-		}
+		if (mInputStream == nullptr)
+			return;
 
-		std::streamoff fileSize = mInputStream.tellg();
-		if (fileSize > std::numeric_limits<UINT32>::max())
+		if (mInputStream->size() > std::numeric_limits<UINT32>::max())
 		{
 			BS_EXCEPT(InternalErrorException,
 				"File size is larger that UINT32 can hold. Ask a programmer to use a bigger data type.");
 		}
-
-		mInputStream.seekg(0, std::ios::beg);
-	}
-
-	FileDecoder::~FileDecoder()
-	{
-		mInputStream.close();
-		mInputStream.clear();
 	}
 
 	SPtr<IReflectable> FileDecoder::decode()
 	{
-		if (mInputStream.fail() || mInputStream.eof())
+		if (mInputStream->eof())
 			return nullptr;
 
 		UINT32 objectSize = 0;
-		mInputStream.read((char*)&objectSize, sizeof(objectSize));
-
-		UINT8* readBuffer = (UINT8*)bs_alloc((UINT32)objectSize); // TODO - Low priority. Consider upgrading BinarySerializer so we don't have to read everything at once
-		mInputStream.read((char*)readBuffer, objectSize);
+		mInputStream->read(&objectSize, sizeof(objectSize));
 
 		BinarySerializer bs;
-		SPtr<IReflectable> object = bs.decode(readBuffer, objectSize);
-
-		bs_free(readBuffer);
+		SPtr<IReflectable> object = bs.decode(mInputStream, objectSize);
 
 		return object;
 	}
 
 	void FileDecoder::skip()
 	{
-		if (mInputStream.eof())
+		if (mInputStream->eof())
 			return;
 
 		UINT32 objectSize = 0;
-		mInputStream.read((char*)&objectSize, sizeof(objectSize));
-
-		mInputStream.seekg(objectSize, std::ios::cur);
+		mInputStream->read(&objectSize, sizeof(objectSize));
+		mInputStream->skip(objectSize);
 	}
 }

+ 0 - 33
Source/BansheeUtility/Source/BsManagedDataBlock.cpp

@@ -1,33 +0,0 @@
-//********************************** Banshee Engine (www.banshee3d.com) **************************************************//
-//**************** Copyright (c) 2016 Marko Pintera ([email protected]). All rights reserved. **********************//
-#include "BsManagedDataBlock.h"
-#include "BsException.h"
-
-namespace BansheeEngine
-{
-	ManagedDataBlock::ManagedDataBlock(UINT8* data, UINT32 size)
-		:mData(data), mSize(size), mManaged(false), mIsDataOwner(true)
-	{ }
-
-	ManagedDataBlock::ManagedDataBlock(UINT32 size)
-		:mSize(size), mManaged(true), mIsDataOwner(true)
-	{
-		mData = (UINT8*)bs_alloc(size);
-	}
-
-	ManagedDataBlock::ManagedDataBlock(const ManagedDataBlock& source)
-	{
-		mData = source.mData;
-		mSize = source.mSize;
-		mManaged = source.mManaged;
-
-		mIsDataOwner = true;
-		source.mIsDataOwner = false;
-	}
-
-	ManagedDataBlock::~ManagedDataBlock()
-	{
-		if(mManaged && mIsDataOwner)
-			bs_free(mData);
-	}
-}

+ 4 - 1
Source/BansheeUtility/Source/BsMemorySerializer.cpp

@@ -5,6 +5,7 @@
 #include "BsException.h"
 #include "BsIReflectable.h"
 #include "BsBinarySerializer.h"
+#include "BsDataStream.h"
 
 using namespace std::placeholders;
 
@@ -60,8 +61,10 @@ namespace BansheeEngine
 
 	SPtr<IReflectable> MemorySerializer::decode(UINT8* buffer, UINT32 bufferSize)
 	{
+		SPtr<MemoryDataStream> stream = bs_shared_ptr_new<MemoryDataStream>(buffer, bufferSize, false);
+
 		BinarySerializer bs;
-		SPtr<IReflectable> object = bs.decode(buffer, (UINT32)bufferSize);
+		SPtr<IReflectable> object = bs.decode(stream, (UINT32)bufferSize);
 
 		return object;
 	}

+ 35 - 0
Source/BansheeUtility/Source/BsSerializedObject.cpp

@@ -25,6 +25,31 @@ namespace BansheeEngine
 		return copy;
 	}
 
+	SPtr<SerializedInstance> SerializedDataBlock::clone(bool cloneData)
+	{
+		SPtr<SerializedDataBlock> copy = bs_shared_ptr_new<SerializedDataBlock>();
+		copy->size = size;
+
+		if (cloneData)
+		{
+			if(stream->isFile())
+				LOGWRN("Cloning a file stream. Streaming is disabled and stream data will be loaded into memory.");
+
+			UINT8* data = (UINT8*)bs_alloc(size);
+			stream->read(data, size);
+
+			copy->stream = bs_shared_ptr_new<MemoryDataStream>(data, size);
+			copy->offset = 0;
+		}
+		else
+		{
+			copy->stream = stream;
+			copy->offset = offset;
+		}
+
+		return copy;
+	}
+
 	SPtr<SerializedInstance> SerializedObject::clone(bool cloneData)
 	{
 		SPtr<SerializedObject> copy = bs_shared_ptr_new<SerializedObject>();
@@ -77,6 +102,16 @@ namespace BansheeEngine
 		return SerializedInstance::getRTTIStatic();
 	}
 
+	RTTITypeBase* SerializedDataBlock::getRTTIStatic()
+	{
+		return SerializedDataBlockRTTI::instance();
+	}
+
+	RTTITypeBase* SerializedDataBlock::getRTTI() const
+	{
+		return SerializedDataBlock::getRTTIStatic();
+	}
+
 	RTTITypeBase* SerializedField::getRTTIStatic()
 	{
 		return SerializedFieldRTTI::instance();

+ 4 - 4
Source/SBansheeEngine/Source/BsManagedDiff.cpp

@@ -17,8 +17,8 @@ namespace BansheeEngine
 
 		// Need to call GameObjectManager because GameObject handles call it during deserialization, but we don't really need it
 		GameObjectManager::instance().startDeserialization();
-		SPtr<ManagedSerializableObject> orgObj = std::static_pointer_cast<ManagedSerializableObject>(bs._decodeIntermediate(orgSerzObj));
-		SPtr<ManagedSerializableObject> newObj = std::static_pointer_cast<ManagedSerializableObject>(bs._decodeIntermediate(newSerzObj));
+		SPtr<ManagedSerializableObject> orgObj = std::static_pointer_cast<ManagedSerializableObject>(bs._decodeFromIntermediate(orgSerzObj));
+		SPtr<ManagedSerializableObject> newObj = std::static_pointer_cast<ManagedSerializableObject>(bs._decodeFromIntermediate(newSerzObj));
 		GameObjectManager::instance().endDeserialization();
 
 		SPtr<ManagedSerializableDiff> diff = ManagedSerializableDiff::create(orgObj, newObj);
@@ -33,7 +33,7 @@ namespace BansheeEngine
 
 		SerializedEntry entry;
 		entry.fieldId = 0;
-		entry.serialized = bs._encodeIntermediate(diff.get());
+		entry.serialized = bs._encodeToIntermediate(diff.get());
 
 		subObject.entries[0] = entry;
 
@@ -46,7 +46,7 @@ namespace BansheeEngine
 		SPtr<SerializedObject> diffObj = std::static_pointer_cast<SerializedObject>(serzDiff->subObjects[0].entries[0].serialized);
 
 		BinarySerializer bs;
-		SPtr<ManagedSerializableDiff> diff = std::static_pointer_cast<ManagedSerializableDiff>(bs._decodeIntermediate(diffObj));
+		SPtr<ManagedSerializableDiff> diff = std::static_pointer_cast<ManagedSerializableDiff>(bs._decodeFromIntermediate(diffObj));
 		
 		if (diff != nullptr)
 		{