Materials properly serialize/deserialize textures

CamelotRenderer/Include/CmMaterialRTTI.h

@@ -113,6 +113,8 @@ namespace CamelotEngine
 			}
 		};
 
+		class MaterialParamsRTTI;
+
 		struct MaterialParams : public IReflectable
 		{
 			map<String, FloatParam>::type mFloatParams;
@@ -125,7 +127,7 @@ namespace CamelotEngine
 			/************************************************************************/
 
 		public:
-			friend class MaterialParamsRTTI;
+			friend class MaterialRTTI::MaterialParamsRTTI;
 
 			static RTTITypeBase* getRTTIStatic();
 			RTTITypeBase* getRTTI() const;
@@ -134,6 +136,12 @@ namespace CamelotEngine
 		class MaterialParamsRTTI : public RTTIType<MaterialParams, IReflectable, MaterialParamsRTTI>
 		{
 		private:
+			struct TempParams
+			{
+				vector<std::shared_ptr<FloatParamKVP>>::type mFloatParams;
+				vector<std::shared_ptr<TexParamKVP>>::type mTexParams;
+			};
+
 			std::shared_ptr<FloatParamKVP> getFloatParam(MaterialParams* obj, UINT32 idx)
 			{
 				UINT32 curIdx = 0;
@@ -152,7 +160,8 @@ namespace CamelotEngine
 
 			void setFloatParam(MaterialParams* obj, UINT32 idx, std::shared_ptr<FloatParamKVP> value)
 			{
-				obj->mFloatParams[value->mKey] = value->mValue;
+				TempParams* tempParams = boost::any_cast<TempParams*>(obj->mRTTIData);
+				tempParams->mFloatParams.push_back(value);
 			}
 
 			void setNumFloatParams(MaterialParams* obj, UINT32 size)
@@ -183,7 +192,8 @@ namespace CamelotEngine
 
 			void setTexParam(MaterialParams* obj, UINT32 idx, std::shared_ptr<TexParamKVP> value)
 			{
-				obj->mTextureParams[value->mKey] = value->mValue;
+				TempParams* tempParams = boost::any_cast<TempParams*>(obj->mRTTIData);
+				tempParams->mTexParams.push_back(value);
 			}
 
 			void setNumTexParams(MaterialParams* obj, UINT32 size)
@@ -225,7 +235,42 @@ namespace CamelotEngine
 					&MaterialParamsRTTI::setFloatParam, &MaterialParamsRTTI::setNumFloatParams);
 				addReflectablePtrArrayField("mTexParams", 1, &MaterialParamsRTTI::getTexParam, &MaterialParamsRTTI::getNumTexParams, 
 					&MaterialParamsRTTI::setTexParam, &MaterialParamsRTTI::setNumTexParams);
-				//addDataBlockField("mFloatBuffer", 2, &MaterialParamsRTTI::getFloatBuffer, &MaterialParamsRTTI::setFloatBuffer);
+				addDataBlockField("mFloatBuffer", 2, &MaterialParamsRTTI::getFloatBuffer, &MaterialParamsRTTI::setFloatBuffer);
+			}
+
+			virtual void onDeserializationStarted(IReflectable* obj)
+			{
+				MaterialParams* materialParams = static_cast<MaterialParams*>(obj);
+				materialParams->mRTTIData = new TempParams();
+			}
+
+			virtual void onDeserializationEnded(IReflectable* obj)
+			{
+				// TODO Low priority:
+				// Due to the way serialization is done I cannot guarantee "value"s key/value fields
+				// have been set, so I need to delay accessing them until the end of serialization.
+				// I'd really like to avoid this as its not intuitive to think in this way.
+
+				MaterialParams* materialParams = static_cast<MaterialParams*>(obj);
+
+				if(materialParams->mRTTIData.empty())
+					return;
+
+				TempParams* tempParams = boost::any_cast<TempParams*>(materialParams->mRTTIData);
+
+				for(auto iter = tempParams->mFloatParams.begin(); iter != tempParams->mFloatParams.end(); ++iter)
+				{
+					std::shared_ptr<FloatParamKVP> floatParam = *iter;
+					materialParams->mFloatParams[floatParam->mKey] = floatParam->mValue;
+				}
+
+				for(auto iter = tempParams->mTexParams.begin(); iter != tempParams->mTexParams.end(); ++iter)
+				{
+					std::shared_ptr<TexParamKVP> texParam = *iter;
+					materialParams->mTextureParams[texParam->mKey] = texParam->mValue;
+				}
+
+				materialParams->mRTTIData = nullptr;
 			}
 
 			virtual const String& getRTTIName()
@@ -265,7 +310,7 @@ namespace CamelotEngine
 
 		void setMaterialParams(Material* obj, std::shared_ptr<MaterialParams> value)
 		{
-			//obj->mRTTIData = value;
+			obj->mRTTIData = value;
 		}
 
 	public:

CamelotRenderer/Include/CmResourceRefRTTI.h

@@ -3,6 +3,7 @@
 #include "CmPrerequisites.h"
 #include "CmRTTIType.h"
 #include "CmResourceRef.h"
+#include "CmResources.h"
 
 namespace CamelotEngine
 {
@@ -13,14 +14,6 @@ namespace CamelotEngine
 		void setUUID(ResourceRefData* obj, String& uuid) 
 		{ 
 			obj->mUUID = uuid; 
-
-			if(uuid != "")
-			{
-				// TODO - I probably want to load the resource here
-				//   - Important: consider that user might just want to load the level meta data and not the actual resources
-				//     (Maybe he wants to stream them in as player goes through the world?)
-				//       - Although this should probably be handled on a higher level. Here we just load.
-			}
 		} 
 	public:
 		ResourceRefDataRTTI()
@@ -56,6 +49,21 @@ namespace CamelotEngine
 			addReflectablePtrField("mData", 0, &ResourceRefRTTI::getResData, &ResourceRefRTTI::setResData);
 		}
 
+		void onDeserializationEnded(IReflectable* obj)
+		{
+			ResourceRefBase* resourceRef = static_cast<ResourceRefBase*>(obj);
+
+			if(resourceRef->mData && resourceRef->mData->mUUID != "")
+			{
+				// NOTE: This will cause Resources::load to be called recursively with resources that contain other
+				// resources. This might cause problems. Keep this note here as a warning until I prove otherwise.
+				BaseResourceRef loadedResource = gResources().loadFromUUID(resourceRef->mData->mUUID);
+
+				if(loadedResource)
+					resourceRef->resolve(loadedResource.getInternalPtr());
+			}
+		}
+
 		virtual const String& getRTTIName()
 		{
 			static String name = "ResourceRefBase";

CamelotRenderer/Source/CmApplication.cpp

@@ -182,7 +182,7 @@ namespace CamelotEngine
 		mDbgTexture = static_resource_cast<Texture>(gResources().load("C:\\ExportTest.tex"));
 		mDbgMesh = static_resource_cast<Mesh>(gResources().load("C:\\ExportMesh.mesh"));
 
-		//mTestMaterial->setTexture("tex", mDbgTexture);
+		mTestMaterial->setTexture("tex", mDbgTexture);
 		gResources().create(mTestMaterial, "C:\\ExportMaterial.mat", true);
 
 		//if(!_CrtCheckMemory())
@@ -190,8 +190,7 @@ namespace CamelotEngine
 		//	assert(false);
 		//}
 
-		MaterialRef newMat = gResources().load("C:\\ExportMaterial.mat");
-//		mTestMaterial = gResources().load("C:\\ExportMaterial.mat");
+		mTestMaterial = gResources().load("C:\\ExportMaterial.mat");
 
 		//if(!_CrtCheckMemory())
 		//{

CamelotRenderer/Source/CmMaterialRTTI.cpp

@@ -99,63 +99,66 @@ namespace CamelotEngine
 	void MaterialRTTI::onDeserializationEnded(IReflectable* obj)
 	{
 		Material* material = static_cast<Material*>(obj);
-		//std::shared_ptr<MaterialParams> params = boost::any_cast<std::shared_ptr<MaterialParams>>(material->mRTTIData);
+		if(material->mRTTIData.empty())
+			return;
 
-		//vector<GpuProgramParametersPtr>::type allParams;
-		//for(size_t i = 0; i < material->mParameters.size(); i++)
-		//{
-		//	if(material->mParameters[i].mFragParams != nullptr)
-		//		allParams.push_back(material->mParameters[i].mFragParams);
+		material->initBestTechnique();
 
-		//	if(material->mParameters[i].mVertParams != nullptr)
-		//		allParams.push_back(material->mParameters[i].mVertParams);
+		std::shared_ptr<MaterialParams> params = boost::any_cast<std::shared_ptr<MaterialParams>>(material->mRTTIData);
 
-		//	if(material->mParameters[i].mGeomParams != nullptr)
-		//		allParams.push_back(material->mParameters[i].mGeomParams);
-		//}
+		vector<GpuProgramParametersPtr>::type allParams;
+		for(size_t i = 0; i < material->mParameters.size(); i++)
+		{
+			if(material->mParameters[i].mFragParams != nullptr)
+				allParams.push_back(material->mParameters[i].mFragParams);
+
+			if(material->mParameters[i].mVertParams != nullptr)
+				allParams.push_back(material->mParameters[i].mVertParams);
 
-		//for(size_t i = 0; i < allParams.size(); i++)
-		//{
-		//	const GpuNamedConstants& namedConstants = allParams[i]->getConstantDefinitions();
+			if(material->mParameters[i].mGeomParams != nullptr)
+				allParams.push_back(material->mParameters[i].mGeomParams);
+		}
+
+		for(size_t i = 0; i < allParams.size(); i++)
+		{
+			const GpuNamedConstants& namedConstants = allParams[i]->getConstantDefinitions();
 
-		//	float tempValue[16];
-		//	for(auto iter = namedConstants.map.begin(); iter != namedConstants.map.end(); ++iter)
-		//	{
-		//		const GpuConstantDefinition& def = iter->second;
+			float tempValue[16];
+			for(auto iter = namedConstants.map.begin(); iter != namedConstants.map.end(); ++iter)
+			{
+				const GpuConstantDefinition& def = iter->second;
 
-		//		if(def.constType == GCT_SAMPLER2D)
-		//		{
-		//			auto iterFind = params->mTextureParams.find(iter->first);
+				if(def.constType == GCT_SAMPLER2D)
+				{
+					auto iterFind = params->mTextureParams.find(iter->first);
 
-		//			if(iterFind != params->mTextureParams.end())
-		//				allParams[i]->setNamedConstant(iter->first, iterFind->second);
-		//		}
-		//		else
-		//		{
-		//			auto iterFind = params->mFloatParams.find(iter->first);
+					if(iterFind != params->mTextureParams.end())
+						allParams[i]->setNamedConstant(iter->first, iterFind->second);
+				}
+				else
+				{
+					auto iterFind = params->mFloatParams.find(iter->first);
 
-		//			if(iterFind != params->mFloatParams.end() && iterFind->second.mType == def.constType)
-		//			{
-		//				FloatParam param = iterFind->second;
-		//				UINT32 fieldSize = def.getElementSize(def.constType, false);
+					if(iterFind != params->mFloatParams.end() && iterFind->second.mType == def.constType)
+					{
+						FloatParam param = iterFind->second;
+						UINT32 fieldSize = def.getElementSize(def.constType, false);
 
-		//				if(fieldSize != param.mCount)
-		//					CM_EXCEPT(InternalErrorException, "Deserializing material parameter but field sizes don't match.");
+						if(fieldSize != param.mCount)
+							CM_EXCEPT(InternalErrorException, "Deserializing material parameter but field sizes don't match.");
 
-		//				if(fieldSize > 16)
-		//					CM_EXCEPT(InternalErrorException, "Field size larger than the supported size.");
+						if(fieldSize > 16)
+							CM_EXCEPT(InternalErrorException, "Field size larger than the supported size.");
 
-		//				for(size_t j = 0; j < fieldSize; j++)
-		//					tempValue[j] = params->mFloatBuffer[param.mBufferIdx + j];
+						for(size_t j = 0; j < fieldSize; j++)
+							tempValue[j] = params->mFloatBuffer[param.mBufferIdx + j];
 
-		//				allParams[i]->_writeRawConstants(def.physicalIndex, tempValue, fieldSize);
-		//			}
-		//		}
-		//	}
-		//}
+						allParams[i]->_writeRawConstants(def.physicalIndex, tempValue, fieldSize);
+					}
+				}
+			}
+		}
 
 		material->mRTTIData = nullptr; // This will delete temporary data as it's stored in a unique ptr
-
-		//material->initBestTechnique();
 	}
 }

CamelotRenderer/TODO.txt

@@ -15,8 +15,9 @@ High-level TODO:
  - DX11 render system
 
 HIGH PRIORITY TODO:
- - I need to store material parameters somewhere as well
-  - GpuProgramParams won't work because I want shader parameters to be stored independantly from the shader itself
+ - Issue with deserialization and value types:
+  - Value types are only set at the end of deserialization, because I want to be sure all of their fields are initialized. However there is nothing stopping a custom RTTI method from accessing a (yet uninitialized) value in a ptr field. (See CmMaterialRTTI, setTexParam). I need to initialize fields in a better order.
+   - Solution?: Set (empty) ptr values immediately, and only load them later
  - GetRenderOperation doesn't consider sub-meshes
  - HLSL & Cg don't handle include files yet
 
@@ -55,6 +56,9 @@ Low priority TODO:
  - D3D9Texture::createTextureResources is commented out at the moment. It gets called on device reset, and at that point I should reload texture resources.
   - I should probably keep all resources by DX managed. OpenGL apparently keeps a mirror of all its resources anyway.
  - Device reset and resource re-loading in general
+ - In RTTIType it is possible to add a non-plain data type using addPlainField. This can cause memory corruption issues
+   if pointer is saved/loaded as a plain field. I need to add a check that ensures the type is POD. 
+   See: http://www.boost.org/doc/libs/1_51_0/boost/mpi/datatype.hpp for a possible implementation of a compile time check.
 
 Optional TODO:
  - Add precompiled headers to all projects

CamelotUtility/Include/CmBinarySerializer.h

@@ -30,6 +30,10 @@ namespace CamelotEngine
 	 * 			Any data the object or its children are pointing to will also be serialized (unless the pointer isn't
 	 * 			registered in RTTIType). Upon decoding the pointer addresses will be set
 	 * 			to proper values.
+	 * 			
+	 * @note	When deserializing make sure not to access any child members of a deserialized object in your RTTIType class.
+	 * 			Objects might be deserialized in an unknown order, so it is not guaranteed that child elements have been
+	 * 			deserialized before their parent. 
 	 */
 	class CM_UTILITY_EXPORT BinarySerializer
 	{
@@ -73,29 +77,7 @@ namespace CamelotEngine
 			std::shared_ptr<IReflectable> object;
 		};
 
-		/**
-		 * @brief	Pointer fields get resolved after everything is loaded. Store their
-		 * 			temporary data here until then.
-		 */
-		struct PtrToResolve
-		{
-			PtrToResolve()
-				:field(nullptr), object(nullptr), id(0)
-			{ }
-
-			PtrToResolve(RTTIReflectablePtrFieldBase* _field, std::shared_ptr<IReflectable> _object, UINT32 _id)
-				:field(_field), object(_object), id(_id), arrIdx(0)
-			{ }
-
-			PtrToResolve(RTTIReflectablePtrFieldBase* _field, std::shared_ptr<IReflectable> _object, UINT32 _id, UINT32 _arrIdx)
-				:field(_field), object(_object), id(_id), arrIdx(_arrIdx)
-			{ }
 
-			RTTIReflectablePtrFieldBase* field;
-			UINT32 arrIdx;
-			std::shared_ptr<IReflectable> object;
-			UINT32 id;
-		};
 
 		struct ObjectMetaData
 		{
@@ -108,9 +90,9 @@ namespace CamelotEngine
 		std::vector<ObjectToEncode> mObjectsToEncode;
 		int mTotalBytesWritten;
 
-		std::vector<PtrToResolve> mPtrsToResolve;
-		std::unordered_map<UINT32, std::shared_ptr<IReflectable>> mDecodedObjects;
-
+		std::unordered_map<UINT32, std::shared_ptr<IReflectable>> mObjectMap;
+		std::vector<std::shared_ptr<IReflectable>> mObjectsToDecode;
+		std::vector<std::shared_ptr<IReflectable>> mDecodedObjects;
 
 		UINT32 getObjectSize(IReflectable* object);
 

CamelotUtility/Source/CmBinarySerializer.cpp

@@ -108,55 +108,31 @@ namespace CamelotEngine
 
 	std::shared_ptr<IReflectable> BinarySerializer::decode(UINT8* data, UINT32 dataLength)
 	{
-		mPtrsToResolve.clear();
+		mObjectMap.clear();
 		mDecodedObjects.clear();
 
-		// Find and instantiate root object
-		if(sizeof(UINT32) > dataLength)
-		{
-			CM_EXCEPT(InternalErrorException, 
-				"Error decoding data.");
-		}
-
-		ObjectMetaData objectMetaData;
-		objectMetaData.objectMeta = 0;
-		objectMetaData.typeId = 0;
-		memcpy(&objectMetaData, data, sizeof(ObjectMetaData));
-
-		UINT32 objectId = 0;
-		UINT32 objectTypeId = 0;
-		bool isBaseClass = false;
-		decodeObjectMetaData(objectMetaData, objectId, objectTypeId, isBaseClass);
-
-		if(isBaseClass)
-		{
-			CM_EXCEPT(InternalErrorException, "Encountered a base-class object while looking for a new object. " \
-				"Base class objects are only supposed to be parts of a larger object.");
-		}
-
-		std::shared_ptr<IReflectable> object = IReflectable::createInstanceFromTypeId(objectTypeId);
-		std::shared_ptr<IReflectable> rootObject = object;
-
-		// Create initial object + all other objects that are being referenced.
-		// Use fields to find the type of referenced object.
+		// Create empty instances of all ptr objects
 		UINT32 bytesRead = 0;
-		UINT8* dataIter = data + bytesRead;
-		while(decodeInternal(object, dataIter, dataLength, bytesRead))
+		UINT8* dataIter = nullptr;
+		std::shared_ptr<IReflectable> rootObject = nullptr;
+		do 
 		{
 			dataIter = data + bytesRead;
 
-			if((bytesRead + sizeof(UINT32)) > dataLength)
+			if(sizeof(UINT32) > dataLength)
 			{
 				CM_EXCEPT(InternalErrorException, 
 					"Error decoding data.");
 			}
 
+			ObjectMetaData objectMetaData;
 			objectMetaData.objectMeta = 0;
 			objectMetaData.typeId = 0;
 			memcpy(&objectMetaData, dataIter, sizeof(ObjectMetaData));
 
-			objectId = 0;
-			objectTypeId = 0;
+			UINT32 objectId = 0;
+			UINT32 objectTypeId = 0;
+			bool isBaseClass = false;
 			decodeObjectMetaData(objectMetaData, objectId, objectTypeId, isBaseClass);
 
 			if(isBaseClass)
@@ -165,34 +141,21 @@ namespace CamelotEngine
 					"Base class objects are only supposed to be parts of a larger object.");
 			}
 
-			object = nullptr;
-			if(objectId != 0)
-			{
-				auto iterFind = std::find_if(mPtrsToResolve.begin(), mPtrsToResolve.end(), [objectId](PtrToResolve x) { return x.id == objectId; });
+			std::shared_ptr<IReflectable> object = IReflectable::createInstanceFromTypeId(objectTypeId);
+			mObjectMap.insert(std::make_pair(objectId, object));
+			mObjectsToDecode.push_back(object);
 
-				if(iterFind != mPtrsToResolve.end())
-				{
-					object = IReflectable::createInstanceFromTypeId(objectTypeId);
-				}
-			}
-		}
+			if(rootObject == nullptr)
+				rootObject = object;
 
-		for(auto iter = mPtrsToResolve.begin(); iter != mPtrsToResolve.end(); ++iter)
-		{
-			std::shared_ptr<IReflectable> resolvedObject = nullptr;
+		} while (decodeInternal(nullptr, dataIter, dataLength, bytesRead));
 
-			PtrToResolve curPtr = *iter;
-			if(curPtr.id != 0)
-			{
-				auto iterFind = mDecodedObjects.find(curPtr.id);
-				if(iterFind != mDecodedObjects.end())
-					resolvedObject = iterFind->second;
-			}
+		bytesRead = 0;
+		for(auto objIter = mObjectsToDecode.begin(); objIter != mObjectsToDecode.end(); ++objIter)
+		{
+			dataIter = data + bytesRead;
 
-			if(curPtr.field->mIsVectorType)
-				curPtr.field->setArrayValue(curPtr.object.get(), curPtr.arrIdx, resolvedObject);
-			else
-				curPtr.field->setValue(curPtr.object.get(), resolvedObject);
+			decodeInternal(*objIter, dataIter, dataLength, bytesRead);
 		}
 
 		// Finish serialization for all objects
@@ -202,7 +165,7 @@ namespace CamelotEngine
 		// pointers at an earlier stage as well)
 		for(auto iter = mDecodedObjects.rbegin(); iter != mDecodedObjects.rend(); ++iter)
 		{
-			std::shared_ptr<IReflectable> resolvedObject = iter->second;
+			std::shared_ptr<IReflectable> resolvedObject = *iter;
 
 			if(resolvedObject != nullptr)
 			{
@@ -216,13 +179,8 @@ namespace CamelotEngine
 			}
 		}
 
-		mPtrsToResolve.clear();
-		//mDecodedObjects.clear();
-
-		while(mDecodedObjects.size() > 0)
-		{
-			mDecodedObjects.erase(mDecodedObjects.begin());
-		}
+		mObjectMap.clear();
+		mDecodedObjects.clear();
 
 		return rootObject;
 	}
@@ -486,8 +444,8 @@ namespace CamelotEngine
 		bool objectIsBaseClass = false;
 		decodeObjectMetaData(objectMetaData, objectId, objectTypeId, objectIsBaseClass);
 
-		if(object != nullptr && objectId != 0 && !objectIsBaseClass)
-			mDecodedObjects.insert(std::make_pair(objectId, object));
+		if(object != nullptr && !objectIsBaseClass)
+			mDecodedObjects.push_back(object);
 		
 		while(bytesRead < dataLength)
 		{
@@ -624,7 +582,15 @@ namespace CamelotEngine
 							bytesRead += COMPLEX_TYPE_FIELD_SIZE;
 
 							if(curField != nullptr)
-								mPtrsToResolve.push_back(PtrToResolve(curField, object, objectId, i));
+							{
+								std::shared_ptr<IReflectable> resolvedObject = nullptr;
+
+								auto findIter = mObjectMap.find(objectId);
+								if(findIter != mObjectMap.end())
+									resolvedObject = findIter->second;
+
+								curField->setArrayValue(object.get(), i, resolvedObject);
+							}
 						}
 
 						break;
@@ -641,12 +607,17 @@ namespace CamelotEngine
 									"Error decoding data.");
 							}
 
-							int complexTypeSize = COMPLEX_TYPE_FIELD_SIZE;
+							int complexTypeSize = 0;
 							if(curField != nullptr)
 							{
 								std::shared_ptr<IReflectable> complexType = complexTypeFromBuffer(curField, data, &complexTypeSize);
 								curField->setArrayValue(object.get(), i, *complexType);
 							}
+							else
+							{
+								memcpy(&complexTypeSize, data, COMPLEX_TYPE_FIELD_SIZE);
+								complexTypeSize += COMPLEX_TYPE_FIELD_SIZE;
+							}
 
 							data += complexTypeSize;
 							bytesRead += complexTypeSize;
@@ -697,7 +668,15 @@ namespace CamelotEngine
 						bytesRead += COMPLEX_TYPE_FIELD_SIZE;
 
 						if(curField != nullptr)
-							mPtrsToResolve.push_back(PtrToResolve(curField, object, objectId));
+						{
+							std::shared_ptr<IReflectable> resolvedObject = nullptr;
+
+							auto findIter = mObjectMap.find(objectId);
+							if(findIter != mObjectMap.end())
+								resolvedObject = findIter->second;
+
+							curField->setValue(object.get(), resolvedObject);
+						}
 
 						break;
 					}
@@ -711,12 +690,17 @@ namespace CamelotEngine
 								"Error decoding data.");
 						}
 
-						int complexTypeSize = COMPLEX_TYPE_FIELD_SIZE;
+						int complexTypeSize = 0;
 						if(curField != nullptr)
 						{
 							std::shared_ptr<IReflectable> complexType = complexTypeFromBuffer(curField, data, &complexTypeSize);
 							curField->setValue(object.get(), *complexType);
 						}
+						else
+						{
+							memcpy(&complexTypeSize, data, COMPLEX_TYPE_FIELD_SIZE);
+							complexTypeSize += COMPLEX_TYPE_FIELD_SIZE;
+						}
 
 						data += complexTypeSize;
 						bytesRead += complexTypeSize;