C# serialization: Structs now work properly with normal objects

BansheeMono/Include/BsMonoField.h

@@ -22,6 +22,7 @@ namespace BansheeEngine
 		MonoClass* getType();
 
 		void getValue(MonoObject* instance, void* outValue);
+		MonoObject* getValueBoxed(MonoObject* instance);
 		void setValue(MonoObject* instance, void* value);
 
 		bool hasAttribute(MonoClass* monoClass);

BansheeMono/Source/BsMonoClass.cpp

@@ -88,26 +88,26 @@ namespace BansheeEngine
 		if(iterFind != mMethods.end())
 			return iterFind->second;
 
-		::MonoMethod* method;
-		void* iter = nullptr;
-
-		const char* rawName = name.c_str();
-		const char* rawSig = signature.c_str();
-		while ((method = mono_class_get_methods(mClass, &iter)))
-		{
-			if (strcmp(rawName, mono_method_get_name(method)) == 0)
-			{
-				const char* curSig = mono_signature_get_desc(mono_method_signature(method), false);
-				if(strcmp(rawSig, curSig) == 0)
-				{
+		::MonoMethod* method;
+		void* iter = nullptr;
+
+		const char* rawName = name.c_str();
+		const char* rawSig = signature.c_str();
+		while ((method = mono_class_get_methods(mClass, &iter)))
+		{
+			if (strcmp(rawName, mono_method_get_name(method)) == 0)
+			{
+				const char* curSig = mono_signature_get_desc(mono_method_signature(method), false);
+				if(strcmp(rawSig, curSig) == 0)
+				{
 					MonoMethod* newMethod = new (cm_alloc<MonoMethod>()) MonoMethod(method);
-					mMethods[mehodId] = newMethod;
-
-					return newMethod;
-				}
-			}
-		}
-
+					mMethods[mehodId] = newMethod;
+
+					return newMethod;
+				}
+			}
+		}
+
 		return nullptr;
 	}
 

BansheeMono/Source/BsMonoField.cpp

@@ -34,6 +34,11 @@ namespace BansheeEngine
 		mono_field_get_value(instance, mField, outValue);
 	}
 
+	MonoObject* MonoField::getValueBoxed(MonoObject* instance)
+	{
+		return mono_field_get_value_object(MonoManager::instance().getDomain(), mField, instance);
+	}
+
 	void MonoField::setValue(MonoObject* instance, void* value)
 	{
 		mono_field_set_value(instance, mField, value);

GameObjectSerialization.txt

@@ -2,16 +2,16 @@ TODO
  - Ability to break external references as a pre-processing step
  - I need to properly unload all assemblies (and possibly shutdown mono runtime?) before I shut down various modules used in ScriptBansheeEngine.
 
+C# Serialization
+ - Ensure that everything still works when a serialized type is removed
+ - Ensure that such components retain their field data so when they are no longer unknown their data may be restored
+
  ---------------------------------------
 
 IMMEDIATE:
-In some places (especially list and dict) I do: if(mono_class_is_valuetype(mono_object_get_class(obj))).
-This will return true for strings, which don't work as intended when unboxed (or at least don't show consistent behaviour with other types)
-
 setValue in ScriptSerializableObject & ScriptSerializableArray should check if value is null and if the field type isn't a value type (can't be null)
 
 LOW PRIORITY
-- Ensure that unknown components retain their field data so when they are no longer unknown their data may be restored
 - Get rid of ScriptObject::createInstance and replace it with parameter in constructor
 - A way to serialize any object into a Resource (and deserialize it from Resource)
 
@@ -19,7 +19,6 @@ LOW PRIORITY
 
  Testing:
 Test how structs work
-Test how things work when class gets removed
 
  ------------------------------------------------------
  General C# component management

MBansheeEngine/DbgComponent.cs

@@ -8,11 +8,15 @@ namespace BansheeEngine
     public class DbgComponent : Component
     {
         public int a;
+        public string b;
         public DbgSerzObj complex = new DbgSerzObj();
+        public DbgSerzCls complex2 = new DbgSerzCls();
+
         public DbgComponent2 otherComponent;
         public SceneObject otherSO;
-        public int[][][] zeArray;
-        public List<DbgSerzObj> zeList;
-        public Dictionary<string, int> zeDict;
+
+        //public int[][][] zeArray;
+        //public List<DbgSerzObj> zeList;
+        //public Dictionary<string, int> zeDict;
     }
 }

MBansheeEngine/DbgSerzCls.cs

@@ -0,0 +1,14 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+
+namespace BansheeEngine
+{
+    [SerializableObject]
+    public class DbgSerzCls
+    {
+        public int someValue2;
+        public string anotherValue2;
+    }
+}

MBansheeEngine/DbgSerzObj.cs

@@ -6,8 +6,14 @@ using System.Text;
 namespace BansheeEngine
 {
     [SerializableObject]
-    public class DbgSerzObj
+    public struct DbgSerzObj
     {
+        public DbgSerzObj(int someValue, string anotherValue)
+        {
+            this.someValue = someValue;
+            this.anotherValue = anotherValue;
+        }
+
         public int someValue;
         public string anotherValue;
     }

MBansheeEngine/MBansheeEngine.csproj

@@ -47,6 +47,7 @@
     <Compile Include="Component.cs" />
     <Compile Include="DbgComponent.cs" />
     <Compile Include="DbgComponent2.cs" />
+    <Compile Include="DbgSerzCls.cs" />
     <Compile Include="DbgSerzObj.cs" />
     <Compile Include="DontSerializeField.cs" />
     <Compile Include="Font.cs" />

MBansheeEngine/Program.cs

@@ -18,47 +18,55 @@ namespace BansheeEngine
             SceneObject so = new SceneObject("TestSO");
             DbgComponent dbgComponent = so.AddComponent<DbgComponent>();
             dbgComponent.a = 5;
+            dbgComponent.b = "SomeTestVal";
             dbgComponent.complex.someValue = 19;
             dbgComponent.complex.anotherValue = "AnotherValue";
+            dbgComponent.complex2.someValue2 = 21;
+            dbgComponent.complex2.anotherValue2 = "AnotherValue2";
+
             dbgComponent.otherComponent = dbgComponent2;
             dbgComponent.otherSO = otherSO;
-            dbgComponent.zeArray = new int[5][][];
-            dbgComponent.zeList = new List<DbgSerzObj>();
-            dbgComponent.zeDict = new Dictionary<string, int>();
 
-            dbgComponent.zeList.Add(new DbgSerzObj());
-            dbgComponent.zeList.Add(new DbgSerzObj());
-            dbgComponent.zeList.Add(new DbgSerzObj());
-            dbgComponent.zeList.Add(new DbgSerzObj());
+            //dbgComponent.zeArray = new int[5][][];
+            //dbgComponent.zeList = new List<DbgSerzObj>();
+            //dbgComponent.zeDict = new Dictionary<string, int>();
 
-            dbgComponent.zeList[2].someValue = 101;
+            //dbgComponent.zeList.Add(new DbgSerzObj());
+            //dbgComponent.zeList.Add(new DbgSerzObj());
+            //dbgComponent.zeList.Add(new DbgSerzObj(101, ""));
+            //dbgComponent.zeList.Add(new DbgSerzObj());
 
-            dbgComponent.zeDict["supSup"] = 10001;
-            dbgComponent.zeDict["lolz"] = 696969;
+            //dbgComponent.zeDict["supSup"] = 10001;
+            //dbgComponent.zeDict["lolz"] = 696969;
 
-            var enumerator = dbgComponent.zeDict.GetEnumerator();
-            int all = 0;
-            while (enumerator.MoveNext())
-            {
-                all += enumerator.Current.Value;
-            }
+            //var enumerator = dbgComponent.zeDict.GetEnumerator();
+            //int all = 0;
+            //while (enumerator.MoveNext())
+            //{
+            //    all += enumerator.Current.Value;
+            //}
 
-            for (int i = 0; i < dbgComponent.zeArray.Length; i++)
-            {
-                dbgComponent.zeArray[i] = new int[6][];
+            //for (int i = 0; i < dbgComponent.zeArray.Length; i++)
+            //{
+            //    dbgComponent.zeArray[i] = new int[6][];
 
-                for (int j = 0; j < dbgComponent.zeArray[i].Length; j++)
-                    dbgComponent.zeArray[i][j] = new int[7];
-            }
+            //    for (int j = 0; j < dbgComponent.zeArray[i].Length; j++)
+            //        dbgComponent.zeArray[i][j] = new int[7];
+            //}
 
-            dbgComponent.zeArray[4][1][3] = 129;
+            //dbgComponent.zeArray[4][1][3] = 129;
 
             dbgTestComponentClone(so);
 
             for (int i = 0; i < so.GetNumChildren(); i++)
             {
                 SceneObject childSO = so.GetChild(i);
-                reportDbgValue(childSO.GetComponent<DbgComponent>().zeDict["lolz"], childSO.GetComponent<DbgComponent>().zeList[2].someValue, childSO.GetComponent<DbgComponent>().zeArray[4][1][3], typeof(DbgComponent));
+
+                DbgComponent otherComponent = childSO.GetComponent<DbgComponent>();
+                reportDbgValue(otherComponent.a, otherComponent.b, otherComponent.complex.someValue,
+                               otherComponent.complex2.anotherValue2);
+
+                //reportDbgValue(childSO.GetComponent<DbgComponent>().zeDict["lolz"], childSO.GetComponent<DbgComponent>().zeList[2].someValue, childSO.GetComponent<DbgComponent>().zeArray[4][1][3], typeof(DbgComponent));
             }
 
             //Color newColor = Color.red;
@@ -72,6 +80,6 @@ namespace BansheeEngine
         private static extern void dbgTestComponentClone(SceneObject so);
 
         [MethodImpl(MethodImplOptions.InternalCall)]
-        private static extern void reportDbgValue(int dbgVal, int dbgVal2, int dbgVal3, Type type);
+        private static extern void reportDbgValue(int a, string b, int a2, string b2);
     }
 }

SBansheeEngine/Include/BsScriptSerializableArray.h

@@ -21,6 +21,7 @@ namespace BansheeEngine
 
 	protected:
 		MonoObject* mManagedInstance;
+		::MonoClass* mElementMonoClass;
 
 		ScriptSerializableTypeInfoArrayPtr mArrayTypeInfo;
 		CM::Vector<ScriptSerializableFieldDataPtr>::type mArrayEntries;
@@ -28,6 +29,8 @@ namespace BansheeEngine
 		CM::Vector<CM::UINT32>::type mNumElements;
 		CM::UINT32 mElemSize;
 
+		void initMonoObjects();
+
 		/**
 		 * @brief	Populates internal field data based on currently active managed instance.
 		 */
@@ -42,7 +45,6 @@ namespace BansheeEngine
 		ScriptSerializableFieldDataPtr getFieldData(CM::UINT32 arrayIdx);
 
 		void setValue(CM::UINT32 arrayIdx, void* val);
-		void* getValue(CM::UINT32 arrayIdx);
 
 		CM::UINT32 toSequentialIdx(const CM::Vector<CM::UINT32>::type& idx) const;
 		CM::UINT32 getLength(CM::UINT32 dimension) const;

SBansheeEngine/Include/BsScriptSerializableObjectInfo.h

@@ -40,7 +40,8 @@ namespace BansheeEngine
 
 		virtual bool matches(const ScriptSerializableTypeInfoPtr& typeInfo) const = 0;
 		virtual bool isTypeLoaded() const = 0;
-		virtual bool isMonoObject() const = 0;
+		virtual bool isRawType() const = 0;
+
 		virtual ::MonoClass* getMonoClass() const = 0;
 
 		/************************************************************************/
@@ -59,7 +60,8 @@ namespace BansheeEngine
 
 		bool matches(const ScriptSerializableTypeInfoPtr& typeInfo) const;
 		bool isTypeLoaded() const;
-		bool isMonoObject() const;
+		bool isRawType() const;
+
 		::MonoClass* getMonoClass() const;
 
 		/************************************************************************/
@@ -80,7 +82,8 @@ namespace BansheeEngine
 
 		bool matches(const ScriptSerializableTypeInfoPtr& typeInfo) const;
 		bool isTypeLoaded() const;
-		bool isMonoObject() const { return true; }
+		bool isRawType() const { return false; }
+
 		::MonoClass* getMonoClass() const;
 
 		/************************************************************************/
@@ -100,7 +103,8 @@ namespace BansheeEngine
 
 		bool matches(const ScriptSerializableTypeInfoPtr& typeInfo) const;
 		bool isTypeLoaded() const;
-		bool isMonoObject() const { return true; }
+		bool isRawType() const { return false; }
+
 		::MonoClass* getMonoClass() const;
 
 		/************************************************************************/
@@ -119,7 +123,8 @@ namespace BansheeEngine
 
 		bool matches(const ScriptSerializableTypeInfoPtr& typeInfo) const;
 		bool isTypeLoaded() const;
-		bool isMonoObject() const { return true; }
+		bool isRawType() const { return false; }
+
 		::MonoClass* getMonoClass() const;
 
 		/************************************************************************/
@@ -139,7 +144,8 @@ namespace BansheeEngine
 
 		bool matches(const ScriptSerializableTypeInfoPtr& typeInfo) const;
 		bool isTypeLoaded() const;
-		bool isMonoObject() const { return true; }
+		bool isRawType() const { return false; }
+
 		::MonoClass* getMonoClass() const;
 
 		/************************************************************************/

SBansheeEngine/Source/BsScriptEnginePlugin.cpp

@@ -8,6 +8,7 @@
 // DEBUG ONLY
 #include "BsScriptSceneObject.h"
 #include "CmSceneObject.h"
+#include "BsMonoUtil.h"
 
 using namespace CamelotFramework;
 
@@ -23,13 +24,12 @@ namespace BansheeEngine
 		cloneSO->setParent(SO);
 	}
 
-	void reportDbgValue(int dbgValue, int dbgValue2, int dbgValue3, MonoReflectionType* type)
+	void reportDbgValue(int a, MonoString* b, int a2, MonoString* b2)
 	{
-		::MonoClass* monoClass = mono_type_get_class(mono_reflection_type_get_type(type));
+		WString bStr = MonoUtil::monoToWString(b);
+		WString b2Str = MonoUtil::monoToWString(b2);
 
-		String className = mono_class_get_name(monoClass);
-		
-		int a = dbgValue;
+		int end = 5;
 	}
 
 	extern "C" BS_SCR_BE_EXPORT const String& getPluginName()

SBansheeEngine/Source/BsScriptSerializableArray.cpp

@@ -11,17 +11,19 @@ using namespace CamelotFramework;
 namespace BansheeEngine
 {
 	ScriptSerializableArray::ScriptSerializableArray(const ConstructPrivately& dummy)
-		:mManagedInstance(nullptr), mElemSize(0)
+		:mManagedInstance(nullptr), mElemSize(0), mElementMonoClass(nullptr)
 	{
 
 	}
 
 	ScriptSerializableArray::ScriptSerializableArray(const ConstructPrivately& dummy, const ScriptSerializableTypeInfoArrayPtr& typeInfo, MonoObject* managedInstance)
-		:mArrayTypeInfo(typeInfo), mManagedInstance(managedInstance), mElemSize(0)
+		:mArrayTypeInfo(typeInfo), mManagedInstance(managedInstance), mElemSize(0), mElementMonoClass(nullptr)
 	{
 		::MonoClass* monoClass = mono_object_get_class(mManagedInstance);
 		mElemSize = mono_array_element_size(monoClass);
 
+		initMonoObjects();
+
 		mNumElements.resize(typeInfo->mRank);
 		for(UINT32 i = 0; i < typeInfo->mRank; i++)
 			mNumElements[i] = getLength(i);
@@ -78,6 +80,7 @@ namespace BansheeEngine
 			mono_type_get_object(MonoManager::instance().getDomain(), mono_class_get_type(mArrayTypeInfo->getMonoClass())), lengthArray };
 
 		mManagedInstance = createInstance->invoke(nullptr, params);
+		initMonoObjects();
 
 		CM::UINT32 idx = 0;
 		for(auto& arrayEntry : mArrayEntries)
@@ -89,18 +92,58 @@ namespace BansheeEngine
 
 	void ScriptSerializableArray::setFieldData(CM::UINT32 arrayIdx, const ScriptSerializableFieldDataPtr& val)
 	{
-		if(!mArrayTypeInfo->mElementType->isMonoObject())
-			setValue(arrayIdx, val->getValue(mArrayTypeInfo->mElementType));
+		bool isBoxedValueType = false;
+		if(rtti_is_of_type<ScriptSerializableTypeInfoObject>(mArrayTypeInfo->mElementType))
+		{
+			ScriptSerializableTypeInfoObjectPtr objTypeInfo = std::static_pointer_cast<ScriptSerializableTypeInfoObject>(mArrayTypeInfo->mElementType);
+			isBoxedValueType = objTypeInfo->mValueType;
+		}
+
+		if(isBoxedValueType)
+		{
+			MonoObject* value = (MonoObject*)val->getValue(mArrayTypeInfo->mElementType);
+
+			if(value != nullptr)
+				setValue(arrayIdx, mono_object_unbox(value)); // Value types need to be set as native unboxed types
+		}
 		else
 		{
-			MonoObject* ptrToObj = (MonoObject*)val->getValue(mArrayTypeInfo->mElementType);
-			setValue(arrayIdx, &ptrToObj);
+			if(mArrayTypeInfo->mElementType->isRawType())
+				setValue(arrayIdx, val->getValue(mArrayTypeInfo->mElementType));
+			else
+			{
+				MonoObject* ptrToObj = (MonoObject*)val->getValue(mArrayTypeInfo->mElementType);
+				setValue(arrayIdx, &ptrToObj);
+			}
 		}
 	}
 
 	ScriptSerializableFieldDataPtr ScriptSerializableArray::getFieldData(CM::UINT32 arrayIdx)
 	{
-		return ScriptSerializableFieldData::create(mArrayTypeInfo->mElementType, getValue(arrayIdx));
+		MonoArray* array = (MonoArray*)mManagedInstance;
+
+		UINT32 numElems = (UINT32)mono_array_length(array);
+		assert(arrayIdx < numElems);
+
+		void* arrayValue = mono_array_addr_with_size(array, mElemSize, arrayIdx);
+
+		if(mArrayTypeInfo->mElementType->isRawType())
+		{
+			return ScriptSerializableFieldData::create(mArrayTypeInfo->mElementType, arrayValue);
+		}
+		else
+		{
+			if(mono_class_is_valuetype(mElementMonoClass))
+			{
+				void* rawObj = *(void**)arrayValue;
+				assert(rawObj != nullptr);
+
+				MonoObject* boxedObj = mono_value_box(MonoManager::instance().getDomain(), mElementMonoClass, rawObj);
+				return ScriptSerializableFieldData::create(mArrayTypeInfo->mElementType, &boxedObj);
+			}
+			else
+				return ScriptSerializableFieldData::create(mArrayTypeInfo->mElementType, arrayValue);
+		}
 	}
 	
 	void ScriptSerializableArray::setValue(CM::UINT32 arrayIdx, void* val)
@@ -113,15 +156,10 @@ namespace BansheeEngine
 		void* elemAddr = mono_array_addr_with_size(array, mElemSize, arrayIdx);
 		memcpy(elemAddr, val, mElemSize);
 	}
-	
-	void* ScriptSerializableArray::getValue(CM::UINT32 arrayIdx)
-	{
-		MonoArray* array = (MonoArray*)mManagedInstance;
 
-		UINT32 numElems = (UINT32)mono_array_length(array);
-		assert(arrayIdx < numElems);
-	
-		return mono_array_addr_with_size(array, mElemSize, arrayIdx);
+	void ScriptSerializableArray::initMonoObjects()
+	{
+		mElementMonoClass = mArrayTypeInfo->mElementType->getMonoClass();
 	}
 
 	UINT32 ScriptSerializableArray::toSequentialIdx(const CM::Vector<CM::UINT32>::type& idx) const

SBansheeEngine/Source/BsScriptSerializableObject.cpp

@@ -87,7 +87,10 @@ namespace BansheeEngine
 		if(!RuntimeScriptObjects::instance().getSerializableObjectInfo(storedObjInfo->mTypeInfo->mTypeNamespace, storedObjInfo->mTypeInfo->mTypeName, currentObjInfo))
 			return;
 
-		mManagedInstance = currentObjInfo->mMonoClass->createInstance();
+		if(storedObjInfo->mTypeInfo->mValueType)
+			mManagedInstance = currentObjInfo->mMonoClass->createInstance(false);
+		else
+			mManagedInstance = currentObjInfo->mMonoClass->createInstance();
 
 		auto findFieldInfoFromKey = [&] (CM::UINT16 typeId, CM::UINT16 fieldId, ScriptSerializableObjectInfoPtr objInfo, 
 			ScriptSerializableFieldInfoPtr& outFieldInfo, ScriptSerializableObjectInfoPtr &outObjInfo) -> bool
@@ -162,15 +165,38 @@ namespace BansheeEngine
 
 	void ScriptSerializableObject::setFieldData(const ScriptSerializableFieldInfoPtr& fieldInfo, const ScriptSerializableFieldDataPtr& val)
 	{
-		fieldInfo->mMonoField->setValue(mManagedInstance, val->getValue(fieldInfo->mTypeInfo));
+		bool isBoxedValueType = false;
+		if(rtti_is_of_type<ScriptSerializableTypeInfoObject>(fieldInfo->mTypeInfo))
+		{
+			ScriptSerializableTypeInfoObjectPtr objTypeInfo = std::static_pointer_cast<ScriptSerializableTypeInfoObject>(fieldInfo->mTypeInfo);
+			isBoxedValueType = objTypeInfo->mValueType;
+		}
+
+		if(isBoxedValueType)
+		{
+			MonoObject* value = (MonoObject*)val->getValue(fieldInfo->mTypeInfo);
+
+			if(value != nullptr)
+				fieldInfo->mMonoField->setValue(mManagedInstance, mono_object_unbox(value)); // Value types need to be set as native unboxed types
+		}
+		else
+			fieldInfo->mMonoField->setValue(mManagedInstance, val->getValue(fieldInfo->mTypeInfo));
 	}
 
 	ScriptSerializableFieldDataPtr ScriptSerializableObject::getFieldData(const ScriptSerializableFieldInfoPtr& fieldInfo)
 	{
-		UINT8 outBuffer[32]; // Buffer more than large enough to hold any pointer or primitive value
-		fieldInfo->mMonoField->getValue(mManagedInstance, outBuffer);
+		MonoObject* fieldValue = fieldInfo->mMonoField->getValueBoxed(mManagedInstance);
+
+		if(fieldInfo->mTypeInfo->isRawType())
+		{
+			assert(fieldValue != nullptr);
 
-		return ScriptSerializableFieldData::create(fieldInfo->mTypeInfo, outBuffer);
+			return ScriptSerializableFieldData::create(fieldInfo->mTypeInfo, mono_object_unbox(fieldValue));
+		}
+		else
+		{
+			return ScriptSerializableFieldData::create(fieldInfo->mTypeInfo, &fieldValue);
+		}			
 	}
 
 	RTTITypeBase* ScriptSerializableObject::getRTTIStatic()

SBansheeEngine/Source/BsScriptSerializableObjectInfo.cpp

@@ -127,7 +127,7 @@ namespace BansheeEngine
 		return nullptr;
 	}
 
-	bool ScriptSerializableTypeInfoPrimitive::isMonoObject() const
+	bool ScriptSerializableTypeInfoPrimitive::isRawType() const
 	{
 		switch(mType)
 		{
@@ -136,10 +136,10 @@ namespace BansheeEngine
 			case ScriptPrimitiveType::TextureRef:
 			case ScriptPrimitiveType::SpriteTextureRef:
 			case ScriptPrimitiveType::String:
-				return true;
+				return false;
 		}
 
-		return false;
+		return true;
 	}
 
 	RTTITypeBase* ScriptSerializableTypeInfoPrimitive::getRTTIStatic()
@@ -159,7 +159,7 @@ namespace BansheeEngine
 
 		auto objTypeInfo = std::static_pointer_cast<ScriptSerializableTypeInfoObject>(typeInfo);
 
-		return objTypeInfo->mTypeNamespace == mTypeNamespace && objTypeInfo->mTypeName == mTypeName;
+		return objTypeInfo->mTypeNamespace == mTypeNamespace && objTypeInfo->mTypeName == mTypeName && objTypeInfo->mValueType == mValueType;
 	}
 
 	bool ScriptSerializableTypeInfoObject::isTypeLoaded() const