Added support for reading UTF8 & UTF16 files

Fixed component foldout remove button style
Made some methods in C# project library accept both relative and absolute paths
Fixed UndoRedo CLR hooks

BansheeEditor/Source/BsGUIComponentFoldout.cpp

@@ -110,6 +110,7 @@ namespace BansheeEngine
 
 			GUILayoutData childData = data;
 			childData.area.x = data.area.x + data.area.width - optimalSize.x - 5; // 5 = arbitrary offset
+			childData.area.width = optimalSize.x;
 			childData.area.y += yOffset;
 			childData.area.height = optimalSize.y;
 
@@ -127,7 +128,7 @@ namespace BansheeEngine
 	void GUIComponentFoldout::styleUpdated()
 	{
 		mToggle->setStyle(getSubStyleName(getFoldoutButtonStyleType()));
-		mRemove->setStyle(getSubStyleName(getFoldoutButtonStyleType()));
+		mRemove->setStyle(getSubStyleName(getFoldoutRemoveButtonStyleType()));
 	}
 
 	const String& GUIComponentFoldout::getGUITypeName()

BansheeEditor/Source/BsProjectLibrary.cpp

@@ -614,6 +614,14 @@ namespace BansheeEngine
 			return;
 
 		Path assetPath = path;
+		if (path.isAbsolute())
+		{
+			if (!getResourcesFolder().includes(path))
+				return;
+
+			assetPath = path.getRelative(getResourcesFolder());
+		}
+
 		assetPath.setExtension(assetPath.getWExtension() + L"." + ResourceImporter::DEFAULT_EXTENSION);
 
 		LibraryEntry* existingEntry = findEntry(assetPath);

BansheeUtility/Include/BsSerializedObject.h

@@ -88,7 +88,6 @@ namespace BansheeEngine
 		virtual RTTITypeBase* getRTTI() const override;
 	};
 
-
 	/**
 	 * @brief	A serialized object consisting of multiple sub-objects,
 	 *			one for each inherited class.

BansheeUtility/Source/BsDataStream.cpp

@@ -1,11 +1,56 @@
 #include "BsDataStream.h"
 #include "BsDebug.h"
 #include "BsException.h"
+#include <codecvt>
 
 namespace BansheeEngine 
 {
 	const UINT32 DataStream::StreamTempSize = 128;
 
+	/**
+	 * @brief	Checks does the provided buffer has an UTF32 byte order mark
+	 *			in little endian order.
+	 */
+	bool isUTF32LE(const UINT8* buffer)
+	{
+		return buffer[0] == 0xFF && buffer[1] == 0xFE && buffer[2] == 0x00 && buffer[3] == 0x00;
+	}
+
+	/**
+	 * @brief	Checks does the provided buffer has an UTF32 byte order mark
+	 *			in big endian order.
+	 */
+	bool isUTF32BE(const UINT8* buffer)
+	{
+		return buffer[0] == 0x00 && buffer[1] == 0x00 && buffer[2] == 0xFE && buffer[3] == 0xFF;
+	}
+
+	/**
+	 * @brief	Checks does the provided buffer has an UTF16 byte order mark
+	 *			in little endian order.
+	 */
+	bool isUTF16LE(const UINT8* buffer)
+	{
+		return buffer[0] == 0xFF && buffer[1] == 0xFE;
+	}
+
+	/**
+	 * @brief	Checks does the provided buffer has an UTF16 byte order mark
+	 *			in big endian order.
+	 */
+	bool isUTF16BE(const UINT8* buffer)
+	{
+		return buffer[0] == 0xFE && buffer[1] == 0xFF;
+	}
+
+	/**
+	 * @brief	Checks does the provided buffer has an UTF8 byte order mark.
+	 */
+	bool isUTF8(const UINT8* buffer)
+	{
+		return (buffer[0] == 0xEF && buffer[1] == 0xBB && buffer[2] == 0xBF);
+	}
+
     template <typename T> DataStream& DataStream::operator>> (T& val)
     {
         read(static_cast<void*>(&val), sizeof(T));
@@ -18,11 +63,12 @@ namespace BansheeEngine
 		// Read the entire buffer - ideally in one read, but if the size of
 		// the buffer is unknown, do multiple fixed size reads.
 		size_t bufSize = (mSize > 0 ? mSize : 4096);
-		StringStream::char_type* tempBuffer = (StringStream::char_type*)bs_alloc(bufSize);
+		std::stringstream::char_type* tempBuffer = (std::stringstream::char_type*)bs_alloc(bufSize);
 
 		// Ensure read from begin of stream
 		seek(0);
-		StringStream result;
+
+		std::stringstream result;
 		while (!eof())
 		{
 			size_t numReadBytes = read(tempBuffer, bufSize);
@@ -30,7 +76,76 @@ namespace BansheeEngine
 		}
 
 		free(tempBuffer);
-		return result.str();
+		std::string string = result.str();
+
+		UINT32 readBytes = (UINT32)string.size();
+		if (readBytes >= 4)
+		{
+			if (isUTF32LE((UINT8*)string.data()))
+			{
+				const std::codecvt_mode convMode = (std::codecvt_mode)(std::consume_header | std::little_endian);
+				typedef std::codecvt_utf8<char32_t, 1114111, convMode> utf8utf32;
+
+				std::wstring_convert<utf8utf32, char32_t> conversion("?");
+				char32_t* start = (char32_t*)string.data();
+				char32_t* end = (start + (string.size() - 1) / 4);
+
+				return conversion.to_bytes(start, end).c_str();
+			}
+			else if (isUTF32BE((UINT8*)string.data()))
+			{
+				const std::codecvt_mode convMode = (std::codecvt_mode)(std::consume_header);
+				typedef std::codecvt_utf8<char32_t, 1114111, convMode> utf8utf32;
+
+				std::wstring_convert<utf8utf32, char32_t> conversion("?");
+				char32_t* start = (char32_t*)string.data();
+				char32_t* end = (start + (string.size() - 1) / 4);
+
+				return conversion.to_bytes(start, end).c_str();
+			}			
+		}
+		
+		if (readBytes >= 3)
+		{
+			if (isUTF8((UINT8*)string.data()))
+			{
+				return string.c_str() + 3;
+			}
+		}
+
+		if (readBytes >= 2)
+		{
+			if (isUTF16LE((UINT8*)string.data()))
+			{
+				const std::codecvt_mode convMode = (std::codecvt_mode)(std::little_endian);
+				typedef std::codecvt_utf8<char16_t, 1114111, convMode> utf8utf16;
+
+				std::wstring_convert<utf8utf16, char16_t> conversion("?");
+				char16_t* start = (char16_t*)(string.data() + 2); // Bug?: std::consume_header seems to be ignored so I manually remove the header
+				char16_t* end = (start + (string.size() - 1) / 2);
+
+				return conversion.to_bytes(start, end).c_str();
+			}
+			else if (isUTF16BE((UINT8*)string.data()))
+			{
+				const std::codecvt_mode convMode = (std::codecvt_mode)(0);
+				typedef std::codecvt_utf8<char16_t, 1114111, convMode> utf8utf16;
+
+				// Bug?: Regardless me of not providing the std::little_endian flag it seems that is how the data is read
+				// so I manually flip it
+				UINT32 numChars = (string.size() - 2) / 2;
+				for (UINT32 i = 0; i < numChars; i++)
+					std::swap(string[i * 2 + 0], string[i * 2 + 1]);
+
+				std::wstring_convert<utf8utf16, char16_t> conversion("?");
+				char16_t* start = (char16_t*)(string.data() + 2); // Bug?: std::consume_header seems to be ignored so I manually remove the header
+				char16_t* end = (start + (string.size() - 1) / 2);
+
+				return conversion.to_bytes(start, end).c_str();
+			}
+		}
+
+		return string.c_str();
 	}
 
 	WString DataStream::getAsWString()
@@ -38,11 +153,12 @@ namespace BansheeEngine
 		// Read the entire buffer - ideally in one read, but if the size of
 		// the buffer is unknown, do multiple fixed size reads.
 		size_t bufSize = (mSize > 0 ? mSize : 4096);
-		WStringStream::char_type* tempBuffer = (WStringStream::char_type*)bs_alloc(bufSize);
+		std::stringstream::char_type* tempBuffer = (std::stringstream::char_type*)bs_alloc(bufSize);
 
 		// Ensure read from begin of stream
 		seek(0);
-		WStringStream result;
+
+		std::stringstream result;
 		while (!eof())
 		{
 			size_t numReadBytes = read(tempBuffer, bufSize);
@@ -50,7 +166,60 @@ namespace BansheeEngine
 		}
 
 		free(tempBuffer);
-		return result.str();
+		std::string string = result.str();
+
+		UINT32 readBytes = (UINT32)string.size();
+		if (readBytes >= 4)
+		{
+			if (isUTF32LE((UINT8*)string.data()))
+			{
+				// Not supported
+			}
+			else if (isUTF32BE((UINT8*)string.data()))
+			{
+				// Not supported
+			}
+		}
+
+		if (readBytes >= 3)
+		{
+			if (isUTF8((UINT8*)string.data()))
+			{
+				const std::codecvt_mode convMode = (std::codecvt_mode)(std::consume_header);
+				typedef std::codecvt_utf8<wchar_t, 1114111, convMode> wcharutf8;
+
+				std::wstring_convert<wcharutf8> conversion("?");
+				return conversion.from_bytes(string).c_str();
+			}
+		}
+
+		if (readBytes >= 2)
+		{
+			if (isUTF16LE((UINT8*)string.data()))
+			{
+				const std::codecvt_mode convMode = (std::codecvt_mode)(std::consume_header | std::little_endian);
+				typedef std::codecvt_utf16 <wchar_t, 1114111, convMode> wcharutf16;
+
+				std::wstring_convert<wcharutf16> conversion("?");
+				return conversion.from_bytes(string).c_str();
+			}
+			else if (isUTF16BE((UINT8*)string.data()))
+			{
+				const std::codecvt_mode convMode = (std::codecvt_mode)(std::consume_header);
+				typedef std::codecvt_utf16<wchar_t, 1114111, convMode> wcharutf16;
+
+				std::wstring_convert<wcharutf16> conversion("?");
+				return conversion.from_bytes(string).c_str();
+			}
+		}
+
+		{
+			const std::codecvt_mode convMode = (std::codecvt_mode)(std::consume_header);
+			typedef std::codecvt_utf8<wchar_t, 1114111, convMode> wcharutf8;
+
+			std::wstring_convert<wcharutf8> conversion("?");
+			return conversion.from_bytes(string).c_str();
+		}
 	}
 
     MemoryDataStream::MemoryDataStream(void* memory, size_t inSize)

MBansheeEditor/Inspector/InspectorWindow.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Data.Common;
+using System.IO;
 using BansheeEngine;
 
 namespace BansheeEditor
@@ -36,6 +37,7 @@ namespace BansheeEditor
         private InspectorResource inspectorResource;
         private GUIScrollArea inspectorScrollArea;
         private GUILayout inspectorLayout;
+        private GUIPanel highlightPanel;
         private GUITexture scrollAreaHighlight;
 
         private SceneObject activeSO;
@@ -104,7 +106,8 @@ namespace BansheeEditor
             scrollAreaHighlight.SetTint(HIGHLIGHT_COLOR);
 
             GUI.AddElement(inspectorScrollArea);
-            GUI.AddElement(scrollAreaHighlight);
+            highlightPanel = GUI.AddPanel(-1);
+            highlightPanel.AddElement(scrollAreaHighlight);
             inspectorLayout = inspectorScrollArea.Layout;
 
             // SceneObject fields
@@ -126,8 +129,10 @@ namespace BansheeEditor
                 data.inspector.Initialize(this, data.panel, allComponents[i]);
 
                 data.foldout.SetExpanded(true);
+
+                Type curComponentType = allComponents[i].GetType();
                 data.foldout.OnToggled += (bool expanded) => OnComponentFoldoutToggled(data, expanded);
-                data.foldout.OnRemoveClicked += () => OnComponentRemoveClicked(allComponents[i].GetType());
+                data.foldout.OnRemoveClicked += () => OnComponentRemoveClicked(curComponentType);
 
                 inspectorComponents.Add(data);
 
@@ -320,7 +325,11 @@ namespace BansheeEditor
                 }
 
                 if (requiresRebuild)
-                    SetObjectToInspect(activeSO);
+                {
+                    SceneObject so = activeSO;
+                    Clear();
+                    SetObjectToInspect(so);
+                }
                 else
                 {
                     RefreshSceneObjectFields(false);
@@ -472,6 +481,12 @@ namespace BansheeEditor
                 scrollAreaHighlight = null;
             }
 
+            if (highlightPanel != null)
+            {
+                highlightPanel.Destroy();
+                highlightPanel = null;
+            }
+
             activeSO = null;
             soNameInput = null;
             soPrefabLayout = null;

MBansheeEditor/ProjectLibrary.cs

@@ -42,9 +42,6 @@ namespace BansheeEditor
 
         public static void Create(Resource resource, string path)
         {
-            if (Path.IsPathRooted(path))
-                throw new ArgumentException("Provided path must be relative.", "path");
-
             Internal_Create(resource, path);
         }
 
@@ -55,9 +52,6 @@ namespace BansheeEditor
 
         public static T Load<T>(string path) where T : Resource
         {
-            if (Path.IsPathRooted(path))
-                throw new ArgumentException("Provided path must be relative.", "path");
-
             return (T) Internal_Load(path);
         }
 

SBansheeEditor/Source/BsScriptGUIComponentFoldout.cpp

@@ -37,7 +37,7 @@ namespace BansheeEngine
 		metaData.scriptClass->addInternalCall("Internal_SetTint", &ScriptGUIComponentFoldout::internal_setTint);
 
 		onToggledThunk = (OnToggledThunkDef)metaData.scriptClass->getMethod("DoOnToggled", 1)->getThunk();
-		onRemoveClickedThunk = (OnRemoveClickedThunkDef)metaData.scriptClass->getMethod("OnRemoveClicked")->getThunk();
+		onRemoveClickedThunk = (OnRemoveClickedThunkDef)metaData.scriptClass->getMethod("DoOnRemoveClicked")->getThunk();
 	}
 
 	void ScriptGUIComponentFoldout::internal_createInstance(MonoObject* instance, MonoObject* content, MonoString* style, MonoArray* guiOptions)

SBansheeEditor/Source/BsScriptUndoRedo.cpp

@@ -25,18 +25,18 @@ namespace BansheeEngine
 
 	void ScriptUndoRedo::initRuntimeData()
 	{
-		metaData.scriptClass->addInternalCall("internal_Undo", &ScriptUndoRedo::internal_Undo);
-		metaData.scriptClass->addInternalCall("internal_Redo", &ScriptUndoRedo::internal_Redo);
-		metaData.scriptClass->addInternalCall("internal_PushGroup", &ScriptUndoRedo::internal_PushGroup);
-		metaData.scriptClass->addInternalCall("internal_PopGroup", &ScriptUndoRedo::internal_PopGroup);
-		metaData.scriptClass->addInternalCall("internal_RecordSO", &ScriptUndoRedo::internal_RecordSO);
-		metaData.scriptClass->addInternalCall("internal_CloneSO", &ScriptUndoRedo::internal_CloneSO);
-		metaData.scriptClass->addInternalCall("internal_CloneSOMulti", &ScriptUndoRedo::internal_CloneSOMulti);
-		metaData.scriptClass->addInternalCall("internal_Instantiate", &ScriptUndoRedo::internal_Instantiate);
-		metaData.scriptClass->addInternalCall("internal_CreateSO", &ScriptUndoRedo::internal_CreateSO);
-		metaData.scriptClass->addInternalCall("internal_DeleteSO", &ScriptUndoRedo::internal_DeleteSO);
-		metaData.scriptClass->addInternalCall("internal_ReparentSO", &ScriptUndoRedo::internal_ReparentSO);
-		metaData.scriptClass->addInternalCall("internal_ReparentSOMulti", &ScriptUndoRedo::internal_ReparentSOMulti);
+		metaData.scriptClass->addInternalCall("Internal_Undo", &ScriptUndoRedo::internal_Undo);
+		metaData.scriptClass->addInternalCall("Internal_Redo", &ScriptUndoRedo::internal_Redo);
+		metaData.scriptClass->addInternalCall("Internal_PushGroup", &ScriptUndoRedo::internal_PushGroup);
+		metaData.scriptClass->addInternalCall("Internal_PopGroup", &ScriptUndoRedo::internal_PopGroup);
+		metaData.scriptClass->addInternalCall("Internal_RecordSO", &ScriptUndoRedo::internal_RecordSO);
+		metaData.scriptClass->addInternalCall("Internal_CloneSO", &ScriptUndoRedo::internal_CloneSO);
+		metaData.scriptClass->addInternalCall("Internal_CloneSOMulti", &ScriptUndoRedo::internal_CloneSOMulti);
+		metaData.scriptClass->addInternalCall("Internal_Instantiate", &ScriptUndoRedo::internal_Instantiate);
+		metaData.scriptClass->addInternalCall("Internal_CreateSO", &ScriptUndoRedo::internal_CreateSO);
+		metaData.scriptClass->addInternalCall("Internal_DeleteSO", &ScriptUndoRedo::internal_DeleteSO);
+		metaData.scriptClass->addInternalCall("Internal_ReparentSO", &ScriptUndoRedo::internal_ReparentSO);
+		metaData.scriptClass->addInternalCall("Internal_ReparentSOMulti", &ScriptUndoRedo::internal_ReparentSOMulti);
 	}
 
 	void ScriptUndoRedo::internal_Undo()

SBansheeEngine/Include/BsScriptObject.h

@@ -203,6 +203,10 @@ namespace BansheeEngine
 		Any data;
 	};
 
+/**
+ * @brief	Helper macro to use with script interop objects that
+ *			form a link between C++ and CLR.
+ */
 #define SCRIPT_OBJ(assembly, namespace, name)		\
 	static String getAssemblyName() { return assembly; }	\
 	static String getNamespace() { return namespace; }		\

SBansheeEngine/Source/BsScriptScriptCode.cpp

@@ -102,9 +102,11 @@ namespace BansheeEngine
 		Vector<FullTypeName> output;
 		Stack<NamespaceData> namespaces;
 
-		// Note: Won't match unicode escape sequences
-		WString identifierPattern = LR"([_@a-zA-Z][\p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}\p{Mn}\p{Mc}\p{Nd}\p{Pc}\p{Cf}]*)";
-		std::wregex identifierRegex(identifierPattern, std::regex_constants::ECMAScript | std::regex_constants::icase);
+		// TODO: Won't match non latin characters because C++ regex doesn't support unicode character classes
+		// and writing out Unicode ranges for all the characters C# supports as identifiers is too tedious at the moment.
+		// Classes that need to match: \p{Lu}\p{Ll}\p{Lt}\p{Lm}\p{Lo}\p{Nl}\p{Mn}\p{Mc}\p{Nd}\p{Pc}\p{Cf}
+		WString identifierPattern = LR"([_@a-zA-Z][_\da-zA-Z]*)";
+		std::wregex identifierRegex(identifierPattern);
 
 		WString nsToken = L"namespace";
 		WString classToken = L"class";
@@ -115,10 +117,10 @@ namespace BansheeEngine
 		{
 			wchar_t ch = *iter;
 			
-			if (code.compare(idx, classToken.size(), classToken))
+			if (code.compare(idx, classToken.size(), classToken) == 0)
 			{
 				std::wsmatch results;
-				if (std::regex_match(iter + classToken.size(), code.end(), results, identifierRegex))
+				if (std::regex_search(iter + classToken.size(), code.end(), results, identifierRegex))
 				{
 					WString ns = L"";
 					if (!namespaces.empty())
@@ -133,10 +135,10 @@ namespace BansheeEngine
 					nsTypePair.second = typeName;
 				}
 			}
-			else if (code.compare(idx, nsToken.size(), nsToken))
+			else if (code.compare(idx, nsToken.size(), nsToken) == 0)
 			{
 				std::wsmatch results;
-				if (std::regex_match(iter + nsToken.size(), code.end(), results, identifierRegex))
+				if (std::regex_search(iter + nsToken.size(), code.end(), results, identifierRegex))
 				{
 					std::wstring tempStr = results[0];
 					WString ns = tempStr.c_str();

TODO.txt

@@ -54,10 +54,6 @@ Code quality improvements:
 ----------------------------------------------------------------------
 Polish
 
-Test:
- - Component foldout X
- - Drag and drop for components
-
  - Duplicating a mesh doesn't properly render the mesh
   - It seems to be due to renderables sharing materials. I generate a separate per-object buffer per each
     renderable (but maybe I shouldn't? Just generate the one and update it before rendering) and then assign
@@ -105,6 +101,7 @@ First screenshot:
    - Possibly create helper objects: Cube, Sphere, Plane, Quad, Capsule, Cylinder
   - Component (also add to inspector context): Camera, Renderable, Point/Spot/Directional light, all other components from scripts
   - Help - About, API Reference (link to site)
+ - Modify inspector looks by scaling down transform input box sizes, and properly aligning component entries
  - (Optionally) New UI look (tabs, component/array containers, better buttons)
    - Foldout in scene tree view is hard to click, make it bigger
  - (Optionally) Console window
@@ -129,6 +126,8 @@ Other polish:
  - Import option inspectors for Texture, Mesh, Font
  - MenuBar - will likely need a way to mark elements as disabled when not appropriate (e.g. no "frame selected unless scene is focused")
    - Likely use a user-provided callback to trigger when populating the menus
+ - CmdRecordSO records an SO and all its children but it should only record a single SO
+   - Also it doesn't record a diff, but instead the whole object
 
 Stage 2 polish:
  - Inject an icon into an .exe (Win32 specific)
@@ -157,6 +156,7 @@ Finalizing:
  - Splash screen
  - Settings/Preferences window
  - Documentation
+ - Rename D3D11RenderSystem (and DX9/GL) project to D3D11RenderAPI
  - (Optionally) GUI tabbing to switch between elements
 
 ----------------------------------------------------------------------