Fixing a scene object transform issue when re-parenting objects

BansheeCore/Source/BsSceneObject.cpp

@@ -9,6 +9,7 @@
 #include "BsPrefab.h"
 #include "BsPrefabUtility.h"
 #include "BsPrefabDiff.h"
+#include "BsMatrix3.h"
 
 namespace BansheeEngine
 {
@@ -240,10 +241,18 @@ namespace BansheeEngine
 	{
 		if (mParent != nullptr)
 		{
-			Matrix4 parentTfrm = mParent->getWorldTfrm();
-			parentTfrm.inverseAffine();
+			Matrix3 rotScale;
+			mParent->getWorldTfrm().extract3x3Matrix(rotScale);
+			rotScale.inverse();
 
-			mScale = parentTfrm.multiplyDirection(scale);
+			Matrix3 scaleMat = Matrix3(Quaternion::IDENTITY, scale);
+			scaleMat = rotScale * scaleMat;
+
+			Quaternion rotation;
+			Vector3 localScale;
+			scaleMat.decomposition(rotation, localScale);
+
+			mScale = localScale;
 		}
 		else
 			mScale = scale;

BansheeEditor/Source/BsEditorApplication.cpp

@@ -44,6 +44,7 @@
 #include "BsEditorSettings.h"
 #include "BsMesh.h"
 #include "BsMath.h"
+#include "BsDebug.h"
 
 namespace BansheeEngine
 {

BansheeUtility/Include/BsMatrix3.h

@@ -43,17 +43,17 @@ namespace BansheeEngine
 		/**
          * @brief	Construct a matrix from a quaternion.
          */
-        explicit Matrix3(const Quaternion& quad)
+        explicit Matrix3(const Quaternion& rotation)
         {
-            fromQuaternion(quad);
+            fromQuaternion(rotation);
         }
 
 		/**
          * @brief	Construct a matrix that performs rotation and scale.
          */
-        explicit Matrix3(const Quaternion& quad, const Vector3 scale)
+        explicit Matrix3(const Quaternion& rotation, const Vector3& scale)
         {
-            fromQuaternion(quad);
+            fromQuaternion(rotation);
 			
 			for (int row = 0; row < 3; row++)
 			{
@@ -182,6 +182,15 @@ namespace BansheeEngine
          */
         float determinant() const;
 
+        /**
+         * @brief	Decompose a Matrix3 to rotation and scale.
+         *
+         * @note	Matrix must consist only of rotation and uniform scale transformations,
+         * 			otherwise accurate results are not guaranteed. Applying non-uniform scale guarantees
+         * 			rotation portion will not be accurate.
+         */
+        void decomposition(Quaternion& rotation, Vector3& scale) const;
+
         /**
          * @brief	Decomposes the matrix into various useful values.
          *

BansheeUtility/Source/BsMatrix3.cpp

@@ -573,6 +573,15 @@ namespace BansheeEngine
         m[2][2] *= invLength;
     }
 
+	void Matrix3::decomposition(Quaternion& rotation, Vector3& scale) const
+	{
+		Matrix3 matQ;
+		Vector3 vecU;
+		QDUDecomposition(matQ, scale, vecU);
+
+		rotation = Quaternion(matQ);
+	}
+
     void Matrix3::QDUDecomposition(Matrix3& matQ, Vector3& vecD, Vector3& vecU) const
     {
         // Build orthogonal matrix Q

MBansheeEditor/Scene/DefaultHandleManager.cs

@@ -10,8 +10,8 @@ namespace BansheeEditor
             public HandledObject(SceneObject so)
             {
                 this.so = so;
-                initialPosition = so.Position;
-                initialRotation = so.Rotation;
+                initialPosition = so.LocalPosition;
+                initialRotation = so.LocalRotation;
                 initialScale = so.LocalScale;
             }
 
@@ -26,6 +26,7 @@ namespace BansheeEditor
 
         private HandledObject[] activeSelection;
         private bool isDragged;
+        private Quaternion initialHandleRotation;
 
         protected override void PreInput()
         {
@@ -108,6 +109,8 @@ namespace BansheeEditor
                         activeSelection = new HandledObject[selectedSceneObjects.Length];
                         for (int i = 0; i < selectedSceneObjects.Length; i++)
                             activeSelection[i] = new HandledObject(selectedSceneObjects[0]);
+
+                        initialHandleRotation = activeHandle.Rotation;
                     }
                 }
                 else
@@ -126,17 +129,37 @@ namespace BansheeEditor
                             MoveHandle moveHandle = (MoveHandle) activeHandle;
 
                             foreach (var selectedObj in activeSelection)
-                                selectedObj.so.Position = selectedObj.initialPosition + moveHandle.Delta;
+                                selectedObj.so.LocalPosition = selectedObj.initialPosition + moveHandle.Delta;
 
                             break;
                         case SceneViewTool.Rotate:
+                        {
                             RotateHandle rotateHandle = (RotateHandle) activeHandle;
 
-                            foreach (var selectedObj in activeSelection)
-                                selectedObj.so.Rotation = selectedObj.initialRotation * rotateHandle.Delta;
+                            // Make sure we transform relative to the handle position
+                            SceneObject temporarySO = new SceneObject("Temp");
+                            temporarySO.Position = activeHandle.Position;
+                            temporarySO.LocalRotation = initialHandleRotation;
+
+                            SceneObject[] originalParents = new SceneObject[activeSelection.Length];
+                            for (int i = 0; i < activeSelection.Length; i++)
+                            {
+                                originalParents[i] = activeSelection[i].so.Parent;
+                                activeSelection[i].so.LocalPosition = activeSelection[i].initialPosition;
+                                activeSelection[i].so.LocalRotation = activeSelection[i].initialRotation;
+                                activeSelection[i].so.Parent = temporarySO;
+                            }
 
+                            temporarySO.LocalRotation *= rotateHandle.Delta;
+
+                            for (int i = 0; i < activeSelection.Length; i++)
+                                activeSelection[i].so.Parent = originalParents[i];
+
+                            temporarySO.Destroy();
+                        }
                             break;
                         case SceneViewTool.Scale:
+                        {
                             ScaleHandle scaleHandle = (ScaleHandle) activeHandle;
 
                             // Make sure we transform relative to the handle position
@@ -159,12 +182,12 @@ namespace BansheeEditor
                             {
                                 Debug.Log("POSITION A: " + activeSelection[i].so.Position);
                                 activeSelection[i].so.Parent = originalParents[i];
-                                Debug.Log("TFRM " + activeSelection[i].so.LocalScale + " - " + activeSelection[i].so.Position);
+                                Debug.Log("TFRM " + activeSelection[i].so.LocalScale + " - " +
+                                          activeSelection[i].so.Position);
                             }
 
-                            
                             temporarySO.Destroy();
-
+                        }
                             break;
                     }
                 }

TODO.txt

@@ -57,22 +57,21 @@ Code quality improvements:
 Polish stage 1
 
 Handles:
+Setting rotation to an SO and then adding a child to it will incorrectly transform the child (scaling/skewing it)
 Move plane handles could be more precise
-Cut-off angle for rotation discs is wrong after they rotate
-Rotation disc normals are wrong on one side (likely it's double-sided)
-Once rotated further rotations seem to be going in wrong direction and gizmo is messed up
-Rotations seems to be going in the wrong direction
 Rotations around center incorrectly move and rotate the rotation gizmos
-Free rotation handle doesn't seem to be implemented
 Investigate rotation local/global (also move, I don't think its implemented properly, I seem to be modifying only Position)
 Investigate scale
 Check multi-select move/Rotate/scale
+Rotation disc normals are wrong on one side (likely it's double-sided)
+Free rotation handle doesn't seem to be implemented
 
 There's a large hang when doing certain operations like selecting a mesh in scene, or switching from center to pivot mode
 Crash on shutdown in mono_gchandle_free
 When selecting an gizmo icon the selection seems delayed and its gizmos flash for a frame before hiding (Can't reproduce atm but I saw it)
 Decent looking default layout
 ProjectLibrary seems to import some files on every start-up
+Selection rendering seems to be delayed 1 frame (noticeable when rotating/moving)
 
 Fix handles
  - Some handle functionality is unfinished