All wire/solid gizmos render properly

Fixed DX9 memory corruption when flipping colors in mesh heap

BansheeCore/Source/BsMeshHeap.cpp

@@ -284,7 +284,7 @@ namespace BansheeEngine
 						continue;
 
 					UINT8* colorData = vertDest + mVertexDesc->getElementOffsetFromStream(VES_COLOR, semanticIdx, i);
-					for(UINT32 j = 0; j < mVertexData->vertexCount; j++)
+					for (UINT32 j = 0; j < meshData->getNumVertices(); j++)
 					{
 						UINT32* curColor = (UINT32*)colorData;
 

BansheeD3D9RenderSystem/Source/BsD3D9IndexBuffer.cpp

@@ -270,33 +270,33 @@ namespace BansheeEngine
 		assert(bufferResources->mBuffer != nullptr);
 		assert(bufferResources->mOutOfDate);
 		
-		if (bufferResources->mLockLength == 0)
-			return true;
+		if (bufferResources->mLockLength != 0)
+		{
+			void* dstBytes;
+			HRESULT hr;
 
-		void* dstBytes;
-		HRESULT hr;
-	
-		// Lock the buffer.
-		hr = bufferResources->mBuffer->Lock(
-			static_cast<UINT>(bufferResources->mLockOffset), 
-			static_cast<UINT>(bufferResources->mLockLength), 
-			&dstBytes,
-			D3D9Mappings::get(bufferResources->mLockOptions, mUsage));
+			// Lock the buffer.
+			hr = bufferResources->mBuffer->Lock(
+				static_cast<UINT>(bufferResources->mLockOffset),
+				static_cast<UINT>(bufferResources->mLockLength),
+				&dstBytes,
+				D3D9Mappings::get(bufferResources->mLockOptions, mUsage));
 
-		if (FAILED(hr))
-		{
-			String msg = DXGetErrorDescription(hr);
-			BS_EXCEPT(RenderingAPIException, "Cannot lock D3D9 vertex buffer: " + msg);
-		}
+			if (FAILED(hr))
+			{
+				String msg = DXGetErrorDescription(hr);
+				BS_EXCEPT(RenderingAPIException, "Cannot lock D3D9 vertex buffer: " + msg);
+			}
 
-		memcpy(dstBytes, systemMemoryBuffer + bufferResources->mLockOffset, bufferResources->mLockLength);
+			memcpy(dstBytes, systemMemoryBuffer + bufferResources->mLockOffset, bufferResources->mLockLength);
 
-		// Unlock the buffer.
-		hr = bufferResources->mBuffer->Unlock();
-		if (FAILED(hr))
-		{
-			String msg = DXGetErrorDescription(hr);
-			BS_EXCEPT(RenderingAPIException, "Cannot unlock D3D9 vertex buffer: " + msg);
+			// Unlock the buffer.
+			hr = bufferResources->mBuffer->Unlock();
+			if (FAILED(hr))
+			{
+				String msg = DXGetErrorDescription(hr);
+				BS_EXCEPT(RenderingAPIException, "Cannot unlock D3D9 vertex buffer: " + msg);
+			}
 		}
 
 		bufferResources->mOutOfDate = false;

BansheeD3D9RenderSystem/Source/BsD3D9VertexBuffer.cpp

@@ -266,33 +266,33 @@ namespace BansheeEngine
 		assert(bufferResources->mBuffer != nullptr);
 		assert(bufferResources->mOutOfDate);
 				
-		if (bufferResources->mLockLength == 0)
-			return true;
+		if (bufferResources->mLockLength != 0)
+		{
+			void* dstBytes;
+			HRESULT hr;
 
-		void* dstBytes;
-		HRESULT hr;
-		
-		// Lock the buffer.
-		hr = bufferResources->mBuffer->Lock(
-			static_cast<UINT>(bufferResources->mLockOffset), 
-			static_cast<UINT>(bufferResources->mLockLength), 
-			&dstBytes,
-			D3D9Mappings::get(bufferResources->mLockOptions, mUsage));
+			// Lock the buffer.
+			hr = bufferResources->mBuffer->Lock(
+				static_cast<UINT>(bufferResources->mLockOffset),
+				static_cast<UINT>(bufferResources->mLockLength),
+				&dstBytes,
+				D3D9Mappings::get(bufferResources->mLockOptions, mUsage));
 
-		if (FAILED(hr))
-		{
-			String msg = DXGetErrorDescription(hr);
-			BS_EXCEPT(RenderingAPIException, "Cannot lock D3D9 vertex buffer: " + msg);
-		}
+			if (FAILED(hr))
+			{
+				String msg = DXGetErrorDescription(hr);
+				BS_EXCEPT(RenderingAPIException, "Cannot lock D3D9 vertex buffer: " + msg);
+			}
 
-		memcpy(dstBytes, systemMemoryBuffer + bufferResources->mLockOffset, bufferResources->mLockLength);
+			memcpy(dstBytes, systemMemoryBuffer + bufferResources->mLockOffset, bufferResources->mLockLength);
 
-		// Unlock the buffer.
-		hr = bufferResources->mBuffer->Unlock();
-		if (FAILED(hr))
-		{
-			String msg = DXGetErrorDescription(hr);
-			BS_EXCEPT(RenderingAPIException, "Cannot unlock D3D9 vertex buffer: " + msg);
+			// Unlock the buffer.
+			hr = bufferResources->mBuffer->Unlock();
+			if (FAILED(hr))
+			{
+				String msg = DXGetErrorDescription(hr);
+				BS_EXCEPT(RenderingAPIException, "Cannot unlock D3D9 vertex buffer: " + msg);
+			}
 		}
 
 		bufferResources->mOutOfDate = false;

BansheeEditor/Include/BsGizmoManager.h

@@ -164,7 +164,6 @@ namespace BansheeEngine
 		{
 			MaterialProxyPtr proxy;
 			GpuParamMat4 mViewProj;
-			GpuParamMat4 mViewIT;
 		};
 
 		struct WireMaterialData

BansheeEditor/Source/BsBuiltinEditorResources.cpp

@@ -1393,7 +1393,6 @@ namespace BansheeEngine
 		mShaderGizmoSolid = Shader::create("GizmoSolid");
 
 		mShaderGizmoSolid->addParameter("matViewProj", "matViewProj", GPDT_MATRIX_4X4);
-		mShaderGizmoSolid->addParameter("matViewIT", "matViewIT", GPDT_MATRIX_4X4);
 
 		TechniquePtr newTechnique = mShaderGizmoSolid->addTechnique(mActiveRenderSystem, RendererInvariant);
 		PassPtr newPass = newTechnique->addPass();

BansheeEditor/Source/BsGizmoManager.cpp

@@ -610,7 +610,6 @@ namespace BansheeEngine
 			GpuParamsPtr vertParams = proxy->params[proxy->passes[0].vertexProgParamsIdx];
 
 			vertParams->getParam("matViewProj", mSolidMaterial.mViewProj);
-			vertParams->getParam("matViewIT", mSolidMaterial.mViewIT);
 		}
 
 		{
@@ -690,10 +689,8 @@ namespace BansheeEngine
 		THROW_IF_NOT_CORE_THREAD;
 
 		Matrix4 viewProjMat = projMatrix * viewMatrix;
-		Matrix4 viewIT = viewMatrix.inverse().transpose();
 
 		mSolidMaterial.mViewProj.set(viewProjMat);
-		mSolidMaterial.mViewIT.set(viewIT);
 
 		Renderer::setPass(*mSolidMaterial.proxy, 0);
 		Renderer::draw(*meshProxy);

BansheeEditor/Source/BsSceneEditorWidget.cpp

@@ -69,8 +69,6 @@ namespace BansheeEngine
 		if (mCameraController)
 		{
 			mCameraController->update();
-
-			LOGWRN(toString(mCameraController->sceneObject()->getWorldPosition()));
 		}
 
 		//// DEBUG ONLY

BansheeEditorExec/BsEditorExec.cpp

@@ -11,7 +11,7 @@ int CALLBACK WinMain(
 	_In_  int nCmdShow
 	)
 {
-	EditorApplication::startUp(RenderSystemPlugin::OpenGL);
+	EditorApplication::startUp(RenderSystemPlugin::DX11);
 	EditorApplication::instance().runMainLoop();
 	EditorApplication::shutDown();
 

BansheeEngine/Include/BsShapeMeshes3D.h

@@ -191,6 +191,7 @@ namespace BansheeEngine
 		 * @brief	Fills the mesh data with vertices representing an outline of an camera frustum. Frustum will
 		 *			be facing -z and be positioned at world origin.
 		 *
+		 * @param	position		Starting point for the frustum.
 		 * @param	aspect			Aspect ratio (width / height).
 		 * @param	FOV				Horizontal field of view angle.
 		 * @param	near			Distance to near clipping plane.
@@ -206,7 +207,7 @@ namespace BansheeEngine
 		 *
 		 *			Primitives are output in the form of a line list.
 		 */
-		static void wireFrustum(float aspect, Degree FOV, float near, float far,
+		static void wireFrustum(const Vector3& position, float aspect, Degree FOV, float near, float far,
 			const MeshDataPtr& meshData, UINT32 vertexOffset, UINT32 indexOffset);
 
 		/**
@@ -541,6 +542,7 @@ namespace BansheeEngine
 		/**
 		 * @brief	Fills the provided buffers with position and index data representing an outline of an axis aligned box.
 		 *
+		 * @param	position		Starting point for the frustum.
 		 * @param	aspect			Aspect ratio (width / height).
 		 * @param	FOV				Horizontal field of view angle.
 		 * @param	near			Distance to near clipping plane.
@@ -551,7 +553,7 @@ namespace BansheeEngine
 		 * @param	outIndices		Output buffer that will store the index data. Indices are 32bit.
 		 * @param	indexOffset 	Offset in number of indices from the start of the buffer to start writing at.
 		 */
-		static void wireFrustum(float aspect, Degree FOV, float near, float far,
+		static void wireFrustum(const Vector3& position, float aspect, Degree FOV, float near, float far,
 			UINT8* outVertices, UINT32 vertexOffset, UINT32 vertexStride, UINT32* outIndices, UINT32 indexOffset);
 
 		/**

BansheeEngine/Source/BsDrawHelper.cpp

@@ -311,20 +311,20 @@ namespace BansheeEngine
 			curIndexOffet += 36;
 		}
 
-		for (auto& discData : mSolidSphereData)
+		for (auto& sphereData : mSolidSphereData)
 		{
 			UINT32 numVertices, numIndices;
-			ShapeMeshes3D::getNumElementsSphere(discData.quality, numVertices, numIndices);
+			ShapeMeshes3D::getNumElementsSphere(sphereData.quality, numVertices, numIndices);
 
-			Sphere sphere(discData.position, discData.radius);
-			ShapeMeshes3D::solidSphere(sphere, meshData, curVertexOffset, curIndexOffet, discData.quality);
+			Sphere sphere(sphereData.position, sphereData.radius);
+			ShapeMeshes3D::solidSphere(sphere, meshData, curVertexOffset, curIndexOffet, sphereData.quality);
 
-			Matrix4 transformIT = discData.transform.inverseAffine().transpose();
-			RGBA color = discData.color.getAsRGBA();
+			Matrix4 transformIT = sphereData.transform.inverseAffine().transpose();
+			RGBA color = sphereData.color.getAsRGBA();
 
 			for (UINT32 i = 0; i < numVertices; i++)
 			{
-				Vector3 worldPos = discData.transform.multiply3x4(positionIter.getValue());
+				Vector3 worldPos = sphereData.transform.multiply3x4(positionIter.getValue());
 				Vector3 worldNormal = transformIT.multiply3x4(normalIter.getValue());
 
 				positionIter.addValue(worldPos);
@@ -336,20 +336,20 @@ namespace BansheeEngine
 			curIndexOffet += numIndices;
 		}
 
-		for (auto& discData : mConeData)
+		for (auto& coneData : mConeData)
 		{
 			UINT32 numVertices, numIndices;
-			ShapeMeshes3D::getNumElementsCone(discData.quality, numVertices, numIndices);
+			ShapeMeshes3D::getNumElementsCone(coneData.quality, numVertices, numIndices);
 
-			ShapeMeshes3D::solidCone(discData.base, discData.normal, discData.height, discData.radius, 
-				meshData, curVertexOffset, curIndexOffet, discData.quality);
+			ShapeMeshes3D::solidCone(coneData.base, coneData.normal, coneData.height, coneData.radius,
+				meshData, curVertexOffset, curIndexOffet, coneData.quality);
 
-			Matrix4 transformIT = discData.transform.inverseAffine().transpose();
-			RGBA color = discData.color.getAsRGBA();
+			Matrix4 transformIT = coneData.transform.inverseAffine().transpose();
+			RGBA color = coneData.color.getAsRGBA();
 
 			for (UINT32 i = 0; i < numVertices; i++)
 			{
-				Vector3 worldPos = discData.transform.multiply3x4(positionIter.getValue());
+				Vector3 worldPos = coneData.transform.multiply3x4(positionIter.getValue());
 				Vector3 worldNormal = transformIT.multiply3x4(normalIter.getValue());
 
 				positionIter.addValue(worldPos);
@@ -506,7 +506,7 @@ namespace BansheeEngine
 
 		for (auto& frustumData : mFrustumData)
 		{
-			ShapeMeshes3D::wireFrustum(frustumData.aspect, frustumData.FOV, frustumData.near,
+			ShapeMeshes3D::wireFrustum(frustumData.position, frustumData.aspect, frustumData.FOV, frustumData.near,
 				frustumData.far, meshData, curVertexOffset, curIndexOffet);
 
 			RGBA color = frustumData.color.getAsRGBA();

BansheeEngine/Source/BsShapeMeshes3D.cpp

@@ -133,7 +133,7 @@ namespace BansheeEngine
 			meshData->getVertexDesc()->getVertexStride(), indexData, indexOffset, quality);
 	}
 
-	void ShapeMeshes3D::wireFrustum(float aspect, Degree FOV, float near, float far,
+	void ShapeMeshes3D::wireFrustum(const Vector3& position, float aspect, Degree FOV, float near, float far,
 		const MeshDataPtr& meshData, UINT32 vertexOffset, UINT32 indexOffset)
 	{
 		UINT32* indexData = meshData->getIndices32();
@@ -142,7 +142,7 @@ namespace BansheeEngine
 		assert((vertexOffset + 8) <= meshData->getNumVertices());
 		assert((indexOffset + 24) <= meshData->getNumIndices());
 
-		wireFrustum(aspect, FOV, near, far, positionData, vertexOffset, meshData->getVertexDesc()->getVertexStride(), indexData, indexOffset);
+		wireFrustum(position, aspect, FOV, near, far, positionData, vertexOffset, meshData->getVertexDesc()->getVertexStride(), indexData, indexOffset);
 	}
 
 	void ShapeMeshes3D::solidCone(const Vector3& base, const Vector3& normal, float height, float radius,
@@ -259,8 +259,9 @@ namespace BansheeEngine
 
 	void ShapeMeshes3D::getNumElementsWireSphere(UINT32 quality, UINT32& numVertices, UINT32& numIndices)
 	{
-		numVertices = 3 * ((quality + 1) * 4);
-		numIndices = 6 * ((quality + 1) * 4);
+		getNumElementsWireArc(quality, numVertices, numIndices);
+		numVertices *= 3;
+		numIndices *= 3;
 	}
 
 	void ShapeMeshes3D::getNumElementsArc(UINT32 quality, UINT32& numVertices, UINT32& numIndices)
@@ -384,19 +385,19 @@ namespace BansheeEngine
 		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::FAR_RIGHT_TOP));
 
 		// Bottom face
-		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::FAR_LEFT_TOP));
-		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::NEAR_LEFT_TOP));
-		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::NEAR_RIGHT_TOP));
-		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::FAR_RIGHT_TOP));
+		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::FAR_LEFT_BOTTOM));
+		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::FAR_RIGHT_BOTTOM));
+		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::NEAR_RIGHT_BOTTOM));
+		outVertices = writeVector3(outVertices, vertexStride, box.getCorner(AABox::NEAR_LEFT_BOTTOM));
 
 		static const Vector3 faceNormals[6] = 
 		{
 			Vector3(0, 0, 1),
 			Vector3(0, 0, -1),
-			Vector3(0, 1, 0),
-			Vector3(0, -1, 0),
+			Vector3(-1, 0, 0),
 			Vector3(1, 0, 0),
-			Vector3(-1, 0, 0)
+			Vector3(0, 1, 0),
+			Vector3(0, -1, 0)
 		};
 
 		outNormals += (vertexOffset * vertexStride);
@@ -413,12 +414,12 @@ namespace BansheeEngine
 		{
 			UINT32 faceVertOffset = vertexOffset + face * 4;
 
-			indices[face * 6 + 0] = faceVertOffset + 0;
+			indices[face * 6 + 0] = faceVertOffset + 2;
 			indices[face * 6 + 1] = faceVertOffset + 1;
-			indices[face * 6 + 2] = faceVertOffset + 2;
-			indices[face * 6 + 3] = faceVertOffset + 2;
+			indices[face * 6 + 2] = faceVertOffset + 0;
+			indices[face * 6 + 3] = faceVertOffset + 0;
 			indices[face * 6 + 4] = faceVertOffset + 3;
-			indices[face * 6 + 5] = faceVertOffset + 0;
+			indices[face * 6 + 5] = faceVertOffset + 2;
 		}
 	}
 
@@ -459,19 +460,19 @@ namespace BansheeEngine
 		for (int i = 0; i < 20; ++i) 
 		{
 			curVertOffset += subdivideTriangleOnSphere(sphere.getCenter(), sphere.getRadius(), quality,
-				vertices[triangles[i][0]], vertices[triangles[i][1]], vertices[triangles[i][2]],
+				vertices[triangles[i][2]], vertices[triangles[i][1]], vertices[triangles[i][0]],
 				outVertices, outNormals, curVertOffset, vertexStride);
 		}
 
 		// Create indices
 		outIndices += indexOffset;
 
-		UINT32 numTriangles = 20 * (3 * ((UINT32)std::pow(4L, (long)quality)));
-		for (UINT32 i = 0; i < numTriangles * 3; i += 3)
+		UINT32 numIndices = 20 * (3 * ((UINT32)std::pow(4L, (long)quality)));
+		for (UINT32 i = 0; i < numIndices; i += 3)
 		{
-			outIndices[i] = vertexOffset + i;
+			outIndices[i] = vertexOffset + i + 2;
 			outIndices[i + 1] = vertexOffset + i + 1;
-			outIndices[i + 2] = vertexOffset + i + 2;
+			outIndices[i + 2] = vertexOffset + i + 0;
 		}
 	}
 
@@ -519,10 +520,10 @@ namespace BansheeEngine
 		}
 	}
 
-	void ShapeMeshes3D::wireFrustum(float aspect, Degree FOV, float near, float far,
+	void ShapeMeshes3D::wireFrustum(const Vector3& position, float aspect, Degree FOV, float near, float far,
 		UINT8* outVertices, UINT32 vertexOffset, UINT32 vertexStride, UINT32* outIndices, UINT32 indexOffset)
 	{
-		float fovTan = Math::tan(FOV);
+		float fovTan = Math::tan(FOV * 0.5f);
 
 		Vector3 nearPoint(0, 0, near);
 		Vector3 nearWidth(near * fovTan * aspect, 0, 0);
@@ -547,7 +548,7 @@ namespace BansheeEngine
 		outVertices += vertexOffset * vertexStride;
 
 		for (UINT32 i = 0; i < 8; i++)
-			outVertices = writeVector3(outVertices, vertexStride, points[i]);
+			outVertices = writeVector3(outVertices, vertexStride, position + points[i]);
 
 		outIndices += indexOffset;
 
@@ -872,16 +873,15 @@ namespace BansheeEngine
 			Vector3 sub3 = Vector3::normalize((c + a) * 0.5f);
 
 			numLevels--;
-			UINT32 curVertOffset = vertexOffset + numVertices;
 
 			numVertices += subdivideTriangleOnSphere(center, radius, numLevels, a, sub1, sub3, outVertices, 
-				outNormals, vertexOffset + numVertices, vertexStride);
+				outNormals, numVertices, vertexStride);
 			numVertices += subdivideTriangleOnSphere(center, radius, numLevels, sub1, b, sub2, outVertices, 
-				outNormals, vertexOffset + numVertices, vertexStride);
+				outNormals, numVertices, vertexStride);
 			numVertices += subdivideTriangleOnSphere(center, radius, numLevels, sub1, sub2, sub3, outVertices, 
-				outNormals, vertexOffset + numVertices, vertexStride);
+				outNormals, numVertices, vertexStride);
 			numVertices += subdivideTriangleOnSphere(center, radius, numLevels, sub3, sub2, c, outVertices, 
-				outNormals, vertexOffset + numVertices, vertexStride);
+				outNormals, numVertices, vertexStride);
 		}
 		else
 		{

BansheeGLRenderSystem/Source/GLSL/src/BsGLSLGpuProgram.cpp

@@ -165,7 +165,7 @@ namespace BansheeEngine
 			}
 
 			mGLHandle = glCreateShaderProgramv(shaderType, (GLsizei)lines.size(), (const GLchar**)lines.data());
-			
+
 			for (auto iter = lines.rbegin(); iter != lines.rend(); ++iter)
 			{
 				stackDeallocLast(*iter);
@@ -186,6 +186,10 @@ namespace BansheeEngine
 				mVertexDeclaration = HardwareBufferManager::instance().createVertexDeclaration(elementList);
 			}
 		}
+		else
+		{
+			LOGWRN("Shader compilation/linking failed: " + mCompileError);
+		}
 
 		BS_INC_RENDER_STAT_CAT(ResCreated, RenderStatObject_GpuProgram);
 		GpuProgram::initialize_internal();

MBansheeEditor/DbgGizmo.cs

@@ -13,6 +13,12 @@ namespace BansheeEditor
         private static void DrawDbgComponentGizmo(DbgGizmoComponent target)
         {
             Gizmos.DrawCube(target.sceneObject.position, new Vector3(1, 1, 1));
+            Gizmos.DrawSphere(target.sceneObject.position + 2 * Vector3.xAxis, 1.0f);
+            Gizmos.DrawWireCube(target.sceneObject.position + 4 * Vector3.xAxis, new Vector3(1, 1, 1));
+            Gizmos.DrawWireSphere(target.sceneObject.position + 6 * Vector3.xAxis, 1.0f);
+            Gizmos.DrawLine(target.sceneObject.position + 7.5f * Vector3.xAxis,
+                target.sceneObject.position + 8.5f * Vector3.xAxis);
+            Gizmos.DrawFrustum(target.sceneObject.position + 10 * Vector3.xAxis, 1920.0f / 1080.0f, 90, 1.0f, 1000.0f);
         }
     }
 }

SceneView.txt

@@ -5,20 +5,18 @@
 	- Then make a Component wrapper around the non-component types, and also a C# wrapper around the same types
 
 TESTING:
-Ensure all 3 render systems compile and run
- - Test selection on all 3 render systems
- - Picking need to test something with alpha
+ - Depth rendering in scene view on OpenGL is all screwed up
+ - Test picking on an object with alpha
  - Ensure that selecting an item in scene properly marks it in scene view
  - Ensure that selecting an item in scene or resource tree view properly updates Selection
 
 Test gizmos
- - START UP GIZMO MANAGER SOMEWHERE
- - Test rendering of basic wire and solid gizmos
+ - Test all solid/wire gizmos shapes on all render systems
  - Test rendering of icon gizmos
- - Test them all from C#
  - HOOK UP GIZMO SELECTION and test it
 
 Test handles
+ - Handle rendering might require similar refactor to GizmoManager
  - Test basic move handle
  - Test a custom handle from C#
  - FINISH HANDLE IMPLEMENTATION AND GLUE EVERYTHING TOGHETHER
@@ -33,6 +31,7 @@ Need a way to drag and drop items from Scene tree view to Scene view
 LATER:
  - Need a way to render text for gizmos and handles, and in scene in general
  - Add drag to select
+ - Need a better system to catch broken shaders. DX11 just draws nothing with depth, DX9 draws all white.
 
 ----------------------------------------------------------------------
 Handles