Fixed DX11 and OpenGL GPU program matrix layout

Picking of solid objects works on DX9/DX11, OpenGL has flipped texture

BansheeD3D11RenderSystem/Source/BsD3D11GpuProgram.cpp

@@ -17,7 +17,7 @@ namespace BansheeEngine
 	D3D11GpuProgram::D3D11GpuProgram(const String& source, const String& entryPoint, GpuProgramType gptype, 
 		GpuProgramProfile profile, const Vector<HGpuProgInclude>* includes, bool isAdjacencyInfoRequired)
 		: GpuProgram(source, entryPoint, gptype, profile, includes, isAdjacencyInfoRequired),
-		mColumnMajorMatrices(true), mEnableBackwardsCompatibility(false), mProgramId(0)
+		mColumnMajorMatrices(false), mEnableBackwardsCompatibility(false), mProgramId(0)
 	{
 
 	}

BansheeEditor/Source/BsBuiltinEditorResources.cpp

@@ -1331,7 +1331,7 @@ namespace BansheeEngine
 		newPass->setFragmentProgram(psProgram);
 
 		RASTERIZER_STATE_DESC rasterizerDesc;
-		rasterizerDesc.scissorEnable = false;
+		rasterizerDesc.scissorEnable = true;
 		rasterizerDesc.cullMode = cullMode;
 
 		HRasterizerState rasterizerState = RasterizerState::create(rasterizerDesc);
@@ -1361,7 +1361,7 @@ namespace BansheeEngine
 		newPass->setFragmentProgram(psProgram);
 
 		RASTERIZER_STATE_DESC rasterizerDesc;
-		rasterizerDesc.scissorEnable = false;
+		rasterizerDesc.scissorEnable = true;
 		rasterizerDesc.cullMode = cullMode;
 
 		HRasterizerState rasterizerState = RasterizerState::create(rasterizerDesc);

BansheeEditor/Source/BsScenePicking.cpp

@@ -245,6 +245,14 @@ namespace BansheeEngine
 		RenderTexturePtr rtt = std::static_pointer_cast<RenderTexture>(rt);
 		TexturePtr outputTexture = rtt->getBindableColorTexture().getInternalPtr();
 
+		if (position.x < 0 || position.x >= (INT32)outputTexture->getWidth() || 
+			position.y < 0 || position.y >= (INT32)outputTexture->getHeight())
+		{
+			asyncOp._completeOperation(Vector<UINT32>());
+			return;
+		}
+
+		rs.beginFrame();
 		rs.setViewport(vp);
 		rs.clearRenderTarget(FBT_COLOR | FBT_DEPTH | FBT_STENCIL, Color::White);
 		rs.setScissorRect(position.x, position.y, position.x + area.x, position.y + area.y);
@@ -253,7 +261,6 @@ namespace BansheeEngine
 		bool activeMaterialIsAlpha = false;
 		CullingMode activeMaterialCull = (CullingMode)0;
 
-		rs.beginFrame();
 		for (auto& renderable : renderables)
 		{
 			if (activeMaterialIsAlpha != renderable.alpha || activeMaterialCull != renderable.cullMode)
@@ -298,12 +305,15 @@ namespace BansheeEngine
 
 		rs.readSubresource(outputTexture, 0, outputData, unused);
 
-		// TODO - Only search scissor rect
 		Map<UINT32, UINT32> selectionScores;
 		UINT32 numPixels = outputPixelData->getWidth() * outputPixelData->getHeight();
-		for (UINT32 y = 0; y < outputPixelData->getHeight(); y++)
+
+		UINT32 maxWidth = std::min((UINT32)(position.x + area.x), outputPixelData->getWidth());
+		UINT32 maxHeight = std::min((UINT32)(position.y + area.y), outputPixelData->getHeight());
+
+		for (UINT32 y = (UINT32)position.y; y < maxHeight; y++)
 		{
-			for (UINT32 x = 0; x < outputPixelData->getWidth(); x++)
+			for (UINT32 x = (UINT32)position.x; x < maxWidth; x++)
 			{
 				Color color = outputPixelData->getColorAt(x, y);
 				UINT32 index = decodeIndex(color);

BansheeEditorExec/BsEditorExec.cpp

@@ -11,7 +11,7 @@ int CALLBACK WinMain(
 	_In_  int nCmdShow
 	)
 {
-	EditorApplication::startUp(RenderSystemPlugin::DX11);
+	EditorApplication::startUp(RenderSystemPlugin::OpenGL);
 	EditorApplication::instance().runMainLoop();
 	EditorApplication::shutDown();
 

BansheeGLRenderSystem/Source/BsGLRenderSystem.cpp

@@ -317,6 +317,8 @@ namespace BansheeEngine
 			const UINT8* ptrData = uniformBufferData + paramDesc.cpuMemOffset * sizeof(UINT32);
 			hasBoundAtLeastOne = true;
 
+			// Note: We don't transpose matrices here even though we don't use column major format
+			// because they are assumed to be pre-transposed in the GpuParams buffer
 			switch(paramDesc.type)
 			{
 			case GPDT_FLOAT1:
@@ -336,15 +338,15 @@ namespace BansheeEngine
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_2X3:
-				glProgramUniformMatrix2x3fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
+				glProgramUniformMatrix3x2fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_2X4:
-				glProgramUniformMatrix2x4fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
+				glProgramUniformMatrix4x2fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_3X2:
-				glProgramUniformMatrix3x2fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
+				glProgramUniformMatrix2x3fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_3X3:
@@ -352,15 +354,15 @@ namespace BansheeEngine
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_3X4:
-				glProgramUniformMatrix3x4fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
+				glProgramUniformMatrix4x3fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_4X2:
-				glProgramUniformMatrix4x2fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
+				glProgramUniformMatrix2x4fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_4X3:
-				glProgramUniformMatrix4x3fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
+				glProgramUniformMatrix3x4fv(glProgram, paramDesc.gpuMemOffset, paramDesc.arraySize, 
 					GL_FALSE, (GLfloat*)ptrData);
 				break;
 			case GPDT_MATRIX_4X4:

BansheeGLRenderSystem/Source/GLSL/include/BsGLSLGpuProgram.h

@@ -41,6 +41,11 @@ namespace BansheeEngine
 		 */
 		const UINT32 getProgramID() const { return mProgramID; }
 
+		/**
+		 * @copydoc	GpuProgram::createParameters
+		 */
+		GpuParamsPtr createParameters();
+
 	private:
 		friend class GLSLProgramFactory;
 

BansheeGLRenderSystem/Source/GLSL/src/BsGLSLGpuProgram.cpp

@@ -7,6 +7,7 @@
 #include "BsHardwareBufferManager.h"
 #include "BsGLSLGpuProgramRTTI.h"
 #include "BsRenderStats.h"
+#include "BsGpuParams.h"
 
 namespace BansheeEngine 
 {
@@ -218,6 +219,13 @@ namespace BansheeEngine
 		return language;
 	}
 
+	GpuParamsPtr GLSLGpuProgram::createParameters()
+	{
+		GpuParamsPtr params = bs_shared_ptr<GpuParams, PoolAlloc>(std::ref(mParametersDesc), true);
+
+		return params;
+	}
+
 	/************************************************************************/
 	/* 								SERIALIZATION                      		*/
 	/************************************************************************/

SceneView.txt

@@ -9,7 +9,6 @@ TESTING:
 Ensure all 3 render systems compile and run
  - Test selection on all 3 render systems
  - RE-ENABLE SCISSOR TEST FOR PICKING SHADERS!!
- - FIX SHUTDOWN AS IT BREAKS BECAUSE OF Selection (and potentially others)
  - Picking need to test something with alpha
  - Ensure that selecting an item in scene properly marks it in scene view
  - Ensure that selecting an item in scene or resource tree view properly updates Selection