Fixed unprojection and picking in new renderer

Core/Contents/Source/PolyCamera.cpp

@@ -510,12 +510,14 @@ void Camera::setOrthoMatrix(Matrix4 &matrix, Number xSize, Number ySize, Number
 
 Vector3 Camera::projectRayFrom2DCoordinate(const Vector2 &coordinate, const Polycode::Rectangle &viewport) {
     Matrix4 camInverse = getConcatenatedMatrix().Inverse();
+    Matrix4 projectionMatrix = getProjectionMatrix();
     
-    Vector3 nearPlane = Renderer::unProject(Vector3(coordinate.x, coordinate.y, 0.0), camInverse, getProjectionMatrix(), viewport);
-    Vector3 farPlane = Renderer::unProject(Vector3(coordinate.x, coordinate.y, 1.0), camInverse, getProjectionMatrix(), viewport);
-    
+    Vector3 nearPlane = Renderer::unProject(Vector3(coordinate.x, coordinate.y, 0.0), camInverse, projectionMatrix, viewport);
+    Vector3 farPlane = Renderer::unProject(Vector3(coordinate.x, coordinate.y, 1.0), camInverse, projectionMatrix, viewport);
+
     Vector3 dirVec = (farPlane) - (nearPlane);
     dirVec.Normalize();
+    
     return dirVec;
 }
 

Core/Contents/Source/PolyRenderer.cpp

@@ -394,8 +394,8 @@ Vector3 Renderer::project(const Vector3 &position, const Matrix4 &modelMatrix, c
     in.y = in.y * 0.5 + 0.5;
     in.z = in.z * 0.5 + 0.5;
     
-    in.x = in.x * (viewport.x + viewport.w) + viewport.x;
-    in.y = in.y * (viewport.y + viewport.h) + viewport.y;
+    in.x = in.x * (viewport.w) + viewport.x;
+    in.y = in.y * (viewport.h) + viewport.y;
     
     return Vector3(in.x, in.y, in.z);
 }
@@ -407,8 +407,10 @@ Vector3 Renderer::unProject(const Vector3 &position, const Matrix4 &modelMatrix,
     
     Vector4 in(position);
     
-    in.x = (in.x - viewport.x) / (viewport.x + viewport.w);
-    in.y = (in.y - viewport.y) / (viewport.y + viewport.h);
+    in.x = (in.x - viewport.x) / (viewport.w);
+    in.y = (in.y - viewport.y) / (viewport.h);
+    
+    in.y = 1.0 - in.y;
     
     in.x = in.x * 2.0 - 1.0;
     in.y = in.y * 2.0 - 1.0;

Core/Contents/Source/PolyScene.cpp

@@ -331,7 +331,7 @@ Ray Scene::projectRayFromCameraAndViewportCoordinate(Camera *camera, Vector2 coo
     
     if(remapMouse) {
         viewport.x = sceneMouseRect.x * renderer->getBackingResolutionScaleX();
-        viewport.y = (core->getYRes() - (sceneMouseRect.y + sceneMouseRect.h)) * renderer->getBackingResolutionScaleY();
+        viewport.y = sceneMouseRect.y * renderer->getBackingResolutionScaleY();
     }
     
     Vector3 dir =  camera->projectRayFrom2DCoordinate(Vector2(coordinate.x *  renderer->getBackingResolutionScaleX(), coordinate.y  * renderer->getBackingResolutionScaleY()), viewport);
@@ -380,7 +380,6 @@ Ray Scene::projectRayFromCameraAndViewportCoordinate(Camera *camera, Vector2 coo
 		break;		
 	}
 
-
 	return Ray(pos, dir);
 }
 
@@ -392,13 +391,14 @@ void Scene::handleEvent(Event *event) {
         }
     } else if(event->getDispatcher() == core->getInput() && rootEntity.processInputEvents) {
 		InputEvent *inputEvent = (InputEvent*) event;
-
+        
         if(constrainPickingToViewport) {
             Polycode::Rectangle v = activeCamera->getViewport();
             if(remapMouse) {
                 v.x = sceneMouseRect.x;
                 v.y = sceneMouseRect.y;
-            }            
+            }
+            
             if(inputEvent->mousePosition.x < v.x || inputEvent->mousePosition.x > v.x+(v.w / renderer->getBackingResolutionScaleX()) || inputEvent->mousePosition.y < v.y || inputEvent->mousePosition.y > v.y + (v.h/renderer->getBackingResolutionScaleY())) {
                     return;
             }

IDE/Contents/Source/PolycodeEntityEditor.cpp

@@ -1932,7 +1932,7 @@ void EntityEditorMainView::Resize(Number width, Number height) {
     
 	viewModeSelector->setPosition(width-viewModeSelector->getWidth()-10.0, 3);
     
-    Vector2 screenPos = renderTextureShape->getScreenPosition(globalScene->getDefaultCamera()->getProjectionMatrix(), globalScene->getDefaultCamera()->getTransformMatrix(), globalScene->getDefaultCamera()->getViewport());
+    Vector2 screenPos = renderTextureShape->getScreenPositionForMainCamera();
     
 	renderTexture->resizeRenderTexture(width, height-60);
 	renderTextureShape->setTexture(renderTexture->getTargetTexture());