Depth buffer reading

Source/BansheeCore/Include/BsPixelData.h

@@ -326,6 +326,18 @@ namespace BansheeEngine
 		 */
 		void setColors(Color* colors, UINT32 numElements);
 
+		/** Returns depth at the specified coordinates. */
+		float getDepthAt(UINT32 x, UINT32 y, UINT32 z = 0) const;
+
+		/** Sets the depth at the specified coordinates. */
+		void setDepthAt(float const &cv, UINT32 x, UINT32 y, UINT32 z = 0);
+
+		/**
+		* Converts all the internal data into an array of float. Array is mapped as such:
+		* arrayIdx = x + y * width + z * width * height.
+		*/
+		Vector<float> getDepths() const;
+
 		/**
 		 * Constructs a new object with an internal buffer capable of holding "extents" volume of pixels, where each pixel
 		 * is of the specified pixel format. Extent offsets are also stored, but are not used internally.

Source/BansheeCore/Include/BsPixelUtil.h

@@ -189,6 +189,12 @@ namespace BansheeEngine
 		 * is not natively floating point a conversion is done in such a way that returned values range [0.0, 1.0].
 		 */
         static void unpackColor(float* r, float* g, float* b, float* a, PixelFormat format, const void* src); 
+
+		/** Writes depth to the provided memory location. */
+		static void packDepth(float depth, const PixelFormat format, void* dest);
+
+		/** Reads the depth from the provided memory location. */
+		static void unpackDepth(float& depth, PixelFormat format, void* src);
         
 		/**
 		 * Converts pixels from one format to another. Provided pixel data objects must have previously allocated buffers

Source/BansheeCore/Source/BsPixelData.cpp

@@ -176,6 +176,58 @@ namespace BansheeEngine
 		setColorsInternal(colors, numElements);
 	}
 
+	float PixelData::getDepthAt(UINT32 x, UINT32 y, UINT32 z) const
+	{
+		float depth;
+
+		UINT32 pixelSize = PixelUtil::getNumElemBytes(mFormat);
+		UINT32 pixelOffset = pixelSize * (z * mSlicePitch + y * mRowPitch + x);
+		PixelUtil::unpackDepth(depth, mFormat, (unsigned char *)getData() + pixelOffset);
+
+		return depth;
+	}
+
+	void PixelData::setDepthAt(float const& cv, UINT32 x, UINT32 y, UINT32 z)
+	{
+		UINT32 pixelSize = PixelUtil::getNumElemBytes(mFormat);
+		UINT32 pixelOffset = pixelSize * (z * mSlicePitch + y * mRowPitch + x);
+		PixelUtil::packDepth(cv, mFormat, (unsigned char *)getData() + pixelOffset);
+	}
+
+	Vector<float> PixelData::getDepths() const
+	{
+		UINT32 depth = mExtents.getDepth();
+		UINT32 height = mExtents.getHeight();
+		UINT32 width = mExtents.getWidth();
+
+		UINT32 pixelSize = PixelUtil::getNumElemBytes(mFormat);
+		UINT8* data = getData();
+
+		Vector<float> depths(width * height * depth);
+		for (UINT32 z = 0; z < depth; z++)
+		{
+			UINT32 zArrayIdx = z * width * height;
+			UINT32 zDataIdx = z * mSlicePitch * pixelSize;
+
+			for (UINT32 y = 0; y < height; y++)
+			{
+				UINT32 yArrayIdx = y * width;
+				UINT32 yDataIdx = y * mRowPitch * pixelSize;
+
+				for (UINT32 x = 0; x < width; x++)
+				{
+					UINT32 arrayIdx = x + yArrayIdx + zArrayIdx;
+					UINT32 dataIdx = x * pixelSize + yDataIdx + zDataIdx;
+
+					UINT8* dest = data + dataIdx;
+					PixelUtil::unpackDepth(depths[arrayIdx], mFormat, dest);
+				}
+			}
+		}
+
+		return depths;
+	}
+
 	SPtr<PixelData> PixelData::create(const PixelVolume &extents, PixelFormat pixelFormat)
 	{
 		SPtr<PixelData> pixelData = bs_shared_ptr_new<PixelData>(extents, pixelFormat);

Source/BansheeCore/Source/BsPixelUtil.cpp

@@ -740,11 +740,11 @@ namespace BansheeEngine
 	//-----------------------------------------------------------------------
 		{"PF_D32_S8X24",
 		/* Bytes per element */
-		4,
+		8,
 		/* Flags */
 		PFF_DEPTH | PFF_FLOAT,
 		/* Component type and count */
-		PCT_FLOAT32, 1,
+		PCT_FLOAT32, 2,
 		/* rbits, gbits, bbits, abits */
 		0, 0, 0, 0,
 		/* Masks and shifts */
@@ -753,11 +753,11 @@ namespace BansheeEngine
 	//-----------------------------------------------------------------------
 		{"PF_D24_S8",
 		/* Bytes per element */
-		8,
+		4,
 		/* Flags */
 		PFF_DEPTH | PFF_FLOAT,
 		/* Component type and count */
-		PCT_FLOAT32, 2,
+		PCT_FLOAT32, 1,
 		/* rbits, gbits, bbits, abits */
 		0, 0, 0, 0,
 		/* Masks and shifts */
@@ -1275,7 +1275,6 @@ namespace BansheeEngine
     void PixelUtil::unpackColor(float* r, float* g, float* b, float* a, PixelFormat format, const void* src)
     {
         const PixelFormatDescription &des = getDescriptionFor(format);
-
         if(des.flags & PFF_NATIVEENDIAN) 
 		{
             // Shortcut for integer formats unpacking
@@ -1359,6 +1358,32 @@ namespace BansheeEngine
         }
     }
 
+	void PixelUtil::packDepth(float depth, const PixelFormat format, void* dest)
+	{
+		if (!isDepth(format))
+		{
+			LOGERR("Cannot convert depth to " + getFormatName(format) + ": it is not a depth format");
+			return;
+		}
+			
+
+
+	}
+
+	void PixelUtil::unpackDepth(float& depth, PixelFormat format, void* src)
+	{
+		const PixelFormatDescription &des = getDescriptionFor(format);
+		if (!isDepth(format))
+		{
+			LOGERR("Cannot unpack from " + getFormatName(format) + ": it is not a depth format");
+			return;
+		}
+		//TODO fancy checks
+		UINT32* color = (UINT32 *)src;
+		float d = static_cast<float>(*color & 0x00FFFFFF);
+		depth = d / (float)16777216;
+	}
+
     void PixelUtil::bulkPixelConversion(const PixelData &src, PixelData &dst)
     {
         assert(src.getWidth() == dst.getWidth() &&

Source/BansheeEditor/Include/BsScenePicking.h

@@ -13,6 +13,13 @@ namespace BansheeEngine
 	 *  @{
 	 */
 
+	 /** Contains the results of a scene picking. */
+	struct PickResults
+	{
+		Vector<UINT32> results;
+		Vector3 pickPosition;
+	};
+
 	class ScenePickingCore;
 
 	/**	Handles picking of scene objects with a pointer in scene view. */

Source/BansheeEditor/Source/BsScenePicking.cpp

@@ -169,7 +169,11 @@ namespace BansheeEngine
 
 		assert(op.hasCompleted());
 
-		Vector<UINT32> selectedObjects = op.getReturnValue<Vector<UINT32>>();
+		PickResults returnValue = op.getReturnValue<PickResults>();
+		Vector3 pos = returnValue.pickPosition;
+		pos = cam->screenToWorldPoint(Vector2I(pos.x, pos.y), 
+			(1.0f / (pos.z + cam->getDeviceZTransform().y)) * cam->getDeviceZTransform().x);
+		Vector<UINT32> selectedObjects = returnValue.results;
 		Vector<HSceneObject> results;
 
 		for (auto& selectedObjectIdx : selectedObjects)
@@ -316,6 +320,10 @@ namespace BansheeEngine
 		const Vector2I& area, AsyncOp& asyncOp)
 	{
 		const RenderTargetProperties& rtProps = target->getProperties();
+		RenderAPICore& rs = RenderAPICore::instance();
+
+		rs.endFrame();
+		rs.setRenderTarget(nullptr);
 
 		if (rtProps.isWindow())
 		{
@@ -325,6 +333,7 @@ namespace BansheeEngine
 
 		SPtr<RenderTextureCore> rtt = std::static_pointer_cast<RenderTextureCore>(target);
 		SPtr<TextureCore> outputTexture = rtt->getBindableColorTexture();
+		SPtr<TextureCore> depthTexture = rtt->getBindableDepthStencilTexture();
 
 		if (position.x < 0 || position.x >= (INT32)outputTexture->getProperties().getWidth() ||
 			position.y < 0 || position.y >= (INT32)outputTexture->getProperties().getHeight())
@@ -334,17 +343,15 @@ namespace BansheeEngine
 		}
 
 		SPtr<PixelData> outputPixelData = outputTexture->getProperties().allocateSubresourceBuffer(0);
-		AsyncOp unused;
-
-		RenderAPICore& rs = RenderAPICore::instance();
+		SPtr<PixelData> depthPixelData = depthTexture->getProperties().allocateSubresourceBuffer(0);
 
-		rs.endFrame();
 		outputTexture->readSubresource(0, *outputPixelData);
 
 		Map<UINT32, UINT32> selectionScores;
 		UINT32 maxWidth = std::min((UINT32)(position.x + area.x), outputPixelData->getWidth());
 		UINT32 maxHeight = std::min((UINT32)(position.y + area.y), outputPixelData->getHeight());
 
+		bool needs = rtProps.requiresTextureFlipping();
 		if (rtProps.requiresTextureFlipping())
 		{
 			UINT32 vertOffset = outputPixelData->getHeight() - 1;
@@ -404,10 +411,20 @@ namespace BansheeEngine
 			return b.score < a.score;
 		});
 
-		Vector<UINT32> results;
+		Vector<UINT32> objects;
 		for (auto& selectedObject : selectedObjects)
-			results.push_back(selectedObject.index);
+			objects.push_back(selectedObject.index);
+		
+		depthTexture->readSubresource(0, *depthPixelData);
+		float depth;
+		if (rtProps.requiresTextureFlipping())
+			depth = depthPixelData->getDepthAt(position.x, depthPixelData->getHeight() - position.y);
+		else
+			depth = depthPixelData->getDepthAt(position.x, position.y);
 
-		asyncOp._completeOperation(results);
+		PickResults result;
+		result.results = objects;
+		result.pickPosition = Vector3(position.x, position.y, depth);
+		asyncOp._completeOperation(result);
 	}
 }

Source/BansheeEngine/Include/BsCamera.h

@@ -415,6 +415,15 @@ namespace BansheeEngine
 		 */
 		Ray screenPointToRay(const Vector2I& screenPoint) const;
 
+		/**
+		* Extracts the necessary values from the projection matrix that allow you to transform device Z value into
+		* world Z value.
+		*
+		* @return					Returns two values that can be used to transform device z to world z using this formula:
+		* 							z = (deviceZ + y) * x.
+		*/
+		Vector2 getDeviceZTransform() const;
+
 		/**	Projects a point from view to normalized device space. */
 		Vector3 projectPoint(const Vector3& point) const;
 

Source/BansheeEngine/Source/BsCamera.cpp

@@ -687,6 +687,45 @@ namespace BansheeEngine
 		return Vector3(dir.x, dir.y, dir.z);
 	}
 
+	Vector2 CameraBase::getDeviceZTransform() const
+	{
+		// Returns a set of values that will transform depth buffer values (e.g. [0, 1] in DX, [-1, 1] in GL) to a distance
+		// in world space. This involes applying the inverse projection transform to the depth value. When you multiply
+		// a vector with the projection matrix you get [clipX, clipY, Az + B, C * z], where we don't care about clipX/clipY.
+		// A is [2, 2], B is [2, 3] and C is [3, 2] elements of the projection matrix (only ones that matter for our depth 
+		// value). The hardware will also automatically divide the z value with w to get the depth, therefore the final 
+		// formula is:
+		// depth = (Az + B) / (C * z)
+
+		// To get the z coordinate back we simply do the opposite: 
+		// z = B / (depth * C - A)
+
+		// However some APIs will also do a transformation on the depth values before storing them to the texture 
+		// (e.g. OpenGL will transform from [-1, 1] to [0, 1]). And we need to reverse that as well. Therefore the final 
+		// formula is:
+		// z = B / ((depth * (maxDepth - minDepth) + minDepth) * C - A)
+
+		// Are we reorganize it because it needs to fit the "(1.0f / (depth + y)) * x" format used in the shader:
+		// z = 1.0f / (depth + minDepth/(maxDepth - minDepth) - A/((maxDepth - minDepth) * C)) * B/((maxDepth - minDepth) * C)
+
+		RenderAPICore& rapi = RenderAPICore::instance();
+		const RenderAPIInfo& rapiInfo = rapi.getAPIInfo();
+		Matrix4 projMatrix = mProjMatrix;
+
+		float depthRange = rapiInfo.getMaximumDepthInputValue() - rapiInfo.getMinimumDepthInputValue();
+		float minDepth = rapiInfo.getMinimumDepthInputValue();
+
+		float a = projMatrix[2][2];
+		float b = projMatrix[2][3];
+		float c = projMatrix[3][2];
+
+		Vector2 output;
+		output.x = b / (depthRange * c);
+		output.y = minDepth / depthRange - a / (depthRange * c);
+
+		return output;
+	}
+
 	CameraCore::~CameraCore()
 	{
 		RendererManager::instance().getActive()->notifyCameraRemoved(this);

Source/RenderBeast/Include/BsRendererCamera.h

@@ -67,16 +67,6 @@ namespace BansheeEngine
 		CameraShaderData getShaderData();
 
 	private:
-		/**
-		 * Extracts the necessary values from the projection matrix that allow you to transform device Z value into
-		 * world Z value.
-		 * 			
-		 * @param[in]	projMatrix	Projection matrix that was used to create the device Z value to transform.
-		 * @return					Returns two values that can be used to transform device z to world z using this formula:
-		 * 							z = (deviceZ + y) * x.
-		 */
-		Vector2 getDeviceZTransform(const Matrix4& projMatrix);
-
 		const CameraCore* mCamera;
 		SPtr<RenderQueue> mOpaqueQueue;
 		SPtr<RenderQueue> mTransparentQueue;

Source/RenderBeast/Source/BsRendererCamera.cpp

@@ -119,44 +119,6 @@ namespace BansheeEngine
 		mTransparentQueue->sort();
 	}
 
-	Vector2 RendererCamera::getDeviceZTransform(const Matrix4& projMatrix)
-	{
-		// Returns a set of values that will transform depth buffer values (e.g. [0, 1] in DX, [-1, 1] in GL) to a distance
-		// in world space. This involes applying the inverse projection transform to the depth value. When you multiply
-		// a vector with the projection matrix you get [clipX, clipY, Az + B, C * z], where we don't care about clipX/clipY.
-		// A is [2, 2], B is [2, 3] and C is [3, 2] elements of the projection matrix (only ones that matter for our depth 
-		// value). The hardware will also automatically divide the z value with w to get the depth, therefore the final 
-		// formula is:
-		// depth = (Az + B) / (C * z)
-
-		// To get the z coordinate back we simply do the opposite: 
-		// z = B / (depth * C - A)
-
-		// However some APIs will also do a transformation on the depth values before storing them to the texture 
-		// (e.g. OpenGL will transform from [-1, 1] to [0, 1]). And we need to reverse that as well. Therefore the final 
-		// formula is:
-		// z = B / ((depth * (maxDepth - minDepth) + minDepth) * C - A)
-
-		// Are we reorganize it because it needs to fit the "(1.0f / (depth + y)) * x" format used in the shader:
-		// z = 1.0f / (depth + minDepth/(maxDepth - minDepth) - A/((maxDepth - minDepth) * C)) * B/((maxDepth - minDepth) * C)
-
-		RenderAPICore& rapi = RenderAPICore::instance();
-		const RenderAPIInfo& rapiInfo = rapi.getAPIInfo();
-
-		float depthRange = rapiInfo.getMaximumDepthInputValue() - rapiInfo.getMinimumDepthInputValue();
-		float minDepth = rapiInfo.getMinimumDepthInputValue();
-
-		float a = projMatrix[2][2];
-		float b = projMatrix[2][3];
-		float c = projMatrix[3][2];
-
-		Vector2 output;
-		output.x = b / (depthRange * c);
-		output.y = minDepth / depthRange - a / (depthRange * c);
-
-		return output;
-	}
-
 	CameraShaderData RendererCamera::getShaderData()
 	{
 		CameraShaderData data;
@@ -180,7 +142,7 @@ namespace BansheeEngine
 		data.screenToWorld = data.invViewProj * projZ;
 		data.viewDir = mCamera->getForward();
 		data.viewOrigin = mCamera->getPosition();
-		data.deviceZToWorldZ = getDeviceZTransform(data.proj);
+		data.deviceZToWorldZ = mCamera->getDeviceZTransform();
 
 		SPtr<ViewportCore> viewport = mCamera->getViewport();
 		SPtr<RenderTargetCore> rt = viewport->getTarget();