Implement clip masks for the GL2 renderer

Implement compiled geometry to enable rendering clip masks. Takes advantage of the new guarantee that the pointers to vertices and indices are stable until the call to release.

Backends/RmlUi_Backend_SDL_GL2.cpp

@@ -52,8 +52,7 @@ private:
 public:
 	RenderInterface_GL2_SDL(SDL_Renderer* renderer) : renderer(renderer) {}
 
-	void RenderGeometry(Rml::Vertex* vertices, int num_vertices, int* indices, int num_indices, Rml::TextureHandle texture,
-		const Rml::Vector2f& translation) override
+	void RenderCompiledGeometry(Rml::CompiledGeometryHandle handle, const Rml::Vector2f& translation, Rml::TextureHandle texture) override
 	{
 		SDL_Texture* sdl_texture = (SDL_Texture*)texture;
 		if (sdl_texture)
@@ -62,7 +61,7 @@ public:
 			texture = RenderInterface_GL2::TextureEnableWithoutBinding;
 		}
 
-		RenderInterface_GL2::RenderGeometry(vertices, num_vertices, indices, num_indices, texture, translation);
+		RenderInterface_GL2::RenderCompiledGeometry(handle, translation, texture);
 
 		if (sdl_texture)
 			SDL_GL_UnbindTexture(sdl_texture);

Backends/RmlUi_Backend_SFML_GL2.cpp

@@ -46,8 +46,7 @@ class RenderInterface_GL2_SFML : public RenderInterface_GL2 {
 public:
 	// -- Inherited from Rml::RenderInterface --
 
-	void RenderGeometry(Rml::Vertex* vertices, int num_vertices, int* indices, int num_indices, Rml::TextureHandle texture,
-		const Rml::Vector2f& translation) override
+	void RenderCompiledGeometry(Rml::CompiledGeometryHandle handle, const Rml::Vector2f& translation, Rml::TextureHandle texture) override
 	{
 		if (texture)
 		{
@@ -55,7 +54,7 @@ public:
 			texture = RenderInterface_GL2::TextureEnableWithoutBinding;
 		}
 
-		RenderInterface_GL2::RenderGeometry(vertices, num_vertices, indices, num_indices, texture, translation);
+		RenderInterface_GL2::RenderCompiledGeometry(handle, translation, texture);
 	}
 
 	bool LoadTexture(Rml::TextureHandle& texture_handle, Rml::Vector2i& texture_dimensions, const Rml::String& source) override

Backends/RmlUi_Renderer_GL2.cpp

@@ -72,6 +72,11 @@ void RenderInterface_GL2::BeginFrame()
 	glEnable(GL_BLEND);
 	glBlendFunc(GL_ONE, GL_ONE_MINUS_SRC_ALPHA);
 
+	glEnable(GL_STENCIL_TEST);
+	glStencilFunc(GL_ALWAYS, 1, GLuint(-1));
+	glStencilMask(GLuint(-1));
+	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+
 	Rml::Matrix4f projection = Rml::Matrix4f::ProjectOrtho(0, (float)viewport_width, (float)viewport_height, 0, -10000, 10000);
 	glMatrixMode(GL_PROJECTION);
 	glLoadMatrixf(projection.data());
@@ -88,13 +93,40 @@ void RenderInterface_GL2::EndFrame() {}
 void RenderInterface_GL2::Clear()
 {
 	glClearStencil(0);
-	glClearColor(0, 0, 0, 1);
+	glClearColor(0, 0, 0, 0);
 	glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
 }
 
-void RenderInterface_GL2::RenderGeometry(Rml::Vertex* vertices, int /*num_vertices*/, int* indices, int num_indices, const Rml::TextureHandle texture,
+void RenderInterface_GL2::RenderGeometry(Rml::Vertex* vertices, int num_vertices, int* indices, int num_indices, const Rml::TextureHandle texture,
 	const Rml::Vector2f& translation)
 {
+	Rml::CompiledGeometryHandle geometry = CompileGeometry(vertices, num_vertices, indices, num_indices);
+
+	if (geometry)
+	{
+		RenderCompiledGeometry(geometry, translation, texture);
+		ReleaseCompiledGeometry(geometry);
+	}
+}
+
+Rml::CompiledGeometryHandle RenderInterface_GL2::CompileGeometry(Rml::Vertex* vertices, int num_vertices, int* indices, int num_indices)
+{
+	GeometryView* data = new GeometryView{vertices, indices, num_vertices, num_indices};
+	return reinterpret_cast<Rml::CompiledGeometryHandle>(data);
+}
+
+void RenderInterface_GL2::ReleaseCompiledGeometry(Rml::CompiledGeometryHandle geometry)
+{
+	delete reinterpret_cast<GeometryView*>(geometry);
+}
+
+void RenderInterface_GL2::RenderCompiledGeometry(Rml::CompiledGeometryHandle handle, const Rml::Vector2f& translation, Rml::TextureHandle texture)
+{
+	const GeometryView* geometry = reinterpret_cast<GeometryView*>(handle);
+	const Rml::Vertex* vertices = geometry->vertices;
+	const int* indices = geometry->indices;
+	const int num_indices = geometry->num_indices;
+
 	glPushMatrix();
 	glTranslatef(translation.x, translation.y, 0);
 
@@ -125,62 +157,71 @@ void RenderInterface_GL2::RenderGeometry(Rml::Vertex* vertices, int /*num_vertic
 void RenderInterface_GL2::EnableScissorRegion(bool enable)
 {
 	if (enable)
-	{
-		if (!transform_enabled)
-		{
-			glEnable(GL_SCISSOR_TEST);
-			glDisable(GL_STENCIL_TEST);
-		}
-		else
-		{
-			glDisable(GL_SCISSOR_TEST);
-			glEnable(GL_STENCIL_TEST);
-		}
-	}
+		glEnable(GL_SCISSOR_TEST);
 	else
-	{
 		glDisable(GL_SCISSOR_TEST);
-		glDisable(GL_STENCIL_TEST);
-	}
 }
 
 void RenderInterface_GL2::SetScissorRegion(int x, int y, int width, int height)
 {
-	if (!transform_enabled)
-	{
-		glScissor(x, viewport_height - (y + height), width, height);
-	}
+	glScissor(x, viewport_height - (y + height), width, height);
+}
+
+void RenderInterface_GL2::EnableClipMask(bool enable)
+{
+	if (enable)
+		glEnable(GL_STENCIL_TEST);
 	else
+		glDisable(GL_STENCIL_TEST);
+}
+
+void RenderInterface_GL2::RenderToClipMask(Rml::ClipMaskOperation operation, Rml::CompiledGeometryHandle geometry, Rml::Vector2f translation)
+{
+	RMLUI_ASSERT(glIsEnabled(GL_STENCIL_TEST));
+	using Rml::ClipMaskOperation;
+
+	const bool clear_stencil = (operation == ClipMaskOperation::Set || operation == ClipMaskOperation::SetInverse);
+	if (clear_stencil)
 	{
-		// clear the stencil buffer
-		glStencilMask(GLuint(-1));
+		// @performance Increment the reference value instead of clearing each time.
 		glClear(GL_STENCIL_BUFFER_BIT);
+	}
 
-		// fill the stencil buffer
-		glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
-		glDepthMask(GL_FALSE);
-		glStencilFunc(GL_NEVER, 1, GLuint(-1));
-		glStencilOp(GL_REPLACE, GL_KEEP, GL_KEEP);
-
-		float fx = (float)x;
-		float fy = (float)y;
-		float fwidth = (float)width;
-		float fheight = (float)height;
-
-		// draw transformed quad
-		GLfloat vertices[] = {fx, fy, 0, fx, fy + fheight, 0, fx + fwidth, fy + fheight, 0, fx + fwidth, fy, 0};
-		glDisableClientState(GL_COLOR_ARRAY);
-		glVertexPointer(3, GL_FLOAT, 0, vertices);
-		GLushort indices[] = {1, 2, 0, 3};
-		glDrawElements(GL_TRIANGLE_STRIP, 4, GL_UNSIGNED_SHORT, indices);
-		glEnableClientState(GL_COLOR_ARRAY);
-
-		// prepare for drawing the real thing
-		glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
-		glDepthMask(GL_TRUE);
-		glStencilMask(0);
-		glStencilFunc(GL_EQUAL, 1, GLuint(-1));
+	GLint stencil_test_value = 0;
+	glGetIntegerv(GL_STENCIL_REF, &stencil_test_value);
+
+	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
+	glStencilFunc(GL_ALWAYS, GLint(1), GLuint(-1));
+
+	switch (operation)
+	{
+	case ClipMaskOperation::Set:
+	{
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		stencil_test_value = 1;
+	}
+	break;
+	case ClipMaskOperation::SetInverse:
+	{
+		glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+		stencil_test_value = 0;
 	}
+	break;
+	case ClipMaskOperation::Intersect:
+	{
+		glStencilOp(GL_KEEP, GL_KEEP, GL_INCR);
+		stencil_test_value += 1;
+	}
+	break;
+	}
+
+	RenderCompiledGeometry(geometry, translation, {});
+
+	// Restore state
+	// @performance Cache state so we don't toggle it unnecessarily.
+	glColorMask(GL_TRUE, GL_TRUE, GL_TRUE, GL_TRUE);
+	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
+	glStencilFunc(GL_EQUAL, stencil_test_value, GLuint(-1));
 }
 
 // Set to byte packing, or the compiler will expand our struct, which means it won't read correctly from file

Backends/RmlUi_Renderer_GL2.h

@@ -49,10 +49,16 @@ public:
 
 	void RenderGeometry(Rml::Vertex* vertices, int num_vertices, int* indices, int num_indices, Rml::TextureHandle texture,
 		const Rml::Vector2f& translation) override;
+	Rml::CompiledGeometryHandle CompileGeometry(Rml::Vertex* vertices, int num_vertices, int* indices, int num_indices) override;
+	void ReleaseCompiledGeometry(Rml::CompiledGeometryHandle geometry) override;
+	void RenderCompiledGeometry(Rml::CompiledGeometryHandle handle, const Rml::Vector2f& translation, Rml::TextureHandle texture) override;
 
 	void EnableScissorRegion(bool enable) override;
 	void SetScissorRegion(int x, int y, int width, int height) override;
 
+	void EnableClipMask(bool enable) override;
+	void RenderToClipMask(Rml::ClipMaskOperation operation, Rml::CompiledGeometryHandle geometry, Rml::Vector2f translation) override;
+
 	bool LoadTexture(Rml::TextureHandle& texture_handle, Rml::Vector2i& texture_dimensions, const Rml::String& source) override;
 	bool GenerateTexture(Rml::TextureHandle& texture_handle, const Rml::byte* source, const Rml::Vector2i& source_dimensions) override;
 	void ReleaseTexture(Rml::TextureHandle texture_handle) override;
@@ -63,6 +69,13 @@ public:
 	static const Rml::TextureHandle TextureEnableWithoutBinding = Rml::TextureHandle(-1);
 
 private:
+	struct GeometryView {
+		Rml::Vertex* vertices = nullptr;
+		int* indices = nullptr;
+		int num_vertices = 0;
+		int num_indices = 0;
+	};
+
 	int viewport_width = 0;
 	int viewport_height = 0;
 	bool transform_enabled = false;