Replaced the old stencil API with love.graphics.drawStencil(stencilfunc), love.graphics.setStencilTest(enable, invert), and love.graphics.getStencilTest.

Draw to the stencil buffer (rather than the color buffer) using love.graphics.drawStencil with a function which draws geometry. Once the stencil buffer has been drawn to, use love.graphics.setStencilTest before drawing to let drawn geometry be affected by it.

love.graphics.clear (and Canvas:clear) will clear both the color and stencil buffers by default. love.graphics.clear("stencil") will clear only the stencil buffer.

--HG--
branch : stencil-functions

src/modules/graphics/Graphics.cpp

@@ -109,6 +109,16 @@ bool Graphics::getConstant(StackType in, const char *&out)
 	return stackTypes.find(in, out);
 }
 
+bool Graphics::getConstant(const char *in, ClearType &out)
+{
+	return clearTypes.find(in, out);
+}
+
+bool Graphics::getConstant(ClearType in, const char *&out)
+{
+	return clearTypes.find(in, out);
+}
+
 StringMap<Graphics::DrawMode, Graphics::DRAW_MAX_ENUM>::Entry Graphics::drawModeEntries[] =
 {
 	{ "line", Graphics::DRAW_LINE },
@@ -183,5 +193,13 @@ StringMap<Graphics::StackType, Graphics::STACK_MAX_ENUM>::Entry Graphics::stackT
 
 StringMap<Graphics::StackType, Graphics::STACK_MAX_ENUM> Graphics::stackTypes(Graphics::stackTypeEntries, sizeof(Graphics::stackTypeEntries));
 
+StringMap<Graphics::ClearType, Graphics::CLEAR_MAX_ENUM>::Entry Graphics::clearTypeEntries[] =
+{
+	{"all", Graphics::CLEAR_ALL},
+	{"stencil", Graphics::CLEAR_STENCIL},
+};
+
+StringMap<Graphics::ClearType, Graphics::CLEAR_MAX_ENUM> Graphics::clearTypes(Graphics::clearTypeEntries, sizeof(Graphics::clearTypeEntries));
+
 } // graphics
 } // love

src/modules/graphics/Graphics.h

@@ -103,6 +103,13 @@ public:
 		STACK_MAX_ENUM
 	};
 
+	enum ClearType
+	{
+		CLEAR_ALL,
+		CLEAR_STENCIL,
+		CLEAR_MAX_ENUM
+	};
+
 	struct RendererInfo
 	{
 		std::string name;
@@ -158,6 +165,9 @@ public:
 	static bool getConstant(const char *in, StackType &out);
 	static bool getConstant(StackType in, const char *&out);
 
+	static bool getConstant(const char *in, ClearType &out);
+	static bool getConstant(ClearType in, const char *&out);
+
 private:
 
 	static StringMap<DrawMode, DRAW_MAX_ENUM>::Entry drawModeEntries[];
@@ -184,6 +194,9 @@ private:
 	static StringMap<StackType, STACK_MAX_ENUM>::Entry stackTypeEntries[];
 	static StringMap<StackType, STACK_MAX_ENUM> stackTypes;
 
+	static StringMap<ClearType, CLEAR_MAX_ENUM>::Entry clearTypeEntries[];
+	static StringMap<ClearType, CLEAR_MAX_ENUM> clearTypes;
+
 }; // Graphics
 
 } // graphics

src/modules/graphics/opengl/Canvas.cpp

@@ -823,7 +823,7 @@ void Canvas::clear(Color c)
 		// Don't use the state-shadowed gl.setClearColor because we want to save
 		// the previous clear color.
 		glClearColor(glcolor[0], glcolor[1], glcolor[2], glcolor[3]);
-		glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT);
+		glClear(GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT | GL_DEPTH_BUFFER_BIT);
 
 		if (attachedCanvases.size() > 0)
 			strategy->setAttachments(attachedCanvases);
@@ -850,6 +850,12 @@ bool Canvas::checkCreateStencil()
 
 	bool success = strategy->createStencil(width, height, msaa_samples, depth_stencil);
 
+	if (success)
+	{
+		// We don't want the stencil buffer filled with garbage.
+		glClear(GL_STENCIL_BUFFER_BIT);
+	}
+
 	if (current && current != this)
 		strategy->bindFBO(current->fbo);
 	else if (!current)

src/modules/graphics/opengl/Graphics.cpp

@@ -47,7 +47,7 @@ Graphics::Graphics()
 	: width(0)
 	, height(0)
 	, created(false)
-	, activeStencil(false)
+	, writingToStencil(false)
 	, displayedMinReqWarning(false)
 {
 	states.reserve(10);
@@ -95,6 +95,8 @@ void Graphics::restoreState(const DisplayState &s)
 	else
 		setScissor();
 
+	setStencilTest(s.stencilTest, s.stencilInvert);
+
 	setFont(s.font.get());
 	setShader(s.shader.get());
 	setCanvas(s.canvases);
@@ -132,6 +134,9 @@ void Graphics::restoreStateChecked(const DisplayState &s)
 			setScissor();
 	}
 
+	if (s.stencilTest != cur.stencilTest || s.stencilInvert != cur.stencilInvert)
+		setStencilTest(s.stencilTest, s.stencilInvert);
+
 	setFont(s.font.get());
 	setShader(s.shader.get());
 
@@ -350,14 +355,27 @@ void Graphics::setDebug(bool enable)
 void Graphics::reset()
 {
 	DisplayState s;
-	discardStencil();
-	origin();
+	drawToStencilBuffer(false);
 	restoreState(s);
+	origin();
 }
 
-void Graphics::clear()
+void Graphics::clear(ClearType type)
 {
-	glClear(GL_COLOR_BUFFER_BIT);
+	GLbitfield mask = 0;
+
+	switch (type)
+	{
+	case CLEAR_ALL:
+	default:
+		mask = GL_COLOR_BUFFER_BIT | GL_STENCIL_BUFFER_BIT | GL_DEPTH_BUFFER_BIT;
+		break;
+	case CLEAR_STENCIL:
+		mask = GL_STENCIL_BUFFER_BIT;
+		break;
+	}
+
+	glClear(mask);
 }
 
 void Graphics::present()
@@ -409,38 +427,68 @@ bool Graphics::getScissor(int &x, int &y, int &width, int &height) const
 	return states.back().scissor;
 }
 
-void Graphics::defineStencil()
+void Graphics::drawToStencilBuffer(bool enable)
 {
+	if (writingToStencil == enable)
+		return;
+
+	writingToStencil = enable;
+
+	if (!enable)
+	{
+		const DisplayState &state = states.back();
+
+		// Revert the color write mask.
+		setColorMask(state.colorMask);
+
+		// Use the user-set stencil test state when writes are disabled.
+		setStencilTest(state.stencilTest, state.stencilInvert);
+		return;
+	}
+
 	// Make sure the active canvas has a stencil buffer.
 	if (Canvas::current)
 		Canvas::current->checkCreateStencil();
 
-	// Disable color writes but don't save the mask values.
+	// Disable color writes but don't save the state for it.
 	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
 
-	glClear(GL_STENCIL_BUFFER_BIT);
+	// The stencil test must be enabled in order to write to the stencil buffer.
 	glEnable(GL_STENCIL_TEST);
+
 	glStencilFunc(GL_ALWAYS, 1, 1);
 	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
-
-	activeStencil = true;
 }
 
-void Graphics::useStencil(bool invert)
+void Graphics::setStencilTest(bool enable, bool invert)
 {
-	glStencilFunc(GL_EQUAL, (GLint)(!invert), 1); // invert ? 0 : 1
+	DisplayState &state = states.back();
+	state.stencilTest = enable;
+	state.stencilInvert = invert;
+
+	if (writingToStencil)
+		return;
+
+	if (!enable)
+	{
+		glDisable(GL_STENCIL_TEST);
+		return;
+	}
+
+	// Make sure the active canvas has a stencil buffer.
+	if (Canvas::current)
+		Canvas::current->checkCreateStencil();
+
+	glEnable(GL_STENCIL_TEST);
+	glStencilFunc(GL_EQUAL, invert ? 0 : 1, 1);
 	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
-	setColorMask(states.back().colorMask);
 }
 
-void Graphics::discardStencil()
+void Graphics::getStencilTest(bool &enable, bool &invert)
 {
-	if (!activeStencil)
-		return;
-
-	setColorMask(states.back().colorMask);
-	glDisable(GL_STENCIL_TEST);
-	activeStencil = false;
+	const DisplayState &state = states.back();
+	enable = state.stencilTest;
+	invert = state.stencilInvert;
 }
 
 Image *Graphics::newImage(love::image::ImageData *data, const Image::Flags &flags)
@@ -1279,6 +1327,8 @@ Graphics::DisplayState::DisplayState()
 	, pointSize(1.0f)
 	, scissor(false)
 	, scissorBox()
+	, stencilTest(false)
+	, stencilInvert(false)
 	, font(nullptr)
 	, shader(nullptr)
 	, wireframe(false)
@@ -1298,6 +1348,8 @@ Graphics::DisplayState::DisplayState(const DisplayState &other)
 	, pointSize(other.pointSize)
 	, scissor(other.scissor)
 	, scissorBox(other.scissorBox)
+	, stencilTest(other.stencilTest)
+	, stencilInvert(other.stencilInvert)
 	, font(other.font)
 	, shader(other.shader)
 	, canvases(other.canvases)
@@ -1322,6 +1374,8 @@ Graphics::DisplayState &Graphics::DisplayState::operator = (const DisplayState &
 	pointSize = other.pointSize;
 	scissor = other.scissor;
 	scissorBox = other.scissorBox;
+	stencilTest = other.stencilTest;
+	stencilInvert = other.stencilInvert;
 
 	font = other.font;
 	shader = other.shader;

src/modules/graphics/opengl/Graphics.h

@@ -80,7 +80,7 @@ public:
 	/**
 	 * Clears the screen.
 	 **/
-	void clear();
+	void clear(ClearType type = CLEAR_ALL);
 
 	/**
 	 * Flips buffers. (Rendered geometry is presented on screen).
@@ -124,22 +124,16 @@ public:
 	bool getScissor(int &x, int &y, int &width, int &height) const;
 
 	/**
-	 * Enables the stencil buffer and set stencil function to fill it
-	 */
-	void defineStencil();
-
-	/**
-	 * Set stencil function to mask the following drawing calls using
-	 * the current stencil buffer
-	 * @param invert Invert the mask, i.e. draw everywhere expect where
-	 *               the mask is defined.
-	 */
-	void useStencil(bool invert = false);
+	 * Enables or disables drawing to the stencil buffer. When enabled, the
+	 * color buffer is disabled.
+	 **/
+	void drawToStencilBuffer(bool enable);
 
 	/**
-	 * Disables the stencil buffer
-	 */
-	void discardStencil();
+	 * Sets whether stencil testing is enabled.
+	 **/
+	void setStencilTest(bool enable, bool invert);
+	void getStencilTest(bool &enable, bool &invert);
 
 	/**
 	 * Creates an Image object with padding and/or optimization.
@@ -443,6 +437,10 @@ private:
 		bool scissor;
 		OpenGL::Viewport scissorBox;
 
+		// Stencil.
+		bool stencilTest;
+		bool stencilInvert;
+
 		Object::StrongRef<Font> font;
 		Object::StrongRef<Shader> shader;
 
@@ -471,7 +469,7 @@ private:
 	int height;
 	bool created;
 
-	bool activeStencil;
+	bool writingToStencil;
 
 	std::vector<DisplayState> states;
 	std::vector<StackType> stackTypes; // Keeps track of the pushed stack types.

src/modules/graphics/opengl/wrap_Graphics.cpp

@@ -44,9 +44,15 @@ int w_reset(lua_State *)
 	return 0;
 }
 
-int w_clear(lua_State *)
+int w_clear(lua_State *L)
 {
-	instance()->clear();
+	Graphics::ClearType type = Graphics::CLEAR_ALL;
+
+	const char *tname = lua_isnoneornil(L, 1) ? nullptr : luaL_checkstring(L, 1);
+	if (tname && !Graphics::getConstant(tname, type))
+		return luaL_error(L, "Invalid graphics clear type: %s", tname);
+
+	instance()->clear(type);
 	return 0;
 }
 
@@ -118,32 +124,35 @@ int w_getScissor(lua_State *L)
 	return 4;
 }
 
-static int setStencil(lua_State *L, bool invert)
+int w_drawStencil(lua_State *L)
 {
-	// no argument -> clear stencil
-	if (lua_isnoneornil(L, 1))
-	{
-		instance()->discardStencil();
-		return 0;
-	}
-
 	luaL_checktype(L, 1, LUA_TFUNCTION);
 
-	instance()->defineStencil();
-	lua_call(L, lua_gettop(L) - 1, 0); // call stencil(...)
-	instance()->useStencil(invert);
+	instance()->drawToStencilBuffer(true);
+
+	// Call stencilfunc(...)
+	lua_call(L, lua_gettop(L) - 1, 0);
+
+	instance()->drawToStencilBuffer(false);
 
 	return 0;
 }
 
-int w_setStencil(lua_State *L)
+int w_setStencilTest(lua_State *L)
 {
-	return setStencil(L, false);
+	bool enable = luax_toboolean(L, 1);
+	bool invert = luax_toboolean(L, 2);
+	instance()->setStencilTest(enable, invert);
+	return 0;
 }
 
-int w_setInvertedStencil(lua_State *L)
+int w_getStencilTest(lua_State *L)
 {
-	return setStencil(L, true);
+	bool enabled, inverted;
+	instance()->getStencilTest(enabled, inverted);
+	luax_pushboolean(L, enabled);
+	luax_pushboolean(L, inverted);
+	return 2;
 }
 
 static const char *imageFlagName(Image::FlagType flagtype)
@@ -860,8 +869,8 @@ int w_newScreenshot(lua_State *L)
 
 int w_setCanvas(lua_State *L)
 {
-	// discard stencil testing
-	instance()->discardStencil();
+	// Disable stencil writes.
+	instance()->drawToStencilBuffer(false);
 
 	// called with none -> reset to default buffer
 	if (lua_isnoneornil(L, 1))
@@ -1411,8 +1420,9 @@ static const luaL_Reg functions[] =
 	{ "setScissor", w_setScissor },
 	{ "getScissor", w_getScissor },
 
-	{ "setStencil", w_setStencil },
-	{ "setInvertedStencil", w_setInvertedStencil },
+	{ "drawStencil", w_drawStencil },
+	{ "setStencilTest", w_setStencilTest },
+	{ "getStencilTest", w_getStencilTest },
 
 	{ "point", w_point },
 	{ "line", w_line },

src/modules/graphics/opengl/wrap_Graphics.h

@@ -48,8 +48,9 @@ int w_getHeight(lua_State *L);
 int w_getDimensions(lua_State *L);
 int w_setScissor(lua_State *L);
 int w_getScissor(lua_State *L);
-int w_setStencil(lua_State *L);
-int w_setInvertedStencil(lua_State *L);
+int w_drawStencil(lua_State *L);
+int w_setStencilTest(lua_State *L);
+int w_getStencilTest(lua_State *L);
 int w_newImage(lua_State *L);
 int w_newQuad(lua_State *L);
 int w_newFont(lua_State *L);