Revamped love.graphics.stencil and love.graphics.setStencilTest:

- Each canvas (and the screen) has an invisible stencil value associated with each pixel. Stencil values are between [0, 255].

- love.graphics.stencil takes a function argument which draws things, and makes the geometry of what's drawn set the stencil values of pixels instead of coloring the pixels.

- Its second argument is a 'stencil action' which determines what happens to the stencil values of pixels. The possible stencil actions are 'replace' (the default), 'invert', 'increment', 'decrement', 'incrementwrap', and 'decrementwrap'. If the stencil action is 'replace', the third argument of love.graphics.stencil is a number between [0, 255] which determines what the stencil value of each pixel that touches drawn objects is replaced with.

The fourth argument is a boolean which determines whether the previous stencil values of all pixels on-screen should be kept (true), or cleared to 0 (false, the default.) love.graphics.clear also clears the stencil values.

- love.graphics.setStencilTest allows for anything drawn to be affected by a comparison between the arguments to setStencilTest and the existing stencil value of the pixels that are touched by what's drawn. Its first argument is a compare mode which can be 'equal', 'notequal', 'less', 'lequal', 'gequal', or 'greater', and the second argument is an integer value (between 0 and 255) used in the comparison with the value in the stencil buffer.

For example, love.graphics.setStencilTest("greater", 0) will cause any geometry drawn afterwards to only appear on pixels whose stencil value is greater than 0.

src/modules/graphics/Graphics.cpp

@@ -102,6 +102,26 @@ bool Graphics::getConstant(LineJoin in, const char *&out)
 	return lineJoins.find(in, out);
 }
 
+bool Graphics::getConstant(const char *in, StencilAction &out)
+{
+	return stencilActions.find(in, out);
+}
+
+bool Graphics::getConstant(StencilAction in, const char *&out)
+{
+	return stencilActions.find(in, out);
+}
+
+bool Graphics::getConstant(const char *in, CompareMode &out)
+{
+	return compareModes.find(in, out);
+}
+
+bool Graphics::getConstant(CompareMode in, const char *&out)
+{
+	return compareModes.find(in, out);
+}
+
 bool Graphics::getConstant(const char *in, Support &out)
 {
 	return support.find(in, out);
@@ -179,6 +199,31 @@ StringMap<Graphics::LineJoin, Graphics::LINE_JOIN_MAX_ENUM>::Entry Graphics::lin
 
 StringMap<Graphics::LineJoin, Graphics::LINE_JOIN_MAX_ENUM> Graphics::lineJoins(Graphics::lineJoinEntries, sizeof(Graphics::lineJoinEntries));
 
+StringMap<Graphics::StencilAction, Graphics::STENCIL_MAX_ENUM>::Entry Graphics::stencilActionEntries[] =
+{
+	{ "replace", STENCIL_REPLACE },
+	{ "increment", STENCIL_INCREMENT },
+	{ "decrement", STENCIL_DECREMENT },
+	{ "incrementwrap", STENCIL_INCREMENT_WRAP },
+	{ "decrementwrap", STENCIL_DECREMENT_WRAP },
+	{ "invert", STENCIL_INVERT },
+};
+
+StringMap<Graphics::StencilAction, Graphics::STENCIL_MAX_ENUM> Graphics::stencilActions(Graphics::stencilActionEntries, sizeof(Graphics::stencilActionEntries));
+
+StringMap<Graphics::CompareMode, Graphics::COMPARE_MAX_ENUM>::Entry Graphics::compareModeEntries[] =
+{
+	{ "less",     COMPARE_LESS     },
+	{ "lequal",   COMPARE_LEQUAL   },
+	{ "equal",    COMPARE_EQUAL    },
+	{ "gequal",   COMPARE_GEQUAL   },
+	{ "greater",  COMPARE_GREATER  },
+	{ "notequal", COMPARE_NOTEQUAL },
+	{ "always",   COMPARE_ALWAYS   },
+};
+
+StringMap<Graphics::CompareMode, Graphics::COMPARE_MAX_ENUM> Graphics::compareModes(Graphics::compareModeEntries, sizeof(Graphics::compareModeEntries));
+
 StringMap<Graphics::Support, Graphics::SUPPORT_MAX_ENUM>::Entry Graphics::supportEntries[] =
 {
 	{ "multicanvasformats", SUPPORT_MULTI_CANVAS_FORMATS },

src/modules/graphics/Graphics.h

@@ -96,6 +96,29 @@ public:
 		LINE_JOIN_MAX_ENUM
 	};
 
+	enum StencilAction
+	{
+		STENCIL_REPLACE,
+		STENCIL_INCREMENT,
+		STENCIL_DECREMENT,
+		STENCIL_INCREMENT_WRAP,
+		STENCIL_DECREMENT_WRAP,
+		STENCIL_INVERT,
+		STENCIL_MAX_ENUM
+	};
+
+	enum CompareMode
+	{
+		COMPARE_LESS,
+		COMPARE_LEQUAL,
+		COMPARE_EQUAL,
+		COMPARE_GEQUAL,
+		COMPARE_GREATER,
+		COMPARE_NOTEQUAL,
+		COMPARE_ALWAYS,
+		COMPARE_MAX_ENUM
+	};
+
 	enum Support
 	{
 		SUPPORT_MULTI_CANVAS_FORMATS,
@@ -238,6 +261,12 @@ public:
 	static bool getConstant(const char *in, LineJoin &out);
 	static bool getConstant(LineJoin in, const char *&out);
 
+	static bool getConstant(const char *in, StencilAction &out);
+	static bool getConstant(StencilAction in, const char *&out);
+
+	static bool getConstant(const char *in, CompareMode &out);
+	static bool getConstant(CompareMode in, const char *&out);
+
 	static bool getConstant(const char *in, Support &out);
 	static bool getConstant(Support in, const char *&out);
 
@@ -264,6 +293,12 @@ private:
 	static StringMap<LineJoin, LINE_JOIN_MAX_ENUM>::Entry lineJoinEntries[];
 	static StringMap<LineJoin, LINE_JOIN_MAX_ENUM> lineJoins;
 
+	static StringMap<StencilAction, STENCIL_MAX_ENUM>::Entry stencilActionEntries[];
+	static StringMap<StencilAction, STENCIL_MAX_ENUM> stencilActions;
+
+	static StringMap<CompareMode, COMPARE_MAX_ENUM>::Entry compareModeEntries[];
+	static StringMap<CompareMode, COMPARE_MAX_ENUM> compareModes;
+
 	static StringMap<Support, SUPPORT_MAX_ENUM>::Entry supportEntries[];
 	static StringMap<Support, SUPPORT_MAX_ENUM> support;
 

src/modules/graphics/opengl/Graphics.cpp

@@ -119,7 +119,7 @@ void Graphics::restoreState(const DisplayState &s)
 	else
 		setScissor();
 
-	setStencilTest(s.stencilTest, s.stencilInvert);
+	setStencilTest(s.stencilCompare, s.stencilTestValue);
 
 	setFont(s.font.get());
 	setShader(s.shader.get());
@@ -160,8 +160,8 @@ void Graphics::restoreStateChecked(const DisplayState &s)
 			setScissor();
 	}
 
-	if (s.stencilTest != cur.stencilTest || s.stencilInvert != cur.stencilInvert)
-		setStencilTest(s.stencilTest, s.stencilInvert);
+	if (s.stencilCompare != cur.stencilCompare || s.stencilTestValue != cur.stencilTestValue)
+		setStencilTest(s.stencilCompare, s.stencilTestValue);
 
 	setFont(s.font.get());
 	setShader(s.shader.get());
@@ -433,7 +433,7 @@ void Graphics::setDebug(bool enable)
 void Graphics::reset()
 {
 	DisplayState s;
-	drawToStencilBuffer(false);
+	stopDrawToStencilBuffer();
 	restoreState(s);
 	origin();
 }
@@ -641,24 +641,9 @@ bool Graphics::getScissor(int &x, int &y, int &width, int &height) const
 	return state.scissor;
 }
 
-void Graphics::drawToStencilBuffer(bool enable)
+void Graphics::drawToStencilBuffer(StencilAction action, int value)
 {
-	if (writingToStencil == enable)
-		return;
-
-	writingToStencil = enable;
-
-	if (!enable)
-	{
-		const DisplayState &state = states.back();
-
-		// Revert the color write mask.
-		setColorMask(state.colorMask);
-
-		// Use the user-set stencil test state when writes are disabled.
-		setStencilTest(state.stencilTest, state.stencilInvert);
-		return;
-	}
+	writingToStencil = true;
 
 	// Make sure the active canvas has a stencil buffer.
 	if (Canvas::current)
@@ -667,23 +652,63 @@ void Graphics::drawToStencilBuffer(bool enable)
 	// Disable color writes but don't save the state for it.
 	glColorMask(GL_FALSE, GL_FALSE, GL_FALSE, GL_FALSE);
 
+	GLenum glaction = GL_REPLACE;
+
+	switch (action)
+	{
+	case STENCIL_REPLACE:
+	default:
+		glaction = GL_REPLACE;
+		break;
+	case STENCIL_INCREMENT:
+		glaction = GL_INCR;
+		break;
+	case STENCIL_DECREMENT:
+		glaction = GL_DECR;
+		break;
+	case STENCIL_INCREMENT_WRAP:
+		glaction = GL_INCR_WRAP;
+		break;
+	case STENCIL_DECREMENT_WRAP:
+		glaction = GL_DECR_WRAP;
+		break;
+	case STENCIL_INVERT:
+		glaction = GL_INVERT;
+		break;
+	}
+
 	// The stencil test must be enabled in order to write to the stencil buffer.
 	glEnable(GL_STENCIL_TEST);
+	glStencilFunc(GL_ALWAYS, value, 0xFFFFFFFF);
+	glStencilOp(GL_KEEP, GL_KEEP, glaction);
+}
 
-	glStencilFunc(GL_ALWAYS, 1, 1);
-	glStencilOp(GL_KEEP, GL_KEEP, GL_REPLACE);
+void Graphics::stopDrawToStencilBuffer()
+{
+	if (!writingToStencil)
+		return;
+
+	writingToStencil = false;
+
+	const DisplayState &state = states.back();
+
+	// Revert the color write mask.
+	setColorMask(state.colorMask);
+
+	// Use the user-set stencil test state when writes are disabled.
+	setStencilTest(state.stencilCompare, state.stencilTestValue);
 }
 
-void Graphics::setStencilTest(bool enable, bool invert)
+void Graphics::setStencilTest(CompareMode compare, int value)
 {
 	DisplayState &state = states.back();
-	state.stencilTest = enable;
-	state.stencilInvert = invert;
+	state.stencilCompare = compare;
+	state.stencilTestValue = value;
 
 	if (writingToStencil)
 		return;
 
-	if (!enable)
+	if (compare == COMPARE_ALWAYS)
 	{
 		glDisable(GL_STENCIL_TEST);
 		return;
@@ -693,16 +718,61 @@ void Graphics::setStencilTest(bool enable, bool invert)
 	if (Canvas::current)
 		Canvas::current->checkCreateStencil();
 
+	GLenum glcompare = GL_EQUAL;
+
+	/**
+	 * Q: Why are some of the compare modes inverted (e.g. COMPARE_LESS becomes
+	 * GL_GREATER)?
+	 *
+	 * A: OpenGL / GPUs do the comparison in the opposite way that makes sense
+	 * for this API. For example, if the compare function is GL_GREATER then the
+	 * stencil test will pass if the reference value is greater than the value
+	 * in the stencil buffer. With our API it's more intuitive to assume that
+	 * setStencilTest(COMPARE_GREATER, 4) will make it pass if the stencil
+	 * buffer has a value greater than 4.
+	 **/
+
+	switch (compare)
+	{
+	case COMPARE_LESS:
+		glcompare = GL_GREATER;
+		break;
+	case COMPARE_LEQUAL:
+		glcompare = GL_GEQUAL;
+		break;
+	case COMPARE_EQUAL:
+	default:
+		glcompare = GL_EQUAL;
+		break;
+	case COMPARE_GEQUAL:
+		glcompare = GL_LEQUAL;
+		break;
+	case COMPARE_GREATER:
+		glcompare = GL_LESS;
+		break;
+	case COMPARE_NOTEQUAL:
+		glcompare = GL_NOTEQUAL;
+		break;
+	case COMPARE_ALWAYS:
+		glcompare = GL_ALWAYS;
+		break;
+	}
+
 	glEnable(GL_STENCIL_TEST);
-	glStencilFunc(GL_EQUAL, invert ? 0 : 1, 1);
+	glStencilFunc(glcompare, value, 0xFFFFFFFF);
 	glStencilOp(GL_KEEP, GL_KEEP, GL_KEEP);
 }
 
-void Graphics::getStencilTest(bool &enable, bool &invert)
+void Graphics::setStencilTest()
+{
+	setStencilTest(COMPARE_ALWAYS, 0);
+}
+
+void Graphics::getStencilTest(CompareMode &compare, int &value)
 {
 	const DisplayState &state = states.back();
-	enable = state.stencilTest;
-	invert = state.stencilInvert;
+	compare = state.stencilCompare;
+	value = state.stencilTestValue;
 }
 
 void Graphics::clearStencil()

src/modules/graphics/opengl/Graphics.h

@@ -146,13 +146,15 @@ public:
 	 * Enables or disables drawing to the stencil buffer. When enabled, the
 	 * color buffer is disabled.
 	 **/
-	void drawToStencilBuffer(bool enable);
+	void drawToStencilBuffer(StencilAction action, int value);
+	void stopDrawToStencilBuffer();
 
 	/**
 	 * Sets whether stencil testing is enabled.
 	 **/
-	void setStencilTest(bool enable, bool invert);
-	void getStencilTest(bool &enable, bool &invert);
+	void setStencilTest(CompareMode compare, int value);
+	void setStencilTest();
+	void getStencilTest(CompareMode &compare, int &value);
 
 	/**
 	 * Clear the stencil buffer in the active Canvas(es.)
@@ -488,8 +490,8 @@ private:
 		ScissorRect scissorRect = ScissorRect();
 
 		// Stencil.
-		bool stencilTest = false;
-		bool stencilInvert = false;
+		CompareMode stencilCompare = COMPARE_ALWAYS;
+		int stencilTestValue = 0;
 
 		StrongRef<Font> font;
 		StrongRef<Shader> shader;

src/modules/graphics/opengl/wrap_Graphics.cpp

@@ -217,35 +217,63 @@ int w_stencil(lua_State *L)
 {
 	luaL_checktype(L, 1, LUA_TFUNCTION);
 
-	// Second argument: whether to keep the contents of the stencil buffer.
-	if (lua_toboolean(L, 2) == 0)
+	Graphics::StencilAction action = Graphics::STENCIL_REPLACE;
+
+	if (!lua_isnoneornil(L, 2))
+	{
+		const char *actionstr = luaL_checkstring(L, 2);
+		if (!Graphics::getConstant(actionstr, action))
+			return luaL_error(L, "Invalid stencil draw action: %s", actionstr);
+	}
+
+	int stencilvalue = (int) luaL_optnumber(L, 3, 1);
+
+	// Fourth argument: whether to keep the contents of the stencil buffer.
+	if (lua_toboolean(L, 4) == 0)
 		instance()->clearStencil();
 
-	instance()->drawToStencilBuffer(true);
+	instance()->drawToStencilBuffer(action, stencilvalue);
 
 	// Call stencilfunc()
 	lua_pushvalue(L, 1);
 	lua_call(L, 0, 0);
 
-	instance()->drawToStencilBuffer(false);
-
+	instance()->stopDrawToStencilBuffer();
 	return 0;
 }
 
 int w_setStencilTest(lua_State *L)
 {
-	bool enable = luax_toboolean(L, 1);
-	bool invert = luax_toboolean(L, 2);
-	instance()->setStencilTest(enable, invert);
+	// COMPARE_ALWAYS effectively disables stencil testing.
+	Graphics::CompareMode compare = Graphics::COMPARE_ALWAYS;
+	int comparevalue = 0;
+
+	if (!lua_isnoneornil(L, 1))
+	{
+		const char *comparestr = luaL_checkstring(L, 1);
+		if (!Graphics::getConstant(comparestr, compare))
+			return luaL_error(L, "Invalid compare mode: %s", comparestr);
+
+		comparevalue = (int) luaL_checknumber(L, 2);
+	}
+
+	instance()->setStencilTest(compare, comparevalue);
 	return 0;
 }
 
 int w_getStencilTest(lua_State *L)
 {
-	bool enabled, inverted;
-	instance()->getStencilTest(enabled, inverted);
-	luax_pushboolean(L, enabled);
-	luax_pushboolean(L, inverted);
+	Graphics::CompareMode compare = Graphics::COMPARE_ALWAYS;
+	int comparevalue = 1;
+
+	instance()->getStencilTest(compare, comparevalue);
+
+	const char *comparestr;
+	if (!Graphics::getConstant(compare, comparestr))
+		return luaL_error(L, "Unknown compare mode.");
+
+	lua_pushstring(L, comparestr);
+	lua_pushnumber(L, comparevalue);
 	return 2;
 }
 
@@ -1201,7 +1229,7 @@ int w_newScreenshot(lua_State *L)
 int w_setCanvas(lua_State *L)
 {
 	// Disable stencil writes.
-	instance()->drawToStencilBuffer(false);
+	instance()->stopDrawToStencilBuffer();
 
 	// called with none -> reset to default buffer
 	if (lua_isnoneornil(L, 1))

src/modules/window/sdl/Window.cpp

@@ -90,7 +90,7 @@ void Window::setGLFramebufferAttributes(int msaa, bool sRGB)
 	SDL_GL_SetAttribute(SDL_GL_BLUE_SIZE, 8);
 	SDL_GL_SetAttribute(SDL_GL_ALPHA_SIZE, 8);
 	SDL_GL_SetAttribute(SDL_GL_DOUBLEBUFFER, 1);
-	SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 1);
+	SDL_GL_SetAttribute(SDL_GL_STENCIL_SIZE, 8);
 	SDL_GL_SetAttribute(SDL_GL_RETAINED_BACKING, 0);
 
 	SDL_GL_SetAttribute(SDL_GL_MULTISAMPLEBUFFERS, (msaa > 0) ? 1 : 0);