Move current love.graphics.setStencilMode functionality to setStencilState.

Add new higher level love.graphics.setStencilMode function.
- Add love.graphics.setStencilMode(mode [, value = 1])
- Add love.graphics.setStencilMode().
- Add mode, value = love.graphics.getStencilMode().

The possible modes are "off" (same as no parameters), "draw", and "test".
The "draw" mode disables color writes and sets the stencil state to ("replace", "always") using the given value.
The "test" mode enables color writes and sets the stencil state to ("keep", "equal") using the given value to compare to.

changes.txt

@@ -68,7 +68,8 @@ Released: N/A
 * Added APIs to override the default orthographic projection: love.graphics.setOrthoProjection, setPerspectiveProjection, and resetProjection.
 * Added ability to set point size within a vertex shader by setting the 'love_PointSize' variable.
 * Added love.graphics.setBlendState, which gives lower level control over blend operations than setBlendMode.
-* Added love.graphics.setStencilMode and getStencilMode. Replaces love.graphics.stencil as well as setStencilTest.
+* Added high level love.graphics.setStencilMode and getStencilMode functions. Replaces love.graphics.stencil and setStencilTest.
+* Added lower level love.graphics.setStencilState and getStencilState functions.
 * Added a variant of love.graphics.setColorMask which accepts a single boolean.
 * Added new 'clampone' wrap mode.
 * Added 'clampone', 'texelbuffer', 'indexbuffer32bit', 'mipmaprange', and 'indirectdraw' graphics feature enums.
@@ -113,8 +114,8 @@ Released: N/A
 * Deprecated love.graphics.setNewFont (use love.graphics.newFont and love.graphics.setFont instead).
 * Deprecated love.graphics.newText (renamed to love.graphics.newTextBatch).
 * Deprecated love.graphics.getImageFormats and love.graphics.getCanvasFormats (replaced by getTextureFormats).
-* Deprecated love.graphics.stencil (replaced by love.graphics.setStencilMode).
-* Deprecated love.graphics.setStencilTest and love.graphics.getStencilTest (replaced by love.graphics.setStencilMode and getStencilMode).
+* Deprecated love.graphics.stencil (replaced by love.graphics.setStencilMode or love.graphics.setStencilState).
+* Deprecated love.graphics.setStencilTest and love.graphics.getStencilTest (replaced by love.graphics.setStencilMode or setStencilState).
 * Deprecated t.window.highdpi in love.conf and the highdpi flag of love.window.setMode (replaced by t.highdpi in love.conf).
 * Deprecated t.accelerometerjoystick in love.conf (replaced by love.sensor module).
 * Deprecated the variants of Mesh:attachAttribute and SpriteBatch:attachAttribute which accept a Mesh (replaced by variants which accept a Buffer).

src/modules/graphics/Graphics.cpp

@@ -700,7 +700,7 @@ void Graphics::restoreState(const DisplayState &s)
 	setShader(s.shader.get());
 	setRenderTargets(s.renderTargets);
 
-	setStencilMode(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
+	setStencilState(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
 	setDepthMode(s.depthTest, s.depthWrite);
 
 	setColorMask(s.colorMask);
@@ -776,7 +776,7 @@ void Graphics::restoreStateChecked(const DisplayState &s)
 		setRenderTargets(s.renderTargets);
 
 	if (!(s.stencil == cur.stencil))
-		setStencilMode(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
+		setStencilState(s.stencil.action, s.stencil.compare, s.stencil.value, s.stencil.readMask, s.stencil.writeMask);
 
 	if (s.depthTest != cur.depthTest || s.depthWrite != cur.depthWrite)
 		setDepthMode(s.depthTest, s.depthWrite);
@@ -1309,12 +1309,37 @@ bool Graphics::getScissor(Rect &rect) const
 	return state.scissor;
 }
 
+void Graphics::setStencilMode(StencilMode mode, int value)
+{
+	StencilState s = computeStencilState(mode, value);
+	setStencilState(s.action, s.compare, s.value, s.readMask, s.writeMask);
+	if (mode == STENCIL_MODE_DRAW)
+		setColorMask({ false, false, false, false });
+	else
+		setColorMask({ true, true, true, true });
+}
+
 void Graphics::setStencilMode()
 {
-	setStencilMode(STENCIL_KEEP, COMPARE_ALWAYS, 0, LOVE_UINT32_MAX, LOVE_UINT32_MAX);
+	StencilState s = computeStencilState(STENCIL_MODE_OFF, 0);
+	setStencilState(s.action, s.compare, s.value, s.readMask, s.writeMask);
+	setColorMask({ true, true, true, true });
+}
+
+StencilMode Graphics::getStencilMode(int &value) const
+{
+	const DisplayState& state = states.back();
+	StencilMode mode = computeStencilMode(state.stencil);
+	value = state.stencil.value;
+	return mode;
+}
+
+void Graphics::setStencilState()
+{
+	setStencilState(STENCIL_KEEP, COMPARE_ALWAYS, 0, LOVE_UINT32_MAX, LOVE_UINT32_MAX);
 }
 
-void Graphics::getStencilMode(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const
+void Graphics::getStencilState(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const
 {
 	const DisplayState &state = states.back();
 	action = state.stencil.action;

src/modules/graphics/Graphics.h

@@ -613,9 +613,13 @@ public:
 	 */
 	bool getScissor(Rect &rect) const;
 
-	virtual void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) = 0;
+	void setStencilMode(StencilMode mode, int value);
 	void setStencilMode();
-	void getStencilMode(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const;
+	StencilMode getStencilMode(int &value) const;
+
+	virtual void setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) = 0;
+	void setStencilState();
+	void getStencilState(StencilAction &action, CompareMode &compare, int &value, uint32 &readmask, uint32 &writemask) const;
 
 	virtual void setDepthMode(CompareMode compare, bool write) = 0;
 	void setDepthMode();

src/modules/graphics/metal/Graphics.h

@@ -96,7 +96,7 @@ public:
 	void setScissor(const Rect &rect) override;
 	void setScissor() override;
 
-	void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
+	void setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
 
 	void setDepthMode(CompareMode compare, bool write) override;
 

src/modules/graphics/metal/Graphics.mm

@@ -846,7 +846,7 @@ id<MTLDepthStencilState> Graphics::getCachedDepthStencilState(const DepthState &
 	 * example, if the compare function is GREATER then the stencil test will
 	 * pass if the reference value is greater than the value in the stencil
 	 * buffer. With our API it's more intuitive to assume that
-	 * setStencilMode(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
+	 * setStencilState(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
 	 * stencil buffer has a value greater than 4.
 	 **/
 	stencildesc.stencilCompareFunction = getMTLCompareFunction(getReversedCompareMode(stencil.compare));
@@ -1760,7 +1760,7 @@ void Graphics::setScissor()
 	}
 }
 
-void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
+void Graphics::setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
 {
 	DisplayState &state = states.back();
 

src/modules/graphics/opengl/Graphics.cpp

@@ -1434,7 +1434,7 @@ void Graphics::setScissor()
 		gl.setEnableState(OpenGL::ENABLE_SCISSOR_TEST, false);
 }
 
-void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
+void Graphics::setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask)
 {
 	DisplayState &state = states.back();
 
@@ -1493,7 +1493,7 @@ void Graphics::setStencilMode(StencilAction action, CompareMode compare, int val
 	 * example, if the compare function is GREATER then the stencil test will
 	 * pass if the reference value is greater than the value in the stencil
 	 * buffer. With our API it's more intuitive to assume that
-	 * setStencilMode(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
+	 * setStencilState(STENCIL_KEEP, COMPARE_GREATER, 4) will make it pass if the
 	 * stencil buffer has a value greater than 4.
 	 **/
 	GLenum glcompare = OpenGL::getGLCompareMode(getReversedCompareMode(compare));

src/modules/graphics/opengl/Graphics.h

@@ -92,7 +92,7 @@ public:
 	void setScissor(const Rect &rect) override;
 	void setScissor() override;
 
-	void setStencilMode(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
+	void setStencilState(StencilAction action, CompareMode compare, int value, uint32 readmask, uint32 writemask) override;
 
 	void setDepthMode(CompareMode compare, bool write) override;
 

src/modules/graphics/renderstate.cpp

@@ -28,7 +28,7 @@ namespace graphics
 
 // These are all with premultiplied alpha. computeBlendState adjusts for
 // alpha-multiply if needed.
-static const BlendState states[BLEND_MAX_ENUM] =
+static const BlendState blendStates[BLEND_MAX_ENUM] =
 {
 	// BLEND_ALPHA
 	{BLENDOP_ADD, BLENDOP_ADD, BLENDFACTOR_ONE, BLENDFACTOR_ONE, BLENDFACTOR_ONE_MINUS_SRC_ALPHA, BLENDFACTOR_ONE_MINUS_SRC_ALPHA},
@@ -63,7 +63,7 @@ static const BlendState states[BLEND_MAX_ENUM] =
 
 BlendState computeBlendState(BlendMode mode, BlendAlpha alphamode)
 {
-	BlendState s = states[mode];
+	BlendState s = blendStates[mode];
 
 	// We can only do alpha-multiplication when srcRGB would have been unmodified.
 	if (s.srcFactorRGB == BLENDFACTOR_ONE && alphamode == BLENDALPHA_MULTIPLY && mode != BLEND_NONE)
@@ -87,7 +87,7 @@ BlendMode computeBlendMode(BlendState s, BlendAlpha &alphamode)
 
 	for (int i = 0; i < (int) BLEND_MAX_ENUM; i++)
 	{
-		if (i != (int) BLEND_CUSTOM && states[i] == s)
+		if (i != (int) BLEND_CUSTOM && blendStates[i] == s)
 		{
 			alphamode = alphamultiply ? BLENDALPHA_MULTIPLY : BLENDALPHA_PREMULTIPLIED;
 			return (BlendMode) i;
@@ -111,6 +111,39 @@ bool isAlphaMultiplyBlendSupported(BlendMode mode)
 	}
 }
 
+static const StencilState stencilStates[STENCIL_MODE_MAX_ENUM] =
+{
+	// STENCIL_MODE_OFF
+	{},
+
+	// STENCIL_MODE_DRAW
+	{COMPARE_ALWAYS, STENCIL_REPLACE},
+
+	// STENCIL_MODE_TEST
+	{COMPARE_EQUAL, STENCIL_KEEP},
+
+	// STENCIL_MODE_CUSTOM - N/A
+	{},
+};
+
+StencilState computeStencilState(StencilMode mode, int value)
+{
+	StencilState s = stencilStates[mode];
+	s.value = value;
+	return s;
+}
+
+StencilMode computeStencilMode(const StencilState &s)
+{
+	for (int i = 0; i < (int)STENCIL_MODE_MAX_ENUM; i++)
+	{
+		if (stencilStates[i].action == s.action && stencilStates[i].compare == s.compare)
+			return (StencilMode) i;
+	}
+
+	return STENCIL_MODE_CUSTOM;
+}
+
 CompareMode getReversedCompareMode(CompareMode mode)
 {
 	switch (mode)
@@ -171,6 +204,15 @@ STRINGMAP_BEGIN(BlendOperation, BLENDOP_MAX_ENUM, blendOperation)
 }
 STRINGMAP_END(BlendOperation, BLENDOP_MAX_ENUM, blendOperation)
 
+STRINGMAP_BEGIN(StencilMode, STENCIL_MODE_MAX_ENUM, stencilMode)
+{
+	{ "off",    STENCIL_MODE_OFF    },
+	{ "draw",   STENCIL_MODE_DRAW   },
+	{ "test",   STENCIL_MODE_TEST   },
+	{ "custom", STENCIL_MODE_CUSTOM },
+}
+STRINGMAP_END(StencilMode, STENCIL_MODE_MAX_ENUM, stencilMode)
+
 STRINGMAP_BEGIN(StencilAction, STENCIL_MAX_ENUM, stencilAction)
 {
 	{ "keep",          STENCIL_KEEP           },

src/modules/graphics/renderstate.h

@@ -83,6 +83,15 @@ enum BlendOperation
 	BLENDOP_MAX_ENUM
 };
 
+enum StencilMode // High level wrappers.
+{
+	STENCIL_MODE_OFF,
+	STENCIL_MODE_DRAW,
+	STENCIL_MODE_TEST,
+	STENCIL_MODE_CUSTOM,
+	STENCIL_MODE_MAX_ENUM
+};
+
 enum StencilAction
 {
 	STENCIL_KEEP,
@@ -194,6 +203,9 @@ BlendState computeBlendState(BlendMode mode, BlendAlpha alphamode);
 BlendMode computeBlendMode(BlendState s, BlendAlpha &alphamode);
 bool isAlphaMultiplyBlendSupported(BlendMode mode);
 
+StencilState computeStencilState(StencilMode mode, int value);
+StencilMode computeStencilMode(const StencilState &s);
+
 /**
  * GPU APIs do the comparison in the opposite way of what makes sense for some
  * of love's APIs. For example in OpenGL if the compare function is GL_GREATER,
@@ -208,6 +220,7 @@ STRINGMAP_DECLARE(BlendMode);
 STRINGMAP_DECLARE(BlendAlpha);
 STRINGMAP_DECLARE(BlendFactor);
 STRINGMAP_DECLARE(BlendOperation);
+STRINGMAP_DECLARE(StencilMode);
 STRINGMAP_DECLARE(StencilAction);
 STRINGMAP_DECLARE(CompareMode);
 

src/modules/graphics/vulkan/Graphics.cpp

@@ -1001,7 +1001,7 @@ void Graphics::setScissor()
 	vkCmdSetScissor(commandBuffers.at(currentFrame), 0, 1, &scissor);
 }
 
-void Graphics::setStencilMode(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask)
+void Graphics::setStencilState(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask)
 {
 	if (action != STENCIL_KEEP)
 	{

src/modules/graphics/vulkan/Graphics.h

@@ -293,7 +293,7 @@ public:
 	void setColor(Colorf c) override;
 	void setScissor(const Rect &rect) override;
 	void setScissor() override;
-	void setStencilMode(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask) override;
+	void setStencilState(StencilAction action, CompareMode compare, int value, love::uint32 readmask, love::uint32 writemask) override;
 	void setDepthMode(CompareMode compare, bool write) override;
 	void setFrontFaceWinding(Winding winding) override;
 	void setColorMask(ColorChannelMask mask) override;

src/modules/graphics/wrap_Graphics.cpp

@@ -608,7 +608,40 @@ int w_setStencilMode(lua_State *L)
 {
 	if (lua_gettop(L) <= 1 && lua_isnoneornil(L, 1))
 	{
-		luax_catchexcept(L, [&](){ instance()->setStencilMode(); });
+		luax_catchexcept(L, [&]() { instance()->setStencilMode(); });
+		return 0;
+	}
+
+	StencilMode mode = STENCIL_MODE_OFF;
+	const char *modestr = luaL_checkstring(L, 1);
+	if (!getConstant(modestr, mode))
+		return luax_enumerror(L, "stencil mode", getConstants(mode), modestr);
+
+	int value = (int) luaL_optinteger(L, 3, 1);
+
+	luax_catchexcept(L, [&]() { instance()->setStencilMode(mode, value); });
+	return 0;
+}
+
+int w_getStencilMode(lua_State *L)
+{
+	int value = 0;
+	StencilMode mode = instance()->getStencilMode(value);
+
+	const char *modestr;
+	if (!getConstant(mode, modestr))
+		return luaL_error(L, "Unknown stencil mode.");
+
+	lua_pushstring(L, modestr);
+	lua_pushinteger(L, value);
+	return 2;
+}
+
+int w_setStencilState(lua_State *L)
+{
+	if (lua_gettop(L) <= 1 && lua_isnoneornil(L, 1))
+	{
+		luax_catchexcept(L, [&](){ instance()->setStencilState(); });
 		return 0;
 	}
 
@@ -627,11 +660,11 @@ int w_setStencilMode(lua_State *L)
 	uint32 readmask = (uint32) luaL_optnumber(L, 4, LOVE_UINT32_MAX);
 	uint32 writemask = (uint32) luaL_optnumber(L, 5, LOVE_UINT32_MAX);
 
-	luax_catchexcept(L, [&](){ instance()->setStencilMode(action, compare, value, readmask, writemask); });
+	luax_catchexcept(L, [&](){ instance()->setStencilState(action, compare, value, readmask, writemask); });
 	return 0;
 }
 
-int w_getStencilMode(lua_State *L)
+int w_getStencilState(lua_State *L)
 {
 	StencilAction action = STENCIL_KEEP;
 	CompareMode compare = COMPARE_ALWAYS;
@@ -639,7 +672,7 @@ int w_getStencilMode(lua_State *L)
 	uint32 readmask = LOVE_UINT32_MAX;
 	uint32 writemask = LOVE_UINT32_MAX;
 
-	instance()->getStencilMode(action, compare, value, readmask, writemask);
+	instance()->getStencilState(action, compare, value, readmask, writemask);
 
 	const char *actionstr;
 	if (!getConstant(action, actionstr))
@@ -651,7 +684,7 @@ int w_getStencilMode(lua_State *L)
 
 	lua_pushstring(L, actionstr);
 	lua_pushstring(L, comparestr);
-	lua_pushnumber(L, value);
+	lua_pushinteger(L, value);
 	lua_pushnumber(L, readmask);
 	lua_pushnumber(L, writemask);
 	return 5;
@@ -3936,6 +3969,8 @@ static const luaL_Reg functions[] =
 
 	{ "setStencilMode", w_setStencilMode },
 	{ "getStencilMode", w_getStencilMode },
+	{ "setStencilState", w_setStencilState },
+	{ "getStencilState", w_getStencilState },
 
 	{ "points", w_points },
 	{ "line", w_line },

src/modules/graphics/wrap_Graphics.lua

@@ -53,7 +53,7 @@ function graphics.newVideo(file, settings)
 end
 
 function graphics.stencil(func, action, value, keepvalues)
-	love.markDeprecated(2, "love.graphics.stencil", "function", "replaced", "love.graphics.setStencilMode")
+	love.markDeprecated(2, "love.graphics.stencil", "function", "replaced", "love.graphics.setStencilMode or setStencilState")
 
 	if not keepvalues then
 		graphics.clear(false, true, false)
@@ -61,15 +61,15 @@ function graphics.stencil(func, action, value, keepvalues)
 
 	if value == nil then value = 1 end
 
-	local action2, mode2, value2, readmask2, writemask2 = graphics.getStencilMode()
+	local action2, mode2, value2, readmask2, writemask2 = graphics.getStencilState()
 	local mr, mg, mb, ma = graphics.getColorMask()
 
-	graphics.setStencilMode(action, "always", value)
+	graphics.setStencilState(action, "always", value)
 	graphics.setColorMask(false)
 
 	local success, err = pcall(func)
 
-	graphics.setStencilMode(action2, mode2, value2, readmask2, writemask2)
+	graphics.setStencilState(action2, mode2, value2, readmask2, writemask2)
 	graphics.setColorMask(mr, mg, mb, ma)
 
 	if not success then
@@ -78,19 +78,19 @@ function graphics.stencil(func, action, value, keepvalues)
 end
 
 function graphics.setStencilTest(mode, value)
-	love.markDeprecated(2, "love.graphics.setStencilTest", "function", "replaced", "love.graphics.setStencilMode")
+	love.markDeprecated(2, "love.graphics.setStencilTest", "function", "replaced", "love.graphics.setStencilMode or setStencilState")
 
 	if mode ~= nil then
-		graphics.setStencilMode("keep", mode, value)
+		graphics.setStencilState("keep", mode, value)
 	else
-		graphics.setStencilMode()
+		graphics.setStencilState()
 	end
 end
 
 function graphics.getStencilTest()
-	love.markDeprecated(2, "love.graphics.getStencilTest", "function", "replaced", "love.graphics.getStencilMode")
+	love.markDeprecated(2, "love.graphics.getStencilTest", "function", "replaced", "love.graphics.getStencilMode or getStencilState")
 
-	local action, mode, value = graphics.getStencilMode()
+	local action, mode, value = graphics.getStencilState()
 	return mode, value
 end
 

testing/tests/graphics.lua

@@ -1929,20 +1929,20 @@ love.test.graphics.getStackDepth = function(test)
 end
 
 
--- love.graphics.getStencilMode
-love.test.graphics.getStencilMode = function(test)
+-- love.graphics.getStencilState
+love.test.graphics.getStencilState = function(test)
   -- check default vals
-  local action, comparemode, value = love.graphics.getStencilMode( )
+  local action, comparemode, value = love.graphics.getStencilState( )
   test:assertEquals('keep', action, 'check default stencil action')
   test:assertEquals('always', comparemode, 'check default stencil compare')
   test:assertEquals(0, value, 'check default stencil value')
   -- check set stencil values is returned
-  love.graphics.setStencilMode('replace', 'less', 255)
-  local action, comparemode, value = love.graphics.getStencilMode()
+  love.graphics.setStencilState('replace', 'less', 255)
+  local action, comparemode, value = love.graphics.getStencilState()
   test:assertEquals('replace', action, 'check changed stencil action')
   test:assertEquals('less', comparemode, 'check changed stencil compare')
   test:assertEquals(255, value, 'check changed stencil value')
-  love.graphics.setStencilMode() -- reset
+  love.graphics.setStencilState() -- reset
 end
 
 
@@ -2387,18 +2387,18 @@ love.test.graphics.setShader = function(test)
 end
 
 
--- love.graphics.setStencilMode
-love.test.graphics.setStencilMode = function(test)
+-- love.graphics.setStencilState
+love.test.graphics.setStencilState = function(test)
   local canvas = love.graphics.newCanvas(16, 16)
   love.graphics.setCanvas({canvas, stencil=true})
     love.graphics.clear(0, 0, 0, 1)
-    love.graphics.setStencilMode('replace', 'always', 1)
+    love.graphics.setStencilState('replace', 'always', 1)
     love.graphics.circle('fill', 8, 8, 6)
-    love.graphics.setStencilMode('keep', 'greater', 0)
+    love.graphics.setStencilState('keep', 'greater', 0)
     love.graphics.setColor(1, 0, 0, 1)
     love.graphics.rectangle('fill', 0, 0, 16, 16)
     love.graphics.setColor(1, 1, 1, 1)
-    love.graphics.setStencilMode()
+    love.graphics.setStencilState()
   love.graphics.setCanvas()
   local imgdata = love.graphics.readbackTexture(canvas)
   test:assertPixels(imgdata, {