Require stencil=true to be specified in setCanvas if stenciling is used while a Canvas is active and there is no custom stencil buffer.

--HG--
branch : minor

src/common/pixelformat.cpp

@@ -131,6 +131,11 @@ bool isPixelFormatDepth(PixelFormat format)
 	return iformat >= (int) PIXELFORMAT_DEPTH16 && iformat <= (int) PIXELFORMAT_DEPTH32F_STENCIL8;
 }
 
+bool isPixelFormatStencil(PixelFormat format)
+{
+	return format == PIXELFORMAT_STENCIL8 || format == PIXELFORMAT_DEPTH24_STENCIL8 || format == PIXELFORMAT_DEPTH32F_STENCIL8;
+}
+
 size_t getPixelFormatSize(PixelFormat format)
 {
 	switch (format)

src/common/pixelformat.h

@@ -124,6 +124,11 @@ bool isPixelFormatDepthStencil(PixelFormat format);
  **/
 bool isPixelFormatDepth(PixelFormat format);
 
+/**
+ * Gets whether the specified pixel format is a stencil type.
+ **/
+bool isPixelFormatStencil(PixelFormat format);
+
 /**
  * Gets the size in bytes of the specified pixel format.
  * NOTE: Currently returns 0 for compressed formats.

src/modules/graphics/Graphics.cpp

@@ -314,6 +314,9 @@ void Graphics::restoreStateChecked(const DisplayState &s)
 		{
 			canvaseschanged = true;
 		}
+
+		if (sRTs.useTemporaryStencil != curRTs.useTemporaryStencil)
+			canvaseschanged = true;
 	}
 
 	if (canvaseschanged)
@@ -406,13 +409,14 @@ love::graphics::Shader *Graphics::getShader() const
 	return states.back().shader.get();
 }
 
-void Graphics::setCanvas(RenderTarget rt)
+void Graphics::setCanvas(RenderTarget rt, bool useStencil)
 {
 	if (rt.canvas == nullptr)
 		return setCanvas();
 
 	RenderTargets rts;
 	rts.colors.push_back(rt);
+	rts.useTemporaryStencil = useStencil;
 
 	setCanvas(rts);
 }
@@ -426,6 +430,7 @@ void Graphics::setCanvas(const RenderTargetsStrongRef &rts)
 		targets.colors.emplace_back(rt.canvas.get(), rt.slice, rt.mipmap);
 
 	targets.depthStencil = RenderTarget(rts.depthStencil.canvas, rts.depthStencil.slice, rts.depthStencil.mipmap);
+	targets.useTemporaryStencil = rts.useTemporaryStencil;
 
 	return setCanvas(targets);
 }
@@ -441,13 +446,15 @@ Graphics::RenderTargets Graphics::getCanvas() const
 		rts.colors.emplace_back(rt.canvas.get(), rt.slice, rt.mipmap);
 
 	rts.depthStencil = RenderTarget(curRTs.depthStencil.canvas, curRTs.depthStencil.slice, curRTs.depthStencil.mipmap);
+	rts.useTemporaryStencil = curRTs.useTemporaryStencil;
 
 	return rts;
 }
 
 bool Graphics::isCanvasActive() const
 {
-	return !states.back().renderTargets.colors.empty();
+	const auto &rts = states.back().renderTargets;
+	return !rts.colors.empty() || rts.depthStencil.canvas != nullptr;
 }
 
 bool Graphics::isCanvasActive(love::graphics::Canvas *canvas) const

src/modules/graphics/Graphics.h

@@ -323,9 +323,11 @@ public:
 	{
 		std::vector<RenderTarget> colors;
 		RenderTarget depthStencil;
+		bool useTemporaryStencil;
 
 		RenderTargets()
 			: depthStencil(nullptr)
+			, useTemporaryStencil(false)
 		{}
 	};
 
@@ -333,9 +335,11 @@ public:
 	{
 		std::vector<RenderTargetStrongRef> colors;
 		RenderTargetStrongRef depthStencil;
+		bool useTemporaryStencil;
 
 		RenderTargetsStrongRef()
 			: depthStencil(nullptr)
+			, useTemporaryStencil(false)
 		{}
 	};
 
@@ -459,7 +463,7 @@ public:
 
 	Shader *getShader() const;
 
-	void setCanvas(RenderTarget rt);
+	void setCanvas(RenderTarget rt, bool useStencil);
 	virtual void setCanvas(const RenderTargets &rts) = 0;
 	void setCanvas(const RenderTargetsStrongRef &rts);
 	virtual void setCanvas() = 0;

src/modules/graphics/opengl/Graphics.cpp

@@ -533,6 +533,9 @@ void Graphics::setCanvas(const RenderTargets &rts)
 			modified = true;
 		}
 
+		if (rts.useTemporaryStencil != prevRTs.useTemporaryStencil)
+			modified = true;
+
 		if (!modified)
 			return;
 	}
@@ -631,6 +634,7 @@ void Graphics::setCanvas(const RenderTargets &rts)
 		refs.colors.emplace_back(c.canvas, c.slice);
 
 	refs.depthStencil = RenderTargetStrongRef(rts.depthStencil.canvas, rts.depthStencil.slice);
+	refs.useTemporaryStencil = rts.useTemporaryStencil;
 
 	std::swap(state.renderTargets, refs);
 
@@ -682,7 +686,7 @@ void Graphics::endPass()
 	love::graphics::Canvas *depthstencil = rts.depthStencil.canvas.get();
 
 	// Discard the stencil buffer if we're using an internal cached one.
-	if (depthstencil == nullptr)
+	if (depthstencil == nullptr && rts.useTemporaryStencil)
 		discard({}, true);
 
 	// Resolve MSAA buffers. MSAA is only supported for 2D render targets so we
@@ -907,6 +911,8 @@ void Graphics::bindCachedFBO(const RenderTargets &targets)
 
 	if (targets.depthStencil.canvas != nullptr)
 		hashtargets[hashcount++] = targets.depthStencil;
+	else if (targets.useTemporaryStencil)
+		hashtargets[hashcount++] = RenderTarget(nullptr, -1, 0);
 
 	uint32 hash = XXH32(hashtargets, sizeof(RenderTarget) * hashcount, 0);
 	GLuint fbo = framebufferObjects[hash];
@@ -925,7 +931,7 @@ void Graphics::bindCachedFBO(const RenderTargets &targets)
 
 		RenderTarget depthstencil = targets.depthStencil;
 
-		if (depthstencil.canvas == nullptr)
+		if (depthstencil.canvas == nullptr && targets.useTemporaryStencil)
 		{
 			depthstencil.canvas = getCachedStencilBuffer(w, h, reqmsaa);
 			depthstencil.slice = 0;
@@ -1176,8 +1182,13 @@ void Graphics::setScissor()
 
 void Graphics::drawToStencilBuffer(StencilAction action, int value)
 {
+	const auto &rts = states.back().renderTargets;
+	love::graphics::Canvas *dscanvas = rts.depthStencil.canvas.get();
+
 	if (!isCanvasActive() && !windowHasStencil)
 		throw love::Exception("The window must have stenciling enabled to draw to the main screen's stencil buffer.");
+	else if (isCanvasActive() && !rts.useTemporaryStencil && (dscanvas == nullptr || !isPixelFormatStencil(dscanvas->getPixelFormat())))
+		throw love::Exception("Drawing to the stencil buffer with a Canvas active requires either stencil=true or a custom stencil-type Canvas to be used, in setCanvas.");
 
 	flushStreamDraws();
 
@@ -1237,9 +1248,6 @@ void Graphics::stopDrawToStencilBuffer()
 
 void Graphics::setStencilTest(CompareMode compare, int value)
 {
-	if (compare != COMPARE_ALWAYS && !isCanvasActive() && !windowHasStencil)
-		throw love::Exception("The window must have stenciling enabled to use setStencilTest on the main screen.");
-
 	DisplayState &state = states.back();
 
 	if (state.stencilCompare != compare || state.stencilTestValue != value)

src/modules/graphics/opengl/Image.cpp

@@ -346,7 +346,7 @@ void Image::unloadVolatile()
 void Image::replacePixels(love::image::ImageDataBase *d, int slice, int mipmap, bool reloadmipmaps)
 {
 	// No effect if the texture hasn't been created yet.
-	if (texture == 0 || usingDefaultTexture)
+	if (getHandle() == 0 || usingDefaultTexture)
 		return;
 
 	if (d->getFormat() != getPixelFormat())

src/modules/graphics/wrap_Canvas.cpp

@@ -65,7 +65,7 @@ int w_Canvas_renderTo(lua_State *L)
 		if (oldtargets.depthStencil.canvas != nullptr)
 			oldtargets.depthStencil.canvas->retain();
 
-		luax_catchexcept(L, [&](){ graphics->setCanvas(rt); });
+		luax_catchexcept(L, [&](){ graphics->setCanvas(rt, false); });
 
 		lua_settop(L, 2); // make sure the function is on top of the stack
 		int status = lua_pcall(L, 0, 0, 0);

src/modules/graphics/wrap_Graphics.cpp

@@ -300,6 +300,9 @@ int w_setCanvas(lua_State *L)
 		else if (!lua_isnoneornil(L, -1))
 			targets.depthStencil.canvas = luax_checkcanvas(L, -1);
 		lua_pop(L, 1);
+
+		if (targets.depthStencil.canvas == nullptr)
+			targets.useTemporaryStencil = luax_boolflag(L, 1, "stencil", false);
 	}
 	else
 	{