review stencil implementation, added directx support

h3d/impl/DirectXDriver.hx

@@ -92,10 +92,11 @@ class DirectXDriver extends h3d.impl.Driver {
 	var currentVBuffers = new hl.NativeArray<dx.Resource>(16);
 	var frame : Int;
 	var currentMaterialBits = -1;
+	var currentStencilRef = 0;
 	var targetsCount = 1;
 	var allowDraw = false;
 
-	var depthStates : Map<Int,DepthStencilState>;
+	var depthStates : Map<Int,{ def : DepthStencilState, stencils : Array<{ op : Int, mask : Int, state : DepthStencilState }> }>;
 	var blendStates : Map<Int,BlendState>;
 	var rasterStates : Map<Int,RasterState>;
 	var samplerStates : Map<Int,SamplerState>;
@@ -138,7 +139,7 @@ class DirectXDriver extends h3d.impl.Driver {
 				s.layout.release();
 			}
 		}
-		if( depthStates != null ) for( s in depthStates ) if( s != null ) s.release();
+		if( depthStates != null ) for( s in depthStates ) { if( s.def != null ) s.def.release(); for( s in s.stencils ) if( s.state != null ) s.state.release(); }
 		if( blendStates != null ) for( s in blendStates ) if( s != null ) s.release();
 		if( rasterStates != null ) for( s in rasterStates ) if( s != null ) s.release();
 		if( samplerStates != null ) for( s in samplerStates ) if( s != null ) s.release();
@@ -542,20 +543,59 @@ class DirectXDriver extends h3d.impl.Driver {
 		currentMaterialBits = bits;
 
 		var depthBits = bits & (Pass.depthWrite_mask | Pass.depthTest_mask);
-		if( pass.stencil != null ) throw "TODO: Stencil support";
-		var depth = depthStates.get(depthBits);
+		var depths = depthStates.get(depthBits);
+		var depth = null;
+		var st = pass.stencil;
+		if( depths != null ) {
+			if( st == null )
+				depth = depths.def;
+			else {
+				for( s in depths.stencils )
+					@:privateAccess if( s.op == st.opBits && s.mask == (st.maskBits & ~h3d.mat.Stencil.reference_mask) ) {
+						depth = s.state;
+						break;
+					}
+			}
+		}
 		if( depth == null ) {
 			var cmp = Pass.getDepthTest(bits);
 			var desc = new DepthStencilDesc();
 			desc.depthEnable = cmp != 0;
 			desc.depthFunc = COMPARE[cmp];
 			desc.depthWrite = Pass.getDepthWrite(bits) != 0;
+			if( st != null ) {
+				desc.stencilEnable = true;
+				desc.stencilReadMask = st.readMask;
+				desc.stencilWriteMask = st.writeMask;
+				desc.frontFaceFunc = COMPARE[st.frontTest.getIndex()];
+				desc.frontFacePass = STENCIL_OP[st.frontPass.getIndex()];
+				desc.frontFaceFail = STENCIL_OP[st.frontSTfail.getIndex()];
+				desc.frontFaceDepthFail = STENCIL_OP[st.frontDPfail.getIndex()];
+				desc.backFaceFunc = COMPARE[st.backTest.getIndex()];
+				desc.backFacePass = STENCIL_OP[st.backPass.getIndex()];
+				desc.backFaceFail = STENCIL_OP[st.backSTfail.getIndex()];
+				desc.backFaceDepthFail = STENCIL_OP[st.backDPfail.getIndex()];
+			}
 			depth = Driver.createDepthStencilState(desc);
-			depthStates.set(depthBits, depth);
+			if( depths == null ) {
+				depths = { def : null, stencils : [] };
+				depthStates.set(depthBits, depths);
+			}
+			if( pass.stencil == null )
+				depths.def = depth;
+			else
+				depths.stencils.push(@:privateAccess { op : st.opBits, mask : st.maskBits & ~h3d.mat.Stencil.reference_mask, state : depth });
 		}
-		if( depth != currentDepthState ) {
+		if( depth != currentDepthState || (st != null && st.reference != currentStencilRef) ) {
+			var ref = st == null ? 0 : st.reference;
 			currentDepthState = depth;
+			currentStencilRef = ref;
+			#if (hldx < "1.7")
+			if( ref != 0 ) throw "DirectX Stencil support requires HL 1.7+";
 			Driver.omSetDepthStencilState(depth);
+			#else
+			Driver.omSetDepthStencilState(depth, ref);
+			#end
 		}
 
 		var rasterBits = bits & (Pass.culling_mask | SCISSOR_BIT);
@@ -1045,6 +1085,17 @@ class DirectXDriver extends h3d.impl.Driver {
 		None,
 	];
 
+	static var STENCIL_OP : Array<StencilOp> = [
+		Keep,
+		Zero,
+		Replace,
+		IncrSat,
+		Incr,
+		DecrSat,
+		Decr,
+		Invert,
+	];
+
 	static var BLEND : Array<Blend> = [
 		One,
 		Zero,

h3d/impl/GlDriver.hx

@@ -144,8 +144,7 @@ class GlDriver extends Driver {
 	var curIndexBuffer : IndexBuffer;
 	var curMatBits : Int = -1;
 	var curStOpBits : Int = -1;
-	var curStFrBits : Int = -1;
-	var curStBrBits : Int = -1;
+	var curStMaskBits : Int = -1;
 	var curStEnabled : Bool = false;
 	var defStencil : Stencil;
 	var programs : Map<Int, CompiledProgram>;
@@ -545,7 +544,7 @@ class GlDriver extends Driver {
 				curStEnabled = true;
 			}
 		}
-		@:privateAccess selectStencilBits(s.opBits, s.frontRefBits, s.backRefBits);
+		@:privateAccess selectStencilBits(s.opBits, s.maskBits);
 		// TODO : Blend Op value sync
 	}
 
@@ -613,54 +612,52 @@ class GlDriver extends Driver {
 		curMatBits = bits;
 	}
 
-	function selectStencilBits( opBits : Int, frBits : Int, brBits : Int ) {
+	function selectStencilBits( opBits : Int, maskBits : Int ) {
 		var diffOp = opBits ^ curStOpBits;
-		var diffFr = frBits ^ curStFrBits;
-		var diffBr = brBits ^ curStBrBits;
+		var diffMask = maskBits ^ curStMaskBits;
 
-		if ( (diffOp | diffFr | diffBr) == 0 ) return;
+		if ( (diffOp | diffMask) == 0 ) return;
 
-		if( diffOp & (Stencil.frontSTfail_mask | Stencil.frontDPfail_mask | Stencil.frontDPpass_mask) != 0 ) {
+		if( diffOp & (Stencil.frontSTfail_mask | Stencil.frontDPfail_mask | Stencil.frontPass_mask) != 0 ) {
 			gl.stencilOpSeparate(
 				FACES[Type.enumIndex(Front)],
 				STENCIL_OP[Stencil.getFrontSTfail(opBits)],
 				STENCIL_OP[Stencil.getFrontDPfail(opBits)],
-				STENCIL_OP[Stencil.getFrontDPpass(opBits)]);
+				STENCIL_OP[Stencil.getFrontPass(opBits)]);
 		}
 
-		if( diffOp & (Stencil.backSTfail_mask | Stencil.backDPfail_mask | Stencil.backDPpass_mask) != 0 ) {
+		if( diffOp & (Stencil.backSTfail_mask | Stencil.backDPfail_mask | Stencil.backPass_mask) != 0 ) {
 			gl.stencilOpSeparate(
 				FACES[Type.enumIndex(Back)],
 				STENCIL_OP[Stencil.getBackSTfail(opBits)],
 				STENCIL_OP[Stencil.getBackDPfail(opBits)],
-				STENCIL_OP[Stencil.getBackDPpass(opBits)]);
+				STENCIL_OP[Stencil.getBackPass(opBits)]);
 		}
 
-		if( (diffOp & Stencil.frontTest_mask) | (diffFr & (Stencil.frontRef_mask | Stencil.frontReadMask_mask)) != 0 ) {
+		if( (diffOp & Stencil.frontTest_mask) | (diffMask & (Stencil.reference_mask | Stencil.readMask_mask)) != 0 ) {
 			gl.stencilFuncSeparate(
 				FACES[Type.enumIndex(Front)],
 				COMPARE[Stencil.getFrontTest(opBits)],
-				Stencil.getFrontRef(frBits),
-				Stencil.getFrontReadMask(frBits));
+				Stencil.getReference(maskBits),
+				Stencil.getReadMask(maskBits));
 		}
 
-		if( (diffOp & Stencil.backTest_mask) | (diffBr & (Stencil.backRef_mask | Stencil.backReadMask_mask)) != 0 ) {
+		if( (diffOp & Stencil.backTest_mask) | (diffMask & (Stencil.reference_mask | Stencil.readMask_mask)) != 0 ) {
 			gl.stencilFuncSeparate(
 				FACES[Type.enumIndex(Back)],
 				COMPARE[Stencil.getBackTest(opBits)],
-				Stencil.getBackRef(brBits),
-				Stencil.getBackReadMask(brBits));
+				Stencil.getReference(maskBits),
+				Stencil.getReadMask(maskBits));
 		}
 
-		if( diffFr & Stencil.frontWriteMask_mask != 0 )
-			gl.stencilMaskSeparate(FACES[Type.enumIndex(Front)], Stencil.getFrontWriteMask(frBits));
-
-		if( diffBr & Stencil.backWriteMask_mask != 0 )
-			gl.stencilMaskSeparate(FACES[Type.enumIndex(Back)], Stencil.getBackWriteMask(brBits));
+		if( diffMask & Stencil.writeMask_mask != 0 ) {
+			var w = Stencil.getWriteMask(maskBits);
+			gl.stencilMaskSeparate(FACES[Type.enumIndex(Front)], w);
+			gl.stencilMaskSeparate(FACES[Type.enumIndex(Back)], w);
+		}
 
 		curStOpBits = opBits;
-		curStFrBits = frBits;
-		curStBrBits = brBits;
+		curStMaskBits = maskBits;
 	}
 
 	override function clear( ?color : h3d.Vector, ?depth : Float, ?stencil : Int ) {
@@ -679,7 +676,7 @@ class GlDriver extends Driver {
 		}
 		if( stencil != null ) {
 			// reset stencyl mask when we allow to change it
-			@:privateAccess selectStencilBits(defStencil.opBits, defStencil.frontRefBits, defStencil.backRefBits);
+			@:privateAccess selectStencilBits(defStencil.opBits, defStencil.maskBits);
 			gl.clearStencil(stencil);
 			bits |= GL.STENCIL_BUFFER_BIT;
 		}

h3d/impl/Stage3dDriver.hx

@@ -62,7 +62,7 @@ class Stage3dDriver extends Driver {
 
 	var curMatBits : Int;
 	var curStOpBits : Int;
-	var curStRefBits : Int;
+	var curStMaskBits : Int;
 	var defStencil : Stencil;
 	var curShader : CompiledShader;
 	var curBuffer : Buffer;
@@ -117,7 +117,7 @@ class Stage3dDriver extends Driver {
 		enableDraw = true;
 		curMatBits = -1;
 		curStOpBits = -1;
-		curStRefBits = -1;
+		curStMaskBits = -1;
 		curShader = null;
 		curBuffer = null;
 		curMultiBuffer[0] = -1;
@@ -375,7 +375,7 @@ class Stage3dDriver extends Driver {
 	override function selectMaterial( pass : Pass ) {
 		selectMaterialBits(@:privateAccess pass.bits);
 		var s = pass.stencil != null ? pass.stencil : defStencil;
-		@:privateAccess selectStencilBits(s.opBits, s.frontRefBits, s.backRefBits);
+		@:privateAccess selectStencilBits(s.opBits, s.maskBits);
 	}
 
 	function selectMaterialBits( bits : Int ) {
@@ -415,41 +415,39 @@ class Stage3dDriver extends Driver {
 		curMatBits = bits;
 	}
 
-	function selectStencilBits( opBits : Int, frBits : Int, brBits : Int ) {
-		if( frBits != brBits ) throw "different stencil ref & mask values per face is not allowed in flash";
-
+	function selectStencilBits( opBits : Int, maskBits : Int ) {
 		var diffOp  = opBits ^ curStOpBits;
-		var diffRef = frBits ^ curStRefBits;
+		var diffMask = frBits ^ curStMaskBits;
 
 		if( (diffOp | diffRef) == 0 ) return;
 
-		if( diffOp & (Stencil.frontTest_mask | Stencil.frontSTfail_mask | Stencil.frontDPfail_mask | Stencil.frontDPpass_mask) != 0 ) {
+		if( diffOp & (Stencil.frontTest_mask | Stencil.frontSTfail_mask | Stencil.frontDPfail_mask | Stencil.frontPass_mask) != 0 ) {
 			ctx.setStencilActions(
 				FACE[Type.enumIndex(Front)],
 				COMPARE[Stencil.getFrontTest(opBits)],
-				STENCIL_OP[Stencil.getFrontDPpass(opBits)],
+				STENCIL_OP[Stencil.getFrontPass(opBits)],
 				STENCIL_OP[Stencil.getFrontDPfail(opBits)],
 				STENCIL_OP[Stencil.getFrontSTfail(opBits)]);
 		}
 
-		if( diffOp & (Stencil.backTest_mask | Stencil.backSTfail_mask | Stencil.backDPfail_mask | Stencil.backDPpass_mask) != 0 ) {
+		if( diffOp & (Stencil.backTest_mask | Stencil.backSTfail_mask | Stencil.backDPfail_mask | Stencil.backPass_mask) != 0 ) {
 			ctx.setStencilActions(
 				FACE[Type.enumIndex(Back)],
 				COMPARE[Stencil.getBackTest(opBits)],
-				STENCIL_OP[Stencil.getBackDPpass(opBits)],
+				STENCIL_OP[Stencil.getBackPass(opBits)],
 				STENCIL_OP[Stencil.getBackDPfail(opBits)],
 				STENCIL_OP[Stencil.getBackSTfail(opBits)]);
 		}
 
-		if( diffRef != 0 ) {
+		if( diffMask != 0 ) {
 			ctx.setStencilReferenceValue(
-				Stencil.getFrontRef(frBits),
-				Stencil.getFrontReadMask(frBits),
-				Stencil.getFrontWriteMask(frBits));
+				Stencil.getReference(maskBits),
+				Stencil.getReadMask(maskBits),
+				Stencil.gettWriteMask(maskBits));
 		}
 
 		curStOpBits = opBits;
-		curStRefBits = frBits;
+		curStMaskBits = maskBits;
 	}
 
 	function compileShader( s : hxsl.RuntimeShader.RuntimeShaderData, usedTextures : Array<Bool> ) {

h3d/mat/Stencil.hx

@@ -5,105 +5,69 @@ import h3d.mat.Data;
 @:build(hxd.impl.BitsBuilder.build())
 class Stencil implements hxd.impl.Serializable {
 
-	@:s var frontRefBits : Int = 0;
-	@:s var backRefBits  : Int = 0;
-	@:s var opBits       : Int = 0;
+	@:s var maskBits  : Int = 0;
+	@:s var opBits    : Int = 0;
+
+	@:bits(maskBits, 8) public var readMask : Int;
+	@:bits(maskBits, 8) public var writeMask : Int;
+	@:bits(maskBits, 8) public var reference : Int;
 
 	@:bits(opBits) public var frontTest : Compare;
+	@:bits(opBits) public var frontPass : StencilOp;
 	@:bits(opBits) public var frontSTfail : StencilOp;
 	@:bits(opBits) public var frontDPfail : StencilOp;
-	@:bits(opBits) public var frontDPpass : StencilOp;
-
-	@:bits(frontRefBits, 8) public var frontRef : Int;
-	@:bits(frontRefBits, 8) public var frontReadMask : Int;
-	@:bits(frontRefBits, 8) public var frontWriteMask : Int;
 
 	@:bits(opBits) public var backTest : Compare;
+	@:bits(opBits) public var backPass : StencilOp;
 	@:bits(opBits) public var backSTfail : StencilOp;
 	@:bits(opBits) public var backDPfail : StencilOp;
-	@:bits(opBits) public var backDPpass : StencilOp;
-
-	@:bits(backRefBits, 8) public var backRef : Int;
-	@:bits(backRefBits, 8) public var backReadMask : Int;
-	@:bits(backRefBits, 8) public var backWriteMask : Int;
 
 	public function new() {
-		setFunc(Both, Always, 0, 0xFF);
-		setOp(Both, Keep, Keep, Keep);
-		setMask(Both, 0xFF);
+		setOp(Keep, Keep, Keep);
+		setFunc(Always);
+	}
+
+	public function setFront( stfail : StencilOp, dpfail : StencilOp, pass : StencilOp ) {
+		frontSTfail = stfail;
+		frontDPfail = dpfail;
+		frontPass   = pass;
 	}
 
-	public function setOp( ?face : Face, stfail : StencilOp, dpfail : StencilOp, dppass : StencilOp ) {
-		if( face == null ) face = Both;
-		switch( face ) {
-			case Front :
-				frontSTfail = stfail;
-				frontDPfail = dpfail;
-				frontDPpass = dppass;
-			case Back :
-				backSTfail  = stfail;
-				backDPfail  = dpfail;
-				backDPpass  = dppass;
-			case Both :
-				frontSTfail = backSTfail = stfail;
-				frontDPfail = backDPfail = dpfail;
-				frontDPpass = backDPpass = dppass;
-			default : throw "Invalid face (" + face + "), should be one of [Front, Back, Both]";
-		}
+	public function setBack( stfail : StencilOp, dpfail : StencilOp, pass : StencilOp ) {
+		backSTfail  = stfail;
+		backDPfail  = dpfail;
+		backPass    = pass;
 	}
 
-	public function setMask( ?face : Face, mask : Int ) {
-		if( face == null ) face = Both;
-		switch( face ) {
-			case Front :
-				frontWriteMask = mask;
-			case Back :
-				backWriteMask  = mask;
-			case Both :
-				frontWriteMask = backWriteMask = mask;
-			default : throw "Invalid face (" + face + "), should be one of [Front, Back, Both]";
-		}
+	public function setOp( stfail : StencilOp, dpfail : StencilOp, pass : StencilOp ) {
+		setFront(stfail, dpfail, pass);
+		setBack(stfail, dpfail, pass);
 	}
 
-	public function setFunc( ?face : Face, test : Compare, ref : Int, mask : Int ) {
-		if( face == null ) face = Both;
-		switch( face ) {
-			case Front :
-				frontTest     = test;
-				frontRef      = ref;
-				frontReadMask = mask;
-			case Back :
-				backTest      = test;
-				backRef       = ref;
-				backReadMask  = mask;
-			case Both :
-				frontTest     = backTest     = test;
-				frontRef      = backRef      = ref;
-				frontReadMask = backReadMask = mask;
-			default : throw "Invalid face (" + face + "), should be one of [Front, Back, Both]";
-		}
+	public function setFunc( f : Compare, reference = 0, readMask = 0xFF, writeMask = 0xFF ) {
+		frontTest = backTest = f;
+		this.reference = reference;
+		this.readMask = readMask;
+		this.writeMask = writeMask;
 	}
 
 	public function clone() {
 		var s = new Stencil();
-		s.frontRefBits = frontRefBits;
-		s.backRefBits = backRefBits;
 		s.opBits = opBits;
+		s.opBits = maskBits;
 		return s;
 	}
 
 	public function load(s : Stencil) {
-		frontRefBits = s.frontRefBits;
-		backRefBits = s.backRefBits;
 		opBits = s.opBits;
+		maskBits = s.maskBits;
 	}
 
 	#if hxbit
 	public function customSerialize( ctx : hxbit.Serializer ) {
 	}
 	public function customUnserialize( ctx : hxbit.Serializer ) {
-		loadFrontRefBits(frontRefBits);
-		loadBackRefBits(backRefBits);
+		loadMaskBits(maskBits);
 		loadOpBits(opBits);
 	}
 	#end

samples/Stencil.hx

@@ -31,8 +31,7 @@ class Stencil extends hxd.App {
 			p.setPassName("alpha");
 			p.culling = Front;
 			var s = new h3d.mat.Stencil();
-			s.setFunc(Both, LessEqual, 1, 0xFF);
-			s.setMask(Both, 0x00);
+			s.setFunc(LessEqual, 1, 0xFF, 0);
 			p.stencil = s;
 		}
 
@@ -49,9 +48,8 @@ class Stencil extends hxd.App {
 			var s = new h3d.mat.Stencil();
 			p.depthWrite = false;
 
-			s.setFunc(Both, Always, 1, 0xFF);
-			s.setOp(Both, Keep, Keep, Replace);
-			s.setMask(Both, 0xFF);
+			s.setFunc(Always, 1);
+			s.setOp(Keep, Keep, Replace);
 			p.stencil = s;
 		}