Stencil (#197)

Added stencil support to h3d.mat.Pass

h3d/impl/GlDriver.hx

@@ -1,6 +1,8 @@
 package h3d.impl;
 import h3d.impl.Driver;
 import h3d.mat.Pass;
+import h3d.mat.Stencil;
+import h3d.mat.Data;
 
 #if (js||cpp||hxsdl)
 
@@ -86,6 +88,11 @@ class GlDriver extends Driver {
 	var curShader : CompiledProgram;
 	var curBuffer : h3d.Buffer;
 	var curMatBits : Int;
+	var curStOpBits : Int;
+	var curStFrBits : Int;
+	var curStBrBits : Int;
+	var curStEnabled : Bool;
+	var defStencil : Stencil;
 	var programs : Map<Int, CompiledProgram>;
 	var frame : Int;
 
@@ -109,6 +116,7 @@ class GlDriver extends Driver {
 		programs = new Map();
 		curAttribs = 0;
 		curMatBits = -1;
+		defStencil = new Stencil();
 	}
 
 	override function logImpl( str : String ) {
@@ -315,6 +323,20 @@ class GlDriver extends Driver {
 
 	override function selectMaterial( pass : Pass ) {
 		selectMaterialBits(@:privateAccess pass.bits);
+		var s = defStencil;
+		if( pass.stencil == null ) {
+			if( curStEnabled ) {
+				gl.disable(GL.STENCIL_TEST);
+				curStEnabled = false;
+			}
+		} else {
+			s = pass.stencil;
+			if( !curStEnabled ) {
+				gl.enable(GL.STENCIL_TEST);
+				curStEnabled = true;
+			}
+		}
+		@:privateAccess selectStencilBits(s.opBits, s.frontRefBits, s.backRefBits);
 		// TODO : Blend Op value sync
 	}
 
@@ -381,6 +403,56 @@ class GlDriver extends Driver {
 		curMatBits = bits;
 	}
 
+	function selectStencilBits( opBits : Int, frBits : Int, brBits : Int ) {
+		var diffOp = opBits ^ curStOpBits;
+		var diffFr = frBits ^ curStFrBits;
+		var diffBr = brBits ^ curStBrBits;
+
+		if ( (diffOp | diffFr | diffBr) == 0 ) return;
+
+		if( diffOp & (Stencil.frontSTfail_mask | Stencil.frontDPfail_mask | Stencil.frontDPpass_mask) != 0 ) {
+			gl.stencilOpSeparate(
+				FACES[Type.enumIndex(Front)],
+				STENCIL_OP[Stencil.getFrontSTfail(opBits)],
+				STENCIL_OP[Stencil.getFrontDPfail(opBits)],
+				STENCIL_OP[Stencil.getFrontDPpass(opBits)]);
+		}
+
+		if( diffOp & (Stencil.backSTfail_mask | Stencil.backDPfail_mask | Stencil.backDPpass_mask) != 0 ) {
+			gl.stencilOpSeparate(
+				FACES[Type.enumIndex(Back)],
+				STENCIL_OP[Stencil.getBackSTfail(opBits)],
+				STENCIL_OP[Stencil.getBackDPfail(opBits)],
+				STENCIL_OP[Stencil.getBackDPpass(opBits)]);
+		}
+
+		if( (diffOp & Stencil.frontTest_mask) | (diffFr & (Stencil.frontRef_mask | Stencil.frontReadMask_mask)) != 0 ) {
+			gl.stencilFuncSeparate(
+				FACES[Type.enumIndex(Front)],
+				COMPARE[Stencil.getFrontTest(opBits)],
+				Stencil.getFrontRef(frBits),
+				Stencil.getFrontReadMask(frBits));
+		}
+
+		if( (diffOp & Stencil.backTest_mask) | (diffBr & (Stencil.backRef_mask | Stencil.backReadMask_mask)) != 0 ) {
+			gl.stencilFuncSeparate(
+				FACES[Type.enumIndex(Back)],
+				COMPARE[Stencil.getBackTest(opBits)],
+				Stencil.getBackRef(brBits),
+				Stencil.getBackReadMask(brBits));
+		}
+
+		if( diffFr & Stencil.frontWriteMask_mask != 0 )
+			gl.stencilMaskSeparate(FACES[Type.enumIndex(Front)], Stencil.getFrontWriteMask(frBits));
+
+		if( diffBr & Stencil.backWriteMask_mask != 0 )
+			gl.stencilMaskSeparate(FACES[Type.enumIndex(Back)], Stencil.getBackWriteMask(brBits));
+
+		curStOpBits = opBits;
+		curStFrBits = frBits;
+		curStBrBits = brBits;
+	}
+
 	override function clear( ?color : h3d.Vector, ?depth : Float, ?stencil : Int ) {
 		var bits = 0;
 		if( color != null ) {
@@ -397,6 +469,7 @@ class GlDriver extends Driver {
 		}
 		if( stencil != null ) {
 			// reset stencyl mask when we allow to change it
+			@:privateAccess selectStencilBits(defStencil.opBits, defStencil.frontRefBits, defStencil.backRefBits);
 			gl.clearStencil(stencil);
 			bits |= GL.STENCIL_BUFFER_BIT;
 		}
@@ -937,6 +1010,17 @@ class GlDriver extends Driver {
 		GL.LEQUAL,
 	];
 
+	static var STENCIL_OP = [
+		GL.KEEP,
+		GL.ZERO,
+		GL.REPLACE,
+		GL.INCR,
+		GL.INCR_WRAP,
+		GL.DECR,
+		GL.DECR_WRAP,
+		GL.INVERT,
+	];
+
 	static var OP = [
 		GL.FUNC_ADD,
 		GL.FUNC_SUBTRACT,

h3d/impl/Stage3dDriver.hx

@@ -1,6 +1,8 @@
 package h3d.impl;
 import h3d.impl.Driver;
 import h3d.mat.Pass;
+import h3d.mat.Stencil;
+import h3d.mat.Data;
 
 #if flash
 
@@ -59,6 +61,9 @@ class Stage3dDriver extends Driver {
 	var onCreateCallback : Bool -> Void;
 
 	var curMatBits : Int;
+	var curStOpBits : Int;
+	var curStRefBits : Int;
+	var defStencil : Stencil;
 	var curShader : CompiledShader;
 	var curBuffer : Buffer;
 	var curManagedBuffer : ManagedBuffer;
@@ -89,6 +94,7 @@ class Stage3dDriver extends Driver {
 		curTextures = [];
 		curSamplerBits = [];
 		curMultiBuffer = [];
+		defStencil = new Stencil();
 	}
 
 	override function logImpl( str : String ) {
@@ -107,6 +113,8 @@ class Stage3dDriver extends Driver {
 	function reset() {
 		enableDraw = true;
 		curMatBits = -1;
+		curStOpBits = -1;
+		curStRefBits = -1;
 		curShader = null;
 		curBuffer = null;
 		curMultiBuffer[0] = -1;
@@ -363,6 +371,8 @@ class Stage3dDriver extends Driver {
 
 	override function selectMaterial( pass : Pass ) {
 		selectMaterialBits(@:privateAccess pass.bits);
+		var s = pass.stencil != null ? pass.stencil : defStencil;
+		@:privateAccess selectStencilBits(s.opBits, s.frontRefBits, s.backRefBits);
 	}
 
 	function selectMaterialBits( bits : Int ) {
@@ -402,6 +412,43 @@ class Stage3dDriver extends Driver {
 		curMatBits = bits;
 	}
 
+	function selectStencilBits( opBits : Int, frBits : Int, brBits : Int ) {
+		if( frBits != brBits ) throw "different stencil ref & mask values per face is not allowed in flash";
+
+		var diffOp  = opBits ^ curStOpBits;
+		var diffRef = frBits ^ curStRefBits;
+
+		if( (diffOp | diffRef) == 0 ) return;
+
+		if( diffOp & (Stencil.frontTest_mask | Stencil.frontSTfail_mask | Stencil.frontDPfail_mask | Stencil.frontDPpass_mask) != 0 ) {
+			ctx.setStencilActions(
+				FACE[Type.enumIndex(Front)],
+				COMPARE[Stencil.getFrontTest(opBits)],
+				STENCIL_OP[Stencil.getFrontDPpass(opBits)],
+				STENCIL_OP[Stencil.getFrontDPfail(opBits)],
+				STENCIL_OP[Stencil.getFrontSTfail(opBits)]);
+		}
+
+		if( diffOp & (Stencil.backTest_mask | Stencil.backSTfail_mask | Stencil.backDPfail_mask | Stencil.backDPpass_mask) != 0 ) {
+			ctx.setStencilActions(
+				FACE[Type.enumIndex(Back)],
+				COMPARE[Stencil.getBackTest(opBits)],
+				STENCIL_OP[Stencil.getBackDPpass(opBits)],
+				STENCIL_OP[Stencil.getBackDPfail(opBits)],
+				STENCIL_OP[Stencil.getBackSTfail(opBits)]);
+		}
+
+		if( diffRef != 0 ) {
+			ctx.setStencilReferenceValue(
+				Stencil.getFrontRef(frBits),
+				Stencil.getFrontReadMask(frBits),
+				Stencil.getFrontWriteMask(frBits));
+		}
+
+		curStOpBits = opBits;
+		curStRefBits = frBits;
+	}
+
 	function compileShader( s : hxsl.RuntimeShader.RuntimeShaderData, usedTextures : Array<Bool> ) {
 		//trace(hxsl.Printer.shaderToString(s.data));
 		var agalVersion = isStandardMode ? 2 : 1;
@@ -836,6 +883,17 @@ class Stage3dDriver extends Driver {
 		flash.display3D.Context3DCompareMode.LESS_EQUAL,
 	];
 
+	static var STENCIL_OP = [
+		flash.display3D.Context3DStencilAction.KEEP,
+		flash.display3D.Context3DStencilAction.ZERO,
+		flash.display3D.Context3DStencilAction.SET,
+		flash.display3D.Context3DStencilAction.INCREMENT_SATURATE,
+		flash.display3D.Context3DStencilAction.INCREMENT_WRAP,
+		flash.display3D.Context3DStencilAction.DECREMENT_SATURATE,
+		flash.display3D.Context3DStencilAction.DECREMENT_WRAP,
+		flash.display3D.Context3DStencilAction.INVERT,
+	];
+
 	static var FORMAT = [
 		flash.display3D.Context3DVertexBufferFormat.BYTES_4,
 		flash.display3D.Context3DVertexBufferFormat.FLOAT_1,

h3d/mat/Data.hx

@@ -37,6 +37,17 @@ enum Compare {
 	LessEqual;
 }
 
+enum StencilOp {
+	Keep;
+	Zero;
+	Replace;
+	Increment;
+	IncrementWrap;
+	Decrement;
+	DecrementWrap;
+	Invert;
+}
+
 enum MipMap {
 	None;
 	Nearest;

h3d/mat/Pass.hx

@@ -20,16 +20,18 @@ class Pass {
 	**/
 	public var dynamicParameters : Bool;
 
-	@:bits public var culling : Face;
-	@:bits public var depthWrite : Bool;
-	@:bits public var depthTest : Compare;
-	@:bits public var blendSrc : Blend;
-	@:bits public var blendDst : Blend;
-	@:bits public var blendAlphaSrc : Blend;
-	@:bits public var blendAlphaDst : Blend;
-	@:bits public var blendOp : Operation;
-	@:bits public var blendAlphaOp : Operation;
-	@:bits(4) public var colorMask : Int;
+	@:bits(bits) public var culling : Face;
+	@:bits(bits) public var depthWrite : Bool;
+	@:bits(bits) public var depthTest : Compare;
+	@:bits(bits) public var blendSrc : Blend;
+	@:bits(bits) public var blendDst : Blend;
+	@:bits(bits) public var blendAlphaSrc : Blend;
+	@:bits(bits) public var blendAlphaDst : Blend;
+	@:bits(bits) public var blendOp : Operation;
+	@:bits(bits) public var blendAlphaOp : Operation;
+	@:bits(bits, 4) public var colorMask : Int;
+
+	public var stencil : Stencil;
 
 	public function new(name, ?shaders, ?parent) {
 		this.parentPass = parent;
@@ -58,6 +60,10 @@ class Pass {
 		blendAlphaDst = p.blendAlphaDst;
 		blendAlphaOp = p.blendAlphaOp;
 		colorMask = p.colorMask;
+		if (p.stencil != null) {
+			if (stencil == null) stencil = new Stencil();
+			stencil.loadProps(p.stencil);
+		}
 	}
 
 	public function setPassName( name : String ) {
@@ -172,6 +178,7 @@ class Pass {
 		var p = new Pass(name, shaders.clone());
 		p.bits = bits;
 		p.enableLights = enableLights;
+		if (stencil != null) p.stencil = stencil.clone();
 		return p;
 	}
 

h3d/mat/Stencil.hx

@@ -0,0 +1,101 @@
+package h3d.mat;
+import h3d.mat.Data;
+
+@:allow(h3d.mat.Material)
+@:build(hxd.impl.BitsBuilder.build())
+class Stencil
+{
+	var frontRefBits : Int = 0;
+	var backRefBits  : Int = 0;
+	var opBits       : Int = 0;
+
+	@:bits(opBits) public var frontTest : Compare;
+	@:bits(opBits) public var frontSTfail : StencilOp;
+	@:bits(opBits) public var frontDPfail : StencilOp;
+	@:bits(opBits) public var frontDPpass : StencilOp;
+
+	@:bits(frontRefBits, 8) public var frontRef : Int;
+	@:bits(frontRefBits, 8) public var frontReadMask : Int;
+	@:bits(frontRefBits, 8) public var frontWriteMask : Int;
+
+	@:bits(opBits) public var backTest : Compare;
+	@:bits(opBits) public var backSTfail : StencilOp;
+	@:bits(opBits) public var backDPfail : StencilOp;
+	@:bits(opBits) public var backDPpass : StencilOp;
+
+	@:bits(backRefBits, 8) public var backRef : Int;
+	@:bits(backRefBits, 8) public var backReadMask : Int;
+	@:bits(backRefBits, 8) public var backWriteMask : Int;
+
+	public function new() {
+		setFunc(Both, Always, 0, 0xFF);
+		setOp(Both, Keep, Keep, Keep);
+		setMask(Both, 0xFF);
+	}
+
+	public function setOp( ?face : Face, stfail : StencilOp, dpfail : StencilOp, dppass : StencilOp ) {
+		if( face == null ) face = Both;
+		switch( face ) {
+			case Front :
+				frontSTfail = stfail;
+				frontDPfail = dpfail;
+				frontDPpass = dppass;
+			case Back :
+				backSTfail  = stfail;
+				backDPfail  = dpfail;
+				backDPpass  = dppass;
+			case Both :
+				frontSTfail = backSTfail = stfail;
+				frontDPfail = backDPfail = dpfail;
+				frontDPpass = backDPpass = dppass;
+			default : throw "Invalid face (" + face + "), should be one of [Front, Back, Both]";
+		}
+	}
+
+	public function setMask( ?face : Face, mask : Int ) {
+		if( face == null ) face = Both;
+		switch( face ) {
+			case Front :
+				frontWriteMask = mask;
+			case Back :
+				backWriteMask  = mask;
+			case Both :
+				frontWriteMask = backWriteMask = mask;
+			default : throw "Invalid face (" + face + "), should be one of [Front, Back, Both]";
+		}
+	}
+
+	public function setFunc( ?face : Face, test : Compare, ref : Int, mask : Int ) {
+		if( face == null ) face = Both;
+		switch( face ) {
+			case Front :
+				frontTest     = test;
+				frontRef      = ref;
+				frontReadMask = mask;
+			case Back :
+				backTest      = test;
+				backRef       = ref;
+				backReadMask  = mask;
+			case Both :
+				frontTest     = backTest     = test;
+				frontRef      = backRef      = ref;
+				frontReadMask = backReadMask = mask;
+			default : throw "Invalid face (" + face + "), should be one of [Front, Back, Both]";
+		}
+	}
+
+	public function clone() {
+		var s = new Stencil();
+		s.frontRefBits = frontRefBits;
+		s.backRefBits = backRefBits;
+		s.opBits = opBits;
+		return s;
+	}
+
+	public function loadProps(s : Stencil) {
+		frontRefBits = s.frontRefBits;
+		backRefBits = s.backRefBits;
+		opBits = s.opBits;
+	}
+
+}

hxd/impl/BitsBuilder.hx

@@ -7,19 +7,36 @@ class BitsBuilder {
 	public static function build() {
 		var fields = Context.getBuildFields();
 		var pos = Context.currentPos();
-		var offset = 0;
+		var offsetMap = new Map<String, Int>();
 		for( f in fields.copy() ) {
 			var bits = 0;
+			var field : String = null;
 			for( m2 in f.meta ) {
 				if( m2.name == ":bits" ) {
-					switch( m2.params ) {
-					case [ { expr : EConst(CInt(v)) } ]: bits = Std.parseInt(v);
-					default: bits = -1;
+					if( m2.params.length < 1 )
+						Context.error("Please specify the bits field", f.pos);
+
+					bits  = -1;
+					field = switch( m2.params[0] ) {
+						case { expr : EConst(CIdent(v)) } : v;
+						default : null;
+					}
+
+					if( m2.params.length > 1 ) {
+						bits = switch( m2.params[1] ) {
+							case { expr : EConst(CInt(v)) } : Std.parseInt(v);
+							default : -1;
+						}
 					}
 					break;
 				}
 			}
+
 			if( bits == 0 ) continue;
+
+			var offset = offsetMap.get(field);
+			if( offset == null ) offset = 0;
+
 			switch( f.kind ) {
 			case FVar(vt = TPath( { pack : pack, name : name }), init):
 				f.kind = FProp("default", "set", vt, init);
@@ -59,10 +76,11 @@ class BitsBuilder {
 					kind : FFun({
 						args : [ { name : "v", type : vt } ],
 						ret : vt,
-						expr : macro { bits = (bits & $v{erase}) | ($expr << $v{offset}); return this.$name = v; }
+						expr : macro { this.$field = (this.$field & $v{erase}) | ($expr << $v{offset}); return this.$name = v; }
 					}),
 					pos : f.pos,
 				});
+
 				fields.push({
 					name : "get" + f.name.charAt(0).toUpperCase() + f.name.substr(1),
 					access : [AStatic,APublic,AInline],
@@ -94,9 +112,11 @@ class BitsBuilder {
 				offset += bits;
 			default:
 			}
+			if( offset > 32 )
+				Context.error(offset + " bits were used while maximum was 32", pos);
+			offsetMap.set(field, offset);
 		}
-		if( offset > 32 )
-			Context.error(offset + " bits were used while maximum was 32", pos);
+
 		return fields;
 	}
 

samples/Stencil.hx

@@ -0,0 +1,74 @@
+import h3d.scene.*;
+
+class Stencil extends hxd.App {
+
+	var time : Float = 0.;
+	var root : Object;
+
+	override function init() {
+		root = new Object(s3d);
+
+		// creates a new unit cube
+		var prim = new h3d.prim.Cube();
+		prim.translate( -0.5, -0.5, 0.0);
+		prim.unindex();
+		prim.addNormals();
+		prim.addUVs();
+		var tex = hxd.Res.hxlogo.toTexture();
+
+		{	// create the top cube
+			var obj = new Mesh(prim, new h3d.mat.MeshMaterial(tex), root);
+			obj.material.mainPass.enableLights = true;
+		}
+
+		{	// create the cube reflection
+			var obj = new Mesh(prim, new h3d.mat.MeshMaterial(tex), root);
+			obj.scaleZ = -1;
+			obj.material.color.setColor(0x55C8FF);
+
+			var p = obj.material.mainPass;
+			var s = new h3d.mat.Stencil();
+			p.culling = Front;
+			p.enableLights = true;
+			s.setFunc(Both, Equal, 1, 0xFF);
+			s.setMask(Both, 0x00);
+			p.stencil = s;
+		}
+
+		{	// create reflection plane
+			var prim = new h3d.prim.Cube(2, 2, 0.0001);
+			prim.addNormals();
+			prim.translate( -1, -1, 0);
+
+			var obj = new Mesh(prim, root);
+			obj.material.color.setColor(0x0080C0);
+
+			var p = obj.material.mainPass;
+			var s = new h3d.mat.Stencil();
+			p.depthWrite = false;
+			p.stencil = new h3d.mat.Stencil();
+			s.setFunc(Both, Always, 1, 0xFF);
+			s.setOp(Both, Keep, Keep, Replace);
+			s.setMask(Both, 0xFF);
+			p.stencil = s;
+		}
+
+		// adds a directional light to the scene
+		var light = new h3d.scene.DirLight(new h3d.Vector(-0.5, -0.5, -0.5), s3d);
+		light.enableSpecular = true;
+		s3d.lightSystem.ambientLight.set(0.3, 0.3, 0.3);
+
+		s3d.camera.pos.set(5, 5, 5);
+	}
+
+	override function update( dt : Float ) {
+		time += 0.01 * dt;
+		root.setRotateAxis (0, 0, 1.0, time);
+	}
+
+	static function main() {
+		hxd.Res.initEmbed();
+		new Stencil();
+	}
+
+}