added Mask smoothAlpha, prevent some invalid mask setup

h2d/filter/AbstractMask.hx

@@ -97,8 +97,14 @@ class AbstractMask extends Filter {
 
 	override function sync( ctx : RenderContext, obj : h2d.Sprite ) {
 		this.obj = obj;
-		if( mask == null || hide.frame != ctx.frame )
+		if( mask == null || hide.frame != ctx.frame ) {
+			var p = obj;
+			while( p != null ) {
+				if( p == mask ) throw "You can't mask with one of the sprite parents";
+				p = p.parent;
+			}
 			hide.input = null;
+		}
 	}
 
 }

h2d/filter/Mask.hx

@@ -8,12 +8,13 @@ private class MaskShader extends h3d.shader.ScreenShader {
 		@param var mask : Sampler2D;
 		@param var maskMatA : Vec3;
 		@param var maskMatB : Vec3;
+		@const var smoothAlpha : Bool;
 
 		function fragment() {
 			var color = texture.get(input.uv);
 			var uv = vec3(input.uv, 1);
 			var k = mask.get( vec2(uv.dot(maskMatA), uv.dot(maskMatB)) );
-			output.color = vec4(color.rgb, color.a * k.a);
+			output.color = vec4(color.rgb, color.a * (smoothAlpha ? k.a : float(k.a>0)));
 		}
 
 	};
@@ -24,14 +25,22 @@ private class MaskShader extends h3d.shader.ScreenShader {
 class Mask extends AbstractMask {
 
 	var pass : h3d.pass.ScreenFx<MaskShader>;
+	public var smoothAlpha(get, set) : Bool;
 
-	public function new(mask, maskVisible=false) {
+	public function new(mask, maskVisible=false, smoothAlpha=false) {
 		super(mask);
 		pass = new h3d.pass.ScreenFx(new MaskShader());
 		this.maskVisible = maskVisible;
+		this.smoothAlpha = smoothAlpha;
 	}
 
+	function get_smoothAlpha() return pass.shader.smoothAlpha;
+	function set_smoothAlpha(v) return pass.shader.smoothAlpha = v;
+
 	override function draw( ctx : RenderContext, t : h2d.Tile ) {
+		var mask = getMaskTexture(t);
+		if( mask == null )
+			throw "Mask should be rendered before masked object";
 		var out = ctx.textures.allocTarget("maskTmp", ctx, t.width, t.height, false);
 		ctx.engine.pushTarget(out);
 		pass.shader.texture = t.getTexture();