[benchs] start working on proper benchmark exec

.gitignore

@@ -116,4 +116,5 @@ dev-display.hxml
 
 .DS_Store
 tests/sourcemaps/bin
-/*_plugin.ml
+/*_plugin.ml
+tests/benchs/export/

tests/benchs/.vscode/settings.json

@@ -0,0 +1,20 @@
+{
+	"haxe.displayConfigurations": [
+		{"label": "Neko", "args": ["build.hxml", "-neko", "export/run.n", "-cmd", "neko export/run.n"]},
+		{"label": "JavaScript", "args": ["build.hxml", "-js","export/run.js", "-lib", "hxnodejs", "-cmd", "node export/run.js"]},
+		{"label": "Python", "args": ["build.hxml", "-python", "export/run.py", "-cmd", "python3 export/run.py"]},
+		{"label": "C++", "args": ["build.hxml", "-cpp", "export/cpp", "-cmd", "cmd /C export\\cpp\\Main.exe"]},
+		{"label": "CPPIA", "args": ["build.hxml", "-cppia","export/cppia.txt"]},
+		{"label": "HashLink/JIT", "args": ["build.hxml", "-hl", "export/run.hl", "-cmd", "hl export/run.hl"]},
+		{"label": "HashLink/Interp", "args": ["build.hxml", "-hl", "export/run.hl", "-D", "interp"]},
+		{"label": "Lua 5.1", "args": ["build.hxml", "-lua", "export/run.lua", "-cmd", "C:\\WINDOWS\\Sysnative\\bash.exe -c 'lua5.1 export/run.lua'"]},
+		{"label": "Lua 5.2", "args": ["build.hxml", "-lua", "export/run.lua", "-cmd", "C:\\WINDOWS\\Sysnative\\bash.exe -c 'lua5.2 export/run.lua'"]},
+		{"label": "LuaJIT", "args": ["build.hxml", "-lua", "export/run.lua", "-D", "luajit", "-cmd", "C:\\WINDOWS\\Sysnative\\bash.exe -c 'luajit export/run.lua'"]},
+		{"label": "PHP", "args": ["build.hxml", "-php", "export/php", "-cmd", "C:\\WINDOWS\\Sysnative\\bash.exe -c 'php export/php/index.php'"]},
+		{"label": "PHP 7", "args": ["build.hxml", "-php", "export/php", "-D", "php7", "-cmd", "C:\\WINDOWS\\Sysnative\\bash.exe -c 'php7.0 export/php/index.php'"]},
+		{"label": "Java", "args": ["build.hxml", "-java", "export/java", "-cmd", "java -jar export/java/Main.jar"]},
+		{"label": "C#", "args": ["build.hxml", "-cs", "export/cs", "-cmd", "cmd /C export\\cs\\bin\\Main.exe"]},
+		{"label": "Interp", "args": ["build.hxml", "--interp"]},
+	],
+	"haxe.enableCompilationServer": false
+}

tests/benchs/.vscode/tasks.json

@@ -0,0 +1,13 @@
+{
+	"version": "2.0.0",
+	"tasks": [
+		{
+			"type": "haxe",
+			"args": "active configuration",
+			"group": {
+				"kind": "build",
+				"isDefault": true
+			}
+		}
+	]
+}

tests/benchs/build.hxml

@@ -0,0 +1,3 @@
+-cp src
+-main Main
+-D test=Calls

tests/benchs/ds/ds.hxml

@@ -1,4 +0,0 @@
--p src
---main Main
--cp ../hxbenchmark
---interp

tests/benchs/mtnetwork/MTNetwork.hx

@@ -1,271 +0,0 @@
-class Const {
-
-	public static var HOST = new neko.net.Host("localhost");
-	public static var PORT = 1234;
-	public static var LOOPS = 3000;
-	public static var CLIENTS = 200;
-	public static var SOCKETS = 5;
-	public static var SERVERS = 16;
-	public static var USE_POLL = true;
-
-}
-
-class Task  {
-
-	var t : neko.vm.Thread;
-	public var done : Bool;
-
-	public function new() {
-		t = neko.vm.Thread.create(run);
-	}
-
-	function run() {
-		try {
-			loop();
-		} catch( e : Dynamic ) {
-			neko.Lib.print(e);
-			neko.Lib.print("\n");
-			neko.Lib.print(haxe.Stack.toString(haxe.Stack.exceptionStack()));
-			neko.Lib.print("\n");
-		}
-		done = true;
-	}
-
-	function loop() {
-		// TODO
-	}
-
-}
-
-class Client extends Task {
-
-	public var time : Float;
-
-	public function new() {
-		time = 0;
-		super();
-	}
-
-	function loop() {
-		var sockets = new Array();
-		for( i in 0...Const.SOCKETS ) {
-			var s = new neko.net.Socket();
-			while( true ) {
-				var ok = true;
-				try {
-					s.connect(Const.HOST,Const.PORT);
-				} catch( e : Dynamic ) {
-					ok = false;
-					neko.Lib.print("C");
-					neko.Sys.sleep(0.1);
-				}
-				if( ok )
-					break;
-			}
-			sockets.push(s);
-		}
-		var event = Math.round(Const.LOOPS / Const.SOCKETS);
-		var t = neko.Sys.time();
-		for( i in 0...Const.LOOPS ) {
-			var c = i % 256;
-			var k = Std.random(sockets.length);
-			var s = sockets[k];
-			s.output.writeChar(c);
-			if( s.input.readChar() != c ) throw "???";
-			if( k > 0 && k == sockets.length - 1 && Std.random(event) == 0 ) {
-				neko.Lib.print("x");
-				s.close();
-				sockets.remove(s);
-			}
-		}
-		time = neko.Sys.time() - t;
-		for( s in sockets ) {
-			neko.Lib.print("x");
-			s.close();
-		}
-	}
-
-}
-
-class ServerSelect extends Task {
-
-	var sockets : Array<neko.net.Socket>;
-
-	public function new(size) {
-		sockets = new Array();
-		super();
-	}
-
-	public function addClient(sock) {
-		t.sendMessage(sock);
-	}
-
-	function loop() {
-		var s : neko.net.Socket = neko.vm.Thread.readMessage(true);
-		sockets.push(s);
-		while( true ) {
-			for( s in neko.net.Socket.select(sockets,null,null,0.1).read ) {
-				try {
-					s.output.writeChar(s.input.readChar());
-				} catch( e : Dynamic ) {
-					neko.Lib.print("-");
-					sockets.remove(s);
-					s.close();
-				}
-			}
-			while( true ) {
-				var s : neko.net.Socket = neko.vm.Thread.readMessage(false);
-				if( s == null ) {
-					if( sockets.length != 0 )
-						break;
-					done = true;
-					s = neko.vm.Thread.readMessage(true);
-					done = false;
-				}
-				sockets.push(s);
-				neko.Lib.print("+");
-			}
-		}
-	}
-
-}
-
-
-class ServerEvents extends Task {
-
-	var sockets : Array<neko.net.Socket>;
-	var poll : neko.net.Poll;
-
-	public function new(size) {
-		sockets = new Array();
-		poll = new neko.net.Poll(size);
-		super();
-	}
-
-	public function addClient(sock) {
-		t.sendMessage(sock);
-	}
-
-	function loop() {
-		var toremove = new Array();
-		while( true ) {
-			poll.events(0.1);
-			var i = 0;
-			var idx = poll.readIndexes;
-			while( true ) {
-				var s = sockets[idx[i]];
-				if( s == null )
-					break;
-				i++;
-				try {
-					s.output.writeChar(s.input.readChar());
-				} catch( e : Dynamic ) {
-					neko.Lib.print("-");
-					toremove.push(s);
-				}
-			}
-			if( toremove.length > 0 ) {
-				for( s in toremove ) {
-					sockets.remove(s);
-					s.close();
-				}
-				poll.prepare(sockets,[]);
-				toremove = new Array();
-			}
-			var mod = false;
-			while( true ) {
-				var s : neko.net.Socket = neko.vm.Thread.readMessage(false);
-				if( s == null ) {
-					if( sockets.length != 0 )
-						break;
-					done = true;
-					s = neko.vm.Thread.readMessage(true);
-					done = false;
-				}
-				sockets.push(s);
-				mod = true;
-				neko.Lib.print("+");
-			}
-			if( mod )
-				poll.prepare(sockets,[]);
-		}
-	}
-
-}
-
-
-class MTNetwork {
-
-	static var cputime : Void -> Float = neko.Lib.load("std","sys_cpu_time",0);
-
-	static function main() {
-
-		var size = Math.round(Const.CLIENTS * Const.SOCKETS / Const.SERVERS);
-		trace("Starting on "+Const.HOST.toString()+":"+Const.PORT);
-		trace("Socket per server "+size);
-
-		var cpu = cputime();
-		var time = neko.Sys.time();
-
-		var sock = new neko.net.Socket();
-		sock.bind(Const.HOST,Const.PORT);
-		sock.listen(Const.CLIENTS * Const.SOCKETS);
-
-		var servers = new Array<{ done : Bool, addClient : neko.net.Socket -> Void }>();
-		for( i in 0...Const.SERVERS )
-			if( Const.USE_POLL )
-				servers.push(new ServerEvents(size));
-			else
-				servers.push(new ServerSelect(size));
-
-		var clients = new Array();
-		for( i in 0...Const.CLIENTS )
-			clients.push(new Client());
-
-		for( cid in 0...Const.CLIENTS * Const.SOCKETS ) {
-			var s = sock.accept();
-			s.setBlocking(false);
-			servers[cid%servers.length].addClient(s);
-		}
-
-		var done = false;
-		while( !done ) {
-			done = true;
-			for( c in clients )
-				if( !c.done ) {
-					done = false;
-					break;
-				}
-			if( !done ) {
-				neko.Sys.sleep(0.5);
-				continue;
-			}
-			for( s in servers )
-				if( !s.done ) {
-					done = false;
-					break;
-				}
-			neko.Sys.sleep(0.5);
-		}
-
-		var cpu2 = cputime();
-		var time2 = neko.Sys.time();
-
-		var st = 0.;
-		var min = 1000.;
-		var max = 0.;
-		for( c in clients ) {
-			st += c.time;
-			if( c.time < min ) min = c.time;
-			if( c.time > max ) max = c.time;
-		}
-		neko.Lib.print("\n");
-		trace("AVG = "+(st / (Const.CLIENTS * Const.LOOPS)));
-		trace("MIN = "+(min / Const.LOOPS));
-		trace("MAX = "+(max / Const.LOOPS));
-		trace("CPU = "+((cpu2 - cpu) / Const.LOOPS));
-		trace("TOTAL TIME = "+((time2 - time) / Const.LOOPS));
-		trace("DONE");
-	}
-
-}

tests/benchs/src/Macro.hx

@@ -0,0 +1,55 @@
+import haxe.macro.Context;
+import haxe.macro.Expr;
+import haxe.io.Path;
+
+using StringTools;
+using sys.FileSystem;
+
+class Macro {
+	static var singleCaseField = null;
+
+	macro static function buildCase():Array<Field> {
+		var measureFields = [];
+		var fields = Context.getBuildFields();
+		for (field in fields) {
+			if (field.name.startsWith("measure")) {
+				if (singleCaseField == null || field.name.startsWith(singleCaseField)) {
+					measureFields.push(macro $i{field.name});
+				}
+			}
+		}
+		fields.push((macro class C {
+			public function new() {
+				this.fieldFunctions = $a{measureFields};
+			}
+		}).fields[0]);
+		return fields;
+	}
+
+	macro static public function getCases(pack:String) {
+		var cases = [];
+		var singleCase = Context.definedValue("test");
+		if (singleCase != null) {
+			var split = singleCase.split(".");
+			singleCase = split[0];
+			singleCaseField = split[1];
+		}
+		function loop(pack:Array<String>) {
+			var path = Context.resolvePath(Path.join(pack));
+			for (file in sys.FileSystem.readDirectory(path)) {
+				if (singleCase != null && !file.endsWith(singleCase + ".hx")) {
+					continue;
+				}
+				var p = new haxe.io.Path(file);
+				if (p.ext == "hx") {
+					var tp = {pack: pack, name: p.file};
+					cases.push(macro { name:$v{tp.name}, exec:new $tp() });
+				} else if(Path.join([path, file]).isDirectory()) {
+					loop(pack.concat([file]));
+				}
+			}
+		}
+		loop(pack.split('.'));
+		return macro $a{cases};
+	}
+}

tests/benchs/src/Main.hx

@@ -0,0 +1,17 @@
+import hxbenchmark.SuiteResult;
+import hxbenchmark.ResultPrinter;
+
+class Main {
+	static function main() {
+		var cases = Macro.getCases("cases");
+		var printer = new ResultPrinter();
+		function print(result:SuiteResult) {
+			Sys.println(printer.print(result));
+		}
+
+		for (benchCase in cases) {
+			Sys.println('Case: ${benchCase.name}');
+			benchCase.exec.run(print);
+		}
+	}
+}

tests/benchs/src/TestCase.hx

@@ -0,0 +1,11 @@
+import hxbenchmark.SuiteResult;
+@:autoBuild(Macro.buildCase())
+class TestCase {
+	var fieldFunctions:Array<() -> SuiteResult>;
+
+	public function run(cb:SuiteResult -> Void) {
+		for (f in fieldFunctions) {
+			cb(f());
+		}
+	}
+}

tests/benchs/src/cases/Calls.hx

@@ -0,0 +1,48 @@
+package cases;
+
+import haxe.ds.Vector;
+import hxbenchmark.Suite;
+
+interface CallInterface {
+	function instanceCall0():String;
+}
+
+class CallClass implements CallInterface {
+	@:pure(false) static public function staticCall0() { return null; }
+	@:pure(false) public function instanceCall0() { return null; }
+	@:pure(false) public final function finalCall0() { return null; }
+	@:pure(false) public function overrideCall0() { return null; }
+
+	public function new() { }
+}
+
+class CallClassChild extends CallClass {
+	override function overrideCall0() { return null; }
+}
+
+class Calls extends TestCase {
+	@:analyzer(ignore)
+	function measureCreate() {
+		var c = new CallClass();
+		var cSub:CallClass = new CallClassChild();
+		var cInterface:CallInterface = c;
+		var cAnon:{ function instanceCall0():String; } = c;
+		var cDynamic:Dynamic = c;
+		var staticClosureCall0 = CallClass.staticCall0;
+		var memberClosureCall0 = c.instanceCall0;
+		function closureCall0() { return null; }
+
+		var suite = new Suite("call with 0 args");
+		suite.add("static", CallClass.staticCall0());
+		suite.add("instance", c.instanceCall0());
+		suite.add("override", cSub.overrideCall0());
+		suite.add("final", cSub.finalCall0());
+		suite.add("interface", cInterface.instanceCall0());
+		suite.add("anon", cAnon.instanceCall0());
+		suite.add("dynamic", cDynamic.instanceCall0());
+		suite.add("local closure", closureCall0());
+		suite.add("static closure", staticClosureCall0());
+		suite.add("field closure", memberClosureCall0());
+		return suite.run();
+	}
+}

tests/benchs/ds/src/Main.hx → tests/benchs/src/cases/Ds.hx

@@ -1,34 +1,21 @@
-import haxe.ds.Vector;
+package cases;
 
+import haxe.ds.Vector;
 import hxbenchmark.Suite;
 
-class Main {
-	static var printer = new hxbenchmark.ResultPrinter();
-
-	static function main() {
-		measureCreate();
-		measureRead();
-		measureWrite();
-		measureReadWrite();
-		measureIterate();
-		measureMap();
-		measureJoin();
-		measureCopy();
-	}
-
-	static function measureCreate() {
+class Ds extends TestCase {
+	function measureCreate() {
 		var suite = new Suite("create");
 		suite.add("Vector", new Vector(0));
 		suite.add("Array", new Array());
 		suite.add("StringMap", new haxe.ds.StringMap());
 		suite.add("IntMap", new haxe.ds.IntMap());
 		suite.add("ObjectMap", new haxe.ds.ObjectMap());
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 
 	@:analyzer(ignore)
-	static function measureRead() {
+	function measureRead() {
 		var suite = new Suite("read");
 		var a = [1];
 		var v = Vector.fromArrayCopy(a);
@@ -41,11 +28,10 @@ class Main {
 		suite.add("StringMap", sm["foo"]);
 		suite.add("IntMap", im[1]);
 		suite.add("ObjectMap", om[key]);
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 
-	static function measureWrite() {
+	function measureWrite() {
 		var suite = new Suite("write");
 		var a = [1];
 		var v = Vector.fromArrayCopy(a);
@@ -58,11 +44,10 @@ class Main {
 		suite.add("StringMap", sm["foo"] = 1);
 		suite.add("IntMap", im[1] = 1);
 		suite.add("ObjectMap", om[key] = 1);
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 
-	static function measureReadWrite() {
+	function measureReadWrite() {
 		var suite = new Suite("read + write");
 		var a = [1];
 		var v = Vector.fromArrayCopy(a);
@@ -75,29 +60,32 @@ class Main {
 		suite.add("StringMap", sm["foo"] += 1);
 		suite.add("IntMap", im[1] += 1);
 		suite.add("ObjectMap", om[key] += 1);
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 
-	static function measureIterate() {
+	function measureIterate() {
 		var suite = new Suite("iterate + write");
 		var a = [];
-		for (i in 0...10000) a[i] = i;
-		var v = Vector.fromArrayCopy(a);
 		var sm = new Map();
 		var im = new Map();
 		var om = new Map();
+		for (i in 0...10000) {
+			a[i] = i;
+			im[i] = i;
+			sm["" + i] = i;
+			om[{foo: i}] = i;
+		}
+		var v = Vector.fromArrayCopy(a);
 		var key = { foo: 1 };
 		suite.add("Vector", for (i in v) { v[i] = i; });
 		suite.add("Array", for (i in a) { a[i] = i; });
 		suite.add("StringMap", for (i in a) { sm["foo"] = i; });
 		suite.add("IntMap", for (i in a) { im[1] = i; });
 		suite.add("ObjectMap", for (i in a) { om[key] = i; });
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 
-	static function measureMap() {
+	function measureMap() {
 		var suite = new Suite("map");
 		var a = [];
 		for (i in 0...10000) a[i] = i;
@@ -105,11 +93,10 @@ class Main {
 		function f(x) return x;
 		suite.add("Vector", v.map(f));
 		suite.add("Array", a.map(f));
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 
-	static function measureJoin() {
+	function measureJoin() {
 		var suite = new Suite("join");
 		var a = [];
 		for (i in 0...10000) a[i] = i;
@@ -117,11 +104,10 @@ class Main {
 		function f(x) return x;
 		suite.add("Vector", v.join(""));
 		suite.add("Array", a.join(""));
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 
-	static function measureCopy() {
+	function measureCopy() {
 		var suite = new Suite("copy");
 		var a = [];
 		var sm = new Map();
@@ -140,7 +126,6 @@ class Main {
 		suite.add("StringMap", sm.copy());
 		suite.add("IntMap", im.copy());
 		suite.add("ObjectMap", om.copy());
-		var stats = suite.run();
-		trace(printer.print(stats));
+		return suite.run();
 	}
 }

tests/benchs/hxbenchmark/hxbenchmark/ResultPrinter.hx → tests/benchs/src/hxbenchmark/ResultPrinter.hx

@@ -21,15 +21,16 @@ class ResultPrinter {
 			}
 		}
 		var best = result.cases[0].numSamples;
+		trace(best);
 		var buf = new StringBuf();
-		buf.add("Suite: " + result.name + "\n");
+		buf.add("  Suite: " + result.name + "\n");
 		for (caseResult in result.cases) {
-			buf.add("\t");
+			buf.add("    ");
 			buf.add(caseResult.name.lpad(" ", maxNameLength));
 			buf.add(": ");
 			buf.add(printNumber(caseResult.numSamples).lpad(" ", maxSampleLength));
 			buf.add(" (");
-			var percentage = round(Std.string(caseResult.numSamples * 100 / best), 2);
+			var percentage = round(Std.string(caseResult.numSamples * 100. / best), 2);
 			buf.add(percentage.lpad(" ", 6));
 			buf.add("%)\n");
 		}