Merge branch 'save-output' into development

.travis.yml

@@ -13,6 +13,8 @@ env:
     - secure: "ETbwZaeRq8wIVZVyUk1IsNctYVuQa/U2biRkF9pQkz3MEXpaneynclVzNjm8rnm8JqfKcjUDUvQJBP1KYrJYq3tAJFhl31YUnS0FsF3sgLIcnHkhbRA24xJdIlCwHP6QUPoiyPbkec43NRwrF0071KOMD51vgUToXRtAe3o/15g="
     - secure: "Fcrrge2f4jFYDOopig2rwkQvgJw6Ra8UK6OwTVk08wecytzVaOJK1TcB22PSvZ+h0ZLJs34T+pXHFjlNuSWm4+CwGSvnltRD1/svjS8zOqK7RzuUdzHz87yruz9PFqV63HTas6qtmgLqp8n/Q6AhtDLF39BTZPyDzEbi9qkwRuI="
     - secure: "VBJDQNJ9uvdt0aszo7oU3txuRvjkuLmuHZGOkrd4wE/5B4sX5jzx/+dnrKcNTXJCmQ/rVLuMu9GyxqVjNHlzce678voxdQNOtNkNgpkr1qN9/A9rRnCp77hH27ErdthpWxbmcnE62hAJ83TIKSvn//5lAkx4sMCKS1NXEWQ5qec="
+    # HAXECI_GH_TOKEN: haxe-ci Github personal access token
+    - secure: "TpEMYTLgNrVD7kR6hs6EwyWNXUxnfV6XO5MGvYQncKXB1N65PG18n4WQFhnKaH8C2QTFE7dq7688ooXGzwWeoT9WAOBey10jP1f7LXEAjMGAUA4vh2zS93qBZ92ZgzCDZnQN7ZOTQGocwU6Xolu+7/6hP2M8041HBixmFuNkXF4="
 
 sudo: false
 addons:

tests/RunCi.hx

@@ -528,7 +528,7 @@ class RunCi {
 	static var sysDir(default, never) = cwd + "sys/";
 	static var optDir(default, never) = cwd + "optimization/";
 	static var miscDir(default, never) = cwd + "misc/";
-	static var gitInfo(get, null):{repo:String, branch:String, commit:String, date:String};
+	static var gitInfo(get, null):{repo:String, branch:String, commit:String, timestamp:Float, date:String};
 	static function get_gitInfo() return if (gitInfo != null) gitInfo else gitInfo = {
 		repo: switch (ci) {
 			case TravisCI:
@@ -547,12 +547,14 @@ class RunCi {
 				commandResult("git", ["rev-parse", "--abbrev-ref", "HEAD"]).stdout.trim();
 		},
 		commit: commandResult("git", ["rev-parse", "HEAD"]).stdout.trim(),
+		timestamp: Std.parseFloat(commandResult("git", ["show", "-s", "--format=%ct", "HEAD"]).stdout),
 		date: {
 			var gitTime = commandResult("git", ["show", "-s", "--format=%ct", "HEAD"]).stdout;
 			var tzd = {
 				var z = Date.fromTime(0);
 				z.getHours() * 60 * 60 * 1000 + z.getMinutes() * 60 * 1000;
 			}
+			// make time in the UTC time zone
 			var time = Date.fromTime(Std.parseFloat(gitTime) * 1000 - tzd);
 			DateTools.format(time, "%FT%TZ");
 		}
@@ -608,6 +610,135 @@ class RunCi {
 		}
 	}
 
+	static function saveOutput():Void {
+		if (Sys.getEnv("HAXECI_GH_TOKEN") == null) {
+			infoMsg("Missing HAXECI_GH_TOKEN. Will not save output.");
+			return;
+		}
+
+		changeDirectory(repoDir);
+		var haxe_output = Path.join([repoDir, "haxe-output"]);
+		var gitInfo = gitInfo;
+		var TEST = Sys.getEnv("TEST");
+		var TRAVIS_OS_NAME = Sys.getEnv("TRAVIS_OS_NAME");
+		var haxe_output_branch = '${gitInfo.branch}_travisci_${TRAVIS_OS_NAME}_${TEST}';
+		var haxe_output_repo = "github.com/HaxeFoundation/haxe-output.git";
+
+		function haxeCommitTime(sha:String):Float {
+			var cwd = Sys.getCwd();
+			Sys.setCwd(repoDir);
+			var time = Std.parseFloat(commandResult("git", ["show", "-s", "--pretty=%ct", sha]).stdout);
+			Sys.setCwd(cwd);
+			return time;
+		}
+
+		function save():Void {
+			// prepare haxe-output repo
+			runCommand("git", ["clone", 'https://${Sys.getEnv("HAXECI_GH_TOKEN")}@${haxe_output_repo}', haxe_output]);
+			changeDirectory(haxe_output);
+			runCommand("git", ["config", "user.email", "[email protected]"]);
+			runCommand("git", ["config", "user.name", "Haxe CI Bot"]);
+
+			// check to see whether the haxe repo branch has been created in haxe-output
+			var firstOutputBranch = switch (Sys.command("git", ["ls-remote", "--exit-code", "origin", haxe_output_branch])) {
+				case 0: false;
+				case 2: true;
+				case exitcode: throw "unknown exit code: " + exitcode;
+			}
+
+			// switch to the branch
+			if (firstOutputBranch) {
+				runCommand("git", ["checkout", "-B", haxe_output_branch]);
+			} else {
+				runCommand("git", ["checkout", "-B", haxe_output_branch, "--track", "origin/" + haxe_output_branch]);
+			}
+
+			// check to see whether this will be the first push of this haxe repo commit
+			var firstOutputCommit = firstOutputBranch || {
+				var lastLog = commandResult("git", ["show", "-s", "--pretty=format:%B", "HEAD"]).stdout;
+				var commitRe = ~/[a-f0-9]{40}/;
+				if (!commitRe.match(lastLog))
+					throw "No commit sha found in log: " + lastLog;
+				var lastCommit = commitRe.matched(0);
+
+				// check to see whether this commit is newer than the last pushed one
+				// in case e.g. the current build is a rebuild of an old commit
+				if (haxeCommitTime(lastCommit) > gitInfo.timestamp) {
+					throw "There exists output with a later commit in the haxe-output repo.";
+				}
+
+				lastCommit != gitInfo.commit;
+			};
+
+			// Remove all the old output first.
+			// It is because there may be files no longer emitted by the current build.
+			if (firstOutputCommit) {
+				runCommand("rm", ["-rf", "tests"]);
+			}
+
+			// get the outputs by listing the files/folders ignored by git
+			changeDirectory(cwd);
+			var stdout = {
+				var proc = new Process("git", ["status", "--ignored", "--porcelain", cwd]);
+				var out = proc.stdout.readAll().toString();
+				proc.close();
+				out;
+			};
+			var outputs = stdout.split("\n")
+				.filter(function(s) return s.startsWith("!! "))
+				.map(function(s) return s.substr("!! ".length));
+
+			// copy all the outputs to haxe-output
+			FileSystem.createDirectory(haxe_output);
+			for (item in outputs) {
+				var orig = Path.join([repoDir, item]);
+				var dest = Path.join([haxe_output, item]);
+				if (FileSystem.isDirectory(orig)) {
+					FileSystem.createDirectory(dest);
+				} else {
+					FileSystem.createDirectory(Path.directory(dest));
+				}
+				runCommand("cp", ["-rf", orig, dest]);
+			}
+			changeDirectory(haxe_output);
+			runCommand("git", ["add", haxe_output]);
+			var commitMsg = [
+				'-m', '${Sys.getEnv("TRAVIS_JOB_NUMBER")} ${TEST} https://github.com/HaxeFoundation/haxe/commit/${gitInfo.commit}',
+				'-m', 'https://travis-ci.org/HaxeFoundation/haxe/jobs/${Sys.getEnv("TRAVIS_JOB_ID")}',
+			];
+			runCommand("git", ["commit", "-q", "--allow-empty"].concat(commitMsg));
+		}
+
+		// try save() for 5 times because the push may fail when the another build push at the same time
+		var pushError = false;
+		for (i in 0...5) {
+			try {
+				save();
+			} catch (e:Dynamic) {
+				infoMsg(e);
+				pushError = false;
+				break;
+			}
+			var pushResult = commandResult("git", ["push", "origin", haxe_output_branch]);
+			if (pushResult.exitCode == 0) {
+				successMsg("push to haxe-output succeed");
+				pushError = false;
+				break;
+			} else {
+				failMsg("push to haxe-output failed");
+				failMsg(pushResult.stderr);
+				pushError = true;
+
+				runCommand("rm", ["-rf", haxe_output]);
+
+				Sys.sleep(Std.random(10));
+			}
+		}
+		if (pushError) {
+			Sys.exit(1);
+		}
+	}
+
 	static function main():Void {
 		Sys.putEnv("OCAMLRUNPARAM", "b");
 
@@ -894,6 +1025,20 @@ class RunCi {
 					throw "unknown target: " + t;
 			}
 		}
+
+		if (
+			(switch [ci, systemName] {
+				case [TravisCI, "Linux"]: true;
+				case _: false;
+			})
+			&&
+			Lambda.exists(tests, function(t) return switch (t) {
+				case Js | Cpp | Cs | Java | Php | Python: true;
+				case _: false;
+			})
+		) {
+			saveOutput();
+		}
 	}
 
 	static function testHxTemplo() {