Browse Source

Upload nightly builds through Travis/AppVeyor (#6253)

Cauê Waneck 8 years ago
parent
commit
fedbe2c22c
4 changed files with 212 additions and 9 deletions
  1. 5 0
      .travis.yml
  2. 6 0
      appveyor.yml
  3. 112 0
      tests/Indexer.hx
  4. 89 9
      tests/RunCi.hx

+ 5 - 0
.travis.yml

@@ -2,6 +2,9 @@ env:
   global:
     # make variables
     - ADD_REVISION=1
+    # nightly builds submit
+    - secure: "UoGjYvQqt66GWmeLC4Pih1iue5AufVgW8XQOd2Bx839NN/2mQQ9bD1HuONJe+taWBJ+PHInkAjYROYYaiCQUA7B1SXs3oQD7Og6arVcR7kY7XOdAQ2t8ZkxJHTnuYGWW/2gNFBESv+3H17bkXG4rzaSn2LV5PJLOxSjw0ziBUMY="
+    - secure: "ugpxt+zeYiAiMYKLK96f5TLSxbQAtmDWiumdwaLHl88fIUeefxJJPIF1Xm0AHeYEJE7sD8dLE1dMbRSzOpXFfTmJoQZv19Wjv+2N5B+DaabKjGj1nZG7q3blGa3nUYzWVfFNFiIpM9c5fvW8yiUFzacZE5itEY8+lZQeGsNh+WQ="
     # SauceLabs
     # - secure: SjyKefmjUEXi0IKHGGpcbLAajU0mLHONg8aA8LoY7Q9nAkSN6Aql+fzS38Boq7w1jWn+2FOpr+4jy0l6wVd/bftsF+huFfYpFJmdh8BlKmE0K71zZAral0H1c7YxkuQpPiJCIFGXqtkvev7SWTy0z31u7kuuQeEyW27boXe5cDA=
     # - secure: sUvWUjCyPuWht4seNa4f2VG9DkvXkhZyLZfjJO9TUAHB2JndS16E2j/qrvKEjycyH6w8tU/B9vnjDRvvGrYXxEXcBEwsJVfkorFnRl9uwGCGIYrzjMhssEl3fMYZK7P304f+gAp5ULrDBX2gIaKeSa8lUNRtz2PsZOieE4kMdhk=
@@ -33,6 +36,7 @@ install_linux: &install_linux
       ocaml-native-compilers
       ocaml-findlib
       camlp4
+      awscli
   - wget https://raw.github.com/ocaml/opam/master/shell/opam_installer.sh -O - | sh -s /usr/local/bin system
   - export OPAMYES=1
   - opam install sedlex camlzip xml-light extlib rope ptmap
@@ -76,6 +80,7 @@ install_osx: &install_osx
   - ls -l out
   - export PATH="$PATH:$TRAVIS_BUILD_DIR"
   - export HAXE_STD_PATH="$TRAVIS_BUILD_DIR/std"
+  - travis_retry brew install awscli
 
 matrix:
   include:

+ 6 - 0
appveyor.yml

@@ -8,9 +8,14 @@ environment:
         MYSQL_PATH: C:\Program Files\MySQL\MySQL Server 5.7
         MYSQL_USER: root
         MYSQL_PASSWORD: Password12!
+        HXBUILDS_AWS_ACCESS_KEY_ID:
+          secure: fggQXlr5xGGl0znUi0UkqPWd6LviHnk0TR6YxJmuV3U=
+        HXBUILDS_AWS_SECRET_ACCESS_KEY:
+          secure: ewwkKcjnSKl/Vtrz1SXmI6XKk1ENmJDyzm5YaR2wi03foRhTke29TvymB21rDTSl
     matrix:
         - TEST: "neko,python,cs,java,macro"
         - TEST: "php7,php"
+          DEPLOY: 1
         - TEST: "cpp"
 
 services:
@@ -50,6 +55,7 @@ install:
     - set PATH=%PATH%;C:\Python34-x64
     # expose the dll files
     - set "PATH=%PATH%;%CYG_ROOT%/usr/i686-w64-mingw32/sys-root/mingw/bin"
+    - choco install awscli
 
 build_script:
     - 'cd %APPVEYOR_BUILD_FOLDER%'

+ 112 - 0
tests/Indexer.hx

@@ -0,0 +1,112 @@
+import sys.io.*;
+using StringTools;
+
+class Indexer
+{
+	public static function index(s3path:String)
+	{
+		var spaceRegex = ~/[ \t]+/g,
+				s3pathRegex = ~/s3:\/\/([^\/]+)/;
+		if (!s3path.endsWith('/')) {
+			s3path = '$s3path/';
+		}
+
+		if (!s3pathRegex.match(s3path)) {
+			throw 'Invalid s3 path $s3path';
+		}
+
+		var basePath = 'http://' + s3pathRegex.matched(1) + '.s3-website-us-east-1.amazonaws.com' + s3pathRegex.matchedRight();
+		var proc = new Process('aws',['s3','ls',s3path,'--region','us-east-1']);
+		var records = [],
+				dirs = [];
+		try
+		{
+			var i = proc.stdout;
+			while(true)
+			{
+				var ln = spaceRegex.split(i.readLine());
+				// trace(ln);
+
+				inline function getPath(path:String)
+				{
+					return basePath + path;
+				}
+				switch(ln[1])
+				{
+					case 'PRE':
+						dirs.push(getPath(ln[2]));
+					case _:
+						var size = ln[2];
+						var path = ln[3];
+						path = getPath(path);
+						records.push({ date: ln[0] + ' ' + ln[1], size: ln[2], path: path, fname:haxe.io.Path.withoutDirectory(path) });
+				}
+			}
+		}
+		catch(e:haxe.io.Eof) {}
+
+		var maxSizes = { date:25, size:15, fname:0 };
+		for (r in records)
+		{
+			if (r.date.length > maxSizes.date)
+				maxSizes.date = r.date.length;
+			if (r.size.length > maxSizes.size)
+				maxSizes.size = r.size.length;
+			if (r.fname.length > maxSizes.fname)
+				maxSizes.fname = r.fname.length;
+		}
+		records.sort(function(v1,v2) return Reflect.compare(v2.date,v1.date));
+		var index = sys.io.File.write('index.html');
+
+		index.writeString(
+'
+<html>
+<head>
+<title>Haxe git builds</title>
+</head>
+<body>
+<!-- Google Tag Manager -->
+<noscript><iframe src="//www.googletagmanager.com/ns.html?id=GTM-M4JZKD"
+height="0" width="0" style="display:none;visibility:hidden"></iframe></noscript>
+<script>(function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({"gtm.start":
+	new Date().getTime(),event:"gtm.js"});var f=d.getElementsByTagName(s)[0],
+	j=d.createElement(s),dl=l!="dataLayer"?"&l="+l:"";j.async=true;j.src=
+	"//www.googletagmanager.com/gtm.js?id="+i+dl;f.parentNode.insertBefore(j,f);
+})(window,document,"script","dataLayer","GTM-M4JZKD");</script>
+<!-- End Google Tag Manager -->
+	<div id="listing">
+	<pre>
+');
+
+		inline function data(date:String,size:String,key:String,path:String)
+		{
+			index.writeString(date.rpad(' ',maxSizes.date));
+			index.writeString(size.rpad(' ',maxSizes.size));
+			if (path != null && path != '')
+				index.writeString('<a href="$path">$key</a>\n');
+			else
+				index.writeString('$key\n');
+		}
+
+		data('Last Modified', 'Size', 'Path','');
+		for (i in 0...(maxSizes.date + maxSizes.size + maxSizes.fname + 5))
+			index.writeString('-');
+		index.writeString('\n\n');
+
+		for (dir in dirs)
+			data('','DIR',haxe.io.Path.withoutDirectory(dir.substr(0,dir.length-1)),dir);
+		for (r in records)
+			if (r.fname != 'index.html')
+				data(r.date,r.size,r.fname,r.path);
+
+		index.writeString(
+'
+</pre>
+</div>
+</body>
+</html>
+');
+
+		index.close();
+	}
+}

+ 89 - 9
tests/RunCi.hx

@@ -61,6 +61,8 @@ class RunCi {
 		throw Fail;
 	}
 
+	static var S3_HXBUILDS_ADDR(default, null) = 's3://hxbuilds/builds/haxe';
+
 	/**
 		Run a command using `Sys.command()`.
 		If the command exits with non-zero code, exit the whole script with the same code.
@@ -532,7 +534,7 @@ class RunCi {
 			}
 			// make time in the UTC time zone
 			var time = Date.fromTime(Std.parseFloat(gitTime) * 1000 - tzd);
-			DateTools.format(time, "%FT%TZ");
+			DateTools.format(time, "%Y-%m-%dT%H:%M:%SZ");
 		}
 	}
 	static var haxeVer(default, never) = {
@@ -564,16 +566,20 @@ class RunCi {
 		) {
 			changeDirectory(repoDir);
 
-			// generate doc
-			runCommand("make", ["-s", "install_dox"]);
-			runCommand("make", ["-s", "package_doc"]);
+			if (Sys.systemName() != 'Windows') {
+				// generate doc
+				runCommand("make", ["-s", "install_dox"]);
+				runCommand("make", ["-s", "package_doc"]);
+
+				// deployBintray();
+				deployApiDoc();
 
-			// deployBintray();
-			deployApiDoc();
+				// disable deployment to ppa:haxe/snapshots for now
+				// because there is no debian sedlex package...
+				// deployPPA();
+			}
 
-			// disable deployment to ppa:haxe/snapshots for now
-			// because there is no debian sedlex package...
-			// deployPPA();
+			deployNightlies();
 		}
 	}
 
@@ -624,6 +630,80 @@ class RunCi {
 		}
 	}
 
+	/**
+		Deploy source package to hxbuilds s3
+	*/
+	static function deployNightlies():Void {
+		var gitTime = commandResult("git", ["show", "-s", "--format=%ct", "HEAD"]).stdout;
+		var tzd = {
+			var z = Date.fromTime(0);
+			z.getHours() * 60 * 60 * 1000 + z.getMinutes() * 60 * 1000;
+		};
+		var time = Date.fromTime(Std.parseFloat(gitTime) * 1000 - tzd);
+		if (
+			(gitInfo.branch == "development" ||
+			gitInfo.branch == "master" ||
+			gitInfo.branch == "nightly-travis") &&
+			Sys.getEnv("HXBUILDS_AWS_ACCESS_KEY_ID") != null &&
+			Sys.getEnv("HXBUILDS_AWS_SECRET_ACCESS_KEY") != null
+		) {
+			if (ci == TravisCI) {
+				runCommand("make", ["-s", "package_unix"]);
+				if (Sys.systemName() == 'Linux') {
+					// source
+					for (file in sys.FileSystem.readDirectory('out')) {
+						if (file.startsWith('haxe') && file.endsWith('_src.tar.gz')) {
+							submitToS3("source", 'out/$file');
+							break;
+						}
+					}
+				}
+				for (file in sys.FileSystem.readDirectory('out')) {
+					if (file.startsWith('haxe') && file.endsWith('_bin.tar.gz')) {
+						var name = Sys.systemName() == "Linux" ? 'linux64' : 'mac';
+						submitToS3(name, 'out/$file');
+						break;
+					}
+				}
+			} else {
+				for (file in sys.FileSystem.readDirectory('out')) {
+					if (file.startsWith('haxe') && file.endsWith('_bin.zip')) {
+						submitToS3('windows', 'out/$file');
+						break;
+					}
+				}
+			}
+		} else {
+			trace('Not deploying nightlies');
+		}
+	}
+
+	static function fileExtension(file:String) {
+		file = haxe.io.Path.withoutDirectory(file);
+		var idx = file.indexOf('.');
+		if (idx < 0) {
+			return '';
+		} else {
+			return file.substr(idx);
+		}
+	}
+
+	static function submitToS3(kind:String, sourceFile:String) {
+		var date = DateTools.format(Date.now(), '%Y-%m-%d');
+		var ext = fileExtension(sourceFile);
+		var fileName = 'haxe_${date}_${gitInfo.branch}_${gitInfo.commit.substr(0,7)}${ext}';
+
+		var changeLatest = gitInfo.branch == "development";
+		Sys.putEnv('AWS_ACCESS_KEY_ID', Sys.getEnv('HXBUILDS_AWS_ACCESS_KEY_ID'));
+		Sys.putEnv('AWS_SECRET_ACCESS_KEY', Sys.getEnv('HXBUILDS_AWS_SECRET_ACCESS_KEY'));
+		runCommand('aws s3 cp --region us-east-1 "$sourceFile" "$S3_HXBUILDS_ADDR/$kind/$fileName"');
+		if (changeLatest) {
+			runCommand('aws s3 cp --region us-east-1 "$sourceFile" "$S3_HXBUILDS_ADDR/$kind/haxe_latest$ext"');
+		}
+		Indexer.index('$S3_HXBUILDS_ADDR/$kind/');
+		runCommand('aws s3 cp --region us-east-1 index.html "$S3_HXBUILDS_ADDR/$kind/index.html"');
+	}
+
 	/**
 		Deploy source package to ppa:haxe/snapshots.
 	*/