Travis cache (#6366)

- use travis cache to speed up the builds
- on osx the builds will definitely fail when no cache is available or neko must be rebuild (just rebuild the specific build afterwards), because when we don't fail early we are not able to create a cache and the build will always fail when it takes longer than 50m.

.travis.yml

@@ -1,3 +1,22 @@
+cache:
+  timeout: 1000
+  directories:
+    - $HOME/.opam
+    - $HOME/neko
+    - $HOME/apt-cache
+    - $HOME/hxcache
+    - $HOME/_hxbuild
+    - $HOME/lua_env
+    - $HOME/.luarocks
+    - $HOME/Library/Caches/Homebrew
+
+before_cache:
+  - if [ $TRAVIS_OS_NAME = 'linux' ]; then
+      sudo apt-get autoclean;
+      sudo rm -f $HOME/apt-cache/lock || true;
+    fi
+  - if [ $TRAVIS_OS_NAME = 'osx' ]; then brew cleanup; fi
+
 env:
   global:
     # make variables
@@ -26,10 +45,18 @@ addons: &addons
     - api.haxe.org
 
 install_linux: &install_linux
+  - if [ ! -d "$HOME/neko" ]; then
+      export CACHE_AVAILABLE=0;
+    elif [ ! -d "$HOME/neko/.git" ]; then
+      export CACHE_AVAILABLE=0;
+    else
+      export CACHE_AVAILABLE=1;
+    fi
+  - export APT_CACHE_DIR=~/apt-cache && mkdir -pv $APT_CACHE_DIR
   # Install dependencies
   - sudo add-apt-repository ppa:haxe/ocaml -y
   - sudo apt-get update
-  - sudo apt-get install -y
+  - sudo apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y
       ocaml
       ocaml-native-compilers
       ocaml-findlib
@@ -43,9 +70,20 @@ install_linux: &install_linux
   - export OPAMYES=1
   - opam install sedlex xml-light extlib rope ptmap
   # Install neko
-  - git clone https://github.com/HaxeFoundation/neko.git ../neko
-  - pushd ../neko
-  - cmake . -DSTATIC_DEPS=all -G Ninja
+  - if [ ! -d "$HOME/neko" ]; then
+      mkdir $HOME/neko;
+      pushd $HOME/neko;
+      git clone https://github.com/HaxeFoundation/neko.git .;
+    else
+      pushd $HOME/neko;
+      if [ ! -d "$HOME/neko/.git" ]; then
+        git clone https://github.com/HaxeFoundation/neko.git .;
+      else
+        git fetch --all; git reset --hard origin/master;
+      fi;
+      git submodule update --init --recursive;
+    fi
+  - cmake . -DSTATIC_DEPS=all -G Ninja || (git clean -dfx && export CACHE_AVAILABLE=0 && cmake . -DSTATIC_DEPS=all -G Ninja)
   # download static dependencies before actual build, with 3 chances to deal with network issues
   - ninja download_static_deps || ninja download_static_deps || ninja download_static_deps
   - ninja -j 4
@@ -60,6 +98,10 @@ install_linux: &install_linux
   - jdk_switcher use oraclejdk7
   - java -version
   # Build haxe
+  - if [ -d "$HOME/_hxbuild" ]; then
+      echo "$HOME/_hxbuild exists";
+      cp -pr ~/_hxbuild ./_build;
+    fi
   - make package_src -s
   - opam config exec -- make -s STATICLINK=1
   - make package_bin -s
@@ -68,8 +110,21 @@ install_linux: &install_linux
   - ldd -v ./haxelib
   - export PATH="$PATH:$TRAVIS_BUILD_DIR"
   - export HAXE_STD_PATH="$TRAVIS_BUILD_DIR/std"
+  - if [ ! -d "$HOME/_hxbuild" ]; then
+        mkdir ~/_hxbuild;
+    fi
+  - cp -prf ./_build/* ~/_hxbuild/
+  - sudo rm -f $APT_CACHE_DIR/lock
+
 
 install_osx: &install_osx
+  - if [ ! -d "$HOME/neko" ]; then
+      export CACHE_AVAILABLE=0;
+    elif [ ! -d "$HOME/neko/.git" ]; then
+      export CACHE_AVAILABLE=0;
+    else
+      export CACHE_AVAILABLE=1;
+    fi
   # Install dependencies
   - brew uninstall --force brew-cask # https://github.com/caskroom/homebrew-cask/pull/15381
   - travis_retry brew update
@@ -80,9 +135,16 @@ install_osx: &install_osx
   - eval `opam config env`
   # Install neko
   - brew upgrade cmake # we need a recent cmake to use CMAKE_OSX_DEPLOYMENT_TARGET
-  - git clone https://github.com/HaxeFoundation/neko.git ../neko
-  - pushd ../neko
-  - cmake . -DSTATIC_DEPS=all -G Ninja
+  - if [ ! -d "$HOME/neko" ]; then
+      mkdir $HOME/neko;
+      pushd $HOME/neko;
+      git clone https://github.com/HaxeFoundation/neko.git .;
+    else
+      pushd $HOME/neko;
+      git pull origin master || git clone https://github.com/HaxeFoundation/neko.git .;
+      git submodule update --init --recursive;
+    fi
+  - cmake . -DSTATIC_DEPS=all -G Ninja || (git clean -dfx && export CACHE_AVAILABLE=0 && cmake . -DSTATIC_DEPS=all -G Ninja)
   # download static dependencies before actual build, with 3 chances to deal with network issues
   - ninja download_static_deps || ninja download_static_deps || ninja download_static_deps
   - ninja -j 4
@@ -96,6 +158,10 @@ install_osx: &install_osx
   - mysql -u root -e "grant all on haxe_test.* to travis@localhost;"
   # Build haxe
   - travis_retry brew install zlib pcre
+  - if [ -d "$HOME/_hxbuild" ]; then
+      echo "$HOME/_hxbuild exists";
+      cp -pr ~/_hxbuild ./_build;
+    fi
   - make package_src -s
   - make -s STATICLINK=1 "LIB_PARAMS=/usr/local/opt/zlib/lib/libz.a /usr/local/lib/libpcre.a"
   - make package_bin -s
@@ -104,6 +170,10 @@ install_osx: &install_osx
   - otool -L ./haxelib
   - export PATH="$PATH:$TRAVIS_BUILD_DIR"
   - export HAXE_STD_PATH="$TRAVIS_BUILD_DIR/std"
+  - if [ ! -d "$HOME/_hxbuild" ]; then
+        mkdir ~/_hxbuild;
+    fi
+  - cp -prf ./_build/* ~/_hxbuild/
   - travis_retry brew install awscli
 
 matrix:
@@ -114,7 +184,8 @@ matrix:
     - os: linux
       env:
         - TEST=macro,neko,js,php,php7,flash9,as3,java,cs,python,hl,lua
-        - DEPLOY=1
+        - DEPLOY_API_DOCS=1
+        - DEPLOY_NIGHTLIES=1
         # - SAUCE=1
         # haxeci_decrypt (Deploy source package to ppa:haxe/snapshots.)
         - secure: "Mw3p6bDZuqVQ6u7GrwLQfje5hhIOA4+mdqqLXYHP79UKdhgqb91Dn6IbG9vQ1VXVe64W4YZbQAMBMMRX5kEPDl6JvTVGSBhg00Mi69oO5qrCMcBI6f9FntG72YaVvLf+PA7co+vKrnJzaP2M9pe4SH9Ztbhy0YNxULp7NQ8FLsM="
@@ -129,14 +200,15 @@ matrix:
         - "sh -e /etc/init.d/xvfb start"
         - "export AUDIODEV=null"
       install: *install_linux
-
     - os: linux
       env:
         - TEST=cpp
+        - HXCPP_COMPILE_THREADS=4
         - HXCPP_COMPILE_CACHE=~/hxcache
       before_install:
+        - export APT_CACHE_DIR=~/apt-cache && mkdir -pv $APT_CACHE_DIR
         - sudo apt-get update -y || true
-        - travis_retry sudo apt-get install -y
+        - travis_retry sudo apt-get -o dir::cache::archives="$APT_CACHE_DIR" install -y
             gcc-multilib
             g++-multilib
       install: *install_linux
@@ -147,24 +219,29 @@ matrix:
     - os: osx
       env:
         - TEST=macro,neko,js,php,flash9,as3,java,cs,python,hl,lua
-        - DEPLOY=1
+        - DEPLOY_NIGHTLIES=1
       install: *install_osx
 
     - os: osx
       env:
         - TEST=cpp
         - HXCPP_COMPILE_CACHE=~/hxcache
+        - HXCPP_COMPILE_THREADS=4
       install: *install_osx
 
 script:
   - eval `ssh-agent -s` # for deployment to haxe.org
-  - pushd tests
-  -   mkdir ~/haxelib && haxelib setup ~/haxelib
-  -   haxelib install record-macros
-  -   haxe -version
-  -   haxe RunCi.hxml
-  -   neko RunCi.n
-  - popd
+  - export CAN_BUILD=`(test $CACHE_AVAILABLE = 1 || test $TRAVIS_OS_NAME = 'linux') && echo 1`
+  - if [ ! $CAN_BUILD ]; then
+      echo "No cache available, but initial cache created, please try restarting this job";
+    fi
+  - test $CAN_BUILD && pushd tests
+  - test $CAN_BUILD && mkdir ~/haxelib && haxelib setup ~/haxelib
+  - test $CAN_BUILD && haxelib install record-macros
+  - test $CAN_BUILD && haxe -version
+  - test $CAN_BUILD && haxe RunCi.hxml
+  - test $CAN_BUILD && neko RunCi.n
+  - test $CAN_BUILD && popd
 
 branches:
   except:

tests/RunCi.hx

@@ -104,8 +104,16 @@ class RunCi {
 
 	static function requireAptPackages(packages:Array<String>):Void {
 		var notYetInstalled = [for (p in packages) if (!isAptPackageInstalled(p)) p];
-		if (notYetInstalled.length > 0)
-			runCommand("sudo", ["apt-get", "install", "-y"].concat(notYetInstalled), true);
+		if (notYetInstalled.length > 0) {
+			var aptCacheDir = Sys.getEnv("APT_CACHE_DIR");
+			var baseCommand = if (aptCacheDir != null) {
+				["apt-get", "-o", 'dir::cache::archives=${aptCacheDir}', "install", "-y"];
+			} else {
+				["apt-get", "install", "-y"];
+			};
+			runCommand("sudo", baseCommand.concat(notYetInstalled), true);
+		}
+
 	}
 
 	static function haxelibInstallGit(account:String, repository:String, ?branch:String, ?srcPath:String, useRetry:Bool = false, ?altName:String):Void {
@@ -347,7 +355,7 @@ class RunCi {
 			haxelibInstallGit("HaxeFoundation", "hxcpp", true);
 			var oldDir = Sys.getCwd();
 			changeDirectory(getHaxelibPath("hxcpp") + "tools/hxcpp/");
-			runCommand("haxe", ["compile.hxml"]);
+			runCommand("haxe", ["-D", "source-header=''", "compile.hxml"]);
 			changeDirectory(oldDir);
 		}
 
@@ -402,13 +410,23 @@ class RunCi {
 	}
 
 	static function installLuaVersionDependencies(lv:String){
-	  if (lv == "-l5.1"){
-	    runCommand("luarocks", ["install", "luabitop", "1.0.2-3", "--server=https://luarocks.org/dev"]);
-	  }
-	  runCommand("luarocks", ["install", "lrexlib-pcre", "2.8.0-1", "--server=https://luarocks.org/dev"]);
-	  runCommand("luarocks", ["install", "luv", "1.9.1-0", "--server=https://luarocks.org/dev"]);
-	  runCommand("luarocks", ["install", "luasocket", "3.0rc1-2", "--server=https://luarocks.org/dev"]);
-	  runCommand("luarocks", ["install", "environ", "0.1.0-1", "--server=https://luarocks.org/dev"]);
+		if (lv == "-l5.1"){
+			if (!commandSucceed("luarocks", ["show", "luabit"])) {
+	    		runCommand("luarocks", ["install", "luabitop", "1.0.2-3", "--server=https://luarocks.org/dev"]);
+		  	}
+	  	}
+		if (!commandSucceed("luarocks", ["show", "lrexlib-pcre"])) {
+	  		runCommand("luarocks", ["install", "lrexlib-pcre", "2.8.0-1", "--server=https://luarocks.org/dev"]);
+		}
+		if (!commandSucceed("luarocks", ["show", "luv"])) {
+	  		runCommand("luarocks", ["install", "luv", "1.9.1-0", "--server=https://luarocks.org/dev"]);
+		}
+		if (!commandSucceed("luarocks", ["show", "luasocket"])) {
+	  		runCommand("luarocks", ["install", "luasocket", "3.0rc1-2", "--server=https://luarocks.org/dev"]);
+		}
+		if (!commandSucceed("luarocks", ["show", "environ"])) {
+	  		runCommand("luarocks", ["install", "environ", "0.1.0-1", "--server=https://luarocks.org/dev"]);
+		}
 	}
 
 	static function getCsDependencies() {
@@ -560,24 +578,22 @@ class RunCi {
 	}
 
 	static function deploy():Void {
-		if (
-			Sys.getEnv("DEPLOY") != null
-		) {
+		if (isDeployApiDocsRequired()) {
 			changeDirectory(repoDir);
-
 			if (Sys.systemName() != 'Windows') {
 				// generate doc
 				runCommand("make", ["-s", "install_dox"]);
 				runCommand("make", ["-s", "package_doc"]);
-
 				// deployBintray();
 				deployApiDoc();
-
 				// disable deployment to ppa:haxe/snapshots for now
 				// because there is no debian sedlex package...
 				// deployPPA();
 			}
-
+		}
+		if (
+			Sys.getEnv("DEPLOY_NIGHTLIES") != null
+		) {
 			deployNightlies();
 		}
 	}
@@ -611,22 +627,23 @@ class RunCi {
 		}
 	}
 
+	static function isDeployApiDocsRequired () {
+		return gitInfo.branch == "development" &&
+			Sys.getEnv("DEPLOY_API_DOCS") != null &&
+			Sys.getEnv("deploy_key_decrypt") != null;
+	}
+
 	/**
 		Deploy doc to api.haxe.org.
 	*/
+
 	static function deployApiDoc():Void {
-		if (
-			gitInfo.branch == "development" &&
-			Sys.getEnv("DEPLOY") != null &&
-			Sys.getEnv("deploy_key_decrypt") != null
-		) {
-			// setup deploy_key
-			runCommand("openssl aes-256-cbc -k \"$deploy_key_decrypt\" -in extra/deploy_key.enc -out extra/deploy_key -d");
-			runCommand("chmod 600 extra/deploy_key");
-			runCommand("ssh-add extra/deploy_key");
+		// setup deploy_key
+		runCommand("openssl aes-256-cbc -k \"$deploy_key_decrypt\" -in extra/deploy_key.enc -out extra/deploy_key -d");
+		runCommand("chmod 600 extra/deploy_key");
+		runCommand("ssh-add extra/deploy_key");
 
-			runCommand("make", ["-s", "deploy_doc"]);
-		}
+		runCommand("make", ["-s", "deploy_doc"]);
 	}
 
 	/**

tests/misc/cppObjc/build.hxml

@@ -1,3 +1,4 @@
+-D source-header=''
 -main TestObjc
 -cpp bin
 -debug

tests/sys/compile-each.hxml

@@ -4,6 +4,6 @@
 -cmd nekotools boot bin/neko/ExitCode.n
 
 --next
-
+-D source-header=''
 -debug
 -cp src

tests/unit/compile-cppia-host.hxml

@@ -1,4 +1,5 @@
 -main cpp.cppia.Host
+-D source-header=''
 -D scriptable
 -D dll_export=bin/cppia.classes
 -debug

tests/unit/compile-each.hxml

@@ -1,3 +1,4 @@
+-D source-header=''
 -debug
 -cp src
 -cp "C:\Program Files\The Haxe Effect\src/dev/null"