Run tests in earthly

Earthfile

@@ -3,14 +3,8 @@ FROM ubuntu:20.04
 WORKDIR /tmp
 
 ENV NEKOPATH=/tmp/neko
-
-build-multiarch:
-    ARG ADD_REVISION
-    BUILD --platform=linux/amd64 --platform=linux/arm64 +build --ADD_REVISION=$ADD_REVISION
         
-install-neko:
-    FROM +install-dependencies
-    
+neko:
     ARG TARGETPLATFORM
     
     IF [ "$TARGETPLATFORM" = "linux/amd64" ]
@@ -21,6 +15,9 @@ install-neko:
         RUN echo "Unsupported platform $TARGETPLATFORM" && exit 1
     END
     
+    RUN apt-get update -qqy \
+        && apt-get install -qqy curl
+    
     RUN set -ex                                                                                                 && \
         curl -sSL https://build.haxe.org/builds/neko/$PLATFORM/neko_latest.tar.gz -o /tmp/neko_latest.tar.gz    && \
         tar -xf /tmp/neko_latest.tar.gz -C /tmp                                                                 && \
@@ -28,28 +25,30 @@ install-neko:
         mkdir -p /usr/local/bin                                                                                 && \
         mkdir -p /usr/local/lib/neko                                                                            && \
         ln -s $NEKOPATH/{neko,nekoc,nekoml,nekotools}  /usr/local/bin/                                          && \
-        ln -s $NEKOPATH/libneko.*                      /usr/local/lib/                                          && \
+        ln -s $NEKOPATH/libneko.*                      /lib/                                                    && \
         ln -s $NEKOPATH/*.ndll                         /usr/local/lib/neko/                                     && \
         PATH=$NEKOPATH:$PATH                                                                                    && \
         neko -version
     
-install-dependencies:
+build-environment:
+    FROM +neko
+    
     RUN set -ex && \
         apt-get update -qqy && \
         apt-get install -qqy software-properties-common && \
         add-apt-repository ppa:avsm/ppa -y && \
         add-apt-repository ppa:haxe/ocaml -y && \
         apt-get update -qqy && \
-        apt-get install -qqy ocaml-nox camlp5 opam libpcre3-dev zlib1g-dev libgtk2.0-dev libmbedtls-dev ninja-build libstring-shellquote-perl libstring-shellquote-perl libipc-system-simple-perl curl
+        apt-get install -qqy ocaml-nox camlp5 opam libpcre3-dev zlib1g-dev libgtk2.0-dev libmbedtls-dev ninja-build libstring-shellquote-perl libstring-shellquote-perl libipc-system-simple-perl
         
 build:
-    FROM +install-neko
+    FROM +build-environment
     
-    ARG ADD_REVISION
     ARG TARGETPLATFORM
+    ARG ADD_REVISION
     
     # Copy files
-    COPY . .
+    COPY --dir extra libs plugins src* std dune* Makefile* opam .
     
     # Install OCaml libraries
     RUN set -ex && \
@@ -75,6 +74,10 @@ build:
     SAVE ARTIFACT ./out/* AS LOCAL out/$TARGETPLATFORM/
     SAVE ARTIFACT ./haxe* AS LOCAL out/$TARGETPLATFORM/
     
+build-multiarch:
+    ARG ADD_REVISION
+    BUILD --platform=linux/amd64 --platform=linux/arm64 +build --ADD_REVISION=$ADD_REVISION
+    
 xmldoc:
     FROM +build
     
@@ -94,4 +97,130 @@ xmldoc:
         
     RUN echo "{\"commit\":\"$COMMIT\",\"branch\":\"$BRANCH\"}" > extra/doc/info.json
     
-    SAVE ARTIFACT ./extra/doc/* AS LOCAL extra/doc/
+    SAVE ARTIFACT ./extra/doc/* AS LOCAL extra/doc/
+    
+# python-test-environment:
+#     FROM python:3
+#     ARG TARGETPLATFORM
+#     DO +INSTALL_NEKO --TARGETPLATFORM=$TARGETPLATFORM --NEKOPATH=$NEKOPATH
+    
+test-environment:
+    FROM +neko
+    
+    ENV DEBIAN_FRONTEND=noninteractive
+    ENV COMMON_PACKAGES=wget git build-essential locales sqlite3
+    
+    # Node.js is required as there are tests that use it (search "-cmd node")
+    RUN curl -fsSL https://deb.nodesource.com/setup_16.x | bash - \
+        && apt-get install -qqy nodejs $COMMON_PACKAGES
+    
+    # set locale
+    RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && locale-gen
+    ENV LANG=en_US.UTF-8  
+    ENV LANGUAGE=en_US:en  
+    ENV LC_ALL=en_US.UTF-8
+    
+INSTALL_PACKAGES:
+    COMMAND
+    ARG PACKAGES
+    RUN set -ex && \
+        apt-get update -qqy && \
+        apt-get install -qqy $PACKAGES && \
+        apt-get clean
+    
+test-environment-java:
+    FROM +test-environment
+    DO +INSTALL_PACKAGES --PACKAGES="default-jdk"
+    
+test-environment-js:
+    # somehow js tests require hxjava which in turns require javac
+    FROM +test-environment-java 
+    
+test-environment-python:
+    FROM +test-environment
+    DO +INSTALL_PACKAGES --PACKAGES="python3"
+    
+test-environment-php:
+    FROM +test-environment
+    DO +INSTALL_PACKAGES --PACKAGES="php-cli php-mbstring php-sqlite3"
+    
+test-environment-cs:
+    FROM +test-environment
+    DO +INSTALL_PACKAGES --PACKAGES="mono-devel mono-mcs"
+    
+test-environment-hl:
+    FROM +test-environment
+    DO +INSTALL_PACKAGES --PACKAGES="libpng-dev libjpeg-turbo8-dev libturbojpeg zlib1g-dev libvorbis-dev cmake ninja-build pkg-config"
+    
+test-environment-lua:
+    # hererocks uses pip
+    FROM +test-environment-python 
+    DO +INSTALL_PACKAGES --PACKAGES="libssl-dev libreadline-dev python3-pip unzip libpcre3-dev cmake"
+    RUN ln -s /root/.local/bin/hererocks /bin/
+    
+test-environment-cpp:
+    FROM +test-environment
+    
+    ARG TARGETPLATFORM
+    
+    IF [ "$TARGETPLATFORM" = "linux/amd64" ]
+        ENV PACKAGES=g++-multilib
+    ELSE IF [ "$TARGETPLATFORM" = "linux/arm64" ]
+        ENV PACKAGES=g++-multilib-arm-linux-gnueabi
+    ELSE 
+        RUN echo "Unsupported platform $TARGETPLATFORM" && exit 1
+    END
+    
+    DO +INSTALL_PACKAGES --PACKAGES=$PACKAGES
+    
+test:
+    ARG TEST # macro, js, hl, cpp, java ,jvm, cs, php, python, lua, neko
+    
+    FROM +test-environment
+    
+    IF [ "$TEST" = "python" ]
+        FROM +test-environment-python
+    ELSE IF [ "$TEST" = "php" ]
+        FROM +test-environment-php
+    ELSE IF [ "$TEST" = "java,jvm" ]
+        FROM +test-environment-java
+    ELSE IF [ "$TEST" = "js" ]
+        FROM +test-environment-js
+    ELSE IF [ "$TEST" = "cs" ]
+        FROM +test-environment-cs
+    ELSE IF [ "$TEST" = "cpp" ]
+        FROM +test-environment-cpp
+    ELSE IF [ "$TEST" = "lua" ]
+        FROM +test-environment-lua
+    ELSE IF [ "$TEST" = "hl" ]
+        FROM +test-environment-hl
+    END
+    
+    ENV HAXE_STD_PATH=/haxe/std
+    WORKDIR /haxe
+    
+    COPY +build/* .
+    COPY --dir tests std .
+    
+    RUN PATH=$PATH:$PWD:$NEKOPATH \
+        && mkdir /haxelib \
+        && haxelib setup /haxelib \
+        && cd tests \
+        && haxe RunCi.hxml
+        
+test-all:
+    ARG TARGETPLATFORM
+    
+    BUILD +test --TEST=macro
+    BUILD +test --TEST=neko
+    BUILD +test --TEST=php
+    BUILD +test --TEST=python
+    BUILD +test --TEST=java,jvm
+    BUILD +test --TEST=cs
+    BUILD +test --TEST=js
+    BUILD +test --TEST=cpp
+    
+    IF [ "$TARGETPLATFORM" = "linux/amd64" ] # FIXME
+        BUILD +test --TEST=hl
+        BUILD +test --TEST=lua
+    END

tests/runci/Linux.hx

@@ -2,11 +2,24 @@ package runci;
 
 import runci.System.*;
 
+using StringTools;
+
 class Linux {
+	static public var arch(get, null):Arch;
+	
+	static function get_arch() {
+		if(arch == null)
+			arch = switch commandResult('arch', []).stdout.replace('\n', '') {
+				case 'arm64' | 'aarch64': Arm64;
+				case _: Amd64;
+			}
+		return arch;
+	}
+	
 	static public function isAptPackageInstalled(aptPackage:String):Bool {
 		return commandSucceed("dpkg-query", ["-W", "-f='${Status}'", aptPackage]);
 	}
-
+	
 	static public function requireAptPackages(packages:Array<String>):Void {
 		var notYetInstalled = [for (p in packages) if (!isAptPackageInstalled(p)) p];
 		if (notYetInstalled.length > 0) {
@@ -19,4 +32,9 @@ class Linux {
 			runCommand("sudo", baseCommand.concat(notYetInstalled), true);
 		}
 	}
+}
+
+enum abstract Arch(Int) {
+	final Arm64;
+	final Amd64;
 }

tests/runci/targets/Cpp.hx

@@ -14,7 +14,10 @@ class Cpp {
 		//hxcpp dependencies
 		switch (systemName) {
 			case "Linux":
-				Linux.requireAptPackages(["gcc-multilib", "g++-multilib"]);
+				Linux.requireAptPackages(["gcc-multilib", switch Linux.arch {
+					case Arm64: "g++-multilib-arm-linux-gnueabi";
+					case Amd64: "g++-multilib";
+				}]);
 			case "Mac":
 				//pass
 		}
@@ -44,7 +47,15 @@ class Cpp {
 	static public function run(args:Array<String>, testCompiled:Bool, testCppia:Bool) {
 		getCppDependencies();
 
-		var archFlag = if (systemName == "Windows") "HXCPP_M32" else "HXCPP_M64";
+		var archFlag = switch systemName {
+			case 'Windows':
+				'HXCPP_M32';
+			case 'Linux' if(Linux.arch == Arm64):
+				'HXCPP_LINUX_ARM64';
+			case _:
+				'HXCPP_M64';
+		}
+				
 		if (testCompiled) {
 			runCommand("rm", ["-rf", "cpp"]);
 			runCommand("haxe", ["compile-cpp.hxml", "-D", archFlag].concat(args));
@@ -52,19 +63,23 @@ class Cpp {
 		}
 
 		if (testCppia) {
-			runCommand("haxe", ["compile-cppia-host.hxml"].concat(args));
+			runCommand("haxe", ["compile-cppia-host.hxml", "-D", archFlag].concat(args));
 			runCommand("haxe", ["compile-cppia.hxml"].concat(args));
 			runCpp("bin/cppia/Host-debug", ["bin/unit.cppia"]);
-			runCpp("bin/cppia/Host-debug", ["bin/unit.cppia", "-jit"]);
+			
+			if(systemName != 'Linux' && Linux.arch != Arm64) // FIXME
+				runCpp("bin/cppia/Host-debug", ["bin/unit.cppia", "-jit"]);
 		}
 
 		changeDirectory(sysDir);
-		runCommand("haxe", ["compile-cpp.hxml"].concat(args));
+		runCommand("haxe", ["-D", archFlag, "--each", "compile-cpp.hxml"].concat(args));
 		runSysTest(FileSystem.fullPath("bin/cpp/Main-debug"));
 
-		changeDirectory(threadsDir);
-		runCommand("haxe", ["build.hxml", "-cpp", "export/cpp"]);
-		runCpp("export/cpp/Main");
+		if(systemName != 'Linux' && Linux.arch != Arm64) { // FIXME 
+			changeDirectory(threadsDir);
+			runCommand("haxe", ["-D", archFlag, "build.hxml", "-cpp", "export/cpp"]);
+			runCpp("export/cpp/Main");
+		}
 
 		// if (Sys.systemName() == "Mac")
 		// {

tests/runci/targets/Php.hx

@@ -30,6 +30,7 @@ class Php {
 		}
 		switch systemName {
 			case "Linux":
+				// TODO: install php-sqlite3?
 				Linux.requireAptPackages(["php-cli", "php-mbstring"]);
 			case "Mac":
 				runCommand("brew", ["install", "php"], true);

tests/runci/targets/Python.hx

@@ -21,7 +21,10 @@ class Python {
 				if (commandSucceed(pypy, ["-V"])) {
 					infoMsg('pypy3 has already been installed.');
 				} else {
-					var pypyVersion = "pypy3.6-v7.3.1-linux64";
+					var pypyVersion = "pypy3.8-v7.3.7-" + (switch Linux.arch {
+						case Arm64: "aarch64";
+						case Amd64: "linux64";
+					});
 					var file = '${pypyVersion}.tar.bz2';
 					if(!FileSystem.exists(file)) {
 						runCommand("wget", ["-nv", 'https://downloads.python.org/pypy/$file'], true);