Script updates for assimp-5.2.5 on Linux aarch64 (ARM64) (#163)

* Support using Docker for linux builds
* Fixes to failing unit tests on Linux
* Update arm64 package version for assimp
* Update to use ninja build to improve build performance
* Fix error handling
* Fix failing unit tests that was caused by floating point differences between x86 and arm64

Signed-off-by: Steve Pham <[email protected]>

package-system/assimp/Dockerfile

@@ -0,0 +1,49 @@
+
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+
+# This docker file uses ubuntu 20.04 as the base image so that the AWS Native C++ libraries will use OpenSSL 3 as the base
+# for its dependencies
+#
+
+FROM public.ecr.aws/ubuntu/ubuntu:20.04_stable
+ 
+WORKDIR /data/workspace
+
+ARG ZLIB_FOLDER_PATH
+
+# Initilize apt cache
+RUN apt-get clean && apt-get update
+
+# Setup time zone and locale data (necessary for SSL and HTTPS packages)
+RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata locales keyboard-configuration
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG=en_US.UTF-8
+
+ENV LANG=en_US.UTF-8
+
+# Install the development packages needed to build Qt from source
+RUN apt-get install -y build-essential \
+                       pkg-config \
+                       libminizip-dev \
+                       git \
+                       cmake \
+                       ninja-build
+
+RUN apt upgrade -y
+
+# Prepare a target folder within the container to install the build artifacts tp
+RUN mkdir -p /data/workspace/build
+
+ENV ZLIB_LIB_PATH=${ZLIB_FOLDER_PATH}
+
+ARG CACHEBUST=1
+
+# Copy the build script specific to this Docker script in order to execute the build
+COPY docker_build_assimp_linux.sh /data/workspace/
+

package-system/assimp/build-linux-aarch64.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+
+# TEMP_FOLDER and TARGET_INSTALL_ROOT get set from the pull_and_build_from_git.py script
+
+ZLIB_PACKAGE=zlib-1.2.11-rev5-linux-aarch64
+
+./build-linux.sh $ZLIB_PACKAGE || exit 1
+
+exit 0

package-system/assimp/build-linux-x86.sh

@@ -0,0 +1,16 @@
+#!/bin/bash
+
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+
+# TEMP_FOLDER and TARGET_INSTALL_ROOT get set from the pull_and_build_from_git.py script
+
+ZLIB_PACKAGE=zlib-1.2.11-rev5-linux
+
+./build-linux.sh $ZLIB_PACKAGE || exit 1
+
+exit 0

package-system/assimp/build-linux.sh

@@ -0,0 +1,95 @@
+#!/bin/bash
+
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+
+# TEMP_FOLDER and TARGET_INSTALL_ROOT get set from the pull_and_build_from_git.py script
+
+# Arg 1: The zlib package name
+ZLIB_FOLDER_NAME=$1
+
+# Make sure docker is installed
+DOCKER_VERSION=$(docker --version)
+if [ $? -ne 0 ]
+then
+    echo "Required package docker is not installed"
+    echo "Follow instructions on https://docs.docker.com/engine/install/ubuntu/ to install docker properly"
+    exit 1
+fi
+echo "Detected Docker Version $DOCKER_VERSION"
+
+echo "Using dependent 3rd Party Library ${ZLIB_FOLDER_NAME}"
+
+
+# Prepare the docker file and use the temp folder as the context root
+cp docker_build_assimp_linux.sh temp/
+
+pushd temp
+
+# Build the Docker Image
+echo "Building the docker build script"
+DOCKER_IMAGE_NAME=assimp_linux_3p
+docker build --build-arg ZLIB_FOLDER_PATH=$ZLIB_FOLDER_NAME -f ../Dockerfile -t ${DOCKER_IMAGE_NAME}:latest . 
+if [ $? -ne 0 ]
+then
+    echo "Error occurred creating Docker image ${DOCKER_IMAGE_NAME}:latest." 
+    exit 1
+fi
+
+
+# Capture the Docker Image ID
+IMAGE_ID=$(docker images -q ${DOCKER_IMAGE_NAME}:latest)
+if [ -z $IMAGE_ID ]
+then
+    echo "Error: Cannot find Image ID for ${DOCKER_IMAGE_NAME}"
+    exit 1
+fi
+
+
+# Run the Docker Image
+echo "Running build script in the docker image"
+docker run -v $TEMP_FOLDER/src:/data/workspace/src -v $TEMP_FOLDER/$ZLIB_FOLDER_NAME:/data/workspace/$ZLIB_FOLDER_NAME --tty ${DOCKER_IMAGE_NAME}:latest /data/workspace/docker_build_assimp_linux.sh 
+if [ $? -ne 0 ]
+then
+    echo Failed to build from docker image ${DOCKER_IMAGE_NAME}:latest
+    exit 1
+fi
+
+
+# Capture the container ID
+echo "Capturing the Container ID"
+CONTAINER_ID=$(docker container ls -l -q --filter "ancestor=${DOCKER_IMAGE_NAME}:latest")
+if [ -z $CONTAINER_ID ]
+then
+    echo "Error: Cannot find Container ID for Image ${DOCKER_IMAGE_NAME}"
+    exit 1
+fi
+
+
+# Copy the build artifacts from the Docker Container
+echo "Copying the built contents from the docker container for image ${DOCKER_IMAGE_NAME}"
+
+mkdir -p build
+docker  cp --quiet $CONTAINER_ID:/data/workspace/build/. build  
+if [ $? -ne 0 ]
+then
+    echo "Error occurred copying build artifacts from Docker image ${DOCKER_IMAGE_NAME}:latest." 
+    exit 1
+fi
+
+
+# Clean up the docker image and container
+echo "Cleaning up container"
+docker container rm $CONTAINER_ID || (echo "Warning: unable to clean up container for image ${DOCKER_IMAGE_NAME}")
+
+echo "Cleaning up image"
+docker rmi --force $IMAGE_ID  || (echo "Warning: unable to clean up image ${DOCKER_IMAGE_NAME}")
+
+popd
+
+exit 0
+

package-system/assimp/build_assimp_unix_like.sh

@@ -11,7 +11,7 @@ cmake_base_command="cmake -S temp/src -B temp/src  -DCMAKE_BUILD_TYPE=Release -D
 
 # On Mac, load the toolchain file to make sure
 # the build matches compatibility with other Mac libraries
-if [ "$(uname)" == "Darwin" ];
+if [ "$(uname)" = "Darwin" ];
 then
     echo "Loading Darwin toolchain file"
     cmake_base_command+=" -DCMAKE_TOOLCHAIN_FILE=$PWD/../../Scripts/cmake/Platform/Mac/Toolchain_mac.cmake"
@@ -24,15 +24,18 @@ echo "Running first cmake command:"
 echo "$cmake_no_shared_libs"
 
 eval "$cmake_no_shared_libs temp/src/CMakeLists.txt" || exit 1
-cmake --build temp/src --config release || exit 1
+cmake --build temp/build --config release || exit 1
 
 echo "Running second cmake command:"
 echo "$cmake_shared_libs"
 
 eval "$cmake_shared_libs temp/src/CMakeLists.txt" || exit 1
-cmake --build temp/src --config release || exit 1
+cmake --build temp/build --config release || exit 1
 
-if [ "$(uname)" == "Darwin" ];
+mkdir temp/build/port/
+cp -R temp/src/port/PyAssimp temp/build/port/
+
+if [ "$(uname)" = "Darwin" ];
 then
     # Printing the minimum OS version here can save some time debugging.
     echo "Min OS version:"

package-system/assimp/build_config.json

@@ -48,14 +48,22 @@
                     ["zlib-1.2.11-rev5-linux", "9be5ea85722fc27a8645a9c8a812669d107c68e6baa2ca0740872eaeb6a8b0fc", ""]
                 ],
                 "custom_build_cmd": [
-                    "./build_assimp_unix_like.sh"
+                    "./build-linux-x86.sh"
                 ],
                 "custom_install_json": [
                     "./install_assimp_linux.json"
+                ]
+            },
+            "Linux-aarch64":{
+                "depends_on_packages" :[
+                    ["zlib-1.2.11-rev5-linux-aarch64", "ce9d1ed2883d77ffc69c7982c078595c1f89ca55ec19d89fe7e6beb05f774775", ""]
                 ],
-                "custom_test_cmd" : [
-                   "./test_assimp_linux.sh"
-               ]
+                "custom_build_cmd": [
+                    "./build-linux-aarch64.sh"
+                ],
+                "custom_install_json": [
+                    "./install_assimp_linux.json"
+                ]
             }
         }
     }

package-system/assimp/docker_build_assimp_linux.sh

@@ -0,0 +1,87 @@
+#! /bin/bash
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+# 
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+
+cd /data/workspace/src
+
+
+CPU_ARCH=$(uname -m)
+
+echo "Detected architecture ${CPU_ARCH}"
+if [ "${CPU_ARCH}" = "aarch64" ]
+then
+    AARCH64_FLAGS="-DCMAKE_CXX_FLAGS_INIT=\"-ffp-contract=off\""
+fi
+
+
+# Since we are mapping in a git repo from outside of the docker, git will report that the folder has a 'dubious' owner
+# which will cause code in the CMakeLists.txt that extracts the commit hash to fail, and thus fail the revision
+# unit test. To prevent this, mark the forlder 'src' as a safe directory for git
+git config --global --add safe.directory /data/workspace/src
+
+GIT_HASH=$(git rev-parse --short=8 HEAD)
+echo "Working with Assimp commit hash ${GIT_HASH}"
+
+echo "Using custom zlib (shared) library at /data/workspace/${ZLIB_LIB_PATH}"
+
+
+cmake -S . -B /data/workspace/build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_MODULE_PATH="/data/workspace/${ZLIB_LIB_PATH}" -DASSIMP_BUILD_ZLIB=OFF -DBUILD_SHARED_LIBS=ON -DASSIMP_BUILD_ASSIMP_TOOLS=ON ${AARCH64_FLAGS} 
+if [ $? -ne 0 ]
+then
+    echo "Failed generating cmake project for assimp/shared."
+    exit 1
+fi
+
+
+cmake --build /data/workspace/build 
+if [ $? -ne 0 ]
+then
+    echo "Failed building cmake project for assimp/shared."
+    exit 1
+fi
+
+
+echo "Using custom zlib (static) library at /data/workspace/${ZLIB_LIB_PATH}"
+
+cmake -S . -B /data/workspace/build -GNinja -DCMAKE_BUILD_TYPE=Release -DCMAKE_MODULE_PATH="/data/workspace/${ZLIB_LIB_PATH}" -DASSIMP_BUILD_ZLIB=OFF -DBUILD_SHARED_LIBS=OFF -DASSIMP_BUILD_ASSIMP_TOOLS=ON ${AARCH64_FLAGS}
+if [ $? -ne 0 ]
+then
+    echo "Failed generating cmake project for assimp/static."
+    exit 1
+fi
+
+
+cmake --build /data/workspace/build
+if [ $? -ne 0 ]
+then
+    echo "Failed building cmake project for assimp/shared." 
+    exit 1
+fi
+
+
+mkdir -p /data/workspace/build/port/
+cp -R /data/workspace/src/port/PyAssimp /data/workspace/build/port/
+
+echo "Running unit test"
+
+cd ..
+mkdir -p test_out
+cd test_out
+
+../build/bin/unit 
+if [ $? -eq 0 ]; then
+    echo "Unit Tests Passed"
+    exit 0
+else
+    echo "Unit Tests Failed"
+    exit 1
+fi
+
+exit 0
+

package-system/assimp/install_assimp_linux.json

@@ -1,5 +1,5 @@
 {
-  "Source_Subfolder": "src",
+  "Source_Subfolder": "build",
   "Install_Paths": [
     "include/",
     "lib/libassimp.a",

package-system/assimp/test_assimp_linux.sh

@@ -13,6 +13,6 @@ rm -rf test_out
 mkdir test_out
 cd test_out
 
-../src/bin/unit || exit 1
+../build/bin/unit || exit 1
 exit 0
 

package_build_list_host_linux-aarch64.json

@@ -4,6 +4,7 @@
     "comment3" : "build_from_folder is package name --> folder containing built image of package",
     "comment4" : "Note:  Build from source occurs before build_from_folder",
     "build_from_source": {
+        "assimp-5.2.5-rev1-linux-aarch64":  "Scripts/extras/pull_and_build_from_git.py ../../package-system/assimp --platform-name Linux-aarch64 --clean",
         "astc-encoder-3.2-rev2-linux-aarch64": "Scripts/extras/pull_and_build_from_git.py ../../package-system/astc-encoder --platform-name Linux-aarch64 --clean",
         "AWSGameLiftServerSDK-3.4.2-rev1-linux-aarch64": "package-system/AWSGameLiftServerSDK/build_package_image.py --platform-name linux-aarch64",
         "AwsIotDeviceSdkCpp-1.15.2-rev1-linux-aarch64": "Scripts/extras/pull_and_build_from_git.py ../../package-system/AwsIotDeviceSdkCpp --platform-name Linux-aarch64 --clean",
@@ -39,6 +40,7 @@
         "zlib-1.2.11-rev5-linux-aarch64": "Scripts/extras/pull_and_build_from_git.py ../../package-system/zlib --platform-name Linux-aarch64 --clean"
     },
     "build_from_folder": {
+        "assimp-5.2.5-rev1-linux-aarch64": "package-system/assimp/temp/assimp-linux-aarch64",
         "astc-encoder-3.2-rev2-linux-aarch64": "package-system/astc-encoder/temp/astc-encoder-linux-aarch64",
         "AWSGameLiftServerSDK-3.4.2-rev1-linux-aarch64": "package-system/AWSGameLiftServerSDK-linux-aarch64",
         "AwsIotDeviceSdkCpp-1.15.2-rev1-linux-aarch64": "package-system/AwsIotDeviceSdkCpp/temp/AwsIotDeviceSdkCpp-linux-aarch64",