Update + Fix 3P package script for Qt (#179)

* Update qt package scripts to use Docker + Update qt build flags
* Fixes for x64 build of qt (from docker
* Update to copy from the container the built artifacts rather than link directly to resolve the fact that the artifacts are built with the root owner in the container
* Remove unused arguments and other cleanups
* Update package-system/Qt/Dockerfile
* Fix error handling for commands

Signed-off-by: Steve Pham <[email protected]>

package-system/Qt/Dockerfile

@@ -0,0 +1,50 @@
+
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+
+# This docker file uses ubuntu 20.04 as the base image to install the dependencies to build Qt from source
+#
+
+FROM public.ecr.aws/ubuntu/ubuntu:20.04_stable
+ 
+WORKDIR /data/workspace
+
+# Initilize apt cache
+RUN apt-get clean && apt-get update
+
+# Setup time zone and locale data (necessary for SSL and HTTPS packages)
+RUN DEBIAN_FRONTEND="noninteractive" apt-get -y install tzdata locales keyboard-configuration
+
+RUN sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen && \
+    dpkg-reconfigure --frontend=noninteractive locales && \
+    update-locale LANG=en_US.UTF-8
+
+ENV LANG=en_US.UTF-8
+
+# Install the development packages needed to build Qt from source
+RUN apt-get install -y qtbase5-dev \ 
+                       build-essential \
+                       perl \
+                       python3 \
+                       git \
+                       '^libxcb.*-dev' \
+                       libx11-xcb-dev \
+                       libglu1-mesa-dev \ 
+                       libxrender-dev \
+                       libxi-dev \
+                       libxkbcommon-dev \
+                       libxkbcommon-x11-dev \
+                       libgbm-dev \
+                       libxext-dev \
+                       libfontconfig1-dev \ 
+                       libssl-dev \
+                       libtiff-dev
+
+# Prepare a target folder within the container to install the build artifacts tp
+RUN mkdir -p /data/workspace/qt
+
+# Copy the build script specific to this Docker script in order to execute the build
+COPY docker_build_qt_linux.sh /data/workspace/

package-system/Qt/build-linux-aarch64.sh

@@ -9,95 +9,9 @@
 
 # TEMP_FOLDER and TARGET_INSTALL_ROOT get set from the pull_and_build_from_git.py script
 
-set -euo pipefail
+TIFF_PACKAGE=tiff-4.2.0.15-rev3-linux-aarch64
+ZLIB_PACKAGE=zlib-1.2.11-rev5-linux-aarch64
 
-MAKE_FLAGS=-j32
+./build-linux.sh $TIFF_PACKAGE $ZLIB_PACKAGE || exit 1
 
-echo ""
-echo "------ BUILDING QT5 FROM SOURCE ------"
-echo ""
-echo "BASIC REQUIREMENTS in case something goes wrong:"
-echo "   - git installed and in PATH"
-echo "   - QT5 packages needed for building (https://wiki.qt.io/Building_Qt_5_from_Git)"
-echo "   - Note: This script is currently written for buildng on Ubuntu Linux only."
-echo "   - Note: installing binaries with pip must result with them being on PATH."
-echo ""
-
-# Make sure we have all the required dev packages
-REQUIRED_DEV_PACKAGES="libx11-xcb-dev libxcb-icccm4-dev libxcb-shm0-dev libxcb-image0 libxcb-image0-dev libxcb-util-dev libxcb-keysyms1-dev libxcb-randr0-dev libxcb-render-util0-dev libxcb-sync-dev libxcb-xinerama0-dev libxcb-glx0-dev libgbm-dev libxcb-shape0-dev libxcb-xfixes0-dev libxcb-xkb-dev libfontconfig1-dev libssl-dev libtiff-dev"
-ALL_PACKAGES=`apt list 2>/dev/null`
-for req_package in $REQUIRED_DEV_PACKAGES
-do
-    PACKAGE_COUNT=`echo $ALL_PACKAGES | grep $req_package | wc -l`
-    if [[ $PACKAGE_COUNT -eq 0 ]]; then
-        echo Missing required package $req_package
-        exit 1
-    fi
-done
-
-# Base the Tiff of the dependent tiff O3DE package (static)
-TIFF_PREFIX=$TEMP_FOLDER/tiff-4.2.0.15-rev3-linux-arm64/tiff
-TIFF_INCDIR=$TIFF_PREFIX/include
-TIFF_LIBDIR=$TIFF_PREFIX/lib
-
-# We need to also bring in the zlib dependency since Tiff is a static lib dependency
-ZLIB_PREFIX=$TEMP_FOLDER/zlib-1.2.11-rev5-linux-arm64/zlib
-ZLIB_INCDIR=$ZLIB_PREFIX/include
-ZLIB_LIBDIR=$ZLIB_PREFIX/lib
-
-BUILD_PATH=$TEMP_FOLDER/build
-
-[[ -d $BUILD_PATH ]] || mkdir $BUILD_PATH
-cd $BUILD_PATH
-
-echo Configuring Qt...
-../src/configure \
--prefix ${TARGET_INSTALL_ROOT} \
--opensource \
--nomake examples \
--nomake tests \
--confirm-license \
--no-icu \
--dbus \
--no-separate-debug-info \
--release \
--force-debug-info \
--qt-libpng \
--qt-libjpeg \
--no-feature-vnc \
--no-feature-linuxfb \
---tiff=system \
--qt-zlib \
--v \
--no-cups \
--no-glib \
--no-feature-renameat2 \
--no-feature-getentropy \
--no-feature-statx \
--no-egl \
--I $TIFF_INCDIR \
--I $ZLIB_INCDIR \
--L $TIFF_LIBDIR \
--L $ZLIB_LIBDIR \
--c++std c++1z \
--openssl \
--fontconfig
-
-
-echo Qt configured, building modules...
-qtarray=(qtbase qtgraphicaleffects qtimageformats qtsvg qttools qtx11extras)
-
-for qtlib in "${qtarray[@]}"; do
-    echo Building $qtlib...
-    make module-$qtlib $MAKE_FLAGS
-    echo Built $qtlib.
-done
-
-echo Finished building modules, installing...
-for qtlib in "${qtarray[@]}"; do
-    echo Installing $qtlib...
-    make module-$qtlib-install_subtargets
-    echo $qtlib installed.
-done
-
-echo Qt installed successfully!
+exit 0

package-system/Qt/build-linux-x86.sh

@@ -0,0 +1,17 @@
+#!/bin/bash
+
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+
+# TEMP_FOLDER and TARGET_INSTALL_ROOT get set from the pull_and_build_from_git.py script
+
+TIFF_PACKAGE=tiff-4.2.0.15-rev3-linux
+ZLIB_PACKAGE=zlib-1.2.11-rev5-linux
+
+./build-linux.sh $TIFF_PACKAGE $ZLIB_PACKAGE || exit 1
+
+exit 0

package-system/Qt/build-linux.sh

@@ -9,96 +9,83 @@
 
 # TEMP_FOLDER and TARGET_INSTALL_ROOT get set from the pull_and_build_from_git.py script
 
-set -euo pipefail
-
-MAKE_FLAGS=-j32
-
-echo ""
-echo "------ BUILDING QT5 FROM SOURCE ------"
-echo ""
-echo "BASIC REQUIREMENTS in case something goes wrong:"
-echo "   - git installed and in PATH"
-echo "   - QT5 packages needed for building (https://wiki.qt.io/Building_Qt_5_from_Git)"
-echo "   - Note: This script is currently written for buildng on Ubuntu Linux only."
-echo "   - Note: installing binaries with pip must result with them being on PATH."
-echo ""
-
-# Make sure we have all the required dev packages
-REQUIRED_DEV_PACKAGES="libx11-xcb-dev libxcb-icccm4-dev libxcb-shm0-dev libxcb-image0 libxcb-image0-dev libxcb-util-dev libxcb-keysyms1-dev libxcb-randr0-dev libxcb-render-util0-dev libxcb-sync-dev libxcb-xinerama0-dev libxcb-glx0-dev libgbm-dev libxcb-shape0-dev sudo apt install libxcb-xfixes0-dev libxcb-xkb-dev"
-ALL_PACKAGES=`apt list 2>/dev/null`
-for req_package in $REQUIRED_DEV_PACKAGES
-do
-    PACKAGE_COUNT=`echo $ALL_PACKAGES | grep $req_package | wc -l`
-    if [[ $PACKAGE_COUNT -eq 0 ]]; then
-        echo Missing required package $req_package
-        exit 1
-    fi
-done
-
-# Base the Tiff of the dependent tiff O3DE package (static)
-TIFF_PREFIX=$TEMP_FOLDER/tiff-4.2.0.15-rev3-linux/tiff
-TIFF_INCDIR=$TIFF_PREFIX/include
-TIFF_LIBDIR=$TIFF_PREFIX/lib
-
-# We need to also bring in the zlib dependency since Tiff is a static lib dependency
-ZLIB_PREFIX=$TEMP_FOLDER/zlib-1.2.11-rev5-linux/zlib
-ZLIB_INCDIR=$ZLIB_PREFIX/include
-ZLIB_LIBDIR=$ZLIB_PREFIX/lib
-
-BUILD_PATH=$TEMP_FOLDER/build
-
-[[ -d $BUILD_PATH ]] || mkdir $BUILD_PATH
-cd $BUILD_PATH
-
-echo Configuring Qt...
-../src/configure \
--prefix ${TARGET_INSTALL_ROOT} \
--opensource \
--nomake examples \
--nomake tests \
--confirm-license \
--no-icu \
--dbus \
--no-separate-debug-info \
--release \
--force-debug-info \
--qt-libpng \
--qt-libjpeg \
--no-feature-vnc \
--no-feature-linuxfb \
---tiff=system \
--qt-zlib \
--v \
--no-cups \
--no-glib \
--no-feature-renameat2 \
--no-feature-getentropy \
--no-feature-statx \
--no-egl \
--I $TIFF_INCDIR \
--I $ZLIB_INCDIR \
--L $TIFF_LIBDIR \
--L $ZLIB_LIBDIR \
--c++std c++1z \
--openssl \
--reduce-relocations \
--fontconfig
-
-
-echo Qt configured, building modules...
-qtarray=(qtbase qtgraphicaleffects qtimageformats qtsvg qttools qtx11extras)
-
-for qtlib in "${qtarray[@]}"; do
-    echo Building $qtlib...
-    make module-$qtlib $MAKE_FLAGS
-    echo Built $qtlib.
-done
-
-echo Finished building modules, installing...
-for qtlib in "${qtarray[@]}"; do
-    echo Installing $qtlib...
-    make module-$qtlib-install_subtargets
-    echo $qtlib installed.
-done
-
-echo Qt installed successfully!
+# Arg 1: The tiff package name
+TIFF_FOLDER_NAME=$1
+
+# Arg 2: The zlib package name
+ZLIB_FOLDER_NAME=$2
+
+# Make sure docker is installed
+DOCKER_VERSION=$(docker --version)
+if [ $? -ne 0 ]
+then
+    echo "Required package docker is not installed"
+    echo "Follow instructions on https://docs.docker.com/engine/install/ubuntu/ to install docker properly"
+    exit 1
+fi
+echo "Detected Docker Version $DOCKER_VERSION"
+
+# Prepare the docker file and use the temp folder as the context root
+cp docker_build_qt_linux.sh temp/
+
+pushd temp
+
+# Build the Docker Image
+echo "Building the docker build script"
+DOCKER_IMAGE_NAME=qt_linux_3p
+docker build -f ../Dockerfile -t ${DOCKER_IMAGE_NAME}:latest . 
+if [ $? -ne 0 ]
+then
+    echo "Error occurred creating Docker image ${DOCKER_IMAGE_NAME}:latest."
+    exit 1
+fi
+
+
+# Capture the Docker Image ID
+IMAGE_ID=$(docker images -q ${DOCKER_IMAGE_NAME}:latest)
+if [ -z $IMAGE_ID ]
+then
+    echo "Error: Cannot find Image ID for ${DOCKER_IMAGE_NAME}"
+    exit 1
+fi
+
+# Run the Docker Image
+echo "Running docker build script"
+docker run -v $TEMP_FOLDER/src:/data/workspace/src -v $TEMP_FOLDER/$TIFF_FOLDER_NAME:/data/workspace/$TIFF_FOLDER_NAME -v $TEMP_FOLDER/$ZLIB_FOLDER_NAME:/data/workspace/$ZLIB_FOLDER_NAME --tty ${DOCKER_IMAGE_NAME}:latest ./docker_build_qt_linux.sh 
+if [ $? -ne 0 ]
+then
+    echo "Error occurred running Docker image ${DOCKER_IMAGE_NAME}:latest." 
+    exit 1
+fi
+
+
+# Capture the container ID
+echo "Capturing the Container ID"
+CONTAINER_ID=$(docker container ls -l -q --filter "ancestor=${DOCKER_IMAGE_NAME}:latest")
+if [ -z $CONTAINER_ID ]
+then
+    echo "Error: Cannot find Container ID for Image ${DOCKER_IMAGE_NAME}"
+    exit 1
+fi
+
+
+# Copy the build artifacts from the Docker Container
+echo "Copying the built contents from the docker container for image ${DOCKER_IMAGE_NAME}"
+docker  cp --quiet $CONTAINER_ID:/data/workspace/qt/. $TARGET_INSTALL_ROOT
+if [ $? -ne 0 ]
+then
+    echo "Error occurred copying build artifacts from Docker image ${DOCKER_IMAGE_NAME}:latest."
+    exit 1
+fi
+
+
+# Clean up the docker image and container
+echo "Cleaning up container"
+docker container rm $CONTAINER_ID || (echo "Warning: Unable to clean up container for image ${DOCKER_IMAGE_NAME}")
+
+echo "Cleaning up image"
+docker rmi --force $IMAGE_ID  || (echo "Warning: Unable to clean up image ${DOCKER_IMAGE_NAME}")
+
+popd
+
+exit 0

package-system/Qt/build_config.json

@@ -53,7 +53,7 @@
                     ["zlib-1.2.11-rev5-linux", "9be5ea85722fc27a8645a9c8a812669d107c68e6baa2ca0740872eaeb6a8b0fc", ""]
                 ],
                 "custom_build_cmd": [
-                    "./build-linux.sh"
+                    "./build-linux-x86.sh"
                 ],
                 "custom_install_cmd": [
                     "{python} copy_platform_cmakes.py"

package-system/Qt/docker_build_qt_linux.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+#
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+
+# TEMP_FOLDER and TARGET_INSTALL_ROOT get set from the pull_and_build_from_git.py script
+
+# Read the dependent 3P library paths from the arguments
+TIFF_PACKAGE_DIR=$1
+ZLIB_PACKAGE_DIR=$2
+
+set -euo pipefail
+
+MAKE_FLAGS=-j32
+
+echo "Building Qt5 from source with dependencies on"
+echo "    " $TIFF_PACKAGE_DIR
+echo "    " $ZLIB_PACKAGE_DIR
+
+
+# Base the Tiff of the dependent tiff O3DE package (static)
+TIFF_PREFIX=$TIFF_PACKAGE_DIR/tiff
+TIFF_INCDIR=$TIFF_PREFIX/include
+TIFF_LIBDIR=$TIFF_PREFIX/lib
+
+# We need to also bring in the zlib dependency since Tiff is a static lib dependency
+ZLIB_PREFIX=$ZLIB_PACKAGE_DIR/zlib
+ZLIB_INCDIR=$ZLIB_PREFIX/include
+ZLIB_LIBDIR=$ZLIB_PREFIX/lib
+
+BUILD_PATH=/data/workspace/build
+INSTALL_PATH=/data/workspace/qt
+
+[[ -d $BUILD_PATH ]] || mkdir $BUILD_PATH
+cd $BUILD_PATH
+
+echo Configuring Qt...
+../src/configure -prefix ${INSTALL_PATH} \
+                 -opensource \
+                 -nomake examples \
+                 -nomake tests \
+                 -confirm-license \
+                 -no-icu \
+                 -dbus \
+                 -no-separate-debug-info \
+                 -release \
+                 -force-debug-info \
+                 -qt-libpng \
+                 -qt-libjpeg \
+                 -no-feature-vnc \
+                 -no-feature-linuxfb \
+                 --tiff=system \
+                 -qt-zlib \
+                 -v \
+                 -no-cups \
+                 -no-glib \
+                 -no-feature-renameat2 \
+                 -no-feature-getentropy \
+                 -no-feature-statx \
+                 -no-egl \
+                 -qpa xcb \
+                 -xcb-xlib \
+                 -I $TIFF_INCDIR \
+                 -I $ZLIB_INCDIR \
+                 -L $TIFF_LIBDIR \
+                 -L $ZLIB_LIBDIR \
+                 -c++std c++1z \
+                 -openssl \
+                 -fontconfig
+if [ $? -ne 0 ]
+then
+    echo "Failed to configure Qt"
+    exit 1
+fi
+
+
+echo Qt configured, building modules...
+qtarray=(qtbase qtgraphicaleffects qtimageformats qtsvg qttools qtx11extras)
+
+for qtlib in "${qtarray[@]}"; do
+    echo Building $qtlib...
+    make module-$qtlib $MAKE_FLAGS
+
+    if [ $? -ne 0 ]
+    then
+        echo "Failed building Qt module $qtlib"
+        exit 1
+    fi
+
+    echo Built $qtlib.
+done
+
+echo Finished building modules, installing...
+for qtlib in "${qtarray[@]}"; do
+    echo Installing $qtlib...
+    make module-$qtlib-install_subtargets
+    
+    if [ ?$ -ne 0 ]
+    then
+        echo "Failed installing Qt module $qtlib"
+        exit 1
+    fi
+
+    echo $qtlib installed.
+done
+
+echo Qt installed successfully!