Script updates for python-3.10.5 on Linux aarch64 (ARM64) (#160)

Signed-off-by: Steve Pham <[email protected]>

package-system/python/build_package_image.py

@@ -14,12 +14,16 @@ import os
 import platform
 
 folder_names = { #   subfolder     interpreter     build script 
-    'darwin'     : ('darwin_x64' , 'Python.framework/Versions/3.10/bin/python3', 'make-python.sh'),
-    'linux'      : ('linux_x64'  , 'python/bin/python', 'make-python.sh'),
-    'windows'    : ('win_x64'    , 'python/python.exe', 'build_python.bat')
+    'darwin'        : ('darwin_x64' , 'Python.framework/Versions/3.10/bin/python3', 'make-python.sh'),
+    'linux'         : ('linux_x64'  , 'python/bin/python', 'make-python.sh'),
+    'linux-aarch64' : ('linux_aarch64'  , 'python/bin/python', 'make-python.sh'),
+    'windows'       : ('win_x64'    , 'python/python.exe', 'build_python.bat')
 }
 
 platformsys = platform.system().lower()
+# For linux, we may support aarch64 architecture as well as the default x86_64
+if platformsys == 'linux' and platform.machine() == 'aarch64':
+    platformsys = 'linux-aarch64'
 
 # intentionally generate a keyerror if its not a good platform:
 subfolder_name, binary_relpath, build_script = folder_names[platformsys]

package-system/python/linux_aarch64/.gitignore

@@ -0,0 +1,2 @@
+temp/
+package/

package-system/python/linux_aarch64/FindPython.cmake

@@ -0,0 +1,11 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+# 
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# force this into config mode, so that it uses the config files instead of module files.
+set(Python_DIR ${CMAKE_CURRENT_LIST_DIR})
+find_package(Python 3.10.5 REQUIRED CONFIG)

package-system/python/linux_aarch64/PackageInfo.json

@@ -0,0 +1,6 @@
+{
+    "PackageName" : "python-3.10.5-rev2-linux-aarch64",
+    "License"     : "PSF-2.0",
+    "URL"         : "https://python.org",
+    "LicenseFile" : "python/LICENSE"
+}

package-system/python/linux_aarch64/make-python.sh

@@ -0,0 +1,254 @@
+#!/bin/bash
+
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+# 
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
+cd $SCRIPT_DIR
+
+echo ""
+echo "--------------- PYTHON PACKAGE BUILD SCRIPT ----------------"
+echo ""
+echo "BASIC REQUIREMENTS in case something goes wrong:"
+echo "   - git installed and in PATH"
+echo "   - packages installed: apt-get libtool autoconf tk8.6-dev python3 libssl-dev tcl8.6-dev libgdbm-compat-dev liblzma-dev libsqlite3-dev libreadline-dev texinfo"
+echo "   - python3 with pip in PATH! (i.e. sudo apt install python3 and sudo apt install python3-pip"
+echo "   - Note: This script is currently written for buildng on Ubuntu Linux only."
+echo "   - Note: installing binaries with pip must result with them being on PATH."
+echo ""
+
+# Make sure we have all the required dev packages
+REQUIRED_DEV_PACKAGES="tk8.6-dev python3 libssl-dev tcl8.6-dev libgdbm-compat-dev liblzma-dev libsqlite3-dev libreadline-dev texinfo"
+ALL_PACKAGES=`apt list 2>/dev/null`
+for req_package in $REQUIRED_DEV_PACKAGES
+do
+    PACKAGE_COUNT=`echo $ALL_PACKAGES | grep $req_package | wc -l`
+    if [[ $PACKAGE_COUNT -eq 0 ]]; then
+        echo Missing required package $req_package
+        exit 1
+    fi
+done
+
+
+if [[ ${PACKAGE_CLEAR_TEMP_FOLDERS} -gt 0 ]]; then
+    echo "   - PACKAGE_CLEAR_TEMP_FOLDERS env var is set > 0, will clear temp folders."
+else
+    echo "   - PACKAGE_CLEAR_TEMP_FOLDERS env var not set or = 0, will not clear temp."
+fi
+echo "   ... this will take about one and a half hours ..."
+echo ""
+
+mkdir -p temp
+
+
+echo ""
+echo "--------------- Cloning python 3.10.5 from git ---------------"
+echo ""
+cd temp
+git clone https://github.com/python/cpython.git --branch v3.10.5 --depth 1
+
+if [[ ! -d "cpython" ]]; then
+    echo "Was unable to create cpython dir via git clone.  Is git installed?"
+    exit 1
+fi
+
+echo ""
+echo "--------------- Cloning libexpat 2.4.6 from git and applying update ---------------"
+echo ""
+git clone https://github.com/libexpat/libexpat.git --branch "R_2_4_6" --depth 1
+
+if [[ ! -d "libexpat" ]]; then
+    echo "Was unable to create libexpat dir via git clone.  Is git installed?"
+    exit 1
+fi
+
+cp -f -v libexpat/expat/lib/*.h cpython/Modules/expat/
+cp -f -v libexpat/expat/lib/*.c cpython/Modules/expat/
+
+
+echo ""
+echo "--------------- Cloning libffi 3.4.2 and building static version ---------------"
+echo ""
+git clone https://github.com/libffi/libffi.git --branch "v3.4.2" --depth 1
+if [[ ! -d "libffi" ]]; then
+    echo "Was unable to create libffi dir via git clone."
+    exit 1
+fi
+
+pushd libffi
+
+# According to the README.md for libffi, we need to run autogen.sh first
+./autogen.sh
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error running autogen.sh for libffi"
+    exit $retVal
+fi
+ 
+./configure --prefix=$SCRIPT_DIR/temp/ffi_lib --enable-shared=no CFLAGS='-fPIC' CPPFLAGS='-fPIC'
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error running configuring for libffi"
+    exit $retVal
+fi
+
+make install
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error building libffi"
+    exit $retVal
+fi
+
+popd
+
+
+echo ""
+echo "--------------- Cloning openssl 1.1.1q and building it externally ---------------"
+echo ""
+git clone https://github.com/openssl/openssl.git --branch "OpenSSL_1_1_1q" --depth 1
+if [[ ! -d "openssl" ]]; then
+    echo "Was unable to create openssl dir via git clone."
+    exit 1
+fi
+
+pushd openssl
+
+echo ./config --prefix=$SCRIPT_DIR/temp/openssl-local/build --openssldir=/etc/ssl LDFLAGS='-Wl,-rpath=\$$ORIGIN'
+./config --prefix=$SCRIPT_DIR/temp/openssl-local/build --openssldir=/etc/ssl LDFLAGS='-Wl,-rpath=\$$ORIGIN'
+
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error building openssl"
+    exit $retVal
+fi
+
+echo make
+make
+if [ $retVal -ne 0 ]; then
+    echo "Error building openssl (build failure)"
+    exit $retVal
+fi
+
+echo make test
+if [ $retVal -ne 0 ]; then
+    echo "Error building openssl (test failure)"
+    exit $retVal
+fi
+
+echo make install
+make install
+if [ $retVal -ne 0 ]; then
+    echo "Error building openssl (install failure)"
+    exit $retVal
+fi
+
+popd
+
+
+cd cpython
+
+echo ""
+echo "--------------- Building cpython from source ---------------"
+echo ""
+
+# Build from the source with optimizations and shared libs enabled , and override the RPATH and bzip include/lib paths
+./configure --prefix=$SCRIPT_DIR/package/python --enable-optimizations --with-openssl=$SCRIPT_DIR/temp/openssl-local/build --enable-shared LDFLAGS='-Wl,-rpath=\$$ORIGIN:\$$ORIGIN/../lib:\$$ORIGIN/../.. -L../ffi_lib/lib' CPPFLAGS='-I../ffi_lib/include' CFLAGS='-I../ffi_lib/include' 
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error running configuring optimized build"
+    exit $retVal
+fi
+
+make
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error compiling optimized build"
+    exit $retVal
+fi
+
+# Prepare the package folder
+cd $SCRIPT_DIR
+
+# Install the newly built python 3.10.5 to the package/python folder
+cd $SCRIPT_DIR
+cd temp
+cd cpython
+
+make install
+retVal=$?
+if [ $retVal -ne 0 ]; then
+    echo "Error installing python to the package folder"
+    exit $retVal
+fi
+
+
+cd $SCRIPT_DIR
+mkdir -p package
+cd package
+
+cp $SCRIPT_DIR/temp/cpython/LICENSE ./python/LICENSE
+cp $SCRIPT_DIR/PackageInfo.json .
+cp $SCRIPT_DIR/*.cmake .
+
+cd $SCRIPT_DIR/package/python/bin
+ln -s python3 python
+cd $SCRIPT_DIR/package
+
+
+# Move the openssl libraries to the local cpython build for portability
+pushd $SCRIPT_DIR/package/python/lib
+cp $SCRIPT_DIR/temp/openssl-local/build/lib/libssl.so.1.1 .
+ln -s libssl.so.1.1 libssl.so.1
+cp $SCRIPT_DIR/temp/openssl-local/build/lib/libcrypto.so.1.1 .
+ln -s libcrypto.so.1.1 libcrypto.so.1
+popd
+
+# Copy the openssl license
+cp $SCRIPT_DIR/temp/openssl/LICENSE $SCRIPT_DIR/package/python/LICENSE.OPENSSL
+
+
+echo ""
+echo "--------------- Upgrading pip ---------------"
+echo ""
+# the pip that may come from the above repo can be broken, so we'll use get-pip
+# and then upgrade it.
+curl https://bootstrap.pypa.io/get-pip.py -o get-pip.py
+./python/bin/python3 get-pip.py
+rm get-pip.py
+PYTHONNOUSERSITE=1 ./python/bin/python3 -m pip install --upgrade pip
+
+
+# installing pip causes it to put absolute paths to python
+# in the pip files (in bin).  For example, pip will have 
+# a line at the top that starts with #!/full/path/to/python 
+# so we fix those up too. 
+# We want to change it from and absolute path to python
+# to a multi-line #! that runs python from the same folder as the file is being called from: 
+#!/bin/sh 
+#"exec" "`dirname $0`/python" "$0" "$@"
+sed -i "1s+.*+\#\!/bin/sh+" ./python/bin/pip* 
+sed -i "2i\\
+\"exec\" \"\`dirname \$0\`/python\" \"\$0\" \"\$\@\" " ./python/bin/pip*
+
+echo ""
+echo "--------------- PYTHON WAS BUILT FROM SOURCE ---------------"
+echo ""
+
+echo "Package has completed building, and is now in $SCRIPT_DIR/package"
+
+if [[ ${PACKAGE_CLEAR_TEMP_FOLDERS} -gt 0 ]]
+    then
+        echo "Deleting temp folders because PACKAGE_CLEAR_TEMP_FOLDERS is set to > 0"
+        rm -rf $SCRIPT_DIR/temp
+    else
+        echo "PACKAGE_CLEAR_TEMP_FOLDERS is unset or zero, temp folder retained."
+        echo "Running this script again without deleting temp will just update the package without"
+        echo "The two hour wait time to build everything from scratch..."
+
+fi
+exit 0

package-system/python/linux_aarch64/open3d_bzip2.patch

@@ -0,0 +1,13 @@
+diff --git a/Makefile b/Makefile
+index f8a1772..56de40c 100644
+--- a/Makefile
++++ b/Makefile
+@@ -21,7 +21,7 @@ RANLIB=ranlib
+ LDFLAGS=
+ 
+ BIGFILES=-D_FILE_OFFSET_BITS=64
+-CFLAGS=-Wall -Winline -O2 -g $(BIGFILES)
++CFLAGS=-Wall -Winline -fPIC -O2 -g $(BIGFILES)
+ 
+ # Where you want it installed when you do 'make install'
+ PREFIX=/usr/local

package-system/python/linux_aarch64/python-config-version.cmake

@@ -0,0 +1,40 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+# 
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# this file is called to make sure that if we request a specific version
+# we respond only to that version
+
+set(PACKAGE_VERSION 3.10.5)
+set(PACKAGE_VERSION_EXACT False)
+set(PACKAGE_VERSION_COMPATIBLE False)
+
+if (NOT ${PACKAGE_FIND_NAME} STREQUAL "Python")
+    return()
+endif()
+
+if (PACKAGE_FIND_VERSION_COUNT GREATER 0 AND NOT PACKAGE_FIND_VERSION_MAJOR EQUAL 3)
+    return()
+endif()
+
+if (PACKAGE_FIND_VERSION_COUNT GREATER 1 AND NOT PACKAGE_FIND_VERSION_MINOR EQUAL 10)
+    return()
+endif()
+
+if (PACKAGE_FIND_VERSION_COUNT GREATER 2 AND NOT PACKAGE_FIND_VERSION_PATCH EQUAL 5)
+    return()
+endif()
+
+if (PACKAGE_FIND_VERSION_COUNT GREATER 3)
+    return()
+endif()
+
+if (PACKAGE_FIND_VERSION VERSION_EQUAL PACKAGE_VERSION)
+    set(PACKAGE_VERSION_EXACT TRUE)
+endif()
+
+set(PACKAGE_VERSION_COMPATIBLE TRUE)

package-system/python/linux_aarch64/python-config.cmake

@@ -0,0 +1,73 @@
+#
+# Copyright (c) Contributors to the Open 3D Engine Project.
+# For complete copyright and license terms please see the LICENSE at the root of this distribution.
+# 
+# SPDX-License-Identifier: Apache-2.0 OR MIT
+#
+#
+
+# this file actually ingests the library and defines targets.
+set(MY "Python")
+set(TARGET_WITH_NAMESPACE "3rdParty::${MY}")
+if (TARGET ${TARGET_WITH_NAMESPACE})
+    return()
+endif()
+
+# this FindPython file is designed to be compatible with the base FindPython
+# to at least some degree.  As such, it uses the same variable names:
+
+# this script defines:
+# Python_EXECUTABLE - full path to executable
+# Python_INTERPRETER_ID - "Python"
+# Python_HOME - Where the python folder root is (ie, folder has subfolder of 'Lib')
+# Python_PATHS - Where sys.path should point at to find modules, libraries, etc.
+# Python_Development_FOUND - The platform we are cross compiling for can link to python
+# and a target called 3rdParty::Python that you can use to depend on
+
+set(${MY}_VERSION 3.10.5)
+set(${MY}_INTERPRETER_ID    "Python")
+set(${MY}_EXECUTABLE        ${CMAKE_CURRENT_LIST_DIR}/python/bin/python)
+set(${MY}_HOME              ${CMAKE_CURRENT_LIST_DIR}/python)
+set(${MY}_PATHS             ${CMAKE_CURRENT_LIST_DIR}/python/lib
+                            ${CMAKE_CURRENT_LIST_DIR}/python/lib/python3.10
+                            ${CMAKE_CURRENT_LIST_DIR}/python/lib/python3.10/lib-dynload
+                            ${CMAKE_CURRENT_LIST_DIR}/python/lib/python3.10/site-packages) 
+
+# only if we're compiling FOR on one of the available platforms, add the target and libraries:
+if (${PAL_PLATFORM_NAME} STREQUAL "Linux" )
+    set(${MY}_Development_FOUND TRUE)
+    # Do not use these  PYTHON_LIBRARY_* or other variables, instead, use the 
+    # target '3rdParty::Python'
+    # note:  we built the shared version of python, you must use the .so
+    # in order to load it.  If you don't, then python itself will load a module
+    # from a .so and that module will try to load the dylib for python itself
+    # and bad things will occur since there'll essentially be both the static 
+    # and dy version of python in one address space.
+    # also, because the regular .so is a symlink, we actually skip that and link
+    # to the real one, so that there's no conflict between what library we copy
+    # and what library we link to.
+    set(${MY}_LIBRARY_DEBUG   ${CMAKE_CURRENT_LIST_DIR}/python/lib/libpython3.10.so.1.0)
+    set(${MY}_LIBRARY_RELEASE ${CMAKE_CURRENT_LIST_DIR}/python/lib/libpython3.10.so.1.0)
+    set(${MY}_INCLUDE_DIR     ${CMAKE_CURRENT_LIST_DIR}/python/include/python3.10)
+    # DYLIBS causes things to be copied to the bin folder to satisfy RPATH origin
+    set(${MY}_DYLIBS_DEBUG    ${CMAKE_CURRENT_LIST_DIR}/python/lib/libpython3.10.so.1.0)
+    set(${MY}_DYLIBS_RELEASE  ${CMAKE_CURRENT_LIST_DIR}/python/lib/libpython3.10.so.1.0)
+
+    set(${MY}_COMPILE_DEFINITIONS DEFAULT_LY_PYTHONHOME="${CMAKE_CURRENT_LIST_DIR}/python")
+
+    # the rest of this file could be reused for almost any target:
+    # we set it to a generator expression for multi-config situations:
+    set(${MY}_LIBRARY "$<IF:$<CONFIG:Debug>,${${MY}_LIBRARY_DEBUG},${${MY}_LIBRARY_RELEASE}>")
+    set(${MY}_DYLIBS "$<IF:$<CONFIG:Debug>,${${MY}_DYLIBS_DEBUG},${${MY}_DYLIBS_RELEASE}>")
+
+    # now set up the target using the above declarations....
+    add_library(${TARGET_WITH_NAMESPACE} INTERFACE IMPORTED GLOBAL)
+    ly_target_include_system_directories(TARGET ${TARGET_WITH_NAMESPACE} INTERFACE ${${MY}_INCLUDE_DIR})
+    target_link_libraries(${TARGET_WITH_NAMESPACE} INTERFACE "${${MY}_LIBRARY}")
+    target_compile_definitions(${TARGET_WITH_NAMESPACE} INTERFACE "${${MY}_COMPILE_DEFINITIONS}")
+
+    # we also need to add the python dlls to be copied to the binaries folder...
+    set_target_properties(${TARGET_WITH_NAMESPACE} PROPERTIES INTERFACE_IMPORTED_LOCATION "${${MY}_DYLIBS}")
+endif()
+
+set(${MY}_FOUND True)

package-system/python/linux_x64/make-python.sh

@@ -235,14 +235,6 @@ sed -i "1s+.*+\#\!/bin/sh+" ./python/bin/pip*
 sed -i "2i\\
 \"exec\" \"\`dirname \$0\`/python\" \"\$0\" \"\$\@\" " ./python/bin/pip*
 
-# https://github.com/o3de/o3de/issues/7281 Reports NVD vulnerability in the wininst-*.exe files that
-# get included in the package. Since this is a linux only package, we can remove them 
-echo "Removing wininst*.exe files"
-rm -v $SCRIPT_DIR/package/python/lib/python3.7/distutils/command/wininst-*.exe
-
-echo "Removing out of date pip*.whl"
-rm -v $SCRIPT_DIR/package/python/lib/python3.7/ensurepip/_bundled/pip-*.whl
-
 echo ""
 echo "--------------- PYTHON WAS BUILT FROM SOURCE ---------------"
 echo ""

package_build_list_host_linux-aarch64.json

@@ -24,6 +24,7 @@
         "PhysX-4.1.2.29882248-rev5-linux-aarch64": "package-system/PhysX/build_package_image.py --platform-name linux-aarch64",
         "png-1.6.37-rev2-linux-aarch64": "Scripts/extras/pull_and_build_from_git.py ../../package-system/libpng --platform-name Linux-aarch64 --clean",
 	"poly2tri-7f0487a-rev1-linux-aarch64": "package-system/poly2tri/build_package_image.py --platform-name linux-aarch64",
+        "python-3.10.5-rev2-linux-aarch64": "package-system/python/build_package_image.py",
         "tiff-4.2.0.15-rev3-linux-aarch64": "Scripts/extras/pull_and_build_from_git.py ../../package-system/tiff --platform-name Linux-aarch64 --clean",
         "zlib-1.2.11-rev5-linux-aarch64": "Scripts/extras/pull_and_build_from_git.py ../../package-system/zlib --platform-name Linux-aarch64 --clean"
     },
@@ -48,6 +49,7 @@
         "PhysX-4.1.2.29882248-rev5-linux-aarch64": "package-system/PhysX-linux-aarch64",
         "png-1.6.37-rev2-linux-aarch64":  "package-system/libpng/temp/png-linux-aarch64",
         "poly2tri-7f0487a-rev1-linux-aarch64": "package-system/poly2tri-linux-aarch64",
+        "python-3.10.5-rev2-linux-aarch64": "package-system/python/linux_aarch64/package",
 	"tiff-4.2.0.15-rev3-linux-aarch64": "package-system/tiff/temp/tiff-linux-aarch64",
         "zlib-1.2.11-rev5-linux-aarch64": "package-system/zlib/temp/zlib-linux-aarch64"
     }