|
@@ -0,0 +1,246 @@
|
|
|
+# This automation builds 3p packages based on a PR
|
|
|
+
|
|
|
+name: Build 3P Packages
|
|
|
+
|
|
|
+on:
|
|
|
+ pull_request:
|
|
|
+ branches:
|
|
|
+ - main
|
|
|
+ - development
|
|
|
+ paths:
|
|
|
+ - 'package_build_list_host_*.json'
|
|
|
+
|
|
|
+jobs:
|
|
|
+ detect-changes:
|
|
|
+ name: Detecting changes in PR to build
|
|
|
+ runs-on: ubuntu-latest
|
|
|
+ outputs:
|
|
|
+ matrix: ${{ steps.detect-platform.outputs.matrix }}
|
|
|
+
|
|
|
+ steps:
|
|
|
+ - name: Checkout 3P source repo
|
|
|
+ uses: actions/checkout@v4
|
|
|
+ with:
|
|
|
+ fetch-depth: 0
|
|
|
+
|
|
|
+ - name: Get package and platform from JSON changes
|
|
|
+ id: detect-platform
|
|
|
+ run: |
|
|
|
+ CHANGED_FILES=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --name-only)
|
|
|
+ # Construct the package and os into a json string to be consumed by Github Actions runners
|
|
|
+ JSON="{\"include\":["
|
|
|
+ for FILE in $CHANGED_FILES; do
|
|
|
+ if [[ $FILE == package_build_list_host_* ]]; then
|
|
|
+ PLATFORM=$(echo $FILE | sed -n 's/package_build_list_host_\(.*\).json/\1/p')
|
|
|
+ case $PLATFORM in
|
|
|
+ linux*)
|
|
|
+ OS_RUNNER="ubuntu-20.04"
|
|
|
+ ;;
|
|
|
+ windows)
|
|
|
+ OS_RUNNER="windows-latest" # This is bundled with VS2022
|
|
|
+ ;;
|
|
|
+ darwin)
|
|
|
+ OS_RUNNER="macos-latest"
|
|
|
+ ;;
|
|
|
+ *)
|
|
|
+ OS_RUNNER="windows-latest" # default
|
|
|
+ ;;
|
|
|
+ esac
|
|
|
+
|
|
|
+ DIFF=$(git diff ${{ github.event.pull_request.base.sha }}...${{ github.event.pull_request.head.sha }} --no-ext-diff --unified=0 \
|
|
|
+ --exit-code -a --no-prefix -- $FILE | egrep "^\+" | grep Scripts) # Get oly the changes that can be built
|
|
|
+
|
|
|
+ PACKAGE=$(echo $DIFF | cut -d'"' -f2)
|
|
|
+ PACKPATH=$(echo $DIFF | egrep -o "package-system/[^ ]*")
|
|
|
+ DOCKER=$(test -f "$PACKPATH/Dockerfile" && echo 1 || echo 0)
|
|
|
+ JSONline="{\"package\": \"$PACKAGE\", \"os\": \"$OS_RUNNER\", \"dockerfile\": \"$DOCKER\"},"
|
|
|
+ if [[ "$JSON" != *"$JSONline"* ]]; then
|
|
|
+ JSON="$JSON$JSONline"
|
|
|
+ fi
|
|
|
+ fi
|
|
|
+ done
|
|
|
+
|
|
|
+ # Remove last "," and add closing brackets
|
|
|
+ if [[ $JSON == *, ]]; then
|
|
|
+ JSON="${JSON%?}"
|
|
|
+ fi
|
|
|
+ JSON="$JSON]}"
|
|
|
+ echo $JSON
|
|
|
+
|
|
|
+ # Set output
|
|
|
+ echo "matrix=$( echo "$JSON" )" >> $GITHUB_OUTPUT
|
|
|
+
|
|
|
+ validate-changes:
|
|
|
+ name: Check changes for issues
|
|
|
+ needs: detect-changes
|
|
|
+ strategy:
|
|
|
+ fail-fast: false
|
|
|
+ matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
|
|
|
+ runs-on: ubuntu-latest
|
|
|
+
|
|
|
+ steps:
|
|
|
+ - name: Checkout 3P source repo
|
|
|
+ uses: actions/checkout@v4
|
|
|
+
|
|
|
+ - name: Check if package already exists in prod
|
|
|
+ env:
|
|
|
+ PROD_CDN: ${{ vars.PROD_CDN }} # Change this to compare on your own endpoint
|
|
|
+ run: |
|
|
|
+ url="${{ env.PROD_CDN }}/${{ matrix.package }}"
|
|
|
+ if curl --head --silent --fail ${url}.tar.xz > /dev/null 2>&1; then
|
|
|
+ echo ${{ matrix.package }} already exists in prod. Check the rev in the json file to ensure it is incremented
|
|
|
+ exit 1
|
|
|
+ else
|
|
|
+ echo ${{ matrix.package }} does not exist in CDN, continuing...
|
|
|
+ exit 0
|
|
|
+ fi
|
|
|
+
|
|
|
+ - name: Malware scan of repo
|
|
|
+ uses: dell/common-github-actions/malware-scanner@main
|
|
|
+ with:
|
|
|
+ directories: .
|
|
|
+ options: -r
|
|
|
+
|
|
|
+ build-on-specific-os:
|
|
|
+ name: Build on "${{ matrix.os }}" for "${{ matrix.package }}"
|
|
|
+ needs: [detect-changes, validate-changes]
|
|
|
+ strategy:
|
|
|
+ fail-fast: false
|
|
|
+ matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
|
|
|
+ runs-on: ${{ matrix.os }}
|
|
|
+
|
|
|
+ steps:
|
|
|
+ - name: Configure
|
|
|
+ id: configure
|
|
|
+ run: |
|
|
|
+ git config --global core.longpaths true
|
|
|
+ echo "uid_gid=$(id -u):$(id -g)" >> $GITHUB_OUTPUT
|
|
|
+
|
|
|
+ - name: Checkout 3P source repo
|
|
|
+ uses: actions/checkout@v4
|
|
|
+ with:
|
|
|
+ path: source
|
|
|
+ fetch-depth: 0
|
|
|
+
|
|
|
+ - name: Checkout 3P scripts repo
|
|
|
+ uses: actions/checkout@v4
|
|
|
+ with:
|
|
|
+ repository: o3de/3p-package-scripts
|
|
|
+ path: scripts
|
|
|
+
|
|
|
+ - name: Update python
|
|
|
+ uses: actions/setup-python@v4
|
|
|
+ with:
|
|
|
+ python-version: '3.10'
|
|
|
+ cache: 'pip'
|
|
|
+
|
|
|
+ - name: Install python dependancies
|
|
|
+ run: |
|
|
|
+ python3 -m pip install boto3 certifi
|
|
|
+
|
|
|
+ - name: Update cmake/ninja
|
|
|
+ uses: lukka/get-cmake@latest
|
|
|
+
|
|
|
+ - name: Update msbuild path
|
|
|
+ if: runner.os == 'Windows'
|
|
|
+ uses: ilammy/[email protected]
|
|
|
+
|
|
|
+ - name: Install clang/gcc
|
|
|
+ if: runner.os == 'Linux'
|
|
|
+ env:
|
|
|
+ CLANG_VER: 12
|
|
|
+ GCC_VER: 9
|
|
|
+ run: |
|
|
|
+ sudo apt-get install -y clang-${{ env.CLANG_VER }} gcc-${{ env.GCC_VER }} g++-${{ env.GCC_VER }}
|
|
|
+ sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-${{ env.CLANG_VER }} 10
|
|
|
+ sudo update-alternatives --install /usr/bin/clang++ clang++ /usr/bin/clang++-${{ env.CLANG_VER }} 10
|
|
|
+ sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-${{ env.GCC_VER }} 10
|
|
|
+ sudo update-alternatives --install /usr/bin/g++ g++ /usr/bin/g++-${{ env.GCC_VER }} 10
|
|
|
+
|
|
|
+ - name: Use sccache
|
|
|
+ uses: hendrikmuhs/[email protected]
|
|
|
+ with:
|
|
|
+ variant: sccache
|
|
|
+ max-size: 2048M
|
|
|
+ key: ${{ matrix.package }}-${{ matrix.os }}
|
|
|
+ restore-keys:
|
|
|
+ ${{ matrix.package }}-${{ matrix.os }}
|
|
|
+
|
|
|
+ - name: Set up QEMU (aarch64) # Only if the package folder contains a Dockerfile
|
|
|
+ if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile == '1') }}
|
|
|
+ run: |
|
|
|
+ sudo apt-get install -y qemu qemu-user-static
|
|
|
+
|
|
|
+ - name: Run build command
|
|
|
+ if: ${{ (!contains(matrix.package, 'aarch64')) || (matrix.dockerfile == '1') }}
|
|
|
+ env:
|
|
|
+ CMAKE_CXX_COMPILER_LAUNCHER: sccache
|
|
|
+ CMAKE_C_COMPILER_LAUNCHER: sccache
|
|
|
+ CMAKE_GENERATOR: Ninja # ccache/sccache cannot be used as the compiler launcher under cmake if the generator is MSBuild
|
|
|
+ run: |
|
|
|
+ python3 scripts/o3de_package_scripts/build_package.py --search_path source ${{ matrix.package }}
|
|
|
+
|
|
|
+ - name: Run build command (aarch64) # Generic build for packages without a Dockerfile
|
|
|
+ if: ${{ (contains(matrix.package, 'aarch64')) && (matrix.dockerfile != '1') }}
|
|
|
+ uses: uraimo/[email protected]
|
|
|
+ with:
|
|
|
+ arch: none
|
|
|
+ distro: none
|
|
|
+ base_image: ghcr.io/${{ github.repository }}/run-on-arch-${{ github.repository_owner }}-${{ github.event.repository.name }}-build-container-aarch64-ubuntu20-04:latest # built from build-container.yaml
|
|
|
+ setup: |
|
|
|
+ grep -q ${{ matrix.package }} ${PWD}/source/package_build_list_host_linux.json || rm ${PWD}/source/package_build_list_host_linux.json
|
|
|
+ dockerRunArgs: |
|
|
|
+ --platform=linux/arm64
|
|
|
+ --user ${{ steps.configure.outputs.uid_gid }}
|
|
|
+ --volume "${PWD}:/workspace"
|
|
|
+ --volume "${PWD}/scripts:/scripts"
|
|
|
+ --volume "${PWD}/source:/source"
|
|
|
+ env: |
|
|
|
+ CMAKE_CXX_COMPILER_LAUNCHER: sccache
|
|
|
+ CMAKE_C_COMPILER_LAUNCHER: sccache
|
|
|
+ SCCACHE_IDLE_TIMEOUT: 0
|
|
|
+ SCCACHE_DIR: /workspace/.sccache
|
|
|
+ SCCACHE_CACHE_SIZE: 2048M
|
|
|
+ shell: /bin/bash
|
|
|
+ run: |
|
|
|
+ lsb_release -a
|
|
|
+ uname -a
|
|
|
+ sccache --start-server
|
|
|
+ sccache -z
|
|
|
+ ls -lah /workspace
|
|
|
+ python3 /scripts/o3de_package_scripts/build_package.py --search_path /source/ ${{ matrix.package }}
|
|
|
+
|
|
|
+ - name: Upload packages
|
|
|
+ uses: actions/upload-artifact@v3
|
|
|
+ with:
|
|
|
+ name: ${{ matrix.package }}
|
|
|
+ path: source/packages/*
|
|
|
+
|
|
|
+ validate-packages:
|
|
|
+ name: Validating ${{ matrix.package }}
|
|
|
+ needs: [detect-changes, build-on-specific-os]
|
|
|
+ runs-on: 'ubuntu-latest'
|
|
|
+ strategy:
|
|
|
+ fail-fast: false
|
|
|
+ matrix: ${{fromJson(needs.detect-changes.outputs.matrix)}}
|
|
|
+ steps:
|
|
|
+ - name: Download packages
|
|
|
+ uses: actions/download-artifact@v3
|
|
|
+ with:
|
|
|
+ name: ${{ matrix.package }}
|
|
|
+
|
|
|
+ - name: Verify SHA256
|
|
|
+ run: |
|
|
|
+ echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS)"
|
|
|
+ echo "$(cat ${{ matrix.package }}.tar.xz.SHA256SUMS | cut -d" " -f1) ${{ matrix.package }}.tar.xz" | sha256sum --check
|
|
|
+
|
|
|
+ - name: Decompress package
|
|
|
+ if: ${{ !contains(matrix.package, 'aarch64') }}
|
|
|
+ run: |
|
|
|
+ tar -xvf ${{ matrix.package }}.tar.xz
|
|
|
+
|
|
|
+ - name: Malware scan
|
|
|
+ uses: dell/common-github-actions/malware-scanner@main
|
|
|
+ with:
|
|
|
+ directories: .
|
|
|
+ options: -r
|