Browse Source

For CI - fix the CI builds on AppVeyor to use the build cache.

Yao Wei Tjong 姚伟忠 8 years ago
parent
commit
c017c2e607
3 changed files with 23 additions and 9 deletions
  1. 6 1
      .appveyor.yml
  2. 7 2
      .travis.yml
  3. 10 6
      Rakefile

+ 6 - 1
.appveyor.yml

@@ -26,8 +26,13 @@ platform:
   - x64
   - x64
 clone_depth: 50
 clone_depth: 50
 cache:
 cache:
+  # AppVeyor seems to restore caches based on the job ordering alone, it does not use the env-vars as the key like Travis
+  # Therefore, we are forced to list out all the cache items for our master CI and MinGW CI builds here
   - Build
   - Build
   - mingw-Build
   - mingw-Build
+  # Cache the MinGW compiler toolchains so that our CI builds do not chalk up their download stats
+  - C:\tools\mingw32 -> .appveyor.yml
+  - C:\tools\mingw64 -> .appveyor.yml
 environment:
 environment:
   GIT_NAME:
   GIT_NAME:
     secure: onlJNy/nGFT1GXNdusL4jokojSPZ732EFaXwdhPnpM0=
     secure: onlJNy/nGFT1GXNdusL4jokojSPZ732EFaXwdhPnpM0=
@@ -62,7 +67,7 @@ install:
         }
         }
   - if "%PLATFORM%" == "x64" set "URHO3D_64BIT=1"
   - if "%PLATFORM%" == "x64" set "URHO3D_64BIT=1"
 before_build:
 before_build:
-  - ps: $env:COMMIT_MESSAGE = $(git log --format=%B -n 1 $env:APPVEYOR_REPO_COMMIT)
+  - ps: if (!$env:APPVEYOR_PULL_REQUEST_NUMBER) { $env:COMMIT_MESSAGE = $(git log --format=%B -n 1 $env:APPVEYOR_REPO_COMMIT) }
   - rake ci_setup_cache
   - rake ci_setup_cache
 build_script:
 build_script:
   - rake ci && if "%PACKAGE_UPLOAD%" == "1" rake ci_package_upload && move %build_tree%\*.zip . && rd /S /Q %build_tree%\_CPack_Packages
   - rake ci && if "%PACKAGE_UPLOAD%" == "1" rake ci_package_upload && move %build_tree%\*.zip . && rd /S /Q %build_tree%\_CPack_Packages

+ 7 - 2
.travis.yml

@@ -104,7 +104,7 @@ matrix:
     # env: WIN32=1 URHO3D_LIB_TYPE=SHARED URHO3D_64BIT=0 URHO3D_D3D11=1
     # env: WIN32=1 URHO3D_LIB_TYPE=SHARED URHO3D_64BIT=0 URHO3D_D3D11=1
 before_script:
 before_script:
   - rake ci_timer
   - rake ci_timer
-  - export COMMIT_MESSAGE=$(git log --format=%B -n 1 $TRAVIS_COMMIT)
+  - if [[ "$TRAVIS_PULL_REQUEST" == "false" ]]; then export COMMIT_MESSAGE=$(git log --format=%B -n 1 $TRAVIS_COMMIT); fi
   - export TAG=$(git describe --exact-match $TRAVIS_COMMIT 2>/dev/null); if [[ $TAG =~ [[:digit:]]+\.[[:digit:]]+ ]]; then export RELEASE_TAG=$TAG; fi
   - export TAG=$(git describe --exact-match $TRAVIS_COMMIT 2>/dev/null); if [[ $TAG =~ [[:digit:]]+\.[[:digit:]]+ ]]; then export RELEASE_TAG=$TAG; fi
   # Travis-CI has a special case handling for release tag where it checks out directly from the tag branch instead of the default 'master' branch
   # Travis-CI has a special case handling for release tag where it checks out directly from the tag branch instead of the default 'master' branch
   - if [ $RELEASE_TAG ] || ([ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]); then if [ ${TRAVIS_JOB_NUMBER##*.} == 1 ]; then export SITE_UPDATE=1; fi; if [ "$CC" == "gcc" ] && ([ $RELEASE_TAG ] || echo $COMMIT_MESSAGE |grep -cq '\[ci package\]'); then export PACKAGE_UPLOAD=1; fi; fi
   - if [ $RELEASE_TAG ] || ([ "$TRAVIS_BRANCH" == "master" ] && [ "$TRAVIS_PULL_REQUEST" == "false" ]); then if [ ${TRAVIS_JOB_NUMBER##*.} == 1 ]; then export SITE_UPDATE=1; fi; if [ "$CC" == "gcc" ] && ([ $RELEASE_TAG ] || echo $COMMIT_MESSAGE |grep -cq '\[ci package\]'); then export PACKAGE_UPLOAD=1; fi; fi
@@ -133,8 +133,13 @@ platform:
   - x64
   - x64
 clone_depth: 50
 clone_depth: 50
 cache:
 cache:
+  # AppVeyor seems to restore caches based on the job ordering alone, it does not use the env-vars as the key like Travis
+  # Therefore, we are forced to list out all the cache items for our master CI and MinGW CI builds here
   - Build
   - Build
   - mingw-Build
   - mingw-Build
+  # Cache the MinGW compiler toolchains so that our CI builds do not chalk up their download stats
+  - C:\tools\mingw32 -> .appveyor.yml
+  - C:\tools\mingw64 -> .appveyor.yml
 environment:
 environment:
   GIT_NAME:
   GIT_NAME:
     secure: onlJNy/nGFT1GXNdusL4jokojSPZ732EFaXwdhPnpM0=
     secure: onlJNy/nGFT1GXNdusL4jokojSPZ732EFaXwdhPnpM0=
@@ -171,7 +176,7 @@ install:
           iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-desktop.ps1'))
           iex ((new-object net.webclient).DownloadString('https://raw.githubusercontent.com/appveyor/ci/master/scripts/enable-desktop.ps1'))
         }
         }
   - if "%PLATFORM%" == "x64" (set "URHO3D_64BIT=1" && set "MINGW_URL=%x86_64-posix-seh%" && set "ARCH=64") else (set "MINGW_URL=%i686-posix-dwarf%" && set "ARCH=32")
   - if "%PLATFORM%" == "x64" (set "URHO3D_64BIT=1" && set "MINGW_URL=%x86_64-posix-seh%" && set "ARCH=64") else (set "MINGW_URL=%i686-posix-dwarf%" && set "ARCH=32")
-  - curl -fsSL -o "mingw.7z" -O "%MINGW_URL%" && 7z x -y -o"C:\tools" "mingw.7z" >nul && echo "Installed MinGW compiler toolchain to C:\tools"
+  - if not exist "C:\tools\mingw%ARCH%" curl -fsSL -o "mingw.7z" -O "%MINGW_URL%" && 7z x -y -o"C:\tools" "mingw.7z" >nul && echo "Installed MinGW compiler toolchain to C:\tools"
   - set "PATH=C:\tools\mingw%ARCH%\bin;%PATH%"
   - set "PATH=C:\tools\mingw%ARCH%\bin;%PATH%"
 before_build:
 before_build:
   - set "APPVEYOR_REPO_COMMIT=%APPVEYOR_REPO_COMMIT%~"
   - set "APPVEYOR_REPO_COMMIT=%APPVEYOR_REPO_COMMIT%~"

+ 10 - 6
Rakefile

@@ -425,9 +425,11 @@ task :ci_setup_cache do
   # AppVeyor on Windows host has different kind of cache mechanism, not based on ccache
   # AppVeyor on Windows host has different kind of cache mechanism, not based on ccache
   if ENV['APPVEYOR']
   if ENV['APPVEYOR']
     system "bash -c 'rm -rf #{ENV['build_tree']}'" if clear
     system "bash -c 'rm -rf #{ENV['build_tree']}'" if clear
-    if File.exists?("#{ENV['build_tree']}/.commit")
-      last_commit = File.read "#{ENV['build_tree']}/.commit"
-      system "bash -c 'find CMakeLists.txt CMake Docs Source |xargs touch -r #{ENV['build_tree']}/CMakeCache.txt && touch $(git diff --name-only #{last_commit} #{ENV['APPVEYOR_REPO_COMMIT']})'"
+    if File.exists?("#{ENV['build_tree']}/.commits")
+      # Find the last valid commit SHA because the recorded commit SHAs may no longer be valid due to git reset/forced push
+      last_commit = File.read("#{ENV['build_tree']}/.commits").split.find { |sha| system "git cat-file -e #{sha}" }
+      # AppVeyor prefers CMD's FIND over MSYS's find, so we have to use fully qualified path to the MSYS's find
+      system "bash -c '/c/Program\\ Files/Git/usr/bin/find CMakeLists.txt CMake Docs Source |xargs touch -r #{ENV['build_tree']}/CMakeCache.txt && touch $(git diff --name-only #{last_commit} #{ENV['APPVEYOR_REPO_COMMIT']})'"
     end
     end
     next
     next
   # Use internal cache store instead of using Travis CI one (this is a workaround for using ccache on Travis CI legacy build infra)
   # Use internal cache store instead of using Travis CI one (this is a workaround for using ccache on Travis CI legacy build infra)
@@ -453,9 +455,11 @@ desc 'Teardown build cache'
 task :ci_teardown_cache do
 task :ci_teardown_cache do
   # AppVeyor on Windows host has different kind of cache mechanism, not based on ccache
   # AppVeyor on Windows host has different kind of cache mechanism, not based on ccache
   if ENV['APPVEYOR']
   if ENV['APPVEYOR']
-    File.write("#{ENV['build_tree']}/.commit", ENV['APPVEYOR_REPO_COMMIT']) if Dir.exist?(ENV['build_tree'])
-    # Temporarily exclude build artifacts from being cached due to cache size limitation
-    system "bash -c 'rm #{ENV['build_tree']}/bin/*.{exe,dll}'"
+    # Keep the last 10 commit SHAs
+    commits = (File.exists?("#{ENV['build_tree']}/.commits") ? File.read("#{ENV['build_tree']}/.commits").split : []).unshift(`git rev-parse #{ENV['APPVEYOR_REPO_COMMIT']}`.chomp).take 10
+    File.open("#{ENV['build_tree']}/.commits", 'w') { |file| file.puts commits } if Dir.exist?(ENV['build_tree'])
+    # Exclude build artifacts from being cached due to cache size limitation
+    system "bash -c 'rm -f #{ENV['build_tree']}/bin/*.{exe,dll}'"
     next
     next
   # Upload cache to internal cache store if it is our own
   # Upload cache to internal cache store if it is our own
   elsif ENV['USE_CCACHE'].to_i == 2
   elsif ENV['USE_CCACHE'].to_i == 2