For Travis CI - fail the job when site update or package upload failed.

For topic and PR CI mirror branches, make another attempt to download the cache from the "base" CI mirror branch before creating a new git branch.

.travis.yml

@@ -112,13 +112,14 @@ before_script:
   - export PATH=$(whereis -b ccache |grep -o '\S*lib\S*'):$PATH
   # For some reason clang compiler toolchain installation in Travis CI VM does not have symlink in the ccache symlinks directory, so workaround it
   - if [ "$CC"  == "clang" ]; then ln -s $(which ccache) $HOME/clang && ln -s $(which ccache) $HOME/clang++ && export PATH=$HOME:$PATH; fi
-  - ccache -z -M 100M
-script: rake ci
-before_cache: ccache -s
-after_success:
+  - rake ci_setup_cache
+script:
+  - rake ci
   - if [ $SITE_UPDATE ]; then rake ci_site_update; fi
   - if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload; if [ $LINUX ] && [ ! "$URHO3D_64BIT" == "0" ]; then rake ci_package_upload URHO3D_USE_LIB64_RPM=1; fi; fi
-after_script: if [ ${TRAVIS_JOB_NUMBER##*.} == 1 ]; then rake ci_create_mirrors; fi
+after_script:
+  - rake ci_teardown_cache
+  - if [ ${TRAVIS_JOB_NUMBER##*.} == 1 ]; then rake ci_create_mirrors; fi
 notifications: {email: {on_success: never, on_failure: change}}
 
 ---
@@ -151,9 +152,9 @@ before_script:
   - export TRAVIS_COMMIT=$TRAVIS_COMMIT~
   - export COMMIT_MESSAGE=$(git log --format=%B -n 1 $TRAVIS_COMMIT)
   - export PATH=$(whereis -b ccache |grep -o '\S*lib\S*'):$PATH
-  - ccache -z -M 100M
+  - rake ci_setup_cache
 script: true
-before_cache: ccache -s
+after_script: rake ci_teardown_cache
 
 ---
 
@@ -220,10 +221,11 @@ before_script:
   - if [ "$ABI" == "arm64-v8a" ]; then export C_PATH=$(pwd)/android-ndk/toolchains/aarch64-linux-android-4.9/prebuilt/linux-x86_64/bin C_PREFIX=aarch64-linux-android; fi
   - if [ "$ABI" == "armeabi-v7a" ]; then export C_PATH=$(pwd)/android-ndk/toolchains/arm-linux-androideabi-4.9/prebuilt/linux-x86_64/bin C_PREFIX=arm-linux-androideabi; fi
   - export PATH=$(whereis -b ccache |grep -o '\S*lib\S*'):$C_PATH:$PATH; for f in $C_PATH/$C_PREFIX-{gcc,g++}; do touch -d "2015-01-01 00:00:00 +0800" $f; done
-  - if [ "$USE_CCACHE" == "1" ]; then ccache -z -M 100M; fi
-script: rake ci
-before_cache: ccache -s
-after_success: if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload; fi
+  - rake ci_setup_cache
+script:
+  - rake ci
+  - if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload; fi
+after_script: rake ci_teardown_cache
 
 ---
 
@@ -259,10 +261,11 @@ before_script:
   - export COMMIT_MESSAGE=$(git log --format=%B -n 1 $TRAVIS_COMMIT)
   - if [ $RELEASE_TAG ] || (! [[ $TRAVIS_BRANCH =~ [^-]+-[^-]+-CI ]] && echo $COMMIT_MESSAGE |grep -cq '\[ci package\]'); then export PACKAGE_UPLOAD=1; fi
   - export PATH=$(whereis -b ccache |grep -o '\S*lib\S*'):${RPI_PREFIX%/*}:$PATH
-  - ccache -z -M 100M
-script: rake ci
-before_cache: ccache -s
-after_success: if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload; fi
+  - rake ci_setup_cache
+script:
+  - rake ci
+  - if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload; fi
+after_script: rake ci_teardown_cache
 
 ---
 
@@ -291,6 +294,7 @@ env:
 matrix:
   fast_finish: true
 before_script:
+  - if [ $IOS ]; then export CI_START_TIME=$(date +%s); fi
   - export TRAVIS_COMMIT=$TRAVIS_COMMIT~
   - export TAG=$(git describe --exact-match $TRAVIS_COMMIT 2>/dev/null); if [[ $TAG =~ [[:digit:]]+\.[[:digit:]]+ ]]; then export RELEASE_TAG=$TAG; fi
   - export COMMIT_MESSAGE=$(git log --format=%B -n 1 $TRAVIS_COMMIT)
@@ -300,10 +304,11 @@ before_script:
   - if [ $PACKAGE_UPLOAD ]; then travis_retry brew install doxygen graphviz; fi
   - export PATH=$(brew info ccache |grep -o '\S*lib\S*'):$PATH
   - if [ $XCODE ]; then cp -p $(which ccache) $(dirname $(xcodebuild -find-executable clang)) && for compiler in clang clang++; do path=$(xcodebuild -find-executable $compiler); mv $path{,.orig} && sudo ln -sf $(dirname $path)/clang.orig /usr/bin/$compiler && ln -s ccache $path; done; fi
-  - if [ $IOS ]; then export CI_START_TIME=$(date +%s); fi
-script: rake ci
-after_success: if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload; fi
-after_script: ccache -s
+  - rake ci_setup_cache
+script:
+  - rake ci
+  - if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload; fi
+after_script: rake ci_teardown_cache
 
 ---
 
@@ -327,6 +332,7 @@ env:
 matrix:
   fast_finish: true
 before_script:
+  - export CI_START_TIME=$(date +%s)
   - export TRAVIS_COMMIT=$TRAVIS_COMMIT~
   - export TAG=$(git describe --exact-match $TRAVIS_COMMIT 2>/dev/null); if [[ $TAG =~ [[:digit:]]+\.[[:digit:]]+ ]]; then export RELEASE_TAG=$TAG; fi
   - export COMMIT_MESSAGE=$(git log --format=%B -n 1 $TRAVIS_COMMIT)
@@ -337,10 +343,11 @@ before_script:
   - if [ $PACKAGE_UPLOAD ]; then travis_retry brew install doxygen graphviz; fi
   - wget https://s3.amazonaws.com/mozilla-games/emscripten/releases/emsdk-portable.tar.gz && tar xfz emsdk-portable.tar.gz && rm emsdk-portable.tar.gz && cd emsdk_portable && ./emsdk update >/dev/null && sed -i.bak 's/-xvf/-xf/g' emsdk && ./emsdk install latest >/dev/null && ./emsdk activate latest >/dev/null && source ./emsdk_env.sh && export EMSCRIPTEN_ROOT_PATH=$EMSCRIPTEN && export EMSCRIPTEN=1
   - export PATH=$(brew info ccache |grep -o '\S*lib\S*'):$EMSCRIPTEN_ROOT_PATH:$PATH
-  - export CI_START_TIME=$(date +%s)
-script: rake ci
-after_success: if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload && rake ci_emscripten_samples_update; fi
-after_script: ccache -s
+  - rake ci_setup_cache
+script:
+  - rake ci
+  - if [ $PACKAGE_UPLOAD ]; then rake ci_package_upload && rake ci_emscripten_samples_update; fi
+after_script: rake ci_teardown_cache
 
 # Below samples are excluded from Emscripten CI build due to build time constraint
 data:

Rakefile

@@ -195,17 +195,6 @@ task :ci do
   if ENV['CI'] && ENV['PACKAGE_UPLOAD'] && !ENV['RELEASE_TAG']
     system 'git fetch --unshallow' or abort 'Failed to unshallow cloned repository'
   end
-  # Use internal cache store instead of using Travis CI one (this is a workaround for using ccache on Travis CI legacy build infra)
-  if ENV['USE_CCACHE'].to_i == 2
-    puts 'Setting up build cache'
-    job_number = ".#{ENV['TRAVIS_JOB_NUMBER'].split('.')[1]}"
-    repo_slug = "#{ENV['TRAVIS_REPO_SLUG'].split('/')[0]}/cache-store.git"
-    # Do not abort even when it fails here
-    system "if ! `git clone -q --depth 1 --branch #{ENV['TRAVIS_BRANCH']}#{job_number} https://github.com/#{repo_slug} ~/.ccache 2>/dev/null`; then git clone -q --depth 1 https://github.com/#{repo_slug} ~/.ccache 2>/dev/null && cd ~/.ccache && git checkout -qf -b #{ENV['TRAVIS_BRANCH']}#{job_number}; fi && ccache -z -M 100M"
-    puts "\n"
-  end
-  # Clear ccache on demand
-  system 'ccache -C' if /\[ccache clear\]/ =~ ENV['COMMIT_MESSAGE']
   # Packaging always use Release configuration
   if ENV['PACKAGE_UPLOAD']
     $configuration = 'Release'
@@ -230,12 +219,38 @@ task :ci do
     # GCC or Clang
     makefile_ci
   end
-  # Upload cache to internal cache store
+end
+
+# Usage: NOT intended to be used manually
+desc 'Setup build cache'
+task :ci_setup_cache do
+  # Use internal cache store instead of using Travis CI one (this is a workaround for using ccache on Travis CI legacy build infra)
+  if ENV['USE_CCACHE'].to_i == 2
+    puts 'Setting up build cache'
+    job_number = ".#{ENV['TRAVIS_JOB_NUMBER'].split('.').last}"
+    repo_slug = "#{ENV['TRAVIS_REPO_SLUG'].split('/').first}/cache-store.git"
+    matched = /.*-([^-]+-[^-]+)$/.match(ENV['TRAVIS_BRANCH'])
+    base_name = matched ? matched[1] : nil
+    # Do not abort even when it fails here
+    system "if ! `git clone -q --depth 1 --branch #{ENV['TRAVIS_BRANCH']}#{job_number} https://github.com/#{repo_slug} ~/.ccache 2>/dev/null`; then if ! `git clone -q --depth 1 --branch #{base_name}#{job_number} https://github.com/#{repo_slug} ~/.ccache 2>/dev/null`; then git clone -q --depth 1 https://github.com/#{repo_slug} ~/.ccache 2>/dev/null && cd ~/.ccache && git checkout -qf -b #{ENV['TRAVIS_BRANCH']}#{job_number}; fi; fi"
+  end
+  system 'ccache -z -M 100M'
+  # Clear ccache on demand
+  system 'ccache -C' if /\[ccache clear\]/ =~ ENV['COMMIT_MESSAGE']
+end
+
+# Usage: NOT intended to be used manually
+desc 'Teardown build cache'
+task :ci_teardown_cache do
+  # Upload cache to internal cache store if it is our own
   if ENV['USE_CCACHE'].to_i == 2
-    puts "\nStoring build cache"
+    puts 'Storing build cache'
+    job_number = ".#{ENV['TRAVIS_JOB_NUMBER'].split('.').last}"
+    repo_slug = "#{ENV['TRAVIS_REPO_SLUG'].split('/').first}/cache-store.git"
     # Do not abort even when it fails here
     system "cd ~/.ccache && git config user.name $GIT_NAME && git config user.email $GIT_EMAIL && git remote set-url --push origin https://[email protected]/#{repo_slug} && git add -A . && git commit --amend -qm 'Travis CI: cache update at #{Time.now.utc}.' && git push -qf -u origin #{ENV['TRAVIS_BRANCH']}#{job_number} >/dev/null 2>&1"
   end
+  system 'ccache -s'
 end
 
 # Usage: NOT intended to be used manually
@@ -260,19 +275,19 @@ task :ci_site_update do
   system "cd ../Build && make -j$NUMJOBS doc >/dev/null 2>&1 && ruby -i -pe 'gsub(/(<\\/?h)3([^>]*?>)/, %q{\\14\\2}); gsub(/(<\\/?h)2([^>]*?>)/, %q{\\13\\2}); gsub(/(<\\/?h)1([^>]*?>)/, %q{\\12\\2})' Docs/html/_*.html && rsync -a --delete Docs/html/ ../doc-Build/documentation/#{release}" or abort 'Failed to generate/rsync doxygen pages'
   # Supply GIT credentials and push site documentation to urho3d/urho3d.github.io.git
   system "cd ../doc-Build && pwd && git config user.name $GIT_NAME && git config user.email $GIT_EMAIL && git remote set-url --push origin https://[email protected]/urho3d/urho3d.github.io.git && git add -A . && ( git commit -qm \"Travis CI: site documentation update at #{Time.now.utc}.\n\nCommit: https://github.com/$TRAVIS_REPO_SLUG/commit/$TRAVIS_COMMIT\n\nMessage: $COMMIT_MESSAGE\" || true) && git push -q >/dev/null 2>&1" or abort 'Failed to update site'
-  # Automatically give instruction to do packaging when API has changed, unless the instruction is already given in this commit
-  if ENV['PACKAGE_UPLOAD']
-    instruction = 'skip'
-  else
-    instruction = 'package'
-  end
-  if !ENV['RELEASE_TAG']
+  unless ENV['RELEASE_TAG'] || `git fetch -qf origin master; git log -1 --pretty=format:'%H' FETCH_HEAD` != ENV['TRAVIS_COMMIT']
+    # Automatically give instruction to do packaging when API has changed, unless the instruction is already given in this commit
+    if ENV['PACKAGE_UPLOAD']
+      instruction = 'skip'
+    else
+      instruction = 'package'
+    end
     # Supply GIT credentials and push API documentation to urho3d/Urho3D.git (the push may not be successful if remote master has already diverged)
     system 'pwd && git config user.name $GIT_NAME && git config user.email $GIT_EMAIL && git remote set-url --push origin https://[email protected]/$TRAVIS_REPO_SLUG.git && git add Docs/*API*'
     if system("git commit -qm 'Travis CI: API documentation update at #{Time.now.utc}.\n[ci #{instruction}]'") && !ENV['PACKAGE_UPLOAD']
       bump_soversion 'Source/Urho3D/.soversion' or abort 'Failed to bump soversion'
       system "git add Source/Urho3D/.soversion && git commit --amend -qm 'Travis CI: API documentation update at #{Time.now.utc}.\n[ci #{instruction}]'" or abort 'Failed to stage .soversion file'
-      system 'git push origin HEAD:master -q >/dev/null 2>&1' or abort 'Failed to update API documentation, most likely due to remote master has diverged, the API documentation update will be performed again in the subsequent CI build'
+      system 'git push origin HEAD:master -q >/dev/null 2>&1' or abort 'Failed to update API documentation'
     end
   end
 end