Merge pull request #1006 from hamiltont/toolset-updates

Rewrite of travis change detection

.travis.yml

@@ -144,9 +144,6 @@ env:
     - TESTDIR=yesod
 
 before_install:
-  # Enables $TRAVIS_COMMIT_RANGE for pull requests
-  - time ./toolset/setup/travis-ci/add_commit_range.sh
-  
   # Configure Travis-CI build environment for TFB
   #   e.g. setup databases, users, etc
   - time ./toolset/run-ci.py cisetup $TESTDIR
@@ -165,17 +162,6 @@ script:
   # Pick one test in this directory and verify
   - time ./toolset/run-ci.py verify $TESTDIR
 
-notifications:
-  irc:
-    channels:
-      - "chat.freenode.net#techempower-fwbm"
-    template:
-      - "%{repository_slug} branch '%{branch}' %{result} (%{author})"
-      - "Build %{build_number} took %{duration}. See %{build_url}"
-    on_success: always
-    on_failure: always
-    skip_join: true
-
 cache:
   directories:
     - installs/mono-3.2.8

toolset/run-ci.py

@@ -34,19 +34,139 @@ class CIRunnner:
     testdir  = framework directory we are running
     '''
 
-    logging.basicConfig(level=logging.INFO)
     self.directory = testdir
     self.mode = mode
+    if mode == "cisetup":
+      logging.basicConfig(level=logging.DEBUG)
+    else:
+      logging.basicConfig(level=logging.INFO)
 
     try:
-      self.commit_range = os.environ['TRAVIS_COMMIT_RANGE']
-      if self.commit_range == "":
-          self.commit_range = "-1 %s" % os.environ['TRAVIS_COMMIT']
+      # NOTE: THIS IS VERY TRICKY TO GET RIGHT!
+      #
+      # Our goal: Look at the files changed and determine if we need to 
+      # run a verification for this folder. For a pull request, we want to 
+      # see the list of files changed by any commit in that PR. For a 
+      # push to master, we want to see a list of files changed by the pushed
+      # commits. If this list of files contains the current directory, or 
+      # contains the toolset/ directory, then we need to run a verification
+      # 
+      # If modifying, please consider: 
+      #  - the commit range for a pull request is the first PR commit to 
+      #    the github auto-merge commit
+      #  - the commits in the commit range may include merge commits
+      #    other than the auto-merge commit. An git log with -m 
+      #    will know that *all* the files in the merge were changed, 
+      #    but that is not the changeset that we care about
+      #  - git diff shows differences, but we care about git log, which
+      #    shows information on what was changed during commits
+      #  - master can (and will!) move during a build. This is one 
+      #    of the biggest problems with using git diff - master will 
+      #    be updated, and those updates will include changes to toolset, 
+      #    and suddenly every job in the build will start to run instead 
+      #    of fast-failing
+      #  - commit_range is not set if there was only one commit pushed, 
+      #    so be sure to test for that on both master and PR
+      #  - commit_range and commit are set very differently for pushes
+      #    to an owned branch versus pushes to a pull request, test
+      #  - For merge commits, the TRAVIS_COMMIT and TRAVIS_COMMIT_RANGE 
+      #    will become invalid if additional commits are pushed while a job is 
+      #    building. See https://github.com/travis-ci/travis-ci/issues/2666
+      #  - If you're really insane, consider that the last commit in a 
+      #    pull request could have been a merge commit. This means that 
+      #    the github auto-merge commit could have more than two parents
+      #  
+      #  - TEST ALL THESE OPTIONS: 
+      #      - On a branch you own (e.g. your fork's master)
+      #          - single commit
+      #          - multiple commits pushed at once
+      #          - commit+push, then commit+push again before the first
+      #            build has finished. Verify all jobs in the first build 
+      #            used the correct commit range
+      #          - multiple commits, including a merge commit. Verify that
+      #            the unrelated merge commit changes are not counted as 
+      #            changes the user made
+      #      - On a pull request
+      #          - repeat all above variations
+      #
+      #
+      # ==== CURRENT SOLUTION FOR PRs ====
+      #
+      # For pull requests, we will examine Github's automerge commit to see
+      # what files would be touched if we merged this into the current master. 
+      # You can't trust the travis variables here, as the automerge commit can
+      # be different for jobs on the same build. See https://github.com/travis-ci/travis-ci/issues/2666
+      # We instead use the FETCH_HEAD, which will always point to the SHA of 
+      # the lastest merge commit. However, if we only used FETCH_HEAD than any
+      # new commits to a pull request would instantly start affecting currently
+      # running jobs and the the list of changed files may become incorrect for
+      # those affected jobs. The solution is to walk backward from the FETCH_HEAD
+      # to the last commit in the pull request. Based on how github currently 
+      # does the automerge, this is the second parent of FETCH_HEAD, and 
+      # therefore we use FETCH_HEAD^2 below
+      #
+      # This may not work perfectly in situations where the user had advanced 
+      # merging happening in their PR. We correctly handle them merging in 
+      # from upstream, but if they do wild stuff then this will likely break
+      # on that. However, it will also likely break by seeing a change in 
+      # toolset and triggering a full run when a partial run would be 
+      # acceptable
+      #
+      # ==== CURRENT SOLUTION FOR OWNED BRANCHES (e.g. master) ====
+      #
+      # This one is fairly simple. Find the commit or commit range, and 
+      # examine the log of files changes. If you encounter any merges, 
+      # then fully explode the two parent commits that made the merge
+      # and look for the files changed there. This is an aggressive 
+      # strategy to ensure that commits to master are always tested 
+      # well
+      log.debug("TRAVIS_COMMIT_RANGE: %s", os.environ['TRAVIS_COMMIT_RANGE'])
+      log.debug("TRAVIS_COMMIT      : %s", os.environ['TRAVIS_COMMIT'])
+
+      is_PR = (os.environ['TRAVIS_PULL_REQUEST'] != "false")
+      if is_PR:
+        log.debug('I am testing a pull request')
+        first_commit = os.environ['TRAVIS_COMMIT_RANGE'].split('...')[0]
+        last_commit = subprocess.check_output("git rev-list -n 1 FETCH_HEAD^2", shell=True).rstrip('\n')
+        log.debug("Guessing that first commit in PR is : %s", first_commit)
+        log.debug("Guessing that final commit in PR is : %s", last_commit)
+
+        if first_commit == "":
+          # Travis-CI is not yet passing a commit range for pull requests
+          # so we must use the automerge's changed file list. This has the 
+          # negative effect that new pushes to the PR will immediately 
+          # start affecting any new jobs, regardless of the build they are on
+          log.debug("No first commit, using Github's automerge commit")
+          self.commit_range = "--first-parent -1 -m FETCH_HEAD"
+        elif first_commit == last_commit:
+          # There is only one commit in the pull request so far, 
+          # or Travis-CI is not yet passing the commit range properly 
+          # for pull requests. We examine just the one commit using -1
+          #
+          # On the oddball chance that it's a merge commit, we pray  
+          # it's a merge from upstream and also pass --first-parent 
+          log.debug("Only one commit in range, examining %s", last_commit)
+          self.commit_range = "-m --first-parent -1 %s" % last_commit
+        else: 
+          # In case they merged in upstream, we only care about the first 
+          # parent. For crazier merges, we hope
+          self.commit_range = "--first-parent %s...%s" % (first_commit, last_commit)
+
+      if not is_PR:
+        log.debug('I am not testing a pull request')
+        # If more than one commit was pushed, examine everything including 
+        # all details on all merges
+        self.commit_range = "-m %s" % os.environ['TRAVIS_COMMIT_RANGE']
+        
+        # If only one commit was pushed, examine that one. If it was a 
+        # merge be sure to show all details
+        if self.commit_range == "":
+          self.commit_range = "-m -1 %s" % os.environ['TRAVIS_COMMIT']
+
     except KeyError:
       log.warning("I should only be used for automated integration tests e.g. Travis-CI")
       log.warning("Were you looking for run-tests.py?")
-      last_commit = subprocess.check_output("git rev-parse HEAD^", shell=True).rstrip('\n')
-      self.commit_range = "%s...HEAD" % last_commit
+      self.commit_range = "-m HEAD^...HEAD"
 
     #
     # Find the one test from benchmark_config that we are going to run
@@ -100,10 +220,11 @@ class CIRunnner:
     def touch(fname):
       open(fname, 'a').close()
 
-    log.info("Using commit range %s", self.commit_range)
-    log.info("Running `git log --name-only -m --pretty=\"format:\" %s`" % self.commit_range)
-    changes = subprocess.check_output("git log --name-only -m --pretty=\"format:\" %s" % self.commit_range, shell=True)
-    log.info(changes)
+    log.debug("Using commit range `%s`", self.commit_range)
+    log.debug("Running `git log --name-only --pretty=\"format:\" %s`" % self.commit_range)
+    changes = subprocess.check_output("git log --name-only --pretty=\"format:\" %s" % self.commit_range, shell=True)
+    changes = os.linesep.join([s for s in changes.splitlines() if s]) # drop empty lines
+    log.debug("Result:\n%s", changes)
 
     # Look for changes to core TFB framework code
     if re.search(r'^toolset/', changes, re.M) is not None: 
@@ -125,7 +246,7 @@ class CIRunnner:
     ''' Do the requested command using TFB  '''
 
     if not self._should_run():
-      log.info("Not running directory %s", self.directory)
+      log.info("I found no changes to `%s` or `toolset/`, aborting verification", self.directory)
       return 0
 
     if self.mode == 'cisetup':
@@ -249,9 +370,11 @@ if __name__ == "__main__":
   except KeyError as ke: 
     log.warning("Environment key missing, are you running inside Travis-CI?")
     print traceback.format_exc()
+    retcode = 1
   except:
     log.critical("Unknown error")
     print traceback.format_exc()
+    retcode = 1
   finally:  # Ensure that logs are printed
     
     # Only print logs if we ran a verify

toolset/setup/linux/bash_functions.sh

@@ -14,10 +14,18 @@ fw_get () {
 fw_untar() {
   echo "Running 'tar xf $@'...please wait"
   tar xf "$@"
+  echo "Removing compressed tar file"
+  
+  # use -f to avoid printing errors if they gave additional arguments
+  rm -f "$@"
 }
 
 fw_unzip() {
-  unzip "$@"
+  echo "Running 'unzip $@'...please wait"
+  unzip -o -q "$@"
+  echo "Removing compressed zip file"
+  # use -f to avoid printing errors if they gave additional arguments
+  rm -f "$@"
 }
 
 # Was there an error for the current dependency?

toolset/setup/linux/frameworks/grails.sh

@@ -3,5 +3,5 @@
 RETCODE=$(fw_exists grails-2.4.2)
 [ ! "$RETCODE" == 0 ] || { return 0; }
 
-fw_get http://dist.springframework.org.s3.amazonaws.com/release/GRAILS/grails-2.4.2.zip
-unzip -o grails-2.4.2.zip
+fw_get http://dist.springframework.org.s3.amazonaws.com/release/GRAILS/grails-2.4.2.zip -O grails-2.4.2.zip
+fw_unzip grails-2.4.2.zip

toolset/setup/linux/frameworks/play2.sh

@@ -3,5 +3,5 @@
 RETCODE=$(fw_exists play-2.2.0)
 [ ! "$RETCODE" == 0 ] || { return 0; }
 
-fw_get http://downloads.typesafe.com/play/2.2.0/play-2.2.0.zip
-fw_unzip play-2.2.0.zip
+fw_get http://downloads.typesafe.com/play/2.2.0/play-2.2.0.zip -O play-2.2.0.zip
+fw_unzip play-2.2.0.zip

toolset/setup/linux/frameworks/treefrog.sh

@@ -5,9 +5,8 @@ RETCODE=$(fw_exists treefrog-1.7.7)
 
 sudo apt-get install -y qt4-qmake libqt4-dev libqt4-sql-mysql libqt4-sql-psql g++
 
-fw_get http://downloads.sourceforge.net/project/treefrog/src/treefrog-1.7.7.tar.gz
+fw_get http://downloads.sourceforge.net/project/treefrog/src/treefrog-1.7.7.tar.gz -O treefrog-1.7.7.tar.gz
 fw_untar treefrog-1.7.7.tar.gz
-rm -f treefrog-1.7.7.tar.gz
 cd treefrog-1.7.7
 ./configure
 
@@ -17,4 +16,4 @@ sudo make install
 
 cd ../tools
 make -j4
-sudo make install
+sudo make install

toolset/setup/linux/frameworks/wt.sh

@@ -10,7 +10,6 @@ sudo apt-get -y install libboost1.54-all-dev
 
 fw_get http://downloads.sourceforge.net/witty/wt-3.3.3.tar.gz -O wt.tar.gz
 fw_untar wt.tar.gz
-rm wt.tar.gz
 
 mv wt-* wt
 cd wt

toolset/setup/linux/languages/pypy.sh

@@ -4,7 +4,7 @@ RETCODE=$(fw_exists pypy)
 [ ! "$RETCODE" == 0 ] || { return 0; }
 
 fw_get https://bitbucket.org/pypy/pypy/downloads/pypy-2.3.1-linux64.tar.bz2 -O pypy-2.3.1-linux64.tar.bz2
-tar vxf pypy-2.3.1-linux64.tar.bz2
+fw_untar pypy-2.3.1-linux64.tar.bz2
 ln -sf pypy-2.3.1-linux64 pypy
 
 fw_get https://bootstrap.pypa.io/get-pip.py

toolset/setup/linux/languages/python2.sh

@@ -4,7 +4,7 @@ RETCODE=$(fw_exists py2)
 [ ! "$RETCODE" == 0 ] || { return 0; }
 
 fw_get http://www.python.org/ftp/python/2.7.7/Python-2.7.7.tgz
-tar vxf Python-2.7.7.tgz
+fw_untar Python-2.7.7.tgz
 pre=$(pwd)
 cd Python-2.7.7
 ./configure --prefix=${pre}/py2 --disable-shared

toolset/setup/linux/languages/python3.sh

@@ -4,7 +4,7 @@ RETCODE=$(fw_exists py3)
 [ ! "$RETCODE" == 0 ] || { return 0; }
 
 fw_get http://www.python.org/ftp/python/3.4.1/Python-3.4.1.tar.xz
-tar vxf Python-3.4.1.tar.xz
+fw_untar Python-3.4.1.tar.xz
 pre=$(pwd)
 cd Python-3.4.1
 ./configure --prefix=${pre}/py3 --disable-shared

toolset/setup/travis-ci/add_commit_range.sh

@@ -1,49 +0,0 @@
-#!/bin/bash
-#
-# Our travis setup relies heavily on rapidly cancelling 
-# jobs that are unneeded. For example, if only files
-# in aspnet/ were modified, then we don't need to run 
-# a job that verifies the framework tests in go/
-#
-# Detecting what files have been changed for a pull 
-# request is a a bit tricky, as Travis-CI currently 
-# does not provide a commit range for pull requests
-# (see travis-ci/travis-ci#1719). This script provides
-# a commit range by adding a branch prbase on the 
-# first commit in the pull request and a branch 
-# prhead on the commit that travis is currently 
-# building. 
-
-# This makes 
-#     git diff prbase:prhead 
-# equivalent to what you'd expect from 
-#     git diff $TRAVIS_COMMIT_RANGE
-
-if [ "${TRAVIS_PULL_REQUEST}" = "false" ]
-  then 
-  echo "This is not a pull request, nothing to do"
-  exit 0
-fi
-
-# Find the first commit in the pull request
-#   - download github PR patch file
-#   - grep commit SHA from first line
-#   - remove newline
-PR_FIRST=$(curl -s https://github.com/${TRAVIS_REPO_SLUG}/pull/${TRAVIS_PULL_REQUEST}.patch | head -1 | grep -o -E '\b[0-9a-f]{40}\b' | tr -d '\n')
-
-# Use the parent of the first commit so we can 
-# easily diff the changes introduced by this PR
-PR_PARENT=$(git rev-parse ${PR_FIRST}^)
-
-# Create new branch for first commit in pull request
-git branch -f prbase ${PR_PARENT}
-
-# Create new branch for last commit in pull request
-# Note: This is the automerge commit that github creates
-#       If you really want the last commit use the 
-#       parent of this commit. It's a merge, so you have to 
-#       decipher which parent commit you want
-git branch -f prhead ${TRAVIS_COMMIT}
-
-echo "Set prbase branch to commit ${PR_PARENT}"
-echo "Set prhead branch to commit ${TRAVIS_COMMIT}"