Travis Smart Diffing & tfb --test-dir argument (#2495)

* trying some stuff

* file permissions

* all the travis work

.travis.yml

@@ -188,23 +188,29 @@ env:
     - "TESTDIR=Ur/urweb"
 
 before_script:
+
+  # Runs travis_diff, printing the output to the terminal, and searches for travis-diff-continue
+  # to determine if the suite should be installed and the current $TESTDIR test should run.
+  - export CONTINUE_TEST=`./toolset/travis/travis_diff.py | tee /dev/tty | grep -q "travis-diff-continue" && echo 1`
   # travis_clean.sh takes care of some services that are baked into the travis
   # build. Using language: generic gets us an ubuntu build with fewer services,
   # but includes database installs, ruby and rvm installs, and others that interfere
   # with running the suite in a clean ubuntu install.
-  - source ./toolset/travis/travis_clean.sh
+  - if [ $CONTINUE_TEST ]; then source ./toolset/travis/travis_clean.sh; else echo 'Skipping travis_clean.sh'; fi
 
   # travis_setup.sh runs all the same commands you would run if you were setting up
   # a development environment via:
   # http://frameworkbenchmarks.readthedocs.io/en/latest/Development/Installation-Guide/
-  - source ./toolset/travis/travis_setup.sh
+  - if [ $CONTINUE_TEST ]; then source ./toolset/travis/travis_setup.sh; else echo 'Skipping travis_setup.sh'; fi
 
 script:
+
   # run-ci.py runs the diffing to see if travis needs to test this framework. Ideally/eventually,
   # we'd like to try and do the diffing before travis_clean & setup.
   # This will run the tests exactly as you would in your own vm:
   # ./toolset/run-tests.py --mode verify --test (all the valid tests for this framework)
-  - ./toolset/run-ci.py verify "$TESTDIR"
+  - if [ $CONTINUE_TEST ]; then tfb --mode verify --test-dir "$TESTDIR"; else echo 'Skipping test verification.'; fi
+  # - tfb --mode verify --test
 
 cache:
   directories:

toolset/benchmark/utils.py

@@ -60,10 +60,18 @@ def gather_tests(include = [], exclude=[], benchmarker=None):
     benchmarker = Benchmarker(defaults)
 
   
-  # Search in both old and new directories
-  fwroot = setup_util.get_fwroot() 
-  config_files = glob.glob("%s/*/benchmark_config.json" % fwroot) 
-  config_files.extend(glob.glob("%s/frameworks/*/*/benchmark_config.json" % fwroot))
+  # Search for configuration files
+  fwroot = setup_util.get_fwroot()
+  config_files = []
+  if benchmarker.test_dir:
+    for test_dir in benchmarker.test_dir:
+        dir_config_files = glob.glob("{!s}/frameworks/{!s}/benchmark_config.json".format(fwroot, test_dir))
+        if len(dir_config_files):
+            config_files.extend(dir_config_files)
+        else:
+            raise Exception("Unable to locate tests in test-dir: {!s}".format(test_dir))
+  else:
+    config_files.extend(glob.glob("{!s}/frameworks/*/*/benchmark_config.json".format(fwroot)))
   
   tests = []
   for config_file_name in config_files:

toolset/run-ci.py

@@ -1,359 +0,0 @@
-#!/usr/bin/env python
-
-import subprocess
-import os
-import sys
-import glob
-import json
-import traceback
-import re
-import logging
-log = logging.getLogger('run-ci')
-import time
-import threading
-from benchmark import framework_test
-from benchmark.utils import gather_tests
-from benchmark.utils import header
-
-# Needed for various imports
-sys.path.append('.')
-sys.path.append('toolset/setup/linux')
-sys.path.append('toolset/benchmark')
-
-from setup.linux import setup_util
-
-class CIRunnner:
-  '''
-  Manages running TFB on the Travis Continuous Integration system. 
-  Makes a best effort to avoid wasting time and resources by running 
-  useless jobs. 
-  
-  Only verifies the first test in each directory 
-  '''
-
-  SUPPORTED_DATABASES = "mysql postgres mongodb cassandra elasticsearch sqlite redis none".split()
-  
-  def __init__(self, mode, testdir=None):
-    '''
-    mode = [verify] for what we want to do
-    testdir  = framework directory we are running
-    '''
-
-    self.directory = testdir
-    self.mode = mode
-    logging.basicConfig(level=logging.INFO)
-
-    try:
-      # NOTE: THIS IS VERY TRICKY TO GET RIGHT!
-      #
-      # Our goal: Look at the files changed and determine if we need to 
-      # run a verification for this folder. For a pull request, we want to 
-      # see the list of files changed by any commit in that PR. For a 
-      # push to master, we want to see a list of files changed by the pushed
-      # commits. If this list of files contains the current directory, or 
-      # contains the toolset/ directory, then we need to run a verification
-      # 
-      # If modifying, please consider: 
-      #  - the commit range for a pull request is the first PR commit to 
-      #    the github auto-merge commit
-      #  - the commits in the commit range may include merge commits
-      #    other than the auto-merge commit. An git log with -m 
-      #    will know that *all* the files in the merge were changed, 
-      #    but that is not the changeset that we care about
-      #  - git diff shows differences, but we care about git log, which
-      #    shows information on what was changed during commits
-      #  - master can (and will!) move during a build. This is one 
-      #    of the biggest problems with using git diff - master will 
-      #    be updated, and those updates will include changes to toolset, 
-      #    and suddenly every job in the build will start to run instead 
-      #    of fast-failing
-      #  - commit_range is not set if there was only one commit pushed, 
-      #    so be sure to test for that on both master and PR
-      #  - commit_range and commit are set very differently for pushes
-      #    to an owned branch versus pushes to a pull request, test
-      #  - For merge commits, the TRAVIS_COMMIT and TRAVIS_COMMIT_RANGE 
-      #    will become invalid if additional commits are pushed while a job is 
-      #    building. See https://github.com/travis-ci/travis-ci/issues/2666
-      #  - If you're really insane, consider that the last commit in a 
-      #    pull request could have been a merge commit. This means that 
-      #    the github auto-merge commit could have more than two parents
-      #  - Travis cannot really support rebasing onto an owned branch, the
-      #    commit_range they provide will include commits that are non-existant
-      #    in the repo cloned on the workers. See https://github.com/travis-ci/travis-ci/issues/2668
-      #  
-      #  - TEST ALL THESE OPTIONS: 
-      #      - On a branch you own (e.g. your fork's master)
-      #          - single commit
-      #          - multiple commits pushed at once
-      #          - commit+push, then commit+push again before the first
-      #            build has finished. Verify all jobs in the first build 
-      #            used the correct commit range
-      #          - multiple commits, including a merge commit. Verify that
-      #            the unrelated merge commit changes are not counted as 
-      #            changes the user made
-      #      - On a pull request
-      #          - repeat all above variations
-      #
-      #
-      # ==== CURRENT SOLUTION FOR PRs ====
-      #
-      # For pull requests, we will examine Github's automerge commit to see
-      # what files would be touched if we merged this into the current master. 
-      # You can't trust the travis variables here, as the automerge commit can
-      # be different for jobs on the same build. See https://github.com/travis-ci/travis-ci/issues/2666
-      # We instead use the FETCH_HEAD, which will always point to the SHA of 
-      # the lastest merge commit. However, if we only used FETCH_HEAD than any
-      # new commits to a pull request would instantly start affecting currently
-      # running jobs and the the list of changed files may become incorrect for
-      # those affected jobs. The solution is to walk backward from the FETCH_HEAD
-      # to the last commit in the pull request. Based on how github currently 
-      # does the automerge, this is the second parent of FETCH_HEAD, and 
-      # therefore we use FETCH_HEAD^2 below
-      #
-      # This may not work perfectly in situations where the user had advanced 
-      # merging happening in their PR. We correctly handle them merging in 
-      # from upstream, but if they do wild stuff then this will likely break
-      # on that. However, it will also likely break by seeing a change in 
-      # toolset and triggering a full run when a partial run would be 
-      # acceptable
-      #
-      # ==== CURRENT SOLUTION FOR OWNED BRANCHES (e.g. master) ====
-      #
-      # This one is fairly simple. Find the commit or commit range, and 
-      # examine the log of files changes. If you encounter any merges, 
-      # then fully explode the two parent commits that made the merge
-      # and look for the files changed there. This is an aggressive 
-      # strategy to ensure that commits to master are always tested 
-      # well
-      log.debug("TRAVIS_COMMIT_RANGE: %s", os.environ['TRAVIS_COMMIT_RANGE'])
-      log.debug("TRAVIS_COMMIT      : %s", os.environ['TRAVIS_COMMIT'])
-
-      is_PR = (os.environ['TRAVIS_PULL_REQUEST'] != "false")
-      if is_PR:
-        log.debug('I am testing a pull request')
-        first_commit = os.environ['TRAVIS_COMMIT_RANGE'].split('...')[0]
-        last_commit = subprocess.check_output("git rev-list -n 1 FETCH_HEAD^2", shell=True).rstrip('\n')
-        log.debug("Guessing that first commit in PR is : %s", first_commit)
-        log.debug("Guessing that final commit in PR is : %s", last_commit)
-
-        if first_commit == "":
-          # Travis-CI is not yet passing a commit range for pull requests
-          # so we must use the automerge's changed file list. This has the 
-          # negative effect that new pushes to the PR will immediately 
-          # start affecting any new jobs, regardless of the build they are on
-          log.debug("No first commit, using Github's automerge commit")
-          self.commit_range = "--first-parent -1 -m FETCH_HEAD"
-        elif first_commit == last_commit:
-          # There is only one commit in the pull request so far, 
-          # or Travis-CI is not yet passing the commit range properly 
-          # for pull requests. We examine just the one commit using -1
-          #
-          # On the oddball chance that it's a merge commit, we pray  
-          # it's a merge from upstream and also pass --first-parent 
-          log.debug("Only one commit in range, examining %s", last_commit)
-          self.commit_range = "-m --first-parent -1 %s" % last_commit
-        else: 
-          # In case they merged in upstream, we only care about the first 
-          # parent. For crazier merges, we hope
-          self.commit_range = "--first-parent %s...%s" % (first_commit, last_commit)
-
-      if not is_PR:
-        log.debug('I am not testing a pull request')
-        # Three main scenarios to consider
-        #  - 1 One non-merge commit pushed to master
-        #  - 2 One merge commit pushed to master (e.g. a PR was merged). 
-        #      This is an example of merging a topic branch
-        #  - 3 Multiple commits pushed to master
-        # 
-        #  1 and 2 are actually handled the same way, by showing the 
-        #  changes being brought into to master when that one commit 
-        #  was merged. Fairly simple, `git log -1 COMMIT`. To handle 
-        #  the potential merge of a topic branch you also include 
-        #  `--first-parent -m`. 
-        #
-        #  3 needs to be handled by comparing all merge children for 
-        #  the entire commit range. The best solution here would *not* 
-        #  use --first-parent because there is no guarantee that it 
-        #  reflects changes brought into master. Unfortunately we have
-        #  no good method inside Travis-CI to easily differentiate 
-        #  scenario 1/2 from scenario 3, so I cannot handle them all 
-        #  separately. 1/2 are the most common cases, 3 with a range 
-        #  of non-merge commits is the next most common, and 3 with 
-        #  a range including merge commits is the least common, so I 
-        #  am choosing to make our Travis-CI setup potential not work 
-        #  properly on the least common case by always using 
-        #  --first-parent 
-        
-        # Handle 3
-        # Note: Also handles 2 because Travis-CI sets COMMIT_RANGE for 
-        # merged PR commits
-        self.commit_range = "--first-parent -m %s" % os.environ['TRAVIS_COMMIT_RANGE']
-
-        # Handle 1
-        if self.commit_range == "":
-          self.commit_range = "--first-parent -m -1 %s" % os.environ['TRAVIS_COMMIT']
-
-    except KeyError:
-      log.warning("I should only be used for automated integration tests e.g. Travis-CI")
-      log.warning("Were you looking for run-tests.py?")
-      self.commit_range = "-m HEAD^...HEAD"
-
-    #
-    # Find the one test from benchmark_config.json that we are going to run
-    #
-
-    tests = gather_tests()
-    self.fwroot = setup_util.get_fwroot()
-    target_dir = self.fwroot + '/frameworks/' + testdir
-    log.debug("Target directory is %s", target_dir)
-    dirtests = [t for t in tests if t.directory == target_dir]
-    
-    # Travis-CI is linux only
-    osvalidtests = [t for t in dirtests if t.os.lower() == "linux"
-                  and (t.database_os.lower() == "linux" or t.database_os.lower() == "none")]
-    
-    # Our Travis-CI only has some databases supported
-    validtests = [t for t in osvalidtests if t.database.lower() in self.SUPPORTED_DATABASES]
-    supported_databases = ','.join(self.SUPPORTED_DATABASES)
-    log.info("Found %s usable tests (%s valid for linux, %s valid for linux and {%s}) in directory '%s'",
-      len(dirtests), len(osvalidtests), len(validtests), supported_databases, '$FWROOT/frameworks/' + testdir)
-    if len(validtests) == 0:
-      log.critical("Found no test that is possible to run in Travis-CI! Aborting!")
-      if len(osvalidtests) != 0:
-        log.critical("Note: Found these tests that could run in Travis-CI if more databases were supported")
-        log.critical("Note: %s", osvalidtests)
-        databases_needed = [t.database for t in osvalidtests]
-        databases_needed = list(set(databases_needed))
-        log.critical("Note: Here are the needed databases:")
-        log.critical("Note: %s", databases_needed)
-      sys.exit(1)
-
-    self.names = [t.name for t in validtests]
-    log.info("Using tests %s to verify directory %s", self.names, '$FWROOT/frameworks/' + testdir)
-
-  def _should_run(self):
-    ''' 
-    Decides if the current framework test should be tested. 
-    Examines git commits included in the latest push to see if any files relevant to 
-    this framework were changed. 
-    If you do rewrite history (e.g. rebase) then it's up to you to ensure that both 
-    old and new (e.g. old...new) are available in the public repository. For simple
-    rebase onto the public master this is not a problem, only more complex rebases 
-    may have issues
-    '''
-    # Don't use git diff multiple times, it's mega slow sometimes\
-    # Put flag on filesystem so that future calls to run-ci see it too
-    if os.path.isfile('.run-ci.should_run'):
-      return True
-    if os.path.isfile('.run-ci.should_not_run'):
-      return False
-
-    def touch(fname):
-      open(fname, 'a').close()
-
-    log.debug("Using commit range `%s`", self.commit_range)
-    log.debug("Running `git log --name-only --pretty=\"format:\" %s`" % self.commit_range)
-    changes = ""
-    try:
-      changes = subprocess.check_output("git log --name-only --pretty=\"format:\" %s" % self.commit_range, shell=True)
-    except subprocess.CalledProcessError, e:
-      log.error("Got errors when using git to detect your changes, assuming that we must run this verification!")
-      log.error("Error was: %s", e.output)
-      log.error("Did you rebase a branch? If so, you can safely disregard this error, it's a Travis limitation")
-      return True
-    changes = os.linesep.join([s for s in changes.splitlines() if s]) # drop empty lines
-    if len(changes.splitlines()) > 1000:
-      log.debug("Change list is >1000 lines, uploading to sprunge.us instead of printing to console")
-      url = subprocess.check_output("git log --name-only %s | curl -F 'sprunge=<-' http://sprunge.us" % self.commit_range, shell=True)
-      log.debug("Uploaded to %s", url)
-    else:
-      log.debug("Result:\n%s", changes)
-
-    # Look for changes to core TFB framework code
-    if re.search(r'^toolset/', changes, re.M) is not None: 
-      log.info("Found changes to core framework code")
-      touch('.run-ci.should_run')
-      return True
-  
-    # Look for changes relevant to this test
-    if re.search("^frameworks/%s/" % re.escape(self.directory), changes, re.M) is None:
-      log.info("No changes found for directory %s", self.directory)
-      touch('.run-ci.should_not_run')
-      return False
-
-    log.info("Changes found for directory %s", self.directory)
-    touch('.run-ci.should_run')
-    return True
-
-  def run(self):
-    ''' Do the requested command using TFB  '''
-
-    if not self._should_run():
-      log.info("I found no changes to `%s` or `toolset/`, aborting verification", self.directory)
-      return 0
-
-    names = ' '.join(self.names)
-    # Assume mode is verify
-    # os.environ["TRAVIS_TESTS"] = "%s" % names
-    command = "toolset/run-tests.py --mode verify --test %s" % names
-
-    # Run the command
-    log.info("Running mode %s with commmand %s", self.mode, command)
-    try:
-        p = subprocess.Popen(command, shell=True)
-        p.wait()
-        return p.returncode
-    except subprocess.CalledProcessError:
-        log.critical("Subprocess Error")
-        print traceback.format_exc()
-        return 1
-    except Exception as err:
-        log.critical("Exception from running+wait on subprocess")
-        log.error(err.child_traceback)
-        return 1
-
-if __name__ == "__main__":
-  args = sys.argv[1:]
-
-  usage = '''Usage: toolset/run-ci.py [verify] <framework-directory>
-    
-    run-ci.py selects one test from <framework-directory>/benchark_config, and 
-    automates a number of calls into run-tests.py specific to the selected test. 
-
-    It is guaranteed to always select the same test from the benchark_config, so 
-    multiple runs with the same <framework-directory> reference the same test. 
-    The name of the selected test will be printed to standard output. 
-
-    verify  - run a verification on the selected test using `--mode verify`
-
-    run-ci.py expects to be run inside the Travis-CI build environment, and 
-    will expect environment variables such as $TRAVIS_BUILD'''
-
-  if len(args) != 2:
-    print usage
-    sys.exit(1)
-
-  mode = args[0]
-  testdir = args[1]
-  if len(args) == 2 and (mode == 'verify'):
-    runner = CIRunnner(mode, testdir)
-  else:
-    print usage
-    sys.exit(1)
-    
-  retcode = 0
-  try:
-    retcode = runner.run()
-  except KeyError as ke: 
-    log.warning("Environment key missing, are you running inside Travis-CI?")
-    print traceback.format_exc()
-    retcode = 1
-  except Exception:
-    log.critical("Unknown error")
-    print traceback.format_exc()
-    retcode = 1
-  finally:
-    sys.exit(retcode)
-
-
-# vim: set sw=2 ts=2 expandtab

toolset/run-tests.py

@@ -143,6 +143,7 @@ def main(argv=None):
 
     # Test options
     parser.add_argument('--test', nargs='+', help='names of tests to run')
+    parser.add_argument('--test-dir', nargs='+', dest='test_dir', help='name of framework directory containing all tests to run')
     parser.add_argument('--exclude', nargs='+', help='names of tests to exclude')
     parser.add_argument('--type', choices=['all', 'json', 'db', 'query', 'fortune', 'update', 'plaintext'], default='all', help='which type of test to run')
     parser.add_argument('-m', '--mode', choices=['benchmark', 'verify'], default='benchmark', help='verify mode will only start up the tests, curl the urls and shutdown')

toolset/travis/travis_diff.py

@@ -0,0 +1,129 @@
+#!/usr/bin/env python
+
+# @file:        toolset/travis/travis_diff.py
+# @author:      Nate Brady
+# @description: This script is only for use within Travis-CI. It is meant to look through the commit history
+#   and determine whether or not the current framework test directory needs to be run.
+#
+# Notes: This script will run in python 2 and 3. print is being used instead of the logging import because
+#   travis does not echo python logging during the before_script lifecycle.
+
+import subprocess
+import os
+import re
+from sets import Set
+
+
+# Returns a unique list of fw_depends changes
+def get_fw_depends_changes(changes_output):
+    return list(Set(re.findall(r"toolset/setup/linux/.+/(.+)\.sh", changes_output, re.M)))
+
+
+# Returns a unique list of frameworks that have been changed
+def fw_found_in_changes(changes_output):
+    return re.search(r"" + re.escape(os.environ['TESTDIR']), changes_output, re.M)
+
+
+# Cleans up diffing and grep output and into an array of strings
+def clean_output(output):
+    return os.linesep.join([s for s in output.splitlines() if s])
+
+
+def quit_diffing(should_test_run):
+    if should_test_run:
+        print("travis-diff-continue")
+    exit(0)
+
+
+# COMMIT MESSAGES:
+# Before any complicated diffing, check for forced runs from the commit message
+last_commit_msg = subprocess.check_output(['bash', '-c', 'git log -1 --pretty=%B'])
+
+# Forced *fw-only* specific tests
+if re.search(r'\[ci fw-only.+\]', last_commit_msg, re.M):
+    if re.search(r'\[ci fw-only(.?)+ ' + re.escape(os.environ['TESTDIR']) + '( .+\]|])', last_commit_msg, re.M):
+        print("This test has been forced to run from the commit message.")
+        quit_diffing(True)
+    else:
+        print("Skipping this test from the commit message.")
+        quit_diffing(False)
+# Forced full run
+if re.search(r'\[ci run-all\]', last_commit_msg, re.M):
+    print("All tests have been forced to run from the commit message.")
+    quit_diffing(True)
+# Forced framework run
+if re.search(r'\[ci fw(.?)+ ' + re.escape(os.environ['TESTDIR']) + '( .+\]|\])', last_commit_msg, re.M):
+    print('This test has been forced to run from the commit message.')
+    quit_diffing(True)
+
+print("TRAVIS_COMMIT_RANGE: {!s}".format(os.environ['TRAVIS_COMMIT_RANGE']))
+print("TRAVIS_COMMIT      : {!s}".format(os.environ['TRAVIS_COMMIT']))
+
+is_PR = (os.environ['TRAVIS_PULL_REQUEST'] != "false")
+commit_range = ""
+
+if is_PR:
+    print('I am testing a pull request')
+    first_commit = os.environ['TRAVIS_COMMIT_RANGE'].split('...')[0]
+    last_commit = subprocess.check_output("git rev-list -n 1 FETCH_HEAD^2", shell=True).rstrip('\n')
+    print("Guessing that first commit in PR is : {!s}".format(first_commit))
+    print("Guessing that final commit in PR is : {!s}".format(last_commit))
+
+    if first_commit == "":
+        print("No first commit, using Github's automerge commit")
+        commit_range = "--first-parent -1 -m FETCH_HEAD"
+    elif first_commit == last_commit:
+        print("Only one commit in range, examining {!s}".format(last_commit))
+        commit_range = "-m --first-parent -1 {!s}".format(last_commit)
+    else:
+        commit_range = "--first-parent {!s}...{!s}".format(first_commit, last_commit)
+
+if not is_PR:
+    print('I am not testing a pull request')
+    commit_range = "--first-parent -m {!s}".format(os.environ['TRAVIS_COMMIT_RANGE'])
+
+    # Handle 1
+    if commit_range == "":
+        commit_range = "--first-parent -m -1 {!s}".format(os.environ['TRAVIS_COMMIT'])
+
+print("Using commit range `{!s}`".format(commit_range))
+print("Running `git log --name-only --pretty=\"format:\" {!s}`".format(commit_range))
+changes = clean_output(subprocess.check_output(['bash', '-c', 'git log --name-only --pretty="format:" {!s}'.format(commit_range)]))
+
+# Satisfies this requirement:
+#   Anything in the toolset/ that isn't in the setup/linux/*/ subdirectory
+#   Any commit message that contains [ci run]
+
+if re.search(r'^toolset/(?!setup/linux/.+/)', changes, re.M) is not None:
+    print("Found changes to core toolset. Running all tests.")
+    quit_diffing(True)
+
+if fw_found_in_changes(changes):
+    print("Found changes that affect this framework. Running test.")
+    quit_diffing(True)
+
+# Satisfies this requirement:
+#   ^toolset/setup/linux/.+/(.+)\.sh
+
+# Determine what has been changed based on initial diffing output
+fw_depends_changes = get_fw_depends_changes(changes)
+
+# For each of these, find the files that depend on them, if we find more fw_depends stuff,
+# add it to the bottom of the list, if it isn't already there.
+i = 0
+while i <= len(fw_depends_changes) - 1:
+
+    # Generates output of files that contain the fw_depends for this dependency
+    more_changes = subprocess.check_output(['bash', '-c', 'grep -RP "fw_depends(.?)+ ' + re.escape(fw_depends_changes[i]) + '( |$)" . || echo ""'])
+    print("more_changes: {!s}".format(more_changes))
+    if fw_found_in_changes(more_changes):
+        print("Found changes that affect this framework. Running test.")
+        quit_diffing(True)
+
+    # Preserves the order of the list, so we can continue with this loop
+    fw_depends_changes.extend(Set(get_fw_depends_changes(more_changes)) - Set(fw_depends_changes))
+    i += 1
+
+# If we get here, there was nothing found
+print("Did not find any changes that affect this framework.")
+quit_diffing(False)