Merge in finished Travis-CI YAML file and run-ci.py script

.travis.yml

@@ -0,0 +1,189 @@
+language: python
+python: 
+  - "2.7"
+
+env:
+  global:
+    - TFB_SERVER_HOST=127.0.0.1
+    - TFB_CLIENT_HOST=127.0.0.1
+    - TFB_DATABASE_HOST=127.0.0.1
+    - TFB_CLIENT_USER=travis
+    - TFB_DATABASE_USER=travis
+    - TFB_CLIENT_IDENTITY_FILE=/home/travis/.ssh/id_rsa
+    - TFB_DATABASE_IDENTITY_FILE=/home/travis/.ssh/id_rsa
+
+    # Login token for github to allow run-ci.py to cancel unneeded jobs
+    # To generate, use travis encrypt GH_TOKEN=<token>
+    - secure: A/gsnHDbTA/uf/zWZPQIM2H69N7+Yn8ZgTjlNJvld6VSxcn+PqAEG9KLi5xGfneziNB2R5vTD1NadUvdggR+S/qz6IMu+HHhJHB8kcS0XTCMebNGxzme5qiopUV1m65V5AyB05ZGG5DIS5Lgz9PDEXoJUMrPdmLjOYgMx7Bw54I=
+
+  matrix:
+    # Run one special job that will examine the files changed 
+    # in this push and cancel any unnecessary jobs in the matrix
+    # Put this first as Travis-CI builds top to bottom
+    - TESTDIR=jobcleaner
+
+
+    # Group tests by directory to logically break up travis-CI build. Otherwise
+    # we end up starting ~200+ different workers. Seems that ~100 is the limit
+    # before their website starts to lag heavily
+    #
+    # Here's a bash one-liner if you need to update this: 
+    # toolset/run-tests.py --list-tests | while read line ; do
+    #    echo "    - TEST=$line"
+    # done    
+    - TESTDIR=activeweb
+    - TESTDIR=aspnet
+    - TESTDIR=aspnet-stripped
+    - TESTDIR=beego
+    - TESTDIR=bottle
+    - TESTDIR=cake
+    - TESTDIR=compojure
+    - TESTDIR=cowboy
+    - TESTDIR=cpoll_cppsp
+    - TESTDIR=curacao
+    - TESTDIR=dancer
+    - TESTDIR=dart
+    - TESTDIR=dart-start
+    - TESTDIR=dart-stream
+    - TESTDIR=django
+    - TESTDIR=dropwizard
+    - TESTDIR=dropwizard-mongodb
+    - TESTDIR=elli
+    - TESTDIR=evhttp-sharp
+    - TESTDIR=express
+    - TESTDIR=falcon
+    - TESTDIR=falcore
+    - TESTDIR=finagle
+    - TESTDIR=flask
+    - TESTDIR=gemini
+    - TESTDIR=go
+    - TESTDIR=grails
+    - TESTDIR=grizzly-bm
+    - TESTDIR=grizzly-jersey
+    - TESTDIR=hapi
+    - TESTDIR=hhvm
+    - TESTDIR=http-kit
+    - TESTDIR=HttpListener
+    - TESTDIR=jester
+    - TESTDIR=jetty-servlet
+    - TESTDIR=kelp
+    - TESTDIR=lapis
+    - TESTDIR=lift-stateless
+    - TESTDIR=luminus
+    - TESTDIR=mojolicious
+    - TESTDIR=nancy
+    - TESTDIR=nawak
+    - TESTDIR=netty
+    - TESTDIR=ninja-resin
+    - TESTDIR=ninja-standalone
+    - TESTDIR=nodejs
+    - TESTDIR=onion
+    - TESTDIR=openresty
+    - TESTDIR=php
+    - TESTDIR=php-codeigniter
+    - TESTDIR=php-fuel
+    - TESTDIR=php-kohana
+    - TESTDIR=php-laravel
+    - TESTDIR=php-lithium
+    - TESTDIR=php-micromvc
+    - TESTDIR=php-phalcon
+    - TESTDIR=php-phalcon-micro
+    - TESTDIR=php-phpixie
+    - TESTDIR=php-pimf
+    - TESTDIR=php-senthot
+    - TESTDIR=php-silex
+    - TESTDIR=php-silex-orm
+    - TESTDIR=php-silica
+    - TESTDIR=php-slim
+    - TESTDIR=php-symfony2
+    - TESTDIR=php-symfony2-stripped
+    - TESTDIR=php-yaf
+    - TESTDIR=php-yii2
+    - TESTDIR=php-zend-framework
+    - TESTDIR=phreeze
+    - TESTDIR=plack
+    - TESTDIR=plain
+    - TESTDIR=play1
+    - TESTDIR=play1siena
+    - TESTDIR=play-activate-mysql
+    - TESTDIR=play-java
+    - TESTDIR=play-java-jpa
+    - TESTDIR=play-scala
+    - TESTDIR=play-scala-mongodb
+    - TESTDIR=play-slick
+    - TESTDIR=pyramid
+    - TESTDIR=rack
+    - TESTDIR=racket-ws
+    - TESTDIR=rails
+    - TESTDIR=rails-stripped
+    - TESTDIR=restexpress
+    - TESTDIR=revel
+    - TESTDIR=revel-jet
+    - TESTDIR=revel-qbs
+    - TESTDIR=ringojs
+    - TESTDIR=ringojs-convenient
+    - TESTDIR=scalatra
+    - TESTDIR=servicestack
+    - TESTDIR=servlet
+    - TESTDIR=servlet3-cass
+    - TESTDIR=sinatra
+    - TESTDIR=snap
+    - TESTDIR=spark
+    - TESTDIR=spray
+    - TESTDIR=spring
+    - TESTDIR=tapestry
+    - TESTDIR=tornado
+    - TESTDIR=treefrog
+    - TESTDIR=undertow
+    - TESTDIR=undertow-edge
+    - TESTDIR=unfiltered
+    - TESTDIR=urweb
+    - TESTDIR=uwsgi
+    - TESTDIR=vertx
+    - TESTDIR=wai
+    - TESTDIR=WeberFramework
+    - TESTDIR=webgo
+    - TESTDIR=web-simple
+    - TESTDIR=wicket
+    - TESTDIR=wildfly-ee7
+    - TESTDIR=wsgi
+    - TESTDIR=wt
+    - TESTDIR=yesod
+
+before_install:
+  - sudo apt-get update
+  - sudo apt-get install openssh-server
+  
+  # Needed to cancel build jobs from run-ci.py
+  - time gem install travis -v 1.6.16 --no-rdoc --no-ri 
+ 
+  # Run as travis use (who has passwordless sudo)
+  - ssh-keygen -f /home/travis/.ssh/id_rsa -N '' -t rsa  
+  - cat /home/travis/.ssh/id_rsa.pub > /home/travis/.ssh/authorized_keys
+  - chmod 600 /home/travis/.ssh/authorized_keys
+  
+  # Setup database manually
+  # NOTE: Do not run database installation! It restarts mysql with a different 
+  # configuration and will break travis's mysql setup
+  - mysql -uroot < config/create.sql
+  
+  # Doesn't work yet
+  # - alias run_tfb="coverage run --parallel-mode --omit installs,results"
+install:
+  - time pip install coveralls
+  
+  # Install server prerequisites
+  - time ./toolset/run-ci.py prereq $TESTDIR
+
+  # Add commit diff to the logs
+  - echo $TRAVIS_COMMIT_RANGE
+  - git diff --name-only $TRAVIS_COMMIT_RANGE
+script: 
+  
+  # Run test verification 
+  - time ./toolset/run-ci.py test $TESTDIR
+
+after_success:
+  - coverage combine
+  - coverage report
+  - coveralls

toolset/run-ci.py

@@ -0,0 +1,261 @@
+#!/usr/bin/env python
+
+import subprocess
+import os
+import sys
+from benchmark import framework_test
+import glob
+import json
+import traceback
+import re
+import logging
+log = logging.getLogger('run-ci')
+import time
+import threading
+
+# Needed for various imports
+sys.path.append('.')
+sys.path.append('toolset/setup/linux')
+sys.path.append('toolset/benchmark')
+
+class CIRunnner:
+  '''
+  Manages running TFB on the Travis Continuous Integration system. 
+  Makes a best effort to avoid wasting time and resources by running 
+  useless jobs. 
+  
+  Only verifies the first test in each directory 
+  '''
+  
+  def __init__(self, mode, test_directory):
+    '''
+    mode = [prereq|install|test] for what we want TFB to do
+    dir  = directory we are running
+    '''
+
+    logging.basicConfig(level=logging.INFO)
+
+    try:
+      self.commit_range = os.environ['TRAVIS_COMMIT_RANGE']
+    except KeyError:
+      log.warning("Run-ci.py should only be used for automated integration tests")
+      last_commit = subprocess.check_output("git rev-parse HEAD^", shell=True).rstrip('\n')
+      self.commit_range = "master...%s" % last_commit
+    
+    if not test_directory == 'jobcleaner':
+      tests = self.gather_tests()
+      
+      # Only run the first test in this directory
+      self.test = [t for t in tests if t.directory == test_directory][0]
+      self.name = self.test.name
+
+    self.mode = mode
+    self.travis = Travis()
+
+  def _should_run(self):
+    ''' 
+    Decides if the current framework test should be tested or if we can cancel it.
+    Examines git commits included in the latest push to see if any files relevant to 
+    this framework were changed. 
+    This is a rather primitive strategy for things like pull requests, where
+    we probably want to examine the entire branch of commits. Also, this cannot handle 
+    history re-writing very well, so avoid rebasing onto any published history
+    '''
+
+    # Look for changes to core TFB framework code
+    find_tool_changes = "git diff --name-only %s | grep toolset | wc -l" % self.commit_range
+    changes = subprocess.check_output(find_tool_changes, shell=True)  
+    if int(changes) != 0:
+      log.info("Found changes to core framework code")
+      return True
+  
+    # Look for changes relevant to this test
+    find_test_changes = "git diff --name-only %s | grep %s | wc -l" % (self.commit_range, self.test.directory)
+    changes = subprocess.check_output(find_test_changes, shell=True)
+    if int(changes) == 0:
+      log.info("No changes found for %s", self.name)
+      return False
+
+    log.info("Changes found for %s", self.name)
+    return True
+
+  def run(self):
+    ''' Do the requested command using TFB  '''
+    if not self._should_run():
+      log.info("Not running %s", self.name)
+      
+      # Cancel ourselves
+      self.travis.cancel(self.travis.jobid)
+      return 0
+
+    log.info("Running %s for %s", self.mode, self.name)
+
+    # Use coverage so we can send code coverate to coveralls.io
+    command = "coverage run --source toolset,%s --parallel-mode " % self.test.directory
+    
+    command = command + 'toolset/run-tests.py --log DEBUG '
+    if mode == 'prereq':
+      command = command + "--install server --test ''"
+    elif mode == 'install':
+      # Just a note that having an install-only mode would integrate nicely with 
+      # Travis-CI's line-folding
+      log.warning('Currently there is no install-only mode available')
+      return 1
+    elif mode == 'test':
+      command = command + "--install server --mode verify --test %s" % self.name
+    else:
+      log.critical('Unknown mode passed')
+      return 1
+    
+    # Run the command
+    log.info("Running %s", command)
+    try:
+      p = subprocess.Popen(command, shell=True)
+      p.wait()
+      return p.returncode  
+    except subprocess.CalledProcessError:
+      log.critical("Subprocess Error")
+      print traceback.format_exc()
+      return 1
+    except Exception as err:
+      log.critical("Subprocess Error")
+      log.error(err.child_traceback)
+      return 1
+
+  def gather_tests(self):
+    ''' Returns all available tests as FrameworkTest list '''
+
+    # Fake benchmarker fields that are used
+    class bench_shim():
+      def __init__(self):
+        self.type = 'all'
+        self.fwroot = os.getcwd()
+        self.install_strategy='pertest'
+
+    # Gather all tests
+    tests = []
+    for config_file_name in glob.glob('*/benchmark_config'):
+      with open(config_file_name, 'r') as config_file:
+        config = json.load(config_file)
+        test = framework_test.parse_config(config, os.path.dirname(config_file_name), bench_shim())
+        tests = tests + test
+    tests.sort(key=lambda x: x.name)
+    return tests
+
+  def cancel_unneeded_jobs(self):
+    log.info("I am jobcleaner")
+
+    # Look for changes to core TFB framework code
+    find_tool_changes = "git diff --name-only %s | grep toolset | wc -l" % self.commit_range
+    changes = subprocess.check_output(find_tool_changes, shell=True)  
+    if int(changes) != 0:
+      log.info("Found changes to core framework code. Running all tests")
+      self.travis.cancel(self.travis.jobid) # Cancel ourselves
+      return 0
+    
+    build = self.travis.build_details()
+    log.info("Build details:\n%s", build)
+    def parse_job_id(directory):
+      for line in build.split('\n'):
+        if "TESTDIR=%s" % directory in line: 
+          job = re.findall("\d+.\d+", line)[0]
+          return job
+    
+    # Build a list of modified directories
+    changes = subprocess.check_output("git diff --name-only %s" % self.commit_range, shell=True)
+    dirchanges = []
+    for line in changes.split('\n'):
+      dirchanges.append(line[0:line.find('/')])
+
+    # For each test, launch a Thread to cancel it's job if 
+    # it's directory has not been modified
+    cancelled_testdirs = []
+    threads = []
+    for test in self.gather_tests():
+      if test.directory not in dirchanges:
+        job = parse_job_id(test.directory)
+        log.info("No changes found for %s (job=%s) (dir=%s)", test.name, job, test.directory)
+        if job and test.directory not in cancelled_testdirs:
+          cancelled_testdirs.append(test.directory)
+          t = threading.Thread(target=self.travis.cancel, args=(job,),
+            name="%s (%s)" % (job, test.name))
+          t.start()
+          threads.append(t)
+
+    # Wait for all threads
+    for t in threads:
+      t.join()
+
+    # Cancel ourselves
+    self.travis.cancel(self.travis.jobid)
+
+
+class Travis():
+  '''Integrates the travis-ci build environment and the travis command line'''
+  def __init__(self):     
+    self.token = os.environ['GH_TOKEN']
+    self.jobid = os.environ['TRAVIS_JOB_NUMBER']
+    self.buildid = os.environ['TRAVIS_BUILD_NUMBER']
+    self._login()
+
+  def _login(self):
+    subprocess.check_call("travis login --no-interactive --github-token %s" % self.token, shell=True)
+    log.info("Logged into travis") # NEVER PRINT OUTPUT, GH_TOKEN MIGHT BE REVEALED
+
+  def cancel(self, job):
+    # Ignore errors in case job is already cancelled
+    try:
+      subprocess.check_call("travis cancel %s --no-interactive" % job, shell=True)
+      log.info("Thread %s: Canceled job %s", threading.current_thread().name, job)
+    except subprocess.CalledProcessError:
+      log.exception("Error halting job %s. Report:", job)
+      subprocess.call("travis report --no-interactive --org", shell=True)
+      log.error("Trying to halt %s one more time", job)
+      subprocess.call("travis cancel %s --no-interactive" % job, shell=True)
+
+  def build_details(self):
+    build = subprocess.check_output("travis show %s" % self.buildid, shell=True)
+    return build
+
+if __name__ == "__main__":
+  args = sys.argv[1:]
+
+  if len(args) != 2 or not (args[0] == "prereq" or args[0] == "install" or args[0] == "test"):
+    print "Usage: toolset/run-ci.py [prereq|install|test] test-name"
+    sys.exit(1)
+
+  mode = args[0]
+  testdir = args[1]
+
+  runner = CIRunnner(mode, testdir)
+  
+  # Watch for the special test name indicating we are 
+  # the test in charge of cancelling unnecessary jobs
+  if testdir == "jobcleaner":
+    try:
+      log.info("Sleeping to ensure Travis-CI has queued all jobs")
+      time.sleep(20)
+      runner.cancel_unneeded_jobs()
+    except KeyError as ke: 
+      log.warning("Environment key missing, are you running inside Travis-CI?")
+    except:
+      log.critical("Unknown error")
+      print traceback.format_exc()
+    finally: 
+      sys.exit(0)
+  
+  retcode = 0
+  try:
+    retcode = runner.run()
+  except KeyError as ke: 
+    log.warning("Environment key missing, are you running inside Travis-CI?")
+  except:
+    log.critical("Unknown error")
+    print traceback.format_exc()
+  finally:
+    sys.exit(retcode)
+
+
+
+
+