Merge branch '939'

.travis.yml

@@ -0,0 +1,191 @@
+language: python
+python: 
+  - "2.7"
+
+env:
+  global:
+    - TFB_SERVER_HOST=127.0.0.1
+    - TFB_CLIENT_HOST=127.0.0.1
+    - TFB_DATABASE_HOST=127.0.0.1
+    - TFB_CLIENT_USER=travis
+    - TFB_DATABASE_USER=travis
+    - TFB_CLIENT_IDENTITY_FILE=/home/travis/.ssh/id_rsa
+    - TFB_DATABASE_IDENTITY_FILE=/home/travis/.ssh/id_rsa
+
+    # Login token for github to allow run-ci.py to cancel unneeded jobs
+    # To generate, use travis encrypt GH_TOKEN=<token>
+    - secure: A/gsnHDbTA/uf/zWZPQIM2H69N7+Yn8ZgTjlNJvld6VSxcn+PqAEG9KLi5xGfneziNB2R5vTD1NadUvdggR+S/qz6IMu+HHhJHB8kcS0XTCMebNGxzme5qiopUV1m65V5AyB05ZGG5DIS5Lgz9PDEXoJUMrPdmLjOYgMx7Bw54I=
+
+  matrix:
+    # Run one special job that will examine the files changed 
+    # in this push and cancel any unnecessary jobs in the matrix
+    # Put this first as Travis-CI builds top to bottom
+    - TESTDIR=jobcleaner
+
+
+    # Group tests by directory to logically break up travis-CI build. Otherwise
+    # we end up starting ~200+ different workers. Seems that ~100 is the limit
+    # before their website starts to lag heavily
+    #
+    # Here's a bash one-liner if you need to update this: 
+    # toolset/run-tests.py --list-tests | while read line ; do
+    #    echo "    - TEST=$line"
+    # done    
+    - TESTDIR=activeweb
+    - TESTDIR=aspnet
+    - TESTDIR=aspnet-stripped
+    - TESTDIR=beego
+    - TESTDIR=bottle
+    - TESTDIR=cake
+    - TESTDIR=compojure
+    - TESTDIR=cowboy
+    - TESTDIR=cpoll_cppsp
+    - TESTDIR=curacao
+    - TESTDIR=dancer
+    - TESTDIR=dart
+    - TESTDIR=dart-start
+    - TESTDIR=dart-stream
+    - TESTDIR=dart-redstone
+    - TESTDIR=django
+    - TESTDIR=dropwizard
+    - TESTDIR=dropwizard-mongodb
+    - TESTDIR=elli
+    - TESTDIR=evhttp-sharp
+    - TESTDIR=express
+    - TESTDIR=falcon
+    - TESTDIR=falcore
+    - TESTDIR=finagle
+    - TESTDIR=flask
+    - TESTDIR=gemini
+    - TESTDIR=go
+    - TESTDIR=grails
+    - TESTDIR=grizzly-bm
+    - TESTDIR=grizzly-jersey
+    - TESTDIR=hapi
+    - TESTDIR=hhvm
+    - TESTDIR=http-kit
+    - TESTDIR=HttpListener
+    - TESTDIR=jester
+    - TESTDIR=jetty-servlet
+    - TESTDIR=kelp
+    - TESTDIR=lapis
+    - TESTDIR=lift-stateless
+    - TESTDIR=luminus
+    - TESTDIR=mojolicious
+    - TESTDIR=nancy
+    - TESTDIR=nawak
+    - TESTDIR=netty
+    - TESTDIR=ninja-resin
+    - TESTDIR=ninja-standalone
+    - TESTDIR=nodejs
+    - TESTDIR=onion
+    - TESTDIR=openresty
+    - TESTDIR=php
+    - TESTDIR=php-codeigniter
+    - TESTDIR=php-fuel
+    - TESTDIR=php-kohana
+    - TESTDIR=php-laravel
+    - TESTDIR=php-lithium
+    - TESTDIR=php-micromvc
+    - TESTDIR=php-phalcon
+    - TESTDIR=php-phalcon-micro
+    - TESTDIR=php-phpixie
+    - TESTDIR=php-pimf
+    - TESTDIR=php-senthot
+    - TESTDIR=php-silex
+    - TESTDIR=php-silex-orm
+    - TESTDIR=php-silica
+    - TESTDIR=php-slim
+    - TESTDIR=php-symfony2
+    - TESTDIR=php-symfony2-stripped
+    - TESTDIR=php-yaf
+    - TESTDIR=php-yii2
+    - TESTDIR=php-zend-framework
+    - TESTDIR=phreeze
+    - TESTDIR=plack
+    - TESTDIR=plain
+    - TESTDIR=play1
+    - TESTDIR=play1siena
+    - TESTDIR=play-activate-mysql
+    - TESTDIR=play-java
+    - TESTDIR=play-java-jpa
+    - TESTDIR=play-scala
+    - TESTDIR=play-scala-mongodb
+    - TESTDIR=play-slick
+    - TESTDIR=pyramid
+    - TESTDIR=rack
+    - TESTDIR=racket-ws
+    - TESTDIR=rails
+    - TESTDIR=rails-stripped
+    - TESTDIR=restexpress
+    - TESTDIR=revel
+    - TESTDIR=revel-jet
+    - TESTDIR=revel-qbs
+    - TESTDIR=ringojs
+    - TESTDIR=ringojs-convenient
+    - TESTDIR=scalatra
+    - TESTDIR=servicestack
+    - TESTDIR=servlet
+    - TESTDIR=servlet3-cass
+    - TESTDIR=sinatra
+    - TESTDIR=snap
+    - TESTDIR=spark
+    - TESTDIR=spray
+    - TESTDIR=spring
+    - TESTDIR=tapestry
+    - TESTDIR=tornado
+    - TESTDIR=treefrog
+    - TESTDIR=ULib
+    - TESTDIR=undertow
+    - TESTDIR=undertow-edge
+    - TESTDIR=unfiltered
+    - TESTDIR=urweb
+    - TESTDIR=uwsgi
+    - TESTDIR=vertx
+    - TESTDIR=wai
+    - TESTDIR=WeberFramework
+    - TESTDIR=webgo
+    - TESTDIR=web-simple
+    - TESTDIR=wicket
+    - TESTDIR=wildfly-ee7
+    - TESTDIR=wsgi
+    - TESTDIR=wt
+    - TESTDIR=yesod
+
+before_install:
+  - sudo apt-get update
+  - sudo apt-get install openssh-server
+  
+  # Needed to cancel build jobs from run-ci.py
+  - time gem install travis -v 1.6.16 --no-rdoc --no-ri 
+ 
+  # Run as travis use (who has passwordless sudo)
+  - ssh-keygen -f /home/travis/.ssh/id_rsa -N '' -t rsa  
+  - cat /home/travis/.ssh/id_rsa.pub > /home/travis/.ssh/authorized_keys
+  - chmod 600 /home/travis/.ssh/authorized_keys
+  
+  # Setup database manually
+  # NOTE: Do not run database installation! It restarts mysql with a different 
+  # configuration and will break travis's mysql setup
+  - mysql -uroot < config/create.sql
+  
+  # Doesn't work yet
+  # - alias run_tfb="coverage run --parallel-mode --omit installs,results"
+install:
+  - time pip install coveralls
+  
+  # Install server prerequisites
+  - time ./toolset/run-ci.py prereq $TESTDIR
+
+  # Add commit diff to the logs
+  - echo $TRAVIS_COMMIT_RANGE
+  - git diff --name-only $TRAVIS_COMMIT_RANGE
+script: 
+  
+  # Run test verification 
+  - time ./toolset/run-ci.py test $TESTDIR
+
+after_success:
+  - coverage combine
+  - coverage report
+  - coveralls

ULib/bash_profile.sh

@@ -12,7 +12,5 @@
 #    . $FWROOT/ULib/install.sh (cwd=$FWROOT//installs)
 #---------------------------------------------------------------------------------------------------------
 export ULIB_VERSION=1.4.1
-export ULIB_ROOT=${IROOT}/ULib
+export ULIB_ROOT=$IROOT/ULib
 export ULIB_DOCUMENT_ROOT=${ULIB_ROOT}/ULIB_DOCUMENT_ROOT
-export ULIB_BUILD_OUTPUT=${ULIB_ROOT}/ULIB_BUILD_OUTPUT.txt
-export ULIB_SERVER_OUTPUT=${ULIB_ROOT}/ULIB_SERVER_OUTPUT.txt

ULib/install.sh

@@ -14,14 +14,18 @@
 #    . $FWROOT/toolset/setup/linux/bash_functions.sh && 
 #    . $FWROOT/ULib/install.sh (cwd=$FWROOT//installs)
 # --------------------------------------------------------------------------------------------------------
-. ${TROOT}/bash_profile.sh
 
+# Chekc if ULib is already installed
+RETCODE=$(fw_exists ulib-${ULIB_VERSION}.installed)
+[ ! "$RETCODE" == 0 ] || { return 0; }
+
+# Create a run directory for ULIB
 if [ ! -d "$ULIB_ROOT" ]; then
   mkdir -p $ULIB_ROOT
 fi
 
+# Add a simple configuration file to it
 cd $ULIB_ROOT
-
 if [ ! -f "benchmark.cfg" ]; then
   cat <<EOF >benchmark.cfg
 userver {
@@ -35,45 +39,39 @@ userver {
 EOF
 fi
 
-if [ -x "bin/userver_tcp" ] && [ -x "${ULIB_DOCUMENT_ROOT}/db.so" ]; then
-  exit 0
-fi
-
 # 1. Download ULib
-if [ ! -f "v${ULIB_VERSION}.tar.gz" ]; then
-	wget -nc --no-check-certificate --trust-server-names -O v${ULIB_VERSION}.tar.gz \
-        https://github.com/stefanocasazza/ULib/archive/v${ULIB_VERSION}.tar.gz 2>/dev/null
+cd $IROOT
+fw_get -O ULib-${ULIB_VERSION}.tar.gz https://github.com/stefanocasazza/ULib/archive/v${ULIB_VERSION}.tar.gz 
+fw_untar ULib-${ULIB_VERSION}.tar.gz
 
-	tar xf v${ULIB_VERSION}.tar.gz 2>/dev/null
-fi
+# 2. Compile application (userver_tcp)
 
 cd ULib-$ULIB_VERSION
 
-# 2. Compile application (userver_tcp)
-# ======================================================================================================
-# TO AVOID configure: error: newly created file is older than distributed files! Check your system clock
-# ======================================================================================================
+# AVOID "configure: error: newly created file is older than distributed files! Check your system clock"
 find . -exec touch {} \;
-# ======================================================================================================
 
-#           --enable-debug \
 LIBS="-lssl -lcrypto -lz" \
 ./configure --prefix=$ULIB_ROOT \
             --disable-static \
             --with-mysql \
             --without-ssl --without-pcre --without-expat \
             --without-libz --without-libuuid --without-magic \
-            --enable-static-orm-driver=mysql --enable-static-server-plugin=http >$ULIB_BUILD_OUTPUT 2>&1
-
-make -j1 install >>$ULIB_BUILD_OUTPUT 2>&1
+            --enable-static-orm-driver=mysql --enable-static-server-plugin=http
+#           --enable-debug \
+make install
 
 # 3. Compile usp pages for benchmark
 cd src/ulib/net/server/plugin/usp
-make -j1 db.la fortunes.la json.la plaintext.la queries.la updates.la >>$ULIB_BUILD_OUTPUT 2>&1
+make db.la fortunes.la json.la plaintext.la queries.la updates.la
 
+# Check that compilation worked
 if [ ! -e .libs/db.so ]; then
    exit 1
 fi
 
 mkdir -p $ULIB_DOCUMENT_ROOT
 cp .libs/db.so .libs/fortunes.so .libs/json.so .libs/plaintext.so .libs/queries.so .libs/updates.so $ULIB_DOCUMENT_ROOT
+
+cd $IROOT
+touch ulib-${ULIB_VERSION}.installed 

ULib/setup.py

@@ -1,160 +1,16 @@
-# setup.py
-
-# THIS CALLING IS WORKING
-# ----------------------------------------------------------------------
-# toolset/run-tests.py --install server --test ULib --type all --verbose
-# ----------------------------------------------------------------------
-# ....
-# ULib: setup.py running - FWROOT is /home/tfb/FrameworkBenchmarks
-# ....
-# INFO:root:Running installation for ULib
-# INSTALL: 
-#    export TROOT=$FWROOT/ULib && 
-#    export IROOT=$FWROOT/installs && 
-#    . $FWROOT/toolset/setup/linux/bash_functions.sh && 
-#    . $FWROOT/ULib/install.sh (cwd=$FWROOT//installs)
-# ....
-# -----------------------------------------------------
-#   Running Test: ULib ...
-# -----------------------------------------------------
-# ULib: TROOT is EMPTY
-# ULib: setup.py START - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
-# ULib: bash_profile.sh script START
-# ULib: trying to start server /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/bin/userver_tcp -c /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/benchmark.cfg
-# ULib: server STARTED
-# {'results': []}
-# ULib: TROOT is EMPTY
-# ULib: setup.py STOP - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
-# ULib: bash_profile.sh script START
-# ----------------------------------------------------------------------
-
-# NOW THIS CALLING IS WORKING TOO BUT IT IS CORRECT TO CALL IT BEFORE INSTALLING...?
-# ----------------------------------------------------------------------------------
-# toolset/run-tests.py --test ULib --type all --verbose
-# ----------------------------------------------------------------------------------
-# ....
-# ULib: setup.py running - FWROOT is /home/tfb/FrameworkBenchmarks
-# ....
-# -----------------------------------------------------
-#   Running Test: ULib ...
-# -----------------------------------------------------
-# ULib: TROOT is EMPTY
-# ULib: setup.py START - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
-# ULib: bash_profile.sh script START
-# ULib: NOT INSTALLED...
-# ULib: install.sh script START
-# ULib: trying to start server /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/bin/userver_tcp -c /home/tfb/FrameworkBenchmarks/installs/pertest/ULib/ULib/benchmark.cfg
-# ULib: server STARTED
-# {'results': []}
-# ULib: TROOT is EMPTY
-# ULib: setup.py STOP - IROOT is /home/tfb/FrameworkBenchmarks/installs/pertest/ULib - TROOT is /home/tfb/FrameworkBenchmarks/ULib
-# ULib: bash_profile.sh script START
-# -----------------------------------------------------
-
 import os
 import sys
 import time
-#import logging
 import setup_util
 import subprocess
 import multiprocessing
 
-#log = logging.getLogger('framework_test')
-
-fwroot = setup_util.get_fwroot()
-
-print(    "ULib: setup.py running - FWROOT is %s" % fwroot)
-#log.info("ULib: setup.py running - FWROOT is %s" % fwroot)
-
-# Queries the shell for the value of IROOT
-def get_iroot():
-  try:
-    # Use printf to avoid getting a newline
-    # Redirect to avoid stderr printing
-    iroot = subprocess.check_output('printf \"$IROOT\" 2>/dev/null', shell=True, executable='/bin/bash')
-    if iroot != "":
-      return iroot
-    print(       'ULib: IROOT is EMPTY')
-#   log.critical('ULib: IROOT is EMPTY')
-    return setup_util.get_fwroot() + "/installs"
-  except subprocess.CalledProcessError:
-    return "";
-
-# Queries the shell for the value of TROOT
-def get_troot():
-  try:
-    # Use printf to avoid getting a newline
-    # Redirect to avoid stderr printing
-    troot = subprocess.check_output('printf \"$TROOT\" 2>/dev/null', shell=True, executable='/bin/bash')
-    if troot != "":
-      return troot
-    print(       'ULib: TROOT is EMPTY')
-#   log.critical('ULib: TROOT is EMPTY')
-    return setup_util.get_fwroot() + "/ULib"
-  except subprocess.CalledProcessError:
-    return "";
-
-def getEnvironmentVar(name, iroot, troot):
-  try:
-    script = """
-export IROOT=%s
-. %s/bash_profile.sh 2>>/tmp/ULib_setup.txt
-printf $%s           2>>/tmp/ULib_setup.txt
-"""
-    value = subprocess.check_output(script % (iroot, troot, name), shell=True, executable='/bin/bash')
-    return value
-  except subprocess.CalledProcessError:
-    return "";
-
-def callScriptAndCheckForInstallation(name, iroot, troot):
-  try:
-    script = """
-export IROOT=%s
-export TROOT=%s
-. %s/%s.sh >>/tmp/ULib_setup.txt 2>&1
-if [ ! -x "${ULIB_ROOT}/bin/userver_tcp" ] || [ ! -e "${ULIB_DOCUMENT_ROOT}/db.so" ] || [ ! -f "${ULIB_ROOT}/benchmark.cfg" ]; then
-  exit 1
-fi
-"""
-    print(   "ULib: %s.sh script START" % name)
-#   log.info("ULib: %s.sh script START" % name)
-    return subprocess.call(script % (iroot, troot, troot, name), shell=True, executable='/bin/bash')
-  except subprocess.CalledProcessError:
-    return 1
-
-# --------------------------------------------------------------------------------------------------------
-# TROOT - Path of this test's directory
-# IROOT - Path of this test's install directory ($FWROOT/installs or $FWROOT/installs/pertest/<test-name>)
-# --------------------------------------------------------------------------------------------------------
-def checkEnvironment(iroot, troot):
-  try:
-    if callScriptAndCheckForInstallation('bash_profile', iroot, troot) != 0:
-      print(       'ULib: NOT INSTALLED...')
-#     log.critical('ULib: NOT INSTALLED...')
-      if callScriptAndCheckForInstallation('install', iroot, troot) != 0:
-        print(   "ULib: install.sh script FAILED")
-#       log.info("ULib: install.sh script FAILED")
-        return False
-    return True
-  except:
-    pass
-  return False
-
-##############
-# start(args)
-##############
 def start(args, logfile, errfile):
-  try:
-    iroot = get_iroot()
-    troot = get_troot()
-
-    print(   "ULib: setup.py START - IROOT is %s - TROOT is %s" % (iroot, troot))
-#   log.info("ULib: setup.py START - IROOT is %s - TROOT is %s" % (iroot, troot))
 
-    if not checkEnvironment(iroot, troot):
-      return 1
+  fwroot = args.fwroot
 
-    ulib_root = getEnvironmentVar('ULIB_ROOT', iroot, troot)
+  try:
+    ulib_root = subprocess.check_output('printf $ULIB_ROOT', shell=True, stderr=errfile)
 
     fcfg = ulib_root + "/benchmark.cfg"
 
@@ -168,44 +24,29 @@ def start(args, logfile, errfile):
     fprg = ulib_root + "/bin/userver_tcp"
 
     # 2. Start ULib Server (userver_tcp)
-    print(   "ULib: trying to start server " + fprg + " -c " + fcfg)
-#   log.info("ULib: trying to start server " + fprg + " -c " + fcfg)
+    logfile.write("ULib: trying to start server %s -c %s\n" % (fprg, fcfg))
 
     # sudo mysqlcheck -v -r -A -u benchmarkdbuser -p
     os.putenv("ORM_DRIVER","mysql")
     os.putenv("ORM_OPTION","host=" + args.database_host + " user=benchmarkdbuser password=benchmarkdbpass dbname=hello_world")
     os.putenv("UMEMPOOL", "1583,1507,-19,45,16458,523,-27,-14,27")
 
-    ulib_server_output = getEnvironmentVar('ULIB_SERVER_OUTPUT', iroot, troot)
-
     # Run in the background, but keep stdout/stderr for easy debugging
-    subprocess.Popen(                      fprg + " -c " + fcfg + " >" + ulib_server_output + " 2>&1", shell=True, stdout=logfile, stderr=errfile)
-  # subprocess.Popen("UTRACE=\"0 50M\" " + fprg + " -c " + fcfg + " >" + ulib_server_output + " 2>&1", shell=True, stdout=logfile, stderr=errfile)
+    subprocess.Popen( "%s -c %s" % (fprg, fcfg), shell=True, stdout=logfile, stderr=errfile)
+    # subprocess.Popen("UTRACE=\"0 50M\" " + fprg + " -c " + fcfg, shell=True, stdout=logfile, stderr=errfile)
 
-    print(   "ULib: server STARTED")
-#   log.info("ULib: server STARTED")
+    logfile.write("ULib: server STARTED\n")
     return 0
   except subprocess.CalledProcessError:
     return 1
 
-##############
-# stop()
-##############
 def stop(logfile, errfile):
   try:
-    iroot = get_iroot()
-    troot = get_troot()
-
-    print(   "ULib: setup.py STOP - IROOT is %s - TROOT is %s" % (iroot, troot))
-#   log.info("ULib: setup.py STOP - IROOT is %s - TROOT is %s" % (iroot, troot))
-
-    if not checkEnvironment(iroot, troot):
-        return 1
+    logfile.write( "ULib: setup.py STOP\n")
 
-    ulib_root = getEnvironmentVar('ULIB_ROOT', iroot, troot)
 
     # Stop ULib Server (userver_tcp)
-    subprocess.check_call("kill -TERM $( cat " + ulib_root + "/userver_tcp.pid )", shell=True, stderr=errfile, stdout=logfile)
+    subprocess.check_call("kill -TERM $( cat $ULIB_ROOT/userver_tcp.pid )", shell=True, stderr=errfile, stdout=logfile)
     time.sleep(2);
     p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
     out, err = p.communicate()
@@ -213,7 +54,7 @@ def stop(logfile, errfile):
        if 'userver_tcp' in line:
          pid = int(line.split(None, 2)[1])
          os.kill(pid, 9)
-    subprocess.call("rm -f " + ulib_root + "/userver_tcp.pid /tmp/ULib_setup.txt", shell=True, stderr=errfile, stdout=logfile)
+    subprocess.call("rm -f $ULIB_ROOT/userver_tcp.pid", shell=True, stderr=errfile, stdout=logfile)
     return 0
   except subprocess.CalledProcessError:
     return 1

bottle/bash_profile.sh

@@ -0,0 +1,16 @@
+
+export PY2_ROOT=$IROOT/py2
+export PY2=$PY2_ROOT/bin/python
+export PY2_PIP=$PY2_ROOT/bin/pip
+export PY2_GUNICORN=$PY2_ROOT/bin/gunicorn
+
+export PYPY_ROOT=$IROOT/pypy
+export PYPY=$PYPY_ROOT/bin/python
+export PYPY_PIP=$PYPY_ROOT/bin/pip
+export PYPY_GUNICORN=$PYPY_ROOT/bin/gunicorn
+
+export PY3_ROOT=$IROOT/py3
+export PY3=$PY3_ROOT/bin/python
+export PY3_PIP=$PY3_ROOT/bin/pip
+export PY3_GUNICORN=$PY3_ROOT/bin/gunicorn
+

bottle/setup.py

@@ -1,19 +1,13 @@
 import subprocess
 import os
 
-BIN = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
 
 proc = None
 
 
 def start(args, logfile, errfile):
     global proc
-    proc = subprocess.Popen(
-        [BIN + "/gunicorn",
-         "-c", "gunicorn_conf.py",
-         "-e", "DBHOSTNAME=" + args.database_host,
-         "app:app"],
-        cwd="bottle", stderr=errfile, stdout=logfile)
+    proc = subprocess.Popen( "$PY2_GUNICORN -c gunicorn_conf.py -e DBHOSTNAME=%s app:app" % args.database_host, cwd="bottle", shell=True, stderr=errfile, stdout=logfile)
     return 0
 
 
@@ -24,4 +18,7 @@ def stop(logfile, errfile):
     proc.terminate()
     proc.wait()
     proc = None
+
+    subprocess.call("sudo pkill gunicorn", shell=True, stderr=errfile, stdout=logfile)
+
     return 0

bottle/setup_py3.py

@@ -8,12 +8,7 @@ proc = None
 
 def start(args, logfile, errfile):
     global proc
-    proc = subprocess.Popen(
-        [BIN + "/gunicorn",
-         "-c", "gunicorn_conf.py",
-         "-e", "DBHOSTNAME=" + args.database_host,
-         "app:app"],
-        cwd="bottle", stderr=errfile, stdout=logfile)
+    proc = subprocess.Popen( "$PY3_GUNICORN -c gunicorn_conf.py -e DBHOSTNAME=%s app:app" % args.database_host, cwd="bottle", shell=True, stderr=errfile, stdout=logfile)
     return 0
 
 
@@ -24,4 +19,7 @@ def stop(logfile, errfile):
     proc.terminate()
     proc.wait()
     proc = None
+
+    subprocess.call("sudo pkill gunicorn", shell=True, stderr=errfile, stdout=logfile)
+
     return 0

bottle/setup_pypy.py

@@ -1,19 +1,12 @@
 import subprocess
 import os
 
-BIN = os.path.expanduser('~/FrameworkBenchmarks/installs/pypy/bin')
 
 proc = None
 
-
 def start(args, logfile, errfile):
     global proc
-    proc = subprocess.Popen(
-        [BIN + "/gunicorn",
-         "-c", "gunicorn_conf.py",
-         "-e", "DBHOSTNAME=" + args.database_host,
-         "app:app"],
-        cwd="bottle", stderr=errfile, stdout=logfile)
+    proc = subprocess.Popen( "$PYPY_GUNICORN -c gunicorn_conf.py -e DBHOSTNAME=%s app:app" % args.database_host, cwd="bottle", shell=True, stderr=errfile, stdout=logfile)
     return 0
 
 
@@ -24,4 +17,7 @@ def stop(logfile, errfile):
     proc.terminate()
     proc.wait()
     proc = None
+
+    subprocess.call("sudo pkill gunicorn", shell=True, stderr=errfile, stdout=logfile)
+
     return 0

compojure/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

compojure/install.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-fw_depends resin java
+fw_depends resin java leiningen

compojure/setup.py

@@ -7,8 +7,8 @@ def start(args, logfile, errfile):
   setup_util.replace_text("compojure/hello/src/hello/handler.clj", ":subname \"//.*:3306", ":subname \"//" + args.database_host + ":3306")
 
   try:
-    subprocess.check_call("lein clean", shell=True, cwd="compojure/hello", stderr=errfile, stdout=logfile)
-    subprocess.check_call("lein ring uberwar", shell=True, cwd="compojure/hello", stderr=errfile, stdout=logfile)
+    subprocess.check_call("$IROOT/bin/lein clean", shell=True, cwd="compojure/hello", stderr=errfile, stdout=logfile)
+    subprocess.check_call("$IROOT/bin/lein ring uberwar", shell=True, cwd="compojure/hello", stderr=errfile, stdout=logfile)
     subprocess.check_call("rm -rf $RESIN_HOME/webapps/*", shell=True, stderr=errfile, stdout=logfile)
     subprocess.check_call("cp compojure/hello/target/hello-compojure-standalone.war $RESIN_HOME/webapps/compojure.war", shell=True, stderr=errfile, stdout=logfile)
     subprocess.check_call("$RESIN_HOME/bin/resinctl start", shell=True, stderr=errfile, stdout=logfile)

dart-redstone/setup.py

@@ -3,6 +3,7 @@ import sys
 import setup_util
 import os
 
+
 def start(args, logfile, errfile):
   setup_util.replace_text('dart-redstone/postgresql.yaml', 'host: .*', 'host: ' + args.database_host)
   setup_util.replace_text('dart-redstone/mongodb.yaml', 'host: .*', 'host: ' + args.database_host)

gemini/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

gemini/setup.py

@@ -4,17 +4,16 @@ import sys
 import setup_util
 from os.path import expanduser
 
-home = expanduser("~")
-
 def start(args, logfile, errfile):
+  
   setup_util.replace_text("gemini/Docroot/WEB-INF/GeminiHello.conf", "db.ConnectString = .*:3306", "db.ConnectString = " + args.database_host + ":3306")
-  setup_util.replace_text("gemini/Docroot/WEB-INF/resin.xml", "root-directory=\".*\/FrameworkBenchmarks", "root-directory=\"" + home + "/FrameworkBenchmarks")
+  setup_util.replace_text("gemini/Docroot/WEB-INF/resin.xml", "root-directory=\".*\/FrameworkBenchmarks", "root-directory=\"%s" % args.fwroot)
   
   try:
     # This was reporting an error because it already exists... not sure.
     #subprocess.call("mkdir classes", shell=True, cwd="gemini/Docroot/WEB-INF", stderr=errfile, stdout=logfile)
     subprocess.check_call("ant compile", shell=True, cwd="gemini", stderr=errfile, stdout=logfile)
-    subprocess.check_call("$RESIN_HOME/bin/resinctl -conf $HOME/FrameworkBenchmarks/gemini/Docroot/WEB-INF/resin.xml start", shell=True, stderr=errfile, stdout=logfile)
+    subprocess.check_call("$RESIN_HOME/bin/resinctl -conf $FWROOT/gemini/Docroot/WEB-INF/resin.xml start", shell=True, stderr=errfile, stdout=logfile)
     return 0
   except subprocess.CalledProcessError:
     return 1

grails/bash_profile.sh

@@ -0,0 +1,5 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36
+
+export GRAILS_PATH=${IROOT}/grails-2.4.2/bin/
+export PATH=${GRAILS_PATH}:$PATH

luminus/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

luminus/install.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-fw_depends resin
+fw_depends resin leiningen

luminus/setup.py

@@ -7,8 +7,8 @@ def start(args, logfile, errfile):
   setup_util.replace_text("luminus/hello/src/hello/models/schema.clj", ":subname \"//.*:3306", ":subname \"//" + args.database_host + ":3306")
 
   try:
-    subprocess.check_call("lein clean", shell=True, cwd="compojure/hello", stderr=errfile, stdout=logfile)
-    subprocess.check_call("lein ring uberwar", shell=True, cwd="luminus/hello", stderr=errfile, stdout=logfile)
+    subprocess.check_call("$IROOT/bin/lein clean", shell=True, cwd="compojure/hello", stderr=errfile, stdout=logfile)
+    subprocess.check_call("$IROOT/bin/lein ring uberwar", shell=True, cwd="luminus/hello", stderr=errfile, stdout=logfile)
     subprocess.check_call("rm -rf $RESIN_HOME/webapps/*", shell=True)
     subprocess.check_call("cp luminus/hello/target/hello-luminus-standalone.war $RESIN_HOME/webapps/luminus.war", shell=True)
     subprocess.check_call("$RESIN_HOME/bin/resinctl start", shell=True)

ninja-resin/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

ninja-resin/benchmark_config

@@ -1,5 +1,5 @@
 {
-  "framework": "ninja",
+  "framework": "ninja-resin",
   "tests": [{
     "default": {
       "setup_file": "setup",

scalatra/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

servlet/bash_profile.sh

@@ -0,0 +1 @@
+export RESIN_HOME=${IROOT}/resin-4.0.36

servlet3-cass/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

spark/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

tapestry/bash_profile.sh

@@ -0,0 +1,2 @@
+export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
+export RESIN_HOME=${IROOT}/resin-4.0.36

toolset/benchmark/benchmarker.py

@@ -125,7 +125,7 @@ class Benchmarker:
         Running Tests ...
       =====================================================
       """)
-    self.__run_tests(all_tests)
+    result = self.__run_tests(all_tests)
 
     ##########################
     # Parse results
@@ -139,6 +139,7 @@ class Benchmarker:
       self.__parse_results(all_tests)
 
     self.__finish()
+    return result
 
   ############################################################
   # End run
@@ -514,12 +515,14 @@ class Benchmarker:
     logging.debug("Start __run_tests.")
     logging.debug("__name__ = %s",__name__)
 
+    error_happened = False
     if self.os.lower() == 'windows':
       logging.debug("Executing __run_tests on Windows")
       for test in tests:
         with open('current_benchmark.txt', 'w') as benchmark_resume_file:
           benchmark_resume_file.write(test.name)
-        self.__run_test(test)
+        if self.__run_test(test) != 0:
+          error_happened = True
     else:
       logging.debug("Executing __run_tests on Linux")
       # These features do not work on Windows
@@ -540,10 +543,16 @@ class Benchmarker:
             logging.debug("Child process for {name} is still alive. Terminating.".format(name=test.name))
             self.__write_intermediate_results(test.name,"__run_test timeout (="+ str(self.run_test_timeout_seconds) + " seconds)")
             test_process.terminate()
+            test_process.join()
+          if test_process.exitcode != 0:
+            error_happened = True
     if os.path.isfile('current_benchmark.txt'):
       os.remove('current_benchmark.txt')
     logging.debug("End __run_tests.")
 
+    if error_happened:
+      return 1
+    return 0
   ############################################################
   # End __run_tests
   ############################################################
@@ -559,6 +568,14 @@ class Benchmarker:
   # are needed.
   ############################################################
   def __run_test(self, test):
+    
+    # Used to capture return values 
+    def exit_with_code(code):
+      if self.os.lower() == 'windows':
+        return code
+      else:
+        sys.exit(code)
+
     try:
       os.makedirs(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name)))
     except:
@@ -568,24 +585,24 @@ class Benchmarker:
       if hasattr(test, 'skip'):
         if test.skip.lower() == "true":
           out.write("Test {name} benchmark_config specifies to skip this test. Skipping.\n".format(name=test.name))
-          return
+          return exit_with_code(0)
 
       if test.os.lower() != self.os.lower() or test.database_os.lower() != self.database_os.lower():
         # the operating system requirements of this test for the
         # application server or the database server don't match
         # our current environment
         out.write("OS or Database OS specified in benchmark_config does not match the current environment. Skipping.\n")
-        return 
+        return exit_with_code(0)
       
       # If the test is in the excludes list, we skip it
       if self.exclude != None and test.name in self.exclude:
         out.write("Test {name} has been added to the excludes list. Skipping.\n".format(name=test.name))
-        return
+        return exit_with_code(0)
       
       # If the test does not contain an implementation of the current test-type, skip it
       if self.type != 'all' and not test.contains_type(self.type):
         out.write("Test type {type} does not contain an implementation of the current test-type. Skipping.\n".format(type=self.type))
-        return
+        return exit_with_code(0)
 
       out.write("test.os.lower() = {os}  test.database_os.lower() = {dbos}\n".format(os=test.os.lower(),dbos=test.database_os.lower()))
       out.write("self.results['frameworks'] != None: {val}\n".format(val=str(self.results['frameworks'] != None)))
@@ -593,7 +610,7 @@ class Benchmarker:
       out.write("self.results['completed']: {completed}\n".format(completed=str(self.results['completed'])))
       if self.results['frameworks'] != None and test.name in self.results['completed']:
         out.write('Framework {name} found in latest saved data. Skipping.\n'.format(name=str(test.name)))
-        return
+        return exit_with_code(1)
 
       out.flush()
 
@@ -632,7 +649,7 @@ class Benchmarker:
             ---------------------------------------------------------
             """.format(name=test.name, port=str(test.port))) )
           err.flush()
-          return
+          return exit_with_code(1)
 
         result = test.start(out, err)
         if result != 0: 
@@ -646,14 +663,14 @@ class Benchmarker:
             """.format(name=test.name)) )
           err.flush()
           self.__write_intermediate_results(test.name,"<setup.py>#start() returned non-zero")
-          return
+          return exit_with_code(1)
         
         time.sleep(self.sleep)
 
         ##########################
         # Verify URLs
         ##########################
-        test.verify_urls(out, err)
+        passed_verify = test.verify_urls(out, err)
         out.flush()
         err.flush()
 
@@ -693,7 +710,7 @@ class Benchmarker:
             -----------------------------------------------------
             """.format(name=test.name, port=str(test.port))) )
           err.flush()
-          return
+          return exit_with_code(1)
 
         out.write( textwrap.dedent("""
         -----------------------------------------------------
@@ -714,6 +731,10 @@ class Benchmarker:
         """.format(name=test.name)) )
         out.flush()
         self.__write_intermediate_results(test.name,time.strftime("%Y%m%d%H%M%S", time.localtime()))
+
+        if self.mode == "verify" and not passed_verify:
+          print "Failed verify!"
+          return exit_with_code(1)
       except (OSError, IOError, subprocess.CalledProcessError) as e:
         self.__write_intermediate_results(test.name,"<setup.py> raised an exception")
         err.write( textwrap.dedent("""
@@ -736,7 +757,12 @@ class Benchmarker:
           {trace}
           """.format(name=test.name, err=e, trace=sys.exc_info()[:2])) )
           err.flush()
-      except (KeyboardInterrupt, SystemExit) as e:
+        out.close()
+        err.close()
+        return exit_with_code(1)
+      # TODO - subprocess should not catch this exception!
+      # Parent process should catch it and cleanup/exit
+      except (KeyboardInterrupt) as e:
         test.stop(out, err)
         out.write( """
         -----------------------------------------------------
@@ -745,10 +771,11 @@ class Benchmarker:
         """)
         out.flush()
         self.__finish()
-        sys.exit()
+        sys.exit(1)
 
       out.close()
       err.close()
+      return exit_with_code(0)
 
   ############################################################
   # End __run_tests

toolset/benchmark/framework_test.py

@@ -291,10 +291,13 @@ class FrameworkTest:
       return (False, err_str)
     try:
       json_load = json.loads(jsonString)
-      if isinstance(json_load, list):
-        err_str += "Expected JSON object, got JSON array. " 
+      if not isinstance(json_load, list):
+        err_str += "Expected JSON array, got {typeObj}. ".format(typeObj=type(json_load))
         return (False, err_str)
-      arr = {k.lower(): v for k,v in json_string.iteritems()}
+      if len(json_load) != 1:
+        err_str += "Expected array of length 1. Got length {length}. ".format(length=len(json_load))
+
+      arr = {k.lower(): v for k,v in json_string[0].iteritems()}
 
       for obj in arr:
         id_ret_val = True
@@ -448,8 +451,10 @@ class FrameworkTest:
       logging.warning("Framework %s does not have a bash_profile" % self.name)
       profile="$FWROOT/config/benchmark_profile"
     
-    set_iroot="export IROOT=%s" % self.install_root
-    setup_util.replace_environ(config=profile, command=set_iroot)
+    test_rel_dir = setup_util.path_relative_to_root(self.directory)
+    setup_util.replace_environ(config=profile, 
+              command='export TROOT=$FWROOT%s && export IROOT=%s' %
+              (test_rel_dir, self.install_root))
 
     return self.setup_module.start(self.benchmarker, out, err)
   ############################################################
@@ -461,6 +466,17 @@ class FrameworkTest:
   # Stops the test using it's setup file
   ############################################################
   def stop(self, out, err):
+    # Load profile for this installation
+    profile="%s/bash_profile.sh" % self.directory
+    if not os.path.exists(profile):
+      logging.warning("Framework %s does not have a bash_profile" % self.name)
+      profile="$FWROOT/config/benchmark_profile"
+    
+    test_rel_dir = setup_util.path_relative_to_root(self.directory)
+    setup_util.replace_environ(config=profile, 
+              command='export TROOT=$FWROOT%s && export IROOT=%s' %
+              (test_rel_dir, self.install_root))
+
     return self.setup_module.stop(out, err)
   ############################################################
   # End stop
@@ -472,8 +488,11 @@ class FrameworkTest:
   # curl the URL and check for it's return status. 
   # For each url, a flag will be set on this object for whether
   # or not it passed
+  # Returns True if all verifications succeeded
   ############################################################
   def verify_urls(self, out, err):
+    result = True
+
     # JSON
     if self.runTests[self.JSON]:
       out.write(textwrap.dedent("""
@@ -493,6 +512,7 @@ class FrameworkTest:
       else:
         self.json_url_passed = False
         out.write("\nFAIL" + ret_tuple[1] + "\n\n")
+        result = False
       out.flush()
 
     # DB
@@ -525,6 +545,7 @@ class FrameworkTest:
         out.write("\n\n")
       else:
         out.write("\nFAIL" + validate_ret_tuple[1])
+        result = False
       out.flush()
 
     # Query
@@ -590,6 +611,7 @@ class FrameworkTest:
         out.write("\n\n")
       else:
         out.write("\nFAIL " + ret_tuple[1] + "\n\n")
+        result = False
       out.flush()
 
     # Fortune
@@ -610,6 +632,7 @@ class FrameworkTest:
       else:
         self.fortune_url_passed = False
         out.write("\nFAIL\n\n")
+        result = False
       out.flush()
 
     # Update
@@ -631,6 +654,7 @@ class FrameworkTest:
       else:
         self.update_url_passed = False
         out.write("\nFAIL " + ret_tuple[1] + "\n\n")
+        result = False
       out.flush()
 
     # plaintext
@@ -652,8 +676,10 @@ class FrameworkTest:
       else:
         self.plaintext_url_passed = False
         out.write("\nFAIL\n\n" + ret_tuple[1] + "\n\n")
+        result = False
       out.flush()
 
+    return result
   ############################################################
   # End verify_urls
   ############################################################

toolset/run-ci.py

@@ -0,0 +1,281 @@
+#!/usr/bin/env python
+
+import subprocess
+import os
+import sys
+from benchmark import framework_test
+import glob
+import json
+import traceback
+import re
+import logging
+log = logging.getLogger('run-ci')
+import time
+import threading
+
+# Needed for various imports
+sys.path.append('.')
+sys.path.append('toolset/setup/linux')
+sys.path.append('toolset/benchmark')
+
+class CIRunnner:
+  '''
+  Manages running TFB on the Travis Continuous Integration system. 
+  Makes a best effort to avoid wasting time and resources by running 
+  useless jobs. 
+  
+  Only verifies the first test in each directory 
+  '''
+  
+  def __init__(self, mode, test_directory):
+    '''
+    mode = [prereq|install|test] for what we want TFB to do
+    dir  = directory we are running
+    '''
+
+    logging.basicConfig(level=logging.INFO)
+
+    try:
+      self.commit_range = os.environ['TRAVIS_COMMIT_RANGE']
+    except KeyError:
+      log.warning("Run-ci.py should only be used for automated integration tests")
+      last_commit = subprocess.check_output("git rev-parse HEAD^", shell=True).rstrip('\n')
+      self.commit_range = "master...%s" % last_commit
+    
+    if not test_directory == 'jobcleaner':
+      tests = self.gather_tests()
+      
+      # Only run the first test in this directory
+      self.test = [t for t in tests if t.directory == test_directory][0]
+      self.name = self.test.name
+
+    self.mode = mode
+    self.travis = Travis()
+
+  def _should_run(self):
+    ''' 
+    Decides if the current framework test should be tested or if we can cancel it.
+    Examines git commits included in the latest push to see if any files relevant to 
+    this framework were changed. 
+    This is a rather primitive strategy for things like pull requests, where
+    we probably want to examine the entire branch of commits. Also, this cannot handle 
+    history re-writing very well, so avoid rebasing onto any published history
+    '''
+
+    # Look for changes to core TFB framework code
+    find_tool_changes = "git diff --name-only %s | grep toolset | wc -l" % self.commit_range
+    changes = subprocess.check_output(find_tool_changes, shell=True)  
+    if int(changes) != 0:
+      log.info("Found changes to core framework code")
+      return True
+  
+    # Look for changes relevant to this test
+    find_test_changes = "git diff --name-only %s | grep %s | wc -l" % (self.commit_range, self.test.directory)
+    changes = subprocess.check_output(find_test_changes, shell=True)
+    if int(changes) == 0:
+      log.info("No changes found for %s", self.name)
+      return False
+
+    log.info("Changes found for %s", self.name)
+    return True
+
+  def run(self):
+    ''' Do the requested command using TFB  '''
+    if not self._should_run():
+      log.info("Not running %s", self.name)
+      
+      # Cancel ourselves
+      self.travis.cancel(self.travis.jobid)
+      return 0
+
+    log.info("Running %s for %s", self.mode, self.name)
+
+    # Use coverage so we can send code coverate to coveralls.io
+    command = "coverage run --source toolset,%s --parallel-mode " % self.test.directory
+    
+    command = command + 'toolset/run-tests.py '
+    if mode == 'prereq':
+      command = command + "--install server --test ''"
+    elif mode == 'install':
+      # Just a note that having an install-only mode would integrate nicely with 
+      # Travis-CI's line-folding
+      log.warning('Currently there is no install-only mode available')
+      return 1
+    elif mode == 'test':
+      command = command + "--install server --mode verify --test %s" % self.name
+    else:
+      log.critical('Unknown mode passed')
+      return 1
+    
+    # Run the command
+    log.info("Running %s", command)
+    try:
+      p = subprocess.Popen(command, shell=True)
+      p.wait()
+      return p.returncode  
+    except subprocess.CalledProcessError:
+      log.critical("Subprocess Error")
+      print traceback.format_exc()
+      return 1
+    except Exception as err:
+      log.critical("Subprocess Error")
+      log.error(err.child_traceback)
+      return 1
+
+  def gather_tests(self):
+    ''' Returns all available tests as FrameworkTest list '''
+
+    # Fake benchmarker fields that are used
+    class bench_shim():
+      def __init__(self):
+        self.type = 'all'
+        self.fwroot = os.getcwd()
+        self.install_strategy='pertest'
+
+    # Gather all tests
+    tests = []
+    for config_file_name in glob.glob('*/benchmark_config'):
+      with open(config_file_name, 'r') as config_file:
+        config = json.load(config_file)
+        test = framework_test.parse_config(config, os.path.dirname(config_file_name), bench_shim())
+        tests = tests + test
+    tests.sort(key=lambda x: x.name)
+    return tests
+
+  def cancel_unneeded_jobs(self):
+    log.info("I am jobcleaner")
+
+    # Look for changes to core TFB framework code
+    find_tool_changes = "git diff --name-only %s | grep toolset | wc -l" % self.commit_range
+    changes = subprocess.check_output(find_tool_changes, shell=True)  
+    if int(changes) != 0:
+      log.info("Found changes to core framework code. Running all tests")
+      self.travis.cancel(self.travis.jobid) # Cancel ourselves
+      return 0
+    
+    build = self.travis.build_details()
+    log.info("Build details:\n%s", build)
+    def parse_job_id(directory):
+      for line in build.split('\n'):
+        if "TESTDIR=%s" % directory in line: 
+          job = re.findall("\d+.\d+", line)[0]
+          return job
+    
+    # Build a list of modified directories
+    changes = subprocess.check_output("git diff --name-only %s" % self.commit_range, shell=True)
+    dirchanges = []
+    for line in changes.split('\n'):
+      dirchanges.append(line[0:line.find('/')])
+
+    # For each test, launch a Thread to cancel it's job if 
+    # it's directory has not been modified
+    cancelled_testdirs = []
+    threads = []
+    for test in self.gather_tests():
+      if test.directory not in dirchanges:
+        job = parse_job_id(test.directory)
+        log.info("No changes found for %s (job=%s) (dir=%s)", test.name, job, test.directory)
+        if job and test.directory not in cancelled_testdirs:
+          cancelled_testdirs.append(test.directory)
+          t = threading.Thread(target=self.travis.cancel, args=(job,),
+            name="%s (%s)" % (job, test.name))
+          t.start()
+          threads.append(t)
+
+    # Wait for all threads
+    for t in threads:
+      t.join()
+
+    # Cancel ourselves
+    self.travis.cancel(self.travis.jobid)
+
+
+class Travis():
+  '''Integrates the travis-ci build environment and the travis command line'''
+  def __init__(self):     
+    self.token = os.environ['GH_TOKEN']
+    self.jobid = os.environ['TRAVIS_JOB_NUMBER']
+    self.buildid = os.environ['TRAVIS_BUILD_NUMBER']
+    self._login()
+
+  def _login(self):
+    subprocess.check_call("travis login --skip-version-check --no-interactive --github-token %s" % self.token, shell=True)
+    log.info("Logged into travis") # NEVER PRINT OUTPUT, GH_TOKEN MIGHT BE REVEALED
+
+  def cancel(self, job):
+    # Ignore errors in case job is already cancelled
+    try:
+      subprocess.check_call("travis cancel %s --skip-version-check --no-interactive" % job, shell=True)
+      log.info("Thread %s: Canceled job %s", threading.current_thread().name, job)
+    except subprocess.CalledProcessError:
+      log.exception("Error halting job %s. Report:", job)
+      subprocess.call("travis report --skip-version-check --no-interactive --org", shell=True)
+      log.error("Trying to halt %s one more time", job)
+      subprocess.call("travis cancel %s --skip-version-check --no-interactive" % job, shell=True)
+
+  def build_details(self):
+    build = subprocess.check_output("travis show %s --skip-version-check" % self.buildid, shell=True)
+    return build
+
+if __name__ == "__main__":
+  args = sys.argv[1:]
+
+  if len(args) != 2 or not (args[0] == "prereq" or args[0] == "install" or args[0] == "test"):
+    print "Usage: toolset/run-ci.py [prereq|install|test] test-name"
+    sys.exit(1)
+
+  mode = args[0]
+  testdir = args[1]
+
+  runner = CIRunnner(mode, testdir)
+  
+  # Watch for the special test name indicating we are 
+  # the test in charge of cancelling unnecessary jobs
+  if testdir == "jobcleaner":
+    try:
+      log.info("Sleeping to ensure Travis-CI has queued all jobs")
+      time.sleep(20)
+      runner.cancel_unneeded_jobs()
+    except KeyError as ke: 
+      log.warning("Environment key missing, are you running inside Travis-CI?")
+    except:
+      log.critical("Unknown error")
+      print traceback.format_exc()
+    finally: 
+      sys.exit(0)
+  
+  retcode = 0
+  try:
+    retcode = runner.run()
+  except KeyError as ke: 
+    log.warning("Environment key missing, are you running inside Travis-CI?")
+  except:
+    log.critical("Unknown error")
+    print traceback.format_exc()
+  finally:
+    log.error("Running inside travis, so I will print err and out to console")
+    log.error("Here is ERR:")
+    
+    try:
+      with open("results/ec2/latest/logs/%s/err.txt" % runner.test.name, 'r') as err:
+        for line in err:
+          log.info(line)
+    except IOError:
+      if mode == "test":
+        log.error("No ERR file found")
+
+    log.error("Here is OUT:")
+    try:
+      with open("results/ec2/latest/logs/%s/out.txt" % runner.test.name, 'r') as out:
+        for line in out:
+          log.info(line)
+    except IOError:
+      if mode == "test":
+        log.error("No OUT file found")
+
+    sys.exit(retcode)
+
+
+
+
+

toolset/run-tests.py

@@ -158,7 +158,7 @@ def main(argv=None):
     elif benchmarker.parse != None:
       benchmarker.parse_timestamp()
     else:
-      benchmarker.run()
+      return benchmarker.run()
 
 if __name__ == "__main__":
     sys.exit(main())

toolset/setup/linux/bash_functions.sh

@@ -59,6 +59,8 @@ fw_depends() {
     trap 'fw_traperror $depend $? $LINENO "$BASH_COMMAND" $(printf ":%s" ${FUNCNAME[@]}) $(printf ":%s" ${BASH_SOURCE[@]}) $(printf ":%s" ${BASH_LINENO[@]})'  ERR
     retcode=0
 
+    # Ensure we are inside the installer root for this framework
+    cd $IROOT
     wd=$(pwd)
     relative_wd=\$FWROOT${wd#$FWROOT}
 
@@ -96,6 +98,9 @@ fw_depends() {
   set +E
   trap - ERR
 
+  # Politely return to IROOT for later install.sh code
+  cd $IROOT  
+
   return $FW_any_errors
 }
 

toolset/setup/linux/frameworks/nawak.sh

@@ -3,6 +3,6 @@
 RETCODE=$(fw_exists nawak)
 [ ! "$RETCODE" == 0 ] || { return 0; }
 
-git clone git://github.com/idlewan/nawak.git nawak/nawak
-cd nawak/nawak
+git clone git://github.com/idlewan/nawak.git nawak
+cd nawak
 git checkout 5e56d718ff327c58cbdca14d44abc327f752681d

toolset/setup/linux/installer.py

@@ -75,7 +75,11 @@ class Installer:
             if not os.path.exists(profile):
               logging.warning("Framework %s does not have a bash_profile"%test_name)
               profile="$FWROOT/config/benchmark_profile"
-            setup_util.replace_environ(config=profile)
+            else:
+              logging.info("Loading environment from %s", profile)
+            setup_util.replace_environ(config=profile, 
+              command='export TROOT=$FWROOT%s && export IROOT=$FWROOT%s' %
+              (test_rel_dir, test_rel_install_dir))
 
             # Find relative installation file
             test_rel_install_file = "$FWROOT%s" % setup_util.path_relative_to_root(test_install_file)

toolset/setup/linux/languages/pypy.sh

@@ -10,4 +10,4 @@ ln -sf pypy-2.3.1-linux64 pypy
 fw_get https://bootstrap.pypa.io/get-pip.py
 pypy/bin/pypy get-pip.py
 
-pypy/bin/pip install -r ../config/requirements-pypy.txt
+pypy/bin/pip install -r $FWROOT/config/requirements-pypy.txt

toolset/setup/linux/languages/python2.sh

@@ -14,4 +14,4 @@ make install
 cd ..
 fw_get https://bootstrap.pypa.io/get-pip.py -O get-pip2.py
 py2/bin/python get-pip2.py
-py2/bin/pip install -r ../config/requirements.txt
+py2/bin/pip install -r $FWROOT/config/requirements.txt

toolset/setup/linux/languages/python3.sh

@@ -12,4 +12,4 @@ make -j4
 make install
 
 cd ..
-py3/bin/pip3 install -r ../config/requirements-py3.txt
+py3/bin/pip3 install -r $FWROOT/config/requirements-py3.txt

toolset/setup/linux/languages/yaf.sh

@@ -3,12 +3,12 @@
 RETCODE=$(fw_exists /usr/local/lib/php/extensions/no-debug-non-zts-20100525/yaf.so)
 [ ! "$RETCODE" == 0 ] || { \
   echo "Enabling yaf in PHP configuration...";
-  sudo cp ../config/php.ini /usr/local/lib/php.ini
-  sudo cp ../config/php-fpm.conf /usr/local/lib/php-fpm.conf
+  sudo cp $FWROOT/config/php.ini /usr/local/lib/php.ini
+  sudo cp $FWROOT/config/php-fpm.conf /usr/local/lib/php-fpm.conf
   return 0; }
 
 fw_depends php
 sudo pecl install -f yaf
-sudo cp ../config/php.ini /usr/local/lib/php.ini
-sudo cp ../config/php-fpm.conf /usr/local/lib/php-fpm.conf
+sudo cp $FWROOT/config/php.ini /usr/local/lib/php.ini
+sudo cp $FWROOT/config/php-fpm.conf /usr/local/lib/php-fpm.conf
 

toolset/setup/linux/webservers/resin.sh

@@ -14,7 +14,7 @@ make
 make install
 
 mv conf/resin.properties conf/resin.properties.orig
-cat ../../config/resin.properties > conf/resin.properties
+cat $FWROOT/config/resin.properties > conf/resin.properties
 
 mv conf/resin.xml conf/resin.xml.orig
-cat ../../config/resin.xml > conf/resin.xml
+cat $FWROOT/config/resin.xml > conf/resin.xml

tornado/bash_profile.sh

@@ -0,0 +1,16 @@
+
+export PY2_ROOT=$IROOT/py2
+export PY2=$PY2_ROOT/bin/python
+export PY2_PIP=$PY2_ROOT/bin/pip
+export PY2_GUNICORN=$PY2_ROOT/bin/gunicorn
+
+export PYPY_ROOT=$IROOT/pypy
+export PYPY=$PYPY_ROOT/bin/python
+export PYPY_PIP=$PYPY_ROOT/bin/pip
+export PYPY_GUNICORN=$PYPY_ROOT/bin/gunicorn
+
+export PY3_ROOT=$IROOT/py3
+export PY3=$PY3_ROOT/bin/python
+export PY3_PIP=$PY3_ROOT/bin/pip
+export PY3_GUNICORN=$PY3_ROOT/bin/gunicorn
+

tornado/setup.py

@@ -5,14 +5,9 @@ import sys
 import time
 
 
-python = expanduser('~/FrameworkBenchmarks/installs/py2/bin/python')
-cwd = expanduser('~/FrameworkBenchmarks/tornado')
-
-
 def start(args, logfile, errfile):
-    subprocess.Popen(
-        python + " server.py --port=8080 --mongo=%s --logging=error" % (args.database_host,),
-        shell=True, cwd=cwd, stderr=errfile, stdout=logfile)
+    subprocess.Popen("$PY2 server.py --port=8080 --mongo=%s --logging=error" % (args.database_host,),
+        shell=True, cwd='tornado', stderr=errfile, stdout=logfile)
     return 0
 
 def stop(logfile, errfile):

tornado/setup_pg.py

@@ -4,16 +4,9 @@ import sys
 import time
 
 
-bin_dir = os.path.expanduser('~/FrameworkBenchmarks/installs/py2/bin')
-python = os.path.expanduser(os.path.join(bin_dir, 'python'))
-pip = os.path.expanduser(os.path.join(bin_dir, 'pip'))
-cwd = os.path.expanduser('~/FrameworkBenchmarks/tornado')
-
-
 def start(args, logfile, errfile):
-    subprocess.Popen(
-        python + ' server.py --port=8080 --postgres=%s --logging=error' % (args.database_host,),
-        shell=True, cwd=cwd, stderr=errfile, stdout=logfile)
+    subprocess.Popen('$PY2 server.py --port=8080 --postgres=%s --logging=error' % (args.database_host,),
+        shell=True, cwd='tornado', stderr=errfile, stdout=logfile)
     return 0
 
 

tornado/setup_py3.py

@@ -5,14 +5,9 @@ import sys
 import time
 
 
-python = expanduser('~/FrameworkBenchmarks/installs/py3/bin/python3')
-cwd = expanduser('~/FrameworkBenchmarks/tornado')
-
-
 def start(args, logfile, errfile):
-    subprocess.Popen(
-        python + " server.py --port=8080 --mongo=%s --logging=error" % (args.database_host,),
-        shell=True, cwd=cwd, stderr=errfile, stdout=logfile)
+    subprocess.Popen("$PY3 server.py --port=8080 --mongo=%s --logging=error" % (args.database_host,),
+        shell=True, cwd='tornado', stderr=errfile, stdout=logfile)
     return 0
 
 def stop(logfile, errfile):

wicket/bash_profile.sh

@@ -0,0 +1 @@
+export RESIN_HOME=${IROOT}/resin-4.0.36