Merge branch 'master' into listargs-1123

Conflicts:
	toolset/benchmark/framework_test.py

.travis.yml

@@ -24,6 +24,7 @@ env:
     #   find . -type d -depth 2 | sed 's|./|    - "TESTDIR=|' | sed 's/$/"/g'
     #
     #  
+    - "TESTDIR=C/lwan"
     - "TESTDIR=C/duda"
     - "TESTDIR=C/onion"
     - "TESTDIR=C#/aspnet"

frameworks/C/lwan/README.md

@@ -0,0 +1,35 @@
+# Lwan
+
+This is the configuration files to benchmark the [Lwan](http://lwan.ws)
+web server/framework.
+
+## Requirements
+
+GCC, SQLite 3.7, Linux 3.0, optionally jemalloc or tcmalloc.
+
+## Tests available
+
+### 1. JSON serialization
+
+URL: /json
+
+### 2. Single database query
+
+URL: /db
+
+### 3. Multiple database queries
+
+URL: /queries?queries=N
+
+### 4. Fortunes
+
+URL: /fortunes
+
+### 6. Plaintext
+
+URL: /plaintext
+
+
+## Contact
+
+Leandro Pereira <[email protected]>

frameworks/C/lwan/bash_profile.sh

@@ -0,0 +1,4 @@
+#!/bin/bash
+
+export LWAN_ROOT=${IROOT}/lwan
+export LWAN_BUILD=${LWAN_ROOT}/build

frameworks/C/lwan/benchmark_config

@@ -0,0 +1,27 @@
+{
+  "framework": "lwan",
+  "tests": [{
+    "raw": {
+      "setup_file": "setup",
+      "db_url": "/db",
+      "query_url": "/queries?queries=",
+      "fortune_url": "/fortunes",
+      "plaintext_url": "/plaintext",
+      "json_url": "/json",
+      "port": 8080,
+      "approach": "Realistic",
+      "classification": "Platform",
+      "database": "SQLite",
+      "framework": "lwan",
+      "language": "C",
+      "orm": "Raw",
+      "platform": "Lwan",
+      "webserver": "Lwan",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "Lwan",
+      "notes": "",
+      "versus": ""
+    }
+  }]
+}

frameworks/C/lwan/install.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+REV='aa6c15fbdf63d9db722ddc72bd736b23381331be'
+
+INSTALLED_FILE="${IROOT}/lwan-${REV}.installed"
+RETCODE=$(fw_exists ${INSTALLED_FILE})
+[ ! "$RETCODE" == 0 ] || { return 0; }
+
+# Lwan is only built during installation as a dependency sanity check.
+sudo apt-get update && \
+	sudo apt-get install libjemalloc-dev && \
+	git clone git://github.com/lpereira/lwan.git && \
+        cd lwan && \
+        git checkout ${REV} && \
+        mkdir build && \
+        cd build && \
+        cmake .. -DCMAKE_BUILD_TYPE=Release && \
+        make techempower && \
+        touch ${INSTALLED_FILE}

frameworks/C/lwan/setup.py

@@ -0,0 +1,23 @@
+import subprocess
+import sys
+import os
+
+def start(args, logfile, errfile):
+  subprocess.call('rm -rf ${LWAN_BUILD}', shell=True, stderr=errfile, stdout=logfile)
+  subprocess.call('mkdir -p ${LWAN_BUILD}', shell=True, stderr=errfile, stdout=logfile)
+  subprocess.call('cmake ${LWAN_ROOT} -DCMAKE_BUILD_TYPE=Release && make techempower',
+      cwd=os.environ['LWAN_BUILD'], shell=True, stderr=errfile, stdout=logfile)
+
+  db_dir = os.path.join(os.environ['LWAN_ROOT'], 'techempower')
+  exe_path = os.path.join(os.environ['LWAN_BUILD'], 'techempower', 'techempower')
+  subprocess.Popen(exe_path, cwd=db_dir, stderr=errfile, stdout=logfile, shell=True)
+
+  return 0
+
+def stop(logfile, errfile):
+  p = subprocess.Popen(['pgrep', 'techempower'], stdout=subprocess.PIPE)
+  out, err = p.communicate()
+  for line in out.splitlines():
+      pid = int(line)
+      os.kill(pid, 2)
+  return 0

frameworks/Dart/dart-redstone/.gitignore

@@ -1 +1,2 @@
 packages/*
+.pub

frameworks/Dart/dart-redstone/README.md

@@ -4,15 +4,15 @@ This test adds [Redstone.dart](http://redstonedart.org), a microframework for Da
 
 ## Versions
 
-* [Dart SDK version >=1.3.0](https://launchpad.net/~hachre/+archive/dart)
+* [Dart SDK version >=1.7.0](http://www.dartlang.org/)
 * [Dart args version 0.10.0+2](http://pub.dartlang.org/packages/args)
 * [Dart crypto version 0.9.0](http://pub.dartlang.org/packages/crypto)
 * [Dart mustache version 0.1.8](http://pub.dartlang.org/packages/mustache)
-* [Dart mongo_dart version 0.1.42](http://pub.dartlang.org/packages/mongo_dart)
+* [Dart mongo_dart version 0.1.44](http://pub.dartlang.org/packages/mongo_dart)
 * [Dart postgresql version 0.2.14](http://pub.dartlang.org/packages/postgresql)
-* [Dart redstone version 0.5.11](http://pub.dartlang.org/packages/redstone)
-* [Dart yaml version 0.9.0](http://pub.dartlang.org/packages/yaml)
-* [Dart redstone_mapper version 0.1.4](http://pub.dartlang.org/packages/redstone_mapper)
+* [Dart redstone version 0.5.18](http://pub.dartlang.org/packages/redstone)
+* [Dart yaml version 2.0.1+1](http://pub.dartlang.org/packages/yaml)
+* [Dart redstone_mapper version 0.1.9](http://pub.dartlang.org/packages/redstone_mapper)
 * [Dart redstone_mapper_mongo version 0.1.1+1](http://pub.dartlang.org/packages/redstone_mapper_mongo)
 * [Dart redstone_mapper_pg version 0.1.1](http://pub.dartlang.org/packages/redstone_mapper_pg)
 

frameworks/Dart/dart-redstone/benchmark_config

@@ -17,7 +17,7 @@
       "language": "Dart",
       "orm": "Micro",
       "platform": "Dart",
-      "webserver": "nginx",
+      "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "redstone-postgresql",
@@ -38,7 +38,7 @@
       "language": "Dart",
       "orm": "Micro",
       "platform": "Dart",
-      "webserver": "nginx",
+      "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "redstone-mongodb",

frameworks/Dart/dart-redstone/install.sh

@@ -1,3 +1,3 @@
 #!/bin/bash
 
-fw_depends dart nginx
+fw_depends dart

frameworks/Dart/dart-redstone/postgresql.yaml

@@ -1,4 +1,4 @@
-host: tfbdata
+host: localhost
 port: 5432
 user: benchmarkdbuser
 password: benchmarkdbpass

frameworks/Dart/dart-redstone/pubspec.yaml

@@ -1,15 +1,15 @@
 name: DartRedstoneBenchmark
 description: A benchmark of Redstone.dart, a server-side microframework for Dart
 environment:
-  sdk: ">=1.3.0 <2.0.0"
+  sdk: ">=1.7.0 <2.0.0"
 dependencies:
   args: 0.10.0+2
   crypto: 0.9.0
-  mongo_dart: 0.1.42
+  mongo_dart: 0.1.44
   mustache: 0.1.8
   postgresql: 0.2.14
-  redstone: 0.5.11
-  redstone_mapper: 0.1.4
+  redstone: 0.5.18
+  redstone_mapper: 0.1.9
   redstone_mapper_mongo: 0.1.1+1
   redstone_mapper_pg: 0.1.1
-  yaml: 0.9.0
+  yaml: 2.0.1+1

frameworks/Dart/dart-redstone/server.dart

@@ -1,5 +1,6 @@
 import "dart:core";
 import "dart:io";
+import "dart:isolate";
 import 'dart:async' show Future;
 import 'dart:math' show Random;
 import "package:redstone/server.dart" as app;
@@ -210,10 +211,33 @@ main(List<String> args) {
   parser.addOption('address', abbr: 'a', defaultsTo: '0.0.0.0');
   parser.addOption('port', abbr: 'p', defaultsTo: '8080');
   parser.addOption('dbconnections', abbr: 'd', defaultsTo: '256');
+  parser.addOption('isolates', abbr: 'i', defaultsTo: '1');
   
   var arguments = parser.parse(args);
-  var dbConnections = int.parse(arguments["dbconnections"]);
+  var isolates = int.parse(arguments['isolates']);
+  var dbConnections = int.parse(arguments['dbconnections']) ~/ isolates;
+
+  ServerSocket.bind(arguments['address'], int.parse(arguments['port']))
+      .then((server) {
+        var ref = server.reference;
+        for (int i = 1; i < isolates; i++) {
+          Isolate.spawn(startInIsolate, [ref, dbConnections]);
+        }
+        _startServer(server, dbConnections);
+      });
   
+}
+
+void startInIsolate(args) {
+  var ref = args[0];
+  var dbConnections = args[1];
+  ref.create().then((server) {
+    _startServer(server, dbConnections);
+  });
+}
+
+_startServer(serverSocket, dbConnections) {
+
   MongoDbManager mongoDbManager;
   PostgreSqlManager pgSqlManager;
   mustache.Template fortunesTemplate;
@@ -255,9 +279,11 @@ main(List<String> args) {
     app.addPlugin(getMapperPlugin());
     
     //start the server
-    app.start(address: arguments["address"], port: int.parse(arguments["port"]));
+    var server = new HttpServer.listenOn(serverSocket);
+    app.serveRequests(server);
     
   });
+
 }
 
 _parseQueriesParam(param) {

frameworks/Dart/dart-redstone/setup.py

@@ -3,7 +3,6 @@ import sys
 import setup_util
 import os
 
-
 def start(args, logfile, errfile):
   setup_util.replace_text('dart-redstone/postgresql.yaml', 'host: .*', 'host: ' + args.database_host)
   setup_util.replace_text('dart-redstone/mongodb.yaml', 'host: .*', 'host: ' + args.database_host)
@@ -15,55 +14,12 @@ def start(args, logfile, errfile):
     #
     # start dart servers
     #
-    for port in range(9001, 9001 + args.max_threads):
-      subprocess.Popen('dart server.dart -a 127.0.0.1 -p ' + str(port) + ' -d ' + str(args.max_concurrency / args.max_threads), shell=True, cwd='dart-redstone', stderr=errfile, stdout=logfile)
-    #
-    # create nginx configuration
-    #
-    conf = []
-    conf.append('worker_processes ' + str(args.max_threads) + ';')
-    conf.append('error_log /dev/null error;')
-    conf.append('events {')
-    conf.append('    worker_connections 1024;')
-    conf.append('}')
-    conf.append('http {')
-    conf.append('    access_log off;')
-    conf.append('    include /usr/local/nginx/conf/mime.types;')
-    conf.append('    default_type application/octet-stream;')
-    conf.append('    sendfile on;')
-    conf.append('    upstream dart_cluster {')
-    for port in range(9001, 9001 + args.max_threads):
-      conf.append('        server 127.0.0.1:' + str(port) + ';')
-    conf.append('        keepalive ' + str(args.max_concurrency / args.max_threads) + ';')
-    conf.append('    }')
-    conf.append('    server {')
-    conf.append('        listen 8080;')
-    conf.append('        location / {')
-    conf.append('            proxy_pass http://dart_cluster;')
-    conf.append('            proxy_http_version 1.1;')
-    conf.append('            proxy_set_header Connection "";')
-    conf.append('        }')
-    conf.append('    }')
-    conf.append('}')
-    #
-    # write nginx configuration to disk
-    #
-    with open('dart-redstone/nginx.conf', 'w') as f:
-      f.write('\n'.join(conf))
-    #
-    # start nginx
-    #
-    subprocess.Popen('sudo /usr/local/nginx/sbin/nginx -c $TROOT/nginx.conf', shell=True, cwd='dart-redstone', stderr=errfile, stdout=logfile);
+    subprocess.Popen('dart server.dart -a 0.0.0.0 -p 8080 -d ' + str(args.max_concurrency) + ' -i ' + str(args.max_threads), shell=True, cwd='dart-redstone', stderr=errfile, stdout=logfile)
     return 0
   except subprocess.CalledProcessError:
     return 1
 
 def stop(logfile, errfile):
-  #
-  # stop nginx
-  #
-  subprocess.check_call('sudo /usr/local/nginx/sbin/nginx -c $TROOT/nginx.conf -s stop', shell=True, cwd='dart-redstone', stderr=errfile, stdout=logfile)
-  os.remove('dart-redstone/nginx.conf')
   #
   # stop dart servers
   #

frameworks/Dart/dart-redstone/source_code

@@ -0,0 +1,5 @@
+./dart-redstone/fortunes.mustache
+./dart-redstone/server.dart
+./dart-redstone/postgresql.yaml
+./dart-redstone/mongodb.yaml
+./dart-redstone/pubspec.yaml

frameworks/Elixir/WeberFramework/benchmark_config

@@ -18,7 +18,6 @@
       "database_os": "Linux",
       "display_name": "weber",
       "notes": "",
-      "versus": "",
-      "skip": "false"
+      "versus": ""
   }}]
 }

frameworks/Go/revel-jet/benchmark_config

@@ -20,8 +20,7 @@
       "database_os": "Linux",
       "display_name": "revel-jet",
       "notes": "",
-      "versus": "go",
-      "skip": "true"
+      "versus": "go"
     }
   }]
 }

frameworks/Perl/plack/setup.py

@@ -15,11 +15,29 @@ def start(args, logfile, errfile):
     return 0
   except subprocess.CalledProcessError:
     return 1
-def stop(logfile, errfile):
-  try:
-    subprocess.call('kill -TERM $(cat $TROOT/app.pid)', shell=True, stderr=errfile, stdout=logfile)
-    subprocess.call("sudo /usr/local/nginx/sbin/nginx -c $TROOT/nginx.conf -s stop", shell=True, stderr=errfile, stdout=logfile)
-    return 0
-  except subprocess.CalledProcessError:
-    return 1
 
+def stop(logfile, errfile):
+   try:
+     subprocess.Popen("kill -TERM $(ps --ppid `cat app.pid` -o pid --no-header)", shell=True, cwd="plack", stderr=errfile, stdout=logfile)
+     # TE - There was an issue on the EC2 machines where this, for reasons unknown,
+     # was not sufficient in cleanly ending plack. In fact, the above would not 
+     # successfully kill the starter process which would result in this 'stop' call
+     # to report success but leave port 8080 bound to a plackup instance. We tried
+     # adding a 'nuke' approach which detects the test's port still being bound
+     # after calling stop and then directly kills those pids a few times to try and
+     # cleanly release any/all ports (over 6000).
+     # Why this only happens on EC2 is just a guess, but somehow the server seems
+     # overwhelmed by the sheer volume of requests from the client and gets into
+     # a deadlock state. Calling "kill -15 [pid]" against the server process does
+     # nothing; so we needed a general way to kill all the processes that were 
+     # spawned by the original process. For plack, this was as simple as the next
+     # subprocess.Popen call (killall -s 9 plackup), but to do this generally is
+     # a bit more difficult.
+ 
+     # TE - In general, no test should ever be forced to use the KILL sigterm;
+     # TERM should be sufficient. However, in this case it seems that the plack
+     # server gets into a deadlock state and will not respond to a TERM sigterm.
+     subprocess.Popen("killall -s 9 plackup")
+     return 0
+   except subprocess.CalledProcessError:
+     return 1

toolset/benchmark/benchmarker.py

@@ -2,6 +2,7 @@ from setup.linux.installer import Installer
 from setup.linux import setup_util
 
 from benchmark import framework_test
+from benchmark.test_types import *
 from utils import header
 from utils import gather_tests
 from utils import gather_frameworks
@@ -17,6 +18,7 @@ import sys
 import logging
 import socket
 import threading
+from pprint import pprint
 
 from multiprocessing import Process
 
@@ -325,7 +327,7 @@ class Benchmarker:
     # off, rather than starting from the beginning
     if os.path.isfile('current_benchmark.txt'):
         with open('current_benchmark.txt', 'r') as interrupted_benchmark:
-            interrupt_bench = interrupted_benchmark.read()
+            interrupt_bench = interrupted_benchmark.read().strip()
             for index, atest in enumerate(tests):
                 if atest.name == interrupt_bench:
                     tests = tests[index:]
@@ -495,15 +497,8 @@ class Benchmarker:
       pass
     with open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'out.txt'), 'w') as out, \
          open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'err.txt'), 'w') as err:
-      if hasattr(test, 'skip'):
-        if test.skip.lower() == "true":
-          out.write("Test {name} benchmark_config specifies to skip this test. Skipping.\n".format(name=test.name))
-          return exit_with_code(0)
 
       if test.os.lower() != self.os.lower() or test.database_os.lower() != self.database_os.lower():
-        # the operating system requirements of this test for the
-        # application server or the database server don't match
-        # our current environment
         out.write("OS or Database OS specified in benchmark_config does not match the current environment. Skipping.\n")
         return exit_with_code(0)
       
@@ -511,11 +506,6 @@ class Benchmarker:
       if self.exclude != None and test.name in self.exclude:
         out.write("Test {name} has been added to the excludes list. Skipping.\n".format(name=test.name))
         return exit_with_code(0)
-      
-      # If the test does not contain an implementation of the current test-type, skip it
-      if self.type != 'all' and not test.contains_type(self.type):
-        out.write("Test type {type} does not contain an implementation of the current test-type. Skipping.\n".format(type=self.type))
-        return exit_with_code(0)
 
       out.write("test.os.lower() = {os}  test.database_os.lower() = {dbos}\n".format(os=test.os.lower(),dbos=test.database_os.lower()))
       out.write("self.results['frameworks'] != None: {val}\n".format(val=str(self.results['frameworks'] != None)))
@@ -561,11 +551,13 @@ class Benchmarker:
           self.__write_intermediate_results(test.name,"<setup.py>#start() returned non-zero")
           return exit_with_code(1)
         
+        logging.info("Sleeping %s seconds to ensure framework is ready" % self.sleep)
         time.sleep(self.sleep)
 
         ##########################
         # Verify URLs
         ##########################
+        logging.info("Verifying framework URLs")
         passed_verify = test.verify_urls(out, err)
         out.flush()
         err.flush()
@@ -574,6 +566,7 @@ class Benchmarker:
         # Benchmark this test
         ##########################
         if self.mode == "benchmark":
+          logging.info("Benchmarking")
           out.write(header("Benchmarking %s" % test.name))
           out.flush()
           test.benchmark(out, err)
@@ -591,9 +584,15 @@ class Benchmarker:
         time.sleep(5)
 
         if self.__is_port_bound(test.port):
-          self.__write_intermediate_results(test.name, "port " + str(test.port) + " was not released by stop")
-          err.write(header("Error: Port %s was not released by stop %s" % (test.port, test.name)))
+          err.write("Port %s was not freed. Attempting to free it." % (test.port, ))
           err.flush()
+          self.__forciblyEndPortBoundProcesses(test.port, out, err)
+          time.sleep(5)
+          if self.__is_port_bound(test.port):
+            err.write(header("Error: Port %s was not released by stop %s" % (test.port, test.name)))
+            err.flush()
+            self.__write_intermediate_results(test.name, "port " + str(test.port) + " was not released by stop")
+
           return exit_with_code(1)
 
         out.write(header("Stopped %s" % test.name))
@@ -680,6 +679,44 @@ class Benchmarker:
   # End __is_port_bound
   ############################################################
 
+  def __forciblyEndPortBoundProcesses(self, test_port, out, err):
+    p = subprocess.Popen(['sudo', 'netstat', '-lnp'], stdout=subprocess.PIPE)
+    out, err = p.communicate()
+    for line in out.splitlines():
+      if 'tcp' in line:
+        splitline = line.split()
+        port = splitline[3].split(':')
+        port = int(port[len(port) - 1].strip())
+        if port > 6000:
+          err.write(textwrap.dedent(
+        """
+        A port that shouldn't be open is open. See the following line for netstat output.
+        {splitline}
+        """.format(splitline=splitline)))
+          err.flush()
+        if port == test_port:
+          try:
+            pid = splitline[6].split('/')[0].strip()
+            ps = subprocess.Popen(['ps','p',pid], stdout=subprocess.PIPE)
+            # Store some info about this process
+            proc = ps.communicate()
+            os.kill(int(pid), 15)
+            # Sleep for 10 sec; kill can be finicky
+            time.sleep(10)
+            # Check that PID again
+            ps = subprocess.Popen(['ps','p',pid], stdout=subprocess.PIPE)
+            dead = ps.communicate()
+            if dead in proc:
+              os.kill(int(pid), 9)
+          except OSError:
+            out.write( textwrap.dedent("""
+              -----------------------------------------------------
+                Error: Could not kill pid {pid}
+              -----------------------------------------------------
+              """.format(pid=str(pid))) )
+            # This is okay; likely we killed a parent that ended
+            # up automatically killing this before we could.
+
   ############################################################
   # __parse_results
   # Ensures that the system has all necessary software to run
@@ -712,22 +749,29 @@ class Benchmarker:
     
     jsonResult = {}
     for framework, testlist in frameworks.iteritems():
+      if not os.path.exists(os.path.join(testlist[0].directory, "source_code")):
+        logging.warn("Cannot count lines of code for %s - no 'source_code' file", framework)
+        continue
+
       # Unfortunately the source_code files use lines like
       # ./cpoll_cppsp/www/fortune_old instead of 
       # ./www/fortune_old
       # so we have to back our working dir up one level
       wd = os.path.dirname(testlist[0].directory)
-
+      
       try:
         command = "cloc --list-file=%s/source_code --yaml" % testlist[0].directory
         # Find the last instance of the word 'code' in the yaml output. This should
         # be the line count for the sum of all listed files or just the line count
         # for the last file in the case where there's only one file listed.
         command = command + "| grep code | tail -1 | cut -d: -f 2"
+        logging.debug("Running \"%s\" (cwd=%s)", command, wd)
         lineCount = subprocess.check_output(command, cwd=wd, shell=True)
         jsonResult[framework] = int(lineCount)
       except subprocess.CalledProcessError:
         continue
+      except ValueError as ve:
+        logging.warn("Unable to get linecount for %s due to error '%s'", framework, ve)
     self.results['rawData']['slocCounts'] = jsonResult
   ############################################################
   # End __count_sloc
@@ -845,7 +889,25 @@ class Benchmarker:
   ############################################################
   def __init__(self, args):
     
+    # Map type strings to their objects
+    types = dict()
+    types['json'] = JsonTestType()
+    types['db'] = DBTestType()
+    types['query'] = QueryTestType()
+    types['fortune'] = FortuneTestType()
+    types['update'] = UpdateTestType()
+    types['plaintext'] = PlaintextTestType()
+
+    # Turn type into a map instead of a string
+    if args['type'] == 'all':
+        args['types'] = types
+    else:
+        args['types'] = { args['type'] : types[args['type']] }
+    del args['type']
+
     self.__dict__.update(args)
+    # pprint(self.__dict__)
+
     self.start_time = time.time()
     self.run_test_timeout_seconds = 3600
 

toolset/benchmark/test_types/__init__.py

@@ -0,0 +1,8 @@
+
+from framework_test_type import *
+from json_type import JsonTestType
+from plaintext_type import PlaintextTestType
+from db_type import DBTestType
+from query_type import QueryTestType
+from update_type import UpdateTestType
+from fortune_type import FortuneTestType

toolset/benchmark/test_types/db_type.py

@@ -0,0 +1,103 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+
+import json
+
+class DBTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['db_url']
+    FrameworkTestType.__init__(self, name='db', 
+      accept_header=self.accept_json,
+      requires_db=True, args=args)
+
+  def get_url(self):
+    return self.db_url
+
+  def verify(self, base_url):
+    '''Ensures body is valid JSON with a key 'id' and a key 
+    'randomNumber', both of which must map to integers
+    '''
+
+    url = base_url + self.db_url
+    body = self._curl(url)
+    
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+
+    problems = []
+
+    # We are allowing the single-object array 
+    # e.g. [{'id':5, 'randomNumber':10}] for now, 
+    # but will likely make this fail at some point
+    if type(response) == list:
+      response = response[0]
+      problems.append( ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url) ) 
+
+      # Make sure there was a JSON object inside the array
+      if type(response) != dict:
+        problems.append( ('fail', 'Response is not a JSON object or an array of JSON objects', url) ) 
+        return problems
+
+    problems += self._verifyObject(response, url)
+
+    if len(problems) == 0:
+      return [('pass','',url)]
+    else:
+      return problems
+
+  def _verifyObject(self, db_object, url):
+    '''Ensure the passed item is a JSON object with 
+    keys 'id' and 'randomNumber' mapping to ints. 
+    Separate method allows the QueryTestType to 
+    reuse these checks'''
+
+    problems = []
+
+    if type(db_object) != dict:
+      got = str(db_object)[:20]
+      if len(str(db_object)) > 20:
+        got = str(db_object)[:17] + '...'
+      return ('fail', "Expected a JSON object, got '%s' instead" % got, url)
+
+    # Make keys case insensitive
+    db_object = {k.lower(): v for k,v in db_object.iteritems()}
+
+    if "id" not in db_object:
+      problems.append( ('fail', "Response has no 'id' key", url) ) 
+    if "randomnumber" not in db_object:
+      problems.append( ('fail', "Response has no 'randomNumber' key", url) ) 
+
+    try:
+      float(db_object["id"])
+    except ValueError as ve:
+      problems.append( ('fail', "Response key 'id' does not map to a number - %s" % ve, url) ) 
+
+    try:
+      float(db_object["randomnumber"])
+    except ValueError as ve:
+      problems.append( ('fail', "Response key 'randomNumber' does not map to a number - %s" % ve, url) ) 
+
+    if type(db_object["id"]) != int:
+      problems.append( ('warn', '''Response key 'id' contains extra quotations or decimal points.
+        This may negatively affect performance during benchmarking''', url) ) 
+
+    # Tests based on the value of the numbers
+    try:
+      response_id = float(db_object["id"])
+      response_rn = float(db_object["randomnumber"])
+
+      if response_id > 10000 or response_id < 1:
+        problems.append( ('warn', "Response key 'id' should be between 1 and 10,000", url)) 
+    except ValueError:
+      pass
+
+    return problems
+

toolset/benchmark/test_types/fortune_type.py

@@ -0,0 +1,35 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.fortune_html_parser import FortuneHTMLParser
+
+class FortuneTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['fortune_url']
+    FrameworkTestType.__init__(self, name='fortune', 
+      requires_db=True, 
+      accept_header=self.accept_html, 
+      args=args)
+
+  def get_url(self):
+    return self.fortune_url
+
+  def verify(self, base_url):
+    '''Parses the given HTML string and asks the 
+    FortuneHTMLParser whether the parsed string is a 
+    valid fortune response
+    '''
+    url = base_url + self.fortune_url
+    body = self._curl(url)
+    
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+
+    parser = FortuneHTMLParser()
+    parser.feed(body)
+    if parser.isValidFortune(self.out):
+      return [('pass','',url)]
+    else:
+      return [('fail','Invalid according to FortuneHTMLParser',url)]
+

toolset/benchmark/test_types/framework_test_type.py

@@ -0,0 +1,110 @@
+import copy
+import sys
+import subprocess
+from subprocess import PIPE
+
+from pprint import pprint
+
+class FrameworkTestType:
+  '''Interface between a test type (json, query, plaintext, etc) and 
+  the rest of TFB. A test type defines a number of keys it expects
+  to find in the benchmark_config, and this base class handles extracting
+  those keys and injecting them into the test. For example, if 
+  benchmark_config contains a line `"spam" : "foobar"` and a subclasses X
+  passes an argument list of ['spam'], then after parsing there will 
+  exist a member `X.spam = 'foobar'`. 
+  '''
+
+  accept_json = "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
+  accept_html = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+  accept_plaintext = "Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
+
+  def __init__(self, name, requires_db = False, accept_header = None, args = []):
+    self.name = name
+    self.requires_db = requires_db
+    self.args = args
+    self.out = sys.stdout
+    self.err = sys.stderr
+    self.accept_header = accept_header
+    if accept_header is None:
+      self.accept_header = self.accept_plaintext
+
+    self.passed = None
+    self.failed = None
+    self.warned = None
+
+  def setup_out_err(self, out, err):
+    '''Sets up file-like objects for logging. Used in 
+    cases where it is hard just return the output. Any
+    output sent to these file objects is also printed to 
+    the console
+
+    NOTE: I detest this. It would be much better to use
+    logging like it's intended
+    '''
+    self.out = out
+    self.err = err
+  
+  def parse(self, test_keys):
+    '''Takes the dict of key/value pairs describing a FrameworkTest 
+    and collects all variables needed by this FrameworkTestType
+
+    Raises AttributeError if required keys are missing
+    '''
+    if all (arg in test_keys for arg in self.args):
+      self.__dict__.update({ arg:test_keys[arg] for arg in self.args})
+      return self
+    else: # This is quite common - most tests don't support all types
+      raise AttributeError("A %s requires the benchmark_config to contain %s"%(self.name,self.args))
+
+  def _curl(self, url):
+    '''Downloads a URL and returns the HTTP body'''
+    # Use -m 15 to make curl stop trying after 15sec.
+    # Use -i to output response with headers
+    # Don't use -f so that the HTTP response code is ignored.
+    # Use -sS to hide progress bar, but show errors.
+    print "Accessing URL %s"
+    p = subprocess.Popen(["curl", "-m", "15", "-i", "-sS", url], stderr=PIPE, stdout=PIPE)
+    (out, err) = p.communicate()
+    self.err.write(err+'\n')
+    self.out.write(out+'\n')
+    if p.returncode != 0:
+      return None
+    # Get response body
+    p = subprocess.Popen(["curl", "-m", "15", "-s", url], stdout=PIPE, stderr=PIPE)
+    (out, err) = p.communicate()
+    self.err.write(err+'\n')
+    self.out.write(out+'\n')
+    return out
+  
+  def verify(self, base_url):
+    '''Accesses URL used by this test type and checks the return 
+    values for correctness. Most test types run multiple checks,
+    so this returns a list of results. Each result is a 3-tuple
+    of (String result, String reason, String urlTested).
+
+    - result : 'pass','warn','fail'
+    - reason : Short human-readable reason if result was 
+        warn or fail. Please do not print the response as part of this, 
+        other parts of TFB will do that based upon the current logging 
+        settings if this method indicates a failure happened
+    - urlTested: The exact URL that was queried
+
+    Subclasses should make a best-effort attempt to report as many
+    failures and warnings as they can to help users avoid needing 
+    to run TFB repeatedly while debugging
+    '''
+    # TODO make String result into an enum to enforce
+    raise NotImplementedError("Subclasses must provide verify")
+
+  def get_url(self):
+    '''Returns the URL for this test, like '/json'''
+    # This is a method because each test type uses a different key
+    # for their URL so the base class can't know which arg is the URL
+    raise NotImplementedError("Subclasses must provide verify")
+
+  def copy(self):
+    '''Returns a copy that can be safely modified. Use before calling 
+    parse'''
+    return copy.copy(self)
+

toolset/benchmark/test_types/json_type.py

@@ -0,0 +1,46 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+
+import json
+
+class JsonTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['json_url']
+    FrameworkTestType.__init__(self, name='json', requires_db=False, accept_header=self.accept_json, args=args)
+
+  def get_url(self):
+    return self.json_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON object of 
+    { 'message' : 'hello, world!' }. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.json_url
+    body = self._curl(url)
+    
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+  
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+    
+    # Make everything case insensitive
+    response = {k.lower(): v.lower() for k,v in response.iteritems()}
+
+    if "message" not in response:
+      return [('fail',"No JSON key 'message'", url)]
+
+    if len(response) != 1:
+      return [('warn',"Too many JSON key/value pairs, expected 1", url)]
+
+    if response['message'] != 'hello, world!':
+      return [('fail',"Expected message of 'hello, world!', got '%s'"%response['message'], url)]
+
+    return [('pass','',url)]

toolset/benchmark/test_types/plaintext_type.py

@@ -0,0 +1,32 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+
+class PlaintextTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['plaintext_url']
+    FrameworkTestType.__init__(self, name='plaintext', requires_db=False, accept_header=self.accept_plaintext, args=args)
+
+  def verify(self, base_url):
+    url = base_url + self.plaintext_url
+    body = self._curl(url)
+
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+
+    # Case insensitive
+    orig = body
+    body = body.lower()
+
+    if "hello, world!" not in body:
+      return [('fail', "Could not find 'Hello, World!' in response", url)]
+
+    if len("hello, world!") < len(body):
+      return [('warn', '''Server is returning more data than is required.
+        This may negatively affect benchmark performance''', url)]
+
+    return [('pass', '', url)]
+
+  def get_url(self):
+    return self.plaintext_url

toolset/benchmark/test_types/query_type.py

@@ -0,0 +1,96 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.db_type import DBTestType
+
+import json
+
+class QueryTestType(DBTestType):
+  def __init__(self):
+    args = ['query_url']
+    FrameworkTestType.__init__(self, name='query', requires_db=True, 
+      accept_header=self.accept_json, args=args)
+
+  def get_url(self):
+    return self.query_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON array of 
+    the proper length, each JSON Object in the array 
+    has keys 'id' and 'randomNumber', and these keys 
+    map to integers. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.query_url
+
+    problems = []
+    
+    body = self._curl(url + '2')
+    problems += self._verifyQueryList(2, body, url + '2')
+
+    body = self._curl(url + '0')
+    problems += self._verifyQueryList(1, body, url + '0', 'warn')
+
+    body = self._curl(url + 'foo')
+    problems += self._verifyQueryList(1, body, url + 'foo')
+
+    body = self._curl(url + '501')
+    problems += self._verifyQueryList(500, body, url + '501')
+
+    if len(problems) == 0:
+      return [('pass','',url + '2'),
+              ('pass','',url + '0'),
+              ('pass','',url + 'foo'),
+              ('pass','',url + '501')]
+    else:
+      return problems
+
+  def _verifyQueryList(self, expectedLength, body, url, incorrect_length_response='fail'):
+    '''Validates high-level structure (array length, object 
+      types, etc) before calling into DBTestType to 
+      validate a few individual JSON objects'''
+
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+  
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+
+    problems = []
+
+    if type(response) != list:
+      problems.append(('warn','Top-level JSON is an object, not an array', url))
+
+      # Verify the one object they gave us before returning
+      problems += self._verifyObject(response, url)
+
+      return problems
+
+    if any(type(item) != dict for item in response):
+      problems.append(('fail','All items JSON array must be JSON objects', url))
+
+    # For some edge cases we only warn
+    if len(response) != expectedLength:
+      problems.append((incorrect_length_response,
+        "JSON array length of %s != expected length of %s" % (len(response), expectedLength), 
+        url))
+
+    # verify individual objects
+    maxBadObjects = 5
+    for item in response:
+      obj_ok = self._verifyObject(item, url)
+      if len(obj_ok) > 0:
+        maxBadObjects -=  1
+        problems += obj_ok
+        if maxBadObjects == 0:
+          break
+
+    return problems
+
+
+

toolset/benchmark/test_types/update_type.py

@@ -0,0 +1,46 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.query_type import QueryTestType
+
+from pprint import pprint
+
+class UpdateTestType(QueryTestType):
+  def __init__(self):
+    args = ['update_url']
+    FrameworkTestType.__init__(self, name='update', requires_db=True, 
+      accept_header=self.accept_json, args=args)
+
+  def get_url(self):
+    return self.update_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON array of 
+    the proper length, each JSON Object in the array 
+    has keys 'id' and 'randomNumber', and these keys 
+    map to integers. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.update_url
+    problems = []
+    
+    body = self._curl(url + '2')
+    problems += self._verifyQueryList(2, body, url + '2')
+
+    body = self._curl(url + '0')
+    problems += self._verifyQueryList(1, body, url + '0', 'warn')
+
+    body = self._curl(url + 'foo')
+    problems += self._verifyQueryList(1, body, url + 'foo', 'warn')
+
+    body = self._curl(url + '501')
+    problems += self._verifyQueryList(500, body, url + '501', 'warn')
+
+    if len(problems) == 0:
+      return [('pass','',url + '2'),
+              ('pass','',url + '0'),
+              ('pass','',url + 'foo'),
+              ('pass','',url + '501')]
+    else:
+      return problems
+
+

toolset/benchmark/utils.py

@@ -115,4 +115,4 @@ def header(message, top='-', bottom='-'):
         result = "%s" % bottomheader
       else:
         result += "\n%s" % bottomheader
-    return result
+    return result + '\n'

toolset/run-ci.py

@@ -33,6 +33,8 @@ class CIRunnner:
   
   Only verifies the first test in each directory 
   '''
+
+  SUPPORTED_DATABASES = "mysql postgres mongodb cassandra sqlite none".split()
   
   def __init__(self, mode, testdir=None):
     '''
@@ -163,14 +165,39 @@ class CIRunnner:
 
       if not is_PR:
         log.debug('I am not testing a pull request')
-        # If more than one commit was pushed, examine everything including 
-        # all details on all merges
-        self.commit_range = "-m %s" % os.environ['TRAVIS_COMMIT_RANGE']
+        # Three main scenarios to consider
+        #  - 1 One non-merge commit pushed to master
+        #  - 2 One merge commit pushed to master (e.g. a PR was merged). 
+        #      This is an example of merging a topic branch
+        #  - 3 Multiple commits pushed to master
+        # 
+        #  1 and 2 are actually handled the same way, by showing the 
+        #  changes being brought into to master when that one commit 
+        #  was merged. Fairly simple, `git log -1 COMMIT`. To handle 
+        #  the potential merge of a topic branch you also include 
+        #  `--first-parent -m`. 
+        #
+        #  3 needs to be handled by comparing all merge children for 
+        #  the entire commit range. The best solution here would *not* 
+        #  use --first-parent because there is no guarantee that it 
+        #  reflects changes brought into master. Unfortunately we have
+        #  no good method inside Travis-CI to easily differentiate 
+        #  scenario 1/2 from scenario 3, so I cannot handle them all 
+        #  separately. 1/2 are the most common cases, 3 with a range 
+        #  of non-merge commits is the next most common, and 3 with 
+        #  a range including merge commits is the least common, so I 
+        #  am choosing to make our Travis-CI setup potential not work 
+        #  properly on the least common case by always using 
+        #  --first-parent 
         
-        # If only one commit was pushed, examine that one. If it was a 
-        # merge be sure to show all details
+        # Handle 3
+        # Note: Also handles 2 because Travis-CI sets COMMIT_RANGE for 
+        # merged PR commits
+        self.commit_range = "--first-parent -m %s" % os.environ['TRAVIS_COMMIT_RANGE']
+
+        # Handle 1
         if self.commit_range == "":
-          self.commit_range = "-m -1 %s" % os.environ['TRAVIS_COMMIT']
+          self.commit_range = "--first-parent -m -1 %s" % os.environ['TRAVIS_COMMIT']
 
     except KeyError:
       log.warning("I should only be used for automated integration tests e.g. Travis-CI")
@@ -192,13 +219,10 @@ class CIRunnner:
                   and (t.database_os.lower() == "linux" or t.database_os.lower() == "none")]
     
     # Our Travis-CI only has some databases supported
-    validtests = [t for t in osvalidtests if t.database.lower() == "mysql"
-                  or t.database.lower() == "postgres"
-                  or t.database.lower() == "mongodb"
-                  or t.database.lower() == "cassandra"
-                  or t.database.lower() == "none"]
-    log.info("Found %s usable tests (%s valid for linux, %s valid for linux and {mysql,postgres,mongodb,cassandra,none}) in directory '%s'", 
-      len(dirtests), len(osvalidtests), len(validtests), '$FWROOT/frameworks/' + testdir)
+    validtests = [t for t in osvalidtests if t.database.lower() in self.SUPPORTED_DATABASES]
+    supported_databases = ','.join(self.SUPPORTED_DATABASES)
+    log.info("Found %s usable tests (%s valid for linux, %s valid for linux and {%s}) in directory '%s'",
+      len(dirtests), len(osvalidtests), len(validtests), supported_databases, '$FWROOT/frameworks/' + testdir)
     if len(validtests) == 0:
       log.critical("Found no test that is possible to run in Travis-CI! Aborting!")
       if len(osvalidtests) != 0:
@@ -324,9 +348,11 @@ class CIRunnner:
     until timeout 15s sudo apt-key adv --keyserver pgp.mit.edu --recv 4BD736A82B5C1B00; do echo 'Waiting for apt-key' ; done
     sudo apt-add-repository  'deb http://www.apache.org/dist/cassandra/debian 20x main'
 
-    # Run installation
+    # Run installation 
+    # DO NOT COPY --force-yes TO ANY NON-TRAVIS-CI SCRIPTS! Seriously, it can cause some 
+    # major damage and should only be used inside a VM or Linux Container
     sudo apt-get -q update
-    sudo apt-get -q -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
+    sudo apt-get -q -y --force-yes install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
       mongodb-org \
       cassandra \
       openssh-server

toolset/run-tests.py

@@ -180,6 +180,8 @@ def main(argv=None):
         print 'Configuration options: '
         pprint(vars(args))
 
+
+
     benchmarker = Benchmarker(vars(args))
 
     # Run the benchmarker in the specified mode

toolset/setup/linux/prerequisites.sh

@@ -37,8 +37,10 @@ sudo apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::=
   libgstreamer-plugins-base0.10-0 libgstreamer0.10-0 \
   liborc-0.4-0 libwxbase2.8-0 libwxgtk2.8-0 libgnutls-dev \
   libjson0-dev libmcrypt-dev libicu-dev gettext \
-  libpq-dev mlton cloc dstat
+  libpq-dev mlton \
+  cloc dstat                        `# Collect resource usage statistics`
 
+# Install gcc-4.8
 sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
 sudo apt-get -y update
 sudo apt-get install -y gcc-4.8 g++-4.8