7 years ago · b8ce59f271
--- a/benchmark.cfg.example
+++ b/benchmark.cfg.example
@@ -1,11 +1,11 @@
 
															 [Defaults]
														
 
															 # Available Keys:
														
 
															 os=linux
														
 
															-server_host=TFB-server
														
 
															-client_host=TFB-client
														
 
															+server_host=127.0.0.1
														
 
															+client_host=127.0.0.1
														
 
															 client_identity_file=/home/techempower/.ssh/id_rsa
														
 
															 client_user=techempower
														
 
															-database_host=TFB-database
														
 
															+database_host=127.0.0.1
														
 
															 database_identity_file=/home/techempower/.ssh/id_rsa
														
 
															 database_os=linux
														
 
															 database_user=techempower
														
--- a/deployment/vagrant/bootstrap.sh
+++ b/deployment/vagrant/bootstrap.sh
@@ -22,11 +22,6 @@ if [ ! -e "~/.firstboot" ]; then
 
															   chmod 600 ~/.ssh/config
														
 
															   echo "vagrant ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers
														
 
															-  # Setting up hosts aliases
														
 
															-  echo 127.0.0.1 TFB-database | sudo tee --append /etc/hosts
														
 
															-  echo 127.0.0.1 TFB-client   | sudo tee --append /etc/hosts
														
 
															-  echo 127.0.0.1 TFB-server   | sudo tee --append /etc/hosts
														
 
															-
														
 
															   # Setting up new FWROOT
														
 
															   export FWROOT="/home/vagrant/FrameworkBenchmarks"
														
 
															   echo `export FWROOT="/home/vagrant/FrameworkBenchmarks"` >> ~/.bashrc
														
--- a/toolset/benchmark/benchmarker.py
+++ b/toolset/benchmark/benchmarker.py
--- a/toolset/benchmark/fortune_html_parser.py
+++ b/toolset/benchmark/fortune_html_parser.py
@@ -1,12 +1,12 @@
 
															 # -*- coding: utf-8
														
 
															-import re
														
 
															 from HTMLParser import HTMLParser
														
 
															 from difflib import unified_diff
														
 
															+
														
 
															 class FortuneHTMLParser(HTMLParser):
														
 
															-  body = []
														
 
															+    body = []
														
 
															-  valid_fortune = '''<!doctype html><html>
														
 
															+    valid_fortune = '''<!doctype html><html>
														
 
															 <head><title>Fortunes</title></head>
														
 
															 <body><table>
														
 
															 <tr><th>id</th><th>message</th></tr>
														
@@ -25,140 +25,160 @@ class FortuneHTMLParser(HTMLParser):
 
															 <tr><td>12</td><td>フレームワークのベンチマーク</td></tr>
														
 
															 </table></body></html>'''
														
 
															-  # Is called when a doctype or other such tag is read in.
														
 
															-  # For our purposes, we assume this is only going to be
														
 
															-  # "DOCTYPE html", so we will surround it with "<!" and ">".
														
 
															-  def handle_decl(self, decl):
														
 
															-    # The spec says that for HTML this is case insensitive,
														
 
															-    # and since we did not specify xml compliance (where
														
 
															-    # incorrect casing would throw a syntax error), we must
														
 
															-    # allow all casings. We will lower for our normalization.
														
 
															-    self.body.append("<!{d}>".format(d=decl.lower()))
														
 
															+    def handle_decl(self, decl):
														
 
															+        '''
														
 
															+        Is called when a doctype or other such tag is read in.
														
 
															+        For our purposes, we assume this is only going to be
														
 
															+        "DOCTYPE html", so we will surround it with "<!" and ">".
														
 
															+        '''
														
 
															+        # The spec says that for HTML this is case insensitive,
														
 
															+        # and since we did not specify xml compliance (where
														
 
															+        # incorrect casing would throw a syntax error), we must
														
 
															+        # allow all casings. We will lower for our normalization.
														
 
															+        self.body.append("<!{d}>".format(d=decl.lower()))
														
 
															-  # This is called when an HTML character is parsed (i.e.
														
 
															-  # &quot;). There are a number of issues to be resolved
														
 
															-  # here. For instance, some tests choose to leave the
														
 
															-  # "+" character as-is, which should be fine as far as
														
 
															-  # character escaping goes, but others choose to use the
														
 
															-  # character reference of "&#43;", which is also fine.
														
 
															-  # Therefore, this method looks for all possible character
														
 
															-  # references and normalizes them so that we can
														
 
															-  # validate the input against a single valid spec string.
														
 
															-  # Another example problem: "&quot;" is valid, but so is
														
 
															-  # "&#34;"
														
 
															-  def handle_charref(self, name):
														
 
															-    val = name.lower()
														
 
															-    # "&#34;" is a valid escaping, but we are normalizing
														
 
															-    # it so that our final parse can just be checked for
														
 
															-    # equality.
														
 
															-    if val == "34" or val == "034" or val == "x22":
														
 
															-      # Append our normalized entity reference to our body.
														
 
															-      self.body.append("&quot;")
														
 
															-    # "&#39;" is a valid escaping of "-", but it is not
														
 
															-    # required, so we normalize for equality checking.
														
 
															-    if val == "39" or val == "039" or val == "x27":
														
 
															-      self.body.append("&apos;")
														
 
															-    # Again, "&#43;" is a valid escaping of the "+", but
														
 
															-    # it is not required, so we need to normalize for out
														
 
															-    # final parse and equality check.
														
 
															-    if val == "43" or val == "043" or val == "x2b":
														
 
															-      self.body.append("+")
														
 
															-    # Again, "&#62;" is a valid escaping of ">", but we
														
 
															-    # need to normalize to "&gt;" for equality checking.
														
 
															-    if val == "62" or val == "062" or val == "x3e":
														
 
															-      self.body.append("&gt;")
														
 
															-    # Again, "&#60;" is a valid escaping of "<", but we
														
 
															-    # need to normalize to "&lt;" for equality checking.
														
 
															-    if val == "60" or val == "060" or val == "x3c":
														
 
															-      self.body.append("&lt;")
														
 
															-    # Not sure why some are escaping '/'
														
 
															-    if val == "47" or val == "047" or val == "x2f":
														
 
															-      self.body.append("/")
														
 
															-    # "&#40;" is a valid escaping of "(", but
														
 
															-    # it is not required, so we need to normalize for out
														
 
															-    # final parse and equality check.
														
 
															-    if val == "40" or val == "040" or val == "x28":
														
 
															-      self.body.append("(")
														
 
															-    # "&#41;" is a valid escaping of ")", but
														
 
															-    # it is not required, so we need to normalize for out
														
 
															-    # final parse and equality check.
														
 
															-    if val == "41" or val == "041" or val == "x29":
														
 
															-      self.body.append(")")
														
 
															+    def handle_charref(self, name):
														
 
															+        '''
														
 
															+        This is called when an HTML character is parsed (i.e.
														
 
															+        &quot;). There are a number of issues to be resolved
														
 
															+        here. For instance, some tests choose to leave the
														
 
															+        "+" character as-is, which should be fine as far as
														
 
															+        character escaping goes, but others choose to use the
														
 
															+        character reference of "&#43;", which is also fine.
														
 
															+        Therefore, this method looks for all possible character
														
 
															+        references and normalizes them so that we can
														
 
															+        validate the input against a single valid spec string.
														
 
															+        Another example problem: "&quot;" is valid, but so is
														
 
															+        "&#34;"
														
 
															+        '''
														
 
															+        val = name.lower()
														
 
															+        # "&#34;" is a valid escaping, but we are normalizing
														
 
															+        # it so that our final parse can just be checked for
														
 
															+        # equality.
														
 
															+        if val == "34" or val == "034" or val == "x22":
														
 
															+            # Append our normalized entity reference to our body.
														
 
															+            self.body.append("&quot;")
														
 
															+        # "&#39;" is a valid escaping of "-", but it is not
														
 
															+        # required, so we normalize for equality checking.
														
 
															+        if val == "39" or val == "039" or val == "x27":
														
 
															+            self.body.append("&apos;")
														
 
															+        # Again, "&#43;" is a valid escaping of the "+", but
														
 
															+        # it is not required, so we need to normalize for out
														
 
															+        # final parse and equality check.
														
 
															+        if val == "43" or val == "043" or val == "x2b":
														
 
															+            self.body.append("+")
														
 
															+        # Again, "&#62;" is a valid escaping of ">", but we
														
 
															+        # need to normalize to "&gt;" for equality checking.
														
 
															+        if val == "62" or val == "062" or val == "x3e":
														
 
															+            self.body.append("&gt;")
														
 
															+        # Again, "&#60;" is a valid escaping of "<", but we
														
 
															+        # need to normalize to "&lt;" for equality checking.
														
 
															+        if val == "60" or val == "060" or val == "x3c":
														
 
															+            self.body.append("&lt;")
														
 
															+        # Not sure why some are escaping '/'
														
 
															+        if val == "47" or val == "047" or val == "x2f":
														
 
															+            self.body.append("/")
														
 
															+        # "&#40;" is a valid escaping of "(", but
														
 
															+        # it is not required, so we need to normalize for out
														
 
															+        # final parse and equality check.
														
 
															+        if val == "40" or val == "040" or val == "x28":
														
 
															+            self.body.append("(")
														
 
															+        # "&#41;" is a valid escaping of ")", but
														
 
															+        # it is not required, so we need to normalize for out
														
 
															+        # final parse and equality check.
														
 
															+        if val == "41" or val == "041" or val == "x29":
														
 
															+            self.body.append(")")
														
 
															-  def handle_entityref(self, name):
														
 
															-    # Again, "&mdash;" is a valid escaping of "—", but we
														
 
															-    # need to normalize to "—" for equality checking.
														
 
															-    if name == "mdash":
														
 
															-      self.body.append("—")
														
 
															-    else:
														
 
															-      self.body.append("&{n};".format(n=name))
														
 
															+    def handle_entityref(self, name):
														
 
															+        '''
														
 
															+        Again, "&mdash;" is a valid escaping of "—", but we
														
 
															+        need to normalize to "—" for equality checking.
														
 
															+        '''
														
 
															+        if name == "mdash":
														
 
															+            self.body.append("—")
														
 
															+        else:
														
 
															+            self.body.append("&{n};".format(n=name))
														
 
															-  # This is called every time a tag is opened. We append
														
 
															-  # each one wrapped in "<" and ">".
														
 
															-  def handle_starttag(self, tag, attrs):
														
 
															-    self.body.append("<{t}>".format(t=tag))
														
 
															+    def handle_starttag(self, tag, attrs):
														
 
															+        '''
														
 
															+        This is called every time a tag is opened. We append
														
 
															+        each one wrapped in "<" and ">".
														
 
															+        '''
														
 
															+        self.body.append("<{t}>".format(t=tag))
														
 
															-    # Append a newline after the <table> and <html>
														
 
															-    if tag.lower() == 'table' or tag.lower() == 'html':
														
 
															-      self.body.append("\n")
														
 
															+        # Append a newline after the <table> and <html>
														
 
															+        if tag.lower() == 'table' or tag.lower() == 'html':
														
 
															+            self.body.append("\n")
														
 
															-  # This is called whenever data is presented inside of a
														
 
															-  # start and end tag. Generally, this will only ever be
														
 
															-  # the contents inside of "<td>" and "</td>", but there
														
 
															-  # are also the "<title>" and "</title>" tags.
														
 
															-  def handle_data (self, data):
														
 
															-    if data.strip() != '':
														
 
															-      # After a LOT of debate, these are now considered
														
 
															-      # valid in data. The reason for this approach is
														
 
															-      # because a few tests use tools which determine
														
 
															-      # at compile time whether or not a string needs
														
 
															-      # a given type of html escaping, and our fortune
														
 
															-      # test has apostrophes and quotes in html data
														
 
															-      # rather than as an html attribute etc.
														
 
															-      # example:
														
 
															-      # <td>A computer scientist is someone who fixes things that aren't broken.</td>
														
 
															-      # Semanticly, that apostrophe does not NEED to
														
 
															-      # be escaped. The same is currently true for our
														
 
															-      # quotes.
														
 
															-      # In fact, in data (read: between two html tags)
														
 
															-      # even the '>' need not be replaced as long as
														
 
															-      # the '<' are all escaped.
														
 
															-      # We replace them with their escapings here in
														
 
															-      # order to have a noramlized string for equality
														
 
															-      # comparison at the end.
														
 
															-      data = data.replace('\'', '&apos;')
														
 
															-      data = data.replace('"', '&quot;')
														
 
															-      data = data.replace('>', '&gt;')
														
 
															+    def handle_data(self, data):
														
 
															+        '''
														
 
															+        This is called whenever data is presented inside of a
														
 
															+        start and end tag. Generally, this will only ever be
														
 
															+        the contents inside of "<td>" and "</td>", but there
														
 
															+        are also the "<title>" and "</title>" tags.
														
 
															+        '''
														
 
															+        if data.strip() != '':
														
 
															+            # After a LOT of debate, these are now considered
														
 
															+            # valid in data. The reason for this approach is
														
 
															+            # because a few tests use tools which determine
														
 
															+            # at compile time whether or not a string needs
														
 
															+            # a given type of html escaping, and our fortune
														
 
															+            # test has apostrophes and quotes in html data
														
 
															+            # rather than as an html attribute etc.
														
 
															+            # example:
														
 
															+            # <td>A computer scientist is someone who fixes things that aren't broken.</td>
														
 
															+            # Semanticly, that apostrophe does not NEED to
														
 
															+            # be escaped. The same is currently true for our
														
 
															+            # quotes.
														
 
															+            # In fact, in data (read: between two html tags)
														
 
															+            # even the '>' need not be replaced as long as
														
 
															+            # the '<' are all escaped.
														
 
															+            # We replace them with their escapings here in
														
 
															+            # order to have a noramlized string for equality
														
 
															+            # comparison at the end.
														
 
															+            data = data.replace('\'', '&apos;')
														
 
															+            data = data.replace('"', '&quot;')
														
 
															+            data = data.replace('>', '&gt;')
														
 
															-      self.body.append("{d}".format(d=data))
														
 
															+            self.body.append("{d}".format(d=data))
														
 
															-  # This is called every time a tag is closed. We append
														
 
															-  # each one wrapped in "</" and ">".
														
 
															-  def handle_endtag(self, tag):
														
 
															-    self.body.append("</{t}>".format(t=tag))
														
 
															+    def handle_endtag(self, tag):
														
 
															+        '''
														
 
															+        This is called every time a tag is closed. We append
														
 
															+        each one wrapped in "</" and ">".
														
 
															+        '''
														
 
															+        self.body.append("</{t}>".format(t=tag))
														
 
															-    # Append a newline after each </tr> and </head>
														
 
															-    if tag.lower() == 'tr' or tag.lower() == 'head':
														
 
															-      self.body.append("\n")
														
 
															+        # Append a newline after each </tr> and </head>
														
 
															+        if tag.lower() == 'tr' or tag.lower() == 'head':
														
 
															+            self.body.append("\n")
														
 
															-  # Returns whether the HTML input parsed by this parser
														
 
															-  # is valid against our known "fortune" spec.
														
 
															-  # The parsed data in 'body' is joined on empty strings
														
 
															-  # and checked for equality against our spec.
														
 
															-  def isValidFortune(self, out):
														
 
															-    body = ''.join(self.body)
														
 
															-    same = self.valid_fortune == body
														
 
															-    diff_lines = []
														
 
															-    if not same:
														
 
															-      output = "Oh no! I compared {!s}\n\n\nto.....{!s}\n".format(self.valid_fortune, body)
														
 
															-      output += "Fortune invalid. Diff following:\n"
														
 
															-      headers_left = 3
														
 
															-      for line in unified_diff(self.valid_fortune.split('\n'), body.split('\n'), fromfile='Valid', tofile='Response', n=0):
														
 
															-        diff_lines.append(line)
														
 
															-        output += line
														
 
															-        headers_left -= 1
														
 
															-        if headers_left <= 0:
														
 
															-          output += "\n"
														
 
															-      print(output)
														
 
															-      out.write(output)
														
 
															-    return (same, diff_lines)
														
 
															+    def isValidFortune(self, out):
														
 
															+        '''
														
 
															+        Returns whether the HTML input parsed by this parser
														
 
															+        is valid against our known "fortune" spec.
														
 
															+        The parsed data in 'body' is joined on empty strings
														
 
															+        and checked for equality against our spec.
														
 
															+        '''
														
 
															+        body = ''.join(self.body)
														
 
															+        same = self.valid_fortune == body
														
 
															+        diff_lines = []
														
 
															+        if not same:
														
 
															+            output = "Oh no! I compared {!s}\n\n\nto.....{!s}\n".format(
														
 
															+                self.valid_fortune, body)
														
 
															+            output += "Fortune invalid. Diff following:\n"
														
 
															+            headers_left = 3
														
 
															+            for line in unified_diff(
														
 
															+                    self.valid_fortune.split('\n'),
														
 
															+                    body.split('\n'),
														
 
															+                    fromfile='Valid',
														
 
															+                    tofile='Response',
														
 
															+                    n=0):
														
 
															+                diff_lines.append(line)
														
 
															+                output += line
														
 
															+                headers_left -= 1
														
 
															+                if headers_left <= 0:
														
 
															+                    output += "\n"
														
 
															+            print(output)
														
 
															+            out.write(output)
														
 
															+        return (same, diff_lines)
														
--- a/toolset/benchmark/framework_test.py
+++ b/toolset/benchmark/framework_test.py
@@ -1,1017 +1,191 @@
 
															-from benchmark.fortune_html_parser import FortuneHTMLParser
														
 
															-from setup.linux import setup_util
														
 
															-from benchmark.test_types import *
														
 
															-
														
 
															-import importlib
														
 
															 import os
														
 
															 import subprocess
														
 
															-import socket
														
 
															-import time
														
 
															-import re
														
 
															-from pprint import pprint
														
 
															 import sys
														
 
															 import traceback
														
 
															-import json
														
 
															 import logging
														
 
															-import csv
														
 
															-import shlex
														
 
															-import math
														
 
															-import multiprocessing
														
 
															-import docker
														
 
															-from collections import OrderedDict
														
 
															 from requests import ConnectionError
														
 
															-from threading import Thread
														
 
															-from threading import Event
														
 
															-from utils import header
														
 
															-from utils import gather_docker_dependencies
														
 
															-from utils import find_docker_file
														
 
															+from toolset.utils.output_helper import header
														
 
															+from toolset.utils import docker_helper
														
 
															 # Cross-platform colored text
														
 
															-from colorama import Fore, Back, Style
														
 
															-from datetime import datetime
														
 
															-from datetime import timedelta
														
 
															-
														
 
															-class FrameworkTest:
														
 
															-  headers_template = "-H 'Host: {server_host}' -H 'Accept: {accept}' -H 'Connection: keep-alive'"
														
 
															-
														
 
															-  # Used for test types that require no pipelining or query string params.
														
 
															-  concurrency_template = """
														
 
															-
														
 
															-    let max_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Running Primer {name}"
														
 
															-    echo " {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
														
 
															-    sleep 5
														
 
															-
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Running Warmup {name}"
														
 
															-    echo " {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}\""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}"
														
 
															-    sleep 5
														
 
															-
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Synchronizing time"
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    ntpdate -s pool.ntp.org
														
 
															-
														
 
															-    for c in {levels}
														
 
															-    do
														
 
															-      echo ""
														
 
															-      echo "---------------------------------------------------------"
														
 
															-      echo " Concurrency: $c for {name}"
														
 
															-      echo " {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t $(($c>$max_threads?$max_threads:$c)) \"http://{server_host}:{port}{url}\""
														
 
															-      echo "---------------------------------------------------------"
														
 
															-      echo ""
														
 
															-      STARTTIME=$(date +"%s")
														
 
															-      {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t "$(($c>$max_threads?$max_threads:$c))" http://{server_host}:{port}{url}
														
 
															-      echo "STARTTIME $STARTTIME"
														
 
															-      echo "ENDTIME $(date +"%s")"
														
 
															-      sleep 2
														
 
															-    done
														
 
															-  """
														
 
															-  # Used for test types that require pipelining.
														
 
															-  pipeline_template = """
														
 
															-
														
 
															-    let max_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Running Primer {name}"
														
 
															-    echo " {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
														
 
															-    sleep 5
														
 
															-
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Running Warmup {name}"
														
 
															-    echo " {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}\""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}"
														
 
															-    sleep 5
														
 
															-
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Synchronizing time"
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    ntpdate -s pool.ntp.org
														
 
															-
														
 
															-    for c in {levels}
														
 
															-    do
														
 
															-      echo ""
														
 
															-      echo "---------------------------------------------------------"
														
 
															-      echo " Concurrency: $c for {name}"
														
 
															-      echo " {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t $(($c>$max_threads?$max_threads:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
														
 
															-      echo "---------------------------------------------------------"
														
 
															-      echo ""
														
 
															-      STARTTIME=$(date +"%s")
														
 
															-      {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t "$(($c>$max_threads?$max_threads:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
														
 
															-      echo "STARTTIME $STARTTIME"
														
 
															-      echo "ENDTIME $(date +"%s")"
														
 
															-      sleep 2
														
 
															-    done
														
 
															-  """
														
 
															-  # Used for test types that require a database -
														
 
															-  # These tests run at a static concurrency level and vary the size of
														
 
															-  # the query sent with each request
														
 
															-  query_template = """
														
 
															-    let max_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Running Primer {name}"
														
 
															-    echo " wrk {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}2\""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    wrk {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}2"
														
 
															-    sleep 5
														
 
															-
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Running Warmup {name}"
														
 
															-    echo " wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}2\""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}2"
														
 
															-    sleep 5
														
 
															-
														
 
															-    echo ""
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo " Synchronizing time"
														
 
															-    echo "---------------------------------------------------------"
														
 
															-    echo ""
														
 
															-    ntpdate -s pool.ntp.org
														
 
															-
														
 
															-    for c in {levels}
														
 
															-    do
														
 
															-      echo ""
														
 
															-      echo "---------------------------------------------------------"
														
 
															-      echo " Queries: $c for {name}"
														
 
															-      echo " wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}$c\""
														
 
															-      echo "---------------------------------------------------------"
														
 
															-      echo ""
														
 
															-      STARTTIME=$(date +"%s")
														
 
															-      wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}$c"
														
 
															-      echo "STARTTIME $STARTTIME"
														
 
															-      echo "ENDTIME $(date +"%s")"
														
 
															-      sleep 2
														
 
															-    done
														
 
															-  """
														
 
															-
														
 
															-  ############################################################
														
 
															-  # start(benchmarker)
														
 
															-  # Start the test using its setup file
														
 
															-  ############################################################
														
 
															-  def start(self, out):
														
 
															-
														
 
															-    # Setup environment variables
														
 
															-    logDir = os.path.join(self.fwroot, self.benchmarker.full_results_directory(), 'logs', self.name.lower())
														
 
															-
														
 
															-    def tee_output(prefix, line):
														
 
															-      # Needs to be one atomic write
														
 
															-      # Explicitly use UTF-8 as it's the most common framework output
														
 
															-      # TODO improve encoding handling
														
 
															-      line = prefix.encode('utf-8') + line
														
 
															-
														
 
															-      # Log to current terminal
														
 
															-      sys.stdout.write(line)
														
 
															-      sys.stdout.flush()
														
 
															-
														
 
															-      out.write(line)
														
 
															-      out.flush()
														
 
															-
														
 
															-    prefix = "Setup %s: " % self.name
														
 
															-
														
 
															-    ###########################
														
 
															-    # Build the Docker images
														
 
															-    ##########################
														
 
															-
														
 
															-    # Build the test docker file based on the test name
														
 
															-    # then build any additional docker files specified in the benchmark_config
														
 
															-    # Note - If you want to be able to stream the output of the build process you have
														
 
															-    # to use the low level API:
														
 
															-    #  https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build
														
 
															-
														
 
															-    prev_line = os.linesep
														
 
															-    def handle_build_output(line):
														
 
															-      if line.startswith('{"stream":'):
														
 
															-        line = json.loads(line)
														
 
															-        line = line[line.keys()[0]].encode('utf-8')
														
 
															-        if prev_line.endswith(os.linesep):
														
 
															-          tee_output(prefix, line)
														
 
															-        else:
														
 
															-          tee_output(line)
														
 
															-        self.prev_line = line
														
 
															-
														
 
															-    docker_buildargs = { 'CPU_COUNT': str(multiprocessing.cpu_count()),
														
 
															-                         'MAX_CONCURRENCY': str(max(self.benchmarker.concurrency_levels)) }
														
 
															-
														
 
															-    test_docker_files = ["%s.dockerfile" % self.name]
														
 
															-    if self.docker_files is not None:
														
 
															-      if type(self.docker_files) is list:
														
 
															-        test_docker_files.extend(self.docker_files)
														
 
															-      else:
														
 
															-        raise Exception("docker_files in benchmark_config.json must be an array")
														
 
															-
														
 
															-    for test_docker_file in test_docker_files:
														
 
															-      deps = list(reversed(gather_docker_dependencies(os.path.join(self.directory, test_docker_file))))
														
 
															-
														
 
															-      docker_dir = os.path.join(setup_util.get_fwroot(), "toolset", "setup", "linux", "docker")
														
 
															-
														
 
															-      for dependency in deps:
														
 
															-        docker_file = os.path.join(self.directory, dependency + ".dockerfile")
														
 
															-        if not docker_file or not os.path.exists(docker_file):
														
 
															-          docker_file = find_docker_file(docker_dir, dependency + ".dockerfile")
														
 
															-        if not docker_file:
														
 
															-          tee_output(prefix, "Docker build failed; %s could not be found; terminating\n" % (dependency + ".dockerfile"))
														
 
															-          return 1
														
 
															-
														
 
															-        # Build the dependency image
														
 
															-        try:
														
 
															-          for line in docker.APIClient(base_url='unix://var/run/docker.sock').build(
														
 
															-            path=os.path.dirname(docker_file),
														
 
															-            dockerfile="%s.dockerfile" % dependency,
														
 
															-            tag="tfb/%s" % dependency,
														
 
															-            buildargs=docker_buildargs,
														
 
															-            forcerm=True
														
 
															-          ):
														
 
															-            handle_build_output(line)
														
 
															-        except Exception as e:
														
 
															-          tee_output(prefix, "Docker dependency build failed; terminating\n")
														
 
															-          print(e)
														
 
															-          return 1
														
 
															+from colorama import Fore, Style
														
 
															-    # Build the test images
														
 
															-    for test_docker_file in test_docker_files:
														
 
															-        try:
														
 
															-          for line in docker.APIClient(base_url='unix://var/run/docker.sock').build(
														
 
															-            path=self.directory,
														
 
															-            dockerfile=test_docker_file,
														
 
															-            tag="tfb/test/%s" % test_docker_file.replace(".dockerfile",""),
														
 
															-            buildargs=docker_buildargs,
														
 
															-            forcerm=True
														
 
															-          ):
														
 
															-            handle_build_output(line)
														
 
															-        except Exception as e:
														
 
															-          tee_output(prefix, "Docker build failed; terminating\n")
														
 
															-          print(e)
														
 
															-          return 1
														
 
															-
														
 
															-    ##########################
														
 
															-    # Run the Docker container
														
 
															-    ##########################
														
 
															-
														
 
															-	client = docker.from_env()
														
 
															-
														
 
															-    for test_docker_file in test_docker_files:
														
 
															-      try:
														
 
															-        def watch_container(container, prefix):
														
 
															-          for line in container.logs(stream=True):
														
 
															-            tee_output(prefix, line)
														
 
															-
														
 
															-        container = client.containers.run(
														
 
															-          "tfb/test/%s" % test_docker_file.replace(".dockerfile", ""),
														
 
															-          network_mode="host",
														
 
															-          privileged=True,
														
 
															-          stderr=True,
														
 
															-          detach=True)
														
 
															-
														
 
															-        prefix = "Server %s: " % self.name
														
 
															-        watch_thread = Thread(target = watch_container, args=(container,prefix))
														
 
															-        watch_thread.daemon = True
														
 
															-        watch_thread.start()
														
 
															-
														
 
															-      except Exception as e:
														
 
															-        tee_output(prefix, "Running docker cointainer: %s failed" % test_docker_file)
														
 
															-        print(e)
														
 
															-        return 1
														
 
															-
														
 
															-    return 0
														
 
															-  ############################################################
														
 
															-  # End start
														
 
															-  ############################################################
														
 
															-
														
 
															-  ############################################################
														
 
															-  # verify_urls
														
 
															-  # Verifys each of the URLs for this test. THis will sinply
														
 
															-  # curl the URL and check for it's return status.
														
 
															-  # For each url, a flag will be set on this object for whether
														
 
															-  # or not it passed
														
 
															-  # Returns True if all verifications succeeded
														
 
															-  ############################################################
														
 
															-  def verify_urls(self, logPath):
														
 
															-    result = True
														
 
															-
														
 
															-    def verify_type(test_type):
														
 
															-      verificationPath = os.path.join(logPath, test_type)
														
 
															-      try:
														
 
															-        os.makedirs(verificationPath)
														
 
															-      except OSError:
														
 
															-        pass
														
 
															-      with open(os.path.join(verificationPath, 'verification.txt'), 'w') as verification:
														
 
															-        test = self.runTests[test_type]
														
 
															-        test.setup_out(verification)
														
 
															-        verification.write(header("VERIFYING %s" % test_type.upper()))
														
 
															-
														
 
															-        base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port)
														
 
															-
														
 
															-        try:
														
 
															-          # Verifies headers from the server. This check is made from the
														
 
															-          # App Server using Pythons requests module. Will do a second check from
														
 
															-          # the client to make sure the server isn't only accepting connections
														
 
															-          # from localhost on a multi-machine setup.
														
 
															-          results = test.verify(base_url)
														
 
															-
														
 
															-          # Now verify that the url is reachable from the client machine, unless
														
 
															-          # we're already failing
														
 
															-          if not any(result == 'fail' for (result, reason, url) in results):
														
 
															-            p = subprocess.call(["ssh", "TFB-client", "curl -sSf %s" % base_url + test.get_url()], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
														
 
															-            if p is not 0:
														
 
															-              results = [('fail', "Server did not respond to request from client machine.", base_url)]
														
 
															-              logging.warning("""This error usually means your server is only accepting
														
 
															+class FrameworkTest:
														
 
															+    def __init__(self, name, directory, benchmarker_config, results, runTests,
														
 
															+                 args):
														
 
															+        '''
														
 
															+        Constructor
														
 
															+        '''
														
 
															+        self.name = name
														
 
															+        self.directory = directory
														
 
															+        self.benchmarker_config = benchmarker_config
														
 
															+        self.results = results
														
 
															+        self.runTests = runTests
														
 
															+        self.fwroot = benchmarker_config.fwroot
														
 
															+        self.approach = ""
														
 
															+        self.classification = ""
														
 
															+        self.database = ""
														
 
															+        self.framework = ""
														
 
															+        self.language = ""
														
 
															+        self.orm = ""
														
 
															+        self.platform = ""
														
 
															+        self.webserver = ""
														
 
															+        self.os = ""
														
 
															+        self.database_os = ""
														
 
															+        self.display_name = ""
														
 
															+        self.notes = ""
														
 
															+        self.port = ""
														
 
															+        self.versus = ""
														
 
															+        self.docker_files = None
														
 
															+
														
 
															+        # setup logging
														
 
															+        logging.basicConfig(stream=sys.stderr, level=logging.INFO)
														
 
															+
														
 
															+        # Used in setup.sh scripts for consistency with
														
 
															+        # the bash environment variables
														
 
															+        self.troot = self.directory
														
 
															+
														
 
															+        self.__dict__.update(args)
														
 
															+
														
 
															+    ##########################################################################################
														
 
															+    # Public Methods
														
 
															+    ##########################################################################################
														
 
															+
														
 
															+    def start(self, out):
														
 
															+        '''
														
 
															+        Start the test implementation
														
 
															+        '''
														
 
															+        test_docker_files = ["%s.dockerfile" % self.name]
														
 
															+        if self.docker_files is not None:
														
 
															+            if type(self.docker_files) is list:
														
 
															+                test_docker_files.extend(self.docker_files)
														
 
															+            else:
														
 
															+                raise Exception(
														
 
															+                    "docker_files in benchmark_config.json must be an array")
														
 
															+
														
 
															+        docker_helper.build(self.benchmarker_config, [self.name], out)
														
 
															+
														
 
															+        docker_helper.run(self.benchmarker_config, test_docker_files, out)
														
 
															+
														
 
															+        return 0
														
 
															+
														
 
															+    def verify_urls(self, logPath):
														
 
															+        '''
														
 
															+        Verifys each of the URLs for this test. THis will sinply curl the URL and 
														
 
															+        check for it's return status. For each url, a flag will be set on this 
														
 
															+        object for whether or not it passed.
														
 
															+        Returns True if all verifications succeeded
														
 
															+        '''
														
 
															+        result = True
														
 
															+
														
 
															+        def verify_type(test_type):
														
 
															+            verificationPath = os.path.join(logPath, test_type)
														
 
															+            try:
														
 
															+                os.makedirs(verificationPath)
														
 
															+            except OSError:
														
 
															+                pass
														
 
															+            with open(os.path.join(verificationPath, 'verification.txt'),
														
 
															+                      'w') as verification:
														
 
															+                test = self.runTests[test_type]
														
 
															+                test.setup_out(verification)
														
 
															+                verification.write(header("VERIFYING %s" % test_type.upper()))
														
 
															+
														
 
															+                base_url = "http://%s:%s" % (
														
 
															+                    self.benchmarker_config.server_host, self.port)
														
 
															+
														
 
															+                try:
														
 
															+                    # Verifies headers from the server. This check is made from the
														
 
															+                    # App Server using Pythons requests module. Will do a second check from
														
 
															+                    # the client to make sure the server isn't only accepting connections
														
 
															+                    # from localhost on a multi-machine setup.
														
 
															+                    results = test.verify(base_url)
														
 
															+
														
 
															+                    # Now verify that the url is reachable from the client machine, unless
														
 
															+                    # we're already failing
														
 
															+                    if not any(result == 'fail'
														
 
															+                               for (result, reason, url) in results):
														
 
															+                        p = subprocess.call(
														
 
															+                            [
														
 
															+                                "ssh", self.benchmarker_config.client_host,
														
 
															+                                "curl -sSf %s" % base_url + test.get_url()
														
 
															+                            ],
														
 
															+                            shell=False,
														
 
															+                            stdout=subprocess.PIPE,
														
 
															+                            stderr=subprocess.PIPE)
														
 
															+                        if p is not 0:
														
 
															+                            results = [(
														
 
															+                                'fail',
														
 
															+                                "Server did not respond to request from client machine.",
														
 
															+                                base_url)]
														
 
															+                            logging.warning(
														
 
															+                                """This error usually means your server is only accepting
														
 
															                 requests from localhost.""")
														
 
															-        except ConnectionError as e:
														
 
															-          results = [('fail',"Server did not respond to request", base_url)]
														
 
															-          logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)
														
 
															-        except Exception as e:
														
 
															-          results = [('fail',"""Caused Exception in TFB
														
 
															+                except ConnectionError as e:
														
 
															+                    results = [('fail', "Server did not respond to request",
														
 
															+                                base_url)]
														
 
															+                    logging.warning(
														
 
															+                        "Verifying test %s for %s caused an exception: %s",
														
 
															+                        test_type, self.name, e)
														
 
															+                except Exception as e:
														
 
															+                    results = [('fail', """Caused Exception in TFB
														
 
															             This almost certainly means your return value is incorrect,
														
 
															             but also that you have found a bug. Please submit an issue
														
 
															             including this message: %s\n%s""" % (e, traceback.format_exc()),
														
 
															-            base_url)]
														
 
															-          logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)
														
 
															-          traceback.format_exc()
														
 
															-
														
 
															-        test.failed = any(result == 'fail' for (result, reason, url) in results)
														
 
															-        test.warned = any(result == 'warn' for (result, reason, url) in results)
														
 
															-        test.passed = all(result == 'pass' for (result, reason, url) in results)
														
 
															-
														
 
															-        def output_result(result, reason, url):
														
 
															-          specific_rules_url = "http://frameworkbenchmarks.readthedocs.org/en/latest/Project-Information/Framework-Tests/#specific-test-requirements"
														
 
															-          color = Fore.GREEN
														
 
															-          if result.upper() == "WARN":
														
 
															-            color = Fore.YELLOW
														
 
															-          elif result.upper() == "FAIL":
														
 
															-            color = Fore.RED
														
 
															-
														
 
															-          verification.write(("   " + color + "%s" + Style.RESET_ALL + " for %s\n") % (result.upper(), url))
														
 
															-          print("   {!s}{!s}{!s} for {!s}\n".format(color, result.upper(), Style.RESET_ALL, url))
														
 
															-          if reason is not None and len(reason) != 0:
														
 
															-            for line in reason.splitlines():
														
 
															-              verification.write("     " + line + '\n')
														
 
															-              print("     " + line)
														
 
															-            if not test.passed:
														
 
															-              verification.write("     See %s\n" % specific_rules_url)
														
 
															-              print("     See {!s}\n".format(specific_rules_url))
														
 
															-
														
 
															-        [output_result(r1,r2,url) for (r1, r2, url) in results]
														
 
															-
														
 
															-        if test.failed:
														
 
															-          self.benchmarker.report_verify_results(self, test_type, 'fail')
														
 
															-        elif test.warned:
														
 
															-          self.benchmarker.report_verify_results(self, test_type, 'warn')
														
 
															-        elif test.passed:
														
 
															-          self.benchmarker.report_verify_results(self, test_type, 'pass')
														
 
															-        else:
														
 
															-          raise Exception("Unknown error - test did not pass,warn,or fail")
														
 
															-
														
 
															-        verification.flush()
														
 
															-
														
 
															-    result = True
														
 
															-    for test_type in self.runTests:
														
 
															-      verify_type(test_type)
														
 
															-      if self.runTests[test_type].failed:
														
 
															-        result = False
														
 
															-
														
 
															-    return result
														
 
															-  ############################################################
														
 
															-  # End verify_urls
														
 
															-  ############################################################
														
 
															-
														
 
															-  ############################################################
														
 
															-  # benchmark
														
 
															-  # Runs the benchmark for each type of test that it implements
														
 
															-  # JSON/DB/Query.
														
 
															-  ############################################################
														
 
															-  def benchmark(self, logPath):
														
 
															-
														
 
															-    def benchmark_type(test_type):
														
 
															-      benchmarkPath = os.path.join(logPath, test_type)
														
 
															-      try:
														
 
															-        os.makedirs(benchmarkPath)
														
 
															-      except OSError:
														
 
															-        pass
														
 
															-      with open(os.path.join(benchmarkPath, 'benchmark.txt'), 'w') as out:
														
 
															-        out.write("BENCHMARKING %s ... " % test_type.upper())
														
 
															-
														
 
															-        test = self.runTests[test_type]
														
 
															-        test.setup_out(out)
														
 
															-        output_file = self.benchmarker.output_file(self.name, test_type)
														
 
															-        if not os.path.exists(output_file):
														
 
															-          # Open to create the empty file
														
 
															-          with open(output_file, 'w'):
														
 
															-            pass
														
 
															-
														
 
															-        if not test.failed:
														
 
															-          if test_type == 'plaintext': # One special case
														
 
															-            remote_script = self.__generate_pipeline_script(test.get_url(), self.port, test.accept_header)
														
 
															-          elif test_type == 'query' or test_type == 'update':
														
 
															-            remote_script = self.__generate_query_script(test.get_url(), self.port, test.accept_header, self.benchmarker.query_levels)
														
 
															-          elif test_type == 'cached_query':
														
 
															-            remote_script = self.__generate_query_script(test.get_url(), self.port, test.accept_header, self.benchmarker.cached_query_levels)
														
 
															-          else:
														
 
															-            remote_script = self.__generate_concurrency_script(test.get_url(), self.port, test.accept_header)
														
 
															-
														
 
															-          # Begin resource usage metrics collection
														
 
															-          self.__begin_logging(test_type)
														
 
															-
														
 
															-          # Run the benchmark
														
 
															-          with open(output_file, 'w') as raw_file:
														
 
															-            p = subprocess.Popen(self.benchmarker.client_ssh_string.split(" "), stdin=subprocess.PIPE, stdout=raw_file, stderr=raw_file)
														
 
															-            p.communicate(remote_script)
														
 
															-            out.flush()
														
 
															-
														
 
															-          # End resource usage metrics collection
														
 
															-          self.__end_logging()
														
 
															-
														
 
															-        results = self.__parse_test(test_type)
														
 
															-        print("Benchmark results:")
														
 
															-        pprint(results)
														
 
															-
														
 
															-        self.benchmarker.report_benchmark_results(framework=self, test=test_type, results=results['results'])
														
 
															-        out.write( "Complete\n" )
														
 
															-        out.flush()
														
 
															-
														
 
															-    for test_type in self.runTests:
														
 
															-      benchmark_type(test_type)
														
 
															-  ############################################################
														
 
															-  # End benchmark
														
 
															-  ############################################################
														
 
															-
														
 
															-  ############################################################
														
 
															-  # parse_all
														
 
															-  # Method meant to be run for a given timestamp
														
 
															-  ############################################################
														
 
															-  def parse_all(self):
														
 
															-    for test_type in self.runTests:
														
 
															-      if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
														
 
															-        results = self.__parse_test(test_type)
														
 
															-        self.benchmarker.report_benchmark_results(framework=self, test=test_type, results=results['results'])
														
 
															-
														
 
															-  ##########################################################################################
														
 
															-  # Private Methods
														
 
															-  ##########################################################################################
														
 
															-
														
 
															-  ############################################################
														
 
															-  # __parse_test(test_type)
														
 
															-  ############################################################
														
 
															-  def __parse_test(self, test_type):
														
 
															-    try:
														
 
															-      results = dict()
														
 
															-      results['results'] = []
														
 
															-      stats = []
														
 
															-
														
 
															-      if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
														
 
															-        with open(self.benchmarker.output_file(self.name, test_type)) as raw_data:
														
 
															-          is_warmup = True
														
 
															-          rawData = None
														
 
															-          for line in raw_data:
														
 
															-
														
 
															-            if "Queries:" in line or "Concurrency:" in line:
														
 
															-              is_warmup = False
														
 
															-              rawData = None
														
 
															-              continue
														
 
															-            if "Warmup" in line or "Primer" in line:
														
 
															-              is_warmup = True
														
 
															-              continue
														
 
															-
														
 
															-            if not is_warmup:
														
 
															-              if rawData == None:
														
 
															-                rawData = dict()
														
 
															-                results['results'].append(rawData)
														
 
															-
														
 
															-              #if "Requests/sec:" in line:
														
 
															-              #  m = re.search("Requests/sec:\s+([0-9]+)", line)
														
 
															-              #  rawData['reportedResults'] = m.group(1)
														
 
															-
														
 
															-              # search for weighttp data such as succeeded and failed.
														
 
															-              if "Latency" in line:
														
 
															-                m = re.findall("([0-9]+\.*[0-9]*[us|ms|s|m|%]+)", line)
														
 
															-                if len(m) == 4:
														
 
															-                  rawData['latencyAvg'] = m[0]
														
 
															-                  rawData['latencyStdev'] = m[1]
														
 
															-                  rawData['latencyMax'] = m[2]
														
 
															-              #    rawData['latencyStdevPercent'] = m[3]
														
 
															-
														
 
															-              #if "Req/Sec" in line:
														
 
															-              #  m = re.findall("([0-9]+\.*[0-9]*[k|%]*)", line)
														
 
															-              #  if len(m) == 4:
														
 
															-              #    rawData['requestsAvg'] = m[0]
														
 
															-              #    rawData['requestsStdev'] = m[1]
														
 
															-              #    rawData['requestsMax'] = m[2]
														
 
															-              #    rawData['requestsStdevPercent'] = m[3]
														
 
															-
														
 
															-              #if "requests in" in line:
														
 
															-              #  m = re.search("requests in ([0-9]+\.*[0-9]*[ms|s|m|h]+)", line)
														
 
															-              #  if m != None:
														
 
															-              #    # parse out the raw time, which may be in minutes or seconds
														
 
															-              #    raw_time = m.group(1)
														
 
															-              #    if "ms" in raw_time:
														
 
															-              #      rawData['total_time'] = float(raw_time[:len(raw_time)-2]) / 1000.0
														
 
															-              #    elif "s" in raw_time:
														
 
															-              #      rawData['total_time'] = float(raw_time[:len(raw_time)-1])
														
 
															-              #    elif "m" in raw_time:
														
 
															-              #      rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 60.0
														
 
															-              #    elif "h" in raw_time:
														
 
															-              #      rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 3600.0
														
 
															-
														
 
															-              if "requests in" in line:
														
 
															-                m = re.search("([0-9]+) requests in", line)
														
 
															-                if m != None:
														
 
															-                  rawData['totalRequests'] = int(m.group(1))
														
 
															-
														
 
															-              if "Socket errors" in line:
														
 
															-                if "connect" in line:
														
 
															-                  m = re.search("connect ([0-9]+)", line)
														
 
															-                  rawData['connect'] = int(m.group(1))
														
 
															-                if "read" in line:
														
 
															-                  m = re.search("read ([0-9]+)", line)
														
 
															-                  rawData['read'] = int(m.group(1))
														
 
															-                if "write" in line:
														
 
															-                  m = re.search("write ([0-9]+)", line)
														
 
															-                  rawData['write'] = int(m.group(1))
														
 
															-                if "timeout" in line:
														
 
															-                  m = re.search("timeout ([0-9]+)", line)
														
 
															-                  rawData['timeout'] = int(m.group(1))
														
 
															-
														
 
															-              if "Non-2xx" in line:
														
 
															-                m = re.search("Non-2xx or 3xx responses: ([0-9]+)", line)
														
 
															-                if m != None:
														
 
															-                  rawData['5xx'] = int(m.group(1))
														
 
															-              if "STARTTIME" in line:
														
 
															-                m = re.search("[0-9]+", line)
														
 
															-                rawData["startTime"] = int(m.group(0))
														
 
															-              if "ENDTIME" in line:
														
 
															-                m = re.search("[0-9]+", line)
														
 
															-                rawData["endTime"] = int(m.group(0))
														
 
															-                test_stats = self.__parse_stats(test_type, rawData["startTime"], rawData["endTime"], 1)
														
 
															-                # rawData["averageStats"] = self.__calculate_average_stats(test_stats)
														
 
															-                stats.append(test_stats)
														
 
															-      with open(self.benchmarker.stats_file(self.name, test_type) + ".json", "w") as stats_file:
														
 
															-        json.dump(stats, stats_file, indent=2)
														
 
															-
														
 
															-
														
 
															-      return results
														
 
															-    except IOError:
														
 
															-      return None
														
 
															-  ############################################################
														
 
															-  # End benchmark
														
 
															-  ############################################################
														
 
															-
														
 
															-  ############################################################
														
 
															-  # __generate_concurrency_script(url, port)
														
 
															-  # Generates the string containing the bash script that will
														
 
															-  # be run on the client to benchmark a single test. This
														
 
															-  # specifically works for the variable concurrency tests (JSON
														
 
															-  # and DB)
														
 
															-  ############################################################
														
 
															-  def __generate_concurrency_script(self, url, port, accept_header, wrk_command="wrk"):
														
 
															-    headers = self.headers_template.format(server_host=self.benchmarker.server_host, accept=accept_header)
														
 
															-    return self.concurrency_template.format(max_concurrency=max(self.benchmarker.concurrency_levels),
														
 
															-      name=self.name, duration=self.benchmarker.duration,
														
 
															-      levels=" ".join("{}".format(item) for item in self.benchmarker.concurrency_levels),
														
 
															-      server_host=self.benchmarker.server_host, port=port, url=url, headers=headers, wrk=wrk_command)
														
 
															-
														
 
															-  ############################################################
														
 
															-  # __generate_pipeline_script(url, port)
														
 
															-  # Generates the string containing the bash script that will
														
 
															-  # be run on the client to benchmark a single pipeline test.
														
 
															-  ############################################################
														
 
															-  def __generate_pipeline_script(self, url, port, accept_header, wrk_command="wrk"):
														
 
															-    headers = self.headers_template.format(server_host=self.benchmarker.server_host, accept=accept_header)
														
 
															-    return self.pipeline_template.format(max_concurrency=max(self.benchmarker.pipeline_concurrency_levels),
														
 
															-      name=self.name, duration=self.benchmarker.duration,
														
 
															-      levels=" ".join("{}".format(item) for item in self.benchmarker.pipeline_concurrency_levels),
														
 
															-      server_host=self.benchmarker.server_host, port=port, url=url, headers=headers, wrk=wrk_command,
														
 
															-      pipeline=16)
														
 
															-
														
 
															-  ############################################################
														
 
															-  # __generate_query_script(url, port)
														
 
															-  # Generates the string containing the bash script that will
														
 
															-  # be run on the client to benchmark a single test. This
														
 
															-  # specifically works for the variable query tests (Query)
														
 
															-  ############################################################
														
 
															-  def __generate_query_script(self, url, port, accept_header, query_levels):
														
 
															-    headers = self.headers_template.format(server_host=self.benchmarker.server_host, accept=accept_header)
														
 
															-    return self.query_template.format(max_concurrency=max(self.benchmarker.concurrency_levels),
														
 
															-      name=self.name, duration=self.benchmarker.duration,
														
 
															-      levels=" ".join("{}".format(item) for item in query_levels),
														
 
															-      server_host=self.benchmarker.server_host, port=port, url=url, headers=headers)
														
 
															-
														
 
															-  ############################################################
														
 
															-  # Returns True if any test type this this framework test will use a DB
														
 
															-  ############################################################
														
 
															-  def requires_database(self):
														
 
															-    '''Returns True/False if this test requires a database'''
														
 
															-    return any(tobj.requires_db for (ttype,tobj) in self.runTests.iteritems())
														
 
															-
														
 
															-  ############################################################
														
 
															-  # __begin_logging
														
 
															-  # Starts a thread to monitor the resource usage, to be synced with the client's time
														
 
															-  # TODO: MySQL and InnoDB are possible. Figure out how to implement them.
														
 
															-  ############################################################
														
 
															-  def __begin_logging(self, test_type):
														
 
															-    output_file = "{file_name}".format(file_name=self.benchmarker.get_stats_file(self.name, test_type))
														
 
															-    dstat_string = "dstat -Tafilmprs --aio --fs --ipc --lock --raw --socket --tcp \
														
 
															-                                      --raw --socket --tcp --udp --unix --vm --disk-util \
														
 
															-                                      --rpc --rpcd --output {output_file}".format(output_file=output_file)
														
 
															-    cmd = shlex.split(dstat_string)
														
 
															-    dev_null = open(os.devnull, "w")
														
 
															-    self.subprocess_handle = subprocess.Popen(cmd, stdout=dev_null, stderr=subprocess.STDOUT)
														
 
															-
														
 
															-  ##############################################################
														
 
															-  # Begin __end_logging
														
 
															-  # Stops the logger thread and blocks until shutdown is complete.
														
 
															-  ##############################################################
														
 
															-  def __end_logging(self):
														
 
															-    self.subprocess_handle.terminate()
														
 
															-    self.subprocess_handle.communicate()
														
 
															-
														
 
															-  ##############################################################
														
 
															-  # Begin __parse_stats
														
 
															-  # For each test type, process all the statistics, and return a multi-layered dictionary
														
 
															-  # that has a structure as follows:
														
 
															-  # (timestamp)
														
 
															-  # | (main header) - group that the stat is in
														
 
															-  # | | (sub header) - title of the stat
														
 
															-  # | | | (stat) - the stat itself, usually a floating point number
														
 
															-  ##############################################################
														
 
															-  def __parse_stats(self, test_type, start_time, end_time, interval):
														
 
															-    stats_dict = dict()
														
 
															-    stats_file = self.benchmarker.stats_file(self.name, test_type)
														
 
															-    with open(stats_file) as stats:
														
 
															-      while(stats.next() != "\n"): # dstat doesn't output a completely compliant CSV file - we need to strip the header
														
 
															-        pass
														
 
															-      stats_reader = csv.reader(stats)
														
 
															-      main_header = stats_reader.next()
														
 
															-      sub_header = stats_reader.next()
														
 
															-      time_row = sub_header.index("epoch")
														
 
															-      int_counter = 0
														
 
															-      for row in stats_reader:
														
 
															-        time = float(row[time_row])
														
 
															-        int_counter+=1
														
 
															-        if time < start_time:
														
 
															-          continue
														
 
															-        elif time > end_time:
														
 
															-          return stats_dict
														
 
															-        if int_counter % interval != 0:
														
 
															-          continue
														
 
															-        row_dict = dict()
														
 
															-        for nextheader in main_header:
														
 
															-          if nextheader != "":
														
 
															-            row_dict[nextheader] = dict()
														
 
															-        header = ""
														
 
															-        for item_num, column in enumerate(row):
														
 
															-          if(len(main_header[item_num]) != 0):
														
 
															-            header = main_header[item_num]
														
 
															-          row_dict[header][sub_header[item_num]] = float(column) # all the stats are numbers, so we want to make sure that they stay that way in json
														
 
															-        stats_dict[time] = row_dict
														
 
															-    return stats_dict
														
 
															-  ##############################################################
														
 
															-  # End __parse_stats
														
 
															-  ##############################################################
														
 
															-
														
 
															-  def __getattr__(self, name):
														
 
															-    """For backwards compatibility, we used to pass benchmarker
														
 
															-    as the argument to the setup.sh files"""
														
 
															-    try:
														
 
															-      x = getattr(self.benchmarker, name)
														
 
															-    except AttributeError:
														
 
															-      print("AttributeError: {!s} not a member of FrameworkTest or Benchmarker".format(name))
														
 
															-      print("This is probably a bug")
														
 
															-      raise
														
 
															-    return x
														
 
															-
														
 
															-  ##############################################################
														
 
															-  # Begin __calculate_average_stats
														
 
															-  # We have a large amount of raw data for the statistics that
														
 
															-  # may be useful for the stats nerds, but most people care about
														
 
															-  # a couple of numbers. For now, we're only going to supply:
														
 
															-  # * Average CPU
														
 
															-  # * Average Memory
														
 
															-  # * Total network use
														
 
															-  # * Total disk use
														
 
															-  # More may be added in the future. If they are, please update
														
 
															-  # the above list.
														
 
															-  # Note: raw_stats is directly from the __parse_stats method.
														
 
															-  # Recall that this consists of a dictionary of timestamps,
														
 
															-  # each of which contain a dictionary of stat categories which
														
 
															-  # contain a dictionary of stats
														
 
															-  ##############################################################
														
 
															-  def __calculate_average_stats(self, raw_stats):
														
 
															-    raw_stat_collection = dict()
														
 
															-
														
 
															-    for timestamp, time_dict in raw_stats.items():
														
 
															-      for main_header, sub_headers in time_dict.items():
														
 
															-        item_to_append = None
														
 
															-        if 'cpu' in main_header:
														
 
															-          # We want to take the idl stat and subtract it from 100
														
 
															-          # to get the time that the CPU is NOT idle.
														
 
															-          item_to_append = sub_headers['idl'] - 100.0
														
 
															-        elif main_header == 'memory usage':
														
 
															-          item_to_append = sub_headers['used']
														
 
															-        elif 'net' in main_header:
														
 
															-          # Network stats have two parts - recieve and send. We'll use a tuple of
														
 
															-          # style (recieve, send)
														
 
															-          item_to_append = (sub_headers['recv'], sub_headers['send'])
														
 
															-        elif 'dsk' or 'io' in main_header:
														
 
															-          # Similar for network, except our tuple looks like (read, write)
														
 
															-          item_to_append = (sub_headers['read'], sub_headers['writ'])
														
 
															-        if item_to_append is not None:
														
 
															-          if main_header not in raw_stat_collection:
														
 
															-            raw_stat_collection[main_header] = list()
														
 
															-          raw_stat_collection[main_header].append(item_to_append)
														
 
															-
														
 
															-    # Simple function to determine human readable size
														
 
															-    # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
														
 
															-    def sizeof_fmt(num):
														
 
															-      # We'll assume that any number we get is convertable to a float, just in case
														
 
															-      num = float(num)
														
 
															-      for x in ['bytes','KB','MB','GB']:
														
 
															-        if num < 1024.0 and num > -1024.0:
														
 
															-          return "%3.1f%s" % (num, x)
														
 
															-        num /= 1024.0
														
 
															-      return "%3.1f%s" % (num, 'TB')
														
 
															-
														
 
															-    # Now we have our raw stats in a readable format - we need to format it for display
														
 
															-    # We need a floating point sum, so the built in sum doesn't cut it
														
 
															-    display_stat_collection = dict()
														
 
															-    for header, values in raw_stat_collection.items():
														
 
															-      display_stat = None
														
 
															-      if 'cpu' in header:
														
 
															-        display_stat = sizeof_fmt(math.fsum(values) / len(values))
														
 
															-      elif main_header == 'memory usage':
														
 
															-        display_stat = sizeof_fmt(math.fsum(values) / len(values))
														
 
															-      elif 'net' in main_header:
														
 
															-        receive, send = zip(*values) # unzip
														
 
															-        display_stat = {'receive': sizeof_fmt(math.fsum(receive)), 'send': sizeof_fmt(math.fsum(send))}
														
 
															-      else: # if 'dsk' or 'io' in header:
														
 
															-        read, write = zip(*values) # unzip
														
 
															-        display_stat = {'read': sizeof_fmt(math.fsum(read)), 'write': sizeof_fmt(math.fsum(write))}
														
 
															-      display_stat_collection[header] = display_stat
														
 
															-    return display_stat
														
 
															-  ###########################################################################################
														
 
															-  # End __calculate_average_stats
														
 
															-  #########################################################################################
														
 
															-
														
 
															-
														
 
															-  ##########################################################################################
														
 
															-  # Constructor
														
 
															-  ##########################################################################################
														
 
															-  def __init__(self, name, directory, benchmarker, runTests, args):
														
 
															-    self.name = name
														
 
															-    self.directory = directory
														
 
															-    self.benchmarker = benchmarker
														
 
															-    self.runTests = runTests
														
 
															-    self.fwroot = benchmarker.fwroot
														
 
															-    self.approach = ""
														
 
															-    self.classification = ""
														
 
															-    self.database = ""
														
 
															-    self.framework = ""
														
 
															-    self.language = ""
														
 
															-    self.orm = ""
														
 
															-    self.platform = ""
														
 
															-    self.webserver = ""
														
 
															-    self.os = ""
														
 
															-    self.database_os = ""
														
 
															-    self.display_name = ""
														
 
															-    self.notes = ""
														
 
															-    self.versus = ""
														
 
															-    self.docker_files = None
														
 
															-
														
 
															-    # setup logging
														
 
															-    logging.basicConfig(stream=sys.stderr, level=logging.INFO)
														
 
															-
														
 
															-    self.install_root="%s/%s" % (self.fwroot, "installs")
														
 
															-
														
 
															-    # Used in setup.sh scripts for consistency with
														
 
															-    # the bash environment variables
														
 
															-    self.troot = self.directory
														
 
															-    self.iroot = self.install_root
														
 
															-
														
 
															-    self.__dict__.update(args)
														
 
															-  ############################################################
														
 
															-  # End __init__
														
 
															-  ############################################################
														
 
															-############################################################
														
 
															-# End FrameworkTest
														
 
															-############################################################
														
 
															-
														
 
															-
														
 
															-# Static methods
														
 
															-
														
 
															-def test_order(type_name):
														
 
															-  """
														
 
															-  This sort ordering is set up specifically to return the length
														
 
															-  of the test name. There were SO many problems involved with
														
 
															-  'plaintext' being run first (rather, just not last) that we
														
 
															-  needed to ensure that it was run last for every framework.
														
 
															-  """
														
 
															-  return len(type_name)
														
 
															-
														
 
															-
														
 
															-def validate_urls(test_name, test_keys):
														
 
															-  """
														
 
															-  Separated from validate_test because urls are not required anywhere. We know a url is incorrect if it is
														
 
															-  empty or does not start with a "/" character. There is no validation done to ensure the url conforms to
														
 
															-  the suggested url specifications, although those suggestions are presented if a url fails validation here.
														
 
															-  """
														
 
															-  example_urls = {
														
 
															-    "json_url":         "/json",
														
 
															-    "db_url":           "/mysql/db",
														
 
															-    "query_url":        "/mysql/queries?queries=  or  /mysql/queries/",
														
 
															-    "fortune_url":      "/mysql/fortunes",
														
 
															-    "update_url":       "/mysql/updates?queries=  or  /mysql/updates/",
														
 
															-    "plaintext_url":    "/plaintext",
														
 
															-    "cached_query_url": "/mysql/cached_queries?queries=  or /mysql/cached_queries"
														
 
															-  }
														
 
															-
														
 
															-  for test_url in ["json_url","db_url","query_url","fortune_url","update_url","plaintext_url","cached_query_url"]:
														
 
															-    key_value = test_keys.get(test_url, None)
														
 
															-    if key_value != None and not key_value.startswith('/'):
														
 
															-      errmsg = """`%s` field in test \"%s\" does not appear to be a valid url: \"%s\"\n
														
 
															-        Example `%s` url: \"%s\"
														
 
															-      """ % (test_url, test_name, key_value, test_url, example_urls[test_url])
														
 
															-      raise Exception(errmsg)
														
 
															-
														
 
															-
														
 
															-def validate_test(test_name, test_keys, directory):
														
 
															-  """
														
 
															-  Validate benchmark config values for this test based on a schema
														
 
															-  """
														
 
															-  # Ensure that each FrameworkTest has a framework property, inheriting from top-level if not
														
 
															-  if not test_keys['framework']:
														
 
															-    test_keys['framework'] = config['framework']
														
 
															-
														
 
															-  recommended_lang = directory.split('/')[-2]
														
 
															-  windows_url = "https://github.com/TechEmpower/FrameworkBenchmarks/issues/1038"
														
 
															-  schema = {
														
 
															-    'language': {
														
 
															-      'help': ('language', 'The language of the framework used, suggestion: %s' % recommended_lang)
														
 
															-    },
														
 
															-    'webserver': {
														
 
															-      'help': ('webserver', 'Name of the webserver also referred to as the "front-end server"')
														
 
															-    },
														
 
															-    'classification': {
														
 
															-      'allowed': [
														
 
															-        ('Fullstack', '...'),
														
 
															-        ('Micro', '...'),
														
 
															-        ('Platform', '...')
														
 
															-      ]
														
 
															-    },
														
 
															-    'database': {
														
 
															-      'allowed': [
														
 
															-        ('MySQL', 'One of the most popular databases around the web and in TFB'),
														
 
															-        ('Postgres', 'An advanced SQL database with a larger feature set than MySQL'),
														
 
															-        ('MongoDB', 'A popular document-store database'),
														
 
															-        ('Cassandra', 'A highly performant and scalable NoSQL database'),
														
 
															-        ('Elasticsearch', 'A distributed RESTful search engine that is used as a database for TFB tests'),
														
 
															-        ('Redis', 'An open-sourced, BSD licensed, advanced key-value cache and store'),
														
 
															-        ('SQLite', 'A network-less database, still supported for backwards compatibility'),
														
 
															-        ('SQLServer', 'Microsoft\'s SQL implementation'),
														
 
															-        ('None', 'No database was used for these tests, as is the case with Json Serialization and Plaintext')
														
 
															-      ]
														
 
															-    },
														
 
															-    'approach': {
														
 
															-      'allowed': [
														
 
															-        ('Realistic', '...'),
														
 
															-        ('Stripped', '...')
														
 
															-      ]
														
 
															-    },
														
 
															-    'orm': {
														
 
															-      'allowed': [
														
 
															-        ('Full', 'Has a full suite of features like lazy loading, caching, multiple language support, sometimes pre-configured with scripts.'),
														
 
															-        ('Micro', 'Has basic database driver capabilities such as establishing a connection and sending queries.'),
														
 
															-        ('Raw', 'Tests that do not use an ORM will be classified as "raw" meaning they use the platform\'s raw database connectivity.')
														
 
															-      ]
														
 
															-    },
														
 
															-    'platform': {
														
 
															-      'help': ('platform', 'Name of the platform this framework runs on, e.g. Node.js, PyPy, hhvm, JRuby ...')
														
 
															-    },
														
 
															-    'framework': {
														
 
															-      # Guranteed to be here and correct at this point
														
 
															-      # key is left here to produce the set of required keys
														
 
															-    },
														
 
															-    'os': {
														
 
															-      'allowed': [
														
 
															-        ('Linux', 'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'),
														
 
															-        ('Windows', 'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s' % windows_url)
														
 
															-      ]
														
 
															-    },
														
 
															-    'database_os': {
														
 
															-      'allowed': [
														
 
															-        ('Linux', 'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'),
														
 
															-        ('Windows', 'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s' % windows_url)
														
 
															-      ]
														
 
															-    }
														
 
															-  }
														
 
															-
														
 
															-  # Confirm required keys are present
														
 
															-  required_keys = schema.keys()
														
 
															-  missing = list(set(required_keys) - set(test_keys))
														
 
															-
														
 
															-  if len(missing) > 0:
														
 
															-    missingstr = (", ").join(map(str, missing))
														
 
															-    raise Exception("benchmark_config.json for test %s is invalid, please amend by adding the following required keys: [%s]"
														
 
															-      % (test_name, missingstr))
														
 
															-
														
 
															-  # Check the (all optional) test urls
														
 
															-  validate_urls(test_name, test_keys)
														
 
															-
														
 
															-  # Check values of keys against schema
														
 
															-  for key in required_keys:
														
 
															-    val = test_keys.get(key, "").lower()
														
 
															-    has_predefined_acceptables = 'allowed' in schema[key]
														
 
															-
														
 
															-    if has_predefined_acceptables:
														
 
															-      allowed = schema[key].get('allowed', [])
														
 
															-      acceptable_values, descriptors = zip(*allowed)
														
 
															-      acceptable_values = [a.lower() for a in acceptable_values]
														
 
															-
														
 
															-      if val not in acceptable_values:
														
 
															-        msg = ("Invalid `%s` value specified for test \"%s\" in framework \"%s\"; suggestions:\n"
														
 
															-          % (key, test_name, test_keys['framework']))
														
 
															-        helpinfo = ('\n').join(["  `%s` -- %s" % (v, desc) for (v, desc) in zip(acceptable_values, descriptors)])
														
 
															-        fullerr = msg + helpinfo + "\n"
														
 
															-        raise Exception(fullerr)
														
 
															-
														
 
															-    elif not has_predefined_acceptables and val == "":
														
 
															-      msg = ("Value for `%s` in test \"%s\" in framework \"%s\" was missing:\n"
														
 
															-        % (key, test_name, test_keys['framework']))
														
 
															-      helpinfo = "  %s -- %s" % schema[key]['help']
														
 
															-      fullerr = msg + helpinfo + '\n'
														
 
															-      raise Exception(fullerr)
														
 
															-
														
 
															-def parse_config(config, directory, benchmarker):
														
 
															-  """
														
 
															-  Parses a config file into a list of FrameworkTest objects
														
 
															-  """
														
 
															-  tests = []
														
 
															-
														
 
															-  # The config object can specify multiple tests
														
 
															-  # Loop over them and parse each into a FrameworkTest
														
 
															-  for test in config['tests']:
														
 
															-
														
 
															-    tests_to_run = [name for (name,keys) in test.iteritems()]
														
 
															-    if "default" not in tests_to_run:
														
 
															-      logging.warn("Framework %s does not define a default test in benchmark_config.json", config['framework'])
														
 
															-
														
 
															-    # Check that each test configuration is acceptable
														
 
															-    # Throw exceptions if a field is missing, or how to improve the field
														
 
															-    for test_name, test_keys in test.iteritems():
														
 
															-      # Validates the benchmark_config entry
														
 
															-      validate_test(test_name, test_keys, directory)
														
 
															-
														
 
															-      # Map test type to a parsed FrameworkTestType object
														
 
															-      runTests = dict()
														
 
															-      for type_name, type_obj in benchmarker.types.iteritems():
														
 
															-        try:
														
 
															-          # Makes a FrameWorkTestType object using some of the keys in config
														
 
															-          # e.g. JsonTestType uses "json_url"
														
 
															-          runTests[type_name] = type_obj.copy().parse(test_keys)
														
 
															-        except AttributeError as ae:
														
 
															-          # This is quite common - most tests don't support all types
														
 
															-          # Quitely log it and move on (debug logging is on in travis and this causes
														
 
															-          # ~1500 lines of debug, so I'm totally ignoring it for now
														
 
															-          # logging.debug("Missing arguments for test type %s for framework test %s", type_name, test_name)
														
 
															-          pass
														
 
															-
														
 
															-      # We need to sort by test_type to run
														
 
															-      sortedTestKeys = sorted(runTests.keys(), key=test_order)
														
 
															-      sortedRunTests = OrderedDict()
														
 
															-      for sortedTestKey in sortedTestKeys:
														
 
															-        sortedRunTests[sortedTestKey] = runTests[sortedTestKey]
														
 
															-
														
 
															-      # Prefix all test names with framework except 'default' test
														
 
															-      # Done at the end so we may still refer to the primary test as `default` in benchmark config error messages
														
 
															-      if test_name == 'default':
														
 
															-        test_name = config['framework']
														
 
															-      else:
														
 
															-        test_name = "%s-%s" % (config['framework'], test_name)
														
 
															-
														
 
															-      # By passing the entire set of keys, each FrameworkTest will have a member for each key
														
 
															-      tests.append(FrameworkTest(test_name, directory, benchmarker, sortedRunTests, test_keys))
														
 
															-
														
 
															-  return tests
														
 
															+                                base_url)]
														
 
															+                    logging.warning(
														
 
															+                        "Verifying test %s for %s caused an exception: %s",
														
 
															+                        test_type, self.name, e)
														
 
															+                    traceback.format_exc()
														
 
															+
														
 
															+                test.failed = any(
														
 
															+                    result == 'fail' for (result, reason, url) in results)
														
 
															+                test.warned = any(
														
 
															+                    result == 'warn' for (result, reason, url) in results)
														
 
															+                test.passed = all(
														
 
															+                    result == 'pass' for (result, reason, url) in results)
														
 
															+
														
 
															+                def output_result(result, reason, url):
														
 
															+                    specific_rules_url = "http://frameworkbenchmarks.readthedocs.org/en/latest/Project-Information/Framework-Tests/#specific-test-requirements"
														
 
															+                    color = Fore.GREEN
														
 
															+                    if result.upper() == "WARN":
														
 
															+                        color = Fore.YELLOW
														
 
															+                    elif result.upper() == "FAIL":
														
 
															+                        color = Fore.RED
														
 
															+
														
 
															+                    verification.write((
														
 
															+                        "   " + color + "%s" + Style.RESET_ALL + " for %s\n") %
														
 
															+                                       (result.upper(), url))
														
 
															+                    print("   {!s}{!s}{!s} for {!s}\n".format(
														
 
															+                        color, result.upper(), Style.RESET_ALL, url))
														
 
															+                    if reason is not None and len(reason) != 0:
														
 
															+                        for line in reason.splitlines():
														
 
															+                            verification.write("     " + line + '\n')
														
 
															+                            print("     " + line)
														
 
															+                        if not test.passed:
														
 
															+                            verification.write(
														
 
															+                                "     See %s\n" % specific_rules_url)
														
 
															+                            print("     See {!s}\n".format(specific_rules_url))
														
 
															+
														
 
															+                [output_result(r1, r2, url) for (r1, r2, url) in results]
														
 
															+
														
 
															+                if test.failed:
														
 
															+                    self.results.report_verify_results(self, test_type, 'fail')
														
 
															+                elif test.warned:
														
 
															+                    self.results.report_verify_results(self, test_type, 'warn')
														
 
															+                elif test.passed:
														
 
															+                    self.results.report_verify_results(self, test_type, 'pass')
														
 
															+                else:
														
 
															+                    raise Exception(
														
 
															+                        "Unknown error - test did not pass,warn,or fail")
														
 
															+
														
 
															+                verification.flush()
														
 
															+
														
 
															+        result = True
														
 
															+        for test_type in self.runTests:
														
 
															+            verify_type(test_type)
														
 
															+            if self.runTests[test_type].failed:
														
 
															+                result = False
														
 
															+
														
 
															+        return result
														
--- a/toolset/benchmark/test_types/cached_query_type.py
+++ b/toolset/benchmark/test_types/cached_query_type.py
@@ -1,21 +1,17 @@
 
															-from benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															-from benchmark.test_types.verifications import (
														
 
															-    verify_headers,
														
 
															-    verify_randomnumber_list,
														
 
															-    verify_query_cases
														
 
															-)
														
 
															+from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															+from toolset.benchmark.test_types.verifications import verify_query_cases
														
 
															 class CachedQueryTestType(FrameworkTestType):
														
 
															-
														
 
															-    def __init__(self):
														
 
															+    def __init__(self, config):
														
 
															+        self.cached_query_url = ""
														
 
															         kwargs = {
														
 
															             'name': 'cached_query',
														
 
															             'accept_header': self.accept('json'),
														
 
															             'requires_db': True,
														
 
															             'args': ['cached_query_url']
														
 
															         }
														
 
															-        FrameworkTestType.__init__(self, **kwargs)
														
 
															+        FrameworkTestType.__init__(self, config, **kwargs)
														
 
															     def get_url(self):
														
 
															         return self.cached_query_url
														
@@ -30,13 +26,8 @@ class CachedQueryTestType(FrameworkTestType):
 
															         '''
														
 
															         url = base_url + self.cached_query_url
														
 
															-        cases = [
														
 
															-            ('2',   'fail'),
														
 
															-            ('0',   'fail'),
														
 
															-            ('foo', 'fail'),
														
 
															-            ('501', 'warn'),
														
 
															-            ('',    'fail')
														
 
															-        ]
														
 
															+        cases = [('2', 'fail'), ('0', 'fail'), ('foo', 'fail'),
														
 
															+                 ('501', 'warn'), ('', 'fail')]
														
 
															         problems = verify_query_cases(self, cases, url)
														
--- a/toolset/benchmark/test_types/db_type.py
+++ b/toolset/benchmark/test_types/db_type.py
@@ -1,25 +1,24 @@
 
															-from benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															-from benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object
														
 
															-
														
 
															-import json
														
 
															+from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															+from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object
														
 
															 class DBTestType(FrameworkTestType):
														
 
															-
														
 
															-    def __init__(self):
														
 
															+    def __init__(self, config):
														
 
															+        self.db_url = ""
														
 
															         kwargs = {
														
 
															             'name': 'db',
														
 
															             'accept_header': self.accept('json'),
														
 
															             'requires_db': True,
														
 
															             'args': ['db_url']
														
 
															         }
														
 
															-        FrameworkTestType.__init__(self, **kwargs)
														
 
															+        FrameworkTestType.__init__(self, config, **kwargs)
														
 
															     def get_url(self):
														
 
															         return self.db_url
														
 
															     def verify(self, base_url):
														
 
															-        '''Ensures body is valid JSON with a key 'id' and a key 
														
 
															+        '''
														
 
															+        Ensures body is valid JSON with a key 'id' and a key 
														
 
															         'randomNumber', both of which must map to integers
														
 
															         '''
														
@@ -29,20 +28,24 @@ class DBTestType(FrameworkTestType):
 
															         response, problems = basic_body_verification(body, url)
														
 
															         if len(problems) > 0:
														
 
															-            return problems 
														
 
															+            return problems
														
 
															         # We are allowing the single-object array
														
 
															         # e.g. [{'id':5, 'randomNumber':10}] for now,
														
 
															         # but will likely make this fail at some point
														
 
															         if type(response) == list:
														
 
															             response = response[0]
														
 
															-            problems.append(
														
 
															-                ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url))
														
 
															+            problems.append((
														
 
															+                'warn',
														
 
															+                'Response is a JSON array. Expected JSON object (e.g. [] vs {})',
														
 
															+                url))
														
 
															             # Make sure there was a JSON object inside the array
														
 
															             if type(response) != dict:
														
 
															-                problems.append(
														
 
															-                    ('fail', 'Response is not a JSON object or an array of JSON objects', url))
														
 
															+                problems.append((
														
 
															+                    'fail',
														
 
															+                    'Response is not a JSON object or an array of JSON objects',
														
 
															+                    url))
														
 
															                 return problems
														
 
															         # Verify response content
														
--- a/toolset/benchmark/test_types/fortune_type.py
+++ b/toolset/benchmark/test_types/fortune_type.py
@@ -1,24 +1,25 @@
 
															-from benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															-from benchmark.fortune_html_parser import FortuneHTMLParser
														
 
															-from benchmark.test_types.verifications import basic_body_verification, verify_headers
														
 
															+from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															+from toolset.benchmark.fortune_html_parser import FortuneHTMLParser
														
 
															+from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers
														
 
															 class FortuneTestType(FrameworkTestType):
														
 
															-
														
 
															-    def __init__(self):
														
 
															+    def __init__(self, config):
														
 
															+        self.fortune_url = ""
														
 
															         kwargs = {
														
 
															             'name': 'fortune',
														
 
															             'accept_header': self.accept('html'),
														
 
															             'requires_db': True,
														
 
															             'args': ['fortune_url']
														
 
															         }
														
 
															-        FrameworkTestType.__init__(self, **kwargs)
														
 
															+        FrameworkTestType.__init__(self, config, **kwargs)
														
 
															     def get_url(self):
														
 
															         return self.fortune_url
														
 
															     def verify(self, base_url):
														
 
															-        '''Parses the given HTML string and asks the 
														
 
															+        '''
														
 
															+        Parses the given HTML string and asks the 
														
 
															         FortuneHTMLParser whether the parsed string is a 
														
 
															         valid fortune response
														
 
															         '''
														
@@ -44,13 +45,14 @@ class FortuneTestType(FrameworkTestType):
 
															                 return problems
														
 
															         else:
														
 
															             failures = []
														
 
															-            failures.append(
														
 
															-                ('fail', 'Invalid according to FortuneHTMLParser', url))
														
 
															+            failures.append(('fail', 'Invalid according to FortuneHTMLParser',
														
 
															+                             url))
														
 
															             failures += self._parseDiffForFailure(diff, failures, url)
														
 
															             return failures
														
 
															     def _parseDiffForFailure(self, diff, failures, url):
														
 
															-        '''Example diff:
														
 
															+        '''
														
 
															+        Example diff:
														
 
															         --- Valid
														
 
															         +++ Response
														
@@ -73,16 +75,13 @@ class FortuneTestType(FrameworkTestType):
 
															                 elif line[0] == '-':
														
 
															                     current_pos.append(line[1:])
														
 
															                 elif line[0] == '@':
														
 
															-                    problems.append(('fail',
														
 
															-                                     "`%s` should be `%s`" % (
														
 
															-                                         ''.join(current_neg), ''.join(current_pos)),
														
 
															-                                     url))
														
 
															+                    problems.append(('fail', "`%s` should be `%s`" %
														
 
															+                                     (''.join(current_neg),
														
 
															+                                      ''.join(current_pos)), url))
														
 
															             if len(current_pos) != 0:
														
 
															-                problems.append(
														
 
															-                    ('fail',
														
 
															-                     "`%s` should be `%s`" % (
														
 
															-                         ''.join(current_neg), ''.join(current_pos)),
														
 
															-                     url))
														
 
															+                problems.append(('fail', "`%s` should be `%s`" %
														
 
															+                                 (''.join(current_neg),
														
 
															+                                  ''.join(current_pos)), url))
														
 
															         except:
														
 
															             # If there were errors reading the diff, then no diff information
														
 
															             pass
														
--- a/toolset/benchmark/test_types/framework_test_type.py
+++ b/toolset/benchmark/test_types/framework_test_type.py
@@ -1,9 +1,6 @@
 
															 import copy
														
 
															 import sys
														
 
															-import os
														
 
															 import json
														
 
															-import subprocess
														
 
															-from subprocess import PIPE
														
 
															 import requests
														
 
															 import MySQLdb
														
 
															 import psycopg2
														
@@ -14,11 +11,8 @@ import pymongo
 
															 import logging
														
 
															 logging.getLogger('urllib3').setLevel(logging.CRITICAL)
														
 
															-from pprint import pprint
														
 
															-
														
 
															 class FrameworkTestType:
														
 
															-
														
 
															     '''
														
 
															     Interface between a test type (json, query, plaintext, etc) and 
														
 
															     the rest of TFB. A test type defines a number of keys it expects
														
@@ -29,7 +23,13 @@ class FrameworkTestType:
 
															     exist a member `X.spam = 'foobar'`. 
														
 
															     '''
														
 
															-    def __init__(self, name, requires_db=False, accept_header=None, args=[]):
														
 
															+    def __init__(self,
														
 
															+                 config,
														
 
															+                 name,
														
 
															+                 requires_db=False,
														
 
															+                 accept_header=None,
														
 
															+                 args=[]):
														
 
															+        self.config = config
														
 
															         self.name = name
														
 
															         self.requires_db = requires_db
														
 
															         self.args = args
														
@@ -47,9 +47,12 @@ class FrameworkTestType:
 
															     def accept(self, content_type):
														
 
															         return {
														
 
															-            'json': 'application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7',
														
 
															-            'html': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
														
 
															-            'plaintext': 'text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7'
														
 
															+            'json':
														
 
															+            'application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7',
														
 
															+            'html':
														
 
															+            'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
														
 
															+            'plaintext':
														
 
															+            'text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7'
														
 
															         }[content_type]
														
 
															     def setup_out(self, out):
														
@@ -76,7 +79,8 @@ class FrameworkTestType:
 
															             return self
														
 
															         else:  # This is quite common - most tests don't support all types
														
 
															             raise AttributeError(
														
 
															-                "A %s requires the benchmark_config.json to contain %s" % (self.name, self.args))
														
 
															+                "A %s requires the benchmark_config.json to contain %s" %
														
 
															+                (self.name, self.args))
														
 
															     def request_headers_and_body(self, url):
														
 
															         '''
														
@@ -94,7 +98,9 @@ class FrameworkTestType:
 
															         self.out.write(str(headers))
														
 
															         self.out.write(body)
														
 
															         b = 40
														
 
															-        print("  Response (trimmed to {:d} bytes): \"{!s}\"".format(b, body.strip()[:b]))
														
 
															+        print("  Response (trimmed to {:d} bytes): \"{!s}\"".format(
														
 
															+            b,
														
 
															+            body.strip()[:b]))
														
 
															         return headers, body
														
 
															     def verify(self, base_url):
														
@@ -146,7 +152,9 @@ class FrameworkTestType:
 
															         if database_name == "mysql":
														
 
															             try:
														
 
															-                db = MySQLdb.connect("TFB-database", "benchmarkdbuser", "benchmarkdbpass", "hello_world")
														
 
															+                db = MySQLdb.connect(self.config.database_host,
														
 
															+                                     "benchmarkdbuser", "benchmarkdbpass",
														
 
															+                                     "hello_world")
														
 
															                 cursor = db.cursor()
														
 
															                 cursor.execute("SELECT * FROM World")
														
 
															                 results = cursor.fetchall()
														
@@ -157,11 +165,12 @@ class FrameworkTestType:
 
															                 print(e)
														
 
															         elif database_name == "postgres":
														
 
															             try:
														
 
															-                db = psycopg2.connect(host="TFB-database",
														
 
															-                                      port="5432",
														
 
															-                                      user="benchmarkdbuser",
														
 
															-                                      password="benchmarkdbpass",
														
 
															-                                      database="hello_world")
														
 
															+                db = psycopg2.connect(
														
 
															+                    host=self.config.database_host,
														
 
															+                    port="5432",
														
 
															+                    user="benchmarkdbuser",
														
 
															+                    password="benchmarkdbpass",
														
 
															+                    database="hello_world")
														
 
															                 cursor = db.cursor()
														
 
															                 cursor.execute("SELECT * FROM \"World\"")
														
 
															                 results = cursor.fetchall()
														
@@ -177,20 +186,24 @@ class FrameworkTestType:
 
															         elif database_name == "mongodb":
														
 
															             try:
														
 
															                 worlds_json = {}
														
 
															-                connection = pymongo.MongoClient(host="TFB-database")
														
 
															+                connection = pymongo.MongoClient(
														
 
															+                    host=self.config.database_host)
														
 
															                 db = connection.hello_world
														
 
															                 for world in db.world.find():
														
 
															                     if "randomNumber" in world:
														
 
															                         if "id" in world:
														
 
															-                            worlds_json[str(int(world["id"]))] = int(world["randomNumber"])
														
 
															+                            worlds_json[str(int(world["id"]))] = int(
														
 
															+                                world["randomNumber"])
														
 
															                         elif "_id" in world:
														
 
															-                            worlds_json[str(int(world["_id"]))] = int(world["randomNumber"])
														
 
															+                            worlds_json[str(int(world["_id"]))] = int(
														
 
															+                                world["randomNumber"])
														
 
															                 results_json.append(worlds_json)
														
 
															                 connection.close()
														
 
															             except Exception as e:
														
 
															                 print("ERROR: Unable to load current MongoDB World table.")
														
 
															                 print(e)
														
 
															         else:
														
 
															-            raise ValueError("Database: {!s} does not exist".format(database_name))
														
 
															+            raise ValueError(
														
 
															+                "Database: {!s} does not exist".format(database_name))
														
 
															         return results_json
														
--- a/toolset/benchmark/test_types/json_type.py
+++ b/toolset/benchmark/test_types/json_type.py
@@ -1,29 +1,24 @@
 
															-from benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															-from benchmark.test_types.verifications import (
														
 
															-    basic_body_verification,
														
 
															-    verify_headers,
														
 
															-    verify_helloworld_object
														
 
															-)
														
 
															-
														
 
															-import json
														
 
															+from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															+from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_helloworld_object
														
 
															 class JsonTestType(FrameworkTestType):
														
 
															-
														
 
															-    def __init__(self):
														
 
															+    def __init__(self, config):
														
 
															+        self.json_url = ""
														
 
															         kwargs = {
														
 
															             'name': 'json',
														
 
															             'accept_header': self.accept('json'),
														
 
															             'requires_db': False,
														
 
															             'args': ['json_url']
														
 
															         }
														
 
															-        FrameworkTestType.__init__(self, **kwargs)
														
 
															+        FrameworkTestType.__init__(self, config, **kwargs)
														
 
															     def get_url(self):
														
 
															         return self.json_url
														
 
															     def verify(self, base_url):
														
 
															-        '''Validates the response is a JSON object of 
														
 
															+        '''
														
 
															+        Validates the response is a JSON object of 
														
 
															         { 'message' : 'hello, world!' }. Case insensitive and 
														
 
															         quoting style is ignored
														
 
															         '''
														
--- a/toolset/benchmark/test_types/plaintext_type.py
+++ b/toolset/benchmark/test_types/plaintext_type.py
@@ -1,17 +1,17 @@
 
															-from benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															-from benchmark.test_types.verifications import basic_body_verification, verify_headers
														
 
															+from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															+from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers
														
 
															 class PlaintextTestType(FrameworkTestType):
														
 
															-
														
 
															-    def __init__(self):
														
 
															+    def __init__(self, config):
														
 
															+        self.plaintext_url = ""
														
 
															         kwargs = {
														
 
															             'name': 'plaintext',
														
 
															             'requires_db': False,
														
 
															             'accept_header': self.accept('plaintext'),
														
 
															             'args': ['plaintext_url']
														
 
															         }
														
 
															-        FrameworkTestType.__init__(self, **kwargs)
														
 
															+        FrameworkTestType.__init__(self, config, **kwargs)
														
 
															     def verify(self, base_url):
														
 
															         url = base_url + self.plaintext_url
														
@@ -29,15 +29,15 @@ class PlaintextTestType(FrameworkTestType):
 
															         extra_bytes = len(body) - len(expected)
														
 
															         if expected not in body:
														
 
															-            return [('fail', "Could not find 'Hello, World!' in response.", url)]
														
 
															+            return [('fail', "Could not find 'Hello, World!' in response.",
														
 
															+                     url)]
														
 
															         if extra_bytes > 0:
														
 
															             problems.append(
														
 
															                 ('warn',
														
 
															                  ("Server is returning %s more bytes than are required. "
														
 
															-                  "This may negatively affect benchmark performance."
														
 
															-                  % (extra_bytes)),
														
 
															-                 url))
														
 
															+                  "This may negatively affect benchmark performance." %
														
 
															+                  (extra_bytes)), url))
														
 
															         problems += verify_headers(headers, url, should_be='plaintext')
														
--- a/toolset/benchmark/test_types/query_type.py
+++ b/toolset/benchmark/test_types/query_type.py
@@ -1,23 +1,17 @@
 
															-from benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															-from benchmark.test_types.verifications import (
														
 
															-    verify_headers,
														
 
															-    verify_randomnumber_list,
														
 
															-    verify_query_cases
														
 
															-)
														
 
															-
														
 
															-import json
														
 
															+from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															+from toolset.benchmark.test_types.verifications import verify_query_cases
														
 
															 class QueryTestType(FrameworkTestType):
														
 
															-
														
 
															-    def __init__(self):
														
 
															+    def __init__(self, config):
														
 
															+        self.query_url = ""
														
 
															         kwargs = {
														
 
															             'name': 'query',
														
 
															             'accept_header': self.accept('json'),
														
 
															             'requires_db': True,
														
 
															             'args': ['query_url']
														
 
															         }
														
 
															-        FrameworkTestType.__init__(self, **kwargs)
														
 
															+        FrameworkTestType.__init__(self, config, **kwargs)
														
 
															     def get_url(self):
														
 
															         return self.query_url
														
@@ -32,13 +26,8 @@ class QueryTestType(FrameworkTestType):
 
															         '''
														
 
															         url = base_url + self.query_url
														
 
															-        cases = [
														
 
															-            ('2',   'fail'),
														
 
															-            ('0',   'fail'),
														
 
															-            ('foo', 'fail'),
														
 
															-            ('501', 'warn'),
														
 
															-            ('',    'fail')
														
 
															-        ]
														
 
															+        cases = [('2', 'fail'), ('0', 'fail'), ('foo', 'fail'),
														
 
															+                 ('501', 'warn'), ('', 'fail')]
														
 
															         problems = verify_query_cases(self, cases, url)
														
--- a/toolset/benchmark/test_types/update_type.py
+++ b/toolset/benchmark/test_types/update_type.py
@@ -1,23 +1,24 @@
 
															-from benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															-from benchmark.test_types.verifications import verify_query_cases
														
 
															+from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
														
 
															+from toolset.benchmark.test_types.verifications import verify_query_cases
														
 
															 class UpdateTestType(FrameworkTestType):
														
 
															-
														
 
															-    def __init__(self):
														
 
															+    def __init__(self, config):
														
 
															+        self.update_url = ""
														
 
															         kwargs = {
														
 
															             'name': 'update',
														
 
															             'accept_header': self.accept('json'),
														
 
															             'requires_db': True,
														
 
															             'args': ['update_url', 'database']
														
 
															         }
														
 
															-        FrameworkTestType.__init__(self, **kwargs)
														
 
															+        FrameworkTestType.__init__(self, config, **kwargs)
														
 
															     def get_url(self):
														
 
															         return self.update_url
														
 
															     def verify(self, base_url):
														
 
															-        '''Validates the response is a JSON array of 
														
 
															+        '''
														
 
															+        Validates the response is a JSON array of 
														
 
															         the proper length, each JSON Object in the array 
														
 
															         has keys 'id' and 'randomNumber', and these keys 
														
 
															         map to integers. Case insensitive and 
														
@@ -25,13 +26,8 @@ class UpdateTestType(FrameworkTestType):
 
															         '''
														
 
															         url = base_url + self.update_url
														
 
															-        cases = [
														
 
															-            ('2',   'fail'),
														
 
															-            ('0',   'fail'),
														
 
															-            ('foo', 'fail'),
														
 
															-            ('501', 'warn'),
														
 
															-            ('',    'fail')
														
 
															-        ]
														
 
															+        cases = [('2', 'fail'), ('0', 'fail'), ('foo', 'fail'),
														
 
															+                 ('501', 'warn'), ('', 'fail')]
														
 
															         problems = verify_query_cases(self, cases, url, True)
														
 
															         if len(problems) == 0:
														
--- a/toolset/benchmark/test_types/verifications.py
+++ b/toolset/benchmark/test_types/verifications.py
@@ -1,6 +1,5 @@
 
															 import json
														
 
															 import re
														
 
															-import math
														
 
															 def basic_body_verification(body, url, is_json_check=True):
														
@@ -48,24 +47,23 @@ def verify_headers(headers, url, should_be='json'):
 
															     problems = []
														
 
															-    for v in (v for v in ('Server', 'Date', 'Content-Type') if v.lower() not in headers):
														
 
															-        problems.append(
														
 
															-            ('warn', 'Required response header missing: %s' % v, url))
														
 
															+    for v in (v for v in ('Server', 'Date', 'Content-Type')
														
 
															+              if v.lower() not in headers):
														
 
															+        problems.append(('warn', 'Required response header missing: %s' % v,
														
 
															+                         url))
														
 
															-    if all(v.lower() not in headers for v in ('Content-Length', 'Transfer-Encoding')):
														
 
															-        problems.append(
														
 
															-            ('warn',
														
 
															-             'Required response size header missing, please include either "Content-Length" or "Transfer-Encoding"',
														
 
															-             url))
														
 
															+    if all(v.lower() not in headers
														
 
															+           for v in ('Content-Length', 'Transfer-Encoding')):
														
 
															+        problems.append((
														
 
															+            'warn',
														
 
															+            'Required response size header missing, please include either "Content-Length" or "Transfer-Encoding"',
														
 
															+            url))
														
 
															     content_type = headers.get('Content-Type', None)
														
 
															     if content_type is None:
														
 
															-        problems.append(
														
 
															-            ('warn',
														
 
															-             'No content encoding found, expected \"%s\"' % (
														
 
															-                 expected_type),
														
 
															-             url))
														
 
															+        problems.append(('warn', 'No content encoding found, expected \"%s\"' %
														
 
															+                         (expected_type), url))
														
 
															     else:
														
 
															         # Split out "charset=utf-8" if it's included
														
 
															         content_type_list = re.split('; *', content_type.lower())
														
@@ -73,31 +71,25 @@ def verify_headers(headers, url, should_be='json'):
 
															         # "text/html" requires charset to be set. The others do not
														
 
															         if expected_type == types['html']:
														
 
															             if expected_type not in content_type_list:
														
 
															-                problems.append(
														
 
															-                    ('warn',
														
 
															-                     'Unexpected content encoding, found \"%s\", expected \"%s\".' % (
														
 
															-                         content_type, expected_type + '; ' + charset),
														
 
															-                     url))
														
 
															+                problems.append((
														
 
															+                    'warn',
														
 
															+                    'Unexpected content encoding, found \"%s\", expected \"%s\".'
														
 
															+                    % (content_type, expected_type + '; ' + charset), url))
														
 
															             elif charset not in content_type_list:
														
 
															-                problems.append(
														
 
															-                    ('warn',
														
 
															-                     ('The \"%s\" content type requires \"charset=utf-8\" to be specified.'
														
 
															-                      % expected_type),
														
 
															-                     url))
														
 
															+                problems.append(('warn', (
														
 
															+                    'The \"%s\" content type requires \"charset=utf-8\" to be specified.'
														
 
															+                    % expected_type), url))
														
 
															         else:
														
 
															             if expected_type not in content_type_list:
														
 
															-                problems.append(
														
 
															-                    ('warn',
														
 
															-                     'Unexpected content encoding, found \"%s\", expected \"%s\"' % (
														
 
															-                         content_type, expected_type),
														
 
															-                     url))
														
 
															+                problems.append((
														
 
															+                    'warn',
														
 
															+                    'Unexpected content encoding, found \"%s\", expected \"%s\"'
														
 
															+                    % (content_type, expected_type), url))
														
 
															             elif charset in content_type_list:
														
 
															-                problems.append(
														
 
															-                    ('warn',
														
 
															-                     ("Content encoding found in \"%s\" where \"%s\" is acceptable.\n"
														
 
															-                      "Additional response bytes may negatively affect benchmark performance."
														
 
															-                      % (content_type, expected_type)),
														
 
															-                     url))
														
 
															+                problems.append(('warn', (
														
 
															+                    "Content encoding found in \"%s\" where \"%s\" is acceptable.\n"
														
 
															+                    "Additional response bytes may negatively affect benchmark performance."
														
 
															+                    % (content_type, expected_type)), url))
														
 
															     return problems
														
@@ -110,8 +102,7 @@ def verify_helloworld_object(json_object, url):
 
															     problems = []
														
 
															     # Make everything case insensitive
														
 
															-    json_object = {k.lower(): v.lower()
														
 
															-                   for k, v in json_object.iteritems()}
														
 
															+    json_object = {k.lower(): v.lower() for k, v in json_object.iteritems()}
														
 
															     if 'message' not in json_object:
														
 
															         return [('fail', "Missing required key 'message'", url)]
														
@@ -121,14 +112,19 @@ def verify_helloworld_object(json_object, url):
 
															             additional = (', ').join(
														
 
															                 [k for k in json_object.keys() if k != 'message'])
														
 
															             problems.append(
														
 
															-                ('warn', "Too many JSON key/value pairs, consider removing: %s" % additional, url))
														
 
															+                ('warn', "Too many JSON key/value pairs, consider removing: %s"
														
 
															+                 % additional, url))
														
 
															         if json_len > 27:
														
 
															             problems.append(
														
 
															-                'warn', "%s additional response byte(s) found. Consider removing unnecessary whitespace." % (json_len - 26))
														
 
															+                'warn',
														
 
															+                "%s additional response byte(s) found. Consider removing unnecessary whitespace."
														
 
															+                % (json_len - 26))
														
 
															         message = json_object['message']
														
 
															         if message != 'hello, world!':
														
 
															-            return [('fail', "Expected message of 'hello, world!', got '%s'" % message, url)]
														
 
															+            return [('fail',
														
 
															+                     "Expected message of 'hello, world!', got '%s'" % message,
														
 
															+                     url)]
														
 
															         return problems
														
@@ -148,7 +144,8 @@ def verify_randomnumber_object(db_object, url, max_infraction='fail'):
 
															         got = str(db_object)[:20]
														
 
															         if len(str(db_object)) > 20:
														
 
															             got = str(db_object)[:17] + '...'
														
 
															-        return [(max_infraction, "Expected a JSON object, got '%s' instead" % got, url)]
														
 
															+        return [(max_infraction,
														
 
															+                 "Expected a JSON object, got '%s' instead" % got, url)]
														
 
															     # Make keys case insensitive
														
 
															     db_object = {k.lower(): v for k, v in db_object.iteritems()}
														
@@ -156,15 +153,14 @@ def verify_randomnumber_object(db_object, url, max_infraction='fail'):
 
															     for v in (v for v in required_keys if v not in db_object):
														
 
															         problems.append(
														
 
															-            (max_infraction, 'Response object was missing required key: %s' % v, url))
														
 
															+            (max_infraction,
														
 
															+             'Response object was missing required key: %s' % v, url))
														
 
															     if len(db_object) > len(required_keys):
														
 
															         extras = set(db_object.keys()) - required_keys
														
 
															         problems.append(
														
 
															-            ('warn',
														
 
															-             'An extra key(s) is being included with the db object: %s' % ', '.join(
														
 
															-                 extras),
														
 
															-             url))
														
 
															+            ('warn', 'An extra key(s) is being included with the db object: %s'
														
 
															+             % ', '.join(extras), url))
														
 
															     # All required keys must be present
														
 
															     if len(problems) > 0:
														
@@ -175,31 +171,37 @@ def verify_randomnumber_object(db_object, url, max_infraction='fail'):
 
															         o_id = int(db_object['id'])
														
 
															         if o_id > 10000 or o_id < 1:
														
 
															-            problems.append(
														
 
															-                ('warn',
														
 
															-                 'Response key id should be between 1 and 10,000: ' +
														
 
															-                 str(o_id),
														
 
															-                 url))
														
 
															+            problems.append((
														
 
															+                'warn',
														
 
															+                'Response key id should be between 1 and 10,000: ' + str(o_id),
														
 
															+                url))
														
 
															     except TypeError as e:
														
 
															         problems.append(
														
 
															-            (max_infraction, "Response key 'id' does not map to an integer - %s" % e, url))
														
 
															+            (max_infraction,
														
 
															+             "Response key 'id' does not map to an integer - %s" % e, url))
														
 
															     try:
														
 
															         o_rn = int(db_object['randomnumber'])
														
 
															         if o_rn > 10000:
														
 
															-            problems.append(
														
 
															-                ('warn',
														
 
															-                 'Response key `randomNumber` is over 10,000. This may negatively affect performance by sending extra bytes',
														
 
															-                 url))
														
 
															+            problems.append((
														
 
															+                'warn',
														
 
															+                'Response key `randomNumber` is over 10,000. This may negatively affect performance by sending extra bytes',
														
 
															+                url))
														
 
															     except TypeError as e:
														
 
															         problems.append(
														
 
															-            (max_infraction, "Response key 'randomnumber' does not map to an integer - %s" % e, url))
														
 
															+            (max_infraction,
														
 
															+             "Response key 'randomnumber' does not map to an integer - %s" % e,
														
 
															+             url))
														
 
															     return problems
														
 
															-def verify_randomnumber_list(expected_len, headers, body, url, max_infraction='fail'):
														
 
															+def verify_randomnumber_list(expected_len,
														
 
															+                             headers,
														
 
															+                             body,
														
 
															+                             url,
														
 
															+                             max_infraction='fail'):
														
 
															     '''
														
 
															     Validates that the object is a list containing a number of
														
 
															     randomnumber object. Should closely resemble:
														
@@ -215,21 +217,20 @@ def verify_randomnumber_list(expected_len, headers, body, url, max_infraction='f
 
															     # rather than a list containing one element. We allow this with a warn,
														
 
															     # then verify the supplied object
														
 
															     if type(response) is not list:
														
 
															-        problems.append(
														
 
															-            ('warn', 'Top-level JSON is an object, not an array', url))
														
 
															+        problems.append(('warn', 'Top-level JSON is an object, not an array',
														
 
															+                         url))
														
 
															         problems += verify_randomnumber_object(response, url, max_infraction)
														
 
															         return problems
														
 
															     if any(type(item) is not dict for item in response):
														
 
															         problems.append(
														
 
															-            (max_infraction, 'Not all items in the JSON array were JSON objects', url))
														
 
															+            (max_infraction,
														
 
															+             'Not all items in the JSON array were JSON objects', url))
														
 
															     if len(response) != expected_len:
														
 
															-        problems.append(
														
 
															-            (max_infraction,
														
 
															-             "JSON array length of %s != expected length of %s" % (
														
 
															-                 len(response), expected_len),
														
 
															-             url))
														
 
															+        problems.append((max_infraction,
														
 
															+                         "JSON array length of %s != expected length of %s" %
														
 
															+                         (len(response), expected_len), url))
														
 
															     # Verify individual objects, arbitrarily stop after 5 bad ones are found
														
 
															     # i.e. to not look at all 500
														
@@ -249,11 +250,12 @@ def verify_randomnumber_list(expected_len, headers, body, url, max_infraction='f
 
															     return problems
														
 
															+
														
 
															 def verify_updates(old_worlds, new_worlds, updates_expected, url):
														
 
															     '''
														
 
															     Validates that the /updates requests actually updated values in the database and didn't
														
 
															     just return a JSON list of the correct number of World items.
														
 
															-sz
														
 
															+
														
 
															     old_worlds  a JSON object containing the state of the Worlds table BEFORE the /updates requests
														
 
															     new_worlds  a JSON object containing the state of the Worlds table AFTER the /updates requests
														
 
															     If no items were updated, this validation test returns a "fail."
														
@@ -270,28 +272,30 @@ sz
 
															         for i in range(1, 10001):
														
 
															             try:
														
 
															                 entry_id = str(i)
														
 
															-                if entry_id in old_worlds[n] and entry_id  in new_worlds[n]:
														
 
															+                if entry_id in old_worlds[n] and entry_id in new_worlds[n]:
														
 
															                     if old_worlds[n][entry_id] != new_worlds[n][entry_id]:
														
 
															                         successful_updates += 1
														
 
															             except Exception as e:
														
 
															-                print e
														
 
															+                print(e)
														
 
															         n += 1
														
 
															     if successful_updates == 0:
														
 
															-        problems.append(
														
 
															-            ("fail", "No items were updated in the database.", url))
														
 
															+        problems.append(("fail", "No items were updated in the database.",
														
 
															+                         url))
														
 
															     elif successful_updates <= (updates_expected * 0.90):
														
 
															-        problems.append(
														
 
															-            ("fail", "Only %s items were updated in the database out of roughly %s expected." % (successful_updates, updates_expected), url))
														
 
															+        problems.append((
														
 
															+            "fail",
														
 
															+            "Only %s items were updated in the database out of roughly %s expected."
														
 
															+            % (successful_updates, updates_expected), url))
														
 
															     elif successful_updates <= (updates_expected * 0.95):
														
 
															-        problems.append(
														
 
															-            ("warn",
														
 
															-             "There may have been an error updating the database. Only %s items were updated in the database out of the roughly %s expected." % (
														
 
															-                 successful_updates, updates_expected),
														
 
															-             url))
														
 
															+        problems.append((
														
 
															+            "warn",
														
 
															+            "There may have been an error updating the database. Only %s items were updated in the database out of the roughly %s expected."
														
 
															+            % (successful_updates, updates_expected), url))
														
 
															     return problems
														
 
															+
														
 
															 def verify_query_cases(self, cases, url, check_updates=False):
														
 
															     '''
														
 
															     The /updates and /queries tests accept a `queries` parameter
														
@@ -336,8 +340,8 @@ def verify_query_cases(self, cases, url, check_updates=False):
 
															             else:
														
 
															                 expected_len = queries
														
 
															-            problems += verify_randomnumber_list(
														
 
															-                expected_len, headers, body, case_url, max_infraction)
														
 
															+            problems += verify_randomnumber_list(expected_len, headers, body,
														
 
															+                                                 case_url, max_infraction)
														
 
															             problems += verify_headers(headers, case_url)
														
 
															             # Only check update changes if we are doing an Update verification and if we're testing
														
@@ -346,7 +350,8 @@ def verify_query_cases(self, cases, url, check_updates=False):
 
															             # previously held
														
 
															             if check_updates and queries >= MAX:
														
 
															                 world_db_after = self.get_current_world_table()
														
 
															-                problems += verify_updates(world_db_before, world_db_after, MAX, case_url)
														
 
															+                problems += verify_updates(world_db_before, world_db_after,
														
 
															+                                           MAX, case_url)
														
 
															         except ValueError:
														
 
															             warning = (
														
@@ -355,15 +360,11 @@ def verify_query_cases(self, cases, url, check_updates=False):
 
															                 '(this will be a failure in future rounds, please fix)')
														
 
															             if body is None:
														
 
															-                problems.append(
														
 
															-                    (max_infraction,
														
 
															-                     warning % ('No response', q),
														
 
															-                     case_url))
														
 
															+                problems.append((max_infraction, warning % ('No response', q),
														
 
															+                                 case_url))
														
 
															             elif len(body) == 0:
														
 
															-                problems.append(
														
 
															-                    (max_infraction,
														
 
															-                     warning % ('Empty response', q),
														
 
															-                     case_url))
														
 
															+                problems.append((max_infraction, warning % ('Empty response',
														
 
															+                                                            q), case_url))
														
 
															             else:
														
 
															                 expected_len = 1
														
 
															                 # Strictness will be upped in a future round, i.e. Frameworks currently do not have
														
--- a/toolset/benchmark/utils.py
+++ b/toolset/benchmark/utils.py
@@ -1,241 +0,0 @@
 
															-import ConfigParser
														
 
															-import os
														
 
															-import glob
														
 
															-import json
														
 
															-import socket
														
 
															-import fnmatch
														
 
															-
														
 
															-from ast import literal_eval
														
 
															-
														
 
															-def find_docker_file(path, pattern):
														
 
															-    for root, dirs, files in os.walk(path):
														
 
															-        for name in files:
														
 
															-            if fnmatch.fnmatch(name, pattern):
														
 
															-                return os.path.join(root, name)
														
 
															-
														
 
															-def gather_docker_dependencies(docker_file):
														
 
															-    '''
														
 
															-    Gathers all the known docker dependencies for the given docker image.
														
 
															-    '''
														
 
															-    # Avoid setting up a circular import
														
 
															-    from setup.linux import setup_util
														
 
															-    deps = []
														
 
															-
														
 
															-    docker_dir = os.path.join(setup_util.get_fwroot(), "toolset", "setup", "linux", "docker")
														
 
															-
														
 
															-    if os.path.exists(docker_file):
														
 
															-        with open(docker_file) as fp:
														
 
															-            for line in fp.readlines():
														
 
															-                tokens = line.strip().split(' ')
														
 
															-                if tokens[0] == "FROM":
														
 
															-                    # This is magic that our base image points to
														
 
															-                    if tokens[1] != "ubuntu:16.04":
														
 
															-                        depToken = tokens[1].strip().split(':')[0].strip().split('/')[1]
														
 
															-                        deps.append(depToken)
														
 
															-                        dep_docker_file = os.path.join(os.path.dirname(docker_file), depToken + ".dockerfile")
														
 
															-                        if not os.path.exists(dep_docker_file):
														
 
															-                            dep_docker_file = find_docker_file(docker_dir, depToken + ".dockerfile")
														
 
															-                        deps.extend(gather_docker_dependencies(dep_docker_file))
														
 
															-
														
 
															-    return deps
														
 
															-
														
 
															-
														
 
															-def gather_langauges():
														
 
															-    '''
														
 
															-    Gathers all the known languages in the suite via the folder names
														
 
															-    beneath FWROOT.
														
 
															-    '''
														
 
															-    # Avoid setting up a circular import
														
 
															-    from setup.linux import setup_util
														
 
															-
														
 
															-    lang_dir = os.path.join(setup_util.get_fwroot(), "frameworks")
														
 
															-    langs = []
														
 
															-    for dir in glob.glob(os.path.join(lang_dir, "*")):
														
 
															-        langs.append(dir.replace(lang_dir,"")[1:])
														
 
															-    return langs
														
 
															-
														
 
															-def gather_tests(include = [], exclude=[], benchmarker=None):
														
 
															-    '''
														
 
															-    Given test names as strings, returns a list of FrameworkTest objects.
														
 
															-    For example, 'aspnet-mysql-raw' turns into a FrameworkTest object with
														
 
															-    variables for checking the test directory, the test database os, and
														
 
															-    other useful items.
														
 
															-
														
 
															-    With no arguments, every test in this framework will be returned.
														
 
															-    With include, only tests with this exact name will be returned.
														
 
															-    With exclude, all tests but those excluded will be returned.
														
 
															-
														
 
															-    A benchmarker is needed to construct full FrameworkTest objects. If
														
 
															-    one is not provided, a default Benchmarker will be created.
														
 
															-    '''
														
 
															-
														
 
															-    # Avoid setting up a circular import
														
 
															-    from benchmark import framework_test
														
 
															-    from benchmark.benchmarker import Benchmarker
														
 
															-    from setup.linux import setup_util
														
 
															-
														
 
															-    # Help callers out a bit
														
 
															-    if include is None:
														
 
															-        include = []
														
 
															-    if exclude is None:
														
 
															-        exclude = []
														
 
															-
														
 
															-    # Old, hacky method to exclude all tests was to
														
 
															-    # request a test known to not exist, such as ''.
														
 
															-    # If test '' was requested, short-circuit and return
														
 
															-    # nothing immediately
														
 
															-    if len(include) == 1 and '' in include:
														
 
															-        return []
														
 
															-
														
 
															-    # Setup default Benchmarker using example configuration
														
 
															-    if benchmarker is None:
														
 
															-        default_config = setup_util.get_fwroot() + "/benchmark.cfg"
														
 
															-        config = ConfigParser.SafeConfigParser()
														
 
															-        config.readfp(open(default_config))
														
 
															-        defaults = dict(config.items("Defaults"))
														
 
															-
														
 
															-        # Convert strings into proper python types
														
 
															-        for k,v in defaults.iteritems():
														
 
															-            try:
														
 
															-                defaults[k] = literal_eval(v)
														
 
															-            except Exception:
														
 
															-                pass
														
 
															-
														
 
															-        # Ensure we only run the __init__ method of Benchmarker
														
 
															-        defaults['install'] = None
														
 
															-        defaults['results_name'] = "(unspecified, datetime = %Y-%m-%d %H:%M:%S)"
														
 
															-        defaults['results_environment'] = "My Server Environment"
														
 
															-        defaults['test_dir'] = None
														
 
															-        defaults['test_lang'] = None
														
 
															-        defaults['quiet'] = True
														
 
															-
														
 
															-        benchmarker = Benchmarker(defaults)
														
 
															-
														
 
															-
														
 
															-    # Search for configuration files
														
 
															-    fwroot = setup_util.get_fwroot()
														
 
															-    config_files = []
														
 
															-
														
 
															-    if benchmarker.test_lang:
														
 
															-        benchmarker.test_dir = []
														
 
															-        for lang in benchmarker.test_lang:
														
 
															-            if os.path.exists("{!s}/frameworks/{!s}".format(fwroot, lang)):
														
 
															-                for test_dir in os.listdir("{!s}/frameworks/{!s}".format(fwroot, lang)):
														
 
															-                    benchmarker.test_dir.append("{!s}/{!s}".format(lang, test_dir))
														
 
															-            else:
														
 
															-                raise Exception("Unable to locate language directory: {!s}".format(lang))
														
 
															-
														
 
															-    if benchmarker.test_dir:
														
 
															-        for test_dir in benchmarker.test_dir:
														
 
															-            dir_config_files = glob.glob("{!s}/frameworks/{!s}/benchmark_config.json".format(fwroot, test_dir))
														
 
															-            if len(dir_config_files):
														
 
															-                config_files.extend(dir_config_files)
														
 
															-            else:
														
 
															-                raise Exception("Unable to locate tests in test-dir: {!s}".format(test_dir))
														
 
															-    else:
														
 
															-        config_files.extend(glob.glob("{!s}/frameworks/*/*/benchmark_config.json".format(fwroot)))
														
 
															-
														
 
															-    tests = []
														
 
															-    for config_file_name in config_files:
														
 
															-        config = None
														
 
															-        with open(config_file_name, 'r') as config_file:
														
 
															-            try:
														
 
															-                config = json.load(config_file)
														
 
															-            except ValueError:
														
 
															-                # User-friendly errors
														
 
															-                print("Error loading '{!s}'.".format(config_file_name))
														
 
															-                raise
														
 
															-
														
 
															-        # Find all tests in the config file
														
 
															-        config_tests = framework_test.parse_config(config,
														
 
															-                                                   os.path.dirname(config_file_name), benchmarker)
														
 
															-
														
 
															-        # Filter
														
 
															-        for test in config_tests:
														
 
															-            if len(include) is 0 and len(exclude) is 0:
														
 
															-                # No filters, we are running everything
														
 
															-                tests.append(test)
														
 
															-            elif test.name in exclude:
														
 
															-                continue
														
 
															-            elif test.name in include:
														
 
															-                tests.append(test)
														
 
															-            else:
														
 
															-                # An include list exists, but this test is
														
 
															-                # not listed there, so we ignore it
														
 
															-                pass
														
 
															-
														
 
															-    # Ensure we were able to locate everything that was
														
 
															-    # explicitly included
														
 
															-    if 0 != len(include):
														
 
															-        names = {test.name for test in tests}
														
 
															-        if 0 != len(set(include) - set(names)):
														
 
															-            missing = list(set(include) - set(names))
														
 
															-            raise Exception("Unable to locate tests %s" % missing)
														
 
															-
														
 
															-    tests.sort(key=lambda x: x.name)
														
 
															-    return tests
														
 
															-
														
 
															-def gather_frameworks(include = [], exclude=[], benchmarker=None):
														
 
															-    '''Return a dictionary mapping frameworks->[test1,test2,test3]
														
 
															-    for quickly grabbing all tests in a grouped manner.
														
 
															-    Args have the same meaning as gather_tests'''
														
 
															-
														
 
															-    tests = gather_tests(include, exclude, benchmarker)
														
 
															-    frameworks = dict()
														
 
															-
														
 
															-    for test in tests:
														
 
															-        if test.framework not in frameworks:
														
 
															-            frameworks[test.framework] = []
														
 
															-        frameworks[test.framework].append(test)
														
 
															-    return frameworks
														
 
															-
														
 
															-def header(message, top='-', bottom='-'):
														
 
															-    '''
														
 
															-    Generates a clean header
														
 
															-    '''
														
 
															-    topheader = (top * 80)[:80]
														
 
															-    bottomheader = (bottom * 80)[:80]
														
 
															-    result = ""
														
 
															-    if topheader != "":
														
 
															-        result += "%s" % topheader
														
 
															-    if message != "":
														
 
															-        if result == "":
														
 
															-            result = "  %s" % message
														
 
															-        else:
														
 
															-            result += "\n  %s" % message
														
 
															-    if bottomheader != "":
														
 
															-        if result == "":
														
 
															-            result = "%s" % bottomheader
														
 
															-        else:
														
 
															-            result += "\n%s" % bottomheader
														
 
															-    return result + '\n'
														
 
															-
														
 
															-def check_services(services):
														
 
															-
														
 
															-    def check_service(address, port):
														
 
															-        try:
														
 
															-            s = socket.socket()
														
 
															-            s.settimeout(20)
														
 
															-            s.connect((address, port))
														
 
															-            return (True, "")
														
 
															-        except Exception as ex:
														
 
															-            return (False, ex)
														
 
															-        finally:
														
 
															-            s.close
														
 
															-
														
 
															-    res = []
														
 
															-    for s in services:
														
 
															-        r = check_service(s[1], s[2])
														
 
															-        res.append((s[0], r[0], str(r[1])))
														
 
															-    return res
														
 
															-
														
 
															-def verify_database_connections(services):
														
 
															-    allGo = True
														
 
															-    messages = []
														
 
															-    for r in check_services(services):
														
 
															-        if r[1]:
														
 
															-            messages.append(r[0] + ": is GO!")
														
 
															-        else:
														
 
															-            messages.append(r[0] + ": is _NO_ GO!: ERROR: " + r[2])
														
 
															-            allGo = False
														
 
															-    return (allGo, messages)
														
--- a/toolset/continuous/tasks/keep-logs.py
+++ b/toolset/continuous/tasks/keep-logs.py
@@ -1,18 +1,16 @@
 
															-#!/usr/bin/python
														
 
															 #
														
 
															-# Archives, to the specified folder, the logged output generated by a benchmark 
														
 
															-# run.  
														
 
															+# Archives, to the specified folder, the logged output generated by a benchmark
														
 
															+# run.
														
 
															 #
														
 
															 # @author A. Shawn Bandy
														
 
															 import os
														
 
															 import zipfile
														
 
															 import datetime
														
 
															 import requests
														
 
															-import shutil
														
 
															 # Follows closely from:
														
 
															 # http://stackoverflow.com/a/34153816
														
 
															 #
														
 
															-# Paths to the log folders are generated by TFB and where those files 
														
 
															+# Paths to the log folders are generated by TFB and where those files
														
 
															 # should be archived.
														
 
															 #
														
 
															 path_in = os.path.abspath(os.path.normpath(os.path.expanduser(os.path.join( \
														
@@ -24,15 +22,15 @@ path_out = os.path.abspath(os.path.join(os.environ['TFB_LOGSFOLDER'], \
 
															     dt_folder))
														
 
															 if not os.path.exists(path_out):
														
 
															-  os.makedirs(path_out)
														
 
															+    os.makedirs(path_out)
														
 
															 zip_path = path_out + '/results.zip'
														
 
															 zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
														
 
															 for root, dirs, files in os.walk(path_in):
														
 
															-  for file in files:
														
 
															-    zip_file.write(os.path.join(root, file))
														
 
															+    for file in files:
														
 
															+        zip_file.write(os.path.join(root, file))
														
 
															 zip_file.close()
														
@@ -40,4 +38,7 @@ results_upload_uri = os.environ['TFB_UPLOADURI']
 
															 if results_upload_uri != None:
														
 
															     with open(zip_path, 'rb') as file_to_upload:
														
 
															-        requests.post(results_upload_uri, headers={ 'Content-Type': 'application/zip' }, data=file_to_upload)
														
 
															+        requests.post(
														
 
															+            results_upload_uri,
														
 
															+            headers={'Content-Type': 'application/zip'},
														
 
															+            data=file_to_upload)
														
--- a/toolset/continuous/tasks/run-benchmarks.sh
+++ b/toolset/continuous/tasks/run-benchmarks.sh
@@ -5,4 +5,4 @@
 
															 echo Change to benchmark root
														
 
															 cd $TFB_REPOPARENT/$TFB_REPONAME
														
 
															 echo Running tests
														
 
															-toolset/run-tests.py
														
 
															+PYTHONPATH=$TFB_REPOPARENT/$TFB_REPONAME python toolset/run-tests.py
														
--- a/toolset/continuous/tasks/run-tasks.sh
+++ b/toolset/continuous/tasks/run-tasks.sh
@@ -20,4 +20,4 @@ $TFB_REPOPARENT/$TFB_REPONAME/toolset/continuous/tasks/run-benchmarks.sh
 
															 #  Tasks after the run   #
														
 
															 ##########################
														
 
															-$TFB_REPOPARENT/$TFB_REPONAME/toolset/continuous/tasks/keep-logs.py
														
 
															+PYTHONPATH=$TFB_REPOPARENT/$TFB_REPONAME python $TFB_REPOPARENT/$TFB_REPONAME/toolset/continuous/tasks/keep-logs.py
														
--- a/toolset/initializer.py
+++ b/toolset/initializer.py
@@ -1,105 +0,0 @@
 
															-import subprocess, os
														
 
															-from setup.linux import setup_util
														
 
															-
														
 
															-DEVNULL = open(os.devnull, 'w')
														
 
															-
														
 
															-def initialize(args):
														
 
															-  fwroot = setup_util.get_fwroot()
														
 
															-  dbuser = args.database_user
														
 
															-  dbhost = args.database_host
														
 
															-  dbiden = args.database_identity_file
														
 
															-  cluser = args.client_user
														
 
															-  clhost = args.client_host
														
 
															-  cliden = args.client_identity_file
														
 
															-  aphost = args.server_host
														
 
															-
														
 
															-  # test ssh connections to all the machines
														
 
															-  client_conn = __check_connection(cluser, clhost, cliden, aphost)
														
 
															-  database_conn = __check_connection(dbuser, dbhost, dbiden, aphost)
														
 
															-
														
 
															-  conn_success = client_conn and database_conn
														
 
															-  if not conn_success and not args.quiet:
														
 
															-    return __print_failure()
														
 
															-  
														
 
															-  # set up client machine
														
 
															-  if not __init_client(fwroot, cluser, clhost, cliden, args.quiet) and not args.quiet:
														
 
															-    return __print_failure()
														
 
															-
														
 
															-
														
 
															-  # set up database software
														
 
															-  if not __init_database(fwroot, dbuser, dbhost, dbiden, args.quiet) and not args.quiet:
														
 
															-    return __print_failure()
														
 
															-
														
 
															-def __print_failure():
														
 
															-  print("""
														
 
															--------------------------------------------------------------------------------
														
 
															-  This wizard is intended to help configure the required software on all the
														
 
															-  machines in the ecosystem specified in benchmark.cfg.
														
 
															-
														
 
															-  Note: It is expected that you have already set up passwordless-sudo on all
														
 
															-  of the machines (app, database, client) as well as identity file based 
														
 
															-  authentication and hostname setup in your hosts file. 
														
 
															-  More information on this required setup can be found at:
														
 
															-
														
 
															-  frameworkbenchmarks.readthedocs.io/en/latest/Development/Installation-Guide/
														
 
															-
														
 
															-  Please ensure that your benchmark.cfg is correctly configured as well as all
														
 
															-  of the machines (app, database, client).
														
 
															--------------------------------------------------------------------------------""")
														
 
															-
														
 
															-def __ssh_string(user, host, identity_file):
														
 
															-  return ["ssh", "-T", "-o", "StrictHostKeyChecking=no", "%s@%s" % (user, host), "-i", identity_file]
														
 
															-  
														
 
															-def __check_connection(user, host, identity_file, app_host):
														
 
															-  ''' 
														
 
															-  Checks that the given user and host are accessible via ssh with the given
														
 
															-  identity file and have the the following permissions:
														
 
															-    1. passwordless sudo
														
 
															-    2. ability to ssh back to app machine
														
 
															-  '''
														
 
															-  client_conn = True
														
 
															-  try:
														
 
															-    p = subprocess.Popen(__ssh_string(user, host, identity_file), 
														
 
															-      stdin=subprocess.PIPE, stdout=DEVNULL, stderr=DEVNULL)
														
 
															-    p.communicate("ssh -T -o StrictHostKeyChecking=no %s" % app_host)
														
 
															-    if p.returncode:
														
 
															-      client_conn = False
														
 
															-  except Exception as e:
														
 
															-    client_conn = False
														
 
															-  return client_conn
														
 
															-
														
 
															-def __init_client(fwroot, user, host, identity_file, quiet):
														
 
															-  '''
														
 
															-  Initializes and configures the software required to run the suite on the 
														
 
															-  client machine.
														
 
															-  '''
														
 
															-  if not quiet:
														
 
															-    print("INSTALL: Installing client software")
														
 
															-  with open (os.path.join(fwroot, "toolset", "setup", "linux", "client.sh"), "r") as myfile:
														
 
															-    remote_script=myfile.read()
														
 
															-    if quiet:
														
 
															-      p = subprocess.Popen(__ssh_string(user, host, identity_file), 
														
 
															-        stdin=subprocess.PIPE, stdout=DEVNULL, stderr=DEVNULL)
														
 
															-    else:
														
 
															-      p = subprocess.Popen(__ssh_string(user, host, identity_file), 
														
 
															-        stdin=subprocess.PIPE)
														
 
															-    p.communicate(remote_script)
														
 
															-    return p.returncode == 0
														
 
															-
														
 
															-def __init_database(fwroot, user, host, identity_file, quiet):
														
 
															-  '''
														
 
															-  Initializes and configures the software required to run the suite on the
														
 
															-  database machine.
														
 
															-  '''
														
 
															-  if not quiet:
														
 
															-    print("INSTALL: Installing database software")
														
 
															-  with open(os.path.join(fwroot, "toolset", "setup", "linux", "database.sh"), "r") as myfile:
														
 
															-    remote_script=myfile.read()
														
 
															-    if quiet:
														
 
															-      p = subprocess.Popen(__ssh_string(user, host, identity_file), 
														
 
															-        stdin=subprocess.PIPE, stdout=DEVNULL, stderr=DEVNULL)
														
 
															-    else:
														
 
															-      p = subprocess.Popen(__ssh_string(user, host, identity_file), 
														
 
															-        stdin=subprocess.PIPE)
														
 
															-    p.communicate(remote_script)
														
 
															-    return p.returncode == 0
														
--- a/toolset/run-tests.py
+++ b/toolset/run-tests.py
@@ -1,37 +1,45 @@
 
															-#!/usr/bin/env python
														
 
															 import argparse
														
 
															 import ConfigParser
														
 
															 import socket
														
 
															 import sys
														
 
															-import time
														
 
															 import os
														
 
															 import platform
														
 
															 import multiprocessing
														
 
															-import itertools
														
 
															-import copy
														
 
															-from benchmark.benchmarker import Benchmarker
														
 
															-from setup.linux.unbuffered import Unbuffered
														
 
															-from setup.linux import setup_util
														
 
															-from scaffolding import Scaffolding
														
 
															-from initializer import initialize
														
 
															+from toolset.benchmark.benchmarker import Benchmarker
														
 
															+from toolset.utils import setup_util
														
 
															+from toolset.utils.unbuffered import Unbuffered
														
 
															+from toolset.utils.scaffolding import Scaffolding
														
 
															+from toolset.utils.initializer import initialize
														
 
															+from toolset.utils import cleaner
														
 
															+from toolset.utils.results_helper import Results
														
 
															+from toolset.utils.benchmark_config import BenchmarkConfig
														
 
															+from toolset.utils import docker_helper
														
 
															+from toolset.utils.metadata_helper import gather_tests
														
 
															 from ast import literal_eval
														
 
															 # Enable cross-platform colored output
														
 
															 from colorama import init
														
 
															 init()
														
 
															+
														
 
															 class StoreSeqAction(argparse.Action):
														
 
															-    '''Helper class for parsing a sequence from the command line'''
														
 
															+    '''
														
 
															+    Helper class for parsing a sequence from the command line
														
 
															+    '''
														
 
															+
														
 
															     def __init__(self, option_strings, dest, nargs=None, **kwargs):
														
 
															-        super(StoreSeqAction, self).__init__(option_strings, dest, type=str, **kwargs)
														
 
															+        super(StoreSeqAction, self).__init__(
														
 
															+            option_strings, dest, type=str, **kwargs)
														
 
															+
														
 
															     def __call__(self, parser, namespace, values, option_string=None):
														
 
															         setattr(namespace, self.dest, self.parse_seq(values))
														
 
															+
														
 
															     def parse_seq(self, argument):
														
 
															         result = argument.split(',')
														
 
															         sequences = [x for x in result if ":" in x]
														
 
															         for sequence in sequences:
														
 
															             try:
														
 
															-                (start,step,end) = sequence.split(':')
														
 
															+                (start, step, end) = sequence.split(':')
														
 
															             except ValueError:
														
 
															                 print("  Invalid: {!s}".format(sequence))
														
 
															                 print("  Requires start:step:end, e.g. 1:2:10")
														
@@ -45,7 +53,8 @@ class StoreSeqAction(argparse.Action):
 
															 # Main
														
 
															 ###################################################################################################
														
 
															 def main(argv=None):
														
 
															-    ''' Runs the program. There are three ways to pass arguments
														
 
															+    '''
														
 
															+    Runs the program. There are three ways to pass arguments
														
 
															     1) environment variables TFB_*
														
 
															     2) configuration file benchmark.cfg
														
 
															     3) command line flags
														
@@ -57,7 +66,7 @@ def main(argv=None):
 
															         argv = sys.argv
														
 
															     # Enable unbuffered output so messages will appear in the proper order with subprocess output.
														
 
															-    sys.stdout=Unbuffered(sys.stdout)
														
 
															+    sys.stdout = Unbuffered(sys.stdout)
														
 
															     # Update python environment
														
 
															     # 1) Ensure the current directory (which should be the benchmark home directory) is in the path so that the tests can be imported.
														
@@ -69,7 +78,9 @@ def main(argv=None):
 
															     os.environ['FWROOT'] = setup_util.get_fwroot()
														
 
															     os.environ['IROOT'] = os.environ['FWROOT'] + '/installs'
														
 
															     # 'Ubuntu', '14.04', 'trusty' respectively
														
 
															-    os.environ['TFB_DISTRIB_ID'], os.environ['TFB_DISTRIB_RELEASE'], os.environ['TFB_DISTRIB_CODENAME'] = platform.linux_distribution()
														
 
															+    os.environ['TFB_DISTRIB_ID'], os.environ[
														
 
															+        'TFB_DISTRIB_RELEASE'], os.environ[
														
 
															+            'TFB_DISTRIB_CODENAME'] = platform.linux_distribution()
														
 
															     # App server cpu count
														
 
															     os.environ['CPU_COUNT'] = str(multiprocessing.cpu_count())
														
@@ -78,21 +89,29 @@ def main(argv=None):
 
															         formatter_class=argparse.RawDescriptionHelpFormatter,
														
 
															         add_help=False)
														
 
															     conf_parser.add_argument(
														
 
															-        '--conf_file', default='benchmark.cfg', metavar='FILE',
														
 
															-        help='Optional configuration file to provide argument defaults. All config options can be overridden using the command line.')
														
 
															+        '--conf_file',
														
 
															+        default='benchmark.cfg',
														
 
															+        metavar='FILE',
														
 
															+        help=
														
 
															+        'Optional configuration file to provide argument defaults. All config options can be overridden using the command line.'
														
 
															+    )
														
 
															     args, remaining_argv = conf_parser.parse_known_args()
														
 
															     defaults = {}
														
 
															     try:
														
 
															-        if not os.path.exists(os.path.join(os.environ['FWROOT'], args.conf_file)) and not os.path.exists(os.path.join(os.environ['FWROOT'] + 'benchmark.cfg')):
														
 
															+        if not os.path.exists(
														
 
															+                os.path.join(
														
 
															+                    os.environ['FWROOT'],
														
 
															+                    args.conf_file)) and not os.path.exists(
														
 
															+                        os.path.join(os.environ['FWROOT'] + 'benchmark.cfg')):
														
 
															             print("No config file found. Aborting!")
														
 
															             exit(1)
														
 
															-        with open (os.path.join(os.environ['FWROOT'], args.conf_file)):
														
 
															+        with open(os.path.join(os.environ['FWROOT'], args.conf_file)):
														
 
															             config = ConfigParser.SafeConfigParser()
														
 
															             config.read([os.path.join(os.environ['FWROOT'], args.conf_file)])
														
 
															             defaults.update(dict(config.items("Defaults")))
														
 
															             # Convert strings into proper python types
														
 
															-            for k, v in defaults.iteritems():
														
 
															+            for k, v in defaults.items():
														
 
															                 try:
														
 
															                     defaults[k] = literal_eval(v)
														
 
															                 except Exception:
														
@@ -126,10 +145,12 @@ def main(argv=None):
 
															     ##########################################################
														
 
															     # Set up argument parser
														
 
															     ##########################################################
														
 
															-    parser = argparse.ArgumentParser(description="Install or run the Framework Benchmarks test suite.",
														
 
															-                                     parents=[conf_parser],
														
 
															-                                     formatter_class=argparse.ArgumentDefaultsHelpFormatter,
														
 
															-                                     epilog='''If an argument includes (type int-sequence), then it accepts integer lists in multiple forms.
														
 
															+    parser = argparse.ArgumentParser(
														
 
															+        description="Install or run the Framework Benchmarks test suite.",
														
 
															+        parents=[conf_parser],
														
 
															+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
														
 
															+        epilog=
														
 
															+        '''If an argument includes (type int-sequence), then it accepts integer lists in multiple forms.
														
 
															         Using a single number e.g. 5 will create a list [5]. Using commas will create a list containing those
														
 
															         values e.g. 1,3,6 creates [1, 3, 6]. Using three colon-separated numbers of start:step:end will create a
														
 
															         list, using the semantics of python's range function, e.g. 1:3:15 creates [1, 4, 7, 10, 13] while
														
@@ -137,57 +158,152 @@ def main(argv=None):
 
															         ''')
														
 
															     # Install options
														
 
															-    parser.add_argument('--init', action='store_true', default=False, help='Initializes the benchmark environment')
														
 
															+    parser.add_argument(
														
 
															+        '--init',
														
 
															+        action='store_true',
														
 
															+        default=False,
														
 
															+        help='Initializes the benchmark environment')
														
 
															     # Suite options
														
 
															-    parser.add_argument('--clean', action='store_true', default=False, help='Removes the results directory')
														
 
															-    parser.add_argument('--new', action='store_true', default=False, help='Initialize a new framework test')
														
 
															-    parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Causes the configuration to print before any other commands are executed.')
														
 
															-    parser.add_argument('--quiet', action='store_true', default=False, help='Only print a limited set of messages to stdout, keep the bulk of messages in log files only')
														
 
															-    parser.add_argument('--results-name', help='Gives a name to this set of results, formatted as a date', default='(unspecified, datetime = %Y-%m-%d %H:%M:%S)')
														
 
															-    parser.add_argument('--results-environment', help='Describes the environment in which these results were gathered', default='(unspecified, hostname = %s)' % socket.gethostname())
														
 
															-    parser.add_argument('--results-upload-uri', default=None, help='A URI where the in-progress results.json file will be POSTed periodically')
														
 
															-    parser.add_argument('--parse', help='Parses the results of the given timestamp and merges that with the latest results')
														
 
															+    parser.add_argument(
														
 
															+        '--build',
														
 
															+        nargs='+',
														
 
															+        help='Builds the dockerfile(s) for the given test(s)')
														
 
															+    parser.add_argument(
														
 
															+        '--clean',
														
 
															+        action='store_true',
														
 
															+        default=False,
														
 
															+        help='Removes the results directory')
														
 
															+    parser.add_argument(
														
 
															+        '--new',
														
 
															+        action='store_true',
														
 
															+        default=False,
														
 
															+        help='Initialize a new framework test')
														
 
															+    parser.add_argument(
														
 
															+        '-v',
														
 
															+        '--verbose',
														
 
															+        action='store_true',
														
 
															+        default=False,
														
 
															+        help=
														
 
															+        'Causes the configuration to print before any other commands are executed.'
														
 
															+    )
														
 
															+    parser.add_argument(
														
 
															+        '--quiet',
														
 
															+        action='store_true',
														
 
															+        default=False,
														
 
															+        help=
														
 
															+        'Only print a limited set of messages to stdout, keep the bulk of messages in log files only'
														
 
															+    )
														
 
															+    parser.add_argument(
														
 
															+        '--results-name',
														
 
															+        help='Gives a name to this set of results, formatted as a date',
														
 
															+        default='(unspecified, datetime = %Y-%m-%d %H:%M:%S)')
														
 
															+    parser.add_argument(
														
 
															+        '--results-environment',
														
 
															+        help='Describes the environment in which these results were gathered',
														
 
															+        default='(unspecified, hostname = %s)' % socket.gethostname())
														
 
															+    parser.add_argument(
														
 
															+        '--results-upload-uri',
														
 
															+        default=None,
														
 
															+        help=
														
 
															+        'A URI where the in-progress results.json file will be POSTed periodically'
														
 
															+    )
														
 
															+    parser.add_argument(
														
 
															+        '--parse',
														
 
															+        help=
														
 
															+        'Parses the results of the given timestamp and merges that with the latest results'
														
 
															+    )
														
 
															     # Test options
														
 
															     parser.add_argument('--test', nargs='+', help='names of tests to run')
														
 
															-    parser.add_argument('--test-dir', nargs='+', dest='test_dir', help='name of framework directory containing all tests to run')
														
 
															-    parser.add_argument('--test-lang', nargs='+', dest='test_lang', help='name of language directory containing all tests to run')
														
 
															-    parser.add_argument('--exclude', nargs='+', help='names of tests to exclude')
														
 
															-    parser.add_argument('--type', choices=['all', 'json', 'db', 'query', 'cached_query', 'fortune', 'update', 'plaintext'], default='all', help='which type of test to run')
														
 
															-    parser.add_argument('-m', '--mode', choices=['benchmark', 'verify', 'debug'], default='benchmark', help='verify mode will only start up the tests, curl the urls and shutdown. debug mode will skip verification and leave the server running.')
														
 
															-    parser.add_argument('--list-tests', action='store_true', default=False, help='lists all the known tests that can run')
														
 
															+    parser.add_argument(
														
 
															+        '--test-dir',
														
 
															+        nargs='+',
														
 
															+        dest='test_dir',
														
 
															+        help='name of framework directory containing all tests to run')
														
 
															+    parser.add_argument(
														
 
															+        '--test-lang',
														
 
															+        nargs='+',
														
 
															+        dest='test_lang',
														
 
															+        help='name of language directory containing all tests to run')
														
 
															+    parser.add_argument(
														
 
															+        '--exclude', nargs='+', help='names of tests to exclude')
														
 
															+    parser.add_argument(
														
 
															+        '--type',
														
 
															+        choices=[
														
 
															+            'all', 'json', 'db', 'query', 'cached_query', 'fortune', 'update',
														
 
															+            'plaintext'
														
 
															+        ],
														
 
															+        default='all',
														
 
															+        help='which type of test to run')
														
 
															+    parser.add_argument(
														
 
															+        '-m',
														
 
															+        '--mode',
														
 
															+        choices=['benchmark', 'verify', 'debug'],
														
 
															+        default='benchmark',
														
 
															+        help=
														
 
															+        'verify mode will only start up the tests, curl the urls and shutdown. debug mode will skip verification and leave the server running.'
														
 
															+    )
														
 
															+    parser.add_argument(
														
 
															+        '--list-tests',
														
 
															+        action='store_true',
														
 
															+        default=False,
														
 
															+        help='lists all the known tests that can run')
														
 
															     # Benchmark options
														
 
															-    parser.add_argument('--duration', default=15, help='Time in seconds that each test should run for.')
														
 
															-    parser.add_argument('--sleep', type=int, default=60, help='the amount of time to sleep after starting each test to allow the server to start up.')
														
 
															+    parser.add_argument(
														
 
															+        '--duration',
														
 
															+        default=15,
														
 
															+        help='Time in seconds that each test should run for.')
														
 
															+    parser.add_argument(
														
 
															+        '--sleep',
														
 
															+        type=int,
														
 
															+        default=60,
														
 
															+        help=
														
 
															+        'the amount of time to sleep after starting each test to allow the server to start up.'
														
 
															+    )
														
 
															-    parser.set_defaults(**defaults) # Must do this after add, or each option's default will override the configuration file default
														
 
															+    parser.set_defaults(**defaults)
														
 
															+    # Must do this after add, or each option's default will override the configuration file default
														
 
															     args = parser.parse_args(remaining_argv)
														
 
															-    if args.new:
														
 
															-        Scaffolding().scaffold()
														
 
															-        return 0
														
 
															+    config = BenchmarkConfig(vars(args))
														
 
															+    results = Results(config)
														
 
															+
														
 
															+    if config.new:
														
 
															+        Scaffolding()
														
 
															-    if args.init:
														
 
															-        initialize(args)
														
 
															-        return 0
														
 
															+    elif config.init:
														
 
															+        initialize(config)
														
 
															-    benchmarker = Benchmarker(vars(args))
														
 
															+    elif config.build:
														
 
															+        docker_helper.build(config, config.build, None)
														
 
															-    if args.clean:
														
 
															-        benchmarker.clean_all()
														
 
															-        return 0
														
 
															+    elif config.clean:
														
 
															+        cleaner.clean(results)
														
 
															+        docker_helper.clean()
														
 
															+
														
 
															+    elif config.list_tests:
														
 
															+        all_tests = gather_tests(benchmarker_config=config)
														
 
															+
														
 
															+        for test in all_tests:
														
 
															+            print(test.name)
														
 
															+
														
 
															+    elif config.parse != None:
														
 
															+        # TODO: broken
														
 
															+        all_tests = gather_tests(benchmarker_config=config)
														
 
															+
														
 
															+        for test in all_tests:
														
 
															+            test.parse_all()
														
 
															+
														
 
															+        results.parse(all_tests)
														
 
															-    # Run the benchmarker in the specified mode
														
 
															-    #   Do not use benchmarker variables for these checks,
														
 
															-    #   they are either str or bool based on the python version
														
 
															-    if args.list_tests:
														
 
															-        benchmarker.run_list_tests()
														
 
															-    elif args.parse != None:
														
 
															-        benchmarker.parse_timestamp()
														
 
															     else:
														
 
															+        benchmarker = Benchmarker(config, results)
														
 
															         return benchmarker.run()
														
 
															+    return 0
														
 
															+
														
 
															+
														
 
															 if __name__ == "__main__":
														
 
															     sys.exit(main())
														
--- a/toolset/scaffolding.py
+++ b/toolset/scaffolding.py
@@ -1,348 +0,0 @@
 
															-# -*- coding: utf-8 -*-
														
 
															-
														
 
															-import sys
														
 
															-import os
														
 
															-import imp
														
 
															-from shutil import copytree
														
 
															-from setup.linux.setup_util import replace_text
														
 
															-from benchmark.utils import gather_frameworks, gather_langauges
														
 
															-
														
 
															-class Scaffolding:
														
 
															-  def scaffold(self):
														
 
															-    print("""
														
 
															--------------------------------------------------------------------------------
														
 
															-    This wizard is intended to help build the scaffolding required for a new 
														
 
															-    test to be benchmarked.
														
 
															-
														
 
															-    From here, you will be prompted for values related to the test you
														
 
															-    wish to add.
														
 
															--------------------------------------------------------------------------------""")
														
 
															-
														
 
															-    try:
														
 
															-      self.__gather_display_name()
														
 
															-      self.__gather_language()
														
 
															-      self.__gather_approach()
														
 
															-      self.__gather_classification()
														
 
															-      self.__gather_orm()
														
 
															-      self.__gather_webserver()
														
 
															-      self.__gather_versus()
														
 
															-      self.__confirm_values()
														
 
															-      self.__print_success()
														
 
															-    except:
														
 
															-      print("")
														
 
															-
														
 
															-  def __gather_display_name(self):
														
 
															-    print("""
														
 
															-  The name of your test as you wish it to be displayed on the results page.
														
 
															-
														
 
															-  Example: Gemini, Gin, Express
														
 
															-    """)
														
 
															-    self.__prompt_display_name()
														
 
															-    while not self.display_name:
														
 
															-      self.__prompt_display_name()
														
 
															-    self.name = self.display_name.lower()
														
 
															-
														
 
															-  def __prompt_display_name(self):
														
 
															-    self.display_name = raw_input("Name: ").strip()
														
 
															-
														
 
															-    found = False
														
 
															-    for framework in gather_frameworks():
														
 
															-      if framework.lower() == self.display_name.lower():
														
 
															-        found = True
														
 
															-
														
 
															-    if found:
														
 
															-      print("""
														
 
															-  It appears that there is already a '%s' framework in the test suite. You will
														
 
															-  have to pick a different name.
														
 
															-      """ % self.display_name)
														
 
															-      self.display_name = None
														
 
															-
														
 
															-  def __gather_language(self):
														
 
															-    print("""
														
 
															-  The language in which your test implementation is written.
														
 
															-
														
 
															-  Example: Java, Go, PHP
														
 
															-    """)
														
 
															-    self.language = None
														
 
															-    while not self.language:
														
 
															-      self.__prompt_language()
														
 
															-
														
 
															-  def __prompt_language(self):
														
 
															-    self.language = raw_input("Language: ").strip()
														
 
															-
														
 
															-    known_languages = gather_langauges()
														
 
															-    language = None
														
 
															-    for lang in known_languages:
														
 
															-      if lang.lower() == self.language.lower():
														
 
															-        language = lang
														
 
															-
														
 
															-    if not language:
														
 
															-      similar = []
														
 
															-      for lang in known_languages:
														
 
															-        if lang.lower()[:1] == self.language.lower()[:1]:
														
 
															-          similar.append(lang)
														
 
															-      similar = ', '.join(similar)
														
 
															-
														
 
															-      print("""
														
 
															-  That language is not currently in our list of known languages.
														
 
															-  
														
 
															-  Here is a list of similar languages present in our benchmark suite that you
														
 
															-  may have meant:
														
 
															-
														
 
															-  %s
														
 
															-      
														
 
															-  Did you mean to add the new language, '%s', to the benchmark suite?
														
 
															-      """ % (similar, self.language))
														
 
															-      valid = self.__prompt_confirm_new_language(known_languages)
														
 
															-      while not valid:
														
 
															-        valid = self.__prompt_confirm_new_language(known_languages)
														
 
															-
														
 
															-      if self.confirm_new_lang == 'n':
														
 
															-        self.language = None
														
 
															-      else:
														
 
															-        self.language = self.language.title()
														
 
															-
														
 
															-    return self.language
														
 
															-
														
 
															-  def __prompt_confirm_new_language(self, known_languages):
														
 
															-    self.confirm_new_lang = raw_input("Create New Language '%s' (y/n): " % self.language).strip().lower()
														
 
															-    return self.confirm_new_lang == 'y' or self.confirm_new_lang == 'n'
														
 
															-
														
 
															-  def __gather_approach(self):
														
 
															-    print("""
														
 
															-  The approach of your test implementation.
														
 
															-
														
 
															-  1) Realistic: Uses the framework with most out-of-the-box functionality 
														
 
															-                enabled. We consider this realistic because most applications 
														
 
															-                built with the framework will leave these features enabled.
														
 
															-  2) Stripped:  Removes or outright avoids implementing features that are
														
 
															-                unnecessary for the particulars of the benchmark exercise. This
														
 
															-                might illuminate the marginal improvement available in fine-
														
 
															-                tuning a framework to your application's use-case.
														
 
															-
														
 
															-  Note: If you are unsure, then your approach is probably Realistic. The
														
 
															-        Stripped approach is seldom used and will not have results displayed
														
 
															-        by default on the results website.
														
 
															-    """)
														
 
															-    valid = self.__prompt_approach()
														
 
															-    while not valid:
														
 
															-      valid = self.__prompt_approach()
														
 
															-
														
 
															-  def __prompt_approach(self):
														
 
															-    self.approach = raw_input("Approach [1/2]: ").strip()
														
 
															-    if self.approach == '1':
														
 
															-      self.approach = 'Realistic'
														
 
															-    if self.approach == '2':
														
 
															-      self.approach = 'Stripped'
														
 
															-    return self.approach == 'Realistic' or self.approach == 'Stripped'
														
 
															-
														
 
															-  def __gather_classification(self):
														
 
															-    print("""
														
 
															-  The classification of your test implementation.
														
 
															-
														
 
															-  1) Fullstack: Robust framework expected to provide high-level functionality 
														
 
															-                for serving as a web application; for example, ability to 
														
 
															-                compose views, provide functions for responding with several 
														
 
															-                data types (json, html, etc), connecting to a database, form 
														
 
															-                processing, etc.
														
 
															-  2) Micro:     Simple framework expected to provide enough middleware to build
														
 
															-                a robust web application such as request routing and some 
														
 
															-                simple plumbing, but may not include built-in functionality 
														
 
															-                such as, for example, server-composed views.
														
 
															-  3) Platform:  Barebones infrastructure for servicing HTTP requests, but does
														
 
															-                not include a framework at all.
														
 
															-    """)
														
 
															-    valid = self.__prompt_classification()
														
 
															-    while not valid:
														
 
															-      valid = self.__prompt_classification()
														
 
															-    if self.classification == 'Platform':
														
 
															-      self.platform = 'None'
														
 
															-      self.framework = 'None'
														
 
															-    else:
														
 
															-      self.framework = self.display_name
														
 
															-      self.__gather_platform()
														
 
															-
														
 
															-  def __prompt_classification(self):
														
 
															-    self.classification = raw_input("Classification [1/2/3]: ").strip()
														
 
															-    if self.classification == '1':
														
 
															-      self.classification = 'Fullstack'
														
 
															-    if self.classification == '2':
														
 
															-      self.classification = 'Micro'
														
 
															-    if self.classification == '3':
														
 
															-      self.classification = 'Platform'
														
 
															-    return self.classification == 'Fullstack' or \
														
 
															-           self.classification == 'Micro' or \
														
 
															-           self.classification == 'Platform'
														
 
															-
														
 
															-  def __gather_platform(self):
														
 
															-    print("""
														
 
															-  The platform of your test implementation.
														
 
															-
														
 
															-  The platform is the low-level software or API used to host web applications 
														
 
															-  for the framework; the platform provides an implementation of the HTTP
														
 
															-  fundamentals.
														
 
															-
														
 
															-  Not all frameworks have a platform and if your programming language provides
														
 
															-  much of that by which we define a platform, leave black.
														
 
															-
														
 
															-  Example: Servlet, Wai, .NET
														
 
															-    """)
														
 
															-    self.__prompt_platform()
														
 
															-    
														
 
															-  def __prompt_platform(self):
														
 
															-    self.platform = raw_input("Platform (optional): ").strip()
														
 
															-    if self.platform == '':
														
 
															-      self.platform = 'None'
														
 
															-
														
 
															-  def __gather_orm(self):
														
 
															-    print("""
														
 
															-  How you would classify the ORM (object relational mapper) of your test?
														
 
															-
														
 
															-  1) Full:  A feature-rich ORM which provides functionality for interacting 
														
 
															-            with a database without writing a query in all but the most edge 
														
 
															-            cases.
														
 
															-  2) Micro: An ORM which provides functionality for interacting with a database
														
 
															-            for many trivial operations (querying, updating), but not more 
														
 
															-            robust cases (for example, gathering relations).
														
 
															-  3) Raw:   No ORM; raw database access.
														
 
															-    """)
														
 
															-    valid = self.__prompt_orm()
														
 
															-    while not valid:
														
 
															-      valid = self.__prompt_orm()
														
 
															-
														
 
															-  def __prompt_orm(self):
														
 
															-    self.orm = raw_input("ORM [1/2/3]: ").strip()
														
 
															-    if self.orm == '1':
														
 
															-      self.orm = 'Full'
														
 
															-    if self.orm == '2':
														
 
															-      self.orm = 'Micro'
														
 
															-    if self.orm == '3':
														
 
															-      self.orm = 'Raw'
														
 
															-    return self.orm == 'Full' or \
														
 
															-           self.orm == 'Micro' or \
														
 
															-           self.orm == 'Raw'
														
 
															-
														
 
															-  def __gather_webserver(self):
														
 
															-    print("""
														
 
															-  Name of the front-end webserver sitting in front of your test implementation.
														
 
															-
														
 
															-  Your test implementation may not use a web-server and may act as its own; you
														
 
															-  can leave this blank in this case.
														
 
															-
														
 
															-  Example: nginx, Meinheld, httplight
														
 
															-    """)
														
 
															-    self.__prompt_webserver()
														
 
															-
														
 
															-  def __prompt_webserver(self):
														
 
															-    self.webserver = raw_input("Webserver (optional): ").strip()
														
 
															-    if self.webserver == '':
														
 
															-      self.webserver = 'None'
														
 
															-
														
 
															-  def __gather_versus(self):
														
 
															-    print("""
														
 
															-  The name of another test (elsewhere in this project) that is a subset of this
														
 
															-  framework.
														
 
															-  This allows for the generation of the framework efficiency chart in the 
														
 
															-  results web site.
														
 
															-  For example, Compojure is compared to "servlet" since Compojure is built on 
														
 
															-  the Servlet platform.
														
 
															-
														
 
															-  Example: Servlet, Wai, Undertow
														
 
															-    """)
														
 
															-    self.__prompt_versus()
														
 
															-
														
 
															-  def __prompt_versus(self):
														
 
															-    self.versus = raw_input("Versus (optional): ").strip()
														
 
															-    if self.versus == '':
														
 
															-      self.versus = 'None'
														
 
															-
														
 
															-  def __confirm_values(self):
														
 
															-    print("""
														
 
															-    Name: %s
														
 
															-    Language: %s
														
 
															-    Approach: %s
														
 
															-    Classification: %s
														
 
															-    Platform: %s
														
 
															-    ORM: %s
														
 
															-    Webserver: %s
														
 
															-    Versus: %s
														
 
															-
														
 
															-  Finalize the initialization of your test given the above values?
														
 
															-
														
 
															-  Note: once you have initialized your test, you can change these values later.
														
 
															-    """ % (self.display_name, 
														
 
															-           self.language, 
														
 
															-           self.approach, 
														
 
															-           self.classification, 
														
 
															-           self.platform,
														
 
															-           self.orm, 
														
 
															-           self.webserver, 
														
 
															-           self.versus))
														
 
															-
														
 
															-    valid = self.__prompt_confirmation()
														
 
															-    while not valid:
														
 
															-      valid = self.__prompt_confirmation()
														
 
															-
														
 
															-    if self.confirmation == 'y':
														
 
															-      self.__build_scaffolding()
														
 
															-    else:
														
 
															-      print('Aborting')
														
 
															-
														
 
															-  def __prompt_confirmation(self):
														
 
															-    self.confirmation = raw_input("Initialize [y/n]: ").strip().lower()
														
 
															-    return self.confirmation == 'y' or self.confirmation == 'n'
														
 
															-
														
 
															-  def __build_scaffolding(self):
														
 
															-    if self.__create_test_folder():
														
 
															-      self.__copy_scaffold_files()
														
 
															-      self.__edit_scaffold_files()
														
 
															-
														
 
															-  def __create_test_folder(self):
														
 
															-    self.language_dir = os.path.join("frameworks", self.language)
														
 
															-    self.test_dir = os.path.join(self.language_dir, self.name)
														
 
															-
														
 
															-    if os.path.exists(self.test_dir):
														
 
															-      print("Test '%s' already exists; aborting." % self.name)
														
 
															-      return False
														
 
															-
														
 
															-    return True
														
 
															-
														
 
															-  def __copy_scaffold_files(self):
														
 
															-    self.scaffold_dir = os.path.join("toolset","setup","scaffolding")
														
 
															-    copytree(self.scaffold_dir, self.test_dir)
														
 
															-
														
 
															-  def __edit_scaffold_files(self):
														
 
															-    for file in os.listdir(os.path.join(self.test_dir)):
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$NAME", self.name)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$DISPLAY_NAME", self.display_name)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$APPROACH", self.approach)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$CLASSIFICATION", self.classification)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$FRAMEWORK", self.framework)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$LANGUAGE", self.language)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$ORM", self.orm)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$PLATFORM", self.platform)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$WEBSERVER", self.webserver)
														
 
															-      replace_text(os.path.join(self.test_dir, file), "\$VERSUS", self.versus)
														
 
															-
														
 
															-  def __print_success(self):
														
 
															-    print("""
														
 
															--------------------------------------------------------------------------------
														
 
															-  Success!
														
 
															-
														
 
															-  Your new test structure has been built to the sepcifications of the suite.
														
 
															-  Here is a brief run-down of what has been built:
														
 
															-
														
 
															-    frameworks
														
 
															-        └─── %s
														
 
															-              └─── %s
														
 
															-                    ├─── .gitignore
														
 
															-                    ├─── benchmark_config.json
														
 
															-                    ├─── README.md
														
 
															-                    ├─── setup.sh
														
 
															-                    ├─── setup_mysql.sh
														
 
															-                    └─── source_code
														
 
															-
														
 
															-  The next step is to read through your README.md and follow the instructions
														
 
															-  provided therein.
														
 
															--------------------------------------------------------------------------------"""
														
 
															-    % (self.language, self.name))
														
--- a/toolset/setup/__init__.py
+++ b/toolset/setup/__init__.py
@@ -1 +0,0 @@
 
															-# Setup
														
--- a/toolset/setup/linux/docker/base.dockerfile
+++ b/toolset/setup/linux/docker/base.dockerfile
@@ -24,6 +24,8 @@ ENV LC_ALL en_US.UTF-8
 
															 ARG CPU_COUNT
														
 
															 ARG MAX_CONCURRENCY
														
 
															+ARG TFB_DATABASE
														
 
															 ENV CPU_COUNT=$CPU_COUNT
														
 
															 ENV MAX_CONCURRENCY=$MAX_CONCURRENCY
														
 
															+ENV DBHOST=$TFB_DATABASE
														
--- a/toolset/setup/linux/docker/databases/mongodb/create.js
+++ b/toolset/setup/linux/docker/databases/mongodb/create.js
--- a/toolset/setup/linux/docker/databases/mongodb/mongodb.dockerfile
+++ b/toolset/setup/linux/docker/databases/mongodb/mongodb.dockerfile
--- a/toolset/setup/linux/docker/databases/mysql/create.sql
+++ b/toolset/setup/linux/docker/databases/mysql/create.sql
--- a/toolset/setup/linux/docker/databases/mysql/my.cnf
+++ b/toolset/setup/linux/docker/databases/mysql/my.cnf
--- a/toolset/setup/linux/docker/databases/mysql/mysql.dockerfile
+++ b/toolset/setup/linux/docker/databases/mysql/mysql.dockerfile
--- a/toolset/setup/linux/docker/databases/mysql/mysql.list
+++ b/toolset/setup/linux/docker/databases/mysql/mysql.list
--- a/toolset/setup/linux/docker/databases/postgres/60-postgresql-shm.conf
+++ b/toolset/setup/linux/docker/databases/postgres/60-postgresql-shm.conf
--- a/toolset/setup/linux/docker/databases/postgres/create-postgres-database.sql
+++ b/toolset/setup/linux/docker/databases/postgres/create-postgres-database.sql
--- a/toolset/setup/linux/docker/databases/postgres/create-postgres.sql
+++ b/toolset/setup/linux/docker/databases/postgres/create-postgres.sql
--- a/toolset/setup/linux/docker/databases/postgres/pg_hba.conf
+++ b/toolset/setup/linux/docker/databases/postgres/pg_hba.conf
--- a/toolset/setup/linux/docker/databases/postgres/postgres.dockerfile
+++ b/toolset/setup/linux/docker/databases/postgres/postgres.dockerfile
--- a/toolset/setup/linux/docker/databases/postgres/postgresql.conf
+++ b/toolset/setup/linux/docker/databases/postgres/postgresql.conf
--- a/toolset/setup/linux/docker/languages/d-lang.dockerfile
+++ b/toolset/setup/linux/docker/languages/d-lang.dockerfile
--- a/toolset/setup/linux/docker/languages/dart-lang.dockerfile
+++ b/toolset/setup/linux/docker/languages/dart-lang.dockerfile
--- a/toolset/setup/linux/docker/languages/elixir.dockerfile
+++ b/toolset/setup/linux/docker/languages/elixir.dockerfile
--- a/toolset/setup/linux/docker/languages/erlang.dockerfile
+++ b/toolset/setup/linux/docker/languages/erlang.dockerfile
--- a/toolset/setup/linux/docker/languages/haskell.dockerfile
+++ b/toolset/setup/linux/docker/languages/haskell.dockerfile
--- a/toolset/setup/linux/docker/languages/java.dockerfile
+++ b/toolset/setup/linux/docker/languages/java.dockerfile
--- a/toolset/setup/linux/docker/languages/java8.dockerfile
+++ b/toolset/setup/linux/docker/languages/java8.dockerfile
--- a/toolset/setup/linux/docker/languages/lua.dockerfile
+++ b/toolset/setup/linux/docker/languages/lua.dockerfile
--- a/toolset/setup/linux/docker/languages/nim.dockerfile
+++ b/toolset/setup/linux/docker/languages/nim.dockerfile
--- a/toolset/setup/linux/docker/languages/ruby-2.4.dockerfile
+++ b/toolset/setup/linux/docker/languages/ruby-2.4.dockerfile
--- a/toolset/setup/linux/docker/languages/rust.dockerfile
+++ b/toolset/setup/linux/docker/languages/rust.dockerfile
--- a/toolset/setup/linux/docker/systools/ant.dockerfile
+++ b/toolset/setup/linux/docker/systools/ant.dockerfile
--- a/toolset/setup/linux/docker/systools/gcc-6.dockerfile
+++ b/toolset/setup/linux/docker/systools/gcc-6.dockerfile
--- a/toolset/setup/linux/docker/systools/luarocks.dockerfile
+++ b/toolset/setup/linux/docker/systools/luarocks.dockerfile
--- a/toolset/setup/linux/docker/systools/maven-java8.dockerfile
+++ b/toolset/setup/linux/docker/systools/maven-java8.dockerfile
--- a/toolset/setup/linux/docker/systools/maven-settings.xml
+++ b/toolset/setup/linux/docker/systools/maven-settings.xml
--- a/toolset/setup/linux/docker/systools/maven.dockerfile
+++ b/toolset/setup/linux/docker/systools/maven.dockerfile
--- a/toolset/setup/linux/docker/systools/nimble.dockerfile
+++ b/toolset/setup/linux/docker/systools/nimble.dockerfile
--- a/toolset/setup/linux/docker/systools/sbt-java8.dockerfile
+++ b/toolset/setup/linux/docker/systools/sbt-java8.dockerfile
--- a/toolset/setup/linux/docker/systools/sbt.dockerfile
+++ b/toolset/setup/linux/docker/systools/sbt.dockerfile
--- a/toolset/setup/linux/docker/webservers/nginx.dockerfile
+++ b/toolset/setup/linux/docker/webservers/nginx.dockerfile
--- a/toolset/setup/linux/docker/webservers/nodejs8.dockerfile
+++ b/toolset/setup/linux/docker/webservers/nodejs8.dockerfile
--- a/toolset/setup/linux/docker/webservers/openresty-server.dockerfile
+++ b/toolset/setup/linux/docker/webservers/openresty-server.dockerfile
--- a/toolset/setup/linux/docker/webservers/resin-java8.dockerfile
+++ b/toolset/setup/linux/docker/webservers/resin-java8.dockerfile
--- a/toolset/setup/linux/docker/webservers/resin.dockerfile
+++ b/toolset/setup/linux/docker/webservers/resin.dockerfile
--- a/toolset/setup/linux/__init__.py
+++ b/toolset/setup/linux/__init__.py
@@ -1 +0,0 @@
 
															-# Linux server and client setup
														
--- a/toolset/setup/linux/database.sh
+++ b/toolset/setup/linux/database.sh
@@ -29,6 +29,7 @@ sudo add-apt-repository \
 
															    $(lsb_release -cs) \
														
 
															    stable"
														
 
															 sudo apt-get update
														
 
															+# TODO: this doesn't work on real machines; only single-server setups which conveniently have docker already
														
 
															 sudo apt-get -qqy install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold"  docker-ce
														
 
															 if ! sudo grep -q -E "^docker:" /etc/group; then
														
 
															   sudo groupadd docker
														
--- a/toolset/setup/linux/prerequisites.sh
+++ b/toolset/setup/linux/prerequisites.sh
@@ -22,27 +22,8 @@ sudo apt-get -qqy install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options:
 
															   ca-certificates               `# Needed for Docker on Ubuntu 14` \
														
 
															   curl                          `# Needed for Docker on Ubuntu 14` \
														
 
															   software-properties-common    `# Needed for Docker on Ubuntu 14`
														
 
															-  # cmake build-essential automake    `# Needed for building code` \
														
 
															-  # wget unzip                   `# Common tools` \
														
 
															-  # mercurial                `# Version control systems` \
														
 
															-  # libpcre3 libpcre3-dev libpcrecpp0 `# Regular expression support` \
														
 
															-  # libssl-dev libcurl4-openssl-dev   `# SSL libraries` \
														
 
															-  # zlib1g-dev \
														
 
															-  # libreadline6-dev \
														
 
															-  # libbz2-dev \
														
 
															-  # libyaml-dev libxml2-dev \
														
 
															-  # libxslt-dev libgdbm-dev ncurses-dev  \
														
 
															-  # libffi-dev htop libtool bison libevent-dev \
														
 
															-  # libgstreamer-plugins-base0.10-0 libgstreamer0.10-0 \
														
 
															-  # liborc-0.4-0 libwxbase2.8-0 libwxgtk2.8-0 libgnutls-dev \
														
 
															-  # libjson0-dev libmcrypt-dev libicu-dev gettext \
														
 
															-  #  mlton \
														
 
															-  # re2c libnuma-dev
														
 
															-sudo pip install colorama==0.3.1
														
 
															-# Version 2.3 has a nice Counter() and other features
														
 
															-# but it requires --allow-external and --allow-unverified
														
 
															-sudo pip install progressbar==2.2 requests MySQL-python psycopg2 pymongo
														
 
															+sudo pip install colorama==0.3.1 requests MySQL-python psycopg2-binary pymongo
														
 
															 #
														
 
															 # Install Docker
														
@@ -76,10 +57,11 @@ sudo sh -c "echo '*            soft    rtprio             99' >> /etc/security/l
 
															 # Create a tfb command alias for running the toolset
														
 
															 # For now, this still ensures you have to be in the framework root to run it
														
 
															+export PWD=$(pwd)
														
 
															 sudo tee /etc/profile.d/tfb.sh <<EOF
														
 
															 #!/bin/bash
														
 
															 tfb() {
														
 
															-  $(pwd)/toolset/run-tests.py "\$@"
														
 
															+  PYTHONPATH=$PWD python $PWD/toolset/run-tests.py "\$@"
														
 
															 }
														
 
															 EOF
														
 
															 source /etc/profile.d/tfb.sh
														
--- a/toolset/setup/linux/unbuffered.py
+++ b/toolset/setup/linux/unbuffered.py
@@ -1,13 +0,0 @@
 
															-# Wrapper for unbuffered stream writing.
														
 
															-# http://stackoverflow.com/a/107717/376366
														
 
															-# Used to make sure print output appears in the correct order
														
 
															-# in log files when spawning subprocesses.
														
 
															-
														
 
															-class Unbuffered:
														
 
															-  def __init__(self, stream):
														
 
															-    self.stream = stream
														
 
															-  def write(self, data):
														
 
															-    self.stream.write(data)
														
 
															-    self.stream.flush()
														
 
															-  def __getattr__(self, attr):
														
 
															-    return getattr(self.stream, attr)
														
--- a/toolset/setup/sqlserver/setup-sqlserver-bootstrap.ps1
+++ b/toolset/setup/sqlserver/setup-sqlserver-bootstrap.ps1
@@ -1,25 +0,0 @@
 
															-# To download and run this script, open an elevated Command Prompt and then run:
														
 
															-#
														
 
															-# powershell -ExecutionPolicy Bypass -Command "iex (New-Object Net.WebClient).DownloadString('https://raw.github.com/TechEmpower/FrameworkBenchmarks/master/toolset/setup/sqlserver/setup-sqlserver-bootstrap.ps1')"
														
 
															-
														
 
															-$basedir = "C:\FrameworkBenchmarks"
														
 
															-$rawRepo = "https://raw.github.com/TechEmpower/FrameworkBenchmarks/master"
														
 
															-
														
 
															-$config_url = $rawRepo + "/config"
														
 
															-$config_local = $basedir + "\config"
														
 
															-$setup_sqlserver_url = $rawRepo + "/toolset/setup/sqlserver/setup-sqlserver.ps1"
														
 
															-$setup_sqlserver_local = $basedir + "\setup-sqlserver.ps1"
														
 
															-$create_sqlserver_login_and_database_url = $config_url + "/create-sqlserver-login-and-database.sql"
														
 
															-$create_sqlserver_login_and_database_local = $config_local + "/create-sqlserver-login-and-database.sql"
														
 
															-$create_sqlserver_url = $config_url + "/create-sqlserver.sql"
														
 
															-$create_sqlserver_local = $config_local + "/create-sqlserver.sql"
														
 
															-
														
 
															-Write-Host "Creating directory: $config_local`n"
														
 
															-New-Item -Path $config_local -Type Directory -Force | Out-Null
														
 
															-
														
 
															-Write-Host "Downloading setup files...`n"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($setup_sqlserver_url, $setup_sqlserver_local)
														
 
															-(New-Object System.Net.WebClient).DownloadFile($create_sqlserver_login_and_database_url, $create_sqlserver_login_and_database_local)
														
 
															-(New-Object System.Net.WebClient).DownloadFile($create_sqlserver_url, $create_sqlserver_local)
														
 
															-
														
 
															-powershell -ExecutionPolicy Bypass -File $setup_sqlserver_local
														
--- a/toolset/setup/sqlserver/setup-sqlserver.ps1
+++ b/toolset/setup/sqlserver/setup-sqlserver.ps1
@@ -1,91 +0,0 @@
 
															-# This script downloads and installs SQL Server and opens it on port 1433.
														
 
															-#
														
 
															-# To run this script, run an elevated Command Prompt and enter:
														
 
															-#
														
 
															-# powershell -ExecutionPolicy Bypass -File <this script's filename>
														
 
															-
														
 
															-$basedir = "C:\FrameworkBenchmarks"
														
 
															-$workdir = "$basedir\installs"
														
 
															-New-Item -Path $workdir -Type directory -Force | Out-Null
														
 
															-
														
 
															-If (-Not (Get-Service | ? Name -Eq "MSSQLSERVER")) {
														
 
															-
														
 
															-  Write-Host "Could not find default SQL Server instance, MSSQLSERVER."
														
 
															-  Write-Host "Downloading SQL Server (several GBs)..."
														
 
															-
														
 
															-  # URLs from http://www.microsoft.com/en-us/download/details.aspx?id=35575
														
 
															-
														
 
															-  $sqlserver_exe_url = "http://download.microsoft.com/download/3/B/D/3BD9DD65-D3E3-43C3-BB50-0ED850A82AD5/SQLServer2012SP1-FullSlipstream-x64-ENU.exe"
														
 
															-  $sqlserver_exe_local = "$workdir\SQLServer2012SP1-FullSlipstream-x64-ENU.exe"
														
 
															-  (New-Object System.Net.WebClient).DownloadFile($sqlserver_exe_url, $sqlserver_exe_local)
														
 
															-
														
 
															-  $sqlserver_box_url = "http://download.microsoft.com/download/3/B/D/3BD9DD65-D3E3-43C3-BB50-0ED850A82AD5/SQLServer2012SP1-FullSlipstream-x64-ENU.box"
														
 
															-  $sqlserver_box_local = "$workdir\SQLServer2012SP1-FullSlipstream-x64-ENU.box"
														
 
															-  (New-Object System.Net.WebClient).DownloadFile($sqlserver_box_url, $sqlserver_box_local)
														
 
															-
														
 
															-  Write-Host "Installing SQL Server..."
														
 
															-
														
 
															-  # Install only the SQL Server database engine.
														
 
															-  # Use a default instance name.
														
 
															-  # Make %COMPUTERNAME%\Administrators have administrative rights.
														
 
															-  
														
 
															-  # The following is not used because this is done in PowerShell below.
														
 
															-  #   /securitymode=SQL /sapwd=S3cr3tS3cr3t /TCPENABLED=1
														
 
															-  #   Allow Windows Authentication or old-style SQL authentication.
														
 
															-  #   The password of the sa account is specified.
														
 
															-  #   SQL Server will be listening on TCP port 1433.
														
 
															-  #
														
 
															-  Start-Process "$sqlserver_exe_local" "/q /action=install /features=SQLEngine /INSTANCENAME=MSSQLSERVER /SQLSYSADMINACCOUNTS=Administrators /IACCEPTSQLSERVERLICENSETERMS" -Wait
														
 
															-}
														
 
															-
														
 
															-# In case we just installed SQL Server and the environment variables haven't been refreshed, manually
														
 
															-# refresh PSModulePath so that Import-Module sqlps will work.
														
 
															-
														
 
															-$env:PSModulePath = [Environment]::GetEnvironmentVariable("PSModulePath", [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-Import-Module sqlps
														
 
															-
														
 
															-Write-Host "Setting SQL Server to start on boot..."
														
 
															-
														
 
															-Set-Service MSSQLSERVER -StartupType Automatic
														
 
															-
														
 
															-Write-Host "Ensuring that SQL Server is started..."
														
 
															-
														
 
															-Start-Service MSSQLSERVER
														
 
															-
														
 
															-Write-Host "Enabling SQL authentication..."
														
 
															-
														
 
															-# Enable SQL authentication
														
 
															-$s = New-Object ('Microsoft.SqlServer.Management.Smo.Server')
														
 
															-$s.Settings.LoginMode = [Microsoft.SqlServer.Management.SMO.ServerLoginMode]::Mixed
														
 
															-$s.Alter()
														
 
															-
														
 
															-Write-Host "Configuring SQL Server to listen on TCP (default port 1433)..."
														
 
															-
														
 
															-# Enable the TCP protocol on the default instance.
														
 
															-$wmi = New-Object ('Microsoft.SqlServer.Management.Smo.Wmi.ManagedComputer')
														
 
															-$uri = "ManagedComputer[@Name='" + (Get-Content env:computername) + "']/ ServerInstance[@Name='MSSQLSERVER']/ServerProtocol[@Name='Tcp']"
														
 
															-$Tcp = $wmi.GetSmoObject($uri)
														
 
															-$Tcp.IsEnabled = $true
														
 
															-$Tcp.Alter()
														
 
															-
														
 
															-Write-Host "Restarting SQL Server..."
														
 
															-
														
 
															-Restart-Service -Name MSSQLSERVER
														
 
															-
														
 
															-If (-Not (Get-NetFirewallPortFilter | ? LocalPort -Eq "1433")) {
														
 
															-  Write-Host "Opening port 1433 in firewall..."
														
 
															-  New-NetFirewallRule -DisplayName "SQL 1433" -Action Allow -Direction Inbound -LocalPort 1433 -Protocol TCP | Out-Null
														
 
															-} else {
														
 
															-  Write-Host "Port 1433 is already configured in firewall."
														
 
															-}
														
 
															-
														
 
															-Write-Host "Creating SQL Server login and populated database..."
														
 
															-
														
 
															-# Connect with Windows Authentication, assuming that we have access.
														
 
															-Invoke-Sqlcmd -InputFile "$basedir\config\create-sqlserver-login-and-database.sql" -OutputSqlErrors $True -QueryTimeout 180
														
 
															-
														
 
															-# Now that benchmarkdbuser has been created, we can connect with those credentials.
														
 
															-Invoke-Sqlcmd -Username benchmarkdbuser -Password B3nchmarkDBPass -Database hello_world -InputFile "$basedir\config\create-sqlserver.sql" -OutputSqlErrors $True -QueryTimeout 180
														
 
															-
														
 
															-Write-Host "Done."
														
--- a/toolset/setup/windows/databases/sqlserver/create-sqlserver-login-and-database.sql
+++ b/toolset/setup/windows/databases/sqlserver/create-sqlserver-login-and-database.sql
@@ -1,28 +0,0 @@
 
															--- This SQL Server T-SQL script creates the database user and hello_world database.
														
 
															---
														
 
															--- To run this script, login to an administrator account in Windows, open a command prompt and run:
														
 
															---
														
 
															--- "%ProgramFiles%\Microsoft SQL Server\110\Tools\binn\sqlcmd.exe" -i <filename of this file>
														
 
															---
														
 
															-
														
 
															-IF EXISTS (SELECT * FROM sys.server_principals WHERE name = 'benchmarkdbuser')
														
 
															-    DROP LOGIN benchmarkdbuser
														
 
															-GO
														
 
															-
														
 
															--- This password has mixed-case and a number to satisfy the Windows password policy
														
 
															-CREATE LOGIN benchmarkdbuser WITH PASSWORD = 'B3nchmarkDBPass'
														
 
															-GO
														
 
															-
														
 
															-IF EXISTS(SELECT * FROM SYS.DATABASES WHERE NAME='hello_world')
														
 
															-    DROP DATABASE hello_world
														
 
															-GO
														
 
															-
														
 
															-CREATE DATABASE hello_world
														
 
															-GO
														
 
															-USE hello_world
														
 
															-GO
														
 
															-
														
 
															--- Give this user total power over the database
														
 
															-CREATE USER benchmarkdbuser FOR LOGIN benchmarkdbuser
														
 
															-EXEC sp_addrolemember 'db_owner', 'benchmarkdbuser'
														
 
															-GO
														
--- a/toolset/setup/windows/databases/sqlserver/create-sqlserver.sql
+++ b/toolset/setup/windows/databases/sqlserver/create-sqlserver.sql
@@ -1,55 +0,0 @@
 
															--- This SQL Server T-SQL script creates and populates the World and Fortune tables.
														
 
															---
														
 
															--- To run this script, make sure that you've already run create-sqlserver-login-and-database.sql
														
 
															--- to create the database user and database, then open a command prompt and run:
														
 
															---
														
 
															--- "%ProgramFiles%\Microsoft SQL Server\110\Tools\binn\sqlcmd.exe" -U benchmarkdbuser -P B3nchmarkDBPass -d hello_world -i <filename of this file>
														
 
															-
														
 
															-IF OBJECT_ID('World', 'U') IS NOT NULL
														
 
															-    DROP TABLE World
														
 
															-GO
														
 
															-
														
 
															-CREATE TABLE World (
														
 
															-  id int NOT NULL IDENTITY PRIMARY KEY,
														
 
															-  randomNumber int NOT NULL default 0
														
 
															-)
														
 
															-GO
														
 
															-
														
 
															--- Populate World table
														
 
															-DECLARE @RowCount INT
														
 
															-DECLARE @Random INT
														
 
															-SET @RowCount = 0
														
 
															-
														
 
															-WHILE @RowCount < 10000
														
 
															-BEGIN
														
 
															-	SELECT @Random = ((10000 + 1) - 1) * RAND() + 1
														
 
															-	INSERT INTO World (randomNumber) VALUES (@Random)
														
 
															-	SET @RowCount = @RowCount + 1
														
 
															-END
														
 
															-
														
 
															-GO
														
 
															-
														
 
															-IF OBJECT_ID('Fortune', 'U') IS NOT NULL
														
 
															-    DROP TABLE Fortune
														
 
															-GO
														
 
															-
														
 
															--- Note that this uses nvarchar to make sure that the column is Unicode.
														
 
															-CREATE TABLE Fortune (
														
 
															-  id int NOT NULL IDENTITY PRIMARY KEY,
														
 
															-  message nvarchar(2048) NOT NULL
														
 
															-)
														
 
															-GO
														
 
															-
														
 
															-INSERT INTO Fortune (message) VALUES (N'fortune: No such file or directory');
														
 
															-INSERT INTO Fortune (message) VALUES (N'A computer scientist is someone who fixes things that aren''t broken.');
														
 
															-INSERT INTO Fortune (message) VALUES (N'After enough decimal places, nobody gives a damn.');
														
 
															-INSERT INTO Fortune (message) VALUES (N'A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1');
														
 
															-INSERT INTO Fortune (message) VALUES (N'A computer program does what you tell it to do, not what you want it to do.');
														
 
															-INSERT INTO Fortune (message) VALUES (N'Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen');
														
 
															-INSERT INTO Fortune (message) VALUES (N'Any program that runs right is obsolete.');
														
 
															-INSERT INTO Fortune (message) VALUES (N'A list is only as strong as its weakest link. — Donald Knuth');
														
 
															-INSERT INTO Fortune (message) VALUES (N'Feature: A bug with seniority.');
														
 
															-INSERT INTO Fortune (message) VALUES (N'Computers make very fast, very accurate mistakes.');
														
 
															-INSERT INTO Fortune (message) VALUES (N'<script>alert("This should not be displayed in a browser alert box.");</script>');
														
 
															-INSERT INTO Fortune (message) VALUES (N'フレームワークのベンチマーク');
														
 
															-GO
														
--- a/toolset/setup/windows/installer-bootstrap.ps1
+++ b/toolset/setup/windows/installer-bootstrap.ps1
@@ -1,42 +0,0 @@
 
															-param($noexit)
														
 
															-
														
 
															-if (!$noexit) {
														
 
															-    Write-Host "`nRestarting PowerShell with -NoExit...`n"
														
 
															-    powershell -NoExit -File $MyInvocation.MyCommand.Path 1
														
 
															-    return
														
 
															-}
														
 
															-
														
 
															-$basedir = "C:\FrameworkBenchmarks"
														
 
															-$workdir = $basedir + "\installs"
														
 
															-
														
 
															-$repo = "https://github.com/TechEmpower/FrameworkBenchmarks"
														
 
															-$installer = $basedir + "\toolset\setup\windows\installer.ps1"
														
 
															-
														
 
															-$git = "C:\Git\bin\git.exe"
														
 
															-$gitinstaller_file = "Git-1.8.1.2-preview20130201.exe"
														
 
															-$gitinstaller_url = "https://msysgit.googlecode.com/files/" + $gitinstaller_file
														
 
															-$gitinstaller_local = $workdir + "\" + $gitinstaller_file
														
 
															-
														
 
															-Write-Host "Creating work directory: $workdir `n"
														
 
															-New-Item -Path $workdir -Type Directory -Force | Out-Null
														
 
															-
														
 
															-Write-Host "Downloading git...`n"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($gitinstaller_url, $gitinstaller_local)
														
 
															-
														
 
															-Write-Host "Installing git...`n"
														
 
															-Start-Process $gitinstaller_local '/silent /dir="C:\Git"' -Wait
														
 
															-$env:Path += ";C:\Git\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-Write-Host "Removing git installation files...`n"
														
 
															-Remove-Item -Recurse -Force $basedir
														
 
															-
														
 
															-if (-not (Test-Path $basedir))
														
 
															-{
														
 
															-    Write-Host "Downloading FrameworkBenchmarks from git...`n"
														
 
															-    &$git "clone" $repo $basedir | Out-Host
														
 
															-}
														
 
															-
														
 
															-
														
 
															-Write-Host "`nLaunching installer...`n"
														
 
															-Set-ExecutionPolicy -ExecutionPolicy Bypass -ErrorAction 'SilentlyContinue'
														
 
															-powershell -NoExit -File $installer
														
--- a/toolset/setup/windows/installer.ps1
+++ b/toolset/setup/windows/installer.ps1
@@ -1,352 +0,0 @@
 
															-#
														
 
															-# Versions of software (will need to be updated from time to time)
														
 
															-#
														
 
															-
														
 
															-$node_installer_file      = "node-v0.10.13-x64.msi"
														
 
															-$node_installer_path      = "v0.10.13/x64/$node_installer_file"
														
 
															-$python_installer_file    = "python-2.7.5.amd64.msi"
														
 
															-$python_installer_path    = "2.7.5/$python_installer_file"
														
 
															-$python_version           = "27"
														
 
															-$wincache_installer_file  = "wincache-1.3.4-5.4-nts-vc9-x86.exe"
														
 
															-$wincache_installer_path  = "wincache-1.3.4/$wincache_installer_file"
														
 
															-$go_installer_file        = "go1.2.windows-amd64.msi"
														
 
															-$jre_installer_file       = "jdk-7u65-windows-x64.exe"
														
 
															-$jdk_installer_file       = "jdk-7u65-windows-x64.exe"
														
 
															-$jdk_master_hash          = "f0270817998c7408b24a2dd9ac420346" 
														
 
															-# http://www.oracle.com/technetwork/java/javase/downloads/java-se-binaries-checksum-1956892.html
														
 
															-$resin_version            = "resin-4.0.41"
														
 
															-$resin_installer_file     = "$resin_version.zip"
														
 
															-$ant_version              = "apache-ant-1.9.2"
														
 
															-$ant_installer_file       = "$ant_version-bin.zip"
														
 
															-$maven_version            = "apache-maven-3.0.5"
														
 
															-$maven_installer_file     = "$maven_version-bin.zip"
														
 
															-$maven_installer_path     = "maven-3/3.0.5/binaries/$maven_installer_file"
														
 
															-$scala_version            = "2.10.2"
														
 
															-$play_version             = "2.2.0"
														
 
															-$play_installer_file      = "play-$play_version.zip"
														
 
															-$sbt_version              = "0.13.5"
														
 
															-$mercurial_installer_file = "mercurial-2.6.1-x64.msi"
														
 
															-$cygwin_installer_file    = "setup-x86_64.exe"
														
 
															-
														
 
															-
														
 
															-$basedir = "C:\FrameworkBenchmarks"
														
 
															-$workdir = "$basedir\installs"
														
 
															-New-Item -Path $workdir -Type directory -Force | Out-Null
														
 
															-
														
 
															-function GetMd5FileHash($fileName) {
														
 
															-    [Reflection.Assembly]::LoadWithPartialName("System.Security") | out-null
														
 
															-    $md5 = [System.Security.Cryptography.MD5]::Create()
														
 
															-
														
 
															-    $file = [System.IO.File]::OpenRead($fileName)
														
 
															-    $hash = $md5.ComputeHash($file)
														
 
															-    $file.Dispose()
														
 
															-
														
 
															-    $sb = New-Object System.Text.StringBuilder
														
 
															-    $hash | % { [Void]$sb.Append($_.ToString("x2")) }
														
 
															-    $sb.ToString()
														
 
															-}
														
 
															-
														
 
															-#
														
 
															-# Chocolatey package manager
														
 
															-#
														
 
															-Write-Host "Installing Chocolatey package manager"
														
 
															-Invoke-Expression ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1'))
														
 
															-
														
 
															-#
														
 
															-# ASP.NET
														
 
															-#
														
 
															-Write-Host "Installing IIS, .NET and ASP.NET..."
														
 
															-
														
 
															-# Enable Windows Update to get rid of the yellow warnings
														
 
															-# But this is not strictly neccessary
														
 
															-$Updates = (New-Object -ComObject "Microsoft.Update.AutoUpdate").Settings
														
 
															-$Updates.NotificationLevel = 2 # Notify before download
														
 
															-$Updates.Save()
														
 
															-$Updates.Refresh()
														
 
															-
														
 
															-Install-WindowsFeature Web-Server
														
 
															-Install-WindowsFeature Web-Mgmt-Console
														
 
															-Install-WindowsFeature NET-Framework-45-ASPNET
														
 
															-Install-WindowsFeature Web-Asp-Net45
														
 
															-
														
 
															-$env:Path += ";C:\Windows\system32\inetsrv"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-# Optimize performance
														
 
															-appcmd set config -section:httpProtocol /allowKeepAlive:true | Out-Null
														
 
															-appcmd set config -section:httpLogging /dontLog:True | Out-Null
														
 
															-# Enable detailed error pages
														
 
															-#appcmd set config -section:system.webServer/httpErrors -errorMode:Detailed | Out-Null
														
 
															-# Increase queue length for DefaultAppPool to avoid HTTP 503 errors coming from HTTP.SYS
														
 
															-appcmd set apppool DefaultAppPool /queueLength:65535
														
 
															-# Increase appConcurrentRequestLimit to avoid HTTP 503.2 errors from IIS http://support.microsoft.com/kb/943891
														
 
															-appcmd set config -section:system.webServer/serverRuntime /appConcurrentRequestLimit:65535
														
 
															-
														
 
															-# URL Rewrite
														
 
															-$rewrite_url = "http://download.microsoft.com/download/6/7/D/67D80164-7DD0-48AF-86E3-DE7A182D6815/rewrite_2.0_rtw_x64.msi"
														
 
															-$rewrite_local = "$workdir\rewrite_2.0_rtw_x64.msi"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($rewrite_url, $rewrite_local)
														
 
															-Start-Process "msiexec" "/i $rewrite_local /passive" -Wait
														
 
															-
														
 
															-#
														
 
															-# Tools for building .NET projects on the server
														
 
															-#
														
 
															-Write-Host "`nInstalling .NET build tools...`n"
														
 
															-
														
 
															-# .NET Framework 4.5 SDK
														
 
															-$sdktools_url = "http://download.microsoft.com/download/F/1/3/F1300C9C-A120-4341-90DF-8A52509B23AC/standalonesdk/sdksetup.exe"
														
 
															-$sdktools_local = "$workdir\sdksetup.exe"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($sdktools_url, $sdktools_local)
														
 
															-Start-Process "$workdir\sdksetup.exe" "/features OptionId.NetFxSoftwareDevelopmentKit /q /layout $workdir\sdksetup" -Wait
														
 
															-Start-Process "msiexec" "/i $workdir\sdksetup\Redistributable\4.5.50710\sdk_tools4.msi VSEXTUI=1" -Wait
														
 
															-
														
 
															-# Web Deploy 3.0
														
 
															-$webdeploy_url = "http://download.microsoft.com/download/1/B/3/1B3F8377-CFE1-4B40-8402-AE1FC6A0A8C3/WebDeploy_amd64_en-US.msi"
														
 
															-$webdeploy_local = "$workdir\WebDeploy_amd64_en-US.msi"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($webdeploy_url, $webdeploy_local)
														
 
															-Start-Process "msiexec" "/i $webdeploy_local /passive" -Wait
														
 
															-
														
 
															-#
														
 
															-# node.js
														
 
															-#
														
 
															-Write-Host "Installing node.js...`n"
														
 
															-$node_installer_url = "http://nodejs.org/dist/$node_installer_path"
														
 
															-$node_installer_local = "$workdir\$node_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($node_installer_url, $node_installer_local)
														
 
															-
														
 
															-Start-Process $node_installer_local '/passive' -Wait
														
 
															-$env:Path += ";C:\Program Files\nodejs"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Python
														
 
															-#
														
 
															-Write-Host "Installing Python...`n"
														
 
															-$python_installer_url = "http://www.python.org/ftp/python/$python_installer_path"
														
 
															-$python_installer_local = "$workdir\$python_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($python_installer_url, $python_installer_local)
														
 
															-
														
 
															-Start-Process $python_installer_local '/passive' -Wait
														
 
															-$env:Path += ";C:\Python$python_version"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# PHP
														
 
															-#
														
 
															-Write-Host "Installing PHP..."
														
 
															-
														
 
															-# Locate current PHP 5.4 release
														
 
															-Write-Host "Looking for current PHP 5.4 release"
														
 
															-$php_download_page_url = 'http://windows.php.net/download/'
														
 
															-$php_download_page_file = [IO.Path]::GetTempFileName()
														
 
															-Write-Host "Downloading from $php_download_page_url into $php_download_page_file"
														
 
															-Try {
														
 
															-    (New-Object System.Net.WebClient).DownloadFile($php_download_page_url, $php_download_page_file)
														
 
															-} Catch {
														
 
															-    Write-Host "ERROR: Could not download from $php_download_page_url."
														
 
															-    Write-Host $_.Exception.Message
														
 
															-    Exit 1
														
 
															-}
														
 
															-$file = (cat $php_download_page_file) -join ""
														
 
															-if ($file -match '(?s)h4 id="php-5.4-nts-VC9-x86".*?href="/downloads/releases/(.*?)">Zip</a>') {
														
 
															-    $php_installer_file = $matches[1]
														
 
															-    $php_installer_url = "http://windows.php.net/downloads/releases/$php_installer_file"
														
 
															-    Write-Host "Current PHP 5.4 release found at $php_installer_url"
														
 
															-}
														
 
															-else {
														
 
															-    Write-Host "ERROR: Current PHP release was not found. Aborting."
														
 
															-    Exit 1
														
 
															-}
														
 
															-
														
 
															-# Download PHP
														
 
															-$php_installer_local = "$workdir\$php_installer_file"
														
 
															-Try {
														
 
															-    (New-Object System.Net.WebClient).DownloadFile($php_installer_url, $php_installer_local)
														
 
															-} Catch {
														
 
															-    Write-Host "ERROR: Could not download from $php_installer_url. "
														
 
															-    Write-Host $_.Exception.Message
														
 
															-    Exit 1
														
 
															-}
														
 
															-
														
 
															-# Install PHP
														
 
															-$php = "C:\PHP"
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($php_installer_local, $php) | Out-Null
														
 
															-$env:Path += ";" + $php; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# php.ini
														
 
															-$phpini = "$php\php.ini"
														
 
															-Copy-Item "$php\php.ini-production" $phpini
														
 
															-(Get-Content $phpini) -Replace ";date.timezone =", "date.timezone = UTC" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "short_open_tag = Off", "short_open_tag = On" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "display_errors = Off", "display_errors = Off" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "log_errors = On", "log_errors = Off" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "output_buffering = 4096", "output_buffering = Off" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";cgi.force_redirect = 1", "cgi.force_redirect = 0" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";fastcgi.impersonate = 1", "fastcgi.impersonate = 0" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";fastcgi.logging = 0", "fastcgi.logging = 0" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace '; extension_dir = "./"', "extension_dir = `"$php\ext`"" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";extension=", "extension=" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "extension=php_(interbase|oci8|oci8_11g|firebird|oci|pspell|sybase_ct|zip|pdo_firebird|pdo_oci|snmp).dll.*", "" | Set-Content $phpini
														
 
															-
														
 
															-# IIS with PHP via FastCGI
														
 
															-Install-WindowsFeature Web-CGI | Out-Null
														
 
															-appcmd set config -section:system.webServer/fastCgi /+"[fullPath='C:\PHP\php-cgi.exe', arguments='', maxInstances='0', instanceMaxRequests='10000', queueLength='1000', rapidFailsPerMinute='10', idleTimeout='300', activityTimeout='30', requestTimeout='90', protocol='NamedPipe', flushNamedPipe='False']" /commit:apphost | Out-Null
														
 
															-appcmd set config -section:system.webServer/fastCgi /+"[fullPath='C:\PHP\php-cgi.exe'].environmentVariables.[name='PHPRC', value='C:\PHP\php.ini']" /commit:apphost | Out-Null
														
 
															-appcmd set config -section:system.webServer/handlers /+"[name='PHP FastCGI', path='*.php', modules='FastCgiModule', verb='*', scriptProcessor='C:\PHP\php-cgi.exe', resourceType='File', requireAccess='Script']" /commit:apphost | Out-Null
														
 
															-
														
 
															-# phpinfo() test file
														
 
															-Set-Content "c:\inetpub\wwwroot\phpinfo.php" "<?php phpinfo(); ?>"
														
 
															-
														
 
															-# wincache
														
 
															-$wincache_url = "http://heanet.dl.sourceforge.net/project/wincache/$wincache_installer_path"
														
 
															-$wincache_local = "$workdir\$wincache_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($wincache_url, $wincache_local)
														
 
															-Start-Process $wincache_local "/q /T:$php\ext" -Wait
														
 
															-Move-Item "$php\ext\wincache*" "c:\inetpub\wwwroot"
														
 
															-Set-ItemProperty "c:\inetpub\wwwroot\wincache.php" -name IsReadOnly -value $false
														
 
															-(Get-Content "c:\inetpub\wwwroot\wincache.php") -Replace "'USE_AUTHENTICATION', 1", "'USE_AUTHENTICATION', 0" | Set-Content "c:\inetpub\wwwroot\wincache.php"
														
 
															-Add-Content $phpini "`n`n[PHP]`n"
														
 
															-Add-Content $phpini "extension=php_wincache.dll"
														
 
															-
														
 
															-# composer
														
 
															-$composer_url = "https://getcomposer.org/Composer-Setup.exe"
														
 
															-$composer_local = "$workdir\Composer-Setup.exe"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($composer_url, $composer_local)
														
 
															-Start-Process $composer_local "/silent" -Wait
														
 
															-$env:Path += ";C:\ProgramData\Composer\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-Write-Host ""
														
 
															-
														
 
															-#
														
 
															-# Go
														
 
															-#
														
 
															-Write-Host "Installing Go...`n"
														
 
															-$go_url = "http://go.googlecode.com/files/$go_installer_file"
														
 
															-$go_local = "$workdir\$go_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($go_url, $go_local)
														
 
															-Start-Process $go_local "/passive" -Wait
														
 
															-$env:Path += ";C:\Go\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Java
														
 
															-#
														
 
															-Write-Host "Installing Java...`n"
														
 
															-
														
 
															-# jre
														
 
															-#Write-Host "Installing JRE...`n"
														
 
															-#$jre_url = "http://img.cs.montana.edu/windows/$jre_installer_file"
														
 
															-#$jre_local = "$workdir\$jre_installer_file"
														
 
															-#$jre_dir = "C:\Java\jre"
														
 
															-#(New-Object System.Net.WebClient).DownloadFile($jre_url, $jre_local)
														
 
															-#Start-Process $jre_local "/s INSTALLDIR=$jre_dir" -Wait
														
 
															-#$env:Path += ";$jre_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-#$env:JAVA_HOME = $jre_dir; [Environment]::SetEnvironmentVariable("JAVA_HOME", $jre_dir, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# jdk
														
 
															-Write-Host "Installing JDK...`n"
														
 
															-$jdk_url = "http://ghaffarian.net/downloads/Java/JDK/$jdk_installer_file"
														
 
															-$jdk_local = "$workdir\$jdk_installer_file"
														
 
															-$jdk_dir = "C:\Java\jdk"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($jdk_url, $jdk_local)
														
 
															-
														
 
															-$jdk_local_hash = GetMd5FileHash($jdk_local)
														
 
															-if ($jdk_master_hash -ne $jdk_local_hash)
														
 
															-{
														
 
															-    Write-Host $jdk_master_hash
														
 
															-    Write-Host $jdk_local_hash
														
 
															-    Write-Host "JDK file checksum mismatch. Aborting!"
														
 
															-    Exit 1
														
 
															-}
														
 
															-
														
 
															-Start-Process $jdk_local "/s INSTALLDIR=$jdk_dir" -Wait
														
 
															-$env:Path += ";$jdk_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-$env:JAVA_HOME = $jdk_dir; [Environment]::SetEnvironmentVariable("JAVA_HOME", $jdk_dir, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# resin
														
 
															-Write-Host "Installing Resin...`n"
														
 
															-$resin_url = "http://www.caucho.com/download/$resin_installer_file"
														
 
															-$resin_local = "$workdir\$resin_installer_file"
														
 
															-$resin_dir = "C:\Java\resin"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($resin_url, $resin_local)
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($resin_local, $workdir) | Out-Null
														
 
															-Move-Item "$workdir\$resin_version" $resin_dir
														
 
															-Copy-Item "$basedir\config\resin.properties" "$resin_dir\conf\resin.properties"
														
 
															-[Environment]::SetEnvironmentVariable("RESIN_HOME", $resin_dir, [System.EnvironmentVariableTarget]::Machine)
														
 
															-#$env:Path += ";$resin_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# ant
														
 
															-#Write-Host "Installing Ant...`n"
														
 
															-#$ant_url = "http://apache.mirrors.hoobly.com//ant/binaries/$ant_installer_file"
														
 
															-#$ant_local = "$workdir\$ant_installer_file"
														
 
															-#$ant_dir = "C:\Java\ant"
														
 
															-#(New-Object System.Net.WebClient).DownloadFile($ant_url, $ant_local)
														
 
															-#[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-#[System.IO.Compression.ZipFile]::ExtractToDirectory($ant_local, $workdir) | Out-Null
														
 
															-#Move-Item "$workdir\$ant_version" $ant_dir
														
 
															-#$env:Path += ";$ant_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# maven
														
 
															-Write-Host "Installing Maven...`n"
														
 
															-$maven_url = "http://mirror.cc.columbia.edu/pub/software/apache/maven/$maven_installer_path"
														
 
															-$maven_local = "$workdir\$maven_installer_file"
														
 
															-$maven_dir = "C:\Java\maven"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($maven_url, $maven_local)
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($maven_local, $workdir) | Out-Null
														
 
															-Move-Item "$workdir\$maven_version" $maven_dir
														
 
															-$env:Path += ";$maven_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# scala
														
 
															-cinst scala -version $scala_version
														
 
															-
														
 
															-# play
														
 
															-$play_url = "http://downloads.typesafe.com/play/$play_version/$play_installer_file"
														
 
															-$play_local = "$workdir\$play_installer_file"
														
 
															-$play_dir = "C:\Java\play"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($play_url, $play_local)
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($play_local, $workdir) | Out-Null
														
 
															-Move-Item "$workdir\play-$play_version" $play_dir
														
 
															-$env:Path += ";$play_dir"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# sbt
														
 
															-$sbt_installer_file = "sbt-$sbt_version.zip"
														
 
															-$sbt_url = "http://dl.bintray.com/sbt/native-packages/sbt/$sbt_version/$sbt_installer_file"
														
 
															-$sbt_local = "$workdir\$sbt_installer_file"
														
 
															-$sbt_dir = "C:\Java\sbt"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($sbt_url, $sbt_local)
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($sbt_local, $workdir) | Out-Null
														
 
															-Move-Item "$workdir\sbt" $sbt_dir
														
 
															-$env:Path += ";$sbt_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Firewall
														
 
															-#
														
 
															-Write-Host "Configuring firewall...`n"
														
 
															-New-NetFirewallRule -DisplayName "HTTP 8080" -Action Allow -Direction Inbound -LocalPort 8080 -Protocol TCP | Out-Null
														
 
															-
														
 
															-#
														
 
															-# Mercurial
														
 
															-#
														
 
															-Write-Host "Installing Mercurial...`n"
														
 
															-$hg_installer_url = "https://bitbucket.org/tortoisehg/files/downloads/$mercurial_installer_file"
														
 
															-$hg_installer_local = "$workdir\$mercurial_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($hg_installer_url, $hg_installer_local)
														
 
															-
														
 
															-Start-Process $hg_installer_local '/passive' -Wait
														
 
															-$env:Path += ";C:\Program Files\Mercurial"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Cygwin (including sftp)
														
 
															-#
														
 
															-Write-Host "Installing Cygwin...`n"
														
 
															-$cygwin_installer_url = "http://cygwin.com/$cygwin_installer_file"
														
 
															-$cygwin_installer_dir = $workdir + "\cygwin-installer"
														
 
															-New-Item -Path $cygwin_installer_dir -Type directory -Force | Out-Null
														
 
															-$cygwin_installer_local = "$cygwin_installer_dir\$cygwin_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($cygwin_installer_url, $cygwin_installer_local)
														
 
															-
														
 
															-$cygwin_install_dir = "C:\Cygwin"
														
 
															-Start-Process $cygwin_installer_local "-q -n -l $cygwin_installer_dir -s http://mirrors.kernel.org/sourceware/cygwin/ -R $cygwin_install_dir -P openssh" -WorkingDirectory "$cygwin_installer_dir" -Wait -RedirectStandardOutput $cygwin_installer_dir\install.log
														
 
															-$env:Path += ";$cygwin_install_dir;$cygwin_install_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-cd $basedir
														
--- a/toolset/setup/windows/installer.ps1~
+++ b/toolset/setup/windows/installer.ps1~
@@ -1,340 +0,0 @@
 
															-#
														
 
															-# Versions of software (will need to be updated from time to time)
														
 
															-#
														
 
															-
														
 
															-$node_installer_file      = "node-v0.10.13-x64.msi"
														
 
															-$node_installer_path      = "v0.10.13/x64/$node_installer_file"
														
 
															-$python_installer_file    = "python-2.7.5.amd64.msi"
														
 
															-$python_installer_path    = "2.7.5/$python_installer_file"
														
 
															-$python_version           = "27"
														
 
															-$wincache_installer_file  = "wincache-1.3.4-5.4-nts-vc9-x86.exe"
														
 
															-$wincache_installer_path  = "wincache-1.3.4/$wincache_installer_file"
														
 
															-$go_installer_file        = "go1.2.windows-amd64.msi"
														
 
															-$jre_installer_file       = "jre-7u25-windows-x64.exe"
														
 
															-$jdk_installer_file       = "jdk-7u45-windows-x64.exe"
														
 
															-$jdk_master_hash          = "943527ed9111cbb746d4ab2bb2c31cd6" 
														
 
															-# https://www.oracle.com/technetwork/java/javase/downloads/java-se-binaries-checksum-1956892.html
														
 
															-$resin_version            = "resin-4.0.36"
														
 
															-$resin_installer_file     = "$resin_version.zip"
														
 
															-$ant_version              = "apache-ant-1.9.2"
														
 
															-$ant_installer_file       = "$ant_version-bin.zip"
														
 
															-$maven_version            = "apache-maven-3.0.5"
														
 
															-$maven_installer_file     = "$maven_version-bin.zip"
														
 
															-$maven_installer_path     = "maven-3/3.0.5/binaries/$maven_installer_file"
														
 
															-$scala_version            = "2.10.2"
														
 
															-$play_version             = "2.2.0"
														
 
															-$play_installer_file      = "play-$play_version.zip"
														
 
															-$mercurial_installer_file = "mercurial-2.6.1-x64.msi"
														
 
															-$cygwin_installer_file    = "setup-x86_64.exe"
														
 
															-
														
 
															-
														
 
															-$basedir = "C:\FrameworkBenchmarks"
														
 
															-$workdir = "$basedir\installs"
														
 
															-New-Item -Path $workdir -Type directory -Force | Out-Null
														
 
															-
														
 
															-function GetMd5FileHash($fileName) {
														
 
															-    [Reflection.Assembly]::LoadWithPartialName("System.Security") | out-null
														
 
															-    $md5 = [System.Security.Cryptography.MD5]::Create()
														
 
															-
														
 
															-    $file = [System.IO.File]::OpenRead($fileName)
														
 
															-    $hash = $md5.ComputeHash($file)
														
 
															-    $file.Dispose()
														
 
															-
														
 
															-    $sb = New-Object System.Text.StringBuilder
														
 
															-    $hash | % { [Void]$sb.Append($_.ToString("x2")) }
														
 
															-    $sb.ToString()
														
 
															-}
														
 
															-
														
 
															-#
														
 
															-# Chocolatey package manager
														
 
															-#
														
 
															-Write-Host "Installing Chocolatey package manager"
														
 
															-Invoke-Expression ((new-object net.webclient).DownloadString('https://chocolatey.org/install.ps1'))
														
 
															-
														
 
															-#
														
 
															-# ASP.NET
														
 
															-#
														
 
															-Write-Host "Installing IIS, .NET and ASP.NET..."
														
 
															-
														
 
															-# Enable Windows Update to get rid of the yellow warnings
														
 
															-# But this is not strictly neccessary
														
 
															-$Updates = (New-Object -ComObject "Microsoft.Update.AutoUpdate").Settings
														
 
															-$Updates.NotificationLevel = 2 # Notify before download
														
 
															-$Updates.Save()
														
 
															-$Updates.Refresh()
														
 
															-
														
 
															-Install-WindowsFeature Web-Server
														
 
															-Install-WindowsFeature Web-Mgmt-Console
														
 
															-Install-WindowsFeature NET-Framework-45-ASPNET
														
 
															-Install-WindowsFeature Web-Asp-Net45
														
 
															-
														
 
															-$env:Path += ";C:\Windows\system32\inetsrv"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-# Optimize performance
														
 
															-appcmd set config -section:httpProtocol /allowKeepAlive:true | Out-Null
														
 
															-appcmd set config -section:httpLogging /dontLog:True | Out-Null
														
 
															-# Enable detailed error pages
														
 
															-#appcmd set config -section:system.webServer/httpErrors -errorMode:Detailed | Out-Null
														
 
															-# Increase queue length for DefaultAppPool to avoid HTTP 503 errors coming from HTTP.SYS
														
 
															-appcmd set apppool DefaultAppPool /queueLength:65535
														
 
															-# Increase appConcurrentRequestLimit to avoid HTTP 503.2 errors from IIS http://support.microsoft.com/kb/943891
														
 
															-appcmd set config -section:system.webServer/serverRuntime /appConcurrentRequestLimit:65535
														
 
															-
														
 
															-# URL Rewrite
														
 
															-$rewrite_url = "http://download.microsoft.com/download/6/7/D/67D80164-7DD0-48AF-86E3-DE7A182D6815/rewrite_2.0_rtw_x64.msi"
														
 
															-$rewrite_local = "$workdir\rewrite_2.0_rtw_x64.msi"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($rewrite_url, $rewrite_local)
														
 
															-Start-Process "msiexec" "/i $rewrite_local /passive" -Wait
														
 
															-
														
 
															-#
														
 
															-# Tools for building .NET projects on the server
														
 
															-#
														
 
															-Write-Host "`nInstalling .NET build tools...`n"
														
 
															-
														
 
															-# .NET Framework 4.5 SDK
														
 
															-$sdktools_url = "http://download.microsoft.com/download/F/1/3/F1300C9C-A120-4341-90DF-8A52509B23AC/standalonesdk/sdksetup.exe"
														
 
															-$sdktools_local = "$workdir\sdksetup.exe"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($sdktools_url, $sdktools_local)
														
 
															-Start-Process "$workdir\sdksetup.exe" "/features OptionId.NetFxSoftwareDevelopmentKit /q /layout $workdir\sdksetup" -Wait
														
 
															-Start-Process "msiexec" "/i $workdir\sdksetup\Redistributable\4.5.50710\sdk_tools4.msi VSEXTUI=1" -Wait
														
 
															-
														
 
															-# Web Deploy 3.0
														
 
															-$webdeploy_url = "http://download.microsoft.com/download/1/B/3/1B3F8377-CFE1-4B40-8402-AE1FC6A0A8C3/WebDeploy_amd64_en-US.msi"
														
 
															-$webdeploy_local = "$workdir\WebDeploy_amd64_en-US.msi"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($webdeploy_url, $webdeploy_local)
														
 
															-Start-Process "msiexec" "/i $webdeploy_local /passive" -Wait
														
 
															-
														
 
															-#
														
 
															-# node.js
														
 
															-#
														
 
															-Write-Host "Installing node.js...`n"
														
 
															-$node_installer_url = "http://nodejs.org/dist/$node_installer_path"
														
 
															-$node_installer_local = "$workdir\$node_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($node_installer_url, $node_installer_local)
														
 
															-
														
 
															-Start-Process $node_installer_local '/passive' -Wait
														
 
															-$env:Path += ";C:\Program Files\nodejs"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Python
														
 
															-#
														
 
															-Write-Host "Installing Python...`n"
														
 
															-$python_installer_url = "http://www.python.org/ftp/python/$python_installer_path"
														
 
															-$python_installer_local = "$workdir\$python_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($python_installer_url, $python_installer_local)
														
 
															-
														
 
															-Start-Process $python_installer_local '/passive' -Wait
														
 
															-$env:Path += ";C:\Python$python_version"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# PHP
														
 
															-#
														
 
															-Write-Host "Installing PHP..."
														
 
															-
														
 
															-# Locate current PHP 5.4 release
														
 
															-Write-Host "Looking for current PHP 5.4 release"
														
 
															-$php_download_page_url = 'http://windows.php.net/download/'
														
 
															-$php_download_page_file = [IO.Path]::GetTempFileName()
														
 
															-Write-Host "Downloading from $php_download_page_url into $php_download_page_file"
														
 
															-Try {
														
 
															-    (New-Object System.Net.WebClient).DownloadFile($php_download_page_url, $php_download_page_file)
														
 
															-} Catch {
														
 
															-    Write-Host "ERROR: Could not download from $php_download_page_url."
														
 
															-    Write-Host $_.Exception.Message
														
 
															-    Exit 1
														
 
															-}
														
 
															-$file = (cat $php_download_page_file) -join ""
														
 
															-if ($file -match '(?s)h4 id="php-5.4-nts-VC9-x86".*?href="/downloads/releases/(.*?)">Zip</a>') {
														
 
															-    $php_installer_file = $matches[1]
														
 
															-    $php_installer_url = "http://windows.php.net/downloads/releases/$php_installer_file"
														
 
															-    Write-Host "Current PHP 5.4 release found at $php_installer_url"
														
 
															-}
														
 
															-else {
														
 
															-    Write-Host "ERROR: Current PHP release was not found. Aborting."
														
 
															-    Exit 1
														
 
															-}
														
 
															-
														
 
															-# Download PHP
														
 
															-$php_installer_local = "$workdir\$php_installer_file"
														
 
															-Try {
														
 
															-    (New-Object System.Net.WebClient).DownloadFile($php_installer_url, $php_installer_local)
														
 
															-} Catch {
														
 
															-    Write-Host "ERROR: Could not download from $php_installer_url. "
														
 
															-    Write-Host $_.Exception.Message
														
 
															-    Exit 1
														
 
															-}
														
 
															-
														
 
															-# Install PHP
														
 
															-$php = "C:\PHP"
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($php_installer_local, $php) | Out-Null
														
 
															-$env:Path += ";" + $php; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# php.ini
														
 
															-$phpini = "$php\php.ini"
														
 
															-Copy-Item "$php\php.ini-production" $phpini
														
 
															-(Get-Content $phpini) -Replace ";date.timezone =", "date.timezone = UTC" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "short_open_tag = Off", "short_open_tag = On" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "display_errors = Off", "display_errors = Off" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "log_errors = On", "log_errors = Off" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "output_buffering = 4096", "output_buffering = Off" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";cgi.force_redirect = 1", "cgi.force_redirect = 0" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";fastcgi.impersonate = 1", "fastcgi.impersonate = 0" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";fastcgi.logging = 0", "fastcgi.logging = 0" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace '; extension_dir = "./"', "extension_dir = `"$php\ext`"" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace ";extension=", "extension=" | Set-Content $phpini
														
 
															-(Get-Content $phpini) -Replace "extension=php_(interbase|oci8|oci8_11g|firebird|oci|pspell|sybase_ct|zip|pdo_firebird|pdo_oci|snmp).dll.*", "" | Set-Content $phpini
														
 
															-
														
 
															-# IIS with PHP via FastCGI
														
 
															-Install-WindowsFeature Web-CGI | Out-Null
														
 
															-appcmd set config -section:system.webServer/fastCgi /+"[fullPath='C:\PHP\php-cgi.exe', arguments='', maxInstances='0', instanceMaxRequests='10000', queueLength='1000', rapidFailsPerMinute='10', idleTimeout='300', activityTimeout='30', requestTimeout='90', protocol='NamedPipe', flushNamedPipe='False']" /commit:apphost | Out-Null
														
 
															-appcmd set config -section:system.webServer/fastCgi /+"[fullPath='C:\PHP\php-cgi.exe'].environmentVariables.[name='PHPRC', value='C:\PHP\php.ini']" /commit:apphost | Out-Null
														
 
															-appcmd set config -section:system.webServer/handlers /+"[name='PHP FastCGI', path='*.php', modules='FastCgiModule', verb='*', scriptProcessor='C:\PHP\php-cgi.exe', resourceType='File', requireAccess='Script']" /commit:apphost | Out-Null
														
 
															-
														
 
															-# phpinfo() test file
														
 
															-Set-Content "c:\inetpub\wwwroot\phpinfo.php" "<?php phpinfo(); ?>"
														
 
															-
														
 
															-# wincache
														
 
															-$wincache_url = "http://heanet.dl.sourceforge.net/project/wincache/$wincache_installer_path"
														
 
															-$wincache_local = "$workdir\$wincache_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($wincache_url, $wincache_local)
														
 
															-Start-Process $wincache_local "/q /T:$php\ext" -Wait
														
 
															-Move-Item "$php\ext\wincache*" "c:\inetpub\wwwroot"
														
 
															-Set-ItemProperty "c:\inetpub\wwwroot\wincache.php" -name IsReadOnly -value $false
														
 
															-(Get-Content "c:\inetpub\wwwroot\wincache.php") -Replace "'USE_AUTHENTICATION', 1", "'USE_AUTHENTICATION', 0" | Set-Content "c:\inetpub\wwwroot\wincache.php"
														
 
															-Add-Content $phpini "`n`n[PHP]`n"
														
 
															-Add-Content $phpini "extension=php_wincache.dll"
														
 
															-
														
 
															-# composer
														
 
															-$composer_url = "https://getcomposer.org/Composer-Setup.exe"
														
 
															-$composer_local = "$workdir\Composer-Setup.exe"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($composer_url, $composer_local)
														
 
															-Start-Process $composer_local "/silent" -Wait
														
 
															-$env:Path += ";C:\ProgramData\Composer\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-Write-Host ""
														
 
															-
														
 
															-#
														
 
															-# Go
														
 
															-#
														
 
															-Write-Host "Installing Go...`n"
														
 
															-$go_url = "http://go.googlecode.com/files/$go_installer_file"
														
 
															-$go_local = "$workdir\$go_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($go_url, $go_local)
														
 
															-Start-Process $go_local "/passive" -Wait
														
 
															-$env:Path += ";C:\Go\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Java
														
 
															-#
														
 
															-Write-Host "Installing Java...`n"
														
 
															-
														
 
															-# jre
														
 
															-#Write-Host "Installing JRE...`n"
														
 
															-#$jre_url = "http://img.cs.montana.edu/windows/$jre_installer_file"
														
 
															-#$jre_local = "$workdir\$jre_installer_file"
														
 
															-#$jre_dir = "C:\Java\jre"
														
 
															-#(New-Object System.Net.WebClient).DownloadFile($jre_url, $jre_local)
														
 
															-#Start-Process $jre_local "/s INSTALLDIR=$jre_dir" -Wait
														
 
															-#$env:Path += ";$jre_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-#$env:JAVA_HOME = $jre_dir; [Environment]::SetEnvironmentVariable("JAVA_HOME", $jre_dir, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# jdk
														
 
															-Write-Host "Installing JDK...`n"
														
 
															-$jdk_url = "http://ghaffarian.net/downloads/Java/JDK/$jdk_installer_file"
														
 
															-$jdk_local = "$workdir\$jdk_installer_file"
														
 
															-$jdk_dir = "C:\Java\jdk"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($jdk_url, $jdk_local)
														
 
															-
														
 
															-$jdk_local_hash = GetMd5FileHash($jdk_local)
														
 
															-if ($jdk_master_hash -ne $jdk_local_hash)
														
 
															-{
														
 
															-    Write-Host $jdk_master_hash
														
 
															-    Write-Host $jdk_local_hash
														
 
															-    Write-Host "JDK file checksum mismatch. Aborting!"
														
 
															-    Exit 1
														
 
															-}
														
 
															-
														
 
															-Start-Process $jdk_local "/s INSTALLDIR=$jdk_dir" -Wait
														
 
															-$env:Path += ";$jdk_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-$env:JAVA_HOME = $jdk_dir; [Environment]::SetEnvironmentVariable("JAVA_HOME", $jdk_dir, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# resin
														
 
															-Write-Host "Installing Resin...`n"
														
 
															-$resin_url = "http://www.caucho.com/download/$resin_installer_file"
														
 
															-$resin_local = "$workdir\$resin_installer_file"
														
 
															-$resin_dir = "C:\Java\resin"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($resin_url, $resin_local)
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($resin_local, $workdir) | Out-Null
														
 
															-Move-Item "$workdir\$resin_version" $resin_dir
														
 
															-Copy-Item "$basedir\config\resin.properties" "$resin_dir\conf\resin.properties"
														
 
															-[Environment]::SetEnvironmentVariable("RESIN_HOME", $resin_dir, [System.EnvironmentVariableTarget]::Machine)
														
 
															-#$env:Path += ";$resin_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# ant
														
 
															-#Write-Host "Installing Ant...`n"
														
 
															-#$ant_url = "http://apache.mirrors.hoobly.com//ant/binaries/$ant_installer_file"
														
 
															-#$ant_local = "$workdir\$ant_installer_file"
														
 
															-#$ant_dir = "C:\Java\ant"
														
 
															-#(New-Object System.Net.WebClient).DownloadFile($ant_url, $ant_local)
														
 
															-#[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-#[System.IO.Compression.ZipFile]::ExtractToDirectory($ant_local, $workdir) | Out-Null
														
 
															-#Move-Item "$workdir\$ant_version" $ant_dir
														
 
															-#$env:Path += ";$ant_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# maven
														
 
															-Write-Host "Installing Maven...`n"
														
 
															-$maven_url = "http://mirror.cc.columbia.edu/pub/software/apache/maven/$maven_installer_path"
														
 
															-$maven_local = "$workdir\$maven_installer_file"
														
 
															-$maven_dir = "C:\Java\maven"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($maven_url, $maven_local)
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($maven_local, $workdir) | Out-Null
														
 
															-Move-Item "$workdir\$maven_version" $maven_dir
														
 
															-$env:Path += ";$maven_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-# scala
														
 
															-cinst scala -version $scala_version
														
 
															-
														
 
															-# play
														
 
															-$play_url = "http://downloads.typesafe.com/play/$play_version/$play_installer_file"
														
 
															-$play_local = "$workdir\$play_installer_file"
														
 
															-$play_dir = "C:\Java\play"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($play_url, $play_local)
														
 
															-[System.Reflection.Assembly]::LoadWithPartialName("System.IO.Compression.FileSystem") | Out-Null
														
 
															-[System.IO.Compression.ZipFile]::ExtractToDirectory($play_local, $workdir) | Out-Null
														
 
															-Move-Item "$workdir\play-$play_version" $play_dir
														
 
															-$env:Path += ";$play_dir"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Firewall
														
 
															-#
														
 
															-Write-Host "Configuring firewall...`n"
														
 
															-New-NetFirewallRule -DisplayName "HTTP 8080" -Action Allow -Direction Inbound -LocalPort 8080 -Protocol TCP | Out-Null
														
 
															-
														
 
															-#
														
 
															-# Mercurial
														
 
															-#
														
 
															-Write-Host "Installing Mercurial...`n"
														
 
															-$hg_installer_url = "https://bitbucket.org/tortoisehg/files/downloads/$mercurial_installer_file"
														
 
															-$hg_installer_local = "$workdir\$mercurial_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($hg_installer_url, $hg_installer_local)
														
 
															-
														
 
															-Start-Process $hg_installer_local '/passive' -Wait
														
 
															-$env:Path += ";C:\Program Files\Mercurial"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-#
														
 
															-# Cygwin (including sftp)
														
 
															-#
														
 
															-Write-Host "Installing Cygwin...`n"
														
 
															-$cygwin_installer_url = "http://cygwin.com/$cygwin_installer_file"
														
 
															-$cygwin_installer_dir = $workdir + "\cygwin-installer"
														
 
															-New-Item -Path $cygwin_installer_dir -Type directory -Force | Out-Null
														
 
															-$cygwin_installer_local = "$cygwin_installer_dir\$cygwin_installer_file"
														
 
															-(New-Object System.Net.WebClient).DownloadFile($cygwin_installer_url, $cygwin_installer_local)
														
 
															-
														
 
															-$cygwin_install_dir = "C:\Cygwin"
														
 
															-Start-Process $cygwin_installer_local "-q -n -l $cygwin_installer_dir -s http://mirrors.kernel.org/sourceware/cygwin/ -R $cygwin_install_dir -P openssh" -WorkingDirectory "$cygwin_installer_dir" -Wait -RedirectStandardOutput $cygwin_installer_dir\install.log
														
 
															-$env:Path += ";$cygwin_install_dir;$cygwin_install_dir\bin"; [Environment]::SetEnvironmentVariable("Path", $env:Path, [System.EnvironmentVariableTarget]::Machine)
														
 
															-
														
 
															-cd $basedir
														
--- a/toolset/travis/travis_diff.py
+++ b/toolset/travis/travis_diff.py
@@ -16,12 +16,16 @@ from sets import Set
 
															 # Returns a unique list of fw_depends changes
														
 
															 def get_fw_depends_changes(changes_output):
														
 
															-    return list(Set(re.findall(r"toolset/setup/linux/.+/(.+)\.sh", changes_output, re.M)))
														
 
															+    return list(
														
 
															+        Set(
														
 
															+            re.findall(r"toolset/setup/linux/.+/(.+)\.sh", changes_output,
														
 
															+                       re.M)))
														
 
															 # Returns a unique list of frameworks that have been changed
														
 
															 def fw_found_in_changes(changes_output):
														
 
															-    return re.search(r"" + re.escape(os.environ['TESTDIR']), changes_output, re.M)
														
 
															+    return re.search(r"" + re.escape(os.environ['TESTDIR']), changes_output,
														
 
															+                     re.M)
														
 
															 # Cleans up diffing and grep output and into an array of strings
														
@@ -34,16 +38,19 @@ def quit_diffing(should_test_run):
 
															         print("travis-diff-continue")
														
 
															     exit(0)
														
 
															+
														
 
															 # TODO: Remove this
														
 
															 quit_diffing(True)
														
 
															 # COMMIT MESSAGES:
														
 
															 # Before any complicated diffing, check for forced runs from the commit message
														
 
															-last_commit_msg = subprocess.check_output(['bash', '-c', 'git log -1 --pretty=%B'])
														
 
															+last_commit_msg = subprocess.check_output(
														
 
															+    ['bash', '-c', 'git log -1 --pretty=%B'])
														
 
															 # Forced *fw-only* specific tests
														
 
															 if re.search(r'\[ci fw-only.+\]', last_commit_msg, re.M):
														
 
															-    if re.search(r'\[ci fw-only(.?)+ ' + re.escape(os.environ['TESTDIR']) + '( .+\]|])', last_commit_msg, re.M):
														
 
															+    if re.search(r'\[ci fw-only(.?)+ ' + re.escape(os.environ['TESTDIR']) +
														
 
															+                 '( .+\]|])', last_commit_msg, re.M):
														
 
															         print("This test has been forced to run from the commit message.")
														
 
															         quit_diffing(True)
														
 
															     else:
														
@@ -54,7 +61,9 @@ if re.search(r'\[ci run-all\]', last_commit_msg, re.M):
 
															     print("All tests have been forced to run from the commit message.")
														
 
															     quit_diffing(True)
														
 
															 # Forced framework run
														
 
															-if re.search(r'\[ci fw(.?)+ ' + re.escape(os.environ['TESTDIR']) + '( .+\]|\])', last_commit_msg, re.M):
														
 
															+if re.search(
														
 
															+        r'\[ci fw(.?)+ ' + re.escape(os.environ['TESTDIR']) + '( .+\]|\])',
														
 
															+        last_commit_msg, re.M):
														
 
															     print('This test has been forced to run from the commit message.')
														
 
															     quit_diffing(True)
														
@@ -67,7 +76,8 @@ commit_range = ""
 
															 if is_PR:
														
 
															     print('I am testing a pull request')
														
 
															     first_commit = os.environ['TRAVIS_COMMIT_RANGE'].split('...')[0]
														
 
															-    last_commit = subprocess.check_output("git rev-list -n 1 FETCH_HEAD^2", shell=True).rstrip('\n')
														
 
															+    last_commit = subprocess.check_output(
														
 
															+        "git rev-list -n 1 FETCH_HEAD^2", shell=True).rstrip('\n')
														
 
															     print("Guessing that first commit in PR is : {!s}".format(first_commit))
														
 
															     print("Guessing that final commit in PR is : {!s}".format(last_commit))
														
@@ -78,19 +88,27 @@ if is_PR:
 
															         print("Only one commit in range, examining {!s}".format(last_commit))
														
 
															         commit_range = "-m --first-parent -1 {!s}".format(last_commit)
														
 
															     else:
														
 
															-        commit_range = "--first-parent {!s}...{!s}".format(first_commit, last_commit)
														
 
															+        commit_range = "--first-parent {!s}...{!s}".format(
														
 
															+            first_commit, last_commit)
														
 
															 if not is_PR:
														
 
															     print('I am not testing a pull request')
														
 
															-    commit_range = "--first-parent -m {!s}".format(os.environ['TRAVIS_COMMIT_RANGE'])
														
 
															+    commit_range = "--first-parent -m {!s}".format(
														
 
															+        os.environ['TRAVIS_COMMIT_RANGE'])
														
 
															     # Handle 1
														
 
															     if commit_range == "":
														
 
															-        commit_range = "--first-parent -m -1 {!s}".format(os.environ['TRAVIS_COMMIT'])
														
 
															+        commit_range = "--first-parent -m -1 {!s}".format(
														
 
															+            os.environ['TRAVIS_COMMIT'])
														
 
															 print("Using commit range `{!s}`".format(commit_range))
														
 
															-print("Running `git log --name-only --pretty=\"format:\" {!s}`".format(commit_range))
														
 
															-changes = clean_output(subprocess.check_output(['bash', '-c', 'git log --name-only --pretty="format:" {!s}'.format(commit_range)]))
														
 
															+print("Running `git log --name-only --pretty=\"format:\" {!s}`".format(
														
 
															+    commit_range))
														
 
															+changes = clean_output(
														
 
															+    subprocess.check_output([
														
 
															+        'bash', '-c',
														
 
															+        'git log --name-only --pretty="format:" {!s}'.format(commit_range)
														
 
															+    ]))
														
 
															 # Satisfies this requirement:
														
 
															 #   Anything in the toolset/ that isn't in the setup/linux/*/ subdirectory
														
@@ -116,14 +134,18 @@ i = 0
 
															 while i <= len(fw_depends_changes) - 1:
														
 
															     # Generates output of files that contain the fw_depends for this dependency
														
 
															-    more_changes = subprocess.check_output(['bash', '-c', 'grep -RP "fw_depends(.?)+ ' + re.escape(fw_depends_changes[i]) + '( |$)" . || echo ""'])
														
 
															+    more_changes = subprocess.check_output([
														
 
															+        'bash', '-c', 'grep -RP "fw_depends(.?)+ ' +
														
 
															+        re.escape(fw_depends_changes[i]) + '( |$)" . || echo ""'
														
 
															+    ])
														
 
															     print("more_changes: {!s}".format(more_changes))
														
 
															     if fw_found_in_changes(more_changes):
														
 
															         print("Found changes that affect this framework. Running test.")
														
 
															         quit_diffing(True)
														
 
															     # Preserves the order of the list, so we can continue with this loop
														
 
															-    fw_depends_changes.extend(Set(get_fw_depends_changes(more_changes)) - Set(fw_depends_changes))
														
 
															+    fw_depends_changes.extend(
														
 
															+        Set(get_fw_depends_changes(more_changes)) - Set(fw_depends_changes))
														
 
															     i += 1
														
 
															 # If we get here, there was nothing found
														
--- a/toolset/travis/travis_setup.sh
+++ b/toolset/travis/travis_setup.sh
@@ -18,10 +18,6 @@ sed -i s/techempower/travis/g ./benchmark.cfg
 
															 echo "travis ALL=(ALL:ALL) NOPASSWD: ALL" | sudo tee -a /etc/sudoers
														
 
															-echo 127.0.0.1 TFB-database | sudo tee --append /etc/hosts
														
 
															-echo 127.0.0.1 TFB-client   | sudo tee --append /etc/hosts
														
 
															-echo 127.0.0.1 TFB-server   | sudo tee --append /etc/hosts
														
 
															-
														
 
															 source ./toolset/setup/linux/prerequisites.sh
														
 
															 tfb --init --quiet
														
--- a/toolset/utils/__init__.py
+++ b/toolset/utils/__init__.py
--- a/toolset/utils/benchmark_config.py
+++ b/toolset/utils/benchmark_config.py
@@ -0,0 +1,86 @@
 
															+from toolset.utils import setup_util
														
 
															+from toolset.benchmark.test_types import *
														
 
															+from toolset.utils.output_helper import QuietOutputStream
														
 
															+
														
 
															+import logging
														
 
															+import time
														
 
															+
														
 
															+
														
 
															+class BenchmarkConfig:
														
 
															+    def __init__(self, args):
														
 
															+        '''
														
 
															+        Configures this BenchmarkConfig given the arguments provided.
														
 
															+        '''
														
 
															+
														
 
															+        # Map type strings to their objects
														
 
															+        types = dict()
														
 
															+        types['json'] = JsonTestType(self)
														
 
															+        types['db'] = DBTestType(self)
														
 
															+        types['query'] = QueryTestType(self)
														
 
															+        types['fortune'] = FortuneTestType(self)
														
 
															+        types['update'] = UpdateTestType(self)
														
 
															+        types['plaintext'] = PlaintextTestType(self)
														
 
															+        types['cached_query'] = CachedQueryTestType(self)
														
 
															+
														
 
															+        # Turn type into a map instead of a string
														
 
															+        if args['type'] == 'all':
														
 
															+            args['types'] = types
														
 
															+        else:
														
 
															+            args['types'] = {args['type']: types[args['type']]}
														
 
															+        del args['type']
														
 
															+
														
 
															+        args['max_concurrency'] = max(args['concurrency_levels'])
														
 
															+        if 'pipeline_concurrency_levels' not in args:
														
 
															+            args['pipeline_concurrency_levels'] = [256, 1024, 4096, 16384]
														
 
															+
														
 
															+        self.quiet = False
														
 
															+        self.client_user = ""
														
 
															+        self.client_host = ""
														
 
															+        self.client_identity_file = ""
														
 
															+        self.database_user = ""
														
 
															+        self.database_host = ""
														
 
															+        self.database_identity_file = ""
														
 
															+        self.parse = False
														
 
															+        self.new = False
														
 
															+        self.init = False
														
 
															+        self.build = False
														
 
															+        self.clean = False
														
 
															+        self.list_tests = False
														
 
															+        self.concurrency_levels = []
														
 
															+        self.pipeline_concurrency_levels = []
														
 
															+
														
 
															+        self.__dict__.update(args)
														
 
															+
														
 
															+        self.quiet_out = QuietOutputStream(self.quiet)
														
 
															+
														
 
															+        self.start_time = time.time()
														
 
															+
														
 
															+        # setup logging
														
 
															+        logging.basicConfig(stream=self.quiet_out, level=logging.INFO)
														
 
															+
														
 
															+        # setup some additional variables
														
 
															+        if self.database_user == None: self.database_user = self.client_user
														
 
															+        if self.database_host == None: self.database_host = self.client_host
														
 
															+        if self.database_identity_file == None:
														
 
															+            self.database_identity_file = self.client_identity_file
														
 
															+
														
 
															+        # Remember root directory
														
 
															+        self.fwroot = setup_util.get_fwroot()
														
 
															+
														
 
															+        # setup current_benchmark.txt location
														
 
															+        self.current_benchmark = "/tmp/current_benchmark.txt"
														
 
															+
														
 
															+        if hasattr(self, 'parse') and self.parse != None:
														
 
															+            self.timestamp = self.parse
														
 
															+        else:
														
 
															+            self.timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
														
 
															+
														
 
															+        # Setup the ssh command string
														
 
															+        self.database_ssh_string = "ssh -T -o StrictHostKeyChecking=no " + self.database_user + "@" + self.database_host
														
 
															+        self.client_ssh_string = "ssh -T -o StrictHostKeyChecking=no " + self.client_user + "@" + self.client_host
														
 
															+        if self.database_identity_file != None:
														
 
															+            self.database_ssh_string = self.database_ssh_string + " -i " + self.database_identity_file
														
 
															+        if self.client_identity_file != None:
														
 
															+            self.client_ssh_string = self.client_ssh_string + " -i " + self.client_identity_file
														
 
															+
														
 
															+        self.run_test_timeout_seconds = 7200
														
--- a/toolset/utils/cleaner.py
+++ b/toolset/utils/cleaner.py
@@ -0,0 +1,15 @@
 
															+import os
														
 
															+import shutil
														
 
															+
														
 
															+
														
 
															+def clean(results):
														
 
															+    '''
														
 
															+    Cleans the given directory of all files and folders
														
 
															+    '''
														
 
															+    results_dir = os.path.dirname(results.directory)
														
 
															+    if os.path.exists(results_dir):
														
 
															+        for file in os.listdir(results_dir):
														
 
															+            if not os.path.exists(os.path.dirname(file)):
														
 
															+                shutil.rmtree(os.path.join(results_dir, file))
														
 
															+            else:
														
 
															+                os.remove(os.path.join(results_dir, file))
														
--- a/toolset/utils/docker_helper.py
+++ b/toolset/utils/docker_helper.py
@@ -0,0 +1,201 @@
 
															+import os
														
 
															+import socket
														
 
															+import fnmatch
														
 
															+import subprocess
														
 
															+import multiprocessing
														
 
															+import json
														
 
															+import docker
														
 
															+
														
 
															+from threading import Thread
														
 
															+
														
 
															+from toolset.utils import setup_util
														
 
															+from toolset.utils.output_helper import tee_output
														
 
															+from toolset.utils.metadata_helper import gather_tests
														
 
															+
														
 
															+
														
 
															+def clean():
														
 
															+    '''
														
 
															+    Cleans all the docker images from the system
														
 
															+    '''
														
 
															+    subprocess.check_call(["docker", "image", "prune", "-f"])
														
 
															+
														
 
															+    docker_ids = subprocess.check_output(["docker", "images",
														
 
															+                                          "-q"]).splitlines()
														
 
															+    for docker_id in docker_ids:
														
 
															+        subprocess.check_call(["docker", "image", "rmi", "-f", docker_id])
														
 
															+
														
 
															+    subprocess.check_call(["docker", "system", "prune", "-a", "-f"])
														
 
															+
														
 
															+
														
 
															+def build(benchmarker_config, test_names, out):
														
 
															+    '''
														
 
															+    Builds the dependency chain as well as the test implementation docker images
														
 
															+    for the given tests.
														
 
															+    '''
														
 
															+    tests = gather_tests(test_names)
														
 
															+
														
 
															+    for test in tests:
														
 
															+        docker_buildargs = {
														
 
															+            'CPU_COUNT': str(multiprocessing.cpu_count()),
														
 
															+            'MAX_CONCURRENCY': str(max(benchmarker_config.concurrency_levels)),
														
 
															+            'TFB_DATABASE': str(benchmarker_config.database_host)
														
 
															+        }
														
 
															+
														
 
															+        test_docker_files = ["%s.dockerfile" % test.name]
														
 
															+        if test.docker_files is not None:
														
 
															+            if type(test.docker_files) is list:
														
 
															+                test_docker_files.extend(test.docker_files)
														
 
															+            else:
														
 
															+                raise Exception(
														
 
															+                    "docker_files in benchmark_config.json must be an array")
														
 
															+
														
 
															+        for test_docker_file in test_docker_files:
														
 
															+            deps = list(
														
 
															+                reversed(
														
 
															+                    gather_dependencies(
														
 
															+                        os.path.join(test.directory, test_docker_file))))
														
 
															+
														
 
															+            docker_dir = os.path.join(setup_util.get_fwroot(), "toolset",
														
 
															+                                      "setup", "docker")
														
 
															+            for dependency in deps:
														
 
															+                docker_file = os.path.join(test.directory,
														
 
															+                                           dependency + ".dockerfile")
														
 
															+                if not docker_file or not os.path.exists(docker_file):
														
 
															+                    docker_file = find(docker_dir, dependency + ".dockerfile")
														
 
															+                if not docker_file:
														
 
															+                    tee_output(
														
 
															+                        out,
														
 
															+                        "Docker build failed; %s could not be found; terminating\n"
														
 
															+                        % (dependency + ".dockerfile"))
														
 
															+                    return 1
														
 
															+
														
 
															+                # Build the dependency image
														
 
															+                try:
														
 
															+                    for line in docker.APIClient(
														
 
															+                            base_url='unix://var/run/docker.sock').build(
														
 
															+                                path=os.path.dirname(docker_file),
														
 
															+                                dockerfile="%s.dockerfile" % dependency,
														
 
															+                                tag="tfb/%s" % dependency,
														
 
															+                                buildargs=docker_buildargs,
														
 
															+                                forcerm=True):
														
 
															+                        prev_line = os.linesep
														
 
															+                        if line.startswith('{"stream":'):
														
 
															+                            line = json.loads(line)
														
 
															+                            line = line[line.keys()[0]].encode('utf-8')
														
 
															+                            if prev_line.endswith(os.linesep):
														
 
															+                                tee_output(out, line)
														
 
															+                            else:
														
 
															+                                tee_output(out, line)
														
 
															+                            prev_line = line
														
 
															+                except Exception as e:
														
 
															+                    tee_output(out,
														
 
															+                               "Docker dependency build failed; terminating\n")
														
 
															+                    print(e)
														
 
															+                    return 1
														
 
															+
														
 
															+        # Build the test images
														
 
															+        for test_docker_file in test_docker_files:
														
 
															+            try:
														
 
															+                for line in docker.APIClient(
														
 
															+                        base_url='unix://var/run/docker.sock').build(
														
 
															+                            path=test.directory,
														
 
															+                            dockerfile=test_docker_file,
														
 
															+                            tag="tfb/test/%s" % test_docker_file.replace(
														
 
															+                                ".dockerfile", ""),
														
 
															+                            buildargs=docker_buildargs,
														
 
															+                            forcerm=True):
														
 
															+                    prev_line = os.linesep
														
 
															+                    if line.startswith('{"stream":'):
														
 
															+                        line = json.loads(line)
														
 
															+                        line = line[line.keys()[0]].encode('utf-8')
														
 
															+                        if prev_line.endswith(os.linesep):
														
 
															+                            tee_output(out, line)
														
 
															+                        else:
														
 
															+                            tee_output(out, line)
														
 
															+                        prev_line = line
														
 
															+            except Exception as e:
														
 
															+                tee_output(out, "Docker build failed; terminating\n")
														
 
															+                print(e)
														
 
															+                return 1
														
 
															+
														
 
															+
														
 
															+def run(benchmarker_config, docker_files, out):
														
 
															+    '''
														
 
															+    Run the given Docker container(s)
														
 
															+    '''
														
 
															+    client = docker.from_env()
														
 
															+
														
 
															+    for docker_file in docker_files:
														
 
															+        try:
														
 
															+
														
 
															+            def watch_container(container):
														
 
															+                for line in container.logs(stream=True):
														
 
															+                    tee_output(out, line)
														
 
															+
														
 
															+            extra_hosts = {
														
 
															+                socket.gethostname(): str(benchmarker_config.server_host),
														
 
															+                'TFB-SERVER': str(benchmarker_config.server_host),
														
 
															+                'TFB-DATABASE': str(benchmarker_config.database_host),
														
 
															+                'TFB-CLIENT': str(benchmarker_config.client_host)
														
 
															+            }
														
 
															+
														
 
															+            container = client.containers.run(
														
 
															+                "tfb/test/%s" % docker_file.replace(".dockerfile", ""),
														
 
															+                network_mode="host",
														
 
															+                privileged=True,
														
 
															+                stderr=True,
														
 
															+                detach=True,
														
 
															+                extra_hosts=extra_hosts)
														
 
															+
														
 
															+            watch_thread = Thread(target=watch_container, args=(container, ))
														
 
															+            watch_thread.daemon = True
														
 
															+            watch_thread.start()
														
 
															+
														
 
															+        except Exception as e:
														
 
															+            tee_output(out,
														
 
															+                       "Running docker cointainer: %s failed" % docker_file)
														
 
															+            print(e)
														
 
															+            return 1
														
 
															+
														
 
															+
														
 
															+def find(path, pattern):
														
 
															+    '''
														
 
															+    Finds and returns all the the files matching the given pattern recursively in
														
 
															+    the given path. 
														
 
															+    '''
														
 
															+    for root, dirs, files in os.walk(path):
														
 
															+        for name in files:
														
 
															+            if fnmatch.fnmatch(name, pattern):
														
 
															+                return os.path.join(root, name)
														
 
															+
														
 
															+
														
 
															+def gather_dependencies(docker_file):
														
 
															+    '''
														
 
															+    Gathers all the known docker dependencies for the given docker image.
														
 
															+    '''
														
 
															+    # Avoid setting up a circular import
														
 
															+    from toolset.utils import setup_util
														
 
															+    deps = []
														
 
															+
														
 
															+    docker_dir = os.path.join(setup_util.get_fwroot(), "toolset", "setup",
														
 
															+                              "docker")
														
 
															+
														
 
															+    if os.path.exists(docker_file):
														
 
															+        with open(docker_file) as fp:
														
 
															+            for line in fp.readlines():
														
 
															+                tokens = line.strip().split(' ')
														
 
															+                if tokens[0] == "FROM":
														
 
															+                    # This is magic that our base image points to
														
 
															+                    if tokens[1] != "ubuntu:16.04":
														
 
															+                        depToken = tokens[1].strip().split(':')[
														
 
															+                            0].strip().split('/')[1]
														
 
															+                        deps.append(depToken)
														
 
															+                        dep_docker_file = os.path.join(
														
 
															+                            os.path.dirname(docker_file),
														
 
															+                            depToken + ".dockerfile")
														
 
															+                        if not os.path.exists(dep_docker_file):
														
 
															+                            dep_docker_file = find(docker_dir,
														
 
															+                                                   depToken + ".dockerfile")
														
 
															+                        deps.extend(gather_dependencies(dep_docker_file))
														
 
															+
														
 
															+    return deps
														
--- a/toolset/utils/initializer.py
+++ b/toolset/utils/initializer.py
@@ -0,0 +1,129 @@
 
															+import subprocess, os
														
 
															+from toolset.utils import setup_util
														
 
															+
														
 
															+DEVNULL = open(os.devnull, 'w')
														
 
															+
														
 
															+
														
 
															+def initialize(args):
														
 
															+    fwroot = setup_util.get_fwroot()
														
 
															+    dbuser = args.database_user
														
 
															+    dbhost = args.database_host
														
 
															+    dbiden = args.database_identity_file
														
 
															+    cluser = args.client_user
														
 
															+    clhost = args.client_host
														
 
															+    cliden = args.client_identity_file
														
 
															+    aphost = args.server_host
														
 
															+
														
 
															+    # test ssh connections to all the machines
														
 
															+    client_conn = __check_connection(cluser, clhost, cliden, aphost)
														
 
															+    database_conn = __check_connection(dbuser, dbhost, dbiden, aphost)
														
 
															+
														
 
															+    conn_success = client_conn and database_conn
														
 
															+    if not conn_success and not args.quiet:
														
 
															+        return __print_failure()
														
 
															+
														
 
															+    # set up client machine
														
 
															+    if not __init_client(fwroot, cluser, clhost, cliden,
														
 
															+                         args.quiet) and not args.quiet:
														
 
															+        return __print_failure()
														
 
															+
														
 
															+    # set up database software
														
 
															+    if not __init_database(fwroot, dbuser, dbhost, dbiden,
														
 
															+                           args.quiet) and not args.quiet:
														
 
															+        return __print_failure()
														
 
															+
														
 
															+
														
 
															+def __print_failure():
														
 
															+    print("""
														
 
															+-------------------------------------------------------------------------------
														
 
															+  This wizard is intended to help configure the required software on all the
														
 
															+  machines in the ecosystem specified in benchmark.cfg.
														
 
															+
														
 
															+  Note: It is expected that you have already set up passwordless-sudo on all
														
 
															+  of the machines (app, database, client) as well as identity file based 
														
 
															+  authentication and hostname setup in your hosts file. 
														
 
															+  More information on this required setup can be found at:
														
 
															+
														
 
															+  frameworkbenchmarks.readthedocs.io/en/latest/Development/Installation-Guide/
														
 
															+
														
 
															+  Please ensure that your benchmark.cfg is correctly configured as well as all
														
 
															+  of the machines (app, database, client).
														
 
															+-------------------------------------------------------------------------------"""
														
 
															+          )
														
 
															+
														
 
															+
														
 
															+def __ssh_string(user, host, identity_file):
														
 
															+    return [
														
 
															+        "ssh", "-T", "-o", "StrictHostKeyChecking=no",
														
 
															+        "%s@%s" % (user, host), "-i", identity_file
														
 
															+    ]
														
 
															+
														
 
															+
														
 
															+def __check_connection(user, host, identity_file, app_host):
														
 
															+    ''' 
														
 
															+  Checks that the given user and host are accessible via ssh with the given
														
 
															+  identity file and have the the following permissions:
														
 
															+    1. passwordless sudo
														
 
															+    2. ability to ssh back to app machine
														
 
															+  '''
														
 
															+    client_conn = True
														
 
															+    try:
														
 
															+        p = subprocess.Popen(
														
 
															+            __ssh_string(user, host, identity_file),
														
 
															+            stdin=subprocess.PIPE,
														
 
															+            stdout=DEVNULL,
														
 
															+            stderr=DEVNULL)
														
 
															+        p.communicate("ssh -T -o StrictHostKeyChecking=no %s" % app_host)
														
 
															+        if p.returncode:
														
 
															+            client_conn = False
														
 
															+    except Exception:
														
 
															+        client_conn = False
														
 
															+    return client_conn
														
 
															+
														
 
															+
														
 
															+def __init_client(fwroot, user, host, identity_file, quiet):
														
 
															+    '''
														
 
															+  Initializes and configures the software required to run the suite on the 
														
 
															+  client machine.
														
 
															+  '''
														
 
															+    if not quiet:
														
 
															+        print("INSTALL: Installing client software")
														
 
															+    with open(
														
 
															+            os.path.join(fwroot, "toolset", "setup", "linux", "client.sh"),
														
 
															+            "r") as myfile:
														
 
															+        remote_script = myfile.read()
														
 
															+        if quiet:
														
 
															+            p = subprocess.Popen(
														
 
															+                __ssh_string(user, host, identity_file),
														
 
															+                stdin=subprocess.PIPE,
														
 
															+                stdout=DEVNULL,
														
 
															+                stderr=DEVNULL)
														
 
															+        else:
														
 
															+            p = subprocess.Popen(
														
 
															+                __ssh_string(user, host, identity_file), stdin=subprocess.PIPE)
														
 
															+        p.communicate(remote_script)
														
 
															+        return p.returncode == 0
														
 
															+
														
 
															+
														
 
															+def __init_database(fwroot, user, host, identity_file, quiet):
														
 
															+    '''
														
 
															+  Initializes and configures the software required to run the suite on the
														
 
															+  database machine.
														
 
															+  '''
														
 
															+    if not quiet:
														
 
															+        print("INSTALL: Installing database software")
														
 
															+    with open(
														
 
															+            os.path.join(fwroot, "toolset", "setup", "linux", "database.sh"),
														
 
															+            "r") as myfile:
														
 
															+        remote_script = myfile.read()
														
 
															+        if quiet:
														
 
															+            p = subprocess.Popen(
														
 
															+                __ssh_string(user, host, identity_file),
														
 
															+                stdin=subprocess.PIPE,
														
 
															+                stdout=DEVNULL,
														
 
															+                stderr=DEVNULL)
														
 
															+        else:
														
 
															+            p = subprocess.Popen(
														
 
															+                __ssh_string(user, host, identity_file), stdin=subprocess.PIPE)
														
 
															+        p.communicate(remote_script)
														
 
															+        return p.returncode == 0
														
--- a/toolset/utils/metadata_helper.py
+++ b/toolset/utils/metadata_helper.py
@@ -0,0 +1,418 @@
 
															+import ConfigParser
														
 
															+import os
														
 
															+import glob
														
 
															+import json
														
 
															+import logging
														
 
															+
														
 
															+from ast import literal_eval
														
 
															+from collections import OrderedDict
														
 
															+
														
 
															+
														
 
															+def gather_langauges():
														
 
															+    '''
														
 
															+    Gathers all the known languages in the suite via the folder names
														
 
															+    beneath FWROOT.
														
 
															+    '''
														
 
															+    # Avoid setting up a circular import
														
 
															+    from toolset.utils import setup_util
														
 
															+
														
 
															+    lang_dir = os.path.join(setup_util.get_fwroot(), "frameworks")
														
 
															+    langs = []
														
 
															+    for dir in glob.glob(os.path.join(lang_dir, "*")):
														
 
															+        langs.append(dir.replace(lang_dir, "")[1:])
														
 
															+    return langs
														
 
															+
														
 
															+
														
 
															+def gather_tests(include=[], exclude=[], benchmarker_config=None,
														
 
															+                 results=None):
														
 
															+    '''
														
 
															+    Given test names as strings, returns a list of FrameworkTest objects.
														
 
															+    For example, 'aspnet-mysql-raw' turns into a FrameworkTest object with
														
 
															+    variables for checking the test directory, the test database os, and
														
 
															+    other useful items.
														
 
															+
														
 
															+    With no arguments, every test in this framework will be returned.
														
 
															+    With include, only tests with this exact name will be returned.
														
 
															+    With exclude, all tests but those excluded will be returned.
														
 
															+
														
 
															+    A config is needed to construct full FrameworkTest objects. If
														
 
															+    one is not provided, a default config will be created.
														
 
															+    '''
														
 
															+    # Avoid setting up a circular import
														
 
															+    from toolset.utils.benchmark_config import BenchmarkConfig
														
 
															+    from toolset.utils import setup_util
														
 
															+
														
 
															+    # Help callers out a bit
														
 
															+    if include is None:
														
 
															+        include = []
														
 
															+    if exclude is None:
														
 
															+        exclude = []
														
 
															+
														
 
															+    # Old, hacky method to exclude all tests was to
														
 
															+    # request a test known to not exist, such as ''.
														
 
															+    # If test '' was requested, short-circuit and return
														
 
															+    # nothing immediately
														
 
															+    if len(include) == 1 and '' in include:
														
 
															+        return []
														
 
															+
														
 
															+    # Setup default BenchmarkerConfig using example configuration
														
 
															+    if benchmarker_config is None:
														
 
															+        default_config = setup_util.get_fwroot() + "/benchmark.cfg"
														
 
															+        config = ConfigParser.SafeConfigParser()
														
 
															+        config.readfp(open(default_config))
														
 
															+        defaults = dict(config.items("Defaults"))
														
 
															+
														
 
															+        # Convert strings into proper python types
														
 
															+        for k, v in defaults.items():
														
 
															+            try:
														
 
															+                defaults[k] = literal_eval(v)
														
 
															+            except Exception:
														
 
															+                pass
														
 
															+
														
 
															+        defaults[
														
 
															+            'results_name'] = "(unspecified, datetime = %Y-%m-%d %H:%M:%S)"
														
 
															+        defaults['results_environment'] = "My Server Environment"
														
 
															+        defaults['test_dir'] = None
														
 
															+        defaults['test_lang'] = None
														
 
															+        defaults['quiet'] = True
														
 
															+
														
 
															+        benchmarker_config = BenchmarkConfig(defaults)
														
 
															+
														
 
															+    # Search for configuration files
														
 
															+    config_files = []
														
 
															+
														
 
															+    if benchmarker_config.test_lang:
														
 
															+        benchmarker_config.test_dir = []
														
 
															+        for lang in benchmarker_config.test_lang:
														
 
															+            if os.path.exists("{!s}/frameworks/{!s}".format(
														
 
															+                    benchmarker_config.fwroot, lang)):
														
 
															+                for test_dir in os.listdir("{!s}/frameworks/{!s}".format(
														
 
															+                        benchmarker_config.fwroot, lang)):
														
 
															+                    benchmarker_config.test_dir.append("{!s}/{!s}".format(
														
 
															+                        lang, test_dir))
														
 
															+            else:
														
 
															+                raise Exception(
														
 
															+                    "Unable to locate language directory: {!s}".format(lang))
														
 
															+
														
 
															+    if benchmarker_config.test_dir:
														
 
															+        for test_dir in benchmarker_config.test_dir:
														
 
															+            dir_config_files = glob.glob(
														
 
															+                "{!s}/frameworks/{!s}/benchmark_config.json".format(
														
 
															+                    benchmarker_config.fwroot, test_dir))
														
 
															+            if len(dir_config_files):
														
 
															+                config_files.extend(dir_config_files)
														
 
															+            else:
														
 
															+                raise Exception(
														
 
															+                    "Unable to locate tests in test-dir: {!s}".format(
														
 
															+                        test_dir))
														
 
															+    else:
														
 
															+        config_files.extend(
														
 
															+            glob.glob("{!s}/frameworks/*/*/benchmark_config.json".format(
														
 
															+                benchmarker_config.fwroot)))
														
 
															+
														
 
															+    tests = []
														
 
															+    for config_file_name in config_files:
														
 
															+        config = None
														
 
															+        with open(config_file_name, 'r') as config_file:
														
 
															+            try:
														
 
															+                config = json.load(config_file)
														
 
															+            except ValueError:
														
 
															+                raise Exception(
														
 
															+                    "Error loading '{!s}'.".format(config_file_name))
														
 
															+
														
 
															+        # Find all tests in the config file
														
 
															+        config_tests = parse_config(config, os.path.dirname(config_file_name),
														
 
															+                                    benchmarker_config, results)
														
 
															+
														
 
															+        # Filter
														
 
															+        for test in config_tests:
														
 
															+            if len(include) is 0 and len(exclude) is 0:
														
 
															+                # No filters, we are running everything
														
 
															+                tests.append(test)
														
 
															+            elif test.name in exclude:
														
 
															+                continue
														
 
															+            elif test.name in include:
														
 
															+                tests.append(test)
														
 
															+            else:
														
 
															+                # An include list exists, but this test is
														
 
															+                # not listed there, so we ignore it
														
 
															+                pass
														
 
															+
														
 
															+    # Ensure we were able to locate everything that was
														
 
															+    # explicitly included
														
 
															+    if 0 != len(include):
														
 
															+        names = {test.name for test in tests}
														
 
															+        if 0 != len(set(include) - set(names)):
														
 
															+            missing = list(set(include) - set(names))
														
 
															+            raise Exception("Unable to locate tests %s" % missing)
														
 
															+
														
 
															+    tests.sort(key=lambda x: x.name)
														
 
															+    return tests
														
 
															+
														
 
															+
														
 
															+def gather_remaining_tests(config, results):
														
 
															+    '''
														
 
															+    Gathers the tests remaining in a current benchmark run.
														
 
															+    '''
														
 
															+    tests = gather_tests(config.test, config.exclude, config, results)
														
 
															+
														
 
															+    # If the tests have been interrupted somehow, then we want to resume them where we left
														
 
															+    # off, rather than starting from the beginning
														
 
															+    if os.path.isfile(config.current_benchmark):
														
 
															+        with open(config.current_benchmark, 'r') as interrupted_benchmark:
														
 
															+            interrupt_bench = interrupted_benchmark.read().strip()
														
 
															+        for index, atest in enumerate(tests):
														
 
															+            if atest.name == interrupt_bench:
														
 
															+                tests = tests[index:]
														
 
															+                break
														
 
															+    return tests
														
 
															+
														
 
															+
														
 
															+def gather_frameworks(include=[], exclude=[], config=None):
														
 
															+    '''
														
 
															+    Return a dictionary mapping frameworks->[test1,test2,test3]
														
 
															+    for quickly grabbing all tests in a grouped manner.
														
 
															+    Args have the same meaning as gather_tests
														
 
															+    '''
														
 
															+    tests = gather_tests(include, exclude, config)
														
 
															+    frameworks = dict()
														
 
															+
														
 
															+    for test in tests:
														
 
															+        if test.framework not in frameworks:
														
 
															+            frameworks[test.framework] = []
														
 
															+        frameworks[test.framework].append(test)
														
 
															+    return frameworks
														
 
															+
														
 
															+
														
 
															+def test_order(type_name):
														
 
															+    """
														
 
															+    This sort ordering is set up specifically to return the length
														
 
															+    of the test name. There were SO many problems involved with
														
 
															+    'plaintext' being run first (rather, just not last) that we
														
 
															+    needed to ensure that it was run last for every framework.
														
 
															+    """
														
 
															+    return len(type_name)
														
 
															+
														
 
															+
														
 
															+def parse_config(config, directory, benchmarker_config, results):
														
 
															+    """
														
 
															+    Parses a config file into a list of FrameworkTest objects
														
 
															+    """
														
 
															+    from toolset.benchmark.framework_test import FrameworkTest
														
 
															+    tests = []
														
 
															+
														
 
															+    # The config object can specify multiple tests
														
 
															+    # Loop over them and parse each into a FrameworkTest
														
 
															+    for test in config['tests']:
														
 
															+
														
 
															+        tests_to_run = [name for (name, keys) in test.iteritems()]
														
 
															+        if "default" not in tests_to_run:
														
 
															+            logging.warn(
														
 
															+                "Framework %s does not define a default test in benchmark_config.json",
														
 
															+                config['framework'])
														
 
															+
														
 
															+        # Check that each test configuration is acceptable
														
 
															+        # Throw exceptions if a field is missing, or how to improve the field
														
 
															+        for test_name, test_keys in test.iteritems():
														
 
															+            # Validates the benchmark_config entry
														
 
															+            validate_test(test_name, test_keys, directory)
														
 
															+
														
 
															+            # Map test type to a parsed FrameworkTestType object
														
 
															+            runTests = dict()
														
 
															+            for type_name, type_obj in benchmarker_config.types.iteritems():
														
 
															+                try:
														
 
															+                    # Makes a FrameWorkTestType object using some of the keys in config
														
 
															+                    # e.g. JsonTestType uses "json_url"
														
 
															+                    runTests[type_name] = type_obj.copy().parse(test_keys)
														
 
															+                except AttributeError:
														
 
															+                    # This is quite common - most tests don't support all types
														
 
															+                    # Quitely log it and move on (debug logging is on in travis and this causes
														
 
															+                    # ~1500 lines of debug, so I'm totally ignoring it for now
														
 
															+                    # logging.debug("Missing arguments for test type %s for framework test %s", type_name, test_name)
														
 
															+                    pass
														
 
															+
														
 
															+            # We need to sort by test_type to run
														
 
															+            sortedTestKeys = sorted(runTests.keys(), key=test_order)
														
 
															+            sortedRunTests = OrderedDict()
														
 
															+            for sortedTestKey in sortedTestKeys:
														
 
															+                sortedRunTests[sortedTestKey] = runTests[sortedTestKey]
														
 
															+
														
 
															+            # Prefix all test names with framework except 'default' test
														
 
															+            # Done at the end so we may still refer to the primary test as `default` in benchmark config error messages
														
 
															+            if test_name == 'default':
														
 
															+                test_name = config['framework']
														
 
															+            else:
														
 
															+                test_name = "%s-%s" % (config['framework'], test_name)
														
 
															+
														
 
															+            # By passing the entire set of keys, each FrameworkTest will have a member for each key
														
 
															+            tests.append(
														
 
															+                FrameworkTest(test_name, directory, benchmarker_config,
														
 
															+                              results, sortedRunTests, test_keys))
														
 
															+
														
 
															+    return tests
														
 
															+
														
 
															+
														
 
															+def validate_test(test_name, test_keys, directory):
														
 
															+    """
														
 
															+    Validate benchmark config values for this test based on a schema
														
 
															+    """
														
 
															+    recommended_lang = directory.split('/')[-2]
														
 
															+    windows_url = "https://github.com/TechEmpower/FrameworkBenchmarks/issues/1038"
														
 
															+    schema = {
														
 
															+        'language': {
														
 
															+            'help':
														
 
															+            ('language', 'The language of the framework used, suggestion: %s' %
														
 
															+             recommended_lang)
														
 
															+        },
														
 
															+        'webserver': {
														
 
															+            'help':
														
 
															+            ('webserver',
														
 
															+             'Name of the webserver also referred to as the "front-end server"'
														
 
															+             )
														
 
															+        },
														
 
															+        'classification': {
														
 
															+            'allowed': [('Fullstack', '...'), ('Micro', '...'), ('Platform',
														
 
															+                                                                 '...')]
														
 
															+        },
														
 
															+        'database': {
														
 
															+            'allowed':
														
 
															+            [('MySQL',
														
 
															+              'One of the most popular databases around the web and in TFB'),
														
 
															+             ('Postgres',
														
 
															+              'An advanced SQL database with a larger feature set than MySQL'),
														
 
															+             ('MongoDB', 'A popular document-store database'),
														
 
															+             ('Cassandra', 'A highly performant and scalable NoSQL database'),
														
 
															+             ('Elasticsearch',
														
 
															+              'A distributed RESTful search engine that is used as a database for TFB tests'
														
 
															+              ),
														
 
															+             ('Redis',
														
 
															+              'An open-sourced, BSD licensed, advanced key-value cache and store'
														
 
															+              ),
														
 
															+             ('SQLite',
														
 
															+              'A network-less database, still supported for backwards compatibility'
														
 
															+              ), ('SQLServer', 'Microsoft\'s SQL implementation'),
														
 
															+             ('None',
														
 
															+              'No database was used for these tests, as is the case with Json Serialization and Plaintext'
														
 
															+              )]
														
 
															+        },
														
 
															+        'approach': {
														
 
															+            'allowed': [('Realistic', '...'), ('Stripped', '...')]
														
 
															+        },
														
 
															+        'orm': {
														
 
															+            'allowed':
														
 
															+            [('Full',
														
 
															+              'Has a full suite of features like lazy loading, caching, multiple language support, sometimes pre-configured with scripts.'
														
 
															+              ),
														
 
															+             ('Micro',
														
 
															+              'Has basic database driver capabilities such as establishing a connection and sending queries.'
														
 
															+              ),
														
 
															+             ('Raw',
														
 
															+              'Tests that do not use an ORM will be classified as "raw" meaning they use the platform\'s raw database connectivity.'
														
 
															+              )]
														
 
															+        },
														
 
															+        'platform': {
														
 
															+            'help':
														
 
															+            ('platform',
														
 
															+             'Name of the platform this framework runs on, e.g. Node.js, PyPy, hhvm, JRuby ...'
														
 
															+             )
														
 
															+        },
														
 
															+        'framework': {
														
 
															+            # Guranteed to be here and correct at this point
														
 
															+            # key is left here to produce the set of required keys
														
 
															+        },
														
 
															+        'os': {
														
 
															+            'allowed':
														
 
															+            [('Linux',
														
 
															+              'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'
														
 
															+              ),
														
 
															+             ('Windows',
														
 
															+              'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s'
														
 
															+              % windows_url)]
														
 
															+        },
														
 
															+        'database_os': {
														
 
															+            'allowed':
														
 
															+            [('Linux',
														
 
															+              'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'
														
 
															+              ),
														
 
															+             ('Windows',
														
 
															+              'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s'
														
 
															+              % windows_url)]
														
 
															+        }
														
 
															+    }
														
 
															+
														
 
															+    # Confirm required keys are present
														
 
															+    required_keys = schema.keys()
														
 
															+    missing = list(set(required_keys) - set(test_keys))
														
 
															+
														
 
															+    if len(missing) > 0:
														
 
															+        missingstr = (", ").join(map(str, missing))
														
 
															+        raise Exception(
														
 
															+            "benchmark_config.json for test %s is invalid, please amend by adding the following required keys: [%s]"
														
 
															+            % (test_name, missingstr))
														
 
															+
														
 
															+    # Check the (all optional) test urls
														
 
															+    validate_urls(test_name, test_keys)
														
 
															+
														
 
															+    # Check values of keys against schema
														
 
															+    for key in required_keys:
														
 
															+        val = test_keys.get(key, "").lower()
														
 
															+        has_predefined_acceptables = 'allowed' in schema[key]
														
 
															+
														
 
															+        if has_predefined_acceptables:
														
 
															+            allowed = schema[key].get('allowed', [])
														
 
															+            acceptable_values, descriptors = zip(*allowed)
														
 
															+            acceptable_values = [a.lower() for a in acceptable_values]
														
 
															+
														
 
															+            if val not in acceptable_values:
														
 
															+                msg = (
														
 
															+                    "Invalid `%s` value specified for test \"%s\" in framework \"%s\"; suggestions:\n"
														
 
															+                    % (key, test_name, test_keys['framework']))
														
 
															+                helpinfo = ('\n').join([
														
 
															+                    "  `%s` -- %s" % (v, desc)
														
 
															+                    for (v, desc) in zip(acceptable_values, descriptors)
														
 
															+                ])
														
 
															+                fullerr = msg + helpinfo + "\n"
														
 
															+                raise Exception(fullerr)
														
 
															+
														
 
															+        elif not has_predefined_acceptables and val == "":
														
 
															+            msg = (
														
 
															+                "Value for `%s` in test \"%s\" in framework \"%s\" was missing:\n"
														
 
															+                % (key, test_name, test_keys['framework']))
														
 
															+            helpinfo = "  %s -- %s" % schema[key]['help']
														
 
															+            fullerr = msg + helpinfo + '\n'
														
 
															+            raise Exception(fullerr)
														
 
															+
														
 
															+
														
 
															+def validate_urls(test_name, test_keys):
														
 
															+    """
														
 
															+    Separated from validate_test because urls are not required anywhere. We know a url is incorrect if it is
														
 
															+    empty or does not start with a "/" character. There is no validation done to ensure the url conforms to
														
 
															+    the suggested url specifications, although those suggestions are presented if a url fails validation here.
														
 
															+    """
														
 
															+    example_urls = {
														
 
															+        "json_url":
														
 
															+        "/json",
														
 
															+        "db_url":
														
 
															+        "/mysql/db",
														
 
															+        "query_url":
														
 
															+        "/mysql/queries?queries=  or  /mysql/queries/",
														
 
															+        "fortune_url":
														
 
															+        "/mysql/fortunes",
														
 
															+        "update_url":
														
 
															+        "/mysql/updates?queries=  or  /mysql/updates/",
														
 
															+        "plaintext_url":
														
 
															+        "/plaintext",
														
 
															+        "cached_query_url":
														
 
															+        "/mysql/cached_queries?queries=  or /mysql/cached_queries"
														
 
															+    }
														
 
															+
														
 
															+    for test_url in [
														
 
															+            "json_url", "db_url", "query_url", "fortune_url", "update_url",
														
 
															+            "plaintext_url", "cached_query_url"
														
 
															+    ]:
														
 
															+        key_value = test_keys.get(test_url, None)
														
 
															+        if key_value != None and not key_value.startswith('/'):
														
 
															+            errmsg = """`%s` field in test \"%s\" does not appear to be a valid url: \"%s\"\n
														
 
															+        Example `%s` url: \"%s\"
														
 
															+      """ % (test_url, test_name, key_value, test_url, example_urls[test_url])
														
 
															+            raise Exception(errmsg)
														
--- a/toolset/utils/output_helper.py
+++ b/toolset/utils/output_helper.py
@@ -0,0 +1,65 @@
 
															+import os, sys
														
 
															+from contextlib import contextmanager
														
 
															+
														
 
															+
														
 
															+def header(message, top='-', bottom='-'):
														
 
															+    '''
														
 
															+    Generates a clean header
														
 
															+    '''
														
 
															+    topheader = (top * 80)[:80]
														
 
															+    bottomheader = (bottom * 80)[:80]
														
 
															+    result = ""
														
 
															+    if topheader != "":
														
 
															+        result += "%s" % topheader
														
 
															+    if message != "":
														
 
															+        if result == "":
														
 
															+            result = "  %s" % message
														
 
															+        else:
														
 
															+            result += "%s  %s" % (os.linesep, message)
														
 
															+    if bottomheader != "":
														
 
															+        if result == "":
														
 
															+            result = "%s" % bottomheader
														
 
															+        else:
														
 
															+            result += "%s%s" % (os.linesep, bottomheader)
														
 
															+    return result + os.linesep
														
 
															+
														
 
															+
														
 
															+def tee_output(out, line):
														
 
															+    '''
														
 
															+    Writes to bouth stdout and the provided out file
														
 
															+    '''
														
 
															+    sys.stdout.write(line)
														
 
															+    sys.stdout.flush()
														
 
															+
														
 
															+    if out is not None:
														
 
															+        out.write(line)
														
 
															+        out.flush()
														
 
															+
														
 
															+
														
 
															+class QuietOutputStream:
														
 
															+    def __init__(self, is_quiet):
														
 
															+        self.is_quiet = is_quiet
														
 
															+        self.null_out = open(os.devnull, 'w')
														
 
															+
														
 
															+    def fileno(self):
														
 
															+        with self.enable():
														
 
															+            return sys.stdout.fileno()
														
 
															+
														
 
															+    def write(self, message):
														
 
															+        with self.enable():
														
 
															+            sys.stdout.write(message)
														
 
															+
														
 
															+    @contextmanager
														
 
															+    def enable(self):
														
 
															+        if self.is_quiet:
														
 
															+            old_out = sys.stdout
														
 
															+            old_err = sys.stderr
														
 
															+            try:
														
 
															+                sys.stdout = self.null_out
														
 
															+                sys.stderr = self.null_out
														
 
															+                yield
														
 
															+            finally:
														
 
															+                sys.stdout = old_out
														
 
															+                sys.stderr = old_err
														
 
															+        else:
														
 
															+            yield
														
--- a/toolset/utils/remote_script_helper.py
+++ b/toolset/utils/remote_script_helper.py
@@ -0,0 +1,211 @@
 
															+def generate_concurrency_script(benchmarker_config,
														
 
															+                                name,
														
 
															+                                url,
														
 
															+                                port,
														
 
															+                                accept_header,
														
 
															+                                wrk_command="wrk"):
														
 
															+    '''
														
 
															+    Generates the string containing the bash script that will be run on the 
														
 
															+    client to benchmark a single test. This specifically works for the variable 
														
 
															+    concurrency tests.
														
 
															+    '''
														
 
															+    headers = headers_template.format(
														
 
															+        server_host=benchmarker_config.server_host, accept=accept_header)
														
 
															+    return concurrency_template.format(
														
 
															+        max_concurrency=max(benchmarker_config.concurrency_levels),
														
 
															+        name=name,
														
 
															+        duration=benchmarker_config.duration,
														
 
															+        levels=" ".join("{}".format(item)
														
 
															+                        for item in benchmarker_config.concurrency_levels),
														
 
															+        server_host=benchmarker_config.server_host,
														
 
															+        port=port,
														
 
															+        url=url,
														
 
															+        headers=headers,
														
 
															+        wrk=wrk_command)
														
 
															+
														
 
															+
														
 
															+def generate_pipeline_script(benchmarker_config,
														
 
															+                             name,
														
 
															+                             url,
														
 
															+                             port,
														
 
															+                             accept_header,
														
 
															+                             wrk_command="wrk"):
														
 
															+    '''
														
 
															+    Generates the string containing the bash script that will be run on the 
														
 
															+    client to benchmark a single pipeline test.
														
 
															+    '''
														
 
															+    headers = headers_template.format(
														
 
															+        server_host=benchmarker_config.server_host, accept=accept_header)
														
 
															+    return pipeline_template.format(
														
 
															+        max_concurrency=max(benchmarker_config.pipeline_concurrency_levels),
														
 
															+        name=name,
														
 
															+        duration=benchmarker_config.duration,
														
 
															+        levels=" ".join(
														
 
															+            "{}".format(item)
														
 
															+            for item in benchmarker_config.pipeline_concurrency_levels),
														
 
															+        server_host=benchmarker_config.server_host,
														
 
															+        port=port,
														
 
															+        url=url,
														
 
															+        headers=headers,
														
 
															+        wrk=wrk_command,
														
 
															+        pipeline=16)
														
 
															+
														
 
															+
														
 
															+def generate_query_script(benchmarker_config, name, url, port, accept_header,
														
 
															+                          query_levels):
														
 
															+    '''
														
 
															+    Generates the string containing the bash script that will be run on the 
														
 
															+    client to benchmark a single test. This specifically works for the variable 
														
 
															+    query tests (Query)
														
 
															+    '''
														
 
															+    headers = headers_template.format(
														
 
															+        server_host=benchmarker_config.server_host, accept=accept_header)
														
 
															+    return query_template.format(
														
 
															+        max_concurrency=max(benchmarker_config.concurrency_levels),
														
 
															+        name=name,
														
 
															+        duration=benchmarker_config.duration,
														
 
															+        levels=" ".join("{}".format(item) for item in query_levels),
														
 
															+        server_host=benchmarker_config.server_host,
														
 
															+        port=port,
														
 
															+        url=url,
														
 
															+        headers=headers)
														
 
															+
														
 
															+
														
 
															+##########################################################################################
														
 
															+# Constants
														
 
															+##########################################################################################
														
 
															+headers_template = "-H 'Host: {server_host}' -H 'Accept: {accept}' -H 'Connection: keep-alive'"
														
 
															+
														
 
															+# Used for test types that require no pipelining or query string params.
														
 
															+concurrency_template = """
														
 
															+    let max_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Running Primer {name}"
														
 
															+    echo " {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
														
 
															+    sleep 5
														
 
															+
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Running Warmup {name}"
														
 
															+    echo " {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}"
														
 
															+    sleep 5
														
 
															+
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Synchronizing time"
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    ntpdate -s pool.ntp.org
														
 
															+
														
 
															+    for c in {levels}
														
 
															+    do
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Concurrency: $c for {name}"
														
 
															+    echo " {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t $(($c>$max_threads?$max_threads:$c)) \"http://{server_host}:{port}{url}\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    STARTTIME=$(date +"%s")
														
 
															+    {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t "$(($c>$max_threads?$max_threads:$c))" http://{server_host}:{port}{url}
														
 
															+    echo "STARTTIME $STARTTIME"
														
 
															+    echo "ENDTIME $(date +"%s")"
														
 
															+    sleep 2
														
 
															+    done
														
 
															+"""
														
 
															+
														
 
															+# Used for test types that require pipelining.
														
 
															+pipeline_template = """
														
 
															+    let max_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Running Primer {name}"
														
 
															+    echo " {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    {wrk} {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}"
														
 
															+    sleep 5
														
 
															+
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Running Warmup {name}"
														
 
															+    echo " {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    {wrk} {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}"
														
 
															+    sleep 5
														
 
															+
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Synchronizing time"
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    ntpdate -s pool.ntp.org
														
 
															+
														
 
															+    for c in {levels}
														
 
															+    do
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Concurrency: $c for {name}"
														
 
															+    echo " {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t $(($c>$max_threads?$max_threads:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    STARTTIME=$(date +"%s")
														
 
															+    {wrk} {headers} --latency -d {duration} -c $c --timeout 8 -t "$(($c>$max_threads?$max_threads:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
														
 
															+    echo "STARTTIME $STARTTIME"
														
 
															+    echo "ENDTIME $(date +"%s")"
														
 
															+    sleep 2
														
 
															+    done
														
 
															+"""
														
 
															+
														
 
															+# Used for test types that require a database -
														
 
															+# These tests run at a static concurrency level and vary the size of
														
 
															+# the query sent with each request
														
 
															+query_template = """
														
 
															+    let max_threads=$(cat /proc/cpuinfo | grep processor | wc -l)
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Running Primer {name}"
														
 
															+    echo " wrk {headers} --latency -d 5 -c 8 --timeout 8 -t 8 \"http://{server_host}:{port}{url}2\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    wrk {headers} --latency -d 5 -c 8 --timeout 8 -t 8 "http://{server_host}:{port}{url}2"
														
 
															+    sleep 5
														
 
															+
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Running Warmup {name}"
														
 
															+    echo " wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}2\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}2"
														
 
															+    sleep 5
														
 
															+
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Synchronizing time"
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    ntpdate -s pool.ntp.org
														
 
															+
														
 
															+    for c in {levels}
														
 
															+    do
														
 
															+    echo ""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo " Queries: $c for {name}"
														
 
															+    echo " wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads \"http://{server_host}:{port}{url}$c\""
														
 
															+    echo "---------------------------------------------------------"
														
 
															+    echo ""
														
 
															+    STARTTIME=$(date +"%s")
														
 
															+    wrk {headers} --latency -d {duration} -c {max_concurrency} --timeout 8 -t $max_threads "http://{server_host}:{port}{url}$c"
														
 
															+    echo "STARTTIME $STARTTIME"
														
 
															+    echo "ENDTIME $(date +"%s")"
														
 
															+    sleep 2
														
 
															+    done
														
 
															+"""
														
--- a/toolset/utils/results_helper.py
+++ b/toolset/utils/results_helper.py
@@ -0,0 +1,596 @@
 
															+from toolset.utils.metadata_helper import gather_remaining_tests, gather_frameworks
														
 
															+from toolset.utils.output_helper import header
														
 
															+
														
 
															+import os
														
 
															+import logging
														
 
															+import subprocess
														
 
															+import uuid
														
 
															+import time
														
 
															+import json
														
 
															+import requests
														
 
															+import threading
														
 
															+import re
														
 
															+import math
														
 
															+import csv
														
 
															+from datetime import datetime
														
 
															+
														
 
															+# Cross-platform colored text
														
 
															+from colorama import Fore, Style
														
 
															+
														
 
															+
														
 
															+class Results:
														
 
															+    def __init__(self, config):
														
 
															+        '''
														
 
															+        Constructor
														
 
															+        '''
														
 
															+        self.config = config
														
 
															+        self.directory = os.path.join(self.config.fwroot, "results",
														
 
															+                                      self.config.timestamp)
														
 
															+        try:
														
 
															+            os.makedirs(self.directory)
														
 
															+        except OSError:
														
 
															+            pass
														
 
															+        self.file = os.path.join(self.directory, "results.json")
														
 
															+
														
 
															+        self.uuid = str(uuid.uuid4())
														
 
															+        self.name = datetime.now().strftime(self.config.results_name)
														
 
															+        self.environmentDescription = self.config.results_environment
														
 
															+        try:
														
 
															+            self.git = dict()
														
 
															+            self.git['commitId'] = self.__get_git_commit_id()
														
 
															+            self.git['repositoryUrl'] = self.__get_git_repository_url()
														
 
															+            self.git['branchName'] = self.__get_git_branch_name()
														
 
															+        except Exception as e:
														
 
															+            logging.debug(
														
 
															+                'Could not read local git repository, which is fine. The error was: %s',
														
 
															+                e)
														
 
															+            self.git = None
														
 
															+        self.startTime = int(round(time.time() * 1000))
														
 
															+        self.completionTime = None
														
 
															+        self.concurrencyLevels = self.config.concurrency_levels
														
 
															+        self.pipelineConcurrencyLevels = self.config.pipeline_concurrency_levels
														
 
															+        self.queryIntervals = self.config.query_levels
														
 
															+        self.cachedQueryIntervals = self.config.cached_query_levels
														
 
															+        self.frameworks = [
														
 
															+            t.name for t in gather_remaining_tests(self.config, self)
														
 
															+        ]
														
 
															+        self.duration = self.config.duration
														
 
															+        self.rawData = dict()
														
 
															+        self.rawData['json'] = dict()
														
 
															+        self.rawData['db'] = dict()
														
 
															+        self.rawData['query'] = dict()
														
 
															+        self.rawData['fortune'] = dict()
														
 
															+        self.rawData['update'] = dict()
														
 
															+        self.rawData['plaintext'] = dict()
														
 
															+        self.rawData['cached_query'] = dict()
														
 
															+        self.completed = dict()
														
 
															+        self.succeeded = dict()
														
 
															+        self.succeeded['json'] = []
														
 
															+        self.succeeded['db'] = []
														
 
															+        self.succeeded['query'] = []
														
 
															+        self.succeeded['fortune'] = []
														
 
															+        self.succeeded['update'] = []
														
 
															+        self.succeeded['plaintext'] = []
														
 
															+        self.succeeded['cached_query'] = []
														
 
															+        self.failed = dict()
														
 
															+        self.failed['json'] = []
														
 
															+        self.failed['db'] = []
														
 
															+        self.failed['query'] = []
														
 
															+        self.failed['fortune'] = []
														
 
															+        self.failed['update'] = []
														
 
															+        self.failed['plaintext'] = []
														
 
															+        self.failed['cached_query'] = []
														
 
															+        self.verify = dict()
														
 
															+
														
 
															+        try:
														
 
															+            with open(os.path.join(self.directory, 'results.json'), 'r') as f:
														
 
															+                # Load json file into results object
														
 
															+                self.__dict__.update(json.load(f))
														
 
															+        except IOError:
														
 
															+            logging.warn("results.json for test not found.")
														
 
															+
														
 
															+    #############################################################################
														
 
															+    # PUBLIC FUNCTIONS
														
 
															+    #############################################################################
														
 
															+
														
 
															+    def parse(self, tests):
														
 
															+        '''
														
 
															+        Ensures that the system has all necessary software to run
														
 
															+        the tests. This does not include that software for the individual
														
 
															+        test, but covers software such as curl and weighttp that
														
 
															+        are needed.
														
 
															+        '''
														
 
															+        # Run the method to get the commmit count of each framework.
														
 
															+        self.__count_commits()
														
 
															+        # Call the method which counts the sloc for each framework
														
 
															+        self.__count_sloc()
														
 
															+
														
 
															+        # Time to create parsed files
														
 
															+        # Aggregate JSON file
														
 
															+        with open(self.file, "w") as f:
														
 
															+            f.write(json.dumps(self.__to_jsonable(), indent=2))
														
 
															+
														
 
															+    def parse_test(self, framework_test, test_type):
														
 
															+        '''
														
 
															+        Parses the given test and test_type from the raw_file.
														
 
															+        '''
														
 
															+        try:
														
 
															+            results = dict()
														
 
															+            results['results'] = []
														
 
															+            stats = []
														
 
															+
														
 
															+            if os.path.exists(
														
 
															+                    self.get_raw_file(framework_test.name, test_type)):
														
 
															+                with open(self.get_raw_file(framework_test.name,
														
 
															+                                            test_type)) as raw_data:
														
 
															+
														
 
															+                    is_warmup = True
														
 
															+                    rawData = None
														
 
															+                    for line in raw_data:
														
 
															+                        if "Queries:" in line or "Concurrency:" in line:
														
 
															+                            is_warmup = False
														
 
															+                            rawData = None
														
 
															+                            continue
														
 
															+                        if "Warmup" in line or "Primer" in line:
														
 
															+                            is_warmup = True
														
 
															+                            continue
														
 
															+                        if not is_warmup:
														
 
															+                            if rawData == None:
														
 
															+                                rawData = dict()
														
 
															+                                results['results'].append(rawData)
														
 
															+                            if "Latency" in line:
														
 
															+                                m = re.findall(
														
 
															+                                    r"([0-9]+\.*[0-9]*[us|ms|s|m|%]+)", line)
														
 
															+                                if len(m) == 4:
														
 
															+                                    rawData['latencyAvg'] = m[0]
														
 
															+                                    rawData['latencyStdev'] = m[1]
														
 
															+                                    rawData['latencyMax'] = m[2]
														
 
															+                            if "requests in" in line:
														
 
															+                                m = re.search("([0-9]+) requests in", line)
														
 
															+                                if m != None:
														
 
															+                                    rawData['totalRequests'] = int(m.group(1))
														
 
															+                            if "Socket errors" in line:
														
 
															+                                if "connect" in line:
														
 
															+                                    m = re.search("connect ([0-9]+)", line)
														
 
															+                                    rawData['connect'] = int(m.group(1))
														
 
															+                                if "read" in line:
														
 
															+                                    m = re.search("read ([0-9]+)", line)
														
 
															+                                    rawData['read'] = int(m.group(1))
														
 
															+                                if "write" in line:
														
 
															+                                    m = re.search("write ([0-9]+)", line)
														
 
															+                                    rawData['write'] = int(m.group(1))
														
 
															+                                if "timeout" in line:
														
 
															+                                    m = re.search("timeout ([0-9]+)", line)
														
 
															+                                    rawData['timeout'] = int(m.group(1))
														
 
															+                            if "Non-2xx" in line:
														
 
															+                                m = re.search(
														
 
															+                                    "Non-2xx or 3xx responses: ([0-9]+)", line)
														
 
															+                                if m != None:
														
 
															+                                    rawData['5xx'] = int(m.group(1))
														
 
															+                            if "STARTTIME" in line:
														
 
															+                                m = re.search("[0-9]+", line)
														
 
															+                                rawData["startTime"] = int(m.group(0))
														
 
															+                            if "ENDTIME" in line:
														
 
															+                                m = re.search("[0-9]+", line)
														
 
															+                                rawData["endTime"] = int(m.group(0))
														
 
															+                                test_stats = self.__parse_stats(
														
 
															+                                    framework_test, test_type,
														
 
															+                                    rawData["startTime"], rawData["endTime"],
														
 
															+                                    1)
														
 
															+                                stats.append(test_stats)
														
 
															+            with open(
														
 
															+                    self.get_stats_file(framework_test.name, test_type) +
														
 
															+                    ".json", "w") as stats_file:
														
 
															+                json.dump(stats, stats_file, indent=2)
														
 
															+
														
 
															+            return results
														
 
															+        except IOError:
														
 
															+            return None
														
 
															+
														
 
															+    def parse_all(self, framework_test):
														
 
															+        '''
														
 
															+        Method meant to be run for a given timestamp
														
 
															+        '''
														
 
															+        for test_type in framework_test.runTests:
														
 
															+            if os.path.exists(
														
 
															+                    self.get_raw_file(framework_test.name, test_type)):
														
 
															+                results = self.parse_test(framework_test, test_type)
														
 
															+                self.report_benchmark_results(framework_test, test_type,
														
 
															+                                              results['results'])
														
 
															+
														
 
															+    def write_intermediate(self, test_name, status_message):
														
 
															+        '''
														
 
															+        Writes the intermediate results for the given test_name and status_message
														
 
															+        '''
														
 
															+        self.completed[test_name] = status_message
														
 
															+        self.__write_results()
														
 
															+
														
 
															+    def set_completion_time(self):
														
 
															+        '''
														
 
															+        Sets the completionTime for these results and writes the results
														
 
															+        '''
														
 
															+        self.completionTime = int(round(time.time() * 1000))
														
 
															+        self.__write_results()
														
 
															+
														
 
															+    def upload(self):
														
 
															+        '''
														
 
															+        Attempts to upload the results.json to the configured results_upload_uri
														
 
															+        '''
														
 
															+        if self.config.results_upload_uri != None:
														
 
															+            try:
														
 
															+                requests.post(
														
 
															+                    self.config.results_upload_uri,
														
 
															+                    headers={'Content-Type': 'application/json'},
														
 
															+                    data=json.dumps(self, indent=2))
														
 
															+            except (Exception):
														
 
															+                logging.error("Error uploading results.json")
														
 
															+
														
 
															+    def load(self):
														
 
															+        '''
														
 
															+        Load the results.json file
														
 
															+        '''
														
 
															+        try:
														
 
															+            with open(self.file) as f:
														
 
															+                self.__dict__.update(json.load(f))
														
 
															+        except (ValueError, IOError):
														
 
															+            pass
														
 
															+
														
 
															+    def get_raw_file(self, test_name, test_type):
														
 
															+        '''
														
 
															+        Returns the output file for this test_name and test_type
														
 
															+        Example: fwroot/results/timestamp/test_type/test_name/raw.txt
														
 
															+        '''
														
 
															+        path = os.path.join(self.directory, test_name, test_type, "raw.txt")
														
 
															+        try:
														
 
															+            os.makedirs(os.path.dirname(path))
														
 
															+        except OSError:
														
 
															+            pass
														
 
															+        return path
														
 
															+
														
 
															+    def get_stats_file(self, test_name, test_type):
														
 
															+        '''
														
 
															+        Returns the stats file name for this test_name and
														
 
															+        Example: fwroot/results/timestamp/test_type/test_name/stats.txt
														
 
															+        '''
														
 
															+        path = os.path.join(self.directory, test_name, test_type, "stats.txt")
														
 
															+        try:
														
 
															+            os.makedirs(os.path.dirname(path))
														
 
															+        except OSError:
														
 
															+            pass
														
 
															+        return path
														
 
															+
														
 
															+    def report_verify_results(self, framework_test, test_type, result):
														
 
															+        '''
														
 
															+        Used by FrameworkTest to add verification details to our results
														
 
															+        
														
 
															+        TODO: Technically this is an IPC violation - we are accessing
														
 
															+        the parent process' memory from the child process
														
 
															+        '''
														
 
															+        if framework_test.name not in self.verify.keys():
														
 
															+            self.verify[framework_test.name] = dict()
														
 
															+        self.verify[framework_test.name][test_type] = result
														
 
															+
														
 
															+    def report_benchmark_results(self, framework_test, test_type, results):
														
 
															+        '''
														
 
															+        Used by FrameworkTest to add benchmark data to this
														
 
															+        
														
 
															+        TODO: Technically this is an IPC violation - we are accessing
														
 
															+        the parent process' memory from the child process
														
 
															+        '''
														
 
															+        if test_type not in self.rawData.keys():
														
 
															+            self.rawData[test_type] = dict()
														
 
															+
														
 
															+        # If results has a size from the parse, then it succeeded.
														
 
															+        if results:
														
 
															+            self.rawData[test_type][framework_test.name] = results
														
 
															+
														
 
															+            # This may already be set for single-tests
														
 
															+            if framework_test.name not in self.succeeded[test_type]:
														
 
															+                self.succeeded[test_type].append(framework_test.name)
														
 
															+        else:
														
 
															+            # This may already be set for single-tests
														
 
															+            if framework_test.name not in self.failed[test_type]:
														
 
															+                self.failed[test_type].append(framework_test.name)
														
 
															+
														
 
															+    def finish(self):
														
 
															+        '''
														
 
															+        Finishes these results.
														
 
															+        '''
														
 
															+        if not self.config.parse:
														
 
															+            tests = gather_remaining_tests(self.config, self)
														
 
															+            # Normally you don't have to use Fore.BLUE before each line, but
														
 
															+            # Travis-CI seems to reset color codes on newline (see travis-ci/travis-ci#2692)
														
 
															+            # or stream flush, so we have to ensure that the color code is printed repeatedly
														
 
															+            prefix = Fore.CYAN
														
 
															+            for line in header(
														
 
															+                    "Verification Summary", top='=', bottom='').split('\n'):
														
 
															+                print(prefix + line)
														
 
															+            for test in tests:
														
 
															+                print(prefix + "| Test: {!s}".format(test.name))
														
 
															+                if test.name in self.verify.keys():
														
 
															+                    for test_type, result in self.verify[
														
 
															+                            test.name].iteritems():
														
 
															+                        if result.upper() == "PASS":
														
 
															+                            color = Fore.GREEN
														
 
															+                        elif result.upper() == "WARN":
														
 
															+                            color = Fore.YELLOW
														
 
															+                        else:
														
 
															+                            color = Fore.RED
														
 
															+                        print(prefix + "|       " + test_type.ljust(13) +
														
 
															+                              ' : ' + color + result.upper())
														
 
															+                else:
														
 
															+                    print(prefix + "|      " + Fore.RED +
														
 
															+                          "NO RESULTS (Did framework launch?)")
														
 
															+            print(prefix + header('', top='', bottom='=') + Style.RESET_ALL)
														
 
															+
														
 
															+        print("Time to complete: " +
														
 
															+              str(int(time.time() - self.config.start_time)) + " seconds")
														
 
															+        print("Results are saved in " + self.directory)
														
 
															+
														
 
															+    #############################################################################
														
 
															+    # PRIVATE FUNCTIONS
														
 
															+    #############################################################################
														
 
															+
														
 
															+    def __to_jsonable(self):
														
 
															+        '''
														
 
															+        Returns a dict suitable for jsonification
														
 
															+        '''
														
 
															+        toRet = dict()
														
 
															+
														
 
															+        toRet['uuid'] = self.uuid
														
 
															+        toRet['name'] = self.name
														
 
															+        toRet['environmentDescription'] = self.environmentDescription
														
 
															+        toRet['git'] = self.git
														
 
															+        toRet['startTime'] = self.startTime
														
 
															+        toRet['completionTime'] = self.completionTime
														
 
															+        toRet['concurrencyLevels'] = self.concurrencyLevels
														
 
															+        toRet['pipelineConcurrencyLevels'] = self.pipelineConcurrencyLevels
														
 
															+        toRet['queryIntervals'] = self.queryIntervals
														
 
															+        toRet['cachedQueryIntervals'] = self.cachedQueryIntervals
														
 
															+        toRet['frameworks'] = self.frameworks
														
 
															+        toRet['duration'] = self.duration
														
 
															+        toRet['rawData'] = self.rawData
														
 
															+        toRet['completed'] = self.completed
														
 
															+        toRet['succeeded'] = self.succeeded
														
 
															+        toRet['failed'] = self.failed
														
 
															+        toRet['verify'] = self.verify
														
 
															+
														
 
															+        return toRet
														
 
															+
														
 
															+    def __write_results(self):
														
 
															+        try:
														
 
															+            with open(self.file, 'w') as f:
														
 
															+                f.write(json.dumps(self.__to_jsonable(), indent=2))
														
 
															+        except (IOError):
														
 
															+            logging.error("Error writing results.json")
														
 
															+
														
 
															+    def __count_sloc(self):
														
 
															+        '''
														
 
															+        Counts the significant lines of code for all tests and stores in results.
														
 
															+        '''
														
 
															+        frameworks = gather_frameworks(self.config.test, self.config.exclude,
														
 
															+                                       self.config)
														
 
															+
														
 
															+        jsonResult = {}
														
 
															+        for framework, testlist in frameworks.items():
														
 
															+            if not os.path.exists(
														
 
															+                    os.path.join(testlist[0].directory, "source_code")):
														
 
															+                logging.warn(
														
 
															+                    "Cannot count lines of code for %s - no 'source_code' file",
														
 
															+                    framework)
														
 
															+                continue
														
 
															+
														
 
															+            # Unfortunately the source_code files use lines like
														
 
															+            # ./cpoll_cppsp/www/fortune_old instead of
														
 
															+            # ./www/fortune_old
														
 
															+            # so we have to back our working dir up one level
														
 
															+            wd = os.path.dirname(testlist[0].directory)
														
 
															+
														
 
															+            try:
														
 
															+                command = "cloc --list-file=%s/source_code --yaml" % testlist[
														
 
															+                    0].directory
														
 
															+
														
 
															+                if os.path.exists(
														
 
															+                        os.path.join(testlist[0].directory, "cloc_defs.txt")):
														
 
															+                    command += " --read-lang-def %s" % os.path.join(
														
 
															+                        testlist[0].directory, "cloc_defs.txt")
														
 
															+                    logging.info("Using custom cloc definitions for %s",
														
 
															+                                 framework)
														
 
															+
														
 
															+                # Find the last instance of the word 'code' in the yaml output. This should
														
 
															+                # be the line count for the sum of all listed files or just the line count
														
 
															+                # for the last file in the case where there's only one file listed.
														
 
															+                command = command + "| grep code | tail -1 | cut -d: -f 2"
														
 
															+                logging.debug("Running \"%s\" (cwd=%s)", command, wd)
														
 
															+                lineCount = subprocess.check_output(
														
 
															+                    command, cwd=wd, shell=True)
														
 
															+                jsonResult[framework] = int(lineCount)
														
 
															+            except subprocess.CalledProcessError:
														
 
															+                continue
														
 
															+            except ValueError as ve:
														
 
															+                logging.warn(
														
 
															+                    "Unable to get linecount for %s due to error '%s'",
														
 
															+                    framework, ve)
														
 
															+        self.rawData['slocCounts'] = jsonResult
														
 
															+
														
 
															+    def __count_commits(self):
														
 
															+        '''
														
 
															+        Count the git commits for all the framework tests
														
 
															+        '''
														
 
															+        frameworks = gather_frameworks(self.config.test, self.config.exclude,
														
 
															+                                       self.config)
														
 
															+
														
 
															+        def count_commit(directory, jsonResult):
														
 
															+            command = "git rev-list HEAD -- " + directory + " | sort -u | wc -l"
														
 
															+            try:
														
 
															+                commitCount = subprocess.check_output(command, shell=True)
														
 
															+                jsonResult[framework] = int(commitCount)
														
 
															+            except subprocess.CalledProcessError:
														
 
															+                pass
														
 
															+
														
 
															+        # Because git can be slow when run in large batches, this
														
 
															+        # calls git up to 4 times in parallel. Normal improvement is ~3-4x
														
 
															+        # in my trials, or ~100 seconds down to ~25
														
 
															+        # This is safe to parallelize as long as each thread only
														
 
															+        # accesses one key in the dictionary
														
 
															+        threads = []
														
 
															+        jsonResult = {}
														
 
															+        # t1 = datetime.now()
														
 
															+        for framework, testlist in frameworks.items():
														
 
															+            directory = testlist[0].directory
														
 
															+            t = threading.Thread(
														
 
															+                target=count_commit, args=(directory, jsonResult))
														
 
															+            t.start()
														
 
															+            threads.append(t)
														
 
															+            # Git has internal locks, full parallel will just cause contention
														
 
															+            # and slowness, so we rate-limit a bit
														
 
															+            if len(threads) >= 4:
														
 
															+                threads[0].join()
														
 
															+                threads.remove(threads[0])
														
 
															+
														
 
															+        # Wait for remaining threads
														
 
															+        for t in threads:
														
 
															+            t.join()
														
 
															+        # t2 = datetime.now()
														
 
															+        # print "Took %s seconds " % (t2 - t1).seconds
														
 
															+
														
 
															+        self.rawData['commitCounts'] = jsonResult
														
 
															+        self.config.commits = jsonResult
														
 
															+
														
 
															+    def __get_git_commit_id(self):
														
 
															+        '''
														
 
															+        Get the git commit id for this benchmark
														
 
															+        '''
														
 
															+        return subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
														
 
															+
														
 
															+    def __get_git_repository_url(self):
														
 
															+        '''
														
 
															+        Gets the git repository url for this benchmark
														
 
															+        '''
														
 
															+        return subprocess.check_output(
														
 
															+            ["git", "config", "--get", "remote.origin.url"]).strip()
														
 
															+
														
 
															+    def __get_git_branch_name(self):
														
 
															+        '''
														
 
															+        Gets the git branch name for this benchmark
														
 
															+        '''
														
 
															+        return subprocess.check_output(
														
 
															+            'git rev-parse --abbrev-ref HEAD', shell=True).strip()
														
 
															+
														
 
															+    def __parse_stats(self, framework_test, test_type, start_time, end_time,
														
 
															+                      interval):
														
 
															+        '''
														
 
															+        For each test type, process all the statistics, and return a multi-layered 
														
 
															+        dictionary that has a structure as follows:
														
 
															+
														
 
															+        (timestamp)
														
 
															+        | (main header) - group that the stat is in
														
 
															+        | | (sub header) - title of the stat
														
 
															+        | | | (stat) - the stat itself, usually a floating point number
														
 
															+        '''
														
 
															+        stats_dict = dict()
														
 
															+        stats_file = self.get_stats_file(framework_test.name, test_type)
														
 
															+        with open(stats_file) as stats:
														
 
															+            # dstat doesn't output a completely compliant CSV file - we need to strip the header
														
 
															+            while (stats.next() != "\n"):
														
 
															+                pass
														
 
															+            stats_reader = csv.reader(stats)
														
 
															+            main_header = stats_reader.next()
														
 
															+            sub_header = stats_reader.next()
														
 
															+            time_row = sub_header.index("epoch")
														
 
															+            int_counter = 0
														
 
															+            for row in stats_reader:
														
 
															+                time = float(row[time_row])
														
 
															+                int_counter += 1
														
 
															+                if time < start_time:
														
 
															+                    continue
														
 
															+                elif time > end_time:
														
 
															+                    return stats_dict
														
 
															+                if int_counter % interval != 0:
														
 
															+                    continue
														
 
															+                row_dict = dict()
														
 
															+                for nextheader in main_header:
														
 
															+                    if nextheader != "":
														
 
															+                        row_dict[nextheader] = dict()
														
 
															+                header = ""
														
 
															+                for item_num, column in enumerate(row):
														
 
															+                    if (len(main_header[item_num]) != 0):
														
 
															+                        header = main_header[item_num]
														
 
															+                    # all the stats are numbers, so we want to make sure that they stay that way in json
														
 
															+                    row_dict[header][sub_header[item_num]] = float(column)
														
 
															+                stats_dict[time] = row_dict
														
 
															+        return stats_dict
														
 
															+
														
 
															+    def __calculate_average_stats(self, raw_stats):
														
 
															+        '''
														
 
															+        We have a large amount of raw data for the statistics that may be useful 
														
 
															+        for the stats nerds, but most people care about a couple of numbers. For 
														
 
															+        now, we're only going to supply:
														
 
															+          * Average CPU
														
 
															+          * Average Memory
														
 
															+          * Total network use
														
 
															+          * Total disk use
														
 
															+        More may be added in the future. If they are, please update the above list.
														
 
															+        
														
 
															+        Note: raw_stats is directly from the __parse_stats method.
														
 
															+        
														
 
															+        Recall that this consists of a dictionary of timestamps, each of which 
														
 
															+        contain a dictionary of stat categories which contain a dictionary of stats
														
 
															+        '''
														
 
															+        raw_stat_collection = dict()
														
 
															+
														
 
															+        for time_dict in raw_stats.items()[1]:
														
 
															+            for main_header, sub_headers in time_dict.items():
														
 
															+                item_to_append = None
														
 
															+                if 'cpu' in main_header:
														
 
															+                    # We want to take the idl stat and subtract it from 100
														
 
															+                    # to get the time that the CPU is NOT idle.
														
 
															+                    item_to_append = sub_headers['idl'] - 100.0
														
 
															+                elif main_header == 'memory usage':
														
 
															+                    item_to_append = sub_headers['used']
														
 
															+                elif 'net' in main_header:
														
 
															+                    # Network stats have two parts - recieve and send. We'll use a tuple of
														
 
															+                    # style (recieve, send)
														
 
															+                    item_to_append = (sub_headers['recv'], sub_headers['send'])
														
 
															+                elif 'dsk' or 'io' in main_header:
														
 
															+                    # Similar for network, except our tuple looks like (read, write)
														
 
															+                    item_to_append = (sub_headers['read'], sub_headers['writ'])
														
 
															+                if item_to_append is not None:
														
 
															+                    if main_header not in raw_stat_collection:
														
 
															+                        raw_stat_collection[main_header] = list()
														
 
															+                    raw_stat_collection[main_header].append(item_to_append)
														
 
															+
														
 
															+        # Simple function to determine human readable size
														
 
															+        # http://stackoverflow.com/questions/1094841/reusable-library-to-get-human-readable-version-of-file-size
														
 
															+        def sizeof_fmt(num):
														
 
															+            # We'll assume that any number we get is convertable to a float, just in case
														
 
															+            num = float(num)
														
 
															+            for x in ['bytes', 'KB', 'MB', 'GB']:
														
 
															+                if num < 1024.0 and num > -1024.0:
														
 
															+                    return "%3.1f%s" % (num, x)
														
 
															+                num /= 1024.0
														
 
															+            return "%3.1f%s" % (num, 'TB')
														
 
															+
														
 
															+        # Now we have our raw stats in a readable format - we need to format it for display
														
 
															+        # We need a floating point sum, so the built in sum doesn't cut it
														
 
															+        display_stat_collection = dict()
														
 
															+        for header, values in raw_stat_collection.items():
														
 
															+            display_stat = None
														
 
															+            if 'cpu' in header:
														
 
															+                display_stat = sizeof_fmt(math.fsum(values) / len(values))
														
 
															+            elif main_header == 'memory usage':
														
 
															+                display_stat = sizeof_fmt(math.fsum(values) / len(values))
														
 
															+            elif 'net' in main_header:
														
 
															+                receive, send = zip(*values)  # unzip
														
 
															+                display_stat = {
														
 
															+                    'receive': sizeof_fmt(math.fsum(receive)),
														
 
															+                    'send': sizeof_fmt(math.fsum(send))
														
 
															+                }
														
 
															+            else:  # if 'dsk' or 'io' in header:
														
 
															+                read, write = zip(*values)  # unzip
														
 
															+                display_stat = {
														
 
															+                    'read': sizeof_fmt(math.fsum(read)),
														
 
															+                    'write': sizeof_fmt(math.fsum(write))
														
 
															+                }
														
 
															+            display_stat_collection[header] = display_stat
														
 
															+        return display_stat
														
--- a/toolset/utils/scaffolding.py
+++ b/toolset/utils/scaffolding.py
@@ -0,0 +1,356 @@
 
															+# -*- coding: utf-8 -*-
														
 
															+import os
														
 
															+from shutil import copytree
														
 
															+from toolset.utils.setup_util import replace_text
														
 
															+from toolset.utils.metadata_helper import gather_frameworks, gather_langauges
														
 
															+
														
 
															+
														
 
															+class Scaffolding:
														
 
															+    def __init__(self):
														
 
															+        print("""
														
 
															+-------------------------------------------------------------------------------
														
 
															+    This wizard is intended to help build the scaffolding required for a new 
														
 
															+    test to be benchmarked.
														
 
															+
														
 
															+    From here, you will be prompted for values related to the test you
														
 
															+    wish to add.
														
 
															+-------------------------------------------------------------------------------"""
														
 
															+              )
														
 
															+
														
 
															+        try:
														
 
															+            self.__gather_display_name()
														
 
															+            self.__gather_language()
														
 
															+            self.__gather_approach()
														
 
															+            self.__gather_classification()
														
 
															+            self.__gather_orm()
														
 
															+            self.__gather_webserver()
														
 
															+            self.__gather_versus()
														
 
															+            self.__confirm_values()
														
 
															+            self.__print_success()
														
 
															+        except:
														
 
															+            print("")
														
 
															+
														
 
															+    def __gather_display_name(self):
														
 
															+        print("""
														
 
															+  The name of your test as you wish it to be displayed on the results page.
														
 
															+
														
 
															+  Example: Gemini, Gin, Express
														
 
															+    """)
														
 
															+        self.__prompt_display_name()
														
 
															+        while not self.display_name:
														
 
															+            self.__prompt_display_name()
														
 
															+        self.name = self.display_name.lower()
														
 
															+
														
 
															+    def __prompt_display_name(self):
														
 
															+        self.display_name = input("Name: ").strip()
														
 
															+
														
 
															+        found = False
														
 
															+        for framework in gather_frameworks():
														
 
															+            if framework.lower() == self.display_name.lower():
														
 
															+                found = True
														
 
															+
														
 
															+        if found:
														
 
															+            print("""
														
 
															+  It appears that there is already a '%s' framework in the test suite. You will
														
 
															+  have to pick a different name.
														
 
															+      """ % self.display_name)
														
 
															+            self.display_name = None
														
 
															+
														
 
															+    def __gather_language(self):
														
 
															+        print("""
														
 
															+  The language in which your test implementation is written.
														
 
															+
														
 
															+  Example: Java, Go, PHP
														
 
															+    """)
														
 
															+        self.language = None
														
 
															+        while not self.language:
														
 
															+            self.__prompt_language()
														
 
															+
														
 
															+    def __prompt_language(self):
														
 
															+        self.language = input("Language: ").strip()
														
 
															+
														
 
															+        known_languages = gather_langauges()
														
 
															+        language = None
														
 
															+        for lang in known_languages:
														
 
															+            if lang.lower() == self.language.lower():
														
 
															+                language = lang
														
 
															+
														
 
															+        if not language:
														
 
															+            similar = []
														
 
															+            for lang in known_languages:
														
 
															+                if lang.lower()[:1] == self.language.lower()[:1]:
														
 
															+                    similar.append(lang)
														
 
															+            similar = ', '.join(similar)
														
 
															+
														
 
															+            print("""
														
 
															+  That language is not currently in our list of known languages.
														
 
															+  
														
 
															+  Here is a list of similar languages present in our benchmark suite that you
														
 
															+  may have meant:
														
 
															+
														
 
															+  %s
														
 
															+      
														
 
															+  Did you mean to add the new language, '%s', to the benchmark suite?
														
 
															+      """ % (similar, self.language))
														
 
															+            valid = self.__prompt_confirm_new_language(known_languages)
														
 
															+            while not valid:
														
 
															+                valid = self.__prompt_confirm_new_language(known_languages)
														
 
															+
														
 
															+            if self.confirm_new_lang == 'n':
														
 
															+                self.language = None
														
 
															+            else:
														
 
															+                self.language = self.language.title()
														
 
															+
														
 
															+        return self.language
														
 
															+
														
 
															+    def __prompt_confirm_new_language(self, known_languages):
														
 
															+        self.confirm_new_lang = input("Create New Language '%s' (y/n): " %
														
 
															+                                      self.language).strip().lower()
														
 
															+        return self.confirm_new_lang == 'y' or self.confirm_new_lang == 'n'
														
 
															+
														
 
															+    def __gather_approach(self):
														
 
															+        print("""
														
 
															+  The approach of your test implementation.
														
 
															+
														
 
															+  1) Realistic: Uses the framework with most out-of-the-box functionality 
														
 
															+                enabled. We consider this realistic because most applications 
														
 
															+                built with the framework will leave these features enabled.
														
 
															+  2) Stripped:  Removes or outright avoids implementing features that are
														
 
															+                unnecessary for the particulars of the benchmark exercise. This
														
 
															+                might illuminate the marginal improvement available in fine-
														
 
															+                tuning a framework to your application's use-case.
														
 
															+
														
 
															+  Note: If you are unsure, then your approach is probably Realistic. The
														
 
															+        Stripped approach is seldom used and will not have results displayed
														
 
															+        by default on the results website.
														
 
															+    """)
														
 
															+        valid = self.__prompt_approach()
														
 
															+        while not valid:
														
 
															+            valid = self.__prompt_approach()
														
 
															+
														
 
															+    def __prompt_approach(self):
														
 
															+        self.approach = input("Approach [1/2]: ").strip()
														
 
															+        if self.approach == '1':
														
 
															+            self.approach = 'Realistic'
														
 
															+        if self.approach == '2':
														
 
															+            self.approach = 'Stripped'
														
 
															+        return self.approach == 'Realistic' or self.approach == 'Stripped'
														
 
															+
														
 
															+    def __gather_classification(self):
														
 
															+        print("""
														
 
															+  The classification of your test implementation.
														
 
															+
														
 
															+  1) Fullstack: Robust framework expected to provide high-level functionality 
														
 
															+                for serving as a web application; for example, ability to 
														
 
															+                compose views, provide functions for responding with several 
														
 
															+                data types (json, html, etc), connecting to a database, form 
														
 
															+                processing, etc.
														
 
															+  2) Micro:     Simple framework expected to provide enough middleware to build
														
 
															+                a robust web application such as request routing and some 
														
 
															+                simple plumbing, but may not include built-in functionality 
														
 
															+                such as, for example, server-composed views.
														
 
															+  3) Platform:  Barebones infrastructure for servicing HTTP requests, but does
														
 
															+                not include a framework at all.
														
 
															+    """)
														
 
															+        valid = self.__prompt_classification()
														
 
															+        while not valid:
														
 
															+            valid = self.__prompt_classification()
														
 
															+        if self.classification == 'Platform':
														
 
															+            self.platform = 'None'
														
 
															+            self.framework = 'None'
														
 
															+        else:
														
 
															+            self.framework = self.display_name
														
 
															+            self.__gather_platform()
														
 
															+
														
 
															+    def __prompt_classification(self):
														
 
															+        self.classification = input("Classification [1/2/3]: ").strip()
														
 
															+        if self.classification == '1':
														
 
															+            self.classification = 'Fullstack'
														
 
															+        if self.classification == '2':
														
 
															+            self.classification = 'Micro'
														
 
															+        if self.classification == '3':
														
 
															+            self.classification = 'Platform'
														
 
															+        return self.classification == 'Fullstack' or \
														
 
															+               self.classification == 'Micro' or \
														
 
															+               self.classification == 'Platform'
														
 
															+
														
 
															+    def __gather_platform(self):
														
 
															+        print("""
														
 
															+  The platform of your test implementation.
														
 
															+
														
 
															+  The platform is the low-level software or API used to host web applications 
														
 
															+  for the framework; the platform provides an implementation of the HTTP
														
 
															+  fundamentals.
														
 
															+
														
 
															+  Not all frameworks have a platform and if your programming language provides
														
 
															+  much of that by which we define a platform, leave black.
														
 
															+
														
 
															+  Example: Servlet, Wai, .NET
														
 
															+    """)
														
 
															+        self.__prompt_platform()
														
 
															+
														
 
															+    def __prompt_platform(self):
														
 
															+        self.platform = input("Platform (optional): ").strip()
														
 
															+        if self.platform == '':
														
 
															+            self.platform = 'None'
														
 
															+
														
 
															+    def __gather_orm(self):
														
 
															+        print("""
														
 
															+  How you would classify the ORM (object relational mapper) of your test?
														
 
															+
														
 
															+  1) Full:  A feature-rich ORM which provides functionality for interacting 
														
 
															+            with a database without writing a query in all but the most edge 
														
 
															+            cases.
														
 
															+  2) Micro: An ORM which provides functionality for interacting with a database
														
 
															+            for many trivial operations (querying, updating), but not more 
														
 
															+            robust cases (for example, gathering relations).
														
 
															+  3) Raw:   No ORM; raw database access.
														
 
															+    """)
														
 
															+        valid = self.__prompt_orm()
														
 
															+        while not valid:
														
 
															+            valid = self.__prompt_orm()
														
 
															+
														
 
															+    def __prompt_orm(self):
														
 
															+        self.orm = input("ORM [1/2/3]: ").strip()
														
 
															+        if self.orm == '1':
														
 
															+            self.orm = 'Full'
														
 
															+        if self.orm == '2':
														
 
															+            self.orm = 'Micro'
														
 
															+        if self.orm == '3':
														
 
															+            self.orm = 'Raw'
														
 
															+        return self.orm == 'Full' or \
														
 
															+               self.orm == 'Micro' or \
														
 
															+               self.orm == 'Raw'
														
 
															+
														
 
															+    def __gather_webserver(self):
														
 
															+        print("""
														
 
															+  Name of the front-end webserver sitting in front of your test implementation.
														
 
															+
														
 
															+  Your test implementation may not use a web-server and may act as its own; you
														
 
															+  can leave this blank in this case.
														
 
															+
														
 
															+  Example: nginx, Meinheld, httplight
														
 
															+    """)
														
 
															+        self.__prompt_webserver()
														
 
															+
														
 
															+    def __prompt_webserver(self):
														
 
															+        self.webserver = input("Webserver (optional): ").strip()
														
 
															+        if self.webserver == '':
														
 
															+            self.webserver = 'None'
														
 
															+
														
 
															+    def __gather_versus(self):
														
 
															+        print("""
														
 
															+  The name of another test (elsewhere in this project) that is a subset of this
														
 
															+  framework.
														
 
															+  This allows for the generation of the framework efficiency chart in the 
														
 
															+  results web site.
														
 
															+  For example, Compojure is compared to "servlet" since Compojure is built on 
														
 
															+  the Servlet platform.
														
 
															+
														
 
															+  Example: Servlet, Wai, Undertow
														
 
															+    """)
														
 
															+        self.__prompt_versus()
														
 
															+
														
 
															+    def __prompt_versus(self):
														
 
															+        self.versus = input("Versus (optional): ").strip()
														
 
															+        if self.versus == '':
														
 
															+            self.versus = 'None'
														
 
															+
														
 
															+    def __confirm_values(self):
														
 
															+        print("""
														
 
															+    Name: %s
														
 
															+    Language: %s
														
 
															+    Approach: %s
														
 
															+    Classification: %s
														
 
															+    Platform: %s
														
 
															+    ORM: %s
														
 
															+    Webserver: %s
														
 
															+    Versus: %s
														
 
															+
														
 
															+  Finalize the initialization of your test given the above values?
														
 
															+
														
 
															+  Note: once you have initialized your test, you can change these values later.
														
 
															+    """ % (self.display_name, self.language, self.approach,
														
 
															+           self.classification, self.platform, self.orm, self.webserver,
														
 
															+           self.versus))
														
 
															+
														
 
															+        valid = self.__prompt_confirmation()
														
 
															+        while not valid:
														
 
															+            valid = self.__prompt_confirmation()
														
 
															+
														
 
															+        if self.confirmation == 'y':
														
 
															+            self.__build_scaffolding()
														
 
															+        else:
														
 
															+            print('Aborting')
														
 
															+
														
 
															+    def __prompt_confirmation(self):
														
 
															+        self.confirmation = input("Initialize [y/n]: ").strip().lower()
														
 
															+        return self.confirmation == 'y' or self.confirmation == 'n'
														
 
															+
														
 
															+    def __build_scaffolding(self):
														
 
															+        if self.__create_test_folder():
														
 
															+            self.__copy_scaffold_files()
														
 
															+            self.__edit_scaffold_files()
														
 
															+
														
 
															+    def __create_test_folder(self):
														
 
															+        self.language_dir = os.path.join("frameworks", self.language)
														
 
															+        self.test_dir = os.path.join(self.language_dir, self.name)
														
 
															+
														
 
															+        if os.path.exists(self.test_dir):
														
 
															+            print("Test '%s' already exists; aborting." % self.name)
														
 
															+            return False
														
 
															+
														
 
															+        return True
														
 
															+
														
 
															+    def __copy_scaffold_files(self):
														
 
															+        self.scaffold_dir = os.path.join("toolset", "setup", "scaffolding")
														
 
															+        copytree(self.scaffold_dir, self.test_dir)
														
 
															+
														
 
															+    def __edit_scaffold_files(self):
														
 
															+        for file in os.listdir(os.path.join(self.test_dir)):
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$NAME", self.name)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$DISPLAY_NAME",
														
 
															+                self.display_name)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$APPROACH", self.approach)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$CLASSIFICATION",
														
 
															+                self.classification)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$FRAMEWORK",
														
 
															+                self.framework)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$LANGUAGE", self.language)
														
 
															+            replace_text(os.path.join(self.test_dir, file), "\$ORM", self.orm)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$PLATFORM", self.platform)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$WEBSERVER",
														
 
															+                self.webserver)
														
 
															+            replace_text(
														
 
															+                os.path.join(self.test_dir, file), "\$VERSUS", self.versus)
														
 
															+
														
 
															+    def __print_success(self):
														
 
															+        print("""
														
 
															+-------------------------------------------------------------------------------
														
 
															+  Success!
														
 
															+
														
 
															+  Your new test structure has been built to the sepcifications of the suite.
														
 
															+  Here is a brief run-down of what has been built:
														
 
															+
														
 
															+    frameworks
														
 
															+        └─── %s
														
 
															+              └─── %s
														
 
															+                    ├─── .gitignore
														
 
															+                    ├─── benchmark_config.json
														
 
															+                    ├─── README.md
														
 
															+                    ├─── setup.sh
														
 
															+                    ├─── setup_mysql.sh
														
 
															+                    └─── source_code
														
 
															+
														
 
															+  The next step is to read through your README.md and follow the instructions
														
 
															+  provided therein.
														
 
															+-------------------------------------------------------------------------------"""
														
 
															+              % (self.language, self.name))
														
--- a/toolset/setup/linux/setup_util.py
+++ b/toolset/setup/linux/setup_util.py
@@ -2,21 +2,22 @@ import re
 
															 import os
														
 
															 import sys
														
 
															 import subprocess
														
 
															-import platform
														
 
															 from threading import Thread
														
 
															 from Queue import Queue, Empty
														
 
															+
														
 
															 class NonBlockingStreamReader:
														
 
															-  '''
														
 
															+    '''
														
 
															   Enables calling readline in a non-blocking manner with a blocking stream, 
														
 
															   such as the ones returned from subprocess.Popen
														
 
															   Originally written by Eyal Arubas, who granted permission to use this inside TFB
														
 
															   See http://eyalarubas.com/python-subproc-nonblock.html
														
 
															   '''
														
 
															-  def __init__(self, stream, eof_message = None):
														
 
															-    '''
														
 
															+
														
 
															+    def __init__(self, stream, eof_message=None):
														
 
															+        '''
														
 
															     stream: the stream to read from.
														
 
															             Usually a process' stdout or stderr.
														
 
															     eof_message: A message to print to stdout as soon
														
@@ -24,38 +25,39 @@ class NonBlockingStreamReader:
 
															       want to track the exact moment a stream terminates
														
 
															     '''
														
 
															-    self._s = stream
														
 
															-    self._q = Queue()
														
 
															-    self._eof_message = eof_message
														
 
															-    self._poisonpill = 'MAGIC_POISONPILL_STRING'
														
 
															-
														
 
															-    def _populateQueue(stream, queue):
														
 
															-      while True:
														
 
															-        line = stream.readline()
														
 
															-        if line: # 'data\n' or '\n'
														
 
															-          queue.put(line)
														
 
															-        else:    # '' e.g. EOF
														
 
															-          if self._eof_message:
														
 
															-            sys.stdout.write(self._eof_message + '\n')
														
 
															-          queue.put(self._poisonpill)
														
 
															-          return
														
 
															-
														
 
															-    self._t = Thread(target = _populateQueue,
														
 
															-            args = (self._s, self._q))
														
 
															-    self._t.daemon = True
														
 
															-    self._t.start()
														
 
															-
														
 
															-  def readline(self, timeout = None):
														
 
															-    try:
														
 
															-      line = self._q.get(block = timeout is not None,
														
 
															-        timeout = timeout)
														
 
															-      if line == self._poisonpill: 
														
 
															-        raise EndOfStream
														
 
															-      return line
														
 
															-    except Empty:
														
 
															-      return None
														
 
															-
														
 
															-class EndOfStream(Exception): pass
														
 
															+        self._s = stream
														
 
															+        self._q = Queue()
														
 
															+        self._eof_message = eof_message
														
 
															+        self._poisonpill = 'MAGIC_POISONPILL_STRING'
														
 
															+
														
 
															+        def _populateQueue(stream, queue):
														
 
															+            while True:
														
 
															+                line = stream.readline()
														
 
															+                if line:  # 'data\n' or '\n'
														
 
															+                    queue.put(line)
														
 
															+                else:  # '' e.g. EOF
														
 
															+                    if self._eof_message:
														
 
															+                        sys.stdout.write(self._eof_message + '\n')
														
 
															+                    queue.put(self._poisonpill)
														
 
															+                    return
														
 
															+
														
 
															+        self._t = Thread(target=_populateQueue, args=(self._s, self._q))
														
 
															+        self._t.daemon = True
														
 
															+        self._t.start()
														
 
															+
														
 
															+    def readline(self, timeout=None):
														
 
															+        try:
														
 
															+            line = self._q.get(block=timeout is not None, timeout=timeout)
														
 
															+            if line == self._poisonpill:
														
 
															+                raise EndOfStream
														
 
															+            return line
														
 
															+        except Empty:
														
 
															+            return None
														
 
															+
														
 
															+
														
 
															+class EndOfStream(Exception):
														
 
															+    pass
														
 
															+
														
 
															 # Replaces all text found using the regular expression to_replace with the supplied replacement.
														
 
															 def replace_text(file, to_replace, replacement):
														
@@ -65,6 +67,7 @@ def replace_text(file, to_replace, replacement):
 
															     with open(file, "w") as f:
														
 
															         f.write(replaced_text)
														
 
															+
														
 
															 # Queries the shell for the value of FWROOT
														
 
															 def get_fwroot():
														
@@ -73,12 +76,16 @@ def get_fwroot():
 
															     else:
														
 
															         return os.getcwd()
														
 
															+
														
 
															 # Turns absolute path into path relative to FWROOT
														
 
															 # Assumes path is underneath FWROOT, not above
														
 
															-# 
														
 
															-# Useful for clean presentation of paths 
														
 
															+#
														
 
															+# Useful for clean presentation of paths
														
 
															 # e.g. /foo/bar/benchmarks/go/install.sh
														
 
															-# v.s. FWROOT/go/install.sh 
														
 
															+# v.s. FWROOT/go/install.sh
														
 
															 def path_relative_to_root(path):
														
 
															     # Requires bash shell parameter expansion
														
 
															-    return subprocess.check_output("D=%s && printf \"${D#%s}\""%(path, get_fwroot()), shell=True, executable='/bin/bash')
														
 
															+    return subprocess.check_output(
														
 
															+        "D=%s && printf \"${D#%s}\"" % (path, get_fwroot()),
														
 
															+        shell=True,
														
 
															+        executable='/bin/bash')
														
--- a/toolset/utils/unbuffered.py
+++ b/toolset/utils/unbuffered.py
@@ -0,0 +1,16 @@
 
															+# Wrapper for unbuffered stream writing.
														
 
															+# http://stackoverflow.com/a/107717/376366
														
 
															+# Used to make sure print output appears in the correct order
														
 
															+# in log files when spawning subprocesses.
														
 
															+
														
 
															+
														
 
															+class Unbuffered:
														
 
															+    def __init__(self, stream):
														
 
															+        self.stream = stream
														
 
															+
														
 
															+    def write(self, data):
														
 
															+        self.stream.write(data)
														
 
															+        self.stream.flush()
														
 
															+
														
 
															+    def __getattr__(self, attr):
														
 
															+        return getattr(self.stream, attr)