Merge pull request #1767 from TechEmpower/requests-not-curl

Replace curl usage with requests

.travis.yml

@@ -176,10 +176,13 @@ env:
     - "TESTDIR=Ur/urweb"
 
 before_install:
-  # Need to install python modules before using
-  # python
-  - pip install -r requirements.txt
+  - pip install colorama==0.3.1
+  # Version 2.3 has a nice Counter() and other features
+  # but it requires —-allow-external and -—allow-unverified
+  - pip install progressbar==2.2
+  - pip install requests
 
+install:
   # Configure Travis-CI build environment for TFB
   #   e.g. setup databases, users, etc
   - ./toolset/run-ci.py cisetup "$TESTDIR"
@@ -187,13 +190,6 @@ before_install:
 addons:
   postgresql: "9.3"
 
-install:
-  # Install prerequisites
-  - ./toolset/run-ci.py prereq "$TESTDIR"
-
-  # Install software for this framework
-  - ./toolset/run-ci.py install "$TESTDIR"
-
 script:
   # Pick one test in this directory and verify
   - time ./toolset/run-ci.py verify "$TESTDIR"

deployment/vagrant-common/bootstrap.sh

@@ -122,7 +122,6 @@ if [ ! -e "~/.firstboot" ]; then
     echo "Cloning project from $GH_REPO $GH_BRANCH"
     git clone -b ${GH_BRANCH} https://github.com/${GH_REPO}.git $FWROOT
   fi
-  sudo pip install -r $FWROOT/requirements.txt
 
   # Everyone gets SSH access to localhost
   echo "Setting up SSH access to localhost"

frameworks/Crystal/moonshine/server-redis.cr

@@ -14,9 +14,9 @@ app = App.new
 
 class CONTENT
   UTF8 = "; charset=UTF-8"
-  JSON = "application/json" + UTF8
+  JSON = "application/json" #+ UTF8
   PLAIN = "text/plain"
-  HTML = "text/html" + UTF8
+  HTML = "text/html" #+ UTF8
 end
 
 ID_MAXIMUM = 10_000

toolset/benchmark/framework_test.py

@@ -396,9 +396,9 @@ class FrameworkTest:
         logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)
         traceback.format_exc()
 
-      test.failed = any(result is 'fail' for (result, reason, url) in results)
-      test.warned = any(result is 'warn' for (result, reason, url) in results)
-      test.passed = all(result is 'pass' for (result, reason, url) in results)
+      test.failed = any(result == 'fail' for (result, reason, url) in results)
+      test.warned = any(result == 'warn' for (result, reason, url) in results)
+      test.passed = all(result == 'pass' for (result, reason, url) in results)
       
       def output_result(result, reason, url):
         specific_rules_url = "http://frameworkbenchmarks.readthedocs.org/en/latest/Project-Information/Framework-Tests/#specific-test-requirements"
@@ -852,165 +852,175 @@ class FrameworkTest:
 # End FrameworkTest
 ############################################################
 
-##########################################################################################
+
 # Static methods
-##########################################################################################
 
-##############################################################
-# parse_config(config, directory, benchmarker)
-# parses a config file and returns a list of FrameworkTest
-# objects based on that config file.
-##############################################################
+def test_order(type_name):
+  """
+  This sort ordering is set up specifically to return the length
+  of the test name. There were SO many problems involved with
+  'plaintext' being run first (rather, just not last) that we
+  needed to ensure that it was run last for every framework.
+  """
+  return len(type_name)
+
+
+def validate_urls(test_name, test_keys):
+  """
+  Separated from validate_test because urls are not required anywhere. We know a url is incorrect if it is
+  empty or does not start with a "/" character. There is no validation done to ensure the url conforms to
+  the suggested url specifications, although those suggestions are presented if a url fails validation here.
+  """
+  example_urls = {
+    "json_url":      "/json",
+    "db_url":        "/mysql/db",
+    "query_url":     "/mysql/queries?queries=  or  /mysql/queries/",
+    "fortune_url":   "/mysql/fortunes",
+    "update_url":    "/mysql/updates?queries=  or  /mysql/updates/",
+    "plaintext_url": "/plaintext"
+  }
+  for test_url in ["json_url","db_url","query_url","fortune_url","update_url","plaintext_url"]:
+    key_value = test_keys.get(test_url, None)
+    if key_value != None and not key_value.startswith('/'):
+      errmsg = """`%s` field in test \"%s\" does not appear to be a valid url: \"%s\"\n
+        Example `%s` url: \"%s\"
+      """ % (test_url, test_name, key_value, test_url, example_urls[test_url])
+      raise Exception(errmsg)
+ 
+
+def validate_test(test_name, test_keys, directory):
+  """
+  Validate benchmark config values for this test based on a schema
+  """
+  # Ensure that each FrameworkTest has a framework property, inheriting from top-level if not
+  if not test_keys['framework']:
+    test_keys['framework'] = config['framework']
+
+  recommended_lang = directory.split('/')[-2]
+  windows_url = "https://github.com/TechEmpower/FrameworkBenchmarks/milestones/Windows%%20Compatibility"
+  schema = {
+    'language': {
+      'help': ('language', 'The language of the framework used, suggestion: %s' % recommended_lang)
+    },
+    'webserver': {
+      'help': ('webserver', 'Name of the webserver also referred to as the "front-end server"')
+    },
+    'classification': {
+      'allowed': [
+        ('Fullstack', '...'),
+        ('Micro', '...'),
+        ('Platform', '...')
+      ]
+    },
+    'database': {
+      'allowed': [
+        ('MySQL', 'One of the most popular databases around the web and in TFB'),
+        ('Postgres', 'An advanced SQL database with a larger feature set than MySQL'),
+        ('MongoDB', 'A popular document-store database'),
+        ('Cassandra', 'A highly performant and scalable NoSQL database'),
+        ('Elasticsearch', 'A distributed RESTful search engine that is used as a database for TFB tests'),
+        ('Redis', 'An open-sourced, BSD licensed, advanced key-value cache and store'),
+        ('SQLite', 'A network-less database, still supported for backwards compatibility'),
+        ('SQLServer', 'Microsoft\'s SQL implementation'),
+        ('None', 'No database was used for these tests, as is the case with Json Serialization and Plaintext')
+      ]
+    },
+    'approach': {
+      'allowed': [
+        ('Realistic', '...'),
+        ('Stripped', '...')
+      ]
+    },
+    'orm': {
+      'allowed': [
+        ('Full', 'Has a full suite of features like lazy loading, caching, multiple language support, sometimes pre-configured with scripts.'),
+        ('Micro', 'Has basic database driver capabilities such as establishing a connection and sending queries.'),
+        ('Raw', 'Tests that do not use an ORM will be classified as "raw" meaning they use the platform\'s raw database connectivity.')
+      ]
+    },
+    'platform': {
+      'help': ('platform', 'Name of the platform this framework runs on, e.g. Node.js, Pypy, hhvm, JRuby ...')
+    },
+    'framework': {
+      # Guranteed to be here and correct at this point
+      # key is left here to produce the set of required keys
+    },
+    'os': {
+      'allowed': [
+        ('Linux', 'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'),
+        ('Windows', 'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s' % windows_url)
+      ]
+    },
+    'database_os': {
+      'allowed': [
+        ('Linux', 'Our best-supported host OS, it is recommended that you build your tests for Linux hosts'),
+        ('Windows', 'TFB is not fully-compatible on windows, contribute towards our work on compatibility: %s' % windows_url)
+      ]
+    }
+  }
+
+  # Confirm required keys are present
+  required_keys = schema.keys()
+  missing = list(set(required_keys) - set(test_keys))
+
+  if len(missing) > 0:
+    missingstr = (", ").join(map(str, missing))
+    raise Exception("benchmark_config.json for test %s is invalid, please amend by adding the following required keys: [%s]"
+      % (test_name, missingstr))
+
+  # Check the (all optional) test urls
+  validate_urls(test_name, test_keys)
+
+  # Check values of keys against schema
+  for key in required_keys:
+    val = test_keys.get(key, "").lower()
+    has_predefined_acceptables = 'allowed' in schema[key]
+
+    if has_predefined_acceptables:
+      allowed = schema[key].get('allowed', [])
+      acceptable_values, descriptors = zip(*allowed)
+      acceptable_values = [a.lower() for a in acceptable_values]
+      
+      if val not in acceptable_values:
+        msg = ("Invalid `%s` value specified for test \"%s\" in framework \"%s\"; suggestions:\n"
+          % (key, test_name, config['framework']))
+        helpinfo = ('\n').join(["  `%s` -- %s" % (v, desc) for (v, desc) in zip(acceptable_values, descriptors)])
+        fullerr = msg + helpinfo + "\n"
+        raise Exception(fullerr)
+    
+    elif not has_predefined_acceptables and val == "":
+      msg = ("Value for `%s` in test \"%s\" in framework \"%s\" was missing:\n"
+        % (key, test_name, config['framework']))
+      helpinfo = "  %s -- %s" % schema[key]['help']
+      fullerr = msg + helpinfo + '\n'
+      raise Exception(fullerr)
+
 def parse_config(config, directory, benchmarker):
+  """
+  Parses a config file into a list of FrameworkTest objects
+  """
   tests = []
 
-  # This sort ordering is set up specifically to return the length
-  # of the test name. There were SO many problems involved with
-  # 'plaintext' being run first (rather, just not last) that we
-  # needed to ensure that it was run last for every framework.
-  def testOrder(type_name):
-    return len(type_name)
-
   # The config object can specify multiple tests
-  #   Loop over them and parse each into a FrameworkTest
+  # Loop over them and parse each into a FrameworkTest
   for test in config['tests']:
 
-    names = [name for (name,keys) in test.iteritems()]
-    if "default" not in names:
+    tests_to_run = [name for (name,keys) in test.iteritems()]
+    if "default" not in tests_to_run:
       logging.warn("Framework %s does not define a default test in benchmark_config.json", config['framework'])
     
     # Check that each test configuration is acceptable
     # Throw exceptions if a field is missing, or how to improve the field
     for test_name, test_keys in test.iteritems():
-      # Ensure that each FrameworkTest has a framework property, inheriting from top-level if not
-      if not test_keys['framework']:
-        test_keys['framework'] = config['framework']
-
-      # Confirm required keys are present
-      required_keys = ['language','webserver','classification','database','approach','orm','framework','os','database_os']
-      missing = list(set(required_keys) - set(test_keys))
-      if len(missing) > 0:
-        missingstr = (", ").join(map(str, missing))
-        raise Exception("benchmark_config.json for test %s is invalid, please amend and add the following required keys: [%s]"
-          % (test_name, missingstr))
-      
-      # Check that test url values are all appropriate
-      example_urls = {
-        "json_url":      "/json",
-        "db_url":        "/mysql/db",
-        "query_url":     "/mysql/queries?queries=  or  /mysql/queries/",
-        "fortune_url":   "/mysql/fortunes",
-        "update_url":    "/mysql/updates?queries=  or  /mysql/updates/",
-        "plaintext_url": "/plaintext"
-      }
-      for test_url in ["json_url","db_url","query_url","fortune_url","update_url","plaintext_url"]:
-        key_value = test_keys.get(test_url, None)
-        if key_value != None and not key_value.startswith('/'):
-          errmsg = """`%s` field in test \"%s\" does not appear to be a valid url: \"%s\"\n
-            Example `%s` url: \"%s\"
-          """ % (test_url, test_name, key_value, test_url, example_urls[test_url])
-          raise Exception(errmsg)
-
-      # Check database type
-      # List adopted from run-ci.py
-      SUPPORTED_DATABASES = ["mysql","postgres","mongodb","cassandra","elasticsearch","redis"]
-      EDGE_CASES = ["sqlite","sqlserver","none"]
-      db_type = test_keys.get("database", None).lower()
-
-      if db_type.lower() not in sum([SUPPORTED_DATABASES, EDGE_CASES], []):
-        supportedstr = (", ").join(map(str, SUPPORTED_DATABASES))
-        edgestr = (", ").join(map(str, EDGE_CASES))
-
-        errmsg = """Invalid db specified for test \"%s\" in framework \"%s\", please specify a supported database or \"None\"\n
-           Supported databases: [%s]\n
-           Edge cases: [%s]\n
-           Supplied (lowercased): \"%s\"
-        """ % (test_name, config["framework"], supportedstr, edgestr, db_type)
-        raise Exception(errmsg)
-
-      # Check language
-      # "Scala" from "/home/vagrant/FrameworkBenchmarks/frameworks/Scala/finagle"
-      recommended_lang = directory.split('/')[-2]
-      
-      if test_keys.get("language", "") == "":
-        raise Exception("Please specify a language for test \"%s\" in framework \"%s\", suggestion: \"%s\""
-          % (test_name, config["framework"], recommended_lang))
-
-      # Check approach
-      SUPPORTED_APPROACHES = ["realistic","stripped"]
-      test_approach = test_keys.get("approach", None).lower()
-      if test_approach not in SUPPORTED_APPROACHES:
-        approachstr = (", ").join(map(str, SUPPORTED_APPROACHES))
-
-        errmsg = """Invalid approach specified for test \"%s\" in framework \"%s\", please specify a supported approach\n
-           Supported approaches: [%s]\n
-           Suggestion: \"Realistic\"\n
-           Supplied (lowercased): \"%s\"
-        """ % (test_name, config["framework"], approachstr, test_approach)
-        raise Exception(errmsg)
-
-      # Check classification
-      SUPPORTED_CLASSIFICATIONS = ["fullstack","micro","platform"]
-      test_classification = test_keys.get("classification", None).lower()
-      if test_classification not in SUPPORTED_CLASSIFICATIONS:
-        classstr = (", ").join(map(str, SUPPORTED_CLASSIFICATIONS))
-
-        errmsg = """Invalid classification specified for test \"%s\" in framework \"%s\", please specify a supported classification\n
-           Supported classifications: [%s]\n
-           Supplied (lowercased): \"%s\"
-        """ % (test_name, config["framework"], classstr, test_classification)
-        raise Exception(errmsg)
-
-      # Check webserver
-      if test_keys.get("webserver", None) == "":
-        raise Exception("Invalid `webserver` specified for test \"%s\" in framework \"%s\", field `webserver` cannot be empty"
-          % (test_name, config["framework"]))
-
-      # Check ORM
-      SUPPORTED_ORMS = ["full","micro","raw"]
-      test_orm = test_keys.get("orm", None).lower()
-      if test_orm not in SUPPORTED_ORMS:
-        ormstr = (", ").join(map(str, SUPPORTED_ORMS))
-
-        errmsg = """Invalid orm specified for test \"%s\" in framework \"%s\", please specify a supported orm type\n
-           Supported classifications: [%s]\n
-           Supplied (lowercased): \"%s\"
-        """ % (test_name, config["framework"], ormstr, test_orm)
-        raise Exception(errmsg)
-
-      # Check OS
-      SUPPORTED_OSES = ["linux","windows"]
-      test_os = test_keys.get("os", None).lower()
-      if test_os not in SUPPORTED_OSES:
-        osstr = (", ").join(map(str, SUPPORTED_OSES))
-
-        errmsg = """Invalid OS specified for test \"%s\" in framework \"%s\", please specify a supported OS\n
-           Supported OS's: [%s]\n
-           Suggestion: \"Linux\"\n
-           Supplied (lowercased): \"%s\"
-        """ % (test_name, config["framework"], osstr, test_os)
-        raise Exception(errmsg)
-
-      # Check Database OS
-      SUPPORTED_DB_OSES = ["linux","windows"]
-      test_db_os = test_keys.get("database_os", None).lower()
-      if test_db_os not in SUPPORTED_DB_OSES:
-        db_osstr = (", ").join(map(str, SUPPORTED_DB_OSES))
-
-        errmsg = """Invalid Database OS specified for test \"%s\" in framework \"%s\", please specify a supported Database OS\n
-           Supported OS's: [%s]\n
-           Suggestion: \"Linux\"\n
-           Supplied (lowercased): \"%s\"
-        """ % (test_name, config["framework"], db_osstr, test_db_os)
-        raise Exception(errmsg)
-
-      ### Done validating benchmark_config values ###
-
+      # Validates the benchmark_config entry
+      validate_test(test_name, test_keys, directory)
       
       # Map test type to a parsed FrameworkTestType object
       runTests = dict()
       for type_name, type_obj in benchmarker.types.iteritems():
         try:
+          # Makes a FrameWorkTestType object using some of the keys in config
+          # e.g. JsonTestType uses "json_url"
           runTests[type_name] = type_obj.copy().parse(test_keys)
         except AttributeError as ae:
           # This is quite common - most tests don't support all types
@@ -1020,7 +1030,7 @@ def parse_config(config, directory, benchmarker):
           pass
 
       # We need to sort by test_type to run
-      sortedTestKeys = sorted(runTests.keys(), key=testOrder)
+      sortedTestKeys = sorted(runTests.keys(), key=test_order)
       sortedRunTests = OrderedDict()
       for sortedTestKey in sortedTestKeys:
         sortedRunTests[sortedTestKey] = runTests[sortedTestKey]
@@ -1036,6 +1046,3 @@ def parse_config(config, directory, benchmarker):
       tests.append(FrameworkTest(test_name, directory, benchmarker, sortedRunTests, test_keys))
 
   return tests
-##############################################################
-# End parse_config
-##############################################################

toolset/benchmark/test_types/db_type.py

@@ -1,117 +1,55 @@
 from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object
 
 import json
 
-class DBTestType(FrameworkTestType):
-  def __init__(self):
-    args = ['db_url']
-    FrameworkTestType.__init__(self, name='db', 
-      accept_header=self.accept_json,
-      requires_db=True, args=args)
-
-  def get_url(self):
-    return self.db_url
-
-  def verify(self, base_url):
-    '''Ensures body is valid JSON with a key 'id' and a key 
-    'randomNumber', both of which must map to integers
-    '''
-
-    url = base_url + self.db_url
-    full_response = self._curl(url)
-    body = self._curl_body(url)
-    
-    # Empty response
-    if body is None:
-      return [('fail','No response', url)]
-    elif len(body) == 0:
-      return [('fail','Empty Response', url)]
-
-    # Valid JSON? 
-    try: 
-      response = json.loads(body)
-    except ValueError as ve:
-      return [('fail',"Invalid JSON - %s" % ve, url)]
-
-    problems = []
-
-    # We are allowing the single-object array 
-    # e.g. [{'id':5, 'randomNumber':10}] for now, 
-    # but will likely make this fail at some point
-    if type(response) == list:
-      response = response[0]
-      problems.append( ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url) ) 
-
-      # Make sure there was a JSON object inside the array
-      if type(response) != dict:
-        problems.append( ('fail', 'Response is not a JSON object or an array of JSON objects', url) ) 
-        return problems
-
-    problems += self._verifyObject(response, url)
-
-    # Ensure required response headers are present
-    if any(v.lower() not in full_response.lower() for v in ('Server','Date','Content-Type: application/json')) \
-       or all(v.lower() not in full_response.lower() for v in ('Content-Length','Transfer-Encoding')):
-      problems.append( ('warn','Required response header missing.',url) )
-
-    if len(problems) == 0:
-      return [('pass','',url)]
-    else:
-      return problems
-
-  def _verifyObject(self, db_object, url, max_infraction='fail'):
-    '''Ensure the passed item is a JSON object with 
-    keys 'id' and 'randomNumber' mapping to ints. 
-    Separate method allows the QueryTestType to 
-    reuse these checks'''
 
-    problems = []
-
-    if type(db_object) != dict:
-      got = str(db_object)[:20]
-      if len(str(db_object)) > 20:
-        got = str(db_object)[:17] + '...'
-      return [(max_infraction, "Expected a JSON object, got '%s' instead" % got, url)]
-
-    # Make keys case insensitive
-    db_object = {k.lower(): v for k,v in db_object.iteritems()}
-
-    if "id" not in db_object:
-      problems.append( (max_infraction, "Response has no 'id' key", url) ) 
-    if "randomnumber" not in db_object:
-      problems.append( (max_infraction, "Response has no 'randomNumber' key", url) )
-    
-    # Ensure we can continue on to use these keys
-    if "id" not in db_object or "randomnumber" not in db_object:
-      return problems
-
-    try:
-      float(db_object["id"])
-    except ValueError as ve:
-      problems.append( (max_infraction, "Response key 'id' does not map to a number - %s" % ve, url) ) 
-
-    try:
-      float(db_object["randomnumber"])
-    except ValueError as ve:
-      problems.append( (max_infraction, "Response key 'randomNumber' does not map to a number - %s" % ve, url) ) 
-
-    if type(db_object["id"]) != int:
-      problems.append( ('warn', '''Response key 'id' contains extra quotations or decimal points.
-        This may negatively affect performance during benchmarking''', url) ) 
-
-    # Tests based on the value of the numbers
-    try:
-      response_id = float(db_object["id"])
-      response_rn = float(db_object["randomnumber"])
-
-      if response_id > 10000 or response_id < 1:
-        problems.append( ('warn', "Response key 'id' should be between 1 and 10,000", url) ) 
-
-      if response_rn > 10000:
-        problems.append( ('warn', '''Response key 'randomNumber' is over 10,000. This may negatively 
-          afect performance by sending extra bytes.''', url) )
-    except ValueError:
-      pass
-
-    return problems
+class DBTestType(FrameworkTestType):
 
+    def __init__(self):
+        kwargs = {
+            'name': 'db',
+            'accept_header': self.accept('json'),
+            'requires_db': True,
+            'args': ['db_url']
+        }
+        FrameworkTestType.__init__(self, **kwargs)
+
+    def get_url(self):
+        return self.db_url
+
+    def verify(self, base_url):
+        '''Ensures body is valid JSON with a key 'id' and a key 
+        'randomNumber', both of which must map to integers
+        '''
+
+        url = base_url + self.db_url
+        headers, body = self.request_headers_and_body(url)
+
+        response, problems = basic_body_verification(body)
+
+        if len(problems) > 0:
+            return problems 
+
+        # We are allowing the single-object array
+        # e.g. [{'id':5, 'randomNumber':10}] for now,
+        # but will likely make this fail at some point
+        if type(response) == list:
+            response = response[0]
+            problems.append(
+                ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url))
+
+            # Make sure there was a JSON object inside the array
+            if type(response) != dict:
+                problems.append(
+                    ('fail', 'Response is not a JSON object or an array of JSON objects', url))
+                return problems
+
+        # Verify response content
+        problems += verify_randomnumber_object(response, url)
+        problems += verify_headers(headers, url, should_be='json')
+
+        if len(problems) == 0:
+            return [('pass', '', url)]
+        else:
+            return problems

toolset/benchmark/test_types/fortune_type.py

@@ -1,71 +1,89 @@
 from benchmark.test_types.framework_test_type import FrameworkTestType
 from benchmark.fortune_html_parser import FortuneHTMLParser
+from benchmark.test_types.verifications import basic_body_verification, verify_headers
+
 
 class FortuneTestType(FrameworkTestType):
-  def __init__(self):
-    args = ['fortune_url']
-    FrameworkTestType.__init__(self, name='fortune', 
-      requires_db=True, 
-      accept_header=self.accept_html, 
-      args=args)
-
-  def get_url(self):
-    return self.fortune_url
-
-  def verify(self, base_url):
-    '''Parses the given HTML string and asks the 
-    FortuneHTMLParser whether the parsed string is a 
-    valid fortune response
-    '''
-    url = base_url + self.fortune_url
-    full_response = self._curl(url)
-    body = self._curl_body(url)
-    
-    # Empty response
-    if body is None:
-      return [('fail','No response', url)]
-    elif len(body) == 0:
-      return [('fail','Empty Response', url)]
-
-    parser = FortuneHTMLParser()
-    parser.feed(body)
-    (valid, diff) = parser.isValidFortune(self.out)
-    if valid:
-      # Ensure required response headers are present
-      if any(v.lower() not in full_response.lower() for v in ('Server','Date','Content-Type: text/html')) \
-         or all(v.lower() not in full_response.lower() for v in ('Content-Length','Transfer-Encoding')):
-        return[('warn','Required response header missing.',url)]
-
-      return [('pass','',url)]
-    else:
-      failures = [('fail','Invalid according to FortuneHTMLParser',url)]
-      # Catch exceptions because we are relying on internal code
-      try:
-        # Parsing this: 
-        # --- Valid
-        # +++ Response
-        # @@ -1 +1 @@
-        #
-        # -<!doctype html><html><head><title>Fortunes</title></head><body><table>
-        # +<!doctype html><html><head><meta></meta><title>Fortunes</title></head><body><div><table>
-        # @@ -16 +16 @@
-        
-        current_neg = []
-        current_pos = []
-        for line in diff[3:]:
-          if line[0] == '+':
-            current_neg.append(line[1:])
-          elif line[0] == '-':
-            current_pos.append(line[1:])
-          elif line[0] == '@':
-            failures.append( ('fail', 
-              "`%s` should be `%s`" % (''.join(current_neg), ''.join(current_pos)),
-              url) )
-        if len(current_pos) != 0:
-          failures.append( ('fail', 
-              "`%s` should be `%s`" % (''.join(current_neg), ''.join(current_pos)),
-              url) )
-      except: 
-        pass
-      return failures 
 
+    def __init__(self):
+        kwargs = {
+            'name': 'fortune',
+            'accept_header': self.accept('html'),
+            'requires_db': True,
+            'args': ['fortune_url']
+        }
+        FrameworkTestType.__init__(self, **kwargs)
+
+    def get_url(self):
+        return self.fortune_url
+
+    def verify(self, base_url):
+        '''Parses the given HTML string and asks the 
+        FortuneHTMLParser whether the parsed string is a 
+        valid fortune response
+        '''
+
+        url = base_url + self.fortune_url
+        headers, body = self.request_headers_and_body(url)
+
+        _, problems = basic_body_verification(body, is_json_check=False)
+
+        if len(problems) > 0:
+            return problems
+
+        parser = FortuneHTMLParser()
+        parser.feed(body)
+        (valid, diff) = parser.isValidFortune(self.out)
+
+        if valid:
+            problems += verify_headers(headers, url, should_be='html')
+
+            if len(problems) == 0:
+                return [('pass', '', url)]
+            else:
+                return problems
+        else:
+            failures = []
+            failures.append(
+                ('fail', 'Invalid according to FortuneHTMLParser', url))
+            failures += self._parseDiffForFailure(diff, failures, url)
+            return failures
+
+    def _parseDiffForFailure(self, diff, failures, url):
+        '''Example diff:
+
+        --- Valid
+        +++ Response
+        @@ -1 +1 @@
+
+        -<!doctype html><html><head><title>Fortunes</title></head><body><table>
+        +<!doctype html><html><head><meta></meta><title>Fortunes</title></head><body><div><table>
+        @@ -16 +16 @@
+        '''
+
+        problems = []
+
+        # Catch exceptions because we are relying on internal code
+        try:
+            current_neg = []
+            current_pos = []
+            for line in diff[3:]:
+                if line[0] == '+':
+                    current_neg.append(line[1:])
+                elif line[0] == '-':
+                    current_pos.append(line[1:])
+                elif line[0] == '@':
+                    problems.append(('fail',
+                                     "`%s` should be `%s`" % (
+                                         ''.join(current_neg), ''.join(current_pos)),
+                                     url))
+            if len(current_pos) != 0:
+                problems.append(
+                    ('fail',
+                     "`%s` should be `%s`" % (
+                         ''.join(current_neg), ''.join(current_pos)),
+                     url))
+        except:
+            # If there were errors reading the diff, then no diff information
+            pass
+        return problems

toolset/benchmark/test_types/framework_test_type.py

@@ -2,117 +2,133 @@ import copy
 import sys
 import subprocess
 from subprocess import PIPE
+import requests
+
+# Requests is built ontop of urllib3,
+# here we prevent general request logging
+import logging
+logging.getLogger('urllib3').setLevel(logging.CRITICAL)
 
 from pprint import pprint
 
+
 class FrameworkTestType:
-  '''Interface between a test type (json, query, plaintext, etc) and 
-  the rest of TFB. A test type defines a number of keys it expects
-  to find in the benchmark_config.json, and this base class handles extracting
-  those keys and injecting them into the test. For example, if 
-  benchmark_config.json contains a line `"spam" : "foobar"` and a subclasses X
-  passes an argument list of ['spam'], then after parsing there will 
-  exist a member `X.spam = 'foobar'`. 
-  '''
-
-  accept_json = "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
-  accept_html = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
-  accept_plaintext = "Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
-
-  def __init__(self, name, requires_db = False, accept_header = None, args = []):
-    self.name = name
-    self.requires_db = requires_db
-    self.args = args
-    self.out = sys.stdout
-    self.err = sys.stderr
-    self.accept_header = accept_header
-    if accept_header is None:
-      self.accept_header = self.accept_plaintext
-
-    self.passed = None
-    self.failed = None
-    self.warned = None
-
-  def setup_out_err(self, out, err):
-    '''Sets up file-like objects for logging. Used in 
-    cases where it is hard just return the output. Any
-    output sent to these file objects is also printed to 
-    the console
-
-    NOTE: I detest this. It would be much better to use
-    logging like it's intended
-    '''
-    self.out = out
-    self.err = err
-  
-  def parse(self, test_keys):
-    '''Takes the dict of key/value pairs describing a FrameworkTest 
-    and collects all variables needed by this FrameworkTestType
-
-    Raises AttributeError if required keys are missing
+
     '''
-    if all (arg in test_keys for arg in self.args):
-      self.__dict__.update({ arg:test_keys[arg] for arg in self.args})
-      return self
-    else: # This is quite common - most tests don't support all types
-      raise AttributeError("A %s requires the benchmark_config.json to contain %s"%(self.name,self.args))
-
-  def _curl(self, url):
-    '''Downloads a URL and returns the HTTP response'''
-    # Use -m 15 to make curl stop trying after 15sec.
-    # Use -i to output response with headers
-    # Don't use -f so that the HTTP response code is ignored.
-    # Use -sS to hide progress bar, but show errors.
-    print "Accessing URL %s:" % url
-    self.err.write("Accessing URL %s \n" % url)
-    self.out.write("Accessing URL %s \n" % url)
-    p = subprocess.Popen(["curl", "-m", "15", "-i", "-sS", url], stderr=PIPE, stdout=PIPE)
-    (out, err) = p.communicate()
-    self.err.write(err+'\n')
-    self.out.write("Response: \n\"" + out.strip() + "\"\n")
-    if p.returncode != 0:
-      return None
-    return out
-
-  def _curl_body(self, url):
-    '''Downloads a URL and returns the HTTP body'''
-    # Use -m 15 to make curl stop trying after 15sec.
-    # Don't use -f so that the HTTP response code is ignored.
-    # Use -s to hide progress bar
-    # Get response body
-    p = subprocess.Popen(["curl", "-m", "15", "-s", url], stdout=PIPE, stderr=PIPE)
-    (out, err) = p.communicate()
-    print "  Response (trimmed to 40 bytes): \"%s\"" % out.strip().replace('\n','')[:40]
-    return out
-  
-  def verify(self, base_url):
-    '''Accesses URL used by this test type and checks the return 
-    values for correctness. Most test types run multiple checks,
-    so this returns a list of results. Each result is a 3-tuple
-    of (String result, String reason, String urlTested).
-
-    - result : 'pass','warn','fail'
-    - reason : Short human-readable reason if result was 
-        warn or fail. Please do not print the response as part of this, 
-        other parts of TFB will do that based upon the current logging 
-        settings if this method indicates a failure happened
-    - urlTested: The exact URL that was queried
-
-    Subclasses should make a best-effort attempt to report as many
-    failures and warnings as they can to help users avoid needing 
-    to run TFB repeatedly while debugging
+    Interface between a test type (json, query, plaintext, etc) and 
+    the rest of TFB. A test type defines a number of keys it expects
+    to find in the benchmark_config.json, and this base class handles extracting
+    those keys and injecting them into the test. For example, if 
+    benchmark_config.json contains a line `"spam" : "foobar"` and a subclasses X
+    passes an argument list of ['spam'], then after parsing there will 
+    exist a member `X.spam = 'foobar'`. 
     '''
-    # TODO make String result into an enum to enforce
-    raise NotImplementedError("Subclasses must provide verify")
 
-  def get_url(self):
-    '''Returns the URL for this test, like '/json'''
-    # This is a method because each test type uses a different key
-    # for their URL so the base class can't know which arg is the URL
-    raise NotImplementedError("Subclasses must provide verify")
+    def __init__(self, name, requires_db=False, accept_header=None, args=[]):
+        self.name = name
+        self.requires_db = requires_db
+        self.args = args
+        self.out = sys.stdout
+        self.err = sys.stderr
+
+        if accept_header is None:
+            self.accept_header = self.accept('json')
+        else:
+            self.accept_header = accept_header
+
+        self.passed = None
+        self.failed = None
+        self.warned = None
+
+    def accept(self, content_type):
+        return {
+            'json': 'application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7',
+            'html': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+            'plaintext': 'text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7'
+        }[content_type]
+
+    def setup_out_err(self, out, err):
+        '''
+        Sets up file-like objects for logging. Used in 
+        cases where it is hard just return the output. Any
+        output sent to these file objects is also printed to 
+        the console
+
+        NOTE: I detest this. It would be much better to use
+        logging like it's intended
+        '''
+        self.out = out
+        self.err = err
+
+    def parse(self, test_keys):
+        '''
+        Takes the dict of key/value pairs describing a FrameworkTest 
+        and collects all variables needed by this FrameworkTestType
+
+        Raises AttributeError if required keys are missing
+        '''
+        if all(arg in test_keys for arg in self.args):
+            self.__dict__.update({arg: test_keys[arg] for arg in self.args})
+            return self
+        else:  # This is quite common - most tests don't support all types
+            raise AttributeError(
+                "A %s requires the benchmark_config.json to contain %s" % (self.name, self.args))
+
+    def request_headers_and_body(self, url):
+        '''
+        Downloads a URL and returns the HTTP response headers
+        and body content as a tuple
+        '''
+        print "Accessing URL %s:" % url
+        self.err.write("Accessing URL %s \n" % url)
+        self.out.write("Accessing URL %s \n" % url)
+
+        headers = {'Accept': self.accept_header}
+        r = requests.get(url, timeout=15, headers=headers)
+
+        try:
+            r.raise_for_status()  # Throws on non-200
+            headers = r.headers
+            body = r.content
+            self.out.write(str(headers))
+            self.out.write(body)
+            b = 40
+            print "  Response (trimmed to %d bytes): \"%s\"" % (b, body.strip()[:b])
+            return headers, body
+        except requests.HTTPError as err:
+            self.err.write(err + '\n')
+            return None, None
+
+    def verify(self, base_url):
+        '''
+        Accesses URL used by this test type and checks the return 
+        values for correctness. Most test types run multiple checks,
+        so this returns a list of results. Each result is a 3-tuple
+        of (String result, String reason, String urlTested).
+
+        - result : 'pass','warn','fail'
+        - reason : Short human-readable reason if result was 
+            warn or fail. Please do not print the response as part of this, 
+            other parts of TFB will do that based upon the current logging 
+            settings if this method indicates a failure happened
+        - urlTested: The exact URL that was queried
+
+        Subclasses should make a best-effort attempt to report as many
+        failures and warnings as they can to help users avoid needing 
+        to run TFB repeatedly while debugging
+        '''
+        # TODO make String result into an enum to enforce
+        raise NotImplementedError("Subclasses must provide verify")
 
-  def copy(self):
-    '''Returns a copy that can be safely modified. Use before calling 
-    parse'''
-    return copy.copy(self)
+    def get_url(self):
+        '''Returns the URL for this test, like '/json'''
+        # This is a method because each test type uses a different key
+        # for their URL so the base class can't know which arg is the URL
+        raise NotImplementedError("Subclasses must provide verify")
 
+    def copy(self):
+        '''
+        Returns a copy that can be safely modified.
+        Use before calling parse
+        '''
+        return copy.copy(self)

toolset/benchmark/test_types/json_type.py

@@ -1,52 +1,45 @@
 from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.verifications import (
+    basic_body_verification,
+    verify_headers,
+    verify_helloworld_object
+)
 
 import json
 
+
 class JsonTestType(FrameworkTestType):
-  def __init__(self):
-    args = ['json_url']
-    FrameworkTestType.__init__(self, name='json', requires_db=False, accept_header=self.accept_json, args=args)
-
-  def get_url(self):
-    return self.json_url
-
-  def verify(self, base_url):
-    '''Validates the response is a JSON object of 
-    { 'message' : 'hello, world!' }. Case insensitive and 
-    quoting style is ignored
-    '''
-
-    url = base_url + self.json_url
-    full_response = self._curl(url)
-    body = self._curl_body(url)
-    
-    # Empty response
-    if body is None:
-      return [('fail','No response', url)]
-    elif len(body) == 0:
-      return [('fail','Empty Response', url)]
-
-    # Valid JSON? 
-    try: 
-      response = json.loads(body)
-    except ValueError as ve:
-      return [('fail',"Invalid JSON - %s" % ve, url)]
-    
-    # Make everything case insensitive
-    response = {k.lower(): v.lower() for k,v in response.iteritems()}
-
-    if "message" not in response:
-      return [('fail',"No JSON key 'message'", url)]
-
-    if len(response) != 1:
-      return [('warn',"Too many JSON key/value pairs, expected 1", url)]
-
-    if response['message'] != 'hello, world!':
-      return [('fail',"Expected message of 'hello, world!', got '%s'"%response['message'], url)]
-
-    # Ensure required response headers are present
-    if any(v.lower() not in full_response.lower() for v in ('Server','Date','Content-Type: application/json')) \
-       or all(v.lower() not in full_response.lower() for v in ('Content-Length','Transfer-Encoding')):
-      return [('warn','Required response header missing.',url)]
-
-    return [('pass','',url)]
+
+    def __init__(self):
+        kwargs = {
+            'name': 'json',
+            'accept_header': self.accept('json'),
+            'requires_db': False,
+            'args': ['json_url']
+        }
+        FrameworkTestType.__init__(self, **kwargs)
+
+    def get_url(self):
+        return self.json_url
+
+    def verify(self, base_url):
+        '''Validates the response is a JSON object of 
+        { 'message' : 'hello, world!' }. Case insensitive and 
+        quoting style is ignored
+        '''
+
+        url = base_url + self.json_url
+        headers, body = self.request_headers_and_body(url)
+
+        response, problems = basic_body_verification(body)
+
+        if len(problems) > 0:
+            return problems
+
+        problems += verify_helloworld_object(response, url)
+        problems += verify_headers(headers, url, should_be='json')
+
+        if len(problems) > 0:
+            return problems
+        else:
+            return [('pass', '', url)]

toolset/benchmark/test_types/plaintext_type.py

@@ -1,38 +1,50 @@
 from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.verifications import basic_body_verification, verify_headers
 
-class PlaintextTestType(FrameworkTestType):
-  def __init__(self):
-    args = ['plaintext_url']
-    FrameworkTestType.__init__(self, name='plaintext', requires_db=False, accept_header=self.accept_plaintext, args=args)
-
-  def verify(self, base_url):
-    url = base_url + self.plaintext_url
-    full_response = self._curl(url)
-    body = self._curl_body(url)
-
-    # Empty response
-    if body is None:
-      return [('fail','No response', url)]
-    elif len(body) == 0:
-      return [('fail','Empty Response', url)]
-
-    # Case insensitive
-    orig = body
-    body = body.lower()
 
-    if "hello, world!" not in body:
-      return [('fail', """Could not find 'Hello, World!' in response.""", url)]
-
-    if len("hello, world!") < len(body):
-      return [('warn', """Server is returning %s more bytes than are required.
-This may negatively affect benchmark performance.""" % (len(body) - len("hello, world!")), url)]
-
-    # Ensure required response headers are present
-    if any(v.lower() not in full_response.lower() for v in ('Server','Date','Content-Type: text/plain')) \
-       or all(v.lower() not in full_response.lower() for v in ('Content-Length','Transfer-Encoding')):
-      return [('warn','Required response header missing.',url)]
-
-    return [('pass', '', url)]
+class PlaintextTestType(FrameworkTestType):
 
-  def get_url(self):
-    return self.plaintext_url
+    def __init__(self):
+        kwargs = {
+            'name': 'plaintext',
+            'requires_db': False,
+            'accept_header': self.accept('plaintext'),
+            'args': ['plaintext_url']
+        }
+        FrameworkTestType.__init__(self, **kwargs)
+
+    def verify(self, base_url):
+        url = base_url + self.plaintext_url
+        headers, body = self.request_headers_and_body(url)
+
+        _, problems = basic_body_verification(body, is_json_check=False)
+
+        if len(problems) > 0:
+            return problems
+
+        # Case insensitive
+        orig = body
+        body = body.lower()
+        expected = "hello, world!"
+        extra_bytes = len(body) - len(expected)
+
+        if expected not in body:
+            return [('fail', "Could not find 'Hello, World!' in response.", url)]
+
+        if extra_bytes > 0:
+            problems.append(
+                ('warn',
+                 ("Server is returning %s more bytes than are required. "
+                  "This may negatively affect benchmark performance."
+                  % (extra_bytes)),
+                 url))
+
+        problems += verify_headers(headers, url, should_be='plaintext')
+
+        if len(problems) == 0:
+            return [('pass', '', url)]
+        else:
+            return problems
+
+    def get_url(self):
+        return self.plaintext_url

toolset/benchmark/test_types/query_type.py

@@ -1,112 +1,48 @@
 from benchmark.test_types.framework_test_type import FrameworkTestType
-from benchmark.test_types.db_type import DBTestType
+from benchmark.test_types.verifications import (
+    verify_headers,
+    verify_randomnumber_list,
+    verify_query_cases
+)
 
 import json
 
-class QueryTestType(DBTestType):
-  def __init__(self):
-    args = ['query_url']
-    FrameworkTestType.__init__(self, name='query', requires_db=True, 
-      accept_header=self.accept_json, args=args)
-
-  def get_url(self):
-    return self.query_url
-
-  def verify(self, base_url):
-    '''Validates the response is a JSON array of 
-    the proper length, each JSON Object in the array 
-    has keys 'id' and 'randomNumber', and these keys 
-    map to integers. Case insensitive and 
-    quoting style is ignored
-    '''
-
-    url = base_url + self.query_url
-
-    problems = []
-    
-    response = self._curl(url + '2')
-    body = self._curl_body(url + '2')
-    problems += self._verifyQueryList(2, response, body, url + '2')
-
-    response = self._curl(url + '0')
-    body = self._curl_body(url + '0')
-    problems += self._verifyQueryList(1, response, body, url + '0', 'warn')
-
-    # Note: A number of tests fail here because they only parse for 
-    # a number and crash on 'foo'. For now we only warn about this
-    response = self._curl(url + 'foo')
-    body = self._curl_body(url + 'foo')
-    if body is None:
-      problems += [('warn','No response (this will be a failure in future rounds, please fix)', url)]
-    elif len(body) == 0:
-      problems += [('warn','Empty response (this will be a failure in future rounds, please fix)', url)]
-    else:
-      problems += self._verifyQueryList(1, response, body, url + 'foo', 'warn')
-
-    response = self._curl(url + '501')
-    body = self._curl_body(url + '501')
-    problems += self._verifyQueryList(500, response, body, url + '501', 'warn')
-
-    if len(problems) == 0:
-      return [('pass','',url + '2'),
-              ('pass','',url + '0'),
-              ('pass','',url + 'foo'),
-              ('pass','',url + '501')]
-    else:
-      return problems
-
-  def _verifyQueryList(self, expectedLength, curlResponse, body, url, max_infraction='fail'):
-    '''Validates high-level structure (array length, object 
-      types, etc) before calling into DBTestType to 
-      validate a few individual JSON objects'''
-
-    # Empty response
-    if body is None:
-      return [(max_infraction,'No response', url)]
-    elif len(body) == 0:
-      return [(max_infraction,'Empty Response', url)]
-  
-    # Valid JSON? 
-    try: 
-      response = json.loads(body)
-    except ValueError as ve:
-      return [(max_infraction,"Invalid JSON - %s" % ve, url)]
-
-    problems = []
-
-    if type(response) != list:
-      problems.append(('warn','Top-level JSON is an object, not an array', url))
-
-      # Verify the one object they gave us before returning
-      problems += self._verifyObject(response, url, max_infraction)
-
-      return problems
-
-    if any(type(item) != dict for item in response):
-      problems.append((max_infraction,'All items JSON array must be JSON objects', url))
-
-    # For some edge cases we only warn
-    if len(response) != expectedLength:
-      problems.append((max_infraction,
-        "JSON array length of %s != expected length of %s" % (len(response), expectedLength), 
-        url))
-
-    # verify individual objects
-    maxBadObjects = 5
-    for item in response:
-      obj_ok = self._verifyObject(item, url, max_infraction)
-      if len(obj_ok) > 0:
-        maxBadObjects -=  1
-        problems += obj_ok
-        if maxBadObjects == 0:
-          break
-
-    # Ensure required response headers are present
-    if any(v.lower() not in curlResponse.lower() for v in ('Server','Date','Content-Type: application/json')) \
-       or all(v.lower() not in curlResponse.lower() for v in ('Content-Length','Transfer-Encoding')):
-      problems.append( ('warn','Required response header missing.',url) )
-
-    return problems
-
-
 
+class QueryTestType(FrameworkTestType):
+
+    def __init__(self):
+        kwargs = {
+            'name': 'query',
+            'accept_header': self.accept('json'),
+            'requires_db': True,
+            'args': ['query_url']
+        }
+        FrameworkTestType.__init__(self, **kwargs)
+
+    def get_url(self):
+        return self.query_url
+
+    def verify(self, base_url):
+        '''
+        Validates the response is a JSON array of 
+        the proper length, each JSON Object in the array 
+        has keys 'id' and 'randomNumber', and these keys 
+        map to integers. Case insensitive and 
+        quoting style is ignored
+        '''
+
+        url = base_url + self.query_url
+        cases = [
+            ('2',   'fail'),
+            ('0',   'warn'),
+            ('foo', 'warn'),
+            ('501', 'warn'),
+            ('',    'warn')
+        ]
+
+        problems = verify_query_cases(self, cases, url)
+
+        if len(problems) == 0:
+            return [('pass', '', url + case) for case, _ in cases]
+        else:
+            return problems

toolset/benchmark/test_types/update_type.py

@@ -1,50 +1,40 @@
 from benchmark.test_types.framework_test_type import FrameworkTestType
-from benchmark.test_types.query_type import QueryTestType
-
-from pprint import pprint
-
-class UpdateTestType(QueryTestType):
-  def __init__(self):
-    args = ['update_url']
-    FrameworkTestType.__init__(self, name='update', requires_db=True, 
-      accept_header=self.accept_json, args=args)
-
-  def get_url(self):
-    return self.update_url
-
-  def verify(self, base_url):
-    '''Validates the response is a JSON array of 
-    the proper length, each JSON Object in the array 
-    has keys 'id' and 'randomNumber', and these keys 
-    map to integers. Case insensitive and 
-    quoting style is ignored
-    '''
-
-    url = base_url + self.update_url
-    problems = []
-    
-    response = self._curl(url + '2')
-    body = self._curl_body(url + '2')
-    problems += self._verifyQueryList(2, response, body, url + '2')
-
-    response = self._curl(url + '0')
-    body = self._curl_body(url + '0')
-    problems += self._verifyQueryList(1, response, body, url + '0', 'warn')
-
-    response = self._curl(url + 'foo')
-    body = self._curl_body(url + 'foo')
-    problems += self._verifyQueryList(1, response, body, url + 'foo', 'warn')
-
-    response = self._curl(url + '501')
-    body = self._curl_body(url + '501')
-    problems += self._verifyQueryList(500, response, body, url + '501', 'warn')
-
-    if len(problems) == 0:
-      return [('pass','',url + '2'),
-              ('pass','',url + '0'),
-              ('pass','',url + 'foo'),
-              ('pass','',url + '501')]
-    else:
-      return problems
-
-
+from benchmark.test_types.verifications import verify_query_cases
+
+
+class UpdateTestType(FrameworkTestType):
+
+    def __init__(self):
+        kwargs = {
+            'name': 'update',
+            'accept_header': self.accept('json'),
+            'requires_db': True,
+            'args': ['update_url']
+        }
+        FrameworkTestType.__init__(self, **kwargs)
+
+    def get_url(self):
+        return self.update_url
+
+    def verify(self, base_url):
+        '''Validates the response is a JSON array of 
+        the proper length, each JSON Object in the array 
+        has keys 'id' and 'randomNumber', and these keys 
+        map to integers. Case insensitive and 
+        quoting style is ignored
+        '''
+
+        url = base_url + self.update_url
+        cases = [
+            ('2',   'fail'),
+            ('0',   'warn'),
+            ('foo', 'warn'),
+            ('501', 'warn'),
+            ('',    'warn')
+        ]
+        problems = verify_query_cases(self, cases, url)
+
+        if len(problems) == 0:
+            return [('pass', '', url + case) for (case, _) in cases]
+        else:
+            return problems

toolset/benchmark/test_types/verifications.py

@@ -0,0 +1,294 @@
+import json
+
+
+def basic_body_verification(body, is_json_check=True):
+    '''
+    Takes in a raw (stringy) response body, checks that it is non-empty,
+    and that it is valid JSON (i.e. can be deserialized into a dict/list of dicts)
+    Returns the deserialized body as a dict (or list of dicts), and also returns any
+    problems encountered, always as a list. If len(problems) > 0,
+    then the response body does not have to be examined further and the caller
+    should handle the failing problem(s).
+
+    Plaintext and Fortunes set `is_json_check` to False
+    '''
+
+    # Empty Response?
+    if body is None:
+        return None, [('fail', 'No response', url)]
+    elif len(body) == 0:
+        return None, [('fail', 'Empty response', url)]
+
+    # Valid JSON?
+    if is_json_check:
+        try:
+            response = json.loads(body)
+            return response, []
+        except ValueError as ve:
+            return None, [('fail', 'Invalid JSON: %s' % ve, url)]
+
+    # Fortunes and Plaintext only use this for the empty response tests
+    # they do not need or expect a dict back
+    return None, []
+
+
+def verify_headers(headers, url, should_be='json'):
+    '''
+    Verifies the headers of a framework response
+    param `should_be` is a switch for the three acceptable content types
+    '''
+
+    types = {
+        'json': 'application/json',
+        'html': 'text/html',
+        'plaintext': 'text/plain'
+    }
+    expected_type = types[should_be]
+    includes_charset = expected_type + '; charset=utf-8'
+
+    problems = []
+
+    if any(v.lower() not in headers for v in ('Server', 'Date', 'Content-Type')):
+        problems.append(
+            ('warn', 'Required response header missing: %s' % v, url))
+    elif all(v.lower() not in headers for v in ('Content-Length', 'Transfer-Encoding')):
+        problems.append(
+            ('warn',
+             'Required response size header missing, please include either "Content-Length" or "Transfer-Encoding"',
+             url))
+    else:
+        content_type = headers.get('Content-Type', None)
+
+        if content_type.lower() == includes_charset:
+            problems.append(
+                ('warn',
+                 ("Content encoding found \"%s\" where \"%s\" is acceptable.\n"
+                  "Additional response bytes may negatively affect benchmark performance."
+                  % (includes_charset, expected_type)),
+                 url))
+        elif content_type != expected_type:
+            problems.append(
+                ('warn',
+                 'Unexpected content encoding, found \"%s\", expected \"%s\"' % (
+                     content_type, expected_type),
+                 url))
+    return problems
+
+
+def verify_helloworld_object(json_object, url):
+    '''
+    Ensure that the JSON object closely resembles
+    { 'message': 'Hello, World!' }
+    '''
+
+    problems = []
+
+    # Make everything case insensitive
+    json_object = {k.lower(): v.lower()
+                   for k, v in json_object.iteritems()}
+
+    if 'message' not in json_object:
+        return [('fail', "Missing required key 'message'", url)]
+    else:
+        if len(json_object) > 1:
+            additional = (', ').join(
+                [k for k in json_object.keys() if k != 'message'])
+            problems.append(
+                ('warn', "Too many JSON key/value pairs, consider removing: %s" % additional, url))
+
+        message = json_object['message']
+
+        if message != 'hello, world!':
+            return [('fail', "Expected message of 'hello, world!', got '%s'" % message)]
+        return problems
+
+
+def verify_randomnumber_object(db_object, url, max_infraction='fail'):
+    '''
+    Ensures that `db_object` is a JSON object with 
+    keys 'id' and 'randomNumber' that both map to ints. 
+    Should closely resemble:
+    { "id": 2354, "randomNumber": 8952 }
+    '''
+
+    problems = []
+
+    # Dict is expected
+    # Produce error for bytes in non-cases
+    if type(db_object) is not dict:
+        got = str(db_object)[:20]
+        if len(str(db_object)) > 20:
+            got = str(db_object)[:17] + '...'
+        return [(max_infraction, "Expected a JSON object, got '%s' instead" % got, url)]
+
+    # Make keys case insensitive
+    db_object = {k.lower(): v for k, v in db_object.iteritems()}
+    required_keys = set(['id', 'randomnumber'])
+
+    if any(v not in db_object for v in required_keys):
+        problems.append(
+            (max_infraction, 'Response object was missing required key: %s' % v, url))
+
+    if len(db_object) > len(required_keys):
+        extras = db_object.keys() - required_keys
+        problems.append(
+            ('warn',
+             'An extra key(s) is being included with the db object: %s' % ', '.join(
+                 extras),
+             url))
+
+    # All required keys must be present
+    if len(problems) > 0:
+        return problems
+
+    # Assert key types and values
+    try:
+        o_id = int(db_object['id'])
+
+        if o_id > 10000 or o_id < 1:
+            problems.append(
+                ('warn',
+                 'Response key id should be between 1 and 10,000: ' +
+                 str(o_id),
+                 url))
+    except TypeError as e:
+        problems.append(
+            (max_infraction, "Response key 'id' does not map to an integer - %s" % e, url))
+
+    try:
+        o_rn = int(db_object['randomnumber'])
+
+        if o_rn > 10000:
+            problems.append(
+                ('warn',
+                 'Response key `randomNumber` is over 10,000. This may negatively affect performance by sending extra bytes',
+                 url))
+    except TypeError as e:
+        problems.append(
+            (max_infraction, "Response key 'randomnumber' does not map to an integer - %s" % e, url))
+
+    return problems
+
+
+def verify_randomnumber_list(expected_len, headers, body, url, max_infraction='fail'):
+    '''
+    Validates that the object is a list containing a number of 
+    randomnumber object. Should closely resemble:
+    [{ "id": 2354, "randomNumber": 8952 }, { "id": 4421, "randomNumber": 32 }, ... ]
+    '''
+    
+    response, problems = basic_body_verification(body)
+
+    if len(problems) > 0:
+        return problems
+
+    # This path will be hit when the framework returns a single JSON object
+    # rather than a list containing one element. We allow this with a warn,
+    # then verify the supplied object
+    if type(response) is not list:
+        problems.append(
+            ('warn', 'Top-level JSON is an object, not an array', url))
+        problems += verify_randomnumber_object(response, url, max_infraction)
+        return problems
+
+    if any(type(item) is not dict for item in response):
+        problems.append(
+            (max_infraction, 'Not all items in the JSON array were JSON objects', url))
+
+    if len(response) != expected_len:
+        problems.append(
+            (max_infraction,
+             "JSON array length of %s != expected length of %s" % (
+                 len(response), expected_len),
+             url))
+
+    # Verify individual objects, arbitrarily stop after 5 bad ones are found
+    # i.e. to not look at all 500
+    badObjectsFound = 0
+    inner_objects = iter(response)
+
+    try:
+        while badObjectsFound < 5:
+            obj = next(inner_objects)
+            findings = verify_randomnumber_object(obj, url, max_infraction)
+
+            if len(findings) > 0:
+                problems += findings
+                badObjectsFound += 1
+    except StopIteration:
+        pass
+
+    return problems
+
+
+def verify_query_cases(self, cases, url):
+    '''
+    The the /updates and /queries tests accept a `queries` parameter
+    that is expected to be between 1-500.
+    This method execises a framework with different `queries` parameter values
+    then verifies that the framework responds appropriately.
+    The `cases` parameter should be a list of 2-tuples containing the query case
+    and the consequence level should the cases fail its verifications, e.g.:
+
+    cases = [
+        ('2',   'fail'),
+        ('0',   'warn'),
+        ('foo', 'warn'),
+        ('501', 'warn'),
+        ('',    'warn')
+    ]
+
+    The reason for using 'warn' is generally for a case that will be allowed in the
+    current run but that may/will be a failing case in future rounds. The cases above
+    suggest that not sanitizing the `queries` parameter against non-int input, or failing
+    to ensure the parameter is between 1-500 will just be a warn,
+    and not prevent the framework from being benchmarked.
+    '''
+    problems = []
+    MAX = 500
+    MIN = 1
+
+    for q, max_infraction in cases:
+        case_url = url + q
+        headers, body = self.request_headers_and_body(case_url)
+
+        try:
+            queries = int(q)  # drops down for 'foo' and ''
+
+            if queries > MAX:
+                expected_len = MAX
+            elif queries < MIN:
+                expected_len = MIN
+            else:
+                expected_len = queries
+
+            problems += verify_randomnumber_list(
+                expected_len, headers, body, case_url, max_infraction)
+            problems += verify_headers(headers, case_url)
+
+        except ValueError:
+            warning = (
+                '%s given for stringy `queries` parameter %s\n'
+                'Suggestion: modify your /queries route to handle this case '
+                '(this will be a failure in future rounds, please fix)')
+
+            if body is None:
+                problems.append(
+                    (max_infraction,
+                     warning % ('No response', q),
+                     case_url))
+            elif len(body) == 0:
+                problems.append(
+                    (max_infraction,
+                     warning % ('Empty response', q),
+                     case_url))
+            else:
+                expected_len = 1
+                # Strictness will be upped in a future round, i.e. Frameworks currently do not have
+                # to gracefully handle absent, or non-intlike `queries`
+                # parameter input
+                problems += verify_randomnumber_list(
+                    expected_len, headers, body, case_url, max_infraction)
+                problems += verify_headers(headers, case_url)
+
+    return problems

toolset/run-ci.py

@@ -38,7 +38,7 @@ class CIRunnner:
   
   def __init__(self, mode, testdir=None):
     '''
-    mode = [cisetup|prereq|install|verify] for what we want to do
+    mode = [cisetup|verify] for what we want to do
     testdir  = framework directory we are running
     '''
 
@@ -303,16 +303,8 @@ class CIRunnner:
       return 0
 
     names = ' '.join(self.names)
-    command = 'toolset/run-tests.py '
-    if self.mode == 'prereq':
-      command = command + "--install server --install-only --test '' --verbose"
-    elif self.mode == 'install':
-      command = command + "--install server --install-only --test %s" % names
-    elif self.mode == 'verify':
-      command = command + "--mode verify --test %s" % names
-    else:
-      log.critical('Unknown mode passed')
-      return 1
+    # Assume mode is verify
+    command = "toolset/run-tests.py --mode verify --test %s" % names
     
     # Run the command
     log.info("Running mode %s with commmand %s", self.mode, command)
@@ -483,7 +475,7 @@ class CIRunnner:
 if __name__ == "__main__":
   args = sys.argv[1:]
 
-  usage = '''Usage: toolset/run-ci.py [cisetup|prereq|install|verify] <framework-directory>
+  usage = '''Usage: toolset/run-ci.py [cisetup|verify] <framework-directory>
     
     run-ci.py selects one test from <framework-directory>/benchark_config, and 
     automates a number of calls into run-tests.py specific to the selected test. 
@@ -493,8 +485,6 @@ if __name__ == "__main__":
     The name of the selected test will be printed to standard output. 
 
     cisetup - configure the Travis-CI environment for our test suite
-    prereq  - trigger standard prerequisite installation
-    install - trigger server installation for the selected test_directory
     verify  - run a verification on the selected test using `--mode verify`
 
     run-ci.py expects to be run inside the Travis-CI build environment, and 
@@ -506,9 +496,7 @@ if __name__ == "__main__":
 
   mode = args[0]
   testdir = args[1]
-  if len(args) == 2 and (mode == "install" 
-    or mode == "verify"
-    or mode == 'prereq'
+  if len(args) == 2 and (mode == 'verify'
     or mode == 'cisetup'):
     runner = CIRunnner(mode, testdir)
   else:

toolset/setup/linux/prerequisites.sh

@@ -60,6 +60,7 @@ sudo pip install colorama==0.3.1
 # Version 2.3 has a nice Counter() and other features
 # but it requires —-allow-external and -—allow-unverified
 sudo pip install progressbar==2.2
+sudo pip install requests
 
 # Install gcc-4.8 and gcc-4.9
 sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y