Merge pull request #1129 from hamiltont/type-refactor

Toolset: Create FrameworkTestType

frameworks/Elixir/WeberFramework/benchmark_config

@@ -18,7 +18,6 @@
       "database_os": "Linux",
       "display_name": "weber",
       "notes": "",
-      "versus": "",
-      "skip": "false"
+      "versus": ""
   }}]
 }

frameworks/Go/revel-jet/benchmark_config

@@ -20,8 +20,7 @@
       "database_os": "Linux",
       "display_name": "revel-jet",
       "notes": "",
-      "versus": "go",
-      "skip": "true"
+      "versus": "go"
     }
   }]
 }

toolset/benchmark/benchmarker.py

@@ -2,6 +2,7 @@ from setup.linux.installer import Installer
 from setup.linux import setup_util
 
 from benchmark import framework_test
+from benchmark.test_types import *
 from utils import header
 from utils import gather_tests
 from utils import gather_frameworks
@@ -17,6 +18,7 @@ import sys
 import logging
 import socket
 import threading
+from pprint import pprint
 
 from multiprocessing import Process
 
@@ -495,15 +497,8 @@ class Benchmarker:
       pass
     with open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'out.txt'), 'w') as out, \
          open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'err.txt'), 'w') as err:
-      if hasattr(test, 'skip'):
-        if test.skip.lower() == "true":
-          out.write("Test {name} benchmark_config specifies to skip this test. Skipping.\n".format(name=test.name))
-          return exit_with_code(0)
 
       if test.os.lower() != self.os.lower() or test.database_os.lower() != self.database_os.lower():
-        # the operating system requirements of this test for the
-        # application server or the database server don't match
-        # our current environment
         out.write("OS or Database OS specified in benchmark_config does not match the current environment. Skipping.\n")
         return exit_with_code(0)
       
@@ -511,11 +506,6 @@ class Benchmarker:
       if self.exclude != None and test.name in self.exclude:
         out.write("Test {name} has been added to the excludes list. Skipping.\n".format(name=test.name))
         return exit_with_code(0)
-      
-      # If the test does not contain an implementation of the current test-type, skip it
-      if self.type != 'all' and not test.contains_type(self.type):
-        out.write("Test type {type} does not contain an implementation of the current test-type. Skipping.\n".format(type=self.type))
-        return exit_with_code(0)
 
       out.write("test.os.lower() = {os}  test.database_os.lower() = {dbos}\n".format(os=test.os.lower(),dbos=test.database_os.lower()))
       out.write("self.results['frameworks'] != None: {val}\n".format(val=str(self.results['frameworks'] != None)))
@@ -561,11 +551,13 @@ class Benchmarker:
           self.__write_intermediate_results(test.name,"<setup.py>#start() returned non-zero")
           return exit_with_code(1)
         
+        logging.info("Sleeping %s seconds to ensure framework is ready" % self.sleep)
         time.sleep(self.sleep)
 
         ##########################
         # Verify URLs
         ##########################
+        logging.info("Verifying framework URLs")
         passed_verify = test.verify_urls(out, err)
         out.flush()
         err.flush()
@@ -574,6 +566,7 @@ class Benchmarker:
         # Benchmark this test
         ##########################
         if self.mode == "benchmark":
+          logging.info("Benchmarking")
           out.write(header("Benchmarking %s" % test.name))
           out.flush()
           test.benchmark(out, err)
@@ -756,22 +749,29 @@ class Benchmarker:
     
     jsonResult = {}
     for framework, testlist in frameworks.iteritems():
+      if not os.path.exists(os.path.join(testlist[0].directory, "source_code")):
+        logging.warn("Cannot count lines of code for %s - no 'source_code' file", framework)
+        continue
+
       # Unfortunately the source_code files use lines like
       # ./cpoll_cppsp/www/fortune_old instead of 
       # ./www/fortune_old
       # so we have to back our working dir up one level
       wd = os.path.dirname(testlist[0].directory)
-
+      
       try:
         command = "cloc --list-file=%s/source_code --yaml" % testlist[0].directory
         # Find the last instance of the word 'code' in the yaml output. This should
         # be the line count for the sum of all listed files or just the line count
         # for the last file in the case where there's only one file listed.
         command = command + "| grep code | tail -1 | cut -d: -f 2"
+        logging.debug("Running \"%s\" (cwd=%s)", command, wd)
         lineCount = subprocess.check_output(command, cwd=wd, shell=True)
         jsonResult[framework] = int(lineCount)
       except subprocess.CalledProcessError:
         continue
+      except ValueError as ve:
+        logging.warn("Unable to get linecount for %s due to error '%s'", framework, ve)
     self.results['rawData']['slocCounts'] = jsonResult
   ############################################################
   # End __count_sloc
@@ -889,7 +889,25 @@ class Benchmarker:
   ############################################################
   def __init__(self, args):
     
+    # Map type strings to their objects
+    types = dict()
+    types['json'] = JsonTestType()
+    types['db'] = DBTestType()
+    types['query'] = QueryTestType()
+    types['fortune'] = FortuneTestType()
+    types['update'] = UpdateTestType()
+    types['plaintext'] = PlaintextTestType()
+
+    # Turn type into a map instead of a string
+    if args['type'] == 'all':
+        args['types'] = types
+    else:
+        args['types'] = { args['type'] : types[args['type']] }
+    del args['type']
+
     self.__dict__.update(args)
+    # pprint(self.__dict__)
+
     self.start_time = time.time()
     self.run_test_timeout_seconds = 3600
 

toolset/benchmark/test_types/__init__.py

@@ -0,0 +1,8 @@
+
+from framework_test_type import *
+from json_type import JsonTestType
+from plaintext_type import PlaintextTestType
+from db_type import DBTestType
+from query_type import QueryTestType
+from update_type import UpdateTestType
+from fortune_type import FortuneTestType

toolset/benchmark/test_types/db_type.py

@@ -0,0 +1,103 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+
+import json
+
+class DBTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['db_url']
+    FrameworkTestType.__init__(self, name='db', 
+      accept_header=self.accept_json,
+      requires_db=True, args=args)
+
+  def get_url(self):
+    return self.db_url
+
+  def verify(self, base_url):
+    '''Ensures body is valid JSON with a key 'id' and a key 
+    'randomNumber', both of which must map to integers
+    '''
+
+    url = base_url + self.db_url
+    body = self._curl(url)
+    
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+
+    problems = []
+
+    # We are allowing the single-object array 
+    # e.g. [{'id':5, 'randomNumber':10}] for now, 
+    # but will likely make this fail at some point
+    if type(response) == list:
+      response = response[0]
+      problems.append( ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url) ) 
+
+      # Make sure there was a JSON object inside the array
+      if type(response) != dict:
+        problems.append( ('fail', 'Response is not a JSON object or an array of JSON objects', url) ) 
+        return problems
+
+    problems += self._verifyObject(response, url)
+
+    if len(problems) == 0:
+      return [('pass','',url)]
+    else:
+      return problems
+
+  def _verifyObject(self, db_object, url):
+    '''Ensure the passed item is a JSON object with 
+    keys 'id' and 'randomNumber' mapping to ints. 
+    Separate method allows the QueryTestType to 
+    reuse these checks'''
+
+    problems = []
+
+    if type(db_object) != dict:
+      got = str(db_object)[:20]
+      if len(str(db_object)) > 20:
+        got = str(db_object)[:17] + '...'
+      return ('fail', "Expected a JSON object, got '%s' instead" % got, url)
+
+    # Make keys case insensitive
+    db_object = {k.lower(): v for k,v in db_object.iteritems()}
+
+    if "id" not in db_object:
+      problems.append( ('fail', "Response has no 'id' key", url) ) 
+    if "randomnumber" not in db_object:
+      problems.append( ('fail', "Response has no 'randomNumber' key", url) ) 
+
+    try:
+      float(db_object["id"])
+    except ValueError as ve:
+      problems.append( ('fail', "Response key 'id' does not map to a number - %s" % ve, url) ) 
+
+    try:
+      float(db_object["randomnumber"])
+    except ValueError as ve:
+      problems.append( ('fail', "Response key 'randomNumber' does not map to a number - %s" % ve, url) ) 
+
+    if type(db_object["id"]) != int:
+      problems.append( ('warn', '''Response key 'id' contains extra quotations or decimal points.
+        This may negatively affect performance during benchmarking''', url) ) 
+
+    # Tests based on the value of the numbers
+    try:
+      response_id = float(db_object["id"])
+      response_rn = float(db_object["randomnumber"])
+
+      if response_id > 10000 or response_id < 1:
+        problems.append( ('warn', "Response key 'id' should be between 1 and 10,000", url)) 
+    except ValueError:
+      pass
+
+    return problems
+

toolset/benchmark/test_types/fortune_type.py

@@ -0,0 +1,35 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.fortune_html_parser import FortuneHTMLParser
+
+class FortuneTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['fortune_url']
+    FrameworkTestType.__init__(self, name='fortune', 
+      requires_db=True, 
+      accept_header=self.accept_html, 
+      args=args)
+
+  def get_url(self):
+    return self.fortune_url
+
+  def verify(self, base_url):
+    '''Parses the given HTML string and asks the 
+    FortuneHTMLParser whether the parsed string is a 
+    valid fortune response
+    '''
+    url = base_url + self.fortune_url
+    body = self._curl(url)
+    
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+
+    parser = FortuneHTMLParser()
+    parser.feed(body)
+    if parser.isValidFortune(self.out):
+      return [('pass','',url)]
+    else:
+      return [('fail','Invalid according to FortuneHTMLParser',url)]
+

toolset/benchmark/test_types/framework_test_type.py

@@ -0,0 +1,110 @@
+import copy
+import sys
+import subprocess
+from subprocess import PIPE
+
+from pprint import pprint
+
+class FrameworkTestType:
+  '''Interface between a test type (json, query, plaintext, etc) and 
+  the rest of TFB. A test type defines a number of keys it expects
+  to find in the benchmark_config, and this base class handles extracting
+  those keys and injecting them into the test. For example, if 
+  benchmark_config contains a line `"spam" : "foobar"` and a subclasses X
+  passes an argument list of ['spam'], then after parsing there will 
+  exist a member `X.spam = 'foobar'`. 
+  '''
+
+  accept_json = "Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
+  accept_html = "Accept: text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"
+  accept_plaintext = "Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
+
+  def __init__(self, name, requires_db = False, accept_header = None, args = []):
+    self.name = name
+    self.requires_db = requires_db
+    self.args = args
+    self.out = sys.stdout
+    self.err = sys.stderr
+    self.accept_header = accept_header
+    if accept_header is None:
+      self.accept_header = self.accept_plaintext
+
+    self.passed = None
+    self.failed = None
+    self.warned = None
+
+  def setup_out_err(self, out, err):
+    '''Sets up file-like objects for logging. Used in 
+    cases where it is hard just return the output. Any
+    output sent to these file objects is also printed to 
+    the console
+
+    NOTE: I detest this. It would be much better to use
+    logging like it's intended
+    '''
+    self.out = out
+    self.err = err
+  
+  def parse(self, test_keys):
+    '''Takes the dict of key/value pairs describing a FrameworkTest 
+    and collects all variables needed by this FrameworkTestType
+
+    Raises AttributeError if required keys are missing
+    '''
+    if all (arg in test_keys for arg in self.args):
+      self.__dict__.update({ arg:test_keys[arg] for arg in self.args})
+      return self
+    else: # This is quite common - most tests don't support all types
+      raise AttributeError("A %s requires the benchmark_config to contain %s"%(self.name,self.args))
+
+  def _curl(self, url):
+    '''Downloads a URL and returns the HTTP body'''
+    # Use -m 15 to make curl stop trying after 15sec.
+    # Use -i to output response with headers
+    # Don't use -f so that the HTTP response code is ignored.
+    # Use -sS to hide progress bar, but show errors.
+    print "Accessing URL %s"
+    p = subprocess.Popen(["curl", "-m", "15", "-i", "-sS", url], stderr=PIPE, stdout=PIPE)
+    (out, err) = p.communicate()
+    self.err.write(err+'\n')
+    self.out.write(out+'\n')
+    if p.returncode != 0:
+      return None
+    # Get response body
+    p = subprocess.Popen(["curl", "-m", "15", "-s", url], stdout=PIPE, stderr=PIPE)
+    (out, err) = p.communicate()
+    self.err.write(err+'\n')
+    self.out.write(out+'\n')
+    return out
+  
+  def verify(self, base_url):
+    '''Accesses URL used by this test type and checks the return 
+    values for correctness. Most test types run multiple checks,
+    so this returns a list of results. Each result is a 3-tuple
+    of (String result, String reason, String urlTested).
+
+    - result : 'pass','warn','fail'
+    - reason : Short human-readable reason if result was 
+        warn or fail. Please do not print the response as part of this, 
+        other parts of TFB will do that based upon the current logging 
+        settings if this method indicates a failure happened
+    - urlTested: The exact URL that was queried
+
+    Subclasses should make a best-effort attempt to report as many
+    failures and warnings as they can to help users avoid needing 
+    to run TFB repeatedly while debugging
+    '''
+    # TODO make String result into an enum to enforce
+    raise NotImplementedError("Subclasses must provide verify")
+
+  def get_url(self):
+    '''Returns the URL for this test, like '/json'''
+    # This is a method because each test type uses a different key
+    # for their URL so the base class can't know which arg is the URL
+    raise NotImplementedError("Subclasses must provide verify")
+
+  def copy(self):
+    '''Returns a copy that can be safely modified. Use before calling 
+    parse'''
+    return copy.copy(self)
+

toolset/benchmark/test_types/json_type.py

@@ -0,0 +1,46 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+
+import json
+
+class JsonTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['json_url']
+    FrameworkTestType.__init__(self, name='json', requires_db=False, accept_header=self.accept_json, args=args)
+
+  def get_url(self):
+    return self.json_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON object of 
+    { 'message' : 'hello, world!' }. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.json_url
+    body = self._curl(url)
+    
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+  
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+    
+    # Make everything case insensitive
+    response = {k.lower(): v.lower() for k,v in response.iteritems()}
+
+    if "message" not in response:
+      return [('fail',"No JSON key 'message'", url)]
+
+    if len(response) != 1:
+      return [('warn',"Too many JSON key/value pairs, expected 1", url)]
+
+    if response['message'] != 'hello, world!':
+      return [('fail',"Expected message of 'hello, world!', got '%s'"%response['message'], url)]
+
+    return [('pass','',url)]

toolset/benchmark/test_types/plaintext_type.py

@@ -0,0 +1,32 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+
+class PlaintextTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['plaintext_url']
+    FrameworkTestType.__init__(self, name='plaintext', requires_db=False, accept_header=self.accept_plaintext, args=args)
+
+  def verify(self, base_url):
+    url = base_url + self.plaintext_url
+    body = self._curl(url)
+
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+
+    # Case insensitive
+    orig = body
+    body = body.lower()
+
+    if "hello, world!" not in body:
+      return [('fail', "Could not find 'Hello, World!' in response", url)]
+
+    if len("hello, world!") < len(body):
+      return [('warn', '''Server is returning more data than is required.
+        This may negatively affect benchmark performance''', url)]
+
+    return [('pass', '', url)]
+
+  def get_url(self):
+    return self.plaintext_url

toolset/benchmark/test_types/query_type.py

@@ -0,0 +1,96 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.db_type import DBTestType
+
+import json
+
+class QueryTestType(DBTestType):
+  def __init__(self):
+    args = ['query_url']
+    FrameworkTestType.__init__(self, name='query', requires_db=True, 
+      accept_header=self.accept_json, args=args)
+
+  def get_url(self):
+    return self.query_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON array of 
+    the proper length, each JSON Object in the array 
+    has keys 'id' and 'randomNumber', and these keys 
+    map to integers. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.query_url
+
+    problems = []
+    
+    body = self._curl(url + '2')
+    problems += self._verifyQueryList(2, body, url + '2')
+
+    body = self._curl(url + '0')
+    problems += self._verifyQueryList(1, body, url + '0', 'warn')
+
+    body = self._curl(url + 'foo')
+    problems += self._verifyQueryList(1, body, url + 'foo')
+
+    body = self._curl(url + '501')
+    problems += self._verifyQueryList(500, body, url + '501')
+
+    if len(problems) == 0:
+      return [('pass','',url + '2'),
+              ('pass','',url + '0'),
+              ('pass','',url + 'foo'),
+              ('pass','',url + '501')]
+    else:
+      return problems
+
+  def _verifyQueryList(self, expectedLength, body, url, incorrect_length_response='fail'):
+    '''Validates high-level structure (array length, object 
+      types, etc) before calling into DBTestType to 
+      validate a few individual JSON objects'''
+
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+  
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+
+    problems = []
+
+    if type(response) != list:
+      problems.append(('warn','Top-level JSON is an object, not an array', url))
+
+      # Verify the one object they gave us before returning
+      problems += self._verifyObject(response, url)
+
+      return problems
+
+    if any(type(item) != dict for item in response):
+      problems.append(('fail','All items JSON array must be JSON objects', url))
+
+    # For some edge cases we only warn
+    if len(response) != expectedLength:
+      problems.append((incorrect_length_response,
+        "JSON array length of %s != expected length of %s" % (len(response), expectedLength), 
+        url))
+
+    # verify individual objects
+    maxBadObjects = 5
+    for item in response:
+      obj_ok = self._verifyObject(item, url)
+      if len(obj_ok) > 0:
+        maxBadObjects -=  1
+        problems += obj_ok
+        if maxBadObjects == 0:
+          break
+
+    return problems
+
+
+

toolset/benchmark/test_types/update_type.py

@@ -0,0 +1,46 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.query_type import QueryTestType
+
+from pprint import pprint
+
+class UpdateTestType(QueryTestType):
+  def __init__(self):
+    args = ['update_url']
+    FrameworkTestType.__init__(self, name='update', requires_db=True, 
+      accept_header=self.accept_json, args=args)
+
+  def get_url(self):
+    return self.update_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON array of 
+    the proper length, each JSON Object in the array 
+    has keys 'id' and 'randomNumber', and these keys 
+    map to integers. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.update_url
+    problems = []
+    
+    body = self._curl(url + '2')
+    problems += self._verifyQueryList(2, body, url + '2')
+
+    body = self._curl(url + '0')
+    problems += self._verifyQueryList(1, body, url + '0', 'warn')
+
+    body = self._curl(url + 'foo')
+    problems += self._verifyQueryList(1, body, url + 'foo', 'warn')
+
+    body = self._curl(url + '501')
+    problems += self._verifyQueryList(500, body, url + '501', 'warn')
+
+    if len(problems) == 0:
+      return [('pass','',url + '2'),
+              ('pass','',url + '0'),
+              ('pass','',url + 'foo'),
+              ('pass','',url + '501')]
+    else:
+      return problems
+
+

toolset/benchmark/utils.py

@@ -114,4 +114,4 @@ def header(message, top='-', bottom='-'):
         result = "%s" % bottomheader
       else:
         result += "\n%s" % bottomheader
-    return result
+    return result + '\n'

toolset/run-ci.py

@@ -348,9 +348,11 @@ class CIRunnner:
     until timeout 15s sudo apt-key adv --keyserver pgp.mit.edu --recv 4BD736A82B5C1B00; do echo 'Waiting for apt-key' ; done
     sudo apt-add-repository  'deb http://www.apache.org/dist/cassandra/debian 20x main'
 
-    # Run installation
+    # Run installation 
+    # DO NOT COPY --force-yes TO ANY NON-TRAVIS-CI SCRIPTS! Seriously, it can cause some 
+    # major damage and should only be used inside a VM or Linux Container
     sudo apt-get -q update
-    sudo apt-get -q -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
+    sudo apt-get -q -y --force-yes install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
       mongodb-org \
       cassandra \
       openssh-server

toolset/run-tests.py

@@ -153,6 +153,8 @@ def main(argv=None):
         print 'Configuration options: '
         pprint(args)
 
+
+
     benchmarker = Benchmarker(vars(args))
 
     # Run the benchmarker in the specified mode

toolset/setup/linux/prerequisites.sh

@@ -37,8 +37,10 @@ sudo apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::=
   libgstreamer-plugins-base0.10-0 libgstreamer0.10-0 \
   liborc-0.4-0 libwxbase2.8-0 libwxgtk2.8-0 libgnutls-dev \
   libjson0-dev libmcrypt-dev libicu-dev gettext \
-  libpq-dev mlton cloc dstat
+  libpq-dev mlton \
+  cloc dstat                        `# Collect resource usage statistics`
 
+# Install gcc-4.8
 sudo add-apt-repository ppa:ubuntu-toolchain-r/test -y
 sudo apt-get -y update
 sudo apt-get install -y gcc-4.8 g++-4.8