Add FrameworkTestTypes for DB, Query, Update

toolset/benchmark/framework_test.py

@@ -128,130 +128,6 @@ class FrameworkTest:
   # Public Methods
   ##########################################################################################
 
-
-  ############################################################
-  # Validates the jsonString is an array with a length of
-  # 2, that each entry in the array is a JSON object, that
-  # each object has an "id" and a "randomNumber" key, and that
-  # both keys map to integers.
-  ############################################################
-  def validateQuery(self, jsonString, out, err):
-    err_str = ""
-    if jsonString is None or len(jsonString) == 0:
-      err_str += "Empty Response"
-      return (False, err_str)
-    try:
-      arr = [{k.lower(): v for k,v in d.iteritems()} for d in json.loads(jsonString)]
-      if len(arr) != 2:
-        err_str += "Expected array of length 2. Got length {length}. ".format(length=len(arr))
-      for obj in arr:
-        id_ret_val = True
-        random_num_ret_val = True
-        if "id" not in obj or "randomnumber" not in obj:
-          err_str += "Expected keys id and randomNumber to be in JSON string. "
-          break
-        try:
-          if not isinstance(float(obj["id"]), float):
-            id_ret_val=False
-        except:
-          id_ret_val=False
-        if not id_ret_val:
-          err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
-        try:
-          if not isinstance(float(obj["randomnumber"]), float):
-            random_num_ret_val=False
-        except:
-          random_num_ret_val=False
-        if not random_num_ret_val:
-          err_str += "Expected randomNumber to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
-    except:
-      err_str += "Got exception when trying to validate the query test: {exception}".format(exception=traceback.format_exc())
-    return (True, ) if len(err_str) == 0 else (False, err_str)
-
-  ############################################################
-  # Validates the jsonString is an array with a length of
-  # 1, that each entry in the array is a JSON object, that
-  # each object has an "id" and a "randomNumber" key, and that
-  # both keys map to integers.
-  ############################################################
-  def validateQueryOneOrLess(self, jsonString, out, err):
-    err_str = ""
-    if jsonString is None or len(jsonString) == 0:
-      err_str += "Empty Response"
-    else:
-      try:
-        json_load = json.loads(jsonString)
-        if not isinstance(json_load, list):
-          err_str += "Expected JSON array, got {typeObj}. ".format(typeObj=type(json_load))
-        if len(json_load) != 1:
-          err_str += "Expected array of length 1. Got length {length}. ".format(length=len(json_load))
-
-        obj = {k.lower(): v for k,v in json_load[0].iteritems()}
-        id_ret_val = True
-        random_num_ret_val = True
-        if "id" not in obj or "randomnumber" not in obj:
-          err_str += "Expected keys id and randomNumber to be in JSON string. "
-        try:
-          if not isinstance(float(obj["id"]), float):
-            id_ret_val=False
-        except:
-          id_ret_val=False
-        if not id_ret_val:
-          err_str += "Expected id to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
-        try:
-          if not isinstance(float(obj["randomnumber"]), float):
-            random_num_ret_val=False
-        except:
-          random_num_ret_val=False
-        if not random_num_ret_val:
-          err_str += "Expected randomNumber to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
-      except:
-        err_str += "Got exception when trying to validate the query test: {exception} ".format(exception=traceback.format_exc())
-
-    return (True, ) if len(err_str) == 0 else (False, err_str)
-
-  ############################################################
-  # Validates the jsonString is an array with a length of
-  # 500, that each entry in the array is a JSON object, that
-  # each object has an "id" and a "randomNumber" key, and that
-  # both keys map to integers.
-  ############################################################
-  def validateQueryFiveHundredOrMore(self, jsonString, out, err):
-    err_str = ""
-    if jsonString is None or len(jsonString) == 0:
-      err_str += "Empty Response"
-      return (False, err_str)
-    try:
-      arr = [{k.lower(): v for k,v in d.iteritems()} for d in json.loads(jsonString)]
-
-      if len(arr) != 500:
-        err_str += "Expected array of length 500. Got length {length}. ".format(length=len(arr))
-        return (False, err_str)
-
-      for obj in arr:
-        id_ret_val = True
-        random_num_ret_val = True
-        if "id" not in obj or "randomnumber" not in obj:
-          err_str += "Expected keys id and randomNumber to be in JSON string. "
-          break
-        try:
-          if not isinstance(float(obj["id"]), float):
-            id_ret_val=False
-        except:
-          id_ret_val=False
-        if not id_ret_val:
-          err_str += "Expected id to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
-        try:
-          if not isinstance(float(obj["randomnumber"]), float):
-            random_num_ret_val=False
-        except:
-          random_num_ret_val=False
-        if not random_num_ret_val:
-          err_str += "Expected randomNumber to be type int or float, got '{rand}'. ".format(rand=obj["randomnumber"])
-    except:
-      err_str += "Got exception when trying to validate the query test: {exception} ".format(exception=traceback.format_exc())
-    return (True, ) if len(err_str) == 0 else (False, err_str)
-
   ############################################################
   # Parses the given HTML string and asks a FortuneHTMLParser
   # whether the parsed string is a valid fortune return.
@@ -429,8 +305,16 @@ class FrameworkTest:
       out.write(header("VERIFYING %s" % test_type.upper()))
       
       base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port)
-      results = test.verify(base_url)
       
+      try:
+        results = test.verify(base_url)
+      except Exception as e:
+        results = [('fail',"""Caused Exception in TFB
+          This almost certainly means your return value is incorrect, 
+          but also that you have found a bug. Please submit an issue
+          including this message: %s""" % e,base_url)]
+        logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)
+
       test.failed = any(result is 'fail' for (result, reason, url) in results)
       test.warned = any(result is 'warn' for (result, reason, url) in results)
       test.passed = all(result is 'pass' for (result, reason, url) in results)

toolset/benchmark/test_types/__init__.py

@@ -2,4 +2,6 @@
 from framework_test_type import *
 from json_type import JsonTestType
 from plaintext_type import PlaintextTestType
-from db_type import DBTestType
+from db_type import DBTestType
+from query_type import QueryTestType
+from update_type import UpdateTestType

toolset/benchmark/test_types/db_type.py

@@ -16,10 +16,10 @@ class DBTestType(FrameworkTestType):
     '''Ensures body is valid JSON with a key 'id' and a key 
     'randomNumber', both of which must map to integers
     '''
+
     url = base_url + self.db_url
     body = self._curl(url)
-    problems = []
-
+    
     # Empty response
     if body is None:
       return [('fail','No response', url)]
@@ -32,6 +32,8 @@ class DBTestType(FrameworkTestType):
     except ValueError as ve:
       return [('fail',"Invalid JSON - %s" % ve, url)]
 
+    problems = []
+
     # We are allowing the single-object array 
     # e.g. [{'id':5, 'randomNumber':10}] for now, 
     # but will likely make this fail at some point
@@ -44,39 +46,58 @@ class DBTestType(FrameworkTestType):
         problems.append( ('fail', 'Response is not a JSON object or an array of JSON objects', url) ) 
         return problems
 
+    problems += self._verifyObject(response, url)
+
+    if len(problems) == 0:
+      return [('pass','',url)]
+    else:
+      return problems
+
+  def _verifyObject(self, db_object, url):
+    '''Ensure the passed item is a JSON object with 
+    keys 'id' and 'randomNumber' mapping to ints. 
+    Separate method allows the QueryTestType to 
+    reuse these checks'''
+
+    problems = []
+
+    if type(db_object) != dict:
+      got = str(db_object)[:20]
+      if len(str(db_object)) > 20:
+        got = str(db_object)[:17] + '...'
+      return ('fail', "Expected a JSON object, got '%s' instead" % got, url)
+
     # Make keys case insensitive
-    response = {k.lower(): v for k,v in response.iteritems()}
+    db_object = {k.lower(): v for k,v in db_object.iteritems()}
 
-    if "id" not in response:
+    if "id" not in db_object:
       problems.append( ('fail', "Response has no 'id' key", url) ) 
-    if "randomnumber" not in response:
+    if "randomnumber" not in db_object:
       problems.append( ('fail', "Response has no 'randomNumber' key", url) ) 
 
     try:
-      float(response["id"])
+      float(db_object["id"])
     except ValueError as ve:
       problems.append( ('fail', "Response key 'id' does not map to a number - %s" % ve, url) ) 
 
     try:
-      float(response["randomnumber"])
+      float(db_object["randomnumber"])
     except ValueError as ve:
       problems.append( ('fail', "Response key 'randomNumber' does not map to a number - %s" % ve, url) ) 
 
-    if type(response["id"]) != int:
+    if type(db_object["id"]) != int:
       problems.append( ('warn', '''Response key 'id' contains extra quotations or decimal points.
         This may negatively affect performance during benchmarking''', url) ) 
 
     # Tests based on the value of the numbers
     try:
-      response_id = float(response["id"])
-      response_rn = float(response["randomnumber"])
+      response_id = float(db_object["id"])
+      response_rn = float(db_object["randomnumber"])
 
       if response_id > 10000 or response_id < 1:
         problems.append( ('warn', "Response key 'id' should be between 1 and 10,000", url)) 
     except ValueError:
       pass
 
-    if len(problems) == 0:
-      return [('pass','',url)]
-    else:
-      return problems
+    return problems
+

toolset/benchmark/test_types/framework_test_type.py

@@ -91,6 +91,10 @@ class FrameworkTestType:
         other parts of TFB will do that based upon the current logging 
         settings if this method indicates a failure happened
     - urlTested: The exact URL that was queried
+
+    Subclasses should make a best-effort attempt to report as many
+    failures and warnings as they can to help users avoid needing 
+    to run TFB repeatedly while debugging
     '''
     # TODO make String result into an enum to enforce
     raise NotImplementedError("Subclasses must provide verify")
@@ -101,19 +105,8 @@ class FrameworkTestType:
     # for their URL so the base class can't know which arg is the URL
     raise NotImplementedError("Subclasses must provide verify")
 
-class QueryTestType(FrameworkTestType):
-  def __init__(self):
-    args = ['query_url']
-    FrameworkTestType.__init__(self, name='query', requires_db=True, args=args)
-
-
 class FortuneTestType(FrameworkTestType):
   def __init__(self):
     args = ['fortune_url']
     FrameworkTestType.__init__(self, name='fortune', requires_db=True, args=args)
 
-class UpdateTestType(FrameworkTestType):
-  def __init__(self):
-    args = ['update_url']
-    FrameworkTestType.__init__(self, name='update', requires_db=True, args=args)
-

toolset/benchmark/test_types/query_type.py

@@ -0,0 +1,91 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.db_type import DBTestType
+
+import json
+
+class QueryTestType(DBTestType):
+  def __init__(self):
+    args = ['query_url']
+    FrameworkTestType.__init__(self, name='query', requires_db=True, 
+      accept_header=self.accept_json, args=args)
+
+  def get_url(self):
+    return self.query_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON array of 
+    the proper length, each JSON Object in the array 
+    has keys 'id' and 'randomNumber', and these keys 
+    map to integers. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.query_url
+
+    problems = []
+    
+    body = self._curl(url + '2')
+    problems += self._verifyQueryList(2, body, url + '2')
+
+    body = self._curl(url + '0')
+    problems += self._verifyQueryList(1, body, url + '0', 'warn')
+
+    body = self._curl(url + 'foo')
+    problems += self._verifyQueryList(1, body, url + 'foo')
+
+    body = self._curl(url + '501')
+    problems += self._verifyQueryList(500, body, url + '501')
+
+    if len(problems) == 0:
+      return [('pass','',url + '2'),
+              ('pass','',url + '0'),
+              ('pass','',url + 'foo'),
+              ('pass','',url + '501')]
+    else:
+      return problems
+
+  def _verifyQueryList(self, expectedLength, body, url, incorrect_length_response='fail'):
+    '''Validates high-level structure (array length, object 
+      types, etc) before calling into DBTestType to 
+      validate a few individual JSON objects'''
+
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+  
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+
+    problems = []
+
+    if type(response) != list:
+      return [('fail','Top-level JSON is an object, not an array', url)]
+
+    if any(type(item) != dict for item in response):
+      problems.append(('fail','All items JSON array must be JSON objects', url))
+
+    # For some edge cases we only warn
+    if len(response) != expectedLength:
+      problems.append((incorrect_length_response,
+        "JSON array length of %s != expected length of %s" % (len(response), expectedLength), 
+        url))
+
+    # verify individual objects
+    maxBadObjects = 5
+    for item in response:
+      obj_ok = self._verifyObject(item, url)
+      if len(obj_ok) > 0:
+        maxBadObjects -=  1
+        problems += obj_ok
+        if maxBadObjects == 0:
+          break
+
+    return problems
+
+
+

toolset/benchmark/test_types/update_type.py

@@ -0,0 +1,48 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+from benchmark.test_types.query_type import QueryTestType
+
+from pprint import pprint
+
+class UpdateTestType(QueryTestType):
+  def __init__(self):
+    args = ['update_url']
+    FrameworkTestType.__init__(self, name='update', requires_db=True, 
+      accept_header=self.accept_json, args=args)
+
+    pprint(self.__dict__)
+
+  def get_url(self):
+    return self.update_url
+
+  def verify(self, base_url):
+    '''Validates the response is a JSON array of 
+    the proper length, each JSON Object in the array 
+    has keys 'id' and 'randomNumber', and these keys 
+    map to integers. Case insensitive and 
+    quoting style is ignored
+    '''
+
+    url = base_url + self.update_url
+    problems = []
+    
+    body = self._curl(url + '2')
+    problems += self._verifyQueryList(2, body, url + '2')
+
+    body = self._curl(url + '0')
+    problems += self._verifyQueryList(1, body, url + '0', 'warn')
+
+    body = self._curl(url + 'foo')
+    problems += self._verifyQueryList(1, body, url + 'foo', 'warn')
+
+    body = self._curl(url + '501')
+    problems += self._verifyQueryList(500, body, url + '501', 'warn')
+
+    if len(problems) == 0:
+      return [('pass','',url + '2'),
+              ('pass','',url + '0'),
+              ('pass','',url + 'foo'),
+              ('pass','',url + '501')]
+    else:
+      return problems
+
+