Add DBTestType

toolset/benchmark/framework_test.py

@@ -129,84 +129,6 @@ class FrameworkTest:
   ##########################################################################################
 
 
-  ############################################################
-  # Validates the jsonString is a JSON object that has an "id"
-  # and a "randomNumber" key, and that both keys map to 
-  # integers.
-  ############################################################
-  def validateDb(self, jsonString, out, err):
-    err_str = ""
-    if jsonString is None or len(jsonString) == 0:
-      err_str += "Empty Response"
-      return (False, err_str)
-    try:
-      obj = {k.lower(): v for k,v in json.loads(jsonString).iteritems()}
-
-      # We are allowing the single-object array for the DB 
-      # test for now, but will likely remove this later.
-      if type(obj) == list:
-        obj = obj[0]
-
-      if "id" not in obj or "randomnumber" not in obj:
-        err_str += "Expected keys id and randomNumber to be in JSON string. "
-        return (False, err_str)
-
-      # This will error out of the value could not parsed to a
-      # float (this will work with ints, but it will turn them
-      # into their float equivalent; i.e. "123" => 123.0)
-      id_ret_val = True
-      try:
-        if not isinstance(float(obj["id"]), float):
-          id_ret_val=False
-      except:
-        id_ret_val=False
-      if not id_ret_val:
-        err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
-      random_num_ret_val = True
-      try:
-        if not isinstance(float(obj["randomnumber"]), float):
-          random_num_ret_val=False
-      except:
-        random_num_ret_val=False
-      if not random_num_ret_val:
-        err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
-    except:
-      err_str += "Got exception when trying to validate the db test: {exception}".format(exception=traceback.format_exc())
-    return (True, ) if len(err_str) == 0 else (False, err_str)
-
-  def validateDbStrict(self, jsonString, out, err):
-    err_str = ""
-    if jsonString is None or len(jsonString) == 0:
-      err_str += "Empty Response "
-      return (False, err_str)
-    try:
-      obj = {k.lower(): v for k,v in json.loads(jsonString).iteritems()}
-
-      # This will error out of the value could not parsed to a
-      # float (this will work with ints, but it will turn them
-      # into their float equivalent; i.e. "123" => 123.0)
-      id_ret_val = True
-      try:
-        if not isinstance(float(obj["id"]), float):
-          id_ret_val=False
-      except:
-        id_ret_val=False
-      if not id_ret_val:
-        err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
-      random_num_ret_val = True
-      try:
-        if not isinstance(float(obj["randomnumber"]), float):
-          random_num_ret_val=False
-      except:
-        random_num_ret_val=False
-      if not random_num_ret_val:
-        err_str += "Expected id to be type int or float, got '{rand}' ".format(rand=obj["randomnumber"])
-      return id_ret_val and random_num_ret_val
-    except:
-      err_str += "Got exception when trying to validate the db test: {exception}".format(exception=traceback.format_exc())
-    return (True, ) if len(err_str) == 0 else (False, err_str)
-
-
   ############################################################
   # Validates the jsonString is an array with a length of
   # 2, that each entry in the array is a JSON object, that

toolset/benchmark/test_types/__init__.py

@@ -1,4 +1,5 @@
 
 from framework_test_type import *
 from json_type import JsonTestType
-from plaintext_type import PlaintextTestType
+from plaintext_type import PlaintextTestType
+from db_type import DBTestType

toolset/benchmark/test_types/db_type.py

@@ -0,0 +1,82 @@
+from benchmark.test_types.framework_test_type import FrameworkTestType
+
+import json
+
+class DBTestType(FrameworkTestType):
+  def __init__(self):
+    args = ['db_url']
+    FrameworkTestType.__init__(self, name='db', 
+      accept_header=self.accept_json,
+      requires_db=True, args=args)
+
+  def get_url(self):
+    return self.db_url
+
+  def verify(self, base_url):
+    '''Ensures body is valid JSON with a key 'id' and a key 
+    'randomNumber', both of which must map to integers
+    '''
+    url = base_url + self.db_url
+    body = self._curl(url)
+    problems = []
+
+    # Empty response
+    if body is None:
+      return [('fail','No response', url)]
+    elif len(body) == 0:
+      return [('fail','Empty Response', url)]
+
+    # Valid JSON? 
+    try: 
+      response = json.loads(body)
+    except ValueError as ve:
+      return [('fail',"Invalid JSON - %s" % ve, url)]
+
+    # We are allowing the single-object array 
+    # e.g. [{'id':5, 'randomNumber':10}] for now, 
+    # but will likely make this fail at some point
+    if type(response) == list:
+      response = response[0]
+      problems.append( ('warn', 'Response is a JSON array. Expected JSON object (e.g. [] vs {})', url) ) 
+
+      # Make sure there was a JSON object inside the array
+      if type(response) != dict:
+        problems.append( ('fail', 'Response is not a JSON object or an array of JSON objects', url) ) 
+        return problems
+
+    # Make keys case insensitive
+    response = {k.lower(): v for k,v in response.iteritems()}
+
+    if "id" not in response:
+      problems.append( ('fail', "Response has no 'id' key", url) ) 
+    if "randomnumber" not in response:
+      problems.append( ('fail', "Response has no 'randomNumber' key", url) ) 
+
+    try:
+      float(response["id"])
+    except ValueError as ve:
+      problems.append( ('fail', "Response key 'id' does not map to a number - %s" % ve, url) ) 
+
+    try:
+      float(response["randomnumber"])
+    except ValueError as ve:
+      problems.append( ('fail', "Response key 'randomNumber' does not map to a number - %s" % ve, url) ) 
+
+    if type(response["id"]) != int:
+      problems.append( ('warn', '''Response key 'id' contains extra quotations or decimal points.
+        This may negatively affect performance during benchmarking''', url) ) 
+
+    # Tests based on the value of the numbers
+    try:
+      response_id = float(response["id"])
+      response_rn = float(response["randomnumber"])
+
+      if response_id > 10000 or response_id < 1:
+        problems.append( ('warn', "Response key 'id' should be between 1 and 10,000", url)) 
+    except ValueError:
+      pass
+
+    if len(problems) == 0:
+      return [('pass','',url)]
+    else:
+      return problems

toolset/benchmark/test_types/framework_test_type.py

@@ -83,11 +83,14 @@ class FrameworkTestType:
     '''Accesses URL used by this test type and checks the return 
     values for correctness. Most test types run multiple checks,
     so this returns a list of results. Each result is a 3-tuple
-    of (String result, String message, String urlTested).
-
-    - Result is always 'pass','warn','fail'
-    - message is a human-readable reason if the result was warn or fail
-    - urlTested is the URL that was queried
+    of (String result, String reason, String urlTested).
+
+    - result : 'pass','warn','fail'
+    - reason : Short human-readable reason if result was 
+        warn or fail. Please do not print the response as part of this, 
+        other parts of TFB will do that based upon the current logging 
+        settings if this method indicates a failure happened
+    - urlTested: The exact URL that was queried
     '''
     # TODO make String result into an enum to enforce
     raise NotImplementedError("Subclasses must provide verify")
@@ -98,11 +101,6 @@ class FrameworkTestType:
     # for their URL so the base class can't know which arg is the URL
     raise NotImplementedError("Subclasses must provide verify")
 
-class DBTestType(FrameworkTestType):
-  def __init__(self):
-    args = ['db_url']
-    FrameworkTestType.__init__(self, name='db', requires_db=True, args=args)
-
 class QueryTestType(FrameworkTestType):
   def __init__(self):
     args = ['query_url']