Parcourir la source

verify Added a check for boundary rules for QUERY

QUERY requires that the request variable for number of queries be
bound between 1 and 500; anything less than 1 should result in 1
result and anything more than 500 should result in 500.

If the test does not clamp the request variable as such, then it
fails validation.
Mike Smith il y a 11 ans
Parent
commit
c40e0e85a6
1 fichiers modifiés avec 78 ajouts et 4 suppressions
  1. 78 4
      toolset/benchmark/framework_test.py

+ 78 - 4
toolset/benchmark/framework_test.py

@@ -146,7 +146,8 @@ class FrameworkTest:
       # This will error out of the value could not parsed to a
       # This will error out of the value could not parsed to a
       # float (this will work with ints, but it will turn them
       # float (this will work with ints, but it will turn them
       # into their float equivalent; i.e. "123" => 123.0)
       # into their float equivalent; i.e. "123" => 123.0)
-      if type(float(obj["id"])) == float and type(float(obj["randomNumber"])) == float:
+      if (type(float(obj["id"])) == float and 
+          type(float(obj["randomNumber"])) == float):
         return True
         return True
     except:
     except:
       err.write(textwrap.dedent("""
       err.write(textwrap.dedent("""
@@ -167,7 +168,10 @@ class FrameworkTest:
     try:
     try:
       arr = json.loads(jsonString)
       arr = json.loads(jsonString)
 
 
-      if type(float(arr[0]["id"])) == float and type(float(arr[0]["randomNumber"])) == float and type(float(arr[1]["id"])) == float and type(float(arr[1]["randomNumber"])) == float:
+      if (type(float(arr[0]["id"])) == float and 
+          type(float(arr[0]["randomNumber"])) == float and 
+          type(float(arr[1]["id"])) == float and 
+          type(float(arr[1]["randomNumber"])) == float):
         return True
         return True
     except:
     except:
       err.write(textwrap.dedent("""
       err.write(textwrap.dedent("""
@@ -178,6 +182,67 @@ class FrameworkTest:
           """.format( trace=sys.exc_info()[:2])))
           """.format( trace=sys.exc_info()[:2])))
     return False
     return False
 
 
+  ############################################################
+  # Validates the jsonString is an array with a length of
+  # 1, that each entry in the array is a JSON object, that
+  # each object has an "id" and a "randomNumber" key, and that
+  # both keys map to integers.
+  ############################################################
+  def validateQueryOneOrLess(self, jsonString, out, err):
+    try:
+      arr = json.loads(jsonString)
+
+      if len(arr) != 1:
+        return False
+
+      for obj in arr:
+        if (type(float(obj["id"])) != float or
+            type(float(obj["randomNumber"])) != float or
+            type(float(obj["id"])) != float or
+            type(float(obj["randomNumber"])) != float):
+          return False
+      # By here, it's passed validation
+      return True
+    except:
+      err.write(textwrap.dedent("""
+          -----------------------------------------------------
+            Error: validateQuery raised exception
+          -----------------------------------------------------
+          {trace}
+          """.format( trace=sys.exc_info()[:2])))
+    return False
+
+
+  ############################################################
+  # Validates the jsonString is an array with a length of
+  # 500, that each entry in the array is a JSON object, that
+  # each object has an "id" and a "randomNumber" key, and that
+  # both keys map to integers.
+  ############################################################
+  def validateQueryFiveHundredOrMore(self, jsonString, out, err):
+    try:
+      arr = json.loads(jsonString)
+
+      if len(arr) != 500:
+        return False
+
+      for obj in arr:
+        if (type(float(obj["id"])) != float or
+            type(float(obj["randomNumber"])) != float or
+            type(float(obj["id"])) != float or
+            type(float(obj["randomNumber"])) != float):
+          return False
+      # By here, it's passed validation
+      return True
+    except:
+      err.write(textwrap.dedent("""
+          -----------------------------------------------------
+            Error: validateQuery raised exception
+          -----------------------------------------------------
+          {trace}
+          """.format( trace=sys.exc_info()[:2])))
+    return False
+
   ############################################################
   ############################################################
   # Parses the given HTML string and asks a FortuneHTMLParser
   # Parses the given HTML string and asks a FortuneHTMLParser
   # whether the parsed string is a valid fortune return.
   # whether the parsed string is a valid fortune return.
@@ -207,7 +272,10 @@ class FrameworkTest:
     try:
     try:
       arr = json.loads(jsonString)
       arr = json.loads(jsonString)
 
 
-      if type(float(arr[0]["id"])) == float and type(float(arr[0]["randomNumber"])) == float and type(float(arr[1]["id"])) == float and type(float(arr[1]["randomNumber"])) == float:
+      if (type(float(arr[0]["id"])) == float and 
+          type(float(arr[0]["randomNumber"])) == float and 
+          type(float(arr[1]["id"])) == float and 
+          type(float(arr[1]["randomNumber"])) == float):
         return True
         return True
     except:
     except:
       err.write(textwrap.dedent("""
       err.write(textwrap.dedent("""
@@ -308,7 +376,13 @@ class FrameworkTest:
       try:
       try:
         url = self.benchmarker.generate_url(self.query_url + "2", self.port)
         url = self.benchmarker.generate_url(self.query_url + "2", self.port)
         output = self.__curl_url(url, self.QUERY, out, err)
         output = self.__curl_url(url, self.QUERY, out, err)
-        if self.validateQuery(output, out, err):
+        url2 = self.benchmarker.generate_url(self.query_url + "-1", self.port)
+        output2 = self.__curl_url(url2, self.QUERY, out, err)
+        url3 = self.benchmarker.generate_url(self.query_url + "501", self.port)
+        output3 = self.__curl_url(url3, self.QUERY, out, err)
+        if (self.validateQuery(output, out, err) and
+            self.validateQueryOneOrLess(output2, out, err) and
+            self.validateQueryFiveHundredOrMore(output3, out, err)):
           self.query_url_passed = True
           self.query_url_passed = True
         else:
         else:
           self.query_url_passed = False
           self.query_url_passed = False