verify Added warning to the results.json

toolset/benchmark/benchmarker.py

@@ -210,6 +210,33 @@ class Benchmarker:
   # End output_file
   ############################################################
 
+  ############################################################
+  # get_warning_file(test_name, test_type)
+  # returns the output file name for this test_name and 
+  # test_type timestamp/test_type/test_name/raw 
+  ############################################################
+  def get_warning_file(self, test_name, test_type):
+    return os.path.join(self.result_directory, self.timestamp, test_type, test_name, "warn")
+  ############################################################
+  # End get_warning_file
+  ############################################################
+
+  ############################################################
+  # warning_file(test_name, test_type)
+  # returns the warning file for this test_name and test_type
+  # timestamp/test_type/test_name/raw 
+  ############################################################
+  def warning_file(self, test_name, test_type):
+    path = self.get_warning_file(test_name, test_type)
+    try:
+      os.makedirs(os.path.dirname(path))
+    except OSError:
+      pass
+    return path
+  ############################################################
+  # End warning_file
+  ############################################################
+
   ############################################################
   # full_results_directory
   ############################################################
@@ -221,7 +248,7 @@ class Benchmarker:
       pass
     return path
   ############################################################
-  # End output_file
+  # End full_results_directory
   ############################################################
 
   ############################################################
@@ -918,6 +945,13 @@ class Benchmarker:
       self.results['failed']['fortune'] = []
       self.results['failed']['update'] = []
       self.results['failed']['plaintext'] = []
+      self.results['warning'] = dict()
+      self.results['warning']['json'] = []
+      self.results['warning']['db'] = []
+      self.results['warning']['query'] = []
+      self.results['warning']['fortune'] = []
+      self.results['warning']['update'] = []
+      self.results['warning']['plaintext'] = []
     else:
       #for x in self.__gather_tests():
       #  if x.name not in self.results['frameworks']:

toolset/benchmark/framework_test.py

@@ -163,6 +163,26 @@ class FrameworkTest:
           """.format( trace=sys.exc_info()[:2])))
     return False
 
+  def validateDbStrict(self, jsonString, out, err):
+    try:
+      obj = json.loads(jsonString)
+
+      # This will error out of the value could not parsed to a
+      # float (this will work with ints, but it will turn them
+      # into their float equivalent; i.e. "123" => 123.0)
+      if (type(float(obj["id"])) == float and 
+          type(float(obj["randomNumber"])) == float):
+        return True
+    except:
+      err.write(textwrap.dedent("""
+          -----------------------------------------------------
+            Error: validateDbStrict raised exception
+          -----------------------------------------------------
+          {trace}
+          """.format( trace=sys.exc_info()[:2])))
+    return False
+
+
   ############################################################
   # Validates the jsonString is an array with a length of
   # 2, that each entry in the array is a JSON object, that
@@ -372,6 +392,10 @@ class FrameworkTest:
           self.db_url_passed = True
         else:
           self.db_url_passed = False
+        if self.validateDbStrict(output, out, err):
+          self.db_url_warn = False
+        else:
+          self.db_url_warn = True
       except (AttributeError, subprocess.CalledProcessError):
         err.write(textwrap.dedent("""
             -----------------------------------------------------
@@ -570,10 +594,14 @@ class FrameworkTest:
         out.flush()
         results = None
         output_file = self.benchmarker.output_file(self.name, self.DB)
+        warning_file = self.benchmarker.warning_file(self.name, self.DB)
         if not os.path.exists(output_file):
           with open(output_file, 'w'):
             # Simply opening the file in write mode should create the empty file.
             pass
+        if self.db_url_warn:
+          with open(warning_file, 'w'):
+            pass
         if self.db_url_passed:
           remote_script = self.__generate_concurrency_script(self.db_url, self.port, self.accept_json)
           self.__run_benchmark(remote_script, output_file, err)
@@ -590,10 +618,14 @@ class FrameworkTest:
         out.flush()
         results = None
         output_file = self.benchmarker.output_file(self.name, self.QUERY)
+        warning_file = self.benchmarker.warning_file(self.name, self.DB)
         if not os.path.exists(output_file):
           with open(output_file, 'w'):
             # Simply opening the file in write mode should create the empty file.
             pass
+        if self.query_url_warn:
+          with open(warning_file, 'w'):
+            pass
         if self.query_url_passed:
           remote_script = self.__generate_query_script(self.query_url, self.port, self.accept_json)
           self.__run_benchmark(remote_script, output_file, err)
@@ -717,6 +749,9 @@ class FrameworkTest:
     try:
       results = dict()
       results['results'] = []
+
+      if os.path.exists(self.benchmarker.get_warning_file(self.name, test_type)):
+        results['results']['warning'][test_type].append(self.name)
       
       if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
         with open(self.benchmarker.output_file(self.name, test_type)) as raw_data: