verify Huge change to parsing and validation

Before this commit, if the test was marked 'json', then a raw file
would be created for that test in the
`results/[timestamp]/json/[your test]/`

However, if a test was marked 'json' and it failed, the raw file
would not be output to the
`results/[timestamp]/json/[your test]/`
dir which would mean there was no way to tell, after a run, whether
the test passed or failed validation.

With this commit, if your test is marked 'json', and it succeeds,
nothing has changed - your test gets a
`results/[timestamp]/json/[your test]/raw`
file with your timing data in it.

If your test is NOT marked 'json', then you will not have a
`results/[timestamp]/json/[your test]/raw`
file. This means you did not 'fail' your json test, but simply did
not have a json test.

If your test is marked 'json' and it failed validation, then your
`results/[timestamp]/json/[your test]/raw`
file will exist, but will be empty. This file existing and being
empty will determine at parse-time whether your test failed
validation or not.

toolset/benchmark/benchmarker.py

@@ -183,13 +183,24 @@ class Benchmarker:
   # End generate_url
   ############################################################
 
+  ############################################################
+  # get_output_file(test_name, test_type)
+  # returns the output file name for this test_name and 
+  # test_type timestamp/test_type/test_name/raw 
+  ############################################################
+  def get_output_file(self, test_name, test_type):
+    return os.path.join(self.result_directory, self.timestamp, test_type, test_name, "raw")
+  ############################################################
+  # End get_output_file
+  ############################################################
+
   ############################################################
   # output_file(test_name, test_type)
   # returns the output file for this test_name and test_type
   # timestamp/test_type/test_name/raw 
   ############################################################
   def output_file(self, test_name, test_type):
-    path = os.path.join(self.result_directory, self.timestamp, test_type, test_name, "raw")
+    path = self.get_output_file(test_name, test_type)
     try:
       os.makedirs(os.path.dirname(path))
     except OSError:
@@ -228,13 +239,13 @@ class Benchmarker:
   ############################################################
   # report_results
   ############################################################
-  def report_results(self, framework, test, results, passed=True):
+  def report_results(self, framework, test, results):
     if test not in self.results['rawData'].keys():
       self.results['rawData'][test] = dict()
 
-    self.results['rawData'][test][framework.name] = results
-
-    if passed:
+    # If results has a size from the parse, then it succeeded.
+    if results:
+      self.results['rawData'][test][framework.name] = results
       self.results['succeeded'][test].append(framework.name)
     else:
       self.results['failed'][test].append(framework.name)

toolset/benchmark/fortune_html_parser.py

@@ -27,26 +27,29 @@ class FortuneHTMLParser(HTMLParser):
     # """ is a valid escaping, but we are normalizing
     # it so that our final parse can just be checked for
     # equality.
-    if name == "34":
+    if name == "34" or name == "x22":
       # Append our normalized entity reference to our body.
       self.body.append(""")
     # "'" is a valid escaping of "-", but it is not
     # required, so we normalize for equality checking.
-    if name == "39":
+    if name == "39" or name == "x27":
       self.body.append("'")
     # Again, "+" is a valid escaping of the "+", but
     # it is not required, so we need to normalize for out
     # final parse and equality check.
-    if name == "43":
+    if name == "43" or name == "x2B":
       self.body.append("+")
     # Again, ">" is a valid escaping of ">", but we
     # need to normalize to ">" for equality checking.
-    if name == "62":
+    if name == "62" or name == "x3E":
       self.body.append(">")
     # Again, "&#60;" is a valid escaping of "<", but we
     # need to nromalize to "&lt;" for equality checking.
-    if name == "60":
+    if name == "60" or name == "x3C":
       self.body.append("&lt;")
+    # Not sure why some are escaping '/'
+    if name == "47" or name == "x2F":
+      self.body.append("/")
 
   def handle_entityref(self, name):
     self.body.append("&{n};".format(n=name))

toolset/benchmark/framework_test.py

@@ -452,14 +452,12 @@ class FrameworkTest:
           out.write("BENCHMARKING JSON ... ") 
           out.flush()
           results = None
+          output_file = self.benchmarker.output_file(self.name, self.JSON)
           if self.json_url_passed:
             remote_script = self.__generate_concurrency_script(self.json_url, self.port, self.accept_json)
-            self.__run_benchmark(remote_script, self.benchmarker.output_file(self.name, self.JSON), err)
-            results = self.__parse_test(self.JSON)
-          else:
-            results = dict()
-            results['results'] = []
-          self.benchmarker.report_results(framework=self, test=self.JSON, results=results['results'], passed=self.json_url_passed)
+            self.__run_benchmark(remote_script, output_file, err)
+          results = self.__parse_test(self.JSON)
+          self.benchmarker.report_results(framework=self, test=self.JSON, results=results['results'])
           out.write( "Complete\n" )
           out.flush()
       except AttributeError:
@@ -472,14 +470,12 @@ class FrameworkTest:
           out.write("BENCHMARKING DB ... ") 
           out.flush()
           results = None
+          output_file = self.benchmarker.output_file(self.name, self.DB)
           if self.db_url_passed:
             remote_script = self.__generate_concurrency_script(self.db_url, self.port, self.accept_json)
-            self.__run_benchmark(remote_script, self.benchmarker.output_file(self.name, self.DB), err)
-            results = self.__parse_test(self.DB)
-          else:
-            results = dict()
-            results['results'] = []
-          self.benchmarker.report_results(framework=self, test=self.DB, results=results['results'], passed=self.db_url_passed)
+            self.__run_benchmark(remote_script, output_file, err)
+          results = self.__parse_test(self.DB)
+          self.benchmarker.report_results(framework=self, test=self.DB, results=results['results'])
           out.write( "Complete\n" )
       except AttributeError:
         pass
@@ -491,14 +487,12 @@ class FrameworkTest:
           out.write("BENCHMARKING Query ... ")
           out.flush()
           results = None
+          output_file = self.benchmarker.output_file(self.name, self.QUERY)
           if self.query_url_passed:
             remote_script = self.__generate_query_script(self.query_url, self.port, self.accept_json)
-            self.__run_benchmark(remote_script, self.benchmarker.output_file(self.name, self.QUERY), err)
-            results = self.__parse_test(self.QUERY)
-          else:
-            results = dict()
-            results['results'] = []
-          self.benchmarker.report_results(framework=self, test=self.QUERY, results=results['results'], passed=self.query_url_passed)
+            self.__run_benchmark(remote_script, output_file, err)
+          results = self.__parse_test(self.QUERY)
+          self.benchmarker.report_results(framework=self, test=self.QUERY, results=results['results'])
           out.write( "Complete\n" )
           out.flush()
       except AttributeError:
@@ -511,14 +505,12 @@ class FrameworkTest:
           out.write("BENCHMARKING Fortune ... ") 
           out.flush()
           results = None
+          output_file = self.benchmarker.output_file(self.name, self.FORTUNE)
           if self.fortune_url_passed:
             remote_script = self.__generate_concurrency_script(self.fortune_url, self.port, self.accept_html)
-            self.__run_benchmark(remote_script, self.benchmarker.output_file(self.name, self.FORTUNE), err)
-            results = self.__parse_test(self.FORTUNE)
-          else:
-            results = dict()
-            results['results'] = []
-          self.benchmarker.report_results(framework=self, test=self.FORTUNE, results=results['results'], passed=self.fortune_url_passed)
+            self.__run_benchmark(remote_script, output_file, err)
+          results = self.__parse_test(self.FORTUNE)
+          self.benchmarker.report_results(framework=self, test=self.FORTUNE, results=results['results'])
           out.write( "Complete\n" )
           out.flush()
       except AttributeError:
@@ -531,14 +523,12 @@ class FrameworkTest:
           out.write("BENCHMARKING Update ... ") 
           out.flush()
           results = None
+          output_file = self.benchmarker.output_file(self.name, self.UPDATE)
           if self.update_url_passed:
             remote_script = self.__generate_query_script(self.update_url, self.port, self.accept_json)
-            self.__run_benchmark(remote_script, self.benchmarker.output_file(self.name, self.UPDATE), err)
-            results = self.__parse_test(self.UPDATE)
-          else:
-            results = dict()
-            results['results'] = []
-          self.benchmarker.report_results(framework=self, test=self.UPDATE, results=results['results'], passed=self.update_url_passed)
+            self.__run_benchmark(remote_script, output_file, err)
+          results = self.__parse_test(self.UPDATE)
+          self.benchmarker.report_results(framework=self, test=self.UPDATE, results=results['results'])
           out.write( "Complete\n" )
           out.flush()
       except AttributeError:
@@ -551,14 +541,12 @@ class FrameworkTest:
           out.write("BENCHMARKING Plaintext ... ")
           out.flush()
           results = None
+          output_file = self.benchmarker.output_file(self.name, self.PLAINTEXT)
           if self.plaintext_url_passed:
             remote_script = self.__generate_concurrency_script(self.plaintext_url, self.port, self.accept_plaintext, wrk_command="wrk-pipeline", intervals=[256,1024,4096,16384], pipeline="--pipeline 16")
-            self.__run_benchmark(remote_script, self.benchmarker.output_file(self.name, self.PLAINTEXT), err)
-            results = self.__parse_test(self.PLAINTEXT)
-          else:
-            results = dict()
-            results['results'] = []
-          self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'], passed=self.plaintext_url_passed)
+            self.__run_benchmark(remote_script, output_file, err)
+          results = self.__parse_test(self.PLAINTEXT)
+          self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'])
           out.write( "Complete\n" )
           out.flush()
       except AttributeError:
@@ -575,32 +563,32 @@ class FrameworkTest:
   ############################################################
   def parse_all(self):
     # JSON
-    if os.path.exists(self.benchmarker.output_file(self.name, self.JSON)):
+    if os.path.exists(self.benchmarker.get_output_file(self.name, self.JSON)):
       results = self.__parse_test(self.JSON)
       self.benchmarker.report_results(framework=self, test=self.JSON, results=results['results'])
     
     # DB
-    if os.path.exists(self.benchmarker.output_file(self.name, self.DB)):
+    if os.path.exists(self.benchmarker.get_output_file(self.name, self.DB)):
       results = self.__parse_test(self.DB)
       self.benchmarker.report_results(framework=self, test=self.DB, results=results['results'])
     
     # Query
-    if os.path.exists(self.benchmarker.output_file(self.name, self.QUERY)):
+    if os.path.exists(self.benchmarker.get_output_file(self.name, self.QUERY)):
       results = self.__parse_test(self.QUERY)
       self.benchmarker.report_results(framework=self, test=self.QUERY, results=results['results'])
 
     # Fortune
-    if os.path.exists(self.benchmarker.output_file(self.name, self.FORTUNE)):
+    if os.path.exists(self.benchmarker.get_output_file(self.name, self.FORTUNE)):
       results = self.__parse_test(self.FORTUNE)
       self.benchmarker.report_results(framework=self, test=self.FORTUNE, results=results['results'])
 
     # Update
-    if os.path.exists(self.benchmarker.output_file(self.name, self.UPDATE)):
+    if os.path.exists(self.benchmarker.get_output_file(self.name, self.UPDATE)):
       results = self.__parse_test(self.UPDATE)
       self.benchmarker.report_results(framework=self, test=self.UPDATE, results=results['results'])
 
     # Plaintext
-    if os.path.exists(self.benchmarker.output_file(self.name, self.PLAINTEXT)):
+    if os.path.exists(self.benchmarker.get_output_file(self.name, self.PLAINTEXT)):
       results = self.__parse_test(self.PLAINTEXT)
       self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'])
   ############################################################
@@ -615,82 +603,83 @@ class FrameworkTest:
       results = dict()
       results['results'] = []
       
-      with open(self.benchmarker.output_file(self.name, test_type)) as raw_data:
-        is_warmup = True
-        rawData = None
-        for line in raw_data:
-
-          if "Queries:" in line or "Concurrency:" in line:
-            is_warmup = False
-            rawData = None
-            continue
-          if "Warmup" in line or "Primer" in line:
-            is_warmup = True
-            continue
-
-          if not is_warmup:
-            if rawData == None:
-              rawData = dict()
-              results['results'].append(rawData)
-
-            #if "Requests/sec:" in line:
-            #  m = re.search("Requests/sec:\s+([0-9]+)", line)
-            #  rawData['reportedResults'] = m.group(1)
+      if os.path.exists(self.benchmarker.get_output_file(self.name, test_type)):
+        with open(self.benchmarker.output_file(self.name, test_type)) as raw_data:
+          is_warmup = True
+          rawData = None
+          for line in raw_data:
+
+            if "Queries:" in line or "Concurrency:" in line:
+              is_warmup = False
+              rawData = None
+              continue
+            if "Warmup" in line or "Primer" in line:
+              is_warmup = True
+              continue
+
+            if not is_warmup:
+              if rawData == None:
+                rawData = dict()
+                results['results'].append(rawData)
+
+              #if "Requests/sec:" in line:
+              #  m = re.search("Requests/sec:\s+([0-9]+)", line)
+              #  rawData['reportedResults'] = m.group(1)
+                
+              # search for weighttp data such as succeeded and failed.
+              if "Latency" in line:
+                m = re.findall("([0-9]+\.*[0-9]*[us|ms|s|m|%]+)", line)
+                if len(m) == 4:
+                  rawData['latencyAvg'] = m[0]
+                  rawData['latencyStdev'] = m[1]
+                  rawData['latencyMax'] = m[2]
+              #    rawData['latencyStdevPercent'] = m[3]
               
-            # search for weighttp data such as succeeded and failed.
-            if "Latency" in line:
-              m = re.findall("([0-9]+\.*[0-9]*[us|ms|s|m|%]+)", line)
-              if len(m) == 4:
-                rawData['latencyAvg'] = m[0]
-                rawData['latencyStdev'] = m[1]
-                rawData['latencyMax'] = m[2]
-            #    rawData['latencyStdevPercent'] = m[3]
-            
-            #if "Req/Sec" in line:
-            #  m = re.findall("([0-9]+\.*[0-9]*[k|%]*)", line)
-            #  if len(m) == 4:
-            #    rawData['requestsAvg'] = m[0]
-            #    rawData['requestsStdev'] = m[1]
-            #    rawData['requestsMax'] = m[2]
-            #    rawData['requestsStdevPercent'] = m[3]
+              #if "Req/Sec" in line:
+              #  m = re.findall("([0-9]+\.*[0-9]*[k|%]*)", line)
+              #  if len(m) == 4:
+              #    rawData['requestsAvg'] = m[0]
+              #    rawData['requestsStdev'] = m[1]
+              #    rawData['requestsMax'] = m[2]
+              #    rawData['requestsStdevPercent'] = m[3]
+                
+              #if "requests in" in line:
+              #  m = re.search("requests in ([0-9]+\.*[0-9]*[ms|s|m|h]+)", line)
+              #  if m != None: 
+              #    # parse out the raw time, which may be in minutes or seconds
+              #    raw_time = m.group(1)
+              #    if "ms" in raw_time:
+              #      rawData['total_time'] = float(raw_time[:len(raw_time)-2]) / 1000.0
+              #    elif "s" in raw_time:
+              #      rawData['total_time'] = float(raw_time[:len(raw_time)-1])
+              #    elif "m" in raw_time:
+              #      rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 60.0
+              #    elif "h" in raw_time:
+              #      rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 3600.0
+             
+              if "requests in" in line:
+                m = re.search("([0-9]+) requests in", line)
+                if m != None: 
+                  rawData['totalRequests'] = int(m.group(1))
               
-            #if "requests in" in line:
-            #  m = re.search("requests in ([0-9]+\.*[0-9]*[ms|s|m|h]+)", line)
-            #  if m != None: 
-            #    # parse out the raw time, which may be in minutes or seconds
-            #    raw_time = m.group(1)
-            #    if "ms" in raw_time:
-            #      rawData['total_time'] = float(raw_time[:len(raw_time)-2]) / 1000.0
-            #    elif "s" in raw_time:
-            #      rawData['total_time'] = float(raw_time[:len(raw_time)-1])
-            #    elif "m" in raw_time:
-            #      rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 60.0
-            #    elif "h" in raw_time:
-            #      rawData['total_time'] = float(raw_time[:len(raw_time)-1]) * 3600.0
-           
-            if "requests in" in line:
-              m = re.search("([0-9]+) requests in", line)
-              if m != None: 
-                rawData['totalRequests'] = int(m.group(1))
-            
-            if "Socket errors" in line:
-              if "connect" in line:
-                m = re.search("connect ([0-9]+)", line)
-                rawData['connect'] = int(m.group(1))
-              if "read" in line:
-                m = re.search("read ([0-9]+)", line)
-                rawData['read'] = int(m.group(1))
-              if "write" in line:
-                m = re.search("write ([0-9]+)", line)
-                rawData['write'] = int(m.group(1))
-              if "timeout" in line:
-                m = re.search("timeout ([0-9]+)", line)
-                rawData['timeout'] = int(m.group(1))
-            
-            if "Non-2xx" in line:
-              m = re.search("Non-2xx or 3xx responses: ([0-9]+)", line)
-              if m != None: 
-                rawData['5xx'] = int(m.group(1))
+              if "Socket errors" in line:
+                if "connect" in line:
+                  m = re.search("connect ([0-9]+)", line)
+                  rawData['connect'] = int(m.group(1))
+                if "read" in line:
+                  m = re.search("read ([0-9]+)", line)
+                  rawData['read'] = int(m.group(1))
+                if "write" in line:
+                  m = re.search("write ([0-9]+)", line)
+                  rawData['write'] = int(m.group(1))
+                if "timeout" in line:
+                  m = re.search("timeout ([0-9]+)", line)
+                  rawData['timeout'] = int(m.group(1))
+              
+              if "Non-2xx" in line:
+                m = re.search("Non-2xx or 3xx responses: ([0-9]+)", line)
+                if m != None: 
+                  rawData['5xx'] = int(m.group(1))
               
 
       return results