Merge pull request #1049 from hamiltont/toolset-423

Toolset: Add verification summary

.travis.yml

@@ -152,19 +152,23 @@ env:
     - "TESTDIR=Ur/urweb"
 
 before_install:
+  # Need to install python modules before using 
+  # python
+  - pip install -r config/python_requirements.txt
+
   # Configure Travis-CI build environment for TFB
   #   e.g. setup databases, users, etc
-  - time ./toolset/run-ci.py cisetup "$TESTDIR"
+  - ./toolset/run-ci.py cisetup "$TESTDIR"
 
 addons:
   postgresql: "9.3" 
 
 install:
   # Install prerequisites
-  - time ./toolset/run-ci.py prereq "$TESTDIR"
+  - ./toolset/run-ci.py prereq "$TESTDIR"
   
   # Install software for this framework  
-  - time ./toolset/run-ci.py install "$TESTDIR"
+  - ./toolset/run-ci.py install "$TESTDIR"
    
 script: 
   # Pick one test in this directory and verify

config/python_requirements.txt

@@ -0,0 +1 @@
+colorama

toolset/benchmark/benchmarker.py

@@ -18,6 +18,9 @@ import socket
 from multiprocessing import Process
 from datetime import datetime
 
+# Cross-platform colored text
+from colorama import Fore, Back, Style
+
 class Benchmarker:
 
   ##########################################################################################
@@ -203,32 +206,6 @@ class Benchmarker:
   # End output_file
   ############################################################
 
-  ############################################################
-  # get_warning_file(test_name, test_type)
-  # returns the output file name for this test_name and 
-  # test_type timestamp/test_type/test_name/raw 
-  ############################################################
-  def get_warning_file(self, test_name, test_type):
-    return os.path.join(self.result_directory, self.timestamp, test_type, test_name, "warn")
-  ############################################################
-  # End get_warning_file
-  ############################################################
-
-  ############################################################
-  # warning_file(test_name, test_type)
-  # returns the warning file for this test_name and test_type
-  # timestamp/test_type/test_name/raw 
-  ############################################################
-  def warning_file(self, test_name, test_type):
-    path = self.get_warning_file(test_name, test_type)
-    try:
-      os.makedirs(os.path.dirname(path))
-    except OSError:
-      pass
-    return path
-  ############################################################
-  # End warning_file
-  ############################################################
 
   ############################################################
   # get_stats_file(test_name, test_type)
@@ -286,22 +263,35 @@ class Benchmarker:
     return path
 
   ############################################################
-  # report_results
+  # report_verify_results
+  # Used by FrameworkTest to add verification details to our results
+  #
+  # TODO: Technically this is an IPC violation - we are accessing
+  # the parent process' memory from the child process
   ############################################################
-  def report_results(self, framework, test, results):
+  def report_verify_results(self, framework, test, result):
+    if framework.name not in self.results['verify'].keys():
+      self.results['verify'][framework.name] = dict()
+    self.results['verify'][framework.name][test] = result
+
+  ############################################################
+  # report_benchmark_results
+  # Used by FrameworkTest to add benchmark data to this
+  #
+  # TODO: Technically this is an IPC violation - we are accessing
+  # the parent process' memory from the child process
+  ############################################################
+  def report_benchmark_results(self, framework, test, results):
     if test not in self.results['rawData'].keys():
       self.results['rawData'][test] = dict()
 
     # If results has a size from the parse, then it succeeded.
     if results:
       self.results['rawData'][test][framework.name] = results
+
       # This may already be set for single-tests
       if framework.name not in self.results['succeeded'][test]:
         self.results['succeeded'][test].append(framework.name)
-      # Add this type
-      if (os.path.exists(self.get_warning_file(framework.name, test)) and
-          framework.name not in self.results['warning'][test]):
-        self.results['warning'][test].append(framework.name)
     else:
       # This may already be set for single-tests
       if framework.name not in self.results['failed'][test]:
@@ -804,6 +794,29 @@ class Benchmarker:
   # __finish
   ############################################################
   def __finish(self):
+    if self.mode == "verify":
+      tests = self.__gather_tests
+      # Normally you don't have to use Fore.BLUE before each line, but 
+      # Travis-CI seems to reset color codes on newline (see travis-ci/travis-ci#2692)
+      # or stream flush, so we have to ensure that the color code is printed repeatedly
+      prefix = Fore.CYAN
+      for line in header("Verification Summary", top='=', bottom='').split('\n'):
+        print prefix + line
+      for test in tests:
+        print prefix + "| Test: %s" % test.name
+        if test.name in self.results['verify'].keys():
+          for test_type, result in self.results['verify'][test.name].iteritems():
+            if result.upper() == "PASS":
+              color = Fore.GREEN
+            elif result.upper() == "WARN":
+              color = Fore.YELLOW
+            else:
+              color = Fore.RED
+            print prefix + "|       " + test_type.ljust(11) + ' : ' + color + result.upper()
+        else:
+          print prefix + "|      " + Fore.RED + "NO RESULTS (Did framework launch?)"
+      print prefix + header('', top='', bottom='=') + Style.RESET_ALL
+
     print "Time to complete: " + str(int(time.time() - self.start_time)) + " seconds"
     print "Results are saved in " + os.path.join(self.result_directory, self.timestamp)
 
@@ -921,13 +934,7 @@ class Benchmarker:
       self.results['failed']['fortune'] = []
       self.results['failed']['update'] = []
       self.results['failed']['plaintext'] = []
-      self.results['warning'] = dict()
-      self.results['warning']['json'] = []
-      self.results['warning']['db'] = []
-      self.results['warning']['query'] = []
-      self.results['warning']['fortune'] = []
-      self.results['warning']['update'] = []
-      self.results['warning']['plaintext'] = []
+      self.results['verify'] = dict()
     else:
       #for x in self.__gather_tests():
       #  if x.name not in self.results['frameworks']:

toolset/benchmark/framework_test.py

@@ -541,9 +541,11 @@ class FrameworkTest:
       if ret_tuple[0]:
         self.json_url_passed = True
         out.write("PASS\n\n")
+        self.benchmarker.report_verify_results(self, self.JSON, 'pass')
       else:
         self.json_url_passed = False
         out.write("\nFAIL" + ret_tuple[1] + "\n\n")
+        self.benchmarker.report_verify_results(self, self.JSON, 'fail')
         result = False
       out.flush()
 
@@ -564,14 +566,16 @@ class FrameworkTest:
         self.db_url_warn = False
       else:
         self.db_url_warn = True
-
       out.write("VALIDATING DB ... ")
       if self.db_url_passed:
         out.write("PASS")
+        self.benchmarker.report_verify_results(self, self.DB, 'pass')
         if self.db_url_warn:
           out.write(" (with warnings) " + validate_strict_ret_tuple[1])
+          self.benchmarker.report_verify_results(self, self.DB, 'warn')
         out.write("\n\n")
       else:
+        self.benchmarker.report_verify_results(self, self.DB, 'fail')
         out.write("\nFAIL" + validate_ret_tuple[1])
         result = False
       out.flush()
@@ -630,11 +634,14 @@ class FrameworkTest:
       out.write("VALIDATING QUERY ... ")
       if self.query_url_passed:
         out.write("PASS")
+        self.benchmarker.report_verify_results(self, self.QUERY, 'pass')
         if self.query_url_warn:
           out.write(" (with warnings)")
+          self.benchmarker.report_verify_results(self, self.QUERY, 'warn')
         out.write("\n\n")
       else:
         out.write("\nFAIL " + ret_tuple[1] + "\n\n")
+        self.benchmarker.report_verify_results(self, self.QUERY, 'fail')
         result = False
       out.flush()
 
@@ -649,9 +656,11 @@ class FrameworkTest:
       if self.validateFortune(output, out, err):
         self.fortune_url_passed = True
         out.write("PASS\n\n")
+        self.benchmarker.report_verify_results(self, self.FORTUNE, 'pass')
       else:
         self.fortune_url_passed = False
         out.write("\nFAIL\n\n")
+        self.benchmarker.report_verify_results(self, self.FORTUNE, 'fail')
         result = False
       out.flush()
 
@@ -667,9 +676,11 @@ class FrameworkTest:
       if ret_tuple[0]:
         self.update_url_passed = True
         out.write("PASS\n\n")
+        self.benchmarker.report_verify_results(self, self.UPDATE, 'pass')
       else:
         self.update_url_passed = False
         out.write("\nFAIL " + ret_tuple[1] + "\n\n")
+        self.benchmarker.report_verify_results(self, self.UPDATE, 'fail')
         result = False
       out.flush()
 
@@ -685,9 +696,11 @@ class FrameworkTest:
       if ret_tuple[0]:
         self.plaintext_url_passed = True
         out.write("PASS\n\n")
+        self.benchmarker.report_verify_results(self, self.PLAINTEXT, 'pass')
       else:
         self.plaintext_url_passed = False
         out.write("\nFAIL\n\n" + ret_tuple[1] + "\n\n")
+        self.benchmarker.report_verify_results(self, self.PLAINTEXT, 'fail')
         result = False
       out.flush()
 
@@ -747,7 +760,7 @@ class FrameworkTest:
           self.__end_logging()
         results = self.__parse_test(self.JSON)
         print results
-        self.benchmarker.report_results(framework=self, test=self.JSON, results=results['results'])
+        self.benchmarker.report_benchmark_results(framework=self, test=self.JSON, results=results['results'])
         out.write( "Complete\n" )
         out.flush()
       except AttributeError:
@@ -760,21 +773,18 @@ class FrameworkTest:
         out.flush()
         results = None
         output_file = self.benchmarker.output_file(self.name, self.DB)
-        warning_file = self.benchmarker.warning_file(self.name, self.DB)
         if not os.path.exists(output_file):
           with open(output_file, 'w'):
             # Simply opening the file in write mode should create the empty file.
             pass
-        if self.db_url_warn:
-          with open(warning_file, 'w'):
-            pass
         if self.db_url_passed:
+          self.benchmarker.report_verify_results(self, self.DB, 'pass')
           remote_script = self.__generate_concurrency_script(self.db_url, self.port, self.accept_json)
           self.__begin_logging(self.DB)
           self.__run_benchmark(remote_script, output_file, err)
           self.__end_logging()
         results = self.__parse_test(self.DB)
-        self.benchmarker.report_results(framework=self, test=self.DB, results=results['results'])
+        self.benchmarker.report_benchmark_results(framework=self, test=self.DB, results=results['results'])
         out.write( "Complete\n" )
       except AttributeError:
         pass
@@ -786,21 +796,19 @@ class FrameworkTest:
         out.flush()
         results = None
         output_file = self.benchmarker.output_file(self.name, self.QUERY)
-        warning_file = self.benchmarker.warning_file(self.name, self.QUERY)
         if not os.path.exists(output_file):
           with open(output_file, 'w'):
             # Simply opening the file in write mode should create the empty file.
             pass
         if self.query_url_warn:
-          with open(warning_file, 'w'):
-            pass
+          self.benchmarker.report_verify_results(framework=self, test=self.QUERY, results=None)
         if self.query_url_passed:
           remote_script = self.__generate_query_script(self.query_url, self.port, self.accept_json)
           self.__begin_logging(self.QUERY)
           self.__run_benchmark(remote_script, output_file, err)
           self.__end_logging()
         results = self.__parse_test(self.QUERY)
-        self.benchmarker.report_results(framework=self, test=self.QUERY, results=results['results'])
+        self.benchmarker.report_benchmark_results(framework=self, test=self.QUERY, results=results['results'])
         out.write( "Complete\n" )
         out.flush()
       except AttributeError:
@@ -823,7 +831,7 @@ class FrameworkTest:
           self.__run_benchmark(remote_script, output_file, err)
           self.__end_logging()
         results = self.__parse_test(self.FORTUNE)
-        self.benchmarker.report_results(framework=self, test=self.FORTUNE, results=results['results'])
+        self.benchmarker.report_benchmark_results(framework=self, test=self.FORTUNE, results=results['results'])
         out.write( "Complete\n" )
         out.flush()
       except AttributeError:
@@ -846,7 +854,7 @@ class FrameworkTest:
           self.__run_benchmark(remote_script, output_file, err)
           self.__end_logging()
         results = self.__parse_test(self.UPDATE)
-        self.benchmarker.report_results(framework=self, test=self.UPDATE, results=results['results'])
+        self.benchmarker.report_benchmark_results(framework=self, test=self.UPDATE, results=results['results'])
         out.write( "Complete\n" )
         out.flush()
       except AttributeError:
@@ -869,7 +877,7 @@ class FrameworkTest:
           self.__run_benchmark(remote_script, output_file, err)
           self.__end_logging()
         results = self.__parse_test(self.PLAINTEXT)
-        self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'])
+        self.benchmarker.report_benchmark_results(framework=self, test=self.PLAINTEXT, results=results['results'])
         out.write( "Complete\n" )
         out.flush()
       except AttributeError:
@@ -888,32 +896,32 @@ class FrameworkTest:
     # JSON
     if os.path.exists(self.benchmarker.get_output_file(self.name, self.JSON)):
       results = self.__parse_test(self.JSON)
-      self.benchmarker.report_results(framework=self, test=self.JSON, results=results['results'])
+      self.benchmarker.report_benchmark_results(framework=self, test=self.JSON, results=results['results'])
     
     # DB
     if os.path.exists(self.benchmarker.get_output_file(self.name, self.DB)):
       results = self.__parse_test(self.DB)
-      self.benchmarker.report_results(framework=self, test=self.DB, results=results['results'])
+      self.benchmarker.report_benchmark_results(framework=self, test=self.DB, results=results['results'])
     
     # Query
     if os.path.exists(self.benchmarker.get_output_file(self.name, self.QUERY)):
       results = self.__parse_test(self.QUERY)
-      self.benchmarker.report_results(framework=self, test=self.QUERY, results=results['results'])
+      self.benchmarker.report_benchmark_results(framework=self, test=self.QUERY, results=results['results'])
 
     # Fortune
     if os.path.exists(self.benchmarker.get_output_file(self.name, self.FORTUNE)):
       results = self.__parse_test(self.FORTUNE)
-      self.benchmarker.report_results(framework=self, test=self.FORTUNE, results=results['results'])
+      self.benchmarker.report_benchmark_results(framework=self, test=self.FORTUNE, results=results['results'])
 
     # Update
     if os.path.exists(self.benchmarker.get_output_file(self.name, self.UPDATE)):
       results = self.__parse_test(self.UPDATE)
-      self.benchmarker.report_results(framework=self, test=self.UPDATE, results=results['results'])
+      self.benchmarker.report_benchmark_results(framework=self, test=self.UPDATE, results=results['results'])
 
     # Plaintext
     if os.path.exists(self.benchmarker.get_output_file(self.name, self.PLAINTEXT)):
       results = self.__parse_test(self.PLAINTEXT)
-      self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'])
+      self.benchmarker.report_benchmark_results(framework=self, test=self.PLAINTEXT, results=results['results'])
   ############################################################
   # End parse_all
   ############################################################

toolset/benchmark/utils.py

@@ -87,4 +87,17 @@ def header(message, top='-', bottom='-'):
     '''
     topheader = (top * 80)[:80]
     bottomheader = (bottom * 80)[:80]
-    return "%s\n  %s\n%s" % (topheader, message, bottomheader)
+    result = ""
+    if topheader != "":
+      result += "%s" % topheader
+    if message != "":
+      if result == "":
+        result = "  %s" % message
+      else:
+        result += "\n  %s" % message
+    if bottomheader != "":
+      if result == "":
+        result = "%s" % bottomheader
+      else:
+        result += "\n%s" % bottomheader
+    return result

toolset/run-ci.py

@@ -13,6 +13,10 @@ import time
 import threading
 from benchmark import framework_test
 from benchmark.utils import gather_tests
+from benchmark.utils import header
+
+# Cross-platform colored text
+from colorama import Fore, Back, Style
 
 # Needed for various imports
 sys.path.append('.')
@@ -427,6 +431,44 @@ if __name__ == "__main__":
       except IOError:
         log.error("No OUT file found")
 
+    log.error("Running inside Travis-CI, so I will print a copy of the verification summary")
+
+    results = None
+    try:
+      with open('results/ec2/latest/results.json', 'r') as f:
+        results = json.load(f)
+    except IOError:
+      log.critical("No results.json found, unable to print verification summary") 
+      sys.exit(retcode)
+
+    target_dir = setup_util.get_fwroot() + '/frameworks/' + testdir
+    dirtests = [t for t in gather_tests() if t.directory == target_dir]
+
+    # Normally you don't have to use Fore.* before each line, but 
+    # Travis-CI seems to reset color codes on newline (see travis-ci/travis-ci#2692)
+    # or stream flush, so we have to ensure that the color code is printed repeatedly
+    prefix = Fore.CYAN
+    for line in header("Verification Summary", top='=', bottom='').split('\n'):
+      print prefix + line
+
+    for test in dirtests:
+      print prefix + "| Test: %s" % test.name
+      if test.name not in runner.names:
+        print prefix + "|      " + Fore.YELLOW + "Unable to verify in Travis-CI"
+      elif test.name in results['verify'].keys():
+        for test_type, result in results['verify'][test.name].iteritems():
+          if result.upper() == "PASS":
+            color = Fore.GREEN
+          elif result.upper() == "WARN":
+            color = Fore.YELLOW
+          else:
+            color = Fore.RED
+          print prefix + "|       " + test_type.ljust(11) + ' : ' + color + result.upper()
+      else:
+        print prefix + "|      " + Fore.RED + "NO RESULTS (Did framework launch?)"
+    print prefix + header('', top='', bottom='=') + Style.RESET_ALL
+
+
     sys.exit(retcode)
 
 

toolset/run-tests.py

@@ -10,6 +10,10 @@ from benchmark.benchmarker import Benchmarker
 from setup.linux.unbuffered import Unbuffered
 from setup.linux import setup_util
 
+# Enable cross-platform colored output
+from colorama import init
+init()
+
 ###################################################################################################
 # Main
 ###################################################################################################