Also print verification summary at end of Travis-CI run

toolset/benchmark/benchmarker.py

@@ -815,7 +815,6 @@ class Benchmarker:
             print prefix + "|       " + test_type.ljust(11) + ' : ' + color + result.upper()
         else:
           print prefix + "|      " + Fore.RED + "NO RESULTS (Did framework launch?)"
-          print prefix + "TESTING BLUE IN TRAVIS"
       print prefix + header('', top='', bottom='=') + Style.RESET_ALL
 
     print "Time to complete: " + str(int(time.time() - self.start_time)) + " seconds"

toolset/run-ci.py

@@ -13,6 +13,10 @@ import time
 import threading
 from benchmark import framework_test
 from benchmark.utils import gather_tests
+from benchmark.utils import header
+
+# Cross-platform colored text
+from colorama import Fore, Back, Style
 
 # Needed for various imports
 sys.path.append('.')
@@ -427,6 +431,44 @@ if __name__ == "__main__":
       except IOError:
         log.error("No OUT file found")
 
+    log.error("Running inside Travis-CI, so I will print a copy of the verification summary")
+
+    results = None
+    try:
+      with open('results/ec2/latest/results.json', 'r') as f:
+        results = json.load(f)
+    except IOError:
+      log.critical("No results.json found, unable to print verification summary") 
+      sys.exit(retcode)
+
+    target_dir = setup_util.get_fwroot() + '/frameworks/' + testdir
+    dirtests = [t for t in gather_tests() if t.directory == target_dir]
+
+    # Normally you don't have to use Fore.* before each line, but 
+    # Travis-CI seems to reset color codes on newline (see travis-ci/travis-ci#2692)
+    # or stream flush, so we have to ensure that the color code is printed repeatedly
+    prefix = Fore.CYAN
+    for line in header("Verification Summary", top='=', bottom='').split('\n'):
+      print prefix + line
+
+    for test in dirtests:
+      print prefix + "| Test: %s" % test.name
+      if test.name not in runner.names:
+        print prefix + "|      " + Fore.YELLOW + "Unable to verify in Travis-CI"
+      elif test.name in results['verify'].keys():
+        for test_type, result in results['verify'][test.name].iteritems():
+          if result.upper() == "PASS":
+            color = Fore.GREEN
+          elif result.upper() == "WARN":
+            color = Fore.YELLOW
+          else:
+            color = Fore.RED
+          print prefix + "|       " + test_type.ljust(11) + ' : ' + color + result.upper()
+      else:
+        print prefix + "|      " + Fore.RED + "NO RESULTS (Did framework launch?)"
+    print prefix + header('', top='', bottom='=') + Style.RESET_ALL
+
+
     sys.exit(retcode)