Optionally upload results to another server (#2593)

This adds another benchmark.cfg / command line parameter named
"results_upload_uri".  If specified, the benchmarker will attempt to
POST the (in-progress) results.json file to that URI between each test.

Additionally, if a "TFB_UPLOADURI" environment variable is defined, then
the continuous benchmarking scripts will attempt to POST the full
results.zip (basically the full results directory, including log files)
to that URI after each full iteration of the entire test suite.

Internally (in the server environments that TechEmpower manages), these
URIs are intended to point at a new, standalone web application (which
is currently in development and not open source) designed to help us
collect and analyze the results.

benchmark.cfg.example

@@ -28,3 +28,4 @@ clean=False
 clean_all=False
 #results_name=My Benchmarking Run %%Y-%%m-%%d %%H:%%M:%%S
 #results_environment=My Server Environment
+#results_upload_uri=https://example.com/uploads

toolset/benchmark/benchmarker.py

@@ -12,6 +12,7 @@ import uuid
 import shutil
 import stat
 import json
+import requests
 import subprocess
 import traceback
 import time
@@ -658,6 +659,12 @@ class Benchmarker:
                 out.flush()
                 self.__write_intermediate_results(test.name,time.strftime("%Y%m%d%H%M%S", time.localtime()))
 
+                ##########################################################
+                # Upload the results thus far to another server (optional)
+                ##########################################################
+
+                self.__upload_results()
+
                 if self.mode == "verify" and not passed_verify:
                     print "Failed verify!"
                     return exit_with_code(1)
@@ -905,6 +912,13 @@ class Benchmarker:
         self.results['completionTime'] = int(round(time.time() * 1000))
         self.__write_results()
 
+    def __upload_results(self):
+        if self.results_upload_uri != None:
+            try:
+                requests.post(self.results_upload_uri, headers={ 'Content-Type': 'application/json' }, data=json.dumps(self.results, indent=2))
+            except (Exception):
+                logging.error("Error uploading results.json")
+
     def __load_results(self):
         try:
             with open(os.path.join(self.full_results_directory(), 'results.json')) as f:

toolset/lifecycle/post-run-tests/keep-logs.py

@@ -7,6 +7,7 @@
 import os
 import zipfile
 import datetime
+import requests
 import shutil
 # Follows closely from:
 # http://stackoverflow.com/a/34153816
@@ -25,10 +26,18 @@ path_out = os.path.abspath(os.path.join(os.environ['TFB_LOGSFOLDER'], \
 if not os.path.exists(path_out):
   os.makedirs(path_out)
 
-zip_file = zipfile.ZipFile(path_out + '/results.zip', 'w', zipfile.ZIP_DEFLATED)
+zip_path = path_out + '/results.zip'
+
+zip_file = zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED)
 
 for root, dirs, files in os.walk(path_in):
   for file in files:
     zip_file.write(os.path.join(root, file))
 
 zip_file.close()
+
+results_upload_uri = os.environ['TFB_UPLOADURI']
+
+if results_upload_uri != None:
+    with open(zip_path, 'rb') as file_to_upload:
+        requests.post(results_upload_uri, headers={ 'Content-Type': 'application/zip' }, data=file_to_upload)

toolset/run-tests.py

@@ -158,6 +158,7 @@ def main(argv=None):
     # Misc Options
     parser.add_argument('--results-name', help='Gives a name to this set of results, formatted as a date', default='(unspecified, datetime = %Y-%m-%d %H:%M:%S)')
     parser.add_argument('--results-environment', help='Describes the environment in which these results were gathered', default='(unspecified, hostname = %s)' % socket.gethostname())
+    parser.add_argument('--results-upload-uri', default=None, help='A URI where the in-progress results.json file will be POSTed periodically')
     parser.add_argument('--parse', help='Parses the results of the given timestamp and merges that with the latest results')
     parser.add_argument('-v', '--verbose', action='store_true', default=False, help='Causes the configuration to print before any other commands are executed.')
     parser.add_argument('--quiet', action='store_true', default=False, help='Only print a limited set of messages to stdout, keep the bulk of messages in log files only')