Docker (#3327)

benchmark.cfg.example

@@ -26,7 +26,6 @@ test=None
 type=all
 verbose=True
 clean=False
-clean_all=False
 ulimit=200000
 #results_name=My Benchmarking Run %%Y-%%m-%%d %%H:%%M:%%S
 #results_environment=My Server Environment

deployment/vagrant/bootstrap.sh

@@ -65,7 +65,6 @@ test=None
 type=all
 verbose=True
 clean=False
-clean_all=False
 ulimit=200000
 EOF
 

toolset/benchmark/benchmarker.py

@@ -23,6 +23,7 @@ import logging
 import socket
 import threading
 import textwrap
+import shutil
 from pprint import pprint
 
 from contextlib import contextmanager
@@ -43,6 +44,20 @@ class Benchmarker:
     # Public methods
     ##########################################################################################
 
+    def clean_all(self):
+        if os.path.exists(self.results_directory):
+            for file in os.listdir(self.results_directory):
+                if not os.path.exists(os.path.dirname(file)):
+                    shutil.rmtree(os.path.join(self.results_directory, file))
+                else:
+                    os.remove(os.path.join(self.results_directory, file))
+
+        subprocess.check_call(["docker", "image", "prune", "-f"])
+
+        docker_ids = subprocess.check_output(["docker", "images", "-q"]).splitlines()
+        for docker_id in docker_ids:
+            subprocess.check_call(["docker", "image", "rmi", "-f", docker_id])
+
     ############################################################
     # Prints all the available tests
     ############################################################
@@ -205,7 +220,7 @@ class Benchmarker:
     # test_type timestamp/test_type/test_name/raw.txt
     ############################################################
     def get_output_file(self, test_name, test_type):
-        return os.path.join(self.result_directory, self.timestamp, test_name, test_type, "raw.txt")
+        return os.path.join(self.results_directory, self.timestamp, test_name, test_type, "raw.txt")
     ############################################################
     # End get_output_file
     ############################################################
@@ -233,7 +248,7 @@ class Benchmarker:
     # test_type timestamp/test_type/test_name/stats.txt
     ############################################################
     def get_stats_file(self, test_name, test_type):
-        return os.path.join(self.result_directory, self.timestamp, test_name, test_type, "stats.txt")
+        return os.path.join(self.results_directory, self.timestamp, test_name, test_type, "stats.txt")
     ############################################################
     # End get_stats_file
     ############################################################
@@ -260,7 +275,7 @@ class Benchmarker:
     # full_results_directory
     ############################################################
     def full_results_directory(self):
-        path = os.path.join(self.fwroot, self.result_directory, self.timestamp)
+        path = os.path.join(self.fwroot, self.results_directory, self.timestamp)
         try:
             os.makedirs(path)
         except OSError:
@@ -742,7 +757,9 @@ class Benchmarker:
                         docker_id = subprocess.check_output(["docker", "ps", "-q"]).strip()
                     # We still need to sleep a bit before removing the image
                     time.sleep(5)
-                    subprocess.check_output(["docker", "image", "rm", "tfb/test/%s" % test.name])
+                    subprocess.check_call(["docker", "image", "rm", "tfb/test/%s" % test.name])
+                    time.sleep(5)
+                    subprocess.check_call(["docker", "image", "prune", "-f"])
     ############################################################
     # End __stop_test
     ############################################################
@@ -930,10 +947,10 @@ class Benchmarker:
             pass
 
     def __get_git_commit_id(self):
-        return subprocess.check_output('git rev-parse HEAD', shell=True).strip()
+        return subprocess.check_output(["git", "rev-parse", "HEAD"]).strip()
 
     def __get_git_repository_url(self):
-        return subprocess.check_output('git config --get remote.origin.url', shell=True).strip()
+        return subprocess.check_output(["git", "config", "--get", "remote.origin.url"]).strip()
 
     ############################################################
     # __finish
@@ -963,7 +980,7 @@ class Benchmarker:
             print(prefix + header('', top='', bottom='=') + Style.RESET_ALL)
 
         print("Time to complete: " + str(int(time.time() - self.start_time)) + " seconds")
-        print("Results are saved in " + os.path.join(self.result_directory, self.timestamp))
+        print("Results are saved in " + os.path.join(self.results_directory, self.timestamp))
 
     ############################################################
     # End __finish
@@ -1001,7 +1018,6 @@ class Benchmarker:
             args['pipeline_concurrency_levels'] = [256,1024,4096,16384]
 
         self.__dict__.update(args)
-        # pprint(self.__dict__)
 
         self.quiet_out = QuietOutputStream(self.quiet)
 
@@ -1028,16 +1044,7 @@ class Benchmarker:
             self.timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
 
         # setup results and latest_results directories
-        self.result_directory = os.path.join(self.fwroot, "results")
-        if (args['clean'] or args['clean_all']) and os.path.exists(os.path.join(self.fwroot, "results")):
-            os.system("sudo rm -rf " + self.result_directory + "/*")
-
-        # TODO: remove this as installs goes away with docker implementation
-        # remove installs directories if --clean-all provided
-        self.install_root = "%s/%s" % (self.fwroot, "installs")
-        if args['clean_all']:
-            os.system("sudo rm -rf " + self.install_root)
-            os.mkdir(self.install_root)
+        self.results_directory = os.path.join(self.fwroot, "results")
 
         self.results = None
         try:

toolset/run-tests.py

@@ -149,9 +149,6 @@ def main(argv=None):
     parser.add_argument('--results-upload-uri', default=None, help='A URI where the in-progress results.json file will be POSTed periodically')
     parser.add_argument('--parse', help='Parses the results of the given timestamp and merges that with the latest results')
 
-    # TODO: remove this; install dir goes away with docker
-    parser.add_argument('--clean-all', action='store_true', dest='clean_all', default=False, help='Removes the results and installs directories')
-
     # Test options
     parser.add_argument('--test', nargs='+', help='names of tests to run')
     parser.add_argument('--test-dir', nargs='+', dest='test_dir', help='name of framework directory containing all tests to run')
@@ -177,6 +174,10 @@ def main(argv=None):
 
     benchmarker = Benchmarker(vars(args))
 
+    if args.clean:
+        benchmarker.clean_all()
+        return 0
+
     # Run the benchmarker in the specified mode
     #   Do not use benchmarker variables for these checks,
     #   they are either str or bool based on the python version