Fixed a bug on multi-env (#3413)

toolset/benchmark/benchmarker.py

@@ -13,8 +13,6 @@ import json
 import shlex
 from pprint import pprint
 
-from multiprocessing import Process
-
 
 class Benchmarker:
     def __init__(self, config, results):
@@ -54,7 +52,7 @@ class Benchmarker:
 
         # Run tests
         print(header("Running Tests...", top='=', bottom='='))
-        result = self.__run_tests(all_tests)
+        self.__run_tests(all_tests)
 
         # Parse results
         if self.config.mode == "benchmark":
@@ -64,7 +62,6 @@ class Benchmarker:
         self.results.set_completion_time()
         self.results.upload()
         self.results.finish()
-        return result
 
     ##########################################################################################
     # Private methods
@@ -285,47 +282,16 @@ class Benchmarker:
         logging.debug("Start __run_tests.")
         logging.debug("__name__ = %s", __name__)
 
-        error_happened = False
-        if self.config.os.lower() == 'windows':
-            logging.debug("Executing __run_tests on Windows")
-            for test in tests:
-                with self.config.quiet_out.enable():
-                    if self.__run_test(test) != 0:
-                        error_happened = True
-        else:
-            logging.debug("Executing __run_tests on Linux")
-
-            # These features do not work on Windows
-            for test in tests:
-                print(header("Running Test: %s" % test.name))
-                with self.config.quiet_out.enable():
-                    test_process = Process(
-                        target=self.__run_test,
-                        name="Test Runner (%s)" % test.name,
-                        args=(test, ))
-                    test_process.start()
-                    test_process.join(self.config.run_test_timeout_seconds)
-                # Load intermediate result from child process
-                self.results.load()
-                if (test_process.is_alive()):
-                    logging.debug(
-                        "Child process for {name} is still alive. Terminating.".
-                        format(name=test.name))
-                    self.results.write_intermediate(
-                        test.name, "__run_test timeout (=" +
-                        str(self.config.run_test_timeout_seconds) +
-                        " seconds)")
-                    test_process.terminate()
-                    test_process.join()
-                if test_process.exitcode != 0:
-                    error_happened = True
+        # These features do not work on Windows
+        for test in tests:
+            print(header("Running Test: %s" % test.name))
+            with self.config.quiet_out.enable():
+                self.__run_test(test)
+            # Load intermediate result from child process
+            self.results.load()
 
         logging.debug("End __run_tests.")
 
-        if error_happened:
-            return 1
-        return 0
-
     def __run_test(self, test):
         '''
         Ensures that the system has all necessary software to run the tests. 
@@ -344,14 +310,14 @@ class Benchmarker:
                 out.write(
                     "OS or Database OS specified in benchmark_config.json does not match the current environment. Skipping.\n"
                 )
-                return sys.exit(0)
+                return
 
             # If the test is in the excludes list, we skip it
             if self.config.exclude != None and test.name in self.config.exclude:
                 out.write(
                     "Test {name} has been added to the excludes list. Skipping.\n".
                     format(name=test.name))
-                return sys.exit(0)
+                return
 
             database_container_id = None
             try:
@@ -368,7 +334,7 @@ class Benchmarker:
                             % (test.port, test.name)))
                     out.flush()
                     print("Error: Unable to recover port, cannot start test")
-                    return sys.exit(1)
+                    return
 
                 # Start database container
                 if test.database.lower() != "none":
@@ -381,33 +347,32 @@ class Benchmarker:
                         out.flush()
                         self.results.write_intermediate(
                             test.name, "ERROR: Problem starting")
-                        return sys.exit(1)
+                        return
 
                 # Start webapp
                 result = test.start(out, database_container_id)
                 if result != 0:
                     docker_helper.stop(self.config, database_container_id,
-                                       test, out)
+                                       test)
                     out.write("ERROR: Problem starting {name}\n".format(
                         name=test.name))
                     out.flush()
                     self.results.write_intermediate(test.name,
                                                     "ERROR: Problem starting")
-                    return sys.exit(1)
+                    return
 
                 slept = 0
                 max_sleep = 60
                 while not test.is_running() and slept < max_sleep:
                     if not docker_helper.successfully_running_containers(
-                            test.get_docker_files(), database_container_id,
-                            out):
+                            test.get_docker_files(), out):
                         docker_helper.stop(self.config, database_container_id,
-                                           test, out)
+                                           test)
                         tee_output(
                             out,
                             "ERROR: One or more expected docker container exited early"
                             + os.linesep)
-                        return sys.exit(1)
+                        return
                     time.sleep(1)
                     slept += 1
 
@@ -431,8 +396,7 @@ class Benchmarker:
                     self.__benchmark(test, logDir)
 
                 # Stop this test
-                docker_helper.stop(self.config, database_container_id, test,
-                                   out)
+                docker_helper.stop(self.config, database_container_id, test)
 
                 # Remove contents of  /tmp folder
                 try:
@@ -455,10 +419,9 @@ class Benchmarker:
 
                 if self.config.mode == "verify" and not passed_verify:
                     print("Failed verify!")
-                    return sys.exit(1)
+                    return
             except KeyboardInterrupt:
-                docker_helper.stop(self.config, database_container_id, test,
-                                   out)
+                docker_helper.stop(self.config, database_container_id, test)
             except (OSError, IOError, subprocess.CalledProcessError) as e:
                 traceback.print_exc()
                 self.results.write_intermediate(
@@ -467,10 +430,9 @@ class Benchmarker:
                 traceback.print_exc(file=out)
                 out.flush()
                 out.close()
-                return sys.exit(1)
+                return
 
             out.close()
-            return sys.exit(0)
 
     def __is_port_bound(self, port):
         '''

toolset/run-tests.py

@@ -5,6 +5,7 @@ import sys
 import os
 import platform
 import multiprocessing
+import signal
 from toolset.benchmark.benchmarker import Benchmarker
 from toolset.utils import setup_util
 from toolset.utils.unbuffered import Unbuffered
@@ -49,6 +50,17 @@ class StoreSeqAction(argparse.Action):
         return [abs(int(item)) for item in result]
 
 
+def __stop(signal, frame):
+    '''
+    Method called on SIGTERM to stop all running containers 
+    '''
+    docker_helper.stop()
+    sys.exit(0)
+
+
+signal.signal(signal.SIGTERM, __stop)
+
+
 ###################################################################################################
 # Main
 ###################################################################################################
@@ -68,12 +80,6 @@ def main(argv=None):
     # Enable unbuffered output so messages will appear in the proper order with subprocess output.
     sys.stdout = Unbuffered(sys.stdout)
 
-    # Update python environment
-    # 1) Ensure the current directory (which should be the benchmark home directory) is in the path so that the tests can be imported.
-    sys.path.append('.')
-    # 2) Ensure toolset/setup/linux is in the path so that the tests can "import setup_util".
-    sys.path.append('toolset/setup/linux')
-
     # Update environment for shell scripts
     os.environ['FWROOT'] = setup_util.get_fwroot()
     os.environ['IROOT'] = os.environ['FWROOT'] + '/installs'
@@ -281,7 +287,7 @@ def main(argv=None):
 
     elif config.clean:
         cleaner.clean(results)
-        docker_helper.clean()
+        docker_helper.clean(config)
 
     elif config.list_tests:
         all_tests = gather_tests(benchmarker_config=config)
@@ -300,7 +306,7 @@ def main(argv=None):
 
     else:
         benchmarker = Benchmarker(config, results)
-        return benchmarker.run()
+        benchmarker.run()
 
     return 0
 

toolset/setup/docker/base.dockerfile

@@ -21,6 +21,7 @@ RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8  
 ENV LANGUAGE en_US:en  
 ENV LC_ALL en_US.UTF-8 
+ENV DEBIAN_FRONTEND noninteractive
 
 ARG CPU_COUNT
 ARG MAX_CONCURRENCY

toolset/setup/linux/prerequisites.sh

@@ -58,10 +58,8 @@ sudo sh -c "echo '*            soft    rtprio             99' >> /etc/security/l
 # Create a tfb command alias for running the toolset
 # For now, this still ensures you have to be in the framework root to run it
 export PWD=$(pwd)
-sudo tee /etc/profile.d/tfb.sh <<EOF
+sudo tee /usr/local/bin/tfb <<EOF
 #!/bin/bash
-tfb() {
-  PYTHONPATH=$PWD python $PWD/toolset/run-tests.py "\$@"
-}
+PYTHONPATH=$PWD python $PWD/toolset/run-tests.py "\$@"
 EOF
-source /etc/profile.d/tfb.sh
+sudo chmod a+x /usr/local/bin/tfb

toolset/utils/benchmark_config.py

@@ -74,7 +74,7 @@ class BenchmarkConfig:
 
         # Setup the ssh commands
         self.client_ssh_command = [
-            'ssh', '-T', 'o', 'StrictHostKeyChecking=no',
+            'ssh', '-T', '-o', 'StrictHostKeyChecking=no',
             self.client_user + "@" + self.client_host
         ]
         if self.client_identity_file != None:

toolset/utils/docker_helper.py

@@ -16,10 +16,11 @@ from toolset.utils.ordered_set import OrderedSet
 from toolset.utils.database_helper import test_database
 
 
-def clean():
+def clean(config):
     '''
     Cleans all the docker images from the system
     '''
+    # Clean the app server images
     subprocess.check_call(["docker", "image", "prune", "-f"])
 
     docker_ids = subprocess.check_output(["docker", "images",
@@ -29,6 +30,24 @@ def clean():
 
     subprocess.check_call(["docker", "system", "prune", "-a", "-f"])
 
+    # Clean the database server images
+    command = list(config.database_ssh_command)
+    command.extend(["docker", "image", "prune", "-f"])
+    subprocess.check_call(command)
+
+    command = list(config.database_ssh_command)
+    command.extend(["docker", "images", "-q"])
+    docker_ids = subprocess.check_output(command).splitlines()
+
+    for docker_id in docker_ids:
+        command = list(config.database_ssh_command)
+        command.extend(["docker", "image", "rmi", "-f", docker_id])
+        subprocess.check_call(command)
+
+    command = list(config.database_ssh_command)
+    command.extend(["docker", "system", "prune", "-a", "-f"])
+    subprocess.check_call(command)
+
 
 def build(benchmarker_config, test_names, out):
     '''
@@ -167,29 +186,33 @@ def run(benchmarker_config, docker_files, out):
     return 0
 
 
-def successfully_running_containers(docker_files, database_container_id, out):
+def successfully_running_containers(docker_files, out):
     '''
     Returns whether all the expected containers for the given docker_files are
     running.
     '''
     client = docker.from_env()
-    running_container_length = len(
-        client.containers.list(filters={'status': 'running'}))
-    expected_length = len(docker_files)
-    if database_container_id is not None:
-        expected_length = expected_length + 1
-    if (running_container_length != expected_length):
-        tee_output(out, "Running Containers (id, name):" + os.linesep)
-        for running_container in client.containers.list():
-            tee_output(out, "%s, %s%s" % (running_container.short_id,
-                                          running_container.image, os.linesep))
-        tee_output(out, "Expected %s running containers; saw %s%s" %
-                   (expected_length, running_container_length, os.linesep))
-        return False
+    expected_running_container_images = []
+    for docker_file in docker_files:
+        # 'gemini.dockerfile' -> 'gemini'
+        image_tag = docker_file.split('.')[0]
+        expected_running_container_images.append(image_tag)
+    running_container_images = []
+    for container in client.containers.list():
+        # 'tfb/test/gemini:latest' -> 'gemini'
+        image_tag = container.image.tags[0].split(':')[0][9:]
+        running_container_images.append(image_tag)
+
+    for image_name in expected_running_container_images:
+        if image_name not in running_container_images:
+            tee_output(out,
+                       "ERROR: Expected tfb/test/%s to be running container" %
+                       image_name)
+            return False
     return True
 
 
-def stop(config, database_container_id, test, out):
+def stop(config=None, database_container_id=None, test=None):
     '''
     Attempts to stop the running test container.
     '''