Toolset cleanup & remove TE database dependencies (#3574)

* Toolset cleanup

toolset/benchmark/benchmarker.py

@@ -39,6 +39,7 @@ class Benchmarker:
         # Get a list of all known  tests that we can run.
         all_tests = gather_remaining_tests(self.config, self.results)
 
+        any_failed = False
         # Run tests
         log("Running Tests...", border='=')
         with open(os.path.join(self.results.directory, 'benchmark.log'),
@@ -46,7 +47,8 @@ class Benchmarker:
             for test in all_tests:
                 log("Running Test: %s" % test.name, border='-')
                 with self.config.quiet_out.enable():
-                    self.__run_test(test, benchmark_log)
+                    if not self.__run_test(test, benchmark_log):
+                        any_failed = True
                 # Load intermediate result from child process
                 self.results.load()
 
@@ -59,6 +61,8 @@ class Benchmarker:
         self.results.upload()
         self.results.finish()
 
+        return any_failed
+
     ##########################################################################################
     # Private methods
     ##########################################################################################
@@ -136,9 +140,9 @@ class Benchmarker:
                     return False
 
             # Start webapp
-            containers = test.start()
-            if containers is None:
-                docker_helper.stop(self.config, containers, database_container,
+            container = test.start()
+            if container is None:
+                docker_helper.stop(self.config, container, database_container,
                                    test)
                 message = "ERROR: Problem starting {name}".format(name=test.name)
                 self.results.write_intermediate(test.name, message)
@@ -153,22 +157,11 @@ class Benchmarker:
             accepting_requests = False
             while not accepting_requests and slept < max_sleep:
                 accepting_requests = test.is_accepting_requests()
-                if not docker_helper.successfully_running_containers(
-                        self.config, test, benchmark_log):
-                    docker_helper.stop(self.config, containers,
-                                       database_container, test)
-                    message = "ERROR: One or more expected docker containers exited early"
-                    self.results.write_intermediate(test.name, message)
-                    log(message,
-                        prefix=log_prefix,
-                        file=benchmark_log,
-                        color=Fore.RED)
-                    return False
                 time.sleep(1)
                 slept += 1
 
             if not accepting_requests:
-                docker_helper.stop(self.config, containers, database_container,
+                docker_helper.stop(self.config, container, database_container,
                                    test)
                 message = "ERROR: Framework is not accepting requests from client machine"
                 self.results.write_intermediate(test.name, message)
@@ -199,7 +192,7 @@ class Benchmarker:
                 self.__benchmark(test, benchmark_log)
 
             # Stop this test
-            docker_helper.stop(self.config, containers, database_container,
+            docker_helper.stop(self.config, container, database_container,
                                test)
 
             # Save results thus far into the latest results directory

toolset/run-tests.py

@@ -235,7 +235,9 @@ def main(argv=None):
 
     else:
         benchmarker = Benchmarker(config, results)
-        benchmarker.run()
+        any_failed = benchmarker.run()
+        if config.mode == "verify":
+            return any_failed
 
     return 0
 

toolset/travis/travis_diff.py

@@ -156,7 +156,7 @@ if re.search(r'\[ci lang .+\]', last_commit_msg, re.M):
 # Also for now, ignore the old linux setup folders, as we don't want to
 # trigger a full run as we remove old fw_depends scripts. [ci run-all] will
 # still work if it's needed.
-if re.search(r'^toolset/(?!(travis/|setup/|continuous/))', changes, re.M) is not None:
+if re.search(r'^toolset/(?!(travis/|continuous/))', changes, re.M) is not None:
     print("Found changes to core toolset. Running all tests.")
     run_tests = test_dirs
     quit_diffing()

toolset/utils/docker_helper.py

@@ -11,7 +11,6 @@ from colorama import Fore, Style
 
 from toolset.utils.output_helper import log
 from toolset.utils.metadata_helper import gather_tests
-from toolset.utils.ordered_set import OrderedSet
 from toolset.utils.database_helper import test_database
 
 
@@ -51,21 +50,12 @@ def build(benchmarker_config, test_names, build_log_dir=os.devnull):
     Builds the dependency chain as well as the test implementation docker images
     for the given tests.
     '''
-    docker_buildargs = {
-        'MAX_CONCURRENCY': str(max(benchmarker_config.concurrency_levels)),
-        'TFB_DATABASE': str(benchmarker_config.database_host)
-    }
-
     tests = gather_tests(
         include=test_names, benchmarker_config=benchmarker_config)
 
     for test in tests:
         log_prefix = "%s: " % test.name
 
-        if __build_dependencies(benchmarker_config, test, docker_buildargs,
-                                build_log_dir) > 0:
-            return 1
-
         # Build the test image
         test_docker_file = "%s.dockerfile" % test.name
         build_log_file = build_log_dir
@@ -81,7 +71,6 @@ def build(benchmarker_config, test_names, build_log_dir=os.devnull):
                             dockerfile=test_docker_file,
                             tag="techempower/tfb.test.%s" %
                             test_docker_file.replace(".dockerfile", ""),
-                            buildargs=docker_buildargs,
                             forcerm=True):
                     if line.startswith('{"stream":'):
                         line = json.loads(line)
@@ -109,16 +98,17 @@ def run(benchmarker_config, test, run_log_dir):
     '''
     client = docker.DockerClient(
         base_url=benchmarker_config.server_docker_host)
-    containers = []
 
     log_prefix = "%s: " % test.name
+    container = None
+
     try:
 
-        def watch_container(container, docker_file):
+        def watch_container(docker_container, docker_file):
             with open(
                     os.path.join(run_log_dir, "%s.log" % docker_file.replace(
                         ".dockerfile", "").lower()), 'w') as run_log:
-                for line in container.logs(stream=True):
+                for line in docker_container.logs(stream=True):
                     log(line, prefix=log_prefix, file=run_log)
 
         extra_hosts = None
@@ -157,8 +147,6 @@ def run(benchmarker_config, test, run_log_dir):
             ulimits=ulimit,
             sysctls=sysctl)
 
-        containers.append(container)
-
         watch_thread = Thread(
             target=watch_container,
             args=(
@@ -178,34 +166,11 @@ def run(benchmarker_config, test, run_log_dir):
                 file=run_log)
             log(tb, prefix=log_prefix, file=run_log)
 
-    return containers
-
-
-def successfully_running_containers(benchmarker_config, test, out):
-    '''
-    Returns whether all the expected containers for the given docker_files are
-    running.
-    '''
-    client = docker.DockerClient(
-        base_url=benchmarker_config.server_docker_host)
-    running_container_images = []
-    for container in client.containers.list():
-        # 'techempower/tfb.test.gemini:0.1' -> 'gemini'
-        image_tag = container.image.tags[0].split(':')[0][21:]
-        running_container_images.append(image_tag)
-
-    if test.name not in running_container_images:
-        log_prefix = "%s: " % test.name
-        log("ERROR: Expected techempower/tfb.test.%s to be running container" %
-            test.name,
-            prefix=log_prefix,
-            file=out)
-        return False
-    return True
+    return container
 
 
 def stop(benchmarker_config=None,
-         containers=None,
+         container=None,
          database_container=None,
          test=None):
     '''
@@ -213,16 +178,15 @@ def stop(benchmarker_config=None,
     '''
     client = docker.DockerClient(
         base_url=benchmarker_config.server_docker_host)
-    if containers is None:
+    if container is None:
         for container in client.containers.list():
             if len(
                     container.image.tags
             ) > 0 and 'techempower' in container.image.tags[0] and 'tfb:latest' not in container.image.tags[0]:
                 container.stop()
     else:
-        # Stop all our running containers
-        for container in containers:
-            container.stop()
+        # Stop the running container
+        container.stop()
 
     database_client = docker.DockerClient(
         base_url=benchmarker_config.database_docker_host)
@@ -261,30 +225,18 @@ def start_database(benchmarker_config, test, database):
     image_name = "techempower/%s:latest" % database
     log_prefix = image_name + ": "
 
-    database_dir = os.path.join(benchmarker_config.fwroot, "toolset", "setup",
-                                "docker", "databases", database)
+    database_dir = os.path.join(benchmarker_config.fwroot, "toolset",
+                                "databases", database)
     docker_file = "%s.dockerfile" % database
 
-    pulled = False
     client = docker.DockerClient(
         base_url=benchmarker_config.database_docker_host)
     try:
         # Don't pull if we have it
         client.images.get(image_name)
-        pulled = True
         log("Found published image; skipping build", prefix=log_prefix)
     except:
-        # Pull the dependency image
-        try:
-            log("Attempting docker pull for image (this can take some time)",
-                prefix=log_prefix)
-            client.images.pull(image_name)
-            pulled = True
-            log("Found published image; skipping build", prefix=log_prefix)
-        except:
-            pass
-
-    if not pulled:
+        # Build the database image
         for line in docker.APIClient(
                 base_url=benchmarker_config.database_docker_host).build(
                     path=database_dir,
@@ -384,134 +336,3 @@ def benchmark(benchmarker_config, script, variables, raw_file):
             stderr=True,
             ulimits=ulimit,
             sysctls=sysctl), raw_file)
-
-
-def __gather_dependencies(benchmarker_config, docker_file):
-    '''
-    Gathers all the known docker dependencies for the given docker image.
-    '''
-    deps = []
-
-    docker_dir = os.path.join(benchmarker_config.fwroot, "toolset", "setup",
-                              "docker")
-
-    if os.path.exists(docker_file):
-        with open(docker_file) as fp:
-            for line in fp.readlines():
-                tokens = line.strip().split(' ')
-                if tokens[0] == "FROM":
-                    # This is magic that our base image points to
-                    if tokens[1].startswith('techempower/'):
-                        dep_ref = tokens[1].strip().split(':')[0].strip()
-                        if '/' not in dep_ref:
-                            raise AttributeError(
-                                "Could not find docker FROM dependency: %s" %
-                                dep_ref)
-                        depToken = dep_ref.split('/')[1]
-                        deps.append(tokens[1])
-                        dep_docker_file = os.path.join(
-                            os.path.dirname(docker_file),
-                            depToken + ".dockerfile")
-                        if not os.path.exists(dep_docker_file):
-                            dep_docker_file = find(docker_dir,
-                                                   depToken + ".dockerfile")
-                        deps.extend(
-                            __gather_dependencies(benchmarker_config,
-                                                  dep_docker_file))
-
-    return deps
-
-
-def __build_dependencies(benchmarker_config,
-                         test,
-                         docker_buildargs,
-                         build_log_dir=os.devnull):
-    '''
-    Builds all the dependency docker images for the given test.
-    Does not build the test docker image.
-    '''
-    dependencies = OrderedSet(
-        list(
-            reversed(
-                __gather_dependencies(
-                    benchmarker_config,
-                    os.path.join(test.directory,
-                                 "%s.dockerfile" % test.name)))))
-
-    docker_dir = os.path.join(benchmarker_config.fwroot, "toolset", "setup",
-                              "docker")
-    for dep in dependencies:
-        log_prefix = dep + ": "
-        pulled = False
-
-        # Do not pull techempower/ images if we are building specifically
-        if not benchmarker_config.build and 'techempower/' not in dep:
-            client = docker.DockerClient(
-                base_url=benchmarker_config.server_docker_host)
-            try:
-                # If we have it, use it
-                client.images.get(dep)
-                pulled = True
-                log("Found published image; skipping build", prefix=log_prefix)
-            except:
-                # Pull the dependency image
-                try:
-                    log("Attempting docker pull for image (this can take some time)",
-                        prefix=log_prefix)
-                    client.images.pull(dep)
-                    pulled = True
-                    log("Found published image; skipping build",
-                        prefix=log_prefix)
-                except:
-                    log("Docker pull failed; %s could not be found; terminating"
-                        % dep,
-                        prefix=log_prefix,
-                        color=Fore.RED)
-                    return 1
-
-        if not pulled:
-            dep_ref = dep.strip().split(':')[0].strip()
-            dependency = dep_ref.split('/')[1]
-            build_log_file = build_log_dir
-            if build_log_dir is not os.devnull:
-                build_log_file = os.path.join(build_log_dir,
-                                              "%s.log" % dependency.lower())
-            with open(build_log_file, 'w') as build_log:
-                docker_file = os.path.join(test.directory,
-                                           dependency + ".dockerfile")
-                if not docker_file or not os.path.exists(docker_file):
-                    docker_file = find(docker_dir, dependency + ".dockerfile")
-                if not docker_file:
-                    log("Docker build failed; %s could not be found; terminating"
-                        % (dependency + ".dockerfile"),
-                        prefix=log_prefix,
-                        file=build_log,
-                        color=Fore.RED)
-                    return 1
-
-                # Build the dependency image
-                try:
-                    for line in docker.APIClient(
-                            base_url=benchmarker_config.server_docker_host
-                    ).build(
-                            path=os.path.dirname(docker_file),
-                            dockerfile="%s.dockerfile" % dependency,
-                            tag=dep,
-                            buildargs=docker_buildargs,
-                            forcerm=True):
-                        if line.startswith('{"stream":'):
-                            line = json.loads(line)
-                            line = line[line.keys()[0]].encode('utf-8')
-                            log(line,
-                                prefix=log_prefix,
-                                file=build_log,
-                                color=Fore.WHITE + Style.BRIGHT \
-                                    if re.match(r'^Step \d+\/\d+', line) else '')
-                except Exception:
-                    tb = traceback.format_exc()
-                    log("Docker dependency build failed; terminating",
-                        prefix=log_prefix,
-                        file=build_log,
-                        color=Fore.RED)
-                    log(tb, prefix=log_prefix, file=build_log)
-                    return 1