Add minimal verification for containers running (#3381)

* Add minimal verification for containers running

* Better error reporting in docker

* Fixed bug with reporting database container

* Stupid equals sign...

* Fixed a couple of verification bugs

toolset/benchmark/benchmarker.py

@@ -358,30 +358,6 @@ class Benchmarker:
                     format(name=test.name))
                 return sys.exit(0)
 
-            out.write(
-                "test.os.lower() = {os}  test.database_os.lower() = {dbos}\n".
-                format(os=test.os.lower(), dbos=test.database_os.lower()))
-            out.write("self.results.frameworks != None: {val}\n".format(
-                val=str(self.results.frameworks != None)))
-            out.write("test.name: {name}\n".format(name=str(test.name)))
-            out.write("self.results.completed.: {completed}\n".format(
-                completed=str(self.results.completed)))
-            if self.results.frameworks != None and test.name in self.results.completed:
-                out.write(
-                    'Framework {name} found in latest saved data. Skipping.\n'.
-                    format(name=str(test.name)))
-                print(
-                    'WARNING: Test {test} exists in the results directory; this must be removed before running a new test.\n'.
-                    format(test=str(test.name)))
-                return sys.exit(1)
-            out.flush()
-
-            out.write(header("Beginning %s" % test.name, top='='))
-            out.flush()
-
-            # Start this test
-            out.write(header("Starting %s" % test.name))
-            out.flush()
             database_container_id = None
             try:
                 if self.__is_port_bound(test.port):
@@ -413,7 +389,7 @@ class Benchmarker:
                         return sys.exit(1)
 
                 # Start webapp
-                result = test.start(out)
+                result = test.start(out, database_container_id)
                 if result != 0:
                     docker_helper.stop(self.config, database_container_id,
                                        test, out)
@@ -451,9 +427,6 @@ class Benchmarker:
                 docker_helper.stop(self.config, database_container_id, test,
                                    out)
 
-                out.write(header("Stopped %s" % test.name))
-                out.flush()
-
                 # Remove contents of  /tmp folder
                 try:
                     subprocess.check_call(
@@ -465,8 +438,6 @@ class Benchmarker:
                     out.write(header("Error: Could not empty /tmp"))
 
                 # Save results thus far into the latest results directory
-                out.write(header("Saving results through %s" % test.name))
-                out.flush()
                 self.results.write_intermediate(test.name,
                                                 time.strftime(
                                                     "%Y%m%d%H%M%S",

toolset/benchmark/framework_test.py

@@ -53,7 +53,7 @@ class FrameworkTest:
     # Public Methods
     ##########################################################################################
 
-    def start(self, out):
+    def start(self, out, database_container_id):
         '''
         Start the test implementation
         '''
@@ -69,7 +69,8 @@ class FrameworkTest:
         if result != 0:
             return result
 
-        return docker_helper.run(self.benchmarker_config, test_docker_files, out)
+        return docker_helper.run(self.benchmarker_config, test_docker_files,
+                                 database_container_id, out)
 
     def verify_urls(self, logPath):
         '''

toolset/utils/docker_helper.py

@@ -11,6 +11,7 @@ from threading import Thread
 from toolset.utils import setup_util
 from toolset.utils.output_helper import tee_output
 from toolset.utils.metadata_helper import gather_tests
+from toolset.utils.ordered_set import OrderedSet
 
 
 def clean():
@@ -50,10 +51,11 @@ def build(benchmarker_config, test_names, out):
                     "docker_files in benchmark_config.json must be an array")
 
         for test_docker_file in test_docker_files:
-            deps = list(
-                reversed(
-                    gather_dependencies(
-                        os.path.join(test.directory, test_docker_file))))
+            deps = OrderedSet(
+                list(
+                    reversed(
+                        __gather_dependencies(
+                            os.path.join(test.directory, test_docker_file)))))
 
             docker_dir = os.path.join(setup_util.get_fwroot(), "toolset",
                                       "setup", "docker")
@@ -121,7 +123,7 @@ def build(benchmarker_config, test_names, out):
     return 0
 
 
-def run(benchmarker_config, docker_files, out):
+def run(benchmarker_config, docker_files, database_container_id, out):
     '''
     Run the given Docker container(s)
     '''
@@ -147,6 +149,7 @@ def run(benchmarker_config, docker_files, out):
                 privileged=True,
                 stderr=True,
                 detach=True,
+                init=True,
                 extra_hosts=extra_hosts)
 
             watch_thread = Thread(target=watch_container, args=(container, ))
@@ -159,6 +162,20 @@ def run(benchmarker_config, docker_files, out):
             print(e)
             return 1
 
+    running_container_length = len(
+        client.containers.list(filters={'status': 'running'}))
+    expected_length = len(docker_files)
+    if database_container_id is not None:
+        expected_length = expected_length + 1
+    if (running_container_length != expected_length):
+        tee_output(out, "Running Containers (id, name):" + os.linesep)
+        for running_container in client.containers.list():
+            tee_output(out, "%s, %s%s" % (running_container.short_id,
+                                          running_container.image, os.linesep))
+        tee_output(out, "Excepted %s running containers; saw %s%s" %
+                   (running_container_length, expected_length, os.linesep))
+        return 1
+
     return 0
 
 
@@ -200,38 +217,6 @@ def find(path, pattern):
                 return os.path.join(root, name)
 
 
-def gather_dependencies(docker_file):
-    '''
-    Gathers all the known docker dependencies for the given docker image.
-    '''
-    # Avoid setting up a circular import
-    from toolset.utils import setup_util
-    deps = []
-
-    docker_dir = os.path.join(setup_util.get_fwroot(), "toolset", "setup",
-                              "docker")
-
-    if os.path.exists(docker_file):
-        with open(docker_file) as fp:
-            for line in fp.readlines():
-                tokens = line.strip().split(' ')
-                if tokens[0] == "FROM":
-                    # This is magic that our base image points to
-                    if tokens[1] != "ubuntu:16.04":
-                        depToken = tokens[1].strip().split(':')[
-                            0].strip().split('/')[1]
-                        deps.append(depToken)
-                        dep_docker_file = os.path.join(
-                            os.path.dirname(docker_file),
-                            depToken + ".dockerfile")
-                        if not os.path.exists(dep_docker_file):
-                            dep_docker_file = find(docker_dir,
-                                                   depToken + ".dockerfile")
-                        deps.extend(gather_dependencies(dep_docker_file))
-
-    return deps
-
-
 def start_database(config, database):
     '''
     Sets up a container for the given database and port, and starts said docker 
@@ -304,4 +289,40 @@ def start_database(config, database):
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT)
     out = p.communicate("docker run -d --rm --network=host %s" % database)[0]
-    return out.splitlines()[len(out.splitlines()) - 1]
+    return out.splitlines()[len(out.splitlines()) - 1]
+
+
+def __gather_dependencies(docker_file):
+    '''
+    Gathers all the known docker dependencies for the given docker image.
+    '''
+    # Avoid setting up a circular import
+    from toolset.utils import setup_util
+    deps = []
+
+    docker_dir = os.path.join(setup_util.get_fwroot(), "toolset", "setup",
+                              "docker")
+
+    if os.path.exists(docker_file):
+        with open(docker_file) as fp:
+            for line in fp.readlines():
+                tokens = line.strip().split(' ')
+                if tokens[0] == "FROM":
+                    # This is magic that our base image points to
+                    if tokens[1] != "ubuntu:16.04":
+                        dep_ref = tokens[1].strip().split(':')[0].strip()
+                        if '/' not in dep_ref:
+                            raise AttributeError(
+                                "Could not find docker FROM dependency: %s" %
+                                dep_ref)
+                        depToken = dep_ref.split('/')[1]
+                        deps.append(depToken)
+                        dep_docker_file = os.path.join(
+                            os.path.dirname(docker_file),
+                            depToken + ".dockerfile")
+                        if not os.path.exists(dep_docker_file):
+                            dep_docker_file = find(docker_dir,
+                                                   depToken + ".dockerfile")
+                        deps.extend(__gather_dependencies(dep_docker_file))
+
+    return deps

toolset/utils/ordered_set.py

@@ -0,0 +1,71 @@
+import collections
+
+
+class OrderedSet(collections.MutableSet):
+    '''
+    From https://code.activestate.com/recipes/576694/
+    '''
+
+    def __init__(self, iterable=None):
+        self.end = end = []
+        end += [None, end, end]  # sentinel node for doubly linked list
+        self.map = {}  # key --> [key, prev, next]
+        if iterable is not None:
+            self |= iterable
+
+    def __len__(self):
+        return len(self.map)
+
+    def __contains__(self, key):
+        return key in self.map
+
+    def add(self, key):
+        if key not in self.map:
+            end = self.end
+            curr = end[1]
+            curr[2] = end[1] = self.map[key] = [key, curr, end]
+
+    def discard(self, key):
+        if key in self.map:
+            key, prev, next = self.map.pop(key)
+            prev[2] = next
+            next[1] = prev
+
+    def __iter__(self):
+        end = self.end
+        curr = end[2]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[2]
+
+    def __reversed__(self):
+        end = self.end
+        curr = end[1]
+        while curr is not end:
+            yield curr[0]
+            curr = curr[1]
+
+    def pop(self, last=True):
+        if not self:
+            raise KeyError('set is empty')
+        key = self.end[1][0] if last else self.end[2][0]
+        self.discard(key)
+        return key
+
+    def __repr__(self):
+        if not self:
+            return '%s()' % (self.__class__.__name__, )
+        return '%s(%r)' % (self.__class__.__name__, list(self))
+
+    def __eq__(self, other):
+        if isinstance(other, OrderedSet):
+            return len(self) == len(other) and list(self) == list(other)
+        return set(self) == set(other)
+
+
+if __name__ == '__main__':
+    s = OrderedSet('abracadaba')
+    t = OrderedSet('simsalabim')
+    print(s | t)
+    print(s & t)
+    print(s - t)