Docker Multiple Container Support + Dart start/stream as examples (#3321)

Duty, decency, reliability, honor, dignity, respect: these are all qualities that TFBReaper not only held in high esteem, but practiced every day during his time reaping. He was a serious and disciplined program, but he could never resist the opportunity to have a laugh with friends and loved ones, given half the chance, by allowing a process to continue when it should have been killed.

It’s difficult to imagine TFBReaper not being around and I’m not sure how we will all cope. It’s strange to think that I can’t just start him up and feel safe knowing that he is there reaping for me. He was an imposing figure of a program, whose reassuring presence we all felt during difficult times.

As we gather here today to remember and commemorate his life, let us bid him farewell as we mourn the loss of a lively, dignified program. A program that brought joy and fulfillment to many, and whose legacy will live on forever.

Press 'F' to pay respects.

.travis.yml

@@ -43,10 +43,10 @@ env:
     # - "TESTDIR=D/vibed"
     # - "TESTDIR=D/hunt"
     # - "TESTDIR=D/collie"
-    # - "TESTDIR=Dart/dart-raw"
-    # - "TESTDIR=Dart/redstone"
-    # - "TESTDIR=Dart/start"
-    # - "TESTDIR=Dart/stream"
+     - "TESTDIR=Dart/dart"
+     - "TESTDIR=Dart/redstone"
+     - "TESTDIR=Dart/start"
+     - "TESTDIR=Dart/stream"
     # - "TESTDIR=Elixir/phoenix"
     # - "TESTDIR=Elixir/cowboy"
     # - "TESTDIR=Erlang/chicagoboss"
@@ -197,7 +197,7 @@ env:
     # - "TESTDIR=Scala/http4s"
     # - "TESTDIR=Scala/finch"
     # - "TESTDIR=Swift/vapor"
-    # - "TESTDIR=Ur/urweb"
+     - "TESTDIR=Ur/urweb"
     # - "TESTDIR=Vala/vsgi"
     # - "TESTDIR=Vala/valum"
 
@@ -230,4 +230,4 @@ script:
 cache:
   directories:
     - $HOME/.m2/repository
-    - $HOME/.cache/pip
+    - $HOME/.cache/pip

frameworks/C/h2o/h2o.dockerfile

@@ -51,4 +51,6 @@ RUN cd "${IROOT}" && \
 
 ENV PATH=${H2O_HOME}/bin:${PATH}
 
-CMD ["./setup.sh"]
+RUN chmod a+wrx start-servers.sh
+
+CMD ["./start-servers.sh"]

frameworks/C/h2o/setup.sh → frameworks/C/h2o/start-servers.sh

@@ -78,4 +78,6 @@ if "$USE_PROCESSES"; then
 else
 	echo "Running h2o_app multithreaded."
 	run_h2o_app 0 "${H2O_APP_HOME}/bin" "${H2O_APP_HOME}/share/h2o_app"
-fi
+fi
+
+wait

frameworks/Dart/dart-raw/.gitignore

@@ -1 +0,0 @@
-packages/*

frameworks/Dart/dart-raw/setup.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-fw_depends postgresql dart
-
-pub upgrade
-
-dart server.dart -a 0.0.0.0 -p 8080 -d ${MAX_CONCURRENCY} -i ${CPU_COUNT} &

frameworks/Dart/dart-raw/benchmark_config.json → frameworks/Dart/dart/benchmark_config.json

@@ -2,7 +2,6 @@
   "framework": "dart",
   "tests": [{
     "default": {
-      "setup_file": "setup",
       "json_url": "/json",
       "db_url": "/db",
       "query_url": "/queries?queries=",

frameworks/Dart/dart/dart.dockerfile

@@ -0,0 +1,7 @@
+FROM tfb/dart-lang:latest
+
+COPY ./ ./
+
+RUN pub upgrade
+
+CMD dart server.dart -a 0.0.0.0 -p 8080 -d $MAX_CONCURRENCY -i $CPU_COUNT

frameworks/Dart/redstone/.gitignore

@@ -1,2 +0,0 @@
-packages/*
-.pub

frameworks/Dart/redstone/benchmark_config.json

@@ -2,7 +2,6 @@
   "framework": "redstone",
   "tests": [{
     "default": {
-      "setup_file": "setup_postgresql",
       "json_url": "/json",
       "db_url": "/pg/db",
       "query_url": "/pg/queries?queries=",
@@ -26,7 +25,6 @@
       "versus": "dart"
     },
     "mongodb": {
-      "setup_file": "setup_mongodb",
       "db_url": "/mongo/db",
       "query_url": "/mongo/queries?queries=",
       "fortune_url": "/mongo/fortunes",

frameworks/Dart/redstone/redstone-mongodb.dockerfile

@@ -0,0 +1,7 @@
+FROM tfb/dart-lang:latest
+
+COPY ./ ./
+
+RUN pub upgrade
+
+CMD dart server.dart -a 0.0.0.0 -p 8080 -d ${CPU_COUNT} -i ${CPU_COUNT}

frameworks/Dart/redstone/redstone.dockerfile

@@ -0,0 +1,7 @@
+FROM tfb/dart-lang:latest
+
+COPY ./ ./
+
+RUN pub upgrade
+
+CMD dart server.dart -a 0.0.0.0 -p 8080 -d ${CPU_COUNT} -i ${CPU_COUNT}

frameworks/Dart/redstone/setup.sh

@@ -1,7 +0,0 @@
-#!/bin/bash
-
-fw_depends dart
-
-pub upgrade
-
-dart server.dart -a 0.0.0.0 -p 8080 -d ${CPU_COUNT} -i ${CPU_COUNT} &

frameworks/Dart/redstone/setup_mongodb.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-fw_depends mongodb
-
-source ./setup.sh

frameworks/Dart/redstone/setup_postgresql.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-fw_depends postgresql
-
-source ./setup.sh

frameworks/Dart/start/.gitignore

@@ -1,2 +0,0 @@
-packages/*
-/nginx.conf

frameworks/Dart/start/benchmark_config.json

@@ -1,8 +1,8 @@
 {
   "framework": "start",
   "tests": [{
-    "default": { 
-      "setup_file": "setup_mongodb",
+    "default": {
+      "docker_files": ["start-nginx.dockerfile"],
       "db_url": "/db-mongo",
       "query_url": "/queries-mongo?queries=",
       "fortune_url": "/fortunes-mongo",

frameworks/Dart/start/setup.sh → frameworks/Dart/start/nginx-conf.sh

@@ -1,20 +1,5 @@
 #!/bin/bash
 
-fw_depends dart nginx
-
-pub upgrade
-
-#
-# start dart servers
-#
-current=9001
-end=$(($current+$CPU_COUNT))
-while [ $current -lt $end ]; do
-  dart server.dart -a 127.0.0.1 -p $current -d ${CPU_COUNT} &
-  let current=current+1
-done
-
-
 #
 # create nginx configuration
 #
@@ -50,5 +35,3 @@ conf+="}"
 # write nginx configuration to disk
 #
 echo -e $conf > nginx.conf
-
-nginx -c $(pwd)/nginx.conf &

frameworks/Dart/start/setup_mongodb.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-fw_depends mongodb
-
-source ./setup.sh

frameworks/Dart/start/setup_postgresql.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-fw_depends postgresql
-
-source ./setup.sh

frameworks/Dart/start/start-nginx.dockerfile

@@ -0,0 +1,8 @@
+FROM tfb/nginx:latest
+
+COPY ./ ./
+
+RUN chmod a+rwx nginx-conf.sh
+RUN ./nginx-conf.sh
+
+CMD ["nginx", "-c", "/nginx.conf", "-g", "daemon off;"]

frameworks/Dart/start/start-servers.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#
+# start dart servers
+#
+current=9001
+end=$(($current+$CPU_COUNT))
+while [ $current -lt $end ]; do
+  dart server.dart -a 127.0.0.1 -p $current -d ${CPU_COUNT} &
+  let current=current+1
+done
+
+wait
+

frameworks/Dart/start/start.dockerfile

@@ -0,0 +1,9 @@
+FROM tfb/dart-lang:latest
+
+COPY ./ ./
+
+RUN pub upgrade
+
+RUN chmod a+rwx start-servers.sh
+
+CMD ["./start-servers.sh"]

frameworks/Dart/stream/.gitignore

@@ -1,2 +0,0 @@
-packages/*
-/nginx.conf

frameworks/Dart/stream/benchmark_config.json

@@ -2,26 +2,26 @@
   "framework": "stream",
   "tests": [{
      "default": {
-      "setup_file": "setup_mongodb",
-      "db_url": "/db-mongo",
-      "query_url": "/queries-mongo?queries=",
-      "fortune_url": "/fortunes-mongo",
-      "update_url": "/updates-mongo?queries=",
-      "port": 8080,
-      "approach": "Realistic",
-      "classification": "Micro",
-      "database": "MongoDB",
-      "framework": "stream",
-      "language": "Dart",
-      "flavor": "None",
-      "orm": "Raw",
-      "platform": "None",
-      "webserver": "nginx",
-      "os": "Linux",
-      "database_os": "Linux",
-      "display_name": "stream",
-      "notes": "",
-      "versus": "dart"
-    }
+       "docker_files": ["stream-nginx.dockerfile"],
+       "db_url": "/db-mongo",
+       "query_url": "/queries-mongo?queries=",
+       "fortune_url": "/fortunes-mongo",
+       "update_url": "/updates-mongo?queries=",
+       "port": 8080,
+       "approach": "Realistic",
+       "classification": "Micro",
+       "database": "MongoDB",
+       "framework": "stream",
+       "language": "Dart",
+       "flavor": "None",
+       "orm": "Raw",
+       "platform": "None",
+       "webserver": "nginx",
+       "os": "Linux",
+       "database_os": "Linux",
+       "display_name": "stream",
+       "notes": "",
+       "versus": "dart"
+     }
   }]
 }

frameworks/Dart/stream/setup.sh → frameworks/Dart/stream/nginx-conf.sh

@@ -1,20 +1,5 @@
 #!/bin/bash
 
-fw_depends dart nginx
-
-pub upgrade
-
-#
-# start dart servers
-#
-current=9001
-end=$(($current+$CPU_COUNT))
-while [ $current -lt $end ]; do
-  dart server.dart -a 127.0.0.1 -p $current -d ${CPU_COUNT} &
-  let current=current+1
-done
-
-
 #
 # create nginx configuration
 #
@@ -50,5 +35,3 @@ conf+="}"
 # write nginx configuration to disk
 #
 echo -e $conf > nginx.conf
-
-nginx -c $(pwd)/nginx.conf &

frameworks/Dart/stream/setup_mongodb.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-fw_depends mongodb
-
-source ./setup.sh

frameworks/Dart/stream/setup_postgresql.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-fw_depends postgresql
-
-source ./setup.sh

frameworks/Dart/stream/start-servers.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+#
+# start dart servers
+#
+current=9001
+end=$(($current+$CPU_COUNT))
+while [ $current -lt $end ]; do
+  dart server.dart -a 127.0.0.1 -p $current -d ${CPU_COUNT} &
+  let current=current+1
+done
+
+wait
+

frameworks/Dart/stream/stream-nginx.dockerfile

@@ -0,0 +1,8 @@
+FROM tfb/nginx:latest
+
+COPY ./ ./
+
+RUN chmod a+rwx nginx-conf.sh
+RUN ./nginx-conf.sh
+
+CMD ["nginx", "-c", "/nginx.conf", "-g", "daemon off;"]

frameworks/Dart/stream/stream.dockerfile

@@ -0,0 +1,9 @@
+FROM tfb/dart-lang:latest
+
+COPY ./ ./
+
+RUN pub upgrade
+
+RUN chmod a+rwx start-servers.sh
+
+CMD ["./start-servers.sh"]

frameworks/JavaScript/nodejs/benchmark_config.json

@@ -150,69 +150,6 @@
       "display_name": "nodejs-mitol",
       "notes": "",
       "versus": "nodejs"
-    },
-    "mongodb-raw-mitol": {
-      "db_url": "/db",
-      "query_url": "/queries?queries=",
-      "update_url": "/updates?queries=",
-      "fortune_url": "/fortunes",
-      "port": 8080,
-      "approach": "Realistic",
-      "classification": "Platform",
-      "database": "MongoDB",
-      "framework": "None",
-      "language": "JavaScript",
-      "flavor": "NodeJS",
-      "orm": "Raw",
-      "platform": "nodejs",
-      "webserver": "None",
-      "os": "Linux",
-      "database_os": "Linux",
-      "display_name": "nodejs-mitol",
-      "notes": "",
-      "versus": "nodejs"
-    },
-    "mysql-mitol": {
-      "db_url": "/db",
-      "query_url": "/queries?queries=",
-      "update_url": "/updates?queries=",
-      "fortune_url": "/fortunes",
-      "port": 8080,
-      "approach": "Realistic",
-      "classification": "Platform",
-      "database": "MySQL",
-      "framework": "None",
-      "language": "JavaScript",
-      "flavor": "NodeJS",
-      "orm": "Full",
-      "platform": "nodejs",
-      "webserver": "None",
-      "os": "Linux",
-      "database_os": "Linux",
-      "display_name": "nodejs-mitol",
-      "notes": "",
-      "versus": "nodejs"
-    },
-    "postgres-mitol": {
-      "db_url": "/db",
-      "query_url": "/queries?queries=",
-      "update_url": "/updates?queries=",
-      "fortune_url": "/fortunes",
-      "port": 8080,
-      "approach": "Realistic",
-      "classification": "Platform",
-      "database": "Postgres",
-      "framework": "None",
-      "language": "JavaScript",
-      "flavor": "NodeJS",
-      "orm": "Full",
-      "platform": "nodejs",
-      "webserver": "None",
-      "os": "Linux",
-      "database_os": "Linux",
-      "display_name": "nodejs-mitol",
-      "notes": "",
-      "versus": "nodejs"
-    }    
+    }
   }]
 }

frameworks/JavaScript/nodejs/nodejs-mongodb-raw-mitol.dockerfile

@@ -1,5 +0,0 @@
-FROM tfb/mitol:latest
-
-ENV NODE_HANDLER mongoose
-
-CMD ["node", "app-mitol.js"]

frameworks/JavaScript/nodejs/nodejs-mysql-mitol.dockerfile

@@ -1,5 +0,0 @@
-FROM tfb/mitol:latest
-
-ENV NODE_HANDLER sequelize
-
-CMD ["node", "app-mitol.js"]

frameworks/JavaScript/nodejs/nodejs-postgres-mitol.dockerfile

@@ -1,5 +0,0 @@
-FROM tfb/mitol:latest
-
-ENV NODE_HANDLER sequelize-postgres
-
-CMD ["node", "app-mitol.js"]

frameworks/Ur/urweb/setup-postgresql.sh

@@ -1,5 +0,0 @@
-#!/bin/bash
-
-fw_depends postgresql
-
-source ./setup.sh

frameworks/Ur/urweb/setup.sh

@@ -1,8 +0,0 @@
-#!/bin/bash
-
-fw_depends urweb
-
-urweb -db "dbname=hello_world user=benchmarkdbuser password=benchmarkdbpass host=${DBHOST}" bench
-
-MAX_THREADS=$((2 * $CPU_COUNT))
-./bench.exe -q -k -t ${MAX_THREADS} &

frameworks/Ur/urweb/setup_mysql.sh

@@ -1,11 +0,0 @@
-#!/bin/bash
-
-fw_depends urweb mysql
-
-export URWEB_HOME=${IROOT}/urweb
-export LD_LIBRARY_PATH=${URWEB_HOME}/lib
-
-${URWEB_HOME}/bin/urweb -dbms mysql -db "dbname=hello_world user=benchmarkdbuser password=benchmarkdbpass host=${DBHOST}" bench
-
-MAX_THREADS=$((2 * $CPU_COUNT))
-./bench.exe -q -k -t ${MAX_THREADS} &

frameworks/Ur/urweb/urweb-base.dockerfile

@@ -0,0 +1,19 @@
+FROM tfb/base:latest
+
+COPY ./ ./
+
+ENV URWEB_VERSION=20160621
+ENV COMPILER=/urweb-build
+
+RUN apt install -yqq mlton libssl-dev libpq-dev libmysqlclient-dev
+RUN mkdir -p $COMPILER && \
+    wget http://www.impredicative.com/ur/urweb-$URWEB_VERSION.tgz && \
+    tar xvf urweb-$URWEB_VERSION.tgz && \
+    cd urweb-$URWEB_VERSION && \
+    ./configure --prefix=$COMPILER && \
+    make && \
+    make install
+
+ENV URWEB_HOME=${COMPILER}
+ENV LD_LIBRARY_PATH=${COMPILER}/lib
+ENV PATH=${COMPILER}/bin:${PATH}

frameworks/Ur/urweb/urweb-mysql.dockerfile

@@ -0,0 +1,5 @@
+FROM tfb/urweb-base:latest
+
+RUN urweb -dbms mysql -db "dbname=hello_world user=benchmarkdbuser password=benchmarkdbpass host=TFB-database" bench
+
+CMD ./bench.exe -q -k -t $((2 * ${CPU_COUNT}))

frameworks/Ur/urweb/urweb.dockerfile

@@ -0,0 +1,5 @@
+FROM tfb/urweb-base:latest
+
+RUN urweb -db "dbname=hello_world user=benchmarkdbuser password=benchmarkdbpass host=TFB-database" bench
+
+CMD ./bench.exe -q -k -t $((2 * ${CPU_COUNT}))

toolset/benchmark/benchmarker.py

@@ -23,6 +23,7 @@ import logging
 import socket
 import threading
 import textwrap
+import docker
 import shutil
 from pprint import pprint
 
@@ -395,7 +396,7 @@ class Benchmarker:
     # Sets up a container for the given database and port, and
     # starts said docker container.
     ############################################################
-    def __setup_database_container(self, database, port):
+    def __setup_database_container(self, database):
         def __is_hex(s):
             try:
                 int(s, 16)
@@ -433,7 +434,7 @@ class Benchmarker:
                 return None
 
         p = subprocess.Popen(self.database_ssh_string, stdin=subprocess.PIPE, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
-        (out,err) = p.communicate("docker run -d --rm -p %s:%s --network=host %s" % (port,port,database))
+        (out,err) = p.communicate("docker run -d --rm --network=host %s" % database)
         return out.splitlines()[len(out.splitlines()) - 1]
     ############################################################
     # End __setup_database_container
@@ -594,13 +595,7 @@ class Benchmarker:
                 # Start database container
                 ##########################
                 if test.database != "None":
-                    # TODO: this is horrible... how should we really do it?
-                    ports = {
-                        "mysql": 3306,
-                        "postgres": 5432,
-                        "mongodb": 27017
-                    }
-                    database_container_id = self.__setup_database_container(test.database.lower(), ports[test.database.lower()])
+                    database_container_id = self.__setup_database_container(test.database.lower())
                     if not database_container_id:
                         out.write("ERROR: Problem building/running database container")
                         out.flush()
@@ -714,22 +709,15 @@ class Benchmarker:
     # Attempts to stop the running test container.
     ############################################################
     def __stop_test(self, database_container_id, test, out):
-        docker_ids = subprocess.check_output(["docker", "ps", "-q"]).splitlines()
-        for docker_id in docker_ids:
-            # This check is in case the database and server machines are the same
-            if docker_id:
-                if not database_container_id or docker_id not in database_container_id:
-                    subprocess.check_output(["docker", "kill", docker_id])
-                    slept = 0
-                    while(slept < 300 and docker_id is ''):
-                        time.sleep(1)
-                        slept += 1
-                        docker_id = subprocess.check_output(["docker", "ps", "-q"]).strip()
-                    # We still need to sleep a bit before removing the image
-                    time.sleep(5)
-                    subprocess.check_call(["docker", "image", "rm", "tfb/test/%s" % test.name])
-                    time.sleep(5)
-                    subprocess.check_call(["docker", "image", "prune", "-f"])
+        client = docker.from_env()
+        # Stop all the containers
+        for container in client.containers.list():
+            if container.status == "running" and container.id != database_container_id:
+              container.stop()
+        # Remove only the tfb/test image for this test
+        client.images.remove("tfb/test/%s" % test.name, force=True)
+        client.images.prune()
+
     ############################################################
     # End __stop_test
     ############################################################

toolset/benchmark/framework_test.py

@@ -17,6 +17,7 @@ import csv
 import shlex
 import math
 import multiprocessing
+import docker
 from collections import OrderedDict
 from requests import ConnectionError
 from threading import Thread
@@ -194,164 +195,110 @@ class FrameworkTest:
 
     ###########################
     # Build the Docker images
-    ###########################
+    ##########################
 
-    test_docker_file = os.path.join(self.directory, "%s.dockerfile" % self.name)
-    deps = list(reversed(gather_docker_dependencies( test_docker_file )))
+    # Build the test docker file based on the test name
+    # then build any additional docker files specified in the benchmark_config
+    # Note - If you want to be able to stream the output of the build process you have
+    # to use the low level API:
+    #  https://docker-py.readthedocs.io/en/stable/api.html#module-docker.api.build
+
+    prev_line = os.linesep
+    def handle_build_output(line):
+      if line.startswith('{"stream":'):
+        line = json.loads(line)
+        line = line[line.keys()[0]].encode('utf-8')
+        if prev_line.endswith(os.linesep):
+          tee_output(prefix, line)
+        else:
+          tee_output(line)
+        self.prev_line = line
 
-    docker_dir = os.path.join(setup_util.get_fwroot(), "toolset", "setup", "linux", "docker")
+    docker_buildargs = { 'CPU_COUNT': str(multiprocessing.cpu_count()),
+                         'MAX_CONCURRENCY': str(max(self.benchmarker.concurrency_levels)) }
 
-    for dependency in deps:
-      docker_file = os.path.join(self.directory, dependency + ".dockerfile")
-      if not docker_file or not os.path.exists(docker_file):
-        docker_file = find_docker_file(docker_dir, dependency + ".dockerfile")
-      if not docker_file:
-        tee_output(prefix, "Docker build failed; %s could not be found; terminating\n" % (dependency + ".dockerfile"))
-        return 1
-      p = subprocess.Popen([
-        "docker", 
-        "build", 
-        "--build-arg",
-        "CPU_COUNT=%s" % str(multiprocessing.cpu_count()),
-        "--build-arg",
-        "MAX_CONCURRENCY=%s" % max(self.benchmarker.concurrency_levels),
-        "-f", 
-        docker_file, 
-        "-t", 
-        "tfb/%s" % dependency, 
-        os.path.dirname(docker_file)],
-          stdout=subprocess.PIPE,
-          stderr=subprocess.STDOUT)
-      nbsr = setup_util.NonBlockingStreamReader(p.stdout)
-      while (p.poll() is None):
-        for i in xrange(10):
-          try:
-            line = nbsr.readline(0.05)
-            if line:
-              tee_output(prefix, line)
-          except setup_util.EndOfStream:
-            break
-      if p.returncode != 0:
-        tee_output(prefix, "Docker build failed; terminating\n")
-        return 1
-    p = subprocess.Popen([
-      "docker", 
-      "build", 
-      "--build-arg",
-      "CPU_COUNT=%s" % str(multiprocessing.cpu_count()),
-      "--build-arg",
-      "MAX_CONCURRENCY=%s" % max(self.benchmarker.concurrency_levels),
-      "-f", 
-      test_docker_file, 
-      "-t", 
-      "tfb/test/%s" % self.name, 
-      self.directory],
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT)
-    nbsr = setup_util.NonBlockingStreamReader(p.stdout)
-    while (p.poll() is None):
-      for i in xrange(10):
+    test_docker_files = ["%s.dockerfile" % self.name]
+    if self.docker_files is not None:
+      if type(self.docker_files) is list:
+        test_docker_files.extend(self.docker_files)
+      else:
+        raise Exception("docker_files in benchmark_config.json must be an array")
+
+    for test_docker_file in test_docker_files:
+      deps = list(reversed(gather_docker_dependencies(os.path.join(self.directory, test_docker_file))))
+
+      docker_dir = os.path.join(setup_util.get_fwroot(), "toolset", "setup", "linux", "docker")
+
+      for dependency in deps:
+        docker_file = os.path.join(self.directory, dependency + ".dockerfile")
+        if not docker_file or not os.path.exists(docker_file):
+          docker_file = find_docker_file(docker_dir, dependency + ".dockerfile")
+        if not docker_file:
+          tee_output(prefix, "Docker build failed; %s could not be found; terminating\n" % (dependency + ".dockerfile"))
+          return 1
+
+        # Build the dependency image
         try:
-          line = nbsr.readline(0.05)
-          if line:
-            tee_output(prefix, line)
-        except setup_util.EndOfStream:
-          break
-    if p.returncode != 0:
-      tee_output(prefix, "Docker build failed; terminating\n")
-      return 1
-        
+          for line in docker.APIClient(base_url='unix://var/run/docker.sock').build(
+            path=os.path.dirname(docker_file),
+            dockerfile="%s.dockerfile" % dependency,
+            tag="tfb/%s" % dependency,
+            buildargs=docker_buildargs,
+            forcerm=True
+          ):
+            handle_build_output(line)
+        except Exception as e:
+          tee_output(prefix, "Docker dependency build failed; terminating\n")
+          print(e)
+          return 1
+
+    # Build the test images
+    for test_docker_file in test_docker_files:
+        try:
+          for line in docker.APIClient(base_url='unix://var/run/docker.sock').build(
+            path=self.directory,
+            dockerfile=test_docker_file,
+            tag="tfb/test/%s" % test_docker_file.replace(".dockerfile",""),
+            buildargs=docker_buildargs,
+            forcerm=True
+          ):
+            handle_build_output(line)
+        except Exception as e:
+          tee_output(prefix, "Docker build failed; terminating\n")
+          print(e)
+          return 1
+
 
     ##########################
     # Run the Docker container
     ##########################
-    p = subprocess.Popen(["docker", "run", "--rm", "-p", "%s:%s" % (self.port, self.port), "--network=host", "tfb/test/%s" % self.name],
-          stdout=subprocess.PIPE,
-          stderr=subprocess.STDOUT)
-    nbsr = setup_util.NonBlockingStreamReader(p.stdout,
-      "%s: framework processes have terminated" % self.name)
-
-    # Set a limit on total execution time of setup.sh
-    timeout = datetime.now() + timedelta(minutes = 105)
-    time_remaining = timeout - datetime.now()
-
-    # Need to print to stdout once every 10 minutes or Travis-CI will abort
-    travis_timeout = datetime.now() + timedelta(minutes = 5)
-
-    # Flush output until docker run work is finished. This is
-    # either a) when docker run exits b) when the port is bound
-    # c) when we run out of time. 
-    prefix = "Server %s: " % self.name
-    while (p.poll() is None
-      and not self.benchmarker.is_port_bound(self.port)
-      and not time_remaining.total_seconds() < 0):
-
-      # The conditions above are slow to check, so
-      # we will delay output substantially if we only
-      # print one line per condition check.
-      # Adding a tight loop here mitigates the effect,
-      # ensuring that most of the output directly from
-      # docker is sent to tee_output before the outer
-      # loop exits and prints things like "docker exited"
-      for i in xrange(10):
-        try:
-          line = nbsr.readline(0.05)
-          if line:
-            tee_output(prefix, line)
 
-            # Reset Travis-CI timer
-            travis_timeout = datetime.now() + timedelta(minutes = 5)
-        except setup_util.EndOfStream:
-          tee_output(prefix, "Docker has terminated\n")
-          break
-      time_remaining = timeout - datetime.now()
-
-      if (travis_timeout - datetime.now()).total_seconds() < 0:
-        sys.stdout.write(prefix + 'Printing so Travis-CI does not time out\n')
-        sys.stdout.write(prefix + "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
-          p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
-        sys.stdout.flush()
-        travis_timeout = datetime.now() + timedelta(minutes = 5)
-
-    # Did we time out?
-    if time_remaining.total_seconds() < 0:
-      tee_output(prefix, "Docker run has timed out!! Aborting...\n" % self.setup_file)
-      p.kill()
-      return 1
-
-    # What's our return code?
-    # If docker run has terminated, use that code
-    # Otherwise, detect if the port was bound
-    tee_output(prefix, "Status: Poll: %s, Port %s bound: %s, Time Left: %s\n" % (
-      p.poll(), self.port, self.benchmarker.is_port_bound(self.port), time_remaining))
-    retcode = (p.poll() if p.poll() is not None else 0 if self.benchmarker.is_port_bound(self.port) else 1)
-    if p.poll() is not None:
-      tee_output(prefix, "Docker run process exited naturally with %s\n" % p.poll())
-    elif self.benchmarker.is_port_bound(self.port):
-      tee_output(prefix, "Bound port detected on %s\n" % self.port)
-
-    # Before we return control to the benchmarker, spin up a
-    # thread to keep an eye on the pipes in case the running
-    # framework uses stdout/stderr. Once all processes accessing
-    # the subprocess.PIPEs are dead, this thread will terminate.
-    # Use a different prefix to indicate this is the framework
-    # speaking
-    def watch_child_pipes(nbsr, prefix):
-      while True:
-        try:
-          line = nbsr.readline(60)
-          if line:
+	client = docker.from_env()
+
+    for test_docker_file in test_docker_files:
+      try:
+        def watch_container(container, prefix):
+          for line in container.logs(stream=True):
             tee_output(prefix, line)
-        except setup_util.EndOfStream:
-          tee_output(prefix, "Framework processes have terminated\n")
-          return
 
-    watch_thread = Thread(target = watch_child_pipes,
-      args = (nbsr, prefix))
-    watch_thread.daemon = True
-    watch_thread.start()
+        container = client.containers.run(
+          "tfb/test/%s" % test_docker_file.replace(".dockerfile", ""),
+          network_mode="host",
+          privileged=True,
+          stderr=True,
+          detach=True)
+
+        prefix = "Server %s: " % self.name
+        watch_thread = Thread(target = watch_container, args=(container,prefix))
+        watch_thread.daemon = True
+        watch_thread.start()
+
+      except Exception as e:
+        tee_output(prefix, "Running docker cointainer: %s failed" % test_docker_file)
+        print(e)
+        return 1
 
-    return retcode
+    return 0
   ############################################################
   # End start
   ############################################################
@@ -852,6 +799,7 @@ class FrameworkTest:
     self.display_name = ""
     self.notes = ""
     self.versus = ""
+    self.docker_files = None
 
     # setup logging
     logging.basicConfig(stream=sys.stderr, level=logging.INFO)

toolset/benchmark/test_types/verifications.py

@@ -267,7 +267,6 @@ sz
 
     n = 0
     while n < len(old_worlds) and successful_updates == 0:
-        print(old_worlds[n]['1'])
         for i in range(1, 10001):
             try:
                 entry_id = str(i)

toolset/setup/linux/docker/TFBReaper/Cargo.toml

@@ -1,12 +0,0 @@
-[package]
-
-name = "tfb_reaper"
-version = "0.0.1"
-authors = [ "Mike Smith <[email protected]>" ]
-
-[dependencies]
-
-prctl = "1.0.0"
-nix = "0.8.1"
-libc = "0.2"
-procinfo = "0.4.2"

toolset/setup/linux/docker/TFBReaper/src/main.rs

@@ -1,118 +0,0 @@
-extern crate prctl;
-extern crate nix;
-extern crate libc;
-extern crate procinfo;
-
-use std::process;
-use std::process::Command;
-use std::env;
-use std::fs;
-
-use nix::sys::signal;
-
-use libc::pause;
-
-/**
- * Recurssively finds all descendant PIDs of the given PID.
- */
-fn find_descendants(pid: i32) -> Vec<i32> {
-  let mut pids: Vec<i32> = vec![];
-
-  for path in fs::read_dir("/proc").unwrap() {
-    let filename = path.unwrap().file_name().into_string().ok().unwrap();
-    let apid = match filename.parse::<i32>() {
-      Ok(apid) => apid,
-      Err(_) => -1
-    };
-    
-    if apid > 0 {
-      let stat = procinfo::pid::stat(apid).unwrap();
-      if stat.ppid == pid {
-
-        pids.push(stat.ppid);
-        pids.append(&mut find_descendants(stat.pid));
-      }
-    }
-  }
-
-  return pids;
-}
-
-/**
- * Reap will kill all descendants of this process.
- */
-extern fn reap(_:i32) {
-  unsafe {
-    let mut done = false;
-    while !done {
-      let pids: Vec<i32> = find_descendants(libc::getpid());
-
-      if pids.len() == 0 {
-        done = true;
-      }
-
-      for pid in pids {
-        libc::kill(pid, libc::SIGKILL);
-      }
-    }
-  }
-
-  process::exit(0);
-}
-
-/**
- * main
- */
-fn main() {
-  // Interrupt SIGTERM and SIGINT and pass to our handler.
-  let sig_action = signal::SigAction::new(
-    signal::SigHandler::Handler(reap),
-    signal::SaFlags::empty(),
-    signal::SigSet::empty());
-  unsafe {
-    signal::sigaction(signal::SIGINT, &sig_action).unwrap();
-    signal::sigaction(signal::SIGTERM, &sig_action).unwrap();
-  }
-
-  // Here is the magic. This sets any child processes to
-  // use THIS process as a 'subreaper'. What that means is
-  // even if the process uses the fork-exit technicque for
-  // running a daemon (which normally orphans the process
-  // and causes init(1) to adopt it, which is problematic
-  // for TFB because we cannot then generally kill the
-  // process since it has lost all context available to us)
-  // the child process will have the parent id of THIS
-  // process, allowing us to kill all the processes started
-  // by the suite in this way generally.
-  //
-  // See: http://man7.org/linux/man-pages/man2/prctl.2.html
-  prctl::set_child_subreaper(true).unwrap();
-
-  // Gather the command line arguments for the pass-through.
-  let args: Vec<_> = env::args().collect();
-  if args.len() > 1 {
-    // This invokes whatever was passed as arguments to TFBReaper
-    // on the system. This program is merely a pass-through to
-    // a shell with the subreaper stuff enabled.
-    let status = Command::new(&args[1])
-            .args(&args[2..])
-            .status()
-            .expect("Failed to execute");
-
-    // We need to wait forever; the suite will clean this 
-    // process up later.
-    if status.success() {
-      loop {
-        unsafe {
-          // Pause to keep us from spiking CPU; whenever a signal
-          // occurs (except SIGTERM etc which will kill this process)
-          // just iterate and pause again.
-          pause();
-        }
-      }
-    }
-
-    // If the scripts failed, we should return that code.
-    process::exit(status.code().unwrap());
-  }
-}

toolset/setup/linux/docker/base.dockerfile

@@ -1,21 +1,15 @@
 FROM ubuntu:16.04
 
 RUN apt-get update
-RUN apt-get install -qqy software-properties-common build-essential curl locales wget git
+RUN apt-get install -qqy software-properties-common build-essential curl locales wget unzip git
 
 RUN locale-gen en_US.UTF-8
 ENV LANG en_US.UTF-8  
 ENV LANGUAGE en_US:en  
 ENV LC_ALL en_US.UTF-8 
 
-ADD TFBReaper TFBReaper
-
-RUN mv TFBReaper/target/debug/tfb_reaper /
-
 ARG CPU_COUNT
 ARG MAX_CONCURRENCY
 
 ENV CPU_COUNT=$CPU_COUNT
 ENV MAX_CONCURRENCY=$MAX_CONCURRENCY
-
-ENTRYPOINT ["/tfb_reaper"]

toolset/setup/linux/docker/languages/dart-lang.dockerfile

@@ -0,0 +1,8 @@
+FROM tfb/base:latest
+
+# Dart version 1.10.0
+RUN wget http://storage.googleapis.com/dart-archive/channels/stable/release/45396/sdk/dartsdk-linux-x64-release.zip
+RUN unzip dartsdk-linux-x64-release.zip
+
+ENV PUB_CACHE=/.pubcache
+ENV PATH=dart-sdk/bin:${PATH}

toolset/setup/linux/docker/webservers/nginx.dockerfile

@@ -0,0 +1,17 @@
+FROM tfb/base:latest
+
+RUN apt install -yqq libpcre3 libpcre3-dev zlib1g-dev
+
+ENV NGINX_HOME="/nginx"
+ENV NGINX_VERSION="1.12.2"
+
+RUN wget http://nginx.org/download/nginx-${NGINX_VERSION}.tar.gz && \
+    tar xvf nginx-${NGINX_VERSION}.tar.gz && \
+    cd nginx-${NGINX_VERSION} && \
+    echo "Configuring nginx..." && \
+    ./configure --prefix=/nginx && \
+    echo "Compiling and installing nginx..." && \
+    make --quiet && \
+    make --quiet install
+
+ENV PATH=/nginx/sbin:${PATH}

toolset/setup/linux/frameworks/urweb.sh

@@ -1,24 +0,0 @@
-#!/bin/bash
-
-VERSION=20160621
-COMPILER=${IROOT}/urweb
-
-RETCODE=$(fw_exists ${COMPILER}.installed)
-[ "$RETCODE" == 0 ] || { \
-  sudo apt-get --assume-yes install mlton
-  cd $IROOT
-  fw_get -O http://www.impredicative.com/ur/urweb-$VERSION.tgz
-  fw_untar urweb-$VERSION.tgz
-  cd urweb-$VERSION
-  ./configure --prefix=$IROOT/urweb
-  make
-  make install
-
-  echo "export URWEB_HOME=${COMPILER}" > $COMPILER.installed
-  echo "export LD_LIBRARY_PATH=${COMPILER}/lib" >> $COMPILER.installed
-  echo -e "export PATH=${COMPILER}/bin:\$PATH" >> $COMPILER.installed
-  cd $TROOT
-}
-
-source $IROOT/urweb.installed
-

toolset/setup/linux/languages/dart.sh

@@ -1,15 +0,0 @@
-#!/bin/bash
-
-fw_installed dart-sdk && return 0
-
-DART_HOME=$IROOT/dart-sdk
-
-# Dart version 1.10.0
-fw_get -O http://storage.googleapis.com/dart-archive/channels/stable/release/45396/sdk/dartsdk-linux-x64-release.zip
-fw_unzip dartsdk-linux-x64-release.zip
-
-echo "export DART_HOME=${DART_HOME}" > $IROOT/dart-sdk.installed
-echo "export PUB_CACHE=${IROOT}/.pubcache" >> $IROOT/dart-sdk.installed
-echo -e "export PATH=\$DART_HOME/bin:\$PATH" >> $IROOT/dart-sdk.installed
-
-source $IROOT/dart-sdk.installed

toolset/setup/linux/prerequisites.sh

@@ -44,18 +44,6 @@ sudo pip install colorama==0.3.1
 # but it requires --allow-external and --allow-unverified
 sudo pip install progressbar==2.2 requests MySQL-python psycopg2 pymongo
 
-#
-# Install Rust
-#
-curl -sL https://sh.rustup.rs -o rustup.sh
-chmod 777 rustup.sh
-./rustup.sh -y
-rm ./rustup.sh
-source ~/.profile
-cd toolset/setup/linux/docker/TFBReaper
-cargo build
-cd ../../../../..
-
 #
 # Install Docker
 #
@@ -69,6 +57,8 @@ sudo apt-get -qqy install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options:
 sudo groupadd docker
 sudo usermod -aG docker $USER
 
+sudo pip install docker==3.1.0
+
 # Get the ulimit from the benchmark config
 if [ -f benchmark.cfg ]; then
   FILE=benchmark.cfg

toolset/setup/linux/webservers/nginx.sh

@@ -1,23 +0,0 @@
-#!/bin/bash
-
-fw_installed nginx && return 0
-
-VERSION="1.12.2"
-NGINX_HOME=$IROOT/nginx
-
-fw_get -O http://nginx.org/download/nginx-${VERSION}.tar.gz
-fw_untar nginx-${VERSION}.tar.gz
-cd nginx-${VERSION}
-
-# There is no --quiet flag that I could find...
-echo "Configuring nginx..."
-./configure --prefix=$NGINX_HOME > /dev/null
-
-echo "Compiling and installing nginx..."
-make --quiet
-make --quiet install
-
-echo "export NGINX_HOME=${NGINX_HOME}" > $IROOT/nginx.installed
-echo -e "export PATH=\$NGINX_HOME/sbin:\$PATH" >> $IROOT/nginx.installed
-
-source $IROOT/nginx.installed