Implement supporting multiple `FROM` directives (#3320)

* Split up lang/systools

* Add the other gemini impls

* I am dumb

* Trying something for Travis-CI

* More Travis-CI testing

* Updated FROM implementation

Also, namespace all tagged images with `tfb/`

* Fixes h2o and libreactor

frameworks/C/duda/duda.dockerfile

@@ -1,4 +1,4 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 RUN apt install git python -y
 

frameworks/C/facil.io/facil.io.dockerfile

@@ -1,4 +1,4 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 COPY ./ ./
 

frameworks/C/h2o/h2o.dockerfile

@@ -1,4 +1,4 @@
-FROM ruby-2.4:latest
+FROM tfb/ruby-2.4:latest
 
 COPY ./ ./
 

frameworks/C/libreactor/libreactor.dockerfile

@@ -1,4 +1,4 @@
-FROM gcc-6:latest
+FROM tfb/gcc-6:latest
 
 COPY ./ ./
 

frameworks/C/octane/octane.dockerfile

@@ -1,4 +1,4 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 COPY ./ ./
 

frameworks/C/onion/onion.dockerfile

@@ -1,4 +1,4 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 COPY ./ ./
 

frameworks/Java/gemini/gemini-mysql.dockerfile

@@ -1,4 +1,4 @@
-FROM resin:latest
+FROM tfb/ant:latest as ant
 
 RUN apt-get install -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
     ant
@@ -13,4 +13,8 @@ RUN cd /gemini/Docroot/WEB-INF; mv gemini-mysql.conf GeminiHello.conf;
 
 RUN cd /gemini; mkdir -p Docroot/WEB-INF/classes; mkdir -p Docroot/WEB-INF/lib; ant resolve; ant compile
 
+FROM tfb/resin:latest
+
+COPY --from=ant /gemini /gemini
+
 CMD ["resinctl", "-conf", "/gemini/Docroot/WEB-INF/resin.xml", "console"]

frameworks/Java/gemini/gemini-postgres.dockerfile

@@ -1,4 +1,4 @@
-FROM resin:latest
+FROM tfb/ant:latest as ant
 
 RUN apt-get install -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
     ant
@@ -13,4 +13,8 @@ RUN cd /gemini/Docroot/WEB-INF; mv gemini-postgres.conf GeminiHello.conf;
 
 RUN cd /gemini; mkdir -p Docroot/WEB-INF/classes; mkdir -p Docroot/WEB-INF/lib; ant resolve; ant compile
 
+FROM tfb/resin:latest
+
+COPY --from=ant /gemini /gemini
+
 CMD ["resinctl", "-conf", "/gemini/Docroot/WEB-INF/resin.xml", "console"]

frameworks/Java/gemini/gemini.dockerfile

@@ -1,4 +1,4 @@
-FROM resin:latest
+FROM tfb/ant:latest as ant
 
 ADD Docroot/ /gemini/Docroot
 ADD Source/ /gemini/Source
@@ -10,4 +10,8 @@ RUN cd /gemini/Docroot/WEB-INF; mv gemini.conf GeminiHello.conf;
 
 RUN cd /gemini; mkdir -p Docroot/WEB-INF/classes; mkdir -p Docroot/WEB-INF/lib; ant resolve; ant compile
 
+FROM tfb/resin:latest
+
+COPY --from=ant /gemini /gemini
+
 CMD ["resinctl", "-conf", "/gemini/Docroot/WEB-INF/resin.xml", "console"]

frameworks/JavaScript/express/express-mongodb.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/express/express-mysql.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/express/express.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/hapi/hapi-mysql.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/hapi/hapi-postgres.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/hapi/hapi.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/koa/koa-mysql.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/koa/koa-postgres.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/koa/koa.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/nodejs/mitol.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/nodejs/nodejs-default-mitol.dockerfile

@@ -1,4 +1,4 @@
-FROM mitol:latest
+FROM tfb/mitol:latest
 
 ENV NODE_HANDLER mysql-raw
 

frameworks/JavaScript/nodejs/nodejs-mongodb-mitol.dockerfile

@@ -1,4 +1,4 @@
-FROM mitol:latest
+FROM tfb/mitol:latest
 
 ENV NODE_HANDLER mongoose
 

frameworks/JavaScript/nodejs/nodejs-mongodb-raw-mitol.dockerfile

@@ -1,4 +1,4 @@
-FROM mitol:latest
+FROM tfb/mitol:latest
 
 ENV NODE_HANDLER mongoose
 

frameworks/JavaScript/nodejs/nodejs-mongodb-raw.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/nodejs/nodejs-mongodb.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/nodejs/nodejs-mysql-mitol.dockerfile

@@ -1,4 +1,4 @@
-FROM mitol:latest
+FROM tfb/mitol:latest
 
 ENV NODE_HANDLER sequelize
 

frameworks/JavaScript/nodejs/nodejs-mysql.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/nodejs/nodejs-postgres-mitol.dockerfile

@@ -1,4 +1,4 @@
-FROM mitol:latest
+FROM tfb/mitol:latest
 
 ENV NODE_HANDLER sequelize-postgres
 

frameworks/JavaScript/nodejs/nodejs-postgres.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/nodejs/nodejs.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/ringojs/ringo-base.dockerfile

@@ -1,4 +1,4 @@
-FROM java:latest
+FROM tfb/java:latest
 
 COPY ./ ./
 

frameworks/JavaScript/ringojs/ringojs-base.dockerfile

@@ -1,4 +1,4 @@
-FROM java:latest
+FROM tfb/java:latest
 
 ENV NODE_ENV=production
 

frameworks/JavaScript/ringojs/ringojs-convenient.dockerfile

@@ -1,4 +1,4 @@
-FROM ringo-base:latest
+FROM tfb/ringo-base:latest
 
 RUN ringo-admin install grob/ringo-sqlstore && \
     ringo-admin install ringo/stick && \

frameworks/JavaScript/ringojs/ringojs.dockerfile

@@ -1,4 +1,4 @@
-FROM ringo-base:latest
+FROM tfb/ringo-base:latest
 
 RUN ringo-admin install oberhamsi/sql-ringojs-client && \
     ringo-admin install orfon/reinhardt && \

frameworks/JavaScript/sailsjs/sailsjs-postgres.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

frameworks/JavaScript/sailsjs/sailsjs.dockerfile

@@ -1,4 +1,4 @@
-FROM nodejs8:latest
+FROM tfb/nodejs8:latest
 
 COPY ./ ./
 

toolset/benchmark/benchmarker.py

@@ -742,7 +742,7 @@ class Benchmarker:
                         docker_id = subprocess.check_output(["docker", "ps", "-q"]).strip()
                     # We still need to sleep a bit before removing the image
                     time.sleep(5)
-                    subprocess.check_output(["docker", "image", "rm", "tfb-test-%s" % test.name])
+                    subprocess.check_output(["docker", "image", "rm", "tfb/test/%s" % test.name])
     ############################################################
     # End __stop_test
     ############################################################

toolset/benchmark/framework_test.py

@@ -217,7 +217,7 @@ class FrameworkTest:
         "-f", 
         docker_file, 
         "-t", 
-        dependency, 
+        "tfb/%s" % dependency, 
         os.path.dirname(docker_file)],
           stdout=subprocess.PIPE,
           stderr=subprocess.STDOUT)
@@ -243,7 +243,7 @@ class FrameworkTest:
       "-f", 
       test_docker_file, 
       "-t", 
-      "tfb-test-%s" % self.name, 
+      "tfb/test/%s" % self.name, 
       self.directory],
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT)
@@ -264,7 +264,7 @@ class FrameworkTest:
     ##########################
     # Run the Docker container
     ##########################
-    p = subprocess.Popen(["docker", "run", "--rm", "-p", "%s:%s" % (self.port, self.port), "--network=host", "tfb-test-%s" % self.name],
+    p = subprocess.Popen(["docker", "run", "--rm", "-p", "%s:%s" % (self.port, self.port), "--network=host", "tfb/test/%s" % self.name],
           stdout=subprocess.PIPE,
           stderr=subprocess.STDOUT)
     nbsr = setup_util.NonBlockingStreamReader(p.stdout,

toolset/benchmark/utils.py

@@ -25,19 +25,16 @@ def gather_docker_dependencies(docker_file):
 
     if os.path.exists(docker_file):
         with open(docker_file) as fp:
-            line = fp.readline()
-            if line:
+            for line in fp.readlines():
                 tokens = line.strip().split(' ')
                 if tokens[0] == "FROM":
                     # This is magic that our base image points to
                     if tokens[1] != "ubuntu:16.04":
-                        depTokens = tokens[1].strip().split(':')
-                        deps.append(depTokens[0])
-                        dep_docker_file = os.path.join(os.path.dirname(docker_file), depTokens[0] + ".dockerfile")
+                        depToken = tokens[1].strip().split(':')[0].strip().split('/')[1]
+                        deps.append(depToken)
+                        dep_docker_file = os.path.join(os.path.dirname(docker_file), depToken + ".dockerfile")
                         if not os.path.exists(dep_docker_file):
-                            # dep_docker_file = os.path.join(setup_util.get_fwroot(),
-                            #     "toolset", "setup", "linux", "docker", depTokens[0] + ".dockerfile")
-                            dep_docker_file = find_docker_file(docker_dir, depTokens[0] + ".dockerfile")
+                            dep_docker_file = find_docker_file(docker_dir, depToken + ".dockerfile")
                         deps.extend(gather_docker_dependencies(dep_docker_file))
 
     return deps

toolset/setup/linux/docker/languages/java.dockerfile

@@ -1,9 +1,9 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 RUN add-apt-repository -y ppa:openjdk-r/ppa
 RUN apt-get update
 RUN apt-get install -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
-    openjdk-8-jdk ant
+    openjdk-8-jdk
 
 # https://bugs.launchpad.net/ubuntu/+source/ca-certificates-java/+bug/1396760
 RUN /var/lib/dpkg/info/ca-certificates-java.postinst configure

toolset/setup/linux/docker/languages/ruby-2.4.dockerfile

@@ -1,4 +1,4 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 RUN apt install -yqq llvm-dev libjemalloc-dev zlib1g-dev libssl-dev libcurl4-openssl-dev bison
 

toolset/setup/linux/docker/systools/ant.dockerfile

@@ -0,0 +1,6 @@
+FROM tfb/java:latest
+
+RUN add-apt-repository -y ppa:openjdk-r/ppa
+RUN apt-get update
+RUN apt-get install -qqy -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
+    ant

toolset/setup/linux/docker/systools/gcc-6.dockerfile

@@ -1,4 +1,4 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 RUN add-apt-repository ppa:ubuntu-toolchain-r/test -y && \
     apt-get -yq update && \

toolset/setup/linux/docker/webservers/nodejs8.dockerfile

@@ -1,4 +1,4 @@
-FROM tfb:latest
+FROM tfb/base:latest
 
 ENV NODE_ENV=production
 

toolset/setup/linux/docker/webservers/resin.dockerfile

@@ -1,4 +1,4 @@
-FROM java:latest
+FROM tfb/java:latest
 
 ENV RESIN_HOME=/resin-4.0.55