[ci skip] Record more kinds of startup failures in results.json (#3541)

This creates an entry for these frameworks in the "completed" section of
results.json.  tfb-status.techempower.com uses this to display its
"X frameworks had problems starting or stopping" stat for each run.

toolset/benchmark/benchmarker.py

@@ -103,8 +103,9 @@ class Benchmarker:
 
         # If the test is in the excludes list, we skip it
         if self.config.exclude != None and test.name in self.config.exclude:
-            log("Test {name} has been added to the excludes list. Skipping.".
-                format(name=test.name),
+            message = "Test {name} has been added to the excludes list. Skipping.".format(name=test.name)
+            self.results.write_intermediate(test.name, message)
+            log(message,
                 prefix=log_prefix,
                 file=benchmark_log)
             return False
@@ -116,10 +117,9 @@ class Benchmarker:
 
             if self.__is_port_bound(test.port):
                 # We gave it our all
-                self.results.write_intermediate(test.name, "port " + str(
-                    test.port) + " is not available before start")
-                log("Error: Port %s is not available, cannot start %s" %
-                    (test.port, test.name),
+                message = "Error: Port %s is not available, cannot start %s" % (test.port, test.name)
+                self.results.write_intermediate(test.name, message)
+                log(message,
                     prefix=log_prefix,
                     file=benchmark_log,
                     color=Fore.RED)
@@ -130,9 +130,9 @@ class Benchmarker:
                 database_container = docker_helper.start_database(
                     self.config, test, test.database.lower())
                 if database_container is None:
-                    self.results.write_intermediate(test.name,
-                                                    "ERROR: Problem starting")
-                    log("ERROR: Problem building/running database container",
+                    message = "ERROR: Problem building/running database container"
+                    self.results.write_intermediate(test.name, message)
+                    log(message,
                         prefix=log_prefix,
                         file=benchmark_log,
                         color=Fore.RED)
@@ -143,9 +143,9 @@ class Benchmarker:
             if containers is None:
                 docker_helper.stop(self.config, containers, database_container,
                                    test)
-                self.results.write_intermediate(test.name,
-                                                "ERROR: Problem starting")
-                log("ERROR: Problem starting {name}".format(name=test.name),
+                message = "ERROR: Problem starting {name}".format(name=test.name)
+                self.results.write_intermediate(test.name, message)
+                log(message,
                     prefix=log_prefix,
                     file=benchmark_log,
                     color=Fore.RED)
@@ -160,7 +160,9 @@ class Benchmarker:
                         self.config, test, benchmark_log):
                     docker_helper.stop(self.config, containers,
                                        database_container, test)
-                    log("ERROR: One or more expected docker container exited early",
+                    message = "ERROR: One or more expected docker containers exited early"
+                    self.results.write_intermediate(test.name, message)
+                    log(message,
                         prefix=log_prefix,
                         file=benchmark_log,
                         color=Fore.RED)
@@ -171,7 +173,9 @@ class Benchmarker:
             if not accepting_requests:
                 docker_helper.stop(self.config, containers, database_container,
                                    test)
-                log("ERROR: Framework is not accepting requests from client machine",
+                message = "ERROR: Framework is not accepting requests from client machine"
+                self.results.write_intermediate(test.name, message)
+                log(message,
                     prefix=log_prefix,
                     file=benchmark_log,
                     color=Fore.RED)