Terminate docker builds after 60 mins (#4569)

toolset/utils/docker_helper.py

@@ -64,6 +64,14 @@ class DockerHelper:
                             file=build_log,
                             color=Fore.WHITE + Style.BRIGHT \
                                 if re.match(r'^Step \d+\/\d+', line) else '')
+                    # Kill docker builds if they exceed 60 mins. This will only
+                    # catch builds that are still printing output.
+                    if self.benchmarker.time_logger.time_since_start() > 3600:
+                        log("Build time exceeded 60 minutes",
+                            prefix=log_prefix,
+                            file=build_log,
+                            color=Fore.RED)
+                        raise Exception
 
                 if buffer:
                     log(buffer,

toolset/utils/time_logger.py

@@ -67,6 +67,9 @@ class TimeLogger:
     def mark_build_start(self):
         self.build_start = time.time()
 
+    def time_since_start(self):
+        return time.time() - self.build_start
+
     def log_build_end(self, log_prefix, file):
         total = int(time.time() - self.build_start)
         self.build_total = self.build_total + total