feat: log testing docker resource usage statistics [ci fw-only Fortran/fortran.io] (#9238)

toolset/benchmark/benchmarker.py

@@ -1,3 +1,6 @@
+import threading
+
+from docker.models.containers import Container
 from toolset.utils.output_helper import log, FNULL
 from toolset.utils.docker_helper import DockerHelper
 from toolset.utils.time_logger import TimeLogger
@@ -263,12 +266,6 @@ class Benchmarker:
             log("BENCHMARKING %s ... " % test_type.upper(), file=benchmark_log)
 
             test = framework_test.runTests[test_type]
-            raw_file = self.results.get_raw_file(framework_test.name,
-                                                 test_type)
-            if not os.path.exists(raw_file):
-                # Open to create the empty file
-                with open(raw_file, 'w'):
-                    pass
 
             if not test.failed:
                 # Begin resource usage metrics collection
@@ -281,8 +278,8 @@ class Benchmarker:
                                                        framework_test.port,
                                                        test.get_url()))
 
-                self.docker_helper.benchmark(script, script_variables,
-                                             raw_file)
+                benchmark_container = self.docker_helper.benchmark(script, script_variables)
+                self.__log_container_output(benchmark_container, framework_test, test_type)
 
                 # End resource usage metrics collection
                 self.__end_logging()
@@ -323,3 +320,31 @@ class Benchmarker:
         self.subprocess_handle.terminate()
         self.subprocess_handle.communicate()
 
+    def __log_container_output(self, container: Container, framework_test, test_type) -> None:
+        def save_docker_logs(stream):
+            raw_file_path = self.results.get_raw_file(framework_test.name, test_type)
+            with open(raw_file_path, 'w') as file:
+                for line in stream:
+                    log(line.decode(), file=file)
+
+        def save_docker_stats(stream):
+            docker_file_path = self.results.get_docker_stats_file(framework_test.name, test_type)
+            with open(docker_file_path, 'w') as file:
+                file.write('[\n')
+                is_first_line = True
+                for line in stream:
+                    if is_first_line:
+                        is_first_line = False
+                    else:
+                        file.write(',')
+                    file.write(line.decode())
+                file.write(']')
+
+        threads = [
+            threading.Thread(target=lambda: save_docker_logs(container.logs(stream=True))),
+            threading.Thread(target=lambda: save_docker_stats(container.stats(stream=True)))
+        ]
+
+        [thread.start() for thread in threads]
+        [thread.join() for thread in threads]
+

toolset/utils/docker_helper.py

@@ -420,16 +420,11 @@ class DockerHelper:
         except:
             return False
 
-    def benchmark(self, script, variables, raw_file):
+    def benchmark(self, script, variables):
         '''
         Runs the given remote_script on the wrk container on the client machine.
         '''
 
-        def watch_container(container):
-            with open(raw_file, 'w') as benchmark_file:
-                for line in container.logs(stream=True):
-                    log(line.decode(), file=benchmark_file)
-
         if self.benchmarker.config.network_mode is None:
             sysctl = {'net.core.somaxconn': 65535}
         else:
@@ -438,8 +433,7 @@ class DockerHelper:
 
         ulimit = [{'name': 'nofile', 'hard': 65535, 'soft': 65535}]
 
-        watch_container(
-            self.client.containers.run(
+        return self.client.containers.run(
                 "techempower/tfb.wrk",
                 "/bin/bash /%s" % script,
                 environment=variables,
@@ -450,4 +444,4 @@ class DockerHelper:
                 ulimits=ulimit,
                 sysctls=sysctl,
                 remove=True,
-                log_config={'type': None}))
+                log_config={'type': None})

toolset/utils/results.py

@@ -212,29 +212,34 @@ class Results:
         except (ValueError, IOError):
             pass
 
+    def __make_dir_for_file(self, test_name: str, test_type: str, file_name: str):
+        path = os.path.join(self.directory, test_name, test_type, file_name)
+        try:
+            os.makedirs(os.path.dirname(path), exist_ok=True)
+        except OSError:
+            pass
+        return path
+
+    def get_docker_stats_file(self, test_name, test_type):
+        '''
+        Returns the stats file name for this test_name and
+        Example: fw_root/results/timestamp/test_type/test_name/stats.txt
+        '''
+        return self.__make_dir_for_file(test_name, test_type, "docker_stats.json")
+
     def get_raw_file(self, test_name, test_type):
         '''
         Returns the output file for this test_name and test_type
         Example: fw_root/results/timestamp/test_type/test_name/raw.txt
         '''
-        path = os.path.join(self.directory, test_name, test_type, "raw.txt")
-        try:
-            os.makedirs(os.path.dirname(path))
-        except OSError:
-            pass
-        return path
+        return self.__make_dir_for_file(test_name, test_type, "raw.txt")
 
     def get_stats_file(self, test_name, test_type):
         '''
         Returns the stats file name for this test_name and
         Example: fw_root/results/timestamp/test_type/test_name/stats.txt
         '''
-        path = os.path.join(self.directory, test_name, test_type, "stats.txt")
-        try:
-            os.makedirs(os.path.dirname(path))
-        except OSError:
-            pass
-        return path
+        return self.__make_dir_for_file(test_name, test_type, "stats.txt")
 
     def report_verify_results(self, framework_test, test_type, result):
         '''