Browse Source

Build wrk and only once at the start of all tests (#3609)

* Quick fix for displaying errors

* remove dupe code and unused params

* build wrk locally

* [ci skip]
Nate 7 years ago
parent
commit
40cc976683
3 changed files with 90 additions and 86 deletions
  1. 5 3
      toolset/benchmark/benchmarker.py
  2. 83 74
      toolset/utils/docker_helper.py
  3. 2 9
      toolset/wrk/wrk.dockerfile

+ 5 - 3
toolset/benchmark/benchmarker.py

@@ -42,6 +42,8 @@ class Benchmarker:
         any_failed = False
         any_failed = False
         # Run tests
         # Run tests
         log("Running Tests...", border='=')
         log("Running Tests...", border='=')
+        docker_helper.build_wrk(self.config)
+
         with open(os.path.join(self.results.directory, 'benchmark.log'),
         with open(os.path.join(self.results.directory, 'benchmark.log'),
                   'w') as benchmark_log:
                   'w') as benchmark_log:
             for test in all_tests:
             for test in all_tests:
@@ -129,7 +131,7 @@ class Benchmarker:
             # Start database container
             # Start database container
             if test.database.lower() != "none":
             if test.database.lower() != "none":
                 database_container = docker_helper.start_database(
                 database_container = docker_helper.start_database(
-                    self.config, test, test.database.lower())
+                    self.config, test.database.lower())
                 if database_container is None:
                 if database_container is None:
                     message = "ERROR: Problem building/running database container"
                     message = "ERROR: Problem building/running database container"
                     self.results.write_intermediate(test.name, message)
                     self.results.write_intermediate(test.name, message)
@@ -271,7 +273,7 @@ class Benchmarker:
 
 
     def __begin_logging(self, framework_test, test_type):
     def __begin_logging(self, framework_test, test_type):
         '''
         '''
-        Starts a thread to monitor the resource usage, to be synced with the 
+        Starts a thread to monitor the resource usage, to be synced with the
         client's time.
         client's time.
         TODO: MySQL and InnoDB are possible. Figure out how to implement them.
         TODO: MySQL and InnoDB are possible. Figure out how to implement them.
         '''
         '''
@@ -295,7 +297,7 @@ class Benchmarker:
 
 
     def __is_port_bound(self, port):
     def __is_port_bound(self, port):
         '''
         '''
-        Check if the requested port is available. If it isn't available, then a 
+        Check if the requested port is available. If it isn't available, then a
         previous test probably didn't shutdown properly.
         previous test probably didn't shutdown properly.
         '''
         '''
         port = int(port)
         port = int(port)

+ 83 - 74
toolset/utils/docker_helper.py

@@ -44,11 +44,58 @@ def clean(benchmarker_config):
                 client.images.remove(image.id, force=True)
                 client.images.remove(image.id, force=True)
     client.images.prune()
     client.images.prune()
 
 
+def __build(base_url, path, build_log_file, log_prefix, dockerfile, tag):
+    '''
+    Builds docker containers using docker-py low-level api
+    '''
+
+    with open(build_log_file, 'w') as build_log:
+        try:
+            client = docker.APIClient(base_url=base_url)
+            output = client.build(
+                path=path,
+                dockerfile=dockerfile,
+                tag=tag,
+                forcerm=True,
+                pull=True)
+            buffer = ""
+            for token in output:
+                if token.startswith('{"stream":'):
+                    token = json.loads(token)
+                    token = token[token.keys()[0]].encode('utf-8')
+                    buffer += token
+                elif token.startswith('{"errorDetail":'):
+                    token = json.loads(token)
+                    raise Exception(token['errorDetail']['message'])
+                while "\n" in buffer:
+                    index = buffer.index("\n")
+                    line = buffer[:index]
+                    buffer = buffer[index + 1:]
+                    log(line,
+                        prefix=log_prefix,
+                        file=build_log,
+                        color=Fore.WHITE + Style.BRIGHT \
+                            if re.match(r'^Step \d+\/\d+', line) else '')
+
+            if buffer:
+                log(buffer,
+                    prefix=log_prefix,
+                    file=build_log,
+                    color=Fore.WHITE + Style.BRIGHT \
+                        if re.match(r'^Step \d+\/\d+', buffer) else '')
+        except Exception:
+            tb = traceback.format_exc()
+            log("Docker build failed; terminating",
+                prefix=log_prefix,
+                file=build_log,
+                color=Fore.RED)
+            log(tb, prefix=log_prefix, file=build_log)
+            raise
+
 
 
 def build(benchmarker_config, test_names, build_log_dir=os.devnull):
 def build(benchmarker_config, test_names, build_log_dir=os.devnull):
     '''
     '''
-    Builds the dependency chain as well as the test implementation docker images
-    for the given tests.
+    Builds the test docker containers
     '''
     '''
     tests = gather_tests(
     tests = gather_tests(
         include=test_names, benchmarker_config=benchmarker_config)
         include=test_names, benchmarker_config=benchmarker_config)
@@ -63,47 +110,18 @@ def build(benchmarker_config, test_names, build_log_dir=os.devnull):
             build_log_file = os.path.join(
             build_log_file = os.path.join(
                 build_log_dir,
                 build_log_dir,
                 "%s.log" % test_docker_file.replace(".dockerfile", "").lower())
                 "%s.log" % test_docker_file.replace(".dockerfile", "").lower())
-        with open(build_log_file, 'w') as build_log:
-            try:
-                client = docker.APIClient(
-                    base_url=benchmarker_config.server_docker_host)
-                output = client.build(
-                    path=test.directory,
-                    dockerfile=test_docker_file,
-                    tag="techempower/tfb.test.%s" %
-                        test_docker_file.replace(".dockerfile", ""),
-                    forcerm=True,
-                    pull=True)
-                buffer = ""
-                for token in output:
-                    if token.startswith('{"stream":'):
-                        token = json.loads(token)
-                        token = token[token.keys()[0]].encode('utf-8')
-                    buffer += token
-                    while "\n" in buffer:
-                        index = buffer.index("\n")
-                        line = buffer[:index]
-                        buffer = buffer[index + 1:]
-                        log(line,
-                            prefix=log_prefix,
-                            file=build_log,
-                            color=Fore.WHITE + Style.BRIGHT \
-                                if re.match(r'^Step \d+\/\d+', line) else '')
-
-                if buffer:
-                    log(buffer,
-                        prefix=log_prefix,
-                        file=build_log,
-                        color=Fore.WHITE + Style.BRIGHT \
-                            if re.match(r'^Step \d+\/\d+', buffer) else '')
-            except Exception:
-                tb = traceback.format_exc()
-                log("Docker build failed; terminating",
-                    prefix=log_prefix,
-                    file=build_log,
-                    color=Fore.RED)
-                log(tb, prefix=log_prefix, file=build_log)
-                return 1
+
+        try:
+            __build(
+                base_url=benchmarker_config.server_docker_host,
+                build_log_file=build_log_file,
+                log_prefix=log_prefix,
+                path=test.directory,
+                dockerfile=test_docker_file,
+                tag="techempower/tfb.test.%s" %
+                    test_docker_file.replace(".dockerfile", ""))
+        except Exception:
+            return 1
 
 
     return 0
     return 0
 
 
@@ -225,7 +243,7 @@ def stop(benchmarker_config=None,
 def find(path, pattern):
 def find(path, pattern):
     '''
     '''
     Finds and returns all the the files matching the given pattern recursively in
     Finds and returns all the the files matching the given pattern recursively in
-    the given path. 
+    the given path.
     '''
     '''
     for root, dirs, files in os.walk(path):
     for root, dirs, files in os.walk(path):
         for name in files:
         for name in files:
@@ -233,9 +251,9 @@ def find(path, pattern):
                 return os.path.join(root, name)
                 return os.path.join(root, name)
 
 
 
 
-def start_database(benchmarker_config, test, database):
+def start_database(benchmarker_config, database):
     '''
     '''
-    Sets up a container for the given database and port, and starts said docker 
+    Sets up a container for the given database and port, and starts said docker
     container.
     container.
     '''
     '''
     image_name = "techempower/%s:latest" % database
     image_name = "techempower/%s:latest" % database
@@ -245,26 +263,13 @@ def start_database(benchmarker_config, test, database):
                                 "databases", database)
                                 "databases", database)
     docker_file = "%s.dockerfile" % database
     docker_file = "%s.dockerfile" % database
 
 
-    client = docker.DockerClient(
-        base_url=benchmarker_config.database_docker_host)
-    try:
-        # Don't pull if we have it
-        client.images.get(image_name)
-        log("Found published image; skipping build", prefix=log_prefix)
-    except:
-        # Build the database image
-        for line in docker.APIClient(
-                base_url=benchmarker_config.database_docker_host).build(
-                    path=database_dir,
-                    dockerfile=docker_file,
-                    tag="techempower/%s" % database):
-            if line.startswith('{"stream":'):
-                line = json.loads(line)
-                line = line[line.keys()[0]].encode('utf-8')
-                log(line,
-                    prefix=log_prefix,
-                    color=Fore.WHITE + Style.BRIGHT \
-                        if re.match(r'^Step \d+\/\d+', line) else '')
+    __build(
+        base_url=benchmarker_config.database_docker_host,
+        path=database_dir,
+        dockerfile=docker_file,
+        log_prefix=log_prefix,
+        build_log_file=os.devnull,
+        tag="techempower/%s" % database)
 
 
     client = docker.DockerClient(
     client = docker.DockerClient(
         base_url=benchmarker_config.database_docker_host)
         base_url=benchmarker_config.database_docker_host)
@@ -296,22 +301,26 @@ def start_database(benchmarker_config, test, database):
 
 
     return container
     return container
 
 
+def build_wrk(benchmarker_config):
+    '''
+    Builds the techempower/tfb.wrk container
+    '''
+    __build(
+        base_url=benchmarker_config.client_docker_host,
+        path=os.path.join(benchmarker_config.fwroot, "toolset", "wrk"),
+        dockerfile="wrk.dockerfile",
+        log_prefix="wrk: ",
+        build_log_file=os.devnull,
+        tag="techempower/tfb.wrk")
 
 
 def test_client_connection(benchmarker_config, url):
 def test_client_connection(benchmarker_config, url):
     '''
     '''
-    Tests that the app server at the given url responds successfully to a 
+    Tests that the app server at the given url responds successfully to a
     request.
     request.
     '''
     '''
     client = docker.DockerClient(
     client = docker.DockerClient(
         base_url=benchmarker_config.client_docker_host)
         base_url=benchmarker_config.client_docker_host)
 
 
-    try:
-        client.images.get('techempower/tfb.wrk:latest')
-    except:
-        log("Attempting docker pull for image (this can take some time)",
-            prefix="techempower/tfb.wrk:latest: ")
-        client.images.pull('techempower/tfb.wrk:latest')
-
     try:
     try:
         client.containers.run(
         client.containers.run(
             'techempower/tfb.wrk',
             'techempower/tfb.wrk',

+ 2 - 9
toolset/wrk/wrk.dockerfile

@@ -1,11 +1,4 @@
-FROM ubuntu:16.04
-
-# One -q produces output suitable for logging (mostly hides
-# progress indicators)
-RUN apt-get -yq update
-
-RUN apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
-    build-essential git libev-dev libpq-dev libreadline6-dev curl
+FROM buildpack-deps:xenial
 
 
 # wrk
 # wrk
 RUN curl -sL -o wrk-4.0.1.tar.gz https://github.com/wg/wrk/archive/4.0.1.tar.gz
 RUN curl -sL -o wrk-4.0.1.tar.gz https://github.com/wg/wrk/archive/4.0.1.tar.gz
@@ -29,4 +22,4 @@ ENV duration duration
 ENV max_concurrency max_concurrency
 ENV max_concurrency max_concurrency
 ENV max_threads max_threads
 ENV max_threads max_threads
 ENV pipeline pipeline
 ENV pipeline pipeline
-ENV accept accept
+ENV accept accept