Sleep more intelligently (#3404)

* Sleep more intelligently

* Fail earlier if a container dies

benchmark.cfg.example

@@ -21,7 +21,6 @@ pipeline_concurrency_levels=[256,1024,4096,16384]
 query_levels=[1,5,10,15,20]
 cached_query_levels=[1,10,20,50,100]
 mode=benchmark
-sleep=60
 test=None
 type=all
 verbose=True

toolset/benchmark/benchmarker.py

@@ -1,4 +1,4 @@
-from toolset.utils.output_helper import header
+from toolset.utils.output_helper import header, tee_output
 from toolset.utils.metadata_helper import gather_tests, gather_remaining_tests
 from toolset.utils import docker_helper
 
@@ -396,20 +396,32 @@ class Benchmarker:
                                                     "ERROR: Problem starting")
                     return sys.exit(1)
 
-                logging.info("Sleeping %s seconds to ensure framework is ready"
-                             % self.config.sleep)
-                time.sleep(self.config.sleep)
+                slept = 0
+                max_sleep = 60
+                while not test.is_running() and slept < max_sleep:
+                    if not docker_helper.successfully_running_containers(
+                            test.get_docker_files(), database_container_id,
+                            out):
+                        tee_output(
+                            out,
+                            "ERROR: One or more expected docker container exited early"
+                            + os.linesep)
+                        return sys.exit(1)
 
-                # Verify URLs
+                    time.sleep(1)
+                    slept += 1
+
+                # Debug mode blocks execution here until ctrl+c
                 if self.config.mode == "debug":
                     logging.info(
                         "Entering debug mode. Server has started. CTRL-c to stop."
                     )
                     while True:
                         time.sleep(1)
-                else:
-                    logging.info("Verifying framework URLs")
-                    passed_verify = test.verify_urls(logDir)
+
+                # Verify URLs
+                logging.info("Verifying framework URLs")
+                passed_verify = test.verify_urls(logDir)
 
                 # Benchmark this test
                 if self.config.mode == "benchmark":

toolset/benchmark/framework_test.py

@@ -57,6 +57,42 @@ class FrameworkTest:
         '''
         Start the test implementation
         '''
+        test_docker_files = self.get_docker_files()
+
+        result = docker_helper.build(self.benchmarker_config, [self.name], out)
+        if result != 0:
+            return result
+
+        return docker_helper.run(self.benchmarker_config, test_docker_files,
+                                 out)
+
+    def is_running(self):
+        '''
+        Determines whether this test implementation is up and accepting 
+        requests.
+        '''
+        test_type = None
+        for any_type in self.runTests:
+            test_type = any_type
+            break
+
+        url = "http://%s:%s/%s" % (self.benchmarker_config.server_host,
+                                   self.port,
+                                   self.runTests[test_type].get_url())
+
+        try:
+            FNULL = open(os.devnull, 'w')
+            subprocess.check_call(
+                ['curl', '-sSfl', url], stdout=FNULL, stderr=subprocess.STDOUT)
+        except:
+            return False
+
+        return True
+
+    def get_docker_files(self):
+        '''
+        Returns all the docker_files for this test.
+        '''
         test_docker_files = ["%s.dockerfile" % self.name]
         if self.docker_files is not None:
             if type(self.docker_files) is list:
@@ -65,12 +101,7 @@ class FrameworkTest:
                 raise Exception(
                     "docker_files in benchmark_config.json must be an array")
 
-        result = docker_helper.build(self.benchmarker_config, [self.name], out)
-        if result != 0:
-            return result
-
-        return docker_helper.run(self.benchmarker_config, test_docker_files,
-                                 database_container_id, out)
+        return test_docker_files
 
     def verify_urls(self, logPath):
         '''

toolset/utils/database_helper.py

@@ -0,0 +1,40 @@
+import MySQLdb
+import psycopg2
+import pymongo
+
+
+def test_database(config, database_name):
+    if database_name == "mysql":
+        try:
+            db = MySQLdb.connect(config.database_host, "benchmarkdbuser",
+                                 "benchmarkdbpass", "hello_world")
+            cursor = db.cursor()
+            cursor.execute("SELECT 1")
+            results = cursor.fetchall()
+            db.close()
+        except:
+            return False
+    elif database_name == "postgres":
+        try:
+            db = psycopg2.connect(
+                host=config.database_host,
+                port="5432",
+                user="benchmarkdbuser",
+                password="benchmarkdbpass",
+                database="hello_world")
+            cursor = db.cursor()
+            cursor.execute("SELECT 1")
+            results = cursor.fetchall()
+            db.close()
+        except:
+            return False
+    elif database_name == "mongodb":
+        try:
+            connection = pymongo.MongoClient(host=config.database_host)
+            db = connection.hello_world
+            db.world.find()
+            db.close()
+        except:
+            return False
+
+    return True

toolset/utils/docker_helper.py

@@ -5,6 +5,7 @@ import subprocess
 import multiprocessing
 import json
 import docker
+import time
 
 from threading import Thread
 
@@ -12,6 +13,7 @@ from toolset.utils import setup_util
 from toolset.utils.output_helper import tee_output
 from toolset.utils.metadata_helper import gather_tests
 from toolset.utils.ordered_set import OrderedSet
+from toolset.utils.database_helper import test_database
 
 
 def clean():
@@ -123,7 +125,7 @@ def build(benchmarker_config, test_names, out):
     return 0
 
 
-def run(benchmarker_config, docker_files, database_container_id, out):
+def run(benchmarker_config, docker_files, out):
     '''
     Run the given Docker container(s)
     '''
@@ -162,6 +164,15 @@ def run(benchmarker_config, docker_files, database_container_id, out):
             print(e)
             return 1
 
+    return 0
+
+
+def successfully_running_containers(docker_files, database_container_id, out):
+    '''
+    Returns whether all the expected containers for the given docker_files are
+    running.
+    '''
+    client = docker.from_env()
     running_container_length = len(
         client.containers.list(filters={'status': 'running'}))
     expected_length = len(docker_files)
@@ -174,9 +185,8 @@ def run(benchmarker_config, docker_files, database_container_id, out):
                                           running_container.image, os.linesep))
         tee_output(out, "Excepted %s running containers; saw %s%s" %
                    (running_container_length, expected_length, os.linesep))
-        return 1
-
-    return 0
+        return False
+    return True
 
 
 def stop(config, database_container_id, test, out):
@@ -269,8 +279,16 @@ def start_database(config, database):
     command = list(config.database_ssh_command)
     command.extend(
         ['docker', 'run', '-d', '--rm', '--init', '--network=host', database])
-    pid = subprocess.check_output(command).strip()
-    return pid
+    docker_id = subprocess.check_output(command).strip()
+
+    # Sleep until the database accepts connections
+    slept = 0
+    max_sleep = 60
+    while not test_database(config, database) and slept < max_sleep:
+        time.sleep(1)
+        slept += 1
+
+    return docker_id
 
 
 def __gather_dependencies(docker_file):