Better logging error checking (#3301)

* Better logging error checking

* No idea how this EVER worked before

Indentation blocks... amirite?

toolset/benchmark/benchmarker.py

@@ -422,11 +422,12 @@ class Benchmarker:
         dbfiles = ""
         for dbfile in os.listdir(dbpath):
             dbfiles += "%s " % os.path.join(dbpath,dbfile)
-        p = subprocess.Popen(__scp_string(dbfiles.split()),
-        stdin=subprocess.PIPE)
+        p = subprocess.Popen(__scp_string(dbfiles.split()), stdin=subprocess.PIPE, stdout=self.quiet_out, stderr=subprocess.STDOUT)
         p.communicate()
-        p = subprocess.Popen(self.database_ssh_string, shell=True, stdin=subprocess.PIPE)
+        p = subprocess.Popen(self.database_ssh_string, shell=True, stdin=subprocess.PIPE, stdout=self.quiet_out, stderr=subprocess.STDOUT)
         p.communicate("docker build -f ~/%s.dockerfile -t %s ~/" % (database, database))
+        if p.returncode != 0:
+            return None
 
         p = subprocess.Popen(self.database_ssh_string, stdin=subprocess.PIPE, shell=True, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
         (out,err) = p.communicate("docker run -d --rm -p %s:%s --network=host %s" % (port,port,database))
@@ -595,6 +596,11 @@ class Benchmarker:
                         "postgres": 5432
                     }
                     database_container_id = self.__setup_database_container(test.database.lower(), ports[test.database.lower()])
+                    if not database_container_id:
+                        out.write("ERROR: Problem building/running database container")
+                        out.flush()
+                        self.__write_intermediate_results(test.name,"ERROR: Problem starting")
+                        return sys.exit(1)
 
                 ##########################
                 # Start webapp

toolset/benchmark/framework_test.py

@@ -174,71 +174,6 @@ class FrameworkTest:
 
     # Setup environment variables
     logDir = os.path.join(self.fwroot, self.benchmarker.full_results_directory(), 'logs', self.name.lower())
-    # bash_functions_path= os.path.join(self.fwroot, 'toolset/setup/linux/bash_functions.sh')
-
-    # os.environ['TROOT'] = self.directory
-    # os.environ['IROOT'] = self.install_root
-    # os.environ['DBHOST'] = socket.gethostbyname(self.database_host)
-    # os.environ['LOGDIR'] = logDir
-    # os.environ['MAX_CONCURRENCY'] = str(max(self.benchmarker.concurrency_levels))
-
-    # Always ensure that IROOT exists
-    # if not os.path.exists(self.install_root):
-    #   os.mkdir(self.install_root)
-
-    # if not os.path.exists(os.path.join(self.install_root,"TFBReaper")):
-    #   subprocess.check_call(['gcc', 
-    #     '-std=c99', 
-    #     '-o%s/TFBReaper' % self.install_root, 
-    #     os.path.join(self.fwroot,'toolset/setup/linux/TFBReaper.c')],
-    #     stderr=out, stdout=out)
-
-    # Check that the client is setup
-    # if not os.path.exists(os.path.join(self.install_root, 'client.installed')):
-    #   print("\nINSTALL: Installing client software\n")    
-    #   # TODO: hax; should dynamically know where this file is
-    #   with open (self.fwroot + "/toolset/setup/linux/client.sh", "r") as myfile:
-    #     remote_script=myfile.read()
-    #     print("\nINSTALL: {!s}".format(self.benchmarker.client_ssh_string))
-    #     p = subprocess.Popen(self.benchmarker.client_ssh_string.split(" ") + ["bash"], stdin=subprocess.PIPE)
-    #     p.communicate(remote_script)
-    #     returncode = p.returncode
-    #     if returncode != 0:
-    #       self.__install_error("status code %s running subprocess '%s'." % (returncode, self.benchmarker.client_ssh_string))
-    #   print("\nINSTALL: Finished installing client software\n")
-    #   subprocess.check_call('touch client.installed', shell=True, cwd=self.install_root, executable='/bin/bash')
-
-    # Run the module start inside parent of TROOT
-    #  - we use the parent as a historical accident, a number of tests
-    # refer to their TROOT maually still
-    # previousDir = os.getcwd()
-    # os.chdir(os.path.dirname(self.troot))
-    # logging.info("Running setup module start (cwd=%s)", self.directory)
-
-    # command = 'bash -exc "source %s && source %s.sh"' % (
-    #   bash_functions_path,
-    #   os.path.join(self.troot, self.setup_file))
-
-    # debug_command = '''\
-    #   export FWROOT=%s          &&  \\
-    #   export TROOT=%s           &&  \\
-    #   export IROOT=%s           &&  \\
-    #   export DBHOST=%s          &&  \\
-    #   export LOGDIR=%s          &&  \\
-    #   export MAX_CONCURRENCY=%s && \\
-    #   cd %s && \\
-    #   %s/TFBReaper "bash -exc \\\"source %s && source %s.sh\\\"''' % (self.fwroot,
-    #     self.directory,
-    #     self.install_root,
-    #     socket.gethostbyname(self.database_host),
-    #     logDir,
-    #     max(self.benchmarker.concurrency_levels),
-    #     self.directory,
-    #     self.install_root,
-    #     bash_functions_path,
-    #     os.path.join(self.troot, self.setup_file))
-    # logging.info("To run %s manually, copy/paste this:\n%s", self.name, debug_command)
-
 
     def tee_output(prefix, line):
       # Needs to be one atomic write
@@ -249,7 +184,6 @@ class FrameworkTest:
       # Log to current terminal
       sys.stdout.write(line)
       sys.stdout.flush()
-      # logging.error("".join([prefix, line]))
 
       out.write(line)
       out.flush()
@@ -280,18 +214,24 @@ class FrameworkTest:
               tee_output(prefix, line)
           except setup_util.EndOfStream:
             break
-      p = subprocess.Popen(["docker", "build", "-f", test_docker_file, "-t", "tfb-test-%s" % self.name, self.directory],
-          stdout=subprocess.PIPE,
-          stderr=subprocess.STDOUT)
-      nbsr = setup_util.NonBlockingStreamReader(p.stdout)
-      while (p.poll() is None):
-        for i in xrange(10):
-          try:
-            line = nbsr.readline(0.05)
-            if line:
-              tee_output(prefix, line)
-          except setup_util.EndOfStream:
-            break
+      if p.returncode != 0:
+        tee_output(prefix, "Docker build failed; terminating\n")
+        return 1
+    p = subprocess.Popen(["docker", "build", "-f", test_docker_file, "-t", "tfb-test-%s" % self.name, self.directory],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT)
+    nbsr = setup_util.NonBlockingStreamReader(p.stdout)
+    while (p.poll() is None):
+      for i in xrange(10):
+        try:
+          line = nbsr.readline(0.05)
+          if line:
+            tee_output(prefix, line)
+        except setup_util.EndOfStream:
+          break
+    if p.returncode != 0:
+      tee_output(prefix, "Docker build failed; terminating\n")
+      return 1
         
 
     ##########################