Direct ssh commands (#3401)

toolset/benchmark/benchmarker.py

@@ -106,7 +106,7 @@ class Benchmarker:
                     # Run the benchmark
                     with open(raw_file, 'w') as raw_file:
                         p = subprocess.Popen(
-                            self.config.client_ssh_string.split(" "),
+                            self.config.client_ssh_command,
                             stdin=subprocess.PIPE,
                             stdout=raw_file,
                             stderr=raw_file)
@@ -236,13 +236,9 @@ class Benchmarker:
         kernel.shm*: http://seriousbirder.com/blogs/linux-understanding-shmmax-and-shmall-settings/
         For kernel.sem: https://access.redhat.com/documentation/en-US/Red_Hat_Enterprise_Linux/5/html/Tuning_and_Optimizing_Red_Hat_Enterprise_Linux_for_Oracle_9i_and_10g_Databases/chap-Oracle_9i_and_10g_Tuning_Guide-Setting_Semaphores.html
         '''
-        p = subprocess.Popen(
-            self.config.database_ssh_string,
-            stdin=subprocess.PIPE,
-            shell=True,
-            stdout=self.config.quiet_out,
-            stderr=subprocess.STDOUT)
-        p.communicate("""
+        command = list(self.config.database_ssh_command)
+        command.extend([
+            """
             sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535
             sudo sysctl -w net.core.somaxconn=65535
             sudo sysctl -w kernel.sched_autogroup_enabled=0
@@ -252,7 +248,9 @@ class Benchmarker:
             sudo sysctl -w kernel.shmmax=2147483648
             sudo sysctl -w kernel.shmall=2097152
             sudo sysctl -w kernel.sem="250 32000 256 512"
-        """)
+            """
+        ])
+        subprocess.check_call(command)
         # TODO - print kernel configuration to file
         # echo "Printing kernel configuration:" && sudo sysctl -a
 
@@ -262,13 +260,9 @@ class Benchmarker:
         before running the tests. Is very similar to the server setup, but may also 
         include client specific changes.
         '''
-        p = subprocess.Popen(
-            self.config.client_ssh_string,
-            stdin=subprocess.PIPE,
-            shell=True,
-            stdout=self.config.quiet_out,
-            stderr=subprocess.STDOUT)
-        p.communicate("""
+        command = list(self.config.client_ssh_command)
+        command.extend([
+            """
             sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535
             sudo sysctl -w net.core.somaxconn=65535
             sudo -s ulimit -n 65535
@@ -276,7 +270,8 @@ class Benchmarker:
             sudo sysctl net.ipv4.tcp_tw_recycle=1
             sudo sysctl -w kernel.shmmax=2147483648
             sudo sysctl -w kernel.shmall=2097152
-        """)
+            """
+        ])
 
     def __run_tests(self, tests):
         '''

toolset/benchmark/framework_test.py

@@ -74,7 +74,7 @@ class FrameworkTest:
 
     def verify_urls(self, logPath):
         '''
-        Verifys each of the URLs for this test. THis will sinply curl the URL and 
+        Verifys each of the URLs for this test. This will simply curl the URL and 
         check for it's return status. For each url, a flag will be set on this 
         object for whether or not it passed.
         Returns True if all verifications succeeded

toolset/utils/benchmark_config.py

@@ -72,12 +72,20 @@ class BenchmarkConfig:
         else:
             self.timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
 
-        # Setup the ssh command string
-        self.database_ssh_string = "ssh -T -o StrictHostKeyChecking=no " + self.database_user + "@" + self.database_host
-        self.client_ssh_string = "ssh -T -o StrictHostKeyChecking=no " + self.client_user + "@" + self.client_host
-        if self.database_identity_file != None:
-            self.database_ssh_string = self.database_ssh_string + " -i " + self.database_identity_file
+        # Setup the ssh commands
+        self.client_ssh_command = [
+            'ssh', '-T', 'o', 'StrictHostKeyChecking=no',
+            self.client_user + "@" + self.client_host
+        ]
         if self.client_identity_file != None:
-            self.client_ssh_string = self.client_ssh_string + " -i " + self.client_identity_file
+            self.client_ssh_command.extend(['-i', self.client_identity_file])
+
+        self.database_ssh_command = [
+            'ssh', '-T', '-o', 'StrictHostKeyChecking=no',
+            self.database_user + "@" + self.database_host
+        ]
+        if self.database_identity_file != None:
+            self.database_ssh_command.extend(
+                ['-i', self.database_identity_file])
 
         self.run_test_timeout_seconds = 7200

toolset/utils/docker_helper.py

@@ -196,13 +196,9 @@ def stop(config, database_container_id, test, out):
         pass
     # Stop the database container
     if database_container_id:
-        p = subprocess.Popen(
-            config.database_ssh_string,
-            stdin=subprocess.PIPE,
-            shell=True,
-            stdout=config.quiet_out,
-            stderr=subprocess.STDOUT)
-        p.communicate("docker stop %s" % database_container_id)
+        command = list(config.database_ssh_command)
+        command.extend(['docker', 'stop', database_container_id])
+        subprocess.check_call(command)
     client.images.prune()
 
 
@@ -230,13 +226,9 @@ def start_database(config, database):
             return False
         return len(s) % 2 == 0
 
-    p = subprocess.Popen(
-        config.database_ssh_string,
-        stdin=subprocess.PIPE,
-        shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT)
-    out = p.communicate("docker images  -q %s" % database)[0]
+    command = list(config.database_ssh_command)
+    command.extend(['docker', 'images', '-q', database])
+    out = subprocess.check_output(command)
     dbid = ''
     if len(out.splitlines()) > 0:
         dbid = out.splitlines()[len(out.splitlines()) - 1]
@@ -245,7 +237,7 @@ def start_database(config, database):
     # fe12ca519b47, and we do not want to rebuild if it exists
     if len(dbid) != 12 and not __is_hex(dbid):
 
-        def __scp_string(files):
+        def __scp_command(files):
             scpstr = ["scp", "-i", config.database_identity_file]
             for file in files:
                 scpstr.append(file)
@@ -253,43 +245,29 @@ def start_database(config, database):
                                            config.database_host, database))
             return scpstr
 
-        p = subprocess.Popen(
-            config.database_ssh_string,
-            shell=True,
-            stdin=subprocess.PIPE,
-            stdout=config.quiet_out,
-            stderr=subprocess.STDOUT)
-        p.communicate("mkdir -p %s" % database)
+        command = list(config.database_ssh_command)
+        command.extend(['mkdir', '-p', database])
+        subprocess.check_call(command)
         dbpath = os.path.join(config.fwroot, "toolset", "setup", "docker",
                               "databases", database)
         dbfiles = ""
         for dbfile in os.listdir(dbpath):
             dbfiles += "%s " % os.path.join(dbpath, dbfile)
-        p = subprocess.Popen(
-            __scp_string(dbfiles.split()),
-            stdin=subprocess.PIPE,
-            stdout=config.quiet_out,
-            stderr=subprocess.STDOUT)
-        p.communicate()
-        p = subprocess.Popen(
-            config.database_ssh_string,
-            shell=True,
-            stdin=subprocess.PIPE,
-            stdout=config.quiet_out,
-            stderr=subprocess.STDOUT)
-        p.communicate("docker build -f ~/%s/%s.dockerfile -t %s ~/%s" %
-                      (database, database, database, database))
-        if p.returncode != 0:
-            return None
-
-    p = subprocess.Popen(
-        config.database_ssh_string,
-        stdin=subprocess.PIPE,
-        shell=True,
-        stdout=subprocess.PIPE,
-        stderr=subprocess.STDOUT)
-    out = p.communicate("docker run -d --rm --network=host %s" % database)[0]
-    return out.splitlines()[len(out.splitlines()) - 1]
+        subprocess.check_call(__scp_command(dbfiles.split()))
+
+        command = list(config.database_ssh_command)
+        command.extend([
+            'docker', 'build', '-f',
+            '~/%s/%s.dockerfile' % (database, database), '-t', database,
+            '~/%s' % database
+        ])
+        subprocess.check_call(command)
+
+    command = list(config.database_ssh_command)
+    command.extend(
+        ['docker', 'run', '-d', '--rm', '--init', '--network=host', database])
+    pid = subprocess.check_output(command).strip()
+    return pid
 
 
 def __gather_dependencies(docker_file):