|
@@ -18,6 +18,7 @@ import sys
|
|
|
import logging
|
|
|
import socket
|
|
|
import threading
|
|
|
+import textwrap
|
|
|
from pprint import pprint
|
|
|
|
|
|
from multiprocessing import Process
|
|
@@ -493,7 +494,7 @@ class Benchmarker:
|
|
|
|
|
|
try:
|
|
|
os.makedirs(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name)))
|
|
|
- except:
|
|
|
+ except Exception:
|
|
|
pass
|
|
|
with open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'out.txt'), 'w') as out, \
|
|
|
open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'err.txt'), 'w') as err:
|
|
@@ -536,10 +537,16 @@ class Benchmarker:
|
|
|
time.sleep(10)
|
|
|
|
|
|
if self.__is_port_bound(test.port):
|
|
|
- self.__write_intermediate_results(test.name, "port " + str(test.port) + " is not available before start")
|
|
|
- err.write(header("Error: Port %s is not available, cannot start %s" % (test.port, test.name)))
|
|
|
+ err.write(header("Error: Port %s is not available, attempting to recover" % test.port))
|
|
|
err.flush()
|
|
|
- return exit_with_code(1)
|
|
|
+ print "Error: Port %s is not available, attempting to recover" % test.port
|
|
|
+ self.__forciblyEndPortBoundProcesses(test.port, out, err)
|
|
|
+ if self.__is_port_bound(test.port):
|
|
|
+ self.__write_intermediate_results(test.name, "port " + str(test.port) + " is not available before start")
|
|
|
+ err.write(header("Error: Port %s is not available, cannot start %s" % (test.port, test.name)))
|
|
|
+ err.flush()
|
|
|
+ print "Error: Unable to recover port, cannot start test"
|
|
|
+ return exit_with_code(1)
|
|
|
|
|
|
result = test.start(out, err)
|
|
|
if result != 0:
|
|
@@ -655,7 +662,7 @@ class Benchmarker:
|
|
|
s.bind(("", port))
|
|
|
# If we get here, we were able to bind successfully,
|
|
|
# which means the port is free.
|
|
|
- except:
|
|
|
+ except Exception:
|
|
|
# If we get an exception, it might be because the port is still bound
|
|
|
# which would be bad, or maybe it is a privileged port (<1024) and we
|
|
|
# are not running as root, or maybe the server is gone, but sockets are
|
|
@@ -666,7 +673,7 @@ class Benchmarker:
|
|
|
# If we get here, we were able to connect to something, which means
|
|
|
# that the port is still bound.
|
|
|
return True
|
|
|
- except:
|
|
|
+ except Exception:
|
|
|
# An exception means that we couldn't connect, so a server probably
|
|
|
# isn't still running on the port.
|
|
|
pass
|
|
@@ -681,41 +688,67 @@ class Benchmarker:
|
|
|
|
|
|
def __forciblyEndPortBoundProcesses(self, test_port, out, err):
|
|
|
p = subprocess.Popen(['sudo', 'netstat', '-lnp'], stdout=subprocess.PIPE)
|
|
|
- out, err = p.communicate()
|
|
|
- for line in out.splitlines():
|
|
|
- if 'tcp' in line:
|
|
|
+ (ns_out, ns_err) = p.communicate()
|
|
|
+ for line in ns_out.splitlines():
|
|
|
+ # Handles tcp, tcp6, udp, udp6
|
|
|
+ if line.startswith('tcp') or line.startswith('udp'):
|
|
|
splitline = line.split()
|
|
|
- port = splitline[3].split(':')
|
|
|
- port = int(port[len(port) - 1].strip())
|
|
|
+ port = int(splitline[3].split(':')[-1])
|
|
|
+ pid = splitline[-1].split('/')[0]
|
|
|
+
|
|
|
+ # Sometimes the last column is just a dash
|
|
|
+ if pid == '-':
|
|
|
+ continue
|
|
|
+
|
|
|
if port > 6000:
|
|
|
+ ps = subprocess.Popen(['ps','p',pid], stdout=subprocess.PIPE)
|
|
|
+ (out_6000, err_6000) = ps.communicate()
|
|
|
err.write(textwrap.dedent(
|
|
|
- """
|
|
|
- A port that shouldn't be open is open. See the following line for netstat output.
|
|
|
- {splitline}
|
|
|
- """.format(splitline=splitline)))
|
|
|
+ """
|
|
|
+ Port {port} should not be open. See the following lines for information
|
|
|
+ {netstat}
|
|
|
+ {ps}
|
|
|
+ """.format(port=port, netstat=line, ps=out_6000)))
|
|
|
err.flush()
|
|
|
+
|
|
|
if port == test_port:
|
|
|
+ err.write( header("Error: Test port %s should not be open" % port, bottom='') )
|
|
|
try:
|
|
|
- pid = splitline[6].split('/')[0].strip()
|
|
|
ps = subprocess.Popen(['ps','p',pid], stdout=subprocess.PIPE)
|
|
|
# Store some info about this process
|
|
|
- proc = ps.communicate()
|
|
|
- os.kill(int(pid), 15)
|
|
|
+ (out_15, err_15) = ps.communicate()
|
|
|
+ children = subprocess.Popen(['ps','--ppid',pid,'-o','ppid'], stdout=subprocess.PIPE)
|
|
|
+ (out_children, err_children) = children.communicate()
|
|
|
+
|
|
|
+ err.write(" Sending SIGTERM to this process:\n %s\n" % out_15)
|
|
|
+ err.write(" Also expecting these child processes to die:\n %s\n" % out_children)
|
|
|
+
|
|
|
+ subprocess.check_output(['sudo','kill',pid])
|
|
|
# Sleep for 10 sec; kill can be finicky
|
|
|
time.sleep(10)
|
|
|
+
|
|
|
# Check that PID again
|
|
|
ps = subprocess.Popen(['ps','p',pid], stdout=subprocess.PIPE)
|
|
|
- dead = ps.communicate()
|
|
|
- if dead in proc:
|
|
|
- os.kill(int(pid), 9)
|
|
|
- except OSError:
|
|
|
- out.write( textwrap.dedent("""
|
|
|
- -----------------------------------------------------
|
|
|
- Error: Could not kill pid {pid}
|
|
|
- -----------------------------------------------------
|
|
|
- """.format(pid=str(pid))) )
|
|
|
- # This is okay; likely we killed a parent that ended
|
|
|
- # up automatically killing this before we could.
|
|
|
+ (out_9, err_9) = ps.communicate()
|
|
|
+ if len(out_9.splitlines()) != 1: # One line for the header row
|
|
|
+ err.write(" Process is still alive!\n")
|
|
|
+ err.write(" Sending SIGKILL to this process:\n %s\n" % out_9)
|
|
|
+ subprocess.check_output(['sudo','kill','-9', pid])
|
|
|
+ else:
|
|
|
+ err.write(" Process has been terminated\n")
|
|
|
+
|
|
|
+ # Ensure all children are dead
|
|
|
+ c_pids = [c_pid.strip() for c_pid in out_children.splitlines()[1:]]
|
|
|
+ for c_pid in c_pids:
|
|
|
+ ps = subprocess.Popen(['ps','p',c_pid], stdout=subprocess.PIPE)
|
|
|
+ (out_9, err_9) = ps.communicate()
|
|
|
+ if len(out_9.splitlines()) != 1: # One line for the header row
|
|
|
+ err.write(" Child Process %s is still alive, sending SIGKILL\n" % c_pid)
|
|
|
+ subprocess.check_output(['sudo','kill','-9', pid])
|
|
|
+ except Exception as e:
|
|
|
+ err.write( " Error: Unknown exception %s\n" % e )
|
|
|
+ err.write( header("Done attempting to recover port %s" % port, top='') )
|
|
|
+
|
|
|
|
|
|
############################################################
|
|
|
# __parse_results
|
|
@@ -904,6 +937,10 @@ class Benchmarker:
|
|
|
else:
|
|
|
args['types'] = { args['type'] : types[args['type']] }
|
|
|
del args['type']
|
|
|
+
|
|
|
+
|
|
|
+ args['max_threads'] = args['threads']
|
|
|
+ args['max_concurrency'] = max(args['concurrency_levels'])
|
|
|
|
|
|
self.__dict__.update(args)
|
|
|
# pprint(self.__dict__)
|
|
@@ -926,7 +963,7 @@ class Benchmarker:
|
|
|
self.result_directory = os.path.join("results", self.name)
|
|
|
self.latest_results_directory = self.latest_results_directory()
|
|
|
|
|
|
- if self.parse != None:
|
|
|
+ if hasattr(self, 'parse') and self.parse != None:
|
|
|
self.timestamp = self.parse
|
|
|
else:
|
|
|
self.timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
|