Increase various limits to eliminate errors

Set most of these config values to 65535 which is pretty big but should be
enough for the high-concurrency plaintext test.

* nginx needs more worker_connections.
* nginx should call listen() with a bigger backlog value.
* uWSGI should call listen() with a bigger backlog value.
* Increase somaxconn (which is a max for the listen() calls).
* tcp_max_syn_backlog seems to act as a max for somaxconn (according to
http://blog.dubbelboer.com/2012/04/09/syn-cookies.html), so increase it.
* Increase max file descriptor limit. For servers this will allow more
concurrent connections and for the wrk client, this will allow 16384
TCP connections plus whatever other file descriptors wrk needs to operate
(such as stdin, stdout, stderr and more).

config/nginx_uwsgi.conf

@@ -4,8 +4,9 @@
 worker_processes auto;
 
 events {
-    # This may need to be increased for the high-concurrency plaintext test.
-    worker_connections  1024;
+    # This needed to be increased because the nginx error log said so.
+    # http://nginx.org/en/docs/ngx_core_module.html#worker_connections
+    worker_connections  65535;
 }
 
 http {
@@ -34,7 +35,8 @@ http {
         # For information on deferred, see:
         # http://nginx.org/en/docs/http/ngx_http_core_module.html#listen
         # http://www.techrepublic.com/article/take-advantage-of-tcp-ip-options-to-optimize-data-transmission/
-        listen       8080 default_server deferred;
+        # The backlog argument to listen() is set to match net.ipv4.tcp_max_syn_backlog and net.core.somaxconn
+        listen       8080 default_server deferred backlog=65535;
         server_name  localhost;
 
         location / {

config/uwsgi.ini

@@ -1,7 +1,8 @@
 [uwsgi]
 master
-; increase listen queue used for nginx connecting to uWSGI
-listen = 5000
+; Increase listen queue used for nginx connecting to uWSGI. This matches
+; net.ipv4.tcp_max_syn_backlog and net.core.somaxconn.
+listen = 65535
 ; for performance
 disable-logging
 ; use UNIX sockets instead of TCP loopback for performance

toolset/benchmark/benchmarker.py

@@ -333,8 +333,9 @@ class Benchmarker:
       if os.name == 'nt':
         return True
       subprocess.check_call(["sudo","bash","-c","cd /sys/devices/system/cpu; ls -d cpu*|while read x; do echo performance > $x/cpufreq/scaling_governor; done"])
-      subprocess.check_call("sudo sysctl -w net.core.somaxconn=5000".rsplit(" "))
-      subprocess.check_call("sudo -s ulimit -n 16384".rsplit(" "))
+      subprocess.check_call("sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535".rsplit(" "))
+      subprocess.check_call("sudo sysctl -w net.core.somaxconn=65535".rsplit(" "))
+      subprocess.check_call("sudo -s ulimit -n 65535".rsplit(" "))
       subprocess.check_call("sudo sysctl net.ipv4.tcp_tw_reuse=1".rsplit(" "))
       subprocess.check_call("sudo sysctl net.ipv4.tcp_tw_recycle=1".rsplit(" "))
       subprocess.check_call("sudo sysctl -w kernel.shmmax=134217728".rsplit(" "))
@@ -354,8 +355,9 @@ class Benchmarker:
   def __setup_database(self):
     p = subprocess.Popen(self.database_ssh_string, stdin=subprocess.PIPE, shell=True)
     p.communicate("""
-      sudo sysctl -w net.core.somaxconn=5000
-      sudo -s ulimit -n 16384
+      sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535
+      sudo sysctl -w net.core.somaxconn=65535
+      sudo -s ulimit -n 65535
       sudo sysctl net.ipv4.tcp_tw_reuse=1
       sudo sysctl net.ipv4.tcp_tw_recycle=1
       sudo sysctl -w kernel.shmmax=2147483648
@@ -374,8 +376,9 @@ class Benchmarker:
   def __setup_client(self):
     p = subprocess.Popen(self.client_ssh_string, stdin=subprocess.PIPE, shell=True)
     p.communicate("""
-      sudo sysctl -w net.core.somaxconn=5000
-      sudo -s ulimit -n 16384
+      sudo sysctl -w net.ipv4.tcp_max_syn_backlog=65535
+      sudo sysctl -w net.core.somaxconn=65535
+      sudo -s ulimit -n 65535
       sudo sysctl net.ipv4.tcp_tw_reuse=1
       sudo sysctl net.ipv4.tcp_tw_recycle=1
       sudo sysctl -w kernel.shmmax=2147483648

toolset/setup/linux/installer.py

@@ -42,7 +42,7 @@ class Installer:
     self.__run_command("cat ../config/benchmark_profile >> ../../.profile")
     self.__run_command("cat ../config/benchmark_profile >> ../../.bashrc")
     self.__run_command("source ../../.profile")
-    self.__run_command("sudo sh -c \"echo '*               -    nofile          16384' >> /etc/security/limits.conf\"")
+    self.__run_command("sudo sh -c \"echo '*               -    nofile          65535' >> /etc/security/limits.conf\"")
 
     ##############################################################
     # System Tools
@@ -401,7 +401,7 @@ class Installer:
     ##############################
     yes | sudo apt-get update
     yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev postgresql
-    sudo sh -c "echo '*               -    nofile          16384' >> /etc/security/limits.conf"
+    sudo sh -c "echo '*               -    nofile          65535' >> /etc/security/limits.conf"
 
     sudo mkdir -p /ssd
     sudo mkdir -p /ssd/log
@@ -485,7 +485,7 @@ class Installer:
     ##############################
     yes | sudo apt-get update
     yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev postgresql
-    sudo sh -c "echo '*               -    nofile          16384' >> /etc/security/limits.conf"
+    sudo sh -c "echo '*               -    nofile          65535' >> /etc/security/limits.conf"
 
     sudo mkdir -p /ssd
     sudo mkdir -p /ssd/log