Merge branch 'master' into python-r10

Conflicts:
	toolset/setup/linux/installer.py

config/10gen.list

@@ -1 +1 @@
-deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen
+deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen

config/benchmark_profile

@@ -20,10 +20,10 @@ export PATH="$PYTHON_HOME/bin:$PATH$JAVA_HOME/bin:$GRAILS_HOME/bin:$PLAY_HOME:$P
 
 export LD_LIBRARY_PATH='$LD_LIBRARY_PATH:/usr/local/apr/lib'
 
-export TFB_SERVER_HOST=172.16.98.122'
+export TFB_SERVER_HOST='172.16.98.122'
 export TFB_CLIENT_HOST='172.16.98.98'
 export TFB_CLIENT_USER='tfb'
 export TFB_CLIENT_IDENTITY_FILE='/home/tfb/.ssh/id_rsa-tfb-1'
 export TFB_DATABASE_HOST='172.16.98.118'
 
-source ~/.rvm/scripts/'rvm'
+. ~/.rvm/scripts/'rvm'

config/database_sftp_batch

@@ -1,3 +1,4 @@
+lcd ..
 put config/my.cnf
 put config/mongodb.conf
 put config/create.sql
@@ -9,3 +10,5 @@ put config/postgresql.conf
 put config/pg_hba.conf
 put config/usr.sbin.mysqld
 put config/60-postgresql-shm.conf
+put config/mysql
+put config/mysql.conf

config/erlang.list

@@ -1,8 +1 @@
-deb http://binaries.erlang-solutions.com/debian quantal contrib
-deb http://binaries.erlang-solutions.com/debian precise contrib
-deb http://binaries.erlang-solutions.com/debian oneiric contrib
-deb http://binaries.erlang-solutions.com/debian natty contrib
-deb http://binaries.erlang-solutions.com/debian maverick contrib
-deb http://binaries.erlang-solutions.com/debian lucid contrib
-
-deb http://binaries.erlang-solutions.com/debian squeeze contrib
+deb http://packages.erlang-solutions.com/debian trusty contrib

config/mysql

@@ -0,0 +1,188 @@
+#!/bin/bash
+#
+### BEGIN INIT INFO
+# Provides:          mysql
+# Required-Start:    $remote_fs $syslog
+# Required-Stop:     $remote_fs $syslog
+# Should-Start:      $network $time
+# Should-Stop:       $network $time
+# Default-Start:     2 3 4 5
+# Default-Stop:      0 1 6
+# Short-Description: Start and stop the mysql database server daemon
+# Description:       Controls the main MySQL database server daemon "mysqld"
+#                    and its wrapper script "mysqld_safe".
+### END INIT INFO
+#
+set -e
+set -u
+${DEBIAN_SCRIPT_DEBUG:+ set -v -x}
+
+test -x /usr/bin/mysqld_safe || exit 0
+
+. /lib/lsb/init-functions
+
+SELF=$(cd $(dirname $0); pwd -P)/$(basename $0)
+CONF=/etc/mysql/my.cnf
+MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
+
+# priority can be overriden and "-s" adds output to stderr
+ERR_LOGGER="logger -p daemon.err -t /etc/init.d/mysql -i"
+
+# Safeguard (relative paths, core dumps..)
+cd /
+umask 077
+
+# mysqladmin likes to read /root/.my.cnf. This is usually not what I want
+# as many admins e.g. only store a password without a username there and
+# so break my scripts.
+export HOME=/etc/mysql/
+
+## Fetch a particular option from mysql's invocation.
+#
+# Usage: void mysqld_get_param option
+mysqld_get_param() {
+	/usr/sbin/mysqld --print-defaults \
+		| tr " " "\n" \
+		| grep -- "--$1" \
+		| tail -n 1 \
+		| cut -d= -f2
+}
+
+## Do some sanity checks before even trying to start mysqld.
+sanity_checks() {
+  # check for config file
+  if [ ! -r /etc/mysql/my.cnf ]; then
+    log_warning_msg "$0: WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz"
+    echo                "WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz" | $ERR_LOGGER
+  fi
+
+  # check for diskspace shortage
+#  datadir=`mysqld_get_param datadir`
+#  if LC_ALL=C BLOCKSIZE= df --portability $datadir/. | tail -n 1 | awk '{ exit ($4>4096) }'; then
+#    log_failure_msg "$0: ERROR: The partition with $datadir is too full!"
+#    echo                "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
+#    exit 1
+#  fi
+}
+
+## Checks if there is a server running and if so if it is accessible.
+#
+# check_alive insists on a pingable server
+# check_dead also fails if there is a lost mysqld in the process list
+#
+# Usage: boolean mysqld_status [check_alive|check_dead] [warn|nowarn]
+mysqld_status () {
+    ping_output=`$MYADMIN ping 2>&1`; ping_alive=$(( ! $? ))
+
+    ps_alive=0
+    pidfile=`mysqld_get_param pid-file`
+    if [ -f "$pidfile" ] && ps `cat $pidfile` >/dev/null 2>&1; then ps_alive=1; fi
+    
+    if [ "$1" = "check_alive"  -a  $ping_alive = 1 ] ||
+       [ "$1" = "check_dead"   -a  $ping_alive = 0  -a  $ps_alive = 0 ]; then
+	return 0 # EXIT_SUCCESS
+    else
+  	if [ "$2" = "warn" ]; then
+  	    echo -e "$ps_alive processes alive and '$MYADMIN ping' resulted in\n$ping_output\n" | $ERR_LOGGER -p daemon.debug
+	fi
+  	return 1 # EXIT_FAILURE
+    fi
+}
+
+#
+# main()
+#
+
+case "${1:-''}" in
+  'start')
+	sanity_checks;
+	# Start daemon
+	log_daemon_msg "Starting MySQL database server" "mysqld"
+	if mysqld_status check_alive nowarn; then
+	   log_progress_msg "already running"
+	   log_end_msg 0
+	else
+	    # Could be removed during boot
+	    test -e /var/run/mysqld || install -m 755 -o mysql -g root -d /var/run/mysqld
+
+	    # Start MySQL! 
+  	    su - mysql -s /bin/sh -c "/usr/bin/mysqld_safe > /dev/null 2>&1 &"
+
+	    # 6s was reported in #352070 to be too few when using ndbcluster
+	    # 14s was reported in #736452 to be too few with large installs
+	    for i in $(seq 1 30); do
+                sleep 1
+	        if mysqld_status check_alive nowarn ; then break; fi
+		log_progress_msg "."
+	    done
+	    if mysqld_status check_alive warn; then
+                log_end_msg 0
+	        # Now start mysqlcheck or whatever the admin wants.
+	        output=$(/etc/mysql/debian-start)
+		[ -n "$output" ] && log_action_msg "$output"
+	    else
+	        log_end_msg 1
+		log_failure_msg "Please take a look at the syslog"
+	    fi
+	fi
+	;;
+
+  'stop')
+	# * As a passwordless mysqladmin (e.g. via ~/.my.cnf) must be possible
+	# at least for cron, we can rely on it here, too. (although we have 
+	# to specify it explicit as e.g. sudo environments points to the normal
+	# users home and not /root)
+	log_daemon_msg "Stopping MySQL database server" "mysqld"
+	if ! mysqld_status check_dead nowarn; then
+	  set +e
+	  shutdown_out=`$MYADMIN shutdown 2>&1`; r=$?
+	  set -e
+	  if [ "$r" -ne 0 ]; then
+	    log_end_msg 1
+	    [ "$VERBOSE" != "no" ] && log_failure_msg "Error: $shutdown_out"
+	    log_daemon_msg "Killing MySQL database server by signal" "mysqld"
+	    killall -15 mysqld
+            server_down=
+	    for i in 1 2 3 4 5 6 7 8 9 10; do
+              sleep 1
+              if mysqld_status check_dead nowarn; then server_down=1; break; fi
+            done
+          if test -z "$server_down"; then killall -9 mysqld; fi
+	  fi
+        fi
+
+        if ! mysqld_status check_dead warn; then
+	  log_end_msg 1
+	  log_failure_msg "Please stop MySQL manually and read /usr/share/doc/mysql-server-5.6/README.Debian.gz!"
+	  exit -1
+	else
+	  log_end_msg 0
+        fi
+	;;
+
+  'restart')
+	set +e; $SELF stop; set -e
+	$SELF start 
+	;;
+
+  'reload'|'force-reload')
+  	log_daemon_msg "Reloading MySQL database server" "mysqld"
+	$MYADMIN reload
+	log_end_msg 0
+	;;
+
+  'status')
+	if mysqld_status check_alive nowarn; then
+	  log_action_msg "$($MYADMIN version)"
+	else
+	  log_action_msg "MySQL is stopped."
+	  exit 3
+	fi
+  	;;
+
+  *)
+	echo "Usage: $SELF start|stop|restart|reload|force-reload|status"
+	exit 1
+	;;
+esac
+

config/mysql.conf

@@ -0,0 +1,63 @@
+description     "MySQL 5.6 Server"
+author          "Mario Limonciello <[email protected]>"
+
+start on runlevel [2345]
+stop on starting rc RUNLEVEL=[016]
+
+respawn
+respawn limit 2 5
+
+env HOME=/etc/mysql
+umask 007
+
+# The default of 5 seconds is too low for mysql which needs to flush buffers
+kill timeout 300
+
+pre-start script
+    ## Fetch a particular option from mysql's invocation.
+    # Usage: void mysqld_get_param option
+    mysqld_get_param() {
+      /usr/sbin/mysqld --print-defaults \
+        | tr " " "\n" \
+        | grep -- "--$1" \
+        | tail -n 1 \
+        | cut -d= -f2
+    }
+
+    # priority can be overriden and "-s" adds output to stderr
+    ERR_LOGGER="logger -p daemon.err -t /etc/init/mysql.conf -i"
+
+    #Sanity checks
+    [ -r $HOME/my.cnf ]
+    [ -d /var/run/mysqld ] || install -m 755 -o mysql -g root -d /var/run/mysqld
+    /lib/init/apparmor-profile-load usr.sbin.mysqld
+
+    # check for diskspace shortage
+#    datadir=`mysqld_get_param datadir`
+#    BLOCKSIZE=`LC_ALL=C df --portability $datadir/. | tail -n 1 | awk '{print $4}'`
+#    if [ $BLOCKSIZE -le 4096 ] ; then
+#      echo "$0: ERROR: The partition with $datadir is too full!" >&2
+#      echo "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
+#      exit 1
+#    fi
+end script
+
+exec /usr/sbin/mysqld
+
+post-start script
+   for i in `seq 1 30` ; do
+        /usr/bin/mysqladmin --defaults-file="${HOME}"/debian.cnf ping && {
+            exec "${HOME}"/debian-start
+            # should not reach this line
+            exit 2
+        }
+        statusnow=`status`
+        if echo $statusnow | grep -q 'stop/' ; then
+            exit 0
+        elif echo $statusnow | grep -q 'respawn/' ; then
+            exit 1
+        fi
+        sleep 1
+    done
+    exit 1
+end script

toolset/benchmark/framework_test.py

@@ -46,10 +46,10 @@ class FrameworkTest:
       echo ""
       echo "---------------------------------------------------------"
       echo " Concurrency: $c for {name}"
-      echo " {wrk} {headers} {pipeline} -d {duration} -c $c -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\""
+      echo " {wrk} {headers} -d {duration} -c $c -t $(($c>{max_threads}?{max_threads}:$c)) \"http://{server_host}:{port}{url}\" -s ~/pipeline.lua -- {pipeline}"
       echo "---------------------------------------------------------"
       echo ""
-      {wrk} {headers} {pipeline} -d {duration} -c "$c" -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url}
+      {wrk} {headers} -d {duration} -c "$c" -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url} -s ~/pipeline.lua -- {pipeline}
       sleep 2
     done
   """
@@ -640,7 +640,7 @@ class FrameworkTest:
             # Simply opening the file in write mode should create the empty file.
             pass
         if self.plaintext_url_passed:
-          remote_script = self.__generate_concurrency_script(self.plaintext_url, self.port, self.accept_plaintext, wrk_command="wrk-pipeline", intervals=[256,1024,4096,16384], pipeline="--pipeline 16")
+          remote_script = self.__generate_concurrency_script(self.plaintext_url, self.port, self.accept_plaintext, wrk_command="wrk", intervals=[256,1024,4096,16384], pipeline="16")
           self.__run_benchmark(remote_script, output_file, err)
         results = self.__parse_test(self.PLAINTEXT)
         self.benchmarker.report_results(framework=self, test=self.PLAINTEXT, results=results['results'])

toolset/setup/README.md

@@ -23,52 +23,109 @@ Refer to [Benchmark Suite Deployment README file](../deployment/README.md) for t
 
 ## Linux server and client setup
 
-To coordinate the tests via scripting, the servers need to be able to work together. So once the instances are running, the first thing you'll want to do is copy your ssh key to the application server instance so that you can ssh between the two machines:
-
-	sftp -i path-to-ssh-key ubuntu@server-instance-ip
-	put path-to-ssh-key .ssh/
-	exit
-
-This will copy your private ssh key file to ~/.ssh on the Linux server, where it will be used to connect to the benchmark client.
-
-Now, connect via ssh to the Linux server:
-
-    ssh -i path-to-ssh-key ubuntu@server-instance-ip
-
-On the Linux server, check out the latest version from GitHub:
-
-	cd ~
-    sudo apt-get install git -qq
-	git clone https://github.com/TechEmpower/FrameworkBenchmarks.git
-	cd FrameworkBenchmarks
-
-As it currently stands, the script that runs the tests makes some assumptions about where the code is placed. We assume that the FrameworkBenchmarks repository will be located in your home directory.
-
-Next, we're going to setup the Linux server and the client with the required software:
-
-	toolset/run-tests.py -s server-private-ip -c client-private-ip -i path-to-ssh-key --install-software --list-tests &> install.log
-    
-This step can take a couple hours. If you like, you can open another shell and watch the progress:
-
-    tail -f ~/FrameworkBenchmarks/install.log | grep '^INSTALL'
-
-After the installation is done, reboot both the Linux server and the Linux client.
-
-When they are online again, connect via ssh to the Linux server and type these commands for additional setup:
-
-    source ~/.bash_profile
-    # For your first time through the tests, set the ulimit for open files
-    ulimit -n 8192
-    # Most software is installed automatically by the script, but running the mongo command below from
-    # the install script was causing some errors. For now this needs to be run manually.
-    cd installs && curl -sS https://getcomposer.org/installer | php -- --install-dir=bin
-    cd ..
-    sudo apt-get remove --purge openjdk-6-jre openjdk-6-jre-headless
-    mongo --host client-private-ip < config/create.js
+**NOTE:** If testing a pull request or doing development, it is usually adequate to only use one computer. In that case, your server, client, and database IPs will be 127.0.0.1
+
+### Installing the Framework Benchmark App Server
+
+* Install [Ubuntu 14.04](http://www.ubuntu.com/download/server) with username `tfb`. Ensure that OpenSSH is selected when you install. If not, run the following command
+```bash
+$ sudo apt-get install openssh-server
+```
+* If Ubuntu is already installed, run the following command and follow the prompts.
+```bash
+$ sudo adduser tfb
+```
+* Log in as `tfb`
+* Fully update **NOTE**: If you update the kernel (linux-firmware), it is generally a good idea to reboot aftewards.
+```bash
+$ sudo apt-get update && sudo apt-get upgrade
+```
+* Run the command: `sudo visudo`
+* Change line 20 in from `%sudo   ALL=(ALL:ALL) ALL` to `%sudo   ALL=NOPASSWD: ALL`
+* Run the following **(Don't enter a password, just hit enter when the prompt pops up)**. **NOTE** This is still necessary if the client and database are on the same computer as the server
+```bash
+$ ssh-keygen
+$ ssh-copy-id <database ip>
+$ ssh-copy-id <client ip>
+```
+* Install git and clone the Framework Benchmarks repository
+```bash
+$ sudo apt-get install git
+$ cd ~
+$ git clone https://github.com/TechEmpower/FrameworkBenchmarks.git
+$ cd FrameworkBenchmarks
+```
+* Install the server software. This will take a long time
+```bash
+$ nohup python toolset/run-tests.py -s <server hostname/ip> -c <client hostname/ip> -u tfb --install-software --install server --list-tests &
+```
+* If you want to view the process of installing, do the following. The session can be interrupted easily so no need to worry about keeping a connection.
+```bash
+$ tail -f nohup.out
+```
+* Reboot when the install is done
+* Edit your ~/.bashrc file to change the following
+ * Change `TFB_SERVER_HOST=<ip address>` to the server's IP address
+ * Change `TFB_CLIENT_HOST=<ip address>` to the client's ip address
+ * Change `TFB_DATABASE_HOST=<ip address>` to the database's ip address.
+ * Change `TFB_CLIENT_IDENTITY_FILE=<path>` to the id file you specified when you ran ssh-keygen (probably /home/tfb/.ssh/id_rsa if you don't know what it is)
+ * Run the command `source ~/.bashrc`
+* If you are setting up any other servers, do so before proceeding.
+* Run the following commands
+```bash
+cd ~/FrameworkBenchmarks
+source ~/.bash_profile
+# For your first time through the tests, set the ulimit for open files
+ulimit -n 8192
+# Most software is installed automatically by the script, but running the mongo command below from
+# the install script was causing some errors. For now this needs to be run manually.
+cd installs && curl -sS https://getcomposer.org/installer | php -- --install-dir=bin
+cd ..
+sudo apt-get remove --purge openjdk-6-jre openjdk-6-jre-headless
+mongo --host database-private-ip < config/create.js
+```
+* Before running the tests, do the following
+```bash
+$ source ~/.bashrc
+```
+
+---
+
+### Installing the Framework Benchmark Database Server
+
+* Install [Ubuntu 14.04](http://www.ubuntu.com/download/server) with username `tfb`
+* Log in as `tfb`
+* Fully update **NOTE**: If you update the kernel (linux-firmware), it is generally a good idea to reboot aftewards.
+```bash
+$ sudo apt-get update && sudo apt-get upgrade
+```
+* Run the command: `sudo visudo`
+* Change line 20 in from `%sudo   ALL=(ALL:ALL) ALL` to `%sudo   ALL=NOPASSWD: ALL`
+* On the app server, run the following from the FrameworkBenchmark directory (this should only take a small amount of time, several minutes or so):
+```bash
+$ toolset/run-tests.py --install-software --install database --list-tests
+```
+
+---
+
+### Installing the Framework Benchmark Load Server
+
+* Install [Ubuntu 14.04](http://www.ubuntu.com/download/server) with username `tfb`
+* Log in as `tfb`
+* Fully update **NOTE**: If you update the kernel (linux-firmware), it is generally a good idea to reboot aftewards.
+```bash
+$ sudo apt-get update && sudo apt-get upgrade
+```
+* Run the command: `sudo visudo`
+* Change line 20 in from `%sudo   ALL=(ALL:ALL) ALL` to `%sudo   ALL=NOPASSWD: ALL`
+* On the app server, run the following from the FrameworkBenchmark directory (this should only take a small amount of time, several minutes or so):
+```bash
+$ toolset/run-tests.py --install-software --install client --list-tests
+```
 
 You can validate that the setup worked by running a smoke test like this:
 
-    toolset/run-tests.py -s server-private-ip -c client-private-ip -i path-to-ssh-key --max-threads 1 --name smoketest --test servlet-raw --type all -m verify
+    toolset/run-tests.py --max-threads 1 --name smoketest --test servlet-raw --type all -m verify
 
 This should run the verification step for a single framework.
 

toolset/setup/linux/installer.py

@@ -27,7 +27,6 @@ class Installer:
   ############################################################
   def __install_server_software(self):
     print("\nINSTALL: Installing server software\n")
-
     #######################################
     # Prerequisites
     #######################################
@@ -41,7 +40,7 @@ class Installer:
     self.__run_command("cp ../config/benchmark_profile ../../.bash_profile")
     self.__run_command("cat ../config/benchmark_profile >> ../../.profile")
     self.__run_command("cat ../config/benchmark_profile >> ../../.bashrc")
-    self.__run_command("source ../../.profile")
+    self.__run_command(". ../../.profile")
     self.__run_command("sudo sh -c \"echo '*               -    nofile          65535' >> /etc/security/limits.conf\"")
 
     ##############################################################
@@ -65,14 +64,13 @@ class Installer:
     #######################################
     # Languages
     #######################################
-
     self._install_python()
 
     #
     # Dart
     #
     self.__download("http://storage.googleapis.com/dart-archive/channels/stable/release/latest/sdk/dartsdk-linux-x64-release.zip")
-    self.__run_command("unzip dartsdk-linux-x64-release.zip")
+    self.__run_command("unzip -qqo dartsdk-linux-x64-release.zip")
 
     #
     # Erlang
@@ -129,7 +127,7 @@ class Installer:
     self.__run_command("./configure --with-pdo-mysql --with-mysql --with-mcrypt --enable-intl --enable-mbstring --enable-fpm --with-fpm-user=www-data --with-fpm-group=www-data --with-openssl", cwd="php-5.4.13")
     self.__run_command("make", cwd="php-5.4.13")
     self.__run_command("sudo make install", cwd="php-5.4.13")
-    self.__run_command("printf \"\\n\" | sudo pecl install apc-beta", cwd="php-5.4.13", retry=True)
+    self.__run_command("printf \"\\n\" | sudo pecl install -f apc-beta", cwd="php-5.4.13", retry=True)
     self.__run_command("sudo cp ../config/php.ini /usr/local/lib/php.ini")
     self.__run_command("sudo cp ../config/php-fpm.conf /usr/local/lib/php-fpm.conf")
 
@@ -142,7 +140,7 @@ class Installer:
     self.__run_command("sudo ./install", cwd="cphalcon/build")
 
     # YAF
-    self.__run_command("sudo pecl install yaf")
+    self.__run_command("sudo pecl install -f yaf")
 
     #
     # Haskell
@@ -178,8 +176,9 @@ class Installer:
     # Nimrod
     #
     self.__run_command("git clone git://github.com/Araq/Nimrod.git nimrod", retry=True)
-    self.__run_command("git reset --hard \"43d12ef7495238977e4f236f6d25bce5bd687967\"", cwd="nimrod")
-    self.__run_command("git clone --depth 1 git://github.com/nimrod-code/csources.git", cwd="nimrod", retry=True)
+    self.__run_command("git checkout 987ac2439a87d74838233a7b188e4db340495ee5", cwd="nimrod")
+    self.__run_command("git clone git://github.com/nimrod-code/csources.git", cwd="nimrod", retry=True)
+    self.__run_command("git checkout 704015887981932c78a033dd5ede623b2ad6ae27", cwd="nimrod/csources")
     self.__run_command("chmod +x build.sh", cwd="nimrod/csources")
     self.__run_command("./build.sh", cwd="nimrod/csources")
     self.__run_command("bin/nimrod c koch", cwd="nimrod")
@@ -207,9 +206,9 @@ class Installer:
     # HHVM
     #
     self.__run_command("wget -O - http://dl.hhvm.com/conf/hhvm.gpg.key | sudo apt-key add -")
-    self.__run_command("echo deb http://dl.hhvm.com/ubuntu precise main | sudo tee /etc/apt/sources.list.d/hhvm.list")
+    self.__run_command("echo deb http://dl.hhvm.com/ubuntu trusty main | sudo tee /etc/apt/sources.list.d/hhvm.list")
     self.__run_command("sudo apt-get update")
-    self.__run_command("sudo apt-get install hhvm")
+    self.__run_command("sudo apt-get install hhvm", True)
 
     #######################################
     # Webservers
@@ -236,7 +235,7 @@ class Installer:
     #
     # Lapis
     #
-    self.__run_command("sudo apt-get install luarocks")
+    self.__run_command("sudo apt-get install luarocks", True)
     self.__run_command("sudo luarocks install http://github.com/leafo/lapis/raw/master/lapis-dev-1.rockspec")
 
 
@@ -263,7 +262,7 @@ class Installer:
     self.__run_command("./configure", cwd="zeromq-4.0.3")
     self.__run_command("make", cwd="zeromq-4.0.3")
     self.__run_command("sudo make install", cwd="zeromq-4.0.3")
-    self.__run_command("sudo apt-get install sqlite3 libsqlite3-dev uuid uuid-runtime uuid-dev")
+    self.__run_command("sudo apt-get install sqlite3 libsqlite3-dev uuid uuid-runtime uuid-dev", True)
     self.__run_command("sudo ldconfig -v")
     self.__download("https://github.com/zedshaw/mongrel2/tarball/v1.8.1", "mongrel2.tar.gz")
     self.__run_command("tar xvf mongrel2.tar.gz")
@@ -351,14 +350,14 @@ class Installer:
     # PyPy 2.3.1
     f = "pypy-2.3.1-linux64.tar.bz2"
     if not os.path.exists(f):
-      self.__download("https://bitbucket.org/pypy/pypy/downloads/" + f)
+      self.__download("https://bitbucket.org/pypy/pypy/downloads/" + f, f)
     self.__run_command("tar xjf " + f)
     self.__run_command('ln -sf pypy-2.3.1-linux64 pypy')
 
     # CPython 2.7.7
     f = "Python-2.7.7.tgz"
     if not os.path.exists(f):
-      self.__download("http://www.python.org/ftp/python/2.7.7/" + f)
+      self.__download("http://www.python.org/ftp/python/2.7.7/" + f, f)
     self.__run_command("tar xf " + f)
     self.__run_command("./configure --prefix=$HOME/FrameworkBenchmarks/installs/py2 --disable-shared", cwd="Python-2.7.7")
     self.__run_command("make -j4", cwd="Python-2.7.7")
@@ -367,23 +366,22 @@ class Installer:
     # CPython 3.4.1
     f = "Python-3.4.1.tar.xz"
     if not os.path.exists(f):
-      self.__download("https://www.python.org/ftp/python/3.4.1/" + f)
+      self.__download("https://www.python.org/ftp/python/3.4.1/" + f, f)
     self.__run_command("tar xf " + f)
     self.__run_command("./configure --prefix=$HOME/FrameworkBenchmarks/installs/py3 --disable-shared", cwd="Python-3.4.1")
     self.__run_command("make -j4", cwd="Python-3.4.1")
     self.__run_command("make install", cwd="Python-3.4.1")
 
     if not os.path.exists("get-pip.py"):
-      self.__download("https://bootstrap.pypa.io/get-pip.py")
+      self.__download("https://bootstrap.pypa.io/get-pip.py", "get-pip.py")
     self.__run_command("py2/bin/python get-pip.py")
-    self.__run_command("py3/bin/python3 get-pip.py")
     self.__run_command("pypy/bin/pypy get-pip.py")
+    # Python 3.4.1 installs pip by default.
 
     self.__run_command('py2/bin/pip install -r ../config/requirements.txt')
     self.__run_command('py3/bin/pip install -r ../config/requirements.txt')
     self.__run_command('pypy/bin/pip install -r ../config/requirements-pypy.txt')
 
-
   ############################################################
   # __install_error
   ############################################################
@@ -400,8 +398,8 @@ class Installer:
   ############################################################
   def __install_database_software(self):
     print("\nINSTALL: Installing database software\n")
-
-    self.__run_command("cd .. && " + self.benchmarker.database_sftp_string(batch_file="config/database_sftp_batch"), True)
+ 
+    self.__run_command("cd .. && " + self.benchmarker.database_sftp_string(batch_file="../config/database_sftp_batch"), True)
 
     remote_script = """
 
@@ -421,9 +419,13 @@ class Installer:
     sudo sh -c "echo mysql-server mysql-server/root_password_again select secret | debconf-set-selections"
     sudo sh -c "echo mysql-server mysql-server/root_password select secret | debconf-set-selections"
 
-    yes | sudo apt-get install mysql-server
+    yes | sudo apt-get install mysql-server-5.6
 
     sudo stop mysql
+    # disable checking of disk size
+    sudo cp mysql /etc/init.d/mysql
+    sudo chmod +x /etc/init.d/mysql
+    sudo cp mysql.conf /etc/init/mysql.conf
     # use the my.cnf file to overwrite /etc/mysql/my.cnf
     sudo mv /etc/mysql/my.cnf /etc/mysql/my.cnf.orig
     sudo mv my.cnf /etc/mysql/my.cnf
@@ -445,10 +447,10 @@ class Installer:
     sudo -u benchmarkdbuser psql hello_world < create-postgres.sql
 
     sudo -u postgres -H /etc/init.d/postgresql stop
-    sudo mv postgresql.conf /etc/postgresql/9.1/main/postgresql.conf
-    sudo mv pg_hba.conf /etc/postgresql/9.1/main/pg_hba.conf
+    sudo mv postgresql.conf /etc/postgresql/9.3/main/postgresql.conf
+    sudo mv pg_hba.conf /etc/postgresql/9.3/main/pg_hba.conf
 
-    sudo cp -R -p /var/lib/postgresql/9.1/main /ssd/postgresql
+    sudo cp -R -p /var/lib/postgresql/9.3/main /ssd/postgresql
     sudo -u postgres -H /etc/init.d/postgresql start
     sudo mv 60-postgresql-shm.conf /etc/sysctl.d/60-postgresql-shm.conf
 
@@ -493,11 +495,9 @@ class Installer:
     # Prerequisites
     ##############################
     yes | sudo apt-get update
-    yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev postgresql
+    yes | sudo apt-get install build-essential git libev-dev libpq-dev libreadline6-dev 
     sudo sh -c "echo '*               -    nofile          65535' >> /etc/security/limits.conf"
 
-    sudo mkdir -p /ssd
-    sudo mkdir -p /ssd/log
 
     ##############################
     # wrk
@@ -508,13 +508,25 @@ class Installer:
     make
     sudo cp wrk /usr/local/bin
     cd ~
-
-    git clone https://github.com/wg/wrk.git wrk-pipeline
-    cd wrk-pipeline
-    git checkout pipeline
-    make
-    sudo cp wrk /usr/local/bin/wrk-pipeline
-    cd ~
+    
+    #############################
+    # pipeline.lua
+    #############################
+cat << EOF | tee pipeline.lua
+init = function(args)
+  wrk.init(args)
+  local r = {}
+  local depth = tonumber(args[1]) or 1
+  for i=1,depth do
+    r[i] = wrk.format()
+  end
+  req = table.concat(r)
+end
+
+request = function()
+  return req
+end
+EOF
     """
     
     print("\nINSTALL: %s" % self.benchmarker.client_ssh_string)
@@ -544,6 +556,9 @@ class Installer:
       max_attempts = 1
     attempt = 1
     delay = 0
+    if send_yes:
+      command = "yes yes | " + command
+        
 
     print("\nINSTALL: %s (cwd=%s)" % (command, cwd))
 
@@ -551,14 +566,7 @@ class Installer:
       error_message = ""
       try:
         # Execute command.
-        if send_yes:
-          process = subprocess.Popen(command, shell=True, stdin=subprocess.PIPE, cwd=cwd)
-          process.communicate("yes")
-          returncode = process.returncode
-          if returncode:
-            raise subprocess.CalledProcessError(returncode, command)
-        else:
-          subprocess.check_call(command, shell=True, cwd=cwd)
+        subprocess.check_call(command, shell=True, cwd=cwd)
         break  # Exit loop if successful.
       except:
         exceptionType, exceptionValue, exceptionTraceBack = sys.exc_info()