Merge branch 'PR501' into development

README.md

@@ -40,58 +40,9 @@ When adding a new framework or new test to an existing framework, please follow
 * Update/add [setup file](#setup-files)
 * When creating a database test, please use the MySQL table hello_world.World, or the MongoDB collection hello_world.world
 
-There are three different tests that we currently run:
+### The Tests
 
-* JSON Response
-* Database (single query)
-* Database (multiple query)
-
-The single query database test can be treated as a special case of the multiple query test with the query-count parameter set to 1.
-
-### JSON Response
-
-This test needs to follow the following conventions:
-
-* The message object should be instantiated as a new object for each request.
-* The test should use a JSON serializer to render the newly instantiated object to JSON.
-* Set the response Content-Type to application/json.
-* The response should be {"message": "Hello, World!"}
-* White space in the response does not matter.
-
-Pseudo-code:
-
-	obj = { message : "Hello, World!" }
-	render json.encode(obj)
-
-### Database (single query)
-
-This test will:
-
-* Access a database table or collection named "World" that is known to contain 10,000 rows/entries.
-* Query for a single row from the table or collection using a randomly generated id (the ids range from 1 to 10,000).
-* Set the response Content-Type to application/json.
-* Serialize the row to JSON and send the resulting string as the response.
-
-By convention, if the test does not use an ORM, and instead uses the raw database connectivity provided by the platform (e.g., JDBC), we append a "-raw" to the test name in the [benchmark_config](#the-benchmark_config-file) file.  For example, "php-raw".
-
-Pseudo-code:
-
-	random_id = random(1, 10000)
-	world = World.find(random_id)
-	render json.encode(world)
-
-### Database (multiple queries)
-
-This test is very similar to the single query test, and in some cases it will be implemented using the same code. A URL parameter is made available to specify the number of queries to run per request. The response is a list of objects resulting from the queries for random rows.
-
-Pseudo-code:
-
-	number_of_queries = get("queries")
-	worlds = []
-	for i = 0; i < number_of_queries; i++
-        random_id = random(1, 10000)
-        worlds[i] = World.find(random_id)
-	render json.encode(worlds)
+For descriptions of the test types that we run against each framework, see the [test requirements section of the Results web site](http://www.techempower.com/benchmarks/#section=code).
 
 ## The benchmark_config File
 

config/benchmark_profile

@@ -1,8 +1,7 @@
 export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
 export RESIN_HOME=~/FrameworkBenchmarks/installs/resin-4.0.36
 export GRAILS_HOME=~/FrameworkBenchmarks/installs/grails-2.1.1
-#export VERTX_HOME=~/FrameworkBenchmarks/installs/vert.x-1.3.1.final
-export VERTX_HOME=~/FrameworkBenchmarks/installs/vert.x-2.0.0-final
+export VERTX_HOME=~/FrameworkBenchmarks/installs/vert.x-2.0.2-final
 export GOROOT=~/FrameworkBenchmarks/installs/go
 export GOPATH=~/FrameworkBenchmarks/go:~/FrameworkBenchmarks/webgo:~/FrameworkBenchmarks/revel
 export TOMCAT_HOME=~/FrameworkBenchmarks/installs/apache-tomcat-7.0.35

results/vertx-pr501/20131008102005/json/vertx/raw

@@ -0,0 +1,120 @@
+Welcome to Ubuntu 12.04.3 LTS (GNU/Linux 3.9.0-030900-generic x86_64)
+
+ * Documentation:  https://help.ubuntu.com/
+
+  System information as of Tue Oct  8 10:21:56 PDT 2013
+
+  System load:  0.0                Processes:           116
+  Usage of /:   5.8% of 129.62GB   Users logged in:     0
+  Memory usage: 1%                 IP address for eth1: 172.16.98.98
+  Swap usage:   0%
+
+  Graph this data and manage this system at https://landscape.canonical.com/
+
+48 packages can be updated.
+15 updates are security updates.
+
+
+---------------------------------------------------------
+ Running Primer vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 60 -c 8 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 5s test @ http://tfbapp:8080/j
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   290.30us    1.11ms  17.97ms   99.55%
+    Req/Sec     4.68k   502.14     6.00k    90.40%
+  169601 requests in 5.00s, 18.28MB read
+Requests/sec:  33921.75
+Transfer/sec:      3.66MB
+
+---------------------------------------------------------
+ Running Warmup vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     1.13ms  428.16us  26.82ms   91.87%
+    Req/Sec    29.86k     2.51k   38.89k    84.33%
+  3365572 requests in 15.00s, 362.69MB read
+Requests/sec: 224381.71
+Transfer/sec:     24.18MB
+
+---------------------------------------------------------
+ Concurrency: 8 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 8 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   216.61us  105.26us   9.70ms   99.61%
+    Req/Sec     4.74k   325.95     6.00k    63.61%
+  539513 requests in 15.00s, 58.14MB read
+Requests/sec:  35968.39
+Transfer/sec:      3.88MB
+
+---------------------------------------------------------
+ Concurrency: 16 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 16 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 16 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   204.59us   76.04us   3.66ms   85.13%
+    Req/Sec     9.96k   693.09    12.56k    66.94%
+  1131535 requests in 15.00s, 121.94MB read
+Requests/sec:  75437.59
+Transfer/sec:      8.13MB
+
+---------------------------------------------------------
+ Concurrency: 32 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 32 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 32 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   192.80us  140.61us   9.74ms   96.09%
+    Req/Sec    21.24k     1.89k   26.00k    75.53%
+  2410319 requests in 15.00s, 259.75MB read
+Requests/sec: 160694.21
+Transfer/sec:     17.32MB
+
+---------------------------------------------------------
+ Concurrency: 64 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 64 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 64 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   308.30us    0.87ms  31.10ms   99.27%
+    Req/Sec    29.53k     3.48k   40.44k    81.50%
+  3336842 requests in 15.00s, 359.60MB read
+Requests/sec: 222472.43
+Transfer/sec:     23.97MB
+
+---------------------------------------------------------
+ Concurrency: 128 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 128 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 128 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   567.51us  329.58us  40.76ms   95.56%
+    Req/Sec    29.76k     2.51k   42.44k    77.07%
+  3371063 requests in 15.00s, 363.28MB read
+Requests/sec: 224764.65
+Transfer/sec:     24.22MB
+
+---------------------------------------------------------
+ Concurrency: 256 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 256 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+

results/vertx-pr501/20131008102405/db/vertx/raw

@@ -0,0 +1,128 @@
+Welcome to Ubuntu 12.04.3 LTS (GNU/Linux 3.9.0-030900-generic x86_64)
+
+ * Documentation:  https://help.ubuntu.com/
+
+  System information as of Tue Oct  8 10:27:38 PDT 2013
+
+  System load:  1.12               Processes:           116
+  Usage of /:   5.8% of 129.62GB   Users logged in:     0
+  Memory usage: 1%                 IP address for eth1: 172.16.98.98
+  Swap usage:   0%
+
+  Graph this data and manage this system at https://landscape.canonical.com/
+
+48 packages can be updated.
+15 updates are security updates.
+
+
+---------------------------------------------------------
+ Running Primer vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 60 -c 8 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 5s test @ http://tfbapp:8080/d
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   526.84us   74.78us   1.94ms   85.43%
+    Req/Sec     2.02k   145.03     2.44k    89.39%
+  72511 requests in 5.00s, 6.29MB read
+Requests/sec:  14503.23
+Transfer/sec:      1.26MB
+
+---------------------------------------------------------
+ Running Warmup vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     5.86ms  645.71us  16.22ms   83.93%
+    Req/Sec     5.73k   472.18     7.82k    77.03%
+  652615 requests in 15.00s, 56.64MB read
+Requests/sec:  43516.13
+Transfer/sec:      3.78MB
+
+---------------------------------------------------------
+ Concurrency: 8 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 8 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   518.82us   76.13us   1.81ms   92.46%
+    Req/Sec     2.05k    96.63     2.22k    80.99%
+  228267 requests in 15.00s, 19.81MB read
+Requests/sec:  15217.84
+Transfer/sec:      1.32MB
+
+---------------------------------------------------------
+ Concurrency: 16 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 16 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d
+  8 threads and 16 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   485.06us  220.13us   9.70ms   96.86%
+    Req/Sec     4.32k   334.35     5.22k    73.26%
+  490921 requests in 15.00s, 42.60MB read
+Requests/sec:  32728.97
+Transfer/sec:      2.84MB
+
+---------------------------------------------------------
+ Concurrency: 32 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 32 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d
+  8 threads and 32 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   739.72us  261.47us   6.39ms   68.66%
+    Req/Sec     5.67k   465.66     7.44k    68.77%
+  644198 requests in 15.00s, 55.91MB read
+Requests/sec:  42949.07
+Transfer/sec:      3.73MB
+
+---------------------------------------------------------
+ Concurrency: 64 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 64 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d
+  8 threads and 64 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     1.49ms  414.60us  11.55ms   86.50%
+    Req/Sec     5.69k   486.91     7.33k    71.86%
+  646628 requests in 15.00s, 56.12MB read
+Requests/sec:  43107.99
+Transfer/sec:      3.74MB
+
+---------------------------------------------------------
+ Concurrency: 128 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 128 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d
+  8 threads and 128 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     3.17ms    3.26ms  56.80ms   99.38%
+    Req/Sec     5.67k   607.47     7.89k    81.62%
+  645153 requests in 15.00s, 55.99MB read
+Requests/sec:  43011.63
+Transfer/sec:      3.73MB
+
+---------------------------------------------------------
+ Concurrency: 256 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 256 -t 8 http://tfbapp:8080/d
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     5.93ms    1.42ms  34.87ms   97.79%
+    Req/Sec     5.73k   556.44     7.70k    77.32%
+  650573 requests in 15.00s, 56.46MB read
+Requests/sec:  43379.25
+Transfer/sec:      3.76MB

results/vertx-pr501/20131008102405/json/vertx/raw

@@ -0,0 +1,128 @@
+Welcome to Ubuntu 12.04.3 LTS (GNU/Linux 3.9.0-030900-generic x86_64)
+
+ * Documentation:  https://help.ubuntu.com/
+
+  System information as of Tue Oct  8 10:25:26 PDT 2013
+
+  System load:  0.32               Processes:           116
+  Usage of /:   5.8% of 129.62GB   Users logged in:     0
+  Memory usage: 1%                 IP address for eth1: 172.16.98.98
+  Swap usage:   0%
+
+  Graph this data and manage this system at https://landscape.canonical.com/
+
+48 packages can be updated.
+15 updates are security updates.
+
+
+---------------------------------------------------------
+ Running Primer vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 60 -c 8 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 5s test @ http://tfbapp:8080/j
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   215.15us   36.75us 479.00us   71.75%
+    Req/Sec     4.72k   410.38     5.89k    89.47%
+  171252 requests in 5.00s, 18.46MB read
+Requests/sec:  34252.48
+Transfer/sec:      3.69MB
+
+---------------------------------------------------------
+ Running Warmup vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     1.14ms  486.42us  17.88ms   92.00%
+    Req/Sec    29.92k     2.72k   41.11k    89.65%
+  3357066 requests in 15.00s, 361.77MB read
+Requests/sec: 223848.87
+Transfer/sec:     24.12MB
+
+---------------------------------------------------------
+ Concurrency: 8 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 8 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   213.72us   72.86us   8.02ms   94.10%
+    Req/Sec     4.81k   331.22     6.78k    78.84%
+  546385 requests in 15.00s, 58.88MB read
+Requests/sec:  36426.71
+Transfer/sec:      3.93MB
+
+---------------------------------------------------------
+ Concurrency: 16 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 16 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 16 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   205.86us   93.09us   7.03ms   92.26%
+    Req/Sec     9.89k   713.96    12.22k    72.84%
+  1123169 requests in 15.00s, 121.04MB read
+Requests/sec:  74879.53
+Transfer/sec:      8.07MB
+
+---------------------------------------------------------
+ Concurrency: 32 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 32 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 32 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   194.08us  140.12us   9.44ms   96.04%
+    Req/Sec    21.18k     1.84k   26.33k    73.21%
+  2400706 requests in 15.00s, 258.71MB read
+Requests/sec: 160055.42
+Transfer/sec:     17.25MB
+
+---------------------------------------------------------
+ Concurrency: 64 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 64 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 64 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   286.17us  331.74us  20.07ms   98.88%
+    Req/Sec    29.52k     3.26k   40.22k    77.68%
+  3340379 requests in 15.00s, 359.98MB read
+Requests/sec: 222701.72
+Transfer/sec:     24.00MB
+
+---------------------------------------------------------
+ Concurrency: 128 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 128 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 128 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   573.14us  389.45us  21.84ms   98.10%
+    Req/Sec    29.65k     2.90k   41.33k    83.02%
+  3354159 requests in 15.00s, 361.46MB read
+Requests/sec: 223625.73
+Transfer/sec:     24.10MB
+
+---------------------------------------------------------
+ Concurrency: 256 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive'  -d 15 -c 256 -t 8 http://tfbapp:8080/j
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/j
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     1.15ms  701.18us  19.53ms   96.37%
+    Req/Sec    29.82k     3.27k   45.78k    94.32%
+  3370562 requests in 15.00s, 363.23MB read
+Requests/sec: 224714.38
+Transfer/sec:     24.22MB

results/vertx-pr501/20131008102405/plaintext/vertx/raw

@@ -0,0 +1,103 @@
+Welcome to Ubuntu 12.04.3 LTS (GNU/Linux 3.9.0-030900-generic x86_64)
+
+ * Documentation:  https://help.ubuntu.com/
+
+  System information as of Tue Oct  8 10:31:46 PDT 2013
+
+  System load:  0.17               Processes:           116
+  Usage of /:   5.8% of 129.62GB   Users logged in:     0
+  Memory usage: 1%                 IP address for eth1: 172.16.98.98
+  Swap usage:   0%
+
+  Graph this data and manage this system at https://landscape.canonical.com/
+
+48 packages can be updated.
+15 updates are security updates.
+
+
+---------------------------------------------------------
+ Running Primer vertx
+ wrk-pipeline -H 'Host: localhost' -H 'Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 60 -c 8 -t 8 http://tfbapp:8080/p
+---------------------------------------------------------
+
+Running 5s test @ http://tfbapp:8080/p
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     1.37ms   15.35ms 204.02ms   99.43%
+    Req/Sec     4.98k   538.16     6.78k    80.63%
+  189477 requests in 5.00s, 17.89MB read
+Requests/sec:  37898.95
+Transfer/sec:      3.58MB
+
+---------------------------------------------------------
+ Running Warmup vertx
+ wrk-pipeline -H 'Host: localhost' -H 'Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/p
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/p
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     1.13ms  453.80us  20.80ms   95.33%
+    Req/Sec    30.19k     2.84k   45.78k    91.92%
+  3400739 requests in 15.00s, 321.08MB read
+Requests/sec: 226760.26
+Transfer/sec:     21.41MB
+
+---------------------------------------------------------
+ Concurrency: 256 for vertx
+ wrk-pipeline -H 'Host: localhost' -H 'Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' --pipeline 16 -d 15 -c 256 -t 8 http://tfbapp:8080/p
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/p
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     5.70ms    2.27ms  28.43ms   90.51%
+    Req/Sec    92.90k     6.45k  134.40k    78.91%
+  10672448 requests in 15.00s, 0.98GB read
+Requests/sec: 711519.11
+Transfer/sec:     67.18MB
+
+---------------------------------------------------------
+ Concurrency: 1024 for vertx
+ wrk-pipeline -H 'Host: localhost' -H 'Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' --pipeline 16 -d 15 -c 1024 -t 8 http://tfbapp:8080/p
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/p
+  8 threads and 1024 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    22.40ms   16.72ms 812.90ms   93.54%
+    Req/Sec    90.98k     7.10k  134.63k    79.54%
+  10667096 requests in 15.00s, 0.98GB read
+  Socket errors: connect 0, read 60, write 0, timeout 47
+Requests/sec: 711137.55
+Transfer/sec:     67.14MB
+
+---------------------------------------------------------
+ Concurrency: 4096 for vertx
+ wrk-pipeline -H 'Host: localhost' -H 'Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' --pipeline 16 -d 15 -c 4096 -t 8 http://tfbapp:8080/p
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/p
+  8 threads and 4096 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   178.93ms  975.52ms   9.38s    98.63%
+    Req/Sec    89.17k    12.86k  121.60k    76.74%
+  10391578 requests in 15.00s, 0.96GB read
+  Socket errors: connect 0, read 551, write 0, timeout 5027
+Requests/sec: 692679.97
+Transfer/sec:     65.40MB
+
+---------------------------------------------------------
+ Concurrency: 16384 for vertx
+ wrk-pipeline -H 'Host: localhost' -H 'Accept: text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' --pipeline 16 -d 15 -c 16384 -t 8 http://tfbapp:8080/p
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/p
+  8 threads and 16384 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     9.39s     6.62s   14.66s    66.71%
+    Req/Sec    86.73k    24.23k  194.04k    76.89%
+  10182173 requests in 15.02s, 0.94GB read
+  Socket errors: connect 11, read 1108, write 0, timeout 77003
+Requests/sec: 677687.84
+Transfer/sec:     63.98MB

results/vertx-pr501/20131008102405/query/vertx/raw

@@ -0,0 +1,114 @@
+Welcome to Ubuntu 12.04.3 LTS (GNU/Linux 3.9.0-030900-generic x86_64)
+
+ * Documentation:  https://help.ubuntu.com/
+
+  System information as of Tue Oct  8 10:29:51 PDT 2013
+
+  System load:  0.73               Processes:           116
+  Usage of /:   5.8% of 129.62GB   Users logged in:     0
+  Memory usage: 1%                 IP address for eth1: 172.16.98.98
+  Swap usage:   0%
+
+  Graph this data and manage this system at https://landscape.canonical.com/
+
+48 packages can be updated.
+15 updates are security updates.
+
+
+---------------------------------------------------------
+ Running Primer vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 5 -c 8 -t 8 http://tfbapp:8080/d?queries=2
+---------------------------------------------------------
+
+Running 5s test @ http://tfbapp:8080/d?queries=2
+  8 threads and 8 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   513.65us  205.27us   7.35ms   94.93%
+    Req/Sec     2.03k   151.96     2.44k    75.95%
+  76836 requests in 5.00s, 7.11MB read
+Requests/sec:  15367.39
+Transfer/sec:      1.42MB
+
+---------------------------------------------------------
+ Running Warmup vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/d?queries=2
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d?queries=2
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    11.74ms    1.89ms  44.92ms   97.05%
+    Req/Sec     2.81k   201.99     3.76k    83.52%
+  327614 requests in 15.00s, 30.31MB read
+Requests/sec:  21844.39
+Transfer/sec:      2.02MB
+
+---------------------------------------------------------
+ Queries: 1 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/d?queries=1
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d?queries=1
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency     5.93ms  751.26us  14.99ms   88.54%
+    Req/Sec     5.68k   589.01     8.00k    83.43%
+  648452 requests in 15.00s, 56.28MB read
+Requests/sec:  43231.27
+Transfer/sec:      3.75MB
+
+---------------------------------------------------------
+ Queries: 5 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/d?queries=5
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d?queries=5
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    29.36ms    2.67ms  63.22ms   90.33%
+    Req/Sec     1.09k   121.83     1.54k    85.29%
+  130024 requests in 15.00s, 13.89MB read
+Requests/sec:   8668.05
+Transfer/sec:      0.93MB
+
+---------------------------------------------------------
+ Queries: 10 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/d?queries=10
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d?queries=10
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    58.95ms    3.35ms  88.38ms   85.35%
+    Req/Sec   543.34     51.37   652.00     89.40%
+  64936 requests in 15.00s, 8.48MB read
+Requests/sec:   4327.78
+Transfer/sec:    579.01KB
+
+---------------------------------------------------------
+ Queries: 15 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/d?queries=15
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d?queries=15
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency    88.27ms    5.24ms 134.22ms   86.22%
+    Req/Sec   363.13     23.51   439.00     77.29%
+  43369 requests in 15.04s, 6.70MB read
+Requests/sec:   2882.69
+Transfer/sec:    456.05KB
+
+---------------------------------------------------------
+ Queries: 20 for vertx
+ wrk -H 'Host: localhost' -H 'Accept: application/json,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7' -H 'Connection: keep-alive' -d 15 -c 256 -t 8 http://tfbapp:8080/d?queries=20
+---------------------------------------------------------
+
+Running 15s test @ http://tfbapp:8080/d?queries=20
+  8 threads and 256 connections
+  Thread Stats   Avg      Stdev     Max   +/- Stdev
+    Latency   118.36ms    6.15ms 168.75ms   89.14%
+    Req/Sec   270.50      9.46   293.00     77.72%
+  32453 requests in 15.00s, 5.82MB read
+Requests/sec:   2162.89
+Transfer/sec:    397.09KB

toolset/README.md

@@ -10,11 +10,11 @@ For deployment instructions, refer to the [Benchmark Suite Deployment README fil
 
 Assuming the deployment finished without error, we're ready to start the test suite. On the Linux server host, type:
 
-	nohup toolset/run-tests.py -s server-private-ip -c client-private-ip -i path-to-ssh-key --max-threads number-of-cores &
+	nohup toolset/run-tests.py -s server-private-ip -c client-private-ip -u username -i path-to-private-ssh-key --max-threads number-of-cores &
 
 For the number-of-cores parameter, you will need to know your application server's core count. For example, Amazon EC2 large instances have 2 cores.
 
-`path-to-ssh-key` is the pathname of the private SSH key file of the Linux client's user. This file is set up during deployment.
+`path-to-private-ssh-key` is the pathname of the private SSH key file of the Linux client's user. This file is set up during deployment.  `username` is the username associated with the private SSH key.
 
 This script will run the full set of tests. Be aware that on Large instances, if you include the slower frameworks (and they are included by default), **the total runtime of a full suite of tests can be measured in days**, not just hours. The EC2 bill isn't going to break the bank, but it's also not going to be chump change.
 
@@ -30,11 +30,11 @@ To run the test suite on the Windows server, log on the server, open the PowerSh
 
 If you use a different configuration than two m1.large instances, please use the --name option to name the results appropriately.
 
-	nohup toolset/run-tests.py -s server-private-ip -c client-private-ip -i path-to-ssh-key --max-threads cores --name ec2-servertype-clienttype &
+	nohup toolset/run-tests.py -s server-private-ip -c client-private-ip -u username -i path-to-private-ssh-key --max-threads cores --name ec2-servertype-clienttype &
 
 So if you were running an m1.large and an m1.medium, it would look like this:
 
-	nohup toolset/run-tests.py -s server-private-ip -c client-private-ip -i path-to-ssh-key --max-threads cores --name ec2-m1.large-m1.medium &
+	nohup toolset/run-tests.py -s server-private-ip -c client-private-ip -u username -i path-to-private-ssh-key --max-threads cores --name ec2-m1.large-m1.medium &
 
 This will allow us to differentiate results.
 
@@ -42,7 +42,7 @@ This will allow us to differentiate results.
 
 If you are making changes to any of the tests, or you simply want to verify a single test, you can run the script with the --test flag. For example, if you only wanted to run the JRuby tests:
 
-	nohup toolset/run-tests.py -s server-ip -c client-ip -i path-to-ssh-key --max-threads cores --name unique-machine-name --test rack-jruby sinatra-jruby rails-jruby
+	nohup toolset/run-tests.py -s server-ip -c client-ip -u username -i path-to-private-ssh-key --max-threads cores --name unique-machine-name --test rack-jruby sinatra-jruby rails-jruby
 
 ## Result Files
 

toolset/deployment/azure/azure-deployment.sh

@@ -217,19 +217,19 @@ function azure_save_deployment_configuration {
     echo "Saving configuration file at $AZURE_CONFIGURATION_OUTPUT_FILE"
 
     cat >$AZURE_CONFIGURATION_OUTPUT_FILE <<_EOF_
-BENCHMARK_LINUX_CLIENT=$CLIENT_VM_NAME.cloudapp.net
-BENCHMARK_LINUX_CLIENT_IP=$CLIENT_IP
-BENCHMARK_LINUX_SERVER=$LINUX_SERVER_VM_NAME.cloudapp.net
-BENCHMARK_LINUX_SERVER_IP=$LINUX_SERVER_IP
-BENCHMARK_LINUX_USER=$AZURE_LINUX_USER
-BENCHMARK_SSH_KEY=$AZURE_KEY_FILE
-BENCHMARK_WINDOWS_SERVER=$WINDOWS_SERVER_VM_NAME.cloudapp.net
-BENCHMARK_WINDOWS_SERVER_USER=$WINDOWS_SERVER_VM_NAME\Administrator
-BENCHMARK_SQL_SERVER=$SQL_SERVER_VM_NAME.cloudapp.net
-BENCHMARK_SQL_SERVER_USER=$SQL_SERVER_VM_NAME\Administrator
-BENCHMARK_WORKING_DIR=$BENCHMARK_WORKING_DIR
-BENCHMARK_REPOSITORY=$BENCHMARK_REPOSITORY
-BENCHMARK_BRANCH=$BENCHMARK_BRANCH
+BENCHMARK_LINUX_CLIENT="$CLIENT_VM_NAME.cloudapp.net"
+BENCHMARK_LINUX_CLIENT_IP="$CLIENT_IP"
+BENCHMARK_LINUX_SERVER="$LINUX_SERVER_VM_NAME.cloudapp.net"
+BENCHMARK_LINUX_SERVER_IP="$LINUX_SERVER_IP"
+BENCHMARK_LINUX_USER="$AZURE_LINUX_USER"
+BENCHMARK_SSH_KEY="$AZURE_KEY_FILE"
+BENCHMARK_WINDOWS_SERVER="$WINDOWS_SERVER_VM_NAME.cloudapp.net"
+BENCHMARK_WINDOWS_SERVER_USER="$WINDOWS_SERVER_VM_NAME\Administrator"
+BENCHMARK_SQL_SERVER="$SQL_SERVER_VM_NAME.cloudapp.net"
+BENCHMARK_SQL_SERVER_USER="$SQL_SERVER_VM_NAME\Administrator"
+BENCHMARK_WORKING_DIR="$BENCHMARK_WORKING_DIR"
+BENCHMARK_REPOSITORY="$BENCHMARK_REPOSITORY"
+BENCHMARK_BRANCH="$BENCHMARK_BRANCH"
 _EOF_
 
     echo ""

toolset/deployment/common/linux-initial-deployment.sh

@@ -22,19 +22,19 @@ information "Benchmark Suite Deployment: Linux server setup"
 information "******************************************************************************"
 
 BENCHMARK_REMOTE_CONFIGURATION_FILE="$BENCHMARK_WORKING_DIR/benchmark-configuration.sh"
-BENCHMARK_REMOTE_KEY_FILE="~/.ssh/benchmark-key"
+BENCHMARK_REMOTE_KEY_FILE="/home/$BENCHMARK_LINUX_USER/.ssh/benchmark-key"
 
 # Create Linux host configuration script.
 echo ""
 echo "Creating Linux host configuration script at $BENCHMARK_REMOTE_CONFIGURATION_FILE"
 cat >$BENCHMARK_REMOTE_CONFIGURATION_FILE <<_EOF_
 #!/bin/bash
-export BENCHMARK_HOME=~/FrameworkBenchmarks
-export BENCHMARK_SERVER_IP=$BENCHMARK_LINUX_SERVER_IP
-export BENCHMARK_CLIENT_IP=$BENCHMARK_LINUX_CLIENT_IP
-export BENCHMARK_KEY_PATH=$BENCHMARK_REMOTE_KEY_FILE
-export BENCHMARK_REPOSITORY=$BENCHMARK_REPOSITORY
-export BENCHMARK_BRANCH=$BENCHMARK_BRANCH
+export BENCHMARK_HOME="/home/$BENCHMARK_LINUX_USER/FrameworkBenchmarks"
+export BENCHMARK_SERVER_IP="$BENCHMARK_LINUX_SERVER_IP"
+export BENCHMARK_CLIENT_IP="$BENCHMARK_LINUX_CLIENT_IP"
+export BENCHMARK_KEY_PATH="$BENCHMARK_REMOTE_KEY_FILE"
+export BENCHMARK_REPOSITORY="$BENCHMARK_REPOSITORY"
+export BENCHMARK_BRANCH="$BENCHMARK_BRANCH"
 _EOF_
 
 # Upload Linux host configuration script.

toolset/setup/README.md

@@ -102,7 +102,7 @@ Now you can run tests:
 * Run a `Command Prompt` as Administrator.
 * Enter this command:
 
-        powershell -ExecutionPolicy Bypass -Command "iex (New-Object Net.WebClient).DownloadString('https://raw.github.com/TechEmpower/FrameworkBenchmarks/master/setup-sqlserver-bootstrap.ps1')"
+        powershell -ExecutionPolicy Bypass -Command "iex (New-Object Net.WebClient).DownloadString('https://raw.github.com/TechEmpower/FrameworkBenchmarks/master/toolset/setup/sqlserver/setup-sqlserver-bootstrap.ps1')"
 
 * This will configure SQL Server, the Windows Firewall, and populate the database.
 

toolset/setup/linux/installer.py

@@ -124,10 +124,10 @@ class Installer:
 
     self.__download("http://downloads.activestate.com/ActivePerl/releases/5.16.3.1603/ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746.tar.gz");
     self.__run_command("tar xzf ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746.tar.gz");
-    self.__run_command("sudo ./install.sh --license-accepted --prefix /opt/ActivePerl-5.16 --no-install-html", cwd="ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746", send_yes=True)
+    self.__run_command("sudo ./install.sh --license-accepted --prefix /opt/ActivePerl-5.16 --no-install-html", cwd="ActivePerl-5.16.3.1603-x86_64-linux-glibc-2.3.5-296746", send_yes=True, retry=True)
     self.__download("http://cpanmin.us", "cpanminus.pl")
-    self.__run_command("perl cpanminus.pl --sudo App::cpanminus")
-    self.__run_command("cpanm -f -S DBI DBD::mysql Kelp Dancer Mojolicious Kelp::Module::JSON::XS Dancer::Plugin::Database Starman Plack JSON Web::Simple DBD::Pg JSON::XS EV HTTP::Parser::XS Monoceros EV IO::Socket::IP IO::Socket::SSL")
+    self.__run_command("perl cpanminus.pl --sudo App::cpanminus", retry=True)
+    self.__run_command("cpanm -f -S DBI DBD::mysql Kelp Dancer Mojolicious Kelp::Module::JSON::XS Dancer::Plugin::Database Starman Plack JSON Web::Simple DBD::Pg JSON::XS EV HTTP::Parser::XS Monoceros EV IO::Socket::IP IO::Socket::SSL", retry=True)
 
     #
     # php
@@ -138,7 +138,7 @@ class Installer:
     self.__run_command("./configure --with-pdo-mysql --with-mysql --with-mcrypt --enable-intl --enable-mbstring --enable-fpm --with-fpm-user=www-data --with-fpm-group=www-data --with-openssl", cwd="php-5.4.13")
     self.__run_command("make", cwd="php-5.4.13")
     self.__run_command("sudo make install", cwd="php-5.4.13")
-    self.__run_command("printf \"\\n\" | sudo pecl install apc-beta", cwd="php-5.4.13")
+    self.__run_command("printf \"\\n\" | sudo pecl install apc-beta", cwd="php-5.4.13", retry=True)
     self.__run_command("sudo cp ../config/php.ini /usr/local/lib/php.ini")
     self.__run_command("sudo cp ../config/php-fpm.conf /usr/local/lib/php-fpm.conf")
     self.__run_command("rm php-5.4.13.tar.gz")
@@ -283,14 +283,14 @@ class Installer:
     #
     # Vert.x
     #
-    self.__download("http://dl.bintray.com/vertx/downloads/vert.x-2.0.0-final.tar.gz")
-    self.__run_command("tar xzf vert.x-2.0.0-final.tar.gz")
+    self.__download("http://dl.bintray.com/vertx/downloads/vert.x-2.0.2-final.tar.gz?direct=true")
+    self.__run_command("tar xzf vert.x-2.0.2-final.tar.gz")
 
     #
     # Yesod
     #
-    self.__run_command("cabal update")
-    self.__run_command("cabal install yesod persistent-mysql")
+    self.__run_command("cabal update", retry=True)
+    self.__run_command("cabal install yesod persistent-mysql", retry=True)
 
     #
     # Jester
@@ -309,9 +309,9 @@ class Installer:
     python3_bin= "~/FrameworkBenchmarks/installs/py3/bin"
     def easy_install(pkg, two=True, three=False, pypy=False):
       cmd = "/easy_install -ZU '" + pkg + "'"
-      if two:   self.__run_command(python_bin + cmd)
-      if three: self.__run_command(python3_bin + cmd)
-      if pypy:  self.__run_command(pypy_bin + cmd)
+      if two:   self.__run_command(python_bin + cmd, retry=True)
+      if three: self.__run_command(python3_bin + cmd, retry=True)
+      if pypy:  self.__run_command(pypy_bin + cmd, retry=True)
 
     self.__download("http://bitbucket.org/pypy/pypy/downloads/pypy-2.1-linux64.tar.bz2")
     self.__run_command("tar xjf pypy-2.1-linux64.tar.bz2")

toolset/setup/sqlserver/setup-sqlserver-bootstrap.ps1

@@ -1,13 +1,13 @@
 # To download and run this script, open an elevated Command Prompt and then run:
 #
-# powershell -ExecutionPolicy Bypass -Command "iex (New-Object Net.WebClient).DownloadString('https://raw.github.com/TechEmpower/FrameworkBenchmarks/master/setup-sqlserver-bootstrap.ps1')"
+# powershell -ExecutionPolicy Bypass -Command "iex (New-Object Net.WebClient).DownloadString('https://raw.github.com/TechEmpower/FrameworkBenchmarks/master/toolset/setup/sqlserver/setup-sqlserver-bootstrap.ps1')"
 
 $basedir = "C:\FrameworkBenchmarks"
 $rawRepo = "https://raw.github.com/TechEmpower/FrameworkBenchmarks/master"
 
 $config_url = $rawRepo + "/config"
 $config_local = $basedir + "\config"
-$setup_sqlserver_url = $rawRepo + "/setup-sqlserver.ps1"
+$setup_sqlserver_url = $rawRepo + "/toolset/setup/sqlserver/setup-sqlserver.ps1"
 $setup_sqlserver_local = $basedir + "\setup-sqlserver.ps1"
 $create_sqlserver_login_and_database_url = $config_url + "/create-sqlserver-login-and-database.sql"
 $create_sqlserver_login_and_database_local = $config_local + "/create-sqlserver-login-and-database.sql"

toolset/setup/windows/installer-bootstrap.ps1

@@ -10,7 +10,7 @@ $basedir = "C:\FrameworkBenchmarks"
 $workdir = $basedir + "\installs"
 
 $repo = "https://github.com/TechEmpower/FrameworkBenchmarks"
-$installer = $basedir + "\installer.ps1"
+$installer = $basedir + "\toolset\setup\windows\installer.ps1"
 
 $git = "C:\Git\bin\git.exe"
 $gitinstaller_file = "Git-1.8.1.2-preview20130201.exe"

vertx/App.groovy

@@ -1,32 +0,0 @@
-// Our application config
-
-def persistorConf = [
-  address: 'hello.persistor',
-  db_name: 'hello_world',
-  host: '127.0.0.1'
-]
-
-def permitted =
-[
-  // Allow calls to get static album data from the persistor
-  //[
-  //  'address' : 'hello.persistor',
-  //  'match' : [
-  //    'action' : 'find',
-  //    'collection' : 'users'
-  //  ]
-  //]
-  [
-    'address' : 'hello.persistor'
-  ]
-]
-
-container.with {
-
-  // Deploy the busmods
-  // deployModule('vertx.mongo-persistor-v1.2.1', persistorConf, 8)
-
-  // Start the web server
-
-  deployVerticle('WebServer', ['permitted': permitted], 8)
-}

vertx/WebServer.java

@@ -1,143 +1,111 @@
-import java.io.IOException;
 import java.nio.charset.*;
+
+import java.util.ArrayList;
+import java.util.Collections;
 import java.util.List;
-import java.util.Map;
-import java.util.HashMap;
 import java.util.Random;
-import java.util.concurrent.CopyOnWriteArrayList;
 import java.util.concurrent.ThreadLocalRandom;
-import java.util.concurrent.atomic.AtomicInteger;
-
-import com.fasterxml.jackson.databind.ObjectMapper;
 import org.vertx.java.core.Handler;
 import org.vertx.java.core.buffer.Buffer;
 import org.vertx.java.core.eventbus.Message;
 import org.vertx.java.core.http.HttpServerRequest;
-import org.vertx.java.core.json.JsonArray;
 import org.vertx.java.core.json.JsonObject;
+import org.vertx.java.core.json.impl.Json;
 import org.vertx.java.platform.Verticle;
 
-public class WebServer
-	extends    Verticle
-	implements Handler<HttpServerRequest>
-{
-  private final ObjectMapper mapper = new ObjectMapper();
+public class WebServer extends Verticle implements Handler<HttpServerRequest> {
+
+  private static String helloWorld = "Hello, World!";
+  private static Buffer helloWorldBuffer = new Buffer(helloWorld);
+  private static String helloWorldContentLength = String.valueOf(helloWorldBuffer.length());
 
   @Override
-  public void start()
-  {
-    this.getVertx().createHttpServer().requestHandler(this).listen(8080);
+  public void start() {
+    vertx.createHttpServer().requestHandler(WebServer.this).listen(8080);
   }
 
   @Override
-  public void handle(HttpServerRequest req)
-  {
-    if (req.path().equals("/json"))
-    {
-      handleJson(req);
-    }
-    else if (req.path().equals("/db"))
-    {
-      handleDb(req);
+  public void handle(HttpServerRequest req) {
+    String path = req.path();
+    switch (path.charAt(1)) {
+      case 'p':
+        handlePlainText(req);
+        break;
+      case 'j':
+        handleJson(req);
+        break;
+      case 'd':
+        handleDbMongo(req);
+        break;
+      default:
+        req.response().setStatusCode(404);
+        req.response().end();
     }
-    else
-    {
-      req.response().setStatusCode(404);
-      req.response().end();
-    }
-  }
 
+  }
 
-  private void handleJson(HttpServerRequest req)
-  {
-    Buffer buffer;
-    try
-    {
-      Map<String, String> data = new HashMap<String, String>();
-      data.put("message", "Hello, world");
-      buffer = new Buffer(mapper.writeValueAsBytes(data));
-    }
-    catch (IOException e)
-    {
-      req.response().setStatusCode(500);
-      req.response().end();
-      return;
-    }
-    
+  private void handlePlainText(HttpServerRequest req) {
+    req.response().putHeader("Content-Type", "application/json; charset=UTF-8");
+    req.response().putHeader("Content-Length", helloWorldContentLength);
+    req.response().end(helloWorldBuffer);
+  }
 
+  private void handleJson(HttpServerRequest req) {
+    String result = Json.encode(Collections.singletonMap("message", "Hello, world!"));
+    int contentLength = result.getBytes(StandardCharsets.UTF_8).length;
     req.response().putHeader("Content-Type", "application/json; charset=UTF-8");
-    req.response().putHeader("Content-Length", Integer.toString(buffer.length()));
-    req.response().write(buffer);
-    req.response().end();
+    req.response().putHeader("Content-Length", String.valueOf(contentLength));
+    req.response().end(result);
   }
 
-  private void handleDb(final HttpServerRequest req)
-  {
+  private void handleDbMongo(final HttpServerRequest req) {
     int queriesParam = 1;
-    try 
-    {
+    try {
       queriesParam = Integer.parseInt(req.params().get("queries"));
-    }
-    catch (NumberFormatException e)
-    {
+    } catch (NumberFormatException e) {
       // do nothing
     }
 
-    final DbHandler dbh = new DbHandler(req, queriesParam);
+    final MongoHandler dbh = new MongoHandler(req, queriesParam);
     final Random random = ThreadLocalRandom.current();
 
-    for (int i = 0; i < queriesParam; i++)
-    {
-      this.getVertx().eventBus().send(
-        "hello.persistor",
-        new JsonObject()
-            .putString("action", "findone")
-            .putString("collection", "world")
-            .putObject("matcher", new JsonObject().putNumber("id", (random.nextInt(10000) + 1))),
-        dbh);
+    for (int i = 0; i < queriesParam; i++) {
+      vertx.eventBus().send(
+          "hello.persistor",
+          new JsonObject()
+              .putString("action", "findone")
+              .putString("collection", "world")
+              .putObject("matcher", new JsonObject().putNumber("id", (random.nextInt(10000) + 1))),
+          dbh);
     }
   }
 
-  class DbHandler implements Handler<Message<JsonObject>>
-  {
+  private static class MongoHandler implements Handler<Message<JsonObject>> {
     private final HttpServerRequest req;
     private final int queries;
-    private final List<Object> worlds = new CopyOnWriteArrayList<>();
+    private final List<Object> worlds = new ArrayList<>();
 
-    public DbHandler(HttpServerRequest request, int queriesParam)
-    {
-   	  this.req = request;
+    public MongoHandler(HttpServerRequest request, int queriesParam) {
+      this.req = request;
       this.queries = queriesParam;
     }
 
     @Override
-    public void handle(Message<JsonObject> reply)
-    {
+    public void handle(Message<JsonObject> reply) {
       final JsonObject body = reply.body();
 
-      if ("ok".equals(body.getString("status")))
-      {
-      	this.worlds.add(body.getObject("result"));
-      }
-
-      if (this.worlds.size() == this.queries)
-      {
-        // All queries have completed; send the response.
-        // final JsonArray arr = new JsonArray(worlds);
-        try
-        {
-          final String result = mapper.writeValueAsString(worlds);
+      if ("ok".equals(body.getString("status"))) {
+        this.worlds.add(body.getObject("result"));
+        if (this.worlds.size() == this.queries) {
+          // All queries have completed; send the response.
+          final String result = Json.encode(worlds);
           final int contentLength = result.getBytes(StandardCharsets.UTF_8).length;
           this.req.response().putHeader("Content-Type", "application/json; charset=UTF-8");
-          this.req.response().putHeader("Content-Length", Integer.toString(contentLength));
-          this.req.response().write(result);
-          this.req.response().end();
-        }
-        catch (IOException e)
-        {
-          req.response().setStatusCode(500);
-          req.response().end();
+          this.req.response().putHeader("Content-Length", String.valueOf(contentLength));
+          this.req.response().end(result);
         }
+      } else {
+        System.err.println("Failed to execute query");
       }
     }
   }

vertx/app.js

@@ -0,0 +1,15 @@
+var container = require('vertx/container')
+
+var persistorConf = {
+  address: 'hello.persistor',
+  db_name: 'hello_world',
+  host: 'localhost'
+}
+
+container.deployModule('io.vertx~mod-mongo-persistor~2.0.0-final', persistorConf, function (err, dep_id) {
+  if (!err) {
+    container.deployVerticle('WebServer.java', 8);
+  } else {
+    err.printStackTrace();
+  }
+});

vertx/benchmark_config

@@ -3,9 +3,10 @@
   "tests": [{
     "default": {
       "setup_file": "setup",
-      "json_url": "/json",
-      "db_url": "/db",
-      "query_url": "/db?queries=",
+      "json_url": "/j",
+      "db_url": "/d",
+      "plaintext_url": "/p",
+      "query_url": "/d?queries=",
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
@@ -22,4 +23,4 @@
       "versus": "netty"
     }
   }]
-}
+}

vertx/setup.py

@@ -7,10 +7,9 @@ import os
 nCpu = multiprocessing.cpu_count()
 
 def start(args):
-  setup_util.replace_text("vertx/App.groovy", "host: '.*'", "host: '" + args.database_host + "'")
-
-  try:    
-    subprocess.Popen("vertx run WebServer.java -instances " + str(nCpu) , shell=True, cwd="vertx")
+  setup_util.replace_text("vertx/app.js", "host: '.*'", "host: '" + args.database_host + "'")
+  try:
+    subprocess.Popen("vertx run app.js", shell=True, cwd="vertx")
     return 0
   except subprocess.CalledProcessError:
     return 1
@@ -18,7 +17,7 @@ def stop():
   p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
   out, err = p.communicate()
   for line in out.splitlines():
-    if 'App.groovy' in line:
+    if 'app.js' in line:
       pid = int(line.split(None, 2)[1])
       os.kill(pid, 9)