Merge branch 'master' of https://github.com/TechEmpower/FrameworkBenchmarks into python-r8

.gitignore

@@ -25,6 +25,7 @@ bin/
 *.swp
 *.diff
 *.patch
+*/bin/
 
 # intellij
 *.iml
@@ -64,3 +65,6 @@ benchmark.cfg
 # java / ruby stuff
 *.lock
 *.war
+
+# go
+go/pkg/

UrWeb/benchmark_config

@@ -11,16 +11,16 @@
       "plaintext_url": "/plaintext",
       "port": 8080,
       "approach": "Realistic",
-      "classification": "Full",
+      "classification": "Fullstack",
       "database": "Postgres",
-      "framework": "Ur/Web",
+      "framework": "ur/web",
       "language": "Ur",
       "orm": "Micro",
       "platform": "Ur/Web",
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "Ur/Web"
+      "display_name": "ur/web"
     }
   }]
 }

config/benchmark_profile

@@ -1,7 +1,7 @@
 export JAVA_HOME=/usr/lib/jvm/java-1.7.0-openjdk-amd64
 export RESIN_HOME=~/FrameworkBenchmarks/installs/resin-4.0.36
 export GRAILS_HOME=~/FrameworkBenchmarks/installs/grails-2.3.3
-export VERTX_HOME=~/FrameworkBenchmarks/installs/vert.x-2.0.2-final
+export VERTX_HOME=~/FrameworkBenchmarks/installs/vert.x-2.1M1
 export GOROOT=~/FrameworkBenchmarks/installs/go
 export GOPATH=~/FrameworkBenchmarks/go:~/FrameworkBenchmarks/webgo:~/FrameworkBenchmarks/revel
 export TOMCAT_HOME=~/FrameworkBenchmarks/installs/apache-tomcat-7.0.35

config/create.js

@@ -1,25 +1,25 @@
 use hello_world
 db.world.drop()
 for (var i = 1; i <= 10000; i++) {
-  db.world.save( { id: i, randomNumber: (Math.floor(Math.random() * 10000) + 1) })
+  db.world.save( { _id: i, randomNumber: (Math.floor(Math.random() * 10000) + 1) })
 }
 
 // http://docs.mongodb.org/manual/applications/optimization/
-db.world.ensureIndex({id: 1})
+db.world.ensureIndex({_id: 1})
 
 db.fortune.drop()
 
-db.fortune.save( {id: 1, message: 'fortune: No such file or directory'} );
-db.fortune.save( {id: 2, message: "A computer scientist is someone who fixes things that aren't broken."} );
-db.fortune.save( {id: 3, message: 'After enough decimal places, nobody gives a damn.'} );
-db.fortune.save( {id: 4, message: 'A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1'} );
-db.fortune.save( {id: 5, message: 'A computer program does what you tell it to do, not what you want it to do.'} );
-db.fortune.save( {id: 6, message: 'Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen'} );
-db.fortune.save( {id: 7, message: 'Any program that runs right is obsolete.'} );
-db.fortune.save( {id: 8, message: 'A list is only as strong as its weakest link. — Donald Knuth'} );
-db.fortune.save( {id: 9, message: 'Feature: A bug with seniority.'} );
-db.fortune.save( {id: 10, message: 'Computers make very fast, very accurate mistakes.'} );
-db.fortune.save( {id: 11, message: '<script>alert("This should not be displayed in a browser alert box.");</script>'} );
-db.fortune.save( {id: 12, message: 'フレームワークのベンチマーク'} );
+db.fortune.save( {_id: 1, message: 'fortune: No such file or directory'} );
+db.fortune.save( {_id: 2, message: "A computer scientist is someone who fixes things that aren't broken."} );
+db.fortune.save( {_id: 3, message: 'After enough decimal places, nobody gives a damn.'} );
+db.fortune.save( {_id: 4, message: 'A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1'} );
+db.fortune.save( {_id: 5, message: 'A computer program does what you tell it to do, not what you want it to do.'} );
+db.fortune.save( {_id: 6, message: 'Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen'} );
+db.fortune.save( {_id: 7, message: 'Any program that runs right is obsolete.'} );
+db.fortune.save( {_id: 8, message: 'A list is only as strong as its weakest link. — Donald Knuth'} );
+db.fortune.save( {_id: 9, message: 'Feature: A bug with seniority.'} );
+db.fortune.save( {_id: 10, message: 'Computers make very fast, very accurate mistakes.'} );
+db.fortune.save( {_id: 11, message: '<script>alert("This should not be displayed in a browser alert box.");</script>'} );
+db.fortune.save( {_id: 12, message: 'フレームワークのベンチマーク'} );
 
-db.fortune.ensureIndex({id: 1})
+db.fortune.ensureIndex({_id: 1})

dart/pubspec.yaml

@@ -1,8 +1,8 @@
 name: dartbenchmark
 description: A benchmark of dart
 dependencies:
-  args: 0.5.9
-  crypto: 0.5.13
-  mustache: 0.1.5
-  postgresql: 0.2.7
-  yaml: 0.5.7
+  args: 0.9.0
+  crypto: 0.9.0
+  mustache: 0.1.6
+  postgresql: 0.2.12
+  yaml: 0.9.0

dart/server.dart

@@ -1,7 +1,6 @@
 import 'dart:async' show Future;
 import 'dart:io';
-import 'dart:utf';
-import 'dart:json' as json;
+import 'dart:convert';
 import 'dart:math' show Random;
 import 'package:args/args.dart' show ArgParser;
 import 'package:mustache/mustache.dart' as mustache;
@@ -13,12 +12,12 @@ import 'package:yaml/yaml.dart' as yaml;
 /// address and port for incoming connections is configurable via command line
 /// arguments, as is the number of database connections to be maintained in the
 /// connection pool.
-main() {
+main(args) {
   var parser = new ArgParser();
   parser.addOption('address', abbr: 'a', defaultsTo: '0.0.0.0');
   parser.addOption('port', abbr: 'p', defaultsTo: '8080');
   parser.addOption('dbconnections', abbr: 'd', defaultsTo: '256');
-  var arguments = parser.parse(new Options().arguments);
+  var arguments = parser.parse(args);
   _startServer(
       arguments['address'],
       int.parse(arguments['port']),
@@ -126,16 +125,11 @@ _parseInt(text) =>
 _sendResponse(request, statusCode, [ type, response ]) {
   request.response.statusCode = statusCode;
   request.response.headers.date = new DateTime.now();
-  //
-  // Prevent GZIP encoding, because it is disallowed in the rules for these
-  // benchmark tests.
-  //
-  request.response.headers.add(HttpHeaders.CONTENT_ENCODING, '');
   if (type != null) {
     request.response.headers.contentType = type;
   }
   if (response != null) {
-    var data = encodeUtf8(response);
+    var data = UTF8.encode(response);
     request.response.contentLength = data.length;
     request.response.add(data);
   } else {
@@ -151,7 +145,7 @@ _sendHtml(request, response) {
 
 /// Completes the given [request] by writing the [response] as JSON.
 _sendJson(request, response) {
-  _sendResponse(request, HttpStatus.OK, _TYPE_JSON, json.stringify(response));
+  _sendResponse(request, HttpStatus.OK, _TYPE_JSON, JSON.encode(response));
 }
 
 /// Completes the given [request] by writing the [response] as plain text.

evhttp-sharp/benchmark_config

@@ -13,7 +13,7 @@
       "language": "C#",
       "orm": "Raw",
       "platform": "Mono",
-      "webserver": "none",
+      "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
       "display_name": "evhttp-sharp",

grizzly-bm/pom.xml

@@ -68,7 +68,7 @@
         <dependency>
             <groupId>org.glassfish.grizzly</groupId>
             <artifactId>grizzly-http-server</artifactId>
-            <version>2.3.5</version>
+            <version>2.3.8-beta1</version>
         </dependency>
         <dependency>
             <groupId>com.fasterxml.jackson.core</groupId>

grizzly-bm/src/main/java/org/glassfish/grizzly/bm/JsonHttpHandler.java

@@ -1,12 +1,10 @@
 package org.glassfish.grizzly.bm;
 
-import com.fasterxml.jackson.core.JsonFactory;
-import com.fasterxml.jackson.core.JsonGenerator;
 import com.fasterxml.jackson.databind.*;
 import java.io.IOException;
-import java.util.concurrent.ExecutorService;
 import org.glassfish.grizzly.http.server.HttpHandler;
 import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.grizzly.http.server.RequestExecutorProvider;
 import org.glassfish.grizzly.http.server.Response;
 import org.glassfish.grizzly.http.util.Header;
 
@@ -15,6 +13,8 @@ import org.glassfish.grizzly.http.util.Header;
  */
 public class JsonHttpHandler extends HttpHandler {
 
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    
     // Response message class.
     public static class HelloMessage {
       public final String message = "Hello, World!";
@@ -26,21 +26,12 @@ public class JsonHttpHandler extends HttpHandler {
         response.setContentType("application/json");
         response.setHeader(Header.Server, Server.SERVER_VERSION);
 
-        ObjectMapper MAPPER = new ObjectMapper();
-
         // Write JSON encoded message to the response.
-        try
-        {
-          MAPPER.writeValue(response.getOutputStream(), new HelloMessage());
-        }
-        catch (IOException ioe) 
-        {
-          // do nothing
-        }
+        MAPPER.writeValue(response.getOutputStream(), new HelloMessage());
     }
 
     @Override
-    protected ExecutorService getThreadPool(Request request) {
-        return null;
+    public RequestExecutorProvider getRequestExecutorProvider() {
+        return Server.EXECUTOR_PROVIDER;
     }
 }

grizzly-bm/src/main/java/org/glassfish/grizzly/bm/PlainTextHttpHandler.java

@@ -1,8 +1,8 @@
 package org.glassfish.grizzly.bm;
 
-import java.util.concurrent.ExecutorService;
 import org.glassfish.grizzly.http.server.HttpHandler;
 import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.grizzly.http.server.RequestExecutorProvider;
 import org.glassfish.grizzly.http.server.Response;
 import org.glassfish.grizzly.http.util.Header;
 
@@ -21,7 +21,7 @@ public class PlainTextHttpHandler extends HttpHandler {
     }
 
     @Override
-    protected ExecutorService getThreadPool(Request request) {
-        return null;
+    public RequestExecutorProvider getRequestExecutorProvider() {
+        return Server.EXECUTOR_PROVIDER;
     }
 }

grizzly-bm/src/main/java/org/glassfish/grizzly/bm/RootHttpHandler.java

@@ -1,8 +1,8 @@
 package org.glassfish.grizzly.bm;
 
-import java.util.concurrent.ExecutorService;
 import org.glassfish.grizzly.http.server.HttpHandler;
 import org.glassfish.grizzly.http.server.Request;
+import org.glassfish.grizzly.http.server.RequestExecutorProvider;
 import org.glassfish.grizzly.http.server.Response;
 import org.glassfish.grizzly.http.util.DataChunk;
 
@@ -28,7 +28,7 @@ public class RootHttpHandler extends HttpHandler {
     }
     
     @Override
-    protected ExecutorService getThreadPool(Request request) {
-        return null;
+    public RequestExecutorProvider getRequestExecutorProvider() {
+        return Server.EXECUTOR_PROVIDER;
     }
 }

grizzly-bm/src/main/java/org/glassfish/grizzly/bm/Server.java

@@ -1,9 +1,9 @@
 package org.glassfish.grizzly.bm;
 
-import java.io.IOException;
 import org.glassfish.grizzly.Grizzly;
 import org.glassfish.grizzly.http.server.HttpServer;
 import org.glassfish.grizzly.http.server.NetworkListener;
+import org.glassfish.grizzly.http.server.RequestExecutorProvider;
 import org.glassfish.grizzly.nio.transport.TCPNIOTransport;
 
 /**
@@ -12,6 +12,11 @@ import org.glassfish.grizzly.nio.transport.TCPNIOTransport;
 public class Server {
     public static final String SERVER_VERSION = "Grizzly/" + Grizzly.getDotedVersion();
     
+    // The RequestExecutorProvider, which will run HTTP request processing
+    // in the same thread
+    static final RequestExecutorProvider EXECUTOR_PROVIDER =
+            new RequestExecutorProvider.SameThreadProvider();
+    
     public static void main(String[] args) throws Exception {
         final int port = args.length > 0
                 ? Integer.parseInt(args[0]) : 8080;
@@ -23,6 +28,7 @@ public class Server {
         
         // force to not initialize worker thread pool
         transport.setWorkerThreadPoolConfig(null);
+        transport.setSelectorRunnersCount(Runtime.getRuntime().availableProcessors());
         
         networkListener.getKeepAlive().setIdleTimeoutInSeconds(-1);
         networkListener.getKeepAlive().setMaxRequestsCount(-1);
@@ -44,7 +50,7 @@ public class Server {
 		Server.class.wait();
             }
         } finally {
-            httpServer.stop();
+            httpServer.shutdown();
         }
     }
 }

jetty-servlet/benchmark_config

@@ -9,9 +9,9 @@
       "approach": "Realistic",
       "classification": "Platform",
       "database": "None",
-      "framework": "Servlet-3.1",
+      "framework": "servlet",
       "language": "Java",
-      "orm": "None",
+      "orm": "Raw",
       "platform": "Jetty",
       "webserver": "Jetty",
       "os": "Linux",

plain/benchmark_config

@@ -20,7 +20,30 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "plain",
+      "display_name": "plain-linux",
+      "notes": "",
+      "versus": ""
+    },
+    "linux-servlet": {
+      "setup_file": "setup",
+      "json_url": "/web-apps/servlet/json",
+      "plaintext_url": "/web-apps/servlet/plaintext",
+      "db_url": "/web-apps/servlet/db",
+      "query_url": "/web-apps/servlet/db?queries=",
+      "update_url": "/web-apps/servlet/update?queries=",
+      "fortune_url": "/web-apps/servlet/fortunes",
+      "port": 9080,
+      "approach": "Realistic",
+      "classification": "Fullstack",
+      "database": "MySQL",
+      "framework": "plain",
+      "language": "Scala",
+      "orm": "Full",
+      "platform": "Plain",
+      "webserver": "None",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "plain-servlet-linux",
       "notes": "",
       "versus": ""
     },
@@ -43,7 +66,30 @@
       "webserver": "None",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "plain-win",
+      "display_name": "plain-windows",
+      "notes": "",
+      "versus": ""
+    },
+    "win-servlet": {
+      "setup_file": "setup",
+      "json_url": "/web-apps/servlet/json",
+      "plaintext_url": "/web-apps/servlet/plaintext",
+      "db_url": "/web-apps/servlet/db",
+      "query_url": "/web-apps/servlet/db?queries=",
+      "update_url": "/web-apps/servlet/update?queries=",
+      "fortune_url": "/web-apps/servlet/fortunes",
+      "port": 9080,
+      "approach": "Realistic",
+      "classification": "Fullstack",
+      "database": "MySQL",
+      "framework": "plain",
+      "language": "Scala",
+      "orm": "Full",
+      "platform": "Plain",
+      "webserver": "None",
+      "os": "Windows",
+      "database_os": "Linux",
+      "display_name": "plain-servlet-windows",
       "notes": "",
       "versus": ""
     }

plain/build.sbt

@@ -1,4 +1,5 @@
 import sbt._
+import Keys._
 import spray.revolver.RevolverPlugin.Revolver
 import sbtassembly.Plugin._
 import AssemblyKeys._
@@ -16,7 +17,7 @@ test in Compile := {}
 mainClass in Compile := Some("com.ibm.plain.bootstrap.Main")
 
 scalacOptions in Compile ++= Seq(
-	"-g:none",
+	"-g:vars",
 	"-encoding", "UTF-8", 
 	"-target:jvm-1.7", 
 	"-deprecation", 
@@ -38,20 +39,22 @@ libraryDependencies ++= Seq(
   "commons-io" % "commons-io" % "2.4",
   "commons-net" % "commons-net" % "3.3",
   "commons-codec" % "commons-codec" % "1.8",
+  "javax.servlet" % "javax.servlet-api" % "3.1.0",
+  "org.glassfish.web" % "javax.servlet.jsp" % "2.3.1",
   "com.googlecode.concurrentlinkedhashmap" % "concurrentlinkedhashmap-lru" % "1.3.2",
   "net.jpountz.lz4" % "lz4" % "1.2.0",
+  "net.lingala.zip4j" % "zip4j" % "1.3.1", 
   "com.fasterxml.jackson.core" % "jackson-databind" % "2.2.3",
   "com.sun.jersey" % "jersey-json" % "1.17.1",
   "org.jvnet.mimepull" % "mimepull" % "1.9.3",
-  "org.apache.derby" % "derby" % "10.10.1.1",
-  "org.apache.derby" % "derbyclient" % "10.10.1.1",
-  "com.h2database" % "h2" % "1.3.172",
-  "mysql" % "mysql-connector-java" % "5.1.26",
-  "javax.servlet" % "javax.servlet-api" % "3.1.0"
+  "mysql" % "mysql-connector-java" % "5.1.27"
 )
 
 Revolver.settings
 
-sbtassembly.Plugin.assemblySettings
-
 scalariformSettings
+
+assemblySettings
+
+test in assembly := {}
+

plain/src/main/resources/application.conf

@@ -14,6 +14,8 @@ plain {
 	http.ignore-accept-header = on
 	http.startup-servers = [ benchmark-server ]
 	jdbc.startup-connection-factories = [ benchmark-mysql ]
+	servlet.web-applications-directory = "../web-apps"
+	servlet.unpack-web-applications-directory = "../web-apps"
 }
 
 benchmark-server {
@@ -27,15 +29,17 @@ benchmark-dispatcher.routes = [
 	{ uri = db, resource-class-name = com.ibm.techempower.Db }
 	{ uri = update, resource-class-name = com.ibm.techempower.Update }
 	{ uri = fortunes, resource-class-name = com.ibm.techempower.Fortunes }
+	{ uri = "web-apps*", resource-class-name = com.ibm.plain.rest.resource.ServletResource }
 ]
 
 benchmark-mysql {
 	name = MysqlBenchmark
+	jndi-lookup-name = "jdbc/hello_world"
 	driver = mysql-5-6-12
 	datasource-settings {	
 		setUrl = "jdbc:mysql://127.0.0.1:3306/hello_world?jdbcCompliantTruncation=false&elideSetAutoCommits=false&dontTrackOpenResources=true&enableQueryTimeouts=false&useLocalSessionState=false&cachePrepStmts=true&cacheCallableStmts=true&alwaysSendSetIsolation=false&prepStmtCacheSize=1024&cacheServerConfiguration=true&prepStmtCacheSqlLimit=2048&zeroDateTimeBehavior=convertToNull&traceProtocol=false&useUnbufferedInput=false&useReadAheadInput=true&maintainTimeStats=false&useServerPrepStmts&cacheRSMetadata=true&poolPreparedStatements=true"
-		setUser = "benchmarkdbuser"
-		setPassword = "benchmarkdbpass"
+		setUser = benchmarkdbuser
+		setPassword = benchmarkdbpass
 	}	
 	min-pool-size = 8
 	max-pool-size = 128

servicestack/setup_nginx.py

@@ -50,7 +50,7 @@ def stop(logfile, errfile):
   p2 = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
   out, err = p2.communicate()
   for line in out.splitlines():
-    if 'sbin/nginx' in line:
+    if 'nginx: master' in line:
       pid = int(line.split(None, 2)[1])
       os.kill(pid, 15)
 

toolset/benchmark/benchmarker.py

@@ -284,7 +284,15 @@ class Benchmarker:
         if config == None:
           continue
 
-        tests = tests + framework_test.parse_config(config, dirname[2:], self)
+        test = framework_test.parse_config(config, dirname[2:], self)
+        # If the user specified which tests to run, then 
+        # we can skip over tests that are not in that list
+        if self.test == None:
+          tests = tests + test
+        else:
+          for atest in test:
+            if atest.name in self.test:
+              tests.append(atest)
 
     tests.sort(key=lambda x: x.name)
     return tests
@@ -457,11 +465,6 @@ class Benchmarker:
       pass
     with open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'out.txt'), 'w') as out, \
          open(os.path.join(self.latest_results_directory, 'logs', "{name}".format(name=test.name), 'err.txt'), 'w') as err:
-      # If the user specified which tests to run, then 
-      # we can skip over tests that are not in that list
-      if self.test != None and test.name not in self.test:
-        return
-
       if hasattr(test, 'skip'):
         if test.skip.lower() == "true":
           out.write("Test {name} benchmark_config specifies to skip this test. Skipping.\n".format(name=test.name))

toolset/setup/linux/installer.py

@@ -290,8 +290,8 @@ class Installer:
     #
     # Vert.x
     #
-    self.__download("http://dl.bintray.com/vertx/downloads/vert.x-2.0.2-final.tar.gz?direct=true", "vert.x-2.0.2-final.tar.gz")
-    self.__run_command("tar xzf vert.x-2.0.2-final.tar.gz")
+    self.__download("http://dl.bintray.com/vertx/downloads/vert.x-2.1M1.tar.gz?direct=true", "vert.x-2.1M1.tar.gz")
+    self.__run_command("tar xzf vert.x-2.1M1.tar.gz")
 
     #
     # Yesod

wai/setup.py

@@ -12,7 +12,7 @@ def start(args, logfile, errfile):
 
   db_host = args.database_host
   threads = str(args.max_threads)
-  subprocess.Popen("dist/build/bench/bench " + threads + " " + db_host + " +RTS -A4M -N -qg2 -I0 -G2 > /dev/null", shell=True, cwd="wai/bench", stderr=errfile, stdout=logfile)
+  subprocess.Popen("dist/build/bench/bench " + threads + " " + db_host + " +RTS -A4M -N -qg2 -I0 -G2", shell=True, cwd="wai/bench", stderr=errfile, stdout=logfile)
   return 0
 
 def stop(logfile, errfile):
@@ -22,7 +22,7 @@ def stop(logfile, errfile):
     if 'bench' in line:
       try:
         pid = int(line.split(None, 2)[1])
-        os.kill(pid, 9)
+        os.kill(pid, 15)
       except OSError:
         pass
 

yesod/setup.py

@@ -14,17 +14,17 @@ def start(args, logfile, errfile):
 
   db_host = args.database_host
   threads = str(args.max_threads)
-  subprocess.Popen("dist/build/bench/bench " + threads + " " + db_host + " +RTS -A4M -N -qg2 -I0 -G2 > /dev/null", shell=True, cwd="yesod/bench", stderr=errfile, stdout=logfile)
+  subprocess.Popen("dist/build/bench/bench " + threads + " " + db_host + " +RTS -A4M -N -qg2 -I0 -G2", shell=True, cwd="yesod/bench", stderr=errfile, stdout=logfile)
   return 0
 
-def stop(logfile):
+def stop(logfile, errfile):
   p = subprocess.Popen(['ps', 'aux'], stdout=subprocess.PIPE)
   out, err = p.communicate()
   for line in out.splitlines():
     if 'bench' in line:
       try:
         pid = int(line.split(None, 2)[1])
-        os.kill(pid, 9)
+        os.kill(pid, 15)
       except OSError:
         pass