Refactor benchmark_configs

This is a work in progress.

1. Standardize "None" as the database type for tests that don't involve a
   database.  Also, use "Linux" as the database OS type for those tests.
   That attribute is really "when do you want to run this test, given the
   current database OS?" and so we'll arbitrarily pick Linux.
2. Specify a display_name for every test so that we don't need any logic to
   come up with a default display name.
3. Add an option to run-tests.py that prints the test metadata as JSON.
   This will be used by the results website.
4. Fix some small errors / typos.

HttpListener/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "http-listener",
       "language": "C#",
       "orm": "",
       "platform": "",
       "webserver": "IIS",
       "os": "Windows",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "http-listener",
       "notes": "",
       "versus": ""
@@ -58,7 +58,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "http-listener",
       "notes": "",
       "versus": "http-listener"
     },
@@ -79,7 +79,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "http-listener",
       "notes": "",
       "versus": "http-listener"
     },

aspnet-stripped/benchmark_config

@@ -8,15 +8,15 @@
       "port": 8080,
       "approach": "Stripped",
       "classification": "Fullstack",
-      "database": "",
+      "database": "None",
       "framework": "aspnet",
       "language": "C#",
       "orm": "Raw",
       "platform": "NET",
       "webserver": "IIS",
       "os": "Windows",
-      "database_os": "",
-      "display_name": "",
+      "database_os": "Linux",
+      "display_name": "aspnet-stripped",
       "notes": "",
       "versus": ""
     },
@@ -37,7 +37,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-stripped-raw",
       "notes": "",
       "versus": ""
     },
@@ -58,7 +58,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-stripped-raw",
       "notes": "",
       "versus": ""
     },
@@ -79,7 +79,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-stripped-raw",
       "notes": "",
       "versus": ""
     },
@@ -100,7 +100,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Windows",
-      "display_name": "",
+      "display_name": "aspnet-stripped-raw",
       "notes": "",
       "versus": ""
     }

aspnet/benchmark_config

@@ -8,7 +8,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "MySQL",
+      "database": "None",
       "framework": "aspnet-mvc",
       "language": "C#",
       "orm": "Raw",
@@ -16,7 +16,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet",
       "notes": "",
       "versus": ""
     },
@@ -26,7 +26,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "MySQL",
+      "database": "None",
       "framework": "aspnet",
       "language": "C#",
       "orm": "Raw",
@@ -44,7 +44,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "MySQL",
+      "database": "None",
       "framework": "aspnet",
       "language": "C#",
       "orm": "Raw",
@@ -73,7 +73,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-mvc-raw",
       "notes": "",
       "versus": ""
     },
@@ -94,7 +94,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-mvc-raw",
       "notes": "",
       "versus": ""
     },
@@ -115,7 +115,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-mvc-raw",
       "notes": "",
       "versus": ""
     },
@@ -136,7 +136,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Windows",
-      "display_name": "",
+      "display_name": "aspnet-mvc-raw",
       "notes": "",
       "versus": ""
     },
@@ -157,7 +157,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-mvc",
       "notes": "",
       "versus": ""
     },
@@ -178,7 +178,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "aspnet-mvc",
       "notes": "",
       "versus": ""
     },
@@ -199,7 +199,7 @@
       "webserver": "IIS",
       "os": "Windows",
       "database_os": "Windows",
-      "display_name": "",
+      "display_name": "aspnet-mvc",
       "notes": "",
       "versus": ""
     },
@@ -210,7 +210,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "MySQL",
+      "database": "None",
       "framework": "aspnet-mvc",
       "language": "C#",
       "orm": "Raw",
@@ -228,7 +228,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "MySQL",
+      "database": "None",
       "framework": "aspnet",
       "language": "C#",
       "orm": "Raw",
@@ -246,7 +246,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "MySQL",
+      "database": "None",
       "framework": "aspnet",
       "language": "C#",
       "orm": "Raw",
@@ -275,7 +275,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "aspnet-mvc-mono",
+      "display_name": "aspnet-mvc-mono-raw",
       "notes": "",
       "versus": ""
     },
@@ -296,7 +296,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "aspnet-mvc-mono",
+      "display_name": "aspnet-mvc-mono-raw",
       "notes": "",
       "versus": ""
     },
@@ -317,7 +317,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "aspnet-mvc-mono",
+      "display_name": "aspnet-mvc-mono-raw",
       "notes": "",
       "versus": ""
     },

beego/benchmark_config

@@ -7,7 +7,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "MySQL",
+      "database": "None",
       "framework": "beego",
       "language": "Go",
       "orm": "Raw",
@@ -15,7 +15,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "beego",
       "notes": "",
       "versus": "go"
     }

bottle/benchmark_config

@@ -20,7 +20,7 @@
       "webserver": "Gunicorn",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "bottle",
       "notes": "",
       "versus": "wsgi"
     },
@@ -43,7 +43,7 @@
       "webserver": "Gunicorn",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "bottle-py3",
       "notes": "",
       "versus": "wsgi"
     },
@@ -66,7 +66,7 @@
       "webserver": "Gunicorn",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "bottle-pypy",
       "notes": "",
       "versus": "wsgi"
     },
@@ -87,7 +87,7 @@
       "webserver": "Gunicorn",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "bottle-raw",
       "notes": "",
       "versus": "wsgi"
     }

cake/benchmark_config

@@ -19,7 +19,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "cake",
       "notes": "",
       "versus": "php"
     }

compojure/benchmark_config

@@ -19,7 +19,7 @@
       "webserver": "Resin",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "compojure",
       "notes": "",
       "versus": "servlet"
     },
@@ -38,7 +38,7 @@
       "webserver": "Resin",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "compojure-raw",
       "notes": "",
       "versus": "servlet"
     }

cowboy/benchmark_config

@@ -17,7 +17,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "cowboy",
       "notes": "",
       "versus": ""
   }}]

cpoll_cppsp/benchmark_config

@@ -8,7 +8,7 @@
       "port": 16969,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "MySQL",
+      "database": "None",
       "framework": "cpoll-cppsp",
       "language": "C++",
       "orm": "Raw",
@@ -16,7 +16,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "cpoll-cppsp",
       "notes": "",
       "versus": ""
     },
@@ -37,7 +37,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "cpoll-cppsp-raw",
       "notes": "",
       "versus": ""
     },
@@ -56,7 +56,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "cpoll-cppsp-raw",
       "notes": "",
       "versus": ""
     },

dancer/benchmark_config

@@ -7,7 +7,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "MySQL",
+      "database": "None",
       "framework": "dancer",
       "language": "Perl",
       "orm": "Full",
@@ -15,7 +15,7 @@
       "webserver": "Starman",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "dancer",
       "notes": "",
       "versus": ""
     },
@@ -34,7 +34,7 @@
       "webserver": "Starman",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "dancer",
       "notes": "",
       "versus": ""
     }

dart-start/benchmark_config

@@ -20,7 +20,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "start",
       "notes": "",
       "versus": "dart"
     },
@@ -41,7 +41,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "start",
       "notes": "",
       "versus": "dart"
     }

dart-stream/benchmark_config

@@ -20,7 +20,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "stream",
       "notes": "",
       "versus": "dart"
     },
@@ -41,7 +41,7 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "stream",
       "notes": "",
       "versus": "dart"
     }

dart/benchmark_config

@@ -20,9 +20,9 @@
       "webserver": "nginx",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "dart",
       "notes": "",
-      "versus": ""
+      "versus": "dart"
     }
   }]
 }

django/benchmark_config

@@ -19,7 +19,7 @@
       "webserver": "Gunicorn",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "django",
       "notes": "",
       "versus": "wsgi"
     },
@@ -63,7 +63,7 @@
       "webserver": "Gunicorn",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "django",
       "notes": "",
       "versus": "wsgi"
     }

dropwizard/benchmark_config

@@ -17,7 +17,7 @@
       "webserver": "Jetty",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "dropwizard",
       "notes": "",
       "versus": ""
     }

elli/benchmark_config

@@ -17,7 +17,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "elli",
       "notes": "",
       "versus": ""
   }}]

express/benchmark_config

@@ -7,15 +7,15 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "express",
       "language": "JavaScript",
       "orm": "Full",
       "platform": "Node.js",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
-      "display_name": "",
+      "database_os": "Linux",
+      "display_name": "express",
       "notes": "",
       "versus": "node"
     },
@@ -35,7 +35,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "express",
       "notes": "",
       "versus": "node"
     },
@@ -56,7 +56,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "express",
       "notes": "",
       "versus": "node"
     }

falcon/benchmark_config

@@ -8,15 +8,15 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "falcon",
       "language": "Python",
       "orm": "Raw",
       "platform": "wsgi",
       "webserver": "Gunicorn",
       "os": "Linux",
-      "database_os": "",
-      "display_name": "",
+      "database_os": "Linux",
+      "display_name": "falcon",
       "notes": "",
       "versus": "wsgi"
     },
@@ -27,14 +27,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "falcon",
       "language": "Python",
       "orm": "Raw",
       "platform": "wsgi",
       "webserver": "Gunicorn",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "falcon-py3",
       "notes": "",
       "versus": "wsgi"
@@ -46,14 +46,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "falcon",
       "language": "Python",
       "orm": "Raw",
       "platform": "wsgi",
       "webserver": "Gunicorn",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "falcon-pypy",
       "notes": "",
       "versus": "wsgi"

falcore/benchmark_config

@@ -20,7 +20,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "falcore",
       "notes": "",
       "versus": "go"
     }

finagle/benchmark_config

@@ -17,7 +17,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "finagle",
       "notes": "",
       "versus": "netty"
     }

go/benchmark_config

@@ -20,7 +20,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "go",
       "notes": "",
       "versus": "go"
     }

grails/benchmark_config

@@ -17,7 +17,7 @@
       "webserver": "Resin",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "",
+      "display_name": "grails",
       "notes": "",
       "versus": "servlet"
     }

grizzly-bm/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "grizzly-bm",
       "language": "Java",
       "orm": "Raw",
       "platform": "Servlet",
       "webserver": "Grizzly",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "grizzly-bm",
       "notes": "",
       "versus": "servlet"

hapi/benchmark_config

@@ -7,7 +7,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "MySQL",
+      "database": "None",
       "framework": "hapi",
       "language": "JavaScript",
       "orm": "Raw",

kelp/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "",
+      "database": "None",
       "framework": "kelp",
       "language": "Perl",
       "orm": "Full",
       "platform": "Plack",
       "webserver": "Starman",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "kelp",
       "notes": "",
       "versus": ""

mojolicious/benchmark_config

@@ -7,7 +7,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "MySQL",
+      "database": "None",
       "framework": "mojolicious",
       "language": "Perl",
       "orm": "Full",

netty/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "netty",
       "language": "Java",
       "orm": "Raw",
       "platform": "Netty",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "netty",
       "notes": "",
       "versus": "netty"

nodejs/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "nodejs",
       "language": "JavaScript",
       "orm": "Raw",
       "platform": "Node.js",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "nodejs",
       "notes": "",
       "versus": "nodejs"

onion/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "onion",
       "language": "C",
       "orm": "Raw",
       "platform": "Onion",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "onion",
       "notes": "",
       "versus": ""

php-codeigniter/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "",
+      "database": "None",
       "framework": "codeigniter",
       "language": "PHP",
       "orm": "Raw",
       "platform": "PHP-FPM",
       "webserver": "nginx",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "codeigniter",
       "notes": "",
       "versus": "php"

php-kohana/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "",
+      "database": "None",
       "framework": "kohana",
       "language": "PHP",
       "orm": "Raw",
       "platform": "PHP-FPM",
       "webserver": "nginx",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "kohana",
       "notes": "",
       "versus": "php"

php-laravel/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "",
+      "database": "None",
       "framework": "laravel",
       "language": "PHP",
       "orm": "Raw",
       "platform": "PHP-FPM",
       "webserver": "nginx",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "laravel",
       "notes": "",
       "versus": "php"

play-java/benchmark_config

@@ -17,7 +17,7 @@
       "webserver": "None",
       "os": "Linux",
       "database_os": "Linux",
-      "display_name": "play",
+      "display_name": "play-java",
       "notes": "",
       "versus": "netty"
     }

rack/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "rack",
       "language": "Ruby",
       "orm": "Full",
       "platform": "Rack",
       "webserver": "Unicorn",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "rack",
       "notes": "",
       "versus": ""
@@ -27,14 +27,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "rack",
       "language": "Ruby",
       "orm": "Full",
       "platform": "JRuby",
       "webserver": "Resin",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "rack",
       "notes": "",
       "versus": ""

restexpress/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "rest-express",
       "language": "Java",
       "orm": "Raw",
       "platform": "Netty",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "rest-express",
       "notes": "",
       "versus": "netty"

ringojs/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "ringo",
       "language": "JavaScript",
       "orm": "Raw",
       "platform": "Jetty",
       "webserver": "Jetty",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "ringojs",
       "notes": "",
       "versus": ""

scalatra/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "scalatra",
       "language": "Scala",
       "orm": "Full",
       "platform": "Servlet",
       "webserver": "Resin",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "scalatra",
       "notes": "",
       "versus": "servlet"

servicestack/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "servicestack",
       "language": "C#",
       "orm": "Raw",
       "platform": "NET",
       "webserver": "IIS",
       "os": "Windows",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "servicestack",
       "notes": "",
       "versus": ""
@@ -90,14 +90,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "servicestack",
       "language": "C#",
       "orm": "Raw",
       "platform": "Mono",
       "webserver": "nginx",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "servicestack",
       "notes": "",
       "versus": ""
@@ -172,14 +172,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "servicestack",
       "language": "C#",
       "orm": "Raw",
       "platform": "Mono",
       "webserver": "XSP",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "servicestack",
       "notes": "",
       "versus": ""

servlet/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "servlet",
       "language": "Java",
       "orm": "Raw",
       "platform": "Servlet",
       "webserver": "Resin",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "servlet",
       "notes": "",
       "versus": "servlet"

spray/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "spray",
       "language": "Scala",
       "orm": "Raw",
       "platform": "Spray",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "spray",
       "notes": "",
       "versus": ""

toolset/benchmark/benchmarker.py

@@ -31,6 +31,21 @@ class Benchmarker:
   ############################################################
   # End run_list_tests
   ############################################################
+
+  ############################################################
+  # Prints the metadata for all the available tests
+  ############################################################
+  def run_list_test_metadata(self):
+    all_tests = self.__gather_tests()
+
+    with open(os.path.join(self.full_results_directory(), "test_metadata.json"), "w") as f:
+      f.write(json.dumps(all_tests))
+
+    self.__finish()
+
+  ############################################################
+  # End run_list_test_metadata
+  ############################################################
   
   ############################################################
   # parse_timestamp
@@ -47,7 +62,7 @@ class Benchmarker:
     self.__finish()
 
   ############################################################
-  # End run_list_tests
+  # End parse_timestamp
   ############################################################
 
   ############################################################
@@ -309,7 +324,7 @@ class Benchmarker:
   ############################################################
   def __run_tests(self, tests):
     for test in tests:
-      if test.application_os != self.application_os or test.database_os != self.database_os:
+      if test.os.lower() != self.os.lower() or test.database_os.lower() != self.database_os.lower():
         # the operating system requirements of this test for the
 		# application server or the database server don't match
 		# our current environment

toolset/benchmark/framework_test.py

@@ -510,7 +510,7 @@ class FrameworkTest:
     self.benchmarker = benchmarker
     self.__dict__.update(args)
 
-    # ensure diretory has __init__.py file so that we can use it as a Python package
+    # ensure directory has __init__.py file so that we can use it as a Python package
     if not os.path.exists(os.path.join(directory, "__init__.py")):
       open(os.path.join(directory, "__init__.py"), 'w').close()
 

toolset/run-tests.py

@@ -37,7 +37,7 @@ parser.add_argument('--exclude', nargs='+', help='names of tests to exclude')
 parser.add_argument('--type', choices=['all', 'json', 'db', 'query', 'fortune', 'update', 'plaintext'], default='all', help='which type of test to run')
 parser.add_argument('-m', '--mode', choices=['benchmark', 'verify'], default='benchmark', help='verify mode will only start up the tests, curl the urls and shutdown')
 parser.add_argument('--list-tests', action='store_true', default=False, help='lists all the known tests that can run')
-parser.add_argument('--next-sort', action='store_true', default=False, help='displays the next value that can be used as a sort value')
+parser.add_argument('--list-test-metadata', action='store_true', default=False, help='writes all the test metadata as a JSON file in the results directory')
 parser.add_argument('--max-concurrency', default=256, help='the maximum concurrency that the tests will run at. The query tests will run at this concurrency', type=int)
 parser.add_argument('--max-queries', default=20, help='The maximum number of queries to run during the query test', type=int)
 parser.add_argument('--query-interval', default=5, type=int)
@@ -56,8 +56,8 @@ benchmarker = Benchmarker(vars(args))
 # Run the benchmarker in the specified mode
 if benchmarker.list_tests:
   benchmarker.run_list_tests()
-elif benchmarker.next_sort:
-  benchmarker.next_sort_value()
+elif benchmarker.list_test_metadata:
+  benchmarker.run_list_test_metadata()
 elif benchmarker.parse != None:
   benchmarker.parse_timestamp()
 else:

tornado/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realitic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "tornado",
       "language": "Python",
       "orm": "Raw",
       "platform": "Tornado",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "tornado",
       "notes": "",
       "versus": ""

undertow/benchmark_config

@@ -9,14 +9,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "undertow",
       "language": "Java",
       "orm": "Raw",
       "platform": "Undertow",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "undertow",
       "notes": "",
       "versus": ""

uwsgi/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "uwsgi",
       "language": "Python",
       "orm": "Raw",
       "platform": "wsgi",
       "webserver": "uwsgi",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "uwsgi",
       "notes": "",
       "versus": ""

wai/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8000,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "MySQL",
+      "database": "None",
       "framework": "wai",
       "language": "Haskell",
       "orm": "Full",
       "platform": "Wai",
       "webserver": "Warp",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "wai",
       "notes": "",
       "versus": "wai"

web-simple/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "",
+      "database": "None",
       "framework": "web-simple",
       "language": "Perl",
       "orm": "Raw",
       "platform": "Plack",
       "webserver": "Starman",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "web-simple",
       "notes": "",
       "versus": ""

webgo/benchmark_config

@@ -7,14 +7,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Micro",
-      "database": "MySQL",
+      "database": "None",
       "framework": "webgo",
       "language": "Go",
       "orm": "Full",
       "platform": "Go",
       "webserver": "None",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "webgo",
       "notes": "",
       "versus": "goo"

wsgi/benchmark_config

@@ -8,14 +8,14 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Platform",
-      "database": "",
+      "database": "None",
       "framework": "wsgi",
       "language": "Python",
       "orm": "Raw",
       "platform": "wsgi",
       "webserver": "Gunicorn",
       "os": "Linux",
-      "database_os": "",
+      "database_os": "Linux",
       "display_name": "wsgi",
       "notes": "",
       "versus": "wsgi"