Added 'skip' attribute in benchmark_config to specify a test to skip. Added back in cowboy and elli with this new attribute added.

cowboy/.gitignore

@@ -0,0 +1,5 @@
+erl_crash.dump
+__init__.py
+deps
+ebin
+*.deb

cowboy/Makefile

@@ -0,0 +1,4 @@
+all:
+	./rebar get-deps
+	./rebar compile
+	erl -pa ebin deps/*/ebin -s hello_world -noshell -detached

cowboy/README.md

@@ -0,0 +1,34 @@
+# Cowboy Benchmarking Test
+
+This is the Cowboy portion of a [benchmarking test suite](../) comparing a variety of web development platforms.
+
+### JSON Encoding Test
+
+* [JSON test controller](src/json_handler.erl)
+
+
+### Data-Store/Database Mapping Test
+Uses the db abstraction class from Kohana
+
+* [DB test controller](src/db_handler.erl)
+
+
+## Infrastructure Software Versions
+The tests were run with:
+
+* [Cowboy 0.8.3](https://github.com/extend/cowboy)
+* [Erlang R16B](http://www.erlang.org/)
+* [MySQL 5.5.29](https://dev.mysql.com/)
+
+## Test URLs
+### JSON Encoding Test
+
+http://localhost/json
+
+### Data-Store/Database Mapping Test
+
+http://localhost/db
+
+### Variable Query Test
+    
+http://localhost/db?queries=2

cowboy/benchmark_config

@@ -0,0 +1,25 @@
+{
+  "framework": "cowboy",
+  "tests": [{
+    "default": {
+      "setup_file": "setup_erlang",
+      "json_url": "/json",
+      "db_url": "/db",
+      "query_url": "/db?queries=",
+      "port": 8080,
+      "approach": "Realistic",
+      "classification": "Platform",
+      "database": "MySQL",
+      "framework": "cowboy",
+      "language": "Erlang",
+      "orm": "Raw",
+      "platform": "Cowboy",
+      "webserver": "None",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "cowboy",
+      "notes": "",
+      "versus": "",
+      "skip": "true"
+  }}]
+}

cowboy/rebar.config

@@ -0,0 +1,7 @@
+{deps,
+ [
+  {jiffy, ".*", {git, "https://github.com/davisp/jiffy.git", {tag, "0.8.3"}}},
+  {mimetypes, ".*", {git, "http://github.com/spawngrid/mimetypes.git", {branch, master}}},
+  {emysql, ".*", {git, "https://github.com/Eonblast/Emysql.git"}},
+  {cowboy, ".*", {git, "https://github.com/extend/cowboy.git", {tag, "0.8.3"}}}
+ ]}.

cowboy/setup_erlang.py

@@ -0,0 +1,20 @@
+import subprocess
+import sys
+import setup_util
+
+def start(args):
+  setup_util.replace_text("cowboy/src/hello_world_app.erl", "\"benchmarkdbpass\", \".*\", 3306", "\"benchmarkdbpass\", \"" + args.database_host + "\", 3306")
+
+  try:
+    subprocess.check_call("./rebar get-deps", shell=True, cwd="cowboy")
+    subprocess.check_call("./rebar compile", shell=True, cwd="cowboy")
+    subprocess.check_call("erl -pa ebin deps/*/ebin +sbwt very_long +swt very_low -s hello_world -noshell -detached", shell=True, cwd="cowboy")
+    return 0
+  except subprocess.CalledProcessError:
+    return 1
+def stop():
+  try:
+    subprocess.check_call("killall beam.smp", shell=True, cwd="/usr/bin")
+    return 0
+  except subprocess.CalledProcessError:
+    return 1

cowboy/source_code

@@ -0,0 +1,7 @@
+./cowboy/src/
+./cowboy/src/hello_world_app.erl
+./cowboy/src/db_handler.erl
+./cowboy/src/hello_world_sup.erl
+./cowboy/src/hello_world.app.src
+./cowboy/src/json_handler.erl
+./cowboy/src/hello_world.erl

cowboy/src/db_handler.erl

@@ -0,0 +1,29 @@
+%% Feel free to use, reuse and abuse the code in this file.
+
+%% @doc Hello world handler.
+-module(db_handler).
+
+-export([init/3]).
+-export([handle/2]).
+-export([terminate/3]).
+
+init(_Transport, Req, []) ->
+	{ok, Req, undefined}.
+
+handle(Req, State) ->
+        random:seed(erlang:now()),
+        {JSON, Req2} = case cowboy_req:qs_val(<<"queries">>, Req) of
+		{undefined, Req1} ->
+			{result_packet, _, _, [[ID, Rand]], _} = emysql:execute(test_pool, db_stmt, [random:uniform(10000)]),
+			{[{[{<<"id">>, ID}, {<<"randomNumber">>, Rand}]}], Req1};
+		{N, Req1} ->
+			I = list_to_integer(binary_to_list(N)),
+			Res = [ {[{<<"id">>, ID}, {<<"randomNumber">>, Rand}]} || 
+			        {result_packet, _, _, [[ID, Rand]], _} <- [emysql:execute(test_pool, db_stmt, [random:uniform(10000)]) || _ <- lists:seq(1, I) ]],
+			{Res, Req1}
+		end,
+	{ok, Req3} = cowboy_req:reply(200, [{<<"Content-Type">>, <<"application/json">>}], jiffy:encode(JSON), Req2),
+	{ok, Req3, State}.
+
+terminate(_Reason, _Req, _State) ->
+	ok.

cowboy/src/hello_world.app.src

@@ -0,0 +1,15 @@
+%% Feel free to use, reuse and abuse the code in this file.
+
+{application, hello_world, [
+	{description, "Cowboy Hello World example."},
+	{vsn, "1"},
+	{modules, []},
+	{registered, []},
+	{applications, [
+		kernel,
+		stdlib,
+		cowboy
+	]},
+	{mod, {hello_world_app, []}},
+	{env, []}
+]}.

cowboy/src/hello_world.erl

@@ -0,0 +1,14 @@
+%% Feel free to use, reuse and abuse the code in this file.
+
+-module(hello_world).
+
+%% API.
+-export([start/0]).
+
+%% API.
+
+start() ->
+	ok = application:start(crypto),
+	ok = application:start(ranch),
+	ok = application:start(cowboy),
+	ok = application:start(hello_world).

cowboy/src/hello_world_app.erl

@@ -0,0 +1,32 @@
+%% Feel free to use, reuse and abuse the code in this file.
+
+%% @private
+-module(hello_world_app).
+-behaviour(application).
+
+%% API.
+-export([start/2]).
+-export([stop/1]).
+
+%% API.
+
+start(_Type, _Args) ->
+        crypto:start(),
+        application:start(emysql),
+        emysql:add_pool(test_pool, 5000,
+          "benchmarkdbuser", "benchmarkdbpass", "localhost", 3306,
+          "hello_world", utf8),
+	emysql:prepare(db_stmt, <<"SELECT * FROM World where id = ?">>),
+	Dispatch = cowboy_router:compile([
+		{'_', [
+			{"/json", json_handler, []},
+			{"/db", db_handler, []}
+		]}
+	]),
+	{ok, _} = cowboy:start_http(http, 5000, [{port, 8080}], [
+		{env, [{dispatch, Dispatch}]}
+	]),
+	hello_world_sup:start_link().
+
+stop(_State) ->
+	ok.

cowboy/src/hello_world_sup.erl

@@ -0,0 +1,23 @@
+%% Feel free to use, reuse and abuse the code in this file.
+
+%% @private
+-module(hello_world_sup).
+-behaviour(supervisor).
+
+%% API.
+-export([start_link/0]).
+
+%% supervisor.
+-export([init/1]).
+
+%% API.
+
+-spec start_link() -> {ok, pid()}.
+start_link() ->
+	supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+%% supervisor.
+
+init([]) ->
+	Procs = [],
+	{ok, {{one_for_one, 10, 10}, Procs}}.

cowboy/src/json_handler.erl

@@ -0,0 +1,18 @@
+%% Feel free to use, reuse and abuse the code in this file.
+
+%% @doc Hello world handler.
+-module(json_handler).
+
+-export([init/3]).
+-export([handle/2]).
+-export([terminate/3]).
+
+init(_Transport, Req, []) ->
+	{ok, Req, undefined}.
+
+handle(Req, State) ->
+	{ok, Req2} = cowboy_req:reply(200, [{<<"Content-Type">>, <<"application/json">>}], jiffy:encode({[{<<"message">>, <<"Hello, World!">>}]}), Req),
+	{ok, Req2, State}.
+
+terminate(_Reason, _Req, _State) ->
+	ok.

elli/.gitignore

@@ -0,0 +1,5 @@
+erl_crash.dump
+__init__.py
+deps
+ebin
+*.deb

elli/README.md

@@ -0,0 +1,33 @@
+# Elli Benchmarking Test
+
+This is the Elli portion of a [benchmarking test suite](../) comparing a variety of web development platforms.
+
+### JSON Encoding Test
+
+* [JSON test controller](src/elli_bench_cb.erl)
+
+
+### Data-Store/Database Mapping Test
+
+* [DB test controller](src/elli_bench_cb.erl)
+
+
+## Infrastructure Software Versions
+The tests were run with:
+
+* [Elli](git://github.com/knutin/elli)
+* [Erlang R16B](http://www.erlang.org/)
+* [MySQL 5.5.29](https://dev.mysql.com/)
+
+## Test URLs
+### JSON Encoding Test
+
+http://localhost/json
+
+### Data-Store/Database Mapping Test
+
+http://localhost/db
+
+### Variable Query Test
+    
+http://localhost/db?queries=2

elli/benchmark_config

@@ -0,0 +1,25 @@
+{
+  "framework": "elli",
+  "tests": [{
+    "default": {
+      "setup_file": "setup_erlang",
+      "json_url": "/json",
+      "db_url": "/db",
+      "query_url": "/db?queries=",
+      "port": 8080,
+      "approach": "Realistic",
+      "classification": "Platform",
+      "database": "MySQL",
+      "framework": "elli",
+      "language": "Erlang",
+      "orm": "Raw",
+      "platform": "elli",
+      "webserver": "None",
+      "os": "Linux",
+      "database_os": "Linux",
+      "display_name": "elli",
+      "notes": "",
+      "versus": "",
+      "skip": "true"
+  }}]
+}

elli/rebar.config

@@ -0,0 +1,6 @@
+{deps,
+ [
+  {jiffy, ".*", {git, "https://github.com/davisp/jiffy.git", {tag, "0.8.3"}}},
+  {emysql, ".*", {git, "https://github.com/Eonblast/Emysql.git"}},
+  {elli, "", {git, "git://github.com/knutin/elli.git"}}
+ ]}.

elli/setup_erlang.py

@@ -0,0 +1,21 @@
+import subprocess
+import sys
+import setup_util
+
+def start(args):
+  setup_util.replace_text("elli/src/elli_bench_sup.erl", "\"benchmarkdbpass\", \".*\", 3306", "\"benchmarkdbpass\", \"" + args.database_host + "\", 3306")
+  
+  try:
+    subprocess.check_call("./rebar get-deps", shell=True, cwd="elli")
+    subprocess.check_call("./rebar compile", shell=True, cwd="elli")
+    # adding +K true seemed to actually slow performance
+    subprocess.check_call("erl -pa ebin deps/*/ebin +sbwt very_long +swt very_low -s elli_bench -noshell -detached", shell=True, cwd="elli")
+    return 0
+  except subprocess.CalledProcessError:
+    return 1
+def stop():
+  try:
+    subprocess.check_call("killall beam.smp", shell=True, cwd="/usr/bin")
+    return 0
+  except subprocess.CalledProcessError:
+    return 1

elli/source_code

@@ -0,0 +1,6 @@
+./elli/src/
+./elli/src/elli_bench.app.src
+./elli/src/elli_bench_cb.erl
+./elli/src/elli_bench_sup.erl
+./elli/src/elli_bench.erl
+./elli/src/elli_bench_app.erl

elli/src/elli_bench.app.src

@@ -0,0 +1,12 @@
+{application, elli_bench,
+ [
+  {description, ""},
+  {vsn, "1"},
+  {registered, []},
+  {applications, [
+                  kernel,
+                  stdlib
+                 ]},
+  {mod, { elli_bench_app, []}},
+  {env, []}
+ ]}.

elli/src/elli_bench.erl

@@ -0,0 +1,5 @@
+-module(elli_bench).
+-export([start/0]).
+
+start() ->
+	application:start(elli_bench).

elli/src/elli_bench_app.erl

@@ -0,0 +1,16 @@
+-module(elli_bench_app).
+
+-behaviour(application).
+
+%% Application callbacks
+-export([start/2, stop/1]).
+
+%% ===================================================================
+%% Application callbacks
+%% ===================================================================
+
+start(_StartType, _StartArgs) ->
+    elli_bench_sup:start_link().
+
+stop(_State) ->
+    ok.

elli/src/elli_bench_cb.erl

@@ -0,0 +1,36 @@
+-module(elli_bench_cb).
+-export([handle/2, handle_event/3]).
+
+-include_lib("elli/include/elli.hrl").
+-behaviour(elli_handler).
+
+handle(Req, _Args) ->
+    %% Delegate to our handler function
+    handle(Req#req.method, elli_request:path(Req), Req).
+
+handle('GET',[<<"json">>], _Req) ->
+    %% Reply with a normal response. 'ok' can be used instead of '200'
+    %% to signal success.
+    {ok, [{<<"Content-Type">>, <<"application/json">>}], jiffy:encode({[{<<"message">>, <<"Hello, World!">>}]})};
+
+handle('GET',[<<"db">>], Req) ->
+        random:seed(erlang:now()),
+        JSON = case elli_request:get_arg(<<"queries">>, Req) of
+		undefined ->
+			{result_packet, _, _, [[ID, Rand]], _} = emysql:execute(test_pool, db_stmt, [random:uniform(10000)]),
+			[{[{<<"id">>, ID}, {<<"randomNumber">>, Rand}]}];
+		N ->
+			I = list_to_integer(binary_to_list(N)),
+			Res = [ {[{<<"id">>, ID}, {<<"randomNumber">>, Rand}]} || 
+			        {result_packet, _, _, [[ID, Rand]], _} <- [emysql:execute(test_pool, db_stmt, [random:uniform(10000)]) || _ <- lists:seq(1, I) ]],
+			Res
+		end,
+    {ok, [{<<"Content-Type">>, <<"application/json">>}], jiffy:encode(JSON)};
+
+handle(_, _, _Req) ->
+    {404, [], <<"Not Found">>}.
+
+%% @doc: Handle request events, like request completed, exception
+%% thrown, client timeout, etc. Must return 'ok'.
+handle_event(_Event, _Data, _Args) ->
+    ok.

elli/src/elli_bench_sup.erl

@@ -0,0 +1,42 @@
+-module(elli_bench_sup).
+
+-behaviour(supervisor).
+
+%% API
+-export([start_link/0]).
+
+%% Supervisor callbacks
+-export([init/1]).
+
+%% Helper macro for declaring children of supervisor
+-define(CHILD(I, Type), {I, {I, start_link, []}, permanent, 5000, Type, [I]}).
+
+%% ===================================================================
+%% API functions
+%% ===================================================================
+
+start_link() ->
+    supervisor:start_link({local, ?MODULE}, ?MODULE, []).
+
+%% ===================================================================
+%% Supervisor callbacks
+%% ===================================================================
+
+init([]) ->
+    crypto:start(),
+    application:start(emysql),
+    emysql:add_pool(test_pool, 5000,
+       "benchmarkdbuser", "benchmarkdbpass", "localhost", 3306,
+       "hello_world", utf8),
+    emysql:prepare(db_stmt, <<"SELECT * FROM World where id = ?">>),
+    ElliOpts = [{callback, elli_bench_cb}, {port, 8080}],
+    ElliSpec = {
+        fancy_http,
+        {elli, start_link, [ElliOpts]},
+        permanent,
+        5000,
+        worker,
+        [elli]},
+
+    {ok, { {one_for_one, 5, 10}, [ElliSpec]} }.
+

toolset/benchmark/benchmarker.py

@@ -430,6 +430,11 @@ class Benchmarker:
       if self.test != None and test.name not in self.test:
         return
 
+      if hasattr(test, 'skip'):
+        if test.skip.lower() == "true":
+          logging.info("Test %s benchmark_config specifies to skip this test. Skipping.", test.name)
+          return
+
       if test.os.lower() != self.os.lower() or test.database_os.lower() != self.database_os.lower():
         # the operating system requirements of this test for the
         # application server or the database server don't match