Several Toolset Verification Updates (#2795)

* mode-verify

* update fortunes test

* fix broken queries

* fix dancer dbhost

* php dancer and fintrospect fixes

* json update

frameworks/Erlang/cowboy/src/query_handler.erl

@@ -12,18 +12,19 @@ init(_Transport, Req, []) ->
 
 handle(Req, State) ->
         random:seed(erlang:now()),
-        {JSON, Req2} = case cowboy_req:qs_val(<<"queries">>, Req) of
-    {undefined, Req1} ->
-      {result_packet, _, _, [[ID, Rand]], _} = emysql:execute(test_pool, db_stmt, [random:uniform(10000)]),
-      {[{[{<<"id">>, ID}, {<<"randomNumber">>, Rand}]}], Req1};
-    {N, Req1} ->
-      I = list_to_integer(binary_to_list(N)),
-      Res = [ {[{<<"id">>, ID}, {<<"randomNumber">>, Rand}]} || 
+    {N, Req1} = cowboy_req:qs_val(<<"queries">>, Req, <<"1">>),
+
+      I = try binary_to_integer(N) of
+            X when X > 500 -> 500;
+            X when X < 1 -> 1;
+            X -> X
+          catch error:badarg -> 1 end,
+
+      JSON = [ {[{<<"id">>, ID}, {<<"randomNumber">>, Rand}]} ||
               {result_packet, _, _, [[ID, Rand]], _} <- [emysql:execute(test_pool, db_stmt, [random:uniform(10000)]) || _ <- lists:seq(1, I) ]],
-      {Res, Req1}
-    end,
-  {ok, Req3} = cowboy_req:reply(200, [{<<"Content-Type">>, <<"application/json">>}], jiffy:encode(JSON), Req2),
-  {ok, Req3, State}.
+
+  {ok, Req2} = cowboy_req:reply(200, [{<<"Content-Type">>, <<"application/json">>}], jiffy:encode(JSON), Req1),
+  {ok, Req2, State}.
 
 terminate(_Reason, _Req, _State) ->
   ok.

frameworks/Erlang/mochiweb/src/web_handler.erl

@@ -55,5 +55,6 @@ queries(Req) ->
     case {is_number(Queries), Queries > 500} of
         {true, true} -> 500;
         {false, _}   -> 1;
+        _ when Queries < 1 -> 1;
         _ -> Queries
     end.

frameworks/Java/restexpress/src/main/java/hello/controller/MongodbController.java

@@ -24,24 +24,26 @@ public class MongodbController
 
 	public Object read(Request request, Response response)
 	{
-		// Get the count of queries to run.
-		int count = 1;
-		String value = request.getHeader("queries");
+    // Get the count of queries to run.
+    int count = 1;
+    try
+    {
+      count = Integer.parseInt(request.getHeader("queries"));
 
-		if (value != null)
-		{
-			count = Integer.parseInt(value);
-		}
-
-		// Bounds check.
-		if (count > 500)
-		{
-			count = 500;
-		}
-		else if (count < 1)
-		{
-			count = 1;
-		}
+      // Bounds check.
+      if (count > 500)
+      {
+        count = 500;
+      }
+      if (count < 1)
+      {
+        count = 1;
+      }
+    }
+    catch(NumberFormatException nfexc)
+    {
+      // do nothing
+    }
 
 		// Fetch some rows from the database.
 		final World[] worlds = new World[count];

frameworks/JavaScript/express/app.js

@@ -114,7 +114,8 @@ if (cluster.isMaster) {
     res.header('Content-Type', 'text/plain').send('Hello, World!'));
 
   app.get('/mongoose', (req, res) => {
-    let queries = isNaN(req.query.queries) ? 1 : parseInt(req.query.queries, 10);
+    let queriesRaw = parseInt(req.query.queries, 10),
+      queries = isNaN(queriesRaw) ? 1 : queriesRaw;
     const queryFunctions = [];
 
     queries = Math.min(Math.max(queries, 1), 500);

frameworks/JavaScript/ringojs/app/views.js

@@ -9,6 +9,17 @@ const fortuneTemplate = module.singleton('fortuneTemplate', function() {
 });
 
 const app = exports.app = Application();
+
+const formatQueries = function(queries) {
+  queries = parseInt(queries, 10) || 1;
+  if (isNaN(queries) || queries < 1) {
+    queries = 1;
+  } else if (queries > 500) {
+    queries = 500;
+  }
+  return queries;
+};
+
 app.configure("params", "route");
 
 app.get('/json', function() {
@@ -17,7 +28,7 @@ app.get('/json', function() {
 });
 
 app.get('/db/:queries?', function(request, queries) {
-   queries = parseInt(queries, 10) || 1;
+   queries = formatQueries(queries);
    let worlds = [];
    for (let i = 0; i < queries; i++) {
       let randId = ((Math.random() * 10000) | 0) + 1;
@@ -45,12 +56,7 @@ app.get('/plaintext', function() {
 });
 
 app.get('/updates/:queries?', function(request, queries) {
-   queries = parseInt(queries, 10) || 1;
-   if (isNaN(queries) || queries < 1) {
-      queries = 1;
-   } else if (queries > 500) {
-      queries = 500;
-   }
+   queries = formatQueries(queries);
    const worlds = [];
    for (let i = 0; i < queries; i++) {
       let randId = ((Math.random() * 10000) | 0) + 1;

frameworks/JavaScript/ringojs/ringo-main.js

@@ -20,7 +20,7 @@ exports.app = function(req) {
      try {
        connection = datasource.getConnection();
        let randId, world;
-       if (queryCount === null) {
+       if (!queryCount || isNaN(queryCount) || queryCount < 1) {
          randId = ((Math.random() * 10000) | 0) + 1;
          world = sql.query(connection, 'select * from World where World.id = ' + randId)[0];
          return {

frameworks/PHP/fuel/fuel/app/classes/controller/bench.php

@@ -12,6 +12,8 @@ class Controller_Bench extends Controller
     public function action_db()
     {
         $queries = Input::get('queries', 1);
+        $queries = is_numeric($queries) ? min(max($queries, 1), 500) : 1;
+
         $worlds = array();
 
         for($i = 0; $i < $queries; ++$i) {

frameworks/PHP/phreeze/libs/Controller/TestController.php

@@ -40,44 +40,45 @@ class TestController extends Controller
 	public function DB()
 	{
 		require_once("Model/World.php");
-
-		$id = mt_rand(1, 10000);
-		$world = $this->Phreezer->Get("World",$id);
+
+		$id = mt_rand(1, 10000);
+		$world = $this->Phreezer->Get("World",$id);
 		$this->RenderJSON($world,'',true);
 	}
 	
-	
-	/**
-	 * Test route that connects to the database and outputs
-	 * the number of rows specified in the querystring argument "queries"
-	 */
-	public function Query()
-	{
-		require_once("Model/World.php");
-	
-		// Read number of queries to run from URL parameter
+	
+	/**
+	 * Test route that connects to the database and outputs
+	 * the number of rows specified in the querystring argument "queries"
+	 */
+	public function Query()
+	{
+		require_once("Model/World.php");
+	
+		// Read number of queries to run from URL parameter
 		$query_count = RequestUtil::Get('queries',1);
-		
-		// make sure the query count paramter is in range
+        $query_count = is_numeric($query_count) ? min(max($query_count, 1), 500) : 1;
+
+        // make sure the query count paramter is in range
 		if (!is_numeric($query_count)) {
 			$query_count = 1;
 		}
 		else {
 			$query_count = max(1,min($query_count,500));
 		}
-		
-		$arr = array();
-			
-		for ($i = 0; $i < $query_count; $i++) {
-				
-			$id = mt_rand(1, 10000);
+		
+		$arr = array();
+			
+		for ($i = 0; $i < $query_count; $i++) {
+				
+			$id = mt_rand(1, 10000);
 
-			// convert the Phreezable object into a simple structure for output
-			$arr[] = $this->Phreezer->Get("World",$id)->ToObject();
-		}
-			
-		$this->RenderJSON($arr);
-	
+			// convert the Phreezable object into a simple structure for output
+			$arr[] = $this->Phreezer->Get("World",$id)->ToObject();
+		}
+			
+		$this->RenderJSON($arr);
+	
 	}
 	
 	/**
@@ -104,47 +105,48 @@ class TestController extends Controller
 		Phreezer::Sort($fortunes);
 		
 		// Render using a template
-		$this->RenderEngine = new PHPRenderEngine('templates');
+		$this->RenderEngine = new PHPRenderEngine('templates');
 		$this->Assign('fortunes',$fortunes);
 		$this->Render('TestFortunes.php');
 	}
 	
-	/**
-	 * Test for performing updates
-	 */
-	public function Updates()
+	/**
+	 * Test for performing updates
+	 */
+	public function Updates()
 	{
-		require_once("Model/World.php");
-		
-		// Read number of queries to run from URL parameter
-		$query_count = RequestUtil::Get('queries',1);
-		
-		$arr = array();
-		
-		for ($i = 0; $i < $query_count; $i++) {
-		
-			$id = mt_rand(1, 10000);
-				
+		require_once("Model/World.php");
+		
+		// Read number of queries to run from URL parameter
+		$query_count = RequestUtil::Get('queries',1);
+        $query_count = is_numeric($query_count) ? min(max($query_count, 1), 500) : 1;
+
+        $arr = array();
+		
+		for ($i = 0; $i < $query_count; $i++) {
+		
+			$id = mt_rand(1, 10000);
+				
 			$world = $this->Phreezer->Get("World",$id);
 			
 			// update the random number and persist the record
 			$world->Randomnumber = mt_rand(1, 10000);
-			$world->Save();
-				
-			// convert the Phreezable object into a simple structure for output
-			$arr[] = array('id'=>$world->Id,'randomNumber'=>$world->Randomnumber);
-		}
-		
-		$this->RenderJSON($arr);
+			$world->Save();
+				
+			// convert the Phreezable object into a simple structure for output
+			$arr[] = array('id'=>$world->Id,'randomNumber'=>$world->Randomnumber);
+		}
+		
+		$this->RenderJSON($arr);
 	}
 	
-	/**
-	 * Test for outputting plaintext
-	 */
-	public function PlainText()
+	/**
+	 * Test for outputting plaintext
+	 */
+	public function PlainText()
 	{
 		header('Content-type: text/plain');
-		echo 'Hello, World!';
+		echo 'Hello, World!';
 	}
 	
 }

frameworks/PHP/symfony2-stripped/src/Skamander/BenchmarkBundle/Controller/BenchController.php

@@ -24,8 +24,7 @@ class BenchController extends Controller
     public function dbAction(Request $request)
     {
         $queries = $request->query->getInt('queries', 1);
-        $queries = max(1, $queries);
-        $queries = min(500, $queries);
+        $queries = is_numeric($queries) ? min(max($queries, 1), 500) : 1;
 
         // possibility for enhancement is the use of SplFixedArray -> http://php.net/manual/de/class.splfixedarray.php
         $worlds = array();
@@ -46,6 +45,7 @@ class BenchController extends Controller
     public function dbRawAction(Request $request)
     {
         $queries = $request->query->getInt('queries', 1);
+        $queries = is_numeric($queries) ? min(max($queries, 1), 500) : 1;
 
         // possibility for enhancement is the use of SplFixedArray -> http://php.net/manual/de/class.splfixedarray.php
         $worlds = array();

frameworks/PHP/symfony2/src/Skamander/BenchmarkBundle/Controller/BenchController.php

@@ -24,8 +24,7 @@ class BenchController extends Controller
     public function dbAction(Request $request)
     {
         $queries = $request->query->getInt('queries', 1);
-        $queries = max(1, $queries);
-        $queries = min(500, $queries);
+        $queries = is_numeric($queries) ? min(max($queries, 1), 500) : 1;
 
         // possibility for enhancement is the use of SplFixedArray -> http://php.net/manual/de/class.splfixedarray.php
         $worlds = array();
@@ -46,6 +45,7 @@ class BenchController extends Controller
     public function dbRawAction(Request $request)
     {
         $queries = $request->query->getInt('queries', 1);
+        $queries = is_numeric($queries) ? min(max($queries, 1), 500) : 1;
 
         // possibility for enhancement is the use of SplFixedArray -> http://php.net/manual/de/class.splfixedarray.php
         $worlds = array();

frameworks/Perl/dancer/app.pl

@@ -8,7 +8,7 @@ use JSON::XS;  # Ensure that the fast implementation of the serializer is instal
 
 set serializer => 'JSON';
 
-my $dsn = "dbi:mysql:database=hello_world;host=localhost;port=3306";
+my $dsn = "dbi:mysql:database=hello_world;host=TFB-database;port=3306";
 my $dbh = DBI->connect( $dsn, 'benchmarkdbuser', 'benchmarkdbpass', {} );
 my $sth = $dbh->prepare("SELECT * FROM World where id = ?");
 
@@ -18,6 +18,9 @@ get '/json' => sub {
 
 get '/db' => sub {
     my $queries = params->{queries} || 1;
+    $queries = 1 if ( $queries !~ /^\d+$/ || $queries < 1 );
+    $queries = 500 if $queries > 500;
+    
     my @response;
     for ( 1 .. $queries ) {
         my $id = int rand 10000 + 1;

frameworks/Perl/dancer/nginx.conf

@@ -19,7 +19,7 @@ http {
   tcp_nodelay      on;
 
   upstream backendurl {
-    server unix:/home/tfb/FrameworkBenchmarks/dancer/frameworks-benchmark.sock;
+    server unix:/tmp/perl-dancer.sock;
   }
 
   server {

frameworks/Perl/dancer/setup.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-sed -i 's|localhost|'"${DBHOST}"'|g' app.pl
-sed -i 's|user .*;|user '"$(id -u -n)"';|g' nginx.conf
-sed -i 's|server unix.*frameworks-benchmark.sock;|server unix:'"${TROOT}"'/frameworks-benchmark.sock;|g' nginx.conf
-
 fw_depends perl nginx
 
 cpanm --notest --no-man-page \
@@ -17,4 +13,4 @@ cpanm --notest --no-man-page \
     
 nginx -c ${TROOT}/nginx.conf
 
-plackup -E production -s Starman --workers=${CPU_COUNT} -l ${TROOT}/frameworks-benchmark.sock -a ./app.pl &
+plackup -E production -s Starman --workers=${CPU_COUNT} -l /tmp/perl-dancer.sock -a ./app.pl &

frameworks/Scala/fintrospect/src/main/scala/DatabaseRoutes.scala

@@ -7,7 +7,7 @@ import com.twitter.finagle.mysql.{Client, IntValue, Result, ResultSet}
 import com.twitter.util.Future.collect
 import io.fintrospect.formats.Jackson.JsonFormat.{array, number, obj}
 import io.fintrospect.formats.Jackson.ResponseBuilder._
-import io.fintrospect.parameters.{ParameterSpec, Query}
+import io.fintrospect.parameters.{ParameterSpec, Query, StringValidations}
 import io.fintrospect.{RouteSpec, ServerRoutes}
 
 import scala.language.reflectiveCalls
@@ -37,7 +37,7 @@ object DatabaseRoutes {
         .map(_.map(Ok(_)).getOrElse(NotFound("")).build())
     }
 
-    val numberOfQueries = Query.optional(ParameterSpec.string().map {
+    val numberOfQueries = Query.optional(ParameterSpec.string(StringValidations.EmptyIsValid).map {
       i => Try(i.toInt).getOrElse(1).max(1).min(500)
     }, "queries")
 

toolset/benchmark/framework_test.py

@@ -384,7 +384,20 @@ class FrameworkTest:
         base_url = "http://%s:%s" % (self.benchmarker.server_host, self.port)
 
         try:
+          # Verifies headers from the server. This check is made from the
+          # App Server using Pythons requests module. Will do a second check from
+          # the client to make sure the server isn't only accepting connections
+          # from localhost on a multi-machine setup.
           results = test.verify(base_url)
+
+          # Now verify that the url is reachable from the client machine, unless
+          # we're already failing
+          if not any(result == 'fail' for (result, reason, url) in results):
+            p = subprocess.call(["ssh", "TFB-client", "curl -sSf %s" % base_url + test.get_url()], shell=False, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
+            if p is not 0:
+              results = [('fail', "Server did not respond to request from client machine.", base_url)]
+              logging.warning("""This error usually means your server is only accepting
+                requests from localhost.""")
         except ConnectionError as e:
           results = [('fail',"Server did not respond to request", base_url)]
           logging.warning("Verifying test %s for %s caused an exception: %s", test_type, self.name, e)

toolset/benchmark/test_types/query_type.py

@@ -34,10 +34,10 @@ class QueryTestType(FrameworkTestType):
         url = base_url + self.query_url
         cases = [
             ('2',   'fail'),
-            ('0',   'warn'),
-            ('foo', 'warn'),
+            ('0',   'fail'),
+            ('foo', 'fail'),
             ('501', 'warn'),
-            ('',    'warn')
+            ('',    'fail')
         ]
 
         problems = verify_query_cases(self, cases, url)

toolset/benchmark/test_types/update_type.py

@@ -27,10 +27,10 @@ class UpdateTestType(FrameworkTestType):
         url = base_url + self.update_url
         cases = [
             ('2',   'fail'),
-            ('0',   'warn'),
-            ('foo', 'warn'),
+            ('0',   'fail'),
+            ('foo', 'fail'),
             ('501', 'warn'),
-            ('',    'warn')
+            ('',    'fail')
         ]
         problems = verify_query_cases(self, cases, url)
 

toolset/benchmark/test_types/verifications.py

@@ -116,12 +116,15 @@ def verify_helloworld_object(json_object, url):
     if 'message' not in json_object:
         return [('fail', "Missing required key 'message'", url)]
     else:
-        if len(json_object) > 1:
+        json_len = len(json_object)
+        if json_len > 1:
             additional = (', ').join(
                 [k for k in json_object.keys() if k != 'message'])
             problems.append(
                 ('warn', "Too many JSON key/value pairs, consider removing: %s" % additional, url))
-
+        if json_len > 27:
+            problems.append(
+                'warn', "%s additional response byte(s) found. Consider removing unnecessary whitespace." % (json_len - 26))
         message = json_object['message']
 
         if message != 'hello, world!':
@@ -258,10 +261,10 @@ def verify_query_cases(self, cases, url):
 
     cases = [
         ('2',   'fail'),
-        ('0',   'warn'),
-        ('foo', 'warn'),
+        ('0',   'fail'),
+        ('foo', 'fail'),
         ('501', 'warn'),
-        ('',    'warn')
+        ('',    'fail')
     ]
 
     The reason for using 'warn' is generally for a case that will be allowed in the