Merge pull request #1126 from hamiltont/listargs-1123

Toolset: Allow arbitrary argument sequences

benchmark.cfg.example

@@ -15,18 +15,16 @@ install_strategy=unified
 install_only=False
 list_test_metadata=False
 list_tests=False
-max_concurrency=256
-max_queries=20
-max_threads=8
+concurrency_levels=[8, 16, 32, 64, 128, 256]
+query_levels=[1, 5,10,15,20]
+threads=8
 mode=benchmark
 name=ec2
 os=linux
 parse=None
 password_prompt=False
-query_interval=5
 server_host=localhost
 sleep=60
-starting_concurrency=8
 test=None
 type=all
 verbose=True

toolset/benchmark/benchmarker.py

@@ -930,25 +930,6 @@ class Benchmarker:
       self.timestamp = self.parse
     else:
       self.timestamp = time.strftime("%Y%m%d%H%M%S", time.localtime())
-
-    # Setup the concurrency levels array. This array goes from
-    # starting_concurrency to max concurrency, doubling each time
-    self.concurrency_levels = []
-    concurrency = self.starting_concurrency
-    while concurrency <= self.max_concurrency:
-      self.concurrency_levels.append(concurrency)
-      concurrency = concurrency * 2
-
-    # Setup query interval array
-    # starts at 1, and goes up to max_queries, using the query_interval
-    self.query_intervals = []
-    queries = 1
-    while queries <= self.max_queries:
-      self.query_intervals.append(queries)
-      if queries == 1:
-        queries = 0
-
-      queries = queries + self.query_interval
     
     # Load the latest data
     #self.latest = None
@@ -982,7 +963,7 @@ class Benchmarker:
       self.results = dict()
       self.results['name'] = self.name
       self.results['concurrencyLevels'] = self.concurrency_levels
-      self.results['queryIntervals'] = self.query_intervals
+      self.results['queryIntervals'] = self.query_levels
       self.results['frameworks'] = [t.name for t in self.__gather_tests]
       self.results['duration'] = self.duration
       self.results['rawData'] = dict()

toolset/benchmark/framework_test.py

@@ -52,7 +52,7 @@ class FrameworkTest:
     echo ""
     ntpdate -s pool.ntp.org
 
-    for c in {interval}
+    for c in {levels}
     do
       echo ""
       echo "---------------------------------------------------------"
@@ -97,7 +97,7 @@ class FrameworkTest:
     echo ""
     ntpdate -s pool.ntp.org
 
-    for c in {interval}
+    for c in {levels}
     do
       echo ""
       echo "---------------------------------------------------------"
@@ -433,7 +433,7 @@ class FrameworkTest:
                 # rawData["averageStats"] = self.__calculate_average_stats(test_stats)
                 stats.append(test_stats)
       with open(self.benchmarker.stats_file(self.name, test_type) + ".json", "w") as stats_file:
-        json.dump(stats, stats_file)
+        json.dump(stats, stats_file, indent=2)
 
 
       return results

toolset/benchmark/utils.py

@@ -33,6 +33,7 @@ def gather_tests(include = [], exclude=[], benchmarker=None):
   
   # Setup default Benchmarker using example configuration
   if benchmarker is None:
+    print "Creating Benchmarker from benchmark.cfg.example"
     default_config = setup_util.get_fwroot() + "/benchmark.cfg.example"
     config = ConfigParser.SafeConfigParser()
     config.readfp(open(default_config))

toolset/run-tests.py

@@ -4,6 +4,8 @@ import ConfigParser
 import sys
 import os
 import multiprocessing
+import itertools
+import copy
 import subprocess
 from pprint import pprint 
 from benchmark.benchmarker import Benchmarker
@@ -14,6 +16,27 @@ from setup.linux import setup_util
 from colorama import init
 init()
 
+class StoreSeqAction(argparse.Action):
+  '''Helper class for parsing a sequence from the command line'''
+  def __init__(self, option_strings, dest, nargs=None, **kwargs):
+     super(StoreSeqAction, self).__init__(option_strings, dest, type=str, **kwargs)
+  def __call__(self, parser, namespace, values, option_string=None):
+    setattr(namespace, self.dest, self.parse_seq(values))
+  def parse_seq(self, argument):
+    result = argument.split(',')
+    sequences = [x for x in result if ":" in x]
+    for sequence in sequences:
+      try:
+        (start,step,end) = sequence.split(':')
+      except ValueError: 
+        print "  Invalid: %s" % sequence
+        print "  Requires start:step:end, e.g. 1:2:10"
+        raise
+      result.remove(sequence)
+      result = result + range(int(start), int(end), int(step))
+    return [abs(int(item)) for item in result]
+
+
 ###################################################################################################
 # Main
 ###################################################################################################
@@ -81,9 +104,15 @@ def main(argv=None):
     ##########################################################
     # Set up argument parser
     ##########################################################
-    parser = argparse.ArgumentParser(description='Run the Framework Benchmarking test suite.',
+    parser = argparse.ArgumentParser(description="Install or run the Framework Benchmarks test suite.",
         parents=[conf_parser],
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter)
+        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
+        epilog='''If an argument includes (type int-sequence), then it accepts integer lists in multiple forms. 
+        Using a single number e.g. 5 will create a list [5]. Using commas will create a list containing those 
+        values e.g. 1,3,6 creates [1, 3, 6]. Using three colon-separated numbers of start:step:end will create a 
+        list, using the semantics of python's range function, e.g. 1:3:15 creates [1, 4, 7, 10, 13] while 
+        0:1:5 creates [0, 1, 2, 3, 4]
+        ''')
 
     # SSH options
     parser.add_argument('-s', '--server-host', default=serverHost, help='The application server.')
@@ -122,12 +151,10 @@ def main(argv=None):
     parser.add_argument('--database-os', choices=['linux', 'windows'], default='linux', help='The operating system of the database server.')
 
     # Benchmark options
-    parser.add_argument('--max-concurrency', default=256, help='the maximum number of HTTP connections that wrk will keep open. The query tests will run at this maximum', type=int)
-    parser.add_argument('--max-queries', default=20, help='The maximum number of queries to run during the query test', type=int)
-    parser.add_argument('--query-interval', default=5, type=int, help='Query tests will go from 1 query to max queries in increments of interval queries')
-    parser.add_argument('--max-threads', default=maxThreads, help='The max number of threads to run wrk at. This should be set to the number of cores for your client system.', type=int)
+    parser.add_argument('--concurrency-levels', default=[8, 16, 32, 64, 128, 256], help='Runs wrk benchmarker with different concurrency value (type int-sequence)', action=StoreSeqAction)
+    parser.add_argument('--query-levels', default=[1, 5,10,15,20], help='Database queries requested per HTTP connection, used during query test (type int-sequence)', action=StoreSeqAction) 
+    parser.add_argument('--threads', default=maxThreads, help='Run wrk benchmarker with this many threads. This should probably be the number of cores for your client system', type=int)
     parser.add_argument('--duration', default=15, help='Time in seconds that each test should run for.')
-    parser.add_argument('--starting-concurrency', default=8, type=int)
     parser.add_argument('--sleep', type=int, default=60, help='the amount of time to sleep after starting each test to allow the server to start up.')
 
     # Misc Options
@@ -151,7 +178,7 @@ def main(argv=None):
 
     if args.verbose:
         print 'Configuration options: '
-        pprint(args)
+        pprint(vars(args))
 
 
 

toolset/test/test-run-tests.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+
+import argparse
+import ConfigParser
+import sys
+import os
+import multiprocessing
+import itertools
+import copy
+import subprocess
+from pprint import pprint 
+from benchmark.benchmarker import Benchmarker
+from setup.linux.unbuffered import Unbuffered
+from setup.linux import setup_util
+
+from run-tests import StoreSeqAction
+
+parser = argparse.ArgumentParser()
+parser.add_argument('--foo', action=StoreSeqAction)
+tests = ["1", "1,", "0.23",                       # Single numbers
+        "1,5,7", "1,2,-3", "1,1000,12,1,1,1,1,1", # Lists
+        "1:2:10", "1:2", "10:-2:0",               # Sequences
+        "1,2:1:5"                                 # Complex
+]
+for test in tests:
+  try:
+    t = "--foo %s" % test
+    print "Testing %s" % test
+    print "  %s" % parser.parse_args(t.split())
+    print "  Success"
+  except Exception as e: 
+    print "  Exception! %s" % e
+    continue
+
+# vim: sw=2