Using wrk with a duration rather than number of runs.

framework_test.py

@@ -16,29 +16,29 @@ class FrameworkTest:
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Primer {name}"
-    echo " wrk -r 1000 -c 8 -t 8 http://{server_host}:{port}{url}"
+    echo " wrk -d 60 -c 8 -t 8 http://{server_host}:{port}{url}"
     echo "---------------------------------------------------------"
     echo ""
-    wrk -r 1000 -c 8 -t 8 http://{server_host}:{port}{url}
+    wrk -d 5 -c 8 -t 8 http://{server_host}:{port}{url}
     sleep 5
     
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Warmup {name}"
-    echo " wrk -r {runs} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}"
+    echo " wrk -d {duration} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}"
     echo "---------------------------------------------------------"
     echo ""
-    wrk -r {runs} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}
+    wrk -d {duration} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}
     sleep 5
     for c in {interval}
     do
       echo ""
       echo "---------------------------------------------------------"
       echo " Concurrency: $c for {name}"
-      echo " wrk -n {runs} -c $c -t $(($c>{max_threads}?{max_threads}:$c)) http://{server_host}:{port}{url}"
+      echo " wrk -d {duration} -c $c -t $(($c>{max_threads}?{max_threads}:$c)) http://{server_host}:{port}{url}"
       echo "---------------------------------------------------------"
       echo ""
-      wrk -r {runs} -c "$c" -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url}
+      wrk -d {duration} -c "$c" -t "$(($c>{max_threads}?{max_threads}:$c))" http://{server_host}:{port}{url}
       sleep 2
     done
   """
@@ -49,29 +49,29 @@ class FrameworkTest:
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Primer {name}"
-    echo " wrk -r 1000 -c 8 -t 8 http://{server_host}:{port}{url}2"
+    echo " wrk -d 5 -c 8 -t 8 http://{server_host}:{port}{url}2"
     echo "---------------------------------------------------------"
     echo ""
-    wrk -r 1000 -c 8 -t 8 http://{server_host}:{port}{url}2
+    wrk -d 5 -c 8 -t 8 http://{server_host}:{port}{url}2
     sleep 5
     
     echo ""
     echo "---------------------------------------------------------"
     echo " Running Warmup {name}"
-    echo " wrk -r {runs} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}2"
+    echo " wrk -d {duration} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}2"
     echo "---------------------------------------------------------"
     echo ""
-    wrk -r {runs} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}2
+    wrk -d {duration} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}2
     sleep 5
     for c in {interval}
     do
       echo ""
       echo "---------------------------------------------------------"
       echo " Queries: $c for {name}"
-      echo " wrk -r {runs} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}$c"
+      echo " wrk -d {duration} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}$c"
       echo "---------------------------------------------------------"
       echo ""
-      wrk -r {runs} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}"$c"
+      wrk -d {duration} -c {max_concurrency} -t {max_threads} http://{server_host}:{port}{url}"$c"
       sleep 2
     done
   """
@@ -320,7 +320,7 @@ class FrameworkTest:
   ############################################################
   def __generate_concurrency_script(self, url, port):
     return self.concurrency_template.format(max_concurrency=self.benchmarker.max_concurrency, 
-      max_threads=self.benchmarker.max_threads, name=self.name, runs=self.benchmarker.number_of_runs, 
+      max_threads=self.benchmarker.max_threads, name=self.name, duration=self.benchmarker.duration, 
       interval=" ".join("{}".format(item) for item in self.benchmarker.concurrency_levels), 
       server_host=self.benchmarker.server_host, port=port, url=url)
   ############################################################
@@ -335,7 +335,7 @@ class FrameworkTest:
   ############################################################
   def __generate_query_script(self, url, port):
     return self.query_template.format(max_concurrency=self.benchmarker.max_concurrency, 
-      max_threads=self.benchmarker.max_threads, name=self.name, runs=self.benchmarker.number_of_runs, 
+      max_threads=self.benchmarker.max_threads, name=self.name, duration=self.benchmarker.duration, 
       interval=" ".join("{}".format(item) for item in self.benchmarker.query_intervals), 
       server_host=self.benchmarker.server_host, port=port, url=url)
   ############################################################

run-tests.py

@@ -27,7 +27,7 @@ parser.add_argument('--max-concurrency', default=256, help='the maximum concurre
 parser.add_argument('--max-queries', default=20, help='The maximum number of queries to run during the query test', type=int)
 parser.add_argument('--query-interval', default=5, type=int)
 parser.add_argument('--max-threads', default=8, help='The max number of threads to run weight at, this shoul dbe set to the number of cores for your system.', type=int)
-parser.add_argument('--number-of-runs', default=100000, help='the number of requests to make per test run.')
+parser.add_argument('--duration', default=60, help='Time in seconds that each test should run for.')
 parser.add_argument('--starting-concurrency', default=8, type=int)
 parser.add_argument('--sleep', type=int, default=60, help='the amount of time to sleep after starting each test to allow the server to start up.')
 parser.add_argument('--parse', help='Parses the results of the given timestamp and merges that with the latest results')