Map test type strings to FrameworkTestType objects

toolset/benchmark/benchmarker.py

@@ -2,6 +2,7 @@ from setup.linux.installer import Installer
 from setup.linux import setup_util
 
 from benchmark import framework_test
+from benchmark.test_types.framework_test_type import *
 from utils import header
 from utils import gather_tests
 from utils import gather_frameworks
@@ -17,6 +18,7 @@ import sys
 import logging
 import socket
 import threading
+from pprint import pprint
 
 from multiprocessing import Process
 
@@ -882,7 +884,25 @@ class Benchmarker:
   ############################################################
   def __init__(self, args):
     
+    # Map type strings to their objects
+    types = dict()
+    types['json'] = JsonTestType()
+    types['db'] = DBTestType()
+    types['query'] = QueryTestType()
+    types['fortune'] = FortuneTestType()
+    types['update'] = UpdateTestType()
+    types['plaintext'] = PlaintextTestType()
+
+    # Turn type into a map instead of a string
+    if args['type'] is 'all':
+        args['types'] = types
+    else:
+        args['types'] = { args['type'] : types[args['type']] }
+    del args['type']
+
     self.__dict__.update(args)
+    # pprint(self.__dict__)
+
     self.start_time = time.time()
     self.run_test_timeout_seconds = 3600
 

toolset/benchmark/framework_test.py

@@ -1,12 +1,13 @@
 from benchmark.fortune_html_parser import FortuneHTMLParser
 from setup.linux import setup_util
+from benchmark.test_types.framework_test_type import *
 
 import importlib
 import os
 import subprocess
 import time
 import re
-import pprint
+from pprint import pprint
 import sys
 import traceback
 import json
@@ -1384,16 +1385,16 @@ def parse_config(config, directory, benchmarker):
       if not all (key in test_keys for key in required):
         raise Exception("benchmark_config for test %s is invalid - missing required keys" % test_name)      
       
-      # Map test type to either boolean False (e.g. don't run)
-      # or to a list of strings containing all the arguments 
-      # needed by this test type
+      # Map test type to a parsed FrameworkTestType object
       runTests = dict()
-      for test_type in benchmarker.types:
-        # Ensure all arguments required for this test are present
-        if all (arg in test_keys for arg in benchmarker.type_args[test_type]):
-          runTests[test_type] = [ test_keys[arg] for arg in benchmarker.type_args[test_type]]
-        else:
-          runTests[test_type] = False
+      for type_name, type_obj in benchmarker.types.iteritems():
+        try:
+          runTests[type_name] = type_obj.copy().parse(test_keys)
+        except AttributeError as ae:
+          # This is quite common - most tests don't support all types
+          # Quitely log it and move on
+          logging.debug("Missing arguments for test type %s for framework test %s", type_name, test_name)
+          pass
 
       # By passing the entire set of keys, each FrameworkTest will have a member for each key
       tests.append(FrameworkTest(test_name, directory, benchmarker, runTests, test_keys))

toolset/run-tests.py

@@ -153,23 +153,7 @@ def main(argv=None):
         print 'Configuration options: '
         pprint(args)
 
-    # Turn type into a list instead of a string
-    if args.type is 'all':
-        args.types = ['json', 'db', 'query', 'fortune', 'update', 'plaintext']
-    else:
-        args.types = [args.type]
-    
 
-    # Add mapping between types and the keys that are needed in 
-    # benchmark_config to run that test type
-    type_args = dict()
-    type_args['json'] = ['json_url']
-    type_args['db'] = ['db_url']
-    type_args['query'] = ['query_url']
-    type_args['fortune'] = ['fortune_url']
-    type_args['update'] = ['update_url']
-    type_args['plaintext'] = ['plaintext_url']
-    args.type_args = type_args
 
     benchmarker = Benchmarker(vars(args))