Toolset updates for Round 20 (#5648)

* Add tag support

* add --list-tag

* filter out broken tests

* work on test modules

* fix test order

* :/

* work on test-modules

* fix class

* add url length checks to verifications

* update cached-query key and vagrant to ubuntu 18.04

deployment/vagrant/core.rb

@@ -12,7 +12,7 @@ end
 def provider_libvirt(config)
   config.vm.provider :libvirt do |virt, override|
     override.vm.hostname = "TFB-all"
-    override.vm.box = "generic/ubuntu1604"
+    override.vm.box = "generic/ubuntu1804"
 
     unless ENV.fetch('TFB_SHOW_VM', false)
       virt.graphics_type = "none"
@@ -28,7 +28,7 @@ end
 def provider_virtualbox(config)
   config.vm.provider :virtualbox do |vb, override|
     override.vm.hostname = "TFB-all"
-    override.vm.box = "ubuntu/xenial64"
+    override.vm.box = "ubuntu/bionic64"
 
     # Allow increase in size for /dev/sda1
     # Would need plugin:

toolset/benchmark/test_types/__init__.py

@@ -1,9 +0,0 @@
-
-from framework_test_type import *
-from json_type import JsonTestType
-from plaintext_type import PlaintextTestType
-from db_type import DBTestType
-from query_type import QueryTestType
-from update_type import UpdateTestType
-from fortune_type import FortuneTestType
-from cached_query_type import CachedQueryTestType

toolset/run-tests.py

@@ -133,7 +133,7 @@ def main(argv=None):
     parser.add_argument(
         '--type',
         choices=[
-            'all', 'json', 'db', 'query', 'cached_query', 'fortune', 'update',
+            'all', 'json', 'db', 'query', 'cached-query', 'fortune', 'update',
             'plaintext'
         ],
         nargs='+',

toolset/test_types/__init__.py

@@ -0,0 +1,15 @@
+import imp
+import re
+
+from glob import glob
+
+test_types = {}
+test_type_folders = glob("/FrameworkBenchmarks/toolset/test_types/*/")
+
+# Loads all the test_types from the folders in this directory
+for folder in test_type_folders:
+    # regex that grabs the characters between "toolset/test_types/"
+    # and the final "/" in the folder string to get the name
+    test_type_name = re.findall(r'.+\/(.+)\/$', folder, re.M)[0]
+    test_type = imp.load_source("TestType", "%s%s.py" % (folder, test_type_name))
+    test_types[test_type_name] = test_type.TestType

toolset/benchmark/test_types/framework_test_type.py → toolset/test_types/abstract_test_type.py

@@ -1,11 +1,11 @@
+import abc
 import copy
 import requests
 
 from colorama import Fore
 from toolset.utils.output_helper import log
 
-
-class FrameworkTestType:
+class AbstractTestType:
     '''
     Interface between a test type (json, query, plaintext, etc) and
     the rest of TFB. A test type defines a number of keys it expects
@@ -15,6 +15,7 @@ class FrameworkTestType:
     passes an argument list of ['spam'], then after parsing there will
     exist a member `X.spam = 'foobar'`.
     '''
+    __metaclass__ = abc.ABCMeta
 
     def __init__(self,
                  config,
@@ -38,6 +39,13 @@ class FrameworkTestType:
         self.failed = None
         self.warned = None
 
+    @classmethod
+    @abc.abstractmethod
+    def url(self):
+        pass
+
+    @classmethod
+    @abc.abstractmethod
     def accept(self, content_type):
         return {
             'json':
@@ -51,7 +59,7 @@ class FrameworkTestType:
     def parse(self, test_keys):
         '''
         Takes the dict of key/value pairs describing a FrameworkTest
-        and collects all variables needed by this FrameworkTestType
+        and collects all variables needed by this AbstractTestType
 
         Raises AttributeError if required keys are missing
         '''

toolset/benchmark/test_types/cached_query_type.py → toolset/test_types/cached-query/cached-query.py

@@ -1,17 +1,17 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import verify_query_cases
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import verify_query_cases
 
 
-class CachedQueryTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.cached_query_url = ""
         kwargs = {
-            'name': 'cached_query',
+            'name': 'cached-query',
             'accept_header': self.accept('json'),
             'requires_db': True,
             'args': ['cached_query_url']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.cached_query_url
@@ -31,6 +31,14 @@ class CachedQueryTestType(FrameworkTestType):
 
         problems = verify_query_cases(self, cases, url)
 
+        # cached_query_url should be at least "/cached-worlds/"
+        # some frameworks use a trailing slash while others use ?q=
+        if len(self.cached_query_url) < 15:
+            problems.append(
+                ("fail",
+                 "Route for cached queries must be at least 8 characters, found '{}' instead".format(self.cached_query_url),
+                 url))
+
         if len(problems) == 0:
             return [('pass', '', url + case) for case, _ in cases]
         else:

toolset/benchmark/test_types/db_type.py → toolset/test_types/db/db.py

@@ -1,8 +1,8 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object, verify_queries_count
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_randomnumber_object, verify_queries_count
 
 
-class DBTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.db_url = ""
         kwargs = {
@@ -11,14 +11,14 @@ class DBTestType(FrameworkTestType):
             'requires_db': True,
             'args': ['db_url', 'database']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.db_url
 
     def verify(self, base_url):
         '''
-        Ensures body is valid JSON with a key 'id' and a key 
+        Ensures body is valid JSON with a key 'id' and a key
         'randomNumber', both of which must map to integers
         '''
 
@@ -32,6 +32,13 @@ class DBTestType(FrameworkTestType):
 
         response, problems = basic_body_verification(body, url)
 
+        # db should be at least "/db"
+        if len(self.db_url) < 3:
+            problems.append(
+                ("fail",
+                 "Route for db must be at least 3 characters, found '{}' instead".format(self.db_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 

toolset/benchmark/test_types/fortune_type.py → toolset/test_types/fortune/fortune.py

@@ -1,9 +1,9 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.fortune_html_parser import FortuneHTMLParser
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_queries_count
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.fortune.fortune_html_parser import FortuneHTMLParser
+from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_queries_count
 
 
-class FortuneTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.fortune_url = ""
         kwargs = {
@@ -12,7 +12,7 @@ class FortuneTestType(FrameworkTestType):
             'requires_db': True,
             'args': ['fortune_url','database']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.fortune_url
@@ -34,6 +34,13 @@ class FortuneTestType(FrameworkTestType):
 
         _, problems = basic_body_verification(body, url, is_json_check=False)
 
+        # fortune_url should be at least "/fortunes"
+        if len(self.fortune_url) < 9:
+            problems.append(
+                ("fail",
+                 "Route for fortunes must be at least 9 characters, found '{}' instead".format(self.fortune_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 

toolset/benchmark/test_types/json_type.py → toolset/test_types/json/json.py

@@ -1,8 +1,7 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers, verify_helloworld_object
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import basic_body_verification, verify_headers, verify_helloworld_object
 
-
-class JsonTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.json_url = ""
         kwargs = {
@@ -11,15 +10,15 @@ class JsonTestType(FrameworkTestType):
             'requires_db': False,
             'args': ['json_url']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.json_url
 
     def verify(self, base_url):
         '''
-        Validates the response is a JSON object of 
-        { 'message' : 'hello, world!' }. Case insensitive and 
+        Validates the response is a JSON object of
+        { 'message' : 'hello, world!' }. Case insensitive and
         quoting style is ignored
         '''
 
@@ -28,6 +27,13 @@ class JsonTestType(FrameworkTestType):
 
         response, problems = basic_body_verification(body, url)
 
+        # json_url should be at least "/json"
+        if len(self.json_url) < 5:
+            problems.append(
+                ("fail",
+                 "Route for json must be at least 5 characters, found '{}' instead".format(self.json_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 

toolset/benchmark/test_types/plaintext_type.py → toolset/test_types/plaintext/plaintext.py

@@ -1,9 +1,8 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import basic_body_verification, verify_headers
-from time import sleep
+from toolset.test_types.verifications import basic_body_verification, verify_headers
+from toolset.test_types.abstract_test_type import AbstractTestType
 
 
-class PlaintextTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.plaintext_url = ""
         kwargs = {
@@ -12,7 +11,7 @@ class PlaintextTestType(FrameworkTestType):
             'accept_header': self.accept('plaintext'),
             'args': ['plaintext_url']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def verify(self, base_url):
         url = base_url + self.plaintext_url
@@ -20,6 +19,13 @@ class PlaintextTestType(FrameworkTestType):
 
         _, problems = basic_body_verification(body, url, is_json_check=False)
 
+        # plaintext_url should be at least "/plaintext"
+        if len(self.plaintext_url) < 10:
+            problems.append(
+                ("fail",
+                 "Route for plaintext must be at least 10 characters, found '{}' instead".format(self.plaintext_url),
+                 url))
+
         if len(problems) > 0:
             return problems
 
@@ -55,20 +61,20 @@ class PlaintextTestType(FrameworkTestType):
     def get_script_variables(self, name, url):
         return {
             'max_concurrency':
-            max(self.config.concurrency_levels),
+                max(self.config.concurrency_levels),
             'name':
-            name,
+                name,
             'duration':
-            self.config.duration,
+                self.config.duration,
             'levels':
-            " ".join("{}".format(item)
-                     for item in self.config.pipeline_concurrency_levels),
+                " ".join("{}".format(item)
+                         for item in self.config.pipeline_concurrency_levels),
             'server_host':
-            self.config.server_host,
+                self.config.server_host,
             'url':
-            url,
+                url,
             'pipeline':
-            16,
+                16,
             'accept':
-            "text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
+                "text/plain,text/html;q=0.9,application/xhtml+xml;q=0.9,application/xml;q=0.8,*/*;q=0.7"
         }

toolset/benchmark/test_types/query_type.py → toolset/test_types/query/query.py

@@ -1,8 +1,8 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import verify_query_cases
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import verify_query_cases
 
 
-class QueryTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.query_url = ""
         kwargs = {
@@ -11,17 +11,17 @@ class QueryTestType(FrameworkTestType):
             'requires_db': True,
             'args': ['query_url', 'database']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.query_url
 
     def verify(self, base_url):
         '''
-        Validates the response is a JSON array of 
-        the proper length, each JSON Object in the array 
-        has keys 'id' and 'randomNumber', and these keys 
-        map to integers. Case insensitive and 
+        Validates the response is a JSON array of
+        the proper length, each JSON Object in the array
+        has keys 'id' and 'randomNumber', and these keys
+        map to integers. Case insensitive and
         quoting style is ignored
         '''
 
@@ -31,6 +31,14 @@ class QueryTestType(FrameworkTestType):
 
         problems = verify_query_cases(self, cases, url, False)
 
+        # queries_url should be at least "/queries/"
+        # some frameworks use a trailing slash while others use ?q=
+        if len(self.query_url) < 9:
+            problems.append(
+                ("fail",
+                 "Route for queries must be at least 9 characters, found '{}' instead".format(self.query_url),
+                 url))
+
         if len(problems) == 0:
             return [('pass', '', url + case) for case, _ in cases]
         else:

toolset/benchmark/test_types/update_type.py → toolset/test_types/update/update.py

@@ -1,8 +1,8 @@
-from toolset.benchmark.test_types.framework_test_type import FrameworkTestType
-from toolset.benchmark.test_types.verifications import verify_query_cases
+from toolset.test_types.abstract_test_type import AbstractTestType
+from toolset.test_types.verifications import verify_query_cases
 
 
-class UpdateTestType(FrameworkTestType):
+class TestType(AbstractTestType):
     def __init__(self, config):
         self.update_url = ""
         kwargs = {
@@ -11,17 +11,17 @@ class UpdateTestType(FrameworkTestType):
             'requires_db': True,
             'args': ['update_url', 'database']
         }
-        FrameworkTestType.__init__(self, config, **kwargs)
+        AbstractTestType.__init__(self, config, **kwargs)
 
     def get_url(self):
         return self.update_url
 
     def verify(self, base_url):
         '''
-        Validates the response is a JSON array of 
-        the proper length, each JSON Object in the array 
-        has keys 'id' and 'randomNumber', and these keys 
-        map to integers. Case insensitive and 
+        Validates the response is a JSON array of
+        the proper length, each JSON Object in the array
+        has keys 'id' and 'randomNumber', and these keys
+        map to integers. Case insensitive and
         quoting style is ignored
         '''
 
@@ -30,6 +30,14 @@ class UpdateTestType(FrameworkTestType):
                  ('501', 'warn'), ('', 'fail')]
         problems = verify_query_cases(self, cases, url, True)
 
+        # update_url should be at least "/update/"
+        # some frameworks use a trailing slash while others use ?q=
+        if len(self.update_url) < 8:
+            problems.append(
+                ("fail",
+                 "Route for update must be at least 8 characters, found '{}' instead".format(self.update_url),
+                 url))
+
         if len(problems) == 0:
             return [('pass', '', url + case) for (case, _) in cases]
         else:

toolset/benchmark/test_types/verifications.py → toolset/test_types/verifications.py

@@ -11,6 +11,7 @@ from time import sleep
 # Cross-platform colored text
 from colorama import Fore, Style
 
+
 def basic_body_verification(body, url, is_json_check=True):
     '''
     Takes in a raw (stringy) response body, checks that it is non-empty,
@@ -81,15 +82,15 @@ def verify_headers(request_headers_and_body, headers, url, should_be='json'):
     date2 = second_headers.get('Date')
     if date == date2:
         problems.append((
-        'fail',
-        'Invalid Cached Date. Found \"%s\" and \"%s\" on separate requests.'
-        % (date, date2), url))
+            'fail',
+            'Invalid Cached Date. Found \"%s\" and \"%s\" on separate requests.'
+            % (date, date2), url))
 
     content_type = headers.get('Content-Type')
     if content_type is not None:
         types = {
-            'json':      '^application/json(; ?charset=(UTF|utf)-8)?$',
-            'html':      '^text/html; ?charset=(UTF|utf)-8$',
+            'json': '^application/json(; ?charset=(UTF|utf)-8)?$',
+            'html': '^text/html; ?charset=(UTF|utf)-8$',
             'plaintext': '^text/plain(; ?charset=(UTF|utf)-8)?$'
         }
         expected_type = types[should_be]
@@ -344,7 +345,7 @@ def verify_query_cases(self, cases, url, check_updates=False):
     world_db_before = {}
     if check_updates:
         world_db_before = databases[self.database.lower()].get_current_world_table(self.config)
-        expected_queries = expected_queries + concurrency * repetitions #eventually bulk updates!
+        expected_queries = expected_queries + concurrency * repetitions  # eventually bulk updates!
 
     for q, max_infraction in cases:
         case_url = url + q
@@ -396,52 +397,63 @@ def verify_query_cases(self, cases, url, check_updates=False):
 
     if hasattr(self, 'database'):
         # verify the number of queries and rows read for 20 queries, with a concurrency level of 512, with 2 repetitions
-        problems += verify_queries_count(self, "world", url+"20", concurrency, repetitions, expected_queries, expected_rows, check_updates)
+        problems += verify_queries_count(self, "world", url + "20", concurrency, repetitions, expected_queries,
+                                         expected_rows, check_updates)
     return problems
 
 
-def verify_queries_count(self, tbl_name, url, concurrency=512, count=2, expected_queries=1024, expected_rows = 1024, check_updates = False):
+def verify_queries_count(self, tbl_name, url, concurrency=512, count=2, expected_queries=1024, expected_rows=1024,
+                         check_updates=False):
     '''
-    Checks that the number of executed queries, at the given concurrency level, 
+    Checks that the number of executed queries, at the given concurrency level,
     corresponds to: the total number of http requests made * the number of queries per request.
     No margin is accepted on the number of queries, which seems reliable.
     On the number of rows read or updated, the margin related to the database applies (1% by default see cls.margin)
-    On updates, if the use of bulk updates is detected (number of requests close to that expected), a margin (5% see bulk_margin) is allowed on the number of updated rows. 
+    On updates, if the use of bulk updates is detected (number of requests close to that expected), a margin
+    (5% see bulk_margin) is allowed on the number of updated rows.
     '''
     log("VERIFYING QUERY COUNT FOR %s" % url, border='-', color=Fore.WHITE + Style.BRIGHT)
 
     problems = []
 
-    queries, rows, rows_updated, margin, trans_failures = databases[self.database.lower()].verify_queries(self.config, tbl_name, url, concurrency, count, check_updates)
+    queries, rows, rows_updated, margin, trans_failures = databases[self.database.lower()].verify_queries(self.config,
+                                                                                                          tbl_name, url,
+                                                                                                          concurrency,
+                                                                                                          count,
+                                                                                                          check_updates)
 
     isBulk = check_updates and (queries < 1.001 * expected_queries) and (queries > 0.999 * expected_queries)
-    
-    if check_updates and not isBulk:#Restore the normal queries number if bulk queries are not used
+
+    if check_updates and not isBulk:  # Restore the normal queries number if bulk queries are not used
         expected_queries = (expected_queries - count * concurrency) * 2
 
-    #Add a margin based on the number of cpu cores
-    queries_margin = 1.015 #For a run on Travis
-    if multiprocessing.cpu_count()>2:
-        queries_margin = 1 # real run (Citrine or Azure) -> no margin on queries
-        #Check for transactions failures (socket errors...)
+    # Add a margin based on the number of cpu cores
+    queries_margin = 1.015  # For a run on Travis
+    if multiprocessing.cpu_count() > 2:
+        queries_margin = 1  # real run (Citrine or Azure) -> no margin on queries
+        # Check for transactions failures (socket errors...)
         if trans_failures > 0:
             problems.append((
-            "fail",
-            "%s failed transactions."
-            % trans_failures, url))
+                "fail",
+                "%s failed transactions."
+                % trans_failures, url))
 
-    problems.append(display_queries_count_result(queries * queries_margin, expected_queries, queries, "executed queries", url))
+    problems.append(
+        display_queries_count_result(queries * queries_margin, expected_queries, queries, "executed queries", url))
 
     problems.append(display_queries_count_result(rows, expected_rows, int(rows / margin), "rows read", url))
 
     if check_updates:
         bulk_margin = 1
-        if isBulk:#Special marge for bulk queries
+        if isBulk:  # Special marge for bulk queries
             bulk_margin = 1.05
-        problems.append(display_queries_count_result(rows_updated * bulk_margin, expected_rows, int(rows_updated / margin), "rows updated", url))
+        problems.append(
+            display_queries_count_result(rows_updated * bulk_margin, expected_rows, int(rows_updated / margin),
+                                         "rows updated", url))
 
     return problems
 
+
 def display_queries_count_result(result, expected_result, displayed_result, caption, url):
     '''
     Returns a single result in counting queries, rows read or updated.
@@ -450,13 +462,13 @@ def display_queries_count_result(result, expected_result, displayed_result, capt
     '''
     if result > expected_result * 1.05:
         return (
-        "warn",
-        "%s %s in the database instead of %s expected. This number is excessively high."
-        % (displayed_result, caption, expected_result), url)
-    elif result < expected_result :
+            "warn",
+            "%s %s in the database instead of %s expected. This number is excessively high."
+            % (displayed_result, caption, expected_result), url)
+    elif result < expected_result:
         return (
-        "fail",
-        "Only %s %s in the database out of roughly %s expected."
-        % (displayed_result, caption, expected_result), url)
+            "fail",
+            "Only %s %s in the database out of roughly %s expected."
+            % (displayed_result, caption, expected_result), url)
     else:
-        return ("pass","%s: %s/%s" % (caption.capitalize(), displayed_result,expected_result), url)
+        return ("pass", "%s: %s/%s" % (caption.capitalize(), displayed_result, expected_result), url)

toolset/utils/benchmark_config.py

@@ -1,5 +1,5 @@
-from toolset.benchmark.test_types import *
 from toolset.utils.output_helper import QuietOutputStream
+from toolset.test_types import test_types
 
 import os
 import time
@@ -12,14 +12,9 @@ class BenchmarkConfig:
         '''
 
         # Map type strings to their objects
-        types = dict()
-        types['json'] = JsonTestType(self)
-        types['db'] = DBTestType(self)
-        types['query'] = QueryTestType(self)
-        types['fortune'] = FortuneTestType(self)
-        types['update'] = UpdateTestType(self)
-        types['plaintext'] = PlaintextTestType(self)
-        types['cached_query'] = CachedQueryTestType(self)
+        types = {}
+        for type in test_types:
+            types[type] = test_types[type](self)
 
         # Turn type into a map instead of a list of strings
         if 'all' in args.type:

toolset/utils/metadata.py

@@ -200,8 +200,9 @@ class Metadata:
 
                 # Map test type to a parsed FrameworkTestType object
                 runTests = dict()
-                for type_name, type_obj in self.benchmarker.config.types.iteritems(
-                ):
+
+                # TODO: remove self.benchmarker.config.types
+                for type_name, type_obj in self.benchmarker.config.types.iteritems():
                     try:
                         # Makes a FrameWorkTestType object using some of the keys in config
                         # e.g. JsonTestType uses "json_url"

toolset/utils/results.py

@@ -1,4 +1,5 @@
 from toolset.utils.output_helper import log
+from toolset.test_types import test_types
 
 import os
 import subprocess
@@ -52,31 +53,14 @@ class Results:
         self.frameworks = [t.name for t in benchmarker.tests]
         self.duration = self.config.duration
         self.rawData = dict()
-        self.rawData['json'] = dict()
-        self.rawData['db'] = dict()
-        self.rawData['query'] = dict()
-        self.rawData['fortune'] = dict()
-        self.rawData['update'] = dict()
-        self.rawData['plaintext'] = dict()
-        self.rawData['cached_query'] = dict()
         self.completed = dict()
         self.succeeded = dict()
-        self.succeeded['json'] = []
-        self.succeeded['db'] = []
-        self.succeeded['query'] = []
-        self.succeeded['fortune'] = []
-        self.succeeded['update'] = []
-        self.succeeded['plaintext'] = []
-        self.succeeded['cached_query'] = []
         self.failed = dict()
-        self.failed['json'] = []
-        self.failed['db'] = []
-        self.failed['query'] = []
-        self.failed['fortune'] = []
-        self.failed['update'] = []
-        self.failed['plaintext'] = []
-        self.failed['cached_query'] = []
         self.verify = dict()
+        for type in test_types:
+            self.rawData[type] = dict()
+            self.failed[type] = []
+            self.succeeded[type] = []
 
     #############################################################################
     # PUBLIC FUNCTIONS