Merge pull request #1279 from hamiltont/check-for-default

Toolset: Warn for no default test, error for test not found

frameworks/PHP/hhvm/benchmark_config

@@ -2,28 +2,28 @@
   "framework": "hhvm",
   "tests":
     [{
-        "raw": {
-              "setup_file"     : "setup_hhvm",
-              "json_url"       : "/json",
-              "db_url"         : "/db",
-              "query_url"      : "/queries?queries=",
-              "fortune_url"    : "/fortunes",
-              "update_url"     : "/updates?queries=",
-              "plaintext_url"  : "/plaintext",
-              "port"           : 8080,
-              "approach"       : "Realistic",
-              "classification" : "Platform",
-              "database"       : "MySQL",
-              "framework"      : "hhvm",
-              "language"       : "PHP",
-              "orm"            : "Raw",
-              "platform"       : "hhvm",
-              "webserver"      : "HPHP",
-              "os"             : "Linux",
-              "database_os"    : "Linux",
-              "display_name"   : "hhvm",
-              "notes"          : "",
-              "versus"         : "php"
-        }
+      "default": {
+        "setup_file"     : "setup_hhvm",
+        "json_url"       : "/json",
+        "db_url"         : "/db",
+        "query_url"      : "/queries?queries=",
+        "fortune_url"    : "/fortunes",
+        "update_url"     : "/updates?queries=",
+        "plaintext_url"  : "/plaintext",
+        "port"           : 8080,
+        "approach"       : "Realistic",
+        "classification" : "Platform",
+        "database"       : "MySQL",
+        "framework"      : "hhvm",
+        "language"       : "PHP",
+        "orm"            : "Raw",
+        "platform"       : "hhvm",
+        "webserver"      : "HPHP",
+        "os"             : "Linux",
+        "database_os"    : "Linux",
+        "display_name"   : "hhvm",
+        "notes"          : "",
+        "versus"         : "php"
+      }
     }]
 }

toolset/benchmark/framework_test.py

@@ -733,6 +733,11 @@ def parse_config(config, directory, benchmarker):
   # The config object can specify multiple tests
   #   Loop over them and parse each into a FrameworkTest
   for test in config['tests']:
+
+    names = [name for (name,keys) in test.iteritems()]
+    if "default" not in names:
+      logging.warn("Framework %s does not define a default test in benchmark_config", config['framework'])
+    
     for test_name, test_keys in test.iteritems():
       # Prefix all test names with framework except 'default' test
       if test_name == 'default': 

toolset/benchmark/utils.py

@@ -71,13 +71,27 @@ def gather_tests(include = [], exclude=[], benchmarker=None):
     # Find all tests in the config file
     config_tests = framework_test.parse_config(config, 
       os.path.dirname(config_file_name), benchmarker)
-    
+        
     # Filter
     for test in config_tests:
-      if test.name in exclude:
+      if len(include) is 0 and len(exclude) is 0:
+        # No filters, we are running everything
+        tests.append(test)
+      elif test.name in exclude:
         continue
-      elif len(include) is 0 or test.name in include:
+      elif test.name in include:
         tests.append(test)
+      else: 
+        # An include list exists, but this test is 
+        # not listed there, so we ignore it
+        pass
+
+  # Ensure we were able to locate everything that was 
+  # explicitly included 
+  names = {test.name for test in tests}
+  if 0 != len(set(include) - set(names)):
+    missing = list(set(include) - set(names))
+    raise Exception("Unable to locate tests %s" % missing)
 
   tests.sort(key=lambda x: x.name)
   return tests