Merge pull request #969 from hamiltont/refactor_gather_tests

Refactor gather tests into static call

benchmark.cfg.example

@@ -1,31 +1,31 @@
 [Defaults]
 # Available Keys: 
-# client_host='localhost'
-# client_identity_file=None
-# client_user=None
-# database_host=None
-# database_identity_file=None
-# database_os='linux'
-# database_user=None
-# duration=60
-# exclude=None
-# install='all'
-# install_error_action='continue'
-# install_software=False
-# list_test_metadata=False
-# list_tests=False
-# max_concurrency=256
-# max_queries=20
-# max_threads=8
-# mode='benchmark'
-# name='ec2'
-# os='linux'
-# parse=None
-# password_prompt=False
-# query_interval=5
-# server_host='localhost'
-# sleep=60
-# starting_concurrency=8
-# test=None
-# type='all'
-# verbose=True
+client_host=localhost
+client_identity_file=None
+client_user=localhost
+database_host=localhost
+database_identity_file=None
+database_os=linux
+database_user=tfb
+duration=60
+exclude=None
+install=server
+install_error_action=continue
+install_strategy=unified
+list_test_metadata=False
+list_tests=False
+max_concurrency=256
+max_queries=20
+max_threads=8
+mode=benchmark
+name=ec2
+os=linux
+parse=None
+password_prompt=False
+query_interval=5
+server_host=localhost
+sleep=60
+starting_concurrency=8
+test=None
+type=all
+verbose=True

toolset/benchmark/benchmarker.py

@@ -3,6 +3,7 @@ from setup.linux import setup_util
 
 from benchmark import framework_test
 from utils import header
+from utils import gather_tests
 
 import os
 import json
@@ -320,54 +321,9 @@ class Benchmarker:
   ############################################################
   @property
   def __gather_tests(self):
-    tests = []
-
-    # Assume we are running from FrameworkBenchmarks
-    config_files = glob.glob('*/benchmark_config')
-
-    for config_file_name in config_files:
-      # Look for the benchmark_config file, this will set up our tests.
-      # Its format looks like this:
-      #
-      # {
-      #   "framework": "nodejs",
-      #   "tests": [{
-      #     "default": {
-      #       "setup_file": "setup",
-      #       "json_url": "/json"
-      #     },
-      #     "mysql": {
-      #       "setup_file": "setup",
-      #       "db_url": "/mysql",
-      #       "query_url": "/mysql?queries="
-      #     },
-      #     ...
-      #   }]
-      # }
-      config = None
-
-      with open(config_file_name, 'r') as config_file:
-        # Load json file into config object
-        try:
-          config = json.load(config_file)
-        except:
-          print("Error loading '%s'." % config_file_name)
-          raise
-
-      if config is None:
-        continue
-
-      test = framework_test.parse_config(config, os.path.dirname(config_file_name), self)
-      # If the user specified which tests to run, then 
-      # we can skip over tests that are not in that list
-      if self.test == None:
-        tests = tests + test
-      else:
-        for atest in test:
-          if atest.name in self.test:
-            tests.append(atest)
-
-    tests.sort(key=lambda x: x.name)
+    tests = gather_tests(include=self.test, 
+      exclude=self.exclude,
+      benchmarker=self)
 
     # If the tests have been interrupted somehow, then we want to resume them where we left
     # off, rather than starting from the beginning

toolset/benchmark/utils.py

@@ -1,3 +1,82 @@
+import ConfigParser
+import os
+import glob
+import json
+
+from ast import literal_eval
+
+def gather_tests(include = [], exclude=[], benchmarker=None):
+  '''
+  Given test names as strings, returns a list of FrameworkTest objects. 
+  For example, 'aspnet-mysql-raw' turns into a FrameworkTest object with
+  variables for checking the test directory, the test database os, and 
+  other useful items. 
+
+  With no arguments, every test in this framework will be returned.  
+  With include, only tests with this exact name will be returned. 
+  With exclude, all tests but those excluded will be returned. 
+
+  A benchmarker is needed to construct full FrameworkTest objects. If
+  one is not provided, a default Benchmarker will be created. 
+  '''
+
+  # Avoid setting up a circular import
+  from benchmark import framework_test
+  from benchmark.benchmarker import Benchmarker
+  from setup.linux import setup_util
+
+  # Help callers out a bit
+  if include is None:
+    include = []
+  if exclude is None:
+    exclude = []
+  
+  # Setup default Benchmarker using example configuration
+  if benchmarker is None:
+    default_config = setup_util.get_fwroot() + "/benchmark.cfg.example"
+    config = ConfigParser.SafeConfigParser()
+    config.readfp(open(default_config))
+    defaults = dict(config.items("Defaults"))
+    
+    # Convert strings into proper python types
+    for k,v in defaults.iteritems():
+      try:
+        defaults[k] = literal_eval(v)
+      except:
+        pass
+
+    # Ensure we only run the __init__ method of Benchmarker
+    defaults['install'] = None
+    
+    benchmarker = Benchmarker(defaults)
+
+  # Assume we are running from FrameworkBenchmarks
+  config_files = glob.glob('*/benchmark_config')
+
+  tests = []
+  for config_file_name in config_files:
+    config = None
+    with open(config_file_name, 'r') as config_file:
+      try:
+        config = json.load(config_file)
+      except:
+        # User-friendly errors
+        print("Error loading '%s'." % config_file_name)
+        raise
+
+    # Find all tests in the config file
+    config_tests = framework_test.parse_config(config, 
+      os.path.dirname(config_file_name), benchmarker)
+    
+    # Filter
+    for test in config_tests:
+      if test.name in exclude:
+        continue
+      elif len(include) is 0 or test.name in include:
+        tests.append(test)
+
+  tests.sort(key=lambda x: x.name)
+  return tests
 
 def header(message, top='-', bottom='-'):
     '''

toolset/run-ci.py

@@ -4,6 +4,7 @@ import subprocess
 import os
 import sys
 from benchmark import framework_test
+from benchmark.utils import gather_tests
 import glob
 import json
 import traceback
@@ -36,7 +37,7 @@ class CIRunnner:
     logging.basicConfig(level=logging.INFO)
     
     if not test_directory == 'jobcleaner':
-      tests = self.gather_tests()
+      tests = gather_tests()
       
       # Run the first linux-only test in this directory
       # At the moment, travis only supports mysql or none!
@@ -149,26 +150,6 @@ class CIRunnner:
       log.error(err.child_traceback)
       return 1
 
-  def gather_tests(self):
-    ''' Returns all available tests as FrameworkTest list '''
-
-    # Fake benchmarker fields that are used
-    class bench_shim():
-      def __init__(self):
-        self.type = 'all'
-        self.fwroot = os.getcwd()
-        self.install_strategy='pertest'
-
-    # Gather all tests
-    tests = []
-    for config_file_name in glob.glob('*/benchmark_config'):
-      with open(config_file_name, 'r') as config_file:
-        config = json.load(config_file)
-        test = framework_test.parse_config(config, os.path.dirname(config_file_name), bench_shim())
-        tests = tests + test
-    tests.sort(key=lambda x: x.name)
-    return tests
-
   def cancel_unneeded_jobs(self):
     log.info("I am jobcleaner")
 

toolset/setup/linux/installer.py

@@ -8,6 +8,8 @@ import glob
 import logging
 import setup_util
 
+from benchmark.utils import gather_tests
+
 class Installer:
 
   ############################################################
@@ -37,66 +39,61 @@ class Installer:
     prereq_path='$FWROOT/toolset/setup/linux/prerequisites.sh'
     self.__run_command(". %s && . %s" % (bash_functions_path, prereq_path))
 
-    # Pull in benchmarker include and exclude list
-    exclude = self.benchmarker.exclude
-    include = self.benchmarker.test
-    if exclude == None:
-        exclude = []
+    tests = gather_tests(include=self.benchmarker.test, 
+      exclude=self.benchmarker.exclude,
+      benchmarker=self.benchmarker)
+    
+    dirs = [t.directory for t in tests]
 
-    # Locate all known tests
+    # Locate all installation files
     install_files = glob.glob("%s/*/install.sh" % self.fwroot)
 
     # Run install for selected tests
     for test_install_file in install_files:
-        test_dir = os.path.dirname(test_install_file)
-        test_name = os.path.basename(test_dir)
-        test_rel_dir = setup_util.path_relative_to_root(test_dir)
-
-        if test_name in exclude:
-            logging.debug("%s has been excluded", test_name)
-            continue
-        elif include is not None and test_name not in include:
-            logging.debug("%s not in include list", test_name)
-            continue
-        else:
-            logging.info("Running installation for %s"%test_name)
-
-            # Find installation directory 
-            # e.g. FWROOT/installs or FWROOT/installs/pertest/<test-name>
-            test_install_dir="%s/%s" % (self.fwroot, self.install_dir)
-            if self.strategy is 'pertest':
-              test_install_dir="%s/pertest/%s" % (test_install_dir, test_name)
-            test_rel_install_dir=setup_util.path_relative_to_root(test_install_dir)
-            if not os.path.exists(test_install_dir):
-              os.makedirs(test_install_dir)
-
-            # Load profile for this installation
-            profile="%s/bash_profile.sh" % test_dir
-            if not os.path.exists(profile):
-              logging.warning("Framework %s does not have a bash_profile"%test_name)
-              profile="$FWROOT/config/benchmark_profile"
-            else:
-              logging.info("Loading environment from %s", profile)
-            setup_util.replace_environ(config=profile, 
-              command='export TROOT=$FWROOT%s && export IROOT=$FWROOT%s' %
-              (test_rel_dir, test_rel_install_dir))
-
-            # Find relative installation file
-            test_rel_install_file = "$FWROOT%s" % setup_util.path_relative_to_root(test_install_file)
-
-            # Then run test installer file
-            # Give all installers a number of variables
-            # FWROOT - Path of the FwBm root
-            # IROOT  - Path of this test's install directory
-            # TROOT  - Path to this test's directory 
-            self.__run_command('''
-              export TROOT=$FWROOT%s && 
-              export IROOT=$FWROOT%s && 
-              . %s && 
-              . %s''' % 
-              (test_rel_dir, test_rel_install_dir, 
-                bash_functions_path, test_rel_install_file),
-                cwd=test_install_dir)
+      test_dir = os.path.basename(os.path.dirname(test_install_file))
+      test_rel_dir = setup_util.path_relative_to_root(test_dir)
+
+      if test_dir not in dirs:
+        continue
+              
+      logging.info("Running installation for directory %s", test_dir)
+
+      # Find installation directory 
+      # e.g. FWROOT/installs or FWROOT/installs/pertest/<test-name>
+      test_install_dir="%s/%s" % (self.fwroot, self.install_dir)
+      if self.strategy is 'pertest':
+        test_install_dir="%s/pertest/%s" % (test_install_dir, test_dir)
+      test_rel_install_dir=setup_util.path_relative_to_root(test_install_dir)
+      if not os.path.exists(test_install_dir):
+        os.makedirs(test_install_dir)
+
+      # Load profile for this installation
+      profile="%s/bash_profile.sh" % test_dir
+      if not os.path.exists(profile):
+        logging.warning("Directory %s does not have a bash_profile"%test_dir)
+        profile="$FWROOT/config/benchmark_profile"
+      else:
+        logging.info("Loading environment from %s", profile)
+      setup_util.replace_environ(config=profile, 
+        command='export TROOT=$FWROOT%s && export IROOT=$FWROOT%s' %
+        (test_rel_dir, test_rel_install_dir))
+
+      # Find relative installation file
+      test_rel_install_file = "$FWROOT%s" % setup_util.path_relative_to_root(test_install_file)
+
+      # Then run test installer file
+      # Give all installers a number of variables
+      # FWROOT - Path of the FwBm root
+      # IROOT  - Path of this test's install directory
+      # TROOT  - Path to this test's directory 
+      self.__run_command('''
+        export TROOT=$FWROOT%s && 
+        export IROOT=$FWROOT%s && 
+        . %s && 
+        . %s''' % 
+        (test_rel_dir, test_rel_install_dir, 
+          bash_functions_path, test_rel_install_file),
+          cwd=test_install_dir)
 
     self.__run_command("sudo apt-get -y autoremove");    
 
@@ -105,7 +102,6 @@ class Installer:
   # End __install_server_software
   ############################################################
 
-
   ############################################################
   # __install_error
   ############################################################

toolset/setup/linux/setup_util.py

@@ -80,7 +80,8 @@ def get_fwroot():
             fwroot = subprocess.check_output('printf $FWROOT 2> /dev/null', shell=True, executable='/bin/bash')
             return fwroot
         except subprocess.CalledProcessError:
-            return "";
+            # Make a last-guess effort ;-)
+            return os.getcwd();
 
 # Turns absolute path into path relative to FWROOT
 # Assumes path is underneath FWROOT, not above