Use TestRunner from FrameworkTest

toolset/benchmark/framework_test.py

@@ -1,5 +1,6 @@
 from benchmark.fortune_html_parser import FortuneHTMLParser
 from setup.linux import setup_util
+from test_runner import TestRunner
 
 import importlib
 import os
@@ -12,6 +13,7 @@ import traceback
 import json
 import logging
 log = logging.getLogger('framework_test')
+import inspect
 
 from utils import WrapLogger
 from utils import Header
@@ -331,7 +333,23 @@ class FrameworkTest:
     
     set_iroot="export IROOT=%s" % self.install_root
     setup_util.replace_environ(config=profile, command=set_iroot)
-
+    
+    # Determine if setup.py contains a subclass of TestRunner
+    self.runner = None
+    for name, obj in inspect.getmembers(self.setup_module, inspect.isclass):
+      try:
+        is_subclass = TestRunner.is_parent_of(obj)
+      except Exception as e: 
+        logger.critical("%s: %s", self.setup_module.__file__, e)
+        return 1
+
+      if is_subclass:
+        logger.debug("Framework %s is using the new setup.py format" % self.name)
+        self.runner = obj(self, self.setup_module, logger)
+        return self.runner.start()
+
+    # If not, call the start function directly
+    logger.warning("Framework %s is using the old setup.py format" % self.name)
     (out, err) = WrapLogger(logger, logging.INFO), WrapLogger(logger, logging.ERROR)
     return self.setup_module.start(self.benchmarker, out, err)
   ############################################################
@@ -344,6 +362,11 @@ class FrameworkTest:
   ############################################################
   def stop(self, logger=log):
     log.info("stop")
+
+    # Are we using a TestRunner
+    if self.runner: 
+      return self.runner.stop()  
+
     (out, err) = WrapLogger(logger, logging.INFO), WrapLogger(logger, logging.ERROR)
     return self.setup_module.stop(out, err)
   ############################################################
@@ -952,4 +975,4 @@ def parse_config(config, directory, benchmarker):
   return tests
 ##############################################################
 # End parse_config
-##############################################################
+##############################################################

toolset/benchmark/test_runner.py

@@ -2,9 +2,7 @@ import time
 import logging
 
 class TestRunner:
-  def __init__(self, directory, stdout):
-    self.dir = directory
-    self.stdout = stdout
+  iAmTestRunnerClass = True
 
   def sh(self, command, **kwargs):
     kwargs.setdefault('cwd', self.dir)
@@ -18,6 +16,10 @@ class TestRunner:
         self.stdout.write("No Output")
     except subprocess.CalledProcessError:
       self.stdout.write("Process Returned non-zero exit code")
+  def __init__(self, test, target, logger):
+    self.test = test
+    self.target = target
+    self.logger = logger
 
   def sh_async(self, command, initial_logs=True, **kwargs):
     kwargs.setdefault('cwd', self.dir)
@@ -42,7 +44,14 @@ class TestRunner:
         self.stdout.write("Initial Error Logs")
 	for line in err:
           self.stdout.write(line.rstrip('\n'))
+    # Create convenience variables to decouple the 
+    # setup.py scripts from the internals of TFB
+    self.benchmarker = test.benchmarker
+    self.database_host = self.benchmarker.database_host
+    self.dir = test.directory
 
+  def start(self):
+    raise NotImplementedError()
 
 from threading import Thread
 from Queue import Queue, Empty
@@ -58,6 +67,8 @@ class NonBlockingStreamReader:
   def __init__(self, stream):
     self._s = stream
     self._q = Queue()
+  def stop(self):
+    raise NotImplementedError()
 
     def _populateQueue(stream, queue):
       for line in iter(stream.readline, b''):
@@ -70,6 +81,9 @@ class NonBlockingStreamReader:
 
   # TODO  - This is only returning one line, if it is available. 
   def readline(self, timeout = None):
+  @staticmethod
+  def is_parent_of(target_class):
+    ''' Checks if provided class object is a subclass of TestRunner '''
     try:
       return self._q.get(block = timeout is not None,
                  timeout = timeout)
@@ -83,3 +97,20 @@ class NonBlockingStreamReader:
       if not line:
         return lines
       lines.append(line)
+      # issubclass will not work, as setup_module is loaded in different 
+      # global context and therefore has a different copy of the module 
+      # test_runner. A cheap trick is just to check for this attribute
+      str(target_class.iAmTestRunnerClass)
+
+      # target_class seems to be an instance of TestRunner. Before returning,
+      # ensure they have not created an __init__ method, as they cannot 
+      # call super.__init__(), and therefore the subclass is doomed to error
+      try:
+        target_class()
+        raise Exception("Subclasses of TestRunner should not define __init__")
+      except TypeError: 
+        # Good, there is no init method defined
+        return True
+
+    except AttributeError:
+      return False

toolset/run-tests.py

@@ -35,6 +35,8 @@ def main(argv=None):
     # 2) Ensure toolset/setup/linux is in the path so that the tests can "import setup_util".
     sys.path.append('toolset/setup/linux')
 
+    sys.path.append('toolset/benchmark')
+
     # Update environment for shell scripts
     fwroot = setup_util.get_fwroot()
     if not fwroot: