Travis and --test-lang option (#3332)

.travis.yml

@@ -11,12 +11,7 @@ python:
 
 env:
   matrix:
-     - "TESTDIR=C/duda"
-     - "TESTDIR=C/facil.io"
-     - "TESTDIR=C/h2o"
-     - "TESTDIR=C/libreactor"
-     - "TESTDIR=C/octane"
-     - "TESTDIR=C/onion"
+     - "TESTLANG=C"
     # - "TESTDIR=CSharp/aspnet"
     # - "TESTDIR=CSharp/aspnetcore"
     # ## - "TESTDIR=CSharp/aspnet-stripped"
@@ -112,12 +107,7 @@ env:
     # - "TESTDIR=Java/vertx-web"
     # - "TESTDIR=Java/wicket"
     # - "TESTDIR=Java/wildfly-ee7"
-     - "TESTDIR=JavaScript/express"
-     - "TESTDIR=JavaScript/hapi"
-     - "TESTDIR=JavaScript/koa"
-     - "TESTDIR=JavaScript/nodejs"
-     - "TESTDIR=JavaScript/ringojs"
-     - "TESTDIR=JavaScript/sailsjs"
+     - "TESTLANG=JavaScript"
     # - "TESTDIR=Kotlin/hexagon"
     # - "TESTDIR=Kotlin/ktor"
     # - "TESTDIR=Kotlin/pronghorn"
@@ -235,7 +225,8 @@ script:
   # we'd like to try and do the diffing before travis_clean & setup.
   # This will run the tests exactly as you would in your own vm:
   # ./toolset/run-tests.py --mode verify --test (all the valid tests for this framework)
-  - if [ $CONTINUE_TEST ]; then tfb --mode verify --test-dir "$TESTDIR"; else echo 'Skipping test verification.'; fi
+  - if [[ $CONTINUE_TEST && "$TESTDIR" ]]; then tfb --mode verify --test-dir "$TESTDIR"; else echo 'Skipping test verification.'; fi
+  - if [[ $CONTINUE_TEST && "$TESTLANG" ]]; then tfb --mode verify --test-lang "$TESTLANG"; else echo 'Skipping test verification.'; fi
   # - tfb --mode verify --test
 
 cache:

toolset/benchmark/framework_test.py

@@ -192,9 +192,10 @@ class FrameworkTest:
 
     prefix = "Setup %s: " % self.name
 
-    ##########################
+    ###########################
     # Build the Docker images
-    ##########################
+    ###########################
+
     test_docker_file = os.path.join(self.directory, "%s.dockerfile" % self.name)
     deps = list(reversed(gather_docker_dependencies( test_docker_file )))
 

toolset/benchmark/utils.py

@@ -106,6 +106,7 @@ def gather_tests(include = [], exclude=[], benchmarker=None):
         defaults['results_name'] = "(unspecified, datetime = %Y-%m-%d %H:%M:%S)"
         defaults['results_environment'] = "My Server Environment"
         defaults['test_dir'] = None
+        defaults['test_lang'] = None
         defaults['quiet'] = True
 
         benchmarker = Benchmarker(defaults)
@@ -114,6 +115,16 @@ def gather_tests(include = [], exclude=[], benchmarker=None):
     # Search for configuration files
     fwroot = setup_util.get_fwroot()
     config_files = []
+
+    if benchmarker.test_lang:
+        benchmarker.test_dir = []
+        for lang in benchmarker.test_lang:
+            if os.path.exists("{!s}/frameworks/{!s}".format(fwroot, lang)):
+                for test_dir in os.listdir("{!s}/frameworks/{!s}".format(fwroot, lang)):
+                    benchmarker.test_dir.append("{!s}/{!s}".format(lang, test_dir))
+            else:
+                raise Exception("Unable to locate language directory: {!s}".format(lang))
+
     if benchmarker.test_dir:
         for test_dir in benchmarker.test_dir:
             dir_config_files = glob.glob("{!s}/frameworks/{!s}/benchmark_config.json".format(fwroot, test_dir))

toolset/run-tests.py

@@ -152,6 +152,7 @@ def main(argv=None):
     # Test options
     parser.add_argument('--test', nargs='+', help='names of tests to run')
     parser.add_argument('--test-dir', nargs='+', dest='test_dir', help='name of framework directory containing all tests to run')
+    parser.add_argument('--test-lang', nargs='+', dest='test_lang', help='name of language directory containing all tests to run')
     parser.add_argument('--exclude', nargs='+', help='names of tests to exclude')
     parser.add_argument('--type', choices=['all', 'json', 'db', 'query', 'cached_query', 'fortune', 'update', 'plaintext'], default='all', help='which type of test to run')
     parser.add_argument('-m', '--mode', choices=['benchmark', 'verify', 'debug'], default='benchmark', help='verify mode will only start up the tests, curl the urls and shutdown. debug mode will skip verification and leave the server running.')

toolset/travis/travis_diff.py

@@ -34,6 +34,8 @@ def quit_diffing(should_test_run):
         print("travis-diff-continue")
     exit(0)
 
+# TODO: Remove this
+quit_diffing(True)
 
 # COMMIT MESSAGES:
 # Before any complicated diffing, check for forced runs from the commit message