Switch to GitHub Actions for CI (again) (#6260)

* Switch to GitHub Actions for CI

* Add new TESTDIR

* Remove comment for Scheme

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,7 @@
 <!--
 Thank you for submitting to the TechEmpower Framework Benchmarks!
 
-If you are submitting a new framework, please make sure that you add the appropriate line in the `.travis.yml` file for proper integration testing. Also please make sure that an appropriate `README.md` is added in your framework directory with information about the framework and a link to its homepage and documentation.
+If you are submitting a new framework, please make sure that you add the appropriate line in the `.github/workflows/build.yml` file for proper integration testing. Also please make sure that an appropriate `README.md` is added in your framework directory with information about the framework and a link to its homepage and documentation.
 
 For new frameworks, please do not include source code that isn't required for the benchmarks.
 

.github/workflows/build.yml

@@ -0,0 +1,195 @@
+name: build
+on:
+  push:
+    branches: [ master ]
+  pull_request:
+    branches: [ master ]
+jobs:
+  setup:
+    runs-on: ubuntu-18.04
+    steps:
+      # Commit branch/name extraction from:
+      # https://github.community/t/accessing-commit-message-in-pull-request-event/17158/2
+      #
+      # We need to fetch more than one commit to be able to access HEAD^2 in case
+      # of a pull request
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 10
+      # In case of a push event, the commit we care about is simply HEAD.
+      # The current branch name can be found by parsing GITHUB_REF, for example,
+      # if we are on the master branch, then GITHUB_REF = refs/heads/master.
+      - name: Get commit branch and commit message from push
+        if: github.event_name == 'push'
+        run: |
+          echo "BRANCH_NAME=$(echo ${GITHUB_REF##*/})" >> $GITHUB_ENV
+          echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(git log --format=%B -n 1 HEAD)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      # In case of a pull_request event, the commit we care about is HEAD^2, that
+      # is, the second parent of the pull request merge commit.
+      # The current branch name is directly given by GITHUB_HEAD_REF
+      - name: Get commit branch and commit message from PR
+        if: github.event_name == 'pull_request'
+        run: |
+          echo "BRANCH_NAME=$GITHUB_HEAD_REF" >> $GITHUB_ENV
+          echo "COMMIT_MESSAGE<<EOF" >> $GITHUB_ENV
+          echo "$(git log --format=%B -n 1 HEAD^2)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - id: out
+        name: Write event outputs
+        run: |
+          # Escape the multiline string for Github Actions, see https://github.community/t/set-output-truncates-multiline-strings/16852/3
+          COMMIT_MESSAGE="${COMMIT_MESSAGE//'%'/'%25'}"
+          COMMIT_MESSAGE="${COMMIT_MESSAGE//$'\n'/'%0A'}"
+          COMMIT_MESSAGE="${COMMIT_MESSAGE//$'\r'/'%0D'}"
+          echo "::set-output name=commit_message::$COMMIT_MESSAGE"
+          echo "::set-output name=branch_name::$BRANCH_NAME"
+    outputs:
+      commit_message: ${{ steps.out.outputs.commit_message }}
+      branch_name: ${{ steps.out.outputs.branch_name }}
+  test:
+    needs: setup
+    if: ${{ !contains(needs.setup.outputs.commit_message, '[ci skip]') }}
+    runs-on: ubuntu-18.04
+    strategy:
+      matrix:
+        include:
+          - TESTLANG: "C"
+          - TESTLANG: "CSharp"
+          - TESTDIR: "C++/cppcms C++/cpoll_cppsp C++/poco"
+          - TESTDIR: "C++/ffead-cpp"
+          - TESTDIR: "C++/cuehttp"
+          - TESTDIR: "C++/cutelyst"
+          - TESTDIR: "C++/libhttpserver"
+          - TESTDIR: "C++/silicon"
+          - TESTDIR: "C++/lithium"
+          - TESTDIR: "C++/treefrog"
+          - TESTDIR: "C++/ulib"
+          - TESTDIR: "C++/wt"
+          - TESTDIR: "C++/drogon"
+          - TESTDIR: "C++/oatpp"
+          - TESTDIR: "C++/libsniper"
+          - TESTLANG: "Clojure"
+          - TESTLANG: "Crystal"
+          - TESTLANG: "D"
+          - TESTLANG: "Dart"
+          - TESTLANG: "Dylan"
+          - TESTLANG: "Elixir"
+          - TESTLANG: "Erlang"
+          - TESTLANG: "FSharp"
+          - TESTDIR: "Go/chi Go/gin Go/goji Go/aah Go/beego Go/echo Go/gnet"
+          - TESTDIR: "Go/falcore Go/fiber Go/kami Go/martini Go/revel Go/webgo"
+          - TESTDIR: "Go/evio Go/fasthttp Go/go-std Go/atreugo Go/gramework"
+          - TESTDIR: "Go/gearbox Go/goframe Go/clevergo"
+          - TESTLANG: "Groovy"
+          - TESTDIR: "Haskell/ihp"
+          - TESTDIR: "Haskell/snap"
+          - TESTDIR: "Haskell/yesod"
+          - TESTDIR: "Haskell/servant"
+          - TESTDIR: "Haskell/spock"
+          - TESTDIR: "Haskell/warp"
+          - TESTDIR: "Haskell/wizzardo-inline"
+          - TESTDIR: "Haskell/postgrest"
+          - TESTDIR: "Java/act Java/comsat Java/edap-http"
+          - TESTDIR: "Java/activeweb Java/armeria Java/baratine Java/bayou Java/blade Java/curacao Java/dropwizard Java/firenio Java/servicetalk Java/voovan"
+          - TESTDIR: "Java/gemini Java/greenlightning Java/grizzly Java/helidon Java/httpserver Java/jetty Java/jlhttp Java/jooby Java/wicket"
+          - TESTDIR: "Java/light-java Java/minijax Java/nanohttpd Java/netty Java/ninja-standalone Java/officefloor Java/proteus Java/quarkus"
+          - TESTDIR: "Java/rapidoid Java/redkale Java/restexpress Java/revenj-jvm Java/servlet Java/servlet3 Java/smart-socket Java/spark"
+          - TESTDIR: "Java/spring Java/spring-webflux Java/t-io Java/tapestry Java/undertow Java/undertow-jersey Java/vertx Java/vertx-web Java/simple-server"
+          - TESTDIR: "Java/javalin Java/jawn Java/ratpack Java/wizzardo-http Java/jersey"
+          - TESTDIR: "Java/play1 Java/play2-java Java/wildfly-ee"
+          - TESTDIR: "JavaScript/0http JavaScript/express JavaScript/fastify JavaScript/hapi JavaScript/koa"
+          - TESTDIR: "JavaScript/nodejs JavaScript/polkadot JavaScript/restana JavaScript/restify JavaScript/sailsjs"
+          - TESTDIR: "JavaScript/es4x JavaScript/ringojs JavaScript/just"
+          - TESTLANG: "Julia"
+          - TESTLANG: "Kotlin"
+          - TESTLANG: "Lisp"
+          - TESTLANG: "Lua"
+          - TESTLANG: "Mumps"
+          - TESTLANG: "Nim"
+          - TESTLANG: "OCaml"
+          - TESTLANG: "Perl"
+          - TESTDIR: "PHP/php"
+          - TESTDIR: "PHP/comet PHP/kumbiaphp PHP/workerman PHP/webman"
+          - TESTDIR: "PHP/cakephp PHP/codeigniter PHP/fat-free PHP/fuel PHP/phpixie PHP/slim PHP/symfony PHP/yii2 PHP/zend PHP/spiral PHP/duckphp"
+          - TESTDIR: "PHP/amp PHP/hhvm PHP/peachpie PHP/php-ngx PHP/phalcon"
+          - TESTDIR: "PHP/hamlet PHP/laravel PHP/lumen PHP/swoole PHP/ubiquity PHP/hyperf PHP/sw-fw-less PHP/imi PHP/simps PHP/one"
+          - TESTLANG: "Prolog"
+          - TESTDIR: "Python/aiohttp Python/api_hour Python/apidaora Python/blacksheep Python/bottle Python/cherrypy Python/crax Python/django Python/emmett Python/eve Python/falcon Python/fastapi Python/flask"
+          - TESTDIR: "Python/hug Python/japronto Python/klein Python/morepath Python/pyramid Python/quart Python/responder Python/sanic Python/spyne Python/starlette"
+          - TESTDIR: "Python/tornado Python/turbogears Python/uvicorn Python/uwsgi Python/vibora Python/web2py Python/webware Python/weppy Python/wsgi"
+          - TESTDIR: "Ruby/agoo Ruby/grape Ruby/h2o_mruby Ruby/padrino Ruby/rack Ruby/rack-sequel"
+          - TESTDIR: "Ruby/rails Ruby/roda-sequel Ruby/sinatra Ruby/sinatra-sequel"
+          - TESTDIR: "Rust/actix Rust/gotham Rust/hyper Rust/iron Rust/saphir"
+          - TESTDIR: "Rust/may-minihttp Rust/nickel Rust/rocket"
+          - TESTDIR: "Rust/rouille Rust/thruster Rust/tokio-minihttp"
+          - TESTDIR: "Rust/warp-rust"
+          - TESTDIR: "Rust/roa"
+          - TESTDIR: "Rust/ntex"
+          - TESTDIR: "Scala/akka-http Scala/blaze Scala/cask Scala/colossus Scala/finagle"
+          - TESTDIR: "Scala/finatra Scala/finch Scala/http4s"
+          - TESTDIR: "Scala/play2-scala Scala/scalene Scala/youi"
+          - TESTDIR: "Scala/snunit"
+          - TESTDIR: "Scala/vertx-web-scala"
+          - TESTLANG: "Swift"
+          - TESTLANG: "TypeScript"
+          - TESTLANG: "Ur"
+          - TESTLANG: "V"
+          - TESTLANG: "Vala"
+          - TESTLANG: "VB"
+          - TESTLANG: "Mumps"
+      # Disable fail-fast to allow all failing frameworks/etc to fail in a
+      # single build, rather than stopping when the first one fails.
+      fail-fast: false
+    env:
+      TESTLANG: ${{ matrix.TESTLANG }}
+      TESTDIR: ${{ matrix.TESTDIR }}
+      COMMIT_MESSAGE: ${{ needs.setup.outputs.commit_message }}
+      BRANCH_NAME: ${{ needs.setup.outputs.branch_name }}
+      PR_NUMBER: ${{ github.event.pull_request.number }}
+    steps:
+      - uses: actions/checkout@v2
+        with:
+          fetch-depth: 10
+      - uses: actions/setup-python@v2
+        with:
+          python-version: '2.7'
+          architecture: 'x64'
+      - id: get_diff
+        name: Get all changes vs master
+        # Runs github_actions_diff, with the the output accessible in later steps
+        run: |
+          # Write the result to env.DIFF for later steps
+          echo "DIFF<<EOF" >> $GITHUB_ENV
+          echo "$(./toolset/github_actions/github_actions_diff.py)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - id: should_run_tests
+        name: Determine which (if any) tests need to be run
+        # Searches for github-actions-diff-continue to determine if the suite should be installed and the current $TESTDIR test should run.
+        run: |
+          # grep returns status code 1 if no matches are found. This fails the
+          # build as it is a non-zero status. But this is an expected
+          # possibility, so `|| true` is used to address/silence that.
+          # Write the result to env.RUN_TESTS for later steps
+          echo "RUN_TESTS<<EOF" >> $GITHUB_ENV
+          echo "$(grep -oP "github-actions-run-tests \K(.*)" <<< $DIFF || true)" >> $GITHUB_ENV
+          echo "EOF" >> $GITHUB_ENV
+      - name: Log status
+        run: |
+          if [ "$RUN_TESTS" ]; then echo "Proceeding to run tests."; else echo 'Skipping test verification.'; fi
+      - name: Build tfb dockerfile
+        if: ${{ env.RUN_TESTS }}
+        run: docker build -t techempower/tfb - < ./Dockerfile;
+      - name: Stop services
+        # Stop services that would claim ports we may need
+        run: |
+          sudo service mysql stop || true
+          sudo service postgresql stop || true
+      - name: Run tests if needed
+        if: ${{ env.RUN_TESTS }}
+        run: |
+          # run-ci.py runs the diffing to see if github actions needs to test this framework. Ideally/eventually,
+          # we'd like to try and do the diffing before github_actions_clean & setup.
+          # This will run the tests exactly as you would in your own vm:
+          docker network create tfb > /dev/null 2>&1 && docker run --network=tfb -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=`pwd`,target=/FrameworkBenchmarks techempower/tfb --mode verify --test-dir $RUN_TESTS --results-environment Github-Actions;

.travis.yml

@@ -1,116 +0,0 @@
-# Travis CI
-#
-#
-
-sudo: required
-dist: bionic
-language: generic
-python:
-  - "2.7"
-services:
-  - docker
-
-env:
-  matrix:
-    - "TESTLANG=C"
-    - "TESTLANG=CSharp"
-    - 'TESTDIR="C++/cppcms C++/cpoll_cppsp C++/poco"'
-    - "TESTDIR=C++/ffead-cpp"
-    - "TESTDIR=C++/cuehttp"
-    - "TESTDIR=C++/cutelyst"
-    - "TESTDIR=C++/libhttpserver"
-    - "TESTDIR=C++/silicon"
-    - "TESTDIR=C++/lithium"
-    - "TESTDIR=C++/treefrog"
-    - "TESTDIR=C++/ulib"
-    - "TESTDIR=C++/wt"
-    - "TESTDIR=C++/drogon"
-    - "TESTDIR=C++/oatpp"
-    - "TESTDIR=C++/libsniper"
-    - "TESTLANG=Clojure"
-    - "TESTLANG=Crystal"
-    - "TESTLANG=D"
-    - "TESTLANG=Dart"
-    - "TESTLANG=Dylan"
-    - "TESTLANG=Elixir"
-    - "TESTLANG=Erlang"
-    - "TESTLANG=FSharp"
-    - 'TESTDIR="Go/chi Go/gin Go/goji Go/aah Go/beego Go/echo Go/gnet"'
-    - 'TESTDIR="Go/falcore Go/fiber Go/kami Go/martini Go/revel Go/webgo"'
-    - 'TESTDIR="Go/evio Go/fasthttp Go/go-std Go/atreugo Go/gramework"'
-    - 'TESTDIR="Go/gearbox Go/goframe Go/clevergo"'
-    - "TESTLANG=Groovy"
-    - "TESTDIR=Haskell/ihp"
-    - "TESTDIR=Haskell/snap"
-    - "TESTDIR=Haskell/yesod"
-    - "TESTDIR=Haskell/servant"
-    - "TESTDIR=Haskell/spock"
-    - "TESTDIR=Haskell/warp"
-    - "TESTDIR=Haskell/wizzardo-inline"
-    - "TESTDIR=Haskell/postgrest"
-    - 'TESTDIR="Java/act Java/comsat Java/edap-http"'
-    - 'TESTDIR="Java/activeweb Java/armeria Java/baratine Java/bayou Java/blade Java/curacao Java/dropwizard Java/firenio Java/servicetalk Java/voovan"'
-    - 'TESTDIR="Java/gemini Java/greenlightning Java/grizzly Java/helidon Java/httpserver Java/jetty Java/jlhttp Java/jooby Java/wicket"'
-    - 'TESTDIR="Java/light-java Java/minijax Java/nanohttpd Java/netty Java/ninja-standalone Java/officefloor Java/proteus Java/quarkus"'
-    - 'TESTDIR="Java/rapidoid Java/redkale Java/restexpress Java/revenj-jvm Java/servlet Java/servlet3 Java/smart-socket Java/spark"'
-    - 'TESTDIR="Java/spring Java/spring-webflux Java/t-io Java/tapestry Java/undertow Java/undertow-jersey Java/vertx Java/vertx-web Java/simple-server"'
-    - 'TESTDIR="Java/javalin Java/jawn Java/ratpack Java/wizzardo-http Java/jersey"'
-    - 'TESTDIR="Java/play1 Java/play2-java Java/wildfly-ee"'
-    - 'TESTDIR="JavaScript/0http JavaScript/express JavaScript/fastify JavaScript/hapi JavaScript/koa"'
-    - 'TESTDIR="JavaScript/nodejs JavaScript/polkadot JavaScript/restana JavaScript/restify JavaScript/sailsjs"'
-    - 'TESTDIR="JavaScript/es4x JavaScript/ringojs JavaScript/just"'
-    - "TESTLANG=Julia"
-    - "TESTLANG=Kotlin"
-    - "TESTLANG=Lisp"
-    - "TESTLANG=Lua"
-    - "TESTLANG=Mumps"
-    - "TESTLANG=Nim"
-    - "TESTLANG=OCaml"
-    - "TESTLANG=Perl"
-    - 'TESTDIR="PHP/php"'
-    - 'TESTDIR="PHP/comet PHP/kumbiaphp PHP/workerman PHP/webman"'
-    - 'TESTDIR="PHP/cakephp PHP/codeigniter PHP/fat-free PHP/fuel PHP/phpixie PHP/slim PHP/symfony PHP/yii2 PHP/zend PHP/spiral PHP/duckphp"'
-    - 'TESTDIR="PHP/amp PHP/hhvm PHP/peachpie PHP/php-ngx PHP/phalcon"'
-    - 'TESTDIR="PHP/hamlet PHP/laravel PHP/lumen PHP/swoole PHP/ubiquity PHP/hyperf PHP/sw-fw-less PHP/imi PHP/simps PHP/one"'
-    - "TESTLANG=Prolog"
-    - 'TESTDIR="Python/aiohttp Python/api_hour Python/apidaora Python/blacksheep Python/bottle Python/cherrypy Python/crax Python/django Python/emmett Python/eve Python/falcon Python/fastapi Python/flask"'
-    - 'TESTDIR="Python/hug Python/japronto Python/klein Python/morepath Python/pyramid Python/quart Python/responder Python/sanic Python/spyne Python/starlette"'
-    - 'TESTDIR="Python/tornado Python/turbogears Python/uvicorn Python/uwsgi Python/vibora Python/web2py Python/webware Python/weppy Python/wsgi"'
-    - 'TESTDIR="Ruby/agoo Ruby/grape Ruby/h2o_mruby Ruby/padrino Ruby/rack Ruby/rack-sequel"'
-    - 'TESTDIR="Ruby/rails Ruby/roda-sequel Ruby/sinatra Ruby/sinatra-sequel"'
-    - 'TESTDIR="Rust/actix Rust/gotham Rust/hyper Rust/iron Rust/saphir"'
-    - 'TESTDIR="Rust/may-minihttp Rust/nickel Rust/rocket"'
-    - 'TESTDIR="Rust/rouille Rust/thruster Rust/tokio-minihttp"'
-    - 'TESTDIR="Rust/warp-rust"'
-    - 'TESTDIR="Rust/roa"'
-    - 'TESTDIR="Rust/ntex"'
-    - 'TESTDIR="Scala/akka-http Scala/blaze Scala/cask Scala/colossus Scala/finagle"'
-    - 'TESTDIR="Scala/finatra Scala/finch Scala/http4s"'
-    - 'TESTDIR="Scala/play2-scala Scala/scalene Scala/youi"'
-    - 'TESTDIR="Scala/snunit"'
-    - 'TESTDIR="Scala/vertx-web-scala"'
-    - "TESTLANG=Scheme"
-    - "TESTLANG=Swift"
-    - "TESTLANG=TypeScript"
-    - "TESTLANG=Ur"
-    - "TESTLANG=V"
-    - "TESTLANG=Vala"
-    - "TESTLANG=VB"
-    - "TESTLANG=Mumps"
-
-before_script:
-  # Runs travis_diff, printing the output to the terminal, and searches for travis-diff-continue
-  # to determine if the suite should be installed and the current $TESTDIR test should run.
-  - export RUN_TESTS=`./toolset/travis/travis_diff.py | tee /dev/tty | grep -oP "travis-run-tests \K(.*)"`
-
-  - if [ "$RUN_TESTS" ]; then docker build -t techempower/tfb - < ./Dockerfile; fi
-
-  # Stop services that would claim ports we may need
-  - sudo service mysql stop || true
-  - sudo service postgresql stop || true
-
-script:
-  # run-ci.py runs the diffing to see if travis needs to test this framework. Ideally/eventually,
-  # we'd like to try and do the diffing before travis_clean & setup.
-  # This will run the tests exactly as you would in your own vm:
-  - if [ "$RUN_TESTS" ]; then docker network create tfb > /dev/null 2>&1 && docker run --network=tfb -v /var/run/docker.sock:/var/run/docker.sock --mount type=bind,source=`pwd`,target=/FrameworkBenchmarks techempower/tfb --mode verify --test-dir $RUN_TESTS --results-environment Travis; else echo 'Skipping test verification.'; fi

README.md

@@ -1,6 +1,6 @@
 # Welcome to [TechEmpower Framework Benchmarks (TFB)](http://www.techempower.com/benchmarks/)
 
-[![Build Status](https://travis-ci.org/TechEmpower/FrameworkBenchmarks.svg?branch=master)](https://travis-ci.org/TechEmpower/FrameworkBenchmarks)
+[![Build Status](https://github.com/TechEmpower/FrameworkBenchmarks/workflows/build/badge.svg?branch=master&event=push)](https://github.com/TechEmpower/FrameworkBenchmarks/actions?query=workflow%3Abuild+branch%3Amaster)
 
 If you're new to the project, welcome! Please feel free to ask questions [here](https://github.com/TechEmpower/FrameworkBenchmarks/issues/2978). We encourage new frameworks and contributors to ask questions. We're here to help!
 

toolset/travis/travis_diff.py → toolset/github_actions/github_actions_diff.py

@@ -1,15 +1,15 @@
 #!/usr/bin/env python
 
-# @file:        toolset/travis/travis_diff.py
+# @file:        toolset/github_actions/github_actions_diff.py
 # @author:      Nate Brady
 #
-# @description: This script is only for use within Travis-CI. It is meant to
-# look through the commit history and determine whether or not the current
+# @description: This script is only for use within Github Actions. It is meant
+# to look through the commit history and determine whether or not the current
 # framework test directory needs to be run. It compares the state of the PR
 # branch against the target branch.
 #
-# Any changes found in the toolset/* directory other than continuous/*, travis/*
-# and scaffolding/* will cause all tests to be run.
+# Any changes found in the toolset/* directory other than continuous/*,
+# github_actions/* and scaffolding/* will cause all tests to be run.
 #
 # The following commands can be put in commit messages to affect which tests
 # will run:
@@ -50,22 +50,23 @@ def clean_output(output):
 
 def quit_diffing():
     if len(run_tests):
-        print("travis-run-tests {!s}".format(" ".join(set(run_tests))))
+        print("github-actions-run-tests {!s}".format(" ".join(set(run_tests))))
     else:
         print("No tests to run.")
     exit(0)
 
 
 curr_branch = ""
-is_PR = (os.getenv("TRAVIS_PULL_REQUEST") != "false")
-# TRAVIS_BRANCH is the target branch when it's a pull request or the name
-# of the branch when it isn't
-is_master = not is_PR and os.getenv("TRAVIS_BRANCH") == "master"
+is_PR = (os.getenv("PR_NUMBER") != "")
+# BRANCH_NAME is the the name of the branch
+is_master = os.getenv("BRANCH_NAME") == "master"
 
 if is_PR:
-    curr_branch = "FETCH_HEAD"
+    curr_branch = "HEAD"
 elif not is_master:
-    curr_branch = os.getenv("TRAVIS_COMMIT")
+    curr_branch = os.getenv("GITHUB_SHA")
+
+if not is_master:
     # Also fetch master to compare against
     subprocess.check_output(['bash', '-c', 'git fetch origin master:master'])
 
@@ -85,7 +86,7 @@ if len(changes.split('\n')) > 10:
 # COMMIT MESSAGES:
 # Before any complicated diffing, check for forced runs from the commit message
 # Use -2 because travis now inserts a merge commit as the last commit
-last_commit_msg = os.getenv("TRAVIS_COMMIT_MESSAGE")
+last_commit_msg = os.getenv("COMMIT_MESSAGE")
 
 test_dirs = []
 run_tests = []
@@ -99,7 +100,7 @@ elif os.getenv("TESTDIR"):
     test_dirs = os.getenv("TESTDIR").split(' ')
 
 # Forced full run
-if is_master or re.search(r'\[ci run-all\]', last_commit_msg, re.M):
+if (not is_PR and is_master) or re.search(r'\[ci run-all\]', last_commit_msg, re.M):
     print("All tests have been forced to run from the commit message.")
     run_tests = test_dirs
     quit_diffing()
@@ -145,8 +146,8 @@ if re.search(r'\[ci lang .+\]', last_commit_msg, re.M):
                 run_tests.append(test)
 
 
-# Ignore travis, continuous and scaffolding changes
-if re.search(r'^toolset\/(?!(travis\/|continuous\/|scaffolding\/))', changes, re.M) is not None:
+# Ignore travis, github_actions, continuous and scaffolding changes
+if re.search(r'^toolset\/(?!(travis\/|github_actions\/|continuous\/|scaffolding\/))', changes, re.M) is not None:
     print("Found changes to core toolset. Running all tests.")
     run_tests = test_dirs
     quit_diffing()

toolset/scaffolding/README.md

@@ -28,9 +28,9 @@ This will run the suite in `verify` mode for your test. This means that no bench
 
 Once you are able to successfully run your test through our suite in this way **and** your test passes our validation, you may move on to the next step.
 
-5. Add your test to `.travis.yml`
+5. Add your test to `.github/workflows/build.yml`
 
-Edit `.travis.yml` to ensure that Travis-CI will automatically run our verification tests against your new test. This file is kept in alphabetical order, so find where `TESTDIR=$LANGUAGE/$NAME` should be inserted under `env > matrix` and put it there.
+Edit `.github/workflows/build.yml` to ensure that Github Actions will automatically run our verification tests against your new test. This file is kept in alphabetical order, so find where `TESTDIR=$LANGUAGE/$NAME` should be inserted under `env > matrix` and put it there.
 
 6. Fix this `README.md` and open a pull request