#!/usr/bin/env python import subprocess import os import sys from benchmark import framework_test from benchmark.utils import gather_tests import glob import json import traceback import re import logging log = logging.getLogger('run-ci') import time import threading # Needed for various imports sys.path.append('.') sys.path.append('toolset/setup/linux') sys.path.append('toolset/benchmark') class CIRunnner: ''' Manages running TFB on the Travis Continuous Integration system. Makes a best effort to avoid wasting time and resources by running useless jobs. Only verifies the first test in each directory ''' def __init__(self, mode, testdir=None): ''' mode = [cisetup|jobcleaner|prereq|install|verify] for what we want to do testdir = framework directory we are running ''' logging.basicConfig(level=logging.INFO) self.directory = testdir self.name = testdir # Temporary value, reset below self.mode = mode self.should_run_cache = None self.travis = Travis() try: # See http://git.io/hs_qRQ # TRAVIS_COMMIT_RANGE is empty for pull requests if self.travis.is_pull_req: self.commit_range = "%s..FETCH_HEAD" % os.environ['TRAVIS_BRANCH'].rstrip('\n') else: self.commit_range = os.environ['TRAVIS_COMMIT_RANGE'] except KeyError: log.warning("I should only be used for automated integration tests e.g. Travis-CI") log.warning("Were you looking for run-tests.py?") last_commit = subprocess.check_output("git rev-parse HEAD^", shell=True).rstrip('\n') self.commit_range = "%s...HEAD" % last_commit log.info("Using commit range %s", self.commit_range) log.info("Running `git diff --name-only %s`" % self.commit_range) changes = subprocess.check_output("git diff --name-only %s" % self.commit_range, shell=True) log.info(changes) # Nothing else to setup if mode == 'cisetup' or mode == 'jobcleaner' or mode == 'prereq': return # Should we bother to continue if not self._should_run(): return # # Find the one test from benchmark_config that we are going to run # tests = gather_tests() dirtests = [t for t in tests if t.directory == testdir] # Travis-CI is linux only osvalidtests = [t for t in dirtests if t.os.lower() == "linux" and (t.database_os.lower() == "linux" or t.database_os.lower() == "none")] # Travis-CI only has some supported databases validtests = [t for t in osvalidtests if t.database.lower() == "mysql" or t.database.lower() == "postgres" or t.database.lower() == "none"] log.info("Found %s tests (%s for linux, %s for linux and mysql) in directory '%s'", len(dirtests), len(osvalidtests), len(validtests), testdir) if len(validtests) == 0: log.critical("Found no test that is possible to run in Travis-CI! Aborting!") if len(osvalidtests) != 0: log.critical("Note: Found these tests that could run in Travis-CI if more databases were supported") log.criticat("Note: %s", osvalidtests) sys.exit(1) # Prefer database tests over 'none' if we have both preferred = [t for t in validtests if t.database.lower() != "none"] if len(preferred) > 0: self.test = preferred[0] else: self.test = validtests[0] self.name = self.test.name log.info("Choosing to run test %s in %s", self.name, testdir) def _should_run(self): ''' Decides if the current framework test should be tested or if we can cancel it. Examines git commits included in the latest push to see if any files relevant to this framework were changed. This is a rather primitive strategy for things like pull requests, where we probably want to examine the entire branch of commits. Also, this cannot handle history re-writing very well, so avoid rebasing onto any published history ''' # Don't use git diff twice, it's mega slow sometimes if self.should_run_cache is not None: return self.should_run_cache # Look for changes to core TFB framework code find_tool_changes = "git diff --name-only %s | grep '^toolset/' | wc -l" % self.commit_range changes = subprocess.check_output(find_tool_changes, shell=True) if int(changes) != 0: log.info("Found changes to core framework code") self.should_run_cache = True return True # Look for changes relevant to this test find_test_changes = "git diff --name-only %s | grep '^%s/' | wc -l" % (self.commit_range, self.directory) changes = subprocess.check_output(find_test_changes, shell=True) if int(changes) == 0: log.info("No changes found for %s", self.name) self.should_run_cache = False return False log.info("Changes found for %s", self.name) self.should_run_cache = True return True def run(self): ''' Do the requested command using TFB ''' if self.mode == 'jobcleaner': self.cancel_unneeded_jobs() return 0 if self.mode == 'cisetup' and self._should_run(): self.run_travis_setup() return 0 if not self._should_run(): log.info("Not running %s", self.name) # Cancel ourselves self.travis.cancel(self.travis.jobid) return 0 command = 'toolset/run-tests.py ' if self.mode == 'prereq': command = command + "--install server --install-only --test ''" elif self.mode == 'install': command = command + "--install server --install-only --test %s" % self.name elif self.mode == 'verify': command = command + "--mode verify --test %s" % self.name else: log.critical('Unknown mode passed') return 1 # Run the command log.info("Running mode %s with commmand %s", self.mode, command) try: p = subprocess.Popen(command, shell=True) p.wait() return p.returncode except subprocess.CalledProcessError: log.critical("Subprocess Error") print traceback.format_exc() return 1 except Exception as err: log.critical("Subprocess Error") log.error(err.child_traceback) return 1 def run_travis_setup(self): log.info("Setting up Travis-CI") script = ''' sudo apt-get update sudo apt-get install openssh-server # Run as travis user (who already has passwordless sudo) ssh-keygen -f /home/travis/.ssh/id_rsa -N '' -t rsa cat /home/travis/.ssh/id_rsa.pub > /home/travis/.ssh/authorized_keys chmod 600 /home/travis/.ssh/authorized_keys # Setup database manually # NOTE: Do not run database installation! It restarts mysql with a different # configuration and will break travis's mysql setup mysql -uroot < config/create.sql # Setup Postgres psql --version sudo useradd benchmarkdbuser -p benchmarkdbpass sudo -u postgres psql template1 < config/create-postgres-database.sql sudo -u benchmarkdbuser psql hello_world < config/create-postgres.sql ''' def sh(command): log.info("Running `%s`", command) subprocess.check_call(command, shell=True) for command in script.split('\n'): command = command.lstrip() if command != "" and command[0] != '#': sh(command.lstrip()) # Needed to cancel build jobs from run-ci.py if not self.travis.is_pull_req: sh('time gem install travis -v 1.6.16 --no-rdoc --no-ri') def cancel_unneeded_jobs(self): log.info("I am jobcleaner") log.info("Sleeping to ensure Travis-CI has queued all jobs") time.sleep(20) # Look for changes to core TFB framework code find_tool_changes = "git diff --name-only %s | grep toolset | wc -l" % self.commit_range changes = subprocess.check_output(find_tool_changes, shell=True) if int(changes) != 0: log.info("Found changes to core framework code. Running all tests") self.travis.cancel(self.travis.jobid) # Cancel ourselves return 0 build = self.travis.build_details() log.info("Build details:\n%s", build) def parse_job_id(directory): for line in build.split('\n'): if "TESTDIR=%s" % directory in line: job = re.findall("\d+.\d+", line)[0] return job # Build a list of modified directories changes = subprocess.check_output("git diff --name-only %s" % self.commit_range, shell=True) dirchanges = [] for line in changes.split('\n'): dirchanges.append(line[0:line.find('/')]) # For each test, launch a Thread to cancel it's job if # it's directory has not been modified cancelled_testdirs = [] threads = [] for test in self.gather_tests(): if test.directory not in dirchanges: job = parse_job_id(test.directory) log.info("No changes found for %s (job=%s) (dir=%s)", test.name, job, test.directory) if job and test.directory not in cancelled_testdirs: cancelled_testdirs.append(test.directory) t = threading.Thread(target=self.travis.cancel, args=(job,), name="%s (%s)" % (job, test.name)) t.start() threads.append(t) # Wait for all threads for t in threads: t.join() # Cancel ourselves self.travis.cancel(self.travis.jobid) class Travis(): '''Integrates the travis-ci build environment and the travis command line''' def __init__(self): self.jobid = os.environ['TRAVIS_JOB_NUMBER'] self.buildid = os.environ['TRAVIS_BUILD_NUMBER'] self.is_pull_req = (os.environ['TRAVIS_PULL_REQUEST'] != "false") self.logged_in = False def _login(self): if self.logged_in: return # If this is a PR, we cannot access the secure variable # GH_TOKEN, and instead must return success for all jobs if not self.is_pull_req: self.token = os.environ['GH_TOKEN'] subprocess.check_call("travis login --skip-version-check --no-interactive --github-token %s" % self.token, shell=True) log.info("Logged into travis") # NEVER PRINT OUTPUT, GH_TOKEN MIGHT BE REVEALED else: log.info("Pull Request Detected. Non-necessary jobs will return pass instead of being canceled") self.logged_in = True def cancel(self, job): self._login() # If this is a pull request, we cannot interact with the CLI if self.is_pull_req: log.info("Thread %s: Return pass for job %s", threading.current_thread().name, job) return # Ignore errors in case job is already cancelled try: subprocess.check_call("travis cancel %s --skip-version-check --no-interactive" % job, shell=True) log.info("Thread %s: Canceled job %s", threading.current_thread().name, job) except subprocess.CalledProcessError: log.exception("Error halting job %s. Report:", job) subprocess.call("travis report --skip-version-check --no-interactive --org", shell=True) log.error("Trying to halt %s one more time", job) subprocess.call("travis cancel %s --skip-version-check --no-interactive" % job, shell=True) def build_details(self): self._login() # If this is a pull request, we cannot interact with the CLI if self.is_pull_req: return "No details available" build = subprocess.check_output("travis show %s --skip-version-check" % self.buildid, shell=True) return build if __name__ == "__main__": args = sys.argv[1:] usage = '''Usage: toolset/run-ci.py [cisetup|prereq] OR toolset/run-ci.py [install|verify] run-ci.py selects one test from /benchark_config, and automates a number of calls into run-tests.py specific to the selected test. It is guaranteed to always select the same test from the benchark_config, so multiple runs with the same reference the same test. The name of the selected test will be printed to standard output. cisetup - configure the Travis-CI environment for our test suite prereq - trigger standard prerequisite installation install - trigger server installation for the selected test_directory verify - run a verification on the selected test using `--mode verify` run-ci.py expects to be run inside the Travis-CI build environment, and will expect environment variables such as $TRAVIS_BUILD''' mode = args[0] if mode == 'cisetup' or mode == 'prereq': runner = CIRunnner(mode) elif len(args) == 2 and args[1] == 'jobcleaner': # Only run jobcleaner once if mode != 'verify': sys.exit(0) # Translate jobcleaner from a directory name to a mode mode = 'jobcleaner' runner = CIRunnner(args[1]) elif len(args) == 2 and (mode == "install" or mode == "verify"): runner = CIRunnner(mode, args[1]) else: print usage sys.exit(1) retcode = 0 try: retcode = runner.run() except KeyError as ke: log.warning("Environment key missing, are you running inside Travis-CI?") print traceback.format_exc() except: log.critical("Unknown error") print traceback.format_exc() finally: # Ensure that logs are printed # Only print logs if we are not jobcleaner and we ran a verify if mode == 'jobcleaner' or mode != 'verify': sys.exit(retcode) log.error("Running inside travis, so I will print err and out to console") try: log.error("Here is ERR:") with open("results/ec2/latest/logs/%s/err.txt" % runner.test.name, 'r') as err: for line in err: log.info(line) except IOError: log.error("No ERR file found") try: log.error("Here is OUT:") with open("results/ec2/latest/logs/%s/out.txt" % runner.test.name, 'r') as out: for line in out: log.info(line) except IOError: log.error("No OUT file found") sys.exit(retcode)