Update Continuous Benchmarking (#2821)

* Update Continuous Benchmarking

* remove the old run-continuously.sh

* fix typo

* Make scripts executable

toolset/continuous/README.md

@@ -0,0 +1,12 @@
+# TFB Continuous Benchmarking
+
+Performs continuous benchmarking of all frameworks and uploads the results to `TFB_UPLOADURI` found in `tfb.conf`.
+
+### Setup
+
+* Move the `tfb.conf` file to `/etc/init`
+* `sudo service tfb start`
+
+### Notes
+
+If `run-continuously.sh` is modified the tfb service will need to be restarted. Alternatively, any new or modified tasks in `run-tasks.sh` will now be updated automatically for each continuous run.

toolset/continuous/run-continuously.sh

@@ -0,0 +1,41 @@
+#!/bin/bash
+#
+# Sets up and runs the benchmarking suite in an infinite loop.  The intent is 
+# this script would be run by a service such as upstart, restarting if the 
+# scripts fail.
+#
+# The following environment variables must
+# be set:
+# $TFB_REPOPARENT   parent folder of the repository
+# $TFB_REPONAME     name of the repository folder
+# $TFB_REPOURI      URI for the git repository
+# $TFB_MAILINGLIST  email address for the mailing list
+# $TFB_LOGSFOLDER   location to transfer logged output
+#
+# @author A. Shawn Bandy
+#
+
+while true
+do
+  # This file is copied outside the repo and run by tfb service
+  # so that it may tear down the environment and run without interruption.
+  # If this file is changed, the tfb service will need to be restarted.
+
+  # Now if tasks are updated in the continuous toolset, they will execute
+  # properly on the next full run.
+
+  # Start by tearing down the environment.
+  # Hold on to the benchmark.cfg it it exists
+  echo Tearing down and rebuilding the environment
+  cp $TFB_REPOPARENT/$TFB_REPONAME/benchmark.cfg $TFB_REPOPARENT/benchmark.cfg 2>/dev/null
+  rm -rf $TFB_REPOPARENT/$TFB_REPONAME
+  cd $TFB_REPOPARENT
+  git clone -b $TFB_REPOBRANCH $TFB_REPOURI $TFB_REPOPARENT/$TFB_REPONAME
+  # Replace the benchmark.cfg
+  cp $TFB_REPOPARENT/benchmark.cfg $TFB_REPOPARENT/$TFB_REPONAME/benchmark.cfg 2>/dev/null
+
+  echo Running continuous tasks
+  $TFB_REPOPARENT/$TFB_REPONAME/toolset/continuous/tasks/run-tasks.sh
+
+  sleep 5
+done

toolset/lifecycle/rebuild-environment.sh → toolset/continuous/tasks/record-git-commit.sh

@@ -1,16 +1,5 @@
 #!/bin/bash
 #
-# rebuild-environment.sh
-#
-# Rebuilds the benchmarking environment.
-#
-cd $TFB_REPOPARENT
-git clone -b $TFB_REPOBRANCH $TFB_REPOURI
-cd $TFB_REPOPARENT/$TFB_REPONAME
-
-#
-# Make the results directory if it doesn't exist.
-#
 # save the current git commit to results/.commit
 #
 echo "Saving current git commit to results directory..."

toolset/continuous/tasks/run-benchmarks.sh

@@ -0,0 +1,8 @@
+#!/bin/bash
+#
+# Start the benchmarks
+
+echo Change to benchmark root
+cd $TFB_REPOPARENT/$TFB_REPONAME
+echo Running tests
+toolset/run-tests.py

toolset/continuous/tasks/run-tasks.sh

@@ -0,0 +1,23 @@
+#!/bin/bash
+#
+# A list of tasks to be run for continuous benchmarking
+#
+
+##########################
+#  Tasks before the run  #
+##########################
+
+$TFB_REPOPARENT/$TFB_REPONAME/continuous/tasks/record-git-commit.sh
+
+
+##########################
+#   Run the benchmarks   #
+##########################
+
+$TFB_REPOPARENT/$TFB_REPONAME/continuous/tasks/run-benchmarks.sh
+
+##########################
+#  Tasks after the run   #
+##########################
+
+$TFB_REPOPARENT/$TFB_REPONAME/continuous/tasks/keep-logs.py

toolset/continuous/tfb.conf

@@ -0,0 +1,29 @@
+# /etc/init/tfb.conf
+env TFB_REPOPARENT="/home/techempower"
+env TFB_CONTINUOUSFOLDER="tfb-continuous"
+env TFB_REPONAME="FrameworkBenchmarks"
+env TFB_REPOURI="https://github.com/TechEmpower/FrameworkBenchmarks.git"
+env TFB_LOGSFOLDER="/home/techempower/logs"
+env TFB_REPOBRANCH="master"
+env TFB_UPLOADURI="http://user:pass@example/upload"
+env HOME="/home/techempower"
+env LANG="en_US.UTF-8"
+setuid techempower
+setgid techempower
+umask 0002
+respawn
+respawn limit 5 2
+#console none
+
+script
+  cd $TFB_REPOPARENT
+  # Clone the repo if it doesn't exist
+  if [ ! -d "$TFB_REPOPARENT/$TFB_REPONAME" ]; then
+    git clone -b $TFB_REPOBRANCH $TFB_REPOURI $TFB_REPOPARENT/$TFB_REPONAME
+  fi
+
+  # Separate the entry point for the continuous run from the repo
+  mkdir -p $TFB_CONTINUOUSFOLDER
+  cp $TFB_REPOPARENT/$TFB_REPONAME/toolset/continuous/run-continuously.sh $TFB_REPOPARENT/$TFB_CONTINUOUSFOLDER
+  exec /$TFB_REPOPARENT/$TFB_CONTINUOUSFOLDER/run-continuously.sh
+end script

toolset/lifecycle/tear-down-environment.sh

@@ -1,13 +0,0 @@
-#!/bin/bash
-#
-# tear-down-environment.sh
-#
-# Tears down the benchmark environment in preparation for another
-# run.
-#
-# @author A. Shawn Bandy
-#
-# $TFB_REPOPARENT parent folder of the repository
-# $TFB_REPONAME   name of the repository folder
-sudo rm -r -f $TFB_REPOPARENT/$TFB_REPONAME
-cd $TFB_REPOPARENT

toolset/run-continuously.sh

@@ -1,58 +0,0 @@
-#!/bin/bash
-#
-# Sets up and runs the benchmarking suite in an infinite loop.  The intent is 
-# this script would be run by a service such as upstart, restarting if the 
-# scripts fail.
-#
-# The following environment variables must
-# be set:
-# $TFB_REPOPARENT   parent folder of the repository
-# $TFB_REPONAME     name of the repository folder
-# $TFB_REPOURI      URI for the git repository
-# $TFB_MAILINGLIST  email address for the mailing list
-# $TFB_LOGSFOLDER   location to transfer logged output
-# 
-# @author A. Shawn Bandy
-#
-while true
-do
-  # Tear down the environment
-  # Note: we tear down first so that we can know the state of the environment
-  # regardless of the outcome of prior runs.
-  if [ -d "$TFB_REPOPARENT/$TFB_REPONAME" ]; then
-    cp $TFB_REPOPARENT/$TFB_REPONAME/benchmark.cfg $TFB_REPOPARENT/
-    cp $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/rebuild-environment.sh $TFB_REPOPARENT/
-    echo Tearing down previous environment
-    $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/tear-down-environment.sh
-  fi
-  # Rebuild environment (i.e. clone the repository)
-  echo Rebuilding environment
-  $TFB_REPOPARENT/rebuild-environment.sh
-  echo Returning benchmark configuration file
-  cp $TFB_REPOPARENT/benchmark.cfg $TFB_REPOPARENT/$TFB_REPONAME/
-  # Handle any preprocessing (e.g. send metadata)
-  echo Running pre-test tasks
-  for SCRIPT in $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/pre-run-tests/*
-    do
-      if [ -f $SCRIPT -a -x $SCRIPT ]
-      then
-        $SCRIPT
-      fi
-    done
-  sleep 5
-  # Run the benchmarks
-  echo Change to benchmark root
-  cd $TFB_REPOPARENT/$TFB_REPONAME
-  echo Run tests
-  toolset/run-tests.py
-  # Handle any postprocessing
-  echo Running post-test tasks
-  for SCRIPT in $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/post-run-tests/*
-    do
-      if [ -f $SCRIPT -a -x $SCRIPT ]
-      then
-        $SCRIPT
-      fi
-    done
-  sleep 5
-done