Add initial continuous benchmarking (#2292)

config/upstart.example/tfb.conf

@@ -0,0 +1,19 @@
+# /etc/init/tfb.conf
+env TFB_REPOPARENT="/private"
+env TFB_REPONAME="FrameworkBenchmarks"
+env TFB_REPOURI="https://github.com/ashawnbandy-te-tfb/FrameworkBenchmarks.git"
+env TFB_MAILINGLIST="[email protected]"
+env TFB_MAILING_FROM="[email protected]"
+env TFB_LOGSFOLDER="/private/logs"
+env TFB_REPOBRANCH="45216-continuousbenchmarking-20160609-asb-2"
+setuid techempower
+setgid techempower
+umask 0002
+respawn
+respawn limit 5 2
+script
+  if [ ! -d "$TFB_REPOPARENT/$TFB_REPONAME" ]; then
+    git clone -b $TFB_REPOBRANCH $TFB_REPOURI $TFB_REPOPARENT/$TFB_REPONAME
+  fi
+  exec /$TFB_REPOPARENT/$TFB_REPONAME/toolset/run-continuously.sh
+end script

toolset/lifecycle/post-run-tests/email-results.py

@@ -0,0 +1,71 @@
+#!/usr/bin/python
+#
+# Zips the results file and emails it to a specified address.
+#
+# @author A. Shawn Bandy
+#
+import smtplib
+from os.path import basename
+from email.mime.application import MIMEApplication
+from email.mime.multipart import MIMEMultipart
+from email.mime.text import MIMEText
+from email.mime.base import MIMEBase
+from email import encoders
+from email.utils import COMMASPACE, formatdate
+import zipfile
+import os
+import subprocess
+import tempfile
+#
+# Generate body of email and attach to message
+#
+message = MIMEMultipart()
+body = "preview run complete. results attached"
+message.attach(MIMEText(body))
+#
+# Get results file, zip it and attach to the email
+#
+resultsFileLocation = os.environ['TFB_REPOPARENT'] + "/" \
+    + os.environ['TFB_REPONAME'] + "/results/latest/results.json"
+temporaryFile = tempfile.TemporaryFile(prefix='results', suffix='.zip')
+zipFile = zipfile.ZipFile(temporaryFile, 'w')
+zipFile.write(resultsFileLocation)
+zipFile.close()
+temporaryFile.seek(0)
+#
+#Set up the email to be sent to the mailing list
+#
+message['From'] = os.environ['TFB_MAILING_FROM']
+message['To'] = os.environ['TFB_MAILINGLIST']
+message['Date'] = formatdate(localtime=True)
+message['Subject'] = subprocess.check_output(["git", "describe", "--always"])
+#
+# Zip in memory and email results
+#
+resultsZipped = MIMEBase('application', 'zip')
+resultsZipped.set_payload(temporaryFile.read())
+encoders.encode_base64(resultsZipped)
+resultsZipped.add_header('Content-Disposition', 'attachment', \
+  filename='results.json.zip')
+message.attach(resultsZipped)
+
+#
+# Attach the .commit file
+#
+
+commitFileLocation = os.environ['TFB_REPOPARENT'] + "/" \
+    + os.environ['TFB_REPONAME'] + "/results/.commit"
+attachment = MIMEBase('application', 'octet-stream')
+attachment.set_payload(open(commitFileLocation, "rb").read())
+encoders.encode_base64(attachment)
+attachment.add_header('Content-Disposition', 'attachment; filename=".commit"')
+message.attach(attachment)
+
+
+#
+# Send the message and close the collection
+#
+smtp = smtplib.SMTP('smtp.techempower.com')
+smtp.sendmail(os.environ['TFB_MAILING_FROM'], os.environ['TFB_MAILINGLIST'], \
+    message.as_string())
+smtp.close()

toolset/lifecycle/post-run-tests/keep-logs.py

@@ -0,0 +1,39 @@
+#!/usr/bin/python
+#
+# Archives, to the specified folder, the logged output generated by a benchmark 
+# run.  
+#
+# @author A. Shawn Bandy
+import os
+import zipfile
+import datetime
+# Follows closely from:
+# http://stackoverflow.com/a/34153816
+#
+# Paths to the log folders are generated by TFB and where those files 
+# should be archived.
+#
+path_in = os.path.abspath(os.path.normpath(os.path.expanduser(os.path.join( \
+    os.environ['TFB_REPOPARENT'], os.environ['TFB_REPONAME'], \
+    'results/latest/logs'))))
+date_time = datetime.datetime.now()
+dt_folder = date_time.strftime('%Y%m%d%H%M%S')
+path_out = os.path.abspath(os.path.join(os.environ['TFB_LOGSFOLDER'], \
+    dt_folder))
+# Step through each folder in the TFB log folder...
+for folder in os.listdir(path_in):
+  if not os.path.exists(path_out):
+    os.makedirs(path_out)
+  zip_file = zipfile.ZipFile(path_out + '/' + folder + '.zip', 'w')
+# ... walk the folder structure ...
+  for root, directories, files in os.walk(os.path.join(path_in, folder), 'w', \
+        zipfile.ZIP_DEFLATED):
+# ... and add to the zip file.   
+    for file in files:
+      try:
+        zip_file.write(os.path.abspath(os.path.join(root, file)), \
+            arcname=file)
+      except OSError as err:
+        print "An OSError occurred while writing to a log zip file for {0}: \
+            {1}".format(file, err)
+  zip_file.close()

toolset/lifecycle/rebuild-environment.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+#
+# rebuild-environment.sh
+#
+# Rebuilds the benchmarking environment.
+#
+cd $TFB_REPOPARENT
+git clone -b $TFB_REPOBRANCH $TFB_REPOURI
+cd $TFB_REPOPARENT/$TFB_REPONAME
+
+#
+# Make the results directory if it doesn't exist.
+#
+# save the current git commit to results/.commit
+#
+echo "Saving current git commit to results directory..."
+GIT_COMMIT=$(git rev-parse HEAD)
+mkdir -p results
+echo $GIT_COMMIT > results/.commit
+echo "Using commit: " $GIT_COMMIT
+

toolset/lifecycle/tear-down-environment.sh

@@ -0,0 +1,13 @@
+#!/bin/bash
+#
+# tear-down-environment.sh
+#
+# Tears down the benchmark environment in preparation for another
+# run.
+#
+# @author A. Shawn Bandy
+#
+# $TFB_REPOPARENT parent folder of the repository
+# $TFB_REPONAME   name of the repository folder
+sudo rm -r -f $TFB_REPOPARENT/$TFB_REPONAME
+cd $TFB_REPOPARENT

toolset/run-continuously.sh

@@ -0,0 +1,58 @@
+#!/bin/bash
+#
+# Sets up and runs the benchmarking suite in an infinite loop.  The intent is 
+# this script would be run by a service such as upstart, restarting if the 
+# scripts fail.
+#
+# The following environment variables must
+# be set:
+# $TFB_REPOPARENT   parent folder of the repository
+# $TFB_REPONAME     name of the repository folder
+# $TFB_REPOURI      URI for the git repository
+# $TFB_MAILINGLIST  email address for the mailing list
+# $TFB_LOGSFOLDER   location to transfer logged output
+# 
+# @author A. Shawn Bandy
+#
+while true
+do
+  # Tear down the environment
+  # Note: we tear down first so that we can know the state of the environment
+  # regardless of the outcome of prior runs.
+  if [ -d "$TFB_REPOPARENT/$TFB_REPONAME" ]; then
+    cp $TFB_REPOPARENT/$TFB_REPONAME/benchmark.cfg $TFB_REPOPARENT/
+    cp $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/rebuild-environment.sh $TFB_REPOPARENT/
+    echo Tearing down previous environment
+    $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/tear-down-environment.sh
+  fi
+  # Rebuild environment (i.e. clone the repository)
+  echo Rebuilding environment
+  $TFB_REPOPARENT/rebuild-environment.sh
+  echo Returning benchmark configuration file
+  cp $TFB_REPOPARENT/benchmark.cfg $TFB_REPOPARENT/$TFB_REPONAME/
+  # Handle any preprocessing (e.g. send metadata)
+  echo Running pre-test tasks
+  for SCRIPT in $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/pre-run-tests/*
+    do
+      if [ -f $SCRIPT -a -x $SCRIPT ]
+      then
+        $SCRIPT
+      fi
+    done
+  sleep 5
+  # Run the benchmarks
+  echo Change to benchmark root
+  cd $TFB_REPOPARENT/$TFB_REPONAME
+  echo Run tests
+  toolset/run-tests.py --mode verify --test undertow
+  # Handle any postprocessing
+  echo Running post-test tasks
+  for SCRIPT in $TFB_REPOPARENT/$TFB_REPONAME/toolset/lifecycle/post-run-tests/*
+    do
+      if [ -f $SCRIPT -a -x $SCRIPT ]
+      then
+        $SCRIPT
+      fi
+    done
+  sleep 5
+done