[ci skip] Update continuous benchmarking scripts (#4516)

* Bumped up the expected required free disk space to complete a run from
  20G to 50G based on observation.
* Build the techempower/tfb docker image before each run.  Previously,
  we were pulling the image from docker hub, but we would not always
  publish a new image to docker hub every time we changed our Dockerfile.
* Move looping behavior from the bash script into the service.  That way,
  we can update the script without restarting the service (which would
  end the current run), and the changes will affect the next run.
* Added more logging.
* Added --benchmark-env flag to tfb command.

toolset/continuous/tfb-shutdown.sh

@@ -3,17 +3,31 @@
 set -e
 
 docker_clean() {
+  echo "restarting docker"
   sudo service docker restart
+
+  echo "running 'docker stop'"
   docker ps --all --quiet | xargs --no-run-if-empty docker stop
+
+  echo "running 'docker rm'"
   docker ps --all --quiet | xargs --no-run-if-empty docker rm --force
 
+  echo "checking disk space"
   # https://stackoverflow.com/a/38183298/359008
   FREE=`df -k --output=avail /var/lib/docker | tail -n1`   # df -k not df -h
-  if [[ $FREE -lt 20971520 ]]; then                        # 20G = 20*1024*1024k
+  if [[ $FREE -lt 52428800 ]]; then                        # 50G = 50*1024*1024k
+    echo "running 'docker system prune'"
     docker system prune --all --force
   fi
 }
 
+echo "running docker_clean on server host"
 docker_clean
+
+echo "running docker_clean on database host"
 ssh techempower@$TFB_DATABASE_HOST "$(typeset -f docker_clean); docker_clean"
+
+echo "running docker_clean on client host"
 ssh techempower@$TFB_CLIENT_HOST "$(typeset -f docker_clean); docker_clean"
+
+echo "done with tfb-shutdown script"

toolset/continuous/tfb-startup.sh

@@ -2,44 +2,51 @@
 
 set -e
 
-while true
-do
-  cd $TFB_REPORTPARENT
-  ./tfb-shutdown.sh
-
-  if [ -d "$TFB_REPOPARENT/$TFB_REPONAME" ]; then
-    sudo rm -rf $TFB_REPOPARENT/$TFB_REPONAME
-  fi
-
-  git clone \
-    -b $TFB_REPOBRANCH \
-    $TFB_REPOURI \
-    $TFB_REPOPARENT/$TFB_REPONAME \
-    --depth 1
-
-  cd $TFB_REPOPARENT/$TFB_REPONAME
-
-  docker run \
-    --network=host \
-    --mount type=bind,source=$TFB_REPOPARENT/$TFB_REPONAME,target=/FrameworkBenchmarks \
-    techempower/tfb \
-    --server-host $TFB_SERVER_HOST \
-    --client-host $TFB_CLIENT_HOST \
-    --database-host $TFB_DATABASE_HOST \
-    --network-mode host \
-    --results-name "$TFB_RUN_NAME" \
-    --results-environment "$TFB_ENVIRONMENT" \
-    --results-upload-uri "$TFB_UPLOAD_URI" \
-    --quiet
-
-  zip -r results.zip results
-
-  curl \
-    -i -v \
-    -X POST \
-    --header "Content-Type: application/zip" \
-    --data-binary @results.zip \
-    $TFB_UPLOAD_URI
-
-  sleep 5
-done
+echo "running tfb-shutdown script"
+./tfb-shutdown.sh
+
+echo "removing old tfb directory if necessary"
+if [ -d "$TFB_REPOPARENT/$TFB_REPONAME" ]; then
+  sudo rm -rf $TFB_REPOPARENT/$TFB_REPONAME
+fi
+
+echo "cloning tfb repository"
+git clone \
+  -b $TFB_REPOBRANCH \
+  $TFB_REPOURI \
+  $TFB_REPOPARENT/$TFB_REPONAME \
+  --depth 1
+
+echo "moving to tfb directory"
+cd $TFB_REPOPARENT/$TFB_REPONAME
+
+echo "building tfb docker image"
+docker build -t techempower/tfb .
+
+echo "running tfb docker image"
+docker run \
+  --network=host \
+  --mount type=bind,source=$TFB_REPOPARENT/$TFB_REPONAME,target=/FrameworkBenchmarks \
+  techempower/tfb \
+  --server-host $TFB_SERVER_HOST \
+  --client-host $TFB_CLIENT_HOST \
+  --database-host $TFB_DATABASE_HOST \
+  --network-mode host \
+  --results-name "$TFB_RUN_NAME" \
+  --results-environment "$TFB_ENVIRONMENT" \
+  --results-upload-uri "$TFB_UPLOAD_URI" \
+  --benchmark-env "$TFB_ENVIRONMENT" \
+  --quiet
+
+echo "zipping the results"
+zip -r results.zip results
+
+echo "uploading the results"
+curl \
+  -i -v \
+  -X POST \
+  --header "Content-Type: application/zip" \
+  --data-binary @results.zip \
+  $TFB_UPLOAD_URI
+
+echo "done uploading results"

toolset/continuous/tfb.service

@@ -1,3 +1,14 @@
+#
+# Put this in /lib/systemd/system/tfb.service.
+#
+# Useful commands:
+#
+#   sudo service tfb start
+#   sudo service tfb stop
+#   sudo service tfb restart
+#   sudo journalctl -u tfb -f
+#
+
 [Service]
 Environment=TFB_SERVER_HOST=127.0.0.1
 Environment=TFB_CLIENT_HOST=127.0.0.1
@@ -18,8 +29,9 @@ LimitNOFILE=1048576
 LimitNPROC=1048576
 UMask=0002
 ExecStart=/home/techempower/tfb-startup.sh
-ExecStop=/home/techempower/tfb-shutdown.sh
 ExecStopPost=/home/techempower/tfb-shutdown.sh
+Restart=on-success
+RestartSec=5s
 
 [Install]
 WantedBy=multi-user.target