Retry limit for starting Mongo and Cassandra

toolset/setup/linux/database.sh

@@ -21,9 +21,8 @@
 # concentrated effort to address these cases, but PR's for specific 
 # problems are welcome
 
-export DB_HOST={database_host}
-
 set -x
+export DB_HOST=${DB_HOST:-127.0.0.1}
 export DEBIAN_FRONTEND=noninteractive
 
 source /etc/lsb-release
@@ -143,10 +142,18 @@ sudo cp -R -p /var/lib/mongodb /ssd/
 sudo cp -R -p /var/log/mongodb /ssd/log/
 sudo service mongod start
 
-until nc -z localhost 27017 ; do echo Waiting for MongoDB; sleep 1; done
-mongo < create.js
-rm create.js
-mongod --version
+for i in {1..45}; do
+  nc -z localhost 27017 && break || sleep 1;
+  echo "Waiting for MongoDB ($i/45}"
+done
+nc -z localhost 27017
+if [ $? -eq 0 ]; then
+  mongo < create.js
+  rm create.js
+  mongod --version
+else
+  >&2 echo "MongoDB did not start, skipping"
+fi
 
 ##############################
 # Apache Cassandra
@@ -168,12 +175,20 @@ mv cassandra/cassandra.yaml apache-cassandra-$CASS_V/conf
 mv cassandra/log4j-server.properties apache-cassandra-$CASS_V/conf
 nohup apache-cassandra-$CASS_V/bin/cassandra -p c.pid > cassandra.log
 
-until nc -z $DB_HOST 9160 ; do echo Waiting for Cassandra; sleep 1; done
-cat cassandra/cleanup-keyspace.cql | apache-cassandra-$CASS_V/bin/cqlsh $DB_HOST
-python cassandra/db-data-gen.py > cassandra/tfb-data.cql
-apache-cassandra-$CASS_V/bin/cqlsh -f cassandra/create-keyspace.cql $DB_HOST
-apache-cassandra-$CASS_V/bin/cqlsh -f cassandra/tfb-data.cql $DB_HOST
-rm -rf apache-cassandra-*-bin.tar.gz cassandra
+for i in {1..45}; do
+  nc -z localhost 9160 && break || sleep 1;
+  echo "Waiting for Cassandra ($i/45}"
+done
+nc -z localhost 9160
+if [ $? -eq 0 ]; then
+  cat cassandra/cleanup-keyspace.cql | apache-cassandra-$CASS_V/bin/cqlsh $DB_HOST
+  python cassandra/db-data-gen.py > cassandra/tfb-data.cql
+  apache-cassandra-$CASS_V/bin/cqlsh -f cassandra/create-keyspace.cql $DB_HOST
+  apache-cassandra-$CASS_V/bin/cqlsh -f cassandra/tfb-data.cql $DB_HOST
+  rm -rf apache-cassandra-*-bin.tar.gz cassandra
+else
+  >&2 echo "Cassandra did not start, skipping"
+fi
 
 ##############################
 # Redis

toolset/setup/linux/installer.py

@@ -26,9 +26,11 @@ class Installer:
       print("\nINSTALL: Installing database software\n")   
       self.__run_command("cd .. && " + self.benchmarker.database_sftp_string(batch_file="../config/database_sftp_batch"), True)
       with open (linux_install_root + "/database.sh", "r") as myfile:
-        remote_script=myfile.read().format(database_host=self.benchmarker.database_host)
+        remote_script = myfile.read()
         print("\nINSTALL: %s" % self.benchmarker.database_ssh_string)
-        p = subprocess.Popen(self.benchmarker.database_ssh_string.split(" ") + ["bash"], stdin=subprocess.PIPE)
+        p = subprocess.Popen(self.benchmarker.database_ssh_string.split(" ") +
+                             ["DB_HOST=" + self.benchmarker.database_host,
+                              "bash"], stdin=subprocess.PIPE)
         p.communicate(remote_script)
         returncode = p.returncode
         if returncode != 0: