DB Setup Changes / New Travis / Vagrant fix - The Promised Land! (#2417)

* changes to db setup

* travis changes

* travis changes

* fix travis setup

* Add DB fw_depends (C through Java) (#2415)

* added DB fw_depends for C

* added DB fw_depends for C++

* added DB fw_depends for CSharp

* added DB fw_depends for Clojure

* added DB fw_depends for Crystal

* added DB fw_depends for D

* added DB fw_depends for Dart

* added DB fw_depends for Elixer

* added DB fw_depends for Erlang

* added DB fw_depends for Go

* added DB fw_depends for Groovy

* added DB fw_depends for Haskell

* added DB fw_depends for Java

* fixed syntax where postgres should be postgresql

* chmod change to run-ci

* remove colorama

* move diffing til after some of the setup

* forget to source

* remove install client

* more travis work

* travis work

* travis work

* travis work

* travis work

* travis work

* travis work

* fix sources.list

* postgresql typos

* force a toolset change

* mongodb 3

* mongo fixes

* force toolset change

* mongodb fix

* mongo fixes

* more mongo

* more mongo tests

* fixes to more fws with db setup

* prereqs has things it shouldn't

* mongo and vagrant fixes

* force full run

.travis.yml

@@ -1,20 +1,15 @@
+# Travis CI
+#
+#
+
 sudo: required
 dist: trusty
-language: python
+language: generic
 python:
   - "2.7"
 
 env:
   matrix:
-    #Group tests by directory to logically break up travis-CI build. Otherwise
-    #we end up starting ~200+ different workers. Seems that ~100 is the limit
-    #before their website starts to lag heavily
-    #Here's the bash if you need to update this. Be sure to maintain the
-    #lines that are currently commented out (these cannot run in Travis)
-    #  cd frameworks
-    #  find . -type d -depth 2 | sed 's|./|    - "TESTDIR=|' | sed 's/$/"/g'
-    #
-    #
     - "TESTDIR=C/libreactor"
     - "TESTDIR=C/lwan"
     - "TESTDIR=C/duda"
@@ -197,36 +192,24 @@ env:
     - "TESTDIR=Scala/finch"
     - "TESTDIR=Ur/urweb"
 
-before_install:
-  - pip install colorama==0.3.1
-  # Version 2.3 has a nice Counter() and other features
-  # but it requires —-allow-external and -—allow-unverified
-  - pip install progressbar==2.2
-  - pip install requests
-  - echo "127.0.0.1 " `hostname` | sudo tee /etc/hosts
-  - echo "127.0.0.1 localhost" | sudo tee /etc/hosts
-
-services:
-  - postgresql
-  - redis-server
-  - mongodb
-  - mysql
-
-addons:
-  postgresql: "9.3"
-  apt:
-    packages:
-      - redis-server
-
 before_script:
-  - sudo sysctl -w net.core.somaxconn=65535
-  - sudo ./config/travis_mysql_setup.sh
-  - mysql -uroot < config/create.sql
-  - sudo ./config/create-redis.sh
+  # travis_clean.sh takes care of some services that are baked into the travis
+  # build. Using language: generic gets us an ubuntu build with fewer services,
+  # but includes database installs, ruby and rvm installs, and others that interfere
+  # with running the suite in a clean ubuntu install.
+  - source ./toolset/travis/travis_clean.sh
+
+  # travis_setup.sh runs all the same commands you would run if you were setting up
+  # a development environment via:
+  # http://frameworkbenchmarks.readthedocs.io/en/latest/Development/Installation-Guide/
+  - source ./toolset/travis/travis_setup.sh
 
 script:
-  # Pick one test in this directory and verify
-  - time ./toolset/run-ci.py verify "$TESTDIR"
+  # run-ci.py runs the diffing to see if travis needs to test this framework. Ideally/eventually,
+  # we'd like to try and do the diffing before travis_clean & setup.
+  # This will run the tests exactly as you would in your own vm:
+  # ./toolset/run-tests.py --mode verify --test (all the valid tests for this framework)
+  - ./toolset/run-ci.py verify "$TESTDIR"
 
 cache:
   directories:

config/mongodb.conf

@@ -1,93 +1,23 @@
-# mongodb.conf
-
-# Where to store the data.
-dbpath=/ssd/mongodb
-
-#where to log
-logpath=/ssd/log/mongodb/mongodb.log
-
-logappend=true
-
-bind_ip = 0.0.0.0
-#port = 27017
-
-# Enable journaling, http://www.mongodb.org/display/DOCS/Journaling
-journal=true
-
-# Enables periodic logging of CPU utilization and I/O wait
-#cpu = true
-
-# Turn on/off security.  Off is currently the default
-#noauth = true
-#auth = true
-
-# Verbose logging output.
-#verbose = true
-
-# Inspect all client data for validity on receipt (useful for
-# developing drivers)
-#objcheck = true
-
-# Enable db quota management
-#quota = true
-
-# Set oplogging level where n is
-#   0=off (default)
-#   1=W
-#   2=R
-#   3=both
-#   7=W+some reads
-#oplog = 0
-
-# Diagnostic/debugging option
-#nocursors = true
-
-# Ignore query hints
-#nohints = true
-
-# Disable the HTTP interface (Defaults to localhost:27018).
-#nohttpinterface = true
-
-# Turns off server-side scripting.  This will result in greatly limited
-# functionality
-#noscripting = true
-
-# Turns off table scans.  Any query that would do a table scan fails.
-#notablescan = true
-
-# Disable data file preallocation.
-#noprealloc = true
-
-# Specify .ns file size for new databases.
-# nssize = <size>
-
-# Accout token for Mongo monitoring server.
-#mms-token = <token>
-
-# Server name for Mongo monitoring server.
-#mms-name = <server-name>
-
-# Ping interval for Mongo monitoring server.
-#mms-interval = <seconds>
-
-# Replication Options
-
-# in replicated mongo databases, specify here whether this is a slave or master
-#slave = true
-#source = master.example.com
-# Slave only: specify a single database to replicate
-#only = master.example.com
-# or
-#master = true
-#source = slave.example.com
-
-# Address of a server to pair with.
-#pairwith = <server:port>
-# Address of arbiter server.
-#arbiter = <server:port>
-# Automatically resync if slave data is stale
-#autoresync
-# Custom size for replication operation log.
-#oplogSize = <MB>
-# Size limit for in-memory storage of op ids.
-#opIdMem = <bytes>
+# mongod.conf
+
+# for documentation of all options, see:
+#   http://docs.mongodb.org/manual/reference/configuration-options/
+
+# Where and how to store data.
+storage:
+  dbPath: "/ssd/mongodb"
+  journal:
+    enabled: true
+#  engine:
+#  mmapv1:
+#  wiredTiger:
+
+# where to write logging data.
+systemLog:
+  destination: file
+  logAppend: true
+  path: "/ssd/log/mongodb/mongod.log"
+
+# network interfaces
+net:
+  port: 27017

config/travis_mysql_setup.sh

@@ -1,4 +0,0 @@
-sed -i 's|\[mysqld\]|\[mysqld\]\
-lower_case_table_names = 1\
-character-set-server=utf8\
-collation-server=utf8_general_ci|g' /etc/mysql/my.cnf

config/travis_setup.sh

@@ -1,133 +0,0 @@
-export DEBIAN_FRONTEND=noninteractive
-
-# Turn on command tracing
-set -x 
-
-# Setup Apt For MongoDB
-#   Due to TechEmpower/FrameworkBenchmarks#989 and travis-ci/travis-ci#2655, 
-#   we put this into a loop
-until timeout 15s sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 7F0CEB10; do echo 'Waiting for apt-key' ; done
-echo 'deb http://downloads-distro.mongodb.org/repo/ubuntu-upstart dist 10gen' | sudo tee /etc/apt/sources.list.d/mongodb.list
-
-# Setup apt for Apache Cassandra
-until timeout 15s sudo apt-key adv --keyserver pgp.mit.edu --recv 4BD736A82B5C1B00; do echo 'Waiting for apt-key' ; done
-sudo apt-add-repository  'deb http://www.apache.org/dist/cassandra/debian 20x main'
-
-# Run installation 
-# DO NOT COPY --force-yes TO ANY NON-TRAVIS-CI SCRIPTS! Seriously, it can cause some 
-# major damage and should only be used inside a VM or Linux Container
-sudo apt-get -q update
-sudo apt-get -q -y --force-yes install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
-  mongodb-org \
-  cassandra \
-  openssh-server \
-  mysql-server
-
-# Run as travis user (who already has passwordless sudo)
-ssh-keygen -f /home/travis/.ssh/id_rsa -N '' -t rsa
-cat /home/travis/.ssh/id_rsa.pub > /home/travis/.ssh/authorized_keys
-chmod 600 /home/travis/.ssh/authorized_keys
-
-# Set up the benchmark.cfg for travis user
-# NOTE: Please don't just copy the example config - it causes unexpected
-#       issues when those example variables change
-echo "[Defaults]"                                       > benchmark.cfg
-echo "client_identity_file=/home/travis/.ssh/id_rsa"   >> benchmark.cfg
-echo "database_identity_file=/home/travis/.ssh/id_rsa" >> benchmark.cfg
-echo "client_host=127.0.0.1"                           >> benchmark.cfg
-echo "database_host=127.0.0.1"                         >> benchmark.cfg
-echo "server_host=127.0.0.1"                           >> benchmark.cfg
-echo "client_user=travis"                              >> benchmark.cfg
-echo "database_user=travis"                            >> benchmark.cfg
-
-mkdir installs
-
-# =============Setup Databases===========================
-# NOTE: Do not run `--install database` in travis-ci! 
-#       It changes DB configuration files and will break everything
-# =======================================================
-
-# Setup MySQL
-echo "Populating MySQL database"
-#sudo mysqladmin -u root password secret
-#sudo mv /etc/mysql/my.cnf /etc/mysql/my.cnf.orig
-#sudo mv config/my.cnf /etc/mysql/my.cnf
-sudo sed -i 's|#max_connections        = 100|max_connections        = 500|g' /etc/mysql/my.cnf
-sudo restart mysql
-#mysql -uroot -psecret < config/create.sql
-
-# Setup Postgres
-echo "Removing Postgres 9.1 from Travis-CI"
-sudo apt-get remove -qy postgresql postgresql-9.1 postgresql-client-9.1
-sudo apt-get install -qy postgresql-9.3 postgresql-client-9.3
-
-echo "Populating Postgres database"
-psql --version
-sudo useradd benchmarkdbuser -p benchmarkdbpass
-sudo -u postgres psql template1 < config/create-postgres-database.sql
-sudo -u postgres psql hello_world < config/create-postgres.sql
-sudo sed -i "s|#listen_addresses = 'localhost'|listen_addresses = '*'|g" /etc/postgresql/9.3/main/postgresql.conf
-sudo sed -i 's|max_connections = 255|max_connections = 500|g' /etc/postgresql/9.3/main/postgresql.conf
-sudo service postgresql stop
-sudo service postgresql start 9.3
-
-# Setup Apache Cassandra
-echo "Populating Apache Cassandra database"
-for i in {1..15}; do
-nc -z localhost 9160 && break || sleep 1;
-echo "Waiting for Cassandra ($i/15}"
-done
-nc -z localhost 9160
-if [ $? -eq 0 ]; then
-cat config/cassandra/cleanup-keyspace.cql | sudo cqlsh
-python config/cassandra/db-data-gen.py > config/cassandra/tfb-data.cql
-sudo cqlsh -f config/cassandra/create-keyspace.cql
-sudo cqlsh -f config/cassandra/tfb-data.cql
-else
->&2 echo "Cassandra did not start, skipping"
-fi
-
-# Setup Elasticsearch
-curl -O https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-1.5.0.deb
-sudo dpkg -i --force-confnew elasticsearch-1.5.0.deb
-sudo update-rc.d elasticsearch defaults 95 10
-sudo service elasticsearch restart
-
-echo "Populating Elasticsearch database"
-for i in {1..15}; do
-nc -z localhost 9200 && break || sleep 1;
-echo "Waiting for Elasticsearch ($i/15}"
-done
-nc -z localhost 9200
-if [ $? -eq 0 ]; then
-curl localhost:9200
-sh config/elasticsearch/es-create-index.sh
-python config/elasticsearch/es-db-data-gen.py > config/elasticsearch/tfb-data.json
-curl -sS -D - -o /dev/null -XPOST localhost:9200/tfb/world/_bulk --data-binary @config/elasticsearch/tfb-data.json
-echo "Elasticsearch DB populated"
-else
->&2 echo "Elasticsearch did not start, skipping"
-fi
-
-# Setup MongoDB
-echo "Populating MongoDB database"
-for i in {1..15}; do
-nc -z localhost 27017 && break || sleep 1;
-echo "Waiting for MongoDB ($i/15}"
-done
-nc -z localhost 27017
-if [ $? -eq 0 ]; then
-mongo < config/create.js
-mongod --version
-else
->&2 echo "MongoDB did not start, skipping"
-fi
-
-# =============Modify Configurations===========================
-# It can be useful to enable debug features for verification 
-# inside Travis-CI
-# =======================================================
-
-sed -i 's|display_errors\] = off|display_errors\] = on|' config/php-fpm.conf
-
-#exit $?

deployment/vagrant-common/bootstrap.sh

@@ -117,6 +117,12 @@ if [ ! -e "~/.firstboot" ]; then
     source ~/FrameworkBenchmarks/toolset/setup/linux/prerequisites.sh
   #fi
 
+ # Everyone gets SSH access to localhost
+ echo "Setting up SSH access to localhost"
+ ssh-keygen -t rsa -N '' -f ~/.ssh/id_rsa
+ cat ~/.ssh/id_rsa.pub >> ~/.ssh/authorized_keys
+ chmod 600 ~/.ssh/authorized_keys
+
   # Enable remote SSH access if we are running production environment
   # Note : this is always copied from the local working copy using a
   #        file provisioner. While they exist in the git clone we just 
@@ -138,11 +144,6 @@ if [ ! -e "~/.firstboot" ]; then
     chmod 600 ~/.ssh/client ~/.ssh/database
   fi
 
-  # Setup 
-  echo "Installing $ROLE software"
-  cd $FWROOT
-  toolset/run-tests.py --verbose --install $ROLE --install-only --test ''
-
   # Setup a nice welcome message for our guest
   echo "Setting up welcome message"
   sudo rm -f /etc/update-motd.d/51-cloudguest

frameworks/C++/cpoll_cppsp/benchmark_config.json

@@ -22,7 +22,7 @@
       "versus": "cpoll_cppsp"
     },
     "raw": {
-      "setup_file": "setup",
+      "setup_file": "setup_mysql",
       "db_url": "/db",
       "query_url": "/db?queries=",
       "fortune_url": "/fortune",
@@ -44,7 +44,7 @@
       "versus": "cpoll_cppsp"
     },
     "postgres-raw": {
-      "setup_file": "setup",
+      "setup_file": "setup_postgresql",
       "db_url": "/db_pg_async",
       "query_url": "/db_pg_async?queries=", 
       "port": 16969,
@@ -64,7 +64,7 @@
       "versus": "cpoll_cppsp"
     },
     "postgres-raw-threadpool": {
-      "setup_file": "setup",
+      "setup_file": "setup_postgresql",
       "db_url": "/db_pg_threadpool",
       "query_url": "/db_pg_threadpool?queries=", 
       "port": 16969,

frameworks/C++/cpoll_cppsp/setup_mysql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mysql
+
+source ./setup.sh

frameworks/C++/cpoll_cppsp/setup_postgresql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends postgresql
+
+source ./setup.sh

frameworks/C++/cutelyst/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends cutelyst
+fw_depends mysql postgresql cutelyst
 
 sed -i 's|DatabaseHostName=.*|DatabaseHostName='"$DBHOST"'|g' config/config.ini
 sed -i 's|SendDate=.*|SendDate=false|g' config/config.ini

frameworks/C++/cutelyst/setup_thread.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends cutelyst
+fw_depends mysql postgresql cutelyst
 
 sed -i 's|DatabaseHostName=.*|DatabaseHostName='"$DBHOST"'|g' config/config.ini
 sed -i 's|SendDate=.*|SendDate=false|g' config/config.ini

frameworks/C++/cutelyst/setup_uwsgi_nginx.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends cutelyst nginx
+fw_depends mysql postgresql cutelyst nginx
 
 sed -i 's|DatabaseHostName=.*|DatabaseHostName='"$DBHOST"'|g' config/config_socket.ini
 sed -i 's|SendDate=.*|SendDate=false|g' config/config_socket.ini

frameworks/C++/ffead-cpp/setup-apache2-mysql.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-fw_depends apache
-fw_depends ffead-cpp-apache
+fw_depends mysql apache ffead-cpp-apache
 
 export FFEAD_CPP_PATH=/var/www/ffead-cpp-2.0
 export LD_LIBRARY_PATH=$IROOT:$FFEAD_CPP_PATH/lib:$LD_LIBRARY_PATH

frameworks/C++/ffead-cpp/setup-apache2-postgresql.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-fw_depends apache
-fw_depends ffead-cpp-apache
+fw_depends postgresql apache ffead-cpp-apache
 
 export FFEAD_CPP_PATH=/var/www/ffead-cpp-2.0
 export LD_LIBRARY_PATH=$IROOT:$FFEAD_CPP_PATH/lib:$LD_LIBRARY_PATH

frameworks/C++/ffead-cpp/setup-apache2.sh

@@ -1,7 +1,6 @@
 #!/bin/bash
 
-fw_depends apache
-fw_depends ffead-cpp-apache
+fw_depends apache ffead-cpp-apache
 
 export FFEAD_CPP_PATH=/var/www/ffead-cpp-2.0
 export LD_LIBRARY_PATH=$IROOT:$FFEAD_CPP_PATH/lib:$LD_LIBRARY_PATH

frameworks/C++/ffead-cpp/setup-mysql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends ffead-cpp
+fw_depends mysql ffead-cpp
 
 export FFEAD_CPP_PATH=$TROOT/ffead-cpp-2.0
 export LD_LIBRARY_PATH=$IROOT:$FFEAD_CPP_PATH/lib:$LD_LIBRARY_PATH

frameworks/C++/ffead-cpp/setup-nginx-mysql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends ffead-cpp-nginx
+fw_depends mysql ffead-cpp-nginx
 
 export FFEAD_CPP_PATH=$TROOT/ffead-cpp-2.0
 export LD_LIBRARY_PATH=$IROOT:$FFEAD_CPP_PATH/lib:$LD_LIBRARY_PATH

frameworks/C++/ffead-cpp/setup-nginx-postgresql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends ffead-cpp-nginx
+fw_depends postgresql ffead-cpp-nginx
 
 export FFEAD_CPP_PATH=$TROOT/ffead-cpp-2.0
 export LD_LIBRARY_PATH=$IROOT:$FFEAD_CPP_PATH/lib:$LD_LIBRARY_PATH

frameworks/C++/ffead-cpp/setup-postgresql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends ffead-cpp
+fw_depends postgresql ffead-cpp
 
 export FFEAD_CPP_PATH=$TROOT/ffead-cpp-2.0
 export LD_LIBRARY_PATH=$IROOT:$FFEAD_CPP_PATH/lib:$LD_LIBRARY_PATH

frameworks/C++/silicon/setup_lwan_mysql.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
  
-fw_depends silicon lwan
+fw_depends mysql silicon lwan
 
 rm -rf build
 mkdir build

frameworks/C++/silicon/setup_mhd_epoll_mysql.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
 
-fw_depends silicon microhttpd
+fw_depends mysql silicon microhttpd
 
 rm -rf build
 mkdir build

frameworks/C++/silicon/setup_mhd_tpc_mysql.sh

@@ -1,6 +1,6 @@
 #! /bin/bash
 
-fw_depends silicon microhttpd
+fw_depends mysql silicon microhttpd
 
 rm -rf build
 mkdir build

frameworks/C++/treefrog/setup-mongodb.sh

@@ -5,7 +5,7 @@ sed -i 's|HostName=.*|HostName='"$DBHOST"'|g' config/mongodb.ini
 sed -i 's|DriverType=.*|DriverType=QMYSQL|g' config/database.ini
 sed -i 's|MultiProcessingModule=.*|MultiProcessingModule=thread|g' config/application.ini
 
-fw_depends treefrog
+fw_depends mongodb treefrog
 
 # 1. Generate Makefile
 qmake -r CONFIG+=release

frameworks/C++/treefrog/setup-postgres.sh

@@ -5,7 +5,7 @@ sed -i 's|HostName=.*|HostName='"$DBHOST"'|g' config/mongodb.ini
 sed -i 's|DriverType=.*|DriverType=QPSQL|g' config/database.ini
 sed -i 's|MultiProcessingModule=.*|MultiProcessingModule=thread|g' config/application.ini
 
-fw_depends treefrog
+fw_depends postgresql treefrog
 
 # 1. Generate Makefile
 qmake -r CONFIG+=release

frameworks/C++/treefrog/setup-thread.sh

@@ -5,7 +5,7 @@ sed -i 's|HostName=.*|HostName='"$DBHOST"'|g' config/mongodb.ini
 sed -i 's|DriverType=.*|DriverType=QMYSQL|g' config/database.ini
 sed -i 's|MultiProcessingModule=.*|MultiProcessingModule=thread|g' config/application.ini
 
-fw_depends treefrog
+fw_depends mysql treefrog
 
 # 1. Generate Makefile
 qmake -r CONFIG+=release

frameworks/C++/treefrog/setup.sh

@@ -5,7 +5,7 @@ sed -i 's|HostName=.*|HostName='"$DBHOST"'|g' config/mongodb.ini
 sed -i 's|DriverType=.*|DriverType=QMYSQL|g' config/database.ini
 sed -i 's|MultiProcessingModule=.*|MultiProcessingModule=hybrid|g' config/application.ini
 
-fw_depends treefrog
+fw_depends mysql treefrog
 
 # 1. Generate Makefile
 qmake -r CONFIG+=release

frameworks/C++/ulib/setup_mongodb.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends ulib
+fw_depends mongodb ulib
 
 # Travis is broken
 if [ "$TRAVIS" != "true" ]; then

frameworks/C++/ulib/setup_mysql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends ulib
+fw_depends mysql ulib
 
 # Travis is broken
 if [ "$TRAVIS" != "true" ]; then

frameworks/C++/ulib/setup_postgres.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends ulib
+fw_depends postgresql ulib
 
 MAX_THREADS=$(( 2 * $MAX_THREADS ))
 

frameworks/C++/wt/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends apache wt
+fw_depends mysql apache wt
 
 sed -i 's|INSERT_DB_HOST_HERE|'"${DBHOST}"'|g' benchmark.cpp
 

frameworks/C++/wt/setup_postgres.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends apache wt
+fw_depends postgresql apache wt
 
 sed -i 's|INSERT_DB_HOST_HERE|'"${DBHOST}"'|g' benchmark.cpp
 

frameworks/C/h2o/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends h2o mustache-c yajl
+fw_depends postgresql h2o mustache-c yajl
 
 H2O_APP_HOME="${IROOT}/h2o_app"
 BUILD_DIR="${H2O_APP_HOME}_build"

frameworks/C/lwan/setup-mysql.sh

@@ -6,7 +6,7 @@ export MYSQL_PASS=benchmarkdbpass
 export MYSQL_HOST=$DBHOST
 export MYSQL_DB=hello_world
 
-fw_depends lwan
+fw_depends mysql lwan
 
 cd $LWAN_ROOT/techempower
 $LWAN_BUILD/techempower/techempower &

frameworks/C/onion/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends onion
+fw_depends mysql onion
 
 sed -i 's|127.0.0.1|'${DBHOST}'|g' hello.c
 

frameworks/CSharp/aspnet/benchmark_config.json

@@ -272,7 +272,7 @@
       "versus": "aspnet-mono"
     },
     "mono-mysql-raw": {
-      "setup_file": "setup_nginx",
+      "setup_file": "setup_mysql",
       "json_url": "/json/default",
       "plaintext_url": "/plaintext",
       "db_url": "/ado/mysql",
@@ -296,7 +296,7 @@
       "versus": "aspnet"
     },
     "mono-postgresql-raw": {
-      "setup_file": "setup_nginx",
+      "setup_file": "setup_postgresql",
       "db_url": "/ado/postgresql",
       "query_url": "/ado/postgresql?queries=",
       "fortune_url": "/ado/postgresql/fortunes",
@@ -318,7 +318,7 @@
       "versus": "aspnet"
     },
     "mono-mongodb-raw": {
-      "setup_file": "setup_nginx",
+      "setup_file": "setup_mongodb",
       "db_url": "/mongodb",
       "query_url": "/mongodb?queries=",
       "fortune_url": "/mongodb/fortunes",
@@ -340,7 +340,7 @@
       "versus": "aspnet"
     },
     "mono-mysql-entityframework": {
-      "setup_file": "setup_nginx",
+      "setup_file": "setup_mysql",
       "db_url": "/entityframework/mysql",
       "query_url": "/entityframework/mysql?queries=",
       "fortune_url": "/entityframework/mysql/fortunes",
@@ -362,7 +362,7 @@
       "versus": "aspnet"
     },
     "mono-postgresql-entityframework": {
-      "setup_file": "setup_nginx",
+      "setup_file": "setup_postgresql",
       "db_url": "/entityframework/postgresql",
       "query_url": "/entityframework/postgresql?queries=",
       "fortune_url": "/entityframework/postgresql/fortunes",

frameworks/CSharp/aspnet/setup_mongodb.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mongodb
+
+source ./setup_nginx.sh

frameworks/CSharp/aspnet/setup_mysql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mysql
+
+source ./setup_nginx.sh

frameworks/CSharp/aspnet/setup_postgresql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends postgresql
+
+source ./setup_nginx.sh

frameworks/CSharp/aspnetcore/setup-dapper.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+fw_depends postgresql
+
 source run-linux.sh dapper $(($(nproc)/2))

frameworks/CSharp/aspnetcore/setup-ef.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+fw_depends postgresql
+
 source run-linux.sh ef $(($(nproc)/2))

frameworks/CSharp/aspnetcore/setup-raw.sh

@@ -1,3 +1,5 @@
 #!/bin/bash
 
+fw_depends postgresql
+
 source run-linux.sh raw $(($(nproc)/2))

frameworks/CSharp/nancy/setup_nginx.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends nginx mono
+fw_depends mysql nginx mono
 
 sed -i 's|localhost|'"${DBHOST}"'|g' src/Web.config
 sed -i 's|include /usr/local/nginx/conf/fastcgi_params;|include '"${NGINX_HOME}"'/conf/fastcgi_params;|g' nginx.conf

frameworks/CSharp/revenj/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java mono dsl_platform
+fw_depends postgresql java mono dsl_platform
 
 echo "Cleaning up..."
 rm -rf $TROOT/exe $TROOT/tmp $TROOT/dsl-clc.jar $TROOT/http-server.zip

frameworks/CSharp/servicestack/setup_nginx.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends nginx mono
+fw_depends mysql postgresql mongodb nginx mono
 
 sed -i 's|localhost|'"$DBHOST"'|g' src/Web.config
 sed -i 's|/usr/local/nginx/|'"${IROOT}"'/nginx/|g' nginx.conf

frameworks/CSharp/servicestack/setup_xsp.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends nginx mono
+fw_depends mysql postgresql mongodb nginx mono
 
 sed -i 's|localhost|'"$DBHOST"'|g' src/Web.config
 # extra cleaning

frameworks/Clojure/compojure/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java resin leiningen
+fw_depends mysql java resin leiningen
 
 sed -i 's|127.0.0.1|'"${DBHOST}"'|g' hello/src/hello/handler.clj
 

frameworks/Clojure/http-kit/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends leiningen java
+fw_depends mysql leiningen java
 
 sed -i 's|127.0.0.1|'"${DBHOST}"'|g' hello/src/hello/handler.clj
 

frameworks/Clojure/luminus/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java resin leiningen
+fw_depends postgresql java resin leiningen
 
 # Update db host in the source file
 sed -i 's|127.0.0.1|'"${DBHOST}"'|g' hello/env/prod/resources/config.edn

frameworks/Clojure/pedestal/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java leiningen
+fw_depends mysql java leiningen
 
 sed -i 's|127.0.0.1:3306|'${DBHOST}':3306|g' src/pedestal/service.clj
 

frameworks/Crystal/kemal/setup-postgres.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends crystal
+fw_depends postgresql crystal
 
 shards install
 

frameworks/D/vibed/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends dlang dub
+fw_depends mongodb dlang dub
 
 # Clean any files from last run
 rm -f fwb

frameworks/D/vibed/setup_ldc.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends dlang dub
+fw_depends mongodb dlang dub
 
 # Clean any files from last run
 rm -f fwb

frameworks/Dart/dart-raw/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|host: .*|host: '"${DBHOST}"'|g' postgresql.yaml
 
-fw_depends dart
+fw_depends postgresql dart
 
 pub upgrade
 

frameworks/Dart/redstone/benchmark_config.json

@@ -2,7 +2,7 @@
   "framework": "redstone",
   "tests": [{
     "default": {
-      "setup_file": "setup",
+      "setup_file": "setup_postgresql",
       "json_url": "/json",
       "db_url": "/pg/db",
       "query_url": "/pg/queries?queries=",
@@ -26,7 +26,7 @@
       "versus": "dart"
     },
     "mongodb": {
-      "setup_file": "setup",
+      "setup_file": "setup_mongodb",
       "db_url": "/mongo/db",
       "query_url": "/mongo/queries?queries=",
       "fortune_url": "/mongo/fortunes",

frameworks/Dart/redstone/setup_mongodb.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mongodb
+
+source ./setup.sh

frameworks/Dart/redstone/setup_postgresql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends postgresql
+
+source ./setup.sh

frameworks/Dart/start/benchmark_config.json

@@ -2,7 +2,7 @@
   "framework": "start",
   "tests": [{
     "default": { 
-      "setup_file": "setup",
+      "setup_file": "setup_postgresql",
       "json_url": "/json",
       "db_url": "/db",
       "query_url": "/queries?queries=",
@@ -26,7 +26,7 @@
       "versus": "dart"
     },
     "mongodb-raw": {
-      "setup_file": "setup",
+      "setup_file": "setup_mongodb",
       "db_url": "/db-mongo",
       "query_url": "/queries-mongo?queries=",
       "fortune_url": "/fortunes-mongo",

frameworks/Dart/start/setup_mongodb.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mongodb
+
+source ./setup.sh

frameworks/Dart/start/setup_postgresql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends postgresql
+
+source ./setup.sh

frameworks/Dart/stream/benchmark_config.json

@@ -2,7 +2,7 @@
   "framework": "stream",
   "tests": [{
     "default": {
-      "setup_file": "setup",
+      "setup_file": "setup_postgresql",
       "json_url": "/json",
       "db_url": "/db",
       "query_url": "/queries?queries=",
@@ -26,7 +26,7 @@
       "versus": "dart"
     },
     "mongodb-raw": {
-      "setup_file": "setup",
+      "setup_file": "setup_mongodb",
       "db_url": "/db-mongo",
       "query_url": "/queries-mongo?queries=",
       "fortune_url": "/fortunes-mongo",

frameworks/Dart/stream/setup_mongodb.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mongodb
+
+source ./setup.sh

frameworks/Dart/stream/setup_postgresql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends postgresql
+
+source ./setup.sh

frameworks/Elixir/phoenix/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends elixir
+fw_depends postgresql elixir
 
 sed -i 's|localhost|'${DBHOST}'|g' config/prod.exs
 

frameworks/Erlang/cowboy/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|"benchmarkdbpass", ".*", 3306|"benchmarkdbpass", "'"${DBHOST}"'", 3306|g' src/hello_world_app.erl
 
-fw_depends erlang
+fw_depends mysql erlang
 
 rm -rf deps/* ebin/*
 rebar get-deps

frameworks/Go/beego/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|tcp(.*:3306)|tcp('"${DBHOST}"':3306)|g' src/hello/hello.go
 
-fw_depends go
+fw_depends mysql go
 
 go get github.com/astaxie/beego
 go get github.com/go-sql-driver/mysql

frameworks/Go/echo/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends go
+fw_depends mysql go
 
 go get github.com/labstack/echo/...
 go get github.com/lib/pq

frameworks/Go/echo/setup_prefork.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends go
+fw_depends postgresql go
 
 go get github.com/labstack/echo/...
 go get github.com/lib/pq

frameworks/Go/echo/setup_std.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends go
+fw_depends postgresql go
 
 go get github.com/labstack/echo/...
 go get github.com/lib/pq

frameworks/Go/falcore/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|tcp(.*:3306)|tcp('"${DBHOST}"':3306)|g' src/framework_benchmarks/falcore.go
 
-fw_depends go
+fw_depends mysql go
 
 go get ./...
 go run src/framework_benchmarks/falcore.go &

frameworks/Go/fasthttp/setup-mysql.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|tcp(.*:3306)|tcp('"${DBHOST}"':3306)|g' src/server-mysql/server.go
 
-fw_depends go
+fw_depends mysql go
 
 GOPATH=`pwd` go get -u github.com/go-sql-driver/mysql
 GOPATH=`pwd` go get -u github.com/valyala/fasthttp/...

frameworks/Go/fasthttp/setup-postgresql.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|localhost|'"${DBHOST}"'|g' src/server-postgresql/server.go
 
-fw_depends go
+fw_depends postgresql go
 
 GOPATH=`pwd` go get -u github.com/jackc/pgx
 GOPATH=`pwd` go get -u github.com/valyala/fasthttp/...

frameworks/Go/gin/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends go
+fw_depends mysql go
 
 go get github.com/gin-gonic/gin
 go get github.com/go-sql-driver/mysql

frameworks/Go/go-std/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends go
+fw_depends mysql go
 
 go get github.com/go-sql-driver/mysql
 

frameworks/Go/go-std/setup_mongo.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|connectionString = "localhost"|connectionString = "'"${DBHOST}"'"|g' hello_mongo.go
 
-fw_depends go libsasl2-dev 
+fw_depends mongodb go libsasl2-dev 
 
 go get gopkg.in/mgo.v2
 

frameworks/Go/go-std/setup_postgres.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends go
+fw_depends postgresql go
 
 go get github.com/lib/pq
 

frameworks/Go/go-std/setup_prefork.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends go
+fw_depends mysql go
 
 go get github.com/go-sql-driver/mysql
 

frameworks/Go/goji/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|tcp(.*:3306)|tcp('"${DBHOST}"':3306)|g' src/goji/server.go
 
-fw_depends go
+fw_depends mysql go
 
 go get github.com/go-sql-driver/mysql
 go get github.com/zenazn/goji

frameworks/Go/kami/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|tcp(.*:3306)|tcp('"${DBHOST}"':3306)|g' src/kami/server.go
 
-fw_depends go
+fw_depends mysql go
 
 go get github.com/go-sql-driver/mysql
 go get github.com/guregu/kami

frameworks/Go/revel/benchmark_config.json

@@ -22,7 +22,7 @@
       "versus": "go"
     },
     "raw": {
-      "setup_file": "setup",
+      "setup_file": "setup_mysql",
       "db_url": "/db",
       "query_url": "/db?queries=",
       "fortune_url": "/fortune",
@@ -44,7 +44,7 @@
       "versus": "go"
     },
     "jet": {
-      "setup_file": "setup",
+      "setup_file": "setup_mysql",
       "db_url": "/jet-db",
       "query_url": "/jet-db?queries=",
       "fortune_url": "/jet-fortune",
@@ -66,7 +66,7 @@
       "versus": "go"
     },
     "qbs": {
-      "setup_file": "setup",
+      "setup_file": "setup_mysql",
       "db_url": "/qbs-db",
       "query_url": "/qbs-db?queries=",
       "fortune_url": "/qbs-fortune",

frameworks/Go/revel/setup_mysql.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mysql
+
+source ./setup.sh

frameworks/Groovy/grails/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|jdbc:mysql://.*:3306|jdbc:mysql://'"${DBHOST}"':3306|g' hello/grails-app/conf/DataSource.groovy
 
-fw_depends java grails resin 
+fw_depends mysql java grails resin 
 
 cd hello
 grails -Dgrails.work.dir=${IROOT}/.grails -non-interactive -plain-output refresh-dependencies

frameworks/Groovy/hot/benchmark_config.json

@@ -21,7 +21,7 @@
       "versus": ""
     },
     "mongodb" : {
-      "setup_file": "setup",
+      "setup_file": "setup_mongodb",
       "db_url": "/rest/db/mongodb",
       "query_url": "/rest/queries/mongodb?queries=",
       "fortune_url": "/rest/fortunes/mongodb",

frameworks/Groovy/hot/setup_mongodb.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+fw_depends mongodb
+
+source ./setup.sh

frameworks/Haskell/servant/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends stack
+fw_depends postgresql stack
 
 ${IROOT}/stack --allow-different-user setup
 ${IROOT}/stack --allow-different-user build

frameworks/Haskell/snap/setup.sh

@@ -2,7 +2,7 @@
 
 sed -i 's|host=".*"|host="'"${DBHOST}"'"|g' bench/cfg/db.cfg
 
-fw_depends stack
+fw_depends mysql stack
 
 cd bench
 

frameworks/Haskell/spock/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends stack
+fw_depends postgresql stack
 
 sed -i 's|PG.connectHost     = "localhost"|PG.connectHost     = "'"${DBHOST}"'"|g' src/Main.hs
 

frameworks/Haskell/yesod/run_yesod_mysql_mongo.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends stack
+fw_depends mysql mongodb stack
 
 cd yesod-mysql-mongo
 

frameworks/Haskell/yesod/run_yesod_postgres.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends stack
+fw_depends postgresql stack
 
 cd yesod-postgres
 

frameworks/Java/activeweb/setup.sh

@@ -3,7 +3,7 @@
 sed -i 's|localhost|'"${DBHOST}"'|g' src/main/webapp/WEB-INF/resin-web.xml
 sed -i 's|localhost|'"${DBHOST}"'|g' src/main/java/app/config/DbConfig.java
 
-fw_depends java resin maven
+fw_depends mysql java resin maven
 
 mvn clean package
 rm -rf $RESIN_HOME/webapps/*

frameworks/Java/beyondj/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java
+fw_depends mysql java
 
 WORKING_DIR=beyondj-launcher/deploy
 

frameworks/Java/dropwizard/benchmark_config.json

@@ -48,7 +48,7 @@
       "versus": ""
     },
     "postgres": {
-      "setup_file": "setup_postgres",
+      "setup_file": "setup_postgresql",
       "db_url": "/db",
       "query_url": "/db?queries=",
       "fortune_url": "/fortunes",

frameworks/Java/dropwizard/setup_mongo.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java maven
+fw_depends mongodb java maven
 
 sed -i 's|host: 127.0.0.1|host: '"${DBHOST}"'|g' hello-world-mongo.yml
 

frameworks/Java/dropwizard/setup_mysql.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java maven
+fw_depends mysql java maven
 
 sed -i 's|url: jdbc:mysql://.*/hello_world|url: jdbc:mysql://'"${DBHOST}"':3306/hello_world|g' hello-world-mysql.yml
 

frameworks/Java/dropwizard/setup_postgres.sh → frameworks/Java/dropwizard/setup_postgresql.sh

@@ -1,9 +1,9 @@
 #!/bin/bash
 
-fw_depends java maven
+fw_depends postgresql java maven
 
 sed -i 's|url: jdbc:postgresql://.*/hello_world|url: jdbc:postgresql://'"${DBHOST}"':5432/hello_world|g' hello-world-postgres.yml
 
-mvn -P postgres clean package
+mvn -P postgresql clean package
 
 java -jar target/hello-world-0.0.1-SNAPSHOT.jar server hello-world-postgres.yml &

frameworks/Java/gemini/benchmark_config.json

@@ -44,7 +44,7 @@
       "versus": "servlet"
     },
     "postgres": {
-      "setup_file": "start_postgres",
+      "setup_file": "start_postgresql",
       "db_url": "/db",
       "query_url": "/query?queries=",
       "fortune_url": "/fortunes",

frameworks/Java/grizzly-jersey/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java maven
+fw_depends mysql java maven
 
 mvn clean package
 

frameworks/Java/jawn/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java
+fw_depends mysql java
 
 ./gradlew clean
 

frameworks/Java/jooby/benchmark_config.json

@@ -9,7 +9,7 @@
       "port": 8080,
       "approach": "Realistic",
       "classification": "Fullstack",
-      "database": "None",
+      "database": "MySQL",
       "framework": "jooby",
       "language": "Java",
       "flavor": "None",

frameworks/Java/jooby/setup.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-fw_depends java maven
+fw_depends mysql java maven
 
 sed -i 's|localhost|'"${DBHOST}"'|g' conf/application.conf
 

frameworks/Java/ninja-standalone/setup.sh

@@ -3,7 +3,7 @@
 sed -i 's|mysql://.*:3306|mysql://'"${DBHOST}"':3306|g' src/main/java/conf/application.conf
 sed -i 's|mysql://.*:3306|mysql://'"${DBHOST}"':3306|g' src/main/resources/META-INF/persistence.xml
 
-fw_depends java maven
+fw_depends mysql java maven
 
 mvn clean compile assembly:single