Toolset and Database Updates (#4984)

* Toolset and Database Updates

* postgres apt source back to bionic

* Fix typo in mysql file

* Allow increase of vagrant disksize

* haskell fixes

Dockerfile

@@ -1,20 +1,22 @@
-FROM ubuntu:16.04
+FROM buildpack-deps:bionic
 
 # One -q produces output suitable for logging (mostly hides
 # progress indicators)
-RUN apt update -yqq
+RUN apt-get -yqq update
 
 # WARNING: DONT PUT A SPACE AFTER ANY BACKSLASH OR APT WILL BREAK
-RUN apt -qqy install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
+RUN apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" \
   git-core \
   cloc dstat                    `# Collect resource usage statistics` \
   python-dev \
   python-pip \
   software-properties-common \
-  libmysqlclient-dev            `# Needed for MySQL-python` \
-  libpq-dev                     `# Needed for psycopg2`
+  libmysqlclient-dev            `# Needed for MySQL-python`
 
-RUN pip install colorama==0.3.1 requests MySQL-python psycopg2-binary pymongo docker==3.5.0 psutil
+RUN pip install colorama==0.3.1 requests MySQL-python psycopg2-binary pymongo docker==4.0.2 psutil
+
+# Fix for docker-py trying to import one package from the wrong location
+RUN cp -r /usr/local/lib/python2.7/dist-packages/backports/ssl_match_hostname/ /usr/lib/python2.7/dist-packages/backports
 
 ENV PYTHONPATH /FrameworkBenchmarks
 ENV FWROOT /FrameworkBenchmarks

deployment/vagrant/core.rb

@@ -1,6 +1,6 @@
 def provision_bootstrap(config)
 
-  # TODO this will break if the environment contains the ' delimiter, 
+  # TODO this will break if the environment contains the ' delimiter,
   # so at some point we need to escape the ' character here and unescape
   # it in bootstrap.sh
   config.vm.provision "shell" do |sh|
@@ -30,6 +30,13 @@ def provider_virtualbox(config)
     override.vm.hostname = "TFB-all"
     override.vm.box = "ubuntu/xenial64"
 
+    # Allow increase in size for /dev/sda1
+    # Would need plugin:
+    #  vagrant plugin install vagrant-disksize
+    if ENV.fetch('TFB_DISKSIZE', "0") != "0"
+      override.disksize.size = ENV.fetch('TFB_DISKSIZE')
+    end
+
     if ENV.fetch('TFB_SHOW_VM', false)
       vb.gui = true
     end
@@ -43,12 +50,12 @@ def provider_virtualbox(config)
     vb.cpus = ENV.fetch('TFB_VB_CPU', 2)
 
     # The VirtualBox file system for shared folders (vboxfs)
-    # does not support posix's chown/chmod - these can only 
+    # does not support posix's chown/chmod - these can only
     # be set at mount time, and they are uniform for the entire
-    # shared directory. To mitigate the effects, we set the 
-    # folders and files to 777 permissions. 
+    # shared directory. To mitigate the effects, we set the
+    # folders and files to 777 permissions.
     # With 777 and owner vagrant *most* of the software works ok.
-    # Occasional issues are still possible. 
+    # Occasional issues are still possible.
     #
     # See mitchellh/vagrant#4997
     # See http://superuser.com/a/640028/136050

frameworks/Haskell/snap/bench/snap-bench.cabal

@@ -8,7 +8,7 @@ Maintainer:          [email protected]
 Stability:           Experimental
 Category:            Web
 Build-type:          Simple
-Cabal-version:       <=1.2
+Cabal-version:       >=1.2
 
 Executable snap-bench
   hs-source-dirs: src

frameworks/Haskell/snap/snap.dockerfile

@@ -1,7 +1,7 @@
-FROM haskell:8.2.1
+FROM haskell:8.6.3
 
 RUN apt update -yqq && apt install -yqq xz-utils make netbase
-RUN apt install -yqq libmysqlclient-dev pkg-config libpcre3 libpcre3-dev
+RUN apt install -yqq default-libmysqlclient-dev pkg-config libpcre3 libpcre3-dev
 
 COPY ./bench /snap
 WORKDIR /snap

frameworks/Haskell/spock/spock.dockerfile

@@ -1,4 +1,4 @@
-FROM haskell:8.2.1
+FROM haskell:8.6.3
 
 RUN apt update -yqq && apt install -yqq xz-utils make
 RUN apt install -yqq libpq-dev

toolset/databases/mongodb/mongodb.dockerfile

@@ -1,13 +1,11 @@
-FROM ubuntu:16.04
+FROM buildpack-deps:bionic
 
 COPY ./ ./
 
-RUN apt-get -y update > /dev/null
-RUN apt-get -y install apt-transport-https > /dev/null
-RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 2930ADAE8CAF5059EE73BB4B58712A2291FA4AD5
-RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu xenial/mongodb-org/3.6 multiverse" | tee /etc/apt/sources.list.d/mongodb-org-3.6.list
-RUN apt-get -y update > /dev/null
-RUN apt-get -y install mongodb-org > /dev/null
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv 68818C72E52529D4
+RUN echo "deb [ arch=amd64,arm64 ] https://repo.mongodb.org/apt/ubuntu bionic/mongodb-org/4.0 multiverse" | tee /etc/apt/sources.list.d/mongodb-org.list
+RUN apt-get -yqq update > /dev/null
+RUN DEBIAN_FRONTEND=noninteractive apt-get -yqq install apt-transport-https mongodb-org > /dev/null
 
 RUN mkdir -p /data/db
 RUN chmod 777 /data/db

toolset/databases/mysql/create.sql

@@ -12,8 +12,10 @@ CREATE TABLE  world (
   PRIMARY KEY  (id)
 )
 ENGINE=INNODB;
-GRANT SELECT, UPDATE ON hello_world.world TO 'benchmarkdbuser'@'%' IDENTIFIED BY 'benchmarkdbpass';
-GRANT SELECT, UPDATE ON hello_world.world TO 'benchmarkdbuser'@'localhost' IDENTIFIED BY 'benchmarkdbpass';
+CREATE USER 'benchmarkdbuser'@'%' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass';
+CREATE USER 'benchmarkdbuser'@'localhost' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass';
+GRANT SELECT, UPDATE ON hello_world.world TO 'benchmarkdbuser'@'%';
+GRANT SELECT, UPDATE ON hello_world.world TO 'benchmarkdbuser'@'localhost';
 
 DELIMITER #
 CREATE PROCEDURE load_data()
@@ -29,7 +31,8 @@ declare v_counter int unsigned default 0;
     SET v_counter=v_counter+1;
   end while;
   commit;
-END #
+END 
+#
 
 DELIMITER ;
 
@@ -41,8 +44,8 @@ CREATE TABLE  fortune (
   PRIMARY KEY  (id)
 )
 ENGINE=INNODB;
-GRANT SELECT ON hello_world.fortune TO 'benchmarkdbuser'@'%' IDENTIFIED BY 'benchmarkdbpass';
-GRANT SELECT ON hello_world.fortune TO 'benchmarkdbuser'@'localhost' IDENTIFIED BY 'benchmarkdbpass';
+GRANT SELECT ON hello_world.fortune TO 'benchmarkdbuser'@'%';
+GRANT SELECT ON hello_world.fortune TO 'benchmarkdbuser'@'localhost';
 
 INSERT INTO fortune (message) VALUES ('fortune: No such file or directory');
 INSERT INTO fortune (message) VALUES ('A computer scientist is someone who fixes things that aren''t broken.');

toolset/databases/mysql/create8.sql

@@ -1,61 +0,0 @@
-# To maintain consistency across servers and fix a problem with the jdbc per
-# http://stackoverflow.com/questions/37719818/the-server-time-zone-value-aest-is-unrecognized-or-represents-more-than-one-ti
-SET GLOBAL time_zone = '+00:00';
-
-# modified from SO answer http://stackoverflow.com/questions/5125096/for-loop-in-mysql
-CREATE DATABASE hello_world;
-USE hello_world;
-
-CREATE TABLE  world (
-  id int(10) unsigned NOT NULL auto_increment,
-  randomNumber int NOT NULL default 0,
-  PRIMARY KEY  (id)
-)
-ENGINE=INNODB;
-CREATE USER 'benchmarkdbuser'@'%' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass';
-CREATE USER 'benchmarkdbuser'@'localhost' IDENTIFIED WITH mysql_native_password BY 'benchmarkdbpass';
-GRANT SELECT, UPDATE ON hello_world.world TO 'benchmarkdbuser'@'%';
-GRANT SELECT, UPDATE ON hello_world.world TO 'benchmarkdbuser'@'localhost';
-
-DELIMITER #
-CREATE PROCEDURE load_data()
-BEGIN
-
-declare v_max int unsigned default 10000;
-declare v_counter int unsigned default 0;
-
-  TRUNCATE TABLE world;
-  START TRANSACTION;
-  while v_counter < v_max do
-    INSERT INTO world (randomNumber) VALUES ( floor(0 + (rand() * 10000)) );
-    SET v_counter=v_counter+1;
-  end while;
-  commit;
-END 
-#
-
-DELIMITER ;
-
-CALL load_data();
-
-CREATE TABLE  fortune (
-  id int(10) unsigned NOT NULL auto_increment,
-  message varchar(2048) CHARACTER SET 'utf8' NOT NULL,
-  PRIMARY KEY  (id)
-)
-ENGINE=INNODB;
-GRANT SELECT ON hello_world.fortune TO 'benchmarkdbuser'@'%';
-GRANT SELECT ON hello_world.fortune TO 'benchmarkdbuser'@'localhost';
-
-INSERT INTO fortune (message) VALUES ('fortune: No such file or directory');
-INSERT INTO fortune (message) VALUES ('A computer scientist is someone who fixes things that aren''t broken.');
-INSERT INTO fortune (message) VALUES ('After enough decimal places, nobody gives a damn.');
-INSERT INTO fortune (message) VALUES ('A bad random number generator: 1, 1, 1, 1, 1, 4.33e+67, 1, 1, 1');
-INSERT INTO fortune (message) VALUES ('A computer program does what you tell it to do, not what you want it to do.');
-INSERT INTO fortune (message) VALUES ('Emacs is a nice operating system, but I prefer UNIX. — Tom Christaensen');
-INSERT INTO fortune (message) VALUES ('Any program that runs right is obsolete.');
-INSERT INTO fortune (message) VALUES ('A list is only as strong as its weakest link. — Donald Knuth');
-INSERT INTO fortune (message) VALUES ('Feature: A bug with seniority.');
-INSERT INTO fortune (message) VALUES ('Computers make very fast, very accurate mistakes.');
-INSERT INTO fortune (message) VALUES ('<script>alert("This should not be displayed in a browser alert box.");</script>');
-INSERT INTO fortune (message) VALUES ('フレームワークのベンチマーク');

toolset/databases/mysql/my.cnf

@@ -62,12 +62,7 @@ sync_binlog=0
 
 #
 # * Query Cache Configuration
-#
-query_cache_type         = 0
-#query_cache_limit        = 1M
-#query_cache_size         = 64M
-#query_cache_size = 0
-#query_cache_min_res_unit = 1K
+
 max_prepared_stmt_count  = 1048576
 
 #######################

toolset/databases/mysql/mysql.dockerfile

@@ -1,4 +1,11 @@
-FROM ubuntu:16.04
+FROM buildpack-deps:bionic
+
+ADD create.sql create.sql
+ADD my.cnf my.cnf
+ADD mysql.list mysql.list
+
+RUN cp mysql.list /etc/apt/sources.list.d/
+RUN apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys 8C718D3B5072E1F5
 
 RUN apt-get update > /dev/null
 RUN apt-get install -yqq locales > /dev/null
@@ -8,13 +15,8 @@ ENV LANG en_US.UTF-8
 ENV LANGUAGE en_US:en
 ENV LC_ALL en_US.UTF-8
 
-ADD create.sql create.sql
-ADD my.cnf my.cnf
-ADD mysql.list mysql.list
-
-RUN cp mysql.list /etc/apt/sources.list.d/
-RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8C718D3B5072E1F5
-RUN apt-get update > /dev/null
+# https://bugs.mysql.com/bug.php?id=90695
+RUN ["/bin/bash", "-c", "debconf-set-selections <<< \"mysql-server mysql-server/lowercase-table-names select Enabled\""]
 RUN ["/bin/bash", "-c", "debconf-set-selections <<< \"mysql-community-server mysql-community-server/data-dir select 'Y'\""]
 RUN ["/bin/bash", "-c", "debconf-set-selections <<< \"mysql-community-server mysql-community-server/root-pass password secret\""]
 RUN ["/bin/bash", "-c", "debconf-set-selections <<< \"mysql-community-server mysql-community-server/re-root-pass password secret\""]
@@ -34,7 +36,7 @@ RUN cp -R -p /var/log/mysql /ssd/log
 # that relies on the mysql server running will explicitly start the server and
 # perform the work required.
 RUN chown -R mysql:mysql /var/lib/mysql /var/log/mysql /var/run/mysqld /ssd && \
-    service mysql start & \
+    mysqld & \
     until mysql -uroot -psecret -e "exit"; do sleep 1; done && \
     mysqladmin -uroot -psecret flush-hosts && \
     mysql -uroot -psecret < create.sql

toolset/databases/mysql/mysql.list

@@ -1,7 +1,7 @@
 # You may comment out entries below, but any other modifications may be lost.
 # Use command 'dpkg-reconfigure mysql-apt-config' as root for modifications.
-deb http://repo.mysql.com/apt/ubuntu/ xenial mysql-apt-config
-deb http://repo.mysql.com/apt/ubuntu/ xenial mysql-5.7
-deb http://repo.mysql.com/apt/ubuntu/ xenial mysql-tools
-#deb http://repo.mysql.com/apt/ubuntu/ xenial mysql-tools-preview
-deb-src http://repo.mysql.com/apt/ubuntu/ xenial mysql-5.7
+deb http://repo.mysql.com/apt/ubuntu/ bionic mysql-apt-config
+deb http://repo.mysql.com/apt/ubuntu/ bionic mysql-8.0
+deb http://repo.mysql.com/apt/ubuntu/ bionic mysql-tools
+#deb http://repo.mysql.com/apt/ubuntu/ bionic mysql-tools-preview
+deb-src http://repo.mysql.com/apt/ubuntu/ bionic mysql-8.0

toolset/databases/postgres/pgdg.list

@@ -1,2 +1,2 @@
-deb http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main
-deb-src http://apt.postgresql.org/pub/repos/apt/ xenial-pgdg main
+deb http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main
+deb-src http://apt.postgresql.org/pub/repos/apt/ bionic-pgdg main

toolset/databases/postgres/postgres.dockerfile

@@ -1,12 +1,4 @@
-FROM ubuntu:16.04
-
-RUN apt-get update > /dev/null
-RUN apt-get install -yqq locales wget > /dev/null
-
-RUN locale-gen en_US.UTF-8
-ENV LANG en_US.UTF-8
-ENV LANGUAGE en_US:en
-ENV LC_ALL en_US.UTF-8
+FROM buildpack-deps:bionic
 
 ADD postgresql.conf postgresql.conf
 ADD pg_hba.conf pg_hba.conf
@@ -19,11 +11,18 @@ ADD pgdg.list pgdg.list
 RUN cp pgdg.list /etc/apt/sources.list.d/
 RUN wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | apt-key add -
 
-# install postgresql on database machine
-RUN apt-get -y update > /dev/null
-RUN apt-get -y install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postgresql > /dev/null
+RUN apt-get -yqq update > /dev/null
+RUN apt-get -yqq install locales
 
 ENV PG_VERSION 11
+RUN locale-gen en_US.UTF-8
+ENV LANG en_US.UTF-8
+ENV LANGUAGE en_US:en
+ENV LC_ALL en_US.UTF-8
+ENV DEBIAN_FRONTEND noninteractive
+
+# install postgresql on database machine
+RUN apt-get -yqq install -o Dpkg::Options::="--force-confdef" -o Dpkg::Options::="--force-confold" postgresql
 
 # Make sure all the configuration files in main belong to postgres
 RUN mv postgresql.conf /etc/postgresql/${PG_VERSION}/main/postgresql.conf

toolset/wrk/wrk.dockerfile

@@ -1,4 +1,4 @@
-FROM buildpack-deps:xenial
+FROM buildpack-deps:bionic
 
 WORKDIR /wrk
 RUN curl -sL https://github.com/wg/wrk/archive/4.1.0.tar.gz | tar xz --strip-components=1