MySQL Update to 5.7 (#2851)

* MySQL Update to 5.7

* add keyserver key for mysql

* mysql now a service

* try and set the password

* if this works, no need to reset?

* trying different selection

* apparmor update for vagrant

toolset/setup/linux/databases/mysql/my.cnf

@@ -1,58 +1,36 @@
-#
-# The MySQL database server configuration file.
-#
-# You can copy this to one of:
-# - "/etc/mysql/my.cnf" to set global options,
-# - "~/.my.cnf" to set user-specific options.
-# 
-# One can use all long options that the program supports.
-# Run program with --help to get a list of available options and with
-# --print-defaults to see which it would actually understand and use.
-#
-# For explanations see
-# http://dev.mysql.com/doc/mysql/en/server-system-variables.html
+#######################
+# client              #
+#######################
 
-# This will be passed to all mysql clients
-# It has been reported that passwords should be enclosed with ticks/quotes
-# escpecially if they contain "#" chars...
-# Remember to edit /etc/mysql/debian.cnf when changing the socket location.
 [client]
 port            = 3306
 socket          = /var/run/mysqld/mysqld.sock
 
-# Here is entries for some specific programs
-# The following values assume you have at least 32M ram
-
-# This was formally known as [safe_mysqld]. Both versions are currently parsed.
-[mysqld_safe]
-socket          = /var/run/mysqld/mysqld.sock
-nice            = 0
+#######################
+# mysqld              #
+#######################
 
 [mysqld]
 #
 # * Basic Settings
 #
+default-storage-engine = innodb
+
 user            = mysql
 pid-file        = /var/run/mysqld/mysqld.pid
 socket          = /var/run/mysqld/mysqld.sock
 port            = 3306
-basedir         = /usr
-datadir         = /ssd/mysql
-tmpdir          = /tmp
-lc-messages-dir = /usr/share/mysql
 skip-external-locking
 skip-name-resolve
 lower_case_table_names = 1
 
 character-set-server=utf8
 collation-server=utf8_general_ci
-#
-# Instead of skip-networking the default is now to listen only on
-# localhost which is more compatible and is not less secure.
-#bind-address           = 127.0.0.1
+
 #
 # * Fine Tuning
 #
+
 key_buffer_size         = 16M
 max_allowed_packet      = 16M
 thread_stack            = 256K
@@ -63,9 +41,19 @@ table_open_cache        = 800
 table_definition_cache  = 800
 max_heap_table_size     = 128M
 tmp_table_size          = 128M
+
+#
+# innodb settings
+#
+
 innodb_use_native_aio   = 1
 # sync for every sec. not for every commit.
 innodb_flush_log_at_trx_commit = 2
+innodb_buffer_pool_size = 13G
+innodb_flush_method=O_DIRECT
+innodb_buffer_pool_instances=14
+
+sync_binlog=0
 
 #
 # * Query Cache Configuration
@@ -76,59 +64,14 @@ query_cache_type         = 0
 #query_cache_size = 0
 #query_cache_min_res_unit = 1K
 max_prepared_stmt_count  = 1048576
-#
-# * Logging and Replication
-#
-# Both location gets rotated by the cronjob.
-# Be aware that this log type is a performance killer.
-# As of 5.1 you can enable the log at runtime!
-#general_log_file        = /var/log/mysql/mysql.log
-#general_log             = 1
-#
-# Error logging goes to syslog due to /etc/mysql/conf.d/mysqld_safe_syslog.cnf.
-#
-# Here you can see queries with especially long duration
-#log_slow_queries       = /var/log/mysql/mysql-slow.log
-#long_query_time = 2
-#log-queries-not-using-indexes
-#
-# The following can be used as easy to replay backup logs or for replication.
-# note: if you are setting up a replication slave, see README.Debian about
-#       other settings you may need to change.
-#server-id              = 1
-#log_bin                        = /var/log/mysql/mysql-bin.log
-expire_logs_days        = 10
-max_binlog_size         = 100M
-#binlog_do_db           = include_database_name
-#binlog_ignore_db       = include_database_name
-#
-# * InnoDB
-#
-# InnoDB is enabled by default with a 10MB datafile in /var/lib/mysql/.
-# Read the manual for more InnoDB related options. There are many!
-#
-# * Security Features
-#
-# Read the manual, too, if you want chroot!
-# chroot = /var/lib/mysql/
-#
-# For generating SSL certificates I recommend the OpenSSL GUI "tinyca".
-#
-# ssl-ca=/etc/mysql/cacert.pem
-# ssl-cert=/etc/mysql/server-cert.pem
-# ssl-key=/etc/mysql/server-key.pem
 
+#######################
+# mysqldump           #
+#######################
 
 [mysqldump]
 quick
 quote-names
 max_allowed_packet      = 16M
 
-[mysql]
-#no-auto-rehash # faster start of mysql but no tab completition
-
-#
-# * IMPORTANT: Additional settings that can override those from this file!
-#   The files must end with '.cnf', otherwise they'll be ignored.
-#
 !includedir /etc/mysql/conf.d/

toolset/setup/linux/databases/mysql/mysql

@@ -1,188 +0,0 @@
-#!/bin/bash
-#
-### BEGIN INIT INFO
-# Provides:          mysql
-# Required-Start:    $remote_fs $syslog
-# Required-Stop:     $remote_fs $syslog
-# Should-Start:      $network $time
-# Should-Stop:       $network $time
-# Default-Start:     2 3 4 5
-# Default-Stop:      0 1 6
-# Short-Description: Start and stop the mysql database server daemon
-# Description:       Controls the main MySQL database server daemon "mysqld"
-#                    and its wrapper script "mysqld_safe".
-### END INIT INFO
-#
-set -e
-set -u
-${DEBIAN_SCRIPT_DEBUG:+ set -v -x}
-
-test -x /usr/bin/mysqld_safe || exit 0
-
-. /lib/lsb/init-functions
-
-SELF=$(cd $(dirname $0); pwd -P)/$(basename $0)
-CONF=/etc/mysql/my.cnf
-MYADMIN="/usr/bin/mysqladmin --defaults-file=/etc/mysql/debian.cnf"
-
-# priority can be overriden and "-s" adds output to stderr
-ERR_LOGGER="logger -p daemon.err -t /etc/init.d/mysql -i"
-
-# Safeguard (relative paths, core dumps..)
-cd /
-umask 077
-
-# mysqladmin likes to read /root/.my.cnf. This is usually not what I want
-# as many admins e.g. only store a password without a username there and
-# so break my scripts.
-export HOME=/etc/mysql/
-
-## Fetch a particular option from mysql's invocation.
-#
-# Usage: void mysqld_get_param option
-mysqld_get_param() {
-	/usr/sbin/mysqld --print-defaults \
-		| tr " " "\n" \
-		| grep -- "--$1" \
-		| tail -n 1 \
-		| cut -d= -f2
-}
-
-## Do some sanity checks before even trying to start mysqld.
-sanity_checks() {
-  # check for config file
-  if [ ! -r /etc/mysql/my.cnf ]; then
-    log_warning_msg "$0: WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz"
-    echo                "WARNING: /etc/mysql/my.cnf cannot be read. See README.Debian.gz" | $ERR_LOGGER
-  fi
-
-  # check for diskspace shortage
-#  datadir=`mysqld_get_param datadir`
-#  if LC_ALL=C BLOCKSIZE= df --portability $datadir/. | tail -n 1 | awk '{ exit ($4>4096) }'; then
-#    log_failure_msg "$0: ERROR: The partition with $datadir is too full!"
-#    echo                "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
-#    exit 1
-#  fi
-}
-
-## Checks if there is a server running and if so if it is accessible.
-#
-# check_alive insists on a pingable server
-# check_dead also fails if there is a lost mysqld in the process list
-#
-# Usage: boolean mysqld_status [check_alive|check_dead] [warn|nowarn]
-mysqld_status () {
-    ping_output=`$MYADMIN ping 2>&1`; ping_alive=$(( ! $? ))
-
-    ps_alive=0
-    pidfile=`mysqld_get_param pid-file`
-    if [ -f "$pidfile" ] && ps `cat $pidfile` >/dev/null 2>&1; then ps_alive=1; fi
-    
-    if [ "$1" = "check_alive"  -a  $ping_alive = 1 ] ||
-       [ "$1" = "check_dead"   -a  $ping_alive = 0  -a  $ps_alive = 0 ]; then
-	return 0 # EXIT_SUCCESS
-    else
-  	if [ "$2" = "warn" ]; then
-  	    echo -e "$ps_alive processes alive and '$MYADMIN ping' resulted in\n$ping_output\n" | $ERR_LOGGER -p daemon.debug
-	fi
-  	return 1 # EXIT_FAILURE
-    fi
-}
-
-#
-# main()
-#
-
-case "${1:-''}" in
-  'start')
-	sanity_checks;
-	# Start daemon
-	log_daemon_msg "Starting MySQL database server" "mysqld"
-	if mysqld_status check_alive nowarn; then
-	   log_progress_msg "already running"
-	   log_end_msg 0
-	else
-	    # Could be removed during boot
-	    test -e /var/run/mysqld || install -m 755 -o mysql -g root -d /var/run/mysqld
-
-	    # Start MySQL! 
-  	    su - mysql -s /bin/sh -c "/usr/bin/mysqld_safe > /dev/null 2>&1 &"
-
-	    # 6s was reported in #352070 to be too few when using ndbcluster
-	    # 14s was reported in #736452 to be too few with large installs
-	    for i in $(seq 1 30); do
-                sleep 1
-	        if mysqld_status check_alive nowarn ; then break; fi
-		log_progress_msg "."
-	    done
-	    if mysqld_status check_alive warn; then
-                log_end_msg 0
-	        # Now start mysqlcheck or whatever the admin wants.
-	        output=$(/etc/mysql/debian-start)
-		[ -n "$output" ] && log_action_msg "$output"
-	    else
-	        log_end_msg 1
-		log_failure_msg "Please take a look at the syslog"
-	    fi
-	fi
-	;;
-
-  'stop')
-	# * As a passwordless mysqladmin (e.g. via ~/.my.cnf) must be possible
-	# at least for cron, we can rely on it here, too. (although we have 
-	# to specify it explicit as e.g. sudo environments points to the normal
-	# users home and not /root)
-	log_daemon_msg "Stopping MySQL database server" "mysqld"
-	if ! mysqld_status check_dead nowarn; then
-	  set +e
-	  shutdown_out=`$MYADMIN shutdown 2>&1`; r=$?
-	  set -e
-	  if [ "$r" -ne 0 ]; then
-	    log_end_msg 1
-	    [ "$VERBOSE" != "no" ] && log_failure_msg "Error: $shutdown_out"
-	    log_daemon_msg "Killing MySQL database server by signal" "mysqld"
-	    killall -15 mysqld
-            server_down=
-	    for i in 1 2 3 4 5 6 7 8 9 10; do
-              sleep 1
-              if mysqld_status check_dead nowarn; then server_down=1; break; fi
-            done
-          if test -z "$server_down"; then killall -9 mysqld; fi
-	  fi
-        fi
-
-        if ! mysqld_status check_dead warn; then
-	  log_end_msg 1
-	  log_failure_msg "Please stop MySQL manually and read /usr/share/doc/mysql-server-5.6/README.Debian.gz!"
-	  exit -1
-	else
-	  log_end_msg 0
-        fi
-	;;
-
-  'restart')
-	set +e; $SELF stop; set -e
-	$SELF start 
-	;;
-
-  'reload'|'force-reload')
-  	log_daemon_msg "Reloading MySQL database server" "mysqld"
-	$MYADMIN reload
-	log_end_msg 0
-	;;
-
-  'status')
-	if mysqld_status check_alive nowarn; then
-	  log_action_msg "$($MYADMIN version)"
-	else
-	  log_action_msg "MySQL is stopped."
-	  exit 3
-	fi
-  	;;
-
-  *)
-	echo "Usage: $SELF start|stop|restart|reload|force-reload|status"
-	exit 1
-	;;
-esac
-

toolset/setup/linux/databases/mysql/mysql.conf

@@ -1,63 +0,0 @@
-description     "MySQL 5.6 Server"
-author          "Mario Limonciello <[email protected]>"
-
-start on runlevel [2345]
-stop on starting rc RUNLEVEL=[016]
-
-respawn
-respawn limit 2 5
-
-env HOME=/etc/mysql
-umask 007
-
-# The default of 5 seconds is too low for mysql which needs to flush buffers
-kill timeout 300
-
-pre-start script
-    ## Fetch a particular option from mysql's invocation.
-    # Usage: void mysqld_get_param option
-    mysqld_get_param() {
-      /usr/sbin/mysqld --print-defaults \
-        | tr " " "\n" \
-        | grep -- "--$1" \
-        | tail -n 1 \
-        | cut -d= -f2
-    }
-
-    # priority can be overriden and "-s" adds output to stderr
-    ERR_LOGGER="logger -p daemon.err -t /etc/init/mysql.conf -i"
-
-    #Sanity checks
-    [ -r $HOME/my.cnf ]
-    [ -d /var/run/mysqld ] || install -m 755 -o mysql -g root -d /var/run/mysqld
-    /lib/init/apparmor-profile-load usr.sbin.mysqld
-
-    # check for diskspace shortage
-#    datadir=`mysqld_get_param datadir`
-#    BLOCKSIZE=`LC_ALL=C df --portability $datadir/. | tail -n 1 | awk '{print $4}'`
-#    if [ $BLOCKSIZE -le 4096 ] ; then
-#      echo "$0: ERROR: The partition with $datadir is too full!" >&2
-#      echo "ERROR: The partition with $datadir is too full!" | $ERR_LOGGER
-#      exit 1
-#    fi
-end script
-
-exec /usr/sbin/mysqld
-
-post-start script
-   for i in `seq 1 30` ; do
-        /usr/bin/mysqladmin --defaults-file="${HOME}"/debian.cnf ping && {
-            exec "${HOME}"/debian-start
-            # should not reach this line
-            exit 2
-        }
-        statusnow=`status`
-        if echo $statusnow | grep -q 'stop/' ; then
-            exit 0
-        elif echo $statusnow | grep -q 'respawn/' ; then
-            exit 1
-        fi
-        sleep 1
-    done
-    exit 1
-end script

toolset/setup/linux/databases/mysql/mysql.list

@@ -0,0 +1,7 @@
+# You may comment out entries below, but any other modifications may be lost.
+# Use command 'dpkg-reconfigure mysql-apt-config' as root for modifications.
+deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-apt-config
+deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7
+deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-tools
+#deb http://repo.mysql.com/apt/ubuntu/ trusty mysql-tools-preview
+deb-src http://repo.mysql.com/apt/ubuntu/ trusty mysql-5.7

toolset/setup/linux/databases/mysql/mysql.sh

@@ -6,23 +6,24 @@ fw_installed mysql && return 0
 
 # send over the required files
 scp $FWROOT/toolset/setup/linux/databases/mysql/create.sql $DBHOST:~/
-scp $FWROOT/toolset/setup/linux/databases/mysql/mysql $DBHOST:~/
-scp $FWROOT/toolset/setup/linux/databases/mysql/mysql.conf $DBHOST:~/
 scp $FWROOT/toolset/setup/linux/databases/mysql/my.cnf $DBHOST:~/
 scp $FWROOT/toolset/setup/linux/databases/mysql/usr.sbin.mysqld $DBHOST:~/
+scp $FWROOT/toolset/setup/linux/databases/mysql/mysql.list $DBHOST:~/
 
 # install mysql on database machine
 ssh $DBHOST 'bash' <<EOF
+sudo cp mysql.list /etc/apt/sources.list.d/
+sudo apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 8C718D3B5072E1F5
+sudo apt-get update
+sudo debconf-set-selections <<< "mysql-community-server mysql-community-server/data-dir select 'Y'"
+sudo debconf-set-selections <<< "mysql-community-server mysql-community-server/root-pass password secret"
+sudo debconf-set-selections <<< "mysql-community-server mysql-community-server/re-root-pass password secret"
 sudo DEBIAN_FRONTEND=noninteractive apt-get -y install mysql-server
 
-sudo stop mysql
-
-sudo mv mysql /etc/init.d/mysql
-sudo chmod +x /etc/init.d/mysql
-sudo mv mysql.conf /etc/init/mysql.conf
+sudo service mysql stop
 
 sudo mv /etc/mysql/my.cnf /etc/mysql/my.cnf.orig
-sudo mv my.cnf /etc/mysql/my.cnf
+sudo cp my.cnf /etc/mysql/my.cnf
 
 sudo rm -rf /ssd/mysql
 sudo rm -rf /ssd/log/mysql
@@ -30,16 +31,11 @@ sudo cp -R -p /var/lib/mysql /ssd/
 sudo cp -R -p /var/log/mysql /ssd/log
 sudo cp usr.sbin.mysqld /etc/apparmor.d/
 sudo /etc/init.d/apparmor reload
-sudo start mysql
-
-mysql -uroot -psecret -e'quit' &> /dev/null || sudo mysqladmin -u root password secret
+sudo service mysql start
 EOF
 
-# Install the mysql client
-sudo apt-get install -y mysql-client
-
 echo -e "ssh \$DBHOST 'bash' <<EOF" > $IROOT/mysql.installed
-echo -e "sudo start mysql || echo 'mysql service already started'" >> $IROOT/mysql.installed
+echo -e "sudo service mysql start || echo 'mysql service already started'" >> $IROOT/mysql.installed
 echo -e "mysqladmin -uroot -psecret flush-hosts" >> $IROOT/mysql.installed
 echo -e "mysql -uroot -psecret < create.sql" >> $IROOT/mysql.installed
 echo -e "EOF" >> $IROOT/mysql.installed

toolset/setup/linux/databases/mysql/usr.sbin.mysqld

@@ -27,6 +27,7 @@
   /usr/lib/mysql/plugin/*.so* mr,
   /usr/sbin/mysqld mr,
   /usr/share/mysql/** r,
+  /var/log/** rwk,
   /var/log/mysql.log rw,
   /var/log/mysql.err rw,
   /var/lib/mysql/ r,
@@ -35,6 +36,7 @@
   /var/log/mysql/* rw,
   /var/run/mysqld/mysqld.pid w,
   /var/run/mysqld/mysqld.sock w,
+  /run/mysqld/** rwk,
   /run/mysqld/mysqld.pid w,
   /run/mysqld/mysqld.sock w,
   /ssd/log/mysql.log rw,