added dockerized tests

docker-compose-test.yml

@@ -0,0 +1,66 @@
+version: "2"
+
+services:
+
+  db:
+    build:
+      context: .
+      dockerfile: misc/dockerfiles/mysql/Dockerfile
+    image: manticore_test_db
+    environment:
+        MYSQL_ROOT_PASSWORD: test
+        MYSQL_USER: test
+        MYSQL_PASSWORD: test
+        MYSQL_DATABASE: test
+    
+  test:
+    build:
+      context: .
+      dockerfile: misc/dockerfiles/test/Dockerfile
+    image: manticore_test_id64
+    volumes:
+      - ./:/search_src
+      - /home/stas/aot/:/aot
+    environment:
+        MYSQL_USER: test
+        MYSQL_PASSWORD: test
+        MYSQL_DATABASE: test
+    
+    # command: tail -f /dev/null
+    command: /search_src/misc/dockerfiles/scripts/smoke.sh
+    links:
+      - db
+
+  test32:
+    build:
+      context: .
+      dockerfile: misc/dockerfiles/test/Dockerfile
+    image: manticore_test_id32    
+    volumes:
+      - ./:/search_src
+      - /home/stas/aot/:/aot
+    environment:
+        MYSQL_USER: test
+        MYSQL_PASSWORD: test
+        MYSQL_DATABASE: test
+    
+    command: /search_src/misc/dockerfiles/scripts/smoke.sh -c disable-id64
+    links:
+      - db
+      
+  test_rt:
+    build:
+      context: .
+      dockerfile: misc/dockerfiles/test/Dockerfile
+    image: manticore_test_rt
+    volumes:
+      - ./:/search_src
+      - /home/stas/aot/:/aot
+    environment:
+        MYSQL_USER: test
+        MYSQL_PASSWORD: test
+        MYSQL_DATABASE: test
+    
+    command: /search_src/misc/dockerfiles/scripts/smoke.sh -t rt
+    links:
+      - db

misc/dockerfiles/README.md

@@ -0,0 +1,55 @@
+# Introduction
+
+This is a direcectory with dockerized smoke tests of search engine.
+It depends only on docker and docker-compose.
+
+### Docker and composer installation
+
+docker installation
+~~~
+$ sudo apt-key adv --keyserver hkp://p80.pool.sks-keyservers.net:80 --recv-keys 58118E89F3A912897C070ADBF76221572C52609D
+$ sudo add-apt-repository "deb https://apt.dockerproject.org/repo/  ubuntu-$(lsb_release -cs)  main"
+$ sudo apt-get update
+$ sudo apt-get install docker-engine
+~~~
+docker-compose installation
+~~~
+$ sudo curl -o /usr/local/bin/docker-compose -L "https://github.com/docker/compose/releases/download/1.11.2/docker-compose-$(uname -s)-$(uname -m)"
+$ sudo chmod +x /usr/local/bin/docker-compose
+~~~
+Add user to docker group to be able to run docker commands
+~~~
+$ sudo usermod -aG docker ${USER}
+~~~
+Check of applications
+~~~
+$ docker version
+$ docker-compose version
+~~~
+# Next commands must be run at the root directory of search project
+
+# Build
+~~~
+$ docker-compose -f docker-compose-test.yml build
+~~~
+build should be done only once at source clone or checkout at the box or in case Dockerfile files got changed. That creates docker images.
+
+# Tests
+These starts docker docntainers and issue tests inside containers. Multiple containers might be started at same time.
+
+Regular test for id64 build
+~~~
+$ docker-compose -f docker-compose-test.yml up test
+~~~
+Tests for id32 build
+~~~
+$ docker-compose -f docker-compose-test.yml up test32
+~~~
+Tests of id64 RT pass
+~~~
+$ docker-compose -f docker-compose-test.yml up test_rt
+~~~
+To shutdown test containers
+~~~
+$ docker-compose -f docker-compose-test.yml down
+~~~

misc/dockerfiles/mysql/Dockerfile

@@ -0,0 +1,13 @@
+FROM mysql:5.7
+
+# default values
+ENV MYSQL_ROOT_PASSWORD test
+ENV MYSQL_USER test
+ENV MYSQL_PASSWORD test
+ENV MYSQL_DATABASE test
+
+VOLUME /var/lib/mysql
+
+ENTRYPOINT ["docker-entrypoint.sh"]
+
+CMD ["mysqld"]

misc/dockerfiles/mysql/docker-entrypoint.sh

@@ -0,0 +1,203 @@
+#!/bin/bash
+set -eo pipefail
+shopt -s nullglob
+
+# if command starts with an option, prepend mysqld
+if [ "${1:0:1}" = '-' ]; then
+	set -- mysqld "$@"
+fi
+
+# skip setup if they want an option that stops mysqld
+wantHelp=
+for arg; do
+	case "$arg" in
+		-'?'|--help|--print-defaults|-V|--version)
+			wantHelp=1
+			break
+			;;
+	esac
+done
+
+# usage: file_env VAR [DEFAULT]
+#    ie: file_env 'XYZ_DB_PASSWORD' 'example'
+# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
+#  "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
+file_env() {
+	local var="$1"
+	local fileVar="${var}_FILE"
+	local def="${2:-}"
+	if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
+		echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
+		exit 1
+	fi
+	local val="$def"
+	if [ "${!var:-}" ]; then
+		val="${!var}"
+	elif [ "${!fileVar:-}" ]; then
+		val="$(< "${!fileVar}")"
+	fi
+	export "$var"="$val"
+	unset "$fileVar"
+}
+
+_check_config() {
+	toRun=( "$@" --verbose --help )
+	if ! errors="$("${toRun[@]}" 2>&1 >/dev/null)"; then
+		cat >&2 <<-EOM
+
+			ERROR: mysqld failed while attempting to check config
+			command was: "${toRun[*]}"
+
+			$errors
+		EOM
+		exit 1
+	fi
+}
+
+# Fetch value from server config
+# We use mysqld --verbose --help instead of my_print_defaults because the
+# latter only show values present in config files, and not server defaults
+_get_config() {
+	local conf="$1"; shift
+	"$@" --verbose --help --log-bin-index="$(mktemp -u)" 2>/dev/null | awk '$1 == "'"$conf"'" { print $2; exit }'
+}
+
+# allow the container to be started with `--user`
+if [ "$1" = 'mysqld' -a -z "$wantHelp" -a "$(id -u)" = '0' ]; then
+	_check_config "$@"
+	DATADIR="$(_get_config 'datadir' "$@")"
+	mkdir -p "$DATADIR"
+	chown -R mysql:mysql "$DATADIR"
+	exec gosu mysql "$BASH_SOURCE" "$@"
+fi
+
+if [ "$1" = 'mysqld' -a -z "$wantHelp" ]; then
+	# still need to check config, container may have started with --user
+	_check_config "$@"
+	# Get config
+	DATADIR="$(_get_config 'datadir' "$@")"
+
+	if [ ! -d "$DATADIR/mysql" ]; then
+		file_env 'MYSQL_ROOT_PASSWORD'
+		if [ -z "$MYSQL_ROOT_PASSWORD" -a -z "$MYSQL_ALLOW_EMPTY_PASSWORD" -a -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then
+			echo >&2 'error: database is uninitialized and password option is not specified '
+			echo >&2 '  You need to specify one of MYSQL_ROOT_PASSWORD, MYSQL_ALLOW_EMPTY_PASSWORD and MYSQL_RANDOM_ROOT_PASSWORD'
+			exit 1
+		fi
+
+		mkdir -p "$DATADIR"
+
+		echo 'Initializing database'
+		"$@" --initialize-insecure
+		echo 'Database initialized'
+
+		if command -v mysql_ssl_rsa_setup > /dev/null && [ ! -e "$DATADIR/server-key.pem" ]; then
+			# https://github.com/mysql/mysql-server/blob/23032807537d8dd8ee4ec1c4d40f0633cd4e12f9/packaging/deb-in/extra/mysql-systemd-start#L81-L84
+			echo 'Initializing certificates'
+			mysql_ssl_rsa_setup --datadir="$DATADIR"
+			echo 'Certificates initialized'
+		fi
+
+		SOCKET="$(_get_config 'socket' "$@")"
+		"$@" --skip-networking --socket="${SOCKET}" &
+		pid="$!"
+
+		mysql=( mysql --protocol=socket -uroot -hlocalhost --socket="${SOCKET}" )
+
+		for i in {30..0}; do
+			if echo 'SELECT 1' | "${mysql[@]}" &> /dev/null; then
+				break
+			fi
+			echo 'MySQL init process in progress...'
+			sleep 1
+		done
+		if [ "$i" = 0 ]; then
+			echo >&2 'MySQL init process failed.'
+			exit 1
+		fi
+
+		if [ -z "$MYSQL_INITDB_SKIP_TZINFO" ]; then
+			# sed is for https://bugs.mysql.com/bug.php?id=20545
+			mysql_tzinfo_to_sql /usr/share/zoneinfo | sed 's/Local time zone must be set--see zic manual page/FCTY/' | "${mysql[@]}" mysql
+		fi
+
+		if [ ! -z "$MYSQL_RANDOM_ROOT_PASSWORD" ]; then
+			export MYSQL_ROOT_PASSWORD="$(pwgen -1 32)"
+			echo "GENERATED ROOT PASSWORD: $MYSQL_ROOT_PASSWORD"
+		fi
+
+		rootCreate=
+		# default root to listen for connections from anywhere
+		file_env 'MYSQL_ROOT_HOST' '%'
+		if [ ! -z "$MYSQL_ROOT_HOST" -a "$MYSQL_ROOT_HOST" != 'localhost' ]; then
+			# no, we don't care if read finds a terminating character in this heredoc
+			# https://unix.stackexchange.com/questions/265149/why-is-set-o-errexit-breaking-this-read-heredoc-expression/265151#265151
+			read -r -d '' rootCreate <<-EOSQL || true
+				CREATE USER 'root'@'${MYSQL_ROOT_HOST}' IDENTIFIED BY '${MYSQL_ROOT_PASSWORD}' ;
+				GRANT ALL ON *.* TO 'root'@'${MYSQL_ROOT_HOST}' WITH GRANT OPTION ;
+			EOSQL
+		fi
+
+		"${mysql[@]}" <<-EOSQL
+			-- What's done in this file shouldn't be replicated
+			--  or products like mysql-fabric won't work
+			SET @@SESSION.SQL_LOG_BIN=0;
+
+			DELETE FROM mysql.user WHERE user NOT IN ('mysql.sys', 'mysqlxsys', 'root') OR host NOT IN ('localhost') ;
+			SET PASSWORD FOR 'root'@'localhost'=PASSWORD('${MYSQL_ROOT_PASSWORD}') ;
+			GRANT ALL ON *.* TO 'root'@'localhost' WITH GRANT OPTION ;
+			${rootCreate}
+			DROP DATABASE IF EXISTS test ;
+			FLUSH PRIVILEGES ;
+		EOSQL
+
+		if [ ! -z "$MYSQL_ROOT_PASSWORD" ]; then
+			mysql+=( -p"${MYSQL_ROOT_PASSWORD}" )
+		fi
+
+		file_env 'MYSQL_DATABASE'
+		if [ "$MYSQL_DATABASE" ]; then
+			echo "CREATE DATABASE IF NOT EXISTS \`$MYSQL_DATABASE\` ;" | "${mysql[@]}"
+			mysql+=( "$MYSQL_DATABASE" )
+		fi
+
+		file_env 'MYSQL_USER'
+		file_env 'MYSQL_PASSWORD'
+		if [ "$MYSQL_USER" -a "$MYSQL_PASSWORD" ]; then
+			echo "CREATE USER '$MYSQL_USER'@'%' IDENTIFIED BY '$MYSQL_PASSWORD' ;" | "${mysql[@]}"
+
+			if [ "$MYSQL_DATABASE" ]; then
+				echo "GRANT ALL ON \`$MYSQL_DATABASE\`.* TO '$MYSQL_USER'@'%' ;" | "${mysql[@]}"
+			fi
+
+			echo 'FLUSH PRIVILEGES ;' | "${mysql[@]}"
+		fi
+
+		echo
+		for f in /docker-entrypoint-initdb.d/*; do
+			case "$f" in
+				*.sh)     echo "$0: running $f"; . "$f" ;;
+				*.sql)    echo "$0: running $f"; "${mysql[@]}" < "$f"; echo ;;
+				*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${mysql[@]}"; echo ;;
+				*)        echo "$0: ignoring $f" ;;
+			esac
+			echo
+		done
+
+		if [ ! -z "$MYSQL_ONETIME_PASSWORD" ]; then
+			"${mysql[@]}" <<-EOSQL
+				ALTER USER 'root'@'%' PASSWORD EXPIRE;
+			EOSQL
+		fi
+		if ! kill -s TERM "$pid" || ! wait "$pid"; then
+			echo >&2 'MySQL init process failed.'
+			exit 1
+		fi
+
+		echo
+		echo 'MySQL init process done. Ready for start up.'
+		echo
+	fi
+fi
+
+exec "$@"

misc/dockerfiles/scripts/smoke.sh

@@ -0,0 +1,101 @@
+#!/bin/sh
+
+#for CONFARGS in "--with-debug" "--with-debug --disable-id64" "--with-debug --with-unixodbc";
+CONFARGS="--with-debug"
+RT=""
+
+while getopts "c:t:" option; do
+  case $option in
+	c) CONFARGS="--with-debug --"$OPTARG ;;
+    t) RT="--"$OPTARG ;;
+  esac
+done
+
+# remove the options from the positional parameters
+shift $(( OPTIND - 1 ))
+
+jobs=$(grep -c processor /proc/cpuinfo)
+[ "z$FAILLOG" = "z" ] && FAILLOG="/tmp/faillog"
+LINE="-------------------------------\n"
+if [ q"1" = q"$jobs" ] ; then
+    jobs=""
+else
+    jobs="-j$jobs"
+fi
+
+echo "configure '"$CONFARGS"'"
+echo "jobs '"$jobs"'"
+echo "test '"$RT"'"
+
+die()
+{
+	cat $FAILLOG
+	echo $LINE
+	[ ! "z$2" = "z" ] && { eval $2; echo "$LINE"; }
+	echo "$1"
+	[ -e "$FAILLOG" ] && rm $FAILLOG
+	exit 1
+}
+
+
+cmd1 ()
+{
+	echo "Executing: $2\n">$FAILLOG
+	eval $2 1>>$1 2>&1 || die "$3" "$4"
+}
+
+cmd ()
+{
+	cmd1 "$FAILLOG" "$1" "$2" "$3" 
+}
+
+cmd "cp -R /search_src/* /search/"
+cmd "cd /search/"
+
+echo "Waiting for db instance"
+until mysql -h db -utest -p"$MYSQL_PASSWORD" &> /dev/null
+do
+  printf "."
+  sleep 1
+done
+
+echo "\t mysql ready"
+
+cat >/root/.sphinx <<EOL
+(
+"db-host"=>"db",
+"db-user"=>"test",
+"db-password"=>"$MYSQL_PASSWORD",
+"data"=>"data",
+"lemmatizer_base"=>"/aot",
+'odbc_driver'=>'MyODBC'
+)
+EOL
+
+cmd "mysql -h db -utest -p"$MYSQL_PASSWORD" test < example.sql" "Documents setup failed. Log in into mysql as admin and perform: CREATE DATABASE test; CREATE USER test@localhost; GRANT ALL PRIVILEGES ON test.* TO test@localhost;"
+
+BANNER="testing $CONFARGS build"
+./configure $CONFARGS
+make clean
+make $jobs
+
+cd ./test
+php ubertest.php t --keep-all $RT
+
+cd ..
+
+cd ./src
+./tests
+cd ..
+
+cd ./api/libsphinxclient
+./smoke_test.sh
+cd ../../ 
+
+zip result.zip ./test/test_*/report.txt ./test/error.txt ./api/libsphinxclient/smoke_diff.txt
+
+make clean 1>/dev/null 2>&1
+[ -e "$FAILLOG" ] && rm "$FAILLOG"
+
+echo "all ok\n"
+exit 0

misc/dockerfiles/test/Dockerfile

@@ -0,0 +1,18 @@
+FROM ubuntu:16.04
+
+WORKDIR /tmp
+RUN apt-get update && apt-get install -y \
+    autoconf automake libtool g++ \
+    build-essential \
+    git \
+    libexpat1-dev \
+    libmysqlclient-dev \
+    mysql-client \
+    php php-curl php-mysql php-xml \
+    python \
+    zip && \
+    apt-get clean
+
+RUN mkdir -p /search
+RUN mkdir -p /search_src
+RUN mkdir -p /aot