CI: updated build/test images (#1904)

- Fixed the ODBC test.
- Added support for Python 3 in ubertest.php.
- Auto-generated /root/.sphinx based on the architecture.
- Minor updates to the manual compilation instructions in the documentation.

.github/workflows/test_template.yml

@@ -44,7 +44,7 @@ jobs:
       run:
         shell: bash
     container:
-      image: manticoresearch/ubertests_ctest:3263_jammy
+      image: manticoresearch/ubertests_ctest:3263_mar_2024
       env:
         DIAGNOSTIC: 1
         CACHEB: ../cache

.gitignore

@@ -67,6 +67,14 @@
 /test/test_*/Conf
 /test/data
 /test/localsettings.inc
+/test/error_all.txt
+/test/guess.txt
+/test/query.log
+/test/searchd.log
+/test/clt-tests/**/*.rep
+/test/clt-tests/**/*.deb
+/clt
+/test/clt-tests/*repo.noarch.deb.*
 
 # /test/ql/
 /test/ql/data/*.log
@@ -74,8 +82,6 @@
 /test/ql/*.class
 /test/ql/*.exe
 
+# misc
+/cache
 /local_*_src.txt
-/test/clt-tests/**/*.rep
-/clt
-/test/clt-tests/**/*.deb
-/test/clt-tests/*repo.noarch.deb.*

dist/build_dockers/cross/external_toolchain/Dockerfile

@@ -1,4 +1,6 @@
 FROM ubuntu:focal
+# Note, bison in Jammy leads to some updated error messages and some of them look wrong, e.g.
+# "expecting end of file" instead of "expecting $end"
 ENV DEBIAN_FRONTEND noninteractive
 
 # 1-st layer, basic toolchain stuff

manual/Installation/Compiling_from_sources.md

@@ -21,7 +21,7 @@ cmake -DPACK=1 /path/to/sources
 cmake --build .
 ```
 
-For example, to create the same RedHat 7 package as the official one, but without the embedded ICU and its large datafile, you can execute the following (assuming that the sources are placed in `/manticore/sources/` on the host):
+For instance, to create a package for RHEL7-compatible operating systems that is similar to the official version Manticore Core Team provides, you should execute the following commands in the directory containing the Manticore Search sources. This directory is the root of a cloned repository from https://github.com/manticoresoftware/manticoresearch:
 
 ```bash
 docker run -it --rm -e SYSROOT_URL=https://repo.manticoresearch.com/repository/sysroots \
@@ -29,15 +29,17 @@ docker run -it --rm -e SYSROOT_URL=https://repo.manticoresearch.com/repository/s
 -e DISTR=rhel7 \
 -e boost=boost_rhel_feb17 \
 -e sysroot=roots_nov22 \
--v /manticore/sources:/manticore_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
+-v $(pwd):/manticore_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa \
 manticoresearch/external_toolchain:clang16_cmake3263 bash
 
 # following is to be run inside docker shell
 cd /manticore_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa/
-RELEASE_TAG="noicu"
 mkdir build && cd build
-cmake -DPACK=1 -DBUILD_TAG=$RELEASE_TAG -DWITH_ICU_FORCE_STATIC=0 ..
-cmake --build . --target package
+cmake -DPACK=1 ..
+
+cmake --build .
+# or if you want to build packages:
+# cmake --build . --target package
 ```
 The long source directory path is required or it may fail to build the sources.
 

misc/ctest/ubertests/Dockerfile

@@ -2,7 +2,7 @@ FROM ubuntu:jammy
 ENV DEBIAN_FRONTEND noninteractive
 ENV DISTR jammy
 
-RUN apt-get update && apt-get install -y wget
+RUN apt-get -y update && apt-get install -y wget
 
 RUN wget https://repo.manticoresearch.com/manticore-dev-repo.noarch.deb && \
     dpkg -i manticore-dev-repo.noarch.deb && \
@@ -67,22 +67,31 @@ chmod +x llvm.sh && \
 ENV CC clang-13
 ENV CXX clang++-13
 
-# mysql
-RUN wget --no-check-certificate -q -O /odbc.tar.gz https://dev.mysql.com/get/Downloads/Connector-ODBC/5.3/mysql-connector-odbc-5.3.9-linux-ubuntu16.04-x86-64bit.tar.gz \
+# Ubertests settings file
+ARG TARGETPLATFORM
+RUN if [ "$TARGETPLATFORM" = "linux/arm64" ]; then ARCHITECTURE=aarch64; else ARCHITECTURE=x86-64bit; fi \
+&& wget --no-check-certificate -q -O /odbc.tar.gz https://dev.mysql.com/get/Downloads/Connector-ODBC/8.3/mysql-connector-odbc-8.3.0-linux-glibc2.28-${ARCHITECTURE}.tar.gz \
     && tar -zxf /odbc.tar.gz \
     && rm /odbc.tar.gz \
     && mkdir -p /var/run/mysqld && chmod a+rwX /var/run/mysqld \
     && { mysqld & } && sleep 30 \
     && mysql -e 'CREATE DATABASE test; CREATE USER test@localhost; GRANT ALL PRIVILEGES ON test.* TO test@localhost;' \
-    && mysqladmin shutdown
-
-# Note: for mysql 8 on x86-64 change url above to https://dev.mysql.com/get/Downloads/Connector-ODBC/8.3/mysql-connector-odbc-8.3.0-linux-glibc2.28-x86-64bit.tar.gz,
-# and 'odbc_driver'=>'/mysql-connector-odbc-8.3.0-linux-glibc2.28-x86-64bit/lib/libmyodbc8w.so;CHARSET=utf8' line in file 'sphinx'
-
-# Note: for mysql 8 on ARM64 change url above to https://dev.mysql.com/get/Downloads/Connector-ODBC/8.3/mysql-connector-odbc-8.3.0-linux-glibc2.28-aarch64.tar.gz,
-# and 'odbc_driver'=>'/mysql-connector-odbc-8.3.0-linux-glibc2.28-aarch64/lib/libmyodbc8w.so;CHARSET=utf8' line in file 'sphinx'
-
-# Note for both: if test 228 fails with broken encoding - try to remove ';CHARSET=utf8' ending in the odbc_driver line above. M.b. better is to probe once on local instance before pushing to dockerhub
+    && mysqladmin shutdown \
+&& <<EOF cat > /root/.sphinx
+(
+    "db-host"=>"127.0.0.1",
+    "db-user"=>"test",
+    'lemmatizer_base'=>'/work/aot',
+    'odbc_driver'=>'/mysql-connector-odbc-8.3.0-linux-glibc2.28-${ARCHITECTURE}/lib/libmyodbc8w.so;CHARSET=utf8'
+)
+EOF
+
+# ODBC notes:
+# * Urls for various odbc mysql driver versions:
+#   - https://dev.mysql.com/get/Downloads/Connector-ODBC/8.3/mysql-connector-odbc-8.3.0-linux-glibc2.28-x86-64bit.tar.gz
+#   - https://dev.mysql.com/get/Downloads/Connector-ODBC/8.3/mysql-connector-odbc-8.3.0-linux-glibc2.28-aarch64.tar.gz
+#   - https://dev.mysql.com/get/Downloads/Connector-ODBC/5.3/mysql-connector-odbc-5.3.9-linux-ubuntu16.04-x86-64bit.tar.gz
+# * If test 228 fails with broken encoding - try to remove ';CHARSET=utf8' ending in the odbc_driver line above. M.b. better is to probe once on local instance before pushing to dockerhub
 
 # AOT
 RUN mkdir -p /work/aot && cd /work/aot && \
@@ -97,7 +106,6 @@ rm *.tar.xz
 # Cleanup
 RUN rm -rf /var/lib/apt/lists/*
 
-ADD sphinx /root/.sphinx
 RUN git config --global --add safe.directory '*'
 ADD entry_point.sh /
 VOLUME [ "/work" ]
@@ -107,7 +115,7 @@ CMD []
 
 # docker buildx create --name manticore_build --platform linux/amd64,linux/arm64
 # docker buildx use manticore_build
-# docker buildx build --platform linux/amd64,linux/arm64 --push -t manticoresearch/ubertests_ctest:3263_jammy .
+# docker buildx build --platform linux/amd64,linux/arm64 --push -t manticoresearch/ubertests_ctest:3263_mar_2024 .
 
 # deprecated:
 # docker build -t registry.gitlab.com/manticoresearch/dev/ubertests_ctest:3263 .

misc/ctest/ubertests/sphinx

@@ -1,6 +0,0 @@
-(
-    "db-host"=>"127.0.0.1",
-    "db-user"=>"test",
-    'lemmatizer_base'=>'/work/aot',
-    'odbc_driver'=>'/mysql-connector-odbc-5.3.9-linux-ubuntu16.04-x86-64bit/lib/libmyodbc5w.so'
-)

test/ubertest.php

@@ -155,10 +155,10 @@ $VLG = getenv('VLG');
 
 $python = getenv('python');
 if (!$python)
-	$python = "/usr/bin/python";
+  $python = file_exists("/usr/bin/python3")?"/usr/bin/python3":"/usr/bin/python";
 
 if (!$windows && !is_executable($python) && !$ctest)
-	die("ubertest needs python support; install python");
+	die("ubertest needs python support; install python\n");
 
 $cygwin = false;
 if ( $locals['scriptdir']!=$locals['testdir'] )