Add testrt project to common codebase

Also this used as test try and example of usage for header generator

src/CMakeLists.txt

@@ -150,8 +150,9 @@ target_link_libraries ( searchd libsphinx ${EXTRA_LIBRARIES} )
 target_link_libraries ( spelldump libsphinx ${EXTRA_LIBRARIES} )
 target_link_libraries ( wordbreaker libsphinx ${EXTRA_LIBRARIES} )
 
+include ( helpers )
+include ( genheader )
 if ( SPLIT_SYMBOLS )
-	include ( helpers )
 	install_dbg ( indexer )
 	install_dbg ( indextool )
 	install_dbg ( searchd )
@@ -195,15 +196,58 @@ if ( NOT DISABLE_TESTING )
 	if ( SPLIT_SYMBOLS )
 		install_dbg ( tests )
 	endif()
-    # testrt is win-only project
-	if ( WIN32 )
-		add_executable ( testrt testrt.cpp )
-		target_link_libraries ( testrt libsphinx ${EXTRA_LIBRARIES} )
-		add_test ( NAME "\"Internal rt src/tests\""
-			#	    	CONFIGURATIONS Debug
-				WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
-				COMMAND "testrt" )
-	endif ( WIN32 )
+
+	# compile rtestconfig.h
+	hsnippet ( rtestconfig testrt.conf "static" )
+
+	set (DATAFLD "${MANTICORE_BINARY_DIR}/test/data")
+	file ( MAKE_DIRECTORY ${DATAFLD} )
+	add_definitions ( -DDATAFLD="${DATAFLD}/" )
+	source_group ( "Testing config" FILES testrt.conf.in )
+	add_executable ( testrt testrt.cpp testrt.conf )
+	target_link_libraries ( testrt libsphinx ${EXTRA_LIBRARIES} )
+
+	# testrt is alive, however it needs database 'lj' accessible
+	# in order to work. So, avoid it's calling till the base is persist
+	# in out CI env
+	if (0)
+
+	add_test ( NAME "\"Remove dump index\""
+			COMMAND ${CMAKE_COMMAND} -E remove "${DATAFLD}/dump.*" )
+	SET_TESTS_PROPERTIES ( "\"Remove dump index\"" PROPERTIES FIXTURES_SETUP TESTRT )
+
+	add_test ( NAME "\"Internal rt src/testrt\""
+			WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+			COMMAND testrt )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt\"" PROPERTIES FIXTURES_REQUIRED TESTRT )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt\"" PROPERTIES RESOURCE_LOCK DbAccess )
+
+	add_test ( NAME "\"Internal rt src/testrt step 1000\""
+			WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+			COMMAND testrt 1000 )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 1000\"" PROPERTIES FIXTURES_REQUIRED TESTRT )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 1000\"" PROPERTIES RESOURCE_LOCK DbAccess )
+
+	add_test ( NAME "\"Internal rt src/testrt step 100\""
+			WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+			COMMAND testrt 100 )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 100\"" PROPERTIES FIXTURES_REQUIRED TESTRT )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 1000\"" PROPERTIES RESOURCE_LOCK DbAccess )
+
+	add_test ( NAME "\"Internal rt src/testrt step 10\""
+			WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+			COMMAND testrt 10 )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 10\"" PROPERTIES FIXTURES_REQUIRED TESTRT )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 1000\"" PROPERTIES RESOURCE_LOCK DbAccess )
+
+	add_test ( NAME "\"Internal rt src/testrt step 1\""
+			WORKING_DIRECTORY "${CMAKE_CURRENT_SOURCE_DIR}"
+			COMMAND testrt 1 )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 1\"" PROPERTIES FIXTURES_REQUIRED TESTRT )
+	SET_TESTS_PROPERTIES ( "\"Internal rt src/testrt step 1000\"" PROPERTIES RESOURCE_LOCK DbAccess )
+
+	endif(0)
+
 	# first check syntax
 	find_package ( PythonInterp QUIET )
 	if ( PYTHONINTERP_FOUND )

src/literal.h.in

@@ -0,0 +1,21 @@
+//
+// Copyright (c) 2017-2018, Manticore Software LTD (http://manticoresearch.com)
+// Copyright (c) 2001-2016, Andrew Aksyonoff
+// Copyright (c) 2008-2016, Sphinx Technologies Inc
+// All rights reserved
+//
+// This program is free software; you can redistribute it and/or modify
+// it under the terms of the GNU General Public License. You should have
+// received a copy of the GPL license along with this program; if you
+// did not, you can find it at http://www.gnu.org/
+
+// WARNING! This is generated file! Don't edit it manually!
+// To populate changes, edit the content of @_SRCFILE@,
+// then re-configure your build with cmake.
+
+// Outline file is src/literal.h.in
+
+// literal @SNIPPETNAME@ generated by CMake
+// from @_SRCFILE@
+
+@STATIC@ const char* @SNIPPETNAME@ = R"@SNIPPETNAME@(@_SNIPPET@)@SNIPPETNAME@";

src/testrt.conf.in

@@ -0,0 +1,26 @@
+#
+# Snippet of source config for testrt
+#
+
+source test1
+{
+	type			= mysql
+@SQL_SETTINGS@
+	sql_db			= lj
+	sql_query_pre	= SET NAMES utf8
+
+	sql_query		= \
+		SELECT id, channel_id, UNIX_TIMESTAMP(published) published, \
+		title, UNCOMPRESS(content) content FROM posting WHERE id<=10000 AND id%2=0
+
+	sql_attr_uint		= channel_id
+	sql_attr_timestamp	= published
+
+}
+
+source test2 : test1
+{
+	sql_query		= \
+		SELECT id, channel_id, UNIX_TIMESTAMP(published) published, \
+		title, UNCOMPRESS(content) content FROM posting WHERE id<=10000 AND id%2=1
+}

src/testrt.cpp

@@ -14,6 +14,16 @@
 #include "sphinxrt.h"
 #include "sphinxutils.h"
 
+#if HAVE_RTESTCONFIG_H
+#include "rtestconfig.h"
+#else
+const char * rtestconfig = "error";
+#endif
+
+#if !defined (DATAFLD)
+#define DATAFLD "data/"
+#endif
+
 #if USE_WINDOWS
 #include "psapi.h"
 #pragma comment(linker, "/defaultlib:psapi.lib")
@@ -44,6 +54,7 @@ void DoSearch ( CSphIndex * pIndex )
 	KillListVector dDummyKlist;
 	CSphMultiQueryArgs tArgs ( dDummyKlist, 1 );
 	tQuery.m_sQuery = "@title cat";
+	tQuery.m_pQueryParser = sphCreatePlainQueryParser ();
 
 	SphQueueSettings_t tQueueSettings ( tQuery, pIndex->GetMatchSchema(), tResult.m_sError, NULL );
 	tQueueSettings.m_bComputeItems = false;
@@ -90,6 +101,9 @@ void DoIndexing ( CSphSource_MySQL * pSrc, ISphRtIndex * pIndex )
 		if ( pSrc->m_tDocInfo.m_uDocID )
 			pIndex->AddDocument ( dFields, pSrc->m_tDocInfo, false, sFilter, NULL, dMvas, sError, sWarning, NULL );
 
+		auto& const_stat = pSrc->GetStats ();
+		++const_cast<CSphSourceStats&>(const_stat).m_iTotalDocuments;
+
 		if ( ( pSrc->GetStats().m_iTotalDocuments % COMMIT_STEP )==0 || !pSrc->m_tDocInfo.m_uDocID )
 		{
 			int64_t tmCommit = sphMicroTimer();
@@ -135,27 +149,412 @@ void DoIndexing ( CSphSource_MySQL * pSrc, ISphRtIndex * pIndex )
 	g_fTotalMB += fTotalMB;
 }
 
+// copy-pasted chunk from indexer.cpp
+// FIXME! it would be good to isolate that code and reuse instead of c-pasting
+
+static bool g_bPrintQueries = false;
+static int g_iMaxFileFieldBuffer = 8 * 1024 * 1024;
+static ESphOnFileFieldError g_eOnFileFieldError = FFE_IGNORE_FIELD;
 
-CSphSource_MySQL * SpawnSource ( const char * sQuery, ISphTokenizer * pTok, CSphDict * pDict )
+/////////////////////////////////////////////////////////////////////////////
+
+/// parse multi-valued attr definition
+bool ParseMultiAttr ( const char * sBuf, CSphColumnInfo &tAttr, const char * sSourceName )
 {
-	CSphSource_MySQL * pSrc = new CSphSource_MySQL ( "test" );
-	pSrc->SetTokenizer ( pTok );
-	pSrc->SetDict ( pDict );
+	// format is as follows:
+	//
+	// multi-valued-attr := ATTR-TYPE ATTR-NAME 'from' SOURCE-TYPE [;QUERY] [;RANGE-QUERY]
+	// ATTR-TYPE := 'uint' | 'timestamp' | 'bigint'
+	// SOURCE-TYPE := 'field' | 'query' | 'ranged-query'
+
+	const char * sTok = NULL;
+	int iTokLen = -1;
+
+#define LOC_ERR( _arg, _pos ) \
+    { \
+        if ( !*(_pos) ) \
+            fprintf ( stdout, "ERROR: source '%s': unexpected end of line in sql_attr_multi.\n", sSourceName ); \
+        else \
+            fprintf ( stdout, "ERROR: source '%s': expected " _arg " in sql_attr_multi, got '%s'.\n", sSourceName, _pos ); \
+        return false; \
+    }
+#define LOC_SPACE0()        { while ( isspace(*sBuf) ) sBuf++; }
+#define LOC_SPACE1()        { if ( !isspace(*sBuf) ) LOC_ERR ( "token", sBuf ) ; LOC_SPACE0(); }
+#define LOC_TOK()            { sTok = sBuf; while ( sphIsAlpha(*sBuf) ) sBuf++; iTokLen = sBuf-sTok; }
+#define LOC_TOKEQ( _arg )        ( iTokLen==(int)strlen(_arg) && strncasecmp ( sTok, _arg, iTokLen )==0 )
+#define LOC_TEXT()            { if ( *sBuf!=';') LOC_ERR ( "';'", sBuf ); sTok = ++sBuf; while ( *sBuf && *sBuf!=';' ) sBuf++; iTokLen = sBuf-sTok; }
+
+	// handle ATTR-TYPE
+	LOC_SPACE0();
+	LOC_TOK();
+	if ( LOC_TOKEQ( "uint" ) )
+		tAttr.m_eAttrType = SPH_ATTR_UINT32SET;
+	else if ( LOC_TOKEQ( "timestamp" ) )
+		tAttr.m_eAttrType = SPH_ATTR_UINT32SET;
+	else if ( LOC_TOKEQ( "bigint" ) )
+		tAttr.m_eAttrType = SPH_ATTR_INT64SET;
+	else LOC_ERR ( "attr type ('uint' or 'timestamp' or 'bigint')", sTok );
+
+	// handle ATTR-NAME
+	LOC_SPACE1();
+	LOC_TOK ();
+	if ( iTokLen )
+		tAttr.m_sName.SetBinary ( sTok, iTokLen );
+	else LOC_ERR ( "attr name", sTok );
+
+	// handle 'from'
+	LOC_SPACE1();
+	LOC_TOK();
+	if ( !LOC_TOKEQ( "from" ) ) LOC_ERR ( "'from' keyword", sTok );
+
+	// handle SOURCE-TYPE
+	LOC_SPACE1();
+	LOC_TOK();
+	LOC_SPACE0();
+	if ( LOC_TOKEQ( "field" ) )
+		tAttr.m_eSrc = SPH_ATTRSRC_FIELD;
+	else if ( LOC_TOKEQ( "query" ) )
+		tAttr.m_eSrc = SPH_ATTRSRC_QUERY;
+	else if ( LOC_TOKEQ( "ranged-query" ) )
+		tAttr.m_eSrc = SPH_ATTRSRC_RANGEDQUERY;
+	else if ( LOC_TOKEQ( "ranged-main-query" ) )
+		tAttr.m_eSrc = SPH_ATTRSRC_RANGEDMAINQUERY;
+	else LOC_ERR ( "value source type ('field', or 'query', or 'ranged-query', or 'ranged-main-query')", sTok );
+
+	if ( tAttr.m_eSrc==SPH_ATTRSRC_FIELD )
+		return true;
+
+	// handle QUERY
+	LOC_TEXT();
+	if ( iTokLen )
+		tAttr.m_sQuery.SetBinary ( sTok, iTokLen );
+	else LOC_ERR ( "query", sTok );
+
+	if ( tAttr.m_eSrc==SPH_ATTRSRC_QUERY || tAttr.m_eSrc==SPH_ATTRSRC_RANGEDMAINQUERY )
+		return true;
+
+	// handle RANGE-QUERY
+	LOC_TEXT();
+	if ( iTokLen )
+		tAttr.m_sQueryRange.SetBinary ( sTok, iTokLen );
+	else LOC_ERR ( "range query", sTok );
+
+#undef LOC_ERR
+#undef LOC_SPACE0
+#undef LOC_SPACE1
+#undef LOC_TOK
+#undef LOC_TOKEQ
+#undef LOC_TEXT
+
+	return true;
+}
+
+
+#define LOC_CHECK( _hash, _key, _msg, _add ) \
+    if (!( _hash.Exists ( _key ) )) \
+    { \
+        fprintf ( stdout, "ERROR: key '%s' not found " _msg "\n", _key, _add ); \
+        return false; \
+    }
+
+// get string
+#define LOC_GETS( _arg, _key ) \
+    if ( hSource.Exists(_key) ) \
+        _arg = hSource[_key].strval();
+
+// get int
+#define LOC_GETI( _arg, _key ) \
+    if ( hSource.Exists(_key) && hSource[_key].intval() ) \
+        _arg = hSource[_key].intval();
+
+// get int64_t
+#define LOC_GETL( _arg, _key ) \
+    if ( hSource.Exists(_key) ) \
+        _arg = hSource[_key].int64val();
+
+// get bool
+#define LOC_GETB( _arg, _key ) \
+    if ( hSource.Exists(_key) ) \
+        _arg = ( hSource[_key].intval()!=0 );
+
+// get array of strings
+#define LOC_GETA( _arg, _key ) \
+    for ( CSphVariant * pVal = hSource(_key); pVal; pVal = pVal->m_pNext ) \
+        _arg.Add ( pVal->cstr() );
+
+void SqlAttrsConfigure ( CSphSourceParams_SQL &tParams, const CSphVariant * pHead, ESphAttr eAttrType
+						 , const char * sSourceName, bool bIndexedAttr = false )
+{
+	for ( const CSphVariant * pCur = pHead; pCur; pCur = pCur->m_pNext )
+	{
+		CSphColumnInfo tCol ( pCur->cstr (), eAttrType );
+		char * pColon = strchr ( const_cast<char *> ( tCol.m_sName.cstr () ), ':' );
+		if ( pColon )
+		{
+			*pColon = '\0';
+
+			if ( eAttrType==SPH_ATTR_INTEGER )
+			{
+				int iBits = strtol ( pColon + 1, NULL, 10 );
+				if ( iBits<=0 || iBits>ROWITEM_BITS )
+				{
+					fprintf ( stdout, "WARNING: source '%s': attribute '%s': invalid bitcount=%d (bitcount ignored)\n"
+							  , sSourceName, tCol.m_sName.cstr (), iBits );
+					iBits = -1;
+				}
+				tCol.m_tLocator.m_iBitCount = iBits;
+
+			} else
+			{
+				fprintf ( stdout, "WARNING: source '%s': attribute '%s': bitcount is only supported for integer types\n"
+						  , sSourceName, tCol.m_sName.cstr () );
+			}
+		}
+		tParams.m_dAttrs.Add ( tCol );
+		if ( bIndexedAttr )
+			tParams.m_dAttrs.Last ().m_bIndexed = true;
+	}
+}
+
+
+#if USE_ZLIB
+
+bool ConfigureUnpack ( CSphVariant * pHead, ESphUnpackFormat eFormat, CSphSourceParams_SQL &tParams, const char * )
+{
+	for ( CSphVariant * pVal = pHead; pVal; pVal = pVal->m_pNext )
+	{
+		CSphUnpackInfo &tUnpack = tParams.m_dUnpack.Add ();
+		tUnpack.m_sName = CSphString ( pVal->cstr () );
+		tUnpack.m_eFormat = eFormat;
+	}
+	return true;
+}
+
+#else
+
+bool ConfigureUnpack ( CSphVariant * pHead, ESphUnpackFormat, CSphSourceParams_SQL &, const char * sSourceName )
+{
+	if ( pHead )
+	{
+		fprintf ( stdout, "ERROR: source '%s': unpack is not supported, rebuild with zlib\n", sSourceName );
+		return false;
+	}
+	return true;
+}
+#endif // USE_ZLIB
+
+
+bool ParseJoinedField ( const char * sBuf, CSphJoinedField * pField, const char * sSourceName )
+{
+	// sanity checks
+	assert ( pField );
+	if ( !sBuf || !sBuf[0] )
+	{
+		fprintf ( stdout, "ERROR: source '%s': sql_joined_field must not be empty.\n", sSourceName );
+		return false;
+	}
+
+#define LOC_ERR( _exp ) \
+    { \
+        fprintf ( stdout, "ERROR: source '%s': expected " _exp " in sql_joined_field, got '%s'.\n", sSourceName, sBuf ); \
+        return false; \
+    }
+#define LOC_TEXT()            { if ( *sBuf!=';') LOC_ERR ( "';'" ); sTmp = ++sBuf; while ( *sBuf && *sBuf!=';' ) sBuf++; iTokLen = sBuf-sTmp; }
+
+	// parse field name
+	while ( isspace ( *sBuf ) )
+		sBuf++;
+
+	const char * sName = sBuf;
+	while ( sphIsAlpha ( *sBuf ) )
+		sBuf++;
+	if ( sBuf==sName ) LOC_ERR ( "field name" );
+	pField->m_sName.SetBinary ( sName, sBuf - sName );
+
+	if ( !isspace ( *sBuf ) ) LOC_ERR ( "space" );
+	while ( isspace ( *sBuf ) )
+		sBuf++;
+
+	// parse 'from'
+	if ( strncasecmp ( sBuf, "from", 4 ) ) LOC_ERR ( "'from'" );
+	sBuf += 4;
+
+	if ( !isspace ( *sBuf ) ) LOC_ERR ( "space" );
+	while ( isspace ( *sBuf ) )
+		sBuf++;
+
+	bool bGotRanged = false;
+	pField->m_bPayload = false;
+	pField->m_bRangedMain = false;
+
+	// parse 'query'
+	if ( strncasecmp ( sBuf, "payload-query", 13 )==0 )
+	{
+		pField->m_bPayload = true;
+		sBuf += 13;
+
+	} else if ( strncasecmp ( sBuf, "query", 5 )==0 )
+	{
+		sBuf += 5;
+
+	} else if ( strncasecmp ( sBuf, "ranged-query", 12 )==0 )
+	{
+		bGotRanged = true;
+		sBuf += 12;
+
+	} else if ( strncasecmp ( sBuf, "ranged-main-query", 17 )==0 )
+	{
+		pField->m_bRangedMain = true;
+		sBuf += 17;
+
+	} else LOC_ERR ( "'query'" );
+
+	// parse ';'
+	while ( isspace ( *sBuf ) && *sBuf!=';' )
+		sBuf++;
+
+	if ( *sBuf!=';' ) LOC_ERR ( "';'" );
+
+	// handle QUERY
+	const char * sTmp = sBuf;
+	int iTokLen = 0;
+	LOC_TEXT();
+	if ( iTokLen )
+		pField->m_sQuery.SetBinary ( sTmp, iTokLen );
+	else LOC_ERR ( "query" );
+
+	if ( !bGotRanged )
+		return true;
+
+	// handle RANGE-QUERY
+	LOC_TEXT();
+	if ( iTokLen )
+		pField->m_sRanged.SetBinary ( sTmp, iTokLen );
+	else LOC_ERR ( "range query" );
+
+#undef LOC_ERR
+#undef LOC_TEXT
+
+	return true;
+}
+
+
+bool SqlParamsConfigure ( CSphSourceParams_SQL &tParams, const CSphConfigSection &hSource, const char * sSourceName )
+{
+	if ( !hSource.Exists (
+		"odbc_dsn" ) ) // in case of odbc source, the host, user, pass and db are not mandatory, since they may be already defined in dsn string.
+	{
+		LOC_CHECK ( hSource, "sql_host", "in source '%s'", sSourceName );
+		LOC_CHECK ( hSource, "sql_user", "in source '%s'", sSourceName );
+		LOC_CHECK ( hSource, "sql_pass", "in source '%s'", sSourceName );
+		LOC_CHECK ( hSource, "sql_db", "in source '%s'", sSourceName );
+	}
+	LOC_CHECK ( hSource, "sql_query", "in source '%s'", sSourceName );
+
+	LOC_GETS ( tParams.m_sHost, "sql_host" );
+	LOC_GETS ( tParams.m_sUser, "sql_user" );
+	LOC_GETS ( tParams.m_sPass, "sql_pass" );
+	LOC_GETS ( tParams.m_sDB, "sql_db" );
+	LOC_GETI ( tParams.m_uPort, "sql_port" );
+
+	LOC_GETS ( tParams.m_sQuery, "sql_query" );
+	LOC_GETA ( tParams.m_dQueryPre, "sql_query_pre" );
+	LOC_GETA ( tParams.m_dQueryPost, "sql_query_post" );
+	LOC_GETS ( tParams.m_sQueryRange, "sql_query_range" );
+	LOC_GETA ( tParams.m_dQueryPostIndex, "sql_query_post_index" );
+	LOC_GETL ( tParams.m_iRangeStep, "sql_range_step" );
+	LOC_GETS ( tParams.m_sQueryKilllist, "sql_query_killlist" );
+	LOC_GETS ( tParams.m_sHookConnect, "hook_connect" );
+	LOC_GETS ( tParams.m_sHookQueryRange, "hook_query_range" );
+	LOC_GETS ( tParams.m_sHookPostIndex, "hook_post_index" );
+
+	LOC_GETI ( tParams.m_iRangedThrottle, "sql_ranged_throttle" );
+
+	SqlAttrsConfigure ( tParams, hSource ( "sql_attr_uint" ), SPH_ATTR_INTEGER, sSourceName );
+	SqlAttrsConfigure ( tParams, hSource ( "sql_attr_timestamp" ), SPH_ATTR_TIMESTAMP, sSourceName );
+	SqlAttrsConfigure ( tParams, hSource ( "sql_attr_bool" ), SPH_ATTR_BOOL, sSourceName );
+	SqlAttrsConfigure ( tParams, hSource ( "sql_attr_float" ), SPH_ATTR_FLOAT, sSourceName );
+	SqlAttrsConfigure ( tParams, hSource ( "sql_attr_bigint" ), SPH_ATTR_BIGINT, sSourceName );
+	SqlAttrsConfigure ( tParams, hSource ( "sql_attr_string" ), SPH_ATTR_STRING, sSourceName );
+	SqlAttrsConfigure ( tParams, hSource ( "sql_attr_json" ), SPH_ATTR_JSON, sSourceName );
+
+	SqlAttrsConfigure ( tParams, hSource ( "sql_field_string" ), SPH_ATTR_STRING, sSourceName, true );
+
+	LOC_GETA ( tParams.m_dFileFields, "sql_file_field" );
+
+	tParams.m_iMaxFileBufferSize = g_iMaxFileFieldBuffer;
+	tParams.m_iRefRangeStep = tParams.m_iRangeStep;
+	tParams.m_eOnFileFieldError = g_eOnFileFieldError;
+
+	// unpack
+	if ( !ConfigureUnpack ( hSource ( "unpack_zlib" ), SPH_UNPACK_ZLIB, tParams, sSourceName ) )
+		return false;
+
+	if ( !ConfigureUnpack ( hSource ( "unpack_mysqlcompress" ), SPH_UNPACK_MYSQL_COMPRESS, tParams, sSourceName ) )
+		return false;
+
+	tParams.m_uUnpackMemoryLimit = hSource.GetSize ( "unpack_mysqlcompress_maxsize", 16777216 );
+
+	// parse multi-attrs
+	for ( CSphVariant * pVal = hSource ( "sql_attr_multi" ); pVal; pVal = pVal->m_pNext )
+	{
+		CSphColumnInfo tAttr;
+		if ( !ParseMultiAttr ( pVal->cstr (), tAttr, sSourceName ) )
+			return false;
+		tParams.m_dAttrs.Add ( tAttr );
+	}
+
+	// parse joined fields
+	for ( CSphVariant * pVal = hSource ( "sql_joined_field" ); pVal; pVal = pVal->m_pNext )
+		if ( !ParseJoinedField ( pVal->cstr (), &tParams.m_dJoinedFields.Add (), sSourceName ) )
+			return false;
+
+	// make sure attr names are unique
+	ARRAY_FOREACH ( i, tParams.m_dAttrs )
+		for ( int j = i + 1; j<tParams.m_dAttrs.GetLength (); j++ )
+		{
+			const CSphString &sName = tParams.m_dAttrs[i].m_sName;
+			if ( sName==tParams.m_dAttrs[j].m_sName )
+			{
+				fprintf ( stdout, "ERROR: duplicate attribute name: %s\n", sName.cstr () );
+				return false;
+			}
+		}
+
+	// additional checks
+	if ( tParams.m_iRangedThrottle<0 )
+	{
+		fprintf ( stdout, "WARNING: sql_ranged_throttle must not be negative; throttling disabled\n" );
+		tParams.m_iRangedThrottle = 0;
+	}
+
+	// debug printer
+	if ( g_bPrintQueries )
+		tParams.m_bPrintQueries = true;
+
+	return true;
+}
+
+
+CSphSource_MySQL * SpawnSource ( const char * sSourceName, const CSphConfigType &hSources, ISphTokenizer * pTok, CSphDict * pDict )
+{
+	const CSphConfigSection &hSource = hSources[sSourceName];
+
+	assert ( hSource["type"]=="mysql" );
 
 	CSphSourceParams_MySQL tParams;
-	tParams.m_sHost = "localhost";
-	tParams.m_sUser = "root";
-	tParams.m_sDB = "lj";
-	tParams.m_dQueryPre.Add ( "SET NAMES utf8" );
-	tParams.m_sQuery = sQuery;
-
-	CSphColumnInfo tCol;
-	tCol.m_eAttrType = SPH_ATTR_INTEGER;
-	tCol.m_sName = "channel_id";
-	tParams.m_dAttrs.Add ( tCol );
-	tCol.m_eAttrType = SPH_ATTR_TIMESTAMP;
-	tCol.m_sName = "published";
-	tParams.m_dAttrs.Add ( tCol );
+	if ( !SqlParamsConfigure ( tParams, hSource, sSourceName ) )
+		return NULL;
+
+	LOC_GETS ( tParams.m_sUsock, "sql_sock" );
+	LOC_GETI ( tParams.m_iFlags, "mysql_connect_flags" );
+	LOC_GETS ( tParams.m_sSslKey, "mysql_ssl_key" );
+	LOC_GETS ( tParams.m_sSslCert, "mysql_ssl_cert" );
+	LOC_GETS ( tParams.m_sSslCA, "mysql_ssl_ca" );
+
+	auto * pSrc = new CSphSource_MySQL ( "sSourceName" );
+	pSrc->SetTokenizer ( pTok );
+	pSrc->SetDict ( pDict );
 
 	SetupIndexing ( pSrc, tParams );
 	return pSrc;
@@ -182,19 +581,22 @@ int main ( int argc, char ** argv )
 	sphThreadInit();
 	MemorizeStack ( &cTopOfMainStack );
 
+	CSphConfigParser cp;
+	CSphConfig &hConf = cp.m_tConf;
+	cp.Parse ("internal", rtestconfig);
+	const CSphConfigType &hSources = hConf["source"];
+
 	CSphString sError;
 	CSphDictSettings tDictSettings;
 	tDictSettings.m_bWordDict = false;
 
 	ISphTokenizerRefPtr_c pTok {sphCreateUTF8Tokenizer()};
 	CSphDictRefPtr_c pDict {sphCreateDictionaryCRC ( tDictSettings, NULL, pTok, "rt1", sError )};
-	CSphSource_MySQL * pSrc = SpawnSource ( "SELECT id, channel_id, UNIX_TIMESTAMP(published) published, "
-		"title, UNCOMPRESS(content) content FROM posting WHERE id<=10000 AND id%2=0", pTok, pDict );
+	CSphSource_MySQL * pSrc = SpawnSource ( "test1", hSources, pTok, pDict );
 
 	ISphTokenizerRefPtr_c pTok2 {sphCreateUTF8Tokenizer()};
 	CSphDictRefPtr_c pDict2 {sphCreateDictionaryCRC ( tDictSettings, NULL, pTok, "rt2", sError )};
-	CSphSource_MySQL * pSrc2 = SpawnSource ( "SELECT id, channel_id, UNIX_TIMESTAMP(published) published, "
-		"title, UNCOMPRESS(content) content FROM posting WHERE id<=10000 AND id%2=1", pTok2, pDict2 );
+	CSphSource_MySQL * pSrc2 = SpawnSource ( "test2", hSources, pTok2, pDict2 );
 
 	CSphSchema tSrcSchema;
 	if ( !pSrc->UpdateSchema ( &tSrcSchema, sError ) )
@@ -215,7 +617,7 @@ int main ( int argc, char ** argv )
 	SmallStringHash_T< CSphIndex * > dTemp;
 	BinlogFlushInfo_t tBinlogFlush;
 	sphReplayBinlog ( dTemp, 0, NULL, tBinlogFlush );
-	ISphRtIndex * pIndex = sphCreateIndexRT ( tSchema, "testrt", 32*1024*1024, "data/dump", false );
+	ISphRtIndex * pIndex = sphCreateIndexRT ( tSchema, "testrt", 32*1024*1024, DATAFLD "dump", false );
 	pIndex->SetTokenizer ( pTok ); // index will own this pair from now on
 	pIndex->SetDictionary ( pDict );
 	if ( !pIndex->Prealloc ( false ) )