merged rel21 branch (up to r3802) back into trunk

git-svn-id: svn://svn.sphinxsearch.com/sphinx/trunk@3803 406a0c4d-033a-0410-8de8-e80135713968

api/libsphinxclient/smoke_ref.txt

@@ -2,7 +2,7 @@ exact_phrase=0
 n=1, res=this is my <b>test</b> <b>text</b> to be highlighted ... 
 n=2, res=another <b>test</b> <b>text</b> to be highlighted, below limit
 n=3, res=<b>test</b> number three, without phrase match
-n=4, res=final <b>test</b>, not only without  ...  with swapped phrase <b>text</b> <b>test</b> as well
+n=4, res= ...  <b>test</b>, not only without ... with swapped phrase <b>text</b> <b>test</b> as well
 
 exact_phrase=1
 n=1, res=this is my <b>test text</b> to be highlighted ... 
@@ -14,7 +14,7 @@ passage_boundary=zone
 n=1, res= ...  manager <b>it</b>. <b>Is</b> Filing this report and. <b>It</b> <b>is</b> signed hereby represent. That <b>it</b> <b>is</b> all information.are <b>It</b> or <b>is</b> cool <b>It</b> <b>is</b> cooler <b>It</b> <b>is</b> another ... 
 
 passage_boundary=sentence
-n=1, res= ... The institutional investment manager <b>it</b>. ... <b>Is</b> Filing this report and. ... <b>It</b> <b>is</b> signed hereby represent. ... That <b>it</b> <b>is</b> all information.are <b>It</b> or <b>is</b> cool <b>It</b> <b>is</b> cooler <b>It</b> <b>is</b> another place! ... 
+n=1, res= ...  institutional investment manager <b>it</b> ... <b>Is</b> Filing this report  ... <b>It</b> <b>is</b> signed hereby  ...  <b>it</b> <b>is</b> all information.are <b>It</b> or <b>is</b> cool <b>It</b> <b>is</b> cooler <b>It</b> <b>is</b> another place! ... 
 
 build_keywords result:
 1. tokenized=hello, normalized=hello, docs=0, hits=0

api/sphinxapi.php

@@ -538,11 +538,10 @@ class SphinxClient
 		}
 				
 		$this->_host = $host;
-		if ( is_int($port) )
-			if ( $port )
-				$this->_port = $port;
+		$port = intval($port);
+		assert ( 0<=$port && $port<65536 );
+		$this->_port = ( $port==0 ) ? 9312 : $port;
 		$this->_path = '';
-
 	}
 
 	/// set server connection timeout (0 to remove)

src/indexer.cpp

@@ -602,7 +602,7 @@ bool SqlParamsConfigure ( CSphSourceParams_SQL & tParams, const CSphConfigSectio
 	SqlAttrsConfigure ( tParams,	hSource("sql_attr_str2wordcount"),	SPH_ATTR_WORDCOUNT,	sSourceName );
 
 	SqlAttrsConfigure ( tParams,	hSource("sql_field_string"),		SPH_ATTR_STRING,	sSourceName, true );
-	SqlAttrsConfigure ( tParams,	hSource("sql_field_str2wordcount"),	SPH_ATTR_STRING,	sSourceName, true );
+	SqlAttrsConfigure ( tParams,	hSource("sql_field_str2wordcount"),	SPH_ATTR_WORDCOUNT,	sSourceName, true );
 
 	LOC_GETA ( tParams.m_dFileFields,			"sql_file_field" );
 

src/indextool.cpp

@@ -157,7 +157,7 @@ void CharsetFold ( CSphIndex * pIndex, FILE * fp )
 			}
 
 			pOut += sphUTF8Encode ( pOut, iCode );
-			if ( pOut >= pOutMax )
+			if ( pOut>=pOutMax )
 			{
 				fwrite ( sBuf2.Begin(), 1, pOut-sBuf2.Begin(), stdout );
 				pOut = sBuf2.Begin();
@@ -857,7 +857,7 @@ int main ( int argc, char ** argv )
 		OPT1 ( "--dumpheader" )		{ eCommand = CMD_DUMPHEADER; sDumpHeader = argv[++i]; }
 		OPT1 ( "--dumpconfig" )		{ eCommand = CMD_DUMPCONFIG; sDumpHeader = argv[++i]; }
 		OPT1 ( "--dumpdocids" )		{ eCommand = CMD_DUMPDOCIDS; sIndex = argv[++i]; }
-		OPT1 ( "--check" )			{ eCommand = CMD_CHECK; sIndex = argv[++i]; }
+		OPT1 ( "--check" )			{ eCommand = CMD_CHECK; sIndex = argv[++i]; sphSetDebugCheck(); }
 		OPT1 ( "--htmlstrip" )		{ eCommand = CMD_STRIP; sIndex = argv[++i]; }
 		OPT1 ( "--build-infixes" )	{ eCommand = CMD_BUILDINFIXES; sIndex = argv[++i]; }
 		OPT1 ( "--build-skips" )	{ eCommand = CMD_BUILDSKIPS; sIndex = argv[++i]; }

src/llsphinxql.c

@@ -1763,7 +1763,7 @@ YY_RULE_SETUP
 case 112:
 YY_RULE_SETUP
 #line 148 "sphinxql.l"
-{ YYSTOREBOUNDS; lvalp->m_sValue = yytext; return TOK_ID; }
+{ YYSTOREBOUNDS; lvalp->m_iStart++; lvalp->m_iEnd--; lvalp->m_sValue.SetBinary ( yytext+1, strlen(yytext)-2 ); return TOK_ID; }
 	YY_BREAK
 case 113:
 YY_RULE_SETUP

src/searchd.cpp

@@ -18705,7 +18705,35 @@ bool CheckConfigChanges ()
 		memset ( &tStat, 0, sizeof ( tStat ) );
 
 	DWORD uCRC32 = 0;
+
+#if !USE_WINDOWS
+	char sBuf [ 8192 ];
+	FILE * fp = NULL;
+
+	fp = fopen ( g_sConfigFile.cstr (), "rb" );
+	if ( !fp )
+		return true;
+	fgets ( sBuf, sizeof(sBuf), fp );
+	fclose ( fp );
+
+	char * p = sBuf;
+	while ( isspace(*p) )
+		p++;
+	if ( p[0]=='#' && p[1]=='!' )
+	{
+		p += 2;
+
+		CSphVector<char> dContent;
+		char sError [ 1024 ];
+		if ( !TryToExec ( p, p+strlen(p), g_sConfigFile.cstr(), dContent, sError, sizeof(sError) ) )
+			return true;
+
+		uCRC32 = sphCRC32 ( (const BYTE*)dContent.Begin(), dContent.GetLength() );
+	} else
+		sphCalcFileCRC32 ( g_sConfigFile.cstr (), uCRC32 );
+#else
 	sphCalcFileCRC32 ( g_sConfigFile.cstr (), uCRC32 );
+#endif
 
 	if ( g_uCfgCRC32==uCRC32 && tStat.st_size==g_tCfgStat.st_size
 		&& tStat.st_mtime==g_tCfgStat.st_mtime && tStat.st_ctime==g_tCfgStat.st_ctime )

src/sphinx.cpp

@@ -180,6 +180,7 @@ static const int	MIN_READ_UNHINTED		= 1024;
 #define READ_NO_SIZE_HINT 0
 
 static bool			g_bSphQuiet					= false;
+static bool			g_bDebugCheck				= false;
 
 static int			g_iReadBuffer			= DEFAULT_READ_BUFFER;
 static int			g_iReadUnhinted			= DEFAULT_READ_UNHINTED;
@@ -8227,6 +8228,7 @@ CSphIndex::CSphIndex ( const char * sIndexName, const char * sFilename )
 	, m_fWriteFactor ( 0.0f )
 	, m_bKeepFilesOpen ( false )
 	, m_bPreloadWordlist ( true )
+	, m_bBinlog ( true )
 	, m_bStripperInited ( true )
 	, m_bEnableStar ( false )
 	, m_bId32to64 ( false )
@@ -8430,7 +8432,7 @@ int CSphIndex_VLN::UpdateAttributes ( const CSphAttrUpdate & tUpd, int iIndex, C
 	if ( !m_iDocinfo || !uRows )
 		return 0;
 
-	if ( g_pBinlog )
+	if ( m_bBinlog && g_pBinlog )
 		g_pBinlog->BinlogUpdateAttributes ( &m_iTID, m_sIndexName.cstr(), tUpd );
 
 	// remap update schema to index schema
@@ -9045,7 +9047,7 @@ bool CSphIndex_VLN::SaveAttributes ( CSphString & sError ) const
 	if ( !JuggleFile ( "spa", sError ) )
 		return false;
 
-	if ( g_pBinlog )
+	if ( m_bBinlog && g_pBinlog )
 		g_pBinlog->NotifyIndexFlush ( m_sIndexName.cstr(), m_iTID, false );
 
 	if ( *m_pAttrsStatus==uAttrStatus )
@@ -15833,7 +15835,7 @@ bool CSphIndex_VLN::Preread ()
 	}
 
 	// build attributes hash
-	if ( m_pDocinfo.GetLength() && m_pDocinfoHash.GetLength() )
+	if ( m_pDocinfo.GetLength() && m_pDocinfoHash.GetLength() && !g_bDebugCheck )
 	{
 		sphLogDebug ( "Hashing docinfo" );
 		assert ( CheckDocsCount ( m_iDocinfo, m_sLastError ) );
@@ -16527,6 +16529,14 @@ static XQNode_t * ExpandKeyword ( XQNode_t * pNode, const CSphIndexSettings & tS
 		pInfix->m_dWords[0].m_sWord.SetSprintf ( "*%s*", pNode->m_dWords[0].m_sWord.cstr() );
 		pInfix->m_pParent = pExpand;
 		pExpand->m_dChildren.Add ( pInfix );
+	} else if ( tSettings.m_iMinPrefixLen>0 )
+	{
+		assert ( pNode->m_dChildren.GetLength()==0 );
+		assert ( pNode->m_dWords.GetLength()==1 );
+		XQNode_t * pPrefix = CloneKeyword ( pNode );
+		pPrefix->m_dWords[0].m_sWord.SetSprintf ( "%s*", pNode->m_dWords[0].m_sWord.cstr() );
+		pPrefix->m_pParent = pExpand;
+		pExpand->m_dChildren.Add ( pPrefix );
 	}
 
 	if ( tSettings.m_bIndexExactWords )
@@ -16545,7 +16555,7 @@ static XQNode_t * ExpandKeyword ( XQNode_t * pNode, const CSphIndexSettings & tS
 XQNode_t * sphQueryExpandKeywords ( XQNode_t * pNode, const CSphIndexSettings & tSettings, bool bStarEnabled )
 {
 	// only if expansion makes sense at all
-	if ( tSettings.m_iMinInfixLen<=0 && !tSettings.m_bIndexExactWords )
+	if ( tSettings.m_iMinInfixLen<=0 && tSettings.m_iMinPrefixLen<=0 && !tSettings.m_bIndexExactWords )
 		return pNode;
 
 	// process children for composite nodes
@@ -19907,7 +19917,7 @@ void CSphDictCRCTraits::AddWordform ( CSphWordforms * pContainer, char * sBuffer
 	if ( !pTokenizer->TokenIsBlended () && pTokenizer->GetToken () )
 	{
 		sphWarning ( "invalid mapping (must be exactly 1 destination keyword) ( wordforms='%s' ). Fix your wordforms file '%s'.",
-					 sBuffer, szFile );
+					sBuffer, szFile );
 		return;
 	}
 
@@ -28340,6 +28350,12 @@ void sphSetQuiet ( bool bQuiet )
 }
 
 
+void sphSetDebugCheck ()
+{
+	g_bDebugCheck = true;
+}
+
+
 void sphSetJsonOptions ( bool bStrict, bool bAutoconvNumbers, bool bKeynamesToLowercase )
 {
 	g_bJsonStrict = bStrict;

src/sphinx.h

@@ -633,7 +633,7 @@ public:
 	virtual uint64_t				GetSettingsFNV () const;
 
 	/// get (readonly) lowercaser
-	const CSphLowercaser &			GetLowercaser() const  { return m_tLC; }
+	const CSphLowercaser &			GetLowercaser() const { return m_tLC; }
 
 protected:
 	virtual bool					RemapCharacters ( const char * sConfig, DWORD uFlags, const char * sSource, bool bCanRemap, CSphString & sError );
@@ -2992,6 +2992,7 @@ public:
 	virtual int					GetKillListSize () const = 0;
 	virtual bool				HasDocid ( SphDocID_t uDocid ) const = 0;
 	virtual bool				IsRT() const { return false; }
+	void						SetBinlog ( bool bBinlog ) { m_bBinlog = bBinlog; }
 	virtual int64_t *			GetFieldLens() const { return NULL; }
 
 	virtual bool				IsStarDict() const { return m_bEnableStar; } // disk index overrides this to support super-old legacy formats
@@ -3115,6 +3116,7 @@ protected:
 
 	bool						m_bKeepFilesOpen;		///< keep files open to avoid race on seamless rotation
 	bool						m_bPreloadWordlist;		///< preload wordlists or keep them on disk
+	bool						m_bBinlog;
 
 	bool						m_bStripperInited;		///< was stripper initialized (old index version (<9) handling)
 	bool						m_bEnableStar;			///< enable star-syntax

src/sphinxexcerpt.cpp

@@ -1506,7 +1506,11 @@ bool ExcerptGen_c::HighlightBestPassages ( const ExcerptQuery_t & tQuery )
 		{
 			// there might be just enough space to partially display this passage
 			if ( ( iTotalCodes + iKeywordsLength )<=tQuery.m_iLimit )
+			{
 				dShow.Add ( tBest );
+				iTotalWords += tBest.m_iWords;
+				iTotalCodes += tBest.m_iCodes;
+			}
 			break;
 		}
 
@@ -1973,7 +1977,7 @@ void SnippetsDocIndex_c::ParseQuery ( const char * sQuery, ISphTokenizer * pToke
 				continue;
 
 			m_dAllKeywords.Add ( (const char*)sWord );
-			if ( sWord[0]=='*' || sWord [ strlen((const char*)sWord)-1 ]=='*' )
+			if ( sWord[0]=='*' || sWord [ strlen ( (const char*)sWord)-1 ]=='*' )
 				AddWordStar ( (const char *)sWord );
 			else
 				AddWord ( uWordID, pTokenizer->GetLastTokenLen() );

src/sphinxexpr.cpp

@@ -1088,7 +1088,8 @@ enum Func_e
 	FUNC_PACKEDFACTORS,
 	FUNC_BM25F,
 	FUNC_INTEGER,
-	FUNC_DOUBLE
+	FUNC_DOUBLE,
+	FUNC_LENGTH
 };
 
 
@@ -1150,7 +1151,8 @@ static FuncDesc_t g_dFuncs[] =
 	{ "packedfactors",	0,	FUNC_PACKEDFACTORS, SPH_ATTR_FACTORS },
 	{ "bm25f",			-1,	FUNC_BM25F,			SPH_ATTR_FLOAT },
 	{ "integer",		1,	FUNC_INTEGER,		SPH_ATTR_INTEGER },
-	{ "double",			1,	FUNC_DOUBLE,		SPH_ATTR_FLOAT }
+	{ "double",			1,	FUNC_DOUBLE,		SPH_ATTR_FLOAT },
+	{ "length",			1,	FUNC_LENGTH,		SPH_ATTR_INTEGER }
 };
 
 
@@ -1217,6 +1219,7 @@ static int FuncHashLookup ( const char * sKey )
 		79, 79, 79, 79, 79, 79, 79, 79, 79, 79,
 		79, 79, 79, 79, 79, 79, 79, 79, 79, 79,
 		79, 79, 79, 79, 79, 79
+
 	};
 
 	const BYTE * s = (const BYTE*) sKey;
@@ -1232,7 +1235,7 @@ static int FuncHashLookup ( const char * sKey )
 
 	static int dIndexes[] =
 	{
-		-1, -1, 6, -1, 17, -1, -1, 28, 20, 18,
+		-1, -1, 6, -1, 17, -1, 42, 28, 20, 18,
 		16, 37, 19, -1, 7, 8, -1, 30, -1, 33,
 		-1, -1, 40, 27, 2, -1, 32, 24, 34, 26,
 		3, -1, 35, 4, 12, 13, -1, -1, 15, 25,
@@ -1461,6 +1464,7 @@ private:
 	ISphExpr *				CreateTree ( int iNode );
 	ISphExpr *				CreateIntervalNode ( int iArgsNode, CSphVector<ISphExpr *> & dArgs );
 	ISphExpr *				CreateInNode ( int iNode );
+	ISphExpr *				CreateLengthNode ( const ExprNode_t & tNode );
 	ISphExpr *				CreateGeodistNode ( int iArgs );
 	ISphExpr *				CreateBitdotNode ( int iArgsNode, CSphVector<ISphExpr *> & dArgs );
 	ISphExpr *				CreateUdfNode ( int iCall, ISphExpr * pLeft );
@@ -2956,6 +2960,7 @@ ISphExpr * ExprParser_t::CreateTree ( int iNode )
 
 					case FUNC_INTERVAL:	return CreateIntervalNode ( tNode.m_iLeft, dArgs );
 					case FUNC_IN:		return CreateInNode ( iNode );
+					case FUNC_LENGTH:	return CreateLengthNode ( tNode );
 					case FUNC_BITDOT:	return CreateBitdotNode ( tNode.m_iLeft, dArgs );
 
 					case FUNC_GEODIST:	return CreateGeodistNode ( tNode.m_iLeft );
@@ -3350,6 +3355,42 @@ int Expr_MVAIn_c<true>::MvaEval ( const DWORD * pMva ) const
 	return 0;
 }
 
+/// LENGTH() evaluator for MVAs
+class Expr_MVALength_c : public ISphExpr
+{
+protected:
+	CSphAttrLocator		m_tLocator;
+	int					m_iLocator;
+	const DWORD *		m_pMvaPool;
+
+public:
+	Expr_MVALength_c ( const CSphAttrLocator & tLoc, int iLocator )
+		: m_tLocator ( tLoc )
+		, m_iLocator ( iLocator )
+		, m_pMvaPool ( NULL )
+	{
+		assert ( tLoc.m_iBitOffset>=0 && tLoc.m_iBitCount>0 );
+	}
+
+	virtual int IntEval ( const CSphMatch & tMatch ) const
+	{
+		const DWORD * pMva = tMatch.GetAttrMVA ( m_tLocator, m_pMvaPool );
+		if ( !pMva )
+			return 0;
+		return (int)*pMva;
+	}
+
+	virtual void Command ( ESphExprCommand eCmd, void * pArg )
+	{
+		if ( eCmd==SPH_EXPR_SET_MVA_POOL )
+			m_pMvaPool = (const DWORD*)pArg;
+		if ( eCmd==SPH_EXPR_GET_DEPENDENT_COLS )
+			static_cast < CSphVector<int>* > ( pArg )->Add ( m_iLocator );
+	}
+
+	virtual float	Eval ( const CSphMatch & tMatch ) const { return (float)IntEval ( tMatch ); }
+};
+
 
 //////////////////////////////////////////////////////////////////////////
 
@@ -3692,6 +3733,22 @@ ISphExpr * ExprParser_t::CreateInNode ( int iNode )
 }
 
 
+ISphExpr * ExprParser_t::CreateLengthNode ( const ExprNode_t & tNode )
+{
+	const ExprNode_t & tLeft = m_dNodes [ tNode.m_iLeft ];
+
+	switch ( tLeft.m_iToken )
+	{
+		case TOK_ATTR_MVA32:
+		case TOK_ATTR_MVA64:
+			return new Expr_MVALength_c ( tLeft.m_tLocator, tLeft.m_iLocator );
+		default:
+			m_sCreateError = "LENGTH() argument must be MVA";
+			return NULL;
+	}
+}
+
+
 ISphExpr * ExprParser_t::CreateGeodistNode ( int iArgs )
 {
 	CSphVector<int> dArgs;
@@ -4016,7 +4073,7 @@ int ExprParser_t::AddNodeFunc ( int iFunc, int iLeft, int iRight )
 		m_sParserError.SetSprintf ( "%s() arguments can not be string", g_dFuncs[iFunc].m_sName );
 		return -1;
 	}
-	if ( bGotMva && !(eFunc==FUNC_IN || eFunc==FUNC_TO_STRING ) )
+	if ( bGotMva && !(eFunc==FUNC_IN || eFunc==FUNC_TO_STRING || eFunc==FUNC_LENGTH ) )
 	{
 		m_sParserError.SetSprintf ( "%s() arguments can not be MVA", g_dFuncs[iFunc].m_sName );
 		return -1;

src/sphinxint.h

@@ -1480,6 +1480,7 @@ CSphString		sphReconstructNode ( const XQNode_t * pNode, const CSphSchema * pSch
 
 void			sphSetUnlinkOld ( bool bUnlink );
 void			sphUnlinkIndex ( const char * sName, bool bForce );
+void			sphSetDebugCheck ();
 
 void			WriteSchema ( CSphWriter & fdInfo, const CSphSchema & tSchema );
 void			ReadSchema ( CSphReader & rdInfo, CSphSchema & m_tSchema, DWORD uVersion, bool bDynamic );

src/sphinxql.l

@@ -145,7 +145,7 @@ EXPONENT			[eE](\+|-)?{DIGIT}+
 [a-zA-Z_0-9]+		{ YYSTOREBOUNDS; return TOK_BAD_NUMERIC; }
 @{ID}				{ YYSTOREBOUNDS; lvalp->m_sValue = yytext; return TOK_USERVAR; }
 @@{ID}				{ YYSTOREBOUNDS; lvalp->m_sValue = yytext; return TOK_SYSVAR; }
-"`ID`"				{ YYSTOREBOUNDS; lvalp->m_sValue = yytext; return TOK_ID; }
+"`ID`"				{ YYSTOREBOUNDS; lvalp->m_iStart++; lvalp->m_iEnd--; lvalp->m_sValue.SetBinary ( yytext+1, strlen(yytext)-2 ); return TOK_ID; }
 `{ID}`				{ YYSTOREBOUNDS; lvalp->m_iStart++; lvalp->m_iEnd--; lvalp->m_sValue.SetBinary ( yytext+1, strlen(yytext)-2 ); return TOK_IDENT; }
 
 {SPACE}+			{ ; }

src/sphinxql.y

@@ -403,6 +403,16 @@ where_item:
 			if ( !pParser->AddFloatRangeFilter ( $1.m_sValue, $3.m_fValue, $5.m_fValue, true ) )
 				YYERROR;
 		}
+	| expr_ident TOK_BETWEEN const_int TOK_AND const_float
+		{
+			if ( !pParser->AddFloatRangeFilter ( $1.m_sValue, $3.m_iValue, $5.m_fValue, true ) )
+				YYERROR;
+		}
+	| expr_ident TOK_BETWEEN const_float TOK_AND const_int
+		{
+			if ( !pParser->AddFloatRangeFilter ( $1.m_sValue, $3.m_fValue, $5.m_iValue, true ) )
+				YYERROR;
+		}
 	| expr_ident '>' const_float
 		{
 			if ( !pParser->AddFloatRangeFilter ( $1.m_sValue, $3.m_fValue, FLT_MAX, false ) )

src/sphinxquery.h

@@ -20,14 +20,6 @@
 
 //////////////////////////////////////////////////////////////////////////////
 
-enum XQStarPosition
-{
-	STAR_NONE	= 0,
-	STAR_FRONT	= 1,
-	STAR_BACK	= 2,
-	STAR_BOTH	= 3
-};
-
 /// extended query word with attached position within atom
 struct XQKeyword_t
 {

src/sphinxrt.cpp

@@ -3787,7 +3787,7 @@ CSphIndex * RtIndex_t::LoadDiskChunk ( const char * sChunk, CSphString & sError
 	MEMORY ( SPH_MEM_IDX_DISK );
 
 	// !COMMIT handle errors gracefully instead of dying
-	CSphIndex * pDiskChunk = sphCreateIndexPhrase ( m_sIndexName.cstr(), sChunk );
+	CSphIndex * pDiskChunk = sphCreateIndexPhrase ( sChunk, sChunk );
 	if ( !pDiskChunk )
 	{
 		sError.SetSprintf ( "disk chunk %s: alloc failed", sChunk );
@@ -3796,6 +3796,7 @@ CSphIndex * RtIndex_t::LoadDiskChunk ( const char * sChunk, CSphString & sError
 
 	pDiskChunk->SetWordlistPreload ( m_bPreloadWordlist );
 	pDiskChunk->m_iExpansionLimit = m_iExpansionLimit;
+	pDiskChunk->SetBinlog ( false );
 	pDiskChunk->SetEnableStar ( m_bEnableStar );
 
 	CSphString sWarning;
@@ -7051,6 +7052,10 @@ bool RtIndex_t::AttachDiskIndex ( CSphIndex * pIndex, CSphString & sError )
 	m_pTokenizer = pIndex->GetTokenizer()->Clone ( SPH_CLONE_INDEX );
 	m_pDict = pIndex->GetDictionary()->Clone ();
 	PostSetup();
+	CSphString sName;
+	sName.SetSprintf ( "%s_%d", m_sIndexName.cstr(), m_pDiskChunks.GetLength() );
+	pIndex->SetName ( sName.cstr() );
+	pIndex->SetBinlog ( false );
 
 	// FIXME? what about copying m_TID etc?
 

src/sphinxutils.cpp

@@ -613,13 +613,13 @@ bool CSphConfigParser::ValidateKey ( const char * sKey )
 
 #if !USE_WINDOWS
 
-bool CSphConfigParser::TryToExec ( char * pBuffer, char * pEnd, const char * szFilename, CSphVector<char> & dResult )
+bool TryToExec ( char * pBuffer, char * pEnd, const char * szFilename, CSphVector<char> & dResult, char * sError, int iErrorLen )
 {
 	int dPipe[2] = { -1, -1 };
 
 	if ( pipe ( dPipe ) )
 	{
-		snprintf ( m_sError, sizeof ( m_sError ), "pipe() failed (error=%s)", strerror(errno) );
+		snprintf ( sError, iErrorLen, "pipe() failed (error=%s)", strerror(errno) );
 		return false;
 	}
 
@@ -660,7 +660,7 @@ bool CSphConfigParser::TryToExec ( char * pBuffer, char * pEnd, const char * szF
 	} else
 		if ( iChild==-1 )
 		{
-			snprintf ( m_sError, sizeof ( m_sError ), "fork failed: [%d] %s", errno, strerror(errno) );
+			snprintf ( sError, iErrorLen, "fork failed: [%d] %s", errno, strerror(errno) );
 			return false;
 		}
 
@@ -701,7 +701,7 @@ bool CSphConfigParser::TryToExec ( char * pBuffer, char * pEnd, const char * szF
 
 		if ( iResult==-1 && errno!=EINTR )
 		{
-			snprintf ( m_sError, sizeof ( m_sError ), "waitpid() failed: [%d] %s", errno, strerror(errno) );
+			snprintf ( sError, iErrorLen, "waitpid() failed: [%d] %s", errno, strerror(errno) );
 			return false;
 		}
 	}
@@ -710,19 +710,19 @@ bool CSphConfigParser::TryToExec ( char * pBuffer, char * pEnd, const char * szF
 	if ( WIFEXITED ( iStatus ) && WEXITSTATUS ( iStatus ) )
 	{
 		// FIXME? read stderr and log that too
-		snprintf ( m_sError, sizeof ( m_sError ), "error executing '%s' status = %d", pBuffer, WEXITSTATUS ( iStatus ) );
+		snprintf ( sError, iErrorLen, "error executing '%s' status = %d", pBuffer, WEXITSTATUS ( iStatus ) );
 		return false;
 	}
 
 	if ( WIFSIGNALED ( iStatus ) )
 	{
-		snprintf ( m_sError, sizeof ( m_sError ), "error executing '%s', killed by signal %d", pBuffer, WTERMSIG ( iStatus ) );
+		snprintf ( sError, iErrorLen, "error executing '%s', killed by signal %d", pBuffer, WTERMSIG ( iStatus ) );
 		return false;
 	}
 
 	if ( iBytesRead < 0 )
 	{
-		snprintf ( m_sError, sizeof ( m_sError ), "pipe read error: [%d] %s", errno, strerror(errno) );
+		snprintf ( sError, iErrorLen, "pipe read error: [%d] %s", errno, strerror(errno) );
 		return false;
 	}
 
@@ -731,6 +731,11 @@ bool CSphConfigParser::TryToExec ( char * pBuffer, char * pEnd, const char * szF
 
 	return true;
 }
+
+bool CSphConfigParser::TryToExec ( char * pBuffer, char * pEnd, const char * szFilename, CSphVector<char> & dResult )
+{
+	return ::TryToExec ( pBuffer, pEnd, szFilename, dResult, m_sError, sizeof(m_sError) );
+}
 #endif
 
 

src/sphinxutils.h

@@ -132,6 +132,10 @@ protected:
 	char *			GetBufferString ( char * szDest, int iMax, const char * & szSource );
 };
 
+#if !USE_WINDOWS
+bool TryToExec ( char * pBuffer, char * pEnd, const char * szFilename, CSphVector<char> & dResult, char * sError, int iErrorLen );
+#endif
+
 /////////////////////////////////////////////////////////////////////////////
 
 enum

test/test_050/test.xml

@@ -29,6 +29,23 @@ index test
 	source	= test
 	path	= <data_path/>/test
 }
+
+source test_flt
+{
+	type = mysql
+	<sql_settings/>
+
+	sql_query = SELECT id, n/1000000 as f, text FROM test_table;
+
+	sql_attr_float	= f
+}
+
+index flt
+{
+	source	= test_flt
+	path	= <data_path/>/flt
+}
+
 </config>
 
 <db_create>
@@ -71,32 +88,32 @@ INSERT INTO test_table (id, a, n) VALUES
 </query_attributes>
 
 <queries>
-	<query />
+	<query  index="test"/>
 
-	<query sortmode="expr" sortby="-n"/>
+	<query sortmode="expr" sortby="-n" index="test"/>
 
-	<query select="a, n, n + a as sel"/> <!-- doc_id=10 will warp-around -->
+	<query select="a, n, n + a as sel" index="test"/> <!-- doc_id=10 will warp-around -->
 
-	<query groupfunc="attr" groupattr="n"/>
-	<query select="abs(n) as sel, *" groupfunc="attr" groupattr="sel"/>
+	<query groupfunc="attr" groupattr="n" index="test"/>
+	<query select="abs(n) as sel, *" groupfunc="attr" groupattr="sel" index="test"/>
 
 	<query select="idiv(abs(n), 10000000000) as sel, *"
 		groupfunc="attr" groupattr="sel"
-		id_range="1 6"/>
+		id_range="1 6" index="test"/>
 
 	<query select="idiv(70000000000, 10000000000) as sel, *"
 		groupfunc="attr" groupattr="sel"
-		id_range="1 6"/>
+		id_range="1 6" index="test"/>
 
-	<query select="min(a, n) as sel, *"/>
+	<query select="min(a, n) as sel, *" index="test"/>
 
-	<query filter="n" filter_value="-1" />
-	<query filter="n" filter_value="-4611686018427387903" />
-	<query filter="n" filter_value="4611686018427387903" />
+	<query filter="n" filter_value="-1" index="test" />
+	<query filter="n" filter_value="-4611686018427387903" index="test" />
+	<query filter="n" filter_value="4611686018427387903" index="test" />
 
-	<query filter="n" filter_range="-100000000000 0"/>
-	<query filter="n" filter_range="0 100000000000"/>
-	<query filter="n" filter_range="-4611686018427387903 -4611686018427387900" />
+	<query filter="n" filter_range="-100000000000 0" index="test"/>
+	<query filter="n" filter_range="0 100000000000" index="test"/>
+	<query filter="n" filter_range="-4611686018427387903 -4611686018427387900" index="test" />
 </queries>
 
 <!-- Regression for issue #1062 -->
@@ -114,5 +131,8 @@ INSERT INTO test_table (id, a, n) VALUES
 <sphinxql>select * from test where n between -100000000000 and 0</sphinxql>
 <sphinxql>select * from test where n between 0 and 100000000000</sphinxql>
 <sphinxql>select * from test where n between -4611686018427387903 and -4611686018427387900</sphinxql>
+<!-- regression BETWEEN int with float mixed -->
+<sphinxql>select * from flt where f between -1.1 and 70001</sphinxql>
+<sphinxql>select * from flt where f between 1 and 70000.1</sphinxql>
 </sphqueries>
 </test>

test/test_089/test.xml

@@ -28,10 +28,11 @@ source src_test2
 {
 	type			= mysql
 	<sql_settings/>
-	sql_query = SELECT id, text, text as ord, text as wc, 1 as gid FROM table2
+	sql_query = SELECT id, text, text as ord, text as wc, text as wc2, 1 as gid FROM table2
 	sql_attr_uint = gid
 	sql_attr_str2ordinal = ord	
 	sql_attr_str2wordcount = wc	
+	sql_field_str2wordcount = wc2
 }
 
 index test2

test/test_126/test.xml

@@ -33,6 +33,23 @@ index test
 	expand_keywords	= 1
 }
 
+source src_prefix
+{
+	type			= mysql
+	<sql_settings/>
+
+	sql_query		= SELECT 1, 'force all' UNION SELECT 2, 'forcing me' UNION SELECT 3, 'foam stuff'
+}
+index prefix
+{
+	source			= src_prefix
+	path			= <data_path/>/prefix
+	charset_type 	= utf-8
+	expand_keywords	= 1
+	min_prefix_len = 2
+	enable_star = 1
+}
+
 index expanded_plain
 {
 	source			= srctest
@@ -104,6 +121,14 @@ index plain_inline
 <query mode="phrase" index="test">dog run</query>
 <query mode="phrase" index="test">=dog =run</query>
 <query mode="phrase" index="test">dog* run</query>
+<!-- regression expand_keywords vs prefix -->
+<query mode="extended2" index="prefix">for</query>
+<query mode="extended2" index="prefix">fo</query>
+<query mode="extended2" index="test">dog run</query>
+<query mode="extended2" index="test">=dog =run</query>
+<query mode="phrase" index="test">dog run</query>
+<query mode="phrase" index="test">=dog =run</query>
+<query mode="phrase" index="test">dog* run</query>
 <!-- regression wrong word expansion -->
 <query mode="extended2" index="expanded_plain">run</query>
 <!-- regression crash of daemon with empty dict=keywords and infixes enabled -->
@@ -146,4 +171,4 @@ INSERT INTO `test_table` VALUES
 ( 4,  'dog run' )
 </db_insert>
 
-</test>
+</test>

test/test_134/model.bin

@@ -14,35 +14,35 @@ That <b>it</b> <b>is</b> all information.
 <--->
 That <b>it is</b> all information.
 <--->
-";}i:2;a:1:{i:0;s:367:"
+";}i:2;a:1:{i:0;s:358:"
 <--->
-<zone_unidentified_table>The institutional investment manager <b>it</b>. <b>Is</b> Filing this report and. 
+<zone_unidentified_table> institutional investment manager <b>it</b>. <b>Is</b> Filing this report and. 
 <--->
-<zone_test><b>It</b> <b>is</b> signed hereby represent. That <b>it</b> <b>is</b> all information. are <b>It</b> or <b>is</b> 
+<zone_test><b>It</b> <b>is</b> signed hereby represent. That <b>it</b> <b>is</b> all information. are <b>It</b> or <b>is</b>
 <--->
-<zone_2>cool <b>It</b> <b>is</b> cooler 
+<zone_2> <b>It</b> <b>is</b> cooler 
 <--->
-<zone_unidentified_table><b>It</b> <b>is</b> another place! 
+<zone_unidentified_table><b>It</b> <b>is</b> another place!
 <--->
-";}i:3;a:1:{i:0;s:318:"
+";}i:3;a:1:{i:0;s:309:"
 <--->
-<zone_unidentified_table>The institutional investment manager <b>it. Is</b> Filing this report and. 
+<zone_unidentified_table> institutional investment manager <b>it. Is</b> Filing this report and. 
 <--->
-<zone_test><b>It is</b> signed hereby represent. That <b>it is</b> all information. are It or is 
+<zone_test><b>It is</b> signed hereby represent. That <b>it is</b> all information. are It or is
 <--->
-<zone_2>cool <b>It is</b> cooler 
+<zone_2> <b>It is</b> cooler 
 <--->
-<zone_unidentified_table><b>It is</b> another place! 
+<zone_unidentified_table><b>It is</b> another place!
 <--->
-";}i:4;a:1:{i:0;s:298:"
+";}i:4;a:1:{i:0;s:289:"
 <--->
-The institutional investment manager <b>it</b>. <b>Is</b> Filing this report and. 
+ institutional investment manager <b>it</b>. <b>Is</b> Filing this report and. 
 <--->
-<b>It</b> <b>is</b> signed hereby represent. That <b>it</b> <b>is</b> all information. are <b>It</b> or <b>is</b> 
+<b>It</b> <b>is</b> signed hereby represent. That <b>it</b> <b>is</b> all information. are <b>It</b> or <b>is</b>
 <--->
-cool <b>It</b> <b>is</b> cooler 
+ <b>It</b> <b>is</b> cooler 
 <--->
-<b>It</b> <b>is</b> another place! 
+<b>It</b> <b>is</b> another place!
 <--->
 ";}i:5;a:1:{i:0;s:193:"The institutional investment manager <b>it</b>.
 <--->
@@ -54,17 +54,17 @@ That <b>it</b> <b>is</b> all information.
 <--->
 ";}i:6;a:1:{i:0;s:156:" The institutional investment manager it. Is Filing this report and. It is signed hereby represent. That it is all information. are It or is cool It 
 <--->
-";}i:7;a:1:{i:0;s:113:"
+";}i:7;a:1:{i:0;s:112:"
 <--->
 The manager <b>it</b>. <b>Is</b> Filing this report and.
 <--->
-<b>Is</b> a signed hereby represent.
+<b>Is</b> a signed hereby represent
 <--->
-";}i:8;a:1:{i:0;s:113:"
+";}i:8;a:1:{i:0;s:112:"
 <--->
 The manager <b>it</b>. <b>Is</b> Filing this report and.
 <--->
-<b>Is</b> a signed hereby represent.
+<b>Is</b> a signed hereby represent
 <--->
 ";}i:9;a:1:{i:0;s:119:"
 <--->
@@ -82,10 +82,10 @@ The manager <b>it</b>.
 <--->
 <b>Is</b> a signed hereby represent.
 <--->
-";}i:11;a:1:{i:0;s:113:"
+";}i:11;a:1:{i:0;s:112:"
 <--->
 The manager <b>it</b>. <b>Is</b> Filing this report and.
 <--->
-<b>Is</b> a signed hereby represent.
+<b>Is</b> a signed hereby represent
 <--->
-";}i:12;a:1:{i:0;s:89:" ...  gains (if any). May <b>fallback</b> or go <b>insolvent</b>. Credit risk of the ... ";}i:13;a:1:{i:0;s:93:"Leverage gains (if any). May <b>fallback</b> or go <b>insolvent</b>. Credit risk of the deal.";}i:14;a:2:{i:0;s:87:" ... and this <b>match</b> <b>pass</b> is  ... split as well as this <b>match</b>  ... ";i:1;s:87:" ... and this <b>match</b> <b>pass</b> is  ... split as well as this <b>match</b>  ... ";}}}}
+";}i:12;a:1:{i:0;s:89:" ...  gains (if any). May <b>fallback</b> or go <b>insolvent</b>. Credit risk of the ... ";}i:13;a:1:{i:0;s:93:"Leverage gains (if any). May <b>fallback</b> or go <b>insolvent</b>. Credit risk of the deal.";}i:14;a:2:{i:0;s:76:" ...  this <b>match</b> <b>pass</b> is ... as well as this <b>match</b> ... ";i:1;s:76:" ...  this <b>match</b> <b>pass</b> is ... as well as this <b>match</b> ... ";}}}}

test/test_186/test.xml

@@ -148,6 +148,7 @@ index rt
 <sphinxql>update rt set gid=3212, bfloat=55.5, bint=2340000000000 where id=1</sphinxql>
 <sphinxql>select * from rt</sphinxql>
 <sphinxql>select 100500some from rt</sphinxql>
+<sphinxql>select `id` from rt</sphinxql>
 </sphqueries>
 
   <DB_Create>

test/test_202/test.xml

@@ -7,7 +7,8 @@
 <non-windows/>
 </requires>
 
-<config>
+<config>#!/usr/bin/python
+print """
 indexer
 {
 	mem_limit = 32M
@@ -55,7 +56,18 @@ index plain2
 	path = <data_path/>/plain2
 	docinfo = none
 }
-
+"""
+
+file_exists = False
+try:
+	with open('part.conf'):
+		file_exists = True
+except IOError:
+	pass
+
+if file_exists==True:
+	with open('part.conf') as f:
+		print f.read()
 </config>
 
 <db_drop>drop table if exists test_table</db_drop>
@@ -86,8 +98,13 @@ insert into test_table values
 
 <custom_test><![CDATA[
 
+$results = array();
 $error = '';
 
+$query = create_function('$client, $index','
+$result = $client->Query ( "first", $index );	if ( $result ) { unset ( $result["time"] ); return $result; } else return $client->GetLastError();
+');
+
 StopSearchd ('config.conf', 'searchd.pid');
 foreach ( array ( 'spa', 'spd', 'spe', 'spi', 'spk', 'spm', 'spp', 'sps', 'sph' ) as $ext )
 {
@@ -99,16 +116,18 @@ foreach ( array ( 'spa', 'spd', 'spe', 'spi', 'spk', 'spm', 'spp', 'sps', 'sph'
 }
 $state = StartSearchd ('config.conf', 'error.txt', 'searchd.pid', $error);
 
-RunIndexer ( $errors, '--rotate "p*lain2"' ); // adding plain2 index
-sleep(2);
 
-$query = create_function('$client','
-$result = $client->Query ( "first", "plain2" );	if ( $result ) { unset ( $result["time"] ); return $result; } else return $client->GetLastError();
-');
+$results[] = $query ( $client, 'plain3' );
+$path = "part.conf";
+file_put_contents ( $path, "index plain3\n{\nsource = sql\npath = data/plain3\n}\n" );
 
-$results = array();
+RunIndexer ( $errors, '--rotate p*lain2 plain3' ); // adding indexes
+sleep(2);
+
+unlink($path);
+$results[] = $query ( $client, 'plain3' );
 
-$results[] = $query ( $client );
+$results[] = $query ( $client, 'plain2' );
 
 ]]></custom_test>