added #1423 sub-select to APIs (php, python, libsphinxclient)

fixed client protocol version ver 1.29 (0x11D)

git-svn-id: svn://svn.sphinxsearch.com/sphinx/trunk@3678 406a0c4d-033a-0410-8de8-e80135713968

api/libsphinxclient/sphinxclient.c

@@ -173,6 +173,7 @@ struct st_sphinx_client
 	const char *			outer_orderby;
 	int						outer_offset;
 	int						outer_limit;
+	sphinx_bool				has_outer;
 
 
 	int						num_reqs;
@@ -261,6 +262,7 @@ sphinx_client * sphinx_create ( sphinx_bool copy_args )
 	client->outer_orderby			= NULL;
 	client->outer_offset			= 0;
 	client->outer_limit				= 0;
+	client->has_outer				= SPH_FALSE;
 
 	client->num_reqs				= 0;
 	client->response_len			= 0;
@@ -944,7 +946,7 @@ void sphinx_reset_query_flag ( sphinx_client * client )
 }
 
 
-sphinx_bool sphinx_set_outer ( sphinx_client * client, const char * orderby, int offset, int limit )
+sphinx_bool sphinx_set_outer_select ( sphinx_client * client, const char * orderby, int offset, int limit )
 {
 	if ( !client )
 		return SPH_FALSE;
@@ -959,10 +961,24 @@ sphinx_bool sphinx_set_outer ( sphinx_client * client, const char * orderby, int
 	client->outer_orderby = strchain ( client, orderby );
 	client->outer_offset = offset;
 	client->outer_limit = limit;
+	client->has_outer = SPH_TRUE;
 	return SPH_TRUE;
 }
 
 
+void sphinx_reset_outer_select ( sphinx_client * client )
+{
+	if ( !client )
+		return;
+
+	unchain ( client, client->outer_orderby );
+	client->outer_orderby = NULL;
+	client->outer_offset = 0;
+	client->outer_limit = 0;
+	client->has_outer = SPH_FALSE;
+}
+
+
 void sphinx_reset_filters ( sphinx_client * client )
 {
 	int i;
@@ -1100,13 +1116,7 @@ static int calc_req_len ( sphinx_client * client, const char * query, const char
 		res += 4 + ( client->predicted_time>0 ? 4 : 0 );
 
 	if ( client->ver_search>=0x11D )
-	{
-		// string outer order by + int outer offset + int outer limit
-		if ( safestrlen ( client->outer_orderby )>0 )
-			res += safestrlen ( client->outer_orderby ) + 12;
-		else
-			res += 4;
-	}
+		res += safestrlen ( client->outer_orderby ) + 16; // string outer order by + int outer offset + int outer limit + has outer flag
 
 	return (int)res;
 }
@@ -1316,15 +1326,10 @@ int sphinx_add_query ( sphinx_client * client, const char * query, const char *
 
 	if ( client->ver_search>=0x11D )
 	{
-		if ( safestrlen ( client->outer_orderby )>0 )
-		{
-			send_str ( &req, client->outer_orderby );
-			send_int ( &req, client->outer_offset );
-			send_int ( &req, client->outer_limit );
-		} else
-		{
-			send_str ( &req, NULL );
-		}
+		send_str ( &req, client->outer_orderby );
+		send_int ( &req, client->outer_offset );
+		send_int ( &req, client->outer_limit );
+		send_int ( &req, client->has_outer );
 	}
 
 	if ( !req )

api/libsphinxclient/sphinxclient.h

@@ -222,7 +222,8 @@ sphinx_bool					sphinx_set_select				( sphinx_client * client, const char * sele
 
 sphinx_bool					sphinx_set_query_flags			( sphinx_client * client, const char * flag_name, sphinx_bool enabled, int max_predicted_msec );
 void						sphinx_reset_query_flags		( sphinx_client * client );
-sphinx_bool					sphinx_set_outer				( sphinx_client * client, const char * orderby, int offset, int limit );
+sphinx_bool					sphinx_set_outer_select			( sphinx_client * client, const char * orderby, int offset, int limit );
+void						sphinx_reset_outer_select		( sphinx_client * client );
 
 void						sphinx_reset_filters			( sphinx_client * client );
 void						sphinx_reset_groupby			( sphinx_client * client );

api/sphinxapi.php

@@ -431,6 +431,7 @@ class SphinxClient
 	var $_outerorderby; ///< outer match sort by
 	var $_outeroffset; ///< outer offset
 	var $_outerlimit; ///< outer limit
+	var $_hasouter;
 
 	var $_error;		///< last error message
 	var $_warning;		///< last warning message
@@ -482,6 +483,10 @@ class SphinxClient
 		$this->_select		= "*";
 		$this->_query_flags = 0;
 		$this->_predictedtime = 0;
+		$this->_outerorderby = "";
+		$this->_outeroffset = 0;
+		$this->_outerlimit = 0;
+		$this->_hasouter = false;
 
 		$this->_error		= ""; // per-reply fields (for single-query case)
 		$this->_warning		= "";
@@ -972,7 +977,7 @@ class SphinxClient
 	}
 	
 	/// set outer order by parameters
-	function SetOuter ( $orderby, $offset, $limit )
+	function SetOuterSelect ( $orderby, $offset, $limit )
 	{
 		assert ( is_string($orderby) );
 		assert ( is_int($offset) );
@@ -983,6 +988,7 @@ class SphinxClient
 		$this->_outerorderby = $orderby;
 		$this->_outeroffset = $offset;
 		$this->_outerlimit = $limit;
+		$this->_hasouter = true;
 	}
 
 	
@@ -1016,6 +1022,14 @@ class SphinxClient
 		$this->_predictedtime = 0;
 	}
 
+	function ResetOuterSelect ()
+	{
+		$this->_outerorderby = '';
+		$this->_outeroffset = 0;
+		$this->_outerlimit = 0;
+		$this->_hasouter = false;
+	}
+
 	//////////////////////////////////////////////////////////////////////////////
 
 	/// connect to searchd server, run given search query through given indexes,
@@ -1160,14 +1174,12 @@ class SphinxClient
 		if ( $this->_predictedtime>0 )
 			$req .= pack ( "N", (int)$this->_predictedtime );
 			
-		if ( empty($this->_outerorderby) )
-		{
+		$req .= pack ( "N", strlen($this->_outerorderby) ) . $this->_outerorderby;
+		$req .= pack ( "NN", $this->_outeroffset, $this->_outerlimit );
+		if ( $this->_hasouter )
+			$req .= pack ( "N", 1 );
+		else
 			$req .= pack ( "N", 0 );
-		} else
-		{		 
-			$req .= pack ( "N", strlen($this->_outerorderby) ) . $this->_outerorderby;
-			$req .= pack ( "NN", $this->_outeroffset, $this->_outerlimit );
-		}
 
 		// mbstring workaround
 		$this->_MBPop ();

api/sphinxapi.py

@@ -151,6 +151,7 @@ class SphinxClient:
 		self._outerorderby = ''							# outer match sort by
 		self._outeroffset = 0								# outer offset
 		self._outerlimit = 0								# outer limit
+		self._hasouter = False							# sub-select enabled
 		
 		self._error			= ''							# last error message
 		self._warning		= ''							# last warning message
@@ -509,7 +510,7 @@ class SphinxClient:
 		if name=="idf":
 			self._query_flags = SetBit ( self._query_flags, 4, value=="plain" )
 
-	def SetOuter ( self, orderby, offset, limit ):
+	def SetOuterSelect ( self, orderby, offset, limit ):
 		assert(isinstance(orderby, str))
 		assert(isinstance(offset, (int, long)))
 		assert(isinstance(limit, (int, long)))
@@ -519,6 +520,7 @@ class SphinxClient:
 		self._outerorderby = orderby
 		self._outeroffset = offset
 		self._outerlimit = limit
+		self._hasouter = True
 			
 	def ResetOverrides (self):
 		self._overrides = {}
@@ -544,6 +546,12 @@ class SphinxClient:
 	def ResetQueryFlag (self):
 		self._query_flags = 0
 		self._predictedtime = 0
+		
+	def ResetOuterSelect (self):
+		self._outerorderby = ''
+		self._outeroffset = 0
+		self._outerlimit = 0
+		self._hasouter = False
 
 	def Query (self, query, index='*', comment=''):
 		"""
@@ -669,11 +677,12 @@ class SphinxClient:
 			req.append ( pack('>L', self._predictedtime ) )
 
 		# outer
-		if len(self._outerorderby) == 0:
-			req.append ( pack ('>L', 0))
+		req.append ( pack('>L',len(self._outerorderby)) + self._outerorderby )
+		req.append ( pack ( '>2L', self._outeroffset, self._outerlimit ) )
+		if self._hasouter:
+			req.append ( pack('>L', 1) )
 		else:
-			req.append ( pack('>L',len(self._outerorderby)) + self._outerorderby )
-			req.append ( pack ( '>2L', self._outeroffset, self._outerlimit ) )
+			req.append ( pack('>L', 0) )
 			
 		# send query, get response
 		req = ''.join(req)

src/searchd.cpp

@@ -5233,7 +5233,7 @@ protected:
 
 int SearchRequestBuilder_t::CalcQueryLen ( const char * sIndexes, const CSphQuery & q ) const
 {
-	int iReqSize = 120 + 2*sizeof(SphDocID_t) + 4*q.m_iWeights
+	int iReqSize = 132 + 2*sizeof(SphDocID_t) + 4*q.m_iWeights
 		+ q.m_sSortBy.Length()
 		+ strlen ( sIndexes )
 		+ q.m_sGroupBy.Length()
@@ -5267,7 +5267,7 @@ int SearchRequestBuilder_t::CalcQueryLen ( const char * sIndexes, const CSphQuer
 	ARRAY_FOREACH ( i, q.m_dOverrides )
 		iReqSize += 12 + q.m_dOverrides[i].m_sAttr.Length() + // string attr-name; int type; int values-count
 			( q.m_dOverrides[i].m_eAttrType==SPH_ATTR_BIGINT ? 16 : 12 )*q.m_dOverrides[i].m_dValues.GetLength(); // ( bigint id; int/float/bigint value )[] values
-	if ( !q.m_sOuterOrderBy.IsEmpty() )
+	if ( q.m_bHasOuter )
 		iReqSize += 4; // outer limit
 	if ( q.m_iMaxPredictedMsec>0 )
 		iReqSize += 4;
@@ -5303,7 +5303,7 @@ void SearchRequestBuilder_t::SendQuery ( const char * sIndexes, NetOutputBuffer_
 
 	// The Search Legacy
 	tOut.SendInt ( 0 ); // offset is 0
-	if ( q.m_sOuterOrderBy.IsEmpty() )
+	if ( !q.m_bHasOuter )
 	{
 		if ( m_iDivideLimits==1 )
 			tOut.SendInt ( q.m_iMaxMatches ); // OPTIMIZE? normally, agent limit is max_matches, even if master limit is less
@@ -5413,13 +5413,16 @@ void SearchRequestBuilder_t::SendQuery ( const char * sIndexes, NetOutputBuffer_
 	if ( q.m_iMaxPredictedMsec>0 )
 		tOut.SendInt ( q.m_iMaxPredictedMsec );
 
-	// emulate empty client order by (client ver 1.29) as master sends fixed outer offset+limits
+	// emulate empty sud-select for agent (client ver 1.29) as master sends fixed outer offset+limits
 	tOut.SendString ( NULL );
+	tOut.SendInt ( 0 );
+	tOut.SendInt ( 0 );
+	tOut.SendInt ( q.m_bHasOuter );
 
 	// master-agent extensions
 	tOut.SendDword ( q.m_eCollation ); // v.1
 	tOut.SendString ( q.m_sOuterOrderBy.cstr() ); // v.2
-	if ( !q.m_sOuterOrderBy.IsEmpty() )
+	if ( q.m_bHasOuter )
 		tOut.SendInt ( q.m_iOuterOffset + q.m_iOuterLimit );
 }
 
@@ -5871,7 +5874,7 @@ void CheckQuery ( const CSphQuery & tQuery, CSphString & sError )
 		sError.SetSprintf ( "retry delay out of bounds (delay=%d)", tQuery.m_iRetryDelay );
 		return;
 	}
-	if ( tQuery.m_iOffset>0 && !tQuery.m_sOuterOrderBy.IsEmpty() )
+	if ( tQuery.m_iOffset>0 && tQuery.m_bHasOuter )
 	{
 		sError.SetSprintf ( "inner offset must be 0 when using outer order by" );
 		return;
@@ -5934,6 +5937,13 @@ void PrepareQueryEmulation ( CSphQuery * pQuery )
 		case SPH_MATCH_PHRASE:	pQuery->m_eRanker = SPH_RANK_PROXIMITY; *szRes++ = '\"'; *szRes = '\0'; break;
 		default:				return;
 	}
+
+	if ( !pQuery->m_bHasOuter )
+	{
+		pQuery->m_sOuterOrderBy = "";
+		pQuery->m_iOuterOffset = 0;
+		pQuery->m_iOuterLimit = 0;
+	}
 }
 
 
@@ -6262,11 +6272,9 @@ bool ParseSearchQuery ( InputBuffer_c & tReq, CSphQuery & tQuery, int iVer, int
 	if ( iVer>=0x11D )
 	{
 		tQuery.m_sOuterOrderBy = tReq.GetString();
-		if ( !tQuery.m_sOuterOrderBy.IsEmpty() )
-		{
-			tQuery.m_iOuterOffset = tReq.GetDword();
-			tQuery.m_iOuterLimit = tReq.GetDword();
-		}
+		tQuery.m_iOuterOffset = tReq.GetDword();
+		tQuery.m_iOuterLimit = tReq.GetDword();
+		tQuery.m_bHasOuter = ( tReq.GetInt()!=0 );
 	}
 
 	// extension v.1
@@ -6278,7 +6286,7 @@ bool ParseSearchQuery ( InputBuffer_c & tReq, CSphQuery & tQuery, int iVer, int
 	if ( iMasterVer>=2 )
 	{
 		tQuery.m_sOuterOrderBy = tReq.GetString();
-		if ( !tQuery.m_sOuterOrderBy.IsEmpty() )
+		if ( tQuery.m_bHasOuter )
 			tQuery.m_iOuterLimit = tReq.GetInt();
 	}
 
@@ -6601,7 +6609,7 @@ void LogQuerySphinxql ( const CSphQuery & q, const CSphQueryResult & tRes, const
 	// format request as SELECT query
 	///////////////////////////////////
 
-	if ( !q.m_sOuterOrderBy.IsEmpty() )
+	if ( q.m_bHasOuter )
 		tBuf.Append ( "SELECT * FROM (" );
 
 	tBuf.Append ( "SELECT %s FROM %s", q.m_sSelect.cstr(), q.m_sIndexes.cstr() );
@@ -6760,9 +6768,11 @@ void LogQuerySphinxql ( const CSphQuery & q, const CSphQueryResult & tRes, const
 	}
 
 	// outer order by, limit
-	if ( !q.m_sOuterOrderBy.IsEmpty() )
+	if ( q.m_bHasOuter )
 	{
-		tBuf.Append ( ") ORDER BY %s", q.m_sOuterOrderBy.cstr() );
+		tBuf.Append ( ")" );
+		if ( !q.m_sOuterOrderBy.IsEmpty() )
+			tBuf.Append ( " ORDER BY %s", q.m_sOuterOrderBy.cstr() );
 		if ( q.m_iOuterOffset>0 )
 			tBuf.Append ( " LIMIT %d, %d", q.m_iOuterOffset, q.m_iOuterLimit );
 		else if ( q.m_iOuterLimit>0 )
@@ -8127,11 +8137,13 @@ bool MinimizeAggrResult ( AggrResult_t & tRes, CSphQuery & tQuery, int iLocals,
 	// this is a good time to apply outer order clause, too
 	if ( tRes.m_iSuccesses>1 )
 	{
+		ESphSortOrder eQuerySort = ( tQuery.m_sOuterOrderBy.IsEmpty() ? SPH_SORT_RELEVANCE : SPH_SORT_EXTENDED );
 		// got outer order? gotta do a couple things
-		if ( !tQuery.m_sOuterOrderBy.IsEmpty() )
+		if ( tQuery.m_bHasOuter )
 		{
 			// first, temporarily patch up sorting clause and max_matches (we will restore them later)
 			Swap ( tQuery.m_sOuterOrderBy, tQuery.m_sGroupBy.IsEmpty() ? tQuery.m_sSortBy : tQuery.m_sGroupSortBy );
+			Swap ( eQuerySort, tQuery.m_eSort );
 			tQuery.m_iMaxMatches *= tRes.m_dMatchCounts.GetLength();
 			// FIXME? probably not right; 20 shards with by 300 matches might be too much
 			// but propagating too small inner max_matches to the outer is not right either
@@ -8163,9 +8175,10 @@ bool MinimizeAggrResult ( AggrResult_t & tRes, CSphQuery & tQuery, int iLocals,
 		ISphMatchSorter * pSorter = sphCreateQueue ( &tQuery, tRes.m_tSchema, tRes.m_sError, false );
 
 		// restore outer order related patches, or it screws up the query log
-		if ( !tQuery.m_sOuterOrderBy.IsEmpty() )
+		if ( tQuery.m_bHasOuter )
 		{
 			Swap ( tQuery.m_sOuterOrderBy, tQuery.m_sGroupBy.IsEmpty() ? tQuery.m_sSortBy : tQuery.m_sGroupSortBy );
+			Swap ( eQuerySort, tQuery.m_eSort );
 			tQuery.m_iMaxMatches /= tRes.m_dMatchCounts.GetLength();
 		}
 
@@ -8200,11 +8213,12 @@ bool MinimizeAggrResult ( AggrResult_t & tRes, CSphQuery & tQuery, int iLocals,
 
 	// apply outer order clause to single result set
 	// (multiple combined sets just got reordered above)
-	if ( tRes.m_iSuccesses==1 && !tQuery.m_sOuterOrderBy.IsEmpty() )
-	{
-		// apply inner limit
+	// apply inner limit first
+	if ( tRes.m_iSuccesses==1 && tQuery.m_bHasOuter )
 		tRes.ClampMatches ( tQuery.m_iLimit, bAllEqual );
 
+	if ( tRes.m_iSuccesses==1 && tQuery.m_bHasOuter && !tQuery.m_sOuterOrderBy.IsEmpty() )
+	{
 		// reorder (aka outer order)
 		ESphSortFunc eFunc;
 		GenericMatchSort_fn tReorder;

src/sphinx.cpp

@@ -5518,6 +5518,7 @@ CSphQuery::CSphQuery ()
 	, m_sSelect			( "" )
 	, m_iOuterOffset	( 0 )
 	, m_iOuterLimit		( 0 )
+	, m_bHasOuter		( false )
 	, m_bReverseScan	( false )
 	, m_bIgnoreNonexistent ( false )
 	, m_bIgnoreNonexistentIndexes ( false )

src/sphinx.h

@@ -2454,6 +2454,7 @@ public:
 	CSphString		m_sOuterOrderBy;	///< temporary (?) subselect hack
 	int				m_iOuterOffset;		///< keep and apply outer offset at master
 	int				m_iOuterLimit;
+	bool			m_bHasOuter;
 
 	bool			m_bReverseScan;		///< perform scan in reverse order
 	bool			m_bIgnoreNonexistent; ///< whether to warning or not about non-existent columns in select list

src/sphinxql.y

@@ -188,14 +188,11 @@ subselect_start:
 
 
 opt_outer_order:
-	// nothing
-		{
-			pParser->m_pQuery->m_sOuterOrderBy = pParser->m_pQuery->m_sOrderBy;
-		}
-	| TOK_ORDER TOK_BY order_items_list
+	TOK_ORDER TOK_BY order_items_list
 		{
 			pParser->m_pQuery->m_sOuterOrderBy.SetBinary ( pParser->m_pBuf+$3.m_iStart,
 				$3.m_iEnd-$3.m_iStart );
+			pParser->m_pQuery->m_bHasOuter = true;
 		}
 	;
 
@@ -204,11 +201,13 @@ opt_outer_limit:
 	| TOK_LIMIT TOK_CONST_INT
 		{
 			pParser->m_pQuery->m_iOuterLimit = $2.m_iValue;
+			pParser->m_pQuery->m_bHasOuter = true;
 		}
 	| TOK_LIMIT TOK_CONST_INT ',' TOK_CONST_INT
 		{
 			pParser->m_pQuery->m_iOuterOffset = $2.m_iValue;
 			pParser->m_pQuery->m_iOuterLimit = $4.m_iValue;
+			pParser->m_pQuery->m_bHasOuter = true;
 		}
 	;