core: option to set number of workers per listen socket

- new cfg global parameter: socket_workers - set the number of worker
  processes for the next listen socket
- used before listen on udp and sctp socket - overwrites
  children/sctp_children value for that socket
- used bofer listen on tcp and tls socket - adds extra tcp workers,
  these handling traffic only on that socket
- socket_workers is reset with next listen socket that is added, thus
  use it for each listen socket where you want custom number of workers
- if this parameter is not used at all, it is the same behaviour as so
  far

Example for udp sockets:

children=4
socket_workers=2
listen=udp:127.0.0.1:5080
listen=udp:127.0.0.1:5070
listen=udp:127.0.0.1:5060

- it will start 2 workers to handle traffic on udp:127.0.0.1:5080 and 4
  for each of udp:127.0.0.1:5070 and udp:127.0.0.1:5060. In total there
  are 10 worker processes

Example for tcp sockets:

children=4
socket_workers=2
listen=tcp:127.0.0.1:5080
listen=tcp:127.0.0.1:5070
listen=tcp:127.0.0.1:5060

- it will start 2 workers to handle traffic on tcp:127.0.0.1:5080 and 4
  to handle traffic on both tcp:127.0.0.1:5070 and tcp:127.0.0.1:5060.
  In total there are 6 worker processes

cfg.lex

@@ -390,6 +390,7 @@ STAT	statistics
 MAXBUFFER maxbuffer
 SQL_BUFFER_SIZE sql_buffer_size
 CHILDREN children
+SOCKET_WORKERS socket_workers
 CHECK_VIA	check_via
 PHONE2TEL	phone2tel
 SYN_BRANCH syn_branch
@@ -782,6 +783,7 @@ IMPORTFILE      "import_file"
 <INITIAL>{MAXBUFFER}	{ count(); yylval.strval=yytext; return MAXBUFFER; }
 <INITIAL>{SQL_BUFFER_SIZE}	{ count(); yylval.strval=yytext; return SQL_BUFFER_SIZE; }
 <INITIAL>{CHILDREN}	{ count(); yylval.strval=yytext; return CHILDREN; }
+<INITIAL>{SOCKET_WORKERS}	{ count(); yylval.strval=yytext; return SOCKET_WORKERS; }
 <INITIAL>{CHECK_VIA}	{ count(); yylval.strval=yytext; return CHECK_VIA; }
 <INITIAL>{PHONE2TEL}	{ count(); yylval.strval=yytext; return PHONE2TEL; }
 <INITIAL>{SYN_BRANCH}	{ count(); yylval.strval=yytext; return SYN_BRANCH; }

cfg.y

@@ -443,6 +443,7 @@ extern char *finame;
 %token PORT
 %token STAT
 %token CHILDREN
+%token SOCKET_WORKERS
 %token CHECK_VIA
 %token PHONE2TEL
 %token SYN_BRANCH
@@ -938,6 +939,8 @@ assign_stm:
 	| PORT EQUAL error    { yyerror("number expected"); }
 	| CHILDREN EQUAL NUMBER { children_no=$3; }
 	| CHILDREN EQUAL error { yyerror("number expected"); }
+	| SOCKET_WORKERS EQUAL NUMBER { socket_workers=$3; }
+	| SOCKET_WORKERS EQUAL error { yyerror("number expected"); }
 	| CHECK_VIA EQUAL NUMBER { check_via=$3; }
 	| CHECK_VIA EQUAL error { yyerror("boolean value expected"); }
 	| PHONE2TEL EQUAL NUMBER { phone2tel=$3; }
@@ -998,7 +1001,7 @@ assign_stm:
 	| TCP_ACCEPT_ALIASES EQUAL error { yyerror("boolean value expected"); }
 	| TCP_CHILDREN EQUAL NUMBER {
 		#ifdef USE_TCP
-			tcp_children_no=$3;
+			tcp_cfg_children_no=$3;
 		#else
 			warn("tcp support not compiled in");
 		#endif

globals.h

@@ -88,8 +88,10 @@ extern struct socket_info* sendipv6_sctp; /* same as above for ipv6 */
 extern unsigned int maxbuffer;
 extern unsigned int sql_buffer_size;
 extern int children_no;
+extern int socket_workers;
 #ifdef USE_TCP
 extern int tcp_main_pid;
+extern int tcp_cfg_children_no;
 extern int tcp_children_no;
 extern int tcp_disable;
 extern enum poll_types tcp_poll_method;

ip_addr.h

@@ -116,6 +116,8 @@ struct socket_info{
 	char proto; /* tcp or udp*/
 	str sock_str; /* Socket proto, ip, and port as string */
 	struct addr_info* addr_info_lst; /* extra addresses (e.g. SCTP mh) */
+	int workers; /* number of worker processes for this socket */
+	int workers_tcpidx; /* index of workers in tcp children array */
 };
 
 

main.c

@@ -341,9 +341,12 @@ unsigned int maxbuffer = MAX_RECV_BUFFER_SIZE; /* maximum buffer size we do
 												  be re-configured */
 unsigned int sql_buffer_size = 65535; /* Size for the SQL buffer. Defaults to 64k. 
                                          This may be re-configured */
-int children_no = 0;			/* number of children processing requests */
+int socket_workers = 0;		/* number of workers processing requests for a socket
+							   - it's reset everytime with a new listen socket */
+int children_no = 0;		/* number of children processing requests */
 #ifdef USE_TCP
-int tcp_children_no = 0;
+int tcp_cfg_children_no = 0; /* set via config or command line option */
+int tcp_children_no = 0; /* based on socket_workers and tcp_cfg_children_no */
 int tcp_disable = 0; /* 1 if tcp is disabled */
 #endif
 #ifdef USE_TLS
@@ -1265,6 +1268,7 @@ int main_loop()
 #ifdef EXTRA_DEBUG
 	int r;
 #endif
+	int nrprocs;
 
 	/* one "main" process and n children handling i/o */
 	if (dont_fork){
@@ -1453,7 +1457,7 @@ int main_loop()
 				sendipv6=si;
 	#endif
 			/* children_no per each socket */
-			cfg_register_child(children_no);
+			cfg_register_child((si->workers>0)?si->workers:children_no);
 		}
 #ifdef USE_RAW_SOCKS
 		/* always try to have a raw socket opened if we are using ipv4 */
@@ -1507,7 +1511,7 @@ int main_loop()
 					sendipv6_sctp=si;
 		#endif
 				/* sctp_children_no per each socket */
-				cfg_register_child(sctp_children_no);
+				cfg_register_child((si->workers>0)?si->workers:sctp_children_no);
 			}
 		}
 #endif /* USE_SCTP */
@@ -1589,7 +1593,8 @@ int main_loop()
 
 		/* udp processes */
 		for(si=udp_listen; si; si=si->next){
-			for(i=0;i<children_no;i++){
+			nrprocs = (si->workers>0)?si->workers:children_no;
+			for(i=0;i<nrprocs;i++){
 				if(si->address.af==AF_INET6) {
 					snprintf(si_desc, MAX_PT_DESC, "udp receiver child=%d "
 						"sock=[%s]:%s",
@@ -1620,7 +1625,8 @@ int main_loop()
 		/* sctp processes */
 		if (!sctp_disable){
 			for(si=sctp_listen; si; si=si->next){
-				for(i=0;i<sctp_children_no;i++){
+				nrprocs = (si->workers>0)?si->workers:sctp_children_no;
+				for(i=0;i<nrprocs;i++){
 					if(si->address.af==AF_INET6) {
 						snprintf(si_desc, MAX_PT_DESC, "sctp receiver child=%d "
 								"sock=[%s]:%s",
@@ -1762,17 +1768,33 @@ static int calc_proc_no(void)
 {
 	int udp_listeners;
 	struct socket_info* si;
+#ifdef USE_TCP
+	int tcp_listeners;
+	int tcp_e_listeners;
+#endif
 #ifdef USE_SCTP
 	int sctp_listeners;
 #endif
 
-	for (si=udp_listen, udp_listeners=0; si; si=si->next, udp_listeners++);
+	for (si=udp_listen, udp_listeners=0; si; si=si->next)
+		udp_listeners += (si->workers>0)?si->workers:children_no;
+#ifdef USE_TCP
+	for (si=tcp_listen, tcp_listeners=0, tcp_e_listeners=0; si; si=si->next) {
+		if(si->workers>0)
+			tcp_listeners += si->workers;
+		else
+			 tcp_e_listeners = tcp_cfg_children_no;
+	}
+	tcp_listeners += tcp_e_listeners;
+	tcp_children_no = tcp_listeners;
+#endif
 #ifdef USE_SCTP
-	for (si=sctp_listen, sctp_listeners=0; si; si=si->next, sctp_listeners++);
+	for (si=sctp_listen, sctp_listeners=0; si; si=si->next)
+		sctp_listeners += (si->workers>0)?si->workers:sctp_children_no;
 #endif
 	return
 		     /* receivers and attendant */
-		(dont_fork ? 1 : children_no * udp_listeners + 1)
+		(dont_fork ? 1 : udp_listeners + 1)
 		     /* timer process */
 		+ 1 /* always, we need it in most cases, and we can't tell here
 		       & now if we don't need it */
@@ -1780,10 +1802,10 @@ static int calc_proc_no(void)
 		+ 1 /* slow timer process */
 #endif
 #ifdef USE_TCP
-		+((!tcp_disable)?( 1/* tcp main */ + tcp_children_no ):0)
+		+((!tcp_disable)?( 1/* tcp main */ + tcp_listeners ):0)
 #endif
 #ifdef USE_SCTP
-		+((!sctp_disable)?sctp_children_no*sctp_listeners:0)
+		+((!sctp_disable)?sctp_listeners:0)
 #endif
 		;
 }
@@ -2142,7 +2164,7 @@ try_again:
 					break;
 			case 'N':
 				#ifdef USE_TCP
-					tcp_children_no=strtol(optarg, &tmp, 10);
+					tcp_cfg_children_no=strtol(optarg, &tmp, 10);
 					if ((tmp==0) ||(*tmp)){
 						fprintf(stderr, "bad process number: -N %s\n",
 									optarg);
@@ -2274,7 +2296,8 @@ try_again:
 	if (children_no<=0) children_no=CHILD_NO;
 #ifdef USE_TCP
 	if (!tcp_disable){
-		if (tcp_children_no<=0) tcp_children_no=children_no;
+		if (tcp_cfg_children_no<=0) tcp_cfg_children_no=children_no;
+		tcp_children_no = tcp_cfg_children_no;
 	}
 #endif
 #ifdef USE_SCTP

socket_info.c

@@ -622,6 +622,10 @@ static struct socket_info* new_sock2list(char* name, struct name_lst* addr_l,
 		LOG(L_ERR, "ERROR: new_sock2list: new_sock_info failed\n");
 		goto error;
 	}
+	if(socket_workers>0) {
+		si->workers = socket_workers;
+		socket_workers = 0;
+	}
 	sock_listadd(list, si);
 	return si;
 error:

tcp_init.h

@@ -50,6 +50,7 @@ struct tcp_child{
 	int proc_no; /* ser proc_no, for debugging */
 	int unix_sock; /* unix "read child" sock fd */
 	int busy;
+	struct socket_info *mysocket; /* listen socket to handle traffic on it */
 	int n_reqs; /* number of requests serviced so far */
 };
 

tcp_main.c

@@ -268,7 +268,7 @@ struct tcp_conn_alias** tcpconn_aliases_hash=0;
 struct tcp_connection** tcpconn_id_hash=0;
 gen_lock_t* tcpconn_lock=0;
 
-struct tcp_child* tcp_children;
+struct tcp_child* tcp_children=0;
 static int* connection_id=0; /*  unique for each connection, used for 
 								quickly finding the corresponding connection
 								for a reply */
@@ -282,6 +282,12 @@ static io_wait_h io_h;
 static struct local_timer tcp_main_ltimer;
 static ticks_t tcp_main_prev_ticks;
 
+/* tell if there are tcp workers that should handle only specific socket
+ * - used to optimize the search of least loaded worker for a tcp socket
+ * - 0 - no workers per tcp sockets have been set
+ * - 1 + generic_workers - when there are workers per tcp sockets
+ */
+static int tcp_sockets_gworkers = 0;
 
 static ticks_t tcpconn_main_timeout(ticks_t , struct timer_ln* , void* );
 
@@ -3890,24 +3896,63 @@ inline static int send2child(struct tcp_connection* tcpconn)
 	int i;
 	int min_busy;
 	int idx;
+	int wfirst;
+	int wlast;
 	static int crt=0; /* current child */
 	int last;
 	
-	min_busy=tcp_children[0].busy;
-	idx=0;
-	last=crt+tcp_children_no;
-	for (; crt<last; crt++){
-		i=crt%tcp_children_no;
-		if (!tcp_children[i].busy){
-			idx=i;
-			min_busy=0;
-			break;
-		}else if (min_busy>tcp_children[i].busy){
-			min_busy=tcp_children[i].busy;
-			idx=i;
+	if(likely(tcp_sockets_gworkers==0)) {
+		/* no child selection based on received socket
+		 * - use least loaded over all */
+		min_busy=tcp_children[0].busy;
+		idx=0;
+		last=crt+tcp_children_no;
+		for (; crt<last; crt++){
+			i=crt%tcp_children_no;
+			if (!tcp_children[i].busy){
+				idx=i;
+				min_busy=0;
+				break;
+			}else if (min_busy>tcp_children[i].busy){
+				min_busy=tcp_children[i].busy;
+				idx=i;
+			}
+		}
+		crt=idx+1; /* next time we start with crt%tcp_children_no */
+	} else {
+		/* child selection based on received socket
+		 * - use least loaded per received socket, starting with the first
+		 *   in its group */
+		if(tcpconn->rcv.bind_address->workers>0) {
+			wfirst = tcpconn->rcv.bind_address->workers_tcpidx;
+			wlast = wfirst + tcpconn->rcv.bind_address->workers;
+			LM_DBG("===== checking per-socket specific workers (%d/%d..%d/%d) [%s]\n",
+					tcp_children[wfirst].pid, tcp_children[wfirst].proc_no,
+					tcp_children[wlast-1].pid, tcp_children[wlast-1].proc_no,
+					tcpconn->rcv.bind_address->sock_str.s);
+		} else {
+			wfirst = 0;
+			wlast = tcp_sockets_gworkers - 1;
+			LM_DBG("+++++ checking per-socket generic workers (%d/%d..%d/%d) [%s]\n",
+					tcp_children[wfirst].pid, tcp_children[wfirst].proc_no,
+					tcp_children[wlast-1].pid, tcp_children[wlast-1].proc_no,
+					tcpconn->rcv.bind_address->sock_str.s);
+		}
+		idx = wfirst;
+		min_busy = tcp_children[idx].busy;
+		for(i=wfirst; i<wlast; i++) {
+			if (!tcp_children[i].busy){
+				idx=i;
+				min_busy=0;
+				break;
+			} else {
+				if (min_busy>tcp_children[i].busy) {
+					min_busy=tcp_children[i].busy;
+					idx=i;
+				}
+			}
 		}
 	}
-	crt=idx+1; /* next time we start with crt%tcp_children_no */
 	
 	tcp_children[idx].busy++;
 	tcp_children[idx].n_reqs++;
@@ -3916,9 +3961,9 @@ inline static int send2child(struct tcp_connection* tcpconn)
 				" connection passed to the least busy one (%d)\n",
 				min_busy);
 	}
-	DBG("send2child: to tcp child %d %d(%ld), %p\n", idx, 
-					tcp_children[idx].proc_no,
-					(long)tcp_children[idx].pid, tcpconn);
+	LM_DBG("selected tcp worker %d %d(%ld) for activity on [%s], %p\n",
+			idx, tcp_children[idx].proc_no, (long)tcp_children[idx].pid,
+			tcpconn->rcv.bind_address->sock_str.s, tcpconn);
 	/* first make sure this child doesn't have pending request for
 	 * tcp_main (to avoid a possible deadlock: e.g. child wants to
 	 * send a release command, but the master fills its socket buffer
@@ -4838,9 +4883,10 @@ int tcp_fix_child_sockets(int* fd)
 /* starts the tcp processes */
 int tcp_init_children()
 {
-	int r;
+	int r, i;
 	int reader_fd_1; /* for comm. with the tcp children read  */
 	pid_t pid;
+	char si_desc[MAX_PT_DESC];
 	struct socket_info *si;
 	
 	/* estimate max fd. no:
@@ -4867,13 +4913,32 @@ int tcp_init_children()
 			LOG(L_ERR, "ERROR: tcp_init_children: out of memory\n");
 			goto error;
 	}
+	memset(tcp_children, 0, sizeof(struct tcp_child)*tcp_children_no);
+	/* assign own socket for tcp workers, if it is the case
+	 * - add them from end to start of tcp children array
+	 * - thus, have generic tcp workers at beginning */
+	i = tcp_children_no-1;
+	for(si=tcp_listen; si; si=si->next) {
+		if(si->workers>0) {
+			si->workers_tcpidx = i - si->workers + 1;
+			for(r=0; r<si->workers; r++) {
+				tcp_children[i].mysocket = si;
+				i--;
+			}
+		}
+	}
+	tcp_sockets_gworkers = (i != tcp_children_no-1)?(1 + i + 1):0;
+
 	/* create the tcp sock_info structures */
 	/* copy the sockets --moved to main_loop*/
 	
 	/* fork children & create the socket pairs*/
 	for(r=0; r<tcp_children_no; r++){
 		child_rank++;
-		pid=fork_tcp_process(child_rank, "tcp receiver", r, &reader_fd_1);
+		snprintf(si_desc, MAX_PT_DESC, "tcp receiver (%s)",
+				(tcp_children[r].mysocket!=NULL)?
+					tcp_children[r].mysocket->sock_str.s:"generic");
+		pid=fork_tcp_process(child_rank, si_desc, r, &reader_fd_1);
 		if (pid<0){
 			LOG(L_ERR, "ERROR: tcp_main: fork failed: %s\n",
 					strerror(errno));