- tcp children control fds are now non-blocking

 - send fd queue is used when a childs queue is full

pass_fd.c

@@ -109,7 +109,8 @@ again:
 	if (n<0){
 			/* error */
 		if (errno==EINTR) goto again; /* signal, try again */
-		LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
+		if ((errno!=EAGAIN) &&(errno!=EWOULDBLOCK))
+			LOG(L_CRIT, "ERROR: send_all: send on %d failed: %s\n",
 					socket, strerror(errno));
 	}
 	return n;
@@ -158,8 +159,9 @@ again:
 	ret=sendmsg(unix_socket, &msg, 0);
 	if (ret<0){
 		if (errno==EINTR) goto again;
-		LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
-				unix_socket, strerror(errno));
+		if ((errno!=EAGAIN) && (errno!=EWOULDBLOCK))
+			LOG(L_CRIT, "ERROR: send_fd: sendmsg failed on %d: %s\n",
+					unix_socket, strerror(errno));
 	}
 	
 	return ret;

tcp_conn.h

@@ -43,7 +43,7 @@
 
 #define TCP_CON_MAX_ALIASES 4 /* maximum number of port aliases */
 
-#define TCP_BUF_SIZE 65535
+#define TCP_BUF_SIZE	4096 
 #define DEFAULT_TCP_CONNECTION_LIFETIME 120 /* in  seconds */
 #define DEFAULT_TCP_SEND_TIMEOUT 10 /* if a send can't write for more then 10s,
 									   timeout */

tcp_main.c

@@ -64,6 +64,8 @@
  *              process all children requests, before attempting to send
  *              them new stuff (fixes some deadlocks) (andrei)
  *  2006-02-03  timers are run only once per s (andrei)
+ *              tcp children fds can be non-blocking; send fds are queues on
+ *              EAGAIN (andrei)
  */
 
 
@@ -123,10 +125,14 @@
 
 #define MAX_TCP_CHILDREN 100
 #define TCP_LISTEN_BACKLOG 1024
-/* #define SEND_FD_QUEUE */ /* queue send fd request, instead of sending 
-							   them immediately */
+#define SEND_FD_QUEUE /* queue send fd requests on EAGAIN, instead of sending 
+							them immediately */
+#define TCP_CHILD_NON_BLOCKING 
 #ifdef SEND_FD_QUEUE
-#define MAX_SEND_FD_QUEUE_SIZE	tcp_max_fd_no
+#ifndef TCP_CHILD_NON_BLOCKING
+#define TCP_CHILD_NON_BLOCKING
+#endif
+#define MAX_SEND_FD_QUEUE_SIZE	1024  /* alternative: tcp_max_fd_no */
 #define SEND_FD_QUEUE_SIZE		128  /* initial size */
 #define MAX_SEND_FD_RETRIES		3	 /* FIXME: increase */
 #endif
@@ -985,8 +991,6 @@ static void send_fd_queue_destroy(struct tcp_send_fd_q *q)
 
 static int init_send_fd_queues()
 {
-	if (send_fd_queue_init(&get_fd_q, SEND_FD_QUEUE_SIZE)!=0)
-		goto error;
 	if (send_fd_queue_init(&send2child_q, SEND_FD_QUEUE_SIZE)!=0)
 		goto error;
 	return 0;
@@ -999,7 +1003,6 @@ error:
 
 static void destroy_send_fd_queues()
 {
-	send_fd_queue_destroy(&get_fd_q);
 	send_fd_queue_destroy(&send2child_q);
 }
 
@@ -1053,13 +1056,14 @@ inline static void send_fd_queue_run(struct tcp_send_fd_q* q)
 	for (p=t=&q->data[0]; p<q->crt; p++){
 		if (send_fd(p->unix_sock, &(p->tcp_conn),
 					sizeof(struct tcp_connection*), p->tcp_conn->s)<=0){
-			if (/*FIXME: E_WOULD_BLOCK && */(p->retries<MAX_SEND_FD_RETRIES)){
+			if ( ((errno==EAGAIN)||(errno==EWOULDBLOCK)) && 
+							(p->retries<MAX_SEND_FD_RETRIES)){
 				/* leave in queue for a future try */
 				*t=*p;
 				t->retries++;
 				t++;
 			}else{
-				LOG(L_ERR, "ERROR: rund_send_fd_queue: send_fd failed"
+				LOG(L_ERR, "ERROR: run_send_fd_queue: send_fd failed"
 						   "on %d socket, %ld queue entry, retries %d \n",
 						   p->unix_sock, p-&q->data[0], p->retries);
 			}
@@ -1378,10 +1382,21 @@ inline static int send2child(struct tcp_connection* tcpconn)
 	while(handle_tcp_child(&tcp_children[idx], -1)>0);
 		
 #ifdef SEND_FD_QUEUE
-	if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
+	/* if queue full, try to queue the io */
+	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
+			tcpconn->s)<=0){
+		if ((errno==EAGAIN)||(errno==EWOULDBLOCK)){
+			/* FIXME: remove after debugging */
+			LOG(L_WARN, "WARNING: tcp child %d, socket %d: queue full\n",
+					idx, tcp_children[idx].unix_sock);
+			if (send_fd_queue_add(&send2child_q, tcp_children[idx].unix_sock, 
 						tcpconn)!=0){
-		LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
-		return -1;
+				LOG(L_ERR, "ERROR: send2child: queue send op. failed\n");
+				return -1;
+			}
+		}else{
+			LOG(L_ERR, "ERROR: send2child: send_fd failed\n");
+		}
 	}
 #else
 	if (send_fd(tcp_children[idx].unix_sock, &tcpconn, sizeof(tcpconn),
@@ -1856,6 +1871,29 @@ error:
 
 
 
+/* returns -1 on error */
+static int set_non_blocking(int s)
+{
+	int flags;
+	/* non-blocking */
+	flags=fcntl(s, F_GETFL);
+	if (flags==-1){
+		LOG(L_ERR, "ERROR: set_non_blocking: fnctl failed: (%d) %s\n",
+				errno, strerror(errno));
+		goto error;
+	}
+	if (fcntl(s, F_SETFL, flags|O_NONBLOCK)==-1){
+		LOG(L_ERR, "ERROR: set_non_blocking: fcntl: set non-blocking failed:"
+				" (%d) %s\n", errno, strerror(errno));
+		goto error;
+	}
+	return 0;
+error:
+	return -1;
+}
+
+
+
 /* starts the tcp processes */
 int tcp_init_children()
 {
@@ -1893,6 +1931,15 @@ int tcp_init_children()
 					strerror(errno));
 			goto error;
 		}
+#ifdef TCP_CHILD_NON_BLOCKING
+		if ((set_non_blocking(reader_fd[0])<0) || 
+			(set_non_blocking(reader_fd[1])<0)){
+			LOG(L_ERR, "ERROR: tcp_main: failed to set non blocking"
+						"on child sockets\n");
+			/* continue, it's not critical (it will go slower under
+			 * very high connection rates) */
+		}
+#endif
 		
 		process_no++;
 		child_rank++;

test/gcc_versions.txt

@@ -6,6 +6,7 @@ egcs-2.91.66
 3.0.4
 gcc (GCC) 3.2
 gcc-3.2 (GCC) 3.2.3 (Debian)
+gcc (GCC) 3.3 20030304 (Apple Computer, Inc. build 1809)
 gcc (GCC) 3.3.3 (NetBSD nb3 20040520)
 gcc (GCC) 3.3.5 (propolice)
 gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-6)
@@ -13,6 +14,7 @@ gcc-3.3 (GCC) 3.3.5 (Debian 1:3.3.5-13)
 gcc (GCC) 3.4.2 [FreeBSD] 20040728
 gcc (GCC) 3.4.3 (csl-sol210-3_4-branch+sol_rpath)
 gcc-3.4 (GCC) 3.4.4 20041218 (prerelease) (Debian 3.4.3-7)
+powerpc-apple-darwin8-gcc-3.5.0 (GCC) 3.5.0 20041026 (Apple Computer, Inc. build 4059)
 powerpc-apple-darwin8-gcc-4.0.0 (GCC) 4.0.0 20041026 (Apple Computer, Inc. build 4061)
 gcc-4.0 (GCC) 4.0.2 20050821 (prerelease) (Debian 4.0.1-6)
 gcc-4.0.1 (GCC) 3.3.6 (Gentoo 3.3.6, ssp-3.3.6-1.0, pie-8.7.8)