Tightening of RPC code

httpserver.js

@@ -0,0 +1,7 @@
+var http = require('http');
+var server = http.createServer(function (request, response) {
+  response.writeHead(200, {"Content-Type": "text/plain"});
+  response.end("welcome to the machine!\n");
+});
+server.listen(8080);
+console.log("Server running!");

make-linux.mk

@@ -95,18 +95,24 @@ one:	$(OBJS) service/OneService.o one.o osdep/LinuxEthernetTap.o
 	ln -sf zerotier-one zerotier-idtool
 	ln -sf zerotier-one zerotier-cli
 
-netcon: $(OBJS)
+netcon: rpc_lib $(OBJS)
 	rm -f *.o
 	# Need to selectively rebuild one.cpp and OneService.cpp with ZT_SERVICE_NETCON and ZT_ONE_NO_ROOT_CHECK defined, and also NetconEthernetTap
-	$(CXX) $(CXXFLAGS) $(LDFLAGS) -DZT_SERVICE_NETCON -DZT_ONE_NO_ROOT_CHECK -Iext/lwip/src/include -Iext/lwip/src/include/ipv4 -Iext/lwip/src/include/ipv6 -o zerotier-netcon-service $(OBJS) service/OneService.cpp netcon/NetconEthernetTap.cpp one.cpp $(LDLIBS) -ldl
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -DZT_SERVICE_NETCON -DZT_ONE_NO_ROOT_CHECK -Iext/lwip/src/include -Iext/lwip/src/include/ipv4 -Iext/lwip/src/include/ipv6 -o zerotier-netcon-service $(OBJS) service/OneService.cpp netcon/NetconEthernetTap.cpp one.cpp $(LDLIBS) -ldl -Lnetcon/ -lrpc
 	# Build netcon/liblwip.so which must be placed in ZT home for zerotier-netcon-service to work
 	cd netcon ; make -f make-liblwip.mk
 	# Use gcc not clang to build standalone intercept library since gcc is typically used for libc and we want to ensure maximal ABI compatibility
-	cd netcon ; gcc -g -O2 -Wall -std=c99 -fPIC -DVERBOSE -DDEBUG_RPC -DCHECKS -D_GNU_SOURCE -DNETCON_INTERCEPT -I. -nostdlib -shared -o libzerotierintercept.so Intercept.c -ldl
+	cd netcon ; gcc -Wl,--whole-archive -g -O2 -Wall -std=c99 -fPIC -DVERBOSE -D_GNU_SOURCE -DNETCON_INTERCEPT -I. -nostdlib -shared librpc.a -o libzerotierintercept.so Intercept.c -ldl
 	cp netcon/libzerotierintercept.so libzerotierintercept.so
 	ln -sf zerotier-netcon-service zerotier-cli
 	ln -sf zerotier-netcon-service zerotier-idtool
 
+
+rpc_lib:
+	g++ -c -fPIC -lpthread netcon/rpc.c -DVERBOSE -o netcon/rpc.o
+	ar -rv netcon/librpc.a netcon/rpc.o
+
+
 selftest:	$(OBJS) selftest.o
 	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-selftest selftest.o $(OBJS) $(LDLIBS)
 	$(STRIP) zerotier-selftest
@@ -115,7 +121,7 @@ installer: one FORCE
 	./ext/installfiles/linux/buildinstaller.sh
 
 clean: FORCE
-	rm -rf *.so *.o node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest zerotier-netcon-service build-* ZeroTierOneInstaller-* *.deb *.rpm
+	rm -rf *.so *.o netcon/*.a node/*.o controller/*.o osdep/*.o service/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o $(OBJS) zerotier-one zerotier-idtool zerotier-cli zerotier-selftest zerotier-netcon-service build-* ZeroTierOneInstaller-* *.deb *.rpm
 	# Remove files from all the funny places we put them for tests
 	find netcon -type f \( -name '*.o' -o -name '*.so' -o -name '*.1.0' -o -name 'zerotier-one' -o -name 'zerotier-cli' -o -name 'zerotier-netcon-service' \) -delete
 	find netcon/docker-test -name "zerotier-intercept" -type f -delete

netcon/Intercept.c

@@ -48,18 +48,16 @@
 #include <sys/poll.h>
 #include <sys/un.h>
 #include <arpa/inet.h>
+#include <sys/resource.h>
+#include <linux/net.h> /* for NPROTO */
 
-#include "Intercept.h"
+#define SOCK_MAX (SOCK_PACKET + 1)
+#define SOCK_TYPE_MASK 0xf
 
+#include "Intercept.h"
+#include "rpc.h"
 #include "common.inc.c"
 
-#ifdef CHECKS
-  #include <sys/resource.h>
-  #include <linux/net.h> /* for NPROTO */
-  #define SOCK_MAX (SOCK_PACKET + 1)
-  #define SOCK_TYPE_MASK 0xf
-#endif
-
 /* Global Declarations */
 static int (*realconnect)(CONNECT_SIG);
 static int (*realbind)(BIND_SIG);
@@ -97,227 +95,64 @@ static int init_service_connection();
 static void load_symbols(void);
 static void set_up_intercept();
 
-#define SERVICE_CONNECT_ATTEMPTS  30
-#define RPC_FD                    1023
-
-static pthread_mutex_t lock;
-static ssize_t sock_fd_read(int sock, void *buf, ssize_t bufsize, int *fd);
-
-void handle_error(char *name, char *info, int err)
-{
-#ifdef ERRORS_ARE_FATAL
-  if(err < 0) {
-    dwr(MSG_DEBUG,"handle_error(%s)=%d: FATAL: %s\n", name, err, info);
-    exit(-1);
-  }
-#endif
-#ifdef VERBOSE
-  dwr(MSG_DEBUG,"%s()=%d\n", name, err);
-#endif
-}
-
 /*------------------------------------------------------------------------------
-------------------- Intercept<--->Service Comm mechanisms-----------------------
+------------------- Intercept<--->Service Comm mechanisms ----------------------
 ------------------------------------------------------------------------------*/
 
-static int is_initialized = 0;
-static int fdret_sock; /* used for fd-transfers */
-static int newfd; /* used for "this_end" socket */
+static int rpcfd = -1; /* used for fd-transfers */
 static int thispid = -1;
 static int instance_count = 0;
 
-/*
- * Check for forking
- */
-static void checkpid()
-{
-  /* Do noting if not configured (sanity check -- should never get here in this case) */
-  if (!getenv("ZT_NC_NETWORK"))
-    return;
-
-  if (thispid != getpid()) {
-    dwr(MSG_DEBUG, "checkpid(): clone/fork detected. Re-initializing this instance.\n");
-    set_up_intercept();
-    fdret_sock = init_service_connection();
-    thispid = getpid();
-  }
-}
-
-
-/*
- * Reads a return value from the service and sets errno (if applicable)
- */
-static int get_retval()
-{
-  dwr(MSG_DEBUG,"get_retval()\n");
-  if(fdret_sock >= 0) {
-    int retval;
-    int sz = sizeof(char) + sizeof(retval) + sizeof(errno);
-    char retbuf[BUF_SZ];
-    memset(&retbuf, '\0', sz);
-    int n_read = read(fdret_sock, &retbuf, sz);
-    if(n_read > 0) {
-      memcpy(&retval, &retbuf[1], sizeof(retval));
-      memcpy(&errno, &retbuf[1+sizeof(retval)], sizeof(errno));
-      dwr(MSG_DEBUG, "get_retval(): ret = %d\n", retval);
-      return retval;
-    }
-  }
-  dwr(MSG_DEBUG,"unable to read return value\n");
-  return -1;
-}
-
-/* Reads a new file descriptor from the service */
-static int get_new_fd(int oversock)
-{
-  char buf[BUF_SZ];
-  int newfd;
-  ssize_t size = sock_fd_read(oversock, buf, sizeof(buf), &newfd);
-  if(size > 0){
-    dwr(MSG_DEBUG, "get_new_fd(): RX: fd = (%d) over (%d)\n", newfd, oversock);
-    return newfd;
-  }
-  dwr(MSG_ERROR, "get_new_fd(): ERROR: unable to read fd over (%d)\n", oversock);
-  return -1;
-}
-
-#ifdef VERBOSE
-  static unsigned long rpc_count = 0;
-#endif
-
-/* Sends an RPC command to the service */
-static int send_cmd(int rpc_fd, char *cmd)
-{
-  pthread_mutex_lock(&lock);
-  char metabuf[BUF_SZ]; // portion of buffer which contains RPC metadata for debugging
-#ifdef VERBOSE
-  /*
-  #define IDX_PID       0
-  #define IDX_TID       sizeof(pid_t)
-  #define IDX_COUNT     IDX_TID + sizeof(pid_t)
-  #define IDX_TIME      IDX_COUNT + sizeof(int)
-  #define IDX_CMD       IDX_TIME + 20 // 20 being the length of the timestamp string
-  #define IDX_PAYLOAD   IDX_TIME + sizeof(char)
-  */
-  /* [pid_t] [pid_t] [rpc_count] [int] [...] */
-  memset(metabuf, '\0', BUF_SZ);
-  pid_t pid = syscall(SYS_getpid);
-  pid_t tid = syscall(SYS_gettid);
-  rpc_count++;
-  char timestring[20];
-  time_t timestamp;
-  timestamp = time(NULL);
-  strftime(timestring, sizeof(timestring), "%H:%M:%S", localtime(&timestamp));
-  memcpy(&metabuf[IDX_PID],     &pid,         sizeof(pid_t)      ); /* pid       */
-  memcpy(&metabuf[IDX_TID],     &tid,         sizeof(pid_t)      ); /* tid       */
-  memcpy(&metabuf[IDX_COUNT],   &rpc_count,   sizeof(rpc_count)  ); /* rpc_count */
-  memcpy(&metabuf[IDX_TIME],    &timestring,   20                ); /* timestamp */
-#endif
-  /* Combine command flag+payload with RPC metadata */
-  memcpy(&metabuf[IDX_PAYLOAD], cmd, PAYLOAD_SZ);
-  int n_write = write(rpc_fd, &metabuf, BUF_SZ);
-  if(n_write < 0){
-    dwr(MSG_DEBUG,"Error writing command to service (CMD = %d)\n", cmd[0]);
-    errno = 0;
-  }
-  int ret = ERR_OK;
-
-  if(n_write > 0) {
-    if(cmd[0]==RPC_SOCKET) {
-    	ret = get_new_fd(fdret_sock);
-    }
-    if(cmd[0]==RPC_MAP_REQ
-      || cmd[0]==RPC_CONNECT
-      || cmd[0]==RPC_BIND
-      || cmd[0]==RPC_LISTEN
-      || cmd[0]==RPC_MAP) {
-    	ret = get_retval();
-    }
-    if(cmd[0]==RPC_GETSOCKNAME) {
-      ret = n_write;
-    }
-  }
-  else {
-    ret = -1;
-  }
-  pthread_mutex_unlock(&lock);
-  return ret;
+static int connected_to_service() {
+  return rpcfd == -1 ? 0 : 1;
 }
 
-
-
 /* Check whether the socket is mapped to the service or not. We
 need to know if this is a regular AF_LOCAL socket or an end of a socketpair
 that the service uses. We don't want to keep state in the intercept, so
 we simply ask the service via an RPC */
 static int is_mapped_to_service(int sockfd)
 {
+  if(rpcfd < 0)
+    return 0; /* no connection obviously implies no mapping */
   dwr(MSG_DEBUG,"is_mapped_to_service()\n");
-  char cmd[BUF_SZ];
-  memset(cmd, '\0', BUF_SZ);
-  cmd[0] = RPC_MAP_REQ;
-  memcpy(&cmd[1], &sockfd, sizeof(sockfd));
-  return send_cmd(fdret_sock, cmd);
+  return rpc_send_command(RPC_MAP_REQ, rpcfd, &sockfd, sizeof(sockfd));
 }
 
-/*------------------------------------------------------------------------------
-----------  Unix-domain socket lazy initializer (for fd-transfers)--------------
-------------------------------------------------------------------------------*/
-
 /* Sets up the connection pipes and sockets to the service */
 static int init_service_connection()
 {
-  struct sockaddr_un addr;
-  int tfd = -1, attempts = 0, conn_err = -1;
   const char *network_id;
-  char af_sock_name[1024];
-
+  char rpcname[1024];
   network_id = getenv("ZT_NC_NETWORK");
-  if (!network_id)
-    return -1;
-  strncpy(af_sock_name,network_id,sizeof(af_sock_name));
-  instance_count++;
-
-  dwr(MSG_DEBUG,"init_service_connection()\n");
-
-  memset(&addr, 0, sizeof(addr));
-  addr.sun_family = AF_UNIX;
-  strncpy(addr.sun_path, af_sock_name, sizeof(addr.sun_path)-1);
-  if((tfd = realsocket(AF_UNIX, SOCK_STREAM, 0)) == -1)
-    return -1;
-
-  while(conn_err < 0 && attempts < SERVICE_CONNECT_ATTEMPTS) {
-    conn_err = realconnect(tfd, (struct sockaddr*)&addr, sizeof(addr));
-    if(conn_err < 0) {
-      dwr(MSG_DEBUG,"re-attempting connection in %ds\n", 1+attempts);
-      sleep(1);
-    }
-    else {
-      dwr(MSG_DEBUG,"AF_UNIX connection established: %d\n", tfd);
-      is_initialized = 1;
-      int newtfd = realdup2(tfd, RPC_FD-instance_count);
-      dwr(MSG_DEBUG,"dup'd to rpc_fd = %d\n", newtfd);
-      close(tfd);
-      return newtfd;
-    }
-    attempts++;
+  /* Do noting if not configured (sanity check -- should never get here in this case) */
+  if (!network_id){
+    fprintf(stderr, "init_service_connection(): ZT_NC_NETWORK not set.\n");
+    exit(0);
   }
-  return -1;
+  if((rpcfd < 0 && instance_count==0) || thispid != getpid())
+    rpc_mutex_init();
+
+  strncpy(rpcname,network_id,sizeof(rpcname));
+  instance_count++;
+  rpcfd = rpc_join(rpcname);
+  fprintf(stderr, "rpc_join = %d\n", rpcfd);
+  return rpcfd;
 }
 
 /*------------------------------------------------------------------------------
-------------------------  ctors and dtors (and friends)-------------------------
+------------------------ ctors and dtors (and friends) ------------------------
 ------------------------------------------------------------------------------*/
 
 static void my_dest(void) __attribute__ ((destructor));
 static void my_dest(void) {
   dwr(MSG_DEBUG,"closing connections to service...\n");
-  pthread_mutex_destroy(&lock);
+  rpc_mutex_destroy();
 }
 
 static void load_symbols(void)
 {
-  if(thispid == getpid()) {
+    if(thispid == getpid()) {
     dwr(MSG_DEBUG,"detected duplicate call to global constructor (pid=%d).\n", thispid);
   }
   thispid = getpid();
@@ -350,12 +185,6 @@ static void set_up_intercept()
     return;
   /* Hook/intercept Posix net API symbols */
   load_symbols();
-  if(pthread_mutex_init(&lock, NULL) != 0) {
-    dwr(MSG_ERROR, "error while initializing service call mutex\n");
-  }
-  if(pthread_mutex_init(&loglock, NULL) != 0) {
-    dwr(MSG_ERROR, "error while initializing log mutex mutex\n");
-  }
 }
 
 /*------------------------------------------------------------------------------
@@ -370,11 +199,7 @@ int setsockopt(SETSOCKOPT_SIG)
     return -1;
   }
   dwr(MSG_DEBUG,"setsockopt(%d)\n", socket);
-  /*
-  if(is_mapped_to_service(socket) < 0) { // First, check if the service manages this
-    return realsetsockopt(socket, level, option_name, option_value, option_len);
-  }
-  */
+  
   /* return(realsetsockopt(socket, level, option_name, option_value, option_len)); */
   if(level == SOL_IPV6 && option_name == IPV6_V6ONLY)
     return 0;
@@ -408,24 +233,16 @@ int getsockopt(GETSOCKOPT_SIG)
   if(is_mapped_to_service(sockfd) <= 0) { // First, check if the service manages this
     return realgetsockopt(sockfd, level, optname, optval, optlen);
   }
-  //return 0;
-
 
   //int err = realgetsockopt(sockfd, level, optname, optval, optlen);
   /* TODO: this condition will need a little more intelligence later on
    -- we will need to know if this fd is a local we are spoofing, or a true local */
 
-  if(optname == SO_TYPE)
-  {
+  if(optname == SO_TYPE) {
     int* val = (int*)optval;
     *val = 2;
     optval = (void*)val;
   }
-  /*
-  if(err < 0){
-    perror("getsockopt():\n");
-  }
-  */
   return 0;
 }
 
@@ -437,31 +254,26 @@ int getsockopt(GETSOCKOPT_SIG)
 /* int socket_family, int socket_type, int protocol
    socket() intercept function */
 int socket(SOCKET_SIG)
-{
-  if(realsocket == NULL){
-    dwr(MSG_ERROR, "socket(): SYMBOL NOT FOUND.\n");
-    return -1;
-  }
+{  
+  if(realsocket == NULL)
+    set_up_intercept();
+  
   dwr(MSG_DEBUG,"socket():\n");
-  int err;
-#ifdef CHECKS
+  int newfd = -1;
   /* Check that type makes sense */
   int flags = socket_type & ~SOCK_TYPE_MASK;
   if (flags & ~(SOCK_CLOEXEC | SOCK_NONBLOCK)) {
       errno = EINVAL;
-      handle_error("socket", "", -1);
       return -1;
   }
   socket_type &= SOCK_TYPE_MASK;
   /* Check protocol is in range */
   if (socket_family < 0 || socket_family >= NPROTO){
     errno = EAFNOSUPPORT;
-    handle_error("socket", "", -1);
     return -1;
   }
   if (socket_type < 0 || socket_type >= SOCK_MAX) {
     errno = EINVAL;
-    handle_error("socket", "", -1);
     return -1;
   }
   /* Check that we haven't hit the soft-limit file descriptors allowed */
@@ -474,59 +286,43 @@ int socket(SOCKET_SIG)
   }
   */
   /* TODO: detect ENFILE condition */
-#endif
-  char cmd[BUF_SZ];
-  fdret_sock = !is_initialized ? init_service_connection() : fdret_sock;
-  if(fdret_sock < 0) {
-    dwr(MSG_DEBUG,"BAD service connection. exiting.\n");
-    handle_error("socket", "", -1);
-    exit(-1);
-  }
+
   if(socket_family == AF_LOCAL
     || socket_family == AF_NETLINK
     || socket_family == AF_UNIX) {
       int err = realsocket(socket_family, socket_type, protocol);
-      dwr(MSG_DEBUG,"realsocket, err = %d\n", err);
-      handle_error("socket", "", err);
+      dwr(MSG_DEBUG,"realsocket() = %d\n", err);
       return err;
   }
+
+  rpcfd = !connected_to_service() ? init_service_connection() : rpcfd;
+  if(rpcfd < 0) {
+    dwr(MSG_DEBUG,"BAD service connection. exiting.\n");
+    exit(-1);
+  }
+
   /* Assemble and send RPC */
   struct socket_st rpc_st;
   rpc_st.socket_family = socket_family;
   rpc_st.socket_type = socket_type;
   rpc_st.protocol = protocol;
   rpc_st.__tid = syscall(SYS_gettid);
-  memset(cmd, '\0', BUF_SZ);
-  cmd[0] = RPC_SOCKET;
-  memcpy(&cmd[1], &rpc_st, sizeof(struct socket_st));
 
-  /* send command and get new fd */
-  newfd = send_cmd(fdret_sock, cmd);
+  newfd = rpc_send_command(RPC_SOCKET, rpcfd, &rpc_st, sizeof(struct socket_st));
   if(newfd > 0)
   {
-    dwr(MSG_DEBUG,"sending fd = %d to Service over (%d)\n", newfd, fdret_sock);
+    dwr(MSG_DEBUG,"sending fd = %d to Service over (%d)\n", newfd, rpcfd);
     /* send our local-fd number back to service so
      it can complete its mapping table entry */
-    memset(cmd, '\0', BUF_SZ);
-    cmd[0] = RPC_MAP;
-    memcpy(&cmd[1], &newfd, sizeof(newfd));
   	/* send fd mapping and get confirmation */
-  	err = send_cmd(fdret_sock, cmd);
-
-	if(err > -1) {
-	  errno = ERR_OK;
-	  dwr(MSG_DEBUG, "RXd fd confirmation. Mapped!\n");
+  	if(rpc_send_command(RPC_MAP, rpcfd, &newfd, sizeof(newfd)) > -1) {
+  	  errno = ERR_OK;
+  	  dwr(MSG_DEBUG, "RXd fd confirmation. Mapped!\n");
       return newfd; /* Mapping complete, everything is OK */
     }
-    else{
-    	dwr(MSG_DEBUG,"Error, service sent bad fd.\n");
-    	return err; /* Mapping failed */
-    }
-  }
-  else {
-    dwr(MSG_DEBUG,"Error while receiving new fd.\n");
-    return newfd;
   }
+  dwr(MSG_DEBUG,"Error while receiving new fd.\n");
+  return -1;
 }
 
 /*------------------------------------------------------------------------------
@@ -546,11 +342,9 @@ int connect(CONNECT_SIG)
   struct sockaddr_in *connaddr;
   connaddr = (struct sockaddr_in *) __addr;
 
-#ifdef CHECKS
   /* Check that this is a valid fd */
   if(fcntl(__fd, F_GETFD) < 0) {
     errno = EBADF;
-    handle_error("connect", "EBADF", -1);
     return -1;
   }
   /* Check that it is a socket */
@@ -558,26 +352,18 @@ int connect(CONNECT_SIG)
   socklen_t sock_type_len = sizeof(sock_type);
   if(getsockopt(__fd, SOL_SOCKET, SO_TYPE, (void *) &sock_type, &sock_type_len) < 0) {
     errno = ENOTSOCK;
-    handle_error("connect", "ENOTSOCK", -1);
     return -1;
   }
   /* Check family */
   if (connaddr->sin_family < 0 || connaddr->sin_family >= NPROTO){
     errno = EAFNOSUPPORT;
-    handle_error("connect", "EAFNOSUPPORT", -1);
     return -1;
   }
   /* FIXME: Check that address is in user space, return EFAULT ? */
-#endif
 
   /* make sure we don't touch any standard outputs */
-  if(__fd == STDIN_FILENO || __fd == STDOUT_FILENO || __fd == STDERR_FILENO){
-    if (realconnect == NULL) {
-      handle_error("connect", "Unresolved symbol [connect]", -1);
-      exit(-1);
-    }
+  if(__fd == STDIN_FILENO || __fd == STDOUT_FILENO || __fd == STDERR_FILENO)
     return(realconnect(__fd, __addr, __len));
-  }
 
   if(__addr != NULL && (connaddr->sin_family == AF_LOCAL
     || connaddr->sin_family == PF_NETLINK
@@ -585,21 +371,17 @@ int connect(CONNECT_SIG)
     || connaddr->sin_family == AF_UNIX)) {
     int err = realconnect(__fd, __addr, __len);
     perror("connect():");
-    /* handle_error("connect", "Cannot connect to local socket", err); */
     return err;
   }
 
   /* Assemble and send RPC */
-  char cmd[BUF_SZ];
-  memset(cmd, '\0', BUF_SZ);
   struct connect_st rpc_st;
   rpc_st.__tid = syscall(SYS_gettid);
   rpc_st.__fd = __fd;
   memcpy(&rpc_st.__addr, __addr, sizeof(struct sockaddr_storage));
   memcpy(&rpc_st.__len, &__len, sizeof(socklen_t));
-  cmd[0] = RPC_CONNECT;
-  memcpy(&cmd[1], &rpc_st, sizeof(struct connect_st));
-  return send_cmd(fdret_sock, cmd);
+
+  return rpc_send_command(RPC_CONNECT, rpcfd, &rpc_st, sizeof(struct connect_st));
 }
 
 /*------------------------------------------------------------------------------
@@ -615,12 +397,9 @@ int bind(BIND_SIG)
     return -1;
   }
   dwr(MSG_DEBUG,"bind(%d):\n", sockfd);
-  /* print_addr(addr); */
-#ifdef CHECKS
   /* Check that this is a valid fd */
   if(fcntl(sockfd, F_GETFD) < 0) {
     errno = EBADF;
-    handle_error("bind", "EBADF", -1);
     return -1;
   }
   /* Check that it is a socket */
@@ -628,10 +407,8 @@ int bind(BIND_SIG)
   socklen_t opt_len;
   if(getsockopt(sockfd, SOL_SOCKET, SO_TYPE, (void *) &opt, &opt_len) < 0) {
     errno = ENOTSOCK;
-    handle_error("bind", "ENOTSOCK", -1);
     return -1;
   }
-#endif
 
   /* make sure we don't touch any standard outputs */
   if(sockfd == STDIN_FILENO || sockfd == STDOUT_FILENO || sockfd == STDERR_FILENO)
@@ -658,17 +435,14 @@ int bind(BIND_SIG)
   d[3] = (ip >> 24) & 0xFF;
   dwr(MSG_DEBUG, "bind(): %d.%d.%d.%d: %d\n", d[0],d[1],d[2],d[3], ntohs(port));
 
-
   /* Assemble and send RPC */
-  char cmd[BUF_SZ];
   struct bind_st rpc_st;
   rpc_st.sockfd = sockfd;
   rpc_st.__tid = syscall(SYS_gettid);
   memcpy(&rpc_st.addr, addr, sizeof(struct sockaddr_storage));
   memcpy(&rpc_st.addrlen, &addrlen, sizeof(socklen_t));
-  cmd[0]=RPC_BIND;
-  memcpy(&cmd[1], &rpc_st, sizeof(struct bind_st));
-  return send_cmd(fdret_sock, cmd);
+
+  return rpc_send_command(RPC_BIND, rpcfd, &rpc_st, sizeof(struct bind_st));
 }
 
 /*------------------------------------------------------------------------------
@@ -689,7 +463,6 @@ int accept4(ACCEPT4_SIG)
   if ((flags & SOCK_NONBLOCK))
     fcntl(sockfd, F_SETFL, O_NONBLOCK);
   int newfd = accept(sockfd, addr, addrlen);
-  handle_error("accept4", "", newfd);
   return newfd;
 }
 
@@ -706,13 +479,11 @@ int accept(ACCEPT_SIG)
     return -1;
   }
   dwr(MSG_DEBUG,"accept(%d):\n", sockfd);
-#ifdef CHECKS
   /* Check that this is a valid fd */
   if(fcntl(sockfd, F_GETFD) < 0) {
     return -1;
     errno = EBADF;
     dwr(MSG_DEBUG,"EBADF\n");
-    handle_error("accept", "EBADF", -1);
     return -1;
   }
   /* Check that it is a socket */
@@ -721,14 +492,12 @@ int accept(ACCEPT_SIG)
   if(getsockopt(sockfd, SOL_SOCKET, SO_TYPE, (void *) &opt, &opt_len) < 0) {
     errno = ENOTSOCK;
     dwr(MSG_DEBUG,"ENOTSOCK\n");
-    handle_error("accept", "ENOTSOCK", -1);
     return -1;
   }
   /* Check that this socket supports accept() */
   if(!(opt && (SOCK_STREAM | SOCK_SEQPACKET))) {
     errno = EOPNOTSUPP;
     dwr(MSG_DEBUG,"EOPNOTSUPP\n");
-    handle_error("accept", "EOPNOTSUPP", -1);
     return -1;
   }
   /* Check that we haven't hit the soft-limit file descriptors allowed */
@@ -737,17 +506,14 @@ int accept(ACCEPT_SIG)
   if(sockfd >= rl.rlim_cur){
     errno = EMFILE;
     dwr(MSG_DEBUG,"EMFILE\n");
-    handle_error("accept", "EMFILE", -1);
     return -1;
   }
   /* Check address length */
   if(addrlen < 0) {
     errno = EINVAL;
     dwr(MSG_DEBUG,"EINVAL\n");
-    handle_error("accept", "EINVAL", -1);
     return -1;
   }
-#endif
 
   /* redirect calls for standard I/O descriptors to kernel */
   if(sockfd == STDIN_FILENO || sockfd == STDOUT_FILENO || sockfd == STDERR_FILENO){
@@ -759,8 +525,6 @@ int accept(ACCEPT_SIG)
     addr->sa_family = AF_INET;
     /* TODO: also get address info */
 
-  char cmd[BUF_SZ];
-
   /* The following line is required for libuv/nodejs to accept connections properly,
   however, this has the side effect of causing certain webservers to max out the CPU
   in an accept loop */
@@ -769,27 +533,16 @@ int accept(ACCEPT_SIG)
 
   if(new_conn_socket > 0)
   {
-    dwr(MSG_DEBUG, "accept(): RX: fd = (%d) over (%d)\n", new_conn_socket, fdret_sock);
+    dwr(MSG_DEBUG, "accept(): RX: fd = (%d) over (%d)\n", new_conn_socket, rpcfd);
     /* Send our local-fd number back to service so it can complete its mapping table */
-    memset(cmd, '\0', BUF_SZ);
-    cmd[0] = RPC_MAP;
-    memcpy(&cmd[1], &new_conn_socket, sizeof(new_conn_socket));
-
     dwr(MSG_DEBUG, "accept(): sending perceived fd (%d) to service.\n", new_conn_socket);
-    send_cmd(fdret_sock, cmd);
-    /*
-    if(n_write < 0) {
-      errno = ECONNABORTED;
-      handle_error("accept", "ECONNABORTED - Error sending perceived FD to service", -1);
-      return -1;
-    }
-    */
-    errno = ERR_OK;
+    rpc_send_command(RPC_MAP, rpcfd, &new_conn_socket, sizeof(new_conn_socket));
     dwr(MSG_DEBUG,"accept()=%d\n", new_conn_socket);
+    errno = ERR_OK;
     return new_conn_socket; /* OK */
   }
-  errno = EAGAIN; /* necessary? */
-  handle_error("accept", "EAGAIN - Error reading signal byte from service", -1);
+  dwr(MSG_DEBUG, "accept(): EAGAIN - Error reading signal byte from service");
+  errno = EAGAIN;
   return -EAGAIN;
 }
 
@@ -809,26 +562,21 @@ int listen(LISTEN_SIG)
   int sock_type;
   socklen_t sock_type_len = sizeof(sock_type);
 
-  #ifdef CHECKS
   /* Check that this is a valid fd */
   if(fcntl(sockfd, F_GETFD) < 0) {
     errno = EBADF;
-    handle_error("listen", "EBADF", -1);
     return -1;
   }
   /* Check that it is a socket */
   if(getsockopt(sockfd, SOL_SOCKET, SO_TYPE, (void *) &sock_type, &sock_type_len) < 0) {
     errno = ENOTSOCK;
-    handle_error("listen", "ENOTSOCK", -1);
     return -1;
   }
   /* Check that this socket supports accept() */
   if(!(sock_type && (SOCK_STREAM | SOCK_SEQPACKET))) {
     errno = EOPNOTSUPP;
-    handle_error("listen", "EOPNOTSUPP", -1);
     return -1;
   }
-  #endif
 
   /* make sure we don't touch any standard outputs */
   if(sockfd == STDIN_FILENO || sockfd == STDOUT_FILENO || sockfd == STDERR_FILENO)
@@ -842,15 +590,12 @@ int listen(LISTEN_SIG)
   }
 
   /* Assemble and send RPC */
-  char cmd[BUF_SZ];
-  memset(cmd, '\0', BUF_SZ);
   struct listen_st rpc_st;
   rpc_st.sockfd = sockfd;
   rpc_st.backlog = backlog;
   rpc_st.__tid = syscall(SYS_gettid);
-  cmd[0] = RPC_LISTEN;
-  memcpy(&cmd[1], &rpc_st, sizeof(struct listen_st));
-  return send_cmd(fdret_sock, cmd);
+
+  return rpc_send_command(RPC_LISTEN, rpcfd, &rpc_st, sizeof(struct listen_st));
 }
 
 /*------------------------------------------------------------------------------
@@ -866,7 +611,7 @@ int clone(CLONE_SIG)
   }
   dwr(MSG_DEBUG,"clone()\n");
   int err = realclone(fn, child_stack, flags, arg);
-  checkpid();
+  init_service_connection();
   return err;
 }
 
@@ -878,12 +623,9 @@ int clone(CLONE_SIG)
 int close(CLOSE_SIG)
 {
   dwr(MSG_DEBUG, "close(%d)\n", fd);
-  if(realclose == NULL){
-    checkpid(); // Add for nginx support, remove for apache support.
-    dwr(MSG_ERROR, "close(%d): SYMBOL NOT FOUND.\n", fd);
-    return -1;
-  }
-  if(fd == fdret_sock)
+  if(realclose == NULL)
+    init_service_connection();    
+  if(fd == rpcfd)
     return -1; /* TODO: Ignore request to shut down our rpc fd, this is *almost always* safe */
   if(fd != STDIN_FILENO && fd != STDOUT_FILENO && fd != STDERR_FILENO)
     return realclose(fd);
@@ -902,15 +644,12 @@ int dup2(DUP2_SIG)
     return -1;
   }
   dwr(MSG_DEBUG,"dup2(%d, %d)\n", oldfd, newfd);
-    if(oldfd == fdret_sock) {
+    if(oldfd == rpcfd) {
     dwr(MSG_DEBUG,"client application attempted to dup2 RPC socket (%d). This is not allowed.\n", oldfd);
     errno = EBADF;
     return -1;
   }
-  //if(oldfd != STDIN_FILENO && oldfd != STDOUT_FILENO && oldfd != STDERR_FILENO)
-  //  if(newfd != STDIN_FILENO && newfd != STDOUT_FILENO && newfd != STDERR_FILENO)
-      return realdup2(oldfd, newfd);
-  return -1;
+  return realdup2(oldfd, newfd);
 }
 
 /*------------------------------------------------------------------------------
@@ -925,14 +664,6 @@ int dup3(DUP3_SIG)
     return -1;
   }
   dwr(MSG_DEBUG,"dup3(%d, %d, %d)\n", oldfd, newfd, flags);
-#ifdef DEBUG
-  /* Only do this check if we want to debug the intercept, otherwise, dont mess with
-   the client application's logging methods */
-  if(newfd == STDIN_FILENO || newfd == STDOUT_FILENO || newfd == STDERR_FILENO)
-    return newfd; /* FIXME: This is to prevent httpd from dup'ing over our stderr
-                   and preventing us from debugging */
-  else
-#endif
   return realdup3(oldfd, newfd, flags);
 }
 
@@ -955,19 +686,16 @@ int getsockname(GETSOCKNAME_SIG)
    * and is an IPv4 address. */
 
   /* assemble and send command */
-  char cmd[BUF_SZ];
   struct getsockname_st rpc_st;
   rpc_st.sockfd = sockfd;
   memcpy(&rpc_st.addr, addr, *addrlen);
   memcpy(&rpc_st.addrlen, &addrlen, sizeof(socklen_t));
-  cmd[0] = RPC_GETSOCKNAME;
-  memcpy(&cmd[1], &rpc_st, sizeof(struct getsockname_st));
-  send_cmd(fdret_sock, cmd);
+  rpc_send_command(RPC_GETSOCKNAME, rpcfd, &rpc_st, sizeof(struct getsockname_st));
 
   /* read address info from service */
   char addrbuf[sizeof(struct sockaddr_storage)];
   memset(&addrbuf, 0, sizeof(struct sockaddr_storage));
-  read(fdret_sock, &addrbuf, sizeof(struct sockaddr_storage));
+  read(rpcfd, &addrbuf, sizeof(struct sockaddr_storage));
   struct sockaddr_storage sock_storage;
   memcpy(&sock_storage, addrbuf, sizeof(struct sockaddr_storage));
   *addrlen = sizeof(struct sockaddr_in);
@@ -981,10 +709,7 @@ int getsockname(GETSOCKNAME_SIG)
 ------------------------------------------------------------------------------*/
 
 long syscall(SYSCALL_SIG){
-  if(realsyscall == NULL){
-    dwr(MSG_ERROR, "syscall(): SYMBOL NOT FOUND.\n");
-    return -1;
-  }
+
   //dwr(MSG_DEBUG_EXTRA,"syscall(%u, ...):\n", number);
 
   va_list ap;
@@ -998,6 +723,9 @@ long syscall(SYSCALL_SIG){
   f=va_arg(ap, uintptr_t);
   va_end(ap);
 
+  if(realsyscall == NULL)
+    return -1;
+
 #if defined(__i386__)
   /* TODO: Implement for 32-bit systems: syscall(__NR_socketcall, 18, args);
   args[0] = (unsigned long) fd;
@@ -1013,10 +741,8 @@ long syscall(SYSCALL_SIG){
     int flags = d;
     int old_errno = errno;
     int err = accept4(sockfd, addr, addrlen, flags);
-
     errno = old_errno;
-    if(err == -EBADF)
-      err = -EAGAIN;
+    err = err == -EBADF ? -EAGAIN : err;
     return err;
   }
 #endif

netcon/Intercept.h

@@ -31,156 +31,16 @@
 
 #include <sys/socket.h>
 
-#define IDX_PID			0
-#define IDX_TID			sizeof(pid_t)
-#define IDX_COUNT		IDX_TID + sizeof(pid_t)
-#define IDX_TIME		IDX_COUNT + sizeof(int)
-#define IDX_PAYLOAD		IDX_TIME + 20 /* 20 being the length of the timestamp string */
-
-#define BUF_SZ          256
-#define PAYLOAD_SZ		223 /* BUF_SZ-IDX_PAYLOAD */
-
-#define ERR_OK          0
-
-/* Userland RPC codes */
-#define RPC_UNDEFINED			 	0
-#define RPC_CONNECT				 	1
-#define RPC_CONNECT_SOCKARG		 	2
-#define RPC_CLOSE				 	3
-#define RPC_READ				 	4
-#define RPC_WRITE				 	5
-#define RPC_BIND				 	6
-#define RPC_ACCEPT			 		7
-#define RPC_LISTEN			 		8
-#define RPC_SOCKET			 		9
-#define RPC_SHUTDOWN		 		10
-#define RPC_GETSOCKNAME				11
-
-/* Administration RPC codes */
-#define RPC_MAP						20	/* Give the service the value we "see" for the new buffer fd */
-#define RPC_MAP_REQ					21  /* A call to determine whether an fd is mapped to the service */
-#define RPC_RETVAL					22	/* not RPC per se, but something we should codify */
-#define RPC_KILL_INTERCEPT			23  /* Tells the service we need to shut down all connections */
-
-/* Connection statuses */
-#define UNSTARTED					0
-#define CONNECTING			 		1
-#define CONNECTED				 	2
-#define SENDING					 	3
-#define RECEIVING				 	4
-#define SENTV4REQ				 	5
-#define GOTV4REQ				 	6
-#define SENTV5METHOD			 	7
-#define GOTV5METHOD				 	8
-#define SENTV5AUTH			 		9
-#define GOTV5AUTH				 	10
-#define SENTV5CONNECT			 	11
-#define GOTV5CONNECT			 	12
-#define DONE					 	13
-#define FAILED				 		14
-
-/* Flags to indicate what events a
-   socket was select()ed for */
-#define READ (POLLIN|POLLRDNORM)
-#define WRITE (POLLOUT|POLLWRNORM|POLLWRBAND)
-#define EXCEPT (POLLRDBAND|POLLPRI)
-#define READWRITE (READ|WRITE)
-#define READWRITEEXCEPT (READ|WRITE|EXCEPT)
-
-
-/* for AF_UNIX sockets */
-#define MAX_PATH_NAME_SIZE 64
-
-/* bind */
-#define BIND_SIG int sockfd, const struct sockaddr *addr, socklen_t addrlen
-struct bind_st
-{
-	int sockfd;
-	struct sockaddr_storage addr;
-	socklen_t addrlen;
-	int __tid;
-};
-
-/* connect */
-#define CONNECT_SIG int __fd, const struct sockaddr * __addr, socklen_t __len
-struct connect_st
-{
-	int __fd;
-	struct sockaddr_storage __addr;
-	socklen_t __len;
-	int __tid;
-};
-
-/* close */
 #define CLOSE_SIG int fd
-struct close_st
-{
-	int fd;
-};
-
-/* read */
-#define DEFAULT_READ_BUFFER_SIZE	1024 * 63
-/* read buffer sizes (on test machine) min: 4096    default: 87380   max:6147872 */
 #define READ_SIG int __fd, void *__buf, size_t __nbytes
-struct read_st
-{
-	int fd;
-	size_t count;
-	unsigned char buf[DEFAULT_READ_BUFFER_SIZE];
-};
-
-/* write */
-#define DEFAULT_WRITE_BUFFER_SIZE	1024 * 63
-/* write buffer sizes (on test machine) min: 4096    default: 16384   max:4194304 */
+#define BIND_SIG int sockfd, const struct sockaddr *addr, socklen_t addrlen
+#define CONNECT_SIG int __fd, const struct sockaddr * __addr, socklen_t __len
 #define WRITE_SIG int __fd, const void *__buf, size_t __n
-struct write_st
-{
-	int fd;
-	size_t count;
-	char buf[DEFAULT_WRITE_BUFFER_SIZE];
-};
-
 #define LISTEN_SIG int sockfd, int backlog
-struct listen_st
-{
-	int sockfd;
-	int backlog;
-	int __tid;
-};
-
 #define SOCKET_SIG int socket_family, int socket_type, int protocol
-struct socket_st
-{
-  int socket_family;
-  int socket_type;
-  int protocol;
-	int __tid;
-};
-
 #define ACCEPT4_SIG int sockfd, struct sockaddr *addr, socklen_t *addrlen, int flags
 #define ACCEPT_SIG int sockfd, struct sockaddr *addr, socklen_t *addrlen
-struct accept_st
-{
-	int sockfd;
-	struct sockaddr_storage addr;
-	socklen_t addrlen;
-	int __tid;
-};
-
 #define SHUTDOWN_SIG int socket, int how
-struct shutdown_st
-{
-	int socket;
-	int how;
-};
-
-struct getsockname_st
-{
-	int sockfd;
-	struct sockaddr_storage addr;
-	socklen_t addrlen;
-};
-
 #define CONNECT_SOCKARG struct sockaddr *
 #define IOCTL_SIG int __fd, unsigned long int __request, ...
 #define FCNTL_SIG int __fd, int __cmd, ...

netcon/NetconEthernetTap.cpp

@@ -48,10 +48,12 @@
 #include "lwip/tcp.h"
 
 #include "common.inc.c"
+#include "rpc.h"
 
 #define APPLICATION_POLL_FREQ 			20
 #define ZT_LWIP_TCP_TIMER_INTERVAL 		5
 #define STATUS_TMR_INTERVAL				3000 // How often we check connection statuses
+#define DEFAULT_READ_BUFFER_SIZE		1024 * 63
 
 namespace ZeroTier {
 
@@ -128,9 +130,7 @@ public:
   PhySocket *rpcSock;
   PhySocket *dataSock;
   struct tcp_pcb *pcb;
-
   struct sockaddr_storage *addr;
-
   unsigned char buf[DEFAULT_READ_BUFFER_SIZE];
   int idx;
 };
@@ -926,6 +926,9 @@ void NetconEthernetTap::nc_err(void *arg, err_t err)
 	if(!l->conn)
 		dwr(MSG_ERROR, "nc_err(): Connection is NULL!\n");
 
+	if(l->conn->listening)
+		return;
+
 	switch(err)
 	{
 		case ERR_MEM:
@@ -1424,7 +1427,8 @@ void NetconEthernetTap::handle_connect(PhySocket *sock, void **uptr, struct conn
 	ip_addr_t conn_addr = convert_ip((struct sockaddr_in *)&connect_rpc->__addr);
 
 	if(conn != NULL) {
-		lwipstack->tcp_sent(conn->pcb, nc_sent);
+		if (!conn->listening)
+			lwipstack->tcp_sent(conn->pcb, nc_sent);
 		lwipstack->tcp_recv(conn->pcb, nc_recved);
 		lwipstack->tcp_err(conn->pcb, nc_err);
 		lwipstack->tcp_poll(conn->pcb, nc_poll, APPLICATION_POLL_FREQ);
@@ -1482,6 +1486,8 @@ void NetconEthernetTap::handle_connect(PhySocket *sock, void **uptr, struct conn
 		}
 		// Everything seems to be ok, but we don't have enough info to retval
 		conn->pending=true;
+		conn->listening=true;
+		send_return_value(conn, -1);
 	}
 	else {
 		dwr(MSG_ERROR, " handle_connect(): could not locate PCB based on their fd\n");
@@ -1515,9 +1521,9 @@ void NetconEthernetTap::handle_write(TcpConnection *conn)
 		if(!conn->listening)
 			lwipstack->_tcp_output(conn->pcb);
 
-		if(conn->dataSock) {
+		if(conn->dataSock && !conn->listening) {
 			int read_fd = _phy.getDescriptor(conn->dataSock);
-			if((r = read(read_fd, (&conn->buf)+conn->idx, sndbuf)) > 0) {
+			if((r = recvfrom(read_fd, (&conn->buf)+conn->idx, sndbuf, MSG_DONTWAIT, NULL, NULL)) > 0) {
 				conn->idx += r;
 				/* Writes data pulled from the client's socket buffer to LWIP. This merely sends the
 				 * data to LWIP to be enqueued and eventually sent to the network. */

netcon/RPC.c

@@ -0,0 +1,254 @@
+#include <stdio.h>
+#include <unistd.h>
+#include <sys/un.h>
+#include <pthread.h>
+#include <errno.h>
+#include <sys/syscall.h>
+
+#include <sys/socket.h>
+#include <strings.h>
+#include "rpc.h"
+
+#define RPC_FD 1023
+#define SERVICE_CONNECT_ATTEMPTS 30
+
+static int instance_count;
+static int rpc_count;
+static pthread_mutex_t lock;
+
+void rpc_mutex_init() {
+  if(pthread_mutex_init(&lock, NULL) != 0) {
+    fprintf(stderr, "error while initializing service call mutex\n");
+  }
+}
+
+void rpc_mutex_destroy() {
+  pthread_mutex_destroy(&lock);
+}
+
+/*
+ * Reads a return value from the service and sets errno (if applicable)
+ */
+int get_retval(int rpc_sock)
+{
+  if(rpc_sock >= 0) {
+    int retval;
+    int sz = sizeof(char) + sizeof(retval) + sizeof(errno);
+    char retbuf[BUF_SZ];
+    memset(&retbuf, 0, sz);
+    int n_read = read(rpc_sock, &retbuf, sz);
+    if(n_read > 0) {
+      memcpy(&retval, &retbuf[1], sizeof(retval));
+      memcpy(&errno, &retbuf[1+sizeof(retval)], sizeof(errno));
+      return retval;
+    }
+  }
+  return -1;
+}
+
+/* 
+ * Reads a new file descriptor from the service 
+ */
+int get_new_fd(int sock)
+{
+  char buf[BUF_SZ];
+  int newfd;
+  ssize_t size = sock_fd_read(sock, buf, sizeof(buf), &newfd);
+  if(size > 0){
+    fprintf(stderr, "get_new_fd(): RX: fd = (%d) over (%d)\n", newfd, sock);
+    return newfd;
+  }
+  fprintf(stderr, "get_new_fd(): ERROR: unable to read fd over (%d)\n", sock);
+  return -1;
+}
+
+int rpc_join(const char * sockname)
+{
+	struct sockaddr_un addr;
+	int conn_err = -1, attempts = 0;
+
+	memset(&addr, 0, sizeof(addr));
+	addr.sun_family = AF_UNIX;
+	strncpy(addr.sun_path, sockname, sizeof(addr.sun_path)-1);
+
+	int sock;
+	if((sock = socket(AF_UNIX, SOCK_STREAM, 0)) < 0){
+		fprintf(stderr, "Error while creating RPC socket\n");
+		return -1;
+	}
+	while((conn_err != 0) && (attempts < SERVICE_CONNECT_ATTEMPTS)){
+		if((conn_err = connect(sock, (struct sockaddr*)&addr, sizeof(addr))) != 0) {
+			fprintf(stderr, "Error while connecting to RPC socket. Re-attempting...\n");
+			sleep(1);
+		}
+		else {
+			int newfd = dup2(sock, RPC_FD-instance_count);
+			close(sock);
+			return newfd;
+		}
+		attempts++;
+	}
+	return -1;
+}
+
+/*
+ * Send a command to the service 
+ */
+int rpc_send_command(int cmd, int rpc_sock, void *data, int len)
+{
+  char cmdbuf[BUF_SZ];
+  cmdbuf[0] = cmd;
+  memcpy(&cmdbuf[1], data, len);
+
+  pthread_mutex_lock(&lock);
+  char metabuf[BUF_SZ]; // portion of buffer which contains RPC metadata for debugging
+#ifdef VERBOSE
+  /*
+  #define IDX_PID       0
+  #define IDX_TID       sizeof(pid_t)
+  #define IDX_COUNT     IDX_TID + sizeof(pid_t)
+  #define IDX_TIME      IDX_COUNT + sizeof(int)
+  #define IDX_CMD       IDX_TIME + 20 // 20 being the length of the timestamp string
+  #define IDX_PAYLOAD   IDX_TIME + sizeof(char)
+  */
+  /* [pid_t] [pid_t] [rpc_count] [int] [...] */
+  memset(metabuf, 0, BUF_SZ);
+  pid_t pid = syscall(SYS_getpid);
+  pid_t tid = syscall(SYS_gettid);
+  rpc_count++;
+  char timestring[20];
+  time_t timestamp;
+  timestamp = time(NULL);
+  strftime(timestring, sizeof(timestring), "%H:%M:%S", localtime(&timestamp));
+  memcpy(&metabuf[IDX_PID],     &pid,         sizeof(pid_t)      ); /* pid       */
+  memcpy(&metabuf[IDX_TID],     &tid,         sizeof(pid_t)      ); /* tid       */
+  memcpy(&metabuf[IDX_COUNT],   &rpc_count,   sizeof(rpc_count)  ); /* rpc_count */
+  memcpy(&metabuf[IDX_TIME],    &timestring,   20                ); /* timestamp */
+#endif
+  /* Combine command flag+payload with RPC metadata */
+  memcpy(&metabuf[IDX_PAYLOAD], cmdbuf, len);
+  int n_write = write(rpc_sock, &metabuf, BUF_SZ);
+  if(n_write < 0) {
+    fprintf(stderr, "Error writing command to service (CMD = %d)\n", cmdbuf[0]);
+    errno = 0;
+  }
+  
+  int ret = ERR_OK;
+  if(n_write > 0) {
+    if(cmdbuf[0]==RPC_SOCKET) {
+    	ret = get_new_fd(rpc_sock);
+    }
+    if(cmdbuf[0]==RPC_MAP_REQ
+      || cmdbuf[0]==RPC_CONNECT
+      || cmdbuf[0]==RPC_BIND
+      || cmdbuf[0]==RPC_LISTEN
+      || cmdbuf[0]==RPC_MAP) {
+    	ret = get_retval(rpc_sock);
+    }
+    if(cmdbuf[0]==RPC_GETSOCKNAME) {
+      ret = n_write;
+    }
+  }
+  else {
+    ret = -1;
+  }
+  pthread_mutex_unlock(&lock);
+  return ret;
+}
+
+
+/* 
+ * Send file descriptor 
+ */
+ssize_t sock_fd_write(int sock, int fd)
+{
+  ssize_t size;
+  struct msghdr msg;
+  struct iovec iov;
+  char buf = '\0';
+  int buflen = 1;
+
+  union {
+        struct cmsghdr  cmsghdr;
+    char control[CMSG_SPACE(sizeof (int))];
+  } cmsgu;
+  struct cmsghdr *cmsg;
+
+  iov.iov_base = &buf;
+  iov.iov_len = buflen;
+
+  msg.msg_name = NULL;
+  msg.msg_namelen = 0;
+  msg.msg_iov = &iov;
+  msg.msg_iovlen = 1;
+
+  if (fd != -1) {
+      msg.msg_control = cmsgu.control;
+      msg.msg_controllen = sizeof(cmsgu.control);
+      cmsg = CMSG_FIRSTHDR(&msg);
+      cmsg->cmsg_len = CMSG_LEN(sizeof (int));
+      cmsg->cmsg_level = SOL_SOCKET;
+      cmsg->cmsg_type = SCM_RIGHTS;
+      *((int *) CMSG_DATA(cmsg)) = fd;
+  } else {
+      msg.msg_control = NULL;
+      msg.msg_controllen = 0;
+  }
+
+  size = sendmsg(sock, &msg, 0);
+  if (size < 0)
+      perror ("sendmsg");
+  return size;
+}
+
+/* 
+ * Read a file descriptor 
+ */
+ssize_t sock_fd_read(int sock, void *buf, ssize_t bufsize, int *fd)
+{
+  ssize_t size;
+  if (fd) {
+    struct msghdr msg;
+    struct iovec iov;
+    union {
+      struct cmsghdr cmsghdr;
+      char control[CMSG_SPACE(sizeof (int))];
+    } cmsgu;
+    struct cmsghdr *cmsg;
+
+    iov.iov_base = buf;
+    iov.iov_len = bufsize;
+
+    msg.msg_name = NULL;
+    msg.msg_namelen = 0;
+    msg.msg_iov = &iov;
+    msg.msg_iovlen = 1;
+    msg.msg_control = cmsgu.control;
+    msg.msg_controllen = sizeof(cmsgu.control);
+    size = recvmsg (sock, &msg, 0);
+    if (size < 0) {
+      fprintf(stderr, "sock_fd_read(): recvmsg: Error\n");
+      return -1;
+    }
+    cmsg = CMSG_FIRSTHDR(&msg);
+    if (cmsg && cmsg->cmsg_len == CMSG_LEN(sizeof(int))) {
+      if (cmsg->cmsg_level != SOL_SOCKET) {
+        fprintf (stderr, "invalid cmsg_level %d\n",cmsg->cmsg_level);
+        return -1;
+      }
+      if (cmsg->cmsg_type != SCM_RIGHTS) {
+          fprintf (stderr, "invalid cmsg_type %d\n",cmsg->cmsg_type);
+          return -1;
+      }
+
+      *fd = *((int *) CMSG_DATA(cmsg));
+    } else *fd = -1;
+  } else {
+    size = read (sock, buf, bufsize);
+    if (size < 0) {
+      fprintf(stderr, "sock_fd_read(): read: Error\n");
+      return -1;
+    }
+  }
+  return size;
+}

netcon/RPC.h

@@ -0,0 +1,107 @@
+#ifndef __RPCLIB_H_
+#define __RPCLIB_H_
+
+#define IDX_PID			0
+#define IDX_TID			sizeof(pid_t)
+#define IDX_COUNT		IDX_TID + sizeof(pid_t)
+#define IDX_TIME		IDX_COUNT + sizeof(int)
+#define IDX_PAYLOAD		IDX_TIME + 20 /* 20 being the length of the timestamp string */
+
+#define BUF_SZ          256
+#define PAYLOAD_SZ		223 /* BUF_SZ-IDX_PAYLOAD */
+
+#define ERR_OK          0
+
+/* RPC codes */
+#define RPC_UNDEFINED			 	0
+#define RPC_CONNECT				 	1
+#define RPC_CONNECT_SOCKARG		 	2
+#define RPC_CLOSE				 	3
+#define RPC_READ				 	4
+#define RPC_WRITE				 	5
+#define RPC_BIND				 	6
+#define RPC_ACCEPT			 		7
+#define RPC_LISTEN			 		8
+#define RPC_SOCKET			 		9
+#define RPC_SHUTDOWN		 		10
+#define RPC_GETSOCKNAME				11
+
+/* Administration RPC codes */
+#define RPC_MAP						20	/* Give the service the value we "see" for the new buffer fd */
+#define RPC_MAP_REQ					21  /* A call to determine whether an fd is mapped to the service */
+#define RPC_RETVAL					22	/* not RPC per se, but something we should codify */
+#define RPC_KILL_INTERCEPT			23  /* Tells the service we need to shut down all connections */
+
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+void rpc_mutex_destroy();
+void rpc_mutex_init();
+
+int get_retval(int);
+int get_new_fd(int);
+
+int rpc_join(const char * sockname);
+int rpc_send_command(int cmd, int rpc_sock, void *data, int len);
+
+ssize_t sock_fd_write(int sock, int fd);
+ssize_t sock_fd_read(int sock, void *buf, ssize_t bufsize, int *fd);
+
+/* Structures used for sending commands via RPC mechanism */
+
+struct bind_st {
+	int sockfd;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	int __tid;
+};
+
+struct connect_st {
+	int __fd;
+	struct sockaddr_storage __addr;
+	socklen_t __len;
+	int __tid;
+};
+
+struct close_st {
+	int fd;
+};
+
+struct listen_st {
+	int sockfd;
+	int backlog;
+	int __tid;
+};
+
+struct socket_st {
+  int socket_family;
+  int socket_type;
+  int protocol;
+	int __tid;
+};
+
+struct accept_st {
+	int sockfd;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+	int __tid;
+};
+
+struct shutdown_st {
+	int socket;
+	int how;
+};
+
+struct getsockname_st {
+	int sockfd;
+	struct sockaddr_storage addr;
+	socklen_t addrlen;
+};
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

netcon/common.inc.c

@@ -37,9 +37,12 @@
 #include <netinet/in.h>
 #include <pthread.h>
 #include <fcntl.h>
+#include <sys/syscall.h>
 
+#ifndef _COMMON_H
+#define _COMMON_H  1
 
-#define DEBUG_LEVEL 4
+#define DEBUG_LEVEL     4
 
 #define MSG_WARNING     4
 #define MSG_ERROR       1 // Errors
@@ -49,8 +52,6 @@
 
 #ifdef NETCON_INTERCEPT
 
-static pthread_mutex_t loglock;
-
 void print_addr(struct sockaddr *addr)
 {
   char *s = NULL;
@@ -91,8 +92,8 @@ void print_addr(struct sockaddr *addr)
     time_t timestamp;
     timestamp = time(NULL);
     strftime(timestring, sizeof(timestring), "%H:%M:%S", localtime(&timestamp));
-    pid_t pid = getpid();
-    fprintf(stderr, "%s [pid=%7d] ", timestring, pid);
+    pid_t tid = syscall(SYS_gettid);
+    fprintf(stderr, "%s [tid=%7d] ", timestring, tid);
   #endif
     vfprintf(stderr, fmt, ap);
     fflush(stderr);
@@ -104,95 +105,4 @@ void print_addr(struct sockaddr *addr)
 }
 #endif
 
-static ssize_t sock_fd_write(int sock, int fd);
-static ssize_t sock_fd_read(int sock, void *buf, ssize_t bufsize, int *fd);
-
-static ssize_t sock_fd_write(int sock, int fd)
-{
-	ssize_t size;
-	struct msghdr msg;
-	struct iovec iov;
-	char buf = '\0';
-	int buflen = 1;
-
-	union {
-        struct cmsghdr  cmsghdr;
-		char control[CMSG_SPACE(sizeof (int))];
-	} cmsgu;
-	struct cmsghdr *cmsg;
-
-	iov.iov_base = &buf;
-	iov.iov_len = buflen;
-
-	msg.msg_name = NULL;
-	msg.msg_namelen = 0;
-	msg.msg_iov = &iov;
-	msg.msg_iovlen = 1;
-
-	if (fd != -1) {
-		  msg.msg_control = cmsgu.control;
-		  msg.msg_controllen = sizeof(cmsgu.control);
-		  cmsg = CMSG_FIRSTHDR(&msg);
-		  cmsg->cmsg_len = CMSG_LEN(sizeof (int));
-		  cmsg->cmsg_level = SOL_SOCKET;
-		  cmsg->cmsg_type = SCM_RIGHTS;
-		  *((int *) CMSG_DATA(cmsg)) = fd;
-	} else {
-		  msg.msg_control = NULL;
-		  msg.msg_controllen = 0;
-	}
-
-	size = sendmsg(sock, &msg, 0);
-	if (size < 0)
-		  perror ("sendmsg");
-	return size;
-}
-
-static ssize_t sock_fd_read(int sock, void *buf, ssize_t bufsize, int *fd)
-{
-  ssize_t size;
-  if (fd) {
-    struct msghdr msg;
-    struct iovec iov;
-    union {
-      struct cmsghdr cmsghdr;
-      char control[CMSG_SPACE(sizeof (int))];
-    } cmsgu;
-    struct cmsghdr *cmsg;
-
-    iov.iov_base = buf;
-    iov.iov_len = bufsize;
-
-    msg.msg_name = NULL;
-    msg.msg_namelen = 0;
-    msg.msg_iov = &iov;
-    msg.msg_iovlen = 1;
-    msg.msg_control = cmsgu.control;
-    msg.msg_controllen = sizeof(cmsgu.control);
-    size = recvmsg (sock, &msg, 0);
-    if (size < 0) {
-      dwr(MSG_DEBUG, "sock_fd_read(): recvmsg: Error\n");
-      return -1;
-    }
-    cmsg = CMSG_FIRSTHDR(&msg);
-    if (cmsg && cmsg->cmsg_len == CMSG_LEN(sizeof(int))) {
-      if (cmsg->cmsg_level != SOL_SOCKET) {
-        fprintf (stderr, "invalid cmsg_level %d\n",cmsg->cmsg_level);
-        return -1;
-      }
-      if (cmsg->cmsg_type != SCM_RIGHTS) {
-          fprintf (stderr, "invalid cmsg_type %d\n",cmsg->cmsg_type);
-          return -1;
-      }
-
-      *fd = *((int *) CMSG_DATA(cmsg));
-    } else *fd = -1;
-  } else {
-    size = read (sock, buf, bufsize);
-    if (size < 0) {
-      dwr(MSG_DEBUG, "sock_fd_read(): read: Error\n");
-      return -1;
-    }
-  }
-  return size;
-}
+#endif

netcon/docker-test/httpd/httpd-2.4.16-1.fc23.x86_64/netcon_entrypoint.sh

@@ -31,8 +31,6 @@ echo '--- Up and running at' $virtip4 ' on network: ' $nwid
 echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
-export ZT_NC_NWID=$dev
-
 # --- Test section ---
 # Generate large random file for transfer test, share md5sum for monitor container to check
 echo '*** Generating ' "$bigfile_size" ' file'
@@ -43,4 +41,7 @@ echo '*** Wrote MD5 sum to ' "$tx_md5sumfile"
 echo '*** Starting application...'
 sleep 0.5
 rm -rf /run/httpd/* /tmp/httpd*
-zerotier-intercept /usr/sbin/httpd -X
+
+export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
+export LD_PRELOAD=./libzerotierintercept.so
+/usr/sbin/httpd -X

netcon/docker-test/httpd/httpd-2.4.16-1.ub14.x86_64/netcon_entrypoint.sh

@@ -31,8 +31,6 @@ echo '--- Up and running at' $virtip4 ' on network: ' $nwid
 echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
-export ZT_NC_NWID=$dev
-
 # --- Test section ---
 # Generate large random file for transfer test, share md5sum for monitor container to check
 echo '*** Generating ' "$bigfile_size" ' file'
@@ -43,4 +41,7 @@ echo '*** Wrote MD5 sum to ' "$tx_md5sumfile"
 echo '*** Starting application...'
 sleep 0.5
 rm -rf /run/httpd/* /tmp/httpd*
-zerotier-intercept /usr/sbin/httpd -X
+
+export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
+export LD_PRELOAD=./libzerotierintercept.so
+/usr/sbin/httpd -X

netcon/docker-test/httpd/httpd-2.4.17-3.fc23.x86_64/netcon_entrypoint.sh

@@ -32,8 +32,6 @@ echo '*** Up and running at' $virtip4 ' on network: ' $nwid
 echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
-export ZT_NC_NWID=$dev
-
 # --- Test section ---
 # Generate large random file for transfer test, share md5sum for monitor container to check
 echo '*** Generating ' "$bigfile_size" ' file'
@@ -45,4 +43,7 @@ echo '*** Wrote MD5 sum to ' "$tx_md5sumfile"
 echo '*** Starting application...'
 sleep 0.5
 rm -rf /run/httpd/* /tmp/httpd*
-zerotier-intercept /usr/sbin/httpd -X
+
+export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
+export LD_PRELOAD=./libzerotierintercept.so
+/usr/sbin/httpd -X

netcon/docker-test/nginx/nginx-1.4.6-1.ub14.x86_64/netcon_entrypoint.sh

@@ -32,8 +32,6 @@ echo '*** Up and running at' $virtip4 ' on network: ' $nwid
 echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
-export ZT_NC_NWID=$dev
-
 # --- Test section ---
 cp -f nginx.conf_ /etc/nginx/nginx.conf
 nginx_html_path=/usr/share/nginx/html/
@@ -46,4 +44,7 @@ echo '*** Wrote MD5 sum to ' "$tx_md5sumfile"
 
 echo '*** Starting application...'
 sleep 0.5
-zerotier-intercept nginx
+
+export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
+export LD_PRELOAD=./libzerotierintercept.so
+nginx

netcon/docker-test/nginx/nginx-1.8.0-13.fc23.x86_64/netcon_entrypoint.sh

@@ -32,8 +32,6 @@ echo '*** Up and running at' $virtip4 ' on network: ' $nwid
 echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
-export ZT_NC_NWID=$dev
-
 # --- Test section ---
 cp -f nginx.conf_ /etc/nginx/nginx.conf
 nginx_html_path=/usr/share/nginx/html/
@@ -46,4 +44,7 @@ echo '*** Wrote MD5 sum to ' "$tx_md5sumfile"
 
 echo '*** Starting application...'
 sleep 0.5
-zerotier-intercept nginx
+
+export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
+export LD_PRELOAD=./libzerotierintercept.so
+nginx

netcon/docker-test/nginx/nginx-1.8.0-14.fc23.x86_64/netcon_entrypoint.sh

@@ -32,8 +32,6 @@ echo '*** Up and running at' $virtip4 ' on network: ' $nwid
 echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
-export ZT_NC_NWID=$dev
-
 # --- Test section ---
 cp -f nginx.conf_ /etc/nginx/nginx.conf
 nginx_html_path=/usr/share/nginx/html/
@@ -46,4 +44,7 @@ echo '*** Wrote MD5 sum to ' "$tx_md5sumfile"
 
 echo '*** Starting application...'
 sleep 0.5
-zerotier-intercept nginx
+
+export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
+export LD_PRELOAD=./libzerotierintercept.so
+nginx

netcon/docker-test/nodejs/nodejs-0.10.36-4.fc23/netcon_entrypoint.sh

@@ -30,8 +30,7 @@ echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
 export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
-echo $ZT_NC_NETWORK
-export LD_PRELOAD ./libzerotierintercept.so
+export LD_PRELOAD=./libzerotierintercept.so
 
 # --- Test section ---
 echo '*** Starting application...'

netcon/docker-test/redis/redis-3.0.4-1.fc23.x86_64/netcon_entrypoint.sh

@@ -29,9 +29,10 @@ echo '*** Up and running at' $virtip4 ' on network: ' $nwid
 echo '*** Writing address to ' "$address_file"
 echo $virtip4 > "$address_file"
 
-export ZT_NC_NWID=$dev
-
 # --- Test section ---
 echo '*** Starting application...'
 sleep 0.5
-zerotier-intercept /usr/bin/redis-server --port 6379
+
+export ZT_NC_NETWORK=/var/lib/zerotier-one/nc_"$dev"
+export LD_PRELOAD=./libzerotierintercept.so
+/usr/bin/redis-server --port 6379

netcon/httpserver.js

@@ -0,0 +1,7 @@
+var http = require('http');
+var server = http.createServer(function (request, response) {
+  response.writeHead(200, {"Content-Type": "text/plain"});
+  response.end("welcome to the machine!\n");
+});
+server.listen(8080);
+console.log("Server running!");

netcon/install-intercept.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+# This script is only needed for debugging purposes
 
 cp libzerotierintercept.so /lib/libzerotierintercept.so
 ln -sf /lib/libzerotierintercept.so /lib/libzerotierintercept