Merge branch 'rtnetlink' into edge

.gitignore

@@ -4,6 +4,7 @@
 /zerotier-cli
 /zerotier-selftest
 /zerotier
+/nltest
 
 # OS-created garbage files from various platforms
 .DS_Store

make-linux.mk

@@ -17,6 +17,10 @@ DESTDIR?=
 
 include objects.mk
 ONE_OBJS+=osdep/LinuxEthernetTap.o
+ONE_OBJS+=osdep/LinuxNetLink.o
+
+NLTEST_OBJS+=osdep/LinuxNetLink.o node/InetAddress.o node/Utils.o node/Salsa20.o
+NLTEST_OBJS+=nltest.o
 
 # Auto-detect miniupnpc and nat-pmp as well and use system libs if present,
 # otherwise build into binary as done on Mac and Windows.
@@ -58,8 +62,8 @@ ifeq ($(ZT_SANITIZE),1)
 	SANFLAGS+=-fsanitize=address -DASAN_OPTIONS=symbolize=1
 endif
 ifeq ($(ZT_DEBUG),1)
-	override CFLAGS+=-Wall -Wno-deprecated -Werror -g -pthread $(INCLUDES) $(DEFS)
-	override CXXFLAGS+=-Wall -Wno-deprecated -Werror -g -std=c++11 -pthread $(INCLUDES) $(DEFS)
+	override CFLAGS+=-Wall -Wno-deprecated -g -pthread $(INCLUDES) $(DEFS)
+	override CXXFLAGS+=-Wall -Wno-deprecated -g -std=c++11 -pthread $(INCLUDES) $(DEFS)
 	ZT_TRACE=1
 	STRIP?=echo
 	# The following line enables optimization for the crypto code, since
@@ -315,6 +319,9 @@ debug:	FORCE
 	make ZT_DEBUG=1 one
 	make ZT_DEBUG=1 selftest
 
+nltest: $(NLTEST_OBJS)
+	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o nltest $(NLTEST_OBJS) $(LDLIBS)
+
 # Note: keep the symlinks in /var/lib/zerotier-one to the binaries since these
 # provide backward compatibility with old releases where the binaries actually
 # lived here. Folks got scripts.

nltest.cpp

@@ -0,0 +1,13 @@
+#include "osdep/LinuxNetLink.hpp"
+
+using namespace ZeroTier;
+
+int main(int argc, char **argv)
+{
+    LinuxNetLink &nl = LinuxNetLink::getInstance();
+
+
+    while(true) {
+        Thread::sleep(1000);
+    }
+}

node/Hashtable.hpp

@@ -399,6 +399,10 @@ private:
 	{
 		return ((unsigned long)i * (unsigned long)0x9e3779b1);
 	}
+	static inline unsigned long _hc(const int i)
+	{
+		return ((unsigned long)i * (unsigned long)0x9e3379b1);
+	}
 
 	inline void _grow()
 	{

osdep/LinuxEthernetTap.cpp

@@ -56,6 +56,7 @@
 #include "../node/Dictionary.hpp"
 #include "OSUtils.hpp"
 #include "LinuxEthernetTap.hpp"
+#include "LinuxNetLink.hpp"
 
 // ff:ff:ff:ff:ff:ff with no ADI
 static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
@@ -97,6 +98,9 @@ LinuxEthernetTap::LinuxEthernetTap(
 	char procpath[128],nwids[32];
 	struct stat sbuf;
 
+	// ensure netlink connection is started
+	(void)LinuxNetLink::getInstance();
+	
 	OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid);
 
 	Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally
@@ -263,18 +267,8 @@ bool LinuxEthernetTap::enabled() const
 
 static bool ___removeIp(const std::string &_dev,const InetAddress &ip)
 {
-	long cpid = (long)vfork();
-	if (cpid == 0) {
-		OSUtils::redirectUnixOutputs("/dev/null",(const char *)0);
-		setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1);
-		char iptmp[128];
-		::execlp("ip","ip","addr","del",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0);
-		::_exit(-1);
-	} else {
-		int exitcode = -1;
-		::waitpid(cpid,&exitcode,0);
-		return (exitcode == 0);
-	}
+	LinuxNetLink::getInstance().removeAddress(ip, _dev.c_str());
+	return true;
 }
 
 #ifdef __SYNOLOGY__
@@ -285,49 +279,32 @@ bool LinuxEthernetTap::addIpSyn(std::vector<InetAddress> ips)
 	std::string cfg_contents = "DEVICE="+_dev+"\nBOOTPROTO=static";
 	int ip4=0,ip6=0,ip4_tot=0,ip6_tot=0;
 
-	long cpid = (long)vfork();
-	if (cpid == 0) {
-		OSUtils::redirectUnixOutputs("/dev/null",(const char *)0);
-		setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1);
-		// We must know if there is at least (one) of each protocol version so we
-		// can properly enumerate address/netmask combinations in the ifcfg-dev file
-		for(int i=0; i<(int)ips.size(); i++) {
-			if (ips[i].isV4())
-				ip4_tot++;
-			else
-				ip6_tot++;
-		}
-		// Assemble and write contents of ifcfg-dev file
-		for(int i=0; i<(int)ips.size(); i++) {
-			if (ips[i].isV4()) {
-				char iptmp[64],iptmp2[64];
-				std::string numstr4 = ip4_tot > 1 ? std::to_string(ip4) : "";
-				cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString(iptmp)
-					+ "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString(iptmp2)+"\n";
-				ip4++;
-			}
-			else {
-				char iptmp[64],iptmp2[64];
-				std::string numstr6 = ip6_tot > 1 ? std::to_string(ip6) : "";
-				cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString(iptmp)
-					+ "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString(iptmp2)+"\n";
-				ip6++;
-			}
-		}
-		OSUtils::writeFile(filepath.c_str(), cfg_contents.c_str(), cfg_contents.length());
-		// Finaly, add IPs
-		for(int i=0; i<(int)ips.size(); i++){
-			char iptmp[128],iptmp2[128];
-			if (ips[i].isV4())
-				::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"broadcast",ips[i].broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0);
-			else
-				::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"dev",_dev.c_str(),(const char *)0);
+	for(int i=0; i<(int)ips.size(); i++) {
+		if (ips[i].isV4())
+			ip4_tot++;
+		else
+			ip6_tot++;
+	}
+	// Assemble and write contents of ifcfg-dev file
+	for(int i=0; i<(int)ips.size(); i++) {
+		if (ips[i].isV4()) {
+			char iptmp[64],iptmp2[64];
+			std::string numstr4 = ip4_tot > 1 ? std::to_string(ip4) : "";
+			cfg_contents += "\nIPADDR"+numstr4+"="+ips[i].toIpString(iptmp)
+				+ "\nNETMASK"+numstr4+"="+ips[i].netmask().toIpString(iptmp2)+"\n";
+			ip4++;
+		} else {
+			char iptmp[64],iptmp2[64];
+			std::string numstr6 = ip6_tot > 1 ? std::to_string(ip6) : "";
+			cfg_contents += "\nIPV6ADDR"+numstr6+"="+ips[i].toIpString(iptmp)
+				+ "\nNETMASK"+numstr6+"="+ips[i].netmask().toIpString(iptmp2)+"\n";
+			ip6++;
 		}
-		::_exit(-1);
-	} else if (cpid > 0) {
-		int exitcode = -1;
-		::waitpid(cpid,&exitcode,0);
-		return (exitcode == 0);
+	}
+	OSUtils::writeFile(filepath.c_str(), cfg_contents.c_str(), cfg_contents.length());
+	// Finaly, add IPs
+	for(int i=0; i<(int)ips.size(); i++){
+		LinuxNetLink::getInstance().addAddress(ips[i], _dev.c_str());
 	}
 	return true;
 }
@@ -348,24 +325,9 @@ bool LinuxEthernetTap::addIp(const InetAddress &ip)
 			___removeIp(_dev,*i);
 	}
 
-	long cpid = (long)vfork();
-	if (cpid == 0) {
-		OSUtils::redirectUnixOutputs("/dev/null",(const char *)0);
-		setenv("PATH", "/sbin:/bin:/usr/sbin:/usr/bin", 1);
-		char iptmp[128],iptmp2[128];
-		if (ip.isV4()) {
-			::execlp("ip","ip","addr","add",ip.toString(iptmp),"broadcast",ip.broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0);
-		} else {
-			::execlp("ip","ip","addr","add",ip.toString(iptmp),"dev",_dev.c_str(),(const char *)0);
-		}
-		::_exit(-1);
-	} else if (cpid > 0) {
-		int exitcode = -1;
-		::waitpid(cpid,&exitcode,0);
-		return (exitcode == 0);
-	}
+	LinuxNetLink::getInstance().addAddress(ip, _dev.c_str());
 
-	return false;
+	return true;
 }
 
 bool LinuxEthernetTap::removeIp(const InetAddress &ip)

osdep/LinuxNetLink.cpp

@@ -0,0 +1,1058 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#include "LinuxNetLink.hpp"
+
+#include <unistd.h>
+#include <linux/if_tun.h>
+
+
+namespace ZeroTier {
+
+struct nl_route_req {
+    struct nlmsghdr nl;
+    struct rtmsg rt;
+    char buf[8192];
+};
+
+struct nl_if_req {
+    struct nlmsghdr nl;
+    struct ifinfomsg ifa;
+    char buf[8192];
+};
+
+struct nl_adr_req {
+	struct nlmsghdr nl;
+	struct ifaddrmsg ifa;
+	char buf[8192];
+};
+
+LinuxNetLink::LinuxNetLink()
+	: _t()
+	, _running(false)
+	, _routes_ipv4()
+	, _rv4_m()
+	, _routes_ipv6()
+	, _rv6_m()
+	, _seq(0)
+	, _interfaces()
+	, _if_m()
+	, _fd(socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE))
+	, _la({0})
+{
+
+	// set socket timeout to 1 sec so we're not permablocking recv() calls
+	_setSocketTimeout(_fd, 1);
+
+	_la.nl_family = AF_NETLINK;
+	_la.nl_pid = getpid()+1;
+	_la.nl_groups = RTMGRP_LINK|RTMGRP_IPV4_IFADDR|RTMGRP_IPV6_IFADDR|RTMGRP_IPV4_ROUTE|RTMGRP_IPV6_ROUTE|RTMGRP_NOTIFY;
+	if (bind(_fd, (struct sockaddr*)&_la, sizeof(_la))) {
+		fprintf(stderr, "Error connecting to RTNETLINK: %s\n", strerror(errno));
+		::exit(1);
+	}
+
+	fprintf(stderr, "Requesting IPV4 Routes\n");
+	_requestIPv4Routes();
+	fprintf(stderr, "Requesting IPV6 Routes\n");
+	_requestIPv6Routes();
+	fprintf(stderr, "Requesting Interface List\n");
+	_requestInterfaceList();
+
+	_running = true;
+	_t = Thread::start(this);
+}
+
+LinuxNetLink::~LinuxNetLink()
+{
+	_running = false;
+	Thread::join(_t);
+
+	::close(_fd);
+}
+
+void LinuxNetLink::_setSocketTimeout(int fd, int seconds) 
+{
+	struct timeval tv;
+	tv.tv_sec = seconds;
+	tv.tv_usec = 0;
+	if(setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv)) != 0) {
+		fprintf(stderr, "setsockopt failed: %s\n", strerror(errno));
+	}
+}
+
+int LinuxNetLink::_doRecv(int fd)
+{
+	char buf[8192];
+	char *p = NULL;
+	struct nlmsghdr *nlp;
+	int nll = 0;
+	int rtn = 0;
+	p = buf;
+
+	while(true) {
+		rtn = recv(fd, p, sizeof(buf) - nll, 0);
+
+		if (rtn > 0) {
+			nlp = (struct nlmsghdr *)p;
+
+			if(nlp->nlmsg_type == NLMSG_ERROR && (nlp->nlmsg_flags & NLM_F_ACK) != NLM_F_ACK) {
+				struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(nlp);
+				if (err->error != 0) {
+					fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error)));
+				}
+				p = buf;
+				nll = 0;
+				break;
+			}
+
+			if (nlp->nlmsg_type == NLMSG_NOOP) {
+				break;
+			}
+
+			if( (nlp->nlmsg_flags & NLM_F_MULTI) == NLM_F_MULTI || (nlp->nlmsg_type == NLMSG_DONE))
+			{
+				if (nlp->nlmsg_type == NLMSG_DONE) {
+					_processMessage(nlp, nll);
+					p = buf;
+					nll = 0;
+					break;
+				}
+				p += rtn;
+				nll += rtn;
+			}
+
+			if (nlp->nlmsg_type == NLMSG_OVERRUN) {
+				fprintf(stderr, "NLMSG_OVERRUN: Data lost\n");
+				p = buf;
+				nll = 0;
+				break;
+			}
+			
+			nll += rtn;
+
+			_processMessage(nlp, nll);
+			p = buf;
+			nll = 0;
+			break;
+		} else {
+			break;
+		}
+	}
+	return rtn;
+}
+
+void LinuxNetLink::threadMain() throw()
+{
+	int rtn = 0;
+
+	while(_running) {
+		rtn = _doRecv(_fd);
+		if (rtn <= 0) {
+			Thread::sleep(100);
+			continue;
+		}
+	}
+}
+
+void LinuxNetLink::_processMessage(struct nlmsghdr *nlp, int nll)
+{
+	for(; NLMSG_OK(nlp, nll); nlp=NLMSG_NEXT(nlp, nll))
+	{
+		switch(nlp->nlmsg_type) 
+		{
+		case RTM_NEWLINK:
+			_linkAdded(nlp);
+			break;
+		case RTM_DELLINK:
+			_linkDeleted(nlp);
+			break;
+		case RTM_NEWADDR:
+			_ipAddressAdded(nlp);
+			break;
+		case RTM_DELADDR:
+			_ipAddressDeleted(nlp);
+			break;
+		case RTM_NEWROUTE:
+			_routeAdded(nlp);
+			break;
+		case RTM_DELROUTE:
+			_routeDeleted(nlp);
+			break;
+		default:
+			fprintf(stderr, "ignore msgtype %d...\n", nlp->nlmsg_type);
+		}
+	}
+}
+
+void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp)
+{
+	struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp);
+	struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap);
+	int ifal = IFA_PAYLOAD(nlp);
+
+	char addr[40] = {0};
+	char local[40] = {0};
+	char label[40] = {0};
+	char bcast[40] = {0};
+	
+	for(;RTA_OK(rtap, ifal); rtap=RTA_NEXT(rtap,ifal))
+	{
+		switch(rtap->rta_type) {
+		case IFA_ADDRESS:
+			inet_ntop(ifap->ifa_family, RTA_DATA(rtap), addr, 40);
+			break;
+		case IFA_LOCAL:
+			inet_ntop(ifap->ifa_family, RTA_DATA(rtap), local, 40);
+			break;
+		case IFA_LABEL:
+			memcpy(label, RTA_DATA(rtap), 40);
+			break;
+		case IFA_BROADCAST:
+			inet_ntop(ifap->ifa_family, RTA_DATA(rtap), bcast, 40);
+			break;
+		}
+	}
+
+	fprintf(stderr, "Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
+}
+
+void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp)
+{
+	struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp);
+	struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap);
+	int ifal = IFA_PAYLOAD(nlp);
+
+	char addr[40] = {0};
+	char local[40] = {0};
+	char label[40] = {0};
+	char bcast[40] = {0};
+	
+	for(;RTA_OK(rtap, ifal); rtap=RTA_NEXT(rtap,ifal))
+	{
+		switch(rtap->rta_type) {
+		case IFA_ADDRESS:
+			inet_ntop(ifap->ifa_family, RTA_DATA(rtap), addr, 40);
+			break;
+		case IFA_LOCAL:
+			inet_ntop(ifap->ifa_family, RTA_DATA(rtap), local, 40);
+			break;
+		case IFA_LABEL:
+			memcpy(label, RTA_DATA(rtap), 40);
+			break;
+		case IFA_BROADCAST:
+			inet_ntop(ifap->ifa_family, RTA_DATA(rtap), bcast, 40);
+			break;
+		}
+	}
+
+	fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
+}
+
+void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp)
+{
+	char dsts[40] = {0};
+	char gws[40] = {0};
+	char srcs[40] = {0};
+	char ifs[16] = {0};
+	char ms[24] = {0};
+
+	struct rtmsg *rtp = (struct rtmsg *) NLMSG_DATA(nlp);
+	struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp);
+	int rtl = RTM_PAYLOAD(nlp);
+
+	for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl))
+	{
+		switch(rtap->rta_type)
+		{
+		case RTA_DST:
+			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40);
+			break;
+		case RTA_SRC:
+			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24: 40);
+			break;
+		case RTA_GATEWAY:
+			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40);
+			break;
+		case RTA_OIF:
+			sprintf(ifs, "%d", *((int*)RTA_DATA(rtap)));
+			break;
+		}
+	}
+	sprintf(ms, "%d", rtp->rtm_dst_len);
+
+	fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
+}
+
+void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp)
+{
+	char dsts[40] = {0};
+	char gws[40] = {0};
+	char srcs[40] = {0};
+	char ifs[16] = {0};
+	char ms[24] = {0};
+
+	struct rtmsg *rtp = (struct rtmsg *) NLMSG_DATA(nlp);
+	struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp);
+	int rtl = RTM_PAYLOAD(nlp);
+
+	for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl))
+	{
+		switch(rtap->rta_type)
+		{
+		case RTA_DST:
+			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40);
+			break;
+		case RTA_SRC:
+			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24 : 40);
+			break;
+		case RTA_GATEWAY:
+			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40);
+			break;
+		case RTA_OIF:
+			sprintf(ifs, "%d", *((int*)RTA_DATA(rtap)));
+			break;
+		}
+	}
+	sprintf(ms, "%d", rtp->rtm_dst_len);
+
+	fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
+}
+
+void LinuxNetLink::_linkAdded(struct nlmsghdr *nlp)
+{
+	char mac[18] = {0};
+	char mac_bin[6] = {0};
+	unsigned int mtu = 0;
+	char ifname[IFNAMSIZ] = {0};
+
+	struct ifinfomsg *ifip = (struct ifinfomsg *)NLMSG_DATA(nlp);
+	struct rtattr *rtap = (struct rtattr *)IFLA_RTA(ifip);
+	int ifil = RTM_PAYLOAD(nlp);
+
+	const char *ptr;
+	unsigned char *ptr2;
+	for(;RTA_OK(rtap, ifil);rtap=RTA_NEXT(rtap, ifil))
+	{
+		switch(rtap->rta_type) {
+		case IFLA_ADDRESS:
+			ptr2 = (unsigned char*)RTA_DATA(rtap);
+			snprintf(mac, 20, "%02x:%02x:%02x:%02x:%02x:%02x",
+				ptr2[0], ptr2[1], ptr2[2], ptr2[3], ptr2[4], ptr2[5]);
+			memcpy(mac_bin, ptr, 6);
+			break;
+		case IFLA_IFNAME:
+			ptr = (const char*)RTA_DATA(rtap);
+			memcpy(ifname, ptr, strlen(ptr));
+			break;
+		case IFLA_MTU:
+			memcpy(&mtu, RTA_DATA(rtap), sizeof(unsigned int));
+			break;
+		}
+	}
+
+	{
+		Mutex::Lock l(_if_m);
+		struct iface_entry &entry = _interfaces[ifip->ifi_index];
+		entry.index = ifip->ifi_index;
+		memcpy(entry.ifacename, ifname, sizeof(ifname));
+		memcpy(entry.mac, mac, sizeof(mac));
+		memcpy(entry.mac_bin, mac_bin, 6);
+		entry.mtu = mtu;
+	}
+
+	fprintf(stderr, "Link Added: %s mac: %s, mtu: %d\n", ifname, mac, mtu);
+}
+
+void LinuxNetLink::_linkDeleted(struct nlmsghdr *nlp)
+{
+	char mac[18] = {0};
+	unsigned int mtu = 0;
+	char ifname[40] = {0};
+
+	struct ifinfomsg *ifip = (struct ifinfomsg *)NLMSG_DATA(nlp);
+	struct rtattr *rtap = (struct rtattr *)IFLA_RTA(ifip);
+	int ifil = RTM_PAYLOAD(nlp);
+
+	const char *ptr;
+	unsigned char *ptr2;
+	for(;RTA_OK(rtap, ifil);rtap=RTA_NEXT(rtap, ifil))
+	{
+		switch(rtap->rta_type) {
+		case IFLA_ADDRESS:
+			ptr2 = (unsigned char*)RTA_DATA(rtap);
+			snprintf(mac, 20, "%02x:%02x:%02x:%02x:%02x:%02x",
+				ptr2[0], ptr2[1], ptr2[2], ptr2[3], ptr2[4], ptr2[5]);
+			break;
+		case IFLA_IFNAME:
+			ptr = (const char*)RTA_DATA(rtap);
+			memcpy(ifname, ptr, strlen(ptr));
+			break;
+		case IFLA_MTU:
+			memcpy(&mtu, RTA_DATA(rtap), sizeof(unsigned int));
+			break;
+		}
+	}
+
+	fprintf(stderr, "Link Deleted: %s mac: %s, mtu: %d\n", ifname, mac, mtu);
+	{
+		Mutex::Lock l(_if_m);
+		if(_interfaces.contains(ifip->ifi_index)) {
+			_interfaces.erase(ifip->ifi_index);
+		}
+	}
+}
+
+void LinuxNetLink::_requestIPv4Routes()
+{
+	int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	if (fd == -1) {
+		fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno));
+		return;
+	}
+
+	_setSocketTimeout(fd);
+
+	struct sockaddr_nl la;
+	la.nl_family = AF_NETLINK;
+	la.nl_pid = getpid();
+	la.nl_groups = RTMGRP_IPV4_ROUTE;
+	if(bind(fd, (struct sockaddr*)&la, sizeof(la))) {
+		fprintf(stderr, "Error binding RTNETLINK: %s\n", strerror(errno));
+		return;
+	}
+
+	struct nl_route_req req;
+	bzero(&req, sizeof(req));
+	req.nl.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
+	req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
+	req.nl.nlmsg_type = RTM_GETROUTE;
+	req.nl.nlmsg_pid = 0;
+	req.nl.nlmsg_seq = ++_seq;
+	req.rt.rtm_family = AF_INET;
+	req.rt.rtm_table = RT_TABLE_MAIN;
+
+	struct sockaddr_nl pa;
+	bzero(&pa, sizeof(pa));
+	pa.nl_family = AF_NETLINK;
+
+	struct msghdr msg;
+	bzero(&msg, sizeof(msg));
+	msg.msg_name = (void*)&pa;
+	msg.msg_namelen = sizeof(pa);
+
+	struct iovec iov;
+	bzero(&iov, sizeof(iov));
+	iov.iov_base = (void*)&req.nl;
+	iov.iov_len = req.nl.nlmsg_len;
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+
+	sendmsg(fd, &msg, 0);
+
+	_doRecv(fd);
+
+	close(fd);
+}
+
+void LinuxNetLink::_requestIPv6Routes()
+{
+	int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	if (fd == -1) {
+		fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno));
+		return;
+	}
+
+	_setSocketTimeout(fd);
+
+	struct sockaddr_nl la;
+	la.nl_family = AF_NETLINK;
+	la.nl_pid = getpid();
+	la.nl_groups = RTMGRP_IPV6_ROUTE;
+	if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) {
+		fprintf(stderr, "Error binding RTNETLINK: %s\n", strerror(errno));
+		return;
+	}
+
+	struct nl_route_req req;
+	bzero(&req, sizeof(req));
+	req.nl.nlmsg_len = NLMSG_LENGTH(sizeof(struct rtmsg));
+	req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
+	req.nl.nlmsg_type = RTM_GETROUTE;
+	req.nl.nlmsg_pid = 0;
+	req.nl.nlmsg_seq = ++_seq;
+	req.rt.rtm_family = AF_INET6;
+	req.rt.rtm_table = RT_TABLE_MAIN;
+
+	struct sockaddr_nl pa;
+	bzero(&pa, sizeof(pa));
+	pa.nl_family = AF_NETLINK;
+
+	struct msghdr msg;
+	bzero(&msg, sizeof(msg));
+	msg.msg_name = (void*)&pa;
+	msg.msg_namelen = sizeof(pa);
+
+	struct iovec iov;
+	bzero(&iov, sizeof(iov));
+	iov.iov_base = (void*)&req.nl;
+	iov.iov_len = req.nl.nlmsg_len;
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+
+	sendmsg(fd, &msg, 0);
+
+	_doRecv(fd);
+
+	close(fd);
+}
+
+void LinuxNetLink::_requestInterfaceList()
+{
+	int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	if (fd == -1) {
+		fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno));
+		return;
+	}
+
+	_setSocketTimeout(fd);
+
+	struct sockaddr_nl la;
+	la.nl_family = AF_NETLINK;
+	la.nl_pid = getpid();
+	la.nl_groups = RTMGRP_LINK;
+	if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) {
+		fprintf(stderr, "Error binding RTNETLINK: %s\n", strerror(errno));
+		return;
+	}
+
+	struct nl_if_req req;
+	bzero(&req, sizeof(req));
+	req.nl.nlmsg_len = NLMSG_LENGTH(sizeof(struct ifinfomsg));
+	req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_DUMP;
+	req.nl.nlmsg_type = RTM_GETLINK;
+	req.nl.nlmsg_pid = 0;
+	req.nl.nlmsg_seq = ++_seq;
+	req.ifa.ifi_family = AF_UNSPEC;
+
+	struct sockaddr_nl pa;
+	bzero(&pa, sizeof(pa));
+	pa.nl_family = AF_NETLINK;
+
+	struct msghdr msg;
+	bzero(&msg, sizeof(msg));
+	msg.msg_name = (void*)&pa;
+	msg.msg_namelen = sizeof(pa);
+
+	struct iovec iov;
+	bzero(&iov, sizeof(iov));
+	iov.iov_base = (void*)&req.nl;
+	iov.iov_len = req.nl.nlmsg_len;
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+	sendmsg(fd, &msg, 0);
+
+	_doRecv(fd);
+
+	close(fd);
+}
+
+void LinuxNetLink::addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName)
+{
+	int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	if (fd == -1) {
+		fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno));
+		return;
+	}
+
+	_setSocketTimeout(fd);
+
+	struct sockaddr_nl la;
+	bzero(&la, sizeof(la));
+	la.nl_family = AF_NETLINK;
+	la.nl_pid = getpid();
+
+	if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) {
+		fprintf(stderr, "Error binding RTNETLINK: %s\n", strerror(errno));
+		return;
+	}
+
+	char  tmp[64];
+	char tmp2[64];
+	char tmp3[64];
+	fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
+
+	if(!target) {
+		fprintf(stderr, "Uhhhh adding an empty route?!?!?");
+		return;
+	}
+
+	int rtl = sizeof(struct rtmsg);
+	struct nl_route_req req;
+	bzero(&req, sizeof(req));
+
+	struct rtattr *rtap = (struct rtattr *)req.buf;
+	rtap->rta_type = RTA_DST;
+	if (target.isV4()) {
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+		memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&target)->sin_addr, sizeof(struct in_addr));
+	} else {
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+		memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&target)->sin6_addr, sizeof(struct in6_addr));
+	}
+	rtl += rtap->rta_len;
+
+	if(via) {
+		rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len);
+		rtap->rta_type = RTA_GATEWAY;
+		if(via.isV4()) {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&via)->sin_addr, sizeof(struct in_addr));
+		} else {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&via)->sin6_addr, sizeof(struct in6_addr));
+		}
+		rtl += rtap->rta_len;
+	} else if (src) {
+		rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len);
+		rtap->rta_type = RTA_SRC;
+		if(src.isV4()) {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&src)->sin_addr, sizeof(struct in_addr));
+			
+		} else {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&src)->sin6_addr, sizeof(struct in6_addr));
+		}
+		req.rt.rtm_src_len = src.netmaskBits();
+	}
+
+	if (ifaceName != NULL) {
+		int interface_index = _indexForInterface(ifaceName);
+		if (interface_index != -1) {
+			rtap = (struct rtattr *) (((char*)rtap) + rtap->rta_len);
+			rtap->rta_type = RTA_OIF;
+			rtap->rta_len = RTA_LENGTH(sizeof(int));
+			memcpy(RTA_DATA(rtap), &interface_index, sizeof(int));
+			rtl += rtap->rta_len;
+		}
+	}
+
+	
+
+	req.nl.nlmsg_len = NLMSG_LENGTH(rtl);
+	req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_EXCL | NLM_F_CREATE | NLM_F_ACK;
+	req.nl.nlmsg_type = RTM_NEWROUTE;
+	req.nl.nlmsg_pid = 0;
+	req.nl.nlmsg_seq = ++_seq;
+	req.rt.rtm_family = target.ss_family;
+	req.rt.rtm_table = RT_TABLE_MAIN;
+	req.rt.rtm_protocol = RTPROT_STATIC;
+	req.rt.rtm_scope = RT_SCOPE_UNIVERSE;
+	req.rt.rtm_type = RTN_UNICAST;
+	req.rt.rtm_dst_len = target.netmaskBits();
+	req.rt.rtm_flags = 0;
+
+	struct sockaddr_nl pa;
+	bzero(&pa, sizeof(pa));
+	pa.nl_family = AF_NETLINK;
+
+	struct msghdr msg;
+	bzero(&msg, sizeof(msg));
+	msg.msg_name = (void*)&pa;
+	msg.msg_namelen = sizeof(pa);
+
+	struct iovec iov;
+	bzero(&iov, sizeof(iov));
+	iov.iov_base = (void*)&req.nl;
+	iov.iov_len = req.nl.nlmsg_len;
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+	sendmsg(fd, &msg, 0);
+
+	_doRecv(fd);
+
+	close(fd);
+}
+
+void LinuxNetLink::delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName)
+{
+	int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	if (fd == -1) {
+		fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno));
+		return;
+	}
+
+	_setSocketTimeout(fd);
+
+	struct sockaddr_nl la;
+	la.nl_family = AF_NETLINK;
+	la.nl_pid = getpid();
+
+	if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) {
+		fprintf(stderr, "Error binding RTNETLINK: %s\n", strerror(errno));
+		return;
+	}
+	char  tmp[64];
+	char tmp2[64];
+	char tmp3[64];
+	fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
+	
+	if(!target) {
+		fprintf(stderr, "Uhhhh deleting an empty route?!?!?");
+		return;
+	}
+
+	int rtl = sizeof(struct rtmsg);
+	struct nl_route_req req;
+	bzero(&req, sizeof(req));
+
+	struct rtattr *rtap = (struct rtattr *)req.buf;
+	rtap->rta_type = RTA_DST;
+	if (target.isV4()) {
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+		memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&target)->sin_addr, sizeof(struct in_addr));
+	} else {
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+		memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&target)->sin6_addr, sizeof(struct in6_addr));
+	}
+	rtl += rtap->rta_len;
+
+	if(via) {
+		rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len);
+		rtap->rta_type = RTA_GATEWAY;
+		if(via.isV4()) {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&via)->sin_addr, sizeof(struct in_addr));
+		} else {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&via)->sin6_addr, sizeof(struct in6_addr));
+		}
+		rtl += rtap->rta_len;
+	} else if (src) {
+		rtap = (struct rtattr *)(((char*)rtap)+rtap->rta_len);
+		rtap->rta_type = RTA_SRC;
+		if(src.isV4()) {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in*)&src)->sin_addr, sizeof(struct in_addr));
+			
+		} else {
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+			memcpy(RTA_DATA(rtap), &((struct sockaddr_in6*)&src)->sin6_addr, sizeof(struct in6_addr));
+		}
+		req.rt.rtm_src_len = src.netmaskBits();
+	}
+
+	if (ifaceName != NULL) {
+		int interface_index = _indexForInterface(ifaceName);
+		if (interface_index != -1) {
+			rtap = (struct rtattr *) (((char*)rtap) + rtap->rta_len);
+			rtap->rta_type = RTA_OIF;
+			rtap->rta_len = RTA_LENGTH(sizeof(int));
+			memcpy(RTA_DATA(rtap), &interface_index, sizeof(int));
+			rtl += rtap->rta_len;
+		}
+	}
+
+	
+
+	req.nl.nlmsg_len = NLMSG_LENGTH(rtl);
+	req.nl.nlmsg_flags = NLM_F_REQUEST;
+	req.nl.nlmsg_type = RTM_DELROUTE;
+	req.nl.nlmsg_pid = 0;
+	req.nl.nlmsg_seq = ++_seq;
+	req.rt.rtm_family = target.ss_family;
+	req.rt.rtm_table = RT_TABLE_MAIN;
+	req.rt.rtm_protocol = RTPROT_STATIC;
+	req.rt.rtm_scope = RT_SCOPE_UNIVERSE;
+	req.rt.rtm_type = RTN_UNICAST;
+	req.rt.rtm_dst_len = target.netmaskBits();
+	req.rt.rtm_flags = 0;
+
+	struct sockaddr_nl pa;
+	bzero(&pa, sizeof(pa));
+	pa.nl_family = AF_NETLINK;
+
+	struct msghdr msg;
+	bzero(&msg, sizeof(msg));
+	msg.msg_name = (void*)&pa;
+	msg.msg_namelen = sizeof(pa);
+
+	struct iovec iov;
+	bzero(&iov, sizeof(iov));
+	iov.iov_base = (void*)&req.nl;
+	iov.iov_len = req.nl.nlmsg_len;
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+	sendmsg(fd, &msg, 0);
+
+	_doRecv(fd);
+
+	close(fd);
+}
+
+void LinuxNetLink::addAddress(const InetAddress &addr, const char *iface)
+{
+	int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	if (fd == -1) {
+		fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno));
+		return;
+	}
+
+	_setSocketTimeout(fd);
+
+	struct sockaddr_nl la;
+	la.nl_family = AF_NETLINK;
+	la.nl_pid = getpid();
+	if (addr.isV4()) {
+		la.nl_groups = RTMGRP_IPV4_IFADDR;
+	} else {
+		la.nl_groups = RTMGRP_IPV6_IFADDR;
+	}
+	if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) {
+		fprintf(stderr, "Error binding RTNETLINK: %s\n", strerror(errno));
+		return;
+	}
+
+	char tmp[128];
+	fprintf(stderr, "Adding IP address %s to interface %s", addr.toString(tmp), iface);
+	int interface_index = _indexForInterface(iface);
+
+	if (interface_index == -1) {
+		fprintf(stderr, "Unable to find index for interface %s\n", iface);
+		return;
+	}
+	
+	int rtl = sizeof(struct ifaddrmsg);
+	struct nl_adr_req req;
+	bzero(&req, sizeof(struct nl_adr_req));
+
+	struct rtattr *rtap = (struct rtattr *)req.buf;;
+	if(addr.isV4()) {
+		struct sockaddr_in *addr_v4 = (struct sockaddr_in*)&addr;
+		rtap->rta_type = IFA_ADDRESS;
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+		memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr));
+		rtl += rtap->rta_len;
+
+		rtap = (struct rtattr*)(((char*)rtap) + rtap->rta_len);
+		rtap->rta_type = IFA_LOCAL;
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+		memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr));
+		rtl += rtap->rta_len;
+
+		InetAddress broadcast = addr.broadcast();
+		if(broadcast) {
+			rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len);
+			struct sockaddr_in *bcast = (struct sockaddr_in*)&broadcast;
+			rtap->rta_type = IFA_BROADCAST;
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+			memcpy(RTA_DATA(rtap), &bcast->sin_addr, sizeof(struct in_addr));
+			rtl += rtap->rta_len;
+		}
+	} else { //V6
+		rtap->rta_type = IFA_ADDRESS;
+		struct sockaddr_in6 *addr_v6 = (struct sockaddr_in6*)&addr;
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+		memcpy(RTA_DATA(rtap), &addr_v6->sin6_addr, sizeof(struct in6_addr));
+		rtl += rtap->rta_len;
+	}
+
+	if (iface) {
+		rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len);
+		rtap->rta_type = IFA_LABEL;
+		rtap->rta_len = RTA_LENGTH(strlen(iface));
+		memcpy(RTA_DATA(rtap), iface, strlen(iface));
+		rtl += rtap->rta_len;
+	}
+
+	req.nl.nlmsg_len = NLMSG_LENGTH(rtl);
+	req.nl.nlmsg_flags = NLM_F_REQUEST | NLM_F_CREATE | NLM_F_EXCL;
+	req.nl.nlmsg_type = RTM_NEWADDR;
+	req.nl.nlmsg_pid = 0;
+	req.nl.nlmsg_seq = ++_seq;
+	req.ifa.ifa_family = addr.ss_family;
+	req.ifa.ifa_prefixlen = addr.port();
+	req.ifa.ifa_flags = IFA_F_PERMANENT;
+	req.ifa.ifa_scope = 0;
+	req.ifa.ifa_index = interface_index;
+
+	struct sockaddr_nl pa;
+	bzero(&pa, sizeof(sockaddr_nl));
+	pa.nl_family = AF_NETLINK;
+
+	struct msghdr msg;
+	bzero(&msg, sizeof(msg));
+	msg.msg_name = (void*)&pa;
+	msg.msg_namelen = sizeof(pa);
+
+	struct iovec iov;
+	iov.iov_base = (void*)&req.nl;
+	iov.iov_len = req.nl.nlmsg_len;
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+	sendmsg(fd, &msg, 0);
+
+	_doRecv(fd);
+
+	close(fd);
+}
+
+void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface)
+{
+	int fd = socket(AF_NETLINK, SOCK_RAW, NETLINK_ROUTE);
+	if (fd == -1) {
+		fprintf(stderr, "Error opening RTNETLINK socket: %s\n", strerror(errno));
+		return;
+	}
+
+	_setSocketTimeout(fd);
+
+	struct sockaddr_nl la;
+	la.nl_family = AF_NETLINK;
+	la.nl_pid = getpid();
+	if (addr.isV4()) {
+		la.nl_groups = RTMGRP_IPV4_IFADDR;
+	} else {
+		la.nl_groups = RTMGRP_IPV6_IFADDR;
+	}
+	if(bind(fd, (struct sockaddr*)&la, sizeof(struct sockaddr_nl))) {
+		fprintf(stderr, "Error binding RTNETLINK: %s\n", strerror(errno));
+		return;
+	}
+
+	char tmp[128];
+	fprintf(stderr, "Removing IP address %s from interface %s", addr.toString(tmp), iface);
+
+	int interface_index = _indexForInterface(iface);
+
+	if (interface_index == -1) {
+		fprintf(stderr, "Unable to find index for interface %s\n", iface);
+		return;
+	}
+	
+	int rtl = sizeof(struct ifaddrmsg);
+	struct nl_adr_req req;
+	bzero(&req, sizeof(struct nl_adr_req));
+
+	struct rtattr *rtap = (struct rtattr *)req.buf;
+	if(addr.isV4()) {
+		struct sockaddr_in *addr_v4 = (struct sockaddr_in*)&addr;
+		rtap->rta_type = IFA_ADDRESS;
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+		memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr));
+		rtl += rtap->rta_len;
+
+		rtap = (struct rtattr*)(((char*)rtap) + rtap->rta_len);
+		rtap->rta_type = IFA_LOCAL;
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+		memcpy(RTA_DATA(rtap), &addr_v4->sin_addr, sizeof(struct in_addr));
+		rtl += rtap->rta_len;
+
+		InetAddress broadcast = addr.broadcast();
+		if(broadcast) {
+			rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len);
+			struct sockaddr_in *bcast = (struct sockaddr_in*)&broadcast;
+			rtap->rta_type = IFA_BROADCAST;
+			rtap->rta_len = RTA_LENGTH(sizeof(struct in_addr));
+			memcpy(RTA_DATA(rtap), &bcast->sin_addr, sizeof(struct in_addr));
+			rtl += rtap->rta_len;
+		}
+	} else { //V6
+		rtap->rta_type = IFA_ADDRESS;
+		struct sockaddr_in6 *addr_v6 = (struct sockaddr_in6*)&addr;
+		rtap->rta_len = RTA_LENGTH(sizeof(struct in6_addr));
+		memcpy(RTA_DATA(rtap), &addr_v6->sin6_addr, sizeof(struct in6_addr));
+		rtl += rtap->rta_len;
+	}
+
+	if (iface) {
+		rtap = (struct rtattr*)(((char*)rtap)+rtap->rta_len);
+		rtap->rta_type = IFA_LABEL;
+		rtap->rta_len = RTA_LENGTH(strlen(iface));
+		memcpy(RTA_DATA(rtap), iface, strlen(iface));
+		rtl += rtap->rta_len;
+	}
+
+	req.nl.nlmsg_len = NLMSG_LENGTH(rtl);
+	req.nl.nlmsg_flags = NLM_F_REQUEST;
+	req.nl.nlmsg_type = RTM_DELADDR;
+	req.nl.nlmsg_pid = 0;
+	req.nl.nlmsg_seq = ++_seq;
+	req.ifa.ifa_family = addr.ss_family;
+	req.ifa.ifa_prefixlen = addr.port();
+	req.ifa.ifa_flags = IFA_F_PERMANENT;
+	req.ifa.ifa_scope = 0;
+	req.ifa.ifa_index = interface_index;
+
+	struct sockaddr_nl pa;
+	bzero(&pa, sizeof(sockaddr_nl));
+	pa.nl_family = AF_NETLINK;
+
+	struct msghdr msg;
+	bzero(&msg, sizeof(msg));
+	msg.msg_name = (void*)&pa;
+	msg.msg_namelen = sizeof(pa);
+
+	struct iovec iov;
+	iov.iov_base = (void*)&req.nl;
+	iov.iov_len = req.nl.nlmsg_len;
+	msg.msg_iov = &iov;
+	msg.msg_iovlen = 1;
+	sendmsg(fd, &msg, 0);
+
+	_doRecv(fd);
+
+	close(fd);
+}
+
+RouteList LinuxNetLink::getIPV4Routes() const 
+{
+	return _routes_ipv4;
+}
+
+RouteList LinuxNetLink::getIPV6Routes() const
+{
+	return _routes_ipv6;
+}
+
+int LinuxNetLink::_indexForInterface(const char *iface)
+{
+	Mutex::Lock l(_if_m);
+	int interface_index = -1;
+	Hashtable<int, iface_entry>::Iterator iter(_interfaces);
+	int *k = NULL;
+	iface_entry *v = NULL;
+	while(iter.next(k,v)) {
+		if(strcmp(iface, v->ifacename) == 0) {
+			interface_index = v->index;
+			break;
+		}
+	}
+	return interface_index;
+}
+
+} // namespace ZeroTier

osdep/LinuxNetLink.hpp

@@ -0,0 +1,129 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#ifndef ZT_LINUX_NETLINK_HPP
+#define ZT_LINUX_NETLINK_HPP
+
+#include <vector>
+
+#include <bits/sockaddr.h>
+#include <asm/types.h>
+#include <linux/rtnetlink.h>
+#include <sys/socket.h>
+#include <linux/if.h>
+
+#include "../node/InetAddress.hpp"
+#include "../node/MAC.hpp"
+#include "Thread.hpp"
+#include "../node/Hashtable.hpp"
+#include "../node/Mutex.hpp"
+
+
+namespace ZeroTier {
+
+struct route_entry {
+	InetAddress target;
+	InetAddress via;
+	int if_index;
+	char iface[IFNAMSIZ];
+};
+typedef std::vector<route_entry> RouteList;
+
+/**
+ * Interface with Linux's RTNETLINK 
+ */
+class LinuxNetLink
+{
+private:
+    LinuxNetLink();
+    ~LinuxNetLink();
+
+public:
+    static LinuxNetLink& getInstance()
+    {
+        static LinuxNetLink instance;
+        return instance;
+    }
+
+    LinuxNetLink(LinuxNetLink const&) = delete;
+    void operator=(LinuxNetLink const&) = delete;
+
+    void addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
+    void delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
+    RouteList getIPV4Routes() const;
+    RouteList getIPV6Routes() const;
+
+    void addAddress(const InetAddress &addr, const char *iface);
+    void removeAddress(const InetAddress &addr, const char *iface);
+
+    void threadMain() throw();
+private:
+    int _doRecv(int fd);
+
+    void _processMessage(struct nlmsghdr *nlp, int nll);
+    void _routeAdded(struct nlmsghdr *nlp);
+    void _routeDeleted(struct nlmsghdr *nlp);
+    void _linkAdded(struct nlmsghdr *nlp);
+    void _linkDeleted(struct nlmsghdr *nlp);
+    void _ipAddressAdded(struct nlmsghdr *nlp);
+    void _ipAddressDeleted(struct nlmsghdr *nlp);
+
+    void _requestInterfaceList();
+    void _requestIPv4Routes();
+    void _requestIPv6Routes();
+
+    int _indexForInterface(const char *iface);
+
+    void _setSocketTimeout(int fd, int seconds = 1);
+
+    Thread _t;
+    bool _running;
+
+    RouteList _routes_ipv4;
+    Mutex _rv4_m;
+    RouteList _routes_ipv6;
+    Mutex _rv6_m;
+
+    uint32_t _seq;
+
+    struct iface_entry {
+        int index;
+        char ifacename[IFNAMSIZ];
+        char mac[18];
+        char mac_bin[6];
+        unsigned int mtu;
+    };
+    Hashtable<int, iface_entry> _interfaces;
+    Mutex _if_m;
+
+    // socket communication vars;
+    int _fd;
+    struct sockaddr_nl _la;
+};    
+
+}
+
+#endif // ZT_LINUX_NETLINK_HPPS

osdep/ManagedRoute.cpp

@@ -48,6 +48,14 @@
 #include <arpa/inet.h>
 #include <net/route.h>
 #include <net/if.h>
+#ifdef __LINUX__
+#include <sys/ioctl.h>
+#include <bits/sockaddr.h>
+#include <asm/types.h>
+#include <linux/rtnetlink.h>
+#include <sys/socket.h>
+#include "../osdep/LinuxNetLink.hpp"
+#endif
 #ifdef __BSD__
 #include <net/if_dl.h>
 #include <sys/sysctl.h>
@@ -277,27 +285,127 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress
 #ifdef __LINUX__ // ----------------------------------------------------------
 #define ZT_ROUTING_SUPPORT_FOUND 1
 
-static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *localInterface)
+static void _routeCmd(const char *op, const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *localInterface) 
 {
-	long p = (long)fork();
-	if (p > 0) {
-		int exitcode = -1;
-		::waitpid(p,&exitcode,0);
-	} else if (p == 0) {
-		::close(STDOUT_FILENO);
-		::close(STDERR_FILENO);
-		char ipbuf[64],ipbuf2[64];
+	char targetStr[64] = {0};
+	char viaStr[64] = {0};
+	InetAddress nmsk = target.netmask();
+	char nmskStr[64] = {0};
+	fprintf(stderr, "Received Route Cmd: %s target: %s via: %s netmask: %s localInterface: %s\n", op, target.toString(targetStr), via.toString(viaStr), nmsk.toString(nmskStr), localInterface);
+
+
+	if ((strcmp(op, "add") == 0 || strcmp(op, "replace") == 0)) {
+		LinuxNetLink::getInstance().addRoute(target, via, src, localInterface);
+	} else if ((strcmp(op, "remove") == 0 || strcmp(op, "del") == 0)) {
+		LinuxNetLink::getInstance().delRoute(target, via, src, localInterface);
+	}
+	return;
+
+	
+	int fd = socket(PF_INET, SOCK_DGRAM, IPPROTO_IP);;
+	struct rtentry route = {0};
+
+	if (target.ss_family == AF_INET) {
+		struct sockaddr_in *target_in = (struct sockaddr_in*)&target;
+		struct sockaddr_in *via_in = (struct sockaddr_in*)&via;
+		InetAddress netmask = target.netmask();
+		struct sockaddr_in *netmask_in = (struct sockaddr_in*)&netmask;
+
+		struct sockaddr_in *addr = NULL;
+
+		// set target
+		addr = (struct sockaddr_in *)&route.rt_dst;
+		addr->sin_family = AF_INET;
+		addr->sin_addr = target_in->sin_addr;
+
+		// set netmask
+		addr = (struct sockaddr_in *)&route.rt_genmask;
+		addr->sin_family = AF_INET;
+		addr->sin_addr = netmask_in->sin_addr;
+
+		route.rt_dev = const_cast<char*>(localInterface);
+
 		if (via) {
-			::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"via",via.toIpString(ipbuf2),(const char *)0);
-			::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"via",via.toIpString(ipbuf2),(const char *)0);
+			// set the gateway
+			addr = (struct sockaddr_in *)&route.rt_gateway;
+			addr->sin_family = AF_INET;
+			addr->sin_addr = via_in->sin_addr;
+
+			route.rt_flags = RTF_UP | RTF_GATEWAY;
 		} else if ((localInterface)&&(localInterface[0])) {
-			::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"dev",localInterface,(const char *)0);
-			::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"dev",localInterface,(const char *)0);
+			route.rt_flags = RTF_UP;//| RTF_HOST;
+		}
+	}
+	else if (target.ss_family == AF_INET6) 
+	{
+		struct sockaddr_in6 *addr = NULL;
+
+		// set target
+		addr = (struct sockaddr_in6 *)&route.rt_dst;
+		addr->sin6_family = AF_INET6;
+		memcpy(&addr->sin6_addr, &((struct sockaddr_in6*)&target)->sin6_addr, sizeof(struct in6_addr));
+
+		//set netmask
+		addr = (struct sockaddr_in6 *)&route.rt_genmask;
+		addr->sin6_family = AF_INET6;
+		InetAddress netmask = target.netmask();
+		memcpy(&addr->sin6_addr, &((struct sockaddr_in6*)&netmask)->sin6_addr, sizeof(struct in6_addr));
+
+		if (via) {
+			// set the gateway
+			addr = (struct sockaddr_in6*)&route.rt_gateway;
+			addr->sin6_family = AF_INET;
+			memcpy(&addr->sin6_addr, &((struct sockaddr_in6*)&via)->sin6_addr, sizeof(struct in6_addr));
+
+			route.rt_flags = RTF_UP | RTF_GATEWAY;
+		} else if ((localInterface)&&(localInterface[0])) {
+			route.rt_dev = const_cast<char*>(localInterface);
+			route.rt_flags = RTF_UP;
 		}
-		::_exit(-1);
 	}
+
+	unsigned long ctl = -1;
+	if (strcmp(op, "add") == 0 || strcmp(op, "replace") == 0) {
+		ctl = SIOCADDRT;
+	} else if (strcmp(op, "remove") == 0 || strcmp(op, "del") == 0) {
+		ctl = SIOCDELRT;
+	} else {
+		close(fd);
+		return;
+	}
+
+	if ( ioctl(fd, ctl, &route)) {
+		fprintf(stderr, "Error adding route: %s\n", strerror(errno));
+		close(fd);
+		::exit(1);
+	}
+	close(fd);
 }
 
+// static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *localInterface)
+// {
+// 	// long p = (long)fork();
+// 	// if (p > 0) {
+// 	// 	int exitcode = -1;
+// 	// 	::waitpid(p,&exitcode,0);
+// 	// } else if (p == 0) {
+// 	// 	::close(STDOUT_FILENO);
+// 	// 	::close(STDERR_FILENO);
+// 		char ipbuf[64],ipbuf2[64];
+
+		
+
+// 		if (via) {
+// 			::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"via",via.toIpString(ipbuf2),(const char *)0);
+// 			::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"via",via.toIpString(ipbuf2),(const char *)0);
+// 		} else if ((localInterface)&&(localInterface[0])) {
+// 			::execl(ZT_LINUX_IP_COMMAND,ZT_LINUX_IP_COMMAND,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"dev",localInterface,(const char *)0);
+// 			::execl(ZT_LINUX_IP_COMMAND_2,ZT_LINUX_IP_COMMAND_2,(target.ss_family == AF_INET6) ? "-6" : "-4","route",op,target.toString(ipbuf),"dev",localInterface,(const char *)0);
+// 		}
+// 	// 	::_exit(-1);
+// 	// }
+// }
+
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------
@@ -494,11 +602,11 @@ bool ManagedRoute::sync()
 
 	if (!_applied.count(leftt)) {
 		_applied[leftt] = false; // boolean unused
-		_routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device);
+		_routeCmd("replace",leftt,_via,_src,_device);
 	}
 	if ((rightt)&&(!_applied.count(rightt))) {
 		_applied[rightt] = false; // boolean unused
-		_routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device);
+		_routeCmd("replace",rightt,_via,_src,_device);
 	}
 
 #endif // __LINUX__ ----------------------------------------------------------
@@ -545,7 +653,7 @@ void ManagedRoute::remove()
 #endif // __BSD__ ------------------------------------------------------------
 
 #ifdef __LINUX__ // ----------------------------------------------------------
-		_routeCmd("del",r->first,_via,(_via) ? (const char *)0 : _device);
+		_routeCmd("del",r->first,_via,_src,_device);
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------

osdep/ManagedRoute.hpp

@@ -49,14 +49,20 @@ class ManagedRoute
 	friend class SharedPtr<ManagedRoute>;
 
 public:
-	ManagedRoute(const InetAddress &target,const InetAddress &via,const char *device)
+	ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device)
 	{
 		_target = target;
 		_via = via;
+		_src = src;
 		if (via.ss_family == AF_INET)
 			_via.setPort(32);
 		else if (via.ss_family == AF_INET6)
 			_via.setPort(128);
+		if (src.ss_family == AF_INET) {
+			_src.setPort(32);
+		} else if (src.ss_family == AF_INET6) {
+			_src.setPort(128);
+		}
 		Utils::scopy(_device,sizeof(_device),device);
 		_systemDevice[0] = (char)0;
 	}
@@ -87,6 +93,7 @@ public:
 
 	inline const InetAddress &target() const { return _target; }
 	inline const InetAddress &via() const { return _via; }
+	inline const InetAddress &src() const { return _src; }
 	inline const char *device() const { return _device; }
 
 private:
@@ -95,6 +102,7 @@ private:
 
 	InetAddress _target;
 	InetAddress _via;
+	InetAddress _src;
 	InetAddress _systemVia; // for route overrides
 	std::map<InetAddress,bool> _applied; // routes currently applied
 	char _device[128];

service/OneService.cpp

@@ -1710,11 +1710,13 @@ public:
 			// Nuke applied routes that are no longer in n.config.routes[] and/or are not allowed
 			for(std::list< SharedPtr<ManagedRoute> >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) {
 				bool haveRoute = false;
+
 				if ( (checkIfManagedIsAllowed(n,(*mr)->target())) && (((*mr)->via().ss_family != (*mr)->target().ss_family)||(!matchIpOnly(myIps,(*mr)->via()))) ) {
 					for(unsigned int i=0;i<n.config.routeCount;++i) {
 						const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target));
 						const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via));
-						if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (tapdev == (*mr)->device()) ) ) {
+
+						if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (strcmp(tapdev,(*mr)->device())==0) ) ) {
 							haveRoute = true;
 							break;
 						}
@@ -1731,6 +1733,17 @@ public:
 			for(unsigned int i=0;i<n.config.routeCount;++i) {
 				const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target));
 				const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via));
+				InetAddress *src = NULL;
+				for (unsigned int j=0; j<n.config.assignedAddressCount; ++j) {
+					const InetAddress *const tmp = reinterpret_cast<const InetAddress *>(&(n.config.assignedAddresses[j]));
+					if (target->isV4() && tmp->isV4()) {
+						src = reinterpret_cast<InetAddress *>(&(n.config.assignedAddresses[j]));
+						break;
+					} else if (target->isV6() && tmp->isV6()) {
+						src = reinterpret_cast<InetAddress *>(&(n.config.assignedAddresses[j]));
+						break;
+					}
+				}
 
 				if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) )
 					continue;
@@ -1762,7 +1775,7 @@ public:
 					continue;
 
 				// Add and apply new routes
-				n.managedRoutes.push_back(SharedPtr<ManagedRoute>(new ManagedRoute(*target,*via,tapdev)));
+				n.managedRoutes.push_back(SharedPtr<ManagedRoute>(new ManagedRoute(*target,*via,*src,tapdev)));
 				if (!n.managedRoutes.back()->sync())
 					n.managedRoutes.pop_back();
 			}