Port back over some improvements from 1.6

core/Utils.cpp

@@ -29,11 +29,43 @@
 #include <wincrypt.h>
 #endif
 
-#if defined(ZT_ARCH_ARM_HAS_NEON) && defined(__LINUX__)
+#ifdef ZT_ARCH_ARM_HAS_NEON
+
+#ifdef __LINUX__
 #include <sys/auxv.h>
 #include <asm/hwcap.h>
 #endif
 
+#if defined(__FreeBSD__)
+#include <elf.h>
+#include <sys/auxv.h>
+static inline long getauxval(int caps)
+{
+	long hwcaps = 0;
+	elf_aux_info(caps, &hwcaps, sizeof(hwcaps));
+	return hwcaps;
+}
+#endif
+
+// If these are not even defined, obviously they are not supported.
+#ifndef HWCAP_AES
+#define HWCAP_AES 0
+#endif
+#ifndef HWCAP_CRC32
+#define HWCAP_CRC32 0
+#endif
+#ifndef HWCAP_PMULL
+#define HWCAP_PMULL 0
+#endif
+#ifndef HWCAP_SHA1
+#define HWCAP_SHA1 0
+#endif
+#ifndef HWCAP_SHA2
+#define HWCAP_SHA2 0
+#endif
+
+#endif // ZT_ARCH_ARM_HAS_NEON
+
 namespace ZeroTier {
 
 namespace Utils {

osdep/LinuxEthernetTap.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)2013-2020 ZeroTier, Inc.
+ * Copyright (c)2019 ZeroTier, Inc.
  *
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
@@ -11,17 +11,17 @@
  */
 /****/
 
-#include "../core/Constants.hpp"
-
-#ifdef __LINUX__
-
-#ifdef __GCC__
+#ifdef __GNUC__
 #pragma GCC diagnostic ignored "-Wrestrict"
 #endif
 
-#include "../core/Utils.hpp"
-#include "../core/Mutex.hpp"
-#include "../core/Dictionary.hpp"
+#include "../node/Constants.hpp"
+
+#ifdef __LINUX__
+
+#include "../node/Utils.hpp"
+#include "../node/Mutex.hpp"
+#include "../node/Dictionary.hpp"
 #include "OSUtils.hpp"
 #include "LinuxEthernetTap.hpp"
 #include "LinuxNetLink.hpp"
@@ -52,13 +52,17 @@
 #include <utility>
 #include <string>
 
+#ifndef IFNAMSIZ
+#define IFNAMSIZ 16
+#endif
+
+#define ZT_TAP_BUF_SIZE 16384
+
 // ff:ff:ff:ff:ff:ff with no ADI
 static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
 
 namespace ZeroTier {
 
-static Mutex __tapCreateLock;
-
 static const char _base32_chars[32] = { 'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','2','3','4','5','6','7' };
 static void _base32_5_to_8(const uint8_t *in,char *out)
 {
@@ -84,21 +88,24 @@ LinuxEthernetTap::LinuxEthernetTap(
 	_handler(handler),
 	_arg(arg),
 	_nwid(nwid),
+	_mac(mac),
 	_homePath(homePath),
 	_mtu(mtu),
 	_fd(0),
 	_enabled(true)
 {
+	static std::mutex s_tapCreateLock;
 	char procpath[128],nwids[32];
 	struct stat sbuf;
 
-	// ensure netlink connection is started
+	// Create only one tap at a time globally.
+	std::lock_guard<std::mutex> tapCreateLock(s_tapCreateLock);
+
+	// Make sure Linux netlink is initialized.
 	(void)LinuxNetLink::getInstance();
 
 	OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid);
 
-	Mutex::Lock _l(__tapCreateLock); // create only one tap at a time, globally
-
 	_fd = ::open("/dev/net/tun",O_RDWR);
 	if (_fd <= 0) {
 		_fd = ::open("/dev/tun",O_RDWR);
@@ -109,7 +116,8 @@ LinuxEthernetTap::LinuxEthernetTap(
 	struct ifreq ifr;
 	memset(&ifr,0,sizeof(ifr));
 
-	// Restore device names from legacy devicemap, but for new devices we use a base32-based canonical naming
+	// Restore device names from legacy devicemap, but for new devices we use a base32-based
+	// canonical device name.
 	std::map<std::string,std::string> globalDeviceMap;
 	FILE *devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"r");
 	if (devmapf) {
@@ -170,83 +178,165 @@ LinuxEthernetTap::LinuxEthernetTap(
 		throw std::runtime_error("unable to configure TUN/TAP device for TAP operation");
 	}
 
-	_dev = ifr.ifr_name;
-
 	::ioctl(_fd,TUNSETPERSIST,0); // valgrind may generate a false alarm here
+	_dev = ifr.ifr_name;
+	::fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC);
 
-	// Open an arbitrary socket to talk to netlink
-	int sock = socket(AF_INET,SOCK_DGRAM,0);
-	if (sock <= 0) {
-		::close(_fd);
-		throw std::runtime_error("unable to open netlink socket");
-	}
+	(void)::pipe(_shutdownSignalPipe);
 
-	// Set MAC address
-	ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
-	mac.copyTo((uint8_t *)ifr.ifr_ifru.ifru_hwaddr.sa_data);
-	if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to configure TAP hardware (MAC) address");
-		return;
-	}
+	_tapReaderThread = std::thread([this]{
+		fd_set readfds,nullfds;
+		int n,nfds,r;
+		void *buf = nullptr;
+		std::vector<void *> buffers;
 
-	// Set MTU
-	ifr.ifr_ifru.ifru_mtu = (int)mtu;
-	if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to configure TAP MTU");
-	}
+		{
+			struct ifreq ifr;
+			memset(&ifr,0,sizeof(ifr));
+			strcpy(ifr.ifr_name,_dev.c_str());
 
-	if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) {
-		::close(_fd);
-		throw std::runtime_error("unable to set flags on file descriptor for TAP device");
-	}
+			const int sock = socket(AF_INET,SOCK_DGRAM,0);
+			if (sock <= 0)
+				return;
 
-	/* Bring interface up */
-	if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to get TAP interface flags");
-	}
-	ifr.ifr_flags |= IFF_UP;
-	if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to set TAP interface flags");
-	}
+			if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) {
+				::close(sock);
+				printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
+				return;
+			}
+			ifr.ifr_flags |= IFF_UP;
+			if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) {
+				::close(sock);
+				printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
+				return;
+			}
 
-	::close(sock);
+			// Some kernel versions seem to require you to yield while the device comes up
+			// before they will accept MTU and MAC. For others it doesn't matter, but is
+			// harmless. This was moved to the worker thread though so as not to block the
+			// main ZeroTier loop.
+			usleep(500000);
+
+			ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
+			_mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6);
+			if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) {
+				::close(sock);
+				printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n");
+				return;
+			}
 
-	// Set close-on-exec so that devices cannot persist if we fork/exec for update
-	::fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC);
+			ifr.ifr_ifru.ifru_mtu = (int)_mtu;
+			if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) {
+				::close(sock);
+				printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n");
+				return;
+			}
 
-	(void)::pipe(_shutdownSignalPipe);
+			fcntl(_fd,F_SETFL,O_NONBLOCK);
 
-	/*
-	globalDeviceMap[nwids] = _dev;
-	devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"w");
-	if (devmapf) {
-		gdmEntry = globalDeviceMap.begin();
-		while (gdmEntry != globalDeviceMap.end()) {
-			fprintf(devmapf,"%s=%s\n",gdmEntry->first.c_str(),gdmEntry->second.c_str());
-			++gdmEntry;
+			::close(sock);
 		}
-		fclose(devmapf);
-	}
-	*/
 
-	_thread = Thread::start(this);
+		FD_ZERO(&readfds);
+		FD_ZERO(&nullfds);
+		nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
+
+		r = 0;
+		for(;;) {
+			FD_SET(_shutdownSignalPipe[0],&readfds);
+			FD_SET(_fd,&readfds);
+			select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
+
+			if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
+				break;
+
+			if (FD_ISSET(_fd,&readfds)) {
+				for(;;) { // read until there are no more packets, then return to outer select() loop
+					if (!buf) {
+						// To reduce use of the mutex, we keep a local buffer vector and
+						// swap (which is a pointer swap) with the global one when it's
+						// empty. This retrieves a batch of buffers to use.
+						if (buffers.empty()) {
+							std::lock_guard<std::mutex> l(_buffers_l);
+							buffers.swap(_buffers);
+						}
+						if (buffers.empty()) {
+							buf = malloc(ZT_TAP_BUF_SIZE);
+							if (!buf)
+								break;
+						} else {
+							buf = buffers.back();
+							buffers.pop_back();
+						}
+					}
+
+					n = (int)::read(_fd,reinterpret_cast<uint8_t *>(buf) + r,ZT_TAP_BUF_SIZE - r);
+
+					if (n > 0) {
+						// Some tap drivers like to send the ethernet frame and the
+						// payload in two chunks, so handle that by accumulating
+						// data until we have at least a frame.
+						r += n;
+						if (r > 14) {
+							if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
+								r = _mtu + 14;
+
+							if (_enabled) {
+								_tapq.post(std::pair<void *,int>(buf,r));
+								buf = nullptr;
+							}
+
+							r = 0;
+						}
+					} else {
+						r = 0;
+						break;
+					}
+				}
+			}
+		}
+	});
+
+	_tapProcessorThread = std::thread([this] {
+		MAC to,from;
+		std::pair<void *,int> qi;
+		while (_tapq.get(qi)) {
+			uint8_t *const b = reinterpret_cast<uint8_t *>(qi.first);
+			if (b) {
+				to.setTo(b, 6);
+				from.setTo(b + 6, 6);
+				unsigned int etherType = Utils::ntoh(((const uint16_t *)b)[6]);
+				_handler(_arg, nullptr, _nwid, from, to, etherType, 0, (const void *)(b + 14),(unsigned int)(qi.second - 14));
+				{
+					std::lock_guard<std::mutex> l(_buffers_l);
+					if (_buffers.size() < 128)
+						_buffers.push_back(qi.first);
+					else free(qi.first);
+				}
+			} else break;
+		}
+	});
 }
 
 LinuxEthernetTap::~LinuxEthernetTap()
 {
-	(void)::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit
-	Thread::join(_thread);
+	(void)::write(_shutdownSignalPipe[1],"\0",1); // causes reader thread to exit
+	_tapq.post(std::pair<void *,int>(nullptr,0)); // causes processor thread to exit
+
 	::close(_fd);
 	::close(_shutdownSignalPipe[0]);
 	::close(_shutdownSignalPipe[1]);
+
+	_tapReaderThread.join();
+	_tapProcessorThread.join();
+
+	for(std::vector<void *>::iterator i(_buffers.begin());i!=_buffers.end();++i)
+		free(*i);
+	std::vector< std::pair<void *,int> > dv(_tapq.drain());
+	for(std::vector< std::pair<void *,int> >::iterator i(dv.begin());i!=dv.end();++i) {
+		if (i->first)
+			free(i->first);
+	}
 }
 
 void LinuxEthernetTap::setEnabled(bool en)
@@ -378,8 +468,8 @@ void LinuxEthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,
 {
 	char putBuf[ZT_MAX_MTU + 64];
 	if ((_fd > 0)&&(len <= _mtu)&&(_enabled)) {
-		to.copyTo((uint8_t *)putBuf);
-		from.copyTo((uint8_t *)(putBuf + 6));
+		to.copyTo(putBuf,6);
+		from.copyTo(putBuf + 6,6);
 		*((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType);
 		memcpy(putBuf + 14,data,len);
 		len += 14;
@@ -419,8 +509,8 @@ void LinuxEthernetTap::scanMulticastGroups(std::vector<MulticastGroup> &added,st
 						mcastmac = f;
 					++fno;
 				}
-				if ((devname)&&(!strcmp(devname,_dev.c_str()))&&(mcastmac)&&(Utils::unhex(mcastmac,strlen(mcastmac),mac,6) == 6))
-					newGroups.push_back(MulticastGroup(MAC(mac),0));
+				if ((devname)&&(!strcmp(devname,_dev.c_str()))&&(mcastmac)&&(Utils::unhex(mcastmac,mac,6) == 6))
+					newGroups.push_back(MulticastGroup(MAC(mac,6),0));
 			}
 		}
 		::close(fd);
@@ -460,58 +550,6 @@ void LinuxEthernetTap::setMtu(unsigned int mtu)
 	}
 }
 
-void LinuxEthernetTap::threadMain()
-	throw()
-{
-	fd_set readfds,nullfds;
-	MAC to,from;
-	int n,nfds,r;
-	char getBuf[ZT_MAX_MTU + 64];
-
-	Thread::sleep(500);
-
-	FD_ZERO(&readfds);
-	FD_ZERO(&nullfds);
-	nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
-
-	r = 0;
-	for(;;) {
-		FD_SET(_shutdownSignalPipe[0],&readfds);
-		FD_SET(_fd,&readfds);
-		select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
-
-		if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
-			break;
-
-		if (FD_ISSET(_fd,&readfds)) {
-			n = (int)::read(_fd,getBuf + r,sizeof(getBuf) - r);
-			if (n < 0) {
-				if ((errno != EINTR)&&(errno != ETIMEDOUT))
-					break;
-			} else {
-				// Some tap drivers like to send the ethernet frame and the
-				// payload in two chunks, so handle that by accumulating
-				// data until we have at least a frame.
-				r += n;
-				if (r > 14) {
-					if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
-						r = _mtu + 14;
-
-					if (_enabled) {
-						to.setTo((uint8_t *)getBuf);
-						from.setTo((uint8_t *)(getBuf + 6));
-						unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]);
-						// TODO: VLAN support
-						_handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14);
-					}
-
-					r = 0;
-				}
-			}
-		}
-	}
-}
-
 } // namespace ZeroTier
 
 #endif // __LINUX__

osdep/LinuxEthernetTap.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)2013-2020 ZeroTier, Inc.
+ * Copyright (c)2019 ZeroTier, Inc.
  *
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
@@ -21,10 +21,12 @@
 #include <vector>
 #include <stdexcept>
 #include <atomic>
-
-#include "../core/MulticastGroup.hpp"
-#include "Thread.hpp"
+#include <array>
+#include <thread>
+#include <mutex>
+#include "../node/MulticastGroup.hpp"
 #include "EthernetTap.hpp"
+#include "BlockingQueue.hpp"
 
 namespace ZeroTier {
 
@@ -54,15 +56,13 @@ public:
 	virtual void setFriendlyName(const char *friendlyName);
 	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
 	virtual void setMtu(unsigned int mtu);
-
-	void threadMain()
-		throw();
+	virtual void setDns(const char *domain, const std::vector<InetAddress> &servers) {}
 
 private:
 	void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int);
 	void *_arg;
 	uint64_t _nwid;
-	Thread _thread;
+	MAC _mac;
 	std::string _homePath;
 	std::string _dev;
 	std::vector<MulticastGroup> _multicastGroups;
@@ -70,6 +70,11 @@ private:
 	int _fd;
 	int _shutdownSignalPipe[2];
 	std::atomic_bool _enabled;
+	std::thread _tapReaderThread;
+	std::thread _tapProcessorThread;
+	std::mutex _buffers_l;
+	std::vector<void *> _buffers;
+	BlockingQueue< std::pair<void *,int> > _tapq;
 };
 
 } // namespace ZeroTier

osdep/LinuxNetLink.cpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)2013-2020 ZeroTier, Inc.
+ * Copyright (c)2019 ZeroTier, Inc.
  *
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
@@ -11,7 +11,9 @@
  */
 /****/
 
-#include "../core/Constants.hpp"
+#include "../node/Constants.hpp"
+
+//#define ZT_NETLINK_TRACE
 
 #ifdef __LINUX__
 
@@ -20,6 +22,10 @@
 #include <unistd.h>
 #include <linux/if_tun.h>
 
+#ifndef IFNAMSIZ
+#define IFNAMSIZ 16
+#endif
+
 namespace ZeroTier {
 
 struct nl_route_req {
@@ -43,10 +49,6 @@ struct nl_adr_req {
 LinuxNetLink::LinuxNetLink()
 	: _t()
 	, _running(false)
-	, _routes_ipv4()
-	, _rv4_m()
-	, _routes_ipv6()
-	, _rv6_m()
 	, _seq(0)
 	, _interfaces()
 	, _if_m()
@@ -85,7 +87,7 @@ void LinuxNetLink::_setSocketTimeout(int fd, int seconds)
 	tv.tv_sec = seconds;
 	tv.tv_usec = 0;
 	if(setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv)) != 0) {
-#ifdef ZT_TRACE
+#ifdef ZT_NETLINK_TRACE
 		fprintf(stderr, "setsockopt failed: %s\n", strerror(errno));
 #endif
 	}
@@ -119,8 +121,8 @@ int LinuxNetLink::_doRecv(int fd)
 			if(nlp->nlmsg_type == NLMSG_ERROR && (nlp->nlmsg_flags & NLM_F_ACK) != NLM_F_ACK) {
 				struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(nlp);
 				if (err->error != 0) {
-#ifdef ZT_TRACE
-					//fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error)));
+#ifdef ZT_NETLINK_TRACE
+					fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error)));
 #endif
 				}
 				p = buf;
@@ -145,9 +147,9 @@ int LinuxNetLink::_doRecv(int fd)
 			}
 
 			if (nlp->nlmsg_type == NLMSG_OVERRUN) {
-//#ifdef ZT_TRACE
+#ifdef ZT_NETLINK_TRACE
 				fprintf(stderr, "NLMSG_OVERRUN: Data lost\n");
-//#endif
+#endif
 				p = buf;
 				nll = 0;
 				break;
@@ -173,11 +175,10 @@ int LinuxNetLink::_doRecv(int fd)
 void LinuxNetLink::threadMain() throw()
 {
 	int rtn = 0;
-
 	while(_running) {
 		rtn = _doRecv(_fd);
 		if (rtn <= 0) {
-			Thread::sleep(100);
+			Thread::sleep(250);
 			continue;
 		}
 	}
@@ -215,6 +216,7 @@ void LinuxNetLink::_processMessage(struct nlmsghdr *nlp, int nll)
 
 void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp)
 {
+#ifdef ZT_NETLINK_TRACE
 	struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp);
 	struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap);
 	int ifal = IFA_PAYLOAD(nlp);
@@ -242,13 +244,13 @@ void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp)
 		}
 	}
 
-#ifdef ZT_TRACE
-	//fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
+	fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
 #endif
 }
 
 void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp)
 {
+#ifdef ZT_NETLINK_TRACE
 	struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp);
 	struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap);
 	int ifal = IFA_PAYLOAD(nlp);
@@ -276,8 +278,7 @@ void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp)
 		}
 	}
 
-#ifdef ZT_TRACE
-	//fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
+	fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
 #endif
 }
 
@@ -293,28 +294,79 @@ void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp)
 	struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp);
 	int rtl = RTM_PAYLOAD(nlp);
 
+	Route r;
+	bool wecare = false;
+
 	for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl))
 	{
 		switch(rtap->rta_type)
 		{
 		case RTA_DST:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_SRC:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24: 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_GATEWAY:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_OIF:
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+				case AF_INET6:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+			}
 			sprintf(ifs, "%d", *((int*)RTA_DATA(rtap)));
 			break;
 		}
 	}
-	sprintf(ms, "%d", rtp->rtm_dst_len);
 
-#ifdef ZT_TRACE
-	//fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
+	if (wecare) {
+		Mutex::Lock rl(_routes_m);
+		_routes[r.target].insert(r);
+	}
+
+#ifdef ZT_NETLINK_TRACE
+	sprintf(ms, "%d", rtp->rtm_dst_len);
+	fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
 #endif
 }
 
@@ -330,28 +382,79 @@ void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp)
 	struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp);
 	int rtl = RTM_PAYLOAD(nlp);
 
+	Route r;
+	bool wecare = false;
+
 	for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl))
 	{
 		switch(rtap->rta_type)
 		{
 		case RTA_DST:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_SRC:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_GATEWAY:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_OIF:
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+				case AF_INET6:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+			}
 			sprintf(ifs, "%d", *((int*)RTA_DATA(rtap)));
 			break;
 		}
 	}
-	sprintf(ms, "%d", rtp->rtm_dst_len);
 
-#ifdef ZT_TRACE
-	//fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
+	if (wecare) {
+		Mutex::Lock rl(_routes_m);
+		_routes[r.target].erase(r);
+	}
+
+#ifdef ZT_NETLINK_TRACE
+	sprintf(ms, "%d", rtp->rtm_dst_len);
+	fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
 #endif
 }
 
@@ -419,7 +522,7 @@ void LinuxNetLink::_linkDeleted(struct nlmsghdr *nlp)
 
 	{
 		Mutex::Lock l(_if_m);
-		if(_interfaces.find(ifip->ifi_index) != _interfaces.end()) {
+		if(_interfaces.contains(ifip->ifi_index)) {
 			_interfaces.erase(ifip->ifi_index);
 		}
 	}
@@ -605,11 +708,11 @@ void LinuxNetLink::addRoute(const InetAddress &target, const InetAddress &via, c
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char  tmp[64];
-	//char tmp2[64];
-	//char tmp3[64];
-	//fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
+#ifdef ZT_NETLINK_TRACE
+	char  tmp[64];
+	char tmp2[64];
+	char tmp3[64];
+	fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
 #endif
 
 	int rtl = sizeof(struct rtmsg);
@@ -668,7 +771,7 @@ void LinuxNetLink::addRoute(const InetAddress &target, const InetAddress &via, c
 	req.nl.nlmsg_type = RTM_NEWROUTE;
 	req.nl.nlmsg_pid = 0;
 	req.nl.nlmsg_seq = ++_seq;
-	req.rt.rtm_family = target.family();
+	req.rt.rtm_family = target.ss_family;
 	req.rt.rtm_table = RT_TABLE_MAIN;
 	req.rt.rtm_protocol = RTPROT_STATIC;
 	req.rt.rtm_scope = RT_SCOPE_UNIVERSE;
@@ -720,11 +823,11 @@ void LinuxNetLink::delRoute(const InetAddress &target, const InetAddress &via, c
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char  tmp[64];
-	//char tmp2[64];
-	//char tmp3[64];
-	//fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
+#ifdef ZT_NETLINK_TRACE
+	char  tmp[64];
+	char tmp2[64];
+	char tmp3[64];
+	fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
 #endif
 
 	int rtl = sizeof(struct rtmsg);
@@ -783,7 +886,7 @@ void LinuxNetLink::delRoute(const InetAddress &target, const InetAddress &via, c
 	req.nl.nlmsg_type = RTM_DELROUTE;
 	req.nl.nlmsg_pid = 0;
 	req.nl.nlmsg_seq = ++_seq;
-	req.rt.rtm_family = target.family();
+	req.rt.rtm_family = target.ss_family;
 	req.rt.rtm_table = RT_TABLE_MAIN;
 	req.rt.rtm_protocol = RTPROT_STATIC;
 	req.rt.rtm_scope = RT_SCOPE_UNIVERSE;
@@ -839,9 +942,9 @@ void LinuxNetLink::addAddress(const InetAddress &addr, const char *iface)
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char tmp[128];
-	//fprintf(stderr, "Adding IP address %s to interface %s", addr.toString(tmp), iface);
+#ifdef ZT_NETLINK_TRACE
+	char tmp[128];
+	fprintf(stderr, "Adding IP address %s to interface %s\n", addr.toString(tmp), iface);
 #endif
 
 	int interface_index = _indexForInterface(iface);
@@ -904,7 +1007,7 @@ void LinuxNetLink::addAddress(const InetAddress &addr, const char *iface)
 	req.nl.nlmsg_type = RTM_NEWADDR;
 	req.nl.nlmsg_pid = 0;
 	req.nl.nlmsg_seq = ++_seq;
-	req.ifa.ifa_family = addr.family();
+	req.ifa.ifa_family = addr.ss_family;
 	req.ifa.ifa_prefixlen = addr.port();
 	req.ifa.ifa_flags = IFA_F_PERMANENT;
 	req.ifa.ifa_scope = 0;
@@ -955,9 +1058,9 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface)
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char tmp[128];
-	//fprintf(stderr, "Removing IP address %s from interface %s", addr.toString(tmp), iface);
+#ifdef ZT_NETLINK_TRACE
+	char tmp[128];
+	fprintf(stderr, "Removing IP address %s from interface %s\n", addr.toString(tmp), iface);
 #endif
 
 	int interface_index = _indexForInterface(iface);
@@ -1016,7 +1119,7 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface)
 	req.nl.nlmsg_type = RTM_DELADDR;
 	req.nl.nlmsg_pid = 0;
 	req.nl.nlmsg_seq = ++_seq;
-	req.ifa.ifa_family = addr.family();
+	req.ifa.ifa_family = addr.ss_family;
 	req.ifa.ifa_prefixlen = addr.port();
 	req.ifa.ifa_flags = IFA_F_PERMANENT;
 	req.ifa.ifa_scope = 0;
@@ -1043,23 +1146,35 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface)
 	close(fd);
 }
 
-RouteList LinuxNetLink::getIPV4Routes() const
+bool LinuxNetLink::routeIsSet(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname)
 {
-	return _routes_ipv4;
-}
-
-RouteList LinuxNetLink::getIPV6Routes() const
-{
-	return _routes_ipv6;
+	Mutex::Lock rl(_routes_m);
+	const std::set<LinuxNetLink::Route> &rs = _routes[target];
+	for(std::set<LinuxNetLink::Route>::const_iterator ri(rs.begin());ri!=rs.end();++ri) {
+		if ((ri->via == via)&&(ri->src == src)) {
+			if (ifname) {
+				Mutex::Lock ifl(_if_m);
+				const iface_entry *ife = _interfaces.get(ri->ifidx);
+				if ((ife)&&(!strncmp(ife->ifacename,ifname,IFNAMSIZ)))
+					return true;
+			} else {
+				return true;
+			}
+		}
+	}
+	return false;
 }
 
 int LinuxNetLink::_indexForInterface(const char *iface)
 {
 	Mutex::Lock l(_if_m);
 	int interface_index = -1;
-	for(std::map<int, iface_entry>::iterator i(_interfaces.begin());i!=_interfaces.end();++i) {
-		if (strcmp(iface, i->second.ifacename) == 0) {
-			interface_index = i->second.index;
+	Hashtable<int, iface_entry>::Iterator iter(_interfaces);
+	int *k = NULL;
+	iface_entry *v = NULL;
+	while(iter.next(k,v)) {
+		if(strcmp(iface, v->ifacename) == 0) {
+			interface_index = v->index;
 			break;
 		}
 	}

osdep/LinuxNetLink.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)2013-2020 ZeroTier, Inc.
+ * Copyright (c)2019 ZeroTier, Inc.
  *
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
@@ -14,32 +14,28 @@
 #ifndef ZT_LINUX_NETLINK_HPP
 #define ZT_LINUX_NETLINK_HPP
 
-#include "../core/Constants.hpp"
+#include "../node/Constants.hpp"
 
 #ifdef __LINUX__
 
 #include <vector>
+#include <map>
+#include <set>
 
 #include <sys/socket.h>
 #include <asm/types.h>
 #include <linux/rtnetlink.h>
 #include <sys/socket.h>
-#include <linux/if.h>
+//#include <linux/if.h>
 
-#include "../core/InetAddress.hpp"
-#include "../core/MAC.hpp"
+#include "../node/InetAddress.hpp"
+#include "../node/MAC.hpp"
 #include "Thread.hpp"
-#include "../core/Mutex.hpp"
+#include "../node/Hashtable.hpp"
+#include "../node/Mutex.hpp"
 
-namespace ZeroTier {
 
-struct route_entry {
-	InetAddress target;
-	InetAddress via;
-	int if_index;
-	char iface[IFNAMSIZ];
-};
-typedef std::vector<route_entry> RouteList;
+namespace ZeroTier {
 
 /**
  * Interface with Linux's RTNETLINK
@@ -51,6 +47,41 @@ private:
 	~LinuxNetLink();
 
 public:
+	struct Route {
+		InetAddress target;
+		InetAddress via;
+		InetAddress src;
+		int ifidx;
+
+		inline bool operator==(const Route &r) const
+		{ return ((target == r.target)&&(via == r.via)&&(src == r.src)&&(ifidx == r.ifidx)); }
+		inline bool operator!=(const Route &r) const
+		{ return (!(*this == r)); }
+		inline bool operator<(const Route &r) const
+		{
+			if (target < r.target) {
+				return true;
+			} else if (target == r.target) {
+				if (via < r.via) {
+					return true;
+				} else if (via == r.via) {
+					if (src < r.src) {
+						return true;
+					} else if (src == r.src) {
+						return (ifidx < r.ifidx);
+					}
+				}
+			}
+			return false;
+		}
+		inline bool operator>(const Route &r) const
+		{ return (r < *this); }
+		inline bool operator<=(const Route &r) const
+		{ return !(r < *this); }
+		inline bool operator>=(const Route &r) const
+		{ return !(*this < r); }
+	};
+
 	static LinuxNetLink& getInstance()
 	{
 		static LinuxNetLink instance;
@@ -62,12 +93,12 @@ public:
 
 	void addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
 	void delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
-	RouteList getIPV4Routes() const;
-	RouteList getIPV6Routes() const;
 
 	void addAddress(const InetAddress &addr, const char *iface);
 	void removeAddress(const InetAddress &addr, const char *iface);
 
+	bool routeIsSet(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname);
+
 	void threadMain() throw();
 
 private:
@@ -92,21 +123,21 @@ private:
 	Thread _t;
 	bool _running;
 
-	RouteList _routes_ipv4;
-	Mutex _rv4_m;
-	RouteList _routes_ipv6;
-	Mutex _rv6_m;
-
 	uint32_t _seq;
 
+	std::map< InetAddress,std::set<LinuxNetLink::Route> > _routes;
+	Mutex _routes_m;
+
 	struct iface_entry {
+		iface_entry()
+		{ memset(this,0,sizeof(iface_entry)); }
 		int index;
-		char ifacename[IFNAMSIZ];
+		char ifacename[16]; // IFNAMSIZ on Linux == 16
 		char mac[18];
 		char mac_bin[6];
 		unsigned int mtu;
 	};
-	std::map<int, iface_entry> _interfaces;
+	Hashtable<int, iface_entry> _interfaces;
 	Mutex _if_m;
 
 	// socket communication vars;
@@ -118,4 +149,4 @@ private:
 
 #endif
 
-#endif // ZT_LINUX_NETLINK_HPPS
+#endif // ZT_LINUX_NETLINK_HPPS