Add custom management plane for 3rd party vendors

make-linux.mk

@@ -16,8 +16,13 @@ DESTDIR?=
 EXTRA_DEPS?=
 
 include objects.mk
-ONE_OBJS+=osdep/LinuxEthernetTap.o
-ONE_OBJS+=osdep/LinuxNetLink.o
+ifeq ($(ZT_EXTOSDEP),1)
+	ONE_OBJS+=osdep/ExtOsdep.o
+	override DEFS += -DZT_EXTOSDEP
+else
+	ONE_OBJS+=osdep/LinuxEthernetTap.o
+	ONE_OBJS+=osdep/LinuxNetLink.o
+endif
 
 # for central controller buildsk
 TIMESTAMP=$(shell date +"%Y%m%d%H%M")
@@ -275,6 +280,10 @@ ifeq ($(CC_MACH),loongarch64)
 	override DEFS+=-DZT_NO_TYPE_PUNNING
 endif
 
+ifeq ($(ZT_EXTOSDEP), 1)
+	ZT_SSO_SUPPORTED=0
+endif
+
 # Fail if system architecture could not be determined
 ifeq ($(ZT_ARCHITECTURE),999)
 ERR=$(error FATAL: architecture could not be determined from $(CC) -dumpmachine: $(CC_MACH))
@@ -339,8 +348,11 @@ ifeq ($(ZT_ARCHITECTURE),3)
 		override CXXFLAGS+=-march=armv5t -mfloat-abi=soft -msoft-float -mno-unaligned-access -marm
 		ZT_USE_ARM32_NEON_ASM_CRYPTO=0
 	else
-		override CFLAGS+=-mfloat-abi=hard -march=armv6zk -marm -mfpu=vfp -mno-unaligned-access -mtp=cp15 -mcpu=arm1176jzf-s
-		override CXXFLAGS+=-mfloat-abi=hard -march=armv6zk -marm -mfpu=vfp -fexceptions -mno-unaligned-access -mtp=cp15 -mcpu=arm1176jzf-s
+		ifeq ($(ZT_EXTOSDEP), 0)
+			override CFLAGS+=-mfloat-abi=hard -march=armv6zk -marm -mfpu=vfp -mno-unaligned-access -mtp=cp15 -mcpu=arm1176jzf-s
+			override CXXFLAGS+=-mfloat-abi=hard -march=armv6zk -marm -mfpu=vfp -fexceptions -mno-unaligned-access -mtp=cp15 -mcpu=arm1176jzf-s
+			override DEFS+=-DZT_NO_PEER_METRICS
+		endif
 		ZT_USE_ARM32_NEON_ASM_CRYPTO=0
 	endif
 endif

one.cpp

@@ -57,6 +57,7 @@
 #include <sys/socket.h>
 #include <ifaddrs.h>
 #include <sys/ioctl.h>
+#include "osdep/ExtOsdep.hpp"
 #ifndef ZT_NO_CAPABILITIES
 #include <linux/capability.h>
 #include <linux/securebits.h>
@@ -2110,6 +2111,17 @@ int main(int argc,char **argv)
 	signal(SIGQUIT,&_sighandlerQuit);
 	signal(SIGINT,&_sighandlerQuit);
 
+#ifdef ZT_EXTOSDEP
+	int extosdepFd1 = -1;
+	int extosdepFd2 = -1;
+	for(int i=1;i<argc;++i) {
+		if (argv[i][0] != '-' || argv[i][1] != 'x') continue;
+		if (sscanf(argv[i] + 2, "%d,%d", &extosdepFd1, &extosdepFd2) == 2) break;
+		fprintf(stderr, "bad extosdepFd\n");
+		return 1;
+	}
+#endif // ZT_EXTOSDEP
+
 	/* Ensure that there are no inherited file descriptors open from a previous
 	 * incarnation. This is a hack to ensure that GitHub issue #61 or variants
 	 * of it do not return, and should not do anything otherwise bad. */
@@ -2117,8 +2129,12 @@ int main(int argc,char **argv)
 		int mfd = STDIN_FILENO;
 		if (STDOUT_FILENO > mfd) mfd = STDOUT_FILENO;
 		if (STDERR_FILENO > mfd) mfd = STDERR_FILENO;
-		for(int f=mfd+1;f<1024;++f)
+		for(int f=mfd+1;f<1024;++f) {
+#ifdef ZT_EXTOSDEP
+			if (f == extosdepFd1 || f == extosdepFd2) continue;
+#endif // ZT_EXTOSDEP
 			::close(f);
+		}
 	}
 
 	bool runAsDaemon = false;
@@ -2224,7 +2240,9 @@ int main(int argc,char **argv)
 						return 0;
 					} break;
 #endif // __WINDOWS__
-
+#ifdef ZT_EXTOSDEP
+				case 'x': break;
+#endif
 				case 'h':
 				case '?':
 				default:
@@ -2354,6 +2372,15 @@ int main(int argc,char **argv)
 	}
 #endif // __UNIX_LIKE__
 
+#ifdef ZT_EXTOSDEP
+	if (extosdepFd1 < 0) {
+		fprintf(stderr, "no extosdepFd specified\n");
+		OSUtils::rm(pidPath.c_str());
+		return 1;
+	}
+	ExtOsdep::init(extosdepFd1, extosdepFd2);
+#endif
+
 	_OneServiceRunner thr(argv[0],homeDir,port);
 	thr.threadMain();
 	//Thread::join(Thread::start(&thr));

osdep/Binder.hpp

@@ -53,6 +53,7 @@
 #include "../node/Utils.hpp"
 #include "OSUtils.hpp"
 #include "Phy.hpp"
+#include "../osdep/ExtOsdep.hpp"
 
 #include <algorithm>
 #include <atomic>
@@ -136,6 +137,25 @@ class Binder {
 		bool interfacesEnumerated = true;
 
 		if (explicitBind.empty()) {
+#ifdef ZT_EXTOSDEP
+                        std::map<InetAddress,std::string> addrs;
+                        interfacesEnumerated = ExtOsdep::getBindAddrs(addrs);
+                        for (auto &a : addrs) {
+                                auto ip = a.first;
+                                switch(ip.ipScope()) {
+                                        default: break;
+                                        case InetAddress::IP_SCOPE_PSEUDOPRIVATE:
+                                        case InetAddress::IP_SCOPE_GLOBAL:
+                                        case InetAddress::IP_SCOPE_SHARED:
+                                        case InetAddress::IP_SCOPE_PRIVATE:
+                                                for(int x=0;x<(int)portCount;++x) {
+                                                        ip.setPort(ports[x]);
+                                                        localIfAddrs.insert(std::pair<InetAddress,std::string>(ip,a.second));
+                                                }
+                                                break;
+                                }
+                        }
+#else // ZT_EXTOSDEP
 #ifdef __WINDOWS__
 
 			char aabuf[32768];
@@ -386,6 +406,8 @@ class Binder {
 #endif
 
 #endif
+
+#endif // ZT_EXTOSDEP
 		}
 		else {
 			for (std::vector<InetAddress>::const_iterator i(explicitBind.begin()); i != explicitBind.end(); ++i) {

osdep/EthernetTap.cpp

@@ -32,6 +32,7 @@
 #endif // __APPLE__
 
 #ifdef __LINUX__
+#include "ExtOsdep.hpp"
 #include "LinuxEthernetTap.hpp"
 #endif // __LINUX__
 
@@ -94,7 +95,11 @@ std::shared_ptr<EthernetTap> EthernetTap::newInstance(
 #endif // __APPLE__
 
 #ifdef __LINUX__
+#ifdef ZT_EXTOSDEP
+	return std::shared_ptr<EthernetTap>(new ExtOsdepTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+#else
 	return std::shared_ptr<EthernetTap>(new LinuxEthernetTap(homePath,concurrency,pinning,mac,mtu,metric,nwid,friendlyName,handler,arg));
+#endif // ZT_EXTOSDEP
 #endif // __LINUX__
 
 #ifdef __WINDOWS__

osdep/ExtOsdep.cpp

@@ -0,0 +1,534 @@
+#include <sys/times.h>
+#include <fcntl.h>
+#include <unistd.h>
+#include <iostream>
+#include "ExtOsdep.hpp"
+#include <list>
+#include "../node/AtomicCounter.hpp"
+
+#define ZT_TAP_BUF_SIZE 16384
+
+namespace ZeroTier {
+
+static int eodFd = -1;
+static Mutex eodMutex;
+static int eodMgmtFd = -1;
+
+struct EodRoute {
+	InetAddress target;
+	InetAddress via;
+	InetAddress src;
+	std::string ifname;
+};
+static std::list<EodRoute> allRoutes;
+
+template<typename T> static void __eodSend(const T &t) {
+	write(eodFd, &t, sizeof(t));
+}
+
+static void strncpyx(char *dest, const char *src, size_t n) {
+	strncpy(dest, src, n);
+	if (n > 1) dest[n - 1] = 0;
+}
+
+static int __eodWait(unsigned char msg, unsigned char *d, unsigned l,
+		unsigned maxl = 0, int *recvfd = nullptr) {
+	if (!maxl) maxl = l;
+	auto start = times(NULL);
+	while (1) {
+		msghdr mh;
+		iovec iov;
+		struct {
+			size_t cmsg_len;
+			int cmsg_level;
+			int cmsg_type;
+			int fd;
+		} __attribute__((packed)) cmsg;
+		memset(&mh, 0, sizeof(mh));
+		mh.msg_iov = &iov;
+		mh.msg_iovlen = 1;
+		if (recvfd) {
+			mh.msg_control = &cmsg;
+			mh.msg_controllen = sizeof(cmsg);
+		}
+		iov.iov_base = d;
+		iov.iov_len = maxl;
+
+		int r = recvmsg(eodFd, &mh, MSG_TRUNC | MSG_CMSG_CLOEXEC);
+		if (r > 0) {
+			if (recvfd && mh.msg_controllen >= sizeof(cmsg)
+				&& cmsg.cmsg_len == sizeof(cmsg)
+				&& cmsg.cmsg_level == SOL_SOCKET
+				&& cmsg.cmsg_type == SCM_RIGHTS) {
+				*recvfd = cmsg.fd;
+				fprintf(stderr, "eodWait: received fd %d\n", *recvfd);
+			}
+
+			if (d[0] != msg) {
+				fprintf(stderr, "eodWait: wrong msg, expected %u got %u\n", msg, d[0]);
+				return -1;
+			}
+			if ((unsigned)r < l || (unsigned)r > maxl) {
+				fprintf(stderr, "eodWait: wrong len, expected %u got %d\n", l, r);
+				return -1;
+			}
+			return r;
+		}
+		if (times(NULL) - start > 500) {
+			fprintf(stderr, "eodWait: timeout\n");
+			return -1;
+		}
+		usleep(100000);
+	}
+}
+
+template<typename T> static bool __eodWait(unsigned msg, T &t) {
+	return __eodWait(msg, (unsigned char *)&t, sizeof(T)) == (int)sizeof(T);
+}
+
+template<typename M, typename R> static bool __eodXchg(const M &m, unsigned rm, R &r) {
+	__eodSend(m);
+	return __eodWait(rm, r);
+}
+
+template<typename M, typename R> static bool eodXchg(const M &m, unsigned rm, R &r) {
+	Mutex::Lock l(eodMutex);
+	return __eodXchg(m, rm, r);
+}
+
+void ExtOsdep::init(int fd1, int fd2) {
+	eodFd = fd1;
+	eodMgmtFd = fd2;
+	fcntl(eodMgmtFd,F_SETFL,O_NONBLOCK);
+}
+
+void ExtOsdep::started(int *f, void **cp) {
+	*f = eodMgmtFd;
+	*cp = (void *)eodMgmtFd;
+
+	unsigned char msg = ZT_EOD_MSG_STARTED;
+	Mutex::Lock l(eodMutex);
+	__eodSend(msg);
+}
+
+static std::string mgmtrd;
+static std::string mgmtwr;
+
+bool ExtOsdep::mgmtWritable(void *cookie) {
+	if (cookie != (void *)eodMgmtFd) return false;
+	if (mgmtwr.size() == 0) return true;
+	auto sz = write(eodMgmtFd, mgmtwr.data(), mgmtwr.size());
+	if (sz <= 0) return false;
+	mgmtwr.erase(mgmtwr.begin(), mgmtwr.begin() + sz);
+	return mgmtwr.empty();
+}
+
+bool ExtOsdep::mgmtRecv(void *cookie, void *data, unsigned long len,
+		std::function<unsigned (unsigned, const std::string &, const std::string &, std::string &)> cb) {
+	if (cookie != (void *)eodMgmtFd) return false;
+	mgmtrd.append((char *)data, len);
+	while (1) {
+		auto req = (zt_eod_mgmt_req *)mgmtrd.data();
+		if (mgmtrd.size() < sizeof(*req)) break;
+		unsigned reqsz = sizeof(*req) + req->pathlen + req->datalen;
+		if (mgmtrd.size() < reqsz) break;
+
+		std::string resp;
+		char *p = (char *)req->data;
+		zt_eod_mgmt_reply rep;
+		rep.scode = cb(req->method, std::string(p, p + req->pathlen),
+			std::string(p + req->pathlen, p + req->pathlen + req->datalen), resp);
+		rep.datalen = resp.size();
+
+		mgmtrd.erase(mgmtrd.begin(), mgmtrd.begin() + reqsz);
+
+		mgmtwr.append((char *)&rep, sizeof(rep));
+		mgmtwr.append(resp);
+
+		auto sz = write(eodMgmtFd, mgmtwr.data(), mgmtwr.size());
+		if (sz > 0) mgmtwr.erase(mgmtwr.begin(), mgmtwr.begin() + sz);
+	}
+	return !mgmtwr.empty();
+}
+
+void ExtOsdep::routeAddDel(bool add, const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname) {
+	Mutex::Lock l(eodMutex);
+
+	std::string ifn;
+	if (ifname) ifn = ifname;
+	if (add) {
+		for (auto x = allRoutes.begin(); x != allRoutes.end(); ++x) {
+			if (x->target == target && x->via == via
+				&& x->src == src && x->ifname == ifn) return;
+		}
+		allRoutes.push_back({target, via, src, ifn});
+	}
+	else {
+		bool found = false;
+		for (auto x = allRoutes.begin(); x != allRoutes.end(); ++x) {
+			if (x->target == target && x->via == via
+				&& x->src == src && x->ifname == ifn) {
+				allRoutes.erase(x);
+				found = true;
+				break;
+			}
+		}
+		if (!found) return;
+	}
+
+	zt_eod_msg_route req;
+	memset(&req, 0, sizeof(req));
+	req.cmd = add ? ZT_EOD_MSG_ADDROUTE : ZT_EOD_MSG_DELROUTE;
+	req.afi = target.isV4() ? 1 : 2;
+	req.dstlen = target.netmaskBits();
+	memcpy(req.dst, target.rawIpData(), target.isV4() ? 4 : 16);
+	if (ifname) strncpyx(req.dev, ifname, sizeof(req.dev));
+	if (via) memcpy(req.gw, via.rawIpData(), target.isV4() ? 4 : 16);
+	if (src) memcpy(req.src, src.rawIpData(), target.isV4() ? 4 : 16);
+
+	unsigned char resp;
+	__eodXchg(req, add ? ZT_EOD_MSG_ADDROUTERESP : ZT_EOD_MSG_DELROUTERESP, resp);
+}
+
+bool ExtOsdep::getBindAddrs(std::map<InetAddress,std::string> &ret) {
+	Mutex::Lock l(eodMutex);
+
+	unsigned char req = ZT_EOD_MSG_GETBINDADDRS;
+	__eodSend(req);
+
+	zt_eod_msg_getbindaddrsresp *resp;
+	unsigned char buf[ZT_EOD_MAXMSGSIZE];
+	int r = __eodWait(ZT_EOD_MSG_GETBINDADDRSRESP, (unsigned char *)buf, sizeof(*resp), sizeof(buf));
+	if (r < (int)sizeof(*resp)) return false;
+
+	int c = (r - (int)sizeof(*resp)) / sizeof(resp->addrs[0]);
+	resp = (zt_eod_msg_getbindaddrsresp *)buf;
+	for (int i = 0; i < c; ++i) {
+		ret[InetAddress(resp->addrs[i].data, resp->addrs[i].afi == 1 ? 4 : 16, resp->addrs[i].len)]
+			= resp->addrs[i].ifname;
+	}
+
+	return resp->result;
+}
+
+ExtOsdepTap::ExtOsdepTap(
+                const char *homePath,
+                const MAC &mac,
+                unsigned int mtu,
+                unsigned int metric,
+                uint64_t nwid,
+                const char *friendlyName,
+                void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
+                void *arg) :
+        _handler(handler),
+        _arg(arg),
+        _nwid(nwid),
+        _mac(mac),
+        _homePath(homePath),
+        _mtu(mtu),
+        _fd(0),
+        _enabled(true),
+        _run(true)
+ {
+	zt_eod_msg_addtap req;
+	req.cmd = ZT_EOD_MSG_ADDTAP;
+	req.nwid = nwid;
+	req.mtu = mtu;
+	req.metric = metric;
+	strncpyx(req.fname, friendlyName, sizeof(req.fname));
+	mac.copyTo(req.mac, 6);
+
+	zt_eod_msg_addtapresp resp;
+
+	Mutex::Lock l(eodMutex);
+	__eodSend(req);
+	_fd = -1;
+	if (__eodWait(ZT_EOD_MSG_ADDTAPRESP, (unsigned char *)&resp, sizeof(resp), sizeof(resp), &_fd) != sizeof(resp))
+		throw std::runtime_error(std::string("could not create TAP"));
+
+	_dev = resp.name;
+	if (_dev.empty() || _fd < 0)
+		throw std::runtime_error(std::string("could not create TAP"));
+
+	fcntl(_fd,F_SETFL,O_NONBLOCK);
+
+	// processing shamelessly copied from LinuxEthernetTap
+        (void)::pipe(_shutdownSignalPipe);
+        for(unsigned int t=0;t<2;++t) {
+                _tapReaderThread[t] = std::thread([this, t]{
+                        fd_set readfds,nullfds;
+                        int n,nfds,r;
+                        void *buf = nullptr;
+                        std::vector<void *> buffers;
+
+                        if (!_run)
+                                return;
+
+                        FD_ZERO(&readfds);
+                        FD_ZERO(&nullfds);
+                        nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
+
+                        r = 0;
+                        for(;;) {
+                                FD_SET(_shutdownSignalPipe[0],&readfds);
+                                FD_SET(_fd,&readfds);
+                                select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
+
+                                if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
+                                        break;
+
+                                if (FD_ISSET(_fd,&readfds)) {
+                                        for(;;) { // read until there are no more packets, then return to outer select() loop
+                                                if (!buf) {
+                                                        // To reduce use of the mutex, we keep a local buffer vector and
+                                                        // swap (which is a pointer swap) with the global one when it's
+                                                        // empty. This retrieves a batch of buffers to use.
+                                                        if (buffers.empty()) {
+                                                                std::lock_guard<std::mutex> l(_buffers_l);
+                                                                buffers.swap(_buffers);
+                                                        }
+                                                        if (buffers.empty()) {
+                                                                buf = malloc(ZT_TAP_BUF_SIZE);
+                                                                if (!buf)
+                                                                        break;
+                                                        } else {
+                                                                buf = buffers.back();
+                                                                buffers.pop_back();
+                                                        }
+                                                }
+
+                                                n = (int)::read(_fd,reinterpret_cast<uint8_t *>(buf) + r,ZT_TAP_BUF_SIZE - r);
+                                                if (n > 0) {
+                                                        // Some tap drivers like to send the ethernet frame and the
+                                                        // payload in two chunks, so handle that by accumulating
+                                                        // data until we have at least a frame.
+                                                        r += n;
+                                                        if (r > 14) {
+                                                                if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
+                                                                        r = _mtu + 14;
+
+                                                                if (_enabled && _tapqsize.load() < 1000) {
+									++_tapqsize;
+                                                                        _tapq.post(std::pair<void *,int>(buf,r));
+                                                                        buf = nullptr;
+                                                                }
+
+                                                                r = 0;
+                                                        }
+                                                } else {
+                                                        r = 0;
+                                                        break;
+                                                }
+                                        }
+                                }
+                        }
+                });
+        }
+
+        _tapProcessorThread = std::thread([this] {
+                MAC to,from;
+                std::pair<void *,int> qi;
+                while (_tapq.get(qi)) {
+			--_tapqsize;
+                        uint8_t *const b = reinterpret_cast<uint8_t *>(qi.first);
+                        if (b) {
+                                to.setTo(b, 6);
+                                from.setTo(b + 6, 6);
+                                unsigned int etherType = Utils::ntoh(((const uint16_t *)b)[6]);
+                                _handler(_arg, nullptr, _nwid, from, to, etherType, 0, (const void *)(b + 14),(unsigned int)(qi.second - 14));
+                                {
+                                        std::lock_guard<std::mutex> l(_buffers_l);
+                                        if (_buffers.size() < 128)
+                                                _buffers.push_back(qi.first);
+                                        else free(qi.first);
+                                }
+                        } else break;
+                }
+        });
+}
+
+ExtOsdepTap::~ExtOsdepTap() {
+        _run = false;
+
+        (void)::write(_shutdownSignalPipe[1],"\0",1); // causes reader thread(s) to exit
+        _tapq.post(std::pair<void *,int>(nullptr,0)); // causes processor thread to exit
+
+        _tapReaderThread[0].join();
+        _tapReaderThread[1].join();
+        _tapProcessorThread.join();
+
+        ::close(_fd);
+        ::close(_shutdownSignalPipe[0]);
+        ::close(_shutdownSignalPipe[1]);
+
+        for(std::vector<void *>::iterator i(_buffers.begin());i!=_buffers.end();++i)
+                free(*i);
+        std::vector< std::pair<void *,int> > dv(_tapq.drain());
+        for(std::vector< std::pair<void *,int> >::iterator i(dv.begin());i!=dv.end();++i) {
+                if (i->first)
+                        free(i->first);
+        }
+
+	zt_eod_msg_deltap req;
+	req.cmd = ZT_EOD_MSG_DELTAP;
+	strcpy(req.name, _dev.c_str());
+
+	unsigned char resp;
+	eodXchg(req, ZT_EOD_MSG_DELTAPRESP, resp);
+}
+
+void ExtOsdepTap::setEnabled(bool en) {
+        _enabled = en;
+}
+
+bool ExtOsdepTap::enabled() const {
+        return _enabled;
+}
+
+void ExtOsdepTap::doRemoveIp(const InetAddress &ip) {
+	zt_eod_msg_ip req;
+	req.cmd = ZT_EOD_MSG_DELIP;
+	strcpy(req.name, _dev.c_str());
+	req.afi = ip.isV4() ? 1 : 2;
+	req.len = ip.netmaskBits();
+	memcpy(req.data, ip.rawIpData(), ip.isV4() ? 4 : 16);
+
+	unsigned char resp;
+	__eodXchg(req, ZT_EOD_MSG_DELIPRESP, resp);
+}
+
+bool ExtOsdepTap::addIp(const InetAddress &ip) {
+	Mutex::Lock l(eodMutex);
+
+        for(auto i = allIps.begin();i!=allIps.end();++i) {
+		if (*i == ip) return true;
+                if (i->ipsEqual(ip)) doRemoveIp(*i);
+        }
+
+	zt_eod_msg_ip req;
+	req.cmd = ZT_EOD_MSG_ADDIP;
+	strcpy(req.name, _dev.c_str());
+	req.afi = ip.isV4() ? 1 : 2;
+	req.len = ip.netmaskBits();
+	memcpy(req.data, ip.rawIpData(), ip.isV4() ? 4 : 16);
+
+	unsigned char resp;
+	__eodXchg(req, ZT_EOD_MSG_ADDIPRESP, resp);
+
+	allIps.push_back(ip);
+
+	return true;
+}
+bool ExtOsdepTap::addIps(std::vector<InetAddress> ips) {
+	return false;
+}
+bool ExtOsdepTap::removeIp(const InetAddress &ip) {
+	Mutex::Lock l(eodMutex);
+        for(auto i = allIps.begin();i!=allIps.end();++i) {
+		if (*i == ip) {
+			doRemoveIp(*i);
+			return true;
+		}
+        }
+	return false;
+}
+std::vector<InetAddress> ExtOsdepTap::ips() const {
+	std::vector<InetAddress> ret;
+
+	Mutex::Lock l(eodMutex);
+
+	zt_eod_msg_getips req;
+	req.cmd = ZT_EOD_MSG_GETIPS;
+	strcpy(req.name, _dev.c_str());
+	__eodSend(req);
+
+	zt_eod_msg_getipsresp *resp;
+	unsigned char buf[ZT_EOD_MAXMSGSIZE];
+	int r = __eodWait(ZT_EOD_MSG_GETIPSRESP, (unsigned char *)buf, sizeof(*resp), sizeof(buf));
+	if (r < (int)sizeof(*resp)) return ret;
+
+	int c = (r - (int)sizeof(*resp)) / sizeof(resp->addrs[0]);
+	resp = (zt_eod_msg_getipsresp *)buf;
+	for (int i = 0; i < c; ++i) {
+		ret.push_back(InetAddress(resp->addrs[i].data, resp->addrs[i].afi == 1 ? 4 : 16, resp->addrs[i].len));
+	}
+
+	return ret;
+}
+void ExtOsdepTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) {
+        char putBuf[ZT_MAX_MTU + 64];
+        if ((_fd > 0)&&(len <= _mtu)&&(_enabled)) {
+                to.copyTo(putBuf,6);
+                from.copyTo(putBuf + 6,6);
+                *((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType);
+                memcpy(putBuf + 14,data,len);
+                len += 14;
+                (void)::write(_fd,putBuf,len);
+        }
+}
+std::string ExtOsdepTap::deviceName() const {
+	return _dev;
+}
+void ExtOsdepTap::setFriendlyName(const char *friendlyName) {}
+
+void ExtOsdepTap::scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed) {
+        char *ptr,*ptr2;
+        unsigned char mac[6];
+        std::vector<MulticastGroup> newGroups;
+
+        int fd = ::open("/proc/net/dev_mcast",O_RDONLY);
+        if (fd > 0) {
+                char buf[131072];
+                int n = (int)::read(fd,buf,sizeof(buf));
+                if ((n > 0)&&(n < (int)sizeof(buf))) {
+                        buf[n] = (char)0;
+                        for(char *l=strtok_r(buf,"\r\n",&ptr);(l);l=strtok_r((char *)0,"\r\n",&ptr)) {
+                                int fno = 0;
+                                char *devname = (char *)0;
+                                char *mcastmac = (char *)0;
+                                for(char *f=strtok_r(l," \t",&ptr2);(f);f=strtok_r((char *)0," \t",&ptr2)) {
+                                        if (fno == 1)
+                                                devname = f;
+                                        else if (fno == 4)
+                                                mcastmac = f;
+                                        ++fno;
+                                }
+                                if ((devname)&&(!strcmp(devname,_dev.c_str()))&&(mcastmac)&&(Utils::unhex(mcastmac,mac,6) == 6))
+                                        newGroups.push_back(MulticastGroup(MAC(mac,6),0));
+                        }
+                }
+                ::close(fd);
+        }
+
+        std::vector<InetAddress> allIps(ips());
+        for(std::vector<InetAddress>::iterator ip(allIps.begin());ip!=allIps.end();++ip)
+                newGroups.push_back(MulticastGroup::deriveMulticastGroupForAddressResolution(*ip));
+
+        std::sort(newGroups.begin(),newGroups.end());
+        newGroups.erase(std::unique(newGroups.begin(),newGroups.end()),newGroups.end());
+
+        for(std::vector<MulticastGroup>::iterator m(newGroups.begin());m!=newGroups.end();++m) {
+                if (!std::binary_search(_multicastGroups.begin(),_multicastGroups.end(),*m))
+                        added.push_back(*m);
+        }
+        for(std::vector<MulticastGroup>::iterator m(_multicastGroups.begin());m!=_multicastGroups.end();++m) {
+                if (!std::binary_search(newGroups.begin(),newGroups.end(),*m))
+                        removed.push_back(*m);
+        }
+
+        _multicastGroups.swap(newGroups);
+}
+void ExtOsdepTap::setMtu(unsigned int mtu) {
+	if (mtu == _mtu) return;
+	_mtu = mtu;
+
+	zt_eod_msg_setmtu req;
+	req.cmd = ZT_EOD_MSG_SETMTU;
+	strcpy(req.name, _dev.c_str());
+	req.mtu = mtu;
+
+	unsigned char resp;
+	eodXchg(req, ZT_EOD_MSG_SETMTURESP, resp);
+}
+
+} // namespace ZeroTier

osdep/ExtOsdep.hpp

@@ -0,0 +1,198 @@
+#ifndef ZT_EXTOSDEP_HPP
+#define ZT_EXTOSDEP_HPP
+
+#ifdef ZT_EXTOSDEP
+
+#define ZT_EOD_MAXMSGSIZE (64 * 1024)
+
+#define ZT_EOD_MSG_STARTED 1 // no data
+#define ZT_EOD_MSG_ADDTAP 2
+#define ZT_EOD_MSG_ADDTAPRESP 3
+#define ZT_EOD_MSG_DELTAP 4
+#define ZT_EOD_MSG_DELTAPRESP 5 // no data
+#define ZT_EOD_MSG_SETMTU 6
+#define ZT_EOD_MSG_SETMTURESP 7
+#define ZT_EOD_MSG_ADDIP 8
+#define ZT_EOD_MSG_ADDIPRESP 9
+#define ZT_EOD_MSG_DELIP 10
+#define ZT_EOD_MSG_DELIPRESP 11
+#define ZT_EOD_MSG_GETIPS 12
+#define ZT_EOD_MSG_GETIPSRESP 13
+#define ZT_EOD_MSG_GETBINDADDRS 14
+#define ZT_EOD_MSG_GETBINDADDRSRESP 15
+#define ZT_EOD_MSG_ADDROUTE 16
+#define ZT_EOD_MSG_ADDROUTERESP 17
+#define ZT_EOD_MSG_DELROUTE 18
+#define ZT_EOD_MSG_DELROUTERESP 19
+
+struct zt_eod_msg_addtap {
+	unsigned char cmd;
+	uint64_t nwid;
+	uint32_t mtu;
+	uint32_t metric;
+	char fname[128];
+	unsigned char mac[6];
+} __attribute__((packed));
+
+struct zt_eod_msg_addtapresp {
+	unsigned char cmd;
+	char name[16];
+} __attribute__((packed));
+
+struct zt_eod_msg_deltap {
+	unsigned char cmd;
+	char name[16];
+} __attribute__((packed));
+
+struct zt_eod_msg_setmtu {
+	unsigned char cmd;
+	char name[16];
+	unsigned mtu;
+} __attribute__((packed));
+
+struct zt_eod_msg_ip {
+	unsigned char cmd;
+	char name[16];
+	unsigned char afi; // 1 ip, 2 ip6
+	unsigned char len; // bits in mask
+	unsigned char data[16];
+} __attribute__((packed));
+
+struct zt_eod_msg_getips {
+	unsigned char cmd;
+	char name[16];
+} __attribute__((packed));
+
+struct zt_eod_msg_getipsresp {
+	unsigned char cmd;
+	struct addr {
+		unsigned char afi;
+		unsigned char len;
+		unsigned char data[16];
+	} __attribute__((packed)) addrs[0];
+} __attribute__((packed));
+#define ZT_EOD_GETIPSRESP_MAXADDRS ((ZT_EOD_MAXMSGSIZE - sizeof(zt_eod_msg_getipsresp)) / sizeof(zt_eod_msg_getipsresp::addr))
+
+struct zt_eod_msg_getbindaddrsresp {
+	unsigned char cmd;
+	unsigned char result;
+	struct addr {
+		unsigned char afi;
+		unsigned char len;
+		unsigned char data[16];
+		char ifname[16];
+	} __attribute__((packed)) addrs[0];
+} __attribute__((packed));
+#define ZT_EOD_GETBINDADDRSRESP_MAXADDRS ((ZT_EOD_MAXMSGSIZE - sizeof(zt_eod_msg_getbindaddrsresp)) / sizeof(zt_eod_msg_getbindaddrsresp::addr))
+
+struct zt_eod_msg_route {
+	unsigned char cmd;
+	unsigned char afi; // 1 ip, 2 ip6
+	unsigned char dstlen;
+	unsigned char dst[16];
+	unsigned char gw[16];
+	char dev[16];
+	unsigned char src[16];
+} __attribute__((packed));
+
+struct zt_eod_mgmt_req {
+	uint32_t method;
+	uint32_t pathlen;
+	uint32_t datalen;
+	unsigned char data[0];
+} __attribute__((packed));
+
+struct zt_eod_mgmt_reply {
+	uint32_t scode;
+	uint32_t datalen;
+	unsigned char data[0];
+} __attribute__((packed));
+
+#ifndef ZT_EXTOSDEP_IFACEONLY
+
+#include "../node/InetAddress.hpp"
+#include "../node/MAC.hpp"
+#include "Thread.hpp"
+#include "../node/Hashtable.hpp"
+#include "../node/Mutex.hpp"
+#include "../node/AtomicCounter.hpp"
+#include "EthernetTap.hpp"
+#include "BlockingQueue.hpp"
+#include <atomic>
+#include <thread>
+#include <mutex>
+#include <functional>
+#include <string>
+
+namespace ZeroTier {
+
+class ExtOsdep {
+public:
+	static void init(int, int);
+	static void started(int *, void **);
+
+	static void routeAddDel(bool, const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
+	static bool getBindAddrs(std::map<InetAddress,std::string> &);
+
+	static bool mgmtRecv(void *cookie, void *data, unsigned long len,
+		std::function<unsigned (unsigned, const std::string &, const std::string &, std::string &)>);
+	static bool mgmtWritable(void *);
+};
+
+class ExtOsdepTap : public EthernetTap
+{
+public:
+	ExtOsdepTap(
+		const char *homePath,
+		const MAC &mac,
+		unsigned int mtu,
+		unsigned int metric,
+		uint64_t nwid,
+		const char *friendlyName,
+		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
+		void *arg);
+
+	virtual ~ExtOsdepTap();
+
+	virtual void setEnabled(bool en);
+	virtual bool enabled() const;
+	virtual bool addIp(const InetAddress &ip);
+	virtual bool addIps(std::vector<InetAddress> ips);
+	virtual bool removeIp(const InetAddress &ip);
+	virtual std::vector<InetAddress> ips() const;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
+	virtual std::string deviceName() const;
+	virtual void setFriendlyName(const char *friendlyName);
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
+	virtual void setMtu(unsigned int mtu);
+	virtual void setDns(const char *domain, const std::vector<InetAddress> &servers) {}
+private:
+        void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int);
+        void *_arg;
+        uint64_t _nwid;
+        MAC _mac;
+        std::string _homePath;
+        std::string _dev;
+        std::vector<MulticastGroup> _multicastGroups;
+        unsigned int _mtu;
+	int _fd;
+        int _shutdownSignalPipe[2];
+        std::atomic_bool _enabled;
+        std::atomic_bool _run;
+        std::thread _tapReaderThread[2];
+        std::thread _tapProcessorThread;
+        std::mutex _buffers_l;
+        std::vector<void *> _buffers;
+        BlockingQueue< std::pair<void *,int> > _tapq;
+	AtomicCounter _tapqsize;
+
+	std::vector<InetAddress> allIps;
+	void doRemoveIp(const InetAddress &);
+};
+
+} // namespace ZeroTier
+
+#endif // ZT_EXTOSDEP_IFACEONLY
+#endif // ZT_EXTOSDEP
+
+#endif

osdep/ManagedRoute.cpp

@@ -51,6 +51,7 @@
 
 #include "ManagedRoute.hpp"
 #ifdef __LINUX__
+#include "ExtOsdep.hpp"
 #include "LinuxNetLink.hpp"
 #endif
 
@@ -550,6 +551,10 @@ bool ManagedRoute::sync()
 
 #ifdef __LINUX__ // ----------------------------------------------------------
 
+#ifdef ZT_EXTOSDEP
+	_applied[_target] = false;
+	ExtOsdep::routeAddDel(true, _target, _via, _src, _device);
+#else
 	if ((leftt)&&(!LinuxNetLink::getInstance().routeIsSet(leftt,_via,_src,_device))) {
 		_applied[leftt] = false; // boolean unused
 		LinuxNetLink::getInstance().addRoute(leftt, _via, _src, _device);
@@ -558,6 +563,7 @@ bool ManagedRoute::sync()
 		_applied[rightt] = false; // boolean unused
 		LinuxNetLink::getInstance().addRoute(rightt, _via, _src, _device);
 	}
+#endif // ZT_EXTOSDEP
 
 #endif // __LINUX__ ----------------------------------------------------------
 
@@ -609,7 +615,11 @@ void ManagedRoute::remove()
 
 #ifdef __LINUX__ // ----------------------------------------------------------
 		//_routeCmd("del",r->first,_via,(_via) ? (const char *)0 : _device);
+#ifdef ZT_EXTOSDEP
+		ExtOsdep::routeAddDel(false, r->first,_via,_src,(_via) ? (const char *)0 : _device);
+#else
 		LinuxNetLink::getInstance().delRoute(r->first,_via,_src,(_via) ? (const char *)0 : _device);
+#endif // ZT_EXTOSDEP
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------

service/OneService.cpp

@@ -55,6 +55,7 @@
 #include "../osdep/Binder.hpp"
 #include "../osdep/ManagedRoute.hpp"
 #include "../osdep/BlockingQueue.hpp"
+#include "../osdep/ExtOsdep.hpp"
 
 #include "OneService.hpp"
 #include "SoftwareUpdater.hpp"
@@ -1125,6 +1126,15 @@ public:
 			}
 #endif
 
+#ifdef ZT_EXTOSDEP
+			{
+				int mgmtfd;
+				void *mgmtcookie;
+				ExtOsdep::started(&mgmtfd, &mgmtcookie);
+				_phy.wrapSocket(mgmtfd, mgmtcookie);
+			}
+#endif
+
 			// Delete legacy iddb.d if present (cleanup)
 			OSUtils::rmDashRf((_homePath + ZT_PATH_SEPARATOR_S "iddb.d").c_str());
 
@@ -2165,7 +2175,6 @@ public:
 		auto statusGet = [&, setContent](const httplib::Request &req, httplib::Response &res) {
             ZT_NodeStatus status;
             _node->status(&status);
-
             auto out = json::object();
             char tmp[256] = {};
 
@@ -2335,7 +2344,9 @@ public:
 			_controller->configureHTTPControlPlane(_controlPlane, _controlPlaneV6, setContent);
 		}
 
+#ifndef ZT_EXTOSDEP
 		_controlPlane.set_pre_routing_handler(authCheck);
+#endif // ZT_EXTOSDEP
 		_controlPlaneV6.set_pre_routing_handler(authCheck);
 
 #if ZT_DEBUG==1
@@ -2351,6 +2362,7 @@ public:
 			exit(-1);
 		}
 
+#ifndef ZT_EXTOSDEP
 		bool v4controlPlaneBound = false;
 		_controlPlane.set_address_family(AF_INET);
 		if(_controlPlane.bind_to_port("0.0.0.0", _primaryPort)) {
@@ -2391,6 +2403,7 @@ public:
 			fprintf(stderr, "ERROR: Could not bind control plane. Exiting...\n");
 			exit(-1);
 		}
+#endif // ZT_EXTOSDEP
     }
 
 	// Must be called after _localConfig is read or modified
@@ -3162,8 +3175,52 @@ public:
 	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
 	inline void phyOnUnixAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN) {}
 	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
-	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
-	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
+	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {
+#ifdef ZT_EXTOSDEP
+		if (ExtOsdep::mgmtRecv(*uptr, data, len, [&](unsigned method, const std::string &path, const std::string &data, std::string &resp) {
+				// fprintf(stderr, "mgmtRecv: %u %s %s\n", method, path.c_str(), data.c_str());
+				httplib::Request req;
+				httplib::Response res;
+				req.path = "/" + path;
+				if (method == 1) req.method = "GET";
+				else if (method == 3) req.method = "POST";
+				else if (method == 0) req.method = "DELETE";
+				struct S : public httplib::Stream {
+					const char *ptr;
+					unsigned size;
+					S(const std::string &s) : ptr(s.c_str()), size(s.size()) {}
+					virtual bool is_readable() const { return true; }
+					virtual bool is_writable() const { return true; }
+					virtual ssize_t read(char *p, size_t sz) {
+						// fprintf(stderr, "S::read %d\n", (int)size);
+						if (sz > (size_t)size) sz = size;
+						memcpy(p, ptr, sz);
+						size -= (unsigned)sz;
+						ptr += sz;
+						return (ssize_t)sz;
+					}
+					virtual ssize_t write(const char *ptr, size_t size) {
+						// fprintf(stderr, "S::write %d\n", (int)size);
+						return size;
+					}
+					virtual void get_remote_ip_and_port(std::string &ip, int &port) const {}
+					virtual void get_local_ip_and_port(std::string &ip, int &port) const {};
+					virtual socket_t socket() const { return 0; }
+				};
+				S s(data);
+
+				bool x = _controlPlane.routing(req, res, s);
+				// fprintf(stderr, "mgmtRecv: done, x %d status %u body %s\n", x, res.status, res.body.c_str());
+				resp = res.body;
+				return res.status;
+			})) _phy.setNotifyWritable(sock,true);
+#endif
+	}
+	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {
+#ifdef ZT_EXTOSDEP
+		if (ExtOsdep::mgmtWritable(*uptr)) _phy.setNotifyWritable(sock,false);
+#endif
+	}
 
 	inline int nodeVirtualNetworkConfigFunction(uint64_t nwid,void **nuptr,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwc)
 	{