Tying up default route and route mgmt loose ends. It now periodically updates shadow routes so hopefully your link will stay up as you move around.

osdep/ManagedRoute.cpp

@@ -273,20 +273,25 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress
 
 bool ManagedRoute::sync()
 {
-	if (_target.isDefaultRoute()) {
-		/* In ZeroTier we use a forked-route trick to override the default
-		 * with a more specific one while leaving the original system route
-		 * intact. We also create a shadow more specific route to the
-		 * original gateway that is device-bound so that ZeroTier's device
-		 * bound ports go via the physical Internet link. This has to be
-		 * done *slightly* differently on different platforms. */
-
+	if ((_target.isDefaultRoute())||((_target.ss_family == AF_INET)&&(_target.netmaskBits() < 32))) {
+		/* In ZeroTier we create two more specific routes for every one route. We
+		 * do this for default routes and IPv4 routes other than /32s. If there
+		 * is a pre-existing system route that this route will override, we create
+		 * two more specific interface-bound shadow routes for it.
+		 *
+		 * This means that ZeroTier can *itself* continue communicating over
+		 * whatever physical routes might be present while simultaneously
+		 * overriding them for general system traffic. This is mostly for
+		 * "full tunnel" VPN modes of operation, but might be useful for
+		 * virtualizing physical networks in a hybrid design as well. */
+
+		// Generate two more specific routes than target with one extra bit
 		InetAddress leftt,rightt;
 		_forkTarget(_target,leftt,rightt);
 
 #ifdef __BSD__ // ------------------------------------------------------------
 
-		// Get system default route information
+		// Find lowest metric system route that this route should override (if any)
 		InetAddress newSystemVia;
 		char newSystemDevice[128];
 		newSystemDevice[0] = (char)0;
@@ -296,12 +301,12 @@ bool ManagedRoute::sync()
 			if (r->via) {
 				if ((!newSystemVia)||(r->metric < systemMetric)) {
 					newSystemVia = r->via;
-					Utils::scopy(_systemDevice,sizeof(_systemDevice),r->device);
+					Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device);
 					systemMetric = r->metric;
 				}
 			}
 		}
-		if (!newSystemDevice[0]) {
+		if ((newSystemVia)&&(!newSystemDevice[0])) {
 			rtes = _getRTEs(newSystemVia,true);
 			for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) {
 				if (r->device[0]) {
@@ -310,10 +315,10 @@ bool ManagedRoute::sync()
 				}
 			}
 		}
-		if ((!newSystemVia)||(!newSystemDevice[0]))
-			return false;
 
-		// If system default route has changed or hasn't been shadowed yet, update shadow
+		// Shadow system route if it exists, also delete any obsolete shadows
+		// and replace them with the new state. sync() is called periodically to
+		// allow us to do that if underlying connectivity changes.
 		if ((_systemVia != newSystemVia)||(!strcmp(_systemDevice,newSystemDevice))) {
 			if ((_systemVia)&&(_systemDevice[0])) {
 				_routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0);
@@ -323,13 +328,15 @@ bool ManagedRoute::sync()
 			_systemVia = newSystemVia;
 			Utils::scopy(_systemDevice,sizeof(_systemDevice),newSystemDevice);
 
-			_routeCmd("add",leftt,_systemVia,_systemDevice,(const char *)0);
-			_routeCmd("change",leftt,_systemVia,_systemDevice,(const char *)0);
-			_routeCmd("add",rightt,_systemVia,_systemDevice,(const char *)0);
-			_routeCmd("change",rightt,_systemVia,_systemDevice,(const char *)0);
+			if ((_systemVia)&&(_systemDevice[0])) {
+				_routeCmd("add",leftt,_systemVia,_systemDevice,(const char *)0);
+				_routeCmd("change",leftt,_systemVia,_systemDevice,(const char *)0);
+				_routeCmd("add",rightt,_systemVia,_systemDevice,(const char *)0);
+				_routeCmd("change",rightt,_systemVia,_systemDevice,(const char *)0);
+			}
 		}
 
-		// Apply overriding routes
+		// Apply overriding non-device-scoped routes
 		if (!_applied) {
 			if (_via) {
 				_routeCmd("add",leftt,_via,(const char *)0,(const char *)0);
@@ -357,11 +364,22 @@ bool ManagedRoute::sync()
 #endif // __WINDOWS__ --------------------------------------------------------
 
 	} else {
-
-		// TODO
+		/* For non-default routes, IPv4 /32, and IPv6 non-default routes, we just
+		 * add the route itself. */
 
 #ifdef __BSD__ // ------------------------------------------------------------
 
+		if (!_applied) {
+			if (_via) {
+				_routeCmd("add",_target,_via,(const char *)0,(const char *)0);
+				_routeCmd("change",_target,_via,(const char *)0,(const char *)0);
+			} else if (_device[0]) {
+				_routeCmd("add",_target,_via,(const char *)0,_device);
+				_routeCmd("change",_target,_via,(const char *)0,_device);
+			}
+			_applied = true;
+		}
+
 #endif // __BSD__ ------------------------------------------------------------
 
 #ifdef __LINUX__ // ----------------------------------------------------------
@@ -418,10 +436,14 @@ void ManagedRoute::remove()
 
 		} else {
 
-			// TODO
-
 #ifdef __BSD__ // ------------------------------------------------------------
 
+		if (_via) {
+			_routeCmd("delete",_target,_via,(const char *)0,(const char *)0);
+		} else if (_device[0]) {
+			_routeCmd("delete",_target,_via,(const char *)0,_device);
+		}
+
 #endif // __BSD__ ------------------------------------------------------------
 
 #ifdef __LINUX__ // ----------------------------------------------------------
@@ -444,12 +466,3 @@ void ManagedRoute::remove()
 }
 
 } // namespace ZeroTier
-
-/*
-int main(int argc,char **argv)
-{
-	ZeroTier::ManagedRoute t;
-	t.set(ZeroTier::InetAddress("0.0.0.0/0"),ZeroTier::InetAddress("10.6.6.112"),"zt2");
-	sleep(10000);
-}
-*/

service/OneService.cpp

@@ -527,6 +527,7 @@ public:
 		NetworkState() : tap((EthernetTap *)0),managedIps(),managedRoutes(),allowManaged(true),allowGlobal(true),allowDefault(true) {}
 
 		EthernetTap *tap;
+		ZT_VirtualNetworkConfig config; // memcpy() of raw config from core
 		std::vector<InetAddress> managedIps;
 		std::list<ManagedRoute> managedRoutes;
 		bool allowManaged; // allow managed addresses and routes
@@ -851,7 +852,7 @@ public:
 					restarted = true;
 				}
 
-				// Refresh bindings in case device's interfaces have changed
+				// Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow routes (e.g. shadow default)
 				if (((now - lastBindRefresh) >= ZT_BINDER_REFRESH_PERIOD)||(restarted)) {
 					lastBindRefresh = now;
 					for(int i=0;i<3;++i) {
@@ -859,6 +860,13 @@ public:
 							_bindings[i].refresh(_phy,_ports[i],*this);
 						}
 					}
+					{
+						Mutex::Lock _l(_nets_m);
+						for(std::map<uint64_t,NetworkState>::iterator n(_nets.begin());n!=_nets.end();++n) {
+							if (n->second.tap)
+								syncManagedStuff(n->second,false,true);
+						}
+					}
 				}
 
 				uint64_t dl = _nextBackgroundTaskDeadline;
@@ -985,6 +993,118 @@ public:
 
 	// Begin private implementation methods
 
+	// Checks if a managed IP or route target is allowed
+	bool checkIfManagedIsAllowed(const NetworkState &n,const InetAddress &addr)
+	{
+		if (!n.allowManaged)
+			return false;
+		if (addr.isDefaultRoute())
+			return n.allowDefault;
+		switch(addr.ipScope()) {
+			case InetAddress::IP_SCOPE_NONE:
+			case InetAddress::IP_SCOPE_MULTICAST:
+			case InetAddress::IP_SCOPE_LOOPBACK:
+			case InetAddress::IP_SCOPE_LINK_LOCAL:
+				return false;
+			case InetAddress::IP_SCOPE_GLOBAL:
+				return n.allowGlobal;
+			default:
+				return true;
+		}
+	}
+
+	// Apply or update managed IPs for a configured network (be sure n.tap exists)
+	void syncManagedStuff(NetworkState &n,bool syncIps,bool syncRoutes)
+	{
+		if (syncIps) {
+			std::vector<InetAddress> newManagedIps;
+			newManagedIps.reserve(n.config.assignedAddressCount);
+			for(unsigned int i=0;i<n.config.assignedAddressCount;++i) {
+				const InetAddress *ii = reinterpret_cast<const InetAddress *>(&(n.config.assignedAddresses[i]));
+				if (checkIfManagedIsAllowed(n,*ii))
+					newManagedIps.push_back(*ii);
+			}
+			std::sort(newManagedIps.begin(),newManagedIps.end());
+			newManagedIps.erase(std::unique(newManagedIps.begin(),newManagedIps.end()),newManagedIps.end());
+
+			for(std::vector<InetAddress>::iterator ip(newManagedIps.begin());ip!=newManagedIps.end();++ip) {
+				if (std::find(n.managedIps.begin(),n.managedIps.end(),*ip) == n.managedIps.end()) {
+					if (!n.tap->addIp(*ip))
+						fprintf(stderr,"ERROR: unable to add ip address %s"ZT_EOL_S, ip->toString().c_str());
+				}
+			}
+			for(std::vector<InetAddress>::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) {
+				if (std::find(newManagedIps.begin(),newManagedIps.end(),*ip) == newManagedIps.end()) {
+					if (!n.tap->removeIp(*ip))
+						fprintf(stderr,"ERROR: unable to remove ip address %s"ZT_EOL_S, ip->toString().c_str());
+				}
+			}
+
+			n.managedIps.swap(newManagedIps);
+		}
+
+		if (syncRoutes) {
+			const std::string tapdev(n.tap->deviceName());
+
+			// Nuke applied routes that are no longer in n.config.routes[] and/or are not allowed
+			for(std::list<ManagedRoute>::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) {
+				bool haveRoute = false;
+				if (checkIfManagedIsAllowed(n,mr->target())) {
+					for(unsigned int i=0;i<n.config.routeCount;++i) {
+						const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target));
+						const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via));
+						if ( (mr->target() == *target) && ( ((via->ss_family == target->ss_family)&&(mr->via() == *via)) || (tapdev == mr->device()) ) ) {
+							haveRoute = true;
+							break;
+						}
+					}
+				}
+				if (haveRoute) {
+					++mr;
+				} else {
+					n.managedRoutes.erase(mr++);
+				}
+			}
+
+			// Apply routes in n.config.routes[] that we haven't applied yet, and sync those we have in case shadow routes need to change
+			for(unsigned int i=0;i<n.config.routeCount;++i) {
+				const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target));
+				const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via));
+
+				if (!checkIfManagedIsAllowed(n,*target))
+					continue;
+
+				bool haveRoute = false;
+
+				// Ignore routes implied by local managed IPs since adding the IP adds the route
+				for(std::vector<InetAddress>::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) {
+					if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) {
+						haveRoute = true;
+						break;
+					}
+				}
+				if (haveRoute)
+					continue;
+
+				// If we've already applied this route, just sync it and continue
+				for(std::list<ManagedRoute>::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();++mr) {
+					if ( (mr->target() == *target) && ( ((via->ss_family == target->ss_family)&&(mr->via() == *via)) || (tapdev == mr->device()) ) ) {
+						haveRoute = true;
+						mr->sync();
+						break;
+					}
+				}
+				if (haveRoute)
+					continue;
+
+				// Add and apply new routes
+				n.managedRoutes.push_back(ManagedRoute());
+				if (!n.managedRoutes.back().set(*target,*via,tapdev.c_str()))
+					n.managedRoutes.pop_back();
+			}
+		}
+	}
+
 	inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *localAddr,const struct sockaddr *from,void *data,unsigned long len)
 	{
 #ifdef ZT_ENABLE_CLUSTER
@@ -1256,129 +1376,9 @@ public:
 				// After setting up tap, fall through to CONFIG_UPDATE since we also want to do this...
 
 			case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE:
+				memcpy(&(n.config),nwc,sizeof(ZT_VirtualNetworkConfig));
 				if (n.tap) { // sanity check
-					if (n.allowManaged) {
-
-						{ // configure managed IP addresses
-							std::vector<InetAddress> newManagedIps;
-							for(unsigned int i=0;i<nwc->assignedAddressCount;++i) {
-								const InetAddress *ii = reinterpret_cast<const InetAddress *>(&(nwc->assignedAddresses[i]));
-								switch(ii->ipScope()) {
-									case InetAddress::IP_SCOPE_NONE:
-									case InetAddress::IP_SCOPE_MULTICAST:
-									case InetAddress::IP_SCOPE_LOOPBACK:
-									case InetAddress::IP_SCOPE_LINK_LOCAL:
-										break; // ignore these -- they shouldn't appear here
-									case InetAddress::IP_SCOPE_GLOBAL:
-										if (!n.allowGlobal)
-											continue; // skip global IP ranges if we haven't given this network permission to assign them
-										// else fall through for PSEUDOPRIVATE, SHARED, PRIVATE
-									default:
-										newManagedIps.push_back(*ii);
-										break;
-								}
-							}
-							std::sort(newManagedIps.begin(),newManagedIps.end());
-							newManagedIps.erase(std::unique(newManagedIps.begin(),newManagedIps.end()),newManagedIps.end());
-
-							for(std::vector<InetAddress>::iterator ip(newManagedIps.begin());ip!=newManagedIps.end();++ip) {
-								if (std::find(n.managedIps.begin(),n.managedIps.end(),*ip) == n.managedIps.end()) {
-									if (!n.tap->addIp(*ip))
-										fprintf(stderr,"ERROR: unable to add ip address %s"ZT_EOL_S, ip->toString().c_str());
-								}
-							}
-							for(std::vector<InetAddress>::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) {
-								if (std::find(newManagedIps.begin(),newManagedIps.end(),*ip) == newManagedIps.end()) {
-									if (!n.tap->removeIp(*ip))
-										fprintf(stderr,"ERROR: unable to remove ip address %s"ZT_EOL_S, ip->toString().c_str());
-								}
-							}
-
-							n.managedIps.swap(newManagedIps);
-						}
-
-						{ // configure managed routes
-							const std::string tapdev(n.tap->deviceName());
-
-							for(std::list<ManagedRoute>::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) {
-								bool haveRoute = false;
-								for(unsigned int i=0;i<nwc->routeCount;++i) {
-									const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(nwc->routes[i].target));
-									const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(nwc->routes[i].via));
-									if (mr->target() == *target) {
-										if ((via->ss_family == target->ss_family)&&(mr->via() == *via)) {
-											haveRoute = true;
-											break;
-										} else if (tapdev == mr->device()) {
-											haveRoute = true;
-											break;
-										}
-									}
-								}
-								if (haveRoute) {
-									++mr;
-								} else {
-									n.managedRoutes.erase(mr++); // also removes route via RAII behavior
-								}
-							}
-
-							for(unsigned int i=0;i<nwc->routeCount;++i) {
-								const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(nwc->routes[i].target));
-								const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(nwc->routes[i].via));
-
-								bool haveRoute = false;
-
-								// We don't need to bother applying local routes to local managed IPs since these are implied by setting the IP
-								for(std::vector<InetAddress>::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) {
-									if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) {
-										haveRoute = true;
-										break;
-									}
-								}
-
-								if (haveRoute)
-									continue;
-
-								for(std::list<ManagedRoute>::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();++mr) {
-									if (mr->target() == *target) {
-										if ((via->ss_family == target->ss_family)&&(mr->via() == *via)) {
-											haveRoute = true;
-											break;
-										} else if (tapdev == mr->device()) {
-											haveRoute = true;
-											break;
-										}
-									}
-								}
-
-								if (haveRoute)
-									continue;
-
-								n.managedRoutes.push_back(ManagedRoute());
-								if ((target->isDefaultRoute())&&(n.allowDefault)) {
-									if (!n.managedRoutes.back().set(*target,*via,tapdev.c_str()))
-										n.managedRoutes.pop_back();
-								} else {
-									switch(target->ipScope()) {
-										case InetAddress::IP_SCOPE_NONE:
-										case InetAddress::IP_SCOPE_MULTICAST:
-										case InetAddress::IP_SCOPE_LOOPBACK:
-										case InetAddress::IP_SCOPE_LINK_LOCAL:
-											break;
-										case InetAddress::IP_SCOPE_GLOBAL:
-											if (!n.allowGlobal)
-												continue; // skip global IP ranges if we haven't given this network permission to assign them
-											// else fall through for PSEUDOPRIVATE, SHARED, PRIVATE
-										default:
-											if (!n.managedRoutes.back().set(*target,*via,tapdev.c_str()))
-												n.managedRoutes.pop_back();
-											break;
-									}
-								}
-							}
-						}
-
-					}
+					syncManagedStuff(n,true,true);
 				} else {
 					_nets.erase(nwid);
 					return -999; // tap init failed