Fix RENDEZVOUS issue and possibly improve GeoIP resolution.

node/Topology.hpp

@@ -52,7 +52,14 @@ private:
 		ZT_ALWAYS_INLINE _RootRankingFunction() : bestRoot(),bestRootLatency(0xffff) {}
 		ZT_ALWAYS_INLINE bool operator()(const SharedPtr<Peer> &peer,const std::vector<InetAddress> &phy)
 		{
+			const unsigned int lat = peer->latency(now);
+			if ((!bestRoot)||((lat <= bestRootLatency)&&(peer->getAppropriatePath(now,false)))) {
+				bestRoot = peer;
+				bestRootLatency = lat;
+			}
+			return true;
 		}
+		int64_t now;
 		SharedPtr<Peer> bestRoot;
 		unsigned int bestRootLatency;
 	};
@@ -61,7 +68,8 @@ public:
 	ZT_ALWAYS_INLINE Topology(const RuntimeEnvironment *renv,const Identity &myId) :
 		RR(renv),
 		_myIdentity(myId),
-		_numConfiguredPhysicalPaths(0) {}
+		_numConfiguredPhysicalPaths(0),
+		_lastUpdatedBestRoot(0) {}
 	ZT_ALWAYS_INLINE ~Topology() {}
 
 	/**
@@ -98,25 +106,10 @@ public:
 	{
 		if (zta == _myIdentity.address())
 			return SharedPtr<Peer>();
-
 		Mutex::Lock l1(_peers_l);
 		const SharedPtr<Peer> *const ap = _peers.get(zta);
 		if (ap)
 			return *ap;
-
-#if 0
-		Mutex::Lock l2(_roots_m);
-		for(std::vector<Root>::const_iterator r(_roots.begin());r!=_roots.end();++r) {
-			if (r->address() == zta) {
-				try {
-					SharedPtr<Peer> rp(new Peer(RR,_myIdentity,r->id()));
-					_peers[zta] = rp;
-					return rp;
-				} catch ( ... ) {}
-			}
-		}
-#endif
-
 		return SharedPtr<Peer>();
 	}
 
@@ -160,13 +153,16 @@ public:
 	 */
 	ZT_ALWAYS_INLINE bool isRoot(const Identity &id) const
 	{
-#if 0
-		Mutex::Lock l(_roots_m);
-		for(std::vector<Root>::const_iterator r(_roots.begin());r!=_roots.end();++r) {
-			if (r->is(id))
+		{
+			Mutex::Lock l(_dynamicRoots_l);
+			if (_dynamicRootIdentities.contains(id))
+				return true;
+		}
+		{
+			Mutex::Lock l(_staticRoots_l);
+			if (_staticRoots.contains(id))
 				return true;
 		}
-#endif
 		return false;
 	}
 
@@ -238,75 +234,6 @@ public:
 		}
 	}
 
-#if 0
-	/**
-	 * Apply a function or function object to all roots
-	 *
-	 * This locks the root list during execution but other operations
-	 * are fine.
-	 *
-	 * @param f Function to apply
-	 * @tparam F function or function object type
-	 */
-	template<typename F>
-	ZT_ALWAYS_INLINE void eachRoot(F f)
-	{
-		Mutex::Lock l(_roots_m);
-		SharedPtr<Peer> rp;
-		for(std::vector<Root>::const_iterator i(_roots.begin());i!=_roots.end();++i) {
-			{
-				Mutex::Lock l2(_peers_l);
-				const SharedPtr<Peer> *const ap = _peers.get(i->address());
-				if (ap) {
-					rp = *ap;
-				} else {
-					rp.set(new Peer(RR,_myIdentity,i->id()));
-					_peers.set(rp->address(),rp);
-				}
-			}
-			f(*i,rp);
-		}
-	}
-
-	/**
-	 * Get the best root, rescanning and re-ranking roots periodically
-	 *
-	 * @param now Current time
-	 * @return Best/fastest currently connected root or NULL if none
-	 */
-	inline SharedPtr<Peer> root(const int64_t now)
-	{
-		Mutex::Lock l(_bestRoot_m);
-		if ((!_bestRoot)||((now - _lastRankedBestRoot) >= ZT_FIND_BEST_ROOT_PERIOD)) {
-			_bestRoot.zero();
-			Mutex::Lock l2(_roots_m);
-			SharedPtr<Peer> rp;
-			long bestQuality = 2147483647;
-			for(std::vector<Root>::const_iterator i(_roots.begin());i!=_roots.end();++i) {
-				{
-					Mutex::Lock l2(_peers_l);
-					const SharedPtr<Peer> *const ap = _peers.get(i->address());
-					if (ap) {
-						rp = *ap;
-					} else {
-						rp.set(new Peer(RR,_myIdentity,i->id()));
-						_peers.set(rp->address(),rp);
-					}
-				}
-				SharedPtr<Path> path(rp->getAppropriatePath(now,false));
-				if (path) {
-					const long pq = path->quality(now);
-					if (pq < bestQuality) {
-						bestQuality = pq;
-						_bestRoot = rp;
-					}
-				}
-			}
-		}
-		return _bestRoot;
-	}
-#endif
-
 	/**
 	 * Apply a function or function object to all roots
 	 *
@@ -319,33 +246,6 @@ public:
 	template<typename F>
 	inline void eachRoot(F f)
 	{
-		{
-			Mutex::Lock l(_staticRoots_l);
-			Hashtable< Identity,std::vector<InetAddress> >::Iterator i(_staticRoots);
-			Identity *k = (Identity *)0;
-			std::vector<InetAddress> *v = (std::vector<InetAddress> *)0;
-			while (i.next(k,v)) {
-				if (!v->empty()) {
-					const SharedPtr<Peer> *ap;
-					{
-						Mutex::Lock l2(_peers_l);
-						ap = _peers.get(k->address());
-					}
-					if (ap) {
-						if (!f(*ap,*v))
-							return;
-					} else {
-						SharedPtr<Peer> p(new Peer(RR,_myIdentity,*k));
-						{
-							Mutex::Lock l2(_peers_l);
-							_peers.set(k->address(),p);
-						}
-						if (!f(p,*v))
-							return;
-					}
-				}
-			}
-		}
 		{
 			Mutex::Lock l(_dynamicRoots_l);
 			Hashtable< Str,Locator >::Iterator i(_dynamicRoots);
@@ -375,21 +275,49 @@ public:
 				}
 			}
 		}
-	}
-
-	inline SharedPtr<Peer> root(const int64_t now)
-	{
-		_RootRankingFunction rrf;
-		eachRoot(rrf);
+		{
+			Mutex::Lock l(_staticRoots_l);
+			Hashtable< Identity,std::vector<InetAddress> >::Iterator i(_staticRoots);
+			Identity *k = (Identity *)0;
+			std::vector<InetAddress> *v = (std::vector<InetAddress> *)0;
+			while (i.next(k,v)) {
+				if (!v->empty()) {
+					const SharedPtr<Peer> *ap;
+					{
+						Mutex::Lock l2(_peers_l);
+						ap = _peers.get(k->address());
+					}
+					if (ap) {
+						if (!f(*ap,*v))
+							return;
+					} else {
+						SharedPtr<Peer> p(new Peer(RR,_myIdentity,*k));
+						{
+							Mutex::Lock l2(_peers_l);
+							_peers.set(k->address(),p);
+						}
+						if (!f(p,*v))
+							return;
+					}
+				}
+			}
+		}
 	}
 
 	/**
-	 * @return Names of dynamic roots currently known by the system
+	 * @return Current best root (updated automatically each second)
 	 */
-	ZT_ALWAYS_INLINE std::vector<Str> dynamicRootNames() const
+	inline SharedPtr<Peer> root(const int64_t now)
 	{
-		Mutex::Lock l(_dynamicRoots_l);
-		return _dynamicRoots.keys();
+		Mutex::Lock l(_bestRoot_l);
+		if ((!_bestRoot)||((now - _lastUpdatedBestRoot) > 1000)) {
+			_lastUpdatedBestRoot = now;
+			_RootRankingFunction rrf;
+			rrf.now = now;
+			eachRoot(rrf);
+			_bestRoot = rrf.bestRoot;
+		}
+		return _bestRoot;
 	}
 
 	/**
@@ -404,6 +332,26 @@ public:
 		_staticRoots[id] = addrs;
 	}
 
+	/**
+	 * Remove a static root
+	 *
+	 * @param id Identity to remove
+	 */
+	ZT_ALWAYS_INLINE void removeStaticRoot(const Identity &id)
+	{
+		Mutex::Lock l(_staticRoots_l);
+		_staticRoots.erase(id);
+	}
+
+	/**
+	 * @return Names of dynamic roots currently known by the system
+	 */
+	ZT_ALWAYS_INLINE std::vector<Str> dynamicRootNames() const
+	{
+		Mutex::Lock l(_dynamicRoots_l);
+		return _dynamicRoots.keys();
+	}
+
 	/**
 	 * Set or update dynamic root if new locator is newer and valid
 	 *
@@ -420,11 +368,34 @@ public:
 		Locator &ll = _dynamicRoots[dnsName];
 		if (ll.timestamp() < latestLocator.timestamp()) {
 			ll = latestLocator;
+			_updateDynamicRootIdentities();
 			return true;
 		}
 		return false;
 	}
 
+	/**
+	 * Remove a dynamic root entry
+	 *
+	 * @param dnsName DNS name to remove
+	 */
+	ZT_ALWAYS_INLINE bool removeDynamicRoot(const Str &dnsName)
+	{
+		Mutex::Lock l(_dynamicRoots_l);
+		_dynamicRoots.erase(dnsName);
+		_updateDynamicRootIdentities();
+	}
+
+	/**
+	 * Remove all dynamic roots
+	 */
+	ZT_ALWAYS_INLINE bool clearDynamicRoots(const Str &dnsName)
+	{
+		Mutex::Lock l(_dynamicRoots_l);
+		_dynamicRoots.clear();
+		_dynamicRootIdentities.clear();
+	}
+
 	/**
 	 * Get the best relay to a given address, which may or may not be a root
 	 *
@@ -557,6 +528,19 @@ public:
 	}
 
 private:
+	inline void _updateDynamicRootIdentities()
+	{
+		// assumes _dynamicRoots_l is locked
+		_dynamicRootIdentities.clear();
+		Hashtable< Str,Locator >::Iterator i(_dynamicRoots);
+		Str *k = (Str *)0;
+		Locator *v = (Locator *)0;
+		while (i.next(k,v)) {
+			if (v->id())
+				_dynamicRootIdentities.set(v->id(),true);
+		}
+	}
+
 	const RuntimeEnvironment *const RR;
 	const Identity _myIdentity;
 
@@ -566,13 +550,18 @@ private:
 	Hashtable< Address,SharedPtr<Peer> > _peers;
 	Hashtable< Path::HashKey,SharedPtr<Path> > _paths;
 
-	Hashtable< Identity,std::vector<InetAddress> > _staticRoots;
 	Hashtable< Str,Locator > _dynamicRoots;
+	Hashtable< Identity,bool > _dynamicRootIdentities;
+	Hashtable< Identity,std::vector<InetAddress> > _staticRoots;
+
+	int64_t _lastUpdatedBestRoot;
+	SharedPtr<Peer> _bestRoot;
 
 	Mutex _peers_l;
 	Mutex _paths_l;
-	Mutex _staticRoots_l;
 	Mutex _dynamicRoots_l;
+	Mutex _staticRoots_l;
+	Mutex _bestRoot_l;
 };
 
 } // namespace ZeroTier

root/root.cpp

@@ -431,7 +431,6 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip
 							const uint64_t nwid = pkt.template at<uint64_t>(ptr);
 							const MulticastGroup mg(MAC(pkt.field(ptr + 8,6),6),pkt.template at<uint32_t>(ptr + 14));
 							s_multicastSubscriptions[nwid][mg][source] = now;
-							//printf("%s %s subscribes to %s/%.8lx on network %.16llx" ZT_EOL_S,ip->toString(ipstr),source.toString(astr),mg.mac().toString(tmpstr),(unsigned long)mg.adi(),(unsigned long long)nwid);
 						}
 					} catch ( ... ) {
 						printf("* unexpected exception handling MULTICAST_LIKE from %s" ZT_EOL_S,ip->toString(ipstr));
@@ -561,7 +560,7 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip
 						dest.appendTo(outp);
 						outp.append((uint16_t)b->second->ip6.port());
 						outp.append((uint8_t)16);
-						outp.append((const uint8_t *)b->second->ip6.rawIpData(),16);
+						outp.append((const uint8_t *)(b->second->ip6.rawIpData()),16);
 						outp.armor((*a)->key,true);
 						sendto(v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&((*a)->ip6),(socklen_t)sizeof(struct sockaddr_in6));
 
@@ -572,9 +571,9 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip
 						outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS);
 						outp.append((uint8_t)0);
 						source.appendTo(outp);
-						outp.append((uint16_t)ip->port());
+						outp.append((uint16_t)(*a)->ip6.port());
 						outp.append((uint8_t)16);
-						outp.append((const uint8_t *)ip->rawIpData(),16);
+						outp.append((const uint8_t *)((*a)->ip6.rawIpData()),16);
 						outp.armor(b->second->key,true);
 						sendto(v6s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip6),(socklen_t)sizeof(struct sockaddr_in6));
 
@@ -601,9 +600,9 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip
 						outp.reset(dest,s_self.address(),Packet::VERB_RENDEZVOUS);
 						outp.append((uint8_t)0);
 						source.appendTo(outp);
-						outp.append((uint16_t)ip->port());
+						outp.append((uint16_t)(*a)->ip4.port());
 						outp.append((uint8_t)4);
-						outp.append((const uint8_t *)ip->rawIpData(),4);
+						outp.append((const uint8_t *)((*a)->ip4.rawIpData()),4);
 						outp.armor(b->second->key,true);
 						sendto(v4s,outp.data(),outp.size(),SENDTO_FLAGS,(const struct sockaddr *)&(b->second->ip4),(socklen_t)sizeof(struct sockaddr_in));
 
@@ -616,7 +615,6 @@ static void handlePacket(const int v4s,const int v6s,const InetAddress *const ip
 	}
 
 	for(auto i=toAddrs.begin();i!=toAddrs.end();++i) {
-		//printf("%s -> %s for %s -> %s (%u bytes)" ZT_EOL_S,ip->toString(ipstr),i->first->toString(ipstr2),source.toString(astr),dest.toString(astr2),pkt.size());
 		if (sendto(i->first->isV4() ? v4s : v6s,pkt.data(),pkt.size(),SENDTO_FLAGS,(const struct sockaddr *)i->first,(socklen_t)(i->first->isV4() ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6))) > 0) {
 			s_outputRate.log(now,pkt.size());
 			s_forwardRate.log(now,pkt.size());
@@ -636,13 +634,13 @@ static int bindSocket(struct sockaddr *const bindAddr)
 		return -1;
 	}
 
-	int f = 4194304;
+	int f = 16777216;
 	while (f > 131072) {
 		if (setsockopt(s,SOL_SOCKET,SO_RCVBUF,(const char *)&f,sizeof(f)) == 0)
 			break;
 		f -= 131072;
 	}
-	f = 4194304;
+	f = 16777216;
 	while (f > 131072) {
 		if (setsockopt(s,SOL_SOCKET,SO_SNDBUF,(const char *)&f,sizeof(f)) == 0)
 			break;
@@ -665,11 +663,13 @@ static int bindSocket(struct sockaddr *const bindAddr)
 	f = IP_PMTUDISC_DONT; setsockopt(s,IPPROTO_IP,IP_MTU_DISCOVER,&f,sizeof(f));
 #endif
 
+/*
 #ifdef SO_NO_CHECK
 	if (bindAddr->sa_family == AF_INET) {
 		f = 1; setsockopt(s,SOL_SOCKET,SO_NO_CHECK,(void *)&f,sizeof(f));
 	}
 #endif
+*/
 
 #if defined(SO_REUSEPORT)
 	f = 1; setsockopt(s,SOL_SOCKET,SO_REUSEPORT,(void *)&f,sizeof(f));
@@ -1026,6 +1026,7 @@ int main(int argc,char **argv)
 				std::pair< uint32_t,uint32_t > k4(0,0xffffffff);
 				std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > > k6;
 				k6.second[0] = 0xffffffffffffffffULL; k6.second[1] = 0xffffffffffffffffULL;
+
 				std::unordered_map< InetAddress,std::set<Address>,InetAddressHasher > ips;
 				{
 					std::lock_guard<std::mutex> l(s_peers_l);
@@ -1036,30 +1037,39 @@ int main(int argc,char **argv)
 							ips[(*p)->ip6].insert((*p)->id.address());
 					}
 				}
+
 				for(auto p=ips.begin();p!=ips.end();++p) {
 					if (p->first.isV4()) {
 						k4.first = ip4ToH32(p->first);
 						auto geo = std::map< std::pair< uint32_t,uint32_t >,std::pair< float,float > >::reverse_iterator(s_geoIp4.upper_bound(k4));
+						uint32_t bestRangeSize = 0xffffffff;
+						std::pair< float,float > bestRangeLatLon;
 						while (geo != s_geoIp4.rend()) {
 							if ((geo->first.first <= k4.first)&&(geo->first.second >= k4.first)) {
-								if (!firstCoord)
-									o << ',';
-								firstCoord = false;
-								o << "{lat:" << geo->second.first << ",lng:" << geo->second.second << ",_l:\"";
-								bool firstAddr = true;
-								for(auto a=p->second.begin();a!=p->second.end();++a) {
-									if (!firstAddr)
-										o << ',';
-									o << a->toString(tmp);
-									firstAddr = false;
+								uint32_t range = geo->first.second - geo->first.first;
+								if (range <= bestRangeSize) {
+									bestRangeSize = range;
+									bestRangeLatLon = geo->second;
 								}
-								o << "\"}";
-								break;
 							} else if ((geo->first.first < k4.first)&&(geo->first.second < k4.first)) {
 								break;
 							}
 							++geo;
 						}
+						if (bestRangeSize != 0xffffffff) {
+							if (!firstCoord)
+								o << ',';
+							firstCoord = false;
+							o << "{lat:" << bestRangeLatLon.first << ",lng:" << bestRangeLatLon.second << ",_l:\"";
+							bool firstAddr = true;
+							for(auto a=p->second.begin();a!=p->second.end();++a) {
+								if (!firstAddr)
+									o << ',';
+								o << a->toString(tmp);
+								firstAddr = false;
+							}
+							o << "\"}";
+						}
 					} else if (p->first.isV6()) {
 						k6.first = ip6ToH128(p->first);
 						auto geo = std::map< std::pair< std::array< uint64_t,2 >,std::array< uint64_t,2 > >,std::pair< float,float > >::reverse_iterator(s_geoIp6.upper_bound(k6));