Merge branch 'edge' into windows-ui

.gitignore

@@ -34,7 +34,7 @@ Thumbs.db
 /world/mkworld
 /world/*.c25519
 
-# Miscellaneous file types that we don't want to check in
+# Miscellaneous temporaries, build files, etc.
 *.log
 *.opensdf
 *.user
@@ -50,10 +50,11 @@ Thumbs.db
 *.autosave
 *.tmp
 node_modules
-
-# cluster-geo stuff
 cluster-geo/cluster-geo/config.js
 cluster-geo/cluster-geo/cache.*
+tests/http/zerotier-one
+tests/http/result_*
+tests/http/big-test-out
 
 # MacGap wrapper build files
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*

README.md

@@ -166,6 +166,10 @@ If you're interested, there's a [technical deep dive about NAT traversal on our
 
 If a firewall between you and the Internet blocks ZeroTier's UDP traffic, you will fall back to last-resort TCP tunneling to rootservers over port 443 (https impersonation). This will work almost anywhere but is *very slow* compared to UDP or direct peer to peer connectivity.
 
+### Contributing
+
+There are three main branches: **edge**, **test**, and **master**. Other branches may be for specific features, tests, or use cases. In general **edge** is "bleeding" and may or may not work, while **test** should be relatively stable and **master** is the latest tagged release. Pull requests should generally be done against **test** or **edge**, since pull requests against **master** may be working against a branch that is somewhat out of date.
+
 ### License
 
 The ZeroTier source code is open source and is licensed under the GNU GPL v3 (not LGPL). If you'd like to embed it in a closed-source commercial product or appliance, please e-mail [[email protected]](mailto:[email protected]) to discuss commercial licensing. Otherwise it can be used for free.

include/ZeroTierOne.h

@@ -141,7 +141,7 @@ extern "C" {
 /**
  * Maximum allowed cluster message length in bytes
  */
-#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1444 * 6)
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48)
 
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
@@ -424,15 +424,6 @@ enum ZT_VirtualNetworkConfigOperation
 	ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4
 };
 
-/**
- * Local interface trust levels
- */
-enum ZT_LocalInterfaceAddressTrust {
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 10,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 20
-};
-
 /**
  * What trust hierarchy role does this peer have?
  */
@@ -1337,11 +1328,6 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
 /**
  * Add a local interface address
  *
- * Local interface addresses may be added if you want remote peers
- * with whom you have a trust relatinship (e.g. common network membership)
- * to receive information about these endpoints as potential endpoints for
- * direct communication.
- *
  * Take care that these are never ZeroTier interface addresses, otherwise
  * strange things might happen or they simply won't work.
  *
@@ -1356,11 +1342,9 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
  * reject bad, empty, and unusable addresses.
  *
  * @param addr Local interface address
- * @param metric Local interface metric
- * @param trust How much do you trust the local network under this interface?
  * @return Boolean: non-zero if address was accepted and added
  */
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust);
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr);
 
 /**
  * Clear local interface addresses

make-mac.mk

@@ -6,7 +6,7 @@ ifeq ($(origin CXX),default)
 endif
 
 INCLUDES=
-DEFS=-DZT_ENABLE_CLUSTER
+DEFS=
 LIBS=
 ARCH_FLAGS=-arch x86_64
 

node/AntiRecursion.hpp

@@ -35,28 +35,28 @@
 
 namespace ZeroTier {
 
-#define ZT_ANTIRECURSION_TAIL_LEN 256
+/**
+ * Size of anti-recursion history
+ */
+#define ZT_ANTIRECURSION_HISTORY_SIZE 16
 
 /**
  * Filter to prevent recursion (ZeroTier-over-ZeroTier)
  *
  * This works by logging ZeroTier packets that we send. It's then invoked
- * again against packets read from local Ethernet taps. If the last N
+ * again against packets read from local Ethernet taps. If the last 32
  * bytes representing the ZeroTier packet match in the tap frame, then
  * the frame is a re-injection of a frame that we sent and is rejected.
  *
  * This means that ZeroTier packets simply will not traverse ZeroTier
  * networks, which would cause all sorts of weird problems.
  *
- * NOTE: this is applied to low-level packets before they are sent to
- * SocketManager and/or sockets, not to fully assembled packets before
- * (possible) fragmentation.
+ * This is highly optimized code since it's checked for every packet.
  */
 class AntiRecursion
 {
 public:
 	AntiRecursion()
-		throw()
 	{
 		memset(_history,0,sizeof(_history));
 		_ptr = 0;
@@ -68,13 +68,20 @@ public:
 	 * @param data ZT packet data
 	 * @param len Length of packet
 	 */
-	inline void logOutgoingZT(const void *data,unsigned int len)
-		throw()
+	inline void logOutgoingZT(const void *const data,const unsigned int len)
 	{
-		ArItem *i = &(_history[_ptr++ % ZT_ANTIRECURSION_HISTORY_SIZE]);
-		const unsigned int tl = (len > ZT_ANTIRECURSION_TAIL_LEN) ? ZT_ANTIRECURSION_TAIL_LEN : len;
-		memcpy(i->tail,((const unsigned char *)data) + (len - tl),tl);
-		i->len = tl;
+		if (len < 32)
+			return;
+#ifdef ZT_NO_TYPE_PUNNING
+		memcpy(_history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail,reinterpret_cast<const uint8_t *>(data) + (len - 32),32);
+#else
+		uint64_t *t = _history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail;
+		const uint64_t *p = reinterpret_cast<const uint64_t *>(reinterpret_cast<const uint8_t *>(data) + (len - 32));
+		*(t++) = *(p++);
+		*(t++) = *(p++);
+		*(t++) = *(p++);
+		*t = *p;
+#endif
 	}
 
 	/**
@@ -84,25 +91,36 @@ public:
 	 * @param len Length of frame
 	 * @return True if frame is OK to be passed, false if it's a ZT frame that we sent
 	 */
-	inline bool checkEthernetFrame(const void *data,unsigned int len)
-		throw()
+	inline bool checkEthernetFrame(const void *const data,const unsigned int len) const
 	{
-		for(unsigned int h=0;h<ZT_ANTIRECURSION_HISTORY_SIZE;++h) {
-			ArItem *i = &(_history[h]);
-			if ((i->len > 0)&&(len >= i->len)&&(!memcmp(((const unsigned char *)data) + (len - i->len),i->tail,i->len)))
+		if (len < 32)
+			return true;
+		const uint8_t *const pp = reinterpret_cast<const uint8_t *>(data) + (len - 32);
+		const _ArItem *i = _history;
+		const _ArItem *const end = i + ZT_ANTIRECURSION_HISTORY_SIZE;
+		while (i != end) {
+#ifdef ZT_NO_TYPE_PUNNING
+			if (!memcmp(pp,i->tail,32))
 				return false;
+#else
+			const uint64_t *t = i->tail;
+			const uint64_t *p = reinterpret_cast<const uint64_t *>(pp);
+			uint64_t bits = *(t++) ^ *(p++);
+			bits |= *(t++) ^ *(p++);
+			bits |= *(t++) ^ *(p++);
+			bits |= *t ^ *p;
+			if (!bits)
+				return false;
+#endif
+			++i;
 		}
 		return true;
 	}
 
 private:
-	struct ArItem
-	{
-		unsigned char tail[ZT_ANTIRECURSION_TAIL_LEN];
-		unsigned int len;
-	};
-	ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE];
-	volatile unsigned int _ptr;
+	struct _ArItem { uint64_t tail[4]; };
+	_ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE];
+	volatile unsigned long _ptr;
 };
 
 } // namespace ZeroTier

node/Cluster.cpp

@@ -44,9 +44,9 @@
 #include "CertificateOfMembership.hpp"
 #include "Salsa20.hpp"
 #include "Poly1305.hpp"
-#include "Packet.hpp"
 #include "Identity.hpp"
-#include "Peer.hpp"
+#include "Topology.hpp"
+#include "Packet.hpp"
 #include "Switch.hpp"
 #include "Node.hpp"
 
@@ -82,7 +82,11 @@ Cluster::Cluster(
 	_z(z),
 	_id(id),
 	_zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints),
-	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS])
+	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
+	_peerAffinities(65536),
+	_lastCleanedPeerAffinities(0),
+	_lastCheckedPeersForAnnounce(0),
+	_lastFlushed(0)
 {
 	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 
@@ -200,36 +204,40 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 								}
 #endif
 							}
-							m.lastReceivedAliveAnnouncement = RR->node->now();
 #ifdef ZT_TRACE
-							TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str());
+							if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) {
+								TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str());
+							}
 #endif
+							m.lastReceivedAliveAnnouncement = RR->node->now();
 						}	break;
 
 						case STATE_MESSAGE_HAVE_PEER: {
-							try {
-								Identity id;
-								ptr += id.deserialize(dmsg,ptr);
-								if (id) {
-									RR->topology->saveIdentity(id);
-
-									{	// Add or update peer affinity entry
-										_PeerAffinity pa(id.address(),fromMemberId,RR->node->now());
-										Mutex::Lock _l2(_peerAffinities_m);
-										std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),pa)); // O(log(n))
-										if ((i != _peerAffinities.end())&&(i->key == pa.key)) {
-											i->timestamp = pa.timestamp;
-										} else {
-											_peerAffinities.push_back(pa);
-											std::sort(_peerAffinities.begin(),_peerAffinities.end()); // probably a more efficient way to insert but okay for now
-										}
-			 						}
-
-			 						TRACE("[%u] has %s",(unsigned int)fromMemberId,id.address().toString().c_str());
-			 					}
-							} catch ( ... ) {
-								// ignore invalid identities
-							}
+							const uint64_t now = RR->node->now();
+							Identity id;
+							InetAddress physicalAddress;
+							ptr += id.deserialize(dmsg,ptr);
+							ptr += physicalAddress.deserialize(dmsg,ptr);
+							if (id) {
+								// Forget any paths that we have to this peer at its address
+								if (physicalAddress) {
+									SharedPtr<Peer> myPeerRecord(RR->topology->getPeerNoCache(id.address(),now));
+									if (myPeerRecord)
+										myPeerRecord->removePathByAddress(physicalAddress);
+								}
+
+								// Always save identity to update file time
+								RR->topology->saveIdentity(id);
+
+								// Set peer affinity to its new home
+								{
+									Mutex::Lock _l2(_peerAffinities_m);
+									_PA &pa = _peerAffinities[id.address()];
+									pa.ts = now;
+									pa.mid = fromMemberId;
+		 						}
+		 						TRACE("[%u] has %s @ %s",(unsigned int)fromMemberId,id.address().toString().c_str(),physicalAddress.toString().c_str());
+		 					}
 						}	break;
 
 						case STATE_MESSAGE_MULTICAST_LIKE: {
@@ -238,113 +246,102 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 							const MAC mac(dmsg.field(ptr,6),6); ptr += 6;
 							const uint32_t adi = dmsg.at<uint32_t>(ptr); ptr += 4;
 							RR->mc->add(RR->node->now(),nwid,MulticastGroup(mac,adi),address);
-							TRACE("[%u] %s likes %s/%u on %.16llu",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid);
+							TRACE("[%u] %s likes %s/%.8x on %.16llx",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid);
 						}	break;
 
 						case STATE_MESSAGE_COM: {
+							/* not currently used so not decoded yet
 							CertificateOfMembership com;
 							ptr += com.deserialize(dmsg,ptr);
 							if (com) {
 								TRACE("[%u] COM for %s on %.16llu rev %llu",(unsigned int)fromMemberId,com.issuedTo().toString().c_str(),com.networkId(),com.revision());
 							}
+							*/
 						}	break;
 
-						case STATE_MESSAGE_RELAY: {
+						case STATE_MESSAGE_PROXY_UNITE: {
+							const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
+							const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 							const unsigned int numRemotePeerPaths = dmsg[ptr++];
 							InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max
 							for(unsigned int i=0;i<numRemotePeerPaths;++i)
 								ptr += remotePeerPaths[i].deserialize(dmsg,ptr);
-							const unsigned int packetLen = dmsg.at<uint16_t>(ptr); ptr += 2;
-							const void *packet = (const void *)dmsg.field(ptr,packetLen); ptr += packetLen;
-
-							if (packetLen >= ZT_PROTO_MIN_FRAGMENT_LENGTH) { // ignore anything too short to contain a dest address
-								const Address destinationAddress(reinterpret_cast<const char *>(packet) + 8,ZT_ADDRESS_LENGTH);
-								TRACE("[%u] relay %u bytes to %s (%u remote paths included)",(unsigned int)fromMemberId,packetLen,destinationAddress.toString().c_str(),numRemotePeerPaths);
-
-								SharedPtr<Peer> destinationPeer(RR->topology->getPeer(destinationAddress));
-								if (destinationPeer) {
-									if (
-									    (destinationPeer->send(RR,packet,packetLen,RR->node->now()))&&
-									    (numRemotePeerPaths > 0)&&
-									    (packetLen >= 18)&&
-									    (reinterpret_cast<const unsigned char *>(packet)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR)
-									   ) {
-										// If remote peer paths were sent with this relayed packet, we do
-										// RENDEZVOUS. It's handled here for cluster-relayed packets since
-										// we don't have both Peer records so this is a different path.
-
-										const Address remotePeerAddress(reinterpret_cast<const char *>(packet) + 13,ZT_ADDRESS_LENGTH);
-
-										InetAddress bestDestV4,bestDestV6;
-										destinationPeer->getBestActiveAddresses(RR->node->now(),bestDestV4,bestDestV6);
-										InetAddress bestRemoteV4,bestRemoteV6;
-										for(unsigned int i=0;i<numRemotePeerPaths;++i) {
-											if ((bestRemoteV4)&&(bestRemoteV6))
-												break;
-											switch(remotePeerPaths[i].ss_family) {
-												case AF_INET:
-													if (!bestRemoteV4)
-														bestRemoteV4 = remotePeerPaths[i];
-													break;
-												case AF_INET6:
-													if (!bestRemoteV6)
-														bestRemoteV6 = remotePeerPaths[i];
-													break;
-											}
-										}
-
-										Packet rendezvousForDest(destinationAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
-										rendezvousForDest.append((uint8_t)0);
-										remotePeerAddress.appendTo(rendezvousForDest);
-
-										Buffer<2048> rendezvousForOtherEnd;
-										remotePeerAddress.appendTo(rendezvousForOtherEnd);
-										rendezvousForOtherEnd.append((uint8_t)Packet::VERB_RENDEZVOUS);
-										const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForOtherEnd.size();
-										rendezvousForOtherEnd.addSize(2); // space for actual packet payload length
-										rendezvousForOtherEnd.append((uint8_t)0); // flags == 0
-										destinationAddress.appendTo(rendezvousForOtherEnd);
-
-										bool haveMatch = false;
-										if ((bestDestV6)&&(bestRemoteV6)) {
-											haveMatch = true;
-
-											rendezvousForDest.append((uint16_t)bestRemoteV6.port());
-											rendezvousForDest.append((uint8_t)16);
-											rendezvousForDest.append(bestRemoteV6.rawIpData(),16);
-
-											rendezvousForOtherEnd.append((uint16_t)bestDestV6.port());
-											rendezvousForOtherEnd.append((uint8_t)16);
-											rendezvousForOtherEnd.append(bestDestV6.rawIpData(),16);
-											rendezvousForOtherEnd.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16));
-										} else if ((bestDestV4)&&(bestRemoteV4)) {
-											haveMatch = true;
-
-											rendezvousForDest.append((uint16_t)bestRemoteV4.port());
-											rendezvousForDest.append((uint8_t)4);
-											rendezvousForDest.append(bestRemoteV4.rawIpData(),4);
-
-											rendezvousForOtherEnd.append((uint16_t)bestDestV4.port());
-											rendezvousForOtherEnd.append((uint8_t)4);
-											rendezvousForOtherEnd.append(bestDestV4.rawIpData(),4);
-											rendezvousForOtherEnd.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4));
-										}
-
-										if (haveMatch) {
-											_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForOtherEnd.data(),rendezvousForOtherEnd.size());
-											RR->sw->send(rendezvousForDest,true,0);
-										}
+
+							TRACE("[%u] requested that we unite local %s with remote %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str());
+
+							const uint64_t now = RR->node->now();
+							SharedPtr<Peer> localPeer(RR->topology->getPeerNoCache(localPeerAddress,now));
+							if ((localPeer)&&(numRemotePeerPaths > 0)) {
+								InetAddress bestLocalV4,bestLocalV6;
+								localPeer->getBestActiveAddresses(now,bestLocalV4,bestLocalV6);
+
+								InetAddress bestRemoteV4,bestRemoteV6;
+								for(unsigned int i=0;i<numRemotePeerPaths;++i) {
+									if ((bestRemoteV4)&&(bestRemoteV6))
+										break;
+									switch(remotePeerPaths[i].ss_family) {
+										case AF_INET:
+											if (!bestRemoteV4)
+												bestRemoteV4 = remotePeerPaths[i];
+											break;
+										case AF_INET6:
+											if (!bestRemoteV6)
+												bestRemoteV6 = remotePeerPaths[i];
+											break;
 									}
 								}
+
+								Packet rendezvousForLocal(localPeerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
+								rendezvousForLocal.append((uint8_t)0);
+								remotePeerAddress.appendTo(rendezvousForLocal);
+
+								Buffer<2048> rendezvousForRemote;
+								remotePeerAddress.appendTo(rendezvousForRemote);
+								rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS);
+								const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForRemote.size();
+								rendezvousForRemote.addSize(2); // space for actual packet payload length
+								rendezvousForRemote.append((uint8_t)0); // flags == 0
+								localPeerAddress.appendTo(rendezvousForRemote);
+
+								bool haveMatch = false;
+								if ((bestLocalV6)&&(bestRemoteV6)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV6.port());
+									rendezvousForLocal.append((uint8_t)16);
+									rendezvousForLocal.append(bestRemoteV6.rawIpData(),16);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV6.port());
+									rendezvousForRemote.append((uint8_t)16);
+									rendezvousForRemote.append(bestLocalV6.rawIpData(),16);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16));
+								} else if ((bestLocalV4)&&(bestRemoteV4)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV4.port());
+									rendezvousForLocal.append((uint8_t)4);
+									rendezvousForLocal.append(bestRemoteV4.rawIpData(),4);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV4.port());
+									rendezvousForRemote.append((uint8_t)4);
+									rendezvousForRemote.append(bestLocalV4.rawIpData(),4);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4));
+								}
+
+								if (haveMatch) {
+									_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
+									_flush(fromMemberId); // we want this to go ASAP, since with port restricted cone NATs success can be timing-sensitive
+									RR->sw->send(rendezvousForLocal,true,0);
+								}
 							}
 						}	break;
 
 						case STATE_MESSAGE_PROXY_SEND: {
-							const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+							const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
 							const Packet::Verb verb = (Packet::Verb)dmsg[ptr++];
 							const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2;
 							Packet outp(rcpt,RR->identity.address(),verb);
-							outp.append(dmsg.field(ptr,len),len);
+							outp.append(dmsg.field(ptr,len),len); ptr += len;
 							RR->sw->send(outp,true,0);
 							TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len);
 						}	break;
@@ -363,81 +360,78 @@ void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
 	}
 }
 
-bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len)
+bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
 {
 	if (len > 16384) // sanity check
 		return false;
 
-	uint64_t mostRecentTimestamp = 0;
-	uint16_t canHasPeer = 0;
+	const uint64_t now = RR->node->now();
+	unsigned int canHasPeer = 0;
 
 	{	// Anyone got this peer?
 		Mutex::Lock _l2(_peerAffinities_m);
-		std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),_PeerAffinity(toPeerAddress,0,0))); // O(log(n))
-		while ((i != _peerAffinities.end())&&(i->address() == toPeerAddress)) {
-			uint16_t mid = i->clusterMemberId();
-			if ((mid != _id)&&(i->timestamp > mostRecentTimestamp)) {
-				mostRecentTimestamp = i->timestamp;
-				canHasPeer = mid;
-			}
-		}
+		_PA *pa = _peerAffinities.get(toPeerAddress);
+		if ((pa)&&(pa->mid != _id)&&((now - pa->ts) < ZT_PEER_ACTIVITY_TIMEOUT))
+			canHasPeer = pa->mid;
+		else return false;
 	}
 
-	const uint64_t now = RR->node->now();
-	if ((now - mostRecentTimestamp) < ZT_PEER_ACTIVITY_TIMEOUT) {
-		Buffer<16384> buf;
-
+	Buffer<1024> buf;
+	if (unite) {
 		InetAddress v4,v6;
 		if (fromPeerAddress) {
-			SharedPtr<Peer> fromPeer(RR->topology->getPeer(fromPeerAddress));
+			SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress,now));
 			if (fromPeer)
 				fromPeer->getBestActiveAddresses(now,v4,v6);
 		}
-		buf.append((uint8_t)( (v4) ? ((v6) ? 2 : 1) : ((v6) ? 1 : 0) ));
+		uint8_t addrCount = 0;
 		if (v4)
-			v4.serialize(buf);
+			++addrCount;
 		if (v6)
-			v6.serialize(buf);
-		buf.append((uint16_t)len);
-		buf.append(data,len);
-
-		{
-			Mutex::Lock _l2(_members[canHasPeer].lock);
-			_send(canHasPeer,STATE_MESSAGE_RELAY,buf.data(),buf.size());
+			++addrCount;
+		if (addrCount) {
+			toPeerAddress.appendTo(buf);
+			fromPeerAddress.appendTo(buf);
+			buf.append(addrCount);
+			if (v4)
+				v4.serialize(buf);
+			if (v6)
+				v6.serialize(buf);
 		}
-
-		TRACE("sendViaCluster(): relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)canHasPeer);
-		return true;
-	} else {
-		TRACE("sendViaCluster(): unable to relay %u bytes from %s to %s since no cluster members seem to have it!",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str());
-		return false;
 	}
+	{
+		Mutex::Lock _l2(_members[canHasPeer].lock);
+		if (buf.size() > 0)
+			_send(canHasPeer,STATE_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
+		if (_members[canHasPeer].zeroTierPhysicalEndpoints.size() > 0)
+			RR->node->putPacket(InetAddress(),_members[canHasPeer].zeroTierPhysicalEndpoints.front(),data,len);
+	}
+
+	TRACE("sendViaCluster(): relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)canHasPeer);
+
+	return true;
 }
 
-void Cluster::replicateHavePeer(const Identity &peerId)
+void Cluster::replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress)
 {
-	{	// Use peer affinity table to track our own last announce time for peers
-		_PeerAffinity pa(peerId.address(),_id,RR->node->now());
+	const uint64_t now = RR->node->now();
+	{
 		Mutex::Lock _l2(_peerAffinities_m);
-		std::vector<_PeerAffinity>::iterator i(std::lower_bound(_peerAffinities.begin(),_peerAffinities.end(),pa)); // O(log(n))
-		if ((i != _peerAffinities.end())&&(i->key == pa.key)) {
-			if ((pa.timestamp - i->timestamp) >= ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) {
-				i->timestamp = pa.timestamp;
-				// continue to announcement
-			} else {
-				// we've already announced this peer recently, so skip
-				return;
-			}
+		_PA &pa = _peerAffinities[peerId.address()];
+		if (pa.mid != _id) {
+			pa.ts = now;
+			pa.mid = _id;
+		} else if ((now - pa.ts) < ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) {
+			return;
 		} else {
-			_peerAffinities.push_back(pa);
-			std::sort(_peerAffinities.begin(),_peerAffinities.end()); // probably a more efficient way to insert but okay for now
-			// continue to announcement
+			pa.ts = now;
 		}
 	}
 
 	// announcement
 	Buffer<4096> buf;
 	peerId.serialize(buf,false);
+	physicalAddress.serialize(buf);
 	{
 		Mutex::Lock _l(_memberIds_m);
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -449,7 +443,7 @@ void Cluster::replicateHavePeer(const Identity &peerId)
 
 void Cluster::replicateMulticastLike(uint64_t nwid,const Address &peerAddress,const MulticastGroup &group)
 {
-	Buffer<2048> buf;
+	Buffer<1024> buf;
 	buf.append((uint64_t)nwid);
 	peerAddress.appendTo(buf);
 	group.mac().appendTo(buf);
@@ -466,7 +460,7 @@ void Cluster::replicateMulticastLike(uint64_t nwid,const Address &peerAddress,co
 
 void Cluster::replicateCertificateOfNetworkMembership(const CertificateOfMembership &com)
 {
-	Buffer<2048> buf;
+	Buffer<4096> buf;
 	com.serialize(buf);
 	TRACE("replicating %s COM for %.16llx to all members",com.issuedTo().toString().c_str(),com.networkId());
 	{
@@ -478,11 +472,47 @@ void Cluster::replicateCertificateOfNetworkMembership(const CertificateOfMembers
 	}
 }
 
+struct _ClusterAnnouncePeers
+{
+	_ClusterAnnouncePeers(const uint64_t now_,Cluster *parent_) : now(now_),parent(parent_) {}
+	const uint64_t now;
+	Cluster *const parent;
+	inline void operator()(const Topology &t,const SharedPtr<Peer> &peer) const
+	{
+		Path *p = peer->getBestPath(now);
+		if (p)
+			parent->replicateHavePeer(peer->identity(),p->address());
+	}
+};
 void Cluster::doPeriodicTasks()
 {
 	const uint64_t now = RR->node->now();
 
-	{
+	// Erase old peer affinity entries just to control table size
+	if ((now - _lastCleanedPeerAffinities) >= (ZT_PEER_ACTIVITY_TIMEOUT * 5)) {
+		_lastCleanedPeerAffinities = now;
+		Address *k = (Address *)0;
+		_PA *v = (_PA *)0;
+		Mutex::Lock _l(_peerAffinities_m);
+		Hashtable< Address,_PA >::Iterator i(_peerAffinities);
+		while (i.next(k,v)) {
+			if ((now - v->ts) >= (ZT_PEER_ACTIVITY_TIMEOUT * 5))
+				_peerAffinities.erase(*k);
+		}
+	}
+
+	// Announce peers that we have active direct paths to -- note that we forget paths
+	// that other cluster members claim they have, which prevents us from fighting
+	// with other cluster members (route flapping) over specific paths.
+	if ((now - _lastCheckedPeersForAnnounce) >= (ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD / 4)) {
+		_lastCheckedPeersForAnnounce = now;
+		_ClusterAnnouncePeers func(now,this);
+		RR->topology->eachPeer<_ClusterAnnouncePeers &>(func);
+	}
+
+	// Flush outgoing packet send queue every doPeriodicTasks()
+	if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) {
+		_lastFlushed = now;
 		Mutex::Lock _l(_memberIds_m);
 		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
 			Mutex::Lock _l2(_members[*mid].lock);
@@ -564,26 +594,22 @@ void Cluster::removeMember(uint16_t memberId)
 	_memberIds = newMemberIds;
 }
 
-bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
+bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
 {
-	if (!peerPhysicalAddress) // sanity check
-		return false;
-
 	if (_addressToLocationFunction) {
 		// Pick based on location if it can be determined
 		int px = 0,py = 0,pz = 0;
 		if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
-			TRACE("no geolocation available for %s",peerPhysicalAddress.toIpString().c_str());
+			TRACE("no geolocation data for %s (geo-lookup is lazy/async so it may work next time)",peerPhysicalAddress.toIpString().c_str());
 			return false;
 		}
 
 		// Find member closest to this peer
 		const uint64_t now = RR->node->now();
-		std::vector<InetAddress> best; // initial "best" is for peer to stay put
+		std::vector<InetAddress> best;
 		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
 		double bestDistance = (offload ? 2147483648.0 : currentDistance);
 		unsigned int bestMember = _id;
-		TRACE("%s is at %d,%d,%d -- looking for anyone closer than %d,%d,%d (%fkm)",peerPhysicalAddress.toString().c_str(),px,py,pz,_x,_y,_z,bestDistance);
 		{
 			Mutex::Lock _l(_memberIds_m);
 			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
@@ -592,7 +618,7 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 
 				// Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to
 				if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) {
-					double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
+					const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
 					if (mdist < bestDistance) {
 						bestDistance = mdist;
 						bestMember = *mid;
@@ -602,36 +628,16 @@ bool Cluster::redirectPeer(const Address &peerAddress,const InetAddress &peerPhy
 			}
 		}
 
-		if (best.size() > 0) {
-			TRACE("%s seems closer to %u at %fkm, suggesting redirect...",peerAddress.toString().c_str(),bestMember,bestDistance);
-
-			/* if (peer->remoteVersionProtocol() >= 5) {
-				// If it's a newer peer send VERB_PUSH_DIRECT_PATHS which is more idiomatic
-			} else { */
-				// Otherwise send VERB_RENDEZVOUS for ourselves, which will trick peers into trying other endpoints for us even if they're too old for PUSH_DIRECT_PATHS
-				for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
-					if ((a->ss_family == AF_INET)||(a->ss_family == AF_INET6)) {
-						Packet outp(peerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
-						outp.append((uint8_t)0); // no flags
-						RR->identity.address().appendTo(outp); // HACK: rendezvous with ourselves! with really old peers this will only work if I'm a root server!
-						outp.append((uint16_t)a->port());
-						if (a->ss_family == AF_INET) {
-							outp.append((uint8_t)4);
-							outp.append(a->rawIpData(),4);
-						} else {
-							outp.append((uint8_t)16);
-							outp.append(a->rawIpData(),16);
-						}
-						RR->sw->send(outp,true,0);
-					}
-				}
-			//}
-
-			return true;
-		} else {
-			TRACE("peer %s is at [%d,%d,%d], distance to us is %f and this seems to be the best",peerAddress.toString().c_str(),px,py,pz,currentDistance);
-			return false;
+		// Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family
+		for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
+			if (a->ss_family == peerPhysicalAddress.ss_family) {
+				TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str());
+				redirectTo = *a;
+				return true;
+			}
 		}
+		TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance);
+		return false;
 	} else {
 		// TODO: pick based on load if no location info?
 		return false;
@@ -653,7 +659,7 @@ void Cluster::status(ZT_ClusterStatus &status) const
 	ms[_id]->x = _x;
 	ms[_id]->y = _y;
 	ms[_id]->z = _z;
-	ms[_id]->peers = RR->topology->countAlive();
+	ms[_id]->peers = RR->topology->countActive();
 	for(std::vector<InetAddress>::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) {
 		if (ms[_id]->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
 			break;
@@ -686,10 +692,12 @@ void Cluster::status(ZT_ClusterStatus &status) const
 
 	{
 		Mutex::Lock _l2(_peerAffinities_m);
-		for(std::vector<_PeerAffinity>::const_iterator pi(_peerAffinities.begin());pi!=_peerAffinities.end();++pi) {
-			unsigned int mid = pi->clusterMemberId();
-			if ((ms[mid])&&(mid != _id)&&((now - pi->timestamp) < ZT_PEER_ACTIVITY_TIMEOUT))
-				++ms[mid]->peers;
+		Address *k = (Address *)0;
+		_PA *v = (_PA *)0;
+		Hashtable< Address,_PA >::Iterator i(const_cast<Cluster *>(this)->_peerAffinities);
+		while (i.next(k,v)) {
+			if ( (ms[v->mid]) && (v->mid != _id) && ((now - v->ts) < ZT_PEER_ACTIVITY_TIMEOUT) )
+				++ms[v->mid]->peers;
 		}
 	}
 }

node/Cluster.hpp

@@ -46,18 +46,26 @@
 
 /**
  * Timeout for cluster members being considered "alive"
+ *
+ * A cluster member is considered dead and will no longer have peers
+ * redirected to it if we have not heard a heartbeat in this long.
  */
 #define ZT_CLUSTER_TIMEOUT 10000
 
 /**
  * How often should we announce that we have a peer?
  */
-#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD 60000
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ZT_PEER_DIRECT_PING_DELAY
 
 /**
  * Desired period between doPeriodicTasks() in milliseconds
  */
-#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 100
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
+
+/**
+ * How often to flush outgoing message queues (maximum interval)
+ */
+#define ZT_CLUSTER_FLUSH_PERIOD 500
 
 namespace ZeroTier {
 
@@ -117,6 +125,10 @@ public:
 		/**
 		 * Cluster member has this peer:
 		 *   <[...] binary serialized peer identity>
+		 *   <[...] binary serialized peer remote physical address>
+		 *
+		 * Clusters send this message when they learn a path to a peer. The
+		 * replicated physical address is the one learned.
 		 */
 		STATE_MESSAGE_HAVE_PEER = 2,
 
@@ -136,13 +148,18 @@ public:
 		STATE_MESSAGE_COM = 4,
 
 		/**
-		 * Relay a packet to a peer:
-		 *   <[1] 8-bit number of sending peer active path addresses>
-		 *   <[...] series of serialized InetAddresses of sending peer's paths>
-		 *   <[2] 16-bit packet length>
-		 *   <[...] packet or packet fragment>
+		 * Request that VERB_RENDEZVOUS be sent to a peer that we have:
+		 *   <[5] ZeroTier address of peer on recipient's side>
+		 *   <[5] ZeroTier address of peer on sender's side>
+		 *   <[1] 8-bit number of sender's peer's active path addresses>
+		 *   <[...] series of serialized InetAddresses of sender's peer's paths>
+		 *
+		 * This requests that we perform NAT-t introduction between a peer that
+		 * we have and one on the sender's side. The sender furnishes contact
+		 * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours
+		 * directly and with PROXY_SEND to theirs.
 		 */
-		STATE_MESSAGE_RELAY = 5,
+		STATE_MESSAGE_PROXY_UNITE = 5,
 
 		/**
 		 * Request that a cluster member send a packet to a locally-known peer:
@@ -158,7 +175,20 @@ public:
 		 * while PROXY_SEND is used to implement proxy sending (which right
 		 * now is only used to send RENDEZVOUS).
 		 */
-		STATE_MESSAGE_PROXY_SEND = 6
+		STATE_MESSAGE_PROXY_SEND = 6,
+
+		/**
+		 * Replicate a network config for a network we belong to:
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit length of network config>
+		 *   <[...] serialized network config>
+		 *
+		 * This is used by clusters to avoid every member having to query
+		 * for the same netconf for networks all members belong to.
+		 *
+		 * TODO: not implemented yet!
+		 */
+		STATE_MESSAGE_NETWORK_CONFIG = 7
 	};
 
 	/**
@@ -198,16 +228,18 @@ public:
 	 * @param toPeerAddress Destination peer address
 	 * @param data Packet or packet fragment data
 	 * @param len Length of packet or fragment
+	 * @param unite If true, also request proxy unite across cluster
 	 * @return True if this data was sent via another cluster member, false if none have this peer
 	 */
-	bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len);
+	bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
 
 	/**
 	 * Advertise to the cluster that we have this peer
 	 *
 	 * @param peerId Identity of peer that we have
+	 * @param physicalAddress Physical address of peer (from our POV)
 	 */
-	void replicateHavePeer(const Identity &peerId);
+	void replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress);
 
 	/**
 	 * Advertise a multicast LIKE to the cluster
@@ -245,14 +277,15 @@ public:
 	void removeMember(uint16_t memberId);
 
 	/**
-	 * Redirect this peer to a better cluster member if needed
+	 * Find a better cluster endpoint for this peer (if any)
 	 *
-	 * @param peerAddress Peer to (possibly) redirect
+	 * @param redirectTo InetAddress to be set to a better endpoint (if there is one)
+	 * @param peerAddress Address of peer to (possibly) redirect
 	 * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address())
 	 * @param offload Always redirect if possible -- can be used to offload peers during shutdown
-	 * @return True if peer was redirected
+	 * @return True if redirectTo was set to a new address, false if redirectTo was not modified
 	 */
-	bool redirectPeer(const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
+	bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
 
 	/**
 	 * Fill out ZT_ClusterStatus structure (from core API)
@@ -265,7 +298,7 @@ private:
 	void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len);
 	void _flush(uint16_t memberId);
 
-	// These are initialized in the constructor and remain static
+	// These are initialized in the constructor and remain immutable
 	uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	const RuntimeEnvironment *RR;
@@ -278,6 +311,7 @@ private:
 	const int32_t _z;
 	const uint16_t _id;
 	const std::vector<InetAddress> _zeroTierPhysicalEndpoints;
+	// end immutable fields
 
 	struct _Member
 	{
@@ -310,31 +344,23 @@ private:
 		_Member() { this->clear(); }
 		~_Member() { Utils::burn(key,sizeof(key)); }
 	};
-
-	_Member *const _members; // cluster IDs can be from 0 to 65535 (16-bit)
+	_Member *const _members;
 
 	std::vector<uint16_t> _memberIds;
 	Mutex _memberIds_m;
 
-	// Record tracking which members have which peers and how recently they claimed this -- also used to track our last claimed time
-	struct _PeerAffinity
+	struct _PA
 	{
-		_PeerAffinity(const Address &a,uint16_t mid,uint64_t ts) :
-			key((a.toInt() << 16) | (uint64_t)mid),
-			timestamp(ts) {}
-
-		uint64_t key;
-		uint64_t timestamp;
-
-		inline Address address() const throw() { return Address(key >> 16); }
-		inline uint16_t clusterMemberId() const throw() { return (uint16_t)(key & 0xffff); }
-
-		inline bool operator<(const _PeerAffinity &pi) const throw() { return (key < pi.key); }
+		_PA() : ts(0),mid(0xffff) {}
+		uint64_t ts;
+		uint16_t mid;
 	};
-
-	// A memory-efficient packed map of _PeerAffinity records searchable with std::binary_search() and std::lower_bound()
-	std::vector<_PeerAffinity> _peerAffinities;
+	Hashtable< Address,_PA > _peerAffinities;
 	Mutex _peerAffinities_m;
+
+	uint64_t _lastCleanedPeerAffinities;
+	uint64_t _lastCheckedPeersForAnnounce;
+	uint64_t _lastFlushed;
 };
 
 } // namespace ZeroTier

node/Constants.hpp

@@ -173,13 +173,8 @@
 
 /**
  * Timeout for receipt of fragmented packets in ms
- *
- * Since there's no retransmits, this is just a really bad case scenario for
- * transit time. It's short enough that a DOS attack from exhausing buffers is
- * very unlikely, as the transfer rate would have to be fast enough to fill
- * system memory in this time.
  */
-#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 1000
+#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 500
 
 /**
  * Length of secret key in bytes -- 256-bit -- do not change
@@ -194,7 +189,7 @@
 /**
  * Overriding granularity for timer tasks to prevent CPU-intensive thrashing on every packet
  */
-#define ZT_CORE_TIMER_TASK_GRANULARITY 1000
+#define ZT_CORE_TIMER_TASK_GRANULARITY 500
 
 /**
  * How long to remember peer records in RAM if they haven't been used
@@ -269,28 +264,17 @@
 /**
  * Delay between ordinary case pings of direct links
  */
-#define ZT_PEER_DIRECT_PING_DELAY 120000
-
-/**
- * Delay between requests for updated network autoconf information
- */
-#define ZT_NETWORK_AUTOCONF_DELAY 60000
+#define ZT_PEER_DIRECT_PING_DELAY 60000
 
 /**
  * Timeout for overall peer activity (measured from last receive)
  */
-#define ZT_PEER_ACTIVITY_TIMEOUT (ZT_PEER_DIRECT_PING_DELAY + (ZT_PING_CHECK_INVERVAL * 3))
+#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 4) + ZT_PING_CHECK_INVERVAL)
 
 /**
- * Stop relaying via peers that have not responded to direct sends
- *
- * When we send something (including frames), we generally expect a response.
- * Switching relays if no response in a short period of time causes more
- * rapid failover if a root server goes down or becomes unreachable. In the
- * mistaken case, little harm is done as it'll pick the next-fastest
- * root server and will switch back eventually.
+ * Delay between requests for updated network autoconf information
  */
-#define ZT_PEER_RELAY_CONVERSATION_LATENCY_THRESHOLD 10000
+#define ZT_NETWORK_AUTOCONF_DELAY 60000
 
 /**
  * Minimum interval between attempts by relays to unite peers
@@ -299,7 +283,7 @@
  * a RENDEZVOUS message no more than this often. This instructs the peers
  * to attempt NAT-t and gives each the other's corresponding IP:port pair.
  */
-#define ZT_MIN_UNITE_INTERVAL 60000
+#define ZT_MIN_UNITE_INTERVAL 30000
 
 /**
  * Delay between initial direct NAT-t packet and more aggressive techniques
@@ -309,25 +293,10 @@
  */
 #define ZT_NAT_T_TACTICAL_ESCALATION_DELAY 1000
 
-/**
- * Size of anti-recursion history (see AntiRecursion.hpp)
- */
-#define ZT_ANTIRECURSION_HISTORY_SIZE 16
-
 /**
  * Minimum delay between attempts to confirm new paths to peers (to avoid HELLO flooding)
  */
-#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 5000
-
-/**
- * Interval between direct path pushes in milliseconds
- */
-#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000
-
-/**
- * Minimum interval between direct path pushes from a given peer or we will ignore them
- */
-#define ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL 2500
+#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 1000
 
 /**
  * How long (max) to remember network certificates of membership?
@@ -353,9 +322,28 @@
 #define ZT_MAX_BRIDGE_SPAM 16
 
 /**
- * Maximum number of endpoints to contact per address type (to limit pushes like GitHub issue #235)
+ * Interval between direct path pushes in milliseconds
+ */
+#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000
+
+/**
+ * Time horizon for push direct paths cutoff
+ */
+#define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000
+
+/**
+ * Maximum number of direct path pushes within cutoff time
+ *
+ * This limits response to PUSH_DIRECT_PATHS to CUTOFF_LIMIT responses
+ * per CUTOFF_TIME milliseconds per peer to prevent this from being
+ * useful for DOS amplification attacks.
+ */
+#define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5
+
+/**
+ * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6)
  */
-#define ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE 8
+#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 1
 
 /**
  * A test pseudo-network-ID that can be joined

node/Hashtable.hpp

@@ -322,7 +322,6 @@ public:
 		b->next = _t[bidx];
 		_t[bidx] = b;
 		++_s;
-
 		return b->v;
 	}
 
@@ -351,7 +350,6 @@ public:
 		b->next = _t[bidx];
 		_t[bidx] = b;
 		++_s;
-
 		return b->v;
 	}
 

node/IncomingPacket.cpp

@@ -44,6 +44,8 @@
 #include "SHA512.hpp"
 #include "World.hpp"
 #include "Cluster.hpp"
+#include "Node.hpp"
+#include "AntiRecursion.hpp"
 
 namespace ZeroTier {
 
@@ -121,7 +123,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 
 			case Packet::ERROR_OBJ_NOT_FOUND:
 				if (inReVerb == Packet::VERB_WHOIS) {
-					if (RR->topology->isRoot(peer->identity()))
+					if (RR->topology->isUpstream(peer->identity()))
 						RR->sw->cancelWhoisRequest(Address(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH));
 				} else if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) {
 					SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
@@ -154,6 +156,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
 						nconf->com().serialize(outp);
 						outp.armor(peer->key(),true);
+						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					}
 				}
@@ -201,13 +204,13 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		const uint64_t timestamp = at<uint64_t>(ZT_PROTO_VERB_HELLO_IDX_TIMESTAMP);
 
 		Identity id;
-		InetAddress destAddr;
+		InetAddress externalSurfaceAddress;
 		uint64_t worldId = ZT_WORLD_ID_NULL;
 		uint64_t worldTimestamp = 0;
 		{
 			unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
 			if (ptr < size()) // ZeroTier One < 1.0.3 did not include physical destination address info
-				ptr += destAddr.deserialize(*this,ptr);
+				ptr += externalSurfaceAddress.deserialize(*this,ptr);
 			if ((ptr + 16) <= size()) { // older versions also did not include World IDs or timestamps
 				worldId = at<uint64_t>(ptr); ptr += 8;
 				worldTimestamp = at<uint64_t>(ptr);
@@ -280,11 +283,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 
 		// VALID -- if we made it here, packet passed identity and authenticity checks!
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
-		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
-
-		if (destAddr)
-			RR->sa->iam(id.address(),_remoteAddress,destAddr,RR->topology->isRoot(id),RR->node->now());
+		if (externalSurfaceAddress)
+			RR->sa->iam(id.address(),_remoteAddress,externalSurfaceAddress,RR->topology->isRoot(id),RR->node->now());
 
 		Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_HELLO);
@@ -294,7 +294,36 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
 		outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
 		outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
-		_remoteAddress.serialize(outp);
+		if (protoVersion >= 5) {
+			_remoteAddress.serialize(outp);
+		} else {
+			/* LEGACY COMPATIBILITY HACK:
+			 *
+			 * For a while now (since 1.0.3), ZeroTier has recognized changes in
+			 * its network environment empirically by examining its external network
+			 * address as reported by trusted peers. In versions prior to 1.1.0
+			 * (protocol version < 5), they did this by saving a snapshot of this
+			 * information (in SelfAwareness.hpp) keyed by reporting device ID and
+			 * address type.
+			 *
+			 * This causes problems when clustering is combined with symmetric NAT.
+			 * Symmetric NAT remaps ports, so different endpoints in a cluster will
+			 * report back different exterior addresses. Since the old code keys
+			 * this by device ID and not sending physical address and compares the
+			 * entire address including port, it constantly thinks its external
+			 * surface is changing and resets connections when talking to a cluster.
+			 *
+			 * In new code we key by sending physical address and device and we also
+			 * take the more conservative position of only interpreting changes in
+			 * IP address (neglecting port) as a change in network topology that
+			 * necessitates a reset. But we can make older clients work here by
+			 * nulling out the port field. Since this info is only used for empirical
+			 * detection of link changes, it doesn't break anything else.
+			 */
+			InetAddress tmpa(_remoteAddress);
+			tmpa.setPort(0);
+			tmpa.serialize(outp);
+		}
 
 		if ((worldId != ZT_WORLD_ID_NULL)&&(RR->topology->worldTimestamp() > worldTimestamp)&&(worldId == RR->topology->worldId())) {
 			World w(RR->topology->world());
@@ -307,7 +336,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		}
 
 		outp.armor(peer->key(),true);
+		RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+
+		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -331,12 +364,17 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				const unsigned int vMinor = (*this)[ZT_PROTO_VERB_HELLO__OK__IDX_MINOR_VERSION];
 				const unsigned int vRevision = at<uint16_t>(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION);
 
+				if (vProto < ZT_PROTO_VERSION_MIN) {
+					TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str());
+					return true;
+				}
+
 				const bool trusted = RR->topology->isRoot(peer->identity());
 
-				InetAddress destAddr;
+				InetAddress externalSurfaceAddress;
 				unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2;
 				if (ptr < size()) // ZeroTier One < 1.0.3 did not include this field
-					ptr += destAddr.deserialize(*this,ptr);
+					ptr += externalSurfaceAddress.deserialize(*this,ptr);
 				if ((trusted)&&((ptr + 2) <= size())) { // older versions also did not include this field, and right now we only use if from a root
 					World worldUpdate;
 					const unsigned int worldLen = at<uint16_t>(ptr); ptr += 2;
@@ -347,18 +385,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					}
 				}
 
-				if (vProto < ZT_PROTO_VERSION_MIN) {
-					TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str());
-					return true;
-				}
-
-				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency,((destAddr) ? destAddr.toString().c_str() : "(none)"));
+				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)"));
 
 				peer->addDirectLatencyMeasurment(latency);
 				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
-				if (destAddr)
-					RR->sa->iam(peer->address(),_remoteAddress,destAddr,trusted,RR->node->now());
+				if (externalSurfaceAddress)
+					RR->sa->iam(peer->address(),_remoteAddress,externalSurfaceAddress,trusted,RR->node->now());
 			}	break;
 
 			case Packet::VERB_WHOIS: {
@@ -442,6 +475,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append(packetId());
 				queried.serialize(outp,false);
 				outp.armor(peer->key(),true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			} else {
 				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
@@ -450,6 +484,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 				outp.append(payload(),ZT_ADDRESS_LENGTH);
 				outp.armor(peer->key(),true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			}
 		} else {
@@ -607,6 +642,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		outp.append((unsigned char)Packet::VERB_ECHO);
 		outp.append((uint64_t)pid);
 		outp.append(field(ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD),size() - ZT_PACKET_IDX_PAYLOAD);
+		RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP);
 	} catch ( ... ) {
@@ -622,7 +658,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 
 		// Iterate through 18-byte network,MAC,ADI tuples
 		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18) {
-			const uint32_t nwid(at<uint64_t>(ptr));
+			const uint64_t nwid = at<uint64_t>(ptr);
 			const MulticastGroup group(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14));
 			RR->mc->add(now,nwid,group,peer->address());
 #ifdef ZT_ENABLE_CLUSTER
@@ -692,6 +728,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 						if (outp.size() > ZT_PROTO_MAX_PACKET_LENGTH) { // sanity check
 							TRACE("NETWORK_CONFIG_REQUEST failed: internal error: netconf size %u is too large",(unsigned int)netconfStr.length());
 						} else {
+							RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 							RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 						}
 					}
@@ -704,6 +741,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				}	break;
 
@@ -714,6 +752,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				} break;
 
@@ -736,6 +775,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
 			outp.append(nwid);
 			outp.armor(peer->key(),true);
+			RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 			RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 		}
 	} catch ( ... ) {
@@ -780,6 +820,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 			outp.append((uint32_t)mg.adi());
 			if (RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit)) {
 				outp.armor(peer->key(),true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			}
 		}
@@ -872,6 +913,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 				outp.append((unsigned char)0x02); // flag 0x02 = contains gather results
 				if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) {
 					outp.armor(peer->key(),true);
+					RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				}
 			}
@@ -888,17 +930,19 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 {
 	try {
 		const uint64_t now = RR->node->now();
-		if ((now - peer->lastDirectPathPushReceived()) >= ZT_DIRECT_PATH_PUSH_MIN_RECEIVE_INTERVAL) {
-			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): too frequent!",source().toString().c_str(),_remoteAddress.toString().c_str());
+
+		// First, subject this to a rate limit
+		if (!peer->shouldRespondToDirectPathPush(now)) {
+			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str());
 			return true;
 		}
-		peer->setLastDirectPathPushReceived(now);
 
-		const RemotePath *currentBest = peer->getBestPath(now);
+		// Second, limit addresses by scope and type
+		uint8_t countPerScope[ZT_INETADDRESS_MAX_SCOPE+1][2]; // [][0] is v4, [][1] is v6
+		memset(countPerScope,0,sizeof(countPerScope));
 
 		unsigned int count = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD);
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD + 2;
-		unsigned int v4Count = 0,v6Count = 0;
 
 		while (count--) { // if ptr overflows Buffer will throw
 			// TODO: some flags are not yet implemented
@@ -913,20 +957,22 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 				case 4: {
 					InetAddress a(field(ptr,4),4,at<uint16_t>(ptr + 4));
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) {
-						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						if (v4Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
-							if ((!currentBest)||(currentBest->address() != a))
-								peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
+							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
+							peer->attemptToContactAt(RR,_localAddress,a,now);
+						} else {
+							TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str());
 						}
 					}
 				}	break;
 				case 6: {
 					InetAddress a(field(ptr,16),16,at<uint16_t>(ptr + 16));
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) ) {
-						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						if (v6Count++ < ZT_PUSH_DIRECT_PATHS_MAX_ENDPOINTS_PER_TYPE) {
-							if ((!currentBest)||(currentBest->address() != a))
-								peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
+						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
+							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
+							peer->attemptToContactAt(RR,_localAddress,a,now);
+						} else {
+							TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str());
 						}
 					}
 				}	break;
@@ -1042,7 +1088,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 				remainingHopsPtr += ZT_ADDRESS_LENGTH;
 				SharedPtr<Peer> nhp(RR->topology->getPeer(nextHop[h]));
 				if (nhp) {
-					RemotePath *const rp = nhp->getBestPath(now);
+					Path *const rp = nhp->getBestPath(now);
 					if (rp)
 						nextHopBestPathAddress[h] = rp->address();
 				}
@@ -1178,6 +1224,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 						outp.append((uint16_t)sizeof(result));
 						outp.append(result,sizeof(result));
 						outp.armor(peer->key(),true);
+						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					} else {
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
@@ -1185,6 +1232,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 						outp.append(pid);
 						outp.append((unsigned char)Packet::ERROR_INVALID_REQUEST);
 						outp.armor(peer->key(),true);
+						RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					}
 				}	break;
@@ -1290,6 +1338,7 @@ void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,cons
 	outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
 	outp.append(nwid);
 	outp.armor(peer->key(),true);
+	RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 	RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 }
 

node/InetAddress.hpp

@@ -42,13 +42,18 @@
 
 namespace ZeroTier {
 
+/**
+ * Maximum integer value of enum IpScope
+ */
+#define ZT_INETADDRESS_MAX_SCOPE 7
+
 /**
  * Extends sockaddr_storage with friendly C++ methods
  *
  * This is basically a "mixin" for sockaddr_storage. It adds methods and
  * operators, but does not modify the structure. This can be cast to/from
- * sockaddr_storage and used interchangeably. Don't change this as it's
- * used in a few places.
+ * sockaddr_storage and used interchangeably. DO NOT change this by e.g.
+ * adding non-static fields, since much code depends on this identity.
  */
 struct InetAddress : public sockaddr_storage
 {
@@ -66,7 +71,8 @@ struct InetAddress : public sockaddr_storage
 	 * IP address scope
 	 *
 	 * Note that these values are in ascending order of path preference and
-	 * MUST remain that way or Path must be changed to reflect.
+	 * MUST remain that way or Path must be changed to reflect. Also be sure
+	 * to change ZT_INETADDRESS_MAX_SCOPE if the max changes.
 	 */
 	enum IpScope
 	{
@@ -320,7 +326,7 @@ struct InetAddress : public sockaddr_storage
 	inline bool isV6() const throw() { return (ss_family == AF_INET6); }
 
 	/**
-	 * @return pointer to raw IP address bytes
+	 * @return pointer to raw address bytes or NULL if not available
 	 */
 	inline const void *rawIpData() const
 		throw()
@@ -333,27 +339,19 @@ struct InetAddress : public sockaddr_storage
 	}
 
 	/**
-	 * @return pointer to raw IP address bytes
-	 */
-	inline void *rawIpData()
-		throw()
-	{
-		switch(ss_family) {
-			case AF_INET: return (void *)&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr);
-			case AF_INET6: return (void *)(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr);
-			default: return 0;
-		}
-	}
-
-	/**
+	 * Performs an IP-only comparison or, if that is impossible, a memcmp()
+	 *
 	 * @param a InetAddress to compare again
 	 * @return True if only IP portions are equal (false for non-IP or null addresses)
 	 */
 	inline bool ipsEqual(const InetAddress &a) const
 	{
-		switch(ss_family) {
-			case AF_INET: return (reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_addr.s_addr);
-			case AF_INET6: return (memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0);
+		if (ss_family == a.ss_family) {
+			if (ss_family == AF_INET)
+				return (reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_addr.s_addr);
+			if (ss_family == AF_INET6)
+				return (memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0);
+			return (memcmp(this,&a,sizeof(InetAddress)) == 0);
 		}
 		return false;
 	}

node/Multicaster.cpp

@@ -37,6 +37,7 @@
 #include "Peer.hpp"
 #include "C25519.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -174,129 +175,130 @@ void Multicaster::send(
 	unsigned long idxbuf[8194];
 	unsigned long *indexes = idxbuf;
 
-	Mutex::Lock _l(_groups_m);
-	MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
-
-	if (!gs.members.empty()) {
-		// Allocate a memory buffer if group is monstrous
-		if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
-			indexes = new unsigned long[gs.members.size()];
-
-		// Generate a random permutation of member indexes
-		for(unsigned long i=0;i<gs.members.size();++i)
-			indexes[i] = i;
-		for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
-			unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
-			unsigned long tmp = indexes[j];
-			indexes[j] = indexes[i];
-			indexes[i] = tmp;
+	try {
+		Mutex::Lock _l(_groups_m);
+		MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
+
+		if (!gs.members.empty()) {
+			// Allocate a memory buffer if group is monstrous
+			if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
+				indexes = new unsigned long[gs.members.size()];
+
+			// Generate a random permutation of member indexes
+			for(unsigned long i=0;i<gs.members.size();++i)
+				indexes[i] = i;
+			for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
+				unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
+				unsigned long tmp = indexes[j];
+				indexes[j] = indexes[i];
+				indexes[i] = tmp;
+			}
 		}
-	}
 
-	if (gs.members.size() >= limit) {
-		// Skip queue if we already have enough members to complete the send operation
-		OutboundMulticast out;
-
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			1, // we'll still gather a little from peers to keep multicast list fresh
-			src,
-			mg,
-			etherType,
-			data,
-			len);
-
-		unsigned int count = 0;
-
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendOnly(RR,*ast);
-				if (++count >= limit)
-					break;
+		if (gs.members.size() >= limit) {
+			// Skip queue if we already have enough members to complete the send operation
+			OutboundMulticast out;
+
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				1, // we'll still gather a little from peers to keep multicast list fresh
+				src,
+				mg,
+				etherType,
+				data,
+				len);
+
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendOnly(RR,*ast); // optimization: don't use dedup log if it's a one-pass send
+					if (++count >= limit)
+						break;
+				}
 			}
-		}
 
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendOnly(RR,ma);
-				++count;
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendOnly(RR,ma); // optimization: don't use dedup log if it's a one-pass send
+					++count;
+				}
 			}
-		}
-	} else {
-		unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
-
-		if ((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY) {
-			gs.lastExplicitGather = now;
-			SharedPtr<Peer> sn(RR->topology->getBestRoot());
-			if (sn) {
-				TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
-
-				const CertificateOfMembership *com = (CertificateOfMembership *)0;
-				SharedPtr<NetworkConfig> nconf;
-				if (sn->needsOurNetworkMembershipCertificate(nwid,now,true)) {
-					SharedPtr<Network> nw(RR->node->network(nwid));
-					if (nw) {
-						nconf = nw->config2();
-						if (nconf)
-							com = &(nconf->com());
+		} else {
+			unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
+
+			if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) {
+				gs.lastExplicitGather = now;
+				SharedPtr<Peer> r(RR->topology->getBestRoot());
+				if (r) {
+					TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
+
+					const CertificateOfMembership *com = (CertificateOfMembership *)0;
+					{
+						SharedPtr<Network> nw(RR->node->network(nwid));
+						if (nw) {
+							SharedPtr<NetworkConfig> nconf(nw->config2());
+							if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true)))
+								com = &(nconf->com());
+						}
 					}
-				}
 
-				Packet outp(sn->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
-				outp.append(nwid);
-				outp.append((uint8_t)(com ? 0x01 : 0x00));
-				mg.mac().appendTo(outp);
-				outp.append((uint32_t)mg.adi());
-				outp.append((uint32_t)gatherLimit);
-				if (com)
-					com->serialize(outp);
-				outp.armor(sn->key(),true);
-				sn->send(RR,outp.data(),outp.size(),now);
+					Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
+					outp.append(nwid);
+					outp.append((uint8_t)(com ? 0x01 : 0x00));
+					mg.mac().appendTo(outp);
+					outp.append((uint32_t)mg.adi());
+					outp.append((uint32_t)gatherLimit);
+					if (com)
+						com->serialize(outp);
+					outp.armor(r->key(),true);
+					r->send(RR,outp.data(),outp.size(),now);
+				}
+				gatherLimit = 0;
 			}
-			gatherLimit = 0;
-		}
 
-		gs.txQueue.push_back(OutboundMulticast());
-		OutboundMulticast &out = gs.txQueue.back();
-
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			gatherLimit,
-			src,
-			mg,
-			etherType,
-			data,
-			len);
-
-		unsigned int count = 0;
-
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendAndLog(RR,*ast);
-				if (++count >= limit)
-					break;
+			gs.txQueue.push_back(OutboundMulticast());
+			OutboundMulticast &out = gs.txQueue.back();
+
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				gatherLimit,
+				src,
+				mg,
+				etherType,
+				data,
+				len);
+
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendAndLog(RR,*ast);
+					if (++count >= limit)
+						break;
+				}
 			}
-		}
 
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendAndLog(RR,ma);
-				++count;
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendAndLog(RR,ma);
+					++count;
+				}
 			}
 		}
-	}
+	} catch ( ... ) {} // this is a sanity check to catch any failures and make sure indexes[] still gets deleted
 
 	// Free allocated memory buffer if any
 	if (indexes != idxbuf)

node/Network.cpp

@@ -37,6 +37,7 @@
 #include "Packet.hpp"
 #include "Buffer.hpp"
 #include "NetworkController.hpp"
+#include "Node.hpp"
 
 #include "../version.h"
 

node/NetworkConfig.cpp

@@ -55,6 +55,9 @@ SharedPtr<NetworkConfig> NetworkConfig::createTestNetworkConfig(const Address &s
 	if ((ip & 0x000000ff) == 0x00000000) ip ^= 0x00000001; // or .0
 	nc->_staticIps.push_back(InetAddress(Utils::hton(ip),8));
 
+	// Assign an RFC4193-compliant IPv6 address -- will never collide
+	nc->_staticIps.push_back(InetAddress::makeIpv6rfc4193(ZT_TEST_NETWORK_ID,self.toInt()));
+
 	return nc;
 }
 

node/Node.cpp

@@ -263,7 +263,7 @@ public:
 			}
 
 			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
-		} else if (p->alive(_now)) {
+		} else if (p->activelyTransferringFrames(_now)) {
 			// Normal nodes get their preferred link kept alive if the node has generated frame traffic recently
 			p->doPingAndKeepalive(RR,_now,0);
 		}
@@ -305,24 +305,13 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
 				(*n)->requestConfiguration();
 
-			// Attempt to contact network preferred relays that we don't have direct links to
-			std::sort(networkRelays.begin(),networkRelays.end());
-			networkRelays.erase(std::unique(networkRelays.begin(),networkRelays.end()),networkRelays.end());
-			for(std::vector< std::pair<Address,InetAddress> >::const_iterator nr(networkRelays.begin());nr!=networkRelays.end();++nr) {
-				if (nr->second) {
-					SharedPtr<Peer> rp(RR->topology->getPeer(nr->first));
-					if ((rp)&&(!rp->hasActiveDirectPath(now)))
-						rp->attemptToContactAt(RR,InetAddress(),nr->second,now);
-				}
-			}
-
-			// Ping living or root server/relay peers
+			// Do pings and keepalives
 			_PingPeersThatNeedPing pfunc(RR,now,networkRelays);
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
 			// Update online status, post status change as event
-			bool oldOnline = _online;
-			_online = ((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT);
+			const bool oldOnline = _online;
+			_online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot()));
 			if (oldOnline != _online)
 				postEvent(_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE);
 		} catch ( ... ) {
@@ -448,10 +437,10 @@ ZT_PeerList *Node::peers() const
 		p->latency = pi->second->latency();
 		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
 
-		std::vector<RemotePath> paths(pi->second->paths());
-		RemotePath *bestPath = pi->second->getBestPath(_now);
+		std::vector<Path> paths(pi->second->paths());
+		Path *bestPath = pi->second->getBestPath(_now);
 		p->pathCount = 0;
-		for(std::vector<RemotePath>::iterator path(paths.begin());path!=paths.end();++path) {
+		for(std::vector<Path>::iterator path(paths.begin());path!=paths.end();++path) {
 			memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage));
 			p->paths[p->pathCount].lastSend = path->lastSend();
 			p->paths[p->pathCount].lastReceive = path->lastReceived();
@@ -499,11 +488,11 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
-		_directPaths.push_back(Path(*(reinterpret_cast<const InetAddress *>(addr)),metric,(Path::Trust)trust));
+		_directPaths.push_back(*(reinterpret_cast<const InetAddress *>(addr)));
 		std::sort(_directPaths.begin(),_directPaths.end());
 		_directPaths.erase(std::unique(_directPaths.begin(),_directPaths.end()),_directPaths.end());
 		return 1;
@@ -900,10 +889,10 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric, enum ZT_LocalInterfaceAddressTrust trust)
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr);
 	} catch ( ... ) {
 		return 0;
 	}

node/Node.hpp

@@ -105,7 +105,7 @@ public:
 	ZT_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
 	ZT_VirtualNetworkList *networks() const;
 	void freeQueryResult(void *qr);
-	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
+	int addLocalInterfaceAddress(const struct sockaddr_storage *addr);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
@@ -207,7 +207,7 @@ public:
 	/**
 	 * @return Potential direct paths to me a.k.a. local interface addresses
 	 */
-	inline std::vector<Path> directPaths() const
+	inline std::vector<InetAddress> directPaths() const
 	{
 		Mutex::Lock _l(_directPaths_m);
 		return _directPaths;
@@ -285,7 +285,7 @@ private:
 	std::vector< ZT_CircuitTest * > _circuitTests;
 	Mutex _circuitTests_m;
 
-	std::vector<Path> _directPaths;
+	std::vector<InetAddress> _directPaths;
 	Mutex _directPaths_m;
 
 	Mutex _backgroundTasksLock;

node/Packet.hpp

@@ -57,10 +57,11 @@
  *   + New crypto completely changes key agreement cipher
  * 4 - 0.6.0 ... 1.0.6
  *   + New identity format based on hashcash design
- * 5 - 1.0.6 ... CURRENT
+ * 5 - 1.1.0 ... CURRENT
  *   + Supports circuit test, proof of work, and echo
- *   + Supports in-band world (root definition) updates
- *   + Otherwise backward compatible with 4
+ *   + Supports in-band world (root server definition) updates
+ *   + Clustering! (Though this will work with protocol v4 clients.)
+ *   + Otherwise backward compatible with protocol v4
  */
 #define ZT_PROTO_VERSION 5
 
@@ -924,9 +925,6 @@ public:
 		 *   0x04 - Disable encryption (trust: privacy)
 		 *   0x08 - Disable encryption and authentication (trust: ultimate)
 		 *
-		 * Address types and addresses are of the same format as the destination
-		 * address type and address in HELLO.
-		 *
 		 * The receiver may, upon receiving a push, attempt to establish a
 		 * direct link to one or more of the indicated addresses. It is the
 		 * responsibility of the sender to limit which peers it pushes direct

node/Path.cpp

@@ -0,0 +1,45 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#include "Path.hpp"
+#include "AntiRecursion.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Node.hpp"
+
+namespace ZeroTier {
+
+bool Path::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+{
+	if (RR->node->putPacket(_localAddress,address(),data,len)) {
+		sent(now);
+		RR->antiRec->logOutgoingZT(data,len);
+		return true;
+	}
+	return false;
+}
+
+} // namespace ZeroTier

node/Path.hpp

@@ -28,12 +28,19 @@
 #ifndef ZT_PATH_HPP
 #define ZT_PATH_HPP
 
+#include <stdint.h>
+#include <string.h>
+
+#include <stdexcept>
+#include <algorithm>
+
 #include "Constants.hpp"
 #include "InetAddress.hpp"
-#include "Utils.hpp"
 
 namespace ZeroTier {
 
+class RuntimeEnvironment;
+
 /**
  * Base class for paths
  *
@@ -42,44 +49,92 @@ namespace ZeroTier {
 class Path
 {
 public:
+	Path() :
+		_lastSend(0),
+		_lastReceived(0),
+		_addr(),
+		_localAddress(),
+		_ipScope(InetAddress::IP_SCOPE_NONE)
+	{
+	}
+
+	Path(const InetAddress &localAddress,const InetAddress &addr) :
+		_lastSend(0),
+		_lastReceived(0),
+		_addr(addr),
+		_localAddress(localAddress),
+		_ipScope(addr.ipScope())
+	{
+	}
+
+	inline Path &operator=(const Path &p)
+		throw()
+	{
+		if (this != &p)
+			memcpy(this,&p,sizeof(Path));
+		return *this;
+	}
+
 	/**
-	 * Path trust category
-	 *
-	 * Note that this is NOT peer trust and has nothing to do with root server
-	 * designations or other trust metrics. This indicates how much we trust
-	 * this path to be secure and/or private. A trust level of normal means
-	 * encrypt and authenticate all traffic. Privacy trust means we can send
-	 * traffic in the clear. Ultimate trust means we don't even need
-	 * authentication. Generally a private path would be a hard-wired local
-	 * LAN, while an ultimate trust path would be a physically isolated private
-	 * server backplane.
+	 * Called when a packet is sent to this remote path
 	 *
-	 * Nearly all paths will be normal trust. The other levels are for high
-	 * performance local SDN use only.
+	 * This is called automatically by Path::send().
 	 *
-	 * These values MUST match ZT_LocalInterfaceAddressTrust in ZeroTierOne.h
+	 * @param t Time of send
 	 */
-	enum Trust // NOTE: max 255
+	inline void sent(uint64_t t)
+		throw()
 	{
-		TRUST_NORMAL = 0,
-		TRUST_PRIVACY = 10,
-		TRUST_ULTIMATE = 20
-	};
+		_lastSend = t;
+	}
 
-	Path() :
-		_addr(),
-		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_trust(TRUST_NORMAL)
+	/**
+	 * Called when a packet is received from this remote path
+	 *
+	 * @param t Time of receive
+	 */
+	inline void received(uint64_t t)
+		throw()
 	{
+		_lastReceived = t;
 	}
 
-	Path(const InetAddress &addr,int metric,Trust trust) :
-		_addr(addr),
-		_ipScope(addr.ipScope()),
-		_trust(trust)
+	/**
+	 * @param now Current time
+	 * @return True if this path appears active
+	 */
+	inline bool active(uint64_t now) const
+		throw()
 	{
+		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
 	}
 
+	/**
+	 * Send a packet via this path
+	 *
+	 * @param RR Runtime environment
+	 * @param data Packet data
+	 * @param len Packet length
+	 * @param now Current time
+	 * @return True if transport reported success
+	 */
+	bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now);
+
+	/**
+	 * @return Address of local side of this path or NULL if unspecified
+	 */
+	inline const InetAddress &localAddress() const throw() { return _localAddress; }
+
+	/**
+	 * @return Time of last send to this path
+	 */
+	inline uint64_t lastSend() const throw() { return _lastSend; }
+
+	/**
+	 * @return Time of last receive from this path
+	 */
+	inline uint64_t lastReceived() const throw() { return _lastReceived; }
+
 	/**
 	 * @return Physical address
 	 */
@@ -104,11 +159,6 @@ public:
 		return ( ((int)_ipScope * 2) + ((_addr.ss_family == AF_INET6) ? 1 : 0) );
 	}
 
-	/**
-	 * @return Path trust level
-	 */
-	inline Trust trust() const throw() { return _trust; }
-
 	/**
 	 * @return True if path is considered reliable (no NAT keepalives etc. are needed)
 	 */
@@ -157,10 +207,36 @@ public:
 		return false;
 	}
 
-protected:
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append((uint8_t)1); // version
+		b.append((uint64_t)_lastSend);
+		b.append((uint64_t)_lastReceived);
+		_addr.serialize(b);
+		_localAddress.serialize(b);
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		if (b[p++] != 1)
+			throw std::invalid_argument("invalid serialized Path");
+		_lastSend = b.template at<uint64_t>(p); p += 8;
+		_lastReceived = b.template at<uint64_t>(p); p += 8;
+		p += _addr.deserialize(b,p);
+		p += _localAddress.deserialize(b,p);
+		_ipScope = _addr.ipScope();
+		return (p - startAt);
+	}
+
+private:
+	uint64_t _lastSend;
+	uint64_t _lastReceived;
 	InetAddress _addr;
+	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
-	Trust _trust;
 };
 
 } // namespace ZeroTier

node/Peer.cpp

@@ -35,6 +35,7 @@
 #include "AntiRecursion.hpp"
 #include "SelfAwareness.hpp"
 #include "Cluster.hpp"
+#include "Packet.hpp"
 
 #include <algorithm>
 
@@ -54,7 +55,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastAnnouncedTo(0),
 	_lastPathConfirmationSent(0),
 	_lastDirectPathPushSent(0),
-	_lastDirectPathPushReceived(0),
+	_lastDirectPathPushReceive(0),
 	_lastPathSort(0),
 	_vProto(0),
 	_vMajor(0),
@@ -63,6 +64,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_id(peerIdentity),
 	_numPaths(0),
 	_latency(0),
+	_directPathPushCutoffCount(0),
 	_networkComs(4),
 	_lastPushedComs(4)
 {
@@ -80,85 +82,130 @@ void Peer::received(
 	uint64_t inRePacketId,
 	Packet::Verb inReVerb)
 {
+#ifdef ZT_ENABLE_CLUSTER
+	InetAddress redirectTo;
+	if ((RR->cluster)&&(hops == 0)) {
+		// Note: findBetterEndpoint() is first since we still want to check
+		// for a better endpoint even if we don't actually send a redirect.
+		if ( (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),remoteAddr,false)) && (verb != Packet::VERB_OK)&&(verb != Packet::VERB_ERROR)&&(verb != Packet::VERB_RENDEZVOUS)&&(verb != Packet::VERB_PUSH_DIRECT_PATHS) ) {
+			if (_vProto >= 5) {
+				// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
+				outp.append((uint16_t)1); // count == 1
+				outp.append((uint8_t)0); // no flags
+				outp.append((uint16_t)0); // no extensions
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append((uint8_t)6);
+					outp.append(redirectTo.rawIpData(),4);
+				} else {
+					outp.append((uint8_t)6);
+					outp.append((uint8_t)18);
+					outp.append(redirectTo.rawIpData(),16);
+				}
+				outp.append((uint16_t)redirectTo.port());
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+			} else {
+				// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
+				outp.append((uint8_t)0); // no flags
+				RR->identity.address().appendTo(outp);
+				outp.append((uint16_t)redirectTo.port());
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append(redirectTo.rawIpData(),4);
+				} else {
+					outp.append((uint8_t)16);
+					outp.append(redirectTo.rawIpData(),16);
+				}
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+			}
+		}
+	}
+#endif
+
 	const uint64_t now = RR->node->now();
 	bool needMulticastGroupAnnounce = false;
 	bool pathIsConfirmed = false;
 
-	{
+	{	// begin _lock
 		Mutex::Lock _l(_lock);
 
 		_lastReceive = now;
+		if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
+			_lastUnicastFrame = now;
+		else if (verb == Packet::VERB_MULTICAST_FRAME)
+			_lastMulticastFrame = now;
 
-		if (!hops) {
-			/* Learn new paths from direct (hops == 0) packets */
-			{
-				unsigned int np = _numPaths;
-				for(unsigned int p=0;p<np;++p) {
-					if ((_paths[p].address() == remoteAddr)&&(_paths[p].localAddress() == localAddr)) {
-						_paths[p].received(now);
-						pathIsConfirmed = true;
-						break;
-					}
+#ifdef ZT_ENABLE_CLUSTER
+		// If we think this peer belongs elsewhere, don't learn this path or
+		// do other connection init stuff.
+		if (redirectTo)
+			return;
+#endif
+
+		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
+			_lastAnnouncedTo = now;
+			needMulticastGroupAnnounce = true;
+		}
+
+		if (hops == 0) {
+			unsigned int np = _numPaths;
+			for(unsigned int p=0;p<np;++p) {
+				if ((_paths[p].address() == remoteAddr)&&(_paths[p].localAddress() == localAddr)) {
+					_paths[p].received(now);
+					pathIsConfirmed = true;
+					break;
 				}
+			}
 
-				if (!pathIsConfirmed) {
-					if ((verb == Packet::VERB_OK)&&((inReVerb == Packet::VERB_HELLO)||(inReVerb == Packet::VERB_ECHO))) {
-
-						// Learn paths if they've been confirmed via a HELLO or an ECHO
-						RemotePath *slot = (RemotePath *)0;
-						if (np < ZT_MAX_PEER_NETWORK_PATHS) {
-							slot = &(_paths[np++]);
-						} else {
-							uint64_t slotLRmin = 0xffffffffffffffffULL;
-							for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
-								if (_paths[p].lastReceived() <= slotLRmin) {
-									slotLRmin = _paths[p].lastReceived();
-									slot = &(_paths[p]);
-								}
-							}
-						}
-						if (slot) {
-							*slot = RemotePath(localAddr,remoteAddr);
-							slot->received(now);
-							_numPaths = np;
-							pathIsConfirmed = true;
-							_sortPaths(now);
-						}
+			if (!pathIsConfirmed) {
+				if ((verb == Packet::VERB_OK)||(RR->topology->amRoot())) {
 
+					Path *slot = (Path *)0;
+					if (np < ZT_MAX_PEER_NETWORK_PATHS) {
+						slot = &(_paths[np++]);
 					} else {
-
-						/* If this path is not known, send a HELLO. We don't learn
-						 * paths without confirming that a bidirectional link is in
-						 * fact present, but any packet that decodes and authenticates
-						 * correctly is considered valid. */
-						if ((now - _lastPathConfirmationSent) >= ZT_MIN_PATH_CONFIRMATION_INTERVAL) {
-							_lastPathConfirmationSent = now;
-							TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
-							attemptToContactAt(RR,localAddr,remoteAddr,now);
+						uint64_t slotLRmin = 0xffffffffffffffffULL;
+						for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
+							if (_paths[p].lastReceived() <= slotLRmin) {
+								slotLRmin = _paths[p].lastReceived();
+								slot = &(_paths[p]);
+							}
 						}
+					}
+					if (slot) {
+						*slot = Path(localAddr,remoteAddr);
+						slot->received(now);
+						_numPaths = np;
+						pathIsConfirmed = true;
+						_sortPaths(now);
+					}
+
+				} else {
 
+					/* If this path is not known, send a HELLO. We don't learn
+					 * paths without confirming that a bidirectional link is in
+					 * fact present, but any packet that decodes and authenticates
+					 * correctly is considered valid. */
+					if ((now - _lastPathConfirmationSent) >= ZT_MIN_PATH_CONFIRMATION_INTERVAL) {
+						_lastPathConfirmationSent = now;
+						TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
+						attemptToContactAt(RR,localAddr,remoteAddr,now);
 					}
+
 				}
 			}
 		}
-
-		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
-			_lastAnnouncedTo = now;
-			needMulticastGroupAnnounce = true;
-		}
-
-		if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
-			_lastUnicastFrame = now;
-		else if (verb == Packet::VERB_MULTICAST_FRAME)
-			_lastMulticastFrame = now;
-	}
+	}	// end _lock
 
 #ifdef ZT_ENABLE_CLUSTER
-	if ((pathIsConfirmed)&&(RR->cluster)) {
-		// Either shuttle this peer off somewhere else or report to other members that we have it
-		if (!RR->cluster->redirectPeer(_id.address(),remoteAddr,false))
-			RR->cluster->replicateHavePeer(_id);
-	}
+	if ((RR->cluster)&&(pathIsConfirmed))
+		RR->cluster->replicateHavePeer(_id,remoteAddr);
 #endif
 
 	if (needMulticastGroupAnnounce) {
@@ -190,7 +237,7 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &lo
 
 bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily)
 {
-	RemotePath *p = (RemotePath *)0;
+	Path *p = (Path *)0;
 
 	Mutex::Lock _l(_lock);
 	if (inetAddressFamily != 0) {
@@ -201,16 +248,16 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 
 	if (p) {
 		if ((now - p->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
-			TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
+			//TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
 			attemptToContactAt(RR,p->localAddress(),p->address(),now);
 			p->sent(now);
 		} else if (((now - p->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!p->reliable())) {
-			TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
+			//TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
 			_natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads
 			RR->node->putPacket(p->localAddress(),p->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
 			p->sent(now);
 		} else {
-			TRACE("no PING or NAT keepalive: addr==%s reliable==%d %llums/%llums send/receive inactivity",p->address().toString().c_str(),(int)p->reliable(),now - p->lastSend(),now - p->lastReceived());
+			//TRACE("no PING or NAT keepalive: addr==%s reliable==%d %llums/%llums send/receive inactivity",p->address().toString().c_str(),(int)p->reliable(),now - p->lastSend(),now - p->lastReceived());
 		}
 		return true;
 	}
@@ -218,7 +265,7 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 	return false;
 }
 
-void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
+void Peer::pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force)
 {
 #ifdef ZT_ENABLE_CLUSTER
 	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
@@ -231,23 +278,23 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	if (((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
 		_lastDirectPathPushSent = now;
 
-		std::vector<Path> dps(RR->node->directPaths());
+		std::vector<InetAddress> dps(RR->node->directPaths());
 		if (dps.empty())
 			return;
 
 #ifdef ZT_TRACE
 		{
 			std::string ps;
-			for(std::vector<Path>::const_iterator p(dps.begin());p!=dps.end();++p) {
+			for(std::vector<InetAddress>::const_iterator p(dps.begin());p!=dps.end();++p) {
 				if (ps.length() > 0)
 					ps.push_back(',');
-				ps.append(p->address().toString());
+				ps.append(p->toString());
 			}
 			TRACE("pushing %u direct paths to %s: %s",(unsigned int)dps.size(),_id.address().toString().c_str(),ps.c_str());
 		}
 #endif
 
-		std::vector<Path>::const_iterator p(dps.begin());
+		std::vector<InetAddress>::const_iterator p(dps.begin());
 		while (p != dps.end()) {
 			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
 			outp.addSize(2); // leave room for count
@@ -255,7 +302,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 			unsigned int count = 0;
 			while ((p != dps.end())&&((outp.size() + 24) < ZT_PROTO_MAX_PACKET_LENGTH)) {
 				uint8_t addressType = 4;
-				switch(p->address().ss_family) {
+				switch(p->ss_family) {
 					case AF_INET:
 						break;
 					case AF_INET6:
@@ -267,6 +314,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 				}
 
 				uint8_t flags = 0;
+				/* TODO: path trust is not implemented yet
 				switch(p->trust()) {
 					default:
 						break;
@@ -277,13 +325,14 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 						flags |= (0x04 | 0x08); // no encryption, no authentication (redundant but go ahead and set both)
 						break;
 				}
+				*/
 
 				outp.append(flags);
 				outp.append((uint16_t)0); // no extensions
 				outp.append(addressType);
 				outp.append((uint8_t)((addressType == 4) ? 6 : 18));
-				outp.append(p->address().rawIpData(),((addressType == 4) ? 4 : 16));
-				outp.append((uint16_t)p->address().port());
+				outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16));
+				outp.append((uint16_t)p->port());
 
 				++count;
 				++p;
@@ -456,7 +505,7 @@ struct _SortPathsByQuality
 {
 	uint64_t _now;
 	_SortPathsByQuality(const uint64_t now) : _now(now) {}
-	inline bool operator()(const RemotePath &a,const RemotePath &b) const
+	inline bool operator()(const Path &a,const Path &b) const
 	{
 		const uint64_t qa = (
 			((uint64_t)a.active(_now) << 63) |
@@ -476,7 +525,7 @@ void Peer::_sortPaths(const uint64_t now)
 	std::sort(&(_paths[0]),&(_paths[_numPaths]),_SortPathsByQuality(now));
 }
 
-RemotePath *Peer::_getBestPath(const uint64_t now)
+Path *Peer::_getBestPath(const uint64_t now)
 {
 	// assumes _lock is locked
 	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
@@ -488,10 +537,10 @@ RemotePath *Peer::_getBestPath(const uint64_t now)
 		if (_paths[0].active(now))
 			return &(_paths[0]);
 	}
-	return (RemotePath *)0;
+	return (Path *)0;
 }
 
-RemotePath *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
+Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
 {
 	// assumes _lock is locked
 	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
@@ -503,7 +552,7 @@ RemotePath *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
 		}
 		_sortPaths(now);
 	}
-	return (RemotePath *)0;
+	return (Path *)0;
 }
 
 } // namespace ZeroTier

node/Peer.hpp

@@ -41,7 +41,7 @@
 
 #include "RuntimeEnvironment.hpp"
 #include "CertificateOfMembership.hpp"
-#include "RemotePath.hpp"
+#include "Path.hpp"
 #include "Address.hpp"
 #include "Utils.hpp"
 #include "Identity.hpp"
@@ -135,7 +135,7 @@ public:
 	 * @param now Current time
 	 * @return Best path or NULL if there are no active direct paths
 	 */
-	inline RemotePath *getBestPath(uint64_t now)
+	inline Path *getBestPath(uint64_t now)
 	{
 		Mutex::Lock _l(_lock);
 		return _getBestPath(now);
@@ -150,14 +150,14 @@ public:
 	 * @param now Current time
 	 * @return Path used on success or NULL on failure
 	 */
-	inline RemotePath *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+	inline Path *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 	{
-		RemotePath *bestPath = getBestPath(now);
+		Path *bestPath = getBestPath(now);
 		if (bestPath) {
 			if (bestPath->send(RR,data,len,now))
 				return bestPath;
 		}
-		return (RemotePath *)0;
+		return (Path *)0;
 	}
 
 	/**
@@ -191,46 +191,20 @@ public:
 	 * @param now Current time
 	 * @param force If true, push regardless of rate limit
 	 */
-	void pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force);
+	void pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force);
 
 	/**
 	 * @return All known direct paths to this peer
 	 */
-	inline std::vector<RemotePath> paths() const
+	inline std::vector<Path> paths() const
 	{
-		std::vector<RemotePath> pp;
+		std::vector<Path> pp;
 		Mutex::Lock _l(_lock);
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
 			pp.push_back(_paths[p]);
 		return pp;
 	}
 
-	/**
-	 * @return Time of last direct packet receive for any path
-	 */
-	inline uint64_t lastDirectReceive() const
-		throw()
-	{
-		Mutex::Lock _l(_lock);
-		uint64_t x = 0;
-		for(unsigned int p=0,np=_numPaths;p<np;++p)
-			x = std::max(x,_paths[p].lastReceived());
-		return x;
-	}
-
-	/**
-	 * @return Time of last direct packet send for any path
-	 */
-	inline uint64_t lastDirectSend() const
-		throw()
-	{
-		Mutex::Lock _l(_lock);
-		uint64_t x = 0;
-		for(unsigned int p=0,np=_numPaths;p<np;++p)
-			x = std::max(x,_paths[p].lastSend());
-		return x;
-	}
-
 	/**
 	 * @return Time of last receive of anything, whether direct or relayed
 	 */
@@ -257,20 +231,38 @@ public:
 	inline uint64_t lastAnnouncedTo() const throw() { return _lastAnnouncedTo; }
 
 	/**
-	 * @return True if peer has received an actual data frame within ZT_PEER_ACTIVITY_TIMEOUT milliseconds
+	 * @return True if this peer is actively sending real network frames
 	 */
-	inline uint64_t alive(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
+	inline uint64_t activelyTransferringFrames(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
 
 	/**
-	 * @return Current latency or 0 if unknown (max: 65535)
+	 * @return Latency in milliseconds or 0 if unknown
 	 */
-	inline unsigned int latency() const
-		throw()
+	inline unsigned int latency() const { return _latency; }
+
+	/**
+	 * This computes a quality score for relays and root servers
+	 *
+	 * If we haven't heard anything from these in ZT_PEER_ACTIVITY_TIMEOUT, they
+	 * receive the worst possible quality (max unsigned int). Otherwise the
+	 * quality is a product of latency and the number of potential missed
+	 * pings. This causes roots and relays to switch over a bit faster if they
+	 * fail.
+	 *
+	 * @return Relay quality score computed from latency and other factors, lower is better
+	 */
+	inline unsigned int relayQuality(const uint64_t now) const
 	{
+		const uint64_t tsr = now - _lastReceive;
+		if (tsr >= ZT_PEER_ACTIVITY_TIMEOUT)
+			return (~(unsigned int)0);
 		unsigned int l = _latency;
-		return std::min(l,(unsigned int)65535);
+		if (!l)
+			l = 0xffff;
+		return (l * (((unsigned int)tsr / (ZT_PEER_DIRECT_PING_DELAY + 1000)) + 1));
 	}
 
+
 	/**
 	 * Update latency with a new direct measurment
 	 *
@@ -285,11 +277,6 @@ public:
 		else _latency = std::min(l,(unsigned int)65535);
 	}
 
-	/**
-	 * @return True if this peer has at least one direct IP address path
-	 */
-	inline bool hasDirectPath() const throw() { return (_numPaths != 0); }
-
 	/**
 	 * @param now Current time
 	 * @return True if this peer has at least one active direct path
@@ -408,19 +395,49 @@ public:
 	bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime);
 
 	/**
-	 * @return Time last direct path push was received
+	 * Perform periodic cleaning operations
 	 */
-	inline uint64_t lastDirectPathPushReceived() const throw() { return _lastDirectPathPushReceived; }
+	void clean(const RuntimeEnvironment *RR,uint64_t now);
 
 	/**
-	 * @param t New time of last direct path push received
+	 * Remove all paths with this remote address
+	 *
+	 * @param addr Remote address to remove
 	 */
-	inline void setLastDirectPathPushReceived(uint64_t t) throw() { _lastDirectPathPushReceived = t; }
+	inline void removePathByAddress(const InetAddress &addr)
+	{
+		Mutex::Lock _l(_lock);
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if (_paths[x].address() != addr)
+				_paths[y++] = _paths[x];
+			++x;
+		}
+		_numPaths = y;
+	}
 
 	/**
-	 * Perform periodic cleaning operations
+	 * Update direct path push stats and return true if we should respond
+	 *
+	 * This is a circuit breaker to make VERB_PUSH_DIRECT_PATHS not particularly
+	 * useful as a DDOS amplification attack vector. Otherwise a malicious peer
+	 * could send loads of these and cause others to bombard arbitrary IPs with
+	 * traffic.
+	 *
+	 * @param now Current time
+	 * @return True if we should respond
 	 */
-	void clean(const RuntimeEnvironment *RR,uint64_t now);
+	inline bool shouldRespondToDirectPathPush(const uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME)
+			++_directPathPushCutoffCount;
+		else _directPathPushCutoffCount = 0;
+		_lastDirectPathPushReceive = now;
+		return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT);
+	}
 
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
@@ -461,13 +478,14 @@ public:
 		b.append((uint64_t)_lastAnnouncedTo);
 		b.append((uint64_t)_lastPathConfirmationSent);
 		b.append((uint64_t)_lastDirectPathPushSent);
-		b.append((uint64_t)_lastDirectPathPushReceived);
+		b.append((uint64_t)_lastDirectPathPushReceive);
 		b.append((uint64_t)_lastPathSort);
 		b.append((uint16_t)_vProto);
 		b.append((uint16_t)_vMajor);
 		b.append((uint16_t)_vMinor);
 		b.append((uint16_t)_vRevision);
 		b.append((uint32_t)_latency);
+		b.append((uint16_t)_directPathPushCutoffCount);
 
 		b.append((uint16_t)_numPaths);
 		for(unsigned int i=0;i<_numPaths;++i)
@@ -531,13 +549,14 @@ public:
 		np->_lastAnnouncedTo = b.template at<uint64_t>(p); p += 8;
 		np->_lastPathConfirmationSent = b.template at<uint64_t>(p); p += 8;
 		np->_lastDirectPathPushSent = b.template at<uint64_t>(p); p += 8;
-		np->_lastDirectPathPushReceived = b.template at<uint64_t>(p); p += 8;
+		np->_lastDirectPathPushReceive = b.template at<uint64_t>(p); p += 8;
 		np->_lastPathSort = b.template at<uint64_t>(p); p += 8;
 		np->_vProto = b.template at<uint16_t>(p); p += 2;
 		np->_vMajor = b.template at<uint16_t>(p); p += 2;
 		np->_vMinor = b.template at<uint16_t>(p); p += 2;
 		np->_vRevision = b.template at<uint16_t>(p); p += 2;
 		np->_latency = b.template at<uint32_t>(p); p += 4;
+		np->_directPathPushCutoffCount = b.template at<uint16_t>(p); p += 2;
 
 		const unsigned int numPaths = b.template at<uint16_t>(p); p += 2;
 		for(unsigned int i=0;i<numPaths;++i) {
@@ -545,7 +564,7 @@ public:
 				p += np->_paths[np->_numPaths++].deserialize(b,p);
 			} else {
 				// Skip any paths beyond max, but still read stream
-				RemotePath foo;
+				Path foo;
 				p += foo.deserialize(b,p);
 			}
 		}
@@ -569,8 +588,8 @@ public:
 
 private:
 	void _sortPaths(const uint64_t now);
-	RemotePath *_getBestPath(const uint64_t now);
-	RemotePath *_getBestPath(const uint64_t now,int inetAddressFamily);
+	Path *_getBestPath(const uint64_t now);
+	Path *_getBestPath(const uint64_t now,int inetAddressFamily);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized
 
@@ -581,16 +600,17 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastPathConfirmationSent;
 	uint64_t _lastDirectPathPushSent;
-	uint64_t _lastDirectPathPushReceived;
+	uint64_t _lastDirectPathPushReceive;
 	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 	Identity _id;
-	RemotePath _paths[ZT_MAX_PEER_NETWORK_PATHS];
+	Path _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	unsigned int _numPaths;
 	unsigned int _latency;
+	unsigned int _directPathPushCutoffCount;
 
 	struct _NetworkCom
 	{

node/RemotePath.hpp

@@ -1,161 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef ZT_REMOTEPATH_HPP
-#define ZT_REMOTEPATH_HPP
-
-#include <stdint.h>
-#include <string.h>
-
-#include <stdexcept>
-#include <algorithm>
-
-#include "Path.hpp"
-#include "Node.hpp"
-#include "AntiRecursion.hpp"
-#include "RuntimeEnvironment.hpp"
-
-namespace ZeroTier {
-
-/**
- * Path to a remote peer
- *
- * This extends Path to include status information about path activity.
- */
-class RemotePath : public Path
-{
-public:
-	RemotePath() :
-		Path(),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(),
-		_flags(0) {}
-
-	RemotePath(const InetAddress &localAddress,const InetAddress &addr) :
-		Path(addr,0,TRUST_NORMAL),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(localAddress),
-		_flags(0) {}
-
-	inline const InetAddress &localAddress() const throw() { return _localAddress; }
-
-	inline uint64_t lastSend() const throw() { return _lastSend; }
-	inline uint64_t lastReceived() const throw() { return _lastReceived; }
-
-	/**
-	 * Called when a packet is sent to this remote path
-	 *
-	 * This is called automatically by RemotePath::send().
-	 *
-	 * @param t Time of send
-	 */
-	inline void sent(uint64_t t)
-		throw()
-	{
-		_lastSend = t;
-	}
-
-	/**
-	 * Called when a packet is received from this remote path
-	 *
-	 * @param t Time of receive
-	 */
-	inline void received(uint64_t t)
-		throw()
-	{
-		_lastReceived = t;
-	}
-
-	/**
-	 * @param now Current time
-	 * @return True if this path appears active
-	 */
-	inline bool active(uint64_t now) const
-		throw()
-	{
-		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
-	}
-
-	/**
-	 * Send a packet via this path
-	 *
-	 * @param RR Runtime environment
-	 * @param data Packet data
-	 * @param len Packet length
-	 * @param now Current time
-	 * @return True if transport reported success
-	 */
-	inline bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
-	{
-		if (RR->node->putPacket(_localAddress,address(),data,len)) {
-			sent(now);
-			RR->antiRec->logOutgoingZT(data,len);
-			return true;
-		}
-		return false;
-	}
-
-	template<unsigned int C>
-	inline void serialize(Buffer<C> &b) const
-	{
-		b.append((uint8_t)1); // version
-		_addr.serialize(b);
-		b.append((uint8_t)_trust);
-		b.append((uint64_t)_lastSend);
-		b.append((uint64_t)_lastReceived);
-		_localAddress.serialize(b);
-		b.append((uint16_t)_flags);
-	}
-
-	template<unsigned int C>
-	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
-	{
-		unsigned int p = startAt;
-		if (b[p++] != 1)
-			throw std::invalid_argument("invalid serialized RemotePath");
-		p += _addr.deserialize(b,p);
-		_ipScope = _addr.ipScope();
-		_trust = (Path::Trust)b[p++];
-		_lastSend = b.template at<uint64_t>(p); p += 8;
-		_lastReceived = b.template at<uint64_t>(p); p += 8;
-		p += _localAddress.deserialize(b,p);
-		_flags = b.template at<uint16_t>(p); p += 2;
-		return (p - startAt);
-	}
-
-protected:
-	uint64_t _lastSend;
-	uint64_t _lastReceived;
-	InetAddress _localAddress;
-	uint16_t _flags;
-};
-
-} // namespace ZeroTier
-
-#endif

node/SelfAwareness.cpp

@@ -36,6 +36,7 @@
 #include "Topology.hpp"
 #include "Packet.hpp"
 #include "Peer.hpp"
+#include "Switch.hpp"
 
 // Entry timeout -- make it fairly long since this is just to prevent stale buildup
 #define ZT_SELFAWARENESS_ENTRY_TIMEOUT 3600000
@@ -65,7 +66,8 @@ private:
 };
 
 SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) :
-	RR(renv)
+	RR(renv),
+	_phy(32)
 {
 }
 
@@ -77,35 +79,35 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 {
 	const InetAddress::IpScope scope = myPhysicalAddress.ipScope();
 
+	// This would be weird, e.g. a public IP talking to 10.0.0.1, so just ignore it.
+	// If your network is this weird it's probably not reliable information.
+	if (scope != reporterPhysicalAddress.ipScope())
+		return;
+
+	// Some scopes we ignore, and global scope IPs are only used for this
+	// mechanism if they come from someone we trust (e.g. a root).
 	switch(scope) {
 		case InetAddress::IP_SCOPE_NONE:
 		case InetAddress::IP_SCOPE_LOOPBACK:
 		case InetAddress::IP_SCOPE_MULTICAST:
 			return;
 		case InetAddress::IP_SCOPE_GLOBAL:
-			if ((!trusted)||(scope != reporterPhysicalAddress.ipScope()))
+			if (!trusted)
 				return;
 			break;
 		default:
-			if (scope != reporterPhysicalAddress.ipScope())
-				return;
 			break;
 	}
 
 	Mutex::Lock _l(_phy_m);
-
 	PhySurfaceEntry &entry = _phy[PhySurfaceKey(reporter,reporterPhysicalAddress,scope)];
 
-	if ((now - entry.ts) >= ZT_SELFAWARENESS_ENTRY_TIMEOUT) {
+	if ( ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) {
 		entry.mySurface = myPhysicalAddress;
 		entry.ts = now;
-		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced <null>)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str());
-	} else if (entry.mySurface != myPhysicalAddress) {
-		entry.mySurface = myPhysicalAddress;
-		entry.ts = now;
-		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced %s, resetting all in scope)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str());
+		TRACE("physical address %s for scope %u as seen from %s(%s) differs from %s, resetting paths in scope",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str());
 
-		// Erase all entries in this scope that were not reported by this remote address to prevent 'thrashing'
+		// Erase all entries in this scope that were not reported from this remote address to prevent 'thrashing'
 		// due to multiple reports of endpoint change.
 		// Don't use 'entry' after this since hash table gets modified.
 		{
@@ -118,26 +120,21 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 			}
 		}
 
+		// Reset all paths within this scope
 		_ResetWithinScope rset(RR,now,(InetAddress::IpScope)scope);
 		RR->topology->eachPeer<_ResetWithinScope &>(rset);
 
-		// For all peers for whom we forgot an address, send a packet indirectly if
-		// they are still considered alive so that we will re-establish direct links.
-		SharedPtr<Peer> sn(RR->topology->getBestRoot());
-		if (sn) {
-			RemotePath *snp = sn->getBestPath(now);
-			if (snp) {
-				for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
-					if ((*p)->alive(now)) {
-						TRACE("sending indirect NOP to %s via %s(%s) to re-establish link",(*p)->address().toString().c_str(),sn->address().toString().c_str(),snp->address().toString().c_str());
-						Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
-						outp.armor((*p)->key(),true);
-						snp->send(RR,outp.data(),outp.size(),now);
-					}
-				}
+		// Send a NOP to all peers for whom we forgot a path. This will cause direct
+		// links to be re-established if possible, possibly using a root server or some
+		// other relay.
+		for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
+			if ((*p)->activelyTransferringFrames(now)) {
+				Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
+				RR->sw->send(outp,true,0);
 			}
 		}
 	} else {
+		entry.mySurface = myPhysicalAddress;
 		entry.ts = now;
 	}
 }

node/Switch.cpp

@@ -295,17 +295,18 @@ void Switch::send(const Packet &packet,bool encrypt,uint64_t nwid)
 		return;
 	}
 
+	//TRACE(">> %s to %s (%u bytes, encrypt==%d, nwid==%.16llx)",Packet::verbString(packet.verb()),packet.destination().toString().c_str(),packet.size(),(int)encrypt,nwid);
+
 	if (!_trySend(packet,encrypt,nwid)) {
 		Mutex::Lock _l(_txQueue_m);
 		_txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt,nwid));
 	}
 }
 
-bool Switch::unite(const Address &p1,const Address &p2,bool force)
+bool Switch::unite(const Address &p1,const Address &p2)
 {
 	if ((p1 == RR->identity.address())||(p2 == RR->identity.address()))
 		return false;
-
 	SharedPtr<Peer> p1p = RR->topology->getPeer(p1);
 	if (!p1p)
 		return false;
@@ -315,14 +316,6 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 
 	const uint64_t now = RR->node->now();
 
-	{
-		Mutex::Lock _l(_lastUniteAttempt_m);
-		uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
-		if (((now - luts) < ZT_MIN_UNITE_INTERVAL)&&(!force))
-			return false;
-		luts = now;
-	}
-
 	std::pair<InetAddress,InetAddress> cg(Peer::findCommonGround(*p1p,*p2p,now));
 	if ((!(cg.first))||(cg.first.ipScope() != cg.second.ipScope()))
 		return false;
@@ -449,8 +442,8 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		Mutex::Lock _l(_contactQueue_m);
 		for(std::list<ContactQueueEntry>::iterator qi(_contactQueue.begin());qi!=_contactQueue.end();) {
 			if (now >= qi->fireAtTime) {
-				if ((!qi->peer->alive(now))||(qi->peer->hasActiveDirectPath(now))) {
-					// Cancel attempt if we've already connected or peer is no longer "alive"
+				if (qi->peer->hasActiveDirectPath(now)) {
+					// Cancel if connection has succeeded
 					_contactQueue.erase(qi++);
 					continue;
 				} else {
@@ -546,7 +539,7 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		_LastUniteKey *k = (_LastUniteKey *)0;
 		uint64_t *v = (uint64_t *)0;
 		while (i.next(k,v)) {
-			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 16))
+			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 8))
 				_lastUniteAttempt.erase(*k);
 		}
 	}
@@ -569,7 +562,7 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
 			if ((!relayTo)||(!relayTo->send(RR,fragment.data(),fragment.size(),RR->node->now()))) {
 #ifdef ZT_ENABLE_CLUSTER
-				if ((RR->cluster)&&(RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size())))
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false)))
 					return; // sent by way of another member of this cluster
 #endif
 
@@ -632,11 +625,17 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 
 void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
-	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,RR->node->now()));
+	const uint64_t now = RR->node->now();
+	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,now));
 
 	Address source(packet->source());
 	Address destination(packet->destination());
 
+	// Catch this and toss it -- it would never work, but it could happen if we somehow
+	// mistakenly guessed an address we're bound to as a destination for another peer.
+	if (source == RR->identity.address())
+		return;
+
 	//TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size());
 
 	if (destination != RR->identity.address()) {
@@ -645,17 +644,18 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 			packet->incrementHops();
 
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),RR->node->now())))) {
-				unite(source,destination,false);
+			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),now)))) {
+				if (_shouldTryUnite(now,source,destination))
+					unite(source,destination);
 			} else {
 #ifdef ZT_ENABLE_CLUSTER
-				if ((RR->cluster)&&(RR->cluster->sendViaCluster(source,destination,packet->data(),packet->size())))
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(source,destination,packet->data(),packet->size(),_shouldTryUnite(now,source,destination))))
 					return; // sent by way of another member of this cluster
 #endif
 
 				relayTo = RR->topology->getBestRoot(&source,1,true);
 				if (relayTo)
-					relayTo->send(RR,packet->data(),packet->size(),RR->node->now());
+					relayTo->send(RR,packet->data(),packet->size(),now);
 			}
 		} else {
 			TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet->source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str());
@@ -670,7 +670,7 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 		if (!dq.creationTime) {
 			// If we have no other fragments yet, create an entry and save the head
 
-			dq.creationTime = RR->node->now();
+			dq.creationTime = now;
 			dq.frag0 = packet;
 			dq.totalFragments = 0; // 0 == unknown, waiting for Packet::Fragment
 			dq.haveFragments = 1; // head is first bit (left to right)
@@ -736,17 +736,20 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // sanity check: unconfigured network? why are we trying to talk to it?
 		}
 
-		RemotePath *viaPath = peer->getBestPath(now);
+		Path *viaPath = peer->getBestPath(now);
 		SharedPtr<Peer> relay;
 		if (!viaPath) {
 			// See if this network has a preferred relay (if packet has an associated network)
 			if (nconf) {
-				unsigned int latency = ~((unsigned int)0);
+				unsigned int bestq = ~((unsigned int)0);
 				for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(nconf->relays().begin());r!=nconf->relays().end();++r) {
 					if (r->first != peer->address()) {
 						SharedPtr<Peer> rp(RR->topology->getPeer(r->first));
-						if ((rp)&&(rp->hasActiveDirectPath(now))&&(rp->latency() <= latency))
+						const unsigned int q = rp->relayQuality(now);
+						if ((rp)&&(q < bestq)) { // SUBTILE: < == don't use these if they are nil quality (unsigned int max), instead use a root
+							bestq = q;
 							rp.swap(relay);
+						}
 					}
 				}
 			}
@@ -798,4 +801,14 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 	return false;
 }
 
+bool Switch::_shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2)
+{
+	Mutex::Lock _l(_lastUniteAttempt_m);
+	uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
+	if ((now - luts) < ZT_MIN_UNITE_INTERVAL)
+		return false;
+	luts = now;
+	return true;
+}
+
 } // namespace ZeroTier

node/Switch.hpp

@@ -127,15 +127,10 @@ public:
 	 * This only works if both peers are known, with known working direct
 	 * links to this peer. The best link for each peer is sent to the other.
 	 *
-	 * A rate limiter is in effect via the _lastUniteAttempt map. If force
-	 * is true, a unite attempt is made even if one has been made less than
-	 * ZT_MIN_UNITE_INTERVAL milliseconds ago.
-	 *
 	 * @param p1 One of two peers (order doesn't matter)
 	 * @param p2 Second of pair
-	 * @param force If true, send now regardless of interval
 	 */
-	bool unite(const Address &p1,const Address &p2,bool force);
+	bool unite(const Address &p1,const Address &p2);
 
 	/**
 	 * Attempt NAT traversal to peer at a given physical address
@@ -185,6 +180,7 @@ private:
 	void _handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len);
 	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(const Packet &packet,bool encrypt,uint64_t nwid);
+	bool _shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2);
 
 	const RuntimeEnvironment *const RR;
 	uint64_t _lastBeaconResponse;

node/Topology.hpp

@@ -78,6 +78,26 @@ public:
 	 */
 	SharedPtr<Peer> getPeer(const Address &zta);
 
+	/**
+	 * Get a peer only if it is presently in memory (no disk cache)
+	 *
+	 * This also does not update the lastUsed() time for peers, which means
+	 * that it won't prevent them from falling out of RAM. This is currently
+	 * used in the Cluster code to update peer info without forcing all peers
+	 * across the entire cluster to remain in memory cache.
+	 *
+	 * @param zta ZeroTier address
+	 * @param now Current time
+	 */
+	inline SharedPtr<Peer> getPeerNoCache(const Address &zta,const uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		const SharedPtr<Peer> *const ap = _peers.get(zta);
+		if (ap)
+			return *ap;
+		return SharedPtr<Peer>();
+	}
+
 	/**
 	 * Get the identity of a peer
 	 *
@@ -96,24 +116,12 @@ public:
 	 */
 	void saveIdentity(const Identity &id);
 
-	/**
-	 * @return Vector of peers that are root servers
-	 */
-	inline std::vector< SharedPtr<Peer> > rootPeers() const
-	{
-		Mutex::Lock _l(_lock);
-		return _rootPeers;
-	}
-
 	/**
 	 * Get the current favorite root server
 	 *
 	 * @return Root server with lowest latency or NULL if none
 	 */
-	inline SharedPtr<Peer> getBestRoot()
-	{
-		return getBestRoot((const Address *)0,0,false);
-	}
+	inline SharedPtr<Peer> getBestRoot() { return getBestRoot((const Address *)0,0,false); }
 
 	/**
 	 * Get the best root server, avoiding root servers listed in an array
@@ -193,9 +201,9 @@ public:
 	void clean(uint64_t now);
 
 	/**
-	 * @return Number of 'alive' peers
+	 * @return Number of peers with active direct paths
 	 */
-	unsigned long countAlive() const;
+	unsigned long countActive() const;
 
 	/**
 	 * Apply a function or function object to all peers
@@ -217,8 +225,15 @@ public:
 		Hashtable< Address,SharedPtr<Peer> >::Iterator i(_peers);
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
-		while (i.next(a,p))
-			f(*this,*p);
+		while (i.next(a,p)) {
+#ifdef ZT_TRACE
+			if (!(*p)) {
+				fprintf(stderr,"FATAL BUG: eachPeer() caught NULL peer for %s -- peer pointers in Topology should NEVER be NULL"ZT_EOL_S,a->toString().c_str());
+				abort();
+			}
+#endif
+			f(*this,*((const SharedPtr<Peer> *)p));
+		}
 	}
 
 	/**

objects.mk

@@ -15,6 +15,7 @@ OBJS=\
 	node/Node.o \
 	node/OutboundMulticast.o \
 	node/Packet.o \
+	node/Path.o \
 	node/Peer.o \
 	node/Poly1305.o \
 	node/Salsa20.o \

osdep/Http.cpp

@@ -100,6 +100,14 @@ struct HttpPhyHandler
 			phy->setNotifyWritable(sock,false);
 	}
 
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
+#ifdef __UNIX_LIKE__
+	inline void phyOnUnixAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN) {}
+	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
+	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
+	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
+#endif // __UNIX_LIKE__
+
 	http_parser parser;
 	std::string currentHeaderField;
 	std::string currentHeaderValue;

osdep/Phy.hpp

@@ -78,11 +78,6 @@
 #define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS
 #define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage
 
-#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-#define ZT_PHY_HAVE_EVENTFD 1
-#include <sys/eventfd.h>
-#endif
-
 #endif // Windows or not
 
 namespace ZeroTier {
@@ -109,6 +104,7 @@ typedef void PhySocket;
  * phyOnTcpClose(PhySocket *sock,void **uptr)
  * phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len)
  * phyOnTcpWritable(PhySocket *sock,void **uptr)
+ * phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable)
  *
  * On Linux/OSX/Unix only (not required/used on Windows or elsewhere):
  *
@@ -116,9 +112,6 @@ typedef void PhySocket;
  * phyOnUnixClose(PhySocket *sock,void **uptr)
  * phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len)
  * phyOnUnixWritable(PhySocket *sock,void **uptr)
- * phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr)
- * phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len)
- * phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr)
  *
  * These templates typically refer to function objects. Templates are used to
  * avoid the call overhead of indirection, which is surprisingly high for high
@@ -154,11 +147,10 @@ private:
 		ZT_PHY_SOCKET_TCP_OUT_CONNECTED = 0x02,
 		ZT_PHY_SOCKET_TCP_IN = 0x03,
 		ZT_PHY_SOCKET_TCP_LISTEN = 0x04,
-		ZT_PHY_SOCKET_RAW = 0x05,
-		ZT_PHY_SOCKET_UDP = 0x06,
+		ZT_PHY_SOCKET_UDP = 0x05,
+		ZT_PHY_SOCKET_FD = 0x06,
 		ZT_PHY_SOCKET_UNIX_IN = 0x07,
-		ZT_PHY_SOCKET_UNIX_LISTEN = 0x08,
-		ZT_PHY_SOCKET_PAIR_ENDPOINT = 0x09
+		ZT_PHY_SOCKET_UNIX_LISTEN = 0x08
 	};
 
 	struct PhySocketImpl
@@ -277,57 +269,47 @@ public:
 	 */
 	inline unsigned long maxCount() const throw() { return ZT_PHY_MAX_SOCKETS; }
 
-#ifdef __UNIX_LIKE__
 	/**
-	 * Create a two-way socket pair
+	 * Wrap a raw file descriptor in a PhySocket structure
 	 *
-	 * This uses socketpair() to create a local domain pair. The returned
-	 * PhySocket holds the local side of the socket pair, while the
-	 * supplied fd variable is set to the descriptor for the remote side.
+	 * This can be used to select/poll on a raw file descriptor as part of this
+	 * class's I/O loop. By default the fd is set for read notification but
+	 * this can be controlled with setNotifyReadable(). When any detected
+	 * condition is present, the phyOnFileDescriptorActivity() callback is
+	 * called with one or both of its arguments 'true'.
 	 *
-	 * The local side is set to O_NONBLOCK to work with our poll loop, but
-	 * the remote descriptor is left untouched. It's up to the caller to
-	 * set any required fcntl(), ioctl(), or setsockopt() settings there.
-	 * It's also up to the caller to close the remote descriptor when
-	 * done, if necessary.
+	 * The Phy<>::close() method *must* be called when you're done with this
+	 * file descriptor to remove it from the select/poll set, but unlike other
+	 * types of sockets Phy<> does not actually close the underlying fd or
+	 * otherwise manage its life cycle. There is also no close notification
+	 * callback for this fd, since Phy<> doesn't actually perform reading or
+	 * writing or detect error conditions. This is only useful for adding a
+	 * file descriptor to Phy<> to select/poll on it.
 	 *
-	 * @param remoteSocketDescriptor Result parameter set to remote end of socket pair's socket FD
-	 * @param uptr Pointer to associate with local side of socket pair
-	 * @return PhySocket for local side of socket pair
+	 * @param fd Raw file descriptor
+	 * @param uptr User pointer to supply to callbacks
+	 * @return PhySocket wrapping fd or NULL on failure (out of memory or too many sockets)
 	 */
-	inline PhySocket *createSocketPair(ZT_PHY_SOCKFD_TYPE &remoteSocketDescriptor,void *uptr = (void *)0)
+	inline PhySocket *wrapSocket(ZT_PHY_SOCKFD_TYPE fd,void *uptr = (void *)0)
 	{
 		if (_socks.size() >= ZT_PHY_MAX_SOCKETS)
 			return (PhySocket *)0;
-
-		int fd[2]; fd[0] = -1; fd[1] = -1;
-		if ((::socketpair(PF_LOCAL,SOCK_STREAM,0,fd) != 0)||(fd[0] <= 0)||(fd[1] <= 0))
-			return (PhySocket *)0;
-		fcntl(fd[0],F_SETFL,O_NONBLOCK);
-
 		try {
 			_socks.push_back(PhySocketImpl());
 		} catch ( ... ) {
-			ZT_PHY_CLOSE_SOCKET(fd[0]);
-			ZT_PHY_CLOSE_SOCKET(fd[1]);
 			return (PhySocket *)0;
 		}
 		PhySocketImpl &sws = _socks.back();
-
-		if ((long)fd[0] > _nfds)
-			_nfds = (long)fd[0];
-		FD_SET(fd[0],&_readfds);
-		sws.type = ZT_PHY_SOCKET_PAIR_ENDPOINT;
-		sws.sock = fd[0];
+		if ((long)fd > _nfds)
+			_nfds = (long)fd;
+		FD_SET(fd,&_readfds);
+		sws.type = ZT_PHY_SOCKET_FD;
+		sws.sock = fd;
 		sws.uptr = uptr;
 		memset(&(sws.saddr),0,sizeof(struct sockaddr_storage));
 		// no sockaddr for this socket type, leave saddr null
-
-		remoteSocketDescriptor = fd[1];
-
 		return (PhySocket *)&sws;
 	}
-#endif // __UNIX_LIKE__
 
 	/**
 	 * Bind a UDP socket
@@ -673,6 +655,36 @@ public:
 		return (PhySocket *)&sws;
 	}
 
+	/**
+	 * Try to set buffer sizes as close to the given value as possible
+	 *
+	 * This will try the specified value and then lower values in 16K increments
+	 * until one works.
+	 *
+	 * @param sock Socket
+	 * @param bufferSize Desired buffer sizes
+	 */
+	inline void setBufferSizes(const PhySocket *sock,int bufferSize)
+	{
+		PhySocketImpl &sws = *(reinterpret_cast<PhySocketImpl *>(sock));
+		if (bufferSize > 0) {
+			int bs = bufferSize;
+			while (bs >= 65536) {
+				int tmpbs = bs;
+				if (::setsockopt(sws.sock,SOL_SOCKET,SO_RCVBUF,(const char *)&tmpbs,sizeof(tmpbs)) == 0)
+					break;
+				bs -= 16384;
+			}
+			bs = bufferSize;
+			while (bs >= 65536) {
+				int tmpbs = bs;
+				if (::setsockopt(sws.sock,SOL_SOCKET,SO_SNDBUF,(const char *)&tmpbs,sizeof(tmpbs)) == 0)
+					break;
+				bs -= 16384;
+			}
+		}
+	}
+
 	/**
 	 * Attempt to send data to a stream socket (non-blocking)
 	 *
@@ -787,6 +799,26 @@ public:
 		}
 	}
 
+	/**
+	 * Set whether we want to be notified that a socket is readable
+	 *
+	 * This is primarily for raw sockets added with wrapSocket(). It could be
+	 * used with others, but doing so would essentially lock them and prevent
+	 * data from being read from them until this is set to 'true' again.
+	 *
+	 * @param sock Socket to modify
+	 * @param notifyReadable True if socket should be monitored for readability
+	 */
+	inline const void setNotifyReadable(PhySocket *sock,bool notifyReadable)
+	{
+		PhySocketImpl &sws = *(reinterpret_cast<PhySocketImpl *>(sock));
+		if (notifyReadable) {
+			FD_SET(sws.sock,&_readfds);
+		} else {
+			FD_CLR(sws.sock,&_readfds);
+		}
+	}
+
 	/**
 	 * Wait for activity and handle one or more events
 	 *
@@ -936,7 +968,7 @@ public:
 					}
 					if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) {
 						try {
-							_handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr));
+							//_handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr));
 						} catch ( ... ) {}
 					}
 #endif // __UNIX_LIKE__
@@ -971,25 +1003,15 @@ public:
 #endif // __UNIX_LIKE__
 					break;
 
-				case ZT_PHY_SOCKET_PAIR_ENDPOINT: {
-#ifdef __UNIX_LIKE__
-					ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable
-					if (FD_ISSET(sock,&rfds)) {
-						long n = (long)::read(sock,buf,sizeof(buf));
-						if (n <= 0) {
-							this->close((PhySocket *)&(*s),true);
-						} else {
-							try {
-								_handler->phyOnSocketPairEndpointData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n);
-							} catch ( ... ) {}
-						}
-					}
-					if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) {
+				case ZT_PHY_SOCKET_FD: {
+					ZT_PHY_SOCKFD_TYPE sock = s->sock;
+					const bool readable = ((FD_ISSET(sock,&rfds))&&(FD_ISSET(sock,&_readfds)));
+					const bool writable = ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds)));
+					if ((readable)||(writable)) {
 						try {
-							_handler->phyOnSocketPairEndpointWritable((PhySocket *)&(*s),&(s->uptr));
+							_handler->phyOnFileDescriptorActivity((PhySocket *)&(*s),&(s->uptr),readable,writable);
 						} catch ( ... ) {}
 					}
-#endif // __UNIX_LIKE__
 				}	break;
 
 				default:
@@ -1021,7 +1043,8 @@ public:
 		FD_CLR(sws.sock,&_exceptfds);
 #endif
 
-		ZT_PHY_CLOSE_SOCKET(sws.sock);
+		if (sws.type != ZT_PHY_SOCKET_FD)
+			ZT_PHY_CLOSE_SOCKET(sws.sock);
 
 #ifdef __UNIX_LIKE__
 		if (sws.type == ZT_PHY_SOCKET_UNIX_LISTEN)
@@ -1048,12 +1071,6 @@ public:
 					} catch ( ... ) {}
 #endif // __UNIX_LIKE__
 					break;
-				case ZT_PHY_SOCKET_PAIR_ENDPOINT:
-#ifdef __UNIX_LIKE__
-					try {
-						_handler->phyOnSocketPairEndpointClose(sock,&(sws.uptr));
-					} catch ( ... ) {}
-#endif // __UNIX_LIKE__
 				default:
 					break;
 			}

selftest.cpp

@@ -593,23 +593,44 @@ static int testOther()
 	std::cout << "[other] Testing Hashtable... "; std::cout.flush();
 	{
 		Hashtable<uint64_t,std::string> ht;
-		Hashtable<uint64_t,std::string> ht2;
 		std::map<uint64_t,std::string> ref; // assume std::map works correctly :)
 		for(int x=0;x<2;++x) {
-			for(int i=0;i<25000;++i) {
+			for(int i=0;i<77777;++i) {
 				uint64_t k = rand();
 				while ((k == 0)||(ref.count(k) > 0))
 					++k;
 				std::string v("!");
 				for(int j=0;j<(int)(k % 64);++j)
 					v.push_back("0123456789"[rand() % 10]);
-				ht.set(k,v);
 				ref[k] = v;
+				ht.set(0xffffffffffffffffULL,v);
+				std::string &vref = ht[k];
+				vref = v;
+				ht.erase(0xffffffffffffffffULL);
 			}
 			if (ht.size() != ref.size()) {
 				std::cout << "FAILED! (size mismatch, original)" << std::endl;
 				return -1;
 			}
+			{
+				Hashtable<uint64_t,std::string>::Iterator i(ht);
+				uint64_t *k = (uint64_t *)0;
+				std::string *v = (std::string *)0;
+				while(i.next(k,v)) {
+					if (ref.find(*k)->second != *v) {
+						std::cout << "FAILED! (data mismatch!)" << std::endl;
+						return -1;
+					}
+				}
+			}
+			for(std::map<uint64_t,std::string>::const_iterator i(ref.begin());i!=ref.end();++i) {
+				if (ht[i->first] != i->second) {
+					std::cout << "FAILED! (data mismatch!)" << std::endl;
+					return -1;
+				}
+			}
+
+			Hashtable<uint64_t,std::string> ht2;
 			ht2 = ht;
 			Hashtable<uint64_t,std::string> ht3(ht2);
 			if (ht2.size() != ref.size()) {
@@ -620,6 +641,7 @@ static int testOther()
 				std::cout << "FAILED! (size mismatch, copied)" << std::endl;
 				return -1;
 			}
+
 			for(std::map<uint64_t,std::string>::iterator i(ref.begin());i!=ref.end();++i) {
 				std::string *v = ht.get(i->first);
 				if (!v) {
@@ -837,9 +859,7 @@ struct TestPhyHandlers
 	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
 	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
-	inline void phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr) {}
-  inline void phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
-  inline void phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr) {}
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
 #endif // __UNIX_LIKE__
 };
 static int testPhy()

service/ControlPlane.cpp

@@ -265,7 +265,7 @@ unsigned int ControlPlane::handleRequest(
 	std::string &responseBody,
 	std::string &responseContentType)
 {
-	char json[1024];
+	char json[8194];
 	unsigned int scode = 404;
 	std::vector<std::string> ps(Utils::split(path.c_str(),"/","",""));
 	std::map<std::string,std::string> urlArgs;
@@ -365,11 +365,12 @@ unsigned int ControlPlane::handleRequest(
 					_node->clusterStatus(&cs);
 
 					if (cs.clusterSize >= 1) {
-						char t[4096];
-						Utils::snprintf(t,sizeof(t),"{\n\t\t\"myId\": %u,\n\t\t\"clusterSize\": %u,\n\t\t\"members: [\n",cs.myId,cs.clusterSize);
+						char t[1024];
+						Utils::snprintf(t,sizeof(t),"{\n\t\t\"myId\": %u,\n\t\t\"clusterSize\": %u,\n\t\t\"members\": [",cs.myId,cs.clusterSize);
 						clusterJson.append(t);
 						for(unsigned int i=0;i<cs.clusterSize;++i) {
-							Utils::snprintf(t,sizeof(t),"\t\t\t{\n\t\t\t\t\"id\": %u,\n\t\t\t\t\"msSinceLastHeartbeat\": %u,\n\t\t\t\t\"alive\": %s,\n\t\t\t\t\"x\": %d,\n\t\t\t\t\"y\": %d,\n\t\t\t\t\"z\": %d,\n\t\t\t\t\"load\": %llu\n\t\t\t\t\"peers\": %llu\n\t\t\t}%s",
+							Utils::snprintf(t,sizeof(t),"%s\t\t\t{\n\t\t\t\t\"id\": %u,\n\t\t\t\t\"msSinceLastHeartbeat\": %u,\n\t\t\t\t\"alive\": %s,\n\t\t\t\t\"x\": %d,\n\t\t\t\t\"y\": %d,\n\t\t\t\t\"z\": %d,\n\t\t\t\t\"load\": %llu,\n\t\t\t\t\"peers\": %llu\n\t\t\t}",
+								((i == 0) ? "\n" : ",\n"),
 								cs.members[i].id,
 								cs.members[i].msSinceLastHeartbeat,
 								(cs.members[i].alive != 0) ? "true" : "false",
@@ -377,8 +378,7 @@ unsigned int ControlPlane::handleRequest(
 								cs.members[i].y,
 								cs.members[i].z,
 								cs.members[i].load,
-								cs.members[i].peers,
-								(i == (cs.clusterSize - 1)) ? "," : "");
+								cs.members[i].peers);
 							clusterJson.append(t);
 						}
 						clusterJson.append(" ]\n\t\t}");

service/OneService.cpp

@@ -731,7 +731,7 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
-						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)));
 #endif
 
 					struct ifaddrs *ifatbl = (struct ifaddrs *)0;
@@ -749,7 +749,7 @@ public:
 								if (!isZT) {
 									InetAddress ip(ifa->ifa_addr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip));
 								}
 							}
 							ifa = ifa->ifa_next;
@@ -867,6 +867,7 @@ public:
 	{
 #ifdef ZT_ENABLE_CLUSTER
 		if (sock == _clusterMessageSocket) {
+			_lastDirectReceiveFromGlobal = OSUtils::now();
 			_node->clusterHandleIncomingMessage(data,len);
 			return;
 		}
@@ -1030,7 +1031,7 @@ public:
 							if (from) {
 								ZT_ResultCode rc = _node->processWirePacket(
 									OSUtils::now(),
-									0,
+									&ZT_SOCKADDR_NULL,
 									reinterpret_cast<struct sockaddr_storage *>(&from),
 									data,
 									plen,
@@ -1084,9 +1085,7 @@ public:
 	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
 	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
-	inline void phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr) {}
-  inline void phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
-  inline void phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr) {}
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
 
 	inline int nodeVirtualNetworkConfigFunction(uint64_t nwid,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwc)
 	{

tests/http/Dockerfile

@@ -0,0 +1,24 @@
+FROM centos:latest
+
+MAINTAINER https://www.zerotier.com/
+
+EXPOSE 9993/udp
+
+ADD nodesource-el.repo /etc/yum.repos.d/nodesource-el.repo
+RUN yum -y update && yum install -y nodejs && yum clean all
+
+RUN mkdir -p /var/lib/zerotier-one
+RUN mkdir -p /var/lib/zerotier-one/networks.d
+RUN touch /var/lib/zerotier-one/networks.d/ffffffffffffffff.conf
+
+ADD package.json /
+RUN npm install
+
+ADD zerotier-one /
+RUN chmod a+x /zerotier-one
+
+ADD agent.js /
+ADD docker-main.sh /
+RUN chmod a+x /docker-main.sh
+
+CMD ["./docker-main.sh"]

tests/http/README.md

@@ -0,0 +1,12 @@
+HTTP one-to-all test
+======
+
+*This is really internal use code. You're free to test it out but expect to do some editing/tweaking to make it work. We used this to run some massive scale tests of our new geo-cluster-based root server infrastructure prior to taking it live.*
+
+Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently. You will also want to edit the Dockerfile if you want to build containers and change the network ID to the network you want to run tests over.
+
+This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+
+The Dockerfile builds an image that launches the agent. The image must be "docker run" with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. (Unfortunately CAP_NET_ADMIN may not work due to a bug in Docker and/or Linux.) You can run a bunch with a command like:
+
+    for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done

tests/http/agent.js

@@ -0,0 +1,277 @@
+// ZeroTier distributed HTTP test agent
+
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+// Maximum interval between test attempts
+var TEST_INTERVAL_MAX = 60000;
+
+// Test timeout in ms
+var TEST_TIMEOUT = 30000;
+
+// Where should I contact to register and query a list of other test agents?
+var SERVER_HOST = '104.238.141.145';
+var SERVER_PORT = 18080;
+
+// Which port should agents use for their HTTP?
+var AGENT_PORT = 18888;
+
+// Payload size in bytes
+var PAYLOAD_SIZE = 10000;
+
+// ---------------------------------------------------------------------------
+
+var ipaddr = require('ipaddr.js');
+var os = require('os');
+var http = require('http');
+var async = require('async');
+
+var express = require('express');
+var app = express();
+
+// Find our ZeroTier-assigned RFC4193 IPv6 address
+var thisAgentId = null;
+var interfaces = os.networkInterfaces();
+if (!interfaces) {
+	console.error('FATAL: os.networkInterfaces() failed.');
+	process.exit(1);
+}
+for(var ifname in interfaces) {
+	var ifaddrs = interfaces[ifname];
+	if (Array.isArray(ifaddrs)) {
+		for(var i=0;i<ifaddrs.length;++i) {
+			if (ifaddrs[i].family == 'IPv6') {
+				try {
+					var ipbytes = ipaddr.parse(ifaddrs[i].address).toByteArray();
+					if ((ipbytes.length === 16)&&(ipbytes[0] == 0xfd)&&(ipbytes[9] == 0x99)&&(ipbytes[10] == 0x93)) {
+						thisAgentId = '';
+						for(var j=0;j<16;++j) {
+							var tmp = ipbytes[j].toString(16);
+							if (tmp.length === 1)
+								thisAgentId += '0';
+							thisAgentId += tmp;
+						}
+					}
+				} catch (e) {
+					console.error(e);
+				}
+			}
+		}
+	}
+}
+if (thisAgentId === null) {
+	console.error('FATAL: no ZeroTier-assigned RFC4193 IPv6 addresses found on any local interface!');
+	process.exit(1);
+}
+
+//console.log(thisAgentId);
+
+// Create a random (and therefore not very compressable) payload
+var payload = new Buffer(PAYLOAD_SIZE);
+for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
+	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
+}
+
+function agentIdToIp(agentId)
+{
+	var ip = '';
+	ip += agentId.substr(0,4);
+	ip += ':';
+	ip += agentId.substr(4,4);
+	ip += ':';
+	ip += agentId.substr(8,4);
+	ip += ':';
+	ip += agentId.substr(12,4);
+	ip += ':';
+	ip += agentId.substr(16,4);
+	ip += ':';
+	ip += agentId.substr(20,4);
+	ip += ':';
+	ip += agentId.substr(24,4);
+	ip += ':';
+	ip += agentId.substr(28,4);
+	return ip;
+};
+
+var lastTestResult = null;
+var allOtherAgents = [];
+
+function doTest()
+{
+	var submit = http.request({
+		host: SERVER_HOST,
+		port: SERVER_PORT,
+		path: '/'+thisAgentId,
+		method: 'POST'
+	},function(res) {
+		var body = '';
+		res.on('data',function(chunk) { body += chunk.toString(); });
+		res.on('end',function() {
+
+			if (body) {
+				try {
+					var peers = JSON.parse(body);
+					if (Array.isArray(peers))
+						allOtherAgents = peers;
+				} catch (e) {}
+			}
+
+			if (allOtherAgents.length > 1) {
+
+				var target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+				while (target === thisAgentId)
+					target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+
+				var testRequest = null;
+				var timeoutId = null;
+				timeoutId = setTimeout(function() {
+					if (testRequest !== null)
+						testRequest.abort();
+					timeoutId = null;
+				},TEST_TIMEOUT);
+				var startTime = Date.now();
+
+				testRequest = http.get({
+					host: agentIdToIp(target),
+					port: AGENT_PORT,
+					path: '/'
+				},function(res) {
+					var bytes = 0;
+					res.on('data',function(chunk) { bytes += chunk.length; });
+					res.on('end',function() {
+						lastTestResult = {
+							source: thisAgentId,
+							target: target,
+							time: (Date.now() - startTime),
+							bytes: bytes,
+							timedOut: (timeoutId === null),
+							error: null
+						};
+						if (timeoutId !== null)
+							clearTimeout(timeoutId);
+						return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+					});
+				}).on('error',function(e) {
+					lastTestResult = {
+						source: thisAgentId,
+						target: target,
+						time: (Date.now() - startTime),
+						bytes: 0,
+						timedOut: (timeoutId === null),
+						error: e.toString()
+					};
+					if (timeoutId !== null)
+						clearTimeout(timeoutId);
+					return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+				});
+
+			} else {
+				return setTimeout(doTest,1000);
+			}
+
+		});
+	}).on('error',function(e) {
+		console.log('POST failed: '+e.toString());
+		return setTimeout(doTest,1000);
+	});
+	if (lastTestResult !== null) {
+		submit.write(JSON.stringify(lastTestResult));
+		lastTestResult = null;
+	}
+	submit.end();
+};
+
+/*
+function performTestOnAllPeers(peers,callback)
+{
+	var allResults = {};
+	var allRequests = [];
+	var timedOut = false;
+	var endOfTestTimer = setTimeout(function() {
+		timedOut = true;
+		for(var x=0;x<allRequests.length;++x)
+			allRequests[x].abort();
+	},TEST_DURATION);
+
+	async.each(peers,function(peer,next) {
+		if (timedOut)
+			return next(null);
+		if (peer.length !== 32)
+			return next(null);
+
+		var connectionStartTime = Date.now();
+		allResults[peer] = {
+			start: connectionStartTime,
+			end: 0,
+			error: null,
+			timedOut: false,
+			bytes: 0
+		};
+
+		allRequests.push(http.get({
+			host: agentIdToIp(peer),
+			port: AGENT_PORT,
+			path: '/'
+		},function(res) {
+			var bytes = 0;
+			res.on('data',function(chunk) {
+				bytes += chunk.length;
+			});
+			res.on('end',function() {
+				allResults[peer] = {
+					start: connectionStartTime,
+					end: Date.now(),
+					error: null,
+					timedOut: timedOut,
+					bytes: bytes
+				};
+				return next(null);
+			});
+		}).on('error',function(e) {
+			allResults[peer] = {
+				start: connectionStartTime,
+				end: Date.now(),
+				error: e.toString(),
+				timedOut: timedOut,
+				bytes: 0
+			};
+			return next(null);
+		}));
+	},function(err) {
+		if (!timedOut)
+			clearTimeout(endOfTestTimer);
+		return callback(allResults);
+	});
+};
+
+function doTestsAndReport()
+{
+	registerAndGetPeers(function(err,peers) {
+		if (err) {
+			console.error('WARNING: skipping test: unable to contact or query server: '+err.toString());
+		} else {
+			performTestOnAllPeers(peers,function(results) {
+				var submit = http.request({
+					host: SERVER_HOST,
+					port: SERVER_PORT,
+					path: '/'+thisAgentId,
+					method: 'POST'
+				},function(res) {
+				}).on('error',function(e) {
+					console.error('WARNING: unable to submit results to server: '+err.toString());
+				});
+				submit.write(JSON.stringify(results));
+				submit.end();
+			});
+		}
+	});
+};
+*/
+
+// Agents just serve up a test payload
+app.get('/',function(req,res) { return res.status(200).send(payload); });
+
+var expressServer = app.listen(AGENT_PORT,function () {
+	// Start timeout-based loop
+	doTest();
+});

tests/http/big-test-hosts

@@ -0,0 +1,2 @@
[email protected]
[email protected]

tests/http/big-test-kill.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+pssh -h big-test-hosts -i -t 0 -p 256 "docker ps -aq | xargs -r docker rm -f"
+
+exit 0

tests/http/big-test-ready.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -i -t 0 -p 256 "docker pull $CONTAINER_IMAGE"
+
+exit 0

tests/http/big-test-start.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=50
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -o big-test-out -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
+
+exit 0

tests/http/docker-main.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
+
+/zerotier-one -d >>zerotier-one.out 2>&1
+
+while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
+	sleep 0.25
+done
+
+sleep 2
+
+exec node --harmony /agent.js >>agent.out 2>&1
+#exec node --harmony /agent.js

tests/http/nodesource-el.repo

@@ -0,0 +1,6 @@
+[nodesource]
+name=Node.js Packages for Enterprise Linux 7 - $basearch
+baseurl=https://rpm.nodesource.com/pub_4.x/el/7/$basearch
+failovermethod=priority
+enabled=1
+gpgcheck=0

tests/http/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "zerotier-test-http",
+  "version": "1.0.0",
+  "description": "ZeroTier in-network HTTP test",
+  "main": "agent.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "ZeroTier, Inc.",
+  "license": "GPL-3.0",
+  "dependencies": {
+    "async": "^1.5.0",
+    "express": "^4.13.3",
+    "ipaddr.js": "^1.0.3"
+  }
+}

tests/http/server.js

@@ -0,0 +1,44 @@
+// ZeroTier distributed HTTP test coordinator and result-reporting server
+
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+var SERVER_PORT = 18080;
+
+// ---------------------------------------------------------------------------
+
+var fs = require('fs');
+
+var express = require('express');
+var app = express();
+
+app.use(function(req,res,next) {
+	req.rawBody = '';
+	req.on('data', function(chunk) { req.rawBody += chunk.toString(); });
+	req.on('end', function() { return next(); });
+});
+
+var knownAgents = {};
+
+app.post('/:agentId',function(req,res) {
+	var agentId = req.params.agentId;
+	if ((!agentId)||(agentId.length !== 32))
+		return res.status(404).send('');
+
+	if (req.rawBody) {
+		var receiveTime = Date.now();
+		var resultData = null;
+		try {
+			resultData = JSON.parse(req.rawBody);
+			console.log(resultData.source+','+resultData.target+','+resultData.time+','+resultData.bytes+','+resultData.timedOut+',"'+((resultData.error) ? resultData.error : '')+'"');
+		} catch (e) {}
+	}
+
+	knownAgents[agentId] = Date.now();
+	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
+});
+
+var expressServer = app.listen(SERVER_PORT,function () {
+	console.log('LISTENING ON '+SERVER_PORT);
+	console.log('');
+});

version.h

@@ -36,11 +36,11 @@
 /**
  * Minor version
  */
-#define ZEROTIER_ONE_VERSION_MINOR 0
+#define ZEROTIER_ONE_VERSION_MINOR 1
 
 /**
  * Revision
  */
-#define ZEROTIER_ONE_VERSION_REVISION 6
+#define ZEROTIER_ONE_VERSION_REVISION 0
 
 #endif

world/alice-test/mkworld.cpp

@@ -151,6 +151,7 @@ int main(int argc,char **argv)
 	roots.back().stableEndpoints.push_back(InetAddress("2400:6180:0:d0::1b:1001/9993")); // Singapore IPv6
 
 	// Bob -- global geo-clustered root #2
+	roots.push_back(World::Root());
 	roots.back().identity = Identity("16ebbd6c5d:0:47d39bca9d0a5cf70148e39f6c45199e17e0e32e4e46cac01ae5bcb21224137b097f40bdd982a921c3aabdcb9ada8b4f2bb0593753bfdb21cf12eac28c8d9042");
 	roots.back().stableEndpoints.push_back(InetAddress("45.33.4.67/9993")); // Dallas IPv4
 	roots.back().stableEndpoints.push_back(InetAddress("2600:3c00::f03c:91ff:fe67:b704/9993")); // Dallas IPv6