Merge branch 'dev' into systemtray

attic/CertificateOfTrust.cpp

@@ -0,0 +1,67 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "CertificateOfTrust.hpp"
+
+#include "RuntimeEnvironment.hpp"
+#include "Topology.hpp"
+#include "Switch.hpp"
+
+namespace ZeroTier {
+
+bool CertificateOfTrust::create(uint64_t ts,uint64_t rls,const Identity &iss,const Identity &tgt,Level l)
+{
+	if ((!iss)||(!iss.hasPrivate()))
+		return false;
+
+	_timestamp = ts;
+	_roles = rls;
+	_issuer = iss.address();
+	_target = tgt;
+	_level = l;
+
+	Buffer<sizeof(Identity) + 64> tmp;
+	tmp.append(_timestamp);
+	tmp.append(_roles);
+	_issuer.appendTo(tmp);
+	_target.serialize(tmp,false);
+	tmp.append((uint16_t)_level);
+	_signature = iss.sign(tmp.data(),tmp.size());
+
+	return true;
+}
+
+int CertificateOfTrust::verify(const RuntimeEnvironment *RR) const
+{
+	const Identity id(RR->topology->getIdentity(_issuer));
+	if (!id) {
+		RR->sw->requestWhois(_issuer);
+		return 1;
+	}
+
+	Buffer<sizeof(Identity) + 64> tmp;
+	tmp.append(_timestamp);
+	tmp.append(_roles);
+	_issuer.appendTo(tmp);
+	_target.serialize(tmp,false);
+	tmp.append((uint16_t)_level);
+
+	return (id.verify(tmp.data(),tmp.size(),_signature) ? 0 : -1);
+}
+
+} // namespace ZeroTier

attic/CertificateOfTrust.hpp

@@ -0,0 +1,155 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef ZT_CERTIFICATEOFTRUST_HPP
+#define ZT_CERTIFICATEOFTRUST_HPP
+
+#include "Constants.hpp"
+#include "Identity.hpp"
+#include "C25519.hpp"
+#include "Buffer.hpp"
+
+namespace ZeroTier {
+
+class RuntimeEnvironment;
+
+/**
+ * Certificate of peer to peer trust
+ */
+class CertificateOfTrust
+{
+public:
+	/**
+	 * Trust levels, with 0 indicating anti-trust
+	 */
+	enum Level
+	{
+		/**
+		 * Negative trust is reserved for informing peers that another peer is misbehaving, etc. Not currently used.
+		 */
+		LEVEL_NEGATIVE = 0,
+
+		/**
+		 * Default trust -- for most peers
+		 */
+		LEVEL_DEFAULT = 1,
+
+		/**
+		 * Above normal trust, e.g. common network membership
+		 */
+		LEVEL_MEDIUM = 25,
+
+		/**
+		 * High trust -- e.g. an upstream or a controller
+		 */
+		LEVEL_HIGH = 50,
+
+		/**
+		 * Right now ultimate is only for roots
+		 */
+		LEVEL_ULTIMATE = 100
+	};
+
+	/**
+	 * Role bit masks
+	 */
+	enum Role
+	{
+		/**
+		 * Target is permitted to represent issuer on the network as a federated root / relay
+		 */
+		ROLE_UPSTREAM = 0x00000001
+	};
+
+	CertificateOfTrust() :
+		_timestamp(0),
+		_roles(0),
+		_issuer(),
+		_target(),
+		_level(LEVEL_DEFAULT),
+		_signature() {}
+
+	/**
+	 * Create and sign this certificate of trust
+	 *
+	 * @param ts Cert timestamp
+	 * @param rls Roles bitmap
+	 * @param iss Issuer identity (must have secret key!)
+	 * @param tgt Target identity
+	 * @param l Trust level
+	 * @return True on successful signature
+	 */
+	bool create(uint64_t ts,uint64_t rls,const Identity &iss,const Identity &tgt,Level l);
+
+	/**
+	 * Verify this COT and its signature
+	 *
+	 * @param RR Runtime environment for looking up peers
+	 * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential
+	 */
+	int verify(const RuntimeEnvironment *RR) const;
+
+	inline bool roleUpstream() const { return ((_roles & (uint64_t)ROLE_UPSTREAM) != 0); }
+
+	inline uint64_t timestamp() const { return _timestamp; }
+	inline uint64_t roles() const { return _roles; }
+	inline const Address &issuer() const { return _issuer; }
+	inline const Identity &target() const { return _target; }
+	inline Level level() const { return _level; }
+
+	inline operator bool() const { return (_issuer); }
+
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append(_timestamp);
+		b.append(_roles);
+		_issuer.appendTo(b);
+		_target.serialize(b);
+		b.append((uint16_t)_level);
+		b.append((uint8_t)1); // 1 == ed25519 signature
+		b.append((uint16_t)ZT_C25519_SIGNATURE_LEN);
+		b.append(_signature.data,ZT_C25519_SIGNATURE_LEN);
+		b.append((uint16_t)0); // length of additional fields
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		_timestamp = b.template at<uint64_t>(p); p += 8;
+		_roles = b.template at<uint64_t>(p); p += 8;
+		_issuer.setTo(b.field(p,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); p += ZT_ADDRESS_LENGTH;
+		p += _target.deserialize(b,p);
+		_level = b.template at<uint16_t>(p); p += 2;
+		p += b.template at<uint16_t>(p); p += 2;
+		return (p - startAt);
+	}
+
+private:
+	uint64_t _timestamp;
+	uint64_t _roles;
+	Address _issuer;
+	Identity _target;
+	Level _level;
+	C25519::Signature _signature;
+};
+
+} // namespace ZeroTier
+
+#endif

controller/EmbeddedNetworkController.cpp

@@ -1776,11 +1776,13 @@ void EmbeddedNetworkController::_pushMemberUpdate(uint64_t now,uint64_t nwid,con
 				std::map<std::pair<uint64_t,uint64_t>,uint64_t>::iterator lrt(_lastRequestTime.find(std::pair<uint64_t,uint64_t>(id.address().toInt(),nwid)));
 				online = ( (lrt != _lastRequestTime.end()) && ((now - lrt->second) < ZT_NETWORK_AUTOCONF_DELAY) );
 			}
-			Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> *metaData = new Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>(mdstr.c_str());
-			try {
-				this->request(nwid,InetAddress(),0,id,*metaData);
-			} catch ( ... ) {}
-			delete metaData;
+			if (online) {
+				Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> *metaData = new Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>(mdstr.c_str());
+				try {
+					this->request(nwid,InetAddress(),0,id,*metaData);
+				} catch ( ... ) {}
+				delete metaData;
+			}
 		}
 	} catch ( ... ) {}
 }

include/ZeroTierOne.h

@@ -1784,6 +1784,17 @@ int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage
  */
 void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node);
 
+/**
+ * Set peer role
+ *
+ * Right now this can only be used to set a peer to either LEAF or
+ * UPSTREAM, since roots are fixed and defined by the World.
+ *
+ * @param ztAddress ZeroTier address (least significant 40 bits)
+ * @param role New peer role (LEAF or UPSTREAM)
+ */
+void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,ZT_PeerRole role);
+
 /**
  * Set a network configuration master instance for this node
  *

node/Constants.hpp

@@ -375,6 +375,26 @@
  */
 #define ZT_PEER_GENERAL_RATE_LIMIT 1000
 
+/**
+ * Don't do expensive identity validation more often than this
+ *
+ * IPv4 and IPv6 address prefixes are hashed down to 14-bit (0-16383) integers
+ * using the first 24 bits for IPv4 or the first 48 bits for IPv6. These are
+ * then rate limited to one identity validation per this often milliseconds.
+ */
+#if (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64) || defined(_M_AMD64))
+// AMD64 machines can do anywhere from one every 50ms to one every 10ms. This provides plenty of margin.
+#define ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT 2000
+#else
+#if (defined(__i386__) || defined(__i486__) || defined(__i586__) || defined(__i686__) || defined(_M_IX86) || defined(_X86_) || defined(__I86__))
+// 32-bit Intel machines usually average about one every 100ms
+#define ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT 5000
+#else
+// This provides a safe margin for ARM, MIPS, etc. that usually average one every 250-400ms
+#define ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT 10000
+#endif
+#endif
+
 /**
  * How long is a path or peer considered to have a trust relationship with us (for e.g. relay policy) since last trusted established packet?
  */

node/IncomingPacket.cpp

@@ -160,7 +160,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 
 			case Packet::ERROR_IDENTITY_COLLISION:
 				// FIXME: for federation this will need a payload with a signature or something.
-				if (RR->topology->isRoot(peer->identity()))
+				if (RR->topology->isUpstream(peer->identity()))
 					RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
 				break;
 
@@ -247,6 +247,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut
 				if (peer->identity() != id) {
 					// Identity is different from the one we already have -- address collision
 
+					// Check rate limits
+					if (!RR->node->rateGateIdentityVerification(now,_path->address()))
+						return true;
+
 					uint8_t key[ZT_PEER_SECRET_KEY_LENGTH];
 					if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
 						if (dearmor(key)) { // ensure packet is authentic, otherwise drop
@@ -275,7 +279,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut
 
 					// Continue at // VALID
 				}
-			} // else continue at // VALID
+			} // else if alreadyAuthenticated then continue at // VALID
 		} else {
 			// We don't already have an identity with this address -- validate and learn it
 
@@ -285,18 +289,23 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut
 				return true;
 			}
 
-			// Check that identity's address is valid as per the derivation function
-			if (!id.locallyValidate()) {
-				TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str());
+			// Check rate limits
+			if (!RR->node->rateGateIdentityVerification(now,_path->address()))
 				return true;
-			}
 
-			// Check packet integrity and authentication
+			// Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap)
 			SharedPtr<Peer> newPeer(new Peer(RR,RR->identity,id));
 			if (!dearmor(newPeer->key())) {
 				TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str());
 				return true;
 			}
+
+			// Check that identity's address is valid as per the derivation function
+			if (!id.locallyValidate()) {
+				TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str());
+				return true;
+			}
+
 			peer = RR->topology->addPeer(newPeer);
 
 			// Continue at // VALID
@@ -508,11 +517,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				id.serialize(outp,false);
 				++count;
 			} else {
-				// If I am not the root and don't know this identity, ask upstream. Downstream
-				// peer may re-request in the future and if so we will be able to provide it.
-				if (!RR->topology->amRoot())
-					RR->sw->requestWhois(addr);
-
+				RR->sw->requestWhois(addr);
 #ifdef ZT_ENABLE_CLUSTER
 				// Distribute WHOIS queries across a cluster if we do not know the ID.
 				// This may result in duplicate OKs to the querying peer, which is fine.
@@ -666,7 +671,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				}
 			}
 
-			if ((flags & 0x10) != 0) {
+			if ((flags & 0x10) != 0) { // ACK requested
 				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
 				outp.append((uint8_t)Packet::VERB_EXT_FRAME);
 				outp.append((uint64_t)packetId());

node/InetAddress.hpp

@@ -427,7 +427,7 @@ struct InetAddress : public sockaddr_storage
 		} else {
 			unsigned long tmp = reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_port;
 			const uint8_t *a = reinterpret_cast<const uint8_t *>(this);
-			for(long i=0;i<sizeof(InetAddress);++i)
+			for(long i=0;i<(long)sizeof(InetAddress);++i)
 				reinterpret_cast<uint8_t *>(&tmp)[i % sizeof(tmp)] ^= a[i];
 			return tmp;
 		}
@@ -449,6 +449,30 @@ struct InetAddress : public sockaddr_storage
 	bool isNetwork() const
 		throw();
 
+	/**
+	 * @return 14-bit (0-16383) hash of this IP's first 24 or 48 bits (for V4 or V6) for rate limiting code, or 0 if non-IP
+	 */
+	inline unsigned long rateGateHash() const
+	{
+		unsigned long h = 0;
+		switch(ss_family) {
+			case AF_INET:
+				h = (Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr) & 0xffffff00) >> 8;
+				h ^= (h >> 14);
+				break;
+			case AF_INET6: {
+				const uint8_t *ip = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr);
+				h = ((unsigned long)ip[0]); h <<= 1;
+				h += ((unsigned long)ip[1]); h <<= 1;
+				h += ((unsigned long)ip[2]); h <<= 1;
+				h += ((unsigned long)ip[3]); h <<= 1;
+				h += ((unsigned long)ip[4]); h <<= 1;
+				h += ((unsigned long)ip[5]);
+			}	break;
+		}
+		return (h & 0x3fff);
+	}
+
 	/**
 	 * @return True if address family is non-zero
 	 */

node/Multicaster.cpp

@@ -229,7 +229,7 @@ void Multicaster::send(
 
 				Address explicitGatherPeers[16];
 				unsigned int numExplicitGatherPeers = 0;
-				SharedPtr<Peer> bestRoot(RR->topology->getBestRoot());
+				SharedPtr<Peer> bestRoot(RR->topology->getUpstreamPeer());
 				if (bestRoot)
 					explicitGatherPeers[numExplicitGatherPeers++] = bestRoot->address();
 				explicitGatherPeers[numExplicitGatherPeers++] = Network::controllerFor(nwid);

node/Node.cpp

@@ -78,6 +78,7 @@ Node::Node(
 
 	memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr));
 	memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo));
+	memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification));
 
 	// Use Salsa20 alone as a high-quality non-crypto PRNG
 	{
@@ -211,8 +212,7 @@ public:
 		}
 
 		if (upstream) {
-			// "Upstream" devices are roots and relays and get special treatment -- they stay alive
-			// forever and we try to keep (if available) both IPv4 and IPv6 channels open to them.
+			// We keep connections to upstream peers alive forever.
 			bool needToContactIndirect = true;
 			if (p->doPingAndKeepalive(_now,AF_INET)) {
 				needToContactIndirect = false;
@@ -231,11 +231,8 @@ public:
 				}
 			}
 
+			// If we don't have a direct path or a static endpoint, send something indirectly to find one.
 			if (needToContactIndirect) {
-				// If this is an upstream and we have no stable endpoint for either IPv4 or IPv6,
-				// send a NOP indirectly if possible to see if we can get to this peer in any
-				// way whatsoever. This will e.g. find network preferred relays that lack
-				// stable endpoints by using root servers.
 				Packet outp(p->address(),RR->identity.address(),Packet::VERB_NOP);
 				RR->sw->send(outp,true);
 			}
@@ -415,7 +412,7 @@ ZT_PeerList *Node::peers() const
 			p->versionRev = -1;
 		}
 		p->latency = pi->second->latency();
-		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
+		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : (RR->topology->isUpstream(pi->second->identity()) ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF);
 
 		std::vector< std::pair< SharedPtr<Path>,bool > > paths(pi->second->paths(_now));
 		SharedPtr<Path> bestp(pi->second->getBestPath(_now,false));
@@ -487,6 +484,11 @@ void Node::clearLocalInterfaceAddresses()
 	_directPaths.clear();
 }
 
+void Node::setRole(uint64_t ztAddress,ZT_PeerRole role)
+{
+	RR->topology->setUpstream(Address(ztAddress),(role == ZT_PEER_ROLE_UPSTREAM));
+}
+
 void Node::setNetconfMaster(void *networkControllerInstance)
 {
 	RR->localNetworkController = reinterpret_cast<NetworkController *>(networkControllerInstance);
@@ -1010,6 +1012,13 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
 	} catch ( ... ) {}
 }
 
+void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,ZT_PeerRole role)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->setRole(ztAddress,role);
+	} catch ( ... ) {}
+}
+
 void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 {
 	try {

node/Node.hpp

@@ -105,6 +105,7 @@ public:
 	void freeQueryResult(void *qr);
 	int addLocalInterfaceAddress(const struct sockaddr_storage *addr);
 	void clearLocalInterfaceAddresses();
+	void setRole(uint64_t ztAddress,ZT_PeerRole role);
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
 	void circuitTestEnd(ZT_CircuitTest *test);
@@ -283,6 +284,23 @@ public:
 		return false;
 	}
 
+	/**
+	 * Check whether we should do potentially expensive identity verification (rate limit)
+	 *
+	 * @param now Current time
+	 * @param from Source address of packet
+	 * @return True if within rate limits
+	 */
+	inline bool rateGateIdentityVerification(const uint64_t now,const InetAddress &from)
+	{
+		unsigned long iph = from.rateGateHash();
+		if ((now - _lastIdentityVerification[iph]) >= ZT_IDENTITY_VALIDATION_SOURCE_RATE_LIMIT) {
+			_lastIdentityVerification[iph] = now;
+			return true;
+		}
+		return false;
+	}
+
 	virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig);
 	virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode);
 
@@ -302,9 +320,13 @@ private:
 
 	void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P
 
+	// For tracking packet IDs to filter out OK/ERROR replies to packets we did not send
 	uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1];
 	uint64_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1];
 
+	// Time of last identity verification indexed by InetAddress.rateGateHash()
+	uint64_t _lastIdentityVerification[16384];
+
 	ZT_DataStoreGetFunction _dataStoreGetFunction;
 	ZT_DataStorePutFunction _dataStorePutFunction;
 	ZT_WirePacketSendFunction _wirePacketSendFunction;

node/Packet.hpp

@@ -617,10 +617,8 @@ public:
 		 *   <[1] protocol address length (4 for IPv4, 16 for IPv6)>
 		 *   <[...] protocol address (network byte order)>
 		 *
-		 * This is sent by a relaying node to initiate NAT traversal between two
-		 * peers that are communicating by way of indirect relay. The relay will
-		 * send this to both peers at the same time on a periodic basis, telling
-		 * each where it might find the other on the network.
+		 * An upstream node can send this to inform both sides of a relay of
+		 * information they might use to establish a direct connection.
 		 *
 		 * Upon receipt a peer sends HELLO to establish a direct link.
 		 *

node/Peer.hpp

@@ -403,26 +403,6 @@ public:
 		return false;
 	}
 
-	/**
-	 * Find a common set of addresses by which two peers can link, if any
-	 *
-	 * @param a Peer A
-	 * @param b Peer B
-	 * @param now Current time
-	 * @return Pair: B's address (to send to A), A's address (to send to B)
-	 */
-	static inline std::pair<InetAddress,InetAddress> findCommonGround(const Peer &a,const Peer &b,uint64_t now)
-	{
-		std::pair<InetAddress,InetAddress> v4,v6;
-		b.getBestActiveAddresses(now,v4.first,v6.first);
-		a.getBestActiveAddresses(now,v4.second,v6.second);
-		if ((v6.first)&&(v6.second)) // prefer IPv6 if both have it since NAT-t is (almost) unnecessary
-			return v6;
-		else if ((v4.first)&&(v4.second))
-			return v4;
-		else return std::pair<InetAddress,InetAddress>();
-	}
-
 private:
 	inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const
 	{

node/Switch.cpp

@@ -131,8 +131,8 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 							}
 #endif
 
-							// Don't know peer or no direct path -- so relay via root server
-							relayTo = RR->topology->getBestRoot();
+							// Don't know peer or no direct path -- so relay via someone upstream
+							relayTo = RR->topology->getUpstreamPeer();
 							if (relayTo)
 								relayTo->sendDirect(fragment.data(),fragment.size(),now,true);
 						}
@@ -237,7 +237,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 							uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)];
 							if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) {
 								luts = now;
-								unite(source,destination);
+								_unite(source,destination);
 							}
 						} else {
 #ifdef ZT_ENABLE_CLUSTER
@@ -254,7 +254,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 								return;
 							}
 #endif
-							relayTo = RR->topology->getBestRoot(&source,1,true);
+							relayTo = RR->topology->getUpstreamPeer(&source,1,true);
 							if (relayTo)
 								relayTo->sendDirect(packet.data(),packet.size(),now,true);
 						}
@@ -590,75 +590,6 @@ void Switch::send(const Packet &packet,bool encrypt)
 	}
 }
 
-bool Switch::unite(const Address &p1,const Address &p2)
-{
-	if ((p1 == RR->identity.address())||(p2 == RR->identity.address()))
-		return false;
-	SharedPtr<Peer> p1p = RR->topology->getPeer(p1);
-	if (!p1p)
-		return false;
-	SharedPtr<Peer> p2p = RR->topology->getPeer(p2);
-	if (!p2p)
-		return false;
-
-	const uint64_t now = RR->node->now();
-
-	std::pair<InetAddress,InetAddress> cg(Peer::findCommonGround(*p1p,*p2p,now));
-	if ((!(cg.first))||(cg.first.ipScope() != cg.second.ipScope()))
-		return false;
-
-	TRACE("unite: %s(%s) <> %s(%s)",p1.toString().c_str(),cg.second.toString().c_str(),p2.toString().c_str(),cg.first.toString().c_str());
-
-	/* Tell P1 where to find P2 and vice versa, sending the packets to P1 and
-	 * P2 in randomized order in terms of which gets sent first. This is done
-	 * since in a few cases NAT-t can be sensitive to slight timing differences
-	 * in terms of when the two peers initiate. Normally this is accounted for
-	 * by the nearly-simultaneous RENDEZVOUS kickoff from the relay, but
-	 * given that relay are hosted on cloud providers this can in some
-	 * cases have a few ms of latency between packet departures. By randomizing
-	 * the order we make each attempted NAT-t favor one or the other going
-	 * first, meaning if it doesn't succeed the first time it might the second
-	 * and so forth. */
-	unsigned int alt = (unsigned int)RR->node->prng() & 1;
-	unsigned int completed = alt + 2;
-	while (alt != completed) {
-		if ((alt & 1) == 0) {
-			// Tell p1 where to find p2.
-			Packet outp(p1,RR->identity.address(),Packet::VERB_RENDEZVOUS);
-			outp.append((unsigned char)0);
-			p2.appendTo(outp);
-			outp.append((uint16_t)cg.first.port());
-			if (cg.first.isV6()) {
-				outp.append((unsigned char)16);
-				outp.append(cg.first.rawIpData(),16);
-			} else {
-				outp.append((unsigned char)4);
-				outp.append(cg.first.rawIpData(),4);
-			}
-			outp.armor(p1p->key(),true);
-			p1p->sendDirect(outp.data(),outp.size(),now,true);
-		} else {
-			// Tell p2 where to find p1.
-			Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS);
-			outp.append((unsigned char)0);
-			p1.appendTo(outp);
-			outp.append((uint16_t)cg.second.port());
-			if (cg.second.isV6()) {
-				outp.append((unsigned char)16);
-				outp.append(cg.second.rawIpData(),16);
-			} else {
-				outp.append((unsigned char)4);
-				outp.append(cg.second.rawIpData(),4);
-			}
-			outp.armor(p2p->key(),true);
-			p2p->sendDirect(outp.data(),outp.size(),now,true);
-		}
-		++alt; // counts up and also flips LSB
-	}
-
-	return true;
-}
-
 void Switch::requestWhois(const Address &addr)
 {
 	bool inserted = false;
@@ -763,7 +694,7 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 
 Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted)
 {
-	SharedPtr<Peer> upstream(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false));
+	SharedPtr<Peer> upstream(RR->topology->getUpstreamPeer(peersAlreadyConsulted,numPeersAlreadyConsulted,false));
 	if (upstream) {
 		Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS);
 		addr.appendTo(outp);
@@ -793,7 +724,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 			viaPath.zero();
 		}
 		if (!viaPath) {
-			SharedPtr<Peer> relay(RR->topology->getBestRoot());
+			SharedPtr<Peer> relay(RR->topology->getUpstreamPeer());
 			if ( (!relay) || (!(viaPath = relay->getBestPath(now,false))) ) {
 				if (!(viaPath = peer->getBestPath(now,true)))
 					return false;
@@ -839,4 +770,96 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 	return false;
 }
 
+bool Switch::_unite(const Address &p1,const Address &p2)
+{
+	if ((p1 == RR->identity.address())||(p2 == RR->identity.address()))
+		return false;
+
+	const uint64_t now = RR->node->now();
+	InetAddress *p1a = (InetAddress *)0;
+	InetAddress *p2a = (InetAddress *)0;
+	InetAddress p1v4,p1v6,p2v4,p2v6,uv4,uv6;
+	{
+		const SharedPtr<Peer> p1p(RR->topology->getPeer(p1));
+		const SharedPtr<Peer> p2p(RR->topology->getPeer(p2));
+		if ((!p1p)&&(!p2p)) return false;
+		if (p1p) p1p->getBestActiveAddresses(now,p1v4,p1v6);
+		if (p2p) p2p->getBestActiveAddresses(now,p2v4,p2v6);
+	}
+	if ((p1v6)&&(p2v6)) {
+		p1a = &p1v6;
+		p2a = &p2v6;
+	} else if ((p1v4)&&(p2v4)) {
+		p1a = &p1v4;
+		p2a = &p2v4;
+	} else {
+		SharedPtr<Peer> upstream(RR->topology->getUpstreamPeer());
+		if (!upstream)
+			return false;
+		upstream->getBestActiveAddresses(now,uv4,uv6);
+		if ((p1v6)&&(uv6)) {
+			p1a = &p1v6;
+			p2a = &uv6;
+		} else if ((p1v4)&&(uv4)) {
+			p1a = &p1v4;
+			p2a = &uv4;
+		} else if ((p2v6)&&(uv6)) {
+			p1a = &p2v6;
+			p2a = &uv6;
+		} else if ((p2v4)&&(uv4)) {
+			p1a = &p2v4;
+			p2a = &uv4;
+		} else return false;
+	}
+
+	TRACE("unite: %s(%s) <> %s(%s)",p1.toString().c_str(),p1a->toString().c_str(),p2.toString().c_str(),p2a->toString().c_str());
+
+	/* Tell P1 where to find P2 and vice versa, sending the packets to P1 and
+	 * P2 in randomized order in terms of which gets sent first. This is done
+	 * since in a few cases NAT-t can be sensitive to slight timing differences
+	 * in terms of when the two peers initiate. Normally this is accounted for
+	 * by the nearly-simultaneous RENDEZVOUS kickoff from the relay, but
+	 * given that relay are hosted on cloud providers this can in some
+	 * cases have a few ms of latency between packet departures. By randomizing
+	 * the order we make each attempted NAT-t favor one or the other going
+	 * first, meaning if it doesn't succeed the first time it might the second
+	 * and so forth. */
+	unsigned int alt = (unsigned int)RR->node->prng() & 1;
+	const unsigned int completed = alt + 2;
+	while (alt != completed) {
+		if ((alt & 1) == 0) {
+			// Tell p1 where to find p2.
+			Packet outp(p1,RR->identity.address(),Packet::VERB_RENDEZVOUS);
+			outp.append((unsigned char)0);
+			p2.appendTo(outp);
+			outp.append((uint16_t)p2a->port());
+			if (p2a->isV6()) {
+				outp.append((unsigned char)16);
+				outp.append(p2a->rawIpData(),16);
+			} else {
+				outp.append((unsigned char)4);
+				outp.append(p2a->rawIpData(),4);
+			}
+			send(outp,true);
+		} else {
+			// Tell p2 where to find p1.
+			Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS);
+			outp.append((unsigned char)0);
+			p1.appendTo(outp);
+			outp.append((uint16_t)p1a->port());
+			if (p1a->isV6()) {
+				outp.append((unsigned char)16);
+				outp.append(p1a->rawIpData(),16);
+			} else {
+				outp.append((unsigned char)4);
+				outp.append(p1a->rawIpData(),4);
+			}
+			send(outp,true);
+		}
+		++alt; // counts up and also flips LSB
+	}
+
+	return true;
+}
+
 } // namespace ZeroTier

node/Switch.hpp

@@ -97,17 +97,6 @@ public:
 	 */
 	void send(const Packet &packet,bool encrypt);
 
-	/**
-	 * Send RENDEZVOUS to two peers to permit them to directly connect
-	 *
-	 * This only works if both peers are known, with known working direct
-	 * links to this peer. The best link for each peer is sent to the other.
-	 *
-	 * @param p1 One of two peers (order doesn't matter)
-	 * @param p2 Second of pair
-	 */
-	bool unite(const Address &p1,const Address &p2);
-
 	/**
 	 * Request WHOIS on a given address
 	 *
@@ -138,6 +127,7 @@ public:
 private:
 	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(const Packet &packet,bool encrypt);
+	bool _unite(const Address &p1,const Address &p2);
 
 	const RuntimeEnvironment *const RR;
 	uint64_t _lastBeaconResponse;

node/Topology.cpp

@@ -23,6 +23,7 @@
 #include "Network.hpp"
 #include "NetworkConfig.hpp"
 #include "Buffer.hpp"
+#include "Switch.hpp"
 
 namespace ZeroTier {
 
@@ -111,9 +112,8 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 	{
 		Mutex::Lock _l(_lock);
 		const SharedPtr<Peer> *const ap = _peers.get(zta);
-		if (ap) {
+		if (ap)
 			return *ap;
-		}
 	}
 
 	try {
@@ -158,7 +158,7 @@ void Topology::saveIdentity(const Identity &id)
 	}
 }
 
-SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCount,bool strictAvoid)
+SharedPtr<Peer> Topology::getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid)
 {
 	const uint64_t now = RR->node->now();
 	Mutex::Lock _l(_lock);
@@ -181,30 +181,42 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 		}
 
 	} else {
-		/* If I am not a root server, the best root server is the active one with
-		 * the lowest quality score. (lower == better) */
+		/* Otherwise pick the best upstream from among roots and any other
+		 * designated upstreams that we trust. */
 
 		unsigned int bestQualityOverall = ~((unsigned int)0);
 		unsigned int bestQualityNotAvoid = ~((unsigned int)0);
 		const SharedPtr<Peer> *bestOverall = (const SharedPtr<Peer> *)0;
 		const SharedPtr<Peer> *bestNotAvoid = (const SharedPtr<Peer> *)0;
 
-		for(std::vector< SharedPtr<Peer> >::const_iterator r(_rootPeers.begin());r!=_rootPeers.end();++r) {
+		for(std::vector<Address>::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) {
+			const SharedPtr<Peer> *p = _peers.get(*a);
+
+			if (!p) {
+				const Identity id(_getIdentity(*a));
+				if (id) {
+					p = &(_peers.set(*a,SharedPtr<Peer>(new Peer(RR,RR->identity,id))));
+				} else {
+					RR->sw->requestWhois(*a);
+				}
+				continue; // always skip since even if we loaded it, it's not going to be ready
+			}
+
 			bool avoiding = false;
 			for(unsigned int i=0;i<avoidCount;++i) {
-				if (avoid[i] == (*r)->address()) {
+				if (avoid[i] == (*p)->address()) {
 					avoiding = true;
 					break;
 				}
 			}
-			const unsigned int q = (*r)->relayQuality(now);
+			const unsigned int q = (*p)->relayQuality(now);
 			if (q <= bestQualityOverall) {
 				bestQualityOverall = q;
-				bestOverall = &(*r);
+				bestOverall = &(*p);
 			}
 			if ((!avoiding)&&(q <= bestQualityNotAvoid)) {
 				bestQualityNotAvoid = q;
-				bestNotAvoid = &(*r);
+				bestNotAvoid = &(*p);
 			}
 		}
 
@@ -219,9 +231,45 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 	return SharedPtr<Peer>();
 }
 
+bool Topology::isRoot(const Identity &id) const
+{
+	Mutex::Lock _l(_lock);
+	return (std::find(_rootAddresses.begin(),_rootAddresses.end(),id.address()) != _rootAddresses.end());
+}
+
 bool Topology::isUpstream(const Identity &id) const
 {
-	return isRoot(id);
+	Mutex::Lock _l(_lock);
+	return (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),id.address()) != _upstreamAddresses.end());
+}
+
+void Topology::setUpstream(const Address &a,bool upstream)
+{
+	Mutex::Lock _l(_lock);
+	if (std::find(_rootAddresses.begin(),_rootAddresses.end(),a) == _rootAddresses.end()) {
+		if (upstream) {
+			if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) {
+				_upstreamAddresses.push_back(a);
+
+				const SharedPtr<Peer> *p = _peers.get(a);
+				if (!p) {
+					const Identity id(_getIdentity(a));
+					if (id) {
+						_peers.set(a,SharedPtr<Peer>(new Peer(RR,RR->identity,id)));
+					} else {
+						RR->sw->requestWhois(a);
+					}
+				}
+			}
+		} else {
+			std::vector<Address> ua;
+			for(std::vector<Address>::iterator i(_upstreamAddresses.begin());i!=_upstreamAddresses.end();++i) {
+				if (a != *i)
+					ua.push_back(*i);
+			}
+			_upstreamAddresses.swap(ua);
+		}
+	}
 }
 
 bool Topology::worldUpdateIfValid(const World &newWorld)
@@ -249,7 +297,7 @@ void Topology::clean(uint64_t now)
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
 		while (i.next(a,p)) {
-			if ( (!(*p)->isAlive(now)) && (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end()) )
+			if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) )
 				_peers.erase(*a);
 		}
 	}
@@ -280,25 +328,31 @@ Identity Topology::_getIdentity(const Address &zta)
 void Topology::_setWorld(const World &newWorld)
 {
 	// assumed _lock is locked (or in constructor)
+
+	std::vector<Address> ua;
+	for(std::vector<Address>::iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) {
+		if (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())
+			ua.push_back(*a);
+	}
+
 	_world = newWorld;
-	_amRoot = false;
 	_rootAddresses.clear();
-	_rootPeers.clear();
+	_amRoot = false;
+
 	for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
 		_rootAddresses.push_back(r->identity.address());
+		if (std::find(ua.begin(),ua.end(),r->identity.address()) == ua.end())
+			ua.push_back(r->identity.address());
 		if (r->identity.address() == RR->identity.address()) {
 			_amRoot = true;
 		} else {
 			SharedPtr<Peer> *rp = _peers.get(r->identity.address());
-			if (rp) {
-				_rootPeers.push_back(*rp);
-			} else {
-				SharedPtr<Peer> newrp(new Peer(RR,RR->identity,r->identity));
-				_peers.set(r->identity.address(),newrp);
-				_rootPeers.push_back(newrp);
-			}
+			if (!rp)
+				_peers.set(r->identity.address(),SharedPtr<Peer>(new Peer(RR,RR->identity,r->identity)));
 		}
 	}
+
+	_upstreamAddresses.swap(ua);
 }
 
 } // namespace ZeroTier

node/Topology.hpp

@@ -125,35 +125,27 @@ public:
 	void saveIdentity(const Identity &id);
 
 	/**
-	 * Get the current favorite root server
+	 * Get the current best upstream peer
 	 *
 	 * @return Root server with lowest latency or NULL if none
 	 */
-	inline SharedPtr<Peer> getBestRoot() { return getBestRoot((const Address *)0,0,false); }
+	inline SharedPtr<Peer> getUpstreamPeer() { return getUpstreamPeer((const Address *)0,0,false); }
 
 	/**
-	 * Get the best root server, avoiding root servers listed in an array
-	 *
-	 * This will get the best root server (lowest latency, etc.) but will
-	 * try to avoid the listed root servers, only using them if no others
-	 * are available.
+	 * Get the current best upstream peer, avoiding those in the supplied avoid list
 	 *
 	 * @param avoid Nodes to avoid
 	 * @param avoidCount Number of nodes to avoid
 	 * @param strictAvoid If false, consider avoided root servers anyway if no non-avoid root servers are available
 	 * @return Root server or NULL if none available
 	 */
-	SharedPtr<Peer> getBestRoot(const Address *avoid,unsigned int avoidCount,bool strictAvoid);
+	SharedPtr<Peer> getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid);
 
 	/**
 	 * @param id Identity to check
 	 * @return True if this is a designated root server in this world
 	 */
-	inline bool isRoot(const Identity &id) const
-	{
-		Mutex::Lock _l(_lock);
-		return (std::find(_rootAddresses.begin(),_rootAddresses.end(),id.address()) != _rootAddresses.end());
-	}
+	bool isRoot(const Identity &id) const;
 
 	/**
 	 * @param id Identity to check
@@ -162,20 +154,22 @@ public:
 	bool isUpstream(const Identity &id) const;
 
 	/**
-	 * @return Vector of root server addresses
+	 * Set whether or not an address is upstream
+	 *
+	 * If the address is a root this does nothing, since roots are fixed.
+	 *
+	 * @param a Target address
+	 * @param upstream New upstream status
 	 */
-	inline std::vector<Address> rootAddresses() const
-	{
-		Mutex::Lock _l(_lock);
-		return _rootAddresses;
-	}
+	void setUpstream(const Address &a,bool upstream);
 
 	/**
 	 * @return Vector of active upstream addresses (including roots)
 	 */
 	inline std::vector<Address> upstreamAddresses() const
 	{
-		return rootAddresses();
+		Mutex::Lock _l(_lock);
+		return _upstreamAddresses;
 	}
 
 	/**
@@ -342,9 +336,9 @@ private:
 	Hashtable< Address,SharedPtr<Peer> > _peers;
 	Hashtable< Path::HashKey,SharedPtr<Path> > _paths;
 
-	std::vector< Address > _rootAddresses;
-	std::vector< SharedPtr<Peer> > _rootPeers;
-	bool _amRoot;
+	std::vector< Address > _upstreamAddresses; // includes roots
+	std::vector< Address > _rootAddresses; // only roots
+	bool _amRoot; // am I a root?
 
 	Mutex _lock;
 };

selftest.cpp

@@ -327,6 +327,17 @@ static int testCrypto()
 	}
 	std::cout << "PASS" << std::endl;
 
+	std::cout << "[crypto] Benchmarking C25519 ECC key agreement... "; std::cout.flush();
+	C25519::Pair bp[8];
+	for(int k=0;k<8;++k)
+		bp[k] = C25519::generate();
+	const uint64_t st = OSUtils::now();
+	for(unsigned int k=0;k<50;++k) {
+		C25519::agree(bp[~k & 7],bp[k & 7].pub,buf1,64);
+	}
+	const uint64_t et = OSUtils::now();
+	std::cout << ((double)(et - st) / 50.0) << "ms per agreement." << std::endl;
+
 	std::cout << "[crypto] Testing Ed25519 ECC signatures... "; std::cout.flush();
 	C25519::Pair didntSign = C25519::generate();
 	for(unsigned int i=0;i<10;++i) {
@@ -376,11 +387,15 @@ static int testIdentity()
 		std::cout << "FAIL (1)" << std::endl;
 		return -1;
 	}
-	if (!id.locallyValidate()) {
-		std::cout << "FAIL (2)" << std::endl;
-		return -1;
+	const uint64_t vst = OSUtils::now();
+	for(int k=0;k<10;++k) {
+		if (!id.locallyValidate()) {
+			std::cout << "FAIL (2)" << std::endl;
+			return -1;
+		}
 	}
-	std::cout << "PASS" << std::endl;
+	const uint64_t vet = OSUtils::now();
+	std::cout << "PASS (" << ((double)(vet - vst) / 10.0) << "ms per validation)" << std::endl;
 
 	std::cout << "[identity] Validate known-bad identity... "; std::cout.flush();
 	if (!id.fromString(KNOWN_BAD_IDENTITY)) {