Browse Source

Refactor: move network COMs out of Network and into Peer in prep for tightening up multicast lookup and other things.

Adam Ierymenko 10 years ago
parent
commit
a3db7d0728
11 changed files with 289 additions and 215 deletions
  1. 7 0
      node/Constants.hpp
  2. 8 17
      node/IncomingPacket.cpp
  3. 6 126
      node/Network.cpp
  4. 10 35
      node/Network.hpp
  5. 10 0
      node/Node.hpp
  6. 3 3
      node/OutboundMulticast.cpp
  7. 0 8
      node/Path.hpp
  8. 181 18
      node/Peer.cpp
  9. 54 3
      node/Peer.hpp
  10. 5 3
      node/Switch.cpp
  11. 5 2
      node/Topology.cpp

+ 7 - 0
node/Constants.hpp

@@ -324,6 +324,13 @@
  */
 #define ZT_DIRECT_PATH_PUSH_INTERVAL 300000
 
+/**
+ * How long (max) to remember network certificates of membership?
+ *
+ * This only applies to networks we don't belong to.
+ */
+#define ZT_PEER_NETWORK_COM_EXPIRATION 3600000
+
 /**
  * Sanity limit on maximum bridge routes
  *

+ 8 - 17
node/IncomingPacket.cpp

@@ -421,9 +421,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					// OK(MULTICAST_FRAME) includes certificate of membership update
 					CertificateOfMembership com;
 					offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS);
-					SharedPtr<Network> network(RR->node->network(nwid));
-					if ((network)&&(com.hasRequiredFields()))
-						network->validateAndAddMembershipCertificate(com);
+					peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
 				}
 
 				if ((flags & 0x02) != 0) {
@@ -511,7 +509,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)));
 		if (network) {
 			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
-				if (!network->isAllowed(peer->address())) {
+				if (!network->isAllowed(peer)) {
 					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
 					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
@@ -552,13 +550,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				if ((flags & 0x01) != 0) {
 					CertificateOfMembership com;
 					comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
-					if (com.hasRequiredFields()) {
-						if (!network->validateAndAddMembershipCertificate(com))
-							comFailed = true; // technically this check is redundant to isAllowed(), but do it anyway for thoroughness
-					}
+					if (!peer->validateAndSetNetworkMembershipCertificate(RR,network->id(),com))
+						comFailed = true;
 				}
 
-				if ((comFailed)||(!network->isAllowed(peer->address()))) {
+				if ((comFailed)||(!network->isAllowed(peer))) {
 					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
 					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
@@ -642,11 +638,7 @@ bool IncomingPacket::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
 		while (ptr < size()) {
 			ptr += com.deserialize(*this,ptr);
-			if (com.hasRequiredFields()) {
-				SharedPtr<Network> network(RR->node->network(com.networkId()));
-				if (network)
-					network->validateAndAddMembershipCertificate(com);
-			}
+			peer->validateAndSetNetworkMembershipCertificate(RR,com.networkId(),com);
 		}
 
 		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
@@ -809,13 +801,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			if ((flags & 0x01) != 0) {
 				CertificateOfMembership com;
 				offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
-				if (com.hasRequiredFields())
-					network->validateAndAddMembershipCertificate(com);
+				peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
 			}
 
 			// Check membership after we've read any included COM, since
 			// that cert might be what we needed.
-			if (!network->isAllowed(peer->address())) {
+			if (!network->isAllowed(peer)) {
 				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
 				_sendErrorNeedCertificate(RR,peer,network->id());
 				return true;

+ 6 - 126
node/Network.cpp

@@ -59,6 +59,9 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 	Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id);
 	Utils::snprintf(mcdbn,sizeof(mcdbn),"networks.d/%.16llx.mcerts",_id);
 
+	// These files are no longer used, so clean them.
+	RR->node->dataStoreDelete(mcdbn);
+
 	if (_id == ZT_TEST_NETWORK_ID) {
 		applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address()));
 
@@ -79,24 +82,6 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 			// Save a one-byte CR to persist membership while we request a real netconf
 			RR->node->dataStorePut(confn,"\n",1,false);
 		}
-
-		try {
-			std::string mcdb(RR->node->dataStoreGet(mcdbn));
-			if (mcdb.length() > 6) {
-				const char *p = mcdb.data();
-				const char *e = p + mcdb.length();
-				if (!memcmp("ZTMCD0",p,6)) {
-					p += 6;
-					while (p != e) {
-						CertificateOfMembership com;
-						com.deserialize2(p,e);
-						if (!com)
-							break;
-						_certInfo[com.issuedTo()].com = com;
-					}
-				}
-			}
-		} catch ( ... ) {} // ignore invalid MCDB, we'll re-learn from peers
 	}
 
 	if (!_portInitialized) {
@@ -115,32 +100,10 @@ Network::~Network()
 	char n[128];
 	if (_destroyed) {
 		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
-
 		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
 		RR->node->dataStoreDelete(n);
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-		RR->node->dataStoreDelete(n);
 	} else {
 		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp);
-
-		clean();
-
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-
-		Mutex::Lock _l(_lock);
-		if ((!_config)||(_config->isPublic())||(_certInfo.empty())) {
-			RR->node->dataStoreDelete(n);
-		} else {
-			std::string buf("ZTMCD0");
-			Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-			Address *a = (Address *)0;
-			_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-			while (i.next(a,ci)) {
-				if (ci->com)
-					ci->com.serialize2(buf);
-			}
-			RR->node->dataStorePut(n,buf,true);
-		}
 	}
 }
 
@@ -281,70 +244,6 @@ void Network::requestConfiguration()
 	RR->sw->send(outp,true,0);
 }
 
-bool Network::validateAndAddMembershipCertificate(const CertificateOfMembership &cert)
-{
-	if (!cert) // sanity check
-		return false;
-
-	Mutex::Lock _l(_lock);
-
-	{
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(cert.issuedTo());
-		if ((ci)&&(ci->com == cert))
-			return true; // we already have it
-	}
-
-	// Check signature, log and return if cert is invalid
-	if (cert.signedBy() != controller()) {
-		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,cert.signedBy().toString().c_str());
-		return false; // invalid signer
-	}
-
-	if (cert.signedBy() == RR->identity.address()) {
-
-		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
-		// So, verify that we signed th cert ourself
-		if (!cert.verify(RR->identity)) {
-			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-
-	} else {
-
-		SharedPtr<Peer> signer(RR->topology->getPeer(cert.signedBy()));
-
-		if (!signer) {
-			// This would be rather odd, since this is our controller... could happen
-			// if we get packets before we've gotten config.
-			RR->sw->requestWhois(cert.signedBy());
-			return false; // signer unknown
-		}
-
-		if (!cert.verify(signer->identity())) {
-			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-	}
-
-	// If we made it past authentication, add or update cert in our cert info store
-	_certInfo[cert.issuedTo()].com = cert;
-
-	return true;
-}
-
-bool Network::peerNeedsOurMembershipCertificate(const Address &to,uint64_t now)
-{
-	Mutex::Lock _l(_lock);
-	if ((_config)&&(!_config->isPublic())&&(_config->com())) {
-		_RemoteMemberCertificateInfo &ci = _certInfo[to];
-		if ((now - ci.lastPushed) > (ZT_NETWORK_AUTOCONF_DELAY / 2)) {
-			ci.lastPushed = now;
-			return true;
-		}
-	}
-	return false;
-}
-
 void Network::clean()
 {
 	const uint64_t now = RR->node->now();
@@ -353,22 +252,6 @@ void Network::clean()
 	if (_destroyed)
 		return;
 
-	if ((_config)&&(_config->isPublic())) {
-		// Open (public) networks do not track certs or cert pushes at all.
-		_certInfo.clear();
-	} else if (_config) {
-		// Clean obsolete entries from private network cert info table
-		Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-		Address *a = (Address *)0;
-		_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-		const uint64_t forgetIfBefore = now - (ZT_PEER_ACTIVITY_TIMEOUT * 16); // arbitrary reasonable cutoff
-		while (i.next(a,ci)) {
-			if ((ci->lastPushed < forgetIfBefore)&&(!ci->com.agreesWith(_config->com())))
-				_certInfo.erase(*a);
-		}
-	}
-
-	// Clean learned multicast groups if we haven't heard from them in a while
 	{
 		Hashtable< MulticastGroup,uint64_t >::Iterator i(_multicastGroupsBehindMe);
 		MulticastGroup *mg = (MulticastGroup *)0;
@@ -494,7 +377,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	} else ec->assignedAddressCount = 0;
 }
 
-bool Network::_isAllowed(const Address &peer) const
+bool Network::_isAllowed(const SharedPtr<Peer> &peer) const
 {
 	// Assumes _lock is locked
 	try {
@@ -502,10 +385,7 @@ bool Network::_isAllowed(const Address &peer) const
 			return false;
 		if (_config->isPublic())
 			return true;
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(peer);
-		if (!ci)
-			return false;
-		return _config->com().agreesWith(ci->com);
+		return ((_config->com())&&(peer->networkMembershipCertificatesAgree(_id,_config->com())));
 	} catch (std::exception &exc) {
 		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -542,7 +422,7 @@ public:
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		if ( ( (p->hasActiveDirectPath(_now)) && ( (_network->_isAllowed(p->address())) || (p->address() == _network->controller()) ) ) || (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end()) ) {
+		if ( ( (p->hasActiveDirectPath(_now)) && ( (_network->_isAllowed(p)) || (p->address() == _network->controller()) ) ) || (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end()) ) {
 			Packet outp(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
 
 			for(std::vector<MulticastGroup>::iterator mg(_allMulticastGroups.begin());mg!=_allMulticastGroups.end();++mg) {

+ 10 - 35
node/Network.hpp

@@ -56,6 +56,7 @@ namespace ZeroTier {
 
 class RuntimeEnvironment;
 class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
+class Peer;
 
 /**
  * A virtual LAN
@@ -94,6 +95,12 @@ public:
 	 */
 	inline Address controller() throw() { return Address(_id >> 24); }
 
+	/**
+	 * @param nwid Network ID
+	 * @return Address of network's controller
+	 */
+	static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); }
+
 	/**
 	 * @return Multicast group memberships for this network's port (local, not learned via bridging)
 	 */
@@ -177,33 +184,10 @@ public:
 	void requestConfiguration();
 
 	/**
-	 * Add or update a membership certificate
-	 *
-	 * @param cert Certificate of membership
-	 * @return True if certificate was accepted as valid
-	 */
-	bool validateAndAddMembershipCertificate(const CertificateOfMembership &cert);
-
-	/**
-	 * Check if we should push membership certificate to a peer, AND update last pushed
-	 *
-	 * If we haven't pushed a cert to this peer in a long enough time, this returns
-	 * true and updates the last pushed time. Otherwise it returns false.
-	 *
-	 * This doesn't actually send anything, since COMs can hitch a ride with several
-	 * different kinds of packets.
-	 *
-	 * @param to Destination peer
-	 * @param now Current time
-	 * @return True if we should include a COM with whatever we're currently sending
-	 */
-	bool peerNeedsOurMembershipCertificate(const Address &to,uint64_t now);
-
-	/**
-	 * @param peer Peer address to check
+	 * @param peer Peer to check
 	 * @return True if peer is allowed to communicate on this network
 	 */
-	inline bool isAllowed(const Address &peer) const
+	inline bool isAllowed(const SharedPtr<Peer> &peer) const
 	{
 		Mutex::Lock _l(_lock);
 		return _isAllowed(peer);
@@ -347,16 +331,9 @@ public:
 	inline bool operator>=(const Network &n) const throw() { return (_id >= n._id); }
 
 private:
-	struct _RemoteMemberCertificateInfo
-	{
-		_RemoteMemberCertificateInfo() : com(),lastPushed(0) {}
-		CertificateOfMembership com; // remote member's COM
-		uint64_t lastPushed; // when did we last push ours to them?
-	};
-
 	ZT_VirtualNetworkStatus _status() const;
 	void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked
-	bool _isAllowed(const Address &peer) const;
+	bool _isAllowed(const SharedPtr<Peer> &peer) const;
 	void _announceMulticastGroups();
 	std::vector<MulticastGroup> _allMulticastGroups() const;
 
@@ -370,8 +347,6 @@ private:
 	Hashtable< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups that seem to be behind us and when we last saw them (if we are a bridge)
 	Hashtable< MAC,Address > _remoteBridgeRoutes; // remote addresses where given MACs are reachable (for tracking devices behind remote bridges)
 
-	Hashtable< Address,_RemoteMemberCertificateInfo > _certInfo;
-
 	SharedPtr<NetworkConfig> _config; // Most recent network configuration, which is an immutable value-object
 	volatile uint64_t _lastConfigUpdate;
 

+ 10 - 0
node/Node.hpp

@@ -168,6 +168,16 @@ public:
 		return _network(nwid);
 	}
 
+	inline bool belongsToNetwork(uint64_t nwid) const
+	{
+		Mutex::Lock _l(_networks_m);
+		for(std::vector< std::pair< uint64_t, SharedPtr<Network> > >::const_iterator i=_networks.begin();i!=_networks.end();++i) {
+			if (i->first == nwid)
+				return true;
+		}
+		return false;
+	}
+
 	inline std::vector< SharedPtr<Network> > allNetworks() const
 	{
 		std::vector< SharedPtr<Network> > nw;

+ 3 - 3
node/OutboundMulticast.cpp

@@ -103,11 +103,11 @@ void OutboundMulticast::init(
 void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr)
 {
 	if (_haveCom) {
-		SharedPtr<Network> network(RR->node->network(_nwid));
-		if ((network)&&(network->peerNeedsOurMembershipCertificate(toAddr,RR->node->now()))) {
+		SharedPtr<Peer> peer(RR->topology->getPeer(toAddr));
+		if ( (!peer) || (peer->needsOurNetworkMembershipCertificate(_nwid,RR->node->now(),true)) ) {
+			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			_packetWithCom.newInitializationVector();
 			_packetWithCom.setDestination(toAddr);
-			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			RR->sw->send(_packetWithCom,true,_nwid);
 			return;
 		}

+ 0 - 8
node/Path.hpp

@@ -122,14 +122,6 @@ public:
 	 */
 	inline operator bool() const throw() { return (_addr); }
 
-	// Comparisons are by address only
-	inline bool operator==(const Path &p) const throw() { return (_addr == p._addr); }
-	inline bool operator!=(const Path &p) const throw() { return (_addr != p._addr); }
-	inline bool operator<(const Path &p) const throw() { return (_addr < p._addr); }
-	inline bool operator>(const Path &p) const throw() { return (_addr > p._addr); }
-	inline bool operator<=(const Path &p) const throw() { return (_addr <= p._addr); }
-	inline bool operator>=(const Path &p) const throw() { return (_addr >= p._addr); }
-
 	/**
 	 * Check whether this address is valid for a ZeroTier path
 	 *

+ 181 - 18
node/Peer.cpp

@@ -37,6 +37,8 @@
 
 #include <algorithm>
 
+#define ZT_PEER_PATH_SORT_INTERVAL 5000
+
 namespace ZeroTier {
 
 // Used to send varying values for NAT keepalive
@@ -51,17 +53,38 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastAnnouncedTo(0),
 	_lastPathConfirmationSent(0),
 	_lastDirectPathPush(0),
+	_lastPathSort(0),
 	_vMajor(0),
 	_vMinor(0),
 	_vRevision(0),
 	_id(peerIdentity),
 	_numPaths(0),
-	_latency(0)
+	_latency(0),
+	_networkComs(4),
+	_lastPushedComs(4)
 {
 	if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
 		throw std::runtime_error("new peer identity key agreement failed");
 }
 
+struct _SortPathsByQuality
+{
+	uint64_t _now;
+	_SortPathsByQuality(const uint64_t now) : _now(now) {}
+	inline bool operator()(const RemotePath &a,const RemotePath &b) const
+	{
+		const uint64_t qa = (
+			((uint64_t)a.active(_now) << 63) |
+			(((uint64_t)(a.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)a.lastReceived() & 0x7ffffffffffffULL) );
+		const uint64_t qb = (
+			((uint64_t)b.active(_now) << 63) |
+			(((uint64_t)(b.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)b.lastReceived() & 0x7ffffffffffffULL) );
+		return (qb < qa); // invert sense to sort in descending order
+	}
+};
+
 void Peer::received(
 	const RuntimeEnvironment *RR,
 	const InetAddress &localAddr,
@@ -73,6 +96,8 @@ void Peer::received(
 	Packet::Verb inReVerb)
 {
 	const uint64_t now = RR->node->now();
+	Mutex::Lock _l(_lock);
+
 	_lastReceive = now;
 
 	if (!hops) {
@@ -91,6 +116,7 @@ void Peer::received(
 
 			if (!pathIsConfirmed) {
 				if ((verb == Packet::VERB_OK)&&(inReVerb == Packet::VERB_HELLO)) {
+
 					// Learn paths if they've been confirmed via a HELLO
 					RemotePath *slot = (RemotePath *)0;
 					if (np < ZT_MAX_PEER_NETWORK_PATHS) {
@@ -111,8 +137,11 @@ void Peer::received(
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
+						_sortPaths(now);
 					}
+
 				} else {
+
 					/* If this path is not known, send a HELLO. We don't learn
 					 * paths without confirming that a bidirectional link is in
 					 * fact present, but any packet that decodes and authenticates
@@ -122,6 +151,7 @@ void Peer::received(
 						TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
 						attemptToContactAt(RR,localAddr,remoteAddr,now);
 					}
+
 				}
 			}
 		}
@@ -129,6 +159,7 @@ void Peer::received(
 		/* Announce multicast groups of interest to direct peers if they are
 		 * considered authorized members of a given network. Also announce to
 		 * root servers and network controllers. */
+		/*
 		if ((pathIsConfirmed)&&((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000))) {
 			_lastAnnouncedTo = now;
 
@@ -158,6 +189,7 @@ void Peer::received(
 				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 			}
 		}
+		*/
 	}
 
 	if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
@@ -166,23 +198,10 @@ void Peer::received(
 		_lastMulticastFrame = now;
 }
 
-RemotePath *Peer::getBestPath(uint64_t now)
-{
-	RemotePath *bestPath = (RemotePath *)0;
-	uint64_t lrMax = 0;
-	int rank = 0;
-	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		if ( (_paths[p].active(now)) && ((_paths[p].lastReceived() >= lrMax)||(_paths[p].preferenceRank() >= rank)) ) {
-			lrMax = _paths[p].lastReceived();
-			rank = _paths[p].preferenceRank();
-			bestPath = &(_paths[p]);
-		}
-	}
-	return bestPath;
-}
-
 void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now)
 {
+	// _lock not required here since _id is immutable and nothing else is accessed
+
 	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
 	outp.append((unsigned char)ZT_PROTO_VERSION);
 	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
@@ -214,7 +233,8 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &lo
 
 void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 {
-	RemotePath *const bestPath = getBestPath(now);
+	Mutex::Lock _l(_lock);
+	RemotePath *const bestPath = _getBestPath(now);
 	if (bestPath) {
 		if ((now - bestPath->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
 			TRACE("PING %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
@@ -231,6 +251,8 @@ void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 
 void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
 {
+	Mutex::Lock _l(_lock);
+
 	if (((now - _lastDirectPathPush) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
 		_lastDirectPathPush = now;
 
@@ -299,13 +321,16 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	}
 }
 
-void Peer::addPath(const RemotePath &newp)
+void Peer::addPath(const RemotePath &newp,uint64_t now)
 {
+	Mutex::Lock _l(_lock);
+
 	unsigned int np = _numPaths;
 
 	for(unsigned int p=0;p<np;++p) {
 		if (_paths[p].address() == newp.address()) {
 			_paths[p].setFixed(newp.fixed());
+			_sortPaths(now);
 			return;
 		}
 	}
@@ -328,6 +353,8 @@ void Peer::addPath(const RemotePath &newp)
 		*slot = newp;
 		_numPaths = np;
 	}
+
+	_sortPaths(now);
 }
 
 void Peer::clearPaths(bool fixedToo)
@@ -349,6 +376,7 @@ void Peer::clearPaths(bool fixedToo)
 
 bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope scope,uint64_t now)
 {
+	Mutex::Lock _l(_lock);
 	unsigned int np = _numPaths;
 	unsigned int x = 0;
 	unsigned int y = 0;
@@ -364,11 +392,13 @@ bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope sc
 		++x;
 	}
 	_numPaths = y;
+	_sortPaths(now);
 	return (y < np);
 }
 
 void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const
 {
+	Mutex::Lock _l(_lock);
 	uint64_t bestV4 = 0,bestV6 = 0;
 	for(unsigned int p=0,np=_numPaths;p<np;++p) {
 		if (_paths[p].active(now)) {
@@ -390,4 +420,137 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6)
 	}
 }
 
+bool Peer::networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const
+{
+	Mutex::Lock _l(_lock);
+	const _NetworkCom *ourCom = _networkComs.get(nwid);
+	if (ourCom)
+		return ourCom->com.agreesWith(com);
+	return false;
+}
+
+bool Peer::validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com)
+{
+	// Sanity checks
+	if ((!com)||(com.issuedTo() != _id.address()))
+		return false;
+
+	// Return true if we already have this *exact* COM
+	{
+		Mutex::Lock _l(_lock);
+		_NetworkCom *ourCom = _networkComs.get(nwid);
+		if ((ourCom)&&(ourCom->com == com))
+			return true;
+	}
+
+	// Check signature, log and return if cert is invalid
+	if (com.signedBy() != Network::controllerFor(nwid)) {
+		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,cert.signedBy().toString().c_str());
+		return false; // invalid signer
+	}
+
+	if (com.signedBy() == RR->identity.address()) {
+
+		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
+		// So, verify that we signed th cert ourself
+		if (!com.verify(RR->identity)) {
+			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+
+	} else {
+
+		SharedPtr<Peer> signer(RR->topology->getPeer(com.signedBy()));
+
+		if (!signer) {
+			// This would be rather odd, since this is our controller... could happen
+			// if we get packets before we've gotten config.
+			RR->sw->requestWhois(com.signedBy());
+			return false; // signer unknown
+		}
+
+		if (!com.verify(signer->identity())) {
+			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+	}
+
+	// If we made it past all those checks, add or update cert in our cert info store
+	{
+		Mutex::Lock _l(_lock);
+		_networkComs.set(nwid,_NetworkCom(RR->node->now(),com));
+	}
+
+	return true;
+}
+
+bool Peer::needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime)
+{
+	Mutex::Lock _l(_lock);
+	uint64_t &lastPushed = _lastPushedComs[nwid];
+	const uint64_t tmp = lastPushed;
+	if (updateLastPushedTime)
+		lastPushed = now;
+	return ((now - tmp) < (ZT_NETWORK_AUTOCONF_DELAY / 2));
+}
+
+void Peer::clean(const RuntimeEnvironment *RR,uint64_t now)
+{
+	Mutex::Lock _l(_lock);
+
+	{
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if (_paths[x].active(now))
+				_paths[y++] = _paths[x];
+			++x;
+		}
+		_numPaths = y;
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		_NetworkCom *v = (_NetworkCom *)0;
+		Hashtable< uint64_t,_NetworkCom >::Iterator i(_networkComs);
+		while (i.next(k,v)) {
+			if ( (!RR->node->belongsToNetwork(*k)) && ((now - v->ts) >= ZT_PEER_NETWORK_COM_EXPIRATION) )
+				_networkComs.erase(*k);
+		}
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		uint64_t *v = (uint64_t *)0;
+		Hashtable< uint64_t,uint64_t >::Iterator i(_lastPushedComs);
+		while (i.next(k,v)) {
+			if ((now - *v) > (ZT_NETWORK_AUTOCONF_DELAY * 2))
+				_lastPushedComs.erase(*k);
+		}
+	}
+}
+
+void Peer::_sortPaths(const uint64_t now)
+{
+	// assumes _lock is locked
+	_lastPathSort = now;
+	std::sort(&(_paths[0]),&(_paths[_numPaths]),_SortPathsByQuality(now));
+}
+
+RemotePath *Peer::_getBestPath(const uint64_t now)
+{
+	// assumes _lock is locked
+	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
+		_sortPaths(now);
+	if (_paths[0].active(now)) {
+		return &(_paths[0]);
+	} else {
+		_sortPaths(now);
+		if (_paths[0].active(now))
+			return &(_paths[0]);
+	}
+	return (RemotePath *)0;
+}
+
 } // namespace ZeroTier

+ 54 - 3
node/Peer.hpp

@@ -49,6 +49,8 @@
 #include "Packet.hpp"
 #include "SharedPtr.hpp"
 #include "AtomicCounter.hpp"
+#include "Hashtable.hpp"
+#include "Mutex.hpp"
 #include "NonCopyable.hpp"
 
 namespace ZeroTier {
@@ -129,7 +131,11 @@ public:
 	 * @param now Current time
 	 * @return Best path or NULL if there are no active (or fixed) direct paths
 	 */
-	RemotePath *getBestPath(uint64_t now);
+	inline RemotePath *getBestPath(uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		return _getBestPath(now);
+	}
 
 	/**
 	 * Send via best path
@@ -293,8 +299,9 @@ public:
 	 * Add a path (if we don't already have it)
 	 *
 	 * @param p New path to add
+	 * @param now Current time
 	 */
-	void addPath(const RemotePath &newp);
+	void addPath(const RemotePath &newp,uint64_t now);
 
 	/**
 	 * Clear paths
@@ -381,6 +388,37 @@ public:
 	 */
 	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
 
+	/**
+	 * Check network COM agreement with this peer
+	 *
+	 * @param nwid Network ID
+	 * @param com Another certificate of membership
+	 * @return True if supplied COM agrees with ours, false if not or if we don't have one
+	 */
+	bool networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const;
+
+	/**
+	 * Check the validity of the COM and add/update if valid and new
+	 *
+	 * @param RR Runtime Environment
+	 * @param nwid Network ID
+	 * @param com Externally supplied COM
+	 */
+	bool validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com);
+
+	/**
+	 * @param nwid Network ID
+	 * @param now Current time
+	 * @param updateLastPushedTime If true, go ahead and update the last pushed time regardless of return value
+	 * @return Whether or not this peer needs another COM push from us
+	 */
+	bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime);
+
+	/**
+	 * Perform periodic cleaning operations
+	 */
+	void clean(const RuntimeEnvironment *RR,uint64_t now);
+
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
 	 *
@@ -402,7 +440,8 @@ public:
 	}
 
 private:
-	void _announceMulticastGroups(const RuntimeEnvironment *RR,uint64_t now);
+	void _sortPaths(const uint64_t now);
+	RemotePath *_getBestPath(const uint64_t now);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	uint64_t _lastUsed;
@@ -412,6 +451,7 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastPathConfirmationSent;
 	uint64_t _lastDirectPathPush;
+	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
@@ -421,6 +461,17 @@ private:
 	unsigned int _numPaths;
 	unsigned int _latency;
 
+	struct _NetworkCom
+	{
+		_NetworkCom() {}
+		_NetworkCom(uint64_t t,const CertificateOfMembership &c) : ts(t),com(c) {}
+		uint64_t ts;
+		CertificateOfMembership com;
+	};
+	Hashtable<uint64_t,_NetworkCom> _networkComs;
+	Hashtable<uint64_t,uint64_t> _lastPushedComs;
+
+	Mutex _lock;
 	AtomicCounter __refCount;
 };
 

+ 5 - 3
node/Switch.cpp

@@ -202,7 +202,8 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		// Destination is another ZeroTier peer on the same network
 
 		Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this
-		const bool includeCom = network->peerNeedsOurMembershipCertificate(toZT,RR->node->now());
+		SharedPtr<Peer> toPeer(RR->topology->getPeer(toZT));
+		const bool includeCom = ((!toPeer)||(toPeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true)));;
 		if ((fromBridged)||(includeCom)) {
 			Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
@@ -267,9 +268,10 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		}
 
 		for(unsigned int b=0;b<numBridges;++b) {
+			SharedPtr<Peer> bridgePeer(RR->topology->getPeer(bridges[b]));
 			Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
-			if (network->peerNeedsOurMembershipCertificate(bridges[b],RR->node->now())) {
+			if ((!bridgePeer)||(bridgePeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) {
 				outp.append((unsigned char)0x01); // 0x01 -- COM included
 				nconf->com().serialize(outp);
 			} else {
@@ -747,7 +749,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // no paths, no root servers?
 		}
 
-		if ((network)&&(relay)&&(network->isAllowed(peer->address()))) {
+		if ((network)&&(relay)&&(network->isAllowed(peer))) {
 			// Push hints for direct connectivity to this peer if we are relaying
 			peer->pushDirectPaths(RR,viaPath,now,false);
 		}

+ 5 - 2
node/Topology.cpp

@@ -62,7 +62,7 @@ void Topology::setRootServers(const std::map< Identity,std::vector<InetAddress>
 			if (!p)
 				p = SharedPtr<Peer>(new Peer(RR->identity,i->first));
 			for(std::vector<InetAddress>::const_iterator j(i->second.begin());j!=i->second.end();++j)
-				p->addPath(RemotePath(InetAddress(),*j,true));
+				p->addPath(RemotePath(InetAddress(),*j,true),now);
 			p->use(now);
 			_rootPeers.push_back(p);
 		}
@@ -252,9 +252,12 @@ void Topology::clean(uint64_t now)
 	Hashtable< Address,SharedPtr<Peer> >::Iterator i(_activePeers);
 	Address *a = (Address *)0;
 	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
-	while (i.next(a,p))
+	while (i.next(a,p)) {
 		if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) {
 			_activePeers.erase(*a);
+		} else {
+			(*p)->clean(RR,now);
+		}
 	}
 }