Rest of big Path canonicalization refactor.

node/Constants.hpp

@@ -238,46 +238,38 @@
  */
 #define ZT_MULTICAST_TRANSMIT_TIMEOUT 5000
 
-/**
- * Default maximum number of peers to address with a single multicast (if unspecified in network config)
- */
-#define ZT_MULTICAST_DEFAULT_LIMIT 32
-
-/**
- * How frequently to send a zero-byte UDP keepalive packet
- *
- * There are NATs with timeouts as short as 20 seconds, so this turns out
- * to be needed.
- */
-#define ZT_NAT_KEEPALIVE_DELAY 19000
-
 /**
  * Delay between scans of the topology active peer DB for peers that need ping
  *
  * This is also how often pings will be retried to upstream peers (relays, roots)
  * constantly until something is heard.
  */
-#define ZT_PING_CHECK_INVERVAL 9500
+#define ZT_PING_CHECK_INVERVAL 8000
 
 /**
- * Delay between ordinary case pings of direct links
+ * How frequently to send heartbeats over in-use paths
  */
-#define ZT_PEER_DIRECT_PING_DELAY 60000
+#define ZT_PATH_HEARTBEAT_PERIOD 18000
 
 /**
- * Timeout for overall peer activity (measured from last receive)
+ * Paths are considered inactive if they have not received traffic in this long
  */
-#define ZT_PEER_ACTIVITY_TIMEOUT 500000
+#define ZT_PATH_ALIVE_TIMEOUT ((ZT_PATH_HEARTBEAT_PERIOD * 2) + 2000)
 
 /**
- * No answer timeout to trigger dead path detection
+ * Delay between full-fledge pings of directly connected peers
  */
-#define ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT 2000
+#define ZT_PEER_PING_PERIOD 60000
 
 /**
- * Probation threshold after which a path becomes dead
+ * Peers forget paths that have not spoken in this long
  */
-#define ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION 3
+#define ZT_PEER_PATH_EXPIRATION ((ZT_PEER_PING_PERIOD * 3) + 3000)
+
+/**
+ * Timeout for overall peer activity (measured from last receive)
+ */
+#define ZT_PEER_ACTIVITY_TIMEOUT 500000
 
 /**
  * Delay between requests for updated network autoconf information

node/IncomingPacket.cpp

@@ -54,11 +54,11 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 			// If this is marked as a packet via a trusted path, check source address and path ID.
 			// Obviously if no trusted paths are configured this always returns false and such
 			// packets are dropped on the floor.
-			if (RR->topology->shouldInboundPathBeTrusted(_remoteAddress,trustedPathId())) {
+			if (RR->topology->shouldInboundPathBeTrusted(_path->address(),trustedPathId())) {
 				trusted = true;
-				TRACE("TRUSTED PATH packet approved from %s(%s), trusted path ID %llx",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str(),trustedPathId());
+				TRACE("TRUSTED PATH packet approved from %s(%s), trusted path ID %llx",sourceAddress.toString().c_str(),_path->address().toString().c_str(),trustedPathId());
 			} else {
-				TRACE("dropped packet from %s(%s), cipher set to trusted path mode but path %llx@%s is not trusted!",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str(),trustedPathId(),_remoteAddress.toString().c_str());
+				TRACE("dropped packet from %s(%s), cipher set to trusted path mode but path %llx@%s is not trusted!",sourceAddress.toString().c_str(),_path->address().toString().c_str(),trustedPathId(),_path->address().toString().c_str());
 				return true;
 			}
 		} else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) {
@@ -73,42 +73,42 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 		if (peer) {
 			if (!trusted) {
 				if (!dearmor(peer->key())) {
-					TRACE("dropped packet from %s(%s), MAC authentication failed (size: %u)",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str(),size());
+					TRACE("dropped packet from %s(%s), MAC authentication failed (size: %u)",sourceAddress.toString().c_str(),_path->address().toString().c_str(),size());
 					return true;
 				}
 			}
 
 			if (!uncompress()) {
-				TRACE("dropped packet from %s(%s), compressed data invalid",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str());
+				TRACE("dropped packet from %s(%s), compressed data invalid",sourceAddress.toString().c_str(),_path->address().toString().c_str());
 				return true;
 			}
 
 			const Packet::Verb v = verb();
-			//TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_remoteAddress.toString().c_str());
+			//TRACE("<< %s from %s(%s)",Packet::verbString(v),sourceAddress.toString().c_str(),_path->address().toString().c_str());
 			switch(v) {
 				//case Packet::VERB_NOP:
 				default: // ignore unknown verbs, but if they pass auth check they are "received"
-					peer->received(_localAddress,_remoteAddress,hops(),packetId(),v,0,Packet::VERB_NOP,false);
+					peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false);
 					return true;
 
-				case Packet::VERB_HELLO:                          return _doHELLO(RR,peer);
-				case Packet::VERB_ERROR:                          return _doERROR(RR,peer);
-				case Packet::VERB_OK:                             return _doOK(RR,peer);
-				case Packet::VERB_WHOIS:                          return _doWHOIS(RR,peer);
-				case Packet::VERB_RENDEZVOUS:                     return _doRENDEZVOUS(RR,peer);
-				case Packet::VERB_FRAME:                          return _doFRAME(RR,peer);
-				case Packet::VERB_EXT_FRAME:                      return _doEXT_FRAME(RR,peer);
-				case Packet::VERB_ECHO:                           return _doECHO(RR,peer);
-				case Packet::VERB_MULTICAST_LIKE:                 return _doMULTICAST_LIKE(RR,peer);
-				case Packet::VERB_NETWORK_CREDENTIALS:            return _doNETWORK_CREDENTIALS(RR,peer);
-				case Packet::VERB_NETWORK_CONFIG_REQUEST:         return _doNETWORK_CONFIG_REQUEST(RR,peer);
-				case Packet::VERB_NETWORK_CONFIG_REFRESH:         return _doNETWORK_CONFIG_REFRESH(RR,peer);
-				case Packet::VERB_MULTICAST_GATHER:               return _doMULTICAST_GATHER(RR,peer);
-				case Packet::VERB_MULTICAST_FRAME:                return _doMULTICAST_FRAME(RR,peer);
-				case Packet::VERB_PUSH_DIRECT_PATHS:              return _doPUSH_DIRECT_PATHS(RR,peer);
-				case Packet::VERB_CIRCUIT_TEST:                   return _doCIRCUIT_TEST(RR,peer);
-				case Packet::VERB_CIRCUIT_TEST_REPORT:            return _doCIRCUIT_TEST_REPORT(RR,peer);
-				case Packet::VERB_REQUEST_PROOF_OF_WORK:          return _doREQUEST_PROOF_OF_WORK(RR,peer);
+				case Packet::VERB_HELLO:                      return _doHELLO(RR,peer);
+				case Packet::VERB_ERROR:                      return _doERROR(RR,peer);
+				case Packet::VERB_OK:                         return _doOK(RR,peer);
+				case Packet::VERB_WHOIS:                      return _doWHOIS(RR,peer);
+				case Packet::VERB_RENDEZVOUS:                 return _doRENDEZVOUS(RR,peer);
+				case Packet::VERB_FRAME:                      return _doFRAME(RR,peer);
+				case Packet::VERB_EXT_FRAME:                  return _doEXT_FRAME(RR,peer);
+				case Packet::VERB_ECHO:                       return _doECHO(RR,peer);
+				case Packet::VERB_MULTICAST_LIKE:             return _doMULTICAST_LIKE(RR,peer);
+				case Packet::VERB_NETWORK_CREDENTIALS:        return _doNETWORK_CREDENTIALS(RR,peer);
+				case Packet::VERB_NETWORK_CONFIG_REQUEST:     return _doNETWORK_CONFIG_REQUEST(RR,peer);
+				case Packet::VERB_NETWORK_CONFIG_REFRESH:     return _doNETWORK_CONFIG_REFRESH(RR,peer);
+				case Packet::VERB_MULTICAST_GATHER:           return _doMULTICAST_GATHER(RR,peer);
+				case Packet::VERB_MULTICAST_FRAME:            return _doMULTICAST_FRAME(RR,peer);
+				case Packet::VERB_PUSH_DIRECT_PATHS:          return _doPUSH_DIRECT_PATHS(RR,peer);
+				case Packet::VERB_CIRCUIT_TEST:               return _doCIRCUIT_TEST(RR,peer);
+				case Packet::VERB_CIRCUIT_TEST_REPORT:        return _doCIRCUIT_TEST_REPORT(RR,peer);
+				case Packet::VERB_REQUEST_PROOF_OF_WORK:      return _doREQUEST_PROOF_OF_WORK(RR,peer);
 				case Packet::VERB_USER_MESSAGE:
 					return true;
 			}
@@ -119,7 +119,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 	} catch ( ... ) {
 		// Exceptions are more informatively caught in _do...() handlers but
 		// this outer try/catch will catch anything else odd.
-		TRACE("dropped ??? from %s(%s): unexpected exception in tryDecode()",sourceAddress.toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped ??? from %s(%s): unexpected exception in tryDecode()",sourceAddress.toString().c_str(),_path->address().toString().c_str());
 		return true;
 	}
 }
@@ -131,7 +131,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		const uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_IN_RE_PACKET_ID);
 		const Packet::ErrorCode errorCode = (Packet::ErrorCode)(*this)[ZT_PROTO_VERB_ERROR_IDX_ERROR_CODE];
 
-		//TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
+		//TRACE("ERROR %s from %s(%s) in-re %s",Packet::errorString(errorCode),peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb));
 
 		switch(errorCode) {
 
@@ -172,9 +172,9 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 			default: break;
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb,false);
 	} catch ( ... ) {
-		TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -216,11 +216,11 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		}
 
 		if (protoVersion < ZT_PROTO_VERSION_MIN) {
-			TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_remoteAddress.toString().c_str());
+			TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str());
 			return true;
 		}
 		if (fromAddress != id.address()) {
-			TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_remoteAddress.toString().c_str());
+			TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_path->address().toString().c_str());
 			return true;
 		}
 
@@ -235,18 +235,18 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 					unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
 					if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
 						if (dearmor(key)) { // ensure packet is authentic, otherwise drop
-							TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_remoteAddress.toString().c_str());
+							TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_path->address().toString().c_str());
 							Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR);
 							outp.append((unsigned char)Packet::VERB_HELLO);
 							outp.append((uint64_t)pid);
 							outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
 							outp.armor(key,true);
-							RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+							_path->send(RR,outp.data(),outp.size(),RR->node->now());
 						} else {
-							TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
+							TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str());
 						}
 					} else {
-						TRACE("rejected HELLO from %s(%s): key agreement failed",id.address().toString().c_str(),_remoteAddress.toString().c_str());
+						TRACE("rejected HELLO from %s(%s): key agreement failed",id.address().toString().c_str(),_path->address().toString().c_str());
 					}
 
 					return true;
@@ -254,7 +254,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 					// Identity is the same as the one we already have -- check packet integrity
 
 					if (!dearmor(peer->key())) {
-						TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
+						TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str());
 						return true;
 					}
 
@@ -265,14 +265,14 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 
 				// Check identity proof of work
 				if (!id.locallyValidate()) {
-					TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_remoteAddress.toString().c_str());
+					TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str());
 					return true;
 				}
 
 				// Check packet integrity and authentication
 				SharedPtr<Peer> newPeer(new Peer(RR,RR->identity,id));
 				if (!dearmor(newPeer->key())) {
-					TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
+					TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str());
 					return true;
 				}
 				peer = RR->topology->addPeer(newPeer);
@@ -284,7 +284,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		}
 
 		if (externalSurfaceAddress)
-			RR->sa->iam(id.address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now());
+			RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now());
 
 		Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_HELLO);
@@ -295,7 +295,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
 		outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
 		if (protoVersion >= 5) {
-			_remoteAddress.serialize(outp);
+			_path->address().serialize(outp);
 		} else {
 			/* LEGACY COMPATIBILITY HACK:
 			 *
@@ -320,7 +320,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 			 * nulling out the port field. Since this info is only used for empirical
 			 * detection of link changes, it doesn't break anything else.
 			 */
-			InetAddress tmpa(_remoteAddress);
+			InetAddress tmpa(_path->address());
 			tmpa.setPort(0);
 			tmpa.serialize(outp);
 		}
@@ -336,12 +336,12 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		}
 
 		outp.armor(peer->key(),true);
-		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+		_path->send(RR,outp.data(),outp.size(),RR->node->now());
 
 		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version
-		peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -352,7 +352,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 		const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB];
 		const uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID);
 
-		//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_remoteAddress.toString().c_str(),Packet::verbString(inReVerb));
+		//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb));
 
 		switch(inReVerb) {
 
@@ -364,7 +364,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				const unsigned int vRevision = at<uint16_t>(ZT_PROTO_VERB_HELLO__OK__IDX_REVISION);
 
 				if (vProto < ZT_PROTO_VERSION_MIN) {
-					TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_remoteAddress.toString().c_str());
+					TRACE("%s(%s): OK(HELLO) dropped, protocol version too old",source().toString().c_str(),_path->address().toString().c_str());
 					return true;
 				}
 
@@ -386,13 +386,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					}
 				}
 
-				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_remoteAddress.toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)"));
+				TRACE("%s(%s): OK(HELLO), version %u.%u.%u, latency %u, reported external address %s",source().toString().c_str(),_path->address().toString().c_str(),vMajor,vMinor,vRevision,latency,((externalSurfaceAddress) ? externalSurfaceAddress.toString().c_str() : "(none)"));
 
 				peer->addDirectLatencyMeasurment(latency);
 				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
 				if (externalSurfaceAddress)
-					RR->sa->iam(peer->address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now());
+					RR->sa->iam(peer->address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now());
 			}	break;
 
 			case Packet::VERB_WHOIS: {
@@ -414,7 +414,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 						totalSize = at<uint32_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen);
 						chunkIndex = at<uint32_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT + chunkLen + 4);
 					}
-					TRACE("%s(%s): OK(NETWORK_CONFIG_REQUEST) chunkLen==%u chunkIndex==%u totalSize==%u",source().toString().c_str(),_remoteAddress.toString().c_str(),chunkLen,chunkIndex,totalSize);
+					TRACE("%s(%s): OK(NETWORK_CONFIG_REQUEST) chunkLen==%u chunkIndex==%u totalSize==%u",source().toString().c_str(),_path->address().toString().c_str(),chunkLen,chunkIndex,totalSize);
 					network->handleInboundConfigChunk(inRePacketId,chunkData,chunkLen,chunkIndex,totalSize);
 				}
 			}	break;
@@ -425,7 +425,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 			case Packet::VERB_MULTICAST_GATHER: {
 				const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID);
 				const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI));
-				//TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,mg.toString().c_str(),size());
+				//TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size());
 				const unsigned int count = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4);
 				RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS));
 			}	break;
@@ -435,7 +435,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID);
 				const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI));
 
-				//TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),nwid,mg.toString().c_str(),flags);
+				//TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags);
 
 				unsigned int offset = 0;
 
@@ -461,9 +461,9 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 			default: break;
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false);
 	} catch ( ... ) {
-		TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -502,12 +502,12 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 
 		if (count > 0) {
 			outp.armor(peer->key(),true);
-			RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+			_path->send(RR,outp.data(),outp.size(),RR->node->now());
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -524,24 +524,22 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<
 				const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
 				if (!RR->topology->isUpstream(peer->identity())) {
 					TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
-				} else if (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,atAddr)) {
-					const uint64_t now = RR->node->now();
-					peer->sendHELLO(_localAddress,atAddr,now,2); // send low-TTL packet to 'open' local NAT(s)
-					peer->sendHELLO(_localAddress,atAddr,now);
+				} else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) {
+					RR->node->putPacket(_path->localAddress(),atAddr,"NATSUX",6,2); // send low-TTL packet to 'open' local NAT(s)
+					peer->sendHELLO(_path->localAddress(),atAddr,RR->node->now());
 					TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
 				} else {
 					TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
 				}
 			} else {
-				TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+				TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str());
 			}
 		} else {
-			TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),with.toString().c_str());
+			TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str());
 		}
-
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped RENDEZVOUS from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -553,8 +551,8 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		if (network) {
 			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 				if (!network->isAllowed(peer)) {
-					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
-					peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,false);
+					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
+					peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,false);
 				} else {
 					const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
 					const MAC sourceMac(peer->address(),network->id());
@@ -562,14 +560,14 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 					const uint8_t *const frameData = reinterpret_cast<const uint8_t *>(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 					if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0)
 						RR->node->putFrame(network->id(),network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen);
-					peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true);
+					peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,true);
 				}
 			}
 		} else {
-			TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
+			TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
 	} catch ( ... ) {
-		TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -591,8 +589,8 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				}
 
 				if (!network->isAllowed(peer)) {
-					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
-					peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false);
+					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id());
+					peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false);
 					return true;
 				}
 
@@ -603,8 +601,8 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen);
 
 				if ((!from)||(from.isMulticast())||(from == network->mac())) {
-					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
-					peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
+					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str());
+					peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 					return true;
 				}
 
@@ -614,14 +612,14 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 							if (network->config().permitsBridging(peer->address())) {
 								network->learnBridgeRoute(from,peer->address());
 							} else {
-								TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id());
-								peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
+								TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id());
+								peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 								return true;
 							}
 						} else if (to != network->mac()) {
 							if (!network->config().permitsBridging(RR->identity.address())) {
-								TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id());
-								peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
+								TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id());
+								peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 								return true;
 							}
 						}
@@ -631,13 +629,13 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 						break;
 				}
 
-				peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true);
+				peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true);
 			}
 		} else {
-			TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
+			TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
 	} catch ( ... ) {
-		TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped EXT_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -652,10 +650,10 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		if (size() > ZT_PACKET_IDX_PAYLOAD)
 			outp.append(reinterpret_cast<const unsigned char *>(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD);
 		outp.armor(peer->key(),true);
-		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
-		peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false);
+		_path->send(RR,outp.data(),outp.size(),RR->node->now());
+		peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -672,9 +670,9 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 			RR->mc->add(now,nwid,group,peer->address());
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -721,9 +719,9 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S
 			}
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -744,7 +742,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 		if (RR->localNetworkController) {
 			NetworkConfig *netconf = new NetworkConfig();
 			try {
-				switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _remoteAddress,RR->identity,peer->identity(),nwid,metaData,*netconf)) {
+				switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _path->address(),RR->identity,peer->identity(),nwid,metaData,*netconf)) {
 
 					case NetworkController::NETCONF_QUERY_OK: {
 						netconfOk = true;
@@ -783,7 +781,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 						outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 						outp.append(nwid);
 						outp.armor(peer->key(),true);
-						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+						_path->send(RR,outp.data(),outp.size(),RR->node->now());
 					}	break;
 
 					case NetworkController::NETCONF_QUERY_ACCESS_DENIED: {
@@ -793,7 +791,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 						outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 						outp.append(nwid);
 						outp.armor(peer->key(),true);
-						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+						_path->send(RR,outp.data(),outp.size(),RR->node->now());
 					} break;
 
 					case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR:
@@ -816,16 +814,16 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
 			outp.append(nwid);
 			outp.armor(peer->key(),true);
-			RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+			_path->send(RR,outp.data(),outp.size(),RR->node->now());
 		}
 
-		peer->received(_localAddress,_remoteAddress,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk);
+		peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk);
 	} catch (std::exception &exc) {
 		fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what());
-		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
+		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what());
 	} catch ( ... ) {
 		fprintf(stderr,"WARNING: network config request failed with exception: unknown exception" ZT_EOL_S);
-		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unknown exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -840,8 +838,8 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 			if (network) {
 				network->requestConfiguration();
 			} else {
-				TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid);
-				peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false);
+				TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid);
+				peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false);
 				return true;
 			}
 
@@ -853,9 +851,9 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 			}
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -868,7 +866,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 		const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI));
 		const unsigned int gatherLimit = at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT);
 
-		//TRACE("<<MC %s(%s) GATHER up to %u in %.16llx/%s",source().toString().c_str(),_remoteAddress.toString().c_str(),gatherLimit,nwid,mg.toString().c_str());
+		//TRACE("<<MC %s(%s) GATHER up to %u in %.16llx/%s",source().toString().c_str(),_path->address().toString().c_str(),gatherLimit,nwid,mg.toString().c_str());
 
 		if ((flags & 0x01) != 0) {
 			try {
@@ -880,7 +878,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 						network->addCredential(com);
 				}
 			} catch ( ... ) {
-				TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+				TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str());
 			}
 		}
 
@@ -892,9 +890,9 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 			mg.mac().appendTo(outp);
 			outp.append((uint32_t)mg.adi());
 			const unsigned int gatheredLocally = RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit);
-			if (gatheredLocally) {
+			if (gatheredLocally > 0) {
 				outp.armor(peer->key(),true);
-				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+				_path->send(RR,outp.data(),outp.size(),RR->node->now());
 			}
 
 			// If we are a member of a cluster, distribute this GATHER across it
@@ -904,9 +902,9 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 #endif
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -933,8 +931,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			// Check membership after we've read any included COM, since
 			// that cert might be what we needed.
 			if (!network->isAllowed(peer)) {
-				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
-				peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false);
+				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
+				peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false);
 				return true;
 			}
 
@@ -960,13 +958,13 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 
 			if ((frameLen > 0)&&(frameLen <= ZT_IF_MTU)) {
 				if (!to.mac().isMulticast()) {
-					TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
-					peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
+					TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str());
+					peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 					return true;
 				}
 				if ((!from)||(from.isMulticast())||(from == network->mac())) {
-					TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
-					peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
+					TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str());
+					peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 					return true;
 				}
 
@@ -974,8 +972,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 					if (network->config().permitsBridging(peer->address())) {
 						network->learnBridgeRoute(from,peer->address());
 					} else {
-						TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id());
-						peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
+						TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id());
+						peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 						return true;
 					}
 				}
@@ -996,16 +994,16 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 				outp.append((unsigned char)0x02); // flag 0x02 = contains gather results
 				if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) {
 					outp.armor(peer->key(),true);
-					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+					_path->send(RR,outp.data(),outp.size(),RR->node->now());
 				}
 			}
 
-			peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true);
+			peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true);
 		} else {
-			peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false);
+			peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false);
 		}
 	} catch ( ... ) {
-		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -1017,8 +1015,8 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 
 		// First, subject this to a rate limit
 		if (!peer->shouldRespondToDirectPathPush(now)) {
-			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_remoteAddress.toString().c_str());
-			peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false);
+			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str());
+			peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false);
 			return true;
 		}
 
@@ -1044,12 +1042,12 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 
 					bool redundant = false;
 					if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) {
-						peer->setClusterOptimalPathForAddressFamily(a);
+						peer->makeExclusive(a);
 					} else {
 						redundant = peer->hasActivePathTo(now,a);
 					}
 
-					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,a)) ) {
+					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) {
 						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->sendHELLO(InetAddress(),a,now);
@@ -1063,12 +1061,12 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 
 					bool redundant = false;
 					if ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_CLUSTER_REDIRECT) != 0) {
-						peer->setClusterOptimalPathForAddressFamily(a);
+						peer->makeExclusive(a);
 					} else {
 						redundant = peer->hasActivePathTo(now,a);
 					}
 
-					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_localAddress,a)) ) {
+					if ( ((flags & ZT_PUSH_DIRECT_PATHS_FLAG_FORGET_PATH) == 0) && (!redundant) && (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),a)) ) {
 						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
 							peer->sendHELLO(InetAddress(),a,now);
@@ -1081,9 +1079,9 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 			ptr += addrLen;
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -1124,8 +1122,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		// Verify signature -- only tests signed by their originators are allowed
 		const unsigned int signatureLength = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 27 + vlf);
 		if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) {
-			TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str());
-			peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
+			TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str());
+			peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
 			return true;
 		}
 		vlf += signatureLength;
@@ -1141,13 +1139,13 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		if (originatorCredentialNetworkId) {
 			SharedPtr<Network> network(RR->node->network(originatorCredentialNetworkId));
 			if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) {
-				TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
-				peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
+				TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
+				peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
 				return true;
 			}
 		} else {
-			TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str());
-			peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
+			TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str());
+			peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
 			return true;
 		}
 
@@ -1165,9 +1163,9 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 				remainingHopsPtr += ZT_ADDRESS_LENGTH;
 				SharedPtr<Peer> nhp(RR->topology->getPeer(nextHop[h]));
 				if (nhp) {
-					Path *const rp = nhp->getBestPath(now);
-					if (rp)
-						nextHopBestPathAddress[h] = rp->address();
+					SharedPtr<Path> nhbp(nhp->getBestPath(now,false));
+					if (nhbp)
+						nextHopBestPathAddress[h] = nhbp->address();
 				}
 			}
 		}
@@ -1190,8 +1188,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 			outp.append((uint64_t)packetId());
 			peer->address().appendTo(outp);
 			outp.append((uint8_t)hops());
-			_localAddress.serialize(outp);
-			_remoteAddress.serialize(outp);
+			_path->localAddress().serialize(outp);
+			_path->address().serialize(outp);
 			outp.append((uint16_t)0); // no additional fields
 			outp.append((uint8_t)breadth);
 			for(unsigned int h=0;h<breadth;++h) {
@@ -1218,9 +1216,9 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 			}
 		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -1263,9 +1261,9 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S
 		}
 
 		RR->node->postCircuitTestReport(&report);
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST_REPORT,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
-		TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped CIRCUIT_TEST_REPORT from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }
@@ -1308,28 +1306,28 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 						outp.append((uint16_t)sizeof(result));
 						outp.append(result,sizeof(result));
 						outp.armor(peer->key(),true);
-						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+						_path->send(RR,outp.data(),outp.size(),RR->node->now());
 					} else {
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
 						outp.append((unsigned char)Packet::VERB_REQUEST_PROOF_OF_WORK);
 						outp.append(pid);
 						outp.append((unsigned char)Packet::ERROR_INVALID_REQUEST);
 						outp.armor(peer->key(),true);
-						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+						_path->send(RR,outp.data(),outp.size(),RR->node->now());
 					}
 				}	break;
 
 				default:
-					TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unrecognized proof of work type",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+					TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unrecognized proof of work type",peer->address().toString().c_str(),_path->address().toString().c_str());
 					break;
 			}
 
-			peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false);
+			peer->received(_path,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP,false);
 		} else {
-			TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+			TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_path->address().toString().c_str());
 		}
 	} catch ( ... ) {
-		TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str());
 	}
 	return true;
 }

node/Node.cpp

@@ -415,16 +415,16 @@ ZT_PeerList *Node::peers() const
 		p->latency = pi->second->latency();
 		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
 
-		std::vector<Path> paths(pi->second->paths());
-		Path *bestPath = pi->second->getBestPath(_now);
+		std::vector< SharedPtr<Path> > paths(pi->second->paths());
+		SharedPtr<Path> bestp(pi->second->getBestPath(_now,true));
 		p->pathCount = 0;
-		for(std::vector<Path>::iterator path(paths.begin());path!=paths.end();++path) {
-			memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage));
-			p->paths[p->pathCount].lastSend = path->lastSend();
-			p->paths[p->pathCount].lastReceive = path->lastReceived();
-			p->paths[p->pathCount].active = path->active(_now) ? 1 : 0;
-			p->paths[p->pathCount].preferred = ((bestPath)&&(*path == *bestPath)) ? 1 : 0;
-			p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust(path->address());
+		for(std::vector< SharedPtr<Path> >::iterator path(paths.begin());path!=paths.end();++path) {
+			memcpy(&(p->paths[p->pathCount].address),&((*path)->address()),sizeof(struct sockaddr_storage));
+			p->paths[p->pathCount].lastSend = (*path)->lastOut();
+			p->paths[p->pathCount].lastReceive = (*path)->lastIn();
+			p->paths[p->pathCount].active = (*path)->alive(_now) ? 1 : 0;
+			p->paths[p->pathCount].preferred = (*path == bestp) ? 1 : 0;
+			p->paths[p->pathCount].trustedPathId = RR->topology->getOutboundPathTrust((*path)->address());
 			++p->pathCount;
 		}
 	}

node/Path.hpp

@@ -105,8 +105,7 @@ public:
 		_lastIn(0),
 		_addr(),
 		_localAddress(),
-		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_clusterSuboptimal(false)
+		_ipScope(InetAddress::IP_SCOPE_NONE)
 	{
 	}
 
@@ -115,27 +114,10 @@ public:
 		_lastIn(0),
 		_addr(addr),
 		_localAddress(localAddress),
-		_ipScope(addr.ipScope()),
-		_clusterSuboptimal(false)
+		_ipScope(addr.ipScope())
 	{
 	}
 
-	inline Path &operator=(const Path &p)
-	{
-		if (this != &p)
-			memcpy(this,&p,sizeof(Path));
-		return *this;
-	}
-
-	/**
-	 * Called when a packet is sent to this remote path
-	 *
-	 * This is called automatically by Path::send().
-	 *
-	 * @param t Time of send
-	 */
-	inline void sent(const uint64_t t) { _lastOut = t; }
-
 	/**
 	 * Called when a packet is received from this remote path, regardless of content
 	 *
@@ -157,37 +139,22 @@ public:
 	/**
 	 * @return Address of local side of this path or NULL if unspecified
 	 */
-	inline const InetAddress &localAddress() const throw() { return _localAddress; }
+	inline const InetAddress &localAddress() const { return _localAddress; }
 
 	/**
 	 * @return Physical address
 	 */
-	inline const InetAddress &address() const throw() { return _addr; }
+	inline const InetAddress &address() const { return _addr; }
 
 	/**
 	 * @return IP scope -- faster shortcut for address().ipScope()
 	 */
-	inline InetAddress::IpScope ipScope() const throw() { return _ipScope; }
-
-	/**
-	 * @param f Is this path cluster-suboptimal?
-	 */
-	inline void setClusterSuboptimal(const bool f) { _clusterSuboptimal = f; }
-
-	/**
-	 * @return True if cluster-suboptimal (for someone)
-	 */
-	inline bool isClusterSuboptimal() const { return _clusterSuboptimal; }
-
-	/**
-	 * @return True if cluster-optimal (for someone) (the default)
-	 */
-	inline bool isClusterOptimal() const { return (!(_clusterSuboptimal)); }
+	inline InetAddress::IpScope ipScope() const { return _ipScope; }
 
 	/**
 	 * @return Preference rank, higher == better (will be less than 255)
 	 */
-	inline unsigned int preferenceRank() const throw()
+	inline unsigned int preferenceRank() const
 	{
 		/* First, since the scope enum values in InetAddress.hpp are in order of
 		 * use preference rank, we take that. Then we multiple by two, yielding
@@ -201,20 +168,9 @@ public:
 	/**
 	 * @return This path's overall quality score (higher is better)
 	 */
-	inline uint64_t score() const throw()
+	inline uint64_t score() const
 	{
-		// This is a little bit convoluted because we try to be branch-free, using multiplication instead of branches for boolean flags
-
-		// Start with the last time this path was active, and add a fudge factor to prevent integer underflow if _lastReceived is 0
-		uint64_t score = _lastIn + (ZT_PEER_DIRECT_PING_DELAY * (ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION + 1));
-
-		// Increase score based on path preference rank, which is based on IP scope and address family
-		score += preferenceRank() * (ZT_PEER_DIRECT_PING_DELAY / ZT_PATH_MAX_PREFERENCE_RANK);
-
-		// Decrease score if this is known to be a sub-optimal path to a cluster
-		score -= ((uint64_t)_clusterSuboptimal) * ZT_PEER_DIRECT_PING_DELAY;
-
-		return score;
+		return (_lastIn + (preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK)));
 	}
 
 	/**
@@ -227,7 +183,6 @@ public:
 	 * @return True if address is good for ZeroTier path use
 	 */
 	static inline bool isAddressValidForPath(const InetAddress &a)
-		throw()
 	{
 		if ((a.ss_family == AF_INET)||(a.ss_family == AF_INET6)) {
 			switch(a.ipScope()) {
@@ -258,6 +213,26 @@ public:
 		return false;
 	}
 
+	/**
+	 * @return True if path appears alive
+	 */
+	inline bool alive(const uint64_t now) const { return ((now - _lastIn) <= ZT_PATH_ALIVE_TIMEOUT); }
+
+	/**
+	 * @return True if this path needs a heartbeat
+	 */
+	inline bool needsHeartbeat(const uint64_t now) const { return ((now - _lastOut) > ZT_PATH_HEARTBEAT_PERIOD); }
+
+	/**
+	 * @return Last time we sent something
+	 */
+	inline uint64_t lastOut() const { return _lastOut; }
+
+	/**
+	 * @return Last time we received anything
+	 */
+	inline uint64_t lastIn() const { return _lastIn; }
+
 private:
 	uint64_t _lastOut;
 	uint64_t _lastIn;
@@ -265,7 +240,6 @@ private:
 	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 	AtomicCounter __refCount;
-	bool _clusterSuboptimal;
 };
 
 } // namespace ZeroTier

node/Peer.cpp

@@ -27,8 +27,6 @@
 #include "Cluster.hpp"
 #include "Packet.hpp"
 
-#include <algorithm>
-
 #define ZT_PEER_PATH_SORT_INTERVAL 5000
 
 namespace ZeroTier {
@@ -45,7 +43,6 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_lastAnnouncedTo(0),
 	_lastDirectPathPushSent(0),
 	_lastDirectPathPushReceive(0),
-	_lastPathSort(0),
 	_vProto(0),
 	_vMajor(0),
 	_vMinor(0),
@@ -60,8 +57,7 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 }
 
 void Peer::received(
-	const InetAddress &localAddr,
-	const InetAddress &remoteAddr,
+	const SharedPtr<Path> &path,
 	unsigned int hops,
 	uint64_t packetId,
 	Packet::Verb verb,
@@ -69,13 +65,15 @@ void Peer::received(
 	Packet::Verb inReVerb,
 	const bool trustEstablished)
 {
+	const uint64_t now = RR->node->now();
+
 #ifdef ZT_ENABLE_CLUSTER
 	bool suboptimalPath = false;
 	if ((RR->cluster)&&(hops == 0)) {
 		// Note: findBetterEndpoint() is first since we still want to check
 		// for a better endpoint even if we don't actually send a redirect.
 		InetAddress redirectTo;
-		if ( (verb != Packet::VERB_OK) && (verb != Packet::VERB_ERROR) && (verb != Packet::VERB_RENDEZVOUS) && (verb != Packet::VERB_PUSH_DIRECT_PATHS) && (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),remoteAddr,false)) ) {
+		if ( (verb != Packet::VERB_OK) && (verb != Packet::VERB_ERROR) && (verb != Packet::VERB_RENDEZVOUS) && (verb != Packet::VERB_PUSH_DIRECT_PATHS) && (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),path->address(),false)) ) {
 			if (_vProto >= 5) {
 				// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
 				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
@@ -93,7 +91,7 @@ void Peer::received(
 				}
 				outp.append((uint16_t)redirectTo.port());
 				outp.armor(_key,true);
-				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+				path->send(RR,outp.data(),outp.size(),now);
 			} else {
 				// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
 				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
@@ -108,14 +106,13 @@ void Peer::received(
 					outp.append(redirectTo.rawIpData(),16);
 				}
 				outp.armor(_key,true);
-				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+				path->send(RR,outp.data(),outp.size(),now);
 			}
 			suboptimalPath = true;
 		}
 	}
 #endif
 
-	const uint64_t now = RR->node->now();
 	_lastReceive = now;
 	if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
 		_lastUnicastFrame = now;
@@ -124,53 +121,47 @@ void Peer::received(
 
 	if (hops == 0) {
 		bool pathIsConfirmed = false;
-		unsigned int np = _numPaths;
-		for(unsigned int p=0;p<np;++p) {
-			if ((_paths[p].address() == remoteAddr)&&(_paths[p].localAddress() == localAddr)) {
-				_paths[p].received(now);
+		{
+			Mutex::Lock _l(_paths_m);
+			for(unsigned int p=0;p<_numPaths;++p) {
+				if (_paths[p].path == path) { // paths are canonicalized so pointer compare is good here
+					_paths[p].lastReceive = now;
 #ifdef ZT_ENABLE_CLUSTER
-				_paths[p].setClusterSuboptimal(suboptimalPath);
+					_paths[p].clusterSuboptimal = suboptimalPath;
 #endif
-				pathIsConfirmed = true;
-				break;
+					pathIsConfirmed = true;
+					break;
+				}
 			}
 		}
 
-		if ((!pathIsConfirmed)&&(RR->node->shouldUsePathForZeroTierTraffic(localAddr,remoteAddr))) {
+		if ((!pathIsConfirmed)&&(RR->node->shouldUsePathForZeroTierTraffic(path->localAddress(),path->address()))) {
 			if (verb == Packet::VERB_OK) {
+				Mutex::Lock _l(_paths_m);
 
-				Path *slot = (Path *)0;
-				if (np < ZT_MAX_PEER_NETWORK_PATHS) {
-					slot = &(_paths[np++]);
+				unsigned int slot = 0;
+				if (_numPaths < ZT_MAX_PEER_NETWORK_PATHS) {
+					slot = _numPaths++;
 				} else {
-					uint64_t slotWorstScore = 0xffffffffffffffffULL;
-					for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
-						if (!_paths[p].active(now)) {
-							slot = &(_paths[p]);
-							break;
-						} else {
-							const uint64_t score = _paths[p].score();
-							if (score <= slotWorstScore) {
-								slotWorstScore = score;
-								slot = &(_paths[p]);
-							}
+					uint64_t oldest = 0ULL;
+					unsigned int oldestPath = 0;
+					for(unsigned int p=0;p<_numPaths;++p) {
+						if (_paths[p].lastReceive < oldest) {
+							oldest = _paths[p].lastReceive;
+							oldestPath = p;
 						}
 					}
-				}
-				if (slot) {
-					*slot = Path(localAddr,remoteAddr);
-					slot->received(now);
-#ifdef ZT_ENABLE_CLUSTER
-					slot->setClusterSuboptimal(suboptimalPath);
-#endif
-					_numPaths = np;
+					slot = oldestPath;
 				}
 
+				_paths[slot].path = path;
+				_paths[slot].lastReceive = now;
 #ifdef ZT_ENABLE_CLUSTER
+				_paths[slot].clusterSuboptimal = suboptimalPath;
+
 				if (RR->cluster)
 					RR->cluster->broadcastHavePeer(_id);
 #endif
-
 			} else {
 
 				TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
@@ -178,15 +169,15 @@ void Peer::received(
 				if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) {
 					Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO);
 					outp.armor(_key,true);
-					RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+					path->send(RR,outp.data(),outp.size(),now);
 				} else {
-					sendHELLO(localAddr,remoteAddr,now);
+					sendHELLO(path->localAddress(),path->address(),now);
 				}
 
 			}
 		}
 	} else if (trustEstablished) {
-		_pushDirectPaths(localAddr,remoteAddr,now);
+		_pushDirectPaths(path,now);
 	}
 
 	if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
@@ -197,7 +188,96 @@ void Peer::received(
 	}
 }
 
-void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl)
+bool Peer::hasActivePathTo(uint64_t now,const InetAddress &addr) const
+{
+	Mutex::Lock _l(_paths_m);
+	for(unsigned int p=0;p<_numPaths;++p) {
+		if ( (_paths[p].path->address() == addr) && (_paths[p].path->alive(now)) )
+			return true; 
+	}
+	return false;
+}
+
+void Peer::makeExclusive(const InetAddress &addr)
+{
+	Mutex::Lock _l(_paths_m);
+
+	bool have = false;
+	for(unsigned int p=0;p<_numPaths;++p) {
+		if (_paths[p].path->address() == addr) {
+			have = true;
+			break;
+		}
+	}
+
+	if (have) {
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if ((_paths[x].path->address().ss_family != addr.ss_family)||(_paths[x].path->address() == addr)) {
+				if (y != x) {
+					_paths[y].path = _paths[x].path;
+					_paths[y].lastReceive = _paths[x].lastReceive;
+	#ifdef ZT_ENABLE_CLUSTER
+					_paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal;
+	#endif
+				}
+				++y;
+			}
+			++x;
+		}
+		_numPaths = y;
+	}
+}
+
+bool Peer::send(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead)
+{
+	Mutex::Lock _l(_paths_m);
+
+	int bestp = -1;
+	uint64_t best = 0ULL;
+	for(unsigned int p=0;p<_numPaths;++p) {
+		if (_paths[p].path->alive(now)||(forceEvenIfDead)) {
+			const uint64_t s = _paths[p].path->score();
+			if (s >= best) {
+				best = s;
+				bestp = (int)p;
+			}
+		}
+	}
+
+	if (bestp >= 0) {
+		return _paths[bestp].path->send(RR,data,len,now);
+	} else {
+		return false;
+	}
+}
+
+SharedPtr<Path> Peer::getBestPath(uint64_t now,bool forceEvenIfDead)
+{
+	Mutex::Lock _l(_paths_m);
+
+	int bestp = -1;
+	uint64_t best = 0ULL;
+	for(unsigned int p=0;p<_numPaths;++p) {
+		if (_paths[p].path->alive(now)||(forceEvenIfDead)) {
+			const uint64_t s = _paths[p].path->score();
+			if (s >= best) {
+				best = s;
+				bestp = (int)p;
+			}
+		}
+	}
+
+	if (bestp >= 0) {
+		return _paths[bestp].path;
+	} else {
+		return SharedPtr<Path>();
+	}
+}
+
+void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now)
 {
 	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
 	outp.append((unsigned char)ZT_PROTO_VERSION);
@@ -209,51 +289,56 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u
 	atAddress.serialize(outp);
 	outp.append((uint64_t)RR->topology->worldId());
 	outp.append((uint64_t)RR->topology->worldTimestamp());
-
 	outp.armor(_key,false); // HELLO is sent in the clear
-	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size(),ttl);
+	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 }
 
 bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily)
 {
-	Path *p = (Path *)0;
-
-	if (inetAddressFamily != 0) {
-		p = _getBestPath(now,inetAddressFamily);
-	} else {
-		p = _getBestPath(now);
-	}
-
-	if (p) {
-		if ((now - p->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
-			//TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
-			sendHELLO(p->localAddress(),p->address(),now);
-			p->sent(now);
-			p->pinged(now);
-		} else if ((now - std::max(p->lastSend(),p->lastKeepalive())) >= ZT_NAT_KEEPALIVE_DELAY) {
-			//TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
+	bool somethingAlive = false;
+	Mutex::Lock _l(_paths_m);
+	for(unsigned int p=0;p<_numPaths;++p) {
+		if ((now - _paths[p].lastReceive) >= ZT_PEER_PING_PERIOD) {
+			sendHELLO(_paths[p].path->localAddress(),_paths[p].path->address(),now);
+		} else if (_paths[p].path->needsHeartbeat(now)) {
 			_natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads
-			RR->node->putPacket(p->localAddress(),p->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
-			p->sentKeepalive(now);
+			_paths[p].path->send(RR,&_natKeepaliveBuf,sizeof(_natKeepaliveBuf),now);
 		}
-		return true;
+		somethingAlive |= _paths[p].path->alive(now);
 	}
+	return somethingAlive;
+}
 
+bool Peer::hasActiveDirectPath(uint64_t now) const
+{
+	Mutex::Lock _l(_paths_m);
+	for(unsigned int p=0;p<_numPaths;++p) {
+		if (_paths[p].path->alive(now))
+			return true;
+	}
 	return false;
 }
 
 bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now)
 {
+	Mutex::Lock _l(_paths_m);
 	unsigned int np = _numPaths;
 	unsigned int x = 0;
 	unsigned int y = 0;
 	while (x < np) {
-		if (_paths[x].address().ipScope() == scope) {
+		if (_paths[x].path->address().ipScope() == scope) {
 			// Resetting a path means sending a HELLO and then forgetting it. If we
 			// get OK(HELLO) then it will be re-learned.
-			sendHELLO(_paths[x].localAddress(),_paths[x].address(),now);
+			sendHELLO(_paths[x].path->localAddress(),_paths[x].path->address(),now);
 		} else {
-			_paths[y++] = _paths[x];
+			if (x != y) {
+				_paths[y].path = _paths[x].path;
+				_paths[y].lastReceive = _paths[x].lastReceive;
+#ifdef ZT_ENABLE_CLUSTER
+				_paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal;
+#endif
+			}
+			++y;
 		}
 		++x;
 	}
@@ -263,114 +348,55 @@ bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now)
 
 void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const
 {
-	uint64_t bestV4 = 0,bestV6 = 0;
-	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		if (_paths[p].active(now)) {
-			uint64_t lr = _paths[p].lastReceived();
-			if (lr) {
-				if (_paths[p].address().isV4()) {
-					if (lr >= bestV4) {
-						bestV4 = lr;
-						v4 = _paths[p].address();
-					}
-				} else if (_paths[p].address().isV6()) {
-					if (lr >= bestV6) {
-						bestV6 = lr;
-						v6 = _paths[p].address();
-					}
-				}
+	Mutex::Lock _l(_paths_m);
+
+	int bestp4 = -1,bestp6 = -1;
+	uint64_t best4 = 0ULL,best6 = 0ULL;
+	for(unsigned int p=0;p<_numPaths;++p) {
+		if (_paths[p].path->address().ss_family == AF_INET) {
+			const uint64_t s = _paths[p].path->score();
+			if (s >= best4) {
+				best4 = s;
+				bestp4 = (int)p;
+			}
+		} else if (_paths[p].path->address().ss_family == AF_INET6) {
+			const uint64_t s = _paths[p].path->score();
+			if (s >= best6) {
+				best6 = s;
+				bestp6 = (int)p;
 			}
 		}
 	}
+
+	if (bestp4 >= 0)
+		v4 = _paths[bestp4].path->address();
+	if (bestp6 >= 0)
+		v6 = _paths[bestp6].path->address();
 }
 
 void Peer::clean(uint64_t now)
 {
+	Mutex::Lock _l(_paths_m);
 	unsigned int np = _numPaths;
 	unsigned int x = 0;
 	unsigned int y = 0;
 	while (x < np) {
-		if (_paths[x].active(now))
-			_paths[y++] = _paths[x];
+		if ((now - _paths[x].lastReceive) <= ZT_PEER_PATH_EXPIRATION) {
+			if (y != x) {
+				_paths[y].path = _paths[x].path;
+				_paths[y].lastReceive = _paths[x].lastReceive;
+#ifdef ZT_ENABLE_CLUSTER
+				_paths[y].clusterSuboptimal = _paths[x].clusterSuboptimal;
+#endif
+			}
+			++y;
+		}
 		++x;
 	}
 	_numPaths = y;
 }
 
-void Peer::_doDeadPathDetection(Path &p,const uint64_t now)
-{
-	/* Dead path detection: if we have sent something to this peer and have not
-	 * yet received a reply, double check this path. The majority of outbound
-	 * packets including Ethernet frames do generate some kind of reply either
-	 * immediately or at some point in the near future. This will occasionally
-	 * (every NO_ANSWER_TIMEOUT ms) check paths unnecessarily if traffic that
-	 * does not generate a response is being sent such as multicast announcements
-	 * or frames belonging to unidirectional UDP protocols, but the cost is very
-	 * tiny and the benefit in reliability is very large. This takes care of many
-	 * failure modes including crap NATs that forget links and spurious changes
-	 * to physical network topology that cannot be otherwise detected.
-	 *
-	 * Each time we do this we increment a probation counter in the path. This
-	 * counter is reset on any packet receive over this path. If it reaches the
-	 * MAX_PROBATION threshold the path is considred dead. */
-
-	if (
-	     (p.lastSend() > p.lastReceived()) &&
-			 ((p.lastSend() - p.lastReceived()) >= ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT) &&
-			 ((now - p.lastPing()) >= ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT) &&
-			 (!p.isClusterSuboptimal()) &&
-			 (!RR->topology->amRoot())
-		 ) {
-		TRACE("%s(%s) does not seem to be answering in a timely manner, checking if dead (probation == %u)",_id.address().toString().c_str(),p.address().toString().c_str(),p.probation());
-
-		if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) {
-			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO);
-			outp.armor(_key,true);
-			p.send(RR,outp.data(),outp.size(),now);
-			p.pinged(now);
-		} else {
-			sendHELLO(p.localAddress(),p.address(),now);
-			p.sent(now);
-			p.pinged(now);
-		}
-
-		p.increaseProbation();
-	}
-}
-
-Path *Peer::_getBestPath(const uint64_t now)
-{
-	Path *bestPath = (Path *)0;
-	uint64_t bestPathScore = 0;
-	for(unsigned int i=0;i<_numPaths;++i) {
-		const uint64_t score = _paths[i].score();
-		if ((score >= bestPathScore)&&(_paths[i].active(now))) {
-			bestPathScore = score;
-			bestPath = &(_paths[i]);
-		}
-	}
-	if (bestPath)
-		_doDeadPathDetection(*bestPath,now);
-	return bestPath;
-}
-
-Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
-{
-	Path *bestPath = (Path *)0;
-	uint64_t bestPathScore = 0;
-	for(unsigned int i=0;i<_numPaths;++i) {
-		const uint64_t score = _paths[i].score();
-		if (((int)_paths[i].address().ss_family == inetAddressFamily)&&(score >= bestPathScore)&&(_paths[i].active(now))) {
-			bestPathScore = score;
-			bestPath = &(_paths[i]);
-		}
-	}
-	if (bestPath)
-		_doDeadPathDetection(*bestPath,now);
-	return bestPath;
-}
-
-bool Peer::_pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now)
+bool Peer::_pushDirectPaths(const SharedPtr<Path> &path,uint64_t now)
 {
 #ifdef ZT_ENABLE_CLUSTER
 	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
@@ -445,7 +471,7 @@ bool Peer::_pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAd
 		if (count) {
 			outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count);
 			outp.armor(_key,true);
-			RR->node->putPacket(localAddr,toAddress,outp.data(),outp.size(),0);
+			path->send(RR,outp.data(),outp.size(),now);
 		}
 	}
 

node/Peer.hpp

@@ -96,9 +96,7 @@ public:
 	 * This is called by the decode pipe when a packet is proven to be authentic
 	 * and appears to be valid.
 	 *
-	 * @param RR Runtime environment
-	 * @param localAddr Local address
-	 * @param remoteAddr Internet address of sender
+	 * @param path Path over which packet was received
 	 * @param hops ZeroTier (not IP) hops
 	 * @param packetId Packet ID
 	 * @param verb Packet verb
@@ -107,8 +105,7 @@ public:
 	 * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established
 	 */
 	void received(
-		const InetAddress &localAddr,
-		const InetAddress &remoteAddr,
+		const SharedPtr<Path> &path,
 		unsigned int hops,
 		uint64_t packetId,
 		Packet::Verb verb,
@@ -116,43 +113,19 @@ public:
 		Packet::Verb inReVerb,
 		const bool trustEstablished);
 
-	/**
-	 * Get the current best direct path to this peer
-	 *
-	 * @param now Current time
-	 * @return Best path or NULL if there are no active direct paths
-	 */
-	inline Path *getBestPath(uint64_t now) { return _getBestPath(now); }
-
 	/**
 	 * @param now Current time
 	 * @param addr Remote address
 	 * @return True if we have an active path to this destination
 	 */
-	inline bool hasActivePathTo(uint64_t now,const InetAddress &addr) const
-	{
-		for(unsigned int p=0;p<_numPaths;++p) {
-			if ((_paths[p].active(now))&&(_paths[p].address() == addr))
-				return true;
-		}
-		return false;
-	}
+	bool hasActivePathTo(uint64_t now,const InetAddress &addr) const;
 
 	/**
-	 * Set all paths in the same ss_family that are not this one to cluster suboptimal
-	 *
-	 * Addresses in other families are not affected.
+	 * If we have a confirmed path to this address, forget all others within the same address family
 	 *
 	 * @param addr Address to make exclusive
 	 */
-	inline void setClusterOptimalPathForAddressFamily(const InetAddress &addr)
-	{
-		for(unsigned int p=0;p<_numPaths;++p) {
-			if (_paths[p].address().ss_family == addr.ss_family) {
-				_paths[p].setClusterSuboptimal(_paths[p].address() != addr);
-			}
-		}
-	}
+	void makeExclusive(const InetAddress &addr);
 
 	/**
 	 * Send via best path
@@ -160,30 +133,30 @@ public:
 	 * @param data Packet data
 	 * @param len Packet length
 	 * @param now Current time
-	 * @return Path used on success or NULL on failure
+	 * @param forceEvenIfDead If true, send even if the path is not 'alive'
+	 * @return True if we actually sent something
 	 */
-	inline Path *send(const void *data,unsigned int len,uint64_t now)
-	{
-		Path *const bestPath = getBestPath(now);
-		if (bestPath) {
-			if (bestPath->send(RR,data,len,now))
-				return bestPath;
-		}
-		return (Path *)0;
-	}
+	bool send(const void *data,unsigned int len,uint64_t now,bool forceEvenIfDead);
+
+	/**
+	 * Get the best current direct path
+	 *
+	 * @param now Current time
+	 * @param forceEvenIfDead If true, pick even if path is not alive
+	 * @return Best current path or NULL if none
+	 */
+	SharedPtr<Path> getBestPath(uint64_t now,bool forceEvenIfDead);
 
 	/**
 	 * Send a HELLO to this peer at a specified physical address
 	 *
-	 * This does not update any statistics. It's used to send initial HELLOs
-	 * for NAT traversal and path verification.
+	 * No statistics or sent times are updated here.
 	 *
 	 * @param localAddr Local address
 	 * @param atAddress Destination address
 	 * @param now Current time
-	 * @param ttl Desired IP TTL (default: 0 to leave alone)
 	 */
-	void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl = 0);
+	void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now);
 
 	/**
 	 * Send pings or keepalives depending on configured timeouts
@@ -194,14 +167,49 @@ public:
 	 */
 	bool doPingAndKeepalive(uint64_t now,int inetAddressFamily);
 
+	/**
+	 * @param now Current time
+	 * @return True if this peer has at least one active direct path
+	 */
+	bool hasActiveDirectPath(uint64_t now) const;
+
+	/**
+	 * Reset paths within a given scope
+	 *
+	 * @param scope IP scope of paths to reset
+	 * @param now Current time
+	 * @return True if at least one path was forgotten
+	 */
+	bool resetWithinScope(InetAddress::IpScope scope,uint64_t now);
+
+	/**
+	 * Get most recently active path addresses for IPv4 and/or IPv6
+	 *
+	 * Note that v4 and v6 are not modified if they are not found, so
+	 * initialize these to a NULL address to be able to check.
+	 *
+	 * @param now Current time
+	 * @param v4 Result parameter to receive active IPv4 address, if any
+	 * @param v6 Result parameter to receive active IPv6 address, if any
+	 */
+	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
+
+	/**
+	 * Perform periodic cleaning operations
+	 *
+	 * @param now Current time
+	 */
+	void clean(uint64_t now);
+
 	/**
 	 * @return All known direct paths to this peer (active or inactive)
 	 */
-	inline std::vector<Path> paths() const
+	inline std::vector< SharedPtr<Path> > paths() const
 	{
-		std::vector<Path> pp;
+		std::vector< SharedPtr<Path> > pp;
+		Mutex::Lock _l(_paths_m);
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
-			pp.push_back(_paths[p]);
+			pp.push_back(_paths[p].path);
 		return pp;
 	}
 
@@ -254,7 +262,7 @@ public:
 		unsigned int l = _latency;
 		if (!l)
 			l = 0xffff;
-		return (l * (((unsigned int)tsr / (ZT_PEER_DIRECT_PING_DELAY + 1000)) + 1));
+		return (l * (((unsigned int)tsr / (ZT_PEER_PING_PERIOD + 1000)) + 1));
 	}
 
 	/**
@@ -270,19 +278,6 @@ public:
 		else _latency = std::min(l,(unsigned int)65535);
 	}
 
-	/**
-	 * @param now Current time
-	 * @return True if this peer has at least one active direct path
-	 */
-	inline bool hasActiveDirectPath(uint64_t now) const
-	{
-		for(unsigned int p=0;p<_numPaths;++p) {
-			if (_paths[p].active(now))
-				return true;
-		}
-		return false;
-	}
-
 #ifdef ZT_ENABLE_CLUSTER
 	/**
 	 * @param now Current time
@@ -291,22 +286,13 @@ public:
 	inline bool hasClusterOptimalPath(uint64_t now) const
 	{
 		for(unsigned int p=0,np=_numPaths;p<np;++p) {
-			if ((_paths[p].active(now))&&(!_paths[p].isClusterSuboptimal()))
+			if ( (_paths[p].path->alive(now)) && (!_paths[p].clusterSuboptimal) )
 				return true;
 		}
 		return false;
 	}
 #endif
 
-	/**
-	 * Reset paths within a given scope
-	 *
-	 * @param scope IP scope of paths to reset
-	 * @param now Current time
-	 * @return True if at least one path was forgotten
-	 */
-	bool resetWithinScope(InetAddress::IpScope scope,uint64_t now);
-
 	/**
 	 * @return 256-bit secret symmetric encryption key
 	 */
@@ -335,25 +321,6 @@ public:
 
 	inline bool remoteVersionKnown() const throw() { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); }
 
-	/**
-	 * Get most recently active path addresses for IPv4 and/or IPv6
-	 *
-	 * Note that v4 and v6 are not modified if they are not found, so
-	 * initialize these to a NULL address to be able to check.
-	 *
-	 * @param now Current time
-	 * @param v4 Result parameter to receive active IPv4 address, if any
-	 * @param v6 Result parameter to receive active IPv6 address, if any
-	 */
-	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
-
-	/**
-	 * Perform periodic cleaning operations
-	 *
-	 * @param now Current time
-	 */
-	void clean(uint64_t now);
-
 	/**
 	 * Update direct path push stats and return true if we should respond
 	 *
@@ -395,10 +362,7 @@ public:
 	}
 
 private:
-	void _doDeadPathDetection(Path &p,const uint64_t now);
-	Path *_getBestPath(const uint64_t now);
-	Path *_getBestPath(const uint64_t now,int inetAddressFamily);
-	bool _pushDirectPaths(const InetAddress &localAddr,const InetAddress &toAddress,uint64_t now);
+	bool _pushDirectPaths(const SharedPtr<Path> &path,uint64_t now);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 
@@ -410,13 +374,19 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastDirectPathPushSent;
 	uint64_t _lastDirectPathPushReceive;
-	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 	Identity _id;
-	Path _paths[ZT_MAX_PEER_NETWORK_PATHS];
+	struct {
+		SharedPtr<Path> path;
+		uint64_t lastReceive;
+#ifdef ZT_ENABLE_CLUSTER
+		bool clusterSuboptimal;
+#endif
+	} _paths[ZT_MAX_PEER_NETWORK_PATHS];
+	Mutex _paths_m;
 	unsigned int _numPaths;
 	unsigned int _latency;
 	unsigned int _directPathPushCutoffCount;

node/Switch.cpp

@@ -112,7 +112,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 						// Note: we don't bother initiating NAT-t for fragments, since heads will set that off.
 						// It wouldn't hurt anything, just redundant and unnecessary.
 						SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-						if ((!relayTo)||(!relayTo->send(fragment.data(),fragment.size(),now))) {
+						if ((!relayTo)||(!relayTo->send(fragment.data(),fragment.size(),now,true))) {
 #ifdef ZT_ENABLE_CLUSTER
 							if (RR->cluster) {
 								RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false);
@@ -123,7 +123,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 							// Don't know peer or no direct path -- so relay via root server
 							relayTo = RR->topology->getBestRoot();
 							if (relayTo)
-								relayTo->send(fragment.data(),fragment.size(),now);
+								relayTo->send(fragment.data(),fragment.size(),now,true);
 						}
 					} else {
 						TRACE("dropped relay [fragment](%s) -> %s, max hops exceeded",fromAddr.toString().c_str(),destination.toString().c_str());
@@ -210,7 +210,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 						packet.incrementHops();
 
 						SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-						if ((relayTo)&&((relayTo->send(packet.data(),packet.size(),now)))) {
+						if ((relayTo)&&((relayTo->send(packet.data(),packet.size(),now,true)))) {
 							Mutex::Lock _l(_lastUniteAttempt_m);
 							uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)];
 							if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) {
@@ -234,7 +234,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 #endif
 							relayTo = RR->topology->getBestRoot(&source,1,true);
 							if (relayTo)
-								relayTo->send(packet.data(),packet.size(),now);
+								relayTo->send(packet.data(),packet.size(),now,true);
 						}
 					} else {
 						TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet.source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str());
@@ -251,7 +251,7 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 
 						rq->timestamp = now;
 						rq->packetId = packetId;
-						rq->frag0.init(data,len,localAddr,fromAddr,now);
+						rq->frag0.init(data,len,path,now);
 						rq->totalFragments = 0;
 						rq->haveFragments = 1;
 						rq->complete = false;
@@ -607,7 +607,7 @@ bool Switch::unite(const Address &p1,const Address &p2)
 				outp.append(cg.first.rawIpData(),4);
 			}
 			outp.armor(p1p->key(),true);
-			p1p->send(outp.data(),outp.size(),now);
+			p1p->send(outp.data(),outp.size(),now,true);
 		} else {
 			// Tell p2 where to find p1.
 			Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS);
@@ -622,7 +622,7 @@ bool Switch::unite(const Address &p1,const Address &p2)
 				outp.append(cg.second.rawIpData(),4);
 			}
 			outp.armor(p2p->key(),true);
-			p2p->send(outp.data(),outp.size(),now);
+			p2p->send(outp.data(),outp.size(),now,true);
 		}
 		++alt; // counts up and also flips LSB
 	}
@@ -739,7 +739,7 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread
 		Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS);
 		addr.appendTo(outp);
 		outp.armor(root->key(),true);
-		if (root->send(outp.data(),outp.size(),RR->node->now()))
+		if (root->send(outp.data(),outp.size(),RR->node->now(),true))
 			return root->address();
 	}
 	return Address();
@@ -752,12 +752,10 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 	if (peer) {
 		const uint64_t now = RR->node->now();
 
-		Path *viaPath = peer->getBestPath(now);
-		SharedPtr<Peer> relay;
-
+		SharedPtr<Path> viaPath(peer->getBestPath(now,false));
 		if (!viaPath) {
-			relay = RR->topology->getBestRoot();
-			if ( (!relay) || (!(viaPath = relay->getBestPath(now))) )
+			SharedPtr<Peer> relay(RR->topology->getBestRoot());
+			if ( (!relay) || (!(viaPath = relay->getBestPath(now,true))) )
 				return false;
 		}