Merge branch 'dev' of http://git.int.zerotier.com/ZeroTier/ZeroTierOne into dev

controller/EmbeddedNetworkController.cpp

@@ -566,42 +566,69 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
 
 	// Determine whether and how member is authorized
 	const char *authorizedBy = (const char *)0;
-	if (!_jB(network["private"],true)) {
+	if (_jB(member["authorized"],false)) {
+		authorizedBy = "memberIsAuthorized";
+	} else if (!_jB(network["private"],true)) {
 		authorizedBy = "networkIsPublic";
-		// If member already has an authorized field, leave it alone. That way its state is
-		// preserved if the user toggles the network back to private. Otherwise set it to
-		// true by default for new members of public nets.
 		if (!member.count("authorized")) {
 			member["authorized"] = true;
-			member["lastAuthorizedTime"] = now;
-			member["lastAuthorizedBy"] = authorizedBy;
+			json ah;
+			ah["a"] = true;
+			ah["by"] = authorizedBy;
+			ah["ts"] = now;
+			ah["ct"] = json();
+			ah["c"] = json();
+			member["authHistory"].push_back(ah);
 			member["lastModified"] = now;
-			auto revj = member["revision"];
+			json &revj = member["revision"];
 			member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
 		}
-	} else if (_jB(member["authorized"],false)) {
-		authorizedBy = "memberIsAuthorized";
 	} else {
-		char atok[256];
-		if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN,atok,sizeof(atok)) > 0) {
-			atok[255] = (char)0; // not necessary but YDIFLO
-			if (strlen(atok) > 0) { // extra sanity check since we never want to compare a null token on either side
-				auto authTokens = network["authTokens"];
+		char presentedAuth[512];
+		if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) {
+			presentedAuth[511] = (char)0; // sanity check
+
+			// Check for bearer token presented by member
+			if ((strlen(presentedAuth) > 6)&&(!strncmp(presentedAuth,"token:",6))) {
+				const char *const presentedToken = presentedAuth + 6;
+
+				json &authTokens = network["authTokens"];
 				if (authTokens.is_array()) {
 					for(unsigned long i=0;i<authTokens.size();++i) {
-						auto at = authTokens[i];
-						if (at.is_object()) {
-							const uint64_t expires = _jI(at["expires"],0ULL);
-							std::string tok = _jS(at["token"],"");
-							if ( ((expires == 0ULL)||(expires > now)) && (tok.length() > 0) && (tok == atok) ) {
-								authorizedBy = "token";
-								member["authorized"] = true; // tokens actually change member authorization state
-								member["lastAuthorizedTime"] = now;
-								member["lastAuthorizedBy"] = authorizedBy;
-								member["lastModified"] = now;
-								auto revj = member["revision"];
-								member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
-								break;
+						json &token = authTokens[i];
+						if (token.is_object()) {
+							const uint64_t expires = _jI(token["expires"],0ULL);
+							const uint64_t maxUses = _jI(token["maxUsesPerMember"],0ULL);
+							std::string tstr = _jS(token["token"],"");
+
+							if (((expires == 0ULL)||(expires > now))&&(tstr == presentedToken)) {
+								bool usable = (maxUses == 0);
+								if (!usable) {
+									uint64_t useCount = 0;
+									json &ahist = member["authHistory"];
+									if (ahist.is_array()) {
+										for(unsigned long j=0;j<ahist.size();++j) {
+											json &ah = ahist[j];
+											if ((_jS(ah["ct"],"") == "token")&&(_jS(ah["c"],"") == tstr)&&(_jB(ah["a"],false)))
+												++useCount;
+										}
+									}
+									usable = (useCount < maxUses);
+								}
+								if (usable) {
+									authorizedBy = "token";
+									member["authorized"] = true;
+									json ah;
+									ah["a"] = true;
+									ah["by"] = authorizedBy;
+									ah["ts"] = now;
+									ah["ct"] = "token";
+									ah["c"] = tstr;
+									member["authHistory"].push_back(ah);
+									member["lastModified"] = now;
+									json &revj = member["revision"];
+									member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
+								}
 							}
 						}
 					}
@@ -924,13 +951,11 @@ NetworkController::ResultCode EmbeddedNetworkController::doNetworkConfigRequest(
 		}
 	}
 
-	if (_jB(network["private"],true)) {
-		CertificateOfMembership com(now,credentialtmd,nwid,identity.address());
-		if (com.sign(signingId)) {
-			nc.com = com;
-		} else {
-			return NETCONF_QUERY_INTERNAL_SERVER_ERROR;
-		}
+	CertificateOfMembership com(now,credentialtmd,nwid,identity.address());
+	if (com.sign(signingId)) {
+		nc.com = com;
+	} else {
+		return NETCONF_QUERY_INTERNAL_SERVER_ERROR;
 	}
 
 	_writeJson(memberJP,member);
@@ -1139,16 +1164,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 
 						if (b.count("authorized")) {
 							const bool newAuth = _jB(b["authorized"],false);
-							const bool oldAuth = _jB(member["authorized"],false);
-							if (newAuth != oldAuth) {
-								if (newAuth) {
-									member["authorized"] = true;
-									member["lastAuthorizedTime"] = now;
-									member["lastAuthorizedBy"] = "user";
-								} else {
-									member["authorized"] = false;
-									member["lastDeauthorizedTime"] = now;
-								}
+							if (newAuth != _jB(member["authorized"],false)) {
+								member["authorized"] = newAuth;
+								json ah;
+								ah["a"] = newAuth;
+								ah["by"] = "api";
+								ah["ts"] = now;
+								ah["ct"] = json();
+								ah["c"] = json();
+								member["authHistory"].push_back(ah);
 							}
 						}
 
@@ -1429,13 +1453,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 						if (authTokens.is_array()) {
 							json nat = json::array();
 							for(unsigned long i=0;i<authTokens.size();++i) {
-								auto token = authTokens[i];
+								json &token = authTokens[i];
 								if (token.is_object()) {
 									std::string tstr = token["token"];
 									if (tstr.length() > 0) {
 										json t = json::object();
 										t["token"] = tstr;
 										t["expires"] = _jI(token["expires"],0ULL);
+										t["maxUsesPerMember"] = _jI(token["maxUsesPerMember"],0ULL);
 										nat.push_back(t);
 									}
 								}
@@ -1585,7 +1610,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes
 		"\t\"upstream\": \"%.10llx\"," ZT_EOL_S
 		"\t\"current\": \"%.10llx\"," ZT_EOL_S
 		"\t\"receivedTimestamp\": %llu," ZT_EOL_S
-		"\t\"remoteTimestamp\": %llu," ZT_EOL_S
 		"\t\"sourcePacketId\": \"%.16llx\"," ZT_EOL_S
 		"\t\"flags\": %llu," ZT_EOL_S
 		"\t\"sourcePacketHopCount\": %u," ZT_EOL_S
@@ -1606,7 +1630,6 @@ void EmbeddedNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTes
 		(unsigned long long)report->upstream,
 		(unsigned long long)report->current,
 		(unsigned long long)OSUtils::now(),
-		(unsigned long long)report->remoteTimestamp,
 		(unsigned long long)report->sourcePacketId,
 		(unsigned long long)report->flags,
 		report->sourcePacketHopCount,

controller/EmbeddedNetworkController.hpp

@@ -143,9 +143,7 @@ private:
 	inline void _initMember(nlohmann::json &member)
 	{
 		if (!member.count("authorized")) member["authorized"] = false;
-		if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL;
-		if (!member.count("lastAuthorizedBy")) member["lastAuthorizedBy"] = "";
-		if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL;
+		if (!member.count("authHistory")) member["authHistory"] = nlohmann::json::array();
  		if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array();
 		if (!member.count("recentLog")) member["recentLog"] = nlohmann::json::array();
 		if (!member.count("activeBridge")) member["activeBridge"] = false;

controller/README.md

@@ -229,9 +229,7 @@ This returns an object containing all currently online members and the most rece
 | nwid                  | string        | 16-digit network ID                               | no       |
 | clock                 | integer       | Current clock, ms since epoch                     | no       |
 | authorized            | boolean       | Is member authorized? (for private networks)      | YES      |
-| lastAuthorizedTime    | integer       | Time 'authorized' was last set to 'true'          | no       |
-| lastAuthorizedBy      | string        | What last set 'authorized' to 'true'?             | no       |
-| lastDeauthorizedTime  | integer       | Time 'authorized' was last set to 'false'         | no       |
+| authHistory           | array[object] | History of auth changes, latest at end            | no       |
 | activeBridge          | boolean       | Member is able to bridge to other Ethernet nets   | YES      |
 | identity              | string        | Member's public ZeroTier identity (if known)      | no       |
 | ipAssignments         | array[string] | Managed IP address assignments                    | YES      |

include/ZeroTierOne.h

@@ -154,6 +154,11 @@ extern "C" {
  */
 #define ZT_CIRCUIT_TEST_MAX_HOP_BREADTH 8
 
+/**
+ * Circuit test report flag: upstream peer authorized in path (e.g. by network COM)
+ */
+#define ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH 0x0000000000000001ULL
+
 /**
  * Maximum number of cluster members (and max member ID plus one)
  */
@@ -865,19 +870,28 @@ enum ZT_VirtualNetworkConfigOperation
 	ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4
 };
 
+enum ZT_RelayPolicy
+{
+	ZT_RELAY_POLICY_NEVER = 0,
+	ZT_RELAY_POLICY_TRUSTED = 1,
+	ZT_RELAY_POLICY_ALWAYS = 2
+};
+
 /**
  * What trust hierarchy role does this peer have?
  */
-enum ZT_PeerRole {
+enum ZT_PeerRole
+{
 	ZT_PEER_ROLE_LEAF = 0,     // ordinary node
-	ZT_PEER_ROLE_RELAY = 1,    // relay node
-	ZT_PEER_ROLE_ROOT = 2      // root server
+	ZT_PEER_ROLE_UPSTREAM = 1, // upstream node
+	ZT_PEER_ROLE_ROOT = 2      // global root
 };
 
 /**
  * Vendor ID
  */
-enum ZT_Vendor {
+enum ZT_Vendor
+{
 	ZT_VENDOR_UNSPECIFIED = 0,
 	ZT_VENDOR_ZEROTIER = 1
 };
@@ -885,7 +899,8 @@ enum ZT_Vendor {
 /**
  * Platform type
  */
-enum ZT_Platform {
+enum ZT_Platform
+{
 	ZT_PLATFORM_UNSPECIFIED = 0,
 	ZT_PLATFORM_LINUX = 1,
 	ZT_PLATFORM_WINDOWS = 2,
@@ -900,13 +915,15 @@ enum ZT_Platform {
 	ZT_PLATFORM_VXWORKS = 11,
 	ZT_PLATFORM_FREERTOS = 12,
 	ZT_PLATFORM_SYSBIOS = 13,
-	ZT_PLATFORM_HURD = 14
+	ZT_PLATFORM_HURD = 14,
+	ZT_PLATFORM_WEB = 15
 };
 
 /**
  * Architecture type
  */
-enum ZT_Architecture {
+enum ZT_Architecture
+{
 	ZT_ARCHITECTURE_UNSPECIFIED = 0,
 	ZT_ARCHITECTURE_X86 = 1,
 	ZT_ARCHITECTURE_X64 = 2,
@@ -921,7 +938,8 @@ enum ZT_Architecture {
 	ZT_ARCHITECTURE_SPARC32 = 11,
 	ZT_ARCHITECTURE_SPARC64 = 12,
 	ZT_ARCHITECTURE_DOTNET_CLR = 13,
-	ZT_ARCHITECTURE_JAVA_JVM = 14
+	ZT_ARCHITECTURE_JAVA_JVM = 14,
+	ZT_ARCHITECTURE_WEB = 15
 };
 
 /**
@@ -959,6 +977,11 @@ typedef struct
 	 */
 	unsigned int mtu;
 
+	/**
+	 * Recommended MTU to avoid fragmentation at the physical layer (hint)
+	 */
+	unsigned int physicalMtu;
+
 	/**
 	 * If nonzero, the network this port belongs to indicates DHCP availability
 	 *
@@ -1218,18 +1241,13 @@ typedef struct {
 	 */
 	uint64_t timestamp;
 
-	/**
-	 * Timestamp on remote device
-	 */
-	uint64_t remoteTimestamp;
-
 	/**
 	 * 64-bit packet ID of packet received by the reporting device
 	 */
 	uint64_t sourcePacketId;
 
 	/**
-	 * Flags (currently unused, will be zero)
+	 * Flags
 	 */
 	uint64_t flags;
 
@@ -1591,6 +1609,9 @@ typedef int (*ZT_PathCheckFunction)(
  * Note that this can take a few seconds the first time it's called, as it
  * will generate an identity.
  *
+ * TODO: should consolidate function pointers into versioned structure for
+ * better API stability.
+ *
  * @param node Result: pointer is set to new node instance on success
  * @param uptr User pointer to pass to functions/callbacks
  * @param now Current clock in milliseconds
@@ -1681,6 +1702,15 @@ enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame(
  */
 enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline);
 
+/**
+ * Set node's relay policy
+ *
+ * @param node Node instance
+ * @param rp New relay policy
+ * @return OK(0) or error code
+ */
+enum ZT_ResultCode ZT_Node_setRelayPolicy(ZT_Node *node,enum ZT_RelayPolicy rp);
+
 /**
  * Join a network
  *

node/Constants.hpp

@@ -236,6 +236,11 @@
  */
 #define ZT_MULTICAST_EXPLICIT_GATHER_DELAY (ZT_MULTICAST_LIKE_EXPIRE / 10)
 
+/**
+ * Expiration for credentials presented for MULTICAST_LIKE or MULTICAST_GATHER (for non-network-members)
+ */
+#define ZT_MULTICAST_CREDENTIAL_EXPIRATON ZT_MULTICAST_LIKE_EXPIRE
+
 /**
  * Timeout for outgoing multicasts
  *
@@ -263,6 +268,11 @@
  */
 #define ZT_PATH_MIN_REACTIVATE_INTERVAL 2500
 
+/**
+ * Do not accept HELLOs over a given path more often than this
+ */
+#define ZT_PATH_HELLO_RATE_LIMIT 1000
+
 /**
  * Delay between full-fledge pings of directly connected peers
  */
@@ -283,6 +293,11 @@
  */
 #define ZT_PEER_ACTIVITY_TIMEOUT 500000
 
+/**
+ * General rate limit timeout for multiple packet types (HELLO, etc.)
+ */
+#define ZT_PEER_GENERAL_INBOUND_RATE_LIMIT 1000
+
 /**
  * Delay between requests for updated network autoconf information
  *
@@ -340,6 +355,26 @@
  */
 #define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 4
 
+/**
+ * Time horizon for VERB_NETWORK_CREDENTIALS cutoff
+ */
+#define ZT_PEER_CREDENTIALS_CUTOFF_TIME 60000
+
+/**
+ * Maximum number of VERB_NETWORK_CREDENTIALS within cutoff time
+ */
+#define ZT_PEER_CREDEITIALS_CUTOFF_LIMIT 15
+
+/**
+ * General rate limit for other kinds of rate-limited packets (HELLO, credential request, etc.) both inbound and outbound
+ */
+#define ZT_PEER_GENERAL_RATE_LIMIT 1000
+
+/**
+ * How long is a path or peer considered to have a trust relationship with us (for e.g. relay policy) since last trusted established packet?
+ */
+#define ZT_TRUST_EXPIRATION 600000
+
 /**
  * Enable support for older network configurations from older (pre-1.1.6) controllers
  */

node/IncomingPacket.cpp

@@ -62,11 +62,8 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 				return true;
 			}
 		} else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) {
-			// A null pointer for peer to _doHELLO() tells it to run its own
-			// special internal authentication logic. This is done for unencrypted
-			// HELLOs to learn new identities, etc.
-			SharedPtr<Peer> tmp;
-			return _doHELLO(RR,tmp);
+			// Only HELLO is allowed in the clear, but will still have a MAC
+			return _doHELLO(RR,false);
 		}
 
 		SharedPtr<Peer> peer(RR->topology->getPeer(sourceAddress));
@@ -91,7 +88,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 					peer->received(_path,hops(),packetId(),v,0,Packet::VERB_NOP,false);
 					return true;
 
-				case Packet::VERB_HELLO:                      return _doHELLO(RR,peer);
+				case Packet::VERB_HELLO:                      return _doHELLO(RR,true);
 				case Packet::VERB_ERROR:                      return _doERROR(RR,peer);
 				case Packet::VERB_OK:                         return _doOK(RR,peer);
 				case Packet::VERB_WHOIS:                      return _doWHOIS(RR,peer);
@@ -136,6 +133,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		switch(errorCode) {
 
 			case Packet::ERROR_OBJ_NOT_FOUND:
+				// Object not found, currently only meaningful from network controllers.
 				if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) {
 					SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
 					if ((network)&&(network->controller() == peer->address()))
@@ -144,6 +142,9 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				break;
 
 			case Packet::ERROR_UNSUPPORTED_OPERATION:
+				// This can be sent in response to any operation, though right now we only
+				// consider it meaningful from network controllers. This would indicate
+				// that the queried node does not support acting as a controller.
 				if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) {
 					SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
 					if ((network)&&(network->controller() == peer->address()))
@@ -152,21 +153,47 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				break;
 
 			case Packet::ERROR_IDENTITY_COLLISION:
+				// Roots are the only peers currently permitted to state authoritatively
+				// that an identity has collided. When this occurs the node should be shut
+				// down and a new identity created. The odds of this ever happening are
+				// very low.
 				if (RR->topology->isRoot(peer->identity()))
 					RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
 				break;
 
+			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
+				// This error can be sent in response to any packet that fails network
+				// authorization. We only listen to it if it's from a peer that has recently
+				// been authorized on this network.
+				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+				if ((network)&&(network->recentlyAllowedOnNetwork(peer))) {
+					const uint64_t now = RR->node->now();
+					if (peer->rateGateComRequest(now)) {
+						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS);
+						network->config().com.serialize(outp);
+						outp.append((uint8_t)0);
+						outp.armor(peer->key(),true);
+						_path->send(RR,outp.data(),outp.size(),now);
+					}
+				}
+			}	break;
+
 			case Packet::ERROR_NETWORK_ACCESS_DENIED_: {
+				// Network controller: network access denied.
 				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
 				if ((network)&&(network->controller() == peer->address()))
 					network->setAccessDenied();
 			}	break;
 
 			case Packet::ERROR_UNWANTED_MULTICAST: {
-				uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD);
-				MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at<uint32_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14));
-				TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",nwid,peer->address().toString().c_str(),mg.toString().c_str());
-				RR->mc->remove(nwid,mg,peer->address());
+				// Members of networks can use this error to indicate that they no longer
+				// want to receive multicasts on a given channel.
+				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+				if ((network)&&(network->gate(peer,verb(),packetId()))) {
+					MulticastGroup mg(MAC(field(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8,6),6),at<uint32_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 14));
+					TRACE("%.16llx: peer %s unsubscrubed from multicast group %s",network->id(),peer->address().toString().c_str(),mg.toString().c_str());
+					RR->mc->remove(network->id(),mg,peer->address());
+				}
 			}	break;
 
 			default: break;
@@ -179,16 +206,11 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 	return true;
 }
 
-bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer)
+bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated)
 {
-	/* Note: this is the only packet ever sent in the clear, and it's also
-	 * the only packet that we authenticate via a different path. Authentication
-	 * occurs here and is based on the validity of the identity and the
-	 * integrity of the packet's MAC, but it must be done after we check
-	 * the identity since HELLO is a mechanism for learning new identities
-	 * in the first place. */
-
 	try {
+		const uint64_t now = RR->node->now();
+
 		const uint64_t pid = packetId();
 		const Address fromAddress(source());
 		const unsigned int protoVersion = (*this)[ZT_PROTO_VERB_HELLO_IDX_PROTOCOL_VERSION];
@@ -215,31 +237,30 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 			}
 		}
 
-		if (protoVersion < ZT_PROTO_VERSION_MIN) {
-			TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str());
-			return true;
-		}
 		if (fromAddress != id.address()) {
 			TRACE("dropped HELLO from %s(%s): identity not for sending address",fromAddress.toString().c_str(),_path->address().toString().c_str());
 			return true;
 		}
+		if (protoVersion < ZT_PROTO_VERSION_MIN) {
+			TRACE("dropped HELLO from %s(%s): protocol version too old",id.address().toString().c_str(),_path->address().toString().c_str());
+			return true;
+		}
 
-		if (!peer) { // peer == NULL is the normal case here
-			peer = RR->topology->getPeer(id.address());
-			if (peer) {
-				// We already have an identity with this address -- check for collisions
-
+		SharedPtr<Peer> peer(RR->topology->getPeer(id.address()));
+		if (peer) {
+			// We already have an identity with this address -- check for collisions
+			if (!alreadyAuthenticated) {
 				if (peer->identity() != id) {
 					// Identity is different from the one we already have -- address collision
 
-					unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
+					uint8_t key[ZT_PEER_SECRET_KEY_LENGTH];
 					if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
 						if (dearmor(key)) { // ensure packet is authentic, otherwise drop
 							TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_path->address().toString().c_str());
 							Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR);
-							outp.append((unsigned char)Packet::VERB_HELLO);
+							outp.append((uint8_t)Packet::VERB_HELLO);
 							outp.append((uint64_t)pid);
-							outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
+							outp.append((uint8_t)Packet::ERROR_IDENTITY_COLLISION);
 							outp.armor(key,true);
 							_path->send(RR,outp.data(),outp.size(),RR->node->now());
 						} else {
@@ -260,31 +281,39 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 
 					// Continue at // VALID
 				}
-			} else {
-				// We don't already have an identity with this address -- validate and learn it
+			} // else continue at // VALID
+		} else {
+			// We don't already have an identity with this address -- validate and learn it
 
-				// Check identity proof of work
-				if (!id.locallyValidate()) {
-					TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str());
-					return true;
-				}
+			// Sanity check: this basically can't happen
+			if (alreadyAuthenticated) {
+				TRACE("dropped HELLO from %s(%s): somehow already authenticated with unknown peer?",id.address().toString().c_str(),_path->address().toString().c_str());
+				return true;
+			}
 
-				// Check packet integrity and authentication
-				SharedPtr<Peer> newPeer(new Peer(RR,RR->identity,id));
-				if (!dearmor(newPeer->key())) {
-					TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str());
-					return true;
-				}
-				peer = RR->topology->addPeer(newPeer);
+			// Check that identity's address is valid as per the derivation function
+			if (!id.locallyValidate()) {
+				TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_path->address().toString().c_str());
+				return true;
+			}
 
-				// Continue at // VALID
+			// Check packet integrity and authentication
+			SharedPtr<Peer> newPeer(new Peer(RR,RR->identity,id));
+			if (!dearmor(newPeer->key())) {
+				TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_path->address().toString().c_str());
+				return true;
 			}
+			peer = RR->topology->addPeer(newPeer);
 
-			// VALID -- if we made it here, packet passed identity and authenticity checks!
+			// Continue at // VALID
 		}
 
+		// VALID -- if we made it here, packet passed identity and authenticity checks!
+
+		// Learn our external surface address from other peers to help us negotiate symmetric NATs
+		// and detect changes to our global IP that can trigger path renegotiation.
 		if ((externalSurfaceAddress)&&(hops() == 0))
-			RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now());
+			RR->sa->iam(id.address(),_path->localAddress(),_path->address(),externalSurfaceAddress,RR->topology->isUpstream(id),now);
 
 		Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_HELLO);
@@ -336,7 +365,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		}
 
 		outp.armor(peer->key(),true);
-		_path->send(RR,outp.data(),outp.size(),RR->node->now());
+		_path->send(RR,outp.data(),outp.size(),now);
 
 		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version
 		peer->received(_path,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP,false);
@@ -351,8 +380,15 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 	try {
 		const Packet::Verb inReVerb = (Packet::Verb)(*this)[ZT_PROTO_VERB_OK_IDX_IN_RE_VERB];
 		const uint64_t inRePacketId = at<uint64_t>(ZT_PROTO_VERB_OK_IDX_IN_RE_PACKET_ID);
+		bool trustEstablished = false;
 
-		//TRACE("%s(%s): OK(%s)",source().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb));
+		// Don't parse OK packets that are not in response to a packet ID we sent
+		if (!RR->node->expectingReplyTo(inRePacketId)) {
+			TRACE("%s(%s): OK(%s) DROPPED: not expecting reply to %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb),packetId());
+			return true;
+		}
+
+		//TRACE("%s(%s): OK(%s)",peer->address().toString().c_str(),_path->address().toString().c_str(),Packet::verbString(inReVerb));
 
 		switch(inReVerb) {
 
@@ -406,6 +442,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_NETWORK_ID);
 				const SharedPtr<Network> network(RR->node->network(nwid));
 				if ((network)&&(network->controller() == peer->address())) {
+					trustEstablished = true;
 					const unsigned int chunkLen = at<uint16_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT_LEN);
 					const void *chunkData = field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST__OK__IDX_DICT,chunkLen);
 					unsigned int chunkIndex = 0;
@@ -424,10 +461,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 
 			case Packet::VERB_MULTICAST_GATHER: {
 				const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_NETWORK_ID);
-				const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI));
-				//TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size());
-				const unsigned int count = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4);
-				RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS));
+				SharedPtr<Network> network(RR->node->network(nwid));
+				if ((network)&&(network->gateMulticastGatherReply(peer,verb(),packetId()))) {
+					trustEstablished = true;
+					const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_ADI));
+					//TRACE("%s(%s): OK(MULTICAST_GATHER) %.16llx/%s length %u",source().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),size());
+					const unsigned int count = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 4);
+					RR->mc->addMultiple(RR->node->now(),nwid,mg,field(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS + 6,count * 5),count,at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER__OK__IDX_GATHER_RESULTS));
+				}
 			}	break;
 
 			case Packet::VERB_MULTICAST_FRAME: {
@@ -437,31 +478,34 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 
 				//TRACE("%s(%s): OK(MULTICAST_FRAME) %.16llx/%s flags %.2x",peer->address().toString().c_str(),_path->address().toString().c_str(),nwid,mg.toString().c_str(),flags);
 
-				unsigned int offset = 0;
+				SharedPtr<Network> network(RR->node->network(nwid));
+				if (network) {
+					unsigned int offset = 0;
 
-				if ((flags & 0x01) != 0) { // deprecated but still used by older peers
-					CertificateOfMembership com;
-					offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS);
-					if (com) {
-						SharedPtr<Network> network(RR->node->network(com.networkId()));
-						if (network)
+					if ((flags & 0x01) != 0) { // deprecated but still used by older peers
+						CertificateOfMembership com;
+						offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS);
+						if (com)
 							network->addCredential(com);
 					}
-				}
 
-				if ((flags & 0x02) != 0) {
-					// OK(MULTICAST_FRAME) includes implicit gather results
-					offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS;
-					unsigned int totalKnown = at<uint32_t>(offset); offset += 4;
-					unsigned int count = at<uint16_t>(offset); offset += 2;
-					RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown);
+					if (network->gateMulticastGatherReply(peer,verb(),packetId())) {
+						trustEstablished = true;
+						if ((flags & 0x02) != 0) {
+							// OK(MULTICAST_FRAME) includes implicit gather results
+							offset += ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS;
+							unsigned int totalKnown = at<uint32_t>(offset); offset += 4;
+							unsigned int count = at<uint16_t>(offset); offset += 2;
+							RR->mc->addMultiple(RR->node->now(),nwid,mg,field(offset,count * 5),count,totalKnown);
+						}
+					}
 				}
 			}	break;
 
 			default: break;
 		}
 
-		peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb,trustEstablished);
 	} catch ( ... ) {
 		TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
@@ -471,6 +515,11 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
+		if (!peer->rateGateInboundWhoisRequest(RR->node->now())) {
+			TRACE("dropped WHOIS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str());
+			return true;
+		}
+
 		Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_WHOIS);
 		outp.append(packetId());
@@ -515,27 +564,29 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
-		const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
-		const SharedPtr<Peer> rendezvousWith(RR->topology->getPeer(with));
-		if (rendezvousWith) {
-			const unsigned int port = at<uint16_t>(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT);
-			const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN];
-			if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
-				const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
-				if (!RR->topology->isUpstream(peer->identity())) {
-					TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since peer is not upstream",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
-				} else if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) {
-					RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls
-					rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now());
-					TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
+		if (!RR->topology->isUpstream(peer->identity())) {
+			TRACE("RENDEZVOUS from %s ignored since source is not upstream",peer->address().toString().c_str());
+		} else {
+			const Address with(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ZTADDRESS,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+			const SharedPtr<Peer> rendezvousWith(RR->topology->getPeer(with));
+			if (rendezvousWith) {
+				const unsigned int port = at<uint16_t>(ZT_PROTO_VERB_RENDEZVOUS_IDX_PORT);
+				const unsigned int addrlen = (*this)[ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRLEN];
+				if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
+					const InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
+					if (RR->node->shouldUsePathForZeroTierTraffic(_path->localAddress(),atAddr)) {
+						RR->node->putPacket(_path->localAddress(),atAddr,"ABRE",4,2); // send low-TTL junk packet to 'open' local NAT(s) and stateful firewalls
+						rendezvousWith->attemptToContactAt(_path->localAddress(),atAddr,RR->node->now());
+						TRACE("RENDEZVOUS from %s says %s might be at %s, sent verification attempt",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
+					} else {
+						TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
+					}
 				} else {
-					TRACE("RENDEZVOUS from %s says %s might be at %s, ignoring since path is not suitable",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
+					TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str());
 				}
 			} else {
-				TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_path->address().toString().c_str());
+				TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str());
 			}
-		} else {
-			TRACE("ignored RENDEZVOUS from %s(%s) to meet unknown peer %s",peer->address().toString().c_str(),_path->address().toString().c_str(),with.toString().c_str());
 		}
 		peer->received(_path,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
@@ -549,25 +600,25 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 	try {
 		const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID);
 		const SharedPtr<Network> network(RR->node->network(nwid));
-		bool approved = false;
+		bool trustEstablished = false;
 		if (network) {
-			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
-				if (!network->isAllowed(peer)) {
-					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
-				} else {
+			if (!network->gate(peer,verb(),packetId())) {
+				TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
+			} else {
+				trustEstablished = true;
+				if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 					const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
 					const MAC sourceMac(peer->address(),nwid);
 					const unsigned int frameLen = size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 					const uint8_t *const frameData = reinterpret_cast<const uint8_t *>(data()) + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD;
 					if (network->filterIncomingPacket(peer,RR->identity.address(),sourceMac,network->mac(),frameData,frameLen,etherType,0) > 0)
 						RR->node->putFrame(nwid,network->userPtr(),sourceMac,network->mac(),etherType,0,(const void *)frameData,frameLen);
-					approved = true; // this means approved on the network in general, not this packet per se
 				}
 			}
 		} else {
 			TRACE("dropped FRAME from %s(%s): we are not a member of network %.16llx",source().toString().c_str(),_path->address().toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
-		peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,approved);
+		peer->received(_path,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished);
 	} catch ( ... ) {
 		TRACE("dropped FRAME from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
@@ -580,23 +631,23 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 		const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID);
 		const SharedPtr<Network> network(RR->node->network(nwid));
 		if (network) {
-			if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
-				const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS];
+			const unsigned int flags = (*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS];
 
-				unsigned int comLen = 0;
-				if ((flags & 0x01) != 0) { // deprecated but still used by old peers
-					CertificateOfMembership com;
-					comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
-					if (com)
-						network->addCredential(com);
-				}
+			unsigned int comLen = 0;
+			if ((flags & 0x01) != 0) { // inline COM with EXT_FRAME is deprecated but still used with old peers
+				CertificateOfMembership com;
+				comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
+				if (com)
+					network->addCredential(com);
+			}
 
-				if (!network->isAllowed(peer)) {
-					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id());
-					peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false);
-					return true;
-				}
+			if (!network->gate(peer,verb(),packetId())) {
+				TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),network->id());
+				peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false);
+				return true;
+			}
 
+			if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
 				const unsigned int etherType = at<uint16_t>(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE);
 				const MAC to(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO);
 				const MAC from(field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM);
@@ -604,7 +655,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen);
 
 				if ((!from)||(from.isMulticast())||(from == network->mac())) {
-					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str());
+					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source MAC %s",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),from.toString().c_str());
 					peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 					return true;
 				}
@@ -620,7 +671,13 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 								return true;
 							}
 						} else if (to != network->mac()) {
-							if (!network->config().permitsBridging(RR->identity.address())) {
+							if (to.isMulticast()) {
+								if (network->config().multicastLimit == 0) {
+									TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: network %.16llx does not allow multicast",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id());
+									peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
+									return true;
+								}
+							} else if (!network->config().permitsBridging(RR->identity.address())) {
 								TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: I cannot bridge to %.16llx or bridging disabled on network",from.toString().c_str(),peer->address().toString().c_str(),_path->address().toString().c_str(),to.toString().c_str(),network->id());
 								peer->received(_path,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true); // trustEstablished because COM is okay
 								return true;
@@ -647,6 +704,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
+		if (!peer->rateGateEchoRequest(RR->node->now())) {
+			TRACE("dropped ECHO from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str());
+			return true;
+		}
+
 		const uint64_t pid = packetId();
 		Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_ECHO);
@@ -655,6 +717,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 			outp.append(reinterpret_cast<const unsigned char *>(data()) + ZT_PACKET_IDX_PAYLOAD,size() - ZT_PACKET_IDX_PAYLOAD);
 		outp.armor(peer->key(),true);
 		_path->send(RR,outp.data(),outp.size(),RR->node->now());
+
 		peer->received(_path,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP,false);
 	} catch ( ... ) {
 		TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
@@ -667,14 +730,41 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 	try {
 		const uint64_t now = RR->node->now();
 
+		uint64_t authOnNetwork[256];
+		unsigned int authOnNetworkCount = 0;
+		SharedPtr<Network> network;
+		bool trustEstablished = false;
+
 		// Iterate through 18-byte network,MAC,ADI tuples
 		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18) {
 			const uint64_t nwid = at<uint64_t>(ptr);
-			const MulticastGroup group(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14));
-			RR->mc->add(now,nwid,group,peer->address());
+
+			bool auth = false;
+			for(unsigned int i=0;i<authOnNetworkCount;++i) {
+				if (nwid == authOnNetwork[i]) {
+					auth = true;
+					break;
+				}
+			}
+			if (!auth) {
+				if ((!network)||(network->id() != nwid))
+					network = RR->node->network(nwid);
+				const bool authOnNet = ((network)&&(network->gate(peer,verb(),packetId())));
+				trustEstablished |= authOnNet;
+				if (authOnNet||RR->mc->cacheAuthorized(peer->address(),nwid,now)) {
+					auth = true;
+					if (authOnNetworkCount < 256) // sanity check, packets can't really be this big
+						authOnNetwork[authOnNetworkCount++] = nwid;
+				}
+			}
+
+			if (auth) {
+				const MulticastGroup group(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14));
+				RR->mc->add(now,nwid,group,peer->address());
+			}
 		}
 
-		peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,trustEstablished);
 	} catch ( ... ) {
 		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
@@ -684,9 +774,15 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
+		if (!peer->rateGateCredentialsReceived(RR->node->now())) {
+			TRACE("dropped NETWORK_CREDENTIALS from %s(%s): rate limit circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str());
+			return true;
+		}
+
 		CertificateOfMembership com;
 		Capability cap;
 		Tag tag;
+		bool trustEstablished = false;
 
 		unsigned int p = ZT_PACKET_IDX_PAYLOAD;
 		while ((p < size())&&((*this)[p])) {
@@ -694,9 +790,11 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S
 			if (com) {
 				SharedPtr<Network> network(RR->node->network(com.networkId()));
 				if (network) {
-					if (network->addCredential(com) == 1)
-						return false; // wait for WHOIS
-				}
+					switch (network->addCredential(com)) {
+						case 0: trustEstablished = true; break;
+						case 1: return false; // wait for WHOIS
+					}
+				} else RR->mc->addCredential(com,false);
 			}
 		}
 		++p; // skip trailing 0 after COMs if present
@@ -707,8 +805,10 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S
 				p += cap.deserialize(*this,p);
 				SharedPtr<Network> network(RR->node->network(cap.networkId()));
 				if (network) {
-					if (network->addCredential(cap) == 1)
-						return false; // wait for WHOIS
+					switch (network->addCredential(cap)) {
+						case 0: trustEstablished = true; break;
+						case 1: return false; // wait for WHOIS
+					}
 				}
 			}
 
@@ -717,13 +817,15 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const S
 				p += tag.deserialize(*this,p);
 				SharedPtr<Network> network(RR->node->network(tag.networkId()));
 				if (network) {
-					if (network->addCredential(tag) == 1)
-						return false; // wait for WHOIS
+					switch (network->addCredential(tag)) {
+						case 0: trustEstablished = true; break;
+						case 1: return false; // wait for WHOIS
+					}
 				}
 			}
 		}
 
-		peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished);
 	} catch ( ... ) {
 		TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
@@ -734,22 +836,21 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 {
 	try {
 		const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_NETWORK_ID);
-
-		const unsigned int metaDataLength = at<uint16_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN);
-		const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength);
-		const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> metaData(metaDataBytes,metaDataLength);
-
 		const unsigned int hopCount = hops();
 		const uint64_t requestPacketId = packetId();
-		bool netconfOk = false;
+		bool trustEstablished = false;
 
 		if (RR->localNetworkController) {
+			const unsigned int metaDataLength = at<uint16_t>(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT_LEN);
+			const char *metaDataBytes = (const char *)field(ZT_PROTO_VERB_NETWORK_CONFIG_REQUEST_IDX_DICT,metaDataLength);
+			const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> metaData(metaDataBytes,metaDataLength);
+
 			NetworkConfig *netconf = new NetworkConfig();
 			try {
 				switch(RR->localNetworkController->doNetworkConfigRequest((hopCount > 0) ? InetAddress() : _path->address(),RR->identity,peer->identity(),nwid,metaData,*netconf)) {
 
 					case NetworkController::NETCONF_QUERY_OK: {
-						netconfOk = true;
+						trustEstablished = true;
 						Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *dconf = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
 						try {
 							if (netconf->toDictionary(*dconf,metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6)) {
@@ -821,7 +922,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			_path->send(RR,outp.data(),outp.size(),RR->node->now());
 		}
 
-		peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,netconfOk);
+		peer->received(_path,hopCount,requestPacketId,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,trustEstablished);
 	} catch (std::exception &exc) {
 		fprintf(stderr,"WARNING: network config request failed with exception: %s" ZT_EOL_S,exc.what());
 		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): %s",source().toString().c_str(),_path->address().toString().c_str(),exc.what());
@@ -836,11 +937,13 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 {
 	try {
 		const uint64_t nwid = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD);
+		bool trustEstablished = false;
 
 		if (Network::controllerFor(nwid) == peer->address()) {
 			SharedPtr<Network> network(RR->node->network(nwid));
 			if (network) {
 				network->requestConfiguration();
+				trustEstablished = true;
 			} else {
 				TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): not a member of %.16llx",source().toString().c_str(),_path->address().toString().c_str(),nwid);
 				peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false);
@@ -855,7 +958,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 			}
 		}
 
-		peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP,trustEstablished);
 	} catch ( ... ) {
 		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_path->address().toString().c_str());
 	}
@@ -872,21 +975,24 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 
 		//TRACE("<<MC %s(%s) GATHER up to %u in %.16llx/%s",source().toString().c_str(),_path->address().toString().c_str(),gatherLimit,nwid,mg.toString().c_str());
 
+		const SharedPtr<Network> network(RR->node->network(nwid));
+
 		if ((flags & 0x01) != 0) {
 			try {
 				CertificateOfMembership com;
 				com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM);
 				if (com) {
-					SharedPtr<Network> network(RR->node->network(nwid));
 					if (network)
 						network->addCredential(com);
+					else RR->mc->addCredential(com,false);
 				}
 			} catch ( ... ) {
 				TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_path->address().toString().c_str());
 			}
 		}
 
-		if (gatherLimit) {
+		const bool trustEstablished = ((network)&&(network->gate(peer,verb(),packetId())));
+		if ( ( trustEstablished || RR->mc->cacheAuthorized(peer->address(),nwid,RR->node->now()) ) && (gatherLimit > 0) ) {
 			Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
 			outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER);
 			outp.append(packetId());
@@ -906,7 +1012,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 #endif
 		}
 
-		peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,false);
+		peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished);
 	} catch ( ... ) {
 		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_path->address().toString().c_str());
 	}
@@ -932,14 +1038,18 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 					network->addCredential(com);
 			}
 
-			// Check membership after we've read any included COM, since
-			// that cert might be what we needed.
-			if (!network->isAllowed(peer)) {
+			if (!network->gate(peer,verb(),packetId())) {
 				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
 				peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false);
 				return true;
 			}
 
+			if (network->config().multicastLimit == 0) {
+				TRACE("dropped MULTICAST_FRAME from %s(%s): network %.16llx does not allow multicast",peer->address().toString().c_str(),_path->address().toString().c_str(),(unsigned long long)network->id());
+				peer->received(_path,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false);
+				return true;
+			}
+
 			unsigned int gatherLimit = 0;
 			if ((flags & 0x02) != 0) {
 				gatherLimit = at<uint32_t>(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT);
@@ -1018,7 +1128,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 		const uint64_t now = RR->node->now();
 
 		// First, subject this to a rate limit
-		if (!peer->shouldRespondToDirectPathPush(now)) {
+		if (!peer->rateGatePushDirectPaths(now)) {
 			TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): circuit breaker tripped",source().toString().c_str(),_path->address().toString().c_str());
 			peer->received(_path,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false);
 			return true;
@@ -1139,6 +1249,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		// Add length of second "additional fields" section.
 		vlf += at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 29 + vlf);
 
+		uint64_t reportFlags = 0;
+
 		// Check credentials (signature already verified)
 		if (originatorCredentialNetworkId) {
 			SharedPtr<Network> network(RR->node->network(originatorCredentialNetworkId));
@@ -1147,6 +1259,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 				peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
 				return true;
 			}
+			if (network->gate(peer,verb(),packetId()))
+				reportFlags |= ZT_CIRCUIT_TEST_REPORT_FLAGS_UPSTREAM_AUTHORIZED_IN_PATH;
 		} else {
 			TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_path->address().toString().c_str(),originatorAddress.toString().c_str());
 			peer->received(_path,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP,false);
@@ -1188,7 +1302,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 			outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED);
 			outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED);
 			outp.append((uint16_t)0); // error code, currently unused
-			outp.append((uint64_t)0); // flags, currently unused
+			outp.append((uint64_t)reportFlags);
 			outp.append((uint64_t)packetId());
 			peer->address().appendTo(outp);
 			outp.append((uint8_t)hops());
@@ -1237,7 +1351,6 @@ bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const S
 		report.upstream = Address(field(ZT_PACKET_IDX_PAYLOAD + 52,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH).toInt();
 		report.testId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 8);
 		report.timestamp = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD);
-		report.remoteTimestamp = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 16);
 		report.sourcePacketId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 44);
 		report.flags = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 36);
 		report.sourcePacketHopCount = (*this)[ZT_PACKET_IDX_PAYLOAD + 57]; // end of fixed length headers: 58

node/IncomingPacket.hpp

@@ -136,7 +136,7 @@ private:
 	// These are called internally to handle packet contents once it has
 	// been authenticated, decrypted, decompressed, and classified.
 	bool _doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
-	bool _doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer); // can be called with NULL peer, while all others cannot
+	bool _doHELLO(const RuntimeEnvironment *RR,const bool alreadyAuthenticated);
 	bool _doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);

node/Membership.cpp

@@ -24,13 +24,13 @@
 #include "Packet.hpp"
 #include "Node.hpp"
 
-#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 4)
+#define ZT_CREDENTIAL_PUSH_EVERY (ZT_NETWORK_AUTOCONF_DELAY / 3)
 
 namespace ZeroTier {
 
 void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint64_t now,const Address &peerAddress,const NetworkConfig &nconf,const Capability *cap)
 {
-	if ((now - _lastPushAttempt) < 1000ULL)
+	if ((now - _lastPushAttempt) < 2000ULL)
 		return;
 	_lastPushAttempt = now;
 
@@ -71,7 +71,7 @@ void Membership::sendCredentialsIfNeeded(const RuntimeEnvironment *RR,const uint
 			}
 			capsAndTags.setAt<uint16_t>(tagCountPos,(uint16_t)appendedTags);
 
-			const bool needCom = ((nconf.isPrivate())&&(nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY));
+			const bool needCom = ((nconf.com)&&((now - _lastPushedCom) >= ZT_CREDENTIAL_PUSH_EVERY));
 			if ( (needCom) || (appendedCaps) || (appendedTags) ) {
 				Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS);
 				if (needCom) {
@@ -99,9 +99,11 @@ int Membership::addCredential(const RuntimeEnvironment *RR,const CertificateOfMe
 	const int vr = com.verify(RR);
 
 	if (vr == 0) {
-		TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId());
-		if (com.timestamp().first > _com.timestamp().first) {
+		if (com.timestamp().first >= _com.timestamp().first) {
+			TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED (new)",com.issuedTo().toString().c_str(),com.networkId());
 			_com = com;
+		} else {
+			TRACE("addCredential(CertificateOfMembership) for %s on %.16llx ACCEPTED but not used (OK but older than current)",com.issuedTo().toString().c_str(),com.networkId());
 		}
 	} else {
 		TRACE("addCredential(CertificateOfMembership) for %s on %.16llx REJECTED (%d)",com.issuedTo().toString().c_str(),com.networkId(),vr);

node/Membership.hpp

@@ -154,6 +154,23 @@ public:
 		return nconf.com.agreesWith(_com);
 	}
 
+	/**
+	 * @return True if this member has been on this network recently (or network is public)
+	 */
+	inline bool recentlyAllowedOnNetwork(const NetworkConfig &nconf) const
+	{
+		if (nconf.isPublic())
+			return true;
+		if (_com) {
+			const uint64_t a = _com.timestamp().first;
+			if ((_blacklistBefore)&&(a <= _blacklistBefore))
+				return false;
+			const uint64_t b = nconf.com.timestamp().first;
+			return ((a <= b) ? ((b - a) <= ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA) : true);
+		}
+		return false;
+	}
+
 	/**
 	 * Check whether a capability or tag is within its max delta from the timestamp of our network config and newer than any blacklist cutoff time
 	 *

node/Multicaster.cpp

@@ -34,8 +34,8 @@ namespace ZeroTier {
 
 Multicaster::Multicaster(const RuntimeEnvironment *renv) :
 	RR(renv),
-	_groups(1024),
-	_groups_m()
+	_groups(256),
+	_gatherAuth(256)
 {
 }
 
@@ -244,7 +244,7 @@ void Multicaster::send(
 				}
 
 				for(unsigned int k=0;k<numExplicitGatherPeers;++k) {
-					const CertificateOfMembership *com = (network) ? (((network->config())&&(network->config().isPrivate())) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0;
+					const CertificateOfMembership *com = (network) ? ((network->config().com) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0;
 					Packet outp(explicitGatherPeers[k],RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
 					outp.append(nwid);
 					outp.append((uint8_t)((com) ? 0x01 : 0x00));
@@ -253,6 +253,7 @@ void Multicaster::send(
 					outp.append((uint32_t)gatherLimit);
 					if (com)
 						com->serialize(outp);
+					RR->node->expectReplyTo(outp.packetId());
 					RR->sw->send(outp,true);
 				}
 			}
@@ -300,42 +301,62 @@ void Multicaster::send(
 
 void Multicaster::clean(uint64_t now)
 {
-	Mutex::Lock _l(_groups_m);
-
-	Multicaster::Key *k = (Multicaster::Key *)0;
-	MulticastGroupStatus *s = (MulticastGroupStatus *)0;
-	Hashtable<Multicaster::Key,MulticastGroupStatus>::Iterator mm(_groups);
-	while (mm.next(k,s)) {
-		for(std::list<OutboundMulticast>::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) {
-			if ((tx->expired(now))||(tx->atLimit()))
-				s->txQueue.erase(tx++);
-			else ++tx;
-		}
+	{
+		Mutex::Lock _l(_groups_m);
+		Multicaster::Key *k = (Multicaster::Key *)0;
+		MulticastGroupStatus *s = (MulticastGroupStatus *)0;
+		Hashtable<Multicaster::Key,MulticastGroupStatus>::Iterator mm(_groups);
+		while (mm.next(k,s)) {
+			for(std::list<OutboundMulticast>::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) {
+				if ((tx->expired(now))||(tx->atLimit()))
+					s->txQueue.erase(tx++);
+				else ++tx;
+			}
 
-		unsigned long count = 0;
-		{
-			std::vector<MulticastGroupMember>::iterator reader(s->members.begin());
-			std::vector<MulticastGroupMember>::iterator writer(reader);
-			while (reader != s->members.end()) {
-				if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
-					*writer = *reader;
-					++writer;
-					++count;
+			unsigned long count = 0;
+			{
+				std::vector<MulticastGroupMember>::iterator reader(s->members.begin());
+				std::vector<MulticastGroupMember>::iterator writer(reader);
+				while (reader != s->members.end()) {
+					if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
+						*writer = *reader;
+						++writer;
+						++count;
+					}
+					++reader;
 				}
-				++reader;
+			}
+
+			if (count) {
+				s->members.resize(count);
+			} else if (s->txQueue.empty()) {
+				_groups.erase(*k);
+			} else {
+				s->members.clear();
 			}
 		}
+	}
 
-		if (count) {
-			s->members.resize(count);
-		} else if (s->txQueue.empty()) {
-			_groups.erase(*k);
-		} else {
-			s->members.clear();
+	{
+		Mutex::Lock _l(_gatherAuth_m);
+		_GatherAuthKey *k = (_GatherAuthKey *)0;
+		uint64_t *ts = (uint64_t *)ts;
+		Hashtable<_GatherAuthKey,uint64_t>::Iterator i(_gatherAuth);
+		while (i.next(k,ts)) {
+			if ((now - *ts) >= ZT_MULTICAST_CREDENTIAL_EXPIRATON)
+				_gatherAuth.erase(*k);
 		}
 	}
 }
 
+void Multicaster::addCredential(const CertificateOfMembership &com,bool alreadyValidated)
+{
+	if ((alreadyValidated)||(com.verify(RR) == 0)) {
+		Mutex::Lock _l(_gatherAuth_m);
+		_gatherAuth[_GatherAuthKey(com.networkId(),com.issuedTo())] = RR->node->now();
+	}
+}
+
 void Multicaster::_add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member)
 {
 	// assumes _groups_m is locked

node/Multicaster.hpp

@@ -179,12 +179,52 @@ public:
 	 */
 	void clean(uint64_t now);
 
+	/**
+	 * Add an authorization credential
+	 *
+	 * The Multicaster keeps its own track of when valid credentials of network
+	 * membership are presented. This allows it to control MULTICAST_LIKE
+	 * GATHER authorization for networks this node does not belong to.
+	 *
+	 * @param com Certificate of membership
+	 * @param alreadyValidated If true, COM has already been checked and found to be valid and signed
+	 */
+	void addCredential(const CertificateOfMembership &com,bool alreadyValidated);
+
+	/**
+	 * Check authorization for GATHER and LIKE for non-network-members
+	 *
+	 * @param a Address of peer
+	 * @param nwid Network ID
+	 * @param now Current time
+	 * @return True if GATHER and LIKE should be allowed
+	 */
+	bool cacheAuthorized(const Address &a,const uint64_t nwid,const uint64_t now) const
+	{
+		Mutex::Lock _l(_gatherAuth_m);
+		const uint64_t *p = _gatherAuth.get(_GatherAuthKey(nwid,a));
+		return ((p)&&((now - *p) < ZT_MULTICAST_CREDENTIAL_EXPIRATON));
+	}
+
 private:
 	void _add(uint64_t now,uint64_t nwid,const MulticastGroup &mg,MulticastGroupStatus &gs,const Address &member);
 
 	const RuntimeEnvironment *RR;
+
 	Hashtable<Multicaster::Key,MulticastGroupStatus> _groups;
 	Mutex _groups_m;
+
+	struct _GatherAuthKey
+	{
+		_GatherAuthKey() : member(0),networkId(0) {}
+		_GatherAuthKey(const uint64_t nwid,const Address &a) : member(a.toInt()),networkId(nwid) {}
+		inline unsigned long hashCode() const { return (member ^ networkId); }
+		inline bool operator==(const _GatherAuthKey &k) const { return ((member == k.member)&&(networkId == k.networkId)); }
+		uint64_t member;
+		uint64_t networkId;
+	};
+	Hashtable< _GatherAuthKey,uint64_t > _gatherAuth;
+	Mutex _gatherAuth_m;
 };
 
 } // namespace ZeroTier

node/Network.cpp

@@ -656,8 +656,12 @@ bool Network::filterOutgoingPacket(
 
 	Mutex::Lock _l(_lock);
 
-	Membership &m = _memberships[ztDest];
-	const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS);
+	Membership *m = (Membership *)0;
+	unsigned int remoteTagCount = 0;
+	if (ztDest) {
+		m = &(_memberships[ztDest]);
+		remoteTagCount = m->getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS);
+	}
 
 	switch(_doZtFilter(RR,_config,false,ztSource,ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) {
 
@@ -737,8 +741,8 @@ bool Network::filterOutgoingPacket(
 			RR->sw->send(outp,true);
 
 			return false; // DROP locally, since we redirected
-		} else if (ztDest) {
-			m.sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,relevantCap);
+		} else if (m) {
+			m->sendCredentialsIfNeeded(RR,RR->node->now(),ztDest,_config,relevantCap);
 		}
 	}
 
@@ -764,7 +768,7 @@ int Network::filterIncomingPacket(
 
 	Mutex::Lock _l(_lock);
 
-	Membership &m = _membership(ztDest);
+	Membership &m = _membership(sourcePeer->address());
 	const unsigned int remoteTagCount = m.getAllTags(_config,remoteTagIds,remoteTagValues,ZT_MAX_NETWORK_TAGS);
 
 	switch (_doZtFilter(RR,_config,true,sourcePeer->address(),ztDest2,macSource,macDest,frameData,frameLen,etherType,vlanId,_config.rules,_config.ruleCount,_config.tags,_config.tagCount,remoteTagIds,remoteTagValues,remoteTagCount,cc,ccLength)) {
@@ -862,31 +866,24 @@ bool Network::subscribedToMulticastGroup(const MulticastGroup &mg,bool includeBr
 		return true;
 	else if (includeBridgedGroups)
 		return _multicastGroupsBehindMe.contains(mg);
-	else return false;
+	return false;
 }
 
 void Network::multicastSubscribe(const MulticastGroup &mg)
 {
-	{
-		Mutex::Lock _l(_lock);
-		if (std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg))
-			return;
-		_myMulticastGroups.push_back(mg);
-		std::sort(_myMulticastGroups.begin(),_myMulticastGroups.end());
-		_announceMulticastGroups(&mg);
+	Mutex::Lock _l(_lock);
+	if (!std::binary_search(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg)) {
+		_myMulticastGroups.insert(std::upper_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg),mg);
+		_sendUpdatesToMembers(&mg);
 	}
 }
 
 void Network::multicastUnsubscribe(const MulticastGroup &mg)
 {
 	Mutex::Lock _l(_lock);
-	std::vector<MulticastGroup> nmg;
-	for(std::vector<MulticastGroup>::const_iterator i(_myMulticastGroups.begin());i!=_myMulticastGroups.end();++i) {
-		if (*i != mg)
-			nmg.push_back(*i);
-	}
-	if (nmg.size() != _myMulticastGroups.size())
-		_myMulticastGroups.swap(nmg);
+	std::vector<MulticastGroup>::iterator i(std::lower_bound(_myMulticastGroups.begin(),_myMulticastGroups.end(),mg));
+	if ( (i != _myMulticastGroups.end()) && (*i == mg) )
+		_myMulticastGroups.erase(i);
 }
 
 bool Network::applyConfiguration(const NetworkConfig &conf)
@@ -1004,6 +1001,7 @@ void Network::requestConfiguration()
 
 	Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> rmd;
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION);
+	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_VENDOR,(uint64_t)ZT_VENDOR_ZEROTIER);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MAJOR);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MINOR);
@@ -1014,6 +1012,7 @@ void Network::requestConfiguration()
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS,(uint64_t)ZT_MAX_NETWORK_TAGS);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS,(uint64_t)0);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,(uint64_t)ZT_RULES_ENGINE_REVISION);
+	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY,(uint64_t)RR->node->relayPolicy());
 
 	if (ctrl == RR->identity.address()) {
 		if (RR->localNetworkController) {
@@ -1050,12 +1049,56 @@ void Network::requestConfiguration()
 	} else {
 		outp.append((unsigned char)0,16);
 	}
+
+	RR->node->expectReplyTo(_inboundConfigPacketId = outp.packetId());
+	_inboundConfigChunks.clear();
+
 	outp.compress();
 	RR->sw->send(outp,true);
+}
 
-	// Expect replies with this in-re packet ID
-	_inboundConfigPacketId = outp.packetId();
-	_inboundConfigChunks.clear();
+bool Network::gate(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uint64_t packetId)
+{
+	const uint64_t now = RR->node->now();
+	Mutex::Lock _l(_lock);
+	try {
+		if (_config) {
+			Membership &m = _membership(peer->address());
+			const bool allow = m.isAllowedOnNetwork(_config);
+			if (allow) {
+				m.sendCredentialsIfNeeded(RR,now,peer->address(),_config,(const Capability *)0);
+				if (m.shouldLikeMulticasts(now)) {
+					_announceMulticastGroupsTo(peer->address(),_allMulticastGroups());
+					m.likingMulticasts(now);
+				}
+			} else if (m.recentlyAllowedOnNetwork(_config)&&peer->rateGateRequestCredentials(now)) {
+				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
+				outp.append((uint8_t)verb);
+				outp.append(packetId);
+				outp.append((uint8_t)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
+				outp.append(_id);
+				RR->sw->send(outp,true);
+			}
+			return allow;
+		}
+	} catch ( ... ) {
+		TRACE("gate() check failed for peer %s: unexpected exception",peer->address().toString().c_str());
+	}
+	return false;
+}
+
+bool Network::gateMulticastGatherReply(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uint64_t packetId)
+{
+	return ( (peer->address() == controller()) || RR->topology->isUpstream(peer->identity()) || gate(peer,verb,packetId) || _config.isAnchor(peer->address()) );
+}
+
+bool Network::recentlyAllowedOnNetwork(const SharedPtr<Peer> &peer) const
+{
+	Mutex::Lock _l(_lock);
+	const Membership *m = _memberships.get(peer->address());
+	if (m)
+		return m->recentlyAllowedOnNetwork(_config);
+	return false;
 }
 
 void Network::clean()
@@ -1131,7 +1174,22 @@ void Network::learnBridgedMulticastGroup(const MulticastGroup &mg,uint64_t now)
 	const unsigned long tmp = (unsigned long)_multicastGroupsBehindMe.size();
 	_multicastGroupsBehindMe.set(mg,now);
 	if (tmp != _multicastGroupsBehindMe.size())
-		_announceMulticastGroups(&mg);
+		_sendUpdatesToMembers(&mg);
+}
+
+int Network::addCredential(const CertificateOfMembership &com)
+{
+	if (com.networkId() != _id)
+		return -1;
+	const Address a(com.issuedTo());
+	Mutex::Lock _l(_lock);
+	Membership &m = _membership(a);
+	const int result = m.addCredential(RR,com);
+	if (result == 0) {
+		m.sendCredentialsIfNeeded(RR,RR->node->now(),a,_config,(const Capability *)0);
+		RR->mc->addCredential(com,true);
+	}
+	return result;
 }
 
 void Network::destroy()
@@ -1168,6 +1226,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	ec->status = _status();
 	ec->type = (_config) ? (_config.isPrivate() ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC) : ZT_NETWORK_TYPE_PRIVATE;
 	ec->mtu = ZT_IF_MTU;
+	ec->physicalMtu = ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 16);
 	ec->dhcp = 0;
 	std::vector<Address> ab(_config.activeBridges());
 	ec->bridge = ((_config.allowPassiveBridging())||(std::find(ab.begin(),ab.end(),RR->identity.address()) != ab.end())) ? 1 : 0;
@@ -1196,40 +1255,25 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	}
 }
 
-bool Network::_isAllowed(const SharedPtr<Peer> &peer) const
-{
-	// Assumes _lock is locked
-	try {
-		if (_config) {
-			const Membership *const m = _memberships.get(peer->address());
-			if (m)
-				return m->isAllowedOnNetwork(_config);
-		}
-	} catch ( ... ) {
-		TRACE("isAllowed() check failed for peer %s: unexpected exception",peer->address().toString().c_str());
-	}
-	return false;
-}
-
-void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis)
+void Network::_sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup)
 {
 	// Assumes _lock is locked
 	const uint64_t now = RR->node->now();
 
 	std::vector<MulticastGroup> groups;
-	if (onlyThis)
-		groups.push_back(*onlyThis);
+	if (newMulticastGroup)
+		groups.push_back(*newMulticastGroup);
 	else groups = _allMulticastGroups();
 
-	if ((onlyThis)||((now - _lastAnnouncedMulticastGroupsUpstream) >= ZT_MULTICAST_ANNOUNCE_PERIOD)) {
-		if (!onlyThis)
+	if ((newMulticastGroup)||((now - _lastAnnouncedMulticastGroupsUpstream) >= ZT_MULTICAST_ANNOUNCE_PERIOD)) {
+		if (!newMulticastGroup)
 			_lastAnnouncedMulticastGroupsUpstream = now;
 
 		// Announce multicast groups to upstream peers (roots, etc.) and also send
 		// them our COM so that MULTICAST_GATHER can be authenticated properly.
 		const std::vector<Address> upstreams(RR->topology->upstreamAddresses());
 		for(std::vector<Address>::const_iterator a(upstreams.begin());a!=upstreams.end();++a) {
-			if ((_config.isPrivate())&&(_config.com)) {
+			if (_config.com) {
 				Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS);
 				_config.com.serialize(outp);
 				outp.append((uint8_t)0x00);
@@ -1238,12 +1282,17 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis)
 			_announceMulticastGroupsTo(*a,groups);
 		}
 
-		// Announce to controller, which does not need our COM since it obviously
-		// knows if we are a member. Of course if we already did or are going to
-		// below then we can skip it here.
+		// Also announce to controller, and send COM to simplify and generalize behavior even though in theory it does not need it
 		const Address c(controller());
-		if ( (std::find(upstreams.begin(),upstreams.end(),c) == upstreams.end()) && (!_memberships.contains(c)) )
+		if ( (std::find(upstreams.begin(),upstreams.end(),c) == upstreams.end()) && (!_memberships.contains(c)) ) {
+			if (_config.com) {
+				Packet outp(c,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS);
+				_config.com.serialize(outp);
+				outp.append((uint8_t)0x00);
+				RR->sw->send(outp,true);
+			}
 			_announceMulticastGroupsTo(c,groups);
+		}
 	}
 
 	// Make sure that all "network anchors" have Membership records so we will
@@ -1251,19 +1300,21 @@ void Network::_announceMulticastGroups(const MulticastGroup *const onlyThis)
 	// piecemeal on-demand fashion.
 	const std::vector<Address> anchors(_config.anchors());
 	for(std::vector<Address>::const_iterator a(anchors.begin());a!=anchors.end();++a)
-		_memberships[*a];
+		_membership(*a);
 
-	// Send MULTICAST_LIKE(s) to all members of this network
+	// Send credentials and multicast LIKEs to members, upstreams, and controller
 	{
 		Address *a = (Address *)0;
 		Membership *m = (Membership *)0;
 		Hashtable<Address,Membership>::Iterator i(_memberships);
 		while (i.next(a,m)) {
-			if ((onlyThis)||(m->shouldLikeMulticasts(now))) {
-				if (!onlyThis)
-					m->likingMulticasts(now);
+			if ( (m->recentlyAllowedOnNetwork(_config)) || (std::find(anchors.begin(),anchors.end(),*a) != anchors.end()) ) {
 				m->sendCredentialsIfNeeded(RR,RR->node->now(),*a,_config,(const Capability *)0);
-				_announceMulticastGroupsTo(*a,groups);
+				if ( ((newMulticastGroup)||(m->shouldLikeMulticasts(now))) && (m->isAllowedOnNetwork(_config)) ) {
+					if (!newMulticastGroup)
+						m->likingMulticasts(now);
+					_announceMulticastGroupsTo(*a,groups);
+				}
 			}
 		}
 	}
@@ -1310,15 +1361,7 @@ std::vector<MulticastGroup> Network::_allMulticastGroups() const
 Membership &Network::_membership(const Address &a)
 {
 	// assumes _lock is locked
-	const unsigned long ms = _memberships.size();
-	Membership &m = _memberships[a];
-	if (ms != _memberships.size()) {
-		const uint64_t now = RR->node->now();
-		m.sendCredentialsIfNeeded(RR,now,a,_config,(const Capability *)0);
-		_announceMulticastGroupsTo(a,_allMulticastGroups());
-		m.likingMulticasts(now);
-	}
-	return m;
+	return _memberships[a];
 }
 
 } // namespace ZeroTier

node/Network.hpp

@@ -48,7 +48,6 @@ namespace ZeroTier {
 
 class RuntimeEnvironment;
 class Peer;
-class _MulticastAnnounceAll;
 
 /**
  * A virtual LAN
@@ -56,7 +55,6 @@ class _MulticastAnnounceAll;
 class Network : NonCopyable
 {
 	friend class SharedPtr<Network>;
-	friend class _MulticastAnnounceAll; // internal function object
 
 public:
 	/**
@@ -250,14 +248,25 @@ public:
 	void requestConfiguration();
 
 	/**
+	 * Membership check gate for incoming packets related to this network
+	 *
 	 * @param peer Peer to check
+	 * @param verb Packet verb
+	 * @param packetId Packet ID
 	 * @return True if peer is allowed to communicate on this network
 	 */
-	inline bool isAllowed(const SharedPtr<Peer> &peer) const
-	{
-		Mutex::Lock _l(_lock);
-		return _isAllowed(peer);
-	}
+	bool gate(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uint64_t packetId);
+
+	/**
+	 * Check whether this peer is allowed to provide multicast info for this network
+	 */
+	bool gateMulticastGatherReply(const SharedPtr<Peer> &peer,const Packet::Verb verb,const uint64_t packetId);
+
+	/**
+	 * @param peer Peer to check
+	 * @return True if peer has recently been a valid member of this network
+	 */
+	bool recentlyAllowedOnNetwork(const SharedPtr<Peer> &peer) const;
 
 	/**
 	 * Perform cleanup and possibly save state
@@ -265,12 +274,12 @@ public:
 	void clean();
 
 	/**
-	 * Announce multicast groups to all members, anchors, etc.
+	 * Push state to members such as multicast group memberships and latest COM (if needed)
 	 */
-	inline void announceMulticastGroups()
+	inline void sendUpdatesToMembers()
 	{
 		Mutex::Lock _l(_lock);
-		_announceMulticastGroups((const MulticastGroup *)0);
+		_sendUpdatesToMembers((const MulticastGroup *)0);
 	}
 
 	/**
@@ -323,9 +332,7 @@ public:
 	{
 		Mutex::Lock _l(_lock);
 		const Address *const br = _remoteBridgeRoutes.get(mac);
-		if (br)
-			return *br;
-		return Address();
+		return ((br) ? *br : Address());
 	}
 
 	/**
@@ -348,13 +355,7 @@ public:
 	 * @param com Certificate of membership
 	 * @return 0 == OK, 1 == waiting for WHOIS, -1 == BAD signature or credential
 	 */
-	inline int addCredential(const CertificateOfMembership &com)
-	{
-		if (com.networkId() != _id)
-			return -1;
-		Mutex::Lock _l(_lock);
-		return _membership(com.issuedTo()).addCredential(RR,com);
-	}
+	int addCredential(const CertificateOfMembership &com);
 
 	/**
 	 * @param cap Capability
@@ -408,11 +409,11 @@ public:
 private:
 	ZT_VirtualNetworkStatus _status() const;
 	void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked
-	bool _isAllowed(const SharedPtr<Peer> &peer) const;
-	void _announceMulticastGroups(const MulticastGroup *const onlyThis);
+	bool _gate(const SharedPtr<Peer> &peer);
+	void _sendUpdatesToMembers(const MulticastGroup *const newMulticastGroup);
 	void _announceMulticastGroupsTo(const Address &peer,const std::vector<MulticastGroup> &allMulticastGroups);
 	std::vector<MulticastGroup> _allMulticastGroups() const;
-	Membership &_membership(const Address &a); // also lazily sends COM and MULTICAST_LIKE(s) if this is a new member
+	Membership &_membership(const Address &a);
 
 	const RuntimeEnvironment *RR;
 	void *_uPtr;

node/NetworkConfig.hpp

@@ -108,9 +108,13 @@ namespace ZeroTier {
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION "v"
 // Protocol version (see Packet.hpp)
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION "pv"
-// Software major, minor, revision
+// Software vendor
+#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_VENDOR "vend"
+// Software major version
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv"
+// Software minor version
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv"
+// Software revision
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv"
 // Rules engine revision
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV "revr"
@@ -123,9 +127,11 @@ namespace ZeroTier {
 // Maximum number of tags this node can accept
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_TAGS "mt"
 // Network join authorization token (if any)
-#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH_TOKEN "atok"
+#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH "a"
 // Network configuration meta-data flags
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_FLAGS "f"
+// Relay policy for this node
+#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_RELAY_POLICY "rp"
 
 // These dictionary keys are short so they don't take up much room.
 // By convention we use upper case for binary blobs, but it doesn't really matter.
@@ -285,6 +291,19 @@ public:
 		return r;
 	}
 
+	/**
+	 * @param a Address to check
+	 * @return True if address is an anchor
+	 */
+	inline bool isAnchor(const Address &a) const
+	{
+		for(unsigned int i=0;i<specialistCount;++i) {
+			if ((a == specialists[i])&&((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR) != 0))
+				return true;
+		}
+		return false;
+	}
+
 	/**
 	 * @param fromPeer Peer attempting to bridge other Ethernet peers onto network
 	 * @return True if this network allows bridging

node/Node.cpp

@@ -71,10 +71,14 @@ Node::Node(
 	_prngStreamPtr(0),
 	_now(now),
 	_lastPingCheck(0),
-	_lastHousekeepingRun(0)
+	_lastHousekeepingRun(0),
+	_relayPolicy(ZT_RELAY_POLICY_TRUSTED)
 {
 	_online = false;
 
+	memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr));
+	memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo));
+
 	// Use Salsa20 alone as a high-quality non-crypto PRNG
 	{
 		char foo[32];
@@ -115,6 +119,9 @@ Node::Node(
 		throw;
 	}
 
+	if (RR->topology->amRoot())
+		_relayPolicy = ZT_RELAY_POLICY_ALWAYS;
+
 	postEvent(ZT_EVENT_UP);
 }
 
@@ -128,6 +135,7 @@ Node::~Node()
 	delete RR->topology;
 	delete RR->mc;
 	delete RR->sw;
+
 #ifdef ZT_ENABLE_CLUSTER
 	delete RR->cluster;
 #endif
@@ -263,7 +271,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 				for(std::vector< std::pair< uint64_t,SharedPtr<Network> > >::const_iterator n(_networks.begin());n!=_networks.end();++n) {
 					if (((now - n->second->lastConfigUpdate()) >= ZT_NETWORK_AUTOCONF_DELAY)||(!n->second->hasConfig()))
 						needConfig.push_back(n->second);
-					n->second->announceMulticastGroups();
+					n->second->sendUpdatesToMembers();
 				}
 			}
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
@@ -316,6 +324,12 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 	return ZT_RESULT_OK;
 }
 
+ZT_ResultCode Node::setRelayPolicy(enum ZT_RelayPolicy rp)
+{
+	_relayPolicy = rp;
+	return ZT_RESULT_OK;
+}
+
 ZT_ResultCode Node::join(uint64_t nwid,void *uptr)
 {
 	Mutex::Lock _l(_networks_m);
@@ -821,6 +835,15 @@ enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,vol
 	}
 }
 
+enum ZT_ResultCode ZT_Node_setRelayPolicy(ZT_Node *node,enum ZT_RelayPolicy rp)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->setRelayPolicy(rp);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
 enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid,void *uptr)
 {
 	try {

node/Node.hpp

@@ -44,6 +44,10 @@
 #define TRACE(f,...) {}
 #endif
 
+// Bit mask for "expecting reply" hash
+#define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255
+#define ZT_EXPECTING_REPLIES_BUCKET_MASK2 31
+
 namespace ZeroTier {
 
 /**
@@ -87,6 +91,7 @@ public:
 		unsigned int frameLength,
 		volatile uint64_t *nextBackgroundTaskDeadline);
 	ZT_ResultCode processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline);
+	ZT_ResultCode setRelayPolicy(enum ZT_RelayPolicy rp);
 	ZT_ResultCode join(uint64_t nwid,void *uptr);
 	ZT_ResultCode leave(uint64_t nwid,void **uptr);
 	ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
@@ -241,6 +246,7 @@ public:
 	inline int configureVirtualNetworkPort(uint64_t nwid,void **nuptr,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _virtualNetworkConfigFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,nwid,nuptr,op,nc); }
 
 	inline bool online() const throw() { return _online; }
+	inline ZT_RelayPolicy relayPolicy() const { return _relayPolicy; }
 
 #ifdef ZT_TRACE
 	void postTrace(const char *module,unsigned int line,const char *fmt,...);
@@ -250,6 +256,33 @@ public:
 	void postCircuitTestReport(const ZT_CircuitTestReport *report);
 	void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count);
 
+	/**
+	 * Register that we are expecting a reply to a packet ID
+	 *
+	 * @param packetId Packet ID to expect reply to
+	 */
+	inline void expectReplyTo(const uint64_t packetId)
+	{
+		const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1);
+		_expectingRepliesTo[bucket][_expectingRepliesToBucketPtr[bucket]++ & ZT_EXPECTING_REPLIES_BUCKET_MASK2] = packetId;
+	}
+
+	/**
+	 * Check whether a given packet ID is something we are expecting a reply to
+	 *
+	 * @param packetId Packet ID to check
+	 * @return True if we're expecting a reply
+	 */
+	inline bool expectingReplyTo(const uint64_t packetId) const
+	{
+		const unsigned long bucket = (unsigned long)(packetId & ZT_EXPECTING_REPLIES_BUCKET_MASK1);
+		for(unsigned long i=0;i<=ZT_EXPECTING_REPLIES_BUCKET_MASK2;++i) {
+			if (_expectingRepliesTo[bucket][i] == packetId)
+				return true;
+		}
+		return false;
+	}
+
 private:
 	inline SharedPtr<Network> _network(uint64_t nwid) const
 	{
@@ -266,6 +299,9 @@ private:
 
 	void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P
 
+	uint8_t _expectingRepliesToBucketPtr[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1];
+	uint64_t _expectingRepliesTo[ZT_EXPECTING_REPLIES_BUCKET_MASK1 + 1][ZT_EXPECTING_REPLIES_BUCKET_MASK2 + 1];
+
 	ZT_DataStoreGetFunction _dataStoreGetFunction;
 	ZT_DataStorePutFunction _dataStorePutFunction;
 	ZT_WirePacketSendFunction _wirePacketSendFunction;
@@ -292,6 +328,7 @@ private:
 	uint64_t _now;
 	uint64_t _lastPingCheck;
 	uint64_t _lastHousekeepingRun;
+	ZT_RelayPolicy _relayPolicy;
 	bool _online;
 };
 

node/OutboundMulticast.cpp

@@ -91,6 +91,7 @@ void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toA
 		//TRACE(">>MC %.16llx -> %s",(unsigned long long)this,toAddr.toString().c_str());
 		_packet.newInitializationVector();
 		_packet.setDestination(toAddr2);
+		RR->node->expectReplyTo(_packet.packetId());
 		RR->sw->send(_packet,true);
 	}
 }

node/Packet.cpp

@@ -62,6 +62,7 @@ const char *Packet::errorString(ErrorCode e)
 		case ERROR_OBJ_NOT_FOUND: return "OBJECT_NOT_FOUND";
 		case ERROR_IDENTITY_COLLISION: return "IDENTITY_COLLISION";
 		case ERROR_UNSUPPORTED_OPERATION: return "UNSUPPORTED_OPERATION";
+		case ERROR_NEED_MEMBERSHIP_CERTIFICATE: return "NEED_MEMBERSHIP_CERTIFICATE";
 		case ERROR_NETWORK_ACCESS_DENIED_: return "NETWORK_ACCESS_DENIED";
 		case ERROR_UNWANTED_MULTICAST: return "UNWANTED_MULTICAST";
 	}

node/Packet.hpp

@@ -965,7 +965,7 @@ public:
 		 *   <[2] 16-bit reporter OS/platform or 0 if not specified>
 		 *   <[2] 16-bit reporter architecture or 0 if not specified>
 		 *   <[2] 16-bit error code (set to 0, currently unused)>
-		 *   <[8] 64-bit report flags (set to 0, currently unused)>
+		 *   <[8] 64-bit report flags>
 		 *   <[8] 64-bit packet ID of received CIRCUIT_TEST packet>
 		 *   <[5] upstream ZeroTier address from which CIRCUIT_TEST was received>
 		 *   <[1] 8-bit packet hop count of received CIRCUIT_TEST>
@@ -980,6 +980,9 @@ public:
 		 *   <[5] ZeroTier address of next hop>
 		 *   <[...] current best direct path address, if any, 0 if none>
 		 *
+		 * Report flags:
+		 *   0x1 - Upstream peer in circuit test path allowed in path (e.g. network COM valid)
+		 *
 		 * Circuit test reports can be sent by hops in a circuit test to report
 		 * back results. They should include information about the sender as well
 		 * as about the paths to which next hops are being sent.
@@ -1067,6 +1070,9 @@ public:
 		/* Verb or use case not supported/enabled by this node */
 		ERROR_UNSUPPORTED_OPERATION = 0x05,
 
+		/* Network membership certificate update needed */
+		ERROR_NEED_MEMBERSHIP_CERTIFICATE = 0x06,
+
 		/* Tried to join network, but you're not a member */
 		ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */
 

node/Path.hpp

@@ -104,6 +104,7 @@ public:
 	Path() :
 		_lastOut(0),
 		_lastIn(0),
+		_lastTrustEstablishedPacketReceived(0),
 		_addr(),
 		_localAddress(),
 		_ipScope(InetAddress::IP_SCOPE_NONE)
@@ -113,6 +114,7 @@ public:
 	Path(const InetAddress &localAddress,const InetAddress &addr) :
 		_lastOut(0),
 		_lastIn(0),
+		_lastTrustEstablishedPacketReceived(0),
 		_addr(addr),
 		_localAddress(localAddress),
 		_ipScope(addr.ipScope())
@@ -126,6 +128,11 @@ public:
 	 */
 	inline void received(const uint64_t t) { _lastIn = t; }
 
+	/**
+	 * Set time last trusted packet was received (done in Peer::received())
+	 */
+	inline void trustedPacketReceived(const uint64_t t) { _lastTrustEstablishedPacketReceived = t; }
+
 	/**
 	 * Send a packet via this path (last out time is also updated)
 	 *
@@ -159,6 +166,11 @@ public:
 	 */
 	inline InetAddress::IpScope ipScope() const { return _ipScope; }
 
+	/**
+	 * @return True if path has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms
+	 */
+	inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); }
+
 	/**
 	 * @return Preference rank, higher == better
 	 */
@@ -232,6 +244,7 @@ public:
 private:
 	uint64_t _lastOut;
 	uint64_t _lastIn;
+	uint64_t _lastTrustEstablishedPacketReceived;
 	InetAddress _addr;
 	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often

node/Peer.cpp

@@ -47,6 +47,12 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_lastMulticastFrame(0),
 	_lastDirectPathPushSent(0),
 	_lastDirectPathPushReceive(0),
+	_lastCredentialRequestSent(0),
+	_lastWhoisRequestReceived(0),
+	_lastEchoRequestReceived(0),
+	_lastComRequestReceived(0),
+	_lastCredentialsReceived(0),
+	_lastTrustEstablishedPacketReceived(0),
 	RR(renv),
 	_remoteClusterOptimal4(0),
 	_vProto(0),
@@ -56,7 +62,8 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_id(peerIdentity),
 	_numPaths(0),
 	_latency(0),
-	_directPathPushCutoffCount(0)
+	_directPathPushCutoffCount(0),
+	_credentialsCutoffCount(0)
 {
 	memset(_remoteClusterOptimal6,0,sizeof(_remoteClusterOptimal6));
 	if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
@@ -126,6 +133,11 @@ void Peer::received(
 	else if (verb == Packet::VERB_MULTICAST_FRAME)
 		_lastMulticastFrame = now;
 
+	if (trustEstablished) {
+		_lastTrustEstablishedPacketReceived = now;
+		path->trustedPacketReceived(now);
+	}
+
 	if (hops == 0) {
 		bool pathIsConfirmed = false;
 		{
@@ -194,7 +206,80 @@ void Peer::received(
 		}
 	} else if (trustEstablished) {
 		// Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership)
-		_pushDirectPaths(path,now);
+#ifdef ZT_ENABLE_CLUSTER
+			// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
+			const bool haveCluster = (RR->cluster);
+#else
+			const bool haveCluster = false;
+#endif
+		if ( ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) && (!haveCluster) ) {
+			_lastDirectPathPushSent = now;
+
+			std::vector<InetAddress> pathsToPush;
+
+			std::vector<InetAddress> dps(RR->node->directPaths());
+			for(std::vector<InetAddress>::const_iterator i(dps.begin());i!=dps.end();++i)
+				pathsToPush.push_back(*i);
+
+			std::vector<InetAddress> sym(RR->sa->getSymmetricNatPredictions());
+			for(unsigned long i=0,added=0;i<sym.size();++i) {
+				InetAddress tmp(sym[(unsigned long)RR->node->prng() % sym.size()]);
+				if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) {
+					pathsToPush.push_back(tmp);
+					if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY)
+						break;
+				}
+			}
+
+			if (pathsToPush.size() > 0) {
+#ifdef ZT_TRACE
+				std::string ps;
+				for(std::vector<InetAddress>::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) {
+					if (ps.length() > 0)
+						ps.push_back(',');
+					ps.append(p->toString());
+				}
+				TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str());
+#endif
+
+				std::vector<InetAddress>::const_iterator p(pathsToPush.begin());
+				while (p != pathsToPush.end()) {
+					Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
+					outp.addSize(2); // leave room for count
+
+					unsigned int count = 0;
+					while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) {
+						uint8_t addressType = 4;
+						switch(p->ss_family) {
+							case AF_INET:
+								break;
+							case AF_INET6:
+								addressType = 6;
+								break;
+							default: // we currently only push IP addresses
+								++p;
+								continue;
+						}
+
+						outp.append((uint8_t)0); // no flags
+						outp.append((uint16_t)0); // no extensions
+						outp.append(addressType);
+						outp.append((uint8_t)((addressType == 4) ? 6 : 18));
+						outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16));
+						outp.append((uint16_t)p->port());
+
+						++count;
+						++p;
+					}
+
+					if (count) {
+						outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count);
+						outp.armor(_key,true);
+						path->send(RR,outp.data(),outp.size(),now);
+					}
+				}
+			}
+		}
 	}
 }
 
@@ -266,6 +351,7 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u
 	atAddress.serialize(outp);
 	outp.append((uint64_t)RR->topology->worldId());
 	outp.append((uint64_t)RR->topology->worldTimestamp());
+	RR->node->expectReplyTo(outp.packetId());
 	outp.armor(_key,false); // HELLO is sent in the clear
 	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 }
@@ -274,6 +360,7 @@ void Peer::attemptToContactAt(const InetAddress &localAddr,const InetAddress &at
 {
 	if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) {
 		Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO);
+		RR->node->expectReplyTo(outp.packetId());
 		outp.armor(_key,true);
 		RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 	} else {
@@ -366,86 +453,4 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6)
 		v6 = _paths[bestp6].path->address();
 }
 
-bool Peer::_pushDirectPaths(const SharedPtr<Path> &path,uint64_t now)
-{
-#ifdef ZT_ENABLE_CLUSTER
-	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
-	if (RR->cluster)
-		return false;
-#endif
-
-	if ((now - _lastDirectPathPushSent) < ZT_DIRECT_PATH_PUSH_INTERVAL)
-		return false;
-	else _lastDirectPathPushSent = now;
-
-	std::vector<InetAddress> pathsToPush;
-
-	std::vector<InetAddress> dps(RR->node->directPaths());
-	for(std::vector<InetAddress>::const_iterator i(dps.begin());i!=dps.end();++i)
-		pathsToPush.push_back(*i);
-
-	std::vector<InetAddress> sym(RR->sa->getSymmetricNatPredictions());
-	for(unsigned long i=0,added=0;i<sym.size();++i) {
-		InetAddress tmp(sym[(unsigned long)RR->node->prng() % sym.size()]);
-		if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) {
-			pathsToPush.push_back(tmp);
-			if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY)
-				break;
-		}
-	}
-	if (pathsToPush.empty())
-		return false;
-
-#ifdef ZT_TRACE
-	{
-		std::string ps;
-		for(std::vector<InetAddress>::const_iterator p(pathsToPush.begin());p!=pathsToPush.end();++p) {
-			if (ps.length() > 0)
-				ps.push_back(',');
-			ps.append(p->toString());
-		}
-		TRACE("pushing %u direct paths to %s: %s",(unsigned int)pathsToPush.size(),_id.address().toString().c_str(),ps.c_str());
-	}
-#endif
-
-	std::vector<InetAddress>::const_iterator p(pathsToPush.begin());
-	while (p != pathsToPush.end()) {
-		Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
-		outp.addSize(2); // leave room for count
-
-		unsigned int count = 0;
-		while ((p != pathsToPush.end())&&((outp.size() + 24) < 1200)) {
-			uint8_t addressType = 4;
-			switch(p->ss_family) {
-				case AF_INET:
-					break;
-				case AF_INET6:
-					addressType = 6;
-					break;
-				default: // we currently only push IP addresses
-					++p;
-					continue;
-			}
-
-			outp.append((uint8_t)0); // no flags
-			outp.append((uint16_t)0); // no extensions
-			outp.append(addressType);
-			outp.append((uint8_t)((addressType == 4) ? 6 : 18));
-			outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16));
-			outp.append((uint16_t)p->port());
-
-			++count;
-			++p;
-		}
-
-		if (count) {
-			outp.setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count);
-			outp.armor(_key,true);
-			path->send(RR,outp.data(),outp.size(),now);
-		}
-	}
-
-	return true;
-}
-
 } // namespace ZeroTier

node/Peer.hpp

@@ -312,7 +312,7 @@ public:
 	/**
 	 * @return 256-bit secret symmetric encryption key
 	 */
-	inline const unsigned char *key() const throw() { return _key; }
+	inline const unsigned char *key() const { return _key; }
 
 	/**
 	 * Set the currently known remote version of this peer's client
@@ -330,25 +330,22 @@ public:
 		_vRevision = (uint16_t)vrev;
 	}
 
-	inline unsigned int remoteVersionProtocol() const throw() { return _vProto; }
-	inline unsigned int remoteVersionMajor() const throw() { return _vMajor; }
-	inline unsigned int remoteVersionMinor() const throw() { return _vMinor; }
-	inline unsigned int remoteVersionRevision() const throw() { return _vRevision; }
+	inline unsigned int remoteVersionProtocol() const { return _vProto; }
+	inline unsigned int remoteVersionMajor() const { return _vMajor; }
+	inline unsigned int remoteVersionMinor() const { return _vMinor; }
+	inline unsigned int remoteVersionRevision() const { return _vRevision; }
 
-	inline bool remoteVersionKnown() const throw() { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); }
+	inline bool remoteVersionKnown() const { return ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)); }
 
 	/**
-	 * Update direct path push stats and return true if we should respond
-	 *
-	 * This is a circuit breaker to make VERB_PUSH_DIRECT_PATHS not particularly
-	 * useful as a DDOS amplification attack vector. Otherwise a malicious peer
-	 * could send loads of these and cause others to bombard arbitrary IPs with
-	 * traffic.
-	 *
-	 * @param now Current time
-	 * @return True if we should respond
+	 * @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms
+	 */
+	inline bool trustEstablished(const uint64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); }
+
+	/**
+	 * Rate limit gate for VERB_PUSH_DIRECT_PATHS
 	 */
-	inline bool shouldRespondToDirectPathPush(const uint64_t now)
+	inline bool rateGatePushDirectPaths(const uint64_t now)
 	{
 		if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME)
 			++_directPathPushCutoffCount;
@@ -357,6 +354,66 @@ public:
 		return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT);
 	}
 
+	/**
+	 * Rate limit gate for VERB_NETWORK_CREDENTIALS
+	 */
+	inline bool rateGateCredentialsReceived(const uint64_t now)
+	{
+		if ((now - _lastCredentialsReceived) <= ZT_PEER_CREDENTIALS_CUTOFF_TIME)
+			++_credentialsCutoffCount;
+		else _credentialsCutoffCount = 0;
+		_lastCredentialsReceived = now;
+		return (_directPathPushCutoffCount < ZT_PEER_CREDEITIALS_CUTOFF_LIMIT);
+	}
+
+	/**
+	 * Rate limit gate for sending of ERROR_NEED_MEMBERSHIP_CERTIFICATE
+	 */
+	inline bool rateGateRequestCredentials(const uint64_t now)
+	{
+		if ((now - _lastCredentialRequestSent) >= ZT_PEER_GENERAL_RATE_LIMIT) {
+			_lastCredentialRequestSent = now;
+			return true;
+		}
+		return false;
+	}
+
+	/**
+	 * Rate limit gate for inbound WHOIS requests
+	 */
+	inline bool rateGateInboundWhoisRequest(const uint64_t now)
+	{
+		if ((now - _lastWhoisRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) {
+			_lastWhoisRequestReceived = now;
+			return true;
+		}
+		return false;
+	}
+
+	/**
+	 * Rate limit gate for inbound ECHO requests
+	 */
+	inline bool rateGateEchoRequest(const uint64_t now)
+	{
+		if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) {
+			_lastEchoRequestReceived = now;
+			return true;
+		}
+		return false;
+	}
+
+	/**
+	 * Rate gate requests for network COM
+	 */
+	inline bool rateGateComRequest(const uint64_t now)
+	{
+		if ((now - _lastComRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) {
+			_lastComRequestReceived = now;
+			return true;
+		}
+		return false;
+	}
+
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
 	 *
@@ -378,8 +435,6 @@ public:
 	}
 
 private:
-	bool _pushDirectPaths(const SharedPtr<Path> &path,uint64_t now);
-
 	inline uint64_t _pathScore(const unsigned int p,const uint64_t now) const
 	{
 		uint64_t s = ZT_PEER_PING_PERIOD + _paths[p].lastReceive + (uint64_t)(_paths[p].path->preferenceRank() * (ZT_PEER_PING_PERIOD / ZT_PATH_MAX_PREFERENCE_RANK));
@@ -415,6 +470,12 @@ private:
 	uint64_t _lastMulticastFrame;
 	uint64_t _lastDirectPathPushSent;
 	uint64_t _lastDirectPathPushReceive;
+	uint64_t _lastCredentialRequestSent;
+	uint64_t _lastWhoisRequestReceived;
+	uint64_t _lastEchoRequestReceived;
+	uint64_t _lastComRequestReceived;
+	uint64_t _lastCredentialsReceived;
+	uint64_t _lastTrustEstablishedPacketReceived;
 	const RuntimeEnvironment *RR;
 	uint32_t _remoteClusterOptimal4;
 	uint16_t _vProto;
@@ -433,6 +494,7 @@ private:
 	unsigned int _numPaths;
 	unsigned int _latency;
 	unsigned int _directPathPushCutoffCount;
+	unsigned int _credentialsCutoffCount;
 
 	AtomicCounter __refCount;
 };

node/Switch.cpp

@@ -105,7 +105,18 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 				const Address destination(fragment.destination());
 
 				if (destination != RR->identity.address()) {
-					// Fragment is not for us, so try to relay it
+					switch(RR->node->relayPolicy()) {
+						case ZT_RELAY_POLICY_ALWAYS:
+							break;
+						case ZT_RELAY_POLICY_TRUSTED:
+							if (!path->trustEstablished(now))
+								return;
+							break;
+						// case ZT_RELAY_POLICY_NEVER:
+						default:
+							return;
+					}
+
 					if (fragment.hops() < ZT_RELAY_MAX_HOPS) {
 						fragment.incrementHops();
 
@@ -203,9 +214,20 @@ void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &from
 				//TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size());
 
 				if (destination != RR->identity.address()) {
+					switch(RR->node->relayPolicy()) {
+						case ZT_RELAY_POLICY_ALWAYS:
+							break;
+						case ZT_RELAY_POLICY_TRUSTED:
+							if (!path->trustEstablished(now))
+								return;
+							break;
+						// case ZT_RELAY_POLICY_NEVER:
+						default:
+							return;
+					}
+
 					Packet packet(data,len);
 
-					// Packet is not for us, so try to relay it
 					if (packet.hops() < ZT_RELAY_MAX_HOPS) {
 						packet.incrementHops();
 
@@ -327,6 +349,11 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 	}
 
 	if (to.isMulticast()) {
+		if (network->config().multicastLimit == 0) {
+			TRACE("%.16llx: dropped multicast: not allowed on network",network->id());
+			return;
+		}
+
 		// Destination is a multicast address (including broadcast)
 		MulticastGroup mg(to,0);
 
@@ -734,13 +761,12 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 
 Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted)
 {
-	SharedPtr<Peer> root(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false));
-	if (root) {
-		Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS);
+	SharedPtr<Peer> upstream(RR->topology->getBestRoot(peersAlreadyConsulted,numPeersAlreadyConsulted,false));
+	if (upstream) {
+		Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS);
 		addr.appendTo(outp);
-		outp.armor(root->key(),true);
-		if (root->sendDirect(outp.data(),outp.size(),RR->node->now(),true))
-			return root->address();
+		RR->node->expectReplyTo(outp.packetId());
+		send(outp,true);
 	}
 	return Address();
 }
@@ -760,11 +786,8 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 
 		SharedPtr<Path> viaPath(peer->getBestPath(now,false));
 		if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isRoot(peer->identity())) ) {
-			if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) >> 2,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL)) {
-				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ECHO);
-				outp.armor(peer->key(),true);
-				viaPath->send(RR,outp.data(),outp.size(),now);
-			}
+			if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL))
+				peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now);
 			viaPath.zero();
 		}
 		if (!viaPath) {

node/World.hpp

@@ -150,7 +150,7 @@ public:
 			if (fullSignatureCheck) {
 				Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
 				update.serialize(tmp,true);
-				return C25519::verify(_updateSigningKey,tmp.data(),tmp.size(),update._signature);
+				return C25519::verify(_updatesMustBeSignedBy,tmp.data(),tmp.size(),update._signature);
 			} else return true;
 		}
 		return false;
@@ -169,7 +169,7 @@ public:
 		b.append((uint8_t)0x01);
 		b.append((uint64_t)_id);
 		b.append((uint64_t)_ts);
-		b.append(_updateSigningKey.data,ZT_C25519_PUBLIC_KEY_LEN);
+		b.append(_updatesMustBeSignedBy.data,ZT_C25519_PUBLIC_KEY_LEN);
 		if (!forSign)
 			b.append(_signature.data,ZT_C25519_SIGNATURE_LEN);
 		b.append((uint8_t)_roots.size());
@@ -195,7 +195,7 @@ public:
 
 		_id = b.template at<uint64_t>(p); p += 8;
 		_ts = b.template at<uint64_t>(p); p += 8;
-		memcpy(_updateSigningKey.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN;
+		memcpy(_updatesMustBeSignedBy.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN;
 		memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN;
 		unsigned int numRoots = b[p++];
 		if (numRoots > ZT_WORLD_MAX_ROOTS)
@@ -216,13 +216,13 @@ public:
 		return (p - startAt);
 	}
 
-	inline bool operator==(const World &w) const throw() { return ((_id == w._id)&&(_ts == w._ts)&&(_updateSigningKey == w._updateSigningKey)&&(_signature == w._signature)&&(_roots == w._roots)); }
+	inline bool operator==(const World &w) const throw() { return ((_id == w._id)&&(_ts == w._ts)&&(_updatesMustBeSignedBy == w._updatesMustBeSignedBy)&&(_signature == w._signature)&&(_roots == w._roots)); }
 	inline bool operator!=(const World &w) const throw() { return (!(*this == w)); }
 
 protected:
 	uint64_t _id;
 	uint64_t _ts;
-	C25519::Public _updateSigningKey;
+	C25519::Public _updatesMustBeSignedBy;
 	C25519::Signature _signature;
 	std::vector<Root> _roots;
 };

osdep/ManagedRoute.cpp

@@ -69,23 +69,28 @@ static void _forkTarget(const InetAddress &t,InetAddress &left,InetAddress &righ
 {
 	const unsigned int bits = t.netmaskBits() + 1;
 	left = t;
-	if ((t.ss_family == AF_INET)&&(bits <= 32)) {
-		left.setPort(bits);
-		right = t;
-		reinterpret_cast<struct sockaddr_in *>(&right)->sin_addr.s_addr ^= Utils::hton((uint32_t)(1 << (32 - bits)));
-		right.setPort(bits);
-	} else if ((t.ss_family == AF_INET6)&&(bits <= 128)) {
-		left.setPort(bits);
-		right = t;
-		uint8_t *b = reinterpret_cast<uint8_t *>(reinterpret_cast<struct sockaddr_in6 *>(&right)->sin6_addr.s6_addr);
-		b[bits / 8] ^= 1 << (8 - (bits % 8));
-		right.setPort(bits);
+	if (t.ss_family == AF_INET) {
+		if (bits <= 32) {
+			left.setPort(bits);
+			right = t;
+			reinterpret_cast<struct sockaddr_in *>(&right)->sin_addr.s_addr ^= Utils::hton((uint32_t)(1 << (32 - bits)));
+			right.setPort(bits);
+		} else {
+			right.zero();
+		}
+	} else if (t.ss_family == AF_INET6) {
+		if (bits <= 128) {
+			left.setPort(bits);
+			right = t;
+			uint8_t *b = reinterpret_cast<uint8_t *>(reinterpret_cast<struct sockaddr_in6 *>(&right)->sin6_addr.s6_addr);
+			b[bits / 8] ^= 1 << (8 - (bits % 8));
+			right.setPort(bits);
+		} else {
+			right.zero();
+		}
 	}
 }
 
-#ifdef __BSD__ // ------------------------------------------------------------
-#define ZT_ROUTING_SUPPORT_FOUND 1
-
 struct _RTE
 {
 	InetAddress target;
@@ -95,6 +100,9 @@ struct _RTE
 	bool ifscope;
 };
 
+#ifdef __BSD__ // ------------------------------------------------------------
+#define ZT_ROUTING_SUPPORT_FOUND 1
+
 static std::vector<_RTE> _getRTEs(const InetAddress &target,bool contains)
 {
 	std::vector<_RTE> rtes;
@@ -232,6 +240,7 @@ static std::vector<_RTE> _getRTEs(const InetAddress &target,bool contains)
 
 static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *ifscope,const char *localInterface)
 {
+	//printf("route %s %s %s %s %s\n",op,target.toString().c_str(),(via) ? via.toString().c_str() : "(null)",(ifscope) ? ifscope : "(null)",(localInterface) ? localInterface : "(null)");
 	long p = (long)fork();
 	if (p > 0) {
 		int exitcode = -1;
@@ -369,145 +378,123 @@ bool ManagedRoute::sync()
 		return false;
 #endif
 
-	if ((_target.isDefaultRoute())||((_target.ss_family == AF_INET)&&(_target.netmaskBits() < 32))) {
-		/* In ZeroTier we create two more specific routes for every one route. We
-		 * do this for default routes and IPv4 routes other than /32s. If there
-		 * is a pre-existing system route that this route will override, we create
-		 * two more specific interface-bound shadow routes for it.
-		 *
-		 * This means that ZeroTier can *itself* continue communicating over
-		 * whatever physical routes might be present while simultaneously
-		 * overriding them for general system traffic. This is mostly for
-		 * "full tunnel" VPN modes of operation, but might be useful for
-		 * virtualizing physical networks in a hybrid design as well. */
-
-		// Generate two more specific routes than target with one extra bit
-		InetAddress leftt,rightt;
-		_forkTarget(_target,leftt,rightt);
+	// Generate two more specific routes than target with one extra bit
+	InetAddress leftt,rightt;
+	_forkTarget(_target,leftt,rightt);
 
 #ifdef __BSD__ // ------------------------------------------------------------
 
-		// Find lowest metric system route that this route should override (if any)
-		InetAddress newSystemVia;
-		char newSystemDevice[128];
-		newSystemDevice[0] = (char)0;
-		int systemMetric = 9999999;
-		std::vector<_RTE> rtes(_getRTEs(_target,false));
-		for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) {
-			if (r->via) {
-				if ((!newSystemVia)||(r->metric < systemMetric)) {
-					newSystemVia = r->via;
-					Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device);
-					systemMetric = r->metric;
-				}
+	// Find lowest metric system route that this route should override (if any)
+	InetAddress newSystemVia;
+	char newSystemDevice[128];
+	newSystemDevice[0] = (char)0;
+	int systemMetric = 9999999;
+	std::vector<_RTE> rtes(_getRTEs(_target,false));
+	for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) {
+		if (r->via) {
+			if ( ((!newSystemVia)||(r->metric < systemMetric)) && (strcmp(r->device,_device) != 0) ) {
+				newSystemVia = r->via;
+				Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device);
+				systemMetric = r->metric;
 			}
 		}
-		if ((newSystemVia)&&(!newSystemDevice[0])) {
-			rtes = _getRTEs(newSystemVia,true);
-			for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) {
-				if (r->device[0]) {
-					Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device);
-					break;
-				}
+	}
+
+	// Get device corresponding to route if we don't have that already
+	if ((newSystemVia)&&(!newSystemDevice[0])) {
+		rtes = _getRTEs(newSystemVia,true);
+		for(std::vector<_RTE>::iterator r(rtes.begin());r!=rtes.end();++r) {
+			if ( (r->device[0]) && (strcmp(r->device,_device) != 0) ) {
+				Utils::scopy(newSystemDevice,sizeof(newSystemDevice),r->device);
+				break;
 			}
 		}
-
-		// Shadow system route if it exists, also delete any obsolete shadows
-		// and replace them with the new state. sync() is called periodically to
-		// allow us to do that if underlying connectivity changes.
-		if ( ((_systemVia != newSystemVia)||(strcmp(_systemDevice,newSystemDevice))) && (strcmp(_device,newSystemDevice)) ) {
-			if ((_systemVia)&&(_systemDevice[0])) {
-				_routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0);
+	}
+	if (!newSystemDevice[0])
+		newSystemVia.zero();
+
+	// Shadow system route if it exists, also delete any obsolete shadows
+	// and replace them with the new state. sync() is called periodically to
+	// allow us to do that if underlying connectivity changes.
+	if ((_systemVia != newSystemVia)||(strcmp(_systemDevice,newSystemDevice) != 0)) {
+		if (_systemVia) {
+			_routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0);
+			if (rightt)
 				_routeCmd("delete",rightt,_systemVia,_systemDevice,(const char *)0);
-			}
+		}
 
-			_systemVia = newSystemVia;
-			Utils::scopy(_systemDevice,sizeof(_systemDevice),newSystemDevice);
+		_systemVia = newSystemVia;
+		Utils::scopy(_systemDevice,sizeof(_systemDevice),newSystemDevice);
 
-			if ((_systemVia)&&(_systemDevice[0])) {
-				_routeCmd("add",leftt,_systemVia,_systemDevice,(const char *)0);
-				_routeCmd("change",leftt,_systemVia,_systemDevice,(const char *)0);
+		if (_systemVia) {
+			_routeCmd("add",leftt,_systemVia,_systemDevice,(const char *)0);
+			_routeCmd("change",leftt,_systemVia,_systemDevice,(const char *)0);
+			if (rightt) {
 				_routeCmd("add",rightt,_systemVia,_systemDevice,(const char *)0);
 				_routeCmd("change",rightt,_systemVia,_systemDevice,(const char *)0);
 			}
 		}
+	}
 
-		// Apply overriding non-device-scoped routes
-		if (!_applied) {
-			if (_via) {
-				_routeCmd("add",leftt,_via,(const char *)0,(const char *)0);
-				_routeCmd("change",leftt,_via,(const char *)0,(const char *)0);
-				_routeCmd("add",rightt,_via,(const char *)0,(const char *)0);
-				_routeCmd("change",rightt,_via,(const char *)0,(const char *)0);
-			} else if (_device[0]) {
-				_routeCmd("add",leftt,_via,(const char *)0,_device);
-				_routeCmd("change",leftt,_via,(const char *)0,_device);
-				_routeCmd("add",rightt,_via,(const char *)0,_device);
-				_routeCmd("change",rightt,_via,(const char *)0,_device);
-			}
-
-			_applied = true;
-		}
-
-#endif // __BSD__ ------------------------------------------------------------
-
-#ifdef __LINUX__ // ----------------------------------------------------------
-
-		if (!_applied) {
-			_routeCmd("replace",leftt,_via,(_via) ? _device : (const char *)0);
-			_routeCmd("replace",rightt,_via,(_via) ? _device : (const char *)0);
-			_applied = true;
-		}
-
-#endif // __LINUX__ ----------------------------------------------------------
-
-#ifdef __WINDOWS__ // --------------------------------------------------------
-
-		if (!_applied) {
-			_winRoute(false,interfaceLuid,interfaceIndex,leftt,_via);
-			_winRoute(false,interfaceLuid,interfaceIndex,rightt,_via);
-			_applied = true;
-		}
-
-#endif // __WINDOWS__ --------------------------------------------------------
-
-	} else {
-
-#ifdef __BSD__ // ------------------------------------------------------------
+	if (!_applied.count(leftt)) {
+		_applied[leftt] = false; // not ifscoped
+		_routeCmd("add",leftt,_via,(const char *)0,(_via) ? (const char *)0 : _device);
+		_routeCmd("change",leftt,_via,(const char *)0,(_via) ? (const char *)0 : _device);
+	}
+	if ((rightt)&&(!_applied.count(rightt))) {
+		_applied[rightt] = false; // not ifscoped
+		_routeCmd("add",rightt,_via,(const char *)0,(_via) ? (const char *)0 : _device);
+		_routeCmd("change",rightt,_via,(const char *)0,(_via) ? (const char *)0 : _device);
+	}
 
-		if (!_applied) {
-			if (_via) {
-				_routeCmd("add",_target,_via,(const char *)0,(const char *)0);
-				_routeCmd("change",_target,_via,(const char *)0,(const char *)0);
-			} else if (_device[0]) {
-				_routeCmd("add",_target,_via,(const char *)0,_device);
-				_routeCmd("change",_target,_via,(const char *)0,_device);
+	// Create a device-bound default target if there is none in the system. This
+	// is to allow e.g. IPv6 default route to work even if there is no native
+	// IPv6 on your LAN.
+	/*
+	if (_target.isDefaultRoute()) {
+		if (_systemVia) {
+			if (_applied.count(_target)) {
+				_applied.erase(_target);
+				_routeCmd("delete",_target,_via,_device,(_via) ? (const char *)0 : _device);
+			}
+		} else {
+			if (!_applied.count(_target)) {
+				_applied[_target] = true; // ifscoped
+				_routeCmd("add",_target,_via,_device,(_via) ? (const char *)0 : _device);
+				_routeCmd("change",_target,_via,_device,(_via) ? (const char *)0 : _device);
 			}
-			_applied = true;
 		}
+	}
+	*/
 
 #endif // __BSD__ ------------------------------------------------------------
 
 #ifdef __LINUX__ // ----------------------------------------------------------
 
-		if (!_applied) {
-			_routeCmd("replace",_target,_via,(_via) ? _device : (const char *)0);
-			_applied = true;
-		}
+	if (!_applied.count(leftt)) {
+		_applied[leftt] = false; // boolean unused
+		_routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device);
+	}
+	if ((rightt)&&(!_applied.count(rightt))) {
+		_applied[rightt] = false; // boolean unused
+		_routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device);
+	}
 
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------
 
-		if (!_applied) {
-			_winRoute(false,interfaceLuid,interfaceIndex,_target,_via);
-			_applied = true;
-		}
+	if (!_applied.count(leftt)) {
+		_applied[leftt] = false; // boolean unused
+		_winRoute(false,interfaceLuid,interfaceIndex,leftt,_via);
+	}
+	if ((rightt)&&(!_applied.count(rightt))) {
+		_applied[rightt] = false; // boolean unused
+		_winRoute(false,interfaceLuid,interfaceIndex,rightt,_via);
+	}
 
 #endif // __WINDOWS__ --------------------------------------------------------
 
-	}
-
 	return true;
 }
 
@@ -521,66 +508,28 @@ void ManagedRoute::remove()
 		return;
 #endif
 
-	if (_applied) {
-		if ((_target.isDefaultRoute())||((_target.ss_family == AF_INET)&&(_target.netmaskBits() < 32))) {
-			InetAddress leftt,rightt;
-			_forkTarget(_target,leftt,rightt);
-
-#ifdef __BSD__ // ------------------------------------------------------------
-
-			if ((_systemVia)&&(_systemDevice[0])) {
-				_routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0);
-				_routeCmd("delete",rightt,_systemVia,_systemDevice,(const char *)0);
-			}
-			if (_via) {
-				_routeCmd("delete",leftt,_via,(const char *)0,(const char *)0);
-				_routeCmd("delete",rightt,_via,(const char *)0,(const char *)0);
-			} else if (_device[0]) {
-				_routeCmd("delete",leftt,_via,(const char *)0,_device);
-				_routeCmd("delete",rightt,_via,(const char *)0,_device);
-			}
-
+#ifdef __BSD__
+	if (_systemVia) {
+		InetAddress leftt,rightt;
+		_forkTarget(_target,leftt,rightt);
+		_routeCmd("delete",leftt,_systemVia,_systemDevice,(const char *)0);
+		if (rightt)
+			_routeCmd("delete",rightt,_systemVia,_systemDevice,(const char *)0);
+	}
 #endif // __BSD__ ------------------------------------------------------------
 
-#ifdef __LINUX__ // ----------------------------------------------------------
-
-			_routeCmd("del",leftt,_via,(_via) ? _device : (const char *)0);
-			_routeCmd("del",rightt,_via,(_via) ? _device : (const char *)0);
-
-#endif // __LINUX__ ----------------------------------------------------------
-
-#ifdef __WINDOWS__ // --------------------------------------------------------
-
-			_winRoute(true,interfaceLuid,interfaceIndex,leftt,_via);
-			_winRoute(true,interfaceLuid,interfaceIndex,rightt,_via);
-
-#endif // __WINDOWS__ --------------------------------------------------------
-
-		} else {
-
+	for(std::map<InetAddress,bool>::iterator r(_applied.begin());r!=_applied.end();++r) {
 #ifdef __BSD__ // ------------------------------------------------------------
-
-		if (_via) {
-			_routeCmd("delete",_target,_via,(const char *)0,(const char *)0);
-		} else if (_device[0]) {
-			_routeCmd("delete",_target,_via,(const char *)0,_device);
-		}
-
+		_routeCmd("delete",r->first,_via,r->second ? _device : (const char *)0,(_via) ? (const char *)0 : _device);
 #endif // __BSD__ ------------------------------------------------------------
 
 #ifdef __LINUX__ // ----------------------------------------------------------
-
-			_routeCmd("del",_target,_via,(_via) ? _device : (const char *)0);
-
+		_routeCmd("del",*r,_via,(_via) ? (const char *)0 : _device);
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------
-
-			_winRoute(true,interfaceLuid,interfaceIndex,_target,_via);
-
+		_winRoute(true,interfaceLuid,interfaceIndex,*r,_via);
 #endif // __WINDOWS__ --------------------------------------------------------
-
-		}
 	}
 
 	_target.zero();
@@ -588,7 +537,7 @@ void ManagedRoute::remove()
 	_systemVia.zero();
 	_device[0] = (char)0;
 	_systemDevice[0] = (char)0;
-	_applied = false;
+	_applied.clear();
 }
 
 } // namespace ZeroTier

osdep/ManagedRoute.hpp

@@ -6,23 +6,34 @@
 
 #include "../node/InetAddress.hpp"
 #include "../node/Utils.hpp"
+#include "../node/SharedPtr.hpp"
+#include "../node/AtomicCounter.hpp"
+#include "../node/NonCopyable.hpp"
 
 #include <stdexcept>
 #include <vector>
+#include <map>
 
 namespace ZeroTier {
 
 /**
  * A ZT-managed route that used C++ RAII semantics to automatically clean itself up on deallocate
  */
-class ManagedRoute
+class ManagedRoute : NonCopyable
 {
+	friend class SharedPtr<ManagedRoute>;
+
 public:
-	ManagedRoute()
+	ManagedRoute(const InetAddress &target,const InetAddress &via,const char *device)
 	{
-		_device[0] = (char)0;
+		_target = target;
+		_via = via;
+		if (via.ss_family == AF_INET)
+			_via.setPort(32);
+		else if (via.ss_family == AF_INET6)
+			_via.setPort(128);
+		Utils::scopy(_device,sizeof(_device),device);
 		_systemDevice[0] = (char)0;
-		_applied = false;
 	}
 
 	~ManagedRoute()
@@ -30,45 +41,6 @@ public:
 		this->remove();
 	}
 
-	ManagedRoute(const ManagedRoute &r)
-	{
-		_applied = false;
-		*this = r;
-	}
-
-	inline ManagedRoute &operator=(const ManagedRoute &r)
-	{
-		if ((!_applied)&&(!r._applied)) {
-			memcpy(this,&r,sizeof(ManagedRoute)); // InetAddress is memcpy'able
-		} else {
-			fprintf(stderr,"Applied ManagedRoute isn't copyable!\n");
-			abort();
-		}
-		return *this;
-	}
-
-	/**
-	 * Initialize object and set route
-	 *
-	 * Note: on Windows, use the interface NET_LUID in hexadecimal as the
-	 * "device name."
-	 *
-	 * @param target Route target (e.g. 0.0.0.0/0 for default)
-	 * @param via Route next L3 hop or NULL InetAddress if local in which case it will be routed via device
-	 * @param device Name or hex LUID of ZeroTier device (e.g. zt#)
-	 * @return True if route was successfully set
-	 */
-	inline bool set(const InetAddress &target,const InetAddress &via,const char *device)
-	{
-		if ((!via)&&(!device[0]))
-			return false;
-		this->remove();
-		_target = target;
-		_via = via;
-		Utils::scopy(_device,sizeof(_device),device);
-		return this->sync();
-	}
-
 	/**
 	 * Set or update currently set route
 	 *
@@ -93,13 +65,14 @@ public:
 	inline const char *device() const { return _device; }
 
 private:
-
 	InetAddress _target;
 	InetAddress _via;
 	InetAddress _systemVia; // for route overrides
+	std::map<InetAddress,bool> _applied; // routes currently applied
 	char _device[128];
 	char _systemDevice[128]; // for route overrides
-	bool _applied;
+
+	AtomicCounter __refCount;
 };
 
 } // namespace ZeroTier

service/ControlPlane.cpp

@@ -215,7 +215,7 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer)
 	const char *prole = "";
 	switch(peer->role) {
 		case ZT_PEER_ROLE_LEAF:  prole = "LEAF"; break;
-		case ZT_PEER_ROLE_RELAY: prole = "RELAY"; break;
+		case ZT_PEER_ROLE_UPSTREAM: prole = "UPSTREAM"; break;
 		case ZT_PEER_ROLE_ROOT:  prole = "ROOT"; break;
 	}
 

service/OneService.cpp

@@ -537,7 +537,7 @@ public:
 		EthernetTap *tap;
 		ZT_VirtualNetworkConfig config; // memcpy() of raw config from core
 		std::vector<InetAddress> managedIps;
-		std::list<ManagedRoute> managedRoutes;
+		std::list< SharedPtr<ManagedRoute> > managedRoutes;
 		NetworkSettings settings;
 	};
 	std::map<uint64_t,NetworkState> _nets;
@@ -1128,13 +1128,13 @@ public:
 			std::vector<InetAddress> myIps(n.tap->ips());
 
 			// Nuke applied routes that are no longer in n.config.routes[] and/or are not allowed
-			for(std::list<ManagedRoute>::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) {
+			for(std::list< SharedPtr<ManagedRoute> >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) {
 				bool haveRoute = false;
-				if ( (checkIfManagedIsAllowed(n,mr->target())) && ((mr->via().ss_family != mr->target().ss_family)||(!matchIpOnly(myIps,mr->via()))) ) {
+				if ( (checkIfManagedIsAllowed(n,(*mr)->target())) && (((*mr)->via().ss_family != (*mr)->target().ss_family)||(!matchIpOnly(myIps,(*mr)->via()))) ) {
 					for(unsigned int i=0;i<n.config.routeCount;++i) {
 						const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target));
 						const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via));
-						if ( (mr->target() == *target) && ( ((via->ss_family == target->ss_family)&&(mr->via() == *via)) || (tapdev == mr->device()) ) ) {
+						if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (tapdev == (*mr)->device()) ) ) {
 							haveRoute = true;
 							break;
 						}
@@ -1168,10 +1168,10 @@ public:
 					continue;
 
 				// If we've already applied this route, just sync it and continue
-				for(std::list<ManagedRoute>::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();++mr) {
-					if ( (mr->target() == *target) && ( ((via->ss_family == target->ss_family)&&(mr->via() == *via)) || (tapdev == mr->device()) ) ) {
+				for(std::list< SharedPtr<ManagedRoute> >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();++mr) {
+					if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (tapdev == (*mr)->device()) ) ) {
 						haveRoute = true;
-						mr->sync();
+						(*mr)->sync();
 						break;
 					}
 				}
@@ -1179,8 +1179,8 @@ public:
 					continue;
 
 				// Add and apply new routes
-				n.managedRoutes.push_back(ManagedRoute());
-				if (!n.managedRoutes.back().set(*target,*via,tapdev))
+				n.managedRoutes.push_back(SharedPtr<ManagedRoute>(new ManagedRoute(*target,*via,tapdev)));
+				if (!n.managedRoutes.back()->sync())
 					n.managedRoutes.pop_back();
 			}
 		}