Tidy up a few minor protocol things, improve documentation in Packet.hpp.

node/IncomingPacket.cpp

@@ -469,29 +469,40 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
-		if (payloadLength() == ZT_ADDRESS_LENGTH) {
-			const Address addr(payload(),ZT_ADDRESS_LENGTH);
+		Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
+		outp.append((unsigned char)Packet::VERB_WHOIS);
+		outp.append(packetId());
+
+		unsigned int count = 0;
+		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
+		while ((ptr + ZT_ADDRESS_LENGTH) <= size()) {
+			const Address addr(field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+			ptr += ZT_ADDRESS_LENGTH;
+
 			const Identity id(RR->topology->getIdentity(addr));
 			if (id) {
-				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
-				outp.append((unsigned char)Packet::VERB_WHOIS);
-				outp.append(packetId());
 				id.serialize(outp,false);
-				outp.armor(peer->key(),true);
-				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+				++count;
 			} else {
+				// If I am not the root and don't know this identity, ask upstream. Downstream
+				// peer may re-request in the future and if so we will be able to provide it.
+				if (!RR->topology->amRoot())
+					RR->sw->requestWhois(addr);
+
 #ifdef ZT_ENABLE_CLUSTER
+				// Distribute WHOIS queries across a cluster if we do not know the ID.
+				// This may result in duplicate OKs to the querying peer, which is fine.
 				if (RR->cluster)
 					RR->cluster->sendDistributedQuery(*this);
 #endif
-				if (!RR->topology->amRoot()) {
-					RR->sw->requestWhois(addr);
-					return false; // packet parse will be attempted again if we get a reply from upstream
-				}
 			}
-		} else {
-			TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str());
 		}
+
+		if (count > 0) {
+			outp.armor(peer->key(),true);
+			RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
+		}
+
 		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
@@ -836,11 +847,26 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 {
 	try {
 		const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_NETWORK_ID);
+		const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS];
 		const MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI));
 		const unsigned int gatherLimit = at<uint32_t>(ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT);
 
 		//TRACE("<<MC %s(%s) GATHER up to %u in %.16llx/%s",source().toString().c_str(),_remoteAddress.toString().c_str(),gatherLimit,nwid,mg.toString().c_str());
 
+		if ((flags & 0x01) != 0) {
+			try {
+				CertificateOfMembership com;
+				com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM);
+				if (com) {
+					SharedPtr<Network> network(RR->node->network(nwid));
+					if (network)
+						network->addCredential(com);
+				}
+			} catch ( ... ) {
+				TRACE("MULTICAST_GATHER from %s(%s): discarded invalid COM",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
+			}
+		}
+
 		if (gatherLimit) {
 			Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
 			outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER);
@@ -854,6 +880,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			}
 
+			// If we are a member of a cluster, distribute this GATHER across it
 #ifdef ZT_ENABLE_CLUSTER
 			if ((RR->cluster)&&(gatheredLocally < gatherLimit))
 				RR->cluster->sendDistributedQuery(*this);
@@ -862,7 +889,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 
 		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP);
 	} catch ( ... ) {
-		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
 	}
 	return true;
 }
@@ -878,7 +905,8 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			// Offset -- size of optional fields added to position of later fields
 			unsigned int offset = 0;
 
-			if ((flags & 0x01) != 0) { // deprecated but still used by older peers
+			if ((flags & 0x01) != 0) {
+				// This is deprecated but may still be sent by old peers
 				CertificateOfMembership com;
 				offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
 				if (com)
@@ -1053,7 +1081,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		// Tracks total length of variable length fields, initialized to originator credential length below
 		unsigned int vlf;
 
-		// Originator credentials
+		// Originator credentials -- right now only a network ID for which the originator is controller or is authorized by controller is allowed
 		const unsigned int originatorCredentialLength = vlf = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 23);
 		uint64_t originatorCredentialNetworkId = 0;
 		if (originatorCredentialLength >= 1) {
@@ -1085,15 +1113,10 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		vlf += at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 29 + vlf);
 
 		// Check credentials (signature already verified)
-		NetworkConfig originatorCredentialNetworkConfig;
 		if (originatorCredentialNetworkId) {
-			if (Network::controllerFor(originatorCredentialNetworkId) == originatorAddress) {
-				if (!RR->node->network(originatorCredentialNetworkId)) {
-					TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we are not a member of that network",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
-					return true;
-				}
-			} else {
-				TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID as credential, is not controller for %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
+			SharedPtr<Network> network(RR->node->network(originatorCredentialNetworkId));
+			if ((!network)||(!network->config().circuitTestingAllowed(originatorAddress))) {
+				TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we don't belong to that network or originator is not allowed'",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
 				return true;
 			}
 		} else {

node/Multicaster.cpp

@@ -224,25 +224,37 @@ void Multicaster::send(
 
 			if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) {
 				gs.lastExplicitGather = now;
-				SharedPtr<Peer> explicitGatherPeers[2];
-				explicitGatherPeers[0] = RR->topology->getBestRoot();
-				const Address nwidc(Network::controllerFor(nwid));
-				if (nwidc != RR->identity.address())
-					explicitGatherPeers[1] = RR->topology->getPeer(nwidc);
-				for(unsigned int k=0;k<2;++k) {
-					const SharedPtr<Peer> &p = explicitGatherPeers[k];
-					if (!p)
-						continue;
-					//TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
-					Packet outp(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
+
+				Address explicitGatherPeers[16];
+				unsigned int numExplicitGatherPeers = 0;
+				SharedPtr<Peer> bestRoot(RR->topology->getBestRoot());
+				if (bestRoot)
+					explicitGatherPeers[numExplicitGatherPeers++] = bestRoot->address();
+				explicitGatherPeers[numExplicitGatherPeers++] = Network::controllerFor(nwid);
+				SharedPtr<Network> network(RR->node->network(nwid));
+				if (network) {
+					std::vector<Address> anchors(network->config().anchors());
+					for(std::vector<Address>::const_iterator a(anchors.begin());a!=anchors.end();++a) {
+						if (*a != RR->identity.address()) {
+							explicitGatherPeers[numExplicitGatherPeers++] = *a;
+							if (numExplicitGatherPeers == 16)
+								break;
+						}
+					}
+				}
+
+				for(unsigned int k=0;k<numExplicitGatherPeers;++k) {
+					const CertificateOfMembership *com = (network) ? (((network->config())&&(network->config().isPrivate())) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0;
+					Packet outp(explicitGatherPeers[k],RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
 					outp.append(nwid);
-					outp.append((uint8_t)0x00);
+					outp.append((uint8_t)((com) ? 0x01 : 0x00));
 					mg.mac().appendTo(outp);
 					outp.append((uint32_t)mg.adi());
 					outp.append((uint32_t)gatherLimit);
+					if (com)
+						com->serialize(outp);
 					RR->sw->send(outp,true);
 				}
-				gatherLimit = 0;
 			}
 
 			gs.txQueue.push_back(OutboundMulticast());

node/NetworkConfig.hpp

@@ -65,6 +65,11 @@
  */
 #define ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR 0x0000040000000000ULL
 
+/**
+ * Device can send CIRCUIT_TESTs for this network
+ */
+#define ZT_NETWORKCONFIG_SPECIALIST_TYPE_CIRCUIT_TESTER 0x0000080000000000ULL
+
 namespace ZeroTier {
 
 // Dictionary capacity needed for max size network config
@@ -273,6 +278,21 @@ public:
 		return false;
 	}
 
+	/**
+	 * @param byPeer Address to check
+	 * @return True if this peer is allowed to do circuit tests on this network (controller is always true)
+	 */
+	inline bool circuitTestingAllowed(const Address &byPeer) const
+	{
+		if (byPeer.toInt() == ((networkId >> 24) & 0xffffffffffULL))
+			return true;
+		for(unsigned int i=0;i<specialistCount;++i) {
+			if ((byPeer == specialists[i])&&((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_CIRCUIT_TESTER) != 0))
+				return true;
+		}
+		return false;
+	}
+
 	/**
 	 * @return True if this network config is non-NULL
 	 */

node/Packet.hpp

@@ -307,6 +307,7 @@
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_FLAGS + 1)
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC + 6)
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI + 4)
+#define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT + 4)
 
 // Note: COM, GATHER_LIMIT, and SOURCE_MAC are optional, and so are specified without size
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
@@ -538,7 +539,7 @@ public:
 		 *   <[8] timestamp (ms since epoch)>
 		 *   <[...] binary serialized identity (see Identity)>
 		 *   <[1] destination address type>
-		 *   [<[...] destination address>]
+		 *   [<[...] destination address to which packet was sent>]
 		 *   <[8] 64-bit world ID of current world>
 		 *   <[8] 64-bit timestamp of current world>
 		 *
@@ -592,20 +593,24 @@ public:
 		/**
 		 * Query an identity by address:
 		 *   <[5] address to look up>
+		 *   [<[...] additional addresses to look up>
 		 *
 		 * OK response payload:
 		 *   <[...] binary serialized identity>
+		 *  [<[...] additional binary serialized identities>]
 		 *
 		 * If querying a cluster, duplicate OK responses may occasionally occur.
-		 * These should be discarded.
+		 * These must be tolerated, which is easy since they'll have info you
+		 * already have.
 		 *
-		 * If the address is not found, no response is generated. WHOIS requests
-		 * will time out much like ARP requests and similar do in L2.
+		 * If the address is not found, no response is generated. The semantics
+		 * of WHOIS is similar to ARP and NDP in that persistent retrying can
+		 * be performed.
 		 */
 		VERB_WHOIS = 0x04,
 
 		/**
-		 * Meet another node at a given protocol address:
+		 * Relay-mediated NAT traversal or firewall punching initiation:
 		 *   <[1] flags (unused, currently 0)>
 		 *   <[5] ZeroTier address of peer that might be found at this address>
 		 *   <[2] 16-bit protocol address port>
@@ -619,15 +624,6 @@ public:
 		 *
 		 * Upon receipt a peer sends HELLO to establish a direct link.
 		 *
-		 * Nodes should implement rate control, limiting the rate at which they
-		 * respond to these packets to prevent their use in DDOS attacks. Nodes
-		 * may also ignore these messages if a peer is not known or is not being
-		 * actively communicated with.
-		 *
-		 * Unfortunately the physical address format in this message pre-dates
-		 * InetAddress's serialization format. :( ZeroTier is four years old and
-		 * yes we've accumulated a tiny bit of cruft here and there.
-		 *
 		 * No OK or ERROR is generated.
 		 */
 		VERB_RENDEZVOUS = 0x05,
@@ -652,7 +648,6 @@ public:
 		 * Full Ethernet frame with MAC addressing and optional fields:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
-		 *  [<[...] certificate of network membership (DEPRECATED)>]
 		 *   <[6] destination MAC or all zero for destination node>
 		 *   <[6] source MAC or all zero for node of origin>
 		 *   <[2] 16-bit ethertype>
@@ -715,6 +710,9 @@ public:
 		 * This is sent in response to ERROR_NEED_MEMBERSHIP_CERTIFICATE and may
 		 * be pushed at any other time to keep exchanged certificates up to date.
 		 *
+		 * COMs and other credentials need not be for the same network, since each
+		 * includes its own network ID and signature.
+		 *
 		 * OK/ERROR are not generated.
 		 */
 		VERB_NETWORK_CREDENTIALS = 0x0a,
@@ -762,10 +760,10 @@ public:
 		 *   <[6] MAC address of multicast group being queried>
 		 *   <[4] 32-bit ADI for multicast group being queried>
 		 *   <[4] 32-bit requested max number of multicast peers>
-		 *  [<[...] network certificate of membership (DEPRECATED)>]
+		 *   [<[...] network certificate of membership>]
 		 *
 		 * Flags:
-		 *   0x01 - COM is attached (DEPRECATED)
+		 *   0x01 - COM is attached
 		 *
 		 * This message asks a peer for additional known endpoints that have
 		 * LIKEd a given multicast group. It's sent when the sender wishes
@@ -775,8 +773,8 @@ public:
 		 * More than one OK response can occur if the response is broken up across
 		 * multiple packets or if querying a clustered node.
 		 *
-		 * Send VERB_NETWORK_CREDENTIALS prior to GATHERing if doing so from
-		 * upstream nodes like root servers that are not involved in our network.
+		 * The COM should be included so that upstream nodes that are not
+		 * members of our network can validate our request.
 		 *
 		 * OK response payload:
 		 *   <[8] 64-bit network ID>
@@ -795,7 +793,6 @@ public:
 		 * Multicast frame:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
-		 *  [<[...] network certificate of membership (DEPRECATED)>]
 		 *  [<[4] 32-bit implicit gather limit>]
 		 *  [<[6] source MAC>]
 		 *   <[6] destination MAC (multicast address)>
@@ -890,7 +887,7 @@ public:
 		 *   <[...] next hop(s) in path>
 		 *
 		 * Flags:
-		 *   0x01 - Report back to originator at middle hops
+		 *   0x01 - Report back to originator at all hops
 		 *   0x02 - Report back to originator at last hop
 		 *
 		 * Originator credential types:
@@ -948,21 +945,21 @@ public:
 
 		/**
 		 * Circuit test hop report:
-		 *   <[8] 64-bit timestamp (from original test)>
-		 *   <[8] 64-bit test ID (from original test)>
+		 *   <[8] 64-bit timestamp (echoed from original test)>
+		 *   <[8] 64-bit test ID (echoed from original test)>
 		 *   <[8] 64-bit reserved field (set to 0, currently unused)>
 		 *   <[1] 8-bit vendor ID (set to 0, currently unused)>
 		 *   <[1] 8-bit reporter protocol version>
-		 *   <[1] 8-bit reporter major version>
-		 *   <[1] 8-bit reporter minor version>
-		 *   <[2] 16-bit reporter revision>
-		 *   <[2] 16-bit reporter OS/platform>
-		 *   <[2] 16-bit reporter architecture>
+		 *   <[1] 8-bit reporter software major version>
+		 *   <[1] 8-bit reporter software minor version>
+		 *   <[2] 16-bit reporter software revision>
+		 *   <[2] 16-bit reporter OS/platform or 0 if not specified>
+		 *   <[2] 16-bit reporter architecture or 0 if not specified>
 		 *   <[2] 16-bit error code (set to 0, currently unused)>
 		 *   <[8] 64-bit report flags (set to 0, currently unused)>
-		 *   <[8] 64-bit source packet ID>
-		 *   <[5] upstream ZeroTier address from which test was received>
-		 *   <[1] 8-bit source packet hop count (ZeroTier hop count)>
+		 *   <[8] 64-bit packet ID of received CIRCUIT_TEST packet>
+		 *   <[5] upstream ZeroTier address from which CIRCUIT_TEST was received>
+		 *   <[1] 8-bit packet hop count of received CIRCUIT_TEST>
 		 *   <[...] local wire address on which packet was received>
 		 *   <[...] remote wire address from which packet was received>
 		 *   <[2] 16-bit length of additional fields>