Reorg multicast packet, and a whole bunch of refactoring around the pushing of certificates of membership.

node/IncomingPacket.cpp

@@ -128,8 +128,15 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 
 			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
 				SharedPtr<Network> network(RR->nc->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
-				if (network)
-					network->pushMembershipCertificate(source(),true,Utils::now());
+				if (network) {
+					SharedPtr<NetworkConfig> nconf(network->config2());
+					if (nconf) {
+						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
+						nconf->com().serialize(outp);
+						outp.armor(peer->key(),true);
+						_fromSock->send(_remoteAddress,outp.data(),outp.size());
+					}
+				}
 			}	break;
 
 			case Packet::ERROR_NETWORK_ACCESS_DENIED_: {
@@ -138,9 +145,9 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 					network->setAccessDenied();
 			}	break;
 
-			// TODO -- send and accept these to cancel multicast "LIKE"s
-			//case Packet::ERROR_UNWANTED_MULTICAST: {
-			//}	break;
+			case Packet::ERROR_UNWANTED_MULTICAST: {
+				// TODO: unsubscribe
+			}	break;
 
 			default: break;
 		}
@@ -330,10 +337,23 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 
 			case Packet::VERB_MULTICAST_FRAME: {
 				unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_FLAGS];
-				if ((flags & 0x01) != 0) { // frame includes implicit gather results
-					uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID);
-					MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI));
-					_parseGatherResults(RR,peer,nwid,mg,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_GATHER_RESULTS);
+				uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID);
+				MulticastGroup mg(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC,6),6),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI));
+
+				unsigned int offset = 0;
+
+				if ((flags & 0x01) != 0) {
+					// OK(MULTICAST_FRAME) includes certificate of membership update
+					CertificateOfMembership com;
+					offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_PAYLOAD);
+					SharedPtr<Network> network(RR->nc->network(nwid));
+					if ((network)&&(com.hasRequiredFields()))
+						network->addMembershipCertificate(com,false);
+				}
+
+				if ((flags & 0x02) != 0) {
+					// OK(MULTICAST_FRAME) includes implicit gather results
+					_parseGatherResults(RR,peer,nwid,mg,offset + ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_PAYLOAD);
 				}
 			}	break;
 
@@ -540,13 +560,13 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 
 bool IncomingPacket::_doP5_MULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
-	// This handles the old deprecated "P5" multicast frame, and will
-	// go away once there are no longer nodes using this on the network.
-	// We handle these old nodes by accepting these as simple multicasts
-	// and if we are a supernode performing individual relaying of them
-	// to all older nodes that expect them. This won't be too expensive
-	// though since there aren't likely to be many older nodes left after
-	// we do a software update.
+	/* This handles the old deprecated "P5" multicast frame, and will
+	 * go away once there are no longer nodes using this on the network.
+	 * We handle these old nodes by accepting these as simple multicasts
+	 * and if we are a supernode performing individual relaying of them
+	 * to all older nodes that expect them. This won't be too expensive
+	 * though since there aren't likely to be many older nodes left after
+	 * we do a software update. */
 
 	// Quick and dirty -- this is all condemned code in any case
 	static uint64_t p5MulticastDedupBuffer[1024];
@@ -1027,73 +1047,83 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
-		if (size() > ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME) {
-			uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID);
-			unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS];
-			unsigned int gatherLimit = at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT);
-
-			SharedPtr<Network> network(RR->nc->network(nwid)); // will be NULL if not a member
-			if (network) {
-				unsigned int comLen = 0;
-				if ((flags & 0x01) != 0) {
-					CertificateOfMembership com;
-					comLen = com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
-					if (com.hasRequiredFields())
-						network->addMembershipCertificate(com,false);
-				}
+		uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID);
+		unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS];
 
-				if (!network->isAllowed(peer->address())) {
-					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
-					_sendErrorNeedCertificate(RR,peer,network->id());
-					return true;
-				}
+		SharedPtr<Network> network(RR->nc->network(nwid)); // will be NULL if not a member
+		if (network) {
+			// Offset -- size of optional fields added to position of later fields
+			unsigned int offset = 0;
 
-				// Everything after gatherLimit is relative to the size of the
-				// attached certificate, if any.
+			if ((flags & 0x01) != 0) {
+				CertificateOfMembership com;
+				offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
+				if (com.hasRequiredFields())
+					network->addMembershipCertificate(com,false);
+			}
 
-				MulticastGroup to(MAC(field(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,6),6),at<uint32_t>(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI));
-				MAC from(field(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,6),6);
-				unsigned int etherType = at<uint16_t>(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE);
-				unsigned int payloadLen = size() - (comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME);
+			// Check membership after we've read any included COM, since
+			// that cert might be what we needed.
+			if (!network->isAllowed(peer->address())) {
+				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
+				_sendErrorNeedCertificate(RR,peer,network->id());
+				return true;
+			}
 
-				if (payloadLen) {
-					if (!to.mac().isMulticast()) {
-						TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
-						return true;
-					}
+			unsigned int gatherLimit = 0;
+			if ((flags & 0x02) != 0) {
+				gatherLimit = at<uint32_t>(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT);
+				offset += 4;
+			}
 
-					if ((!from)||(from.isMulticast())||(from == network->mac())) {
-						TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
-						return true;
-					}
+			MAC from;
+			if ((flags & 0x04) != 0) {
+				from.setTo(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,6),6);
+				offset += 6;
+			} else {
+				from.fromAddress(source(),nwid);
+			}
 
-					if (from != MAC(peer->address(),network->id())) {
-						if (network->permitsBridging(peer->address())) {
-							network->learnBridgeRoute(from,peer->address());
-						} else {
-							TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id());
-							return true;
-						}
-					}
+			MulticastGroup to(MAC(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,6),6),at<uint32_t>(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI));
+			unsigned int etherType = at<uint16_t>(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE);
+			unsigned int payloadLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME);
 
-					network->tapPut(from,to.mac(),etherType,field(comLen + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,payloadLen),payloadLen);
+			if ((payloadLen > 0)&&(payloadLen < ZT_IF_MTU)) {
+				if (!to.mac().isMulticast()) {
+					TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: destination is unicast, must use FRAME or EXT_FRAME",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
+					return true;
+				}
+				if ((!from)||(from.isMulticast())||(from == network->mac())) {
+					TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: invalid source MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
+					return true;
 				}
 
-				if (gatherLimit) {
-					Packet outp(source(),RR->identity.address(),Packet::VERB_OK);
-					outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME);
-					outp.append(packetId());
-					outp.append(nwid);
-					to.mac().appendTo(outp);
-					outp.append((uint32_t)to.adi());
-					outp.append((unsigned char)0x01); // flag 0x01 = contains gather results
-					if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) {
-						outp.armor(peer->key(),true);
-						_fromSock->send(_remoteAddress,outp.data(),outp.size());
+				if (from != MAC(peer->address(),network->id())) {
+					if (network->permitsBridging(peer->address())) {
+						network->learnBridgeRoute(from,peer->address());
+					} else {
+						TRACE("dropped MULTICAST_FRAME from %s@%s(%s) to %s: sender not allowed to bridge into %.16llx",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str(),network->id());
+						return true;
 					}
 				}
+
+				network->tapPut(from,to.mac(),etherType,field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,payloadLen),offset);
 			}
-		}
+
+			if (gatherLimit) {
+				Packet outp(source(),RR->identity.address(),Packet::VERB_OK);
+				outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME);
+				outp.append(packetId());
+				outp.append(nwid);
+				to.mac().appendTo(outp);
+				outp.append((uint32_t)to.adi());
+				outp.append((unsigned char)0x02); // flag 0x02 = contains gather results
+				if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) {
+					outp.armor(peer->key(),true);
+					_fromSock->send(_remoteAddress,outp.data(),outp.size());
+				}
+			}
+		} // else ignore -- not a member of this network
 
 		peer->receive(RR,_fromSock,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,Utils::now());
 	} catch (std::exception &exc) {

node/Multicaster.cpp

@@ -174,7 +174,7 @@ void Multicaster::send(
 
 			if (count++ >= limit)
 				break;
-			out.sendOnly(*(RR->sw),*ast);
+			out.sendOnly(RR,*ast);
 		}
 
 		for(std::vector<MulticastGroupMember>::const_reverse_iterator m(gs.members.rbegin());m!=gs.members.rend();++m) {
@@ -187,7 +187,7 @@ void Multicaster::send(
 			if (count++ >= limit)
 				break;
 			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),m->address) == alwaysSendTo.end())
-				out.sendOnly(*(RR->sw),m->address);
+				out.sendOnly(RR,m->address);
 		}
 	} else {
 		unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
@@ -235,7 +235,7 @@ void Multicaster::send(
 					continue;
 			}
 
-			out.sendAndLog(*(RR->sw),*ast);
+			out.sendAndLog(RR,*ast);
 		}
 
 		for(std::vector<MulticastGroupMember>::const_reverse_iterator m(gs.members.rbegin());m!=gs.members.rend();++m) {
@@ -246,7 +246,7 @@ void Multicaster::send(
 			}
 
 			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),m->address) == alwaysSendTo.end())
-				out.sendAndLog(*(RR->sw),m->address);
+				out.sendAndLog(RR,m->address);
 		}
 	}
 
@@ -384,7 +384,7 @@ void Multicaster::_add(uint64_t now,uint64_t nwid,MulticastGroupStatus &gs,const
 				continue;
 		}
 
-		tx->sendIfNew(*(RR->sw),member);
+		tx->sendIfNew(RR,member);
 		if (tx->atLimit())
 			gs.txQueue.erase(tx++);
 		else ++tx;

node/Network.cpp

@@ -331,6 +331,26 @@ void Network::addMembershipCertificate(const CertificateOfMembership &cert,bool
 	}
 }
 
+bool Network::peerNeedsOurMembershipCertificate(const Address &to,uint64_t now)
+{
+	Mutex::Lock _l(_lock);
+	if ((_config)&&(!_config->isPublic())&&(_config->com())) {
+		uint64_t pushInterval = _config->com().timestampMaxDelta() / 2;
+		if (pushInterval) {
+			// Give a 1s margin around +/- 1/2 max delta to account for network latency
+			if (pushInterval > 1000)
+				pushInterval -= 1000;
+
+			uint64_t &lastPushed = _lastPushedMembershipCertificate[to];
+			if ((now - lastPushed) > pushInterval) {
+				lastPushed = now;
+				return true;
+			}
+		}
+	}
+	return false;
+}
+
 bool Network::isAllowed(const Address &peer) const
 {
 	try {
@@ -344,6 +364,7 @@ bool Network::isAllowed(const Address &peer) const
 		std::map<Address,CertificateOfMembership>::const_iterator pc(_membershipCertificates.find(peer));
 		if (pc == _membershipCertificates.end())
 			return false; // no certificate on file
+
 		return _config->com().agreesWith(pc->second); // is other cert valid against ours?
 	} catch (std::exception &exc) {
 		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer.toString().c_str(),exc.what());
@@ -522,31 +543,6 @@ void Network::_CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned
 	}
 }
 
-void Network::_pushMembershipCertificate(const Address &peer,bool force,uint64_t now)
-{
-	// assumes _lock is locked and _config is not null
-
-	uint64_t pushTimeout = _config->com().timestampMaxDelta() / 2;
-
-	// Zero means we're still waiting on our own cert
-	if (!pushTimeout)
-		return;
-
-	// Give a 1s margin around +/- 1/2 max delta to account for latency
-	if (pushTimeout > 1000)
-		pushTimeout -= 1000;
-
-	uint64_t &lastPushed = _lastPushedMembershipCertificate[peer];
-	if ((force)||((now - lastPushed) > pushTimeout)) {
-		lastPushed = now;
-		TRACE("pushing membership cert for %.16llx to %s",(unsigned long long)_id,peer.toString().c_str());
-
-		Packet outp(peer,RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
-		_config->com().serialize(outp);
-		RR->sw->send(outp,true);
-	}
-}
-
 void Network::_restoreState()
 {
 	Buffer<ZT_NETWORK_CERT_WRITE_BUF_SIZE> buf;

node/Network.hpp

@@ -215,18 +215,19 @@ public:
 	void addMembershipCertificate(const CertificateOfMembership &cert,bool forceAccept);
 
 	/**
-	 * Push our membership certificate to a peer
+	 * Check if we should push membership certificate to a peer, and update last pushed
 	 *
-	 * @param peer Destination peer address
-	 * @param force If true, push even if we've already done so within required time frame
+	 * If we haven't pushed a cert to this peer in a long enough time, this returns
+	 * true and updates the last pushed time. Otherwise it returns false.
+	 *
+	 * This doesn't actually send anything, since COMs can hitch a ride with several
+	 * different kinds of packets.
+	 *
+	 * @param to Destination peer
 	 * @param now Current time
+	 * @return True if we should include a COM with whatever we're currently sending
 	 */
-	inline void pushMembershipCertificate(const Address &peer,bool force,uint64_t now)
-	{
-		Mutex::Lock _l(_lock);
-		if ((_config)&&(!_config->isPublic())&&(_config->com()))
-			_pushMembershipCertificate(peer,force,now);
-	}
+	bool peerNeedsOurMembershipCertificate(const Address &to,uint64_t now);
 
 	/**
 	 * @param peer Peer address to check
@@ -445,7 +446,6 @@ public:
 private:
 	static void _CBhandleTapData(void *arg,const MAC &from,const MAC &to,unsigned int etherType,const Buffer<4096> &data);
 
-	void _pushMembershipCertificate(const Address &peer,bool force,uint64_t now);
 	void _restoreState();
 	void _dumpMembershipCerts();
 

node/OutboundMulticast.cpp

@@ -26,9 +26,13 @@
  */
 
 #include "Constants.hpp"
+#include "RuntimeEnvironment.hpp"
 #include "OutboundMulticast.hpp"
 #include "Switch.hpp"
+#include "NodeConfig.hpp"
+#include "Network.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Utils.hpp"
 
 namespace ZeroTier {
 
@@ -47,31 +51,57 @@ void OutboundMulticast::init(
 {
 	_timestamp = timestamp;
 	_nwid = nwid;
-	_source = src;
-	_destination = dest;
 	_limit = limit;
-	_etherType = etherType;
 
-	_packet.setSource(self);
-	_packet.setVerb(Packet::VERB_MULTICAST_FRAME);
+	uint8_t flags = 0;
+	if (gatherLimit) flags |= 0x02;
+	if (src) flags |= 0x04;
 
-	self.appendTo(_packet);
-	_packet.append((uint64_t)nwid);
-	_packet.append((uint8_t)((com) ? 0x01 : 0x00));
-	_packet.append((uint32_t)gatherLimit);
-	if (com) com->serialize(_packet);
-	_packet.append((uint32_t)dest.adi());
-	dest.mac().appendTo(_packet);
-	src.appendTo(_packet);
-	_packet.append((uint16_t)etherType);
-	_packet.append(payload,len);
+	_packetNoCom.setSource(self);
+	_packetNoCom.setVerb(Packet::VERB_MULTICAST_FRAME);
+	_packetNoCom.append((uint64_t)nwid);
+	_packetNoCom.append(flags);
+	if (gatherLimit) _packetNoCom.append((uint32_t)gatherLimit);
+	if (src) src.appendTo(_packetNoCom);
+	dest.mac().appendTo(_packetNoCom);
+	_packetNoCom.append((uint32_t)dest.adi());
+	_packetNoCom.append((uint16_t)etherType);
+	_packetNoCom.append(payload,len);
+	_packetNoCom.compress();
 
-	_packet.compress();
+	if (com) {
+		_haveCom = true;
+		flags |= 0x01;
+
+		_packetWithCom.setSource(self);
+		_packetWithCom.setVerb(Packet::VERB_MULTICAST_FRAME);
+		_packetWithCom.append((uint64_t)nwid);
+		_packetWithCom.append(flags);
+		com->serialize(_packetWithCom);
+		if (gatherLimit) _packetWithCom.append((uint32_t)gatherLimit);
+		if (src) src.appendTo(_packetWithCom);
+		dest.mac().appendTo(_packetWithCom);
+		_packetWithCom.append((uint32_t)dest.adi());
+		_packetWithCom.append((uint16_t)etherType);
+		_packetWithCom.append(payload,len);
+		_packetWithCom.compress();
+	} else _haveCom = false;
 }
 
-void OutboundMulticast::sendOnly(Switch &sw,const Address &toAddr)
+void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr)
 {
-	sw.send(Packet(_packet,toAddr),true);
+	if (_haveCom) {
+		SharedPtr<Network> network(RR->nc->network(_nwid));
+		if (network->peerNeedsOurMembershipCertificate(toAddr,Utils::now())) {
+			_packetWithCom.newInitializationVector();
+			_packetWithCom.setDestination(toAddr);
+			RR->sw->send(_packetWithCom,true);
+			return;
+		}
+	}
+	_packetNoCom.newInitializationVector();
+	_packetNoCom.setDestination(toAddr);
+	RR->sw->send(_packetNoCom,true);
 }
 
 } // namespace ZeroTier

node/OutboundMulticast.hpp

@@ -41,8 +41,8 @@
 
 namespace ZeroTier {
 
-class Switch;
 class CertificateOfMembership;
+class RuntimeEnvironment;
 
 /**
  * An outbound multicast packet
@@ -65,10 +65,10 @@ public:
 	 * @param timestamp Creation time
 	 * @param self My ZeroTier address
 	 * @param nwid Network ID
-	 * @param com Certificate of membership to attach or NULL to omit
+	 * @param com Certificate of membership or NULL if none available
 	 * @param limit Multicast limit for desired number of packets to send
 	 * @param gatherLimit Number to lazily/implicitly gather with this frame or 0 for none
-	 * @param src Source MAC address of frame
+	 * @param src Source MAC address of frame or NULL to imply compute from sender ZT address
 	 * @param dest Destination multicast group (MAC + ADI)
 	 * @param etherType 16-bit Ethernet type ID
 	 * @param payload Data
@@ -107,49 +107,48 @@ public:
 	/**
 	 * Just send without checking log
 	 *
-	 * @param sw Switch instance to send packets
+	 * @param RR Runtime environment
 	 * @param toAddr Destination address
 	 */
-	void sendOnly(Switch &sw,const Address &toAddr);
+	void sendOnly(const RuntimeEnvironment *RR,const Address &toAddr);
 
 	/**
 	 * Just send and log but do not check sent log
 	 *
-	 * @param sw Switch instance to send packets
+	 * @param RR Runtime environment
 	 * @param toAddr Destination address
 	 */
-	inline void sendAndLog(Switch &sw,const Address &toAddr)
+	inline void sendAndLog(const RuntimeEnvironment *RR,const Address &toAddr)
 	{
 		_alreadySentTo.push_back(toAddr);
-		sendOnly(sw,toAddr);
+		sendOnly(RR,toAddr);
 	}
 
 	/**
 	 * Try to send this to a given peer if it hasn't been sent to them already
 	 *
-	 * @param sw Switch instance to send packets
+	 * @param RR Runtime environment
 	 * @param toAddr Destination address
 	 * @return True if address is new and packet was sent to switch, false if duplicate
 	 */
-	inline bool sendIfNew(Switch &sw,const Address &toAddr)
+	inline bool sendIfNew(const RuntimeEnvironment *RR,const Address &toAddr)
 	{
 		for(std::vector<Address>::iterator a(_alreadySentTo.begin());a!=_alreadySentTo.end();++a) {
 			if (*a == toAddr)
 				return false;
 		}
-		sendAndLog(sw,toAddr);
+		sendAndLog(RR,toAddr);
 		return true;
 	}
 
 private:
 	uint64_t _timestamp;
 	uint64_t _nwid;
-	MAC _source;
-	MulticastGroup _destination;
 	unsigned int _limit;
-	unsigned int _etherType;
-	Packet _packet; // packet contains basic structure of MULTICAST_FRAME and payload, is re-used with new IV and addressing each time
+	Packet _packetNoCom;
+	Packet _packetWithCom;
 	std::vector<Address> _alreadySentTo;
+	bool _haveCom;
 };
 
 } // namespace ZeroTier

node/Packet.hpp

@@ -268,15 +268,15 @@
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_MAC + 6)
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI + 4)
 
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ORIGIN (ZT_PACKET_IDX_PAYLOAD)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ORIGIN + 5)
+// Note: COM, GATHER_LIMIT, and SOURCE_MAC are optional, and so are specified without size
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID + 8)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT + 4)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI + 4)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC + 6)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC + 6)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC + 6)
+#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI + 4)
 #define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE + 2)
 
 #define ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP (ZT_PROTO_VERB_OK_IDX_PAYLOAD)
@@ -302,7 +302,7 @@
 #define ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC (ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_NETWORK_ID + 8)
 #define ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI (ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_MAC + 6)
 #define ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_FLAGS (ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_ADI + 4)
-#define ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_GATHER_RESULTS (ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_FLAGS + 1)
+#define ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_PAYLOAD (ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_FLAGS + 1)
 
 // ---------------------------------------------------------------------------
 
@@ -769,48 +769,43 @@ public:
 		VERB_MULTICAST_GATHER = 13,
 
 		/* Multicast frame:
-		 *   <[5] ZT address of original source of multicast frame>
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
-		 *   <[4] 32-bit (suggested) gather limit or 0 for no implicit gathering>
-		 *  [<[...] network certificate of membership if included>]
-		 *   <[4] 32-bit multicast ADI (note that this is out of order here -- it precedes MAC)>
-		 *   <[6] destination MAC or all zero for destination node>
-		 *   <[6] source MAC or all zero for node of origin>
+		 *  [<[...] network certificate of membership>]
+		 *  [<[4] 32-bit implicit gather limit>]
+		 *  [<[6] source MAC>]
+		 *   <[6] destination MAC (multicast address)>
+		 *   <[4] 32-bit multicast ADI (multicast address extension)>
 		 *   <[2] 16-bit ethertype>
 		 *   <[...] ethernet payload>
 		 *
 		 * Flags:
 		 *   0x01 - Network certificate of membership is attached
+		 *   0x02 - Implicit gather limit field is present
+		 *   0x04 - Source MAC is specified -- otherwise it's computed from sender
 		 *
-		 * This is similar to EXT_FRAME but carries a multicast, and is sent
-		 * out to recipients on a multicast list. It may also specify a desired
-		 * number of multicast peers to gather if additional multicast peers
-		 * for this group are desired.
-		 *
-		 * (ADI precedes MAC here so that everything from destination MAC forward
-		 * could be treated as a raw Ethernet frame.)
-		 *
-		 * OK responses are optional and are currently only returned if gathering
-		 * of additional multicast peers is requested.
+		 * OK and ERROR responses are optional. OK may be generated if there are
+		 * implicit gather results or if the recipient wants to send its own
+		 * updated certificate of network membership to the sender. ERROR may be
+		 * generated if a certificate is needed or if multicasts to this group
+		 * are no longer wanted (multicast unsubscribe).
 		 *
 		 * OK response payload:
 		 *   <[8] 64-bit network ID>
 		 *   <[6] MAC address of multicast group>
 		 *   <[4] 32-bit ADI for multicast group>
 		 *   <[1] flags>
+		 *  [<[...] network certficate of membership>]
 		 *  [<[...] implicit gather results if flag 0x01 is set>]
 		 *
-		 * Flags:
-		 *   0x01 - OK include implicit gather results
+		 * OK flags (same bits as request flags):
+		 *   0x01 - OK includes certificate of network membership
+		 *   0x02 - OK includes implicit gather results
 		 *
 		 * ERROR response payload:
 		 *   <[8] 64-bit network ID>
 		 *   <[6] multicast group MAC>
 		 *   <[4] 32-bit multicast group ADI>
-		 *
-		 * ERRORs are optional and can be generated if a certificate is needed or if
-		 * multicasts for this multicast group are no longer wanted.
 		 */
 		VERB_MULTICAST_FRAME = 14
 	};

node/Switch.cpp

@@ -170,11 +170,16 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 
 		Address toZT(to.toAddress(network->id()));
 		if (network->isAllowed(toZT)) {
-			// TODO: we can refactor this to push certificates with EXT_FRAME
-			network->pushMembershipCertificate(toZT,false,Utils::now());
+			if (network->peerNeedsOurMembershipCertificate(toZT,Utils::now())) {
+				// TODO: once there are no more <1.0.0 nodes around, we can
+				// bundle this with EXT_FRAME instead of sending two packets.
+				Packet outp(toZT,RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
+				nconf->com().serialize(outp);
+				send(outp,true);
+			}
 
 			if (fromBridged) {
-				// Must use EXT_FRAME if source is not myself
+				// EXT_FRAME is used for bridging or if we want to include a COM
 				Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME);
 				outp.append(network->id());
 				outp.append((unsigned char)0);
@@ -185,7 +190,7 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 				outp.compress();
 				send(outp,true);
 			} else {
-				// VERB_FRAME is really just lighter weight EXT_FRAME, can use for direct-to-direct (before bridging this was the only unicast method)
+				// FRAME is a shorter version that can be used when there's no bridging and no COM
 				Packet outp(toZT,RR->identity.address(),Packet::VERB_FRAME);
 				outp.append(network->id());
 				outp.append((uint16_t)etherType);