Yet more multicast work.

node/CMakeLists.txt

@@ -30,7 +30,6 @@ set(core_headers
 	Network.hpp
 	NetworkConfig.hpp
 	Node.hpp
-	OutboundMulticast.hpp
 	Packet.hpp
 	Path.hpp
 	Peer.hpp
@@ -62,7 +61,6 @@ set(core_src
 	Network.cpp
 	NetworkConfig.cpp
 	Node.cpp
-	OutboundMulticast.cpp
 	Packet.cpp
 	Path.cpp
 	Peer.cpp

node/Multicaster.cpp

@@ -16,14 +16,9 @@
 #include "Constants.hpp"
 #include "RuntimeEnvironment.hpp"
 #include "Multicaster.hpp"
+#include "Network.hpp"
 #include "Topology.hpp"
 #include "Switch.hpp"
-#include "Packet.hpp"
-#include "Peer.hpp"
-#include "C25519.hpp"
-#include "CertificateOfMembership.hpp"
-#include "Node.hpp"
-#include "Network.hpp"
 
 namespace ZeroTier {
 
@@ -37,224 +32,140 @@ void Multicaster::send(
 	void *tPtr,
 	int64_t now,
 	const SharedPtr<Network> &network,
-	const Address &origin,
 	const MulticastGroup &mg,
 	const MAC &src,
 	unsigned int etherType,
-	const void *data,
+	const unsigned int existingBloomMultiplier,
+	const uint8_t existingBloom[ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS / 8],
+	const void *const data,
 	unsigned int len)
 {
-#if 0
-	unsigned long idxbuf[4096];
-	unsigned long *indexes = idxbuf;
-
-	try {
-		Mutex::Lock _l(_groups_m);
-		MulticastGroupStatus &gs = _groups[Multicaster::Key(network->id(),mg)];
-
-		if (!gs.members.empty()) {
-			// Allocate a memory buffer if group is monstrous
-			if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
-				indexes = new unsigned long[gs.members.size()];
-
-			// Generate a random permutation of member indexes
-			for(unsigned long i=0;i<gs.members.size();++i)
-				indexes[i] = i;
-			for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
-				unsigned long j = (unsigned long)Utils::random() % (i + 1);
-				unsigned long tmp = indexes[j];
-				indexes[j] = indexes[i];
-				indexes[i] = tmp;
-			}
-		}
-
-		Address activeBridges[ZT_MAX_NETWORK_SPECIALISTS];
-		const unsigned int activeBridgeCount = network->config().activeBridges(activeBridges);
-		const unsigned int limit = network->config().multicastLimit;
-
-		if (gs.members.size() >= limit) {
-			// Skip queue if we already have enough members to complete the send operation
-			OutboundMulticast out;
+	static const unsigned int PRIMES[16] = { 2,3,5,7,11,13,17,19,23,29,31,37,41,43,47,53 };
+
+	if (unlikely(len > ZT_MAX_MTU)) return; // sanity check
+
+	const NetworkConfig &config = network->config();
+	if (config.multicastLimit == 0) return; // multicast disabled
+	Address bridges[ZT_MAX_NETWORK_SPECIALISTS],multicastReplicators[ZT_MAX_NETWORK_SPECIALISTS];
+	unsigned int bridgeCount = 0,multicastReplicatorCount = 0;
+	for(unsigned int i=0;i<config.specialistCount;++i) {
+		if ((config.specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0)
+			bridges[bridgeCount++] = config.specialists[i];
+		if ((config.specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR) != 0)
+			multicastReplicators[multicastReplicatorCount++] = config.specialists[i];
+	}
 
-			out.init(
-				RR,
-				now,
-				network->id(),
-				network->config().disableCompression(),
-				limit,
-				1, // we'll still gather a little from peers to keep multicast list fresh
-				src,
-				mg,
-				etherType,
-				data,
-				len);
+	std::vector< std::pair<int64_t,Address> > recipients;
+	bool needMoar = false;
+	for(unsigned int i=0;i<bridgeCount;++i)
+		recipients.push_back(std::pair<int64_t,Address>(9223372036854775807LL,bridges[i]));
+	{
+		Mutex::Lock l2(_groups_l);
+		_getMembersByTime(network->id(),mg,recipients);
+	}
+	std::sort(recipients.begin() + bridgeCount,recipients.end(),std::greater< std::pair<int64_t,Address> >());
+	recipients.erase(std::unique(recipients.begin(),recipients.end()),recipients.end());
+	if (recipients.size() > config.multicastLimit) {
+		recipients.resize(config.multicastLimit);
+	} else if (recipients.size() < config.multicastLimit) {
+		needMoar = true;
+	}
 
-			unsigned int count = 0;
+	_txQueue_l.lock();
+	_OM *om = &(_txQueue[_txQueuePtr++ % ZT_TX_QUEUE_SIZE]);
+	Mutex::Lock ql(om->lock);
+	_txQueue_l.unlock();
+
+	om->nwid = network->id();
+	om->src = src;
+	om->mg = mg;
+	om->etherType = etherType;
+	om->dataSize = len;
+	memcpy(om->data,data,len);
+
+	if (existingBloom) {
+		om->bloomFilterMultiplier = existingBloomMultiplier;
+		memcpy(om->bloomFilter,existingBloom,sizeof(om->bloomFilter));
+	} else {
+		om->bloomFilterMultiplier = 1;
+		memset(om->bloomFilter,0,sizeof(om->bloomFilter));
+
+		if (recipients.size() > 1) {
+			unsigned int mult = 1;
+			unsigned int bestMultColl = 0xffffffff;
+			for(int k=0;k<16;++k) { // 16 == arbitrary limit on iterations for this search, also must be <= size of PRIMES
+				unsigned int coll = 0;
+				for(std::vector< std::pair<int64_t,Address> >::const_iterator r(recipients.begin());r!=recipients.end();++r) {
+					const unsigned int bfi = mult * (unsigned int)r->second.toInt();
+					const unsigned int byte = (bfi >> 3) % sizeof(om->bloomFilter);
+					const uint8_t bit = 1 << (bfi & 7);
+					coll += ((om->bloomFilter[byte] & bit) != 0);
+					om->bloomFilter[byte] |= bit;
+				}
+				memset(om->bloomFilter,0,sizeof(om->bloomFilter));
 
-			for(unsigned int i=0;i<activeBridgeCount;++i) {
-				if ((activeBridges[i] != RR->identity.address())&&(activeBridges[i] != origin)) {
-					out.sendOnly(RR,tPtr,activeBridges[i]); // optimization: don't use dedup log if it's a one-pass send
-					if (++count >= limit)
+				if (coll <= bestMultColl) {
+					om->bloomFilterMultiplier = mult;
+					if (coll == 0) // perfect score, no need to continue searching
 						break;
+					bestMultColl = coll;
 				}
-			}
 
-			unsigned long idx = 0;
-			while ((count < limit)&&(idx < gs.members.size())) {
-				const Address ma(gs.members[indexes[idx++]].address);
-				if ((std::find(activeBridges,activeBridges + activeBridgeCount,ma) == (activeBridges + activeBridgeCount))&&(ma != origin)) {
-					out.sendOnly(RR,tPtr,ma); // optimization: don't use dedup log if it's a one-pass send
-					++count;
-				}
-			}
-		} else {
-			if (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) {
-				RR->t->outgoingNetworkFrameDropped(tPtr,network,src,mg.mac(),etherType,0,len,"multicast TX queue is full");
-				return;
+				mult = PRIMES[k];
 			}
+		}
+	}
 
-			const unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
-
-			if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) {
-				gs.lastExplicitGather = now;
-
-				Address explicitGatherPeers[16];
-				unsigned int numExplicitGatherPeers = 0;
-
-				explicitGatherPeers[numExplicitGatherPeers++] = network->controller();
-
-				/*
-				Address ac[ZT_MAX_NETWORK_SPECIALISTS];
-				const unsigned int accnt = network->config().alwaysContactAddresses(ac);
-				unsigned int shuffled[ZT_MAX_NETWORK_SPECIALISTS];
-				for(unsigned int i=0;i<accnt;++i)
-					shuffled[i] = i;
-				for(unsigned int i=0,k=accnt>>1;i<k;++i) {
-					const uint64_t x = Utils::random();
-					const unsigned int x1 = shuffled[(unsigned int)x % accnt];
-					const unsigned int x2 = shuffled[(unsigned int)(x >> 32) % accnt];
-					const unsigned int tmp = shuffled[x1];
-					shuffled[x1] = shuffled[x2];
-					shuffled[x2] = tmp;
-				}
-				for(unsigned int i=0;i<accnt;++i) {
-					explicitGatherPeers[numExplicitGatherPeers++] = ac[shuffled[i]];
-					if (numExplicitGatherPeers == 16)
-						break;
-				}
-				*/
+	if (multicastReplicatorCount > 0) {
+		// SEND
+		return;
+	}
 
-				/*
-				std::vector<Address> anchors(network->config().anchors());
-				for(std::vector<Address>::const_iterator a(anchors.begin());a!=anchors.end();++a) {
-					if (*a != RR->identity.address()) {
-						explicitGatherPeers[numExplicitGatherPeers++] = *a;
-						if (numExplicitGatherPeers == 16)
+	SharedPtr<Peer> nextHops[2]; // these by definition are protocol version >= 11
+	unsigned int nextHopsBestLatency[2] = { 0xffff,0xffff };
+	for(std::vector< std::pair<int64_t,Address> >::const_iterator r(recipients.begin());r!=recipients.end();++r) {
+		const unsigned int bfi = om->bloomFilterMultiplier * (unsigned int)r->second.toInt();
+		const unsigned int bfbyte = (bfi >> 3) % sizeof(om->bloomFilter);
+		const uint8_t bfbit = 1 << (bfi & 7);
+		if ((om->bloomFilter[bfbyte] & bfbit) != 0) {
+			continue;
+		} else {
+			SharedPtr<Peer> peer(RR->topology->get(r->second));
+			if (peer) {
+				if (peer->remoteVersionProtocol() < 11) {
+					// SEND
+
+					om->bloomFilter[bfbyte] |= bfbit;
+					continue;
+				} else {
+					const unsigned int lat = peer->latency(now);
+					for(unsigned int nh=0;nh<2;++nh) {
+						if (lat <= nextHopsBestLatency[nh]) {
+							nextHopsBestLatency[nh] = lat;
+							nextHops[nh] = peer;
 							break;
+						}
 					}
 				}
-				*/
-
-				for(unsigned int k=0;k<numExplicitGatherPeers;++k) {
-					const CertificateOfMembership *com = (network) ? ((network->config().com) ? &(network->config().com) : (const CertificateOfMembership *)0) : (const CertificateOfMembership *)0;
-					Packet outp(explicitGatherPeers[k],RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
-					outp.append(network->id());
-					outp.append((uint8_t)((com) ? 0x01 : 0x00));
-					mg.mac().appendTo(outp);
-					outp.append((uint32_t)mg.adi());
-					outp.append((uint32_t)gatherLimit);
-					if (com)
-						com->serialize(outp);
-					RR->node->expectReplyTo(outp.packetId());
-					RR->sw->send(tPtr,outp,true);
-				}
-			}
-
-			gs.txQueue.push_back(OutboundMulticast());
-			OutboundMulticast &out = gs.txQueue.back();
-
-			out.init(
-				RR,
-				now,
-				network->id(),
-				network->config().disableCompression(),
-				limit,
-				gatherLimit,
-				src,
-				mg,
-				etherType,
-				data,
-				len);
-
-			if (origin)
-				out.logAsSent(origin);
-
-			unsigned int count = 0;
-
-			for(unsigned int i=0;i<activeBridgeCount;++i) {
-				if (activeBridges[i] != RR->identity.address()) {
-					out.sendAndLog(RR,tPtr,activeBridges[i]);
-					if (++count >= limit)
-						break;
-				}
 			}
+		}
+	}
 
-			unsigned long idx = 0;
-			while ((count < limit)&&(idx < gs.members.size())) {
-				Address ma(gs.members[indexes[idx++]].address);
-				if (std::find(activeBridges,activeBridges + activeBridgeCount,ma) == (activeBridges + activeBridgeCount)) {
-					out.sendAndLog(RR,tPtr,ma);
-					++count;
-				}
-			}
+	for(unsigned int nh=0;nh<2;++nh) {
+		if (nextHops[nh]) {
+			const unsigned int bfi = om->bloomFilterMultiplier * (unsigned int)nextHops[nh]->address().toInt();
+			om->bloomFilter[(bfi >> 3) % sizeof(om->bloomFilter)] |= 1 << (bfi & 7);
 		}
-	} catch ( ... ) {} // this is a sanity check to catch any failures and make sure indexes[] still gets deleted
+	}
 
-	// Free allocated memory buffer if any
-	if (indexes != idxbuf)
-		delete [] indexes;
-#endif
+	for(unsigned int nh=0;nh<2;++nh) {
+		if (nextHops[nh]) {
+		}
+	}
 }
 
 void Multicaster::clean(int64_t now)
 {
-#if 0
-	{
-		Mutex::Lock _l(_groups_m);
-		Multicaster::Key *k = (Multicaster::Key *)0;
-		MulticastGroupStatus *s = (MulticastGroupStatus *)0;
-		Hashtable<Multicaster::Key,MulticastGroupStatus>::Iterator mm(_groups);
-		while (mm.next(k,s)) {
-			for(std::list<OutboundMulticast>::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) {
-				if ((tx->expired(now))||(tx->atLimit()))
-					s->txQueue.erase(tx++);
-				else ++tx;
-			}
-
-			unsigned long count = 0;
-			{
-				std::vector<MulticastGroupMember>::iterator reader(s->members.begin());
-				std::vector<MulticastGroupMember>::iterator writer(reader);
-				while (reader != s->members.end()) {
-					if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
-						*writer = *reader;
-						++writer;
-						++count;
-					}
-					++reader;
-				}
-			}
-
-			if (count) {
-				s->members.resize(count);
-			} else if (s->txQueue.empty()) {
-				_groups.erase(*k);
-			} else {
-				s->members.clear();
-			}
-		}
-	}
-#endif
 }
 
 } // namespace ZeroTier

node/Multicaster.hpp

@@ -25,10 +25,13 @@
 #include "Address.hpp"
 #include "MAC.hpp"
 #include "MulticastGroup.hpp"
-#include "OutboundMulticast.hpp"
 #include "Utils.hpp"
 #include "Mutex.hpp"
 #include "SharedPtr.hpp"
+#include "Packet.hpp"
+
+// Size in bits -- this is pretty close to the maximum allowed by the protocol
+#define ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS 16384
 
 namespace ZeroTier {
 
@@ -54,10 +57,10 @@ public:
 	 * @param mg Multicast group
 	 * @param member New member address
 	 */
-	inline void add(const int64_t now,const uint64_t nwid,const MulticastGroup &mg,const Address &member)
+	ZT_ALWAYS_INLINE void add(const int64_t now,const uint64_t nwid,const MulticastGroup &mg,const Address &member)
 	{
 		Mutex::Lock l(_groups_l);
-		_groups[Multicaster::Key(nwid,mg)].set(member,now);
+		_groups[_K(nwid,mg)].set(member,now);
 	}
 
 	/**
@@ -73,11 +76,11 @@ public:
 	 * @param count Number of addresses
 	 * @param totalKnown Total number of known addresses as reported by peer
 	 */
-	inline void addMultiple(const int64_t now,const uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,const unsigned int totalKnown)
+	ZT_ALWAYS_INLINE void addMultiple(const int64_t now,const uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,const unsigned int totalKnown)
 	{
 		Mutex::Lock l(_groups_l);
 		const uint8_t *a = (const uint8_t *)addresses;
-		Hashtable< Address,int64_t > &members = _groups[Multicaster::Key(nwid,mg)];
+		Hashtable< Address,int64_t > &members = _groups[_K(nwid,mg)];
 		while (count--) {
 			members.set(Address(a,ZT_ADDRESS_LENGTH),now);
 			a += ZT_ADDRESS_LENGTH;
@@ -91,10 +94,10 @@ public:
 	 * @param mg Multicast group
 	 * @param member Member to unsubscribe
 	 */
-	inline void remove(const uint64_t nwid,const MulticastGroup &mg,const Address &member)
+	ZT_ALWAYS_INLINE void remove(const uint64_t nwid,const MulticastGroup &mg,const Address &member)
 	{
 		Mutex::Lock l(_groups_l);
-		const Multicaster::Key gk(nwid,mg);
+		const _K gk(nwid,mg);
 		Hashtable< Address,int64_t > *const members = _groups.get(gk);
 		if (members) {
 			members->erase(member);
@@ -115,26 +118,12 @@ public:
 	 * @return Total number of known members (regardless of when function aborted)
 	 */
 	template<typename F>
-	inline unsigned long eachMember(const uint64_t nwid,const MulticastGroup &mg,F func) const
+	ZT_ALWAYS_INLINE unsigned long eachMember(const uint64_t nwid,const MulticastGroup &mg,F func) const
 	{
 		std::vector< std::pair<int64_t,Address> > sortedByTime;
-		{
-			Mutex::Lock l(_groups_l);
-			const Multicaster::Key gk(nwid,mg);
-			const Hashtable< Address,int64_t > *const members = _groups.get(gk);
-			if (members) {
-				totalKnown = members->size();
-				sortedByTime.reserve(totalKnown);
-				{
-					Hashtable< Address,int64_t >::Iterator mi(*const_cast<Hashtable< Address,int64_t > *>(members));
-					Address *mik = nullptr;
-					int64_t *miv = nullptr;
-					while (mi.next(mik,miv))
-						sortedByTime.push_back(std::pair<int64_t,Address>(*miv,*mik));
-				}
-				std::sort(sortedByTime.begin(),sortedByTime.end());
-			}
-		}
+		Mutex::Lock l(_groups_l);
+		_getMembersByTime(nwid,mg,sortedByTime);
+		std::sort(sortedByTime.begin(),sortedByTime.end());
 		for(std::vector< std::pair<int64_t,Address> >::const_reverse_iterator i(sortedByTime.begin());i!=sortedByTime.end();++i) {
 			if (!func(i->second))
 				break;
@@ -148,10 +137,11 @@ public:
 	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param now Current time
 	 * @param network Network
-	 * @param origin Origin of multicast (to not return to sender) or NULL if none
 	 * @param mg Multicast group
 	 * @param src Source Ethernet MAC address or NULL to skip in packet and compute from ZT address (non-bridged mode)
 	 * @param etherType Ethernet frame type
+	 * @param existingBloomMultiplier Existing bloom filter multiplier or 0 if none
+	 * @param existingBloom Existing bloom filter or NULL if none
 	 * @param data Packet data
 	 * @param len Length of packet data
 	 */
@@ -159,11 +149,12 @@ public:
 		void *tPtr,
 		int64_t now,
 		const SharedPtr<Network> &network,
-		const Address &origin,
 		const MulticastGroup &mg,
 		const MAC &src,
 		unsigned int etherType,
-		const void *data,
+		const unsigned int existingBloomMultiplier,
+		const uint8_t existingBloom[ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS / 8],
+		const void *const data,
 		unsigned int len);
 
 	/**
@@ -175,27 +166,74 @@ public:
 	void clean(int64_t now);
 
 private:
-	struct Key
+	ZT_ALWAYS_INLINE void _getMembersByTime(const uint64_t nwid,const MulticastGroup &mg,std::vector< std::pair<int64_t,Address> > &byTime)
 	{
-		ZT_ALWAYS_INLINE Key() : nwid(0),mg() {}
-		ZT_ALWAYS_INLINE Key(const uint64_t n,const MulticastGroup &g) : nwid(n),mg(g) {}
+		// assumes _groups_l is locked
+		const _K gk(nwid,mg);
+		const Hashtable< Address,int64_t > *const members = _groups.get(gk);
+		if (members) {
+			byTime.reserve(members->size());
+			{
+				Hashtable< Address,int64_t >::Iterator mi(*const_cast<Hashtable< Address,int64_t > *>(members));
+				Address *mik = nullptr;
+				int64_t *miv = nullptr;
+				while (mi.next(mik,miv))
+					byTime.push_back(std::pair<int64_t,Address>(*miv,*mik));
+			}
+		}
+	}
 
+	struct _K
+	{
 		uint64_t nwid;
 		MulticastGroup mg;
 
-		ZT_ALWAYS_INLINE bool operator==(const Key &k) const { return ((nwid == k.nwid)&&(mg == k.mg)); }
-		ZT_ALWAYS_INLINE bool operator!=(const Key &k) const { return ((nwid != k.nwid)||(mg != k.mg)); }
-
+		ZT_ALWAYS_INLINE _K() : nwid(0),mg() {}
+		ZT_ALWAYS_INLINE _K(const uint64_t n,const MulticastGroup &g) : nwid(n),mg(g) {}
+		ZT_ALWAYS_INLINE bool operator==(const _K &k) const { return ((nwid == k.nwid)&&(mg == k.mg)); }
+		ZT_ALWAYS_INLINE bool operator!=(const _K &k) const { return ((nwid != k.nwid)||(mg != k.mg)); }
 		ZT_ALWAYS_INLINE unsigned long hashCode() const { return (mg.hashCode() ^ (unsigned long)(nwid ^ (nwid >> 32))); }
 	};
 
+	/*
+		 * Multicast frame:
+		 *   <[8] 64-bit network ID>
+		 *   <[1] flags>
+		 *  [<[...] network certificate of membership (DEPRECATED)>]
+		 *  [<[4] 32-bit implicit gather limit (DEPRECATED)>]
+		 *  [<[5] ZeroTier address of originating sender (including w/0x08)>]
+		 *  [<[2] 16-bit bloom filter multiplier>]
+		 *  [<[2] 16-bit length of propagation bloom filter in bytes]
+		 *  [<[...] propagation bloom filter>]
+		 *  [<[6] source MAC>]
+		 *   <[6] destination MAC (multicast address)>
+		 *   <[4] 32-bit multicast ADI (multicast address extension)>
+		 *   <[2] 16-bit ethertype>
+		 *   <[...] ethernet payload>
+		 *  [<[2] 16-bit length of signature>]
+		 *  [<[...] signature (algorithm depends on sender identity)>]
+	 */
+
+	struct _OM
+	{
+		uint64_t nwid;
+		MAC src;
+		MulticastGroup mg;
+		unsigned int etherType;
+		unsigned int dataSize;
+		unsigned int bloomFilterMultiplier;
+		uint8_t bloomFilter[ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS / 8];
+		uint8_t data[ZT_MAX_MTU];
+		Mutex lock;
+	};
+
 	const RuntimeEnvironment *const RR;
 
-	OutboundMulticast _txQueue[ZT_TX_QUEUE_SIZE];
+	_OM _txQueue[ZT_TX_QUEUE_SIZE];
 	unsigned int _txQueuePtr;
 	Mutex _txQueue_l;
 
-	Hashtable< Multicaster::Key,Hashtable< Address,int64_t > > _groups;
+	Hashtable< _K,Hashtable< Address,int64_t > > _groups;
 	Mutex _groups_l;
 };
 

node/Network.cpp

@@ -1233,7 +1233,6 @@ void Network::_requestConfiguration(void *tPtr)
 	const Address ctrl(controller());
 
 	ScopedPtr< Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> > rmd(new Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>());
-	rmd->add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION);
 	rmd->add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_VENDOR,(uint64_t)ZT_VENDOR_ZEROTIER);
 	rmd->add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION,(uint64_t)ZT_PROTO_VERSION);
 	rmd->add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MAJOR);

node/NetworkConfig.cpp

@@ -29,7 +29,6 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 
 	// Try to put the more human-readable fields first
 
-	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_VERSION,(uint64_t)ZT_NETWORKCONFIG_VERSION)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,this->networkId)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta)) return false;

node/NetworkConfig.hpp

@@ -39,6 +39,8 @@
 #include "Utils.hpp"
 #include "Trace.hpp"
 
+namespace ZeroTier {
+
 /**
  * Default maximum time delta for COMs, tags, and capabilities
  *
@@ -80,7 +82,10 @@
  */
 #define ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE 0x0000020000000000ULL
 
-namespace ZeroTier {
+/**
+ * Device that replicates multicasts
+ */
+#define ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR 0x0000040000000000ULL
 
 // Dictionary capacity needed for max size network config
 #define ZT_NETWORKCONFIG_DICT_CAPACITY (1024 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
@@ -88,13 +93,8 @@ namespace ZeroTier {
 // Dictionary capacity needed for max size network meta-data
 #define ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY 8192
 
-// Network config version
-#define ZT_NETWORKCONFIG_VERSION 7
-
 // Fields for meta-data sent with network config requests
 
-// Network config version
-#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION "v"
 // Protocol version (see Packet.hpp)
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_PROTOCOL_VERSION "pv"
 // Software vendor
@@ -166,29 +166,6 @@ namespace ZeroTier {
 // tags (binary blobs)
 #define ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP "COO"
 
-// Legacy fields -- these are obsoleted but are included when older clients query
-
-// boolean (now a flag)
-#define ZT_NETWORKCONFIG_DICT_KEY_ENABLE_BROADCAST_OLD "eb"
-// IP/bits[,IP/bits,...]
-// Note that IPs that end in all zeroes are routes with no assignment in them.
-#define ZT_NETWORKCONFIG_DICT_KEY_IPV4_STATIC_OLD "v4s"
-// IP/bits[,IP/bits,...]
-// Note that IPs that end in all zeroes are routes with no assignment in them.
-#define ZT_NETWORKCONFIG_DICT_KEY_IPV6_STATIC_OLD "v6s"
-// 0/1
-#define ZT_NETWORKCONFIG_DICT_KEY_PRIVATE_OLD "p"
-// integer(hex)[,integer(hex),...]
-#define ZT_NETWORKCONFIG_DICT_KEY_ALLOWED_ETHERNET_TYPES_OLD "et"
-// string-serialized CertificateOfMembership
-#define ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATE_OF_MEMBERSHIP_OLD "com"
-// node[,node,...]
-#define ZT_NETWORKCONFIG_DICT_KEY_ACTIVE_BRIDGES_OLD "ab"
-// node;IP/port[,node;IP/port]
-#define ZT_NETWORKCONFIG_DICT_KEY_RELAYS_OLD "rl"
-
-// End legacy fields
-
 /**
  * Network configuration received from network controller nodes
  *
@@ -197,7 +174,7 @@ namespace ZeroTier {
  */
 struct NetworkConfig
 {
-	inline NetworkConfig() :
+	ZT_ALWAYS_INLINE NetworkConfig() :
 		networkId(0),
 		timestamp(0),
 		credentialTimeMaxDelta(0),
@@ -240,17 +217,17 @@ struct NetworkConfig
 	/**
 	 * @return True if broadcast (ff:ff:ff:ff:ff:ff) address should work on this network
 	 */
-	inline bool enableBroadcast() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST) != 0); }
+	ZT_ALWAYS_INLINE bool enableBroadcast() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_BROADCAST) != 0); }
 
 	/**
 	 * @return True if IPv6 NDP emulation should be allowed for certain "magic" IPv6 address patterns
 	 */
-	inline bool ndpEmulation() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION) != 0); }
+	ZT_ALWAYS_INLINE bool ndpEmulation() const { return ((this->flags & ZT_NETWORKCONFIG_FLAG_ENABLE_IPV6_NDP_EMULATION) != 0); }
 
 	/**
 	 * @return True if frames should not be compressed
 	 */
-	inline bool disableCompression() const
+	ZT_ALWAYS_INLINE bool disableCompression() const
 	{
 #ifndef ZT_DISABLE_COMPRESSION
 		return ((this->flags & ZT_NETWORKCONFIG_FLAG_DISABLE_COMPRESSION) != 0);
@@ -266,50 +243,18 @@ struct NetworkConfig
 	/**
 	 * @return Network type is public (no access control)
 	 */
-	inline bool isPublic() const { return (this->type == ZT_NETWORK_TYPE_PUBLIC); }
+	ZT_ALWAYS_INLINE bool isPublic() const { return (this->type == ZT_NETWORK_TYPE_PUBLIC); }
 
 	/**
 	 * @return Network type is private (certificate access control)
 	 */
-	inline bool isPrivate() const { return (this->type == ZT_NETWORK_TYPE_PRIVATE); }
-
-	/**
-	 * @return ZeroTier addresses of devices on this network designated as active bridges
-	 */
-	inline std::vector<Address> activeBridges() const
-	{
-		std::vector<Address> r;
-		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0)
-				r.push_back(Address(specialists[i]));
-		}
-		return r;
-	}
-
-	inline unsigned int activeBridges(Address ab[ZT_MAX_NETWORK_SPECIALISTS]) const
-	{
-		unsigned int c = 0;
-		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0)
-				ab[c++] = specialists[i];
-		}
-		return c;
-	}
-
-	inline bool isActiveBridge(const Address &a) const
-	{
-		for(unsigned int i=0;i<specialistCount;++i) {
-			if (((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0)&&(a == specialists[i]))
-				return true;
-		}
-		return false;
-	}
+	ZT_ALWAYS_INLINE bool isPrivate() const { return (this->type == ZT_NETWORK_TYPE_PRIVATE); }
 
 	/**
 	 * @param fromPeer Peer attempting to bridge other Ethernet peers onto network
 	 * @return True if this network allows bridging
 	 */
-	inline bool permitsBridging(const Address &fromPeer) const
+	ZT_ALWAYS_INLINE bool permitsBridging(const Address &fromPeer) const
 	{
 		for(unsigned int i=0;i<specialistCount;++i) {
 			if ((fromPeer == specialists[i])&&((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0))
@@ -318,9 +263,9 @@ struct NetworkConfig
 		return false;
 	}
 
-	inline operator bool() const { return (networkId != 0); }
-	inline bool operator==(const NetworkConfig &nc) const { return (memcmp(this,&nc,sizeof(NetworkConfig)) == 0); }
-	inline bool operator!=(const NetworkConfig &nc) const { return (!(*this == nc)); }
+	ZT_ALWAYS_INLINE operator bool() const { return (networkId != 0); }
+	ZT_ALWAYS_INLINE bool operator==(const NetworkConfig &nc) const { return (memcmp(this,&nc,sizeof(NetworkConfig)) == 0); }
+	ZT_ALWAYS_INLINE bool operator!=(const NetworkConfig &nc) const { return (!(*this == nc)); }
 
 	/**
 	 * Add a specialist or mask flags if already present
@@ -332,7 +277,7 @@ struct NetworkConfig
 	 * @param f Flags (OR of specialist role/type flags)
 	 * @return True if successfully masked or added
 	 */
-	inline bool addSpecialist(const Address &a,const uint64_t f)
+	ZT_ALWAYS_INLINE bool addSpecialist(const Address &a,const uint64_t f)
 	{
 		const uint64_t aint = a.toInt();
 		for(unsigned int i=0;i<specialistCount;++i) {
@@ -348,7 +293,7 @@ struct NetworkConfig
 		return false;
 	}
 
-	const Capability *capability(const uint32_t id) const
+	ZT_ALWAYS_INLINE Capability *capability(const uint32_t id) const
 	{
 		for(unsigned int i=0;i<capabilityCount;++i) {
 			if (capabilities[i].id() == id)
@@ -357,7 +302,7 @@ struct NetworkConfig
 		return (Capability *)0;
 	}
 
-	const Tag *tag(const uint32_t id) const
+	ZT_ALWAYS_INLINE Tag *tag(const uint32_t id) const
 	{
 		for(unsigned int i=0;i<tagCount;++i) {
 			if (tags[i].id() == id)

node/OutboundMulticast.cpp

@@ -1,79 +0,0 @@
-/*
- * Copyright (c)2019 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2023-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#include "Constants.hpp"
-#include "RuntimeEnvironment.hpp"
-#include "OutboundMulticast.hpp"
-#include "Switch.hpp"
-#include "Network.hpp"
-#include "Node.hpp"
-#include "Peer.hpp"
-#include "Topology.hpp"
-
-namespace ZeroTier {
-
-void OutboundMulticast::init(
-	const RuntimeEnvironment *RR,
-	uint64_t timestamp,
-	uint64_t nwid,
-	bool disableCompression,
-	const MAC &src,
-	const MulticastGroup &dest,
-	unsigned int etherType,
-	const void *payload,
-	unsigned int len)
-{
-	uint8_t flags = 0;
-
-	_timestamp = timestamp;
-	_nwid = nwid;
-	if (src) {
-		_macSrc = src;
-		flags |= 0x04;
-	} else {
-		_macSrc.fromAddress(RR->identity.address(),nwid);
-	}
-	_macDest = dest.mac();
-	_frameLen = (len < ZT_MAX_MTU) ? len : ZT_MAX_MTU;
-	_etherType = etherType;
-
-	_packet.setSource(RR->identity.address());
-	_packet.setVerb(Packet::VERB_MULTICAST_FRAME);
-	_packet.append((uint64_t)nwid);
-	_packet.append(flags);
-	if (src) src.appendTo(_packet);
-	dest.mac().appendTo(_packet);
-	_packet.append((uint32_t)dest.adi());
-	_packet.append((uint16_t)etherType);
-	_packet.append(payload,_frameLen);
-	if (!disableCompression)
-		_packet.compress();
-
-	memcpy(_frameData,payload,_frameLen);
-}
-
-void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr)
-{
-	const SharedPtr<Network> nw(RR->node->network(_nwid));
-	uint8_t QoSBucket = 255; // Dummy value
-	if ((nw)&&(nw->filterOutgoingPacket(tPtr,true,RR->identity.address(),toAddr,_macSrc,_macDest,_frameData,_frameLen,_etherType,0,QoSBucket))) {
-		nw->pushCredentialsIfNeeded(tPtr,toAddr,RR->node->now());
-		_packet.newInitializationVector();
-		_packet.setDestination(toAddr);
-		RR->node->expectReplyTo(_packet.packetId());
-		_tmp = _packet;
-		RR->sw->send(tPtr,_tmp,true);
-	}
-}
-
-} // namespace ZeroTier

node/OutboundMulticast.hpp

@@ -1,154 +0,0 @@
-/*
- * Copyright (c)2019 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2023-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#ifndef ZT_OUTBOUNDMULTICAST_HPP
-#define ZT_OUTBOUNDMULTICAST_HPP
-
-#include <stdint.h>
-
-#include <vector>
-#include <algorithm>
-
-#include "Constants.hpp"
-#include "MAC.hpp"
-#include "MulticastGroup.hpp"
-#include "Address.hpp"
-#include "Packet.hpp"
-
-namespace ZeroTier {
-
-class CertificateOfMembership;
-class RuntimeEnvironment;
-
-/**
- * An outbound multicast packet
- *
- * This object isn't guarded by a mutex; caller must synchronize access.
- */
-class OutboundMulticast
-{
-public:
-	/**
-	 * Create an uninitialized outbound multicast
-	 *
-	 * It must be initialized with init().
-	 */
-	ZT_ALWAYS_INLINE OutboundMulticast() {}
-
-	/**
-	 * Initialize outbound multicast
-	 *
-	 * @param RR Runtime environment
-	 * @param timestamp Creation time
-	 * @param nwid Network ID
-	 * @param disableCompression Disable compression of frame payload
-	 * @param limit Multicast limit for desired number of packets to send
-	 * @param src Source MAC address of frame or NULL to imply compute from sender ZT address
-	 * @param dest Destination multicast group (MAC + ADI)
-	 * @param etherType 16-bit Ethernet type ID
-	 * @param payload Data
-	 * @param len Length of data
-	 * @throws std::out_of_range Data too large to fit in a MULTICAST_FRAME
-	 */
-	void init(
-		const RuntimeEnvironment *RR,
-		uint64_t timestamp,
-		uint64_t nwid,
-		bool disableCompression,
-		const MAC &src,
-		const MulticastGroup &dest,
-		unsigned int etherType,
-		const void *payload,
-		unsigned int len);
-
-	/**
-	 * @return Multicast creation time
-	 */
-	ZT_ALWAYS_INLINE uint64_t timestamp() const { return _timestamp; }
-
-	/**
-	 * @param now Current time
-	 * @return True if this multicast is expired (has exceeded transmit timeout)
-	 */
-	ZT_ALWAYS_INLINE bool expired(int64_t now) const { return ((now - _timestamp) >= ZT_MULTICAST_TRANSMIT_TIMEOUT); }
-
-	/**
-	 * @return True if this outbound multicast has been sent to enough peers
-	 */
-	ZT_ALWAYS_INLINE bool atLimit() const { return (_alreadySentTo.size() >= _limit); }
-
-	/**
-	 * Just send without checking log
-	 *
-	 * @param RR Runtime environment
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param toAddr Destination address
-	 */
-	void sendOnly(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr);
-
-	/**
-	 * Just send and log but do not check sent log
-	 *
-	 * @param RR Runtime environment
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param toAddr Destination address
-	 */
-	ZT_ALWAYS_INLINE void sendAndLog(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr)
-	{
-		_alreadySentTo.insert(std::upper_bound(_alreadySentTo.begin(),_alreadySentTo.end(),toAddr),toAddr); // sorted insert
-		sendOnly(RR,tPtr,toAddr);
-	}
-
-	/**
-	 * Log an address as having been used so we will not send there in the future
-	 *
-	 * @param toAddr Address to log as sent
-	 */
-	ZT_ALWAYS_INLINE void logAsSent(const Address &toAddr)
-	{
-		_alreadySentTo.insert(std::upper_bound(_alreadySentTo.begin(),_alreadySentTo.end(),toAddr),toAddr); // sorted insert
-	}
-
-	/**
-	 * Try to send this to a given peer if it hasn't been sent to them already
-	 *
-	 * @param RR Runtime environment
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param toAddr Destination address
-	 * @return True if address is new and packet was sent to switch, false if duplicate
-	 */
-	ZT_ALWAYS_INLINE bool sendIfNew(const RuntimeEnvironment *RR,void *tPtr,const Address &toAddr)
-	{
-		if (!std::binary_search(_alreadySentTo.begin(),_alreadySentTo.end(),toAddr)) {
-			sendAndLog(RR,tPtr,toAddr);
-			return true;
-		} else {
-			return false;
-		}
-	}
-
-private:
-	uint64_t _timestamp;
-	uint64_t _nwid;
-	MAC _macSrc;
-	MAC _macDest;
-	unsigned int _frameLen;
-	unsigned int _etherType;
-	Packet _packet,_tmp;
-	std::vector<Address> _alreadySentTo;
-	uint8_t _frameData[ZT_MAX_MTU];
-};
-
-} // namespace ZeroTier
-
-#endif

node/Packet.hpp

@@ -750,6 +750,8 @@ public:
 		 * multicasts propagated cooperatively, since unlike sender side
 		 * replication the message MAC alone cannot be used for this. This
 		 * imposes a non-trivial CPU cost on the sender and so it's optional.
+		 * Note that the bloom filter itself is not included in the signature
+		 * because it can be changed in transit.
 		 *
 		 * OK is not sent.
 		 *

node/Peer.hpp

@@ -481,7 +481,7 @@ public:
 	/**
 	 * @return Whether this peer is reachable via an aggregate link
 	 */
-	inline bool hasAggregateLink()
+	ZT_ALWAYS_INLINE bool hasAggregateLink() const
 	{
 		return _localMultipathSupported && _remoteMultipathSupported && _remotePeerMultipathEnabled;
 	}