work in progress

controller/EmbeddedNetworkController.cpp

@@ -1310,20 +1310,6 @@ void EmbeddedNetworkController::_request(
 	nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU);
 	nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL);
 
-	std::string rtt(OSUtils::jsonString(member["remoteTraceTarget"],""));
-	if (rtt.length() == 10) {
-		nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str()));
-		nc->remoteTraceLevel = (Trace::Level)OSUtils::jsonInt(member["remoteTraceLevel"],0ULL);
-	} else {
-		rtt = OSUtils::jsonString(network["remoteTraceTarget"],"");
-		if (rtt.length() == 10) {
-			nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str()));
-		} else {
-			nc->remoteTraceTarget.zero();
-		}
-		nc->remoteTraceLevel = (Trace::Level)OSUtils::jsonInt(network["remoteTraceLevel"],0ULL);
-	}
-
 	for(std::vector<Address>::const_iterator ab(ns.activeBridges.begin());ab!=ns.activeBridges.end();++ab)
 		nc->addSpecialist(*ab,ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE);
 

node/Address.hpp

@@ -105,7 +105,7 @@ public:
 	/**
 	 * @return Hash code for use with Hashtable
 	 */
-	ZT_ALWAYS_INLINE unsigned long hashCode() const { return (unsigned long)_a; }
+	ZT_ALWAYS_INLINE unsigned long hashCode() const { return reinterpret_cast<unsigned long>(_a); }
 
 	/**
 	 * @return Hexadecimal string
@@ -134,6 +134,10 @@ public:
 	 */
 	ZT_ALWAYS_INLINE uint8_t operator[](unsigned int i) const { return (uint8_t)(_a >> (32 - (i * 8))); }
 
+	ZT_ALWAYS_INLINE operator unsigned int() const { return reinterpret_cast<unsigned int>(_a); }
+	ZT_ALWAYS_INLINE operator unsigned long() const { return reinterpret_cast<unsigned long>(_a); }
+	ZT_ALWAYS_INLINE operator unsigned long long() const { return reinterpret_cast<unsigned long long>(_a); }
+
 	ZT_ALWAYS_INLINE void zero() { _a = 0; }
 
 	ZT_ALWAYS_INLINE bool operator==(const uint64_t &a) const { return (_a == (a & 0xffffffffffULL)); }

node/Buffer.hpp

@@ -237,7 +237,7 @@ public:
 	 * @param n Number of times to append
 	 * @throws std::out_of_range Attempt to append beyond capacity
 	 */
-	inline void append(unsigned char c,unsigned int n)
+	inline void append(uint8_t c,unsigned int n)
 	{
 		if (unlikely((_l + n) > C))
 			throw ZT_EXCEPTION_OUT_OF_BOUNDS;

node/CMakeLists.txt

@@ -24,7 +24,6 @@ set(core_headers
 	Locator.hpp
 	MAC.hpp
 	Membership.hpp
-	Multicaster.hpp
 	MulticastGroup.hpp
 	Mutex.hpp
 	Network.hpp
@@ -59,7 +58,6 @@ set(core_src
 	IncomingPacket.cpp
 	InetAddress.cpp
 	Membership.cpp
-	Multicaster.cpp
 	Network.cpp
 	NetworkConfig.cpp
 	Node.cpp

node/Constants.hpp

@@ -14,6 +14,10 @@
 #ifndef ZT_CONSTANTS_HPP
 #define ZT_CONSTANTS_HPP
 
+/****************************************************************************/
+/* Core includes and OS/platform setup stuff                                */
+/****************************************************************************/
+
 #include "../include/ZeroTierCore.h"
 
 #if __has_include("version.h")
@@ -174,6 +178,10 @@
 #define ZT_INVALID_SOCKET -1
 #endif
 
+/****************************************************************************/
+/* Internal ZeroTier constants                                              */
+/****************************************************************************/
+
 /**
  * Length of a ZeroTier address in bytes
  */
@@ -288,6 +296,11 @@
  */
 #define ZT_MULTICAST_GATHER_PERIOD ZT_MULTICAST_ANNOUNCE_PERIOD
 
+/**
+ * Period for multicast GATHER if there are no known recipients
+ */
+#define ZT_MULTICAST_GATHER_PERIOD_WHEN_NO_RECIPIENTS 2500
+
 /**
  * Timeout for outgoing multicasts
  *

node/Identity.hpp

@@ -115,26 +115,19 @@ public:
 	}
 
 	/**
-	 * Compute a 128-bit short hash of this identity's public key
-	 *
-	 * This is the first 128 bits of a SHA384 hash and is the hash used
-	 * in VERB_WILL_RELAY to report reachability.
-	 *
-	 * @param h 128-bit buffer to receive hash (must be 16 bytes in size)
+	 * @param h Buffer to receive SHA384 of public key(s)
 	 */
-	ZT_ALWAYS_INLINE void publicKeyHash128(void *const h) const
+	ZT_ALWAYS_INLINE bool hash(uint8_t h[48]) const
 	{
-		uint8_t tmp[48];
 		switch(_type) {
 			case C25519:
-				SHA384(tmp,_pub.c25519,ZT_C25519_PUBLIC_KEY_LEN);
-				break;
+				SHA384(h,_pub.c25519,ZT_C25519_PUBLIC_KEY_LEN);
+				return true;
 			case P384:
-				SHA384(tmp,&_pub,sizeof(_pub));
-				break;
+				SHA384(h,&_pub,sizeof(_pub));
+				return true;
 		}
-		for(int i=0;i<16;++i)
-			((uint8_t *)h)[i] = tmp[i];
+		return false;
 	}
 
 	/**

node/MulticastGroup.hpp

@@ -41,13 +41,8 @@ namespace ZeroTier {
 class MulticastGroup
 {
 public:
-	ZT_ALWAYS_INLINE MulticastGroup() :
-		_mac(),
-		_adi(0) {}
-
-	ZT_ALWAYS_INLINE MulticastGroup(const MAC &m,uint32_t a) :
-		_mac(m),
-		_adi(a) {}
+	ZT_ALWAYS_INLINE MulticastGroup() : _mac(),_adi(0) {}
+	ZT_ALWAYS_INLINE MulticastGroup(const MAC &m,uint32_t a) : _mac(m),_adi(a) {}
 
 	/**
 	 * Derive the multicast group used for address resolution (ARP/NDP) for an IP
@@ -74,10 +69,36 @@ public:
 		return MulticastGroup();
 	}
 
+	/**
+	 * @return Ethernet MAC portion of multicast group
+	 */
 	ZT_ALWAYS_INLINE const MAC &mac() const { return _mac; }
+
+	/**
+	 * @return Additional distinguishing information, which is normally zero except for IPv4 ARP where it's the IPv4 address
+	 */
 	ZT_ALWAYS_INLINE uint32_t adi() const { return _adi; }
 
-	ZT_ALWAYS_INLINE unsigned long hashCode() const { return (_mac.hashCode() ^ (unsigned long)_adi); }
+	/**
+	 * @return 32-bit hash ID of this multicast group
+	 */
+	ZT_ALWAYS_INLINE uint32_t id() const
+	{
+		uint64_t m = _mac.toInt();
+		uint32_t x1 = _adi;
+		uint32_t x2 = (uint32_t)(m >> 32);
+		uint32_t x3 = (uint32_t)m;
+		x1 = ((x1 >> 16) ^ x1) * 0x45d9f3b;
+		x2 = ((x2 >> 16) ^ x2) * 0x45d9f3b;
+		x3 = ((x3 >> 16) ^ x3) * 0x45d9f3b;
+		x1 = ((x1 >> 16) ^ x1) * 0x45d9f3b;
+		x2 = ((x2 >> 16) ^ x2) * 0x45d9f3b;
+		x3 = ((x3 >> 16) ^ x3) * 0x45d9f3b;
+		x1 = (x1 >> 16) ^ x1;
+		x2 = (x2 >> 16) ^ x2;
+		x3 = (x3 >> 16) ^ x3;
+		return (x1 ^ x2 ^ x3);
+	}
 
 	ZT_ALWAYS_INLINE bool operator==(const MulticastGroup &g) const { return ((_mac == g._mac)&&(_adi == g._adi)); }
 	ZT_ALWAYS_INLINE bool operator!=(const MulticastGroup &g) const { return ((_mac != g._mac)||(_adi != g._adi)); }
@@ -93,6 +114,8 @@ public:
 	ZT_ALWAYS_INLINE bool operator<=(const MulticastGroup &g) const { return !(g < *this); }
 	ZT_ALWAYS_INLINE bool operator>=(const MulticastGroup &g) const { return !(*this < g); }
 
+	ZT_ALWAYS_INLINE unsigned long hashCode() const { return (_mac.hashCode() ^ (unsigned long)_adi); }
+
 private:
 	MAC _mac;
 	uint32_t _adi;

node/Multicaster.cpp

@@ -1,326 +0,0 @@
-/*
- * Copyright (c)2019 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2023-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#include <algorithm>
-
-#include "Constants.hpp"
-#include "RuntimeEnvironment.hpp"
-#include "Multicaster.hpp"
-#include "Network.hpp"
-#include "Membership.hpp"
-#include "Topology.hpp"
-#include "Switch.hpp"
-
-namespace ZeroTier {
-
-Multicaster::Multicaster(const RuntimeEnvironment *renv) :
-	RR(renv),
-	_groups(32) {}
-
-Multicaster::~Multicaster() {}
-
-unsigned int Multicaster::send(
-	void *tPtr,
-	int64_t now,
-	const SharedPtr<Network> &network,
-	const MulticastGroup &mg,
-	const MAC &src,
-	unsigned int etherType,
-	const unsigned int existingBloomMultiplier,
-	const uint8_t existingBloom[ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS / 8],
-	const void *const data,
-	unsigned int len)
-{
-	static const unsigned int PRIMES[16] = { 3,5,7,11,13,17,19,23,29,31,37,41,43,47,53,59 }; // 2 is skipped as it's even
-
-	std::vector< std::pair<int64_t,Address> > recipients;
-
-	const NetworkConfig &config = network->config();
-	if (config.multicastLimit == 0) return 0; // multicast disabled
-
-	Address specialists[ZT_MAX_NETWORK_SPECIALISTS],multicastReplicators[ZT_MAX_NETWORK_SPECIALISTS];
-	unsigned int specialistCount = 0,multicastReplicatorCount = 0,bridgeCount = 0;
-	bool amMulticastReplicator = false;
-	for(unsigned int i=0;i<config.specialistCount;++i) {
-		if (RR->identity.address() == config.specialists[i]) {
-			amMulticastReplicator |= ((config.specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR) != 0);
-		} else {
-			specialists[specialistCount++] = config.specialists[i];
-			if ((config.specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ACTIVE_BRIDGE) != 0) {
-				recipients.push_back(std::pair<int64_t,Address>(0,config.specialists[i]));
-				++bridgeCount;
-			} if ((config.specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR) != 0) {
-				multicastReplicators[multicastReplicatorCount++] = config.specialists[i];
-			}
-		}
-	}
-	std::sort(&(specialists[0]),&(specialists[specialistCount])); // for binary search
-
-	int64_t lastGather = 0;
-	_K groupKey(network->id(),mg);
-	{
-		Mutex::Lock l(_groups_l);
-		const _G *const g = _groups.get(groupKey);
-		if (g) {
-			lastGather = g->lastGather;
-			recipients.reserve(recipients.size() + g->members.size());
-			Hashtable< Address,int64_t >::Iterator mi(const_cast<_G *>(g)->members);
-			Address *mik = nullptr;
-			int64_t *miv = nullptr;
-			while (mi.next(mik,miv)) {
-				if (!std::binary_search(&(specialists[0]),&(specialists[specialistCount]),*mik))
-					recipients.push_back(std::pair<int64_t,Address>(*miv,*mik));
-			}
-		}
-	}
-
-	// Sort recipients, maintaining bridges first in list
-	std::sort(recipients.begin() + bridgeCount,recipients.end(),std::greater< std::pair<int64_t,Address> >());
-
-	// Gather new recipients periodically, being more aggressive if we have none.
-	if ((now - lastGather) > (recipients.empty() ? 5000 : ZT_MULTICAST_GATHER_PERIOD)) {
-		{
-			Mutex::Lock l(_groups_l);
-			_groups[groupKey].lastGather = now;
-		}
-
-		Packet outp(network->controller(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
-		outp.append(network->id());
-		outp.append((uint8_t)0);
-		mg.mac().appendTo(outp);
-		outp.append(mg.adi());
-		outp.append((uint32_t)0xffffffff);
-		RR->sw->send(tPtr,outp,true);
-
-		for(unsigned int i=0;i<specialistCount;++i) {
-			outp.newInitializationVector();
-			outp.setDestination(specialists[i]);
-			RR->sw->send(tPtr,outp,true);
-		}
-
-		// LEGACY: roots may know about older versions' multicast subscriptions but
-		// the root's role here is being phased out.
-		SharedPtr<Peer> root(RR->topology->root(now));
-		if (root) {
-			outp.newInitializationVector();
-			outp.setDestination(root->address());
-			outp.armor(root->key(),true);
-			root->sendDirect(tPtr,outp.data(),outp.size(),now,true);
-		}
-	}
-
-	if (recipients.empty())
-		return 0;
-
-	unsigned int sentCount = 0;
-
-	uint64_t bloomFilter[ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS / 64];
-	unsigned int bloomMultiplier;
-	if (existingBloom) {
-		memcpy(bloomFilter,existingBloom,sizeof(bloomFilter));
-		bloomMultiplier = existingBloomMultiplier;
-	} else {
-		memset(bloomFilter,0,sizeof(bloomFilter));
-		bloomMultiplier = 1;
-
-		// Iteratively search for a bloom multiplier that results in no collisions
-		// among known recipients. Usually the first iteration is good unless
-		// the recipient set is quite large.
-		if (recipients.size() > 1) {
-			unsigned long bestMultColl = 0xffffffff;
-			for(int k=0;k<16;++k) { // 16 == arbitrary limit on iterations for this search, also must be <= size of PRIMES
-				const unsigned int mult = PRIMES[k];
-				unsigned long coll = 0;
-				for(std::vector< std::pair<int64_t,Address> >::const_iterator r(recipients.begin());r!=recipients.end();++r) {
-					const unsigned int bfi = mult * (unsigned int)r->second.toInt();
-					const unsigned int byte = (bfi >> 3) % sizeof(bloomFilter);
-					const uint8_t bit = 1 << (bfi & 7);
-					coll += ((((uint8_t *)bloomFilter)[byte] & bit) != 0);
-					((uint8_t *)bloomFilter)[byte] |= bit;
-				}
-				memset(bloomFilter,0,sizeof(bloomFilter));
-
-				if (coll <= bestMultColl) {
-					bloomMultiplier = mult;
-					if (coll == 0) // perfect score, no need to continue searching
-						break;
-					bestMultColl = coll;
-				}
-			}
-		}
-	}
-
-	// See if there is a multicast replicator, trying to pick the fastest/best one.
-	Address bestReplicator;
-	if (multicastReplicatorCount > 0) {
-		unsigned int bestReplicatorLatency = 0xffff;
-		for(unsigned int i=0;i<multicastReplicatorCount;++i) {
-			const unsigned int bfi = bloomMultiplier * (unsigned int)multicastReplicators[i].toInt();
-			if ((((uint8_t *)bloomFilter)[(bfi >> 3) % sizeof(bloomFilter)] & (1 << (bfi & 7))) == 0) {
-				SharedPtr<Peer> peer(RR->topology->get(multicastReplicators[i]));
-				if (peer) {
-					const unsigned int lat = peer->latency(now);
-					if (lat <= bestReplicatorLatency) {
-						bestReplicator = peer->address();
-						bestReplicatorLatency = lat;
-					}
-				} else if (!bestReplicator) {
-					bestReplicator = multicastReplicators[i];
-				}
-			}
-		}
-	}
-
-	// If this is a multicast replicator, aggressively replicate. Multicast
-	// replicators are not subject to send count limits.
-	if (amMulticastReplicator) {
-		std::vector< std::pair< int,Address > > byLatency;
-		for(std::vector< std::pair<int64_t,Address> >::const_iterator r(recipients.begin());r!=recipients.end();++r) {
-			const unsigned int bfi = bloomMultiplier * (unsigned int)r->second.toInt();
-			if ((((uint8_t *)bloomFilter)[(bfi >> 3) % sizeof(bloomFilter)] & (1 << (bfi & 7))) == 0) {
-				SharedPtr<Peer> peer(RR->topology->get(r->second));
-				byLatency.push_back(std::pair< int,Address >((peer) ? (int)peer->latency(now) : 0xffff,r->second));
-			}
-		}
-		std::sort(byLatency.begin(),byLatency.end());
-
-		unsigned long cnt = byLatency.size();
-		if (bestReplicator)
-			cnt /= 2; // send to only the best half of the latency-sorted population if there are more replicators
-		for(unsigned long i=0;i<cnt;++i) {
-			const unsigned int bfi = bloomMultiplier * (unsigned int)byLatency[i].second.toInt();
-			((uint8_t *)bloomFilter)[(bfi >> 3) % sizeof(bloomFilter)] |= 1 << (bfi & 7);
-
-			Packet outp(byLatency[i].second,RR->identity.address(),Packet::VERB_MULTICAST_FRAME);
-			outp.append(network->id());
-			outp.append((uint8_t)0x04);
-			src.appendTo(outp);
-			mg.mac().appendTo(outp);
-			outp.append(mg.adi());
-			outp.append((uint16_t)etherType);
-			outp.append(data,len);
-			outp.compress();
-			RR->sw->send(tPtr,outp,true);
-
-			++sentCount;
-		}
-	}
-
-	// Forward to the next multicast replicator, if any.
-	if (bestReplicator) {
-		const unsigned int bfi = bloomMultiplier * (unsigned int)bestReplicator.toInt();
-		((uint8_t *)bloomFilter)[(bfi >> 3) % sizeof(bloomFilter)] |= 1 << (bfi & 7);
-
-		Packet outp(bestReplicator,RR->identity.address(),Packet::VERB_MULTICAST_FRAME);
-		outp.append((uint8_t)(0x04 | 0x08));
-		RR->identity.address().appendTo(outp);
-		outp.append((uint16_t)bloomMultiplier);
-		outp.append((uint16_t)sizeof(bloomFilter));
-		outp.append(((uint8_t *)bloomFilter),sizeof(bloomFilter));
-		src.appendTo(outp);
-		mg.mac().appendTo(outp);
-		outp.append(mg.adi());
-		outp.append((uint16_t)etherType);
-		outp.append(data,len);
-		outp.compress();
-		RR->sw->send(tPtr,outp,true);
-
-		++sentCount;
-	}
-
-	// If this is a multicast replicator, we've already replicated.
-	if (amMulticastReplicator)
-		return (unsigned int)recipients.size();
-
-	// Find the two best next hops (that have never seen this multicast)
-	// that are newer version nodes.
-	SharedPtr<Peer> nextHops[2];
-	unsigned int nextHopsBestLatency[2] = { 0xffff,0xffff };
-	for(std::vector< std::pair<int64_t,Address> >::iterator r(recipients.begin());r!=recipients.end();++r) {
-		if (r->first >= 0) {
-			const unsigned int bfi = bloomMultiplier * (unsigned int)r->second.toInt();
-			if ((((uint8_t *)bloomFilter)[(bfi >> 3) % sizeof(bloomFilter)] & (1 << (bfi & 7))) == 0) {
-				const SharedPtr<Peer> peer(RR->topology->get(r->second));
-				if ((peer)&&(peer->remoteVersionProtocol() >= 11)) {
-					r->first = -1; // use this field now to flag as non-legacy
-					const unsigned int lat = peer->latency(now);
-					for(unsigned int nh=0;nh<2;++nh) {
-						if (lat <= nextHopsBestLatency[nh]) {
-							nextHopsBestLatency[nh] = lat;
-							nextHops[nh] = peer;
-							break;
-						}
-					}
-				}
-			}
-		}
-	}
-
-	// Set bits for next hops in bloom filter
-	for(unsigned int nh=0;nh<2;++nh) {
-		if (nextHops[nh]) {
-			const unsigned int bfi = bloomMultiplier * (unsigned int)nextHops[nh]->address().toInt();
-			((uint8_t *)bloomFilter)[(bfi >> 3) % sizeof(bloomFilter)] |= 1 << (bfi & 7);
-			++sentCount;
-		}
-	}
-
-	// Send to legacy peers and flag these in bloom filter
-	const unsigned int limit = config.multicastLimit + bridgeCount;
-	for(std::vector< std::pair<int64_t,Address> >::const_iterator r(recipients.begin());(r!=recipients.end())&&(sentCount<limit);++r) {
-		if (r->first >= 0) {
-			const unsigned int bfi = bloomMultiplier * (unsigned int)r->second.toInt();
-			((uint8_t *)bloomFilter)[(bfi >> 3) % sizeof(bloomFilter)] |= 1 << (bfi & 7);
-
-			Packet outp(r->second,RR->identity.address(),Packet::VERB_MULTICAST_FRAME);
-			outp.append(network->id());
-			outp.append((uint8_t)0x04);
-			src.appendTo(outp);
-			mg.mac().appendTo(outp);
-			outp.append(mg.adi());
-			outp.append((uint16_t)etherType);
-			outp.append(data,len);
-			outp.compress();
-			RR->sw->send(tPtr,outp,true);
-
-			++sentCount;
-		}
-	}
-
-	// Send to next hops for P2P propagation
-	for(unsigned int nh=0;nh<2;++nh) {
-		if (nextHops[nh]) {
-			Packet outp(nextHops[nh]->address(),RR->identity.address(),Packet::VERB_MULTICAST_FRAME);
-			outp.append((uint8_t)(0x04 | 0x08));
-			RR->identity.address().appendTo(outp);
-			outp.append((uint16_t)bloomMultiplier);
-			outp.append((uint16_t)sizeof(bloomFilter));
-			outp.append(((uint8_t *)bloomFilter),sizeof(bloomFilter));
-			src.appendTo(outp);
-			mg.mac().appendTo(outp);
-			outp.append(mg.adi());
-			outp.append((uint16_t)etherType);
-			outp.append(data,len);
-			outp.compress();
-			RR->sw->send(tPtr,outp,true);
-		}
-	}
-
-	return (unsigned int)recipients.size();
-}
-
-void Multicaster::clean(int64_t now)
-{
-}
-
-} // namespace ZeroTier

node/Multicaster.hpp

@@ -1,236 +0,0 @@
-/*
- * Copyright (c)2019 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2023-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#ifndef ZT_MULTICASTER_HPP
-#define ZT_MULTICASTER_HPP
-
-#include <stdint.h>
-#include <string.h>
-
-#include <map>
-#include <vector>
-
-#include "Constants.hpp"
-#include "Hashtable.hpp"
-#include "Address.hpp"
-#include "MAC.hpp"
-#include "MulticastGroup.hpp"
-#include "Utils.hpp"
-#include "Mutex.hpp"
-#include "SharedPtr.hpp"
-#include "Packet.hpp"
-
-// Size in bits -- do not change as this is about as large as we can support
-// This leaves room for up to 10000 MTU data (max supported MTU) and header
-// information in a maximum supported size packet. Note that data compression
-// will practically reduce this size in transit for sparse or saturated fields.
-#define ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS 50048
-
-namespace ZeroTier {
-
-class RuntimeEnvironment;
-class CertificateOfMembership;
-class Packet;
-class Network;
-
-/**
- * Multicast database and outbound multicast logic
- */
-class Multicaster
-{
-private:
-	// Composite key of network ID and multicast group
-	struct _K
-	{
-		uint64_t nwid;
-		MulticastGroup mg;
-
-		ZT_ALWAYS_INLINE _K() : nwid(0),mg() {}
-		ZT_ALWAYS_INLINE _K(const uint64_t n,const MulticastGroup &g) : nwid(n),mg(g) {}
-		ZT_ALWAYS_INLINE bool operator==(const _K &k) const { return ((nwid == k.nwid)&&(mg == k.mg)); }
-		ZT_ALWAYS_INLINE bool operator!=(const _K &k) const { return ((nwid != k.nwid)||(mg != k.mg)); }
-		ZT_ALWAYS_INLINE unsigned long hashCode() const { return (mg.hashCode() ^ (unsigned long)(nwid ^ (nwid >> 32))); }
-	};
-
-	// Multicast group info
-	struct _G
-	{
-		ZT_ALWAYS_INLINE _G() : lastGather(0),members(16) {}
-		int64_t lastGather;
-		Hashtable< Address,int64_t > members;
-	};
-
-	// Outbound multicast
-	struct _OM
-	{
-		uint64_t nwid;
-		MAC src;
-		MulticastGroup mg;
-		unsigned int etherType;
-		unsigned int dataSize;
-		unsigned int count;
-		unsigned int limit;
-		unsigned int bloomFilterMultiplier;
-		uint64_t bloomFilter[ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS / 64];
-		uint8_t data[ZT_MAX_MTU];
-		Mutex lock;
-	};
-
-public:
-	Multicaster(const RuntimeEnvironment *renv);
-	~Multicaster();
-
-	/**
-	 * Add or update a member in a multicast group
-	 *
-	 * @param now Current time
-	 * @param nwid Network ID
-	 * @param mg Multicast group
-	 * @param member New member address
-	 */
-	ZT_ALWAYS_INLINE void add(const int64_t now,const uint64_t nwid,const MulticastGroup &mg,const Address &member)
-	{
-		Mutex::Lock l(_groups_l);
-		_groups[_K(nwid,mg)].members.set(member,now);
-	}
-
-	/**
-	 * Add multiple addresses from a binary array of 5-byte address fields
-	 *
-	 * It's up to the caller to check bounds on the array before calling this.
-	 *
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param now Current time
-	 * @param nwid Network ID
-	 * @param mg Multicast group
-	 * @param addresses Raw binary addresses in big-endian format, as a series of 5-byte fields
-	 * @param count Number of addresses
-	 * @param totalKnown Total number of known addresses as reported by peer
-	 */
-	ZT_ALWAYS_INLINE void addMultiple(const int64_t now,const uint64_t nwid,const MulticastGroup &mg,const void *addresses,unsigned int count,const unsigned int totalKnown)
-	{
-		Mutex::Lock l(_groups_l);
-		const uint8_t *a = (const uint8_t *)addresses;
-		_G &g = _groups[_K(nwid,mg)];
-		while (count--) {
-			g.members.set(Address(a,ZT_ADDRESS_LENGTH),now);
-			a += ZT_ADDRESS_LENGTH;
-		}
-	}
-
-	/**
-	 * Remove a multicast group member (if present)
-	 *
-	 * @param nwid Network ID
-	 * @param mg Multicast group
-	 * @param member Member to unsubscribe
-	 */
-	ZT_ALWAYS_INLINE void remove(const uint64_t nwid,const MulticastGroup &mg,const Address &member)
-	{
-		Mutex::Lock l(_groups_l);
-		const _K gk(nwid,mg);
-		_G *const g = _groups.get(gk);
-		if (g) {
-			g->members.erase(member);
-			if (g->members.empty())
-				_groups.erase(gk);
-		}
-	}
-
-	/**
-	 * Iterate over members of a multicast group until function returns false
-	 *
-	 * Iteration order is in inverse order of most recent receipt of a LIKE
-	 * for a given membership.
-	 *
-	 * @param nwid Network ID
-	 * @param mg Multicast group
-	 * @param func f(Address)
-	 * @return Total number of known members (regardless of when function aborted)
-	 */
-	template<typename F>
-	ZT_ALWAYS_INLINE unsigned long eachMember(const uint64_t nwid,const MulticastGroup &mg,F func) const
-	{
-		std::vector< std::pair<int64_t,Address> > sortedByTime;
-		{
-			Mutex::Lock l(_groups_l);
-			const _K gk(nwid,mg);
-			const _G *const g = _groups.get(gk);
-			if (g) {
-				sortedByTime.reserve(g->members.size());
-				{
-					Hashtable< Address,int64_t >::Iterator mi(const_cast<_G *>(g)->members);
-					Address *mik = nullptr;
-					int64_t *miv = nullptr;
-					while (mi.next(mik,miv))
-					sortedByTime.push_back(std::pair<int64_t,Address>(*miv,*mik));
-				}
-			}
-		}
-		std::sort(sortedByTime.begin(),sortedByTime.end());
-		for(std::vector< std::pair<int64_t,Address> >::const_reverse_iterator i(sortedByTime.rbegin());i!=sortedByTime.rend();++i) {
-			if (!func(i->second))
-				break;
-		}
-		return sortedByTime.size();
-	}
-
-	/**
-	 * Send a multicast
-	 *
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param now Current time
-	 * @param network Network
-	 * @param mg Multicast group
-	 * @param src Source Ethernet MAC address or NULL to skip in packet and compute from ZT address (non-bridged mode)
-	 * @param etherType Ethernet frame type
-	 * @param existingBloomMultiplier Existing bloom filter multiplier or 0 if none
-	 * @param existingBloom Existing bloom filter or NULL if none
-	 * @param data Packet data
-	 * @param len Length of packet data
-	 * @return Number of known recipients for multicast (including bridges and replicators)
-	 */
-	unsigned int send(
-		void *tPtr,
-		int64_t now,
-		const SharedPtr<Network> &network,
-		const MulticastGroup &mg,
-		const MAC &src,
-		unsigned int etherType,
-		const unsigned int existingBloomMultiplier,
-		const uint8_t existingBloom[ZT_MULTICAST_BLOOM_FILTER_SIZE_BITS / 8],
-		const void *const data,
-		unsigned int len);
-
-	/**
-	 * Clean up database
-	 *
-	 * @param RR Runtime environment
-	 * @param now Current time
-	 */
-	void clean(int64_t now);
-
-private:
-	const RuntimeEnvironment *const RR;
-
-	_OM _txQueue[ZT_TX_QUEUE_SIZE];
-	unsigned int _txQueuePtr;
-	Mutex _txQueue_l;
-
-	Hashtable< _K,_G > _groups;
-	Mutex _groups_l;
-};
-
-} // namespace ZeroTier
-
-#endif

node/Network.hpp

@@ -33,7 +33,6 @@
 #include "MulticastGroup.hpp"
 #include "MAC.hpp"
 #include "Dictionary.hpp"
-#include "Multicaster.hpp"
 #include "Membership.hpp"
 #include "NetworkConfig.hpp"
 #include "CertificateOfMembership.hpp"

node/NetworkConfig.cpp

@@ -34,9 +34,8 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo.toString(tmp2))) return false;
-	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET,this->remoteTraceTarget.toString(tmp2))) return false;
-	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_LEVEL,(uint64_t)this->remoteTraceLevel)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false;
+	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TOKEN,this->token)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false;
 	if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name)) return false;
@@ -125,8 +124,6 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 		this->issuedTo = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,0);
 		if (!this->issuedTo)
 			return false;
-		this->remoteTraceTarget = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET);
-		this->remoteTraceLevel = (Trace::Level)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_LEVEL);
 		this->multicastLimit = (unsigned int)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,0);
 		d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME,this->name,sizeof(this->name));
 
@@ -141,6 +138,7 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 		} else {
 			// Otherwise we can use the new fields
 			this->flags = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,0);
+			this->token = d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TOKEN,0);
 			this->type = (ZT_VirtualNetworkType)d.getUI(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)ZT_NETWORK_TYPE_PRIVATE);
 
 			if (d.get(ZT_NETWORKCONFIG_DICT_KEY_COM,*tmp))

node/NetworkConfig.hpp

@@ -87,6 +87,11 @@ namespace ZeroTier {
  */
 #define ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR 0x0000040000000000ULL
 
+/**
+ * Device that can probe and receive remote trace info about this network
+ */
+#define ZT_NETWORKCONFIG_SPECIALIST_TYPE_DIAGNOSTICIAN 0x0000080000000000ULL
+
 // Dictionary capacity needed for max size network config
 #define ZT_NETWORKCONFIG_DICT_CAPACITY (1024 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
 
@@ -133,14 +138,12 @@ namespace ZeroTier {
 #define ZT_NETWORKCONFIG_DICT_KEY_REVISION "r"
 // address of member
 #define ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO "id"
-// remote trace target
-#define ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET "tt"
-// remote trace level
-#define ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_LEVEL "tl"
 // flags(hex)
 #define ZT_NETWORKCONFIG_DICT_KEY_FLAGS "f"
 // integer(hex)
 #define ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT "ml"
+// integer(hex)
+#define ZT_NETWORKCONFIG_DICT_KEY_TOKEN "k"
 // network type (hex)
 #define ZT_NETWORKCONFIG_DICT_KEY_TYPE "t"
 // text
@@ -180,9 +183,7 @@ struct NetworkConfig
 		credentialTimeMaxDelta(0),
 		revision(0),
 		issuedTo(),
-		remoteTraceTarget(),
 		flags(0),
-		remoteTraceLevel(Trace::LEVEL_NORMAL),
 		mtu(0),
 		multicastLimit(0),
 		specialistCount(0),
@@ -336,20 +337,15 @@ struct NetworkConfig
 	 */
 	Address issuedTo;
 
-	/**
-	 * If non-NULL, remote traces related to this network are sent here
-	 */
-	Address remoteTraceTarget;
-
 	/**
 	 * Flags (64-bit)
 	 */
 	uint64_t flags;
 
 	/**
-	 * Remote trace level
+	 * Token (64-bit token known only to network members)
 	 */
-	Trace::Level remoteTraceLevel;
+	uint64_t token;
 
 	/**
 	 * Network MTU

node/Node.cpp

@@ -23,7 +23,6 @@
 #include "RuntimeEnvironment.hpp"
 #include "NetworkController.hpp"
 #include "Switch.hpp"
-#include "Multicaster.hpp"
 #include "Topology.hpp"
 #include "Buffer.hpp"
 #include "Packet.hpp"
@@ -93,11 +92,10 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 	try {
 		const unsigned long ts = sizeof(Trace) + (((sizeof(Trace) & 0xf) != 0) ? (16 - (sizeof(Trace) & 0xf)) : 0);
 		const unsigned long sws = sizeof(Switch) + (((sizeof(Switch) & 0xf) != 0) ? (16 - (sizeof(Switch) & 0xf)) : 0);
-		const unsigned long mcs = sizeof(Multicaster) + (((sizeof(Multicaster) & 0xf) != 0) ? (16 - (sizeof(Multicaster) & 0xf)) : 0);
 		const unsigned long topologys = sizeof(Topology) + (((sizeof(Topology) & 0xf) != 0) ? (16 - (sizeof(Topology) & 0xf)) : 0);
 		const unsigned long sas = sizeof(SelfAwareness) + (((sizeof(SelfAwareness) & 0xf) != 0) ? (16 - (sizeof(SelfAwareness) & 0xf)) : 0);
 
-		m = reinterpret_cast<char *>(::malloc(16 + ts + sws + mcs + topologys + sas));
+		m = reinterpret_cast<char *>(::malloc(16 + ts + sws + topologys + sas));
 		if (!m)
 			throw std::bad_alloc();
 		RR->rtmem = m;
@@ -107,15 +105,12 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		m += ts;
 		RR->sw = new (m) Switch(RR);
 		m += sws;
-		RR->mc = new (m) Multicaster(RR);
-		m += mcs;
 		RR->topology = new (m) Topology(RR,RR->identity);
 		m += topologys;
 		RR->sa = new (m) SelfAwareness(RR);
 	} catch ( ... ) {
 		if (RR->sa) RR->sa->~SelfAwareness();
 		if (RR->topology) RR->topology->~Topology();
-		if (RR->mc) RR->mc->~Multicaster();
 		if (RR->sw) RR->sw->~Switch();
 		if (RR->t) RR->t->~Trace();
 		::free(m);
@@ -133,7 +128,6 @@ Node::~Node()
 	}
 	if (RR->sa) RR->sa->~SelfAwareness();
 	if (RR->topology) RR->topology->~Topology();
-	if (RR->mc) RR->mc->~Multicaster();
 	if (RR->sw) RR->sw->~Switch();
 	if (RR->t) RR->t->~Trace();
 	::free(RR->rtmem);
@@ -309,7 +303,6 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 				(*network)->doPeriodicTasks(tptr,now);
 			}
 		}
-		RR->t->updateMemoizedSettings();
 	}
 
 	if ((now - _lastHousekeepingRun) >= ZT_HOUSEKEEPING_PERIOD) {
@@ -333,7 +326,6 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 
 			RR->topology->doPeriodicTasks(now);
 			RR->sa->clean(now);
-			RR->mc->clean(now);
 		} catch ( ... ) {
 			return ZT_RESULT_FATAL_ERROR_INTERNAL;
 		}

node/Packet.hpp

@@ -545,13 +545,15 @@ public:
 		 *   0x7 - (reserved for future use)
 		 *
 		 * An extended frame carries full MAC addressing, making it a
-		 * superset of VERB_FRAME. It is used for bridged traffic,
-		 * redirected or observed traffic via rules, and can in theory
-		 * be used for multicast though MULTICAST_FRAME exists for that
-		 * purpose and has additional options and capabilities.
+		 * superset of VERB_FRAME. If 0x20 is set then p2p or hub and
+		 * spoke multicast propagation is requested.
 		 *
 		 * OK payload (if ACK flag is set):
 		 *   <[8] 64-bit network ID>
+		 *   <[1] flags>
+		 *   <[6] destination MAC or all zero for destination node>
+		 *   <[6] source MAC or all zero for node of origin>
+		 *   <[2] 16-bit ethertype>
 		 */
 		VERB_EXT_FRAME = 0x07,
 
@@ -566,7 +568,7 @@ public:
 		VERB_ECHO = 0x08,
 
 		/**
-		 * Announce interest in multicast group(s):
+		 * Announce interest in multicast group(s) (DEPRECATED):
 		 *   <[8] 64-bit network ID>
 		 *   <[6] multicast Ethernet address>
 		 *   <[4] multicast additional distinguishing information (ADI)>
@@ -680,108 +682,19 @@ public:
 		 */
 		VERB_NETWORK_CONFIG = 0x0c,
 
-		/**
-		 * Request endpoints for multicast distribution:
-		 *   <[8] 64-bit network ID>
-		 *   <[1] flags (unused, must be 0)>
-		 *   <[6] MAC address of multicast group being queried>
-		 *   <[4] 32-bit ADI for multicast group being queried>
-		 *   <[4] 32-bit requested max number of multicast peers>
-		 *
-		 * More than one OK response can occur if the response is broken up across
-		 * multiple packets or if querying a clustered node.
-		 *
-		 * OK response payload:
-		 *   <[8] 64-bit network ID>
-		 *   <[6] MAC address of multicast group being queried>
-		 *   <[4] 32-bit ADI for multicast group being queried>
-		 *   [begin gather results -- these same fields can be in OK(MULTICAST_FRAME)]
-		 *   <[4] 32-bit total number of known members in this multicast group>
-		 *   <[2] 16-bit number of members enumerated in this packet>
-		 *   <[...] series of 5-byte ZeroTier addresses of enumerated members>
-		 *
-		 * ERROR is not generated; queries that return no response are dropped.
-		 */
-		VERB_MULTICAST_GATHER = 0x0d,
-
-		/**
-		 * Multicast frame:
-		 *   <[8] 64-bit network ID>
-		 *   <[1] flags>
-		 *  [<[...] network certificate of membership (DEPRECATED)>]
-		 *  [<[4] 32-bit implicit gather limit (DEPRECATED)>]
-		 *  [<[5] ZeroTier address of originating sender (including w/0x08)>]
-		 *  [<[2] 16-bit bloom filter multiplier>]
-		 *  [<[2] 16-bit length of propagation bloom filter in bytes]
-		 *  [<[...] propagation bloom filter>]
-		 *  [<[6] source MAC>]
-		 *   <[6] destination MAC (multicast address)>
-		 *   <[4] 32-bit multicast ADI (multicast address extension)>
-		 *   <[2] 16-bit ethertype>
-		 *   <[...] ethernet payload>
-		 *  [<[2] 16-bit length of signature>]
-		 *  [<[...] signature (algorithm depends on sender identity)>]
-		 *
-		 * Flags:
-		 *   0x01 - Network certificate of membership attached (DEPRECATED)
-		 *   0x02 - Implicit gather limit field is present (DEPRECATED)
-		 *   0x04 - Source MAC is specified -- otherwise it's computed from sender
-		 *   0x08 - Propagation bloom filter is included
-		 *   0x10 - Signature by sending identity is included
-		 *
-		 * Version 1.x only supports sender-side replication. Version 2.x also
-		 * supports peer to peer and hub and spoke models. For that there is
-		 * a new field: a bloom filter that tracks recipients by ZeroTier address.
-		 *
-		 * Bits in the bloom filter are set by multiplying the address by the
-		 * indicated multiplier and then taking that modulo the number of bits
-		 * in the filter. Both the length of the filter and this multiplier are
-		 * variable and can be selected based on the sender's knowledge of
-		 * the total recipient set to minimize the chance of collision, as a
-		 * collision would result in a multicast not reaching one particular
-		 * recipient. The algorithm for selecting these is not defined by the
-		 * protocol.
-		 *
-		 * The ZeroTier address of the originating sender is also included
-		 * before the bloom filter if flag bit 0x08 is set.
-		 *
-		 * Version 2.x also supports an optional signature of the packet's
-		 * payload by the sending ZeroTier node. This can be used to validate
-		 * multicasts propagated cooperatively, since unlike sender side
-		 * replication the message MAC alone cannot be used for this. This
-		 * imposes a non-trivial CPU cost on the sender and so it's optional.
-		 * Note that the bloom filter itself is not included in the signature
-		 * because it can be changed in transit.
-		 *
-		 * OK is not sent.
-		 *
-		 * ERROR_MULTICAST_STFU is generated if a recipient no longer wishes to
-		 * receive these multicasts. It's essentially a source quench. Its
-		 * payload is:
-		 *
-		 * ERROR response payload:
-		 *   <[8] 64-bit network ID>
-		 *   <[6] multicast group MAC>
-		 *   <[4] 32-bit multicast group ADI>
-		 */
-		VERB_MULTICAST_FRAME = 0x0e,
-
 		/**
 		 * Push of potential endpoints for direct communication:
 		 *   <[2] 16-bit number of paths>
 		 *   <[...] paths>
 		 *
 		 * Path record format:
-		 *   <[1] 8-bit path flags>
+		 *   <[1] 8-bit path flags (always 0, currently unused)>
 		 *   <[2] length of extended path characteristics or 0 for none>
 		 *   <[...] extended path characteristics>
 		 *   <[1] address type>
 		 *   <[1] address length in bytes>
 		 *   <[...] address>
 		 *
-		 * Path record flags:
-		 *   0x01 - Forget this path if currently known (not implemented yet)
-		 *
 		 * The receiver may, upon receiving a push, attempt to establish a
 		 * direct link to one or more of the indicated addresses. It is the
 		 * responsibility of the sender to limit which peers it pushes direct
@@ -911,37 +824,40 @@ public:
   	VERB_WILL_RELAY = 0x17,
 
 		/**
-		 * A push of one or more ephemeral key pairs:
-		 *   <[1] 8-bit length of random padding>
-		 *   <[...] random padding>
-		 *   <[1] 8-bit number of keys in message>
-		 *   [... begin keys ...]
-		 *   <[1] 8-bit key type>
-		 *   <[...] public key (length determined by type)>
-		 *  [<[...] additional keys ...>]
-		 *   [... end keys ...]
-		 *
-		 * This verb is used to push ephemeral keys. A node replies to each
-		 * ephemeral key push with an OK message containing its own current
-		 * ephemeral keys that it wants to use for p2p communication.
-		 *
-		 * These are ephemeral public keys. Currently keys of type C25519
-		 * and P-384 are supported and both will be pushed.
-		 *
-		 * If more than one key is pushed, key agreement is performed using
-		 * all keys for which both peers pushed the same key type. The raw
-		 * results of these keys are then hashed together in order of key
-		 * type ID with SHA384 to yield a session key. If the desired session
-		 * key is shorter than 384 bits the first N bits are used.
-		 *
-		 * The random padding component can be used to ranomize the length
-		 * of these packets so adversaries can't easily selectively block
-		 * ephemeral key exchange by exploiting a fixed packet length.
-		 *
-		 * OK response payload:
-		 *   <[...] responder's keys, same format as verb payload>
+		 * Multicast frame (since 2.x, 0x0e is deprecated multicast frame):
+		 *   <[1] 8-bit propagation depth or 0xff to not propagate>
+		 *   <[1] 8-bit flags>
+		 *   <[8] 64-bit timestamp>
+		 *   <[5] 40-bit address of sending member>
+		 *   <[8] 64-bit network ID>
+		 *   <[6] MAC address of multicast group>
+		 *   <[4] 32-bit ADI of multicast group>
+		 *   <[6] 48-bit source MAC of packet or all 0 if from sender>
+		 *   <[2] 16-bit ethertype>
+		 *   <[2] 16-bit length of payload>
+		 *   <[...] ethernet payload>
+		 *   <[2] 16-bit length of signature or 0 if not present>
+		 *   <[...] signature of fields after propagation depth>
+		 */
+		VERB_MULTICAST = 0x18,
+
+		/**
+		 * Multicast subscription/unsubscription request:
+		 *   <[1] 8-bit propagation depth of 0xff to not propagate>
+		 *   <[1] 8-bit flags>
+		 *   <[8] 64-bit timestamp>
+		 *   <[5] 40-bit address of subscribing/unsubscribing member>
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit number of multicast group IDs to subscribe>
+		 *   <[...] series of 32-bit multicast group IDs>
+		 *   <[2] 16-bit number of multicast group IDs to unsubscribe>
+		 *   <[...] series of 32-bit multicast group IDs>
+		 *   <[2] 16-bit length of signature or 0 if not present>
+		 *   <[...] signature of fields after propagation depth>
 		 */
-		VERB_EPHEMERAL_KEY = 0x18
+		VERB_MULTICAST_SUBSCRIBE = 0x19,
+
+		// protocol max: 0x1f
 	};
 
 	/**
@@ -973,9 +889,6 @@ public:
 		/* Tried to join network, but you're not a member */
 		ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */
 
-		/* Multicasts to this group are not wanted */
-		ERROR_MULTICAST_STFU = 0x08,
-
 		/* Cannot deliver a forwarded ZeroTier packet (e.g. hops exceeded, no routes) */
 		/* Payload: <packet ID>, <destination>, <... additional packet ID / destinations> */
 		ERROR_CANNOT_DELIVER = 0x09

node/RuntimeEnvironment.hpp

@@ -26,7 +26,6 @@ class NodeConfig;
 class Switch;
 class Topology;
 class Node;
-class Multicaster;
 class NetworkController;
 class SelfAwareness;
 class Trace;
@@ -42,7 +41,6 @@ public:
 		,localNetworkController((NetworkController *)0)
 		,rtmem((void *)0)
 		,sw((Switch *)0)
-		,mc((Multicaster *)0)
 		,topology((Topology *)0)
 		,sa((SelfAwareness *)0)
 	{
@@ -69,7 +67,6 @@ public:
 
 	Trace *t;
 	Switch *sw;
-	Multicaster *mc;
 	Topology *topology;
 	SelfAwareness *sa;
 

node/SHA512.hpp

@@ -18,11 +18,11 @@
 
 #ifdef __APPLE__
 #include <CommonCrypto/CommonDigest.h>
-#endif
-
+#else
 #ifdef ZT_USE_LIBCRYPTO
 #include <openssl/sha.h>
 #endif
+#endif
 
 #define ZT_SHA512_DIGEST_LEN 64
 #define ZT_SHA384_DIGEST_LEN 48
@@ -66,6 +66,7 @@ static ZT_ALWAYS_INLINE void SHA384(void *digest,const void *data0,unsigned int
 }
 #endif
 
+#ifndef ZT_HAVE_NATIVE_SHA512
 #ifdef ZT_USE_LIBCRYPTO
 #define ZT_HAVE_NATIVE_SHA512 1
 static ZT_ALWAYS_INLINE void SHA512(void *digest,const void *data,unsigned int len)
@@ -91,6 +92,7 @@ static ZT_ALWAYS_INLINE void SHA384(void *digest,const void *data0,unsigned int
 	SHA384_Final(reinterpret_cast<unsigned char *>(digest),&ctx);
 }
 #endif
+#endif
 
 #ifndef ZT_HAVE_NATIVE_SHA512
 void SHA512(void *digest,const void *data,unsigned int len);

node/Trace.cpp

@@ -410,23 +410,6 @@ void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char *
 	}
 }
 
-void Trace::updateMemoizedSettings()
-{
-	const std::vector< SharedPtr<Network> > nws(RR->node->allNetworks());
-	{
-		Mutex::Lock l(_byNet_m);
-		_byNet.clear();
-		for(std::vector< SharedPtr<Network> >::const_iterator n(nws.begin());n!=nws.end();++n) {
-			const Address dest((*n)->config().remoteTraceTarget);
-			if (dest) {
-				std::pair<Address,Trace::Level> &m = _byNet[(*n)->id()];
-				m.first = dest;
-				m.second = (*n)->config().remoteTraceLevel;
-			}
-		}
-	}
-}
-
 void Trace::_send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d,const Address &dest)
 {
 	Packet outp(dest,RR->identity.address(),Packet::VERB_REMOTE_TRACE);

node/Trace.hpp

@@ -138,8 +138,6 @@ public:
 	void credentialRejected(void *const tPtr,const Tag &c,const char *reason);
 	void credentialRejected(void *const tPtr,const Revocation &c,const char *reason);
 
-	void updateMemoizedSettings();
-
 private:
 	const RuntimeEnvironment *const RR;