Add capability for P2P multicast propagation, and some cleanup.

controller/EmbeddedNetworkController.cpp

@@ -1093,59 +1093,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
 	return 404;
 }
 
-void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt)
-{
-	static volatile unsigned long idCounter = 0;
-	char id[128],tmp[128];
-	std::string k,v;
-
-	if (!_db)
-		return;
-
-	try {
-		// Convert Dictionary into JSON object
-		json d;
-		char *saveptr = (char *)0;
-		for(char *l=Utils::stok(rt.data,"\n",&saveptr);(l);l=Utils::stok((char *)0,"\n",&saveptr)) {
-			char *eq = strchr(l,'=');
-			if (eq > l) {
-				k.assign(l,(unsigned long)(eq - l));
-				v.clear();
-				++eq;
-				while (*eq) {
-					if (*eq == '\\') {
-						++eq;
-						if (*eq) {
-							switch(*eq) {
-								case 'r': v.push_back('\r'); break;
-								case 'n': v.push_back('\n'); break;
-								case '0': v.push_back((char)0); break;
-								case 'e': v.push_back('='); break;
-								default: v.push_back(*eq); break;
-							}
-							++eq;
-						}
-					} else {
-						v.push_back(*(eq++));
-					}
-				}
-				if ((k.length() > 0)&&(v.length() > 0))
-					d[k] = v;
-			}
-		}
-
-		const int64_t now = OSUtils::now();
-		OSUtils::ztsnprintf(id,sizeof(id),"%.10llx-%.16llx-%.10llx-%.4x",_signingId.address().toInt(),now,rt.origin,(unsigned int)(idCounter++ & 0xffff));
-		d["id"] = id;
-		d["objtype"] = "trace";
-		d["ts"] = now;
-		d["nodeId"] = Utils::hex10(rt.origin,tmp);
-		_db->save((nlohmann::json *)0,d);
-	} catch ( ... ) {
-		// drop invalid trace messages if an error occurs
-	}
-}
-
 void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId)
 {
 	// Send an update to all members of the network that are online

controller/EmbeddedNetworkController.hpp

@@ -103,8 +103,6 @@ public:
 		std::string &responseBody,
 		std::string &responseContentType);
 
-	void handleRemoteTrace(const ZT_RemoteTrace &rt);
-
 	// Called on update via POST or by JSONDB on external update of network or network member records
 	void onNetworkUpdate(const uint64_t networkId);
 	void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId);

controller/FileDB.cpp

@@ -32,14 +32,12 @@ namespace ZeroTier
 FileDB::FileDB(EmbeddedNetworkController *const nc,const Identity &myId,const char *path) :
 	DB(nc,myId,path),
 	_networksPath(_path + ZT_PATH_SEPARATOR_S + "network"),
-	_tracePath(_path + ZT_PATH_SEPARATOR_S + "trace"),
 	_onlineChanged(false),
 	_running(true)
 {
 	OSUtils::mkdir(_path.c_str());
 	OSUtils::lockDownFile(_path.c_str(),true);
 	OSUtils::mkdir(_networksPath.c_str());
-	OSUtils::mkdir(_tracePath.c_str());
 
 	std::vector<std::string> networks(OSUtils::listDirectory(_networksPath.c_str(),false));
 	std::string buf;
@@ -178,12 +176,6 @@ void FileDB::save(nlohmann::json *orig,nlohmann::json &record)
 					_memberChanged(old,record,true);
 				}
 			}
-		} else if (objtype == "trace") {
-			const std::string id = record["id"];
-			if (id.length() > 0) {
-				OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%s.json",_tracePath.c_str(),id.c_str());
-				OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1));
-			}
 		}
 	} catch ( ... ) {} // drop invalid records missing fields
 }

controller/FileDB.hpp

@@ -47,7 +47,6 @@ public:
 
 protected:
 	std::string _networksPath;
-	std::string _tracePath;
 	std::thread _onlineUpdateThread;
 	std::map< uint64_t,std::map<uint64_t,std::map<int64_t,InetAddress> > > _online;
 	std::mutex _online_l;

node/IncomingPacket.cpp

@@ -28,6 +28,8 @@
 #include <string.h>
 #include <stdlib.h>
 
+#include <list>
+
 #include "../version.h"
 #include "../include/ZeroTierOne.h"
 
@@ -168,13 +170,14 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 			break;
 
 		case Packet::ERROR_IDENTITY_COLLISION:
-			// FIXME: for federation this will need a payload with a signature or something.
+			// This is a trusted upstream telling us our 5-digit ID is taken. This
+			// causes the node to generate another.
 			if (RR->topology->isUpstream(peer->identity()))
 				RR->node->postEvent(tPtr,ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
 			break;
 
 		case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
-			// Peers can send this in response to frames if they do not have a recent enough COM from us
+			// Peers can send this to ask for a cert for a network.
 			networkId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD);
 			const SharedPtr<Network> network(RR->node->network(networkId));
 			const int64_t now = RR->node->now();
@@ -1000,18 +1003,16 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr
 
 bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
 {
-	const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID);
-	const unsigned int flags = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS];
+	unsigned int offset = ZT_PACKET_IDX_PAYLOAD;
+	const uint64_t nwid = at<uint64_t>(offset); offset += 8;
+	const unsigned int flags = (*this)[offset]; ++offset;
 
 	const SharedPtr<Network> network(RR->node->network(nwid));
 	if (network) {
-		// Offset -- size of optional fields added to position of later fields
-		unsigned int offset = 0;
-
 		if ((flags & 0x01) != 0) {
 			// This is deprecated but may still be sent by old peers
 			CertificateOfMembership com;
-			offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
+			offset += com.deserialize(*this,offset);
 			if (com)
 				network->addCredential(tPtr,com);
 		}
@@ -1023,60 +1024,87 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,
 
 		unsigned int gatherLimit = 0;
 		if ((flags & 0x02) != 0) {
-			gatherLimit = at<uint32_t>(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT);
-			offset += 4;
+			gatherLimit = at<uint32_t>(offset); offset += 4;
 		}
 
 		MAC from;
 		if ((flags & 0x04) != 0) {
-			from.setTo(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,6),6);
-			offset += 6;
+			from.setTo(field(offset,6),6); offset += 6;
 		} else {
 			from.fromAddress(peer->address(),nwid);
 		}
 
-		const MulticastGroup to(MAC(field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,6),6),at<uint32_t>(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI));
-		const unsigned int etherType = at<uint16_t>(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE);
-		const unsigned int frameLen = size() - (offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME);
+		const unsigned int recipientsOffset = offset;
+		std::list<Address> recipients;
+		if ((flags & 0x08) != 0) {
+			const unsigned int rc = at<uint16_t>(offset); offset += 2;
+			for(unsigned int i=0;i<rc;++i) {
+				const Address a(field(offset,5),5);
+				if ((a != peer->address())&&(a != RR->identity.address())) {
+					recipients.push_back(a);
+				}
+				offset += 5;
+			}
+		}
+		const unsigned int afterRecipientsOffset = offset;
+
+		const MulticastGroup to(MAC(field(offset,6),6),at<uint32_t>(offset + 6)); offset += 10;
+		const unsigned int etherType = at<uint16_t>(offset); offset += 2;
+		const unsigned int frameLen = size() - offset;
 
 		if (network->config().multicastLimit == 0) {
 			RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"multicast disabled");
-			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,false,nwid);
+			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
+			return true;
+		}
+		if (!to.mac().isMulticast()) {
+			RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"destination not multicast");
+			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
+			return true;
+		}
+		if ((!from)||(from.isMulticast())||(from == network->mac())) {
+			RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC");
+			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
 			return true;
 		}
 
 		if ((frameLen > 0)&&(frameLen <= ZT_MAX_MTU)) {
-			if (!to.mac().isMulticast()) {
-				RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"destination not multicast");
-				peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay
-				return true;
-			}
-			if ((!from)||(from.isMulticast())||(from == network->mac())) {
-				RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC");
-				peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay
-				return true;
+			const uint8_t *const frameData = ((const uint8_t *)unsafeData()) + offset;
+			if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0) {
+				RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen);
 			}
+		}
 
-			const uint8_t *const frameData = (const uint8_t *)field(offset + ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen);
-
-			if ((flags & 0x08)&&(network->config().isMulticastReplicator(RR->identity.address())))
-				RR->mc->send(tPtr,RR->node->now(),network,peer->address(),to,from,etherType,frameData,frameLen);
+		if (!recipients.empty()) {
+			const std::vector<Address> anchors = network->config().anchors();
+			const bool amAnchor = (std::find(anchors.begin(),anchors.end(),RR->identity.address()) != anchors.end());
+
+			for(std::list<Address>::iterator ra(recipients.begin());ra!=recipients.end();) {
+				SharedPtr<Peer> recipient(RR->topology->getPeer(tPtr,*ra));
+				if ((recipient)&&((recipient->remoteVersionProtocol() < 10)||(amAnchor))) {
+					Packet outp(*ra,RR->identity.address(),Packet::VERB_MULTICAST_FRAME);
+					outp.append(field(ZT_PACKET_IDX_PAYLOAD,recipientsOffset - ZT_PACKET_IDX_PAYLOAD),recipientsOffset - ZT_PACKET_IDX_PAYLOAD);
+					outp.append(field(afterRecipientsOffset,size() - afterRecipientsOffset),size() - afterRecipientsOffset);
+					RR->sw->send(tPtr,outp,true);
+					recipients.erase(ra++);
+				} else ++ra;
+			}
 
-			if (from != MAC(peer->address(),nwid)) {
-				if (network->config().permitsBridging(peer->address())) {
-					network->learnBridgeRoute(from,peer->address());
-				} else {
-					RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"bridging not allowed (remote)");
-					peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay
-					return true;
+			if (!recipients.empty()) {
+				Packet outp(recipients.front(),RR->identity.address(),Packet::VERB_MULTICAST_FRAME);
+				recipients.pop_front();
+				outp.append(field(ZT_PACKET_IDX_PAYLOAD,recipientsOffset - ZT_PACKET_IDX_PAYLOAD),recipientsOffset - ZT_PACKET_IDX_PAYLOAD);
+				if (!recipients.empty()) {
+					outp.append((uint16_t)recipients.size());
+					for(std::list<Address>::iterator ra(recipients.begin());ra!=recipients.end();++ra)
+						ra->appendTo(outp);
 				}
+				outp.append(field(afterRecipientsOffset,size() - afterRecipientsOffset),size() - afterRecipientsOffset);
+				RR->sw->send(tPtr,outp,true);
 			}
-
-			if (network->filterIncomingPacket(tPtr,peer,RR->identity.address(),from,to.mac(),frameData,frameLen,etherType,0) > 0)
-				RR->node->putFrame(tPtr,nwid,network->userPtr(),from,to.mac(),etherType,0,(const void *)frameData,frameLen);
 		}
 
-		if (gatherLimit) {
+		if (gatherLimit) { // DEPRECATED but still supported
 			Packet outp(source(),RR->identity.address(),Packet::VERB_OK);
 			outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME);
 			outp.append(packetId());
@@ -1091,12 +1119,11 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,
 		}
 
 		peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
+		return true;
 	} else {
 		_sendErrorNeedCredentials(RR,tPtr,peer,nwid);
 		return false;
 	}
-
-	return true;
 }
 
 bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)

node/MulticastGroup.hpp

@@ -31,6 +31,7 @@
 
 #include "MAC.hpp"
 #include "InetAddress.hpp"
+#include "Utils.hpp"
 
 namespace ZeroTier {
 

node/Multicaster.cpp

@@ -170,51 +170,6 @@ void Multicaster::send(
 	unsigned long idxbuf[4096];
 	unsigned long *indexes = idxbuf;
 
-	// If we're in hub-and-spoke designated multicast replication mode, see if we
-	// have a multicast replicator active. If so, pick the best and send it
-	// there. If we are a multicast replicator or if none are alive, fall back
-	// to sender replication. Note that bridges do not do this since this would
-	// break bridge route learning. This is sort of an edge case limitation of
-	// the current protocol and could be fixed, but fixing it would add more
-	// complexity than the fix is probably worth. Bridges are generally high
-	// bandwidth nodes.
-	if (!network->config().isActiveBridge(RR->identity.address())) {
-		Address multicastReplicators[ZT_MAX_NETWORK_SPECIALISTS];
-		const unsigned int multicastReplicatorCount = network->config().multicastReplicators(multicastReplicators);
-		if (multicastReplicatorCount) {
-			if (std::find(multicastReplicators,multicastReplicators + multicastReplicatorCount,RR->identity.address()) == (multicastReplicators + multicastReplicatorCount)) {
-				SharedPtr<Peer> bestMulticastReplicator;
-				SharedPtr<Path> bestMulticastReplicatorPath;
-				unsigned int bestMulticastReplicatorLatency = 0xffff;
-				for(unsigned int i=0;i<multicastReplicatorCount;++i) {
-					const SharedPtr<Peer> p(RR->topology->getPeerNoCache(multicastReplicators[i]));
-					if ((p)&&(p->isAlive(now))) {
-						const SharedPtr<Path> pp(p->getAppropriatePath(now,false));
-						if ((pp)&&(pp->latency() < bestMulticastReplicatorLatency)) {
-							bestMulticastReplicatorLatency = pp->latency();
-							bestMulticastReplicatorPath = pp;
-							bestMulticastReplicator = p;
-						}
-					}
-				}
-				if (bestMulticastReplicator) {
-					Packet outp(bestMulticastReplicator->address(),RR->identity.address(),Packet::VERB_MULTICAST_FRAME);
-					outp.append((uint64_t)network->id());
-					outp.append((uint8_t)0x0c); // includes source MAC | please replicate
-					((src) ? src : MAC(RR->identity.address(),network->id())).appendTo(outp);
-					mg.mac().appendTo(outp);
-					outp.append((uint32_t)mg.adi());
-					outp.append((uint16_t)etherType);
-					outp.append(data,len);
-					if (!network->config().disableCompression()) outp.compress();
-					outp.armor(bestMulticastReplicator->key(),true);
-					bestMulticastReplicatorPath->send(RR,tPtr,outp.data(),outp.size(),now);
-					return;
-				}
-			}
-		}
-	}
-
 	try {
 		Mutex::Lock _l(_groups_m);
 		MulticastGroupStatus &gs = _groups[Multicaster::Key(network->id(),mg)];

node/Network.cpp

@@ -1461,22 +1461,8 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu
 		}
 		std::sort(alwaysAnnounceTo.begin(),alwaysAnnounceTo.end());
 
-		for(std::vector<Address>::const_iterator a(alwaysAnnounceTo.begin());a!=alwaysAnnounceTo.end();++a) {
-			/*
-			// push COM to non-members so they can do multicast request auth
-			if ( (_config.com) && (!_memberships.contains(*a)) && (*a != RR->identity.address()) ) {
-				Packet outp(*a,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS);
-				_config.com.serialize(outp);
-				outp.append((uint8_t)0x00);
-				outp.append((uint16_t)0); // no capabilities
-				outp.append((uint16_t)0); // no tags
-				outp.append((uint16_t)0); // no revocations
-				outp.append((uint16_t)0); // no certificates of ownership
-				RR->sw->send(tPtr,outp,true);
-			}
-			*/
+		for(std::vector<Address>::const_iterator a(alwaysAnnounceTo.begin());a!=alwaysAnnounceTo.end();++a)
 			_announceMulticastGroupsTo(tPtr,*a,groups);
-		}
 	}
 
 	{

node/NetworkConfig.hpp

@@ -98,11 +98,6 @@
  */
 #define ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR 0x0000040000000000ULL
 
-/**
- * Designated multicast replicators replicate multicast in place of sender-side replication
- */
-#define ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR 0x0000080000000000ULL
-
 namespace ZeroTier {
 
 // Dictionary capacity needed for max size network config
@@ -335,40 +330,11 @@ public:
 		return r;
 	}
 
-	inline std::vector<Address> multicastReplicators() const
-	{
-		std::vector<Address> r;
-		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR) != 0)
-				r.push_back(Address(specialists[i]));
-		}
-		return r;
-	}
-
-	inline unsigned int multicastReplicators(Address mr[ZT_MAX_NETWORK_SPECIALISTS]) const
-	{
-		unsigned int c = 0;
-		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR) != 0)
-				mr[c++] = specialists[i];
-		}
-		return c;
-	}
-
-	inline bool isMulticastReplicator(const Address &a) const
-	{
-		for(unsigned int i=0;i<specialistCount;++i) {
-			if (((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR) != 0)&&(a == specialists[i]))
-				return true;
-		}
-		return false;
-	}
-
 	inline std::vector<Address> alwaysContactAddresses() const
 	{
 		std::vector<Address> r;
 		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & (ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR | ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR)) != 0)
+			if ((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR) != 0)
 				r.push_back(Address(specialists[i]));
 		}
 		return r;
@@ -378,7 +344,7 @@ public:
 	{
 		unsigned int c = 0;
 		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & (ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR | ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR)) != 0)
+			if ((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR) != 0)
 				ac[c++] = specialists[i];
 		}
 		return c;
@@ -387,7 +353,7 @@ public:
 	inline void alwaysContactAddresses(Hashtable< Address,std::vector<InetAddress> > &a) const
 	{
 		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & (ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR | ZT_NETWORKCONFIG_SPECIALIST_TYPE_MULTICAST_REPLICATOR)) != 0) {
+			if ((specialists[i] & ZT_NETWORKCONFIG_SPECIALIST_TYPE_ANCHOR) != 0) {
 				a[Address(specialists[i])];
 			}
 		}

node/Node.cpp

@@ -269,15 +269,6 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 			Hashtable< Address,std::vector<InetAddress> > alwaysContact;
 			RR->topology->getUpstreamsToContact(alwaysContact);
 
-			// Uncomment to dump stats
-			/*
-			for(unsigned int i=0;i<32;i++) {
-				if (_stats.inVerbCounts[i] > 0)
-					printf("%.2x\t%12lld %lld\n",i,(unsigned long long)_stats.inVerbCounts[i],(unsigned long long)_stats.inVerbBytes[i]);
-			}
-			printf("\n");
-			*/
-
 			// Check last receive time on designated upstreams to see if we seem to be online
 			int64_t lastReceivedFromUpstream = 0;
 			{

node/Packet.hpp

@@ -70,6 +70,7 @@
  * 9  - 1.2.0 ... 1.2.14
  * 10 - 1.4.0 ... CURRENT
  *    + Multipath capability and load balancing
+ *    + Peer-to-peer multicast replication (optional)
  */
 #define ZT_PROTO_VERSION 10
 
@@ -309,17 +310,6 @@
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_ADI + 4)
 #define ZT_PROTO_VERB_MULTICAST_GATHER_IDX_COM (ZT_PROTO_VERB_MULTICAST_GATHER_IDX_GATHER_LIMIT + 4)
 
-// Note: COM, GATHER_LIMIT, and SOURCE_MAC are optional, and so are specified without size
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID (ZT_PACKET_IDX_PAYLOAD)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_NETWORK_ID + 8)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GATHER_LIMIT (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FLAGS + 1)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC + 6)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI + 4)
-#define ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE + 2)
-
 #define ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP (ZT_PROTO_VERB_OK_IDX_PAYLOAD)
 #define ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION (ZT_PROTO_VERB_HELLO__OK__IDX_TIMESTAMP + 8)
 #define ZT_PROTO_VERB_HELLO__OK__IDX_MAJOR_VERSION (ZT_PROTO_VERB_HELLO__OK__IDX_PROTOCOL_VERSION + 1)
@@ -710,10 +700,6 @@ public:
 		 * controllers and root servers. In the current network, root servers
 		 * will provide the service of final multicast cache.
 		 *
-		 * VERB_NETWORK_CREDENTIALS should be pushed along with this, especially
-		 * if using upstream (e.g. root) nodes as multicast databases. This allows
-		 * GATHERs to be authenticated.
-		 *
 		 * OK/ERROR are not generated.
 		 */
 		VERB_MULTICAST_LIKE = 0x09,
@@ -832,19 +818,11 @@ public:
 		 *   [<[...] network certificate of membership>]
 		 *
 		 * Flags:
-		 *   0x01 - COM is attached
-		 *
-		 * This message asks a peer for additional known endpoints that have
-		 * LIKEd a given multicast group. It's sent when the sender wishes
-		 * to send multicast but does not have the desired number of recipient
-		 * peers.
-		 *
+		 *   0x01 - COM is attached (DEPRECATED)
+		 * 
 		 * More than one OK response can occur if the response is broken up across
 		 * multiple packets or if querying a clustered node.
 		 *
-		 * The COM should be included so that upstream nodes that are not
-		 * members of our network can validate our request.
-		 *
 		 * OK response payload:
 		 *   <[8] 64-bit network ID>
 		 *   <[6] MAC address of multicast group being queried>
@@ -864,6 +842,8 @@ public:
 		 *   <[1] flags>
 		 *  [<[4] 32-bit implicit gather limit>]
 		 *  [<[6] source MAC>]
+		 *  [<[2] number of explicitly specified recipients>]
+		 *  [<[...] series of 5-byte explicitly specified recipients>]
 		 *   <[6] destination MAC (multicast address)>
 		 *   <[4] 32-bit multicast ADI (multicast address extension)>
 		 *   <[2] 16-bit ethertype>
@@ -871,15 +851,12 @@ public:
 		 *
 		 * Flags:
 		 *   0x01 - Network certificate of membership attached (DEPRECATED)
-		 *   0x02 - Implicit gather limit field is present
+		 *   0x02 - Implicit gather limit field is present (DEPRECATED)
 		 *   0x04 - Source MAC is specified -- otherwise it's computed from sender
-		 *   0x08 - Please replicate (sent to multicast replicators)
-		 *
-		 * OK and ERROR responses are optional. OK may be generated if there are
-		 * implicit gather results or if the recipient wants to send its own
-		 * updated certificate of network membership to the sender. ERROR may be
-		 * generated if a certificate is needed or if multicasts to this group
-		 * are no longer wanted (multicast unsubscribe).
+		 *   0x08 - Explicit recipient list included for P2P/HS replication
+		 * 
+		 * Explicit recipient lists are used for peer to peer or hub and spoke
+		 * replication.
 		 *
 		 * OK response payload:
 		 *   <[8] 64-bit network ID>
@@ -1004,10 +981,6 @@ public:
 		 * be sent to observers configured at the network level for those that
 		 * pertain directly to activity on a network, or to global observers if
 		 * locally configured.
-		 *
-		 * The instance ID is a random 64-bit value generated by each ZeroTier
-		 * node on startup. This is helpful in identifying traces from different
-		 * members of a cluster.
 		 */
 		VERB_REMOTE_TRACE = 0x15
 	};

node/Peer.hpp

@@ -540,7 +540,8 @@ public:
 	/**
 	 * @return Whether this peer is reachable via an aggregate link
 	 */
-	inline bool hasAggregateLink() {
+	inline bool hasAggregateLink()
+	{
 		return _localMultipathSupported && _remoteMultipathSupported && _remotePeerMultipathEnabled;
 	}
 

node/Utils.cpp

@@ -55,6 +55,9 @@ namespace ZeroTier {
 
 const char Utils::HEXCHARS[16] = { '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' };
 
+const char Utils::BASE32CHARS[32] = { 'a','b','c','d','e','f','g','h','i','j','k','l','m','n','o','p','q','r','s','t','u','v','w','x','y','z','2','3','4','5','6','7' };
+const uint8_t Utils::BASE32BITS[256] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,26,27,28,29,30,31,0,0,0,0,0,0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,0,0,0,0,0,0,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
+
 // Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers.
 static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len)
 {

node/Utils.hpp

@@ -442,10 +442,35 @@ public:
 	}
 	static inline int64_t ntoh(int64_t n) { return (int64_t)ntoh((uint64_t)n); }
 
+	static inline void base325to8(const uint8_t *const in,char *const out)
+	{
+		out[0] = BASE32CHARS[(in[0]) >> 3];
+		out[1] = BASE32CHARS[(in[0] & 0x07) << 2 | (in[1] & 0xc0) >> 6];
+		out[2] = BASE32CHARS[(in[1] & 0x3e) >> 1];
+		out[3] = BASE32CHARS[(in[1] & 0x01) << 4 | (in[2] & 0xf0) >> 4];
+		out[4] = BASE32CHARS[(in[2] & 0x0f) << 1 | (in[3] & 0x80) >> 7];
+		out[5] = BASE32CHARS[(in[3] & 0x7c) >> 2];
+		out[6] = BASE32CHARS[(in[3] & 0x03) << 3 | (in[4] & 0xe0) >> 5];
+		out[7] = BASE32CHARS[(in[4] & 0x1f)];
+	}
+	
+	static inline void base328to5(const char *const in,uint8_t *const out)
+	{
+		out[0] = ((BASE32BITS[(unsigned int)in[0]]) << 3) | (BASE32BITS[(unsigned int)in[1]] & 0x1C) >> 2;
+		out[1] = ((BASE32BITS[(unsigned int)in[1]] & 0x03) << 6) | (BASE32BITS[(unsigned int)in[2]]) << 1 | (BASE32BITS[(unsigned int)in[3]] & 0x10) >> 4;
+		out[2] = ((BASE32BITS[(unsigned int)in[3]] & 0x0F) << 4) | (BASE32BITS[(unsigned int)in[4]] & 0x1E) >> 1;
+		out[3] = ((BASE32BITS[(unsigned int)in[4]] & 0x01) << 7) | (BASE32BITS[(unsigned int)in[5]]) << 2 | (BASE32BITS[(unsigned int)in[6]] & 0x18) >> 3;
+		out[4] = ((BASE32BITS[(unsigned int)in[6]] & 0x07) << 5) | (BASE32BITS[(unsigned int)in[7]]);
+	}
+	
 	/**
 	 * Hexadecimal characters 0-f
 	 */
 	static const char HEXCHARS[16];
+
+private:
+	static const char BASE32CHARS[32];
+	static const uint8_t BASE32BITS[256];
 };
 
 } // namespace ZeroTier

service/OneService.cpp

@@ -2266,9 +2266,7 @@ public:
 			}	break;
 
 			case ZT_EVENT_REMOTE_TRACE: {
-				const ZT_RemoteTrace *rt = reinterpret_cast<const ZT_RemoteTrace *>(metaData);
-				if ((rt)&&(rt->len > 0)&&(rt->len <= ZT_MAX_REMOTE_TRACE_SIZE)&&(rt->data))
-					_controller->handleRemoteTrace(*rt);
+				// TODO
 			}
 
 			default: