Bridging pretty much ready to test! Got Switch all wired up. Also fix a latent probably-never-triggered bug in MULTICAST_FRAME handling. GitHub issue #68

node/Address.hpp

@@ -177,7 +177,8 @@ public:
 	 * @param prefixBits Number of bits in prefix bit pattern
 	 * @return True if address is within prefix
 	 */
-	inline bool withinMulticastPropagationPrefix(uint64_t prefix,unsigned int prefixBits)
+	inline bool withinMulticastPropagationPrefix(uint64_t prefix,unsigned int prefixBits) const
+		throw()
 	{
 		return ((_a & (0xffffffffffffffffULL >> (64 - prefixBits))) == prefix);
 	}

node/Constants.hpp

@@ -410,6 +410,6 @@ error_no_byte_order_defined;
 /**
  * If there is no known route, spam to up to this many active bridges
  */
-#define ZT_MAX_BRIDGE_SPAM 4
+#define ZT_MAX_BRIDGE_SPAM 16
 
 #endif

node/PacketDecoder.cpp

@@ -457,9 +457,15 @@ bool PacketDecoder::_doEXT_FRAME(const RuntimeEnvironment *_r,const SharedPtr<Pe
 	try {
 		SharedPtr<Network> network(_r->nc->network(at<uint64_t>(ZT_PROTO_VERB_EXT_FRAME_IDX_NETWORK_ID)));
 		if (network) {
+			if ((*this)[ZT_PROTO_VERB_EXT_FRAME_IDX_FLAGS] != 0) {
+				TRACE("dropped EXT_FRAME due to unknown flags");
+				return true;
+			}
+
 			const MAC to(field(ZT_PROTO_VERB_EXT_FRAME_IDX_TO,ZT_PROTO_VERB_EXT_FRAME_LEN_TO),ZT_PROTO_VERB_EXT_FRAME_LEN_TO);
 			const MAC from(field(ZT_PROTO_VERB_EXT_FRAME_IDX_FROM,ZT_PROTO_VERB_EXT_FRAME_LEN_FROM),ZT_PROTO_VERB_EXT_FRAME_LEN_FROM);
 			unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_EXT_FRAME_IDX_ETHERTYPE);
+
 			if (size() > ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD) {
 				if ((!from)||(from.isMulticast())||(!to)) {
 					TRACE("dropped EXT_FRAME from %s@%s(%s) to %s: invalid source or destination MAC",from.toString().c_str(),peer->address().toString().c_str(),_remoteAddress.toString().c_str(),to.toString().c_str());
@@ -657,46 +663,44 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 			if (network) {
 				SharedPtr<NetworkConfig> nconf(network->config2());
 				if (nconf) {
+					// Learn real maxDepth from netconf
 					maxDepth = std::min((unsigned int)ZT_MULTICAST_GLOBAL_MAX_DEPTH,nconf->multicastDepth());
 					if (!maxDepth)
 						maxDepth = ZT_MULTICAST_GLOBAL_MAX_DEPTH;
 
 					if (!network->isAllowed(origin)) {
-						TRACE("didn't inject MULTICAST_FRAME from %s(%s) into %.16llx: sender %s not allowed or we don't have a certificate",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,origin.toString().c_str());
-
-						// Tell them we need a certificate
+						// Papers, please...
 						Packet outp(source(),_r->identity.address(),Packet::VERB_ERROR);
-						outp.append((unsigned char)Packet::VERB_FRAME);
+						outp.append((unsigned char)Packet::VERB_MULTICAST_FRAME);
 						outp.append(packetId());
 						outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
 						outp.append(nwid);
 						outp.armor(peer->key(),true);
 						_fromSock->send(_remoteAddress,outp.data(),outp.size());
+						TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: sender %s not allowed or we don't have a certificate",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,origin.toString().c_str());
+						return true;
+					}
 
-						// We do not terminate here, since if the member just has an out of
-						// date cert or hasn't sent us a cert yet we still want to propagate
-						// the message so multicast keeps working downstream.
-					} else if ((!nconf->permitsBridging(origin))&&(MAC(origin,network->id()) != sourceMac)) {
-						// This *does* terminate propagation, since it's technically a
-						// security violation of the network's bridging policy. But if we
-						// were to keep propagating it wouldn't hurt anything, just waste
-						// bandwidth as everyone else would reject it too.
+					if (MAC(origin,network->id()) != sourceMac) {
+						if (!nconf->permitsBridging(origin)) {
 #ifdef ZT_TRACE_MULTICAST
-						Utils::snprintf(mct,sizeof(mct),
-							"%.16llx %.2u %.3u%s %c %s dropped: bridging not allowed",
-							guid,
-							prefix,
-							depth,
-							mctdepth,
-							(_r->topology->amSupernode() ? 'S' : '-'),
-							_r->identity.address().toString().c_str());
-						_r->sm->sendUdp(ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct));
+							Utils::snprintf(mct,sizeof(mct),
+								"%.16llx %.2u %.3u%s %c %s dropped: bridging not allowed",
+								guid,
+								prefix,
+								depth,
+								mctdepth,
+								(_r->topology->amSupernode() ? 'S' : '-'),
+								_r->identity.address().toString().c_str());
+							_r->sm->sendUdp(ZT_DEFAULTS.multicastTraceWatcher,mct,strlen(mct));
 #endif
-						TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: source mac %s doesn't belong to %s, and bridging is not supported on network",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,sourceMac.toString().c_str(),origin.toString().c_str());
-						return true;
-					} else if (!nconf->permitsEtherType(etherType)) {
-						// Ditto for this-- halt propagation if this is for an ethertype
-						// this network doesn't allow. Same principle as bridging test.
+							TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: source mac %s doesn't belong to %s, and bridging is not supported on network",source().toString().c_str(),_remoteAddress.toString().c_str(),nwid,sourceMac.toString().c_str(),origin.toString().c_str());
+							return true;
+						}
+						network->learnBridgeRoute(sourceMac,origin);
+					}
+
+					if (!nconf->permitsEtherType(etherType)) {
 #ifdef ZT_TRACE_MULTICAST
 						Utils::snprintf(mct,sizeof(mct),
 							"%.16llx %.2u %.3u%s %c %s dropped: ethertype not allowed",
@@ -710,7 +714,9 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 #endif
 						TRACE("dropped MULTICAST_FRAME from %s(%s) into %.16llx: ethertype %u is not allowed",source().toString().c_str(),nwid,_remoteAddress.toString().c_str(),etherType);
 						return true;
-					} else if (!network->updateAndCheckMulticastBalance(origin,dest,frameLen)) {
+					}
+
+					if (!network->updateAndCheckMulticastBalance(origin,dest,frameLen)) {
 						// Rate limits can only be checked by members of this network, but
 						// there should be enough of them that over-limit multicasts get
 						// their propagation aborted.
@@ -727,9 +733,9 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 #endif
 						TRACE("dropped MULTICAST_FRAME from %s(%s): rate limits exceeded for sender %s",source().toString().c_str(),_remoteAddress.toString().c_str(),origin.toString().c_str());
 						return true;
-					} else {
-						network->tapPut(sourceMac,dest.mac(),etherType,frame,frameLen);
 					}
+
+					network->tapPut(sourceMac,dest.mac(),etherType,frame,frameLen);
 				}
 			}
 		}

node/Switch.cpp

@@ -89,11 +89,9 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 	if (!nconf)
 		return;
 
-	// This should not happen
-	if (to == network->mac()) {
-		LOG("%s: frame received from self, ignoring (bridge loop? OS bug?)",network->tapDeviceName().c_str());
+	// Sanity check -- bridge loop? OS problem?
+	if (to == network->mac())
 		return;
-	}
 
 	// Check anti-recursion module to ensure that this is not ZeroTier talking over its own links
 	if (!_r->antiRec->checkEthernetFrame(data.data(),data.size())) {
@@ -111,12 +109,13 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 	bool fromBridged = false;
 	if (from != network->mac()) {
 		if (!network->permitsBridging(_r->identity.address())) {
-			LOG("%s: UNICAST %s -> %s %s dropped, bridging disabled on network %.16llx",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),network->id());
+			LOG("%s: %s -> %s %s not forwarded, bridging disabled on %.16llx or this peer not a bridge",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),network->id());
 			return;
 		}
 		fromBridged = true;
 	}
 
+	// Multicast, either bridged or local source
 	if (to.isMulticast()) {
 		MulticastGroup mg(to,0);
 
@@ -167,6 +166,8 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 			// Then visit next hops according to multicaster (if there's room... almost certainly will be)
 			if (fifoPtr != fifoEnd) {
 				_r->mc->getNextHops(network->id(),mg,Multicaster::AddToPropagationQueue(&fifoPtr,fifoEnd,bloom,bloomNonce,_r->identity.address(),nconf->multicastPrefixBits(),prefix));
+
+				// Pad remainder of FIFO with zeroes
 				while (fifoPtr != fifoEnd)
 					*(fifoPtr++) = (unsigned char)0;
 			}
@@ -176,7 +177,7 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 			if (!firstHop) {
 				if (supernode)
 					firstHop = supernode->address();
-				else continue; // no first hop = nowhere to go, try next bit prefix
+				else continue; // nowhere to go
 			}
 
 			Packet outp(firstHop,_r->identity.address(),Packet::VERB_MULTICAST_FRAME);
@@ -193,7 +194,7 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 			outp.append((unsigned char)((mcid >> 8) & 0xff));
 			outp.append((unsigned char)(mcid & 0xff));
 			from.appendTo(outp);
-			to.appendTo(outp);
+			mg.mac().appendTo(outp);
 			outp.append(mg.adi());
 			outp.append((uint16_t)etherType);
 			outp.append((uint16_t)data.size());
@@ -212,8 +213,12 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 			outp.compress();
 			send(outp,true);
 		}
-	} else if (to[0] == MAC::firstOctetForNetwork(network->id())) {
-		// Simple unicast frame from us to another node on the same virtual network
+
+		return;
+	}
+
+	// Unicast from local peer to another non-bridged ZeroTier node
+	if ((!fromBridged)&&(to[0] == MAC::firstOctetForNetwork(network->id()))) {
 		Address toZT(to.toAddress(network->id()));
 		if (network->isAllowed(toZT)) {
 			network->pushMembershipCertificate(toZT,false,Utils::now());
@@ -227,44 +232,45 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		} else {
 			TRACE("%s: UNICAST: %s -> %s %s dropped, destination not a member of closed network %.16llx",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),network->id());
 		}
-	} else {
-		// Simple unicast from us to another node behind another bridge
-		Address bridges[ZT_MAX_BRIDGE_SPAM];
-		unsigned int numBridges = 0;
-
-		bridges[0] = network->findBridgeTo(to));
-		if ((bridges[0])&&(bridges[0] != _r->identity.address())&&(network->isAllowed(bridges[0]))&&(network->permitsBridging(bridges[0]))) {
-			++numBridges;
-		} else if (!nconf->activeBridges().empty()) {
-			// If there is no known route, spam to up to ZT_MAX_BRIDGE_SPAM active bridges
-			std::set<Address>::const_iterator ab(nconf->activeBridges().begin());
-			if (nconf->activeBridges().size() <= ZT_MAX_BRIDGE_SPAM) {
-				// If there are <= ZT_MAX_BRIDGE_SPAM active bridges, just take them all
-				while (numBridges < nconf->activeBridges().size())
+	}
+
+	// Unicast to another node behind another bridge, whether from us or not
+
+	Address bridges[ZT_MAX_BRIDGE_SPAM];
+	unsigned int numBridges = 0;
+
+	bridges[0] = network->findBridgeTo(to);
+	if ((bridges[0])&&(bridges[0] != _r->identity.address())&&(network->isAllowed(bridges[0]))&&(network->permitsBridging(bridges[0]))) {
+		++numBridges;
+	} else if (!nconf->activeBridges().empty()) {
+		// If there is no known route, spam to up to ZT_MAX_BRIDGE_SPAM active bridges
+		std::set<Address>::const_iterator ab(nconf->activeBridges().begin());
+		if (nconf->activeBridges().size() <= ZT_MAX_BRIDGE_SPAM) {
+			// If there are <= ZT_MAX_BRIDGE_SPAM active bridges, just take them all
+			while (numBridges < nconf->activeBridges().size())
+				bridges[numBridges++] = *(ab++);
+		} else {
+			// Otherwise do this less efficient multipass thing to pick randomly from an ordered set until we have enough
+			while (numBridges < ZT_MAX_BRIDGE_SPAM) {
+				if (ab == nconf->activeBridges().end())
+					ab = nconf->activeBridges().begin();
+				if (((unsigned long)_r->prng->next32() % (unsigned long)nconf->activeBridges().size()) == 0)
 					bridges[numBridges++] = *(ab++);
-			} else {
-				// Otherwise do this less efficient multipass thing to pick randomly from an ordered set until we have enough
-				while (numBridges < ZT_MAX_BRIDGE_SPAM) {
-					if (ab == nconf->activeBridges().end())
-						ab = nconf->activeBridges().begin();
-					if (((unsigned long)_r->prng->next32() % (unsigned long)nconf->activeBridges().size()) == 0)
-						bridges[numBridges++] = *(ab++);
-					else ++ab;
-				}
+				else ++ab;
 			}
 		}
+	}
 
-		for(unsigned int b=0;b<numBridges;++b) {
-			Packet outp(bridges[b],_r->identity.address(),Packet::VERB_EXT_FRAME);
-			outp.append(network->id());
-			outp.append((unsigned char)0);
-			to.appendTo(outp);
-			from.appendTo(outp);
-			outp.append((uint16_t)etherType);
-			outp.append(data);
-			outp.compress();
-			send(outp,true);
-		}
+	for(unsigned int b=0;b<numBridges;++b) {
+		Packet outp(bridges[b],_r->identity.address(),Packet::VERB_EXT_FRAME);
+		outp.append(network->id());
+		outp.append((unsigned char)0);
+		to.appendTo(outp);
+		from.appendTo(outp);
+		outp.append((uint16_t)etherType);
+		outp.append(data);
+		outp.compress();
+		send(outp,true);
 	}
 }