GitHub Issue #69 - make MAC assignment schema differ between virtual networks.

node/Address.hpp

@@ -37,7 +37,6 @@
 
 #include "Constants.hpp"
 #include "Utils.hpp"
-#include "MAC.hpp"
 #include "Buffer.hpp"
 
 namespace ZeroTier {
@@ -167,35 +166,6 @@ public:
 		return _a;
 	}
 
-	/**
-	 * Derive a MAC whose first octet is the ZeroTier LAN standard
-	 * 
-	 * @return Ethernet MAC derived from address
-	 */
-	inline MAC toMAC() const
-		throw()
-	{
-		MAC m;
-		m.data[0] = ZT_MAC_FIRST_OCTET;
-		copyTo(m.data + 1,ZT_ADDRESS_LENGTH);
-		return m;
-	}
-
-	/**
-	 * @param mac MAC address to check
-	 * @return True if this address would have this MAC
-	 */
-	inline bool wouldHaveMac(const MAC &mac) const
-		throw()
-	{
-		return ((mac.data[0] == ZT_MAC_FIRST_OCTET)&&
-		        (mac.data[1] == (unsigned char)((_a >> 32) & 0xff))&&
-		        (mac.data[2] == (unsigned char)((_a >> 24) & 0xff))&&
-		        (mac.data[3] == (unsigned char)((_a >> 16) & 0xff))&&
-		        (mac.data[4] == (unsigned char)((_a >> 8) & 0xff))&&
-		        (mac.data[5] == (unsigned char)(_a & 0xff)));
-	}
-
 	/**
 	 * @return Hexadecimal string
 	 */

node/Constants.hpp

@@ -170,14 +170,6 @@ error_no_byte_order_defined;
  */
 #define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 1000
 
-/**
- * First byte of MAC addresses derived from ZeroTier addresses
- * 
- * This has the 0x02 bit set, which indicates a locally administrered
- * MAC address rather than one with a known HW ID.
- */
-#define ZT_MAC_FIRST_OCTET 0x32
-
 /**
  * Length of secret key in bytes -- 256-bit for Salsa20
  */

node/InetAddress.hpp

@@ -413,14 +413,14 @@ public:
 		ip._sa.sin6.sin6_addr.s6_addr[5] = 0x00;
 		ip._sa.sin6.sin6_addr.s6_addr[6] = 0x00;
 		ip._sa.sin6.sin6_addr.s6_addr[7] = 0x00;
-		ip._sa.sin6.sin6_addr.s6_addr[8] = mac.data[0] & 0xfd;
-		ip._sa.sin6.sin6_addr.s6_addr[9] = mac.data[1];
-		ip._sa.sin6.sin6_addr.s6_addr[10] = mac.data[2];
+		ip._sa.sin6.sin6_addr.s6_addr[8] = mac[0] & 0xfd;
+		ip._sa.sin6.sin6_addr.s6_addr[9] = mac[1];
+		ip._sa.sin6.sin6_addr.s6_addr[10] = mac[2];
 		ip._sa.sin6.sin6_addr.s6_addr[11] = 0xff;
 		ip._sa.sin6.sin6_addr.s6_addr[12] = 0xfe;
-		ip._sa.sin6.sin6_addr.s6_addr[13] = mac.data[3];
-		ip._sa.sin6.sin6_addr.s6_addr[14] = mac.data[4];
-		ip._sa.sin6.sin6_addr.s6_addr[15] = mac.data[5];
+		ip._sa.sin6.sin6_addr.s6_addr[13] = mac[3];
+		ip._sa.sin6.sin6_addr.s6_addr[14] = mac[4];
+		ip._sa.sin6.sin6_addr.s6_addr[15] = mac[5];
 		ip._sa.sin6.sin6_port = Utils::hton((uint16_t)64);
 		return ip;
 	}

node/MAC.hpp

@@ -30,125 +30,231 @@
 
 #include <stdio.h>
 #include <stdlib.h>
+#include <stdint.h>
 
 #include "Constants.hpp"
-#include "Array.hpp"
 #include "Utils.hpp"
+#include "Address.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
 /**
- * An Ethernet MAC address
+ * 48-byte Ethernet MAC address
  */
-class MAC : public Array<unsigned char,6>
+class MAC
 {
 public:
+	MAC() throw() : _m(0ULL) {}
+	MAC(const MAC &m) throw() : _m(m._m) {}
+
+	/**
+	 * @param octet Single octet to fill entire MAC with (e.g. 0xff for broadcast)
+	 */
+	MAC(const unsigned char octet) throw() :
+		_m( ((((uint64_t)octet) & 0xffULL) << 40) |
+		    ((((uint64_t)octet) & 0xffULL) << 32) |
+		    ((((uint64_t)octet) & 0xffULL) << 24) |
+		    ((((uint64_t)octet) & 0xffULL) << 16) |
+		    ((((uint64_t)octet) & 0xffULL) << 8) |
+		    (((uint64_t)octet) & 0xffULL) ) {}
+
+	MAC(const unsigned char a,const unsigned char b,const unsigned char c,const unsigned char d,const unsigned char e,const unsigned char f) throw() :
+		_m( ((((uint64_t)a) & 0xffULL) << 40) |
+		    ((((uint64_t)b) & 0xffULL) << 32) |
+		    ((((uint64_t)c) & 0xffULL) << 24) |
+		    ((((uint64_t)d) & 0xffULL) << 16) |
+		    ((((uint64_t)e) & 0xffULL) << 8) |
+		    (((uint64_t)f) & 0xffULL) ) {}
+
+	MAC(const void *bits,unsigned int len) throw() { setTo(bits,len); }
+
+	MAC(const Address &ztaddr,uint64_t nwid) throw() { fromAddress(ztaddr,nwid); }
+
+	/**
+	 * Set MAC to zero
+	 */
+	inline void zero() { _m = 0ULL; }
+
+	/**
+	 * @return True if MAC is non-zero
+	 */
+	inline operator bool() const throw() { return (_m != 0ULL); }
+
 	/**
-	 * Create a zero/null MAC
+	 * @param bits Raw MAC in big-endian byte order
+	 * @param len Length, must be >= 6 or result is zero
 	 */
-	MAC()
+	inline void setTo(const void *bits,unsigned int len)
 		throw()
 	{
-		for(unsigned int i=0;i<6;++i)
-			data[i] = 0;
+		if (len < 6) {
+			_m = 0ULL;
+			return;
+		}
+		const unsigned char *b = (const unsigned char *)bits;
+		_m =  ((((uint64_t)*b) & 0xff) << 40); ++b;
+		_m |= ((((uint64_t)*b) & 0xff) << 32); ++b;
+		_m |= ((((uint64_t)*b) & 0xff) << 24); ++b;
+		_m |= ((((uint64_t)*b) & 0xff) << 16); ++b;
+		_m |= ((((uint64_t)*b) & 0xff) << 8); ++b;
+		_m |= (((uint64_t)*b) & 0xff);
 	}
 
 	/**
-	 * Create a MAC consisting of only this octet
-	 *
-	 * @param octet Octet to fill MAC with (e.g. 0xff for broadcast-all)
+	 * @param buf Destination buffer for MAC in big-endian byte order
+	 * @param len Length of buffer, must be >= 6 or nothing is copied
 	 */
-	MAC(const unsigned char octet)
+	inline void copyTo(void *buf,unsigned int len) const
 		throw()
 	{
-		for(unsigned int i=0;i<6;++i)
-			data[i] = octet;
+		if (len < 6)
+			return;
+		unsigned char *b = (unsigned char *)buf;
+		*(b++) = (unsigned char)((_m >> 40) & 0xff);
+		*(b++) = (unsigned char)((_m >> 32) & 0xff);
+		*(b++) = (unsigned char)((_m >> 24) & 0xff);
+		*(b++) = (unsigned char)((_m >> 16) & 0xff);
+		*(b++) = (unsigned char)((_m >> 8) & 0xff);
+		*b = (unsigned char)(_m & 0xff);
 	}
 
 	/**
-	 * Create a MAC from raw bits
+	 * Append to a buffer in big-endian byte order
 	 *
-	 * @param bits 6 bytes of MAC address data
+	 * @param b Buffer to append to
 	 */
-	MAC(const void *bits)
-		throw()
+	template<unsigned int C>
+	inline void appendTo(Buffer<C> &b) const
+		throw(std::out_of_range)
 	{
-		for(unsigned int i=0;i<6;++i)
-			data[i] = ((const unsigned char *)bits)[i];
+		unsigned char *p = (unsigned char *)b.appendField(6);
+		*(p++) = (unsigned char)((_m >> 40) & 0xff);
+		*(p++) = (unsigned char)((_m >> 32) & 0xff);
+		*(p++) = (unsigned char)((_m >> 24) & 0xff);
+		*(p++) = (unsigned char)((_m >> 16) & 0xff);
+		*(p++) = (unsigned char)((_m >> 8) & 0xff);
+		*p = (unsigned char)(_m & 0xff);
 	}
 
 	/**
-	 * @return True if non-NULL (not all zero)
+	 * @return True if this is broadcast (all 0xff)
 	 */
-	inline operator bool() const
-		throw()
+	inline bool isBroadcast() const throw() { return (_m == 0xffffffffffffULL); }
+
+	/**
+	 * @return True if this is a multicast MAC
+	 */
+	inline bool isMulticast() const throw() { return ((_m & 0x010000000000ULL) != 0ULL); }
+
+	/**
+	 * @param True if this is a locally-administered MAC
+	 */
+	inline bool isLocallyAdministered() const throw() { return ((_m & 0x020000000000ULL) != 0ULL); }
+
+	/**
+	 * @param s Hex MAC, with or without : delimiters
+	 */
+	inline void fromString(const char *s)
 	{
-		for(unsigned int i=0;i<6;++i) {
-			if (data[i])
-				return true;
-		}
-		return false;
+		char tmp[8];
+		Utils::unhex(s,tmp,6);
+		setTo(tmp,6);
 	}
 
 	/**
-	 * @return True if this is the broadcast-all MAC (0xff:0xff:...)
+	 * @return MAC address in standard :-delimited hex format
 	 */
-	inline bool isBroadcast() const
-		throw()
+	inline std::string toString() const
 	{
-		for(unsigned int i=0;i<6;++i) {
-			if (data[i] != 0xff)
-				return false;
+		char tmp[24];
+		std::string s;
+		Utils::snprintf(tmp,sizeof(tmp),"%.12llx",_m);
+		for(int i=0;i<12;++i) {
+			if ((i > 0)&&((i % 2) == 0))
+				s.push_back(':');
+			s.push_back(tmp[i]);
 		}
-		return true;
+		return s;
 	}
 
 	/**
-	 * @return True if this is a multicast/broadcast address
+	 * Set this MAC to a MAC derived from an address and a network ID
+	 *
+	 * @param ztaddr ZeroTier address
+	 * @param nwid 64-bit network ID
 	 */
-	inline bool isMulticast() const
+	inline void fromAddress(const Address &ztaddr,uint64_t nwid)
 		throw()
 	{
-		return ((data[0] & 1));
+		uint64_t m = ((uint64_t)firstOctetForNetwork(nwid)) << 40;
+		uint64_t a = ztaddr.toInt();
+		m |= a; // a is 40 bits
+		m ^= ((nwid >> 8) & 0xff) << 32;
+		m ^= ((nwid >> 16) & 0xff) << 24;
+		m ^= ((nwid >> 24) & 0xff) << 16;
+		m ^= ((nwid >> 32) & 0xff) << 8;
+		m ^= (nwid >> 40) & 0xff;
+		_m = m;
 	}
 
 	/**
-	 * @return True if this is a ZeroTier unicast MAC
+	 * Get the ZeroTier address for this MAC on this network (assuming no bridging of course, basic unicast)
+	 *
+	 * This just XORs the next-lest-significant 5 bytes of the network ID again to unmask.
+	 *
+	 * @param nwid Network ID
 	 */
-	inline bool isZeroTier() const
+	inline Address toAddress(uint64_t nwid) const
 		throw()
 	{
-		return (data[0] == ZT_MAC_FIRST_OCTET);
+		uint64_t a = _m & 0xffffffffffULL;
+		a ^= ((nwid >> 8) & 0xff) << 32;
+		a ^= ((nwid >> 16) & 0xff) << 24;
+		a ^= ((nwid >> 24) & 0xff) << 16;
+		a ^= ((nwid >> 32) & 0xff) << 8;
+		a ^= (nwid >> 40) & 0xff;
+		return Address(a);
 	}
 
 	/**
-	 * Zero this MAC
+	 * @param nwid Network ID
+	 * @return First octet of MAC for this network
 	 */
-	inline void zero()
+	static inline unsigned char firstOctetForNetwork(uint64_t nwid)
 		throw()
 	{
-		for(unsigned int i=0;i<6;++i)
-			data[i] = 0;
+		unsigned char a = ((unsigned char)(nwid & 0xfe) | 0x02); // locally administered, not multicast, from LSB of network ID
+		return ((a == 0x52) ? 0x32 : a); // blacklist 0x52 since it's used by KVM
 	}
 
 	/**
-	 * @param s String hex representation (with or without :'s)
-	 * @return True if string decoded into a full-length MAC
+	 * @param i Value from 0 to 5 (inclusive)
+	 * @return Byte at said position (address interpreted in big-endian order)
 	 */
-	inline void fromString(const char *s)
-	{
-		Utils::unhex(s,data,6);
-	}
+	inline unsigned char operator[](unsigned int i) const throw() { return (unsigned char)((_m >> (40 - (i * 8))) & 0xff); }
 
 	/**
-	 * @return MAC address in standard :-delimited hex format
+	 * @return 6, which is the number of bytes in a MAC, for container compliance
 	 */
-	inline std::string toString() const
+	inline unsigned int size() const throw() { return 6; }
+
+	inline MAC &operator=(const MAC &m)
+		throw()
 	{
-		char tmp[32];
-		Utils::snprintf(tmp,sizeof(tmp),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)data[0],(int)data[1],(int)data[2],(int)data[3],(int)data[4],(int)data[5]);
-		return std::string(tmp);
+		_m = m._m;
+		return *this;
 	}
+
+	inline bool operator==(const MAC &m) const throw() { return (_m == m._m); }
+	inline bool operator!=(const MAC &m) const throw() { return (_m != m._m); }
+	inline bool operator<(const MAC &m) const throw() { return (_m < m._m); }
+	inline bool operator<=(const MAC &m) const throw() { return (_m <= m._m); }
+	inline bool operator>(const MAC &m) const throw() { return (_m > m._m); }
+	inline bool operator>=(const MAC &m) const throw() { return (_m >= m._m); }
+
+private:
+	uint64_t _m;
 };
 
 } // namespace ZeroTier

node/MulticastGroup.hpp

@@ -100,14 +100,7 @@ public:
 			// 24 bits of uniqueness. Collisions aren't likely to be common enough
 			// to care about.
 			const unsigned char *a = (const unsigned char *)ip.rawIpData();
-			MAC m;
-			m.data[0] = 0x33;
-			m.data[1] = 0x33;
-			m.data[2] = 0xff;
-			m.data[3] = a[13];
-			m.data[4] = a[14];
-			m.data[5] = a[15];
-			return MulticastGroup(m,0);
+			return MulticastGroup(MAC(0x33,0x33,0xff,a[13],a[14],a[15]),0);
 		}
 		return MulticastGroup();
 	}
@@ -118,7 +111,7 @@ public:
 	inline std::string toString() const
 	{
 		char buf[64];
-		Utils::snprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%lx",(unsigned int)_mac.data[0],(unsigned int)_mac.data[1],(unsigned int)_mac.data[2],(unsigned int)_mac.data[3],(unsigned int)_mac.data[4],(unsigned int)_mac.data[5],(unsigned long)_adi);
+		Utils::snprintf(buf,sizeof(buf),"%.2x%.2x%.2x%.2x%.2x%.2x/%lx",(unsigned int)_mac[0],(unsigned int)_mac[1],(unsigned int)_mac[2],(unsigned int)_mac[3],(unsigned int)_mac[4],(unsigned int)_mac[5],(unsigned long)_adi);
 		return std::string(buf);
 	}
 

node/Network.cpp

@@ -102,7 +102,7 @@ SharedPtr<Network> Network::newInstance(const RuntimeEnvironment *renv,NodeConfi
 	SharedPtr<Network> nw(new Network());
 	nw->_id = id;
 	nw->_nc = nc;
-	nw->_mac = renv->identity.address().toMAC();
+	nw->_mac.fromAddress(renv->identity.address(),id);
 	nw->_r = renv;
 	nw->_tap = (EthernetTap *)0;
 	nw->_lastConfigUpdate = 0;

node/PacketDecoder.cpp

@@ -414,7 +414,11 @@ bool PacketDecoder::_doFRAME(const RuntimeEnvironment *_r,const SharedPtr<Peer>
 				unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
 				if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 					if (network->config()->permitsEtherType(etherType)) {
-						network->tapPut(source().toMAC(),etherType,data() + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD);
+						network->tapPut(
+							MAC(source(),network->id()),
+							etherType,
+							data() + ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,
+							size() - ZT_PROTO_VERB_FRAME_IDX_PAYLOAD);
 					} else {
 						TRACE("dropped FRAME from %s: ethernet type %u not allowed on network %.16llx",source().toString().c_str(),etherType,(unsigned long long)network->id());
 						return true;
@@ -481,8 +485,8 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 		const unsigned int prefixBits = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_PREFIX_BITS];
 		const unsigned int prefix = (*this)[ZT_PROTO_VERB_MULTICAST_FRAME_IDX_PROPAGATION_PREFIX];
 		const uint64_t guid = at<uint64_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_GUID);
-		const MAC sourceMac(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_SOURCE_MAC));
-		const MulticastGroup dest(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_DEST_MAC)),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI));
+		const MAC sourceMac(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_SOURCE_MAC,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_SOURCE_MAC),ZT_PROTO_VERB_MULTICAST_FRAME_LEN_SOURCE_MAC);
+		const MulticastGroup dest(MAC(field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_MAC,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_DEST_MAC),ZT_PROTO_VERB_MULTICAST_FRAME_LEN_DEST_MAC),at<uint32_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_DEST_ADI));
 		const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_ETHERTYPE);
 		const unsigned int frameLen = at<uint16_t>(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME_LEN);
 		const unsigned char *const frame = field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME,frameLen);
@@ -635,7 +639,7 @@ bool PacketDecoder::_doMULTICAST_FRAME(const RuntimeEnvironment *_r,const Shared
 						// We do not terminate here, since if the member just has an out of
 						// date cert or hasn't sent us a cert yet we still want to propagate
 						// the message so multicast keeps working downstream.
-					} else if ((!nconf->permitsBridging(origin))&&(!origin.wouldHaveMac(sourceMac))) {
+					} else if ((!nconf->permitsBridging(origin))&&(MAC(origin,network->id()) != sourceMac)) {
 						// This *does* terminate propagation, since it's technically a
 						// security violation of the network's bridging policy. But if we
 						// were to keep propagating it wouldn't hurt anything, just waste
@@ -829,7 +833,7 @@ bool PacketDecoder::_doMULTICAST_LIKE(const RuntimeEnvironment *_r,const SharedP
 			uint64_t nwid = at<uint64_t>(ptr);
 			SharedPtr<Network> network(_r->nc->network(nwid));
 			if ((_r->topology->amSupernode())||((network)&&(network->isAllowed(peer->address())))) {
-				_r->mc->likesGroup(nwid,src,MulticastGroup(MAC(field(ptr + 8,6)),at<uint32_t>(ptr + 14)),now);
+				_r->mc->likesGroup(nwid,src,MulticastGroup(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)),now);
 				if (network)
 					network->pushMembershipCertificate(peer->address(),false,now);
 			}

node/Switch.cpp

@@ -98,103 +98,103 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		LOG("%s: frame received from self, ignoring (bridge loop? OS bug?)",network->tapDeviceName().c_str());
 		return;
 	}
-	if (from != network->mac()) {
-		LOG("%s: ignored tap: %s -> %s %s (bridging not supported)",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType));
-		return;
-	}
 
 	if (!nconf->permitsEtherType(etherType)) {
 		LOG("%s: ignored tap: %s -> %s: ethertype %s not allowed on network %.16llx",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),(unsigned long long)network->id());
 		return;
 	}
 
-	if (to.isMulticast()) {
-		MulticastGroup mg(to,0);
-
-		if (to.isBroadcast()) {
-			// Cram IPv4 IP into ADI field to make IPv4 ARP broadcast channel specific and scalable
-			if ((etherType == ZT_ETHERTYPE_ARP)&&(data.size() == 28)&&(data[2] == 0x08)&&(data[3] == 0x00)&&(data[4] == 6)&&(data[5] == 4)&&(data[7] == 0x01))
-				mg = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(data.field(24,4),4,0));
-		}
+	if (from == network->mac()) {
+		if (to.isMulticast()) {
+			MulticastGroup mg(to,0);
 
-		if (!network->updateAndCheckMulticastBalance(_r->identity.address(),mg,data.size())) {
-			TRACE("%s: didn't multicast %d bytes, quota exceeded for multicast group %s",network->tapDeviceName().c_str(),(int)data.size(),mg.toString().c_str());
-			return;
-		}
+			if (to.isBroadcast()) {
+				// Cram IPv4 IP into ADI field to make IPv4 ARP broadcast channel specific and scalable
+				if ((etherType == ZT_ETHERTYPE_ARP)&&(data.size() == 28)&&(data[2] == 0x08)&&(data[3] == 0x00)&&(data[4] == 6)&&(data[5] == 4)&&(data[7] == 0x01))
+					mg = MulticastGroup::deriveMulticastGroupForAddressResolution(InetAddress(data.field(24,4),4,0));
+			}
 
-		const unsigned int mcid = ++_multicastIdCounter & 0xffffff;
-		const uint16_t bloomNonce = (uint16_t)(_r->prng->next32() & 0xffff); // doesn't need to be cryptographically strong
-		unsigned char bloom[ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM];
-		unsigned char fifo[ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO + ZT_ADDRESS_LENGTH];
-		unsigned char *const fifoEnd = fifo + sizeof(fifo);
-		const unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION) + data.size();
-		const SharedPtr<Peer> supernode(_r->topology->getBestSupernode());
-
-		for(unsigned int prefix=0,np=((unsigned int)2 << (nconf->multicastPrefixBits() - 1));prefix<np;++prefix) {
-			memset(bloom,0,sizeof(bloom));
-
-			unsigned char *fifoPtr = fifo;
-			_r->mc->getNextHops(network->id(),mg,Multicaster::AddToPropagationQueue(&fifoPtr,fifoEnd,bloom,bloomNonce,_r->identity.address(),nconf->multicastPrefixBits(),prefix));
-			while (fifoPtr != fifoEnd)
-				*(fifoPtr++) = (unsigned char)0;
-
-			Address firstHop(fifo,ZT_ADDRESS_LENGTH); // fifo is +1 in size, with first element being used here
-			if (!firstHop) {
-				if (supernode)
-					firstHop = supernode->address();
-				else continue;
+			if (!network->updateAndCheckMulticastBalance(_r->identity.address(),mg,data.size())) {
+				TRACE("%s: didn't multicast %d bytes, quota exceeded for multicast group %s",network->tapDeviceName().c_str(),(int)data.size(),mg.toString().c_str());
+				return;
 			}
 
-			Packet outp(firstHop,_r->identity.address(),Packet::VERB_MULTICAST_FRAME);
-			outp.append((uint16_t)0);
-			outp.append(fifo + ZT_ADDRESS_LENGTH,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO); // remainder of fifo is loaded into packet
-			outp.append(bloom,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM);
-			outp.append((nconf->com()) ? (unsigned char)ZT_PROTO_VERB_MULTICAST_FRAME_FLAGS_HAS_MEMBERSHIP_CERTIFICATE : (unsigned char)0);
-			outp.append(network->id());
-			outp.append(bloomNonce);
-			outp.append((unsigned char)nconf->multicastPrefixBits());
-			outp.append((unsigned char)prefix);
-			_r->identity.address().appendTo(outp);
-			outp.append((unsigned char)((mcid >> 16) & 0xff));
-			outp.append((unsigned char)((mcid >> 8) & 0xff));
-			outp.append((unsigned char)(mcid & 0xff));
-			outp.append(from.data,6);
-			outp.append(mg.mac().data,6);
-			outp.append(mg.adi());
-			outp.append((uint16_t)etherType);
-			outp.append((uint16_t)data.size());
-			outp.append(data);
-
-			C25519::Signature sig(_r->identity.sign(outp.field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION,signedPartLen),signedPartLen));
-			outp.append((uint16_t)sig.size());
-			outp.append(sig.data,(unsigned int)sig.size());
-
-			// FIXME: now we send the netconf cert with every single multicast,
-			// which pretty much ensures everyone has it ahead of time but adds
-			// some redundant payload. Maybe think abouut this in the future.
-			if (nconf->com())
-				nconf->com().serialize(outp);
-
-			outp.compress();
-			send(outp,true);
-		}
-	} else if (to.isZeroTier()) {
-		// Simple unicast frame from us to another node
-		Address toZT(to.data + 1,ZT_ADDRESS_LENGTH);
-		if (network->isAllowed(toZT)) {
-			network->pushMembershipCertificate(toZT,false,Utils::now());
-
-			Packet outp(toZT,_r->identity.address(),Packet::VERB_FRAME);
-			outp.append(network->id());
-			outp.append((uint16_t)etherType);
-			outp.append(data);
-			outp.compress();
-			send(outp,true);
+			const unsigned int mcid = ++_multicastIdCounter & 0xffffff;
+			const uint16_t bloomNonce = (uint16_t)(_r->prng->next32() & 0xffff); // doesn't need to be cryptographically strong
+			unsigned char bloom[ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM];
+			unsigned char fifo[ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO + ZT_ADDRESS_LENGTH];
+			unsigned char *const fifoEnd = fifo + sizeof(fifo);
+			const unsigned int signedPartLen = (ZT_PROTO_VERB_MULTICAST_FRAME_IDX_FRAME - ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION) + data.size();
+			const SharedPtr<Peer> supernode(_r->topology->getBestSupernode());
+
+			for(unsigned int prefix=0,np=((unsigned int)2 << (nconf->multicastPrefixBits() - 1));prefix<np;++prefix) {
+				memset(bloom,0,sizeof(bloom));
+
+				unsigned char *fifoPtr = fifo;
+				_r->mc->getNextHops(network->id(),mg,Multicaster::AddToPropagationQueue(&fifoPtr,fifoEnd,bloom,bloomNonce,_r->identity.address(),nconf->multicastPrefixBits(),prefix));
+				while (fifoPtr != fifoEnd)
+					*(fifoPtr++) = (unsigned char)0;
+
+				Address firstHop(fifo,ZT_ADDRESS_LENGTH); // fifo is +1 in size, with first element being used here
+				if (!firstHop) {
+					if (supernode)
+						firstHop = supernode->address();
+					else continue;
+				}
+
+				Packet outp(firstHop,_r->identity.address(),Packet::VERB_MULTICAST_FRAME);
+				outp.append((uint16_t)0);
+				outp.append(fifo + ZT_ADDRESS_LENGTH,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_FIFO); // remainder of fifo is loaded into packet
+				outp.append(bloom,ZT_PROTO_VERB_MULTICAST_FRAME_LEN_PROPAGATION_BLOOM);
+				outp.append((nconf->com()) ? (unsigned char)ZT_PROTO_VERB_MULTICAST_FRAME_FLAGS_HAS_MEMBERSHIP_CERTIFICATE : (unsigned char)0);
+				outp.append(network->id());
+				outp.append(bloomNonce);
+				outp.append((unsigned char)nconf->multicastPrefixBits());
+				outp.append((unsigned char)prefix);
+				_r->identity.address().appendTo(outp);
+				outp.append((unsigned char)((mcid >> 16) & 0xff));
+				outp.append((unsigned char)((mcid >> 8) & 0xff));
+				outp.append((unsigned char)(mcid & 0xff));
+				from.appendTo(outp);
+				mg.mac().appendTo(outp);
+				outp.append(mg.adi());
+				outp.append((uint16_t)etherType);
+				outp.append((uint16_t)data.size());
+				outp.append(data);
+
+				C25519::Signature sig(_r->identity.sign(outp.field(ZT_PROTO_VERB_MULTICAST_FRAME_IDX__START_OF_SIGNED_PORTION,signedPartLen),signedPartLen));
+				outp.append((uint16_t)sig.size());
+				outp.append(sig.data,(unsigned int)sig.size());
+
+				// FIXME: now we send the netconf cert with every single multicast,
+				// which pretty much ensures everyone has it ahead of time but adds
+				// some redundant payload. Maybe think abouut this in the future.
+				if (nconf->com())
+					nconf->com().serialize(outp);
+
+				outp.compress();
+				send(outp,true);
+			}
+		} else if (to[0] == MAC::firstOctetForNetwork(network->id())) {
+			// Simple unicast frame from us to another node on the same virtual network
+			Address toZT(to.toAddress(network->id()));
+			if (network->isAllowed(toZT)) {
+				network->pushMembershipCertificate(toZT,false,Utils::now());
+
+				Packet outp(toZT,_r->identity.address(),Packet::VERB_FRAME);
+				outp.append(network->id());
+				outp.append((uint16_t)etherType);
+				outp.append(data);
+				outp.compress();
+				send(outp,true);
+			} else {
+				TRACE("%s: UNICAST: %s -> %s %s dropped, destination not a member of closed network %.16llx",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),network->id());
+			}
 		} else {
-			TRACE("UNICAST: %s -> %s %s (dropped, destination not a member of closed network %llu)",from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),network->id());
+			LOG("%s: UNICAST %s -> %s %s dropped, bridging disabled, unicast destination not on network %.16llx",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),network->id());
 		}
 	} else {
-		TRACE("UNICAST: %s -> %s %s (dropped, destination MAC not ZeroTier)",from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType));
+		LOG("%s: UNICAST %s -> %s %s dropped, bridging disabled, unicast source not on network %.16llx",network->tapDeviceName().c_str(),from.toString().c_str(),to.toString().c_str(),etherTypeName(etherType),network->id());
 	}
 }
 
@@ -458,7 +458,7 @@ void Switch::announceMulticastGroups(const std::map< SharedPtr<Network>,std::set
 
 					// network ID, MAC, ADI
 					outp.append((uint64_t)nwmgs->first->id());
-					outp.append(mg->mac().data,6);
+					mg->mac().appendTo(outp);
 					outp.append((uint32_t)mg->adi());
 				}
 			}
@@ -487,7 +487,7 @@ void Switch::announceMulticastGroups(const SharedPtr<Peer> &peer)
 
 				// network ID, MAC, ADI
 				outp.append((uint64_t)(*n)->id());
-				outp.append(mg->mac().data,6);
+				mg->mac().appendTo(outp);
 				outp.append((uint32_t)mg->adi());
 			}
 		}

node/UnixEthernetTap.cpp

@@ -644,10 +644,8 @@ void UnixEthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,c
 {
 	char putBuf[4096 + 14];
 	if ((_fd > 0)&&(len <= _mtu)) {
-		for(int i=0;i<6;++i)
-			putBuf[i] = to.data[i];
-		for(int i=0;i<6;++i)
-			putBuf[i+6] = from.data[i];
+		to.copyTo(putBuf,6);
+		from.copyTo(putBuf + 6,6);
 		*((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType);
 		memcpy(putBuf + 14,data,len);
 		len += 14;
@@ -921,7 +919,7 @@ bool UnixEthernetTap::updateMulticastGroups(std::set<MulticastGroup> &groups)
 				struct sockaddr_dl *in = (struct sockaddr_dl *)p->ifma_name;
 				struct sockaddr_dl *la = (struct sockaddr_dl *)p->ifma_addr;
 				if ((la->sdl_alen == 6)&&(in->sdl_nlen <= _dev.length())&&(!memcmp(_dev.data(),in->sdl_data,in->sdl_nlen)))
-					newGroups.insert(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen),0));
+					newGroups.insert(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen,6),0));
 			}
 			p = p->ifma_next;
 		}
@@ -996,10 +994,8 @@ void UnixEthernetTap::threadMain()
 				if (r > 14) {
 					if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
 						r = _mtu + 14;
-					for(int i=0;i<6;++i)
-						to.data[i] = (unsigned char)getBuf[i];
-					for(int i=0;i<6;++i)
-						from.data[i] = (unsigned char)getBuf[i + 6];
+					to.setTo(getBuf,6);
+					from.setTo(getBuf + 6,6);
 					unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]);
 					if (etherType != 0x8100) { // VLAN tagged frames are not supported!
 						data.copyFrom(getBuf + 14,(unsigned int)r - 14);