Bunch of work on pushing and replication of tags and capabilities, and protocol cleanup.

include/ZeroTierOne.h

@@ -96,11 +96,6 @@ extern "C" {
  */
 #define ZT_MAX_NETWORK_SPECIALISTS 256
 
-/**
- * Maximum number of static physical to ZeroTier address mappings (typically relays, etc.)
- */
-#define ZT_MAX_NETWORK_PINNED 16
-
 /**
  * Maximum number of multicast group subscriptions per network
  */
@@ -111,6 +106,16 @@ extern "C" {
  */
 #define ZT_MAX_NETWORK_RULES 256
 
+/**
+ * Maximum number of per-node capabilities per network
+ */
+#define ZT_MAX_NETWORK_CAPABILITIES 64
+
+/**
+ * Maximum number of per-node tags per network
+ */
+#define ZT_MAX_NETWORK_TAGS 16
+
 /**
  * Maximum number of direct network paths to a given peer
  */
@@ -126,11 +131,6 @@ extern "C" {
  */
 #define ZT_MAX_CAPABILITY_RULES 64
 
-/**
- * Maximum length of a capbility's short descriptive name
- */
-#define ZT_MAX_CAPABILITY_NAME_LENGTH 63
-
 /**
  * Global maximum length for capability chain of custody (including initial issue)
  */

node/Capability.hpp

@@ -109,6 +109,11 @@ public:
 	 */
 	inline uint64_t networkId() const { return _nwid; }
 
+	/**
+	 * @return Expiration time relative to network config timestamp
+	 */
+	inline uint64_t expiration() const { return _expiration; }
+
 	/**
 	 * Sign this capability and add signature to its chain of custody
 	 *

node/Filter.cpp

@@ -19,15 +19,8 @@
 #include <stdint.h>
 
 #include "Constants.hpp"
-#include "RuntimeEnvironment.hpp"
-#include "Address.hpp"
-#include "MAC.hpp"
-#include "InetAddress.hpp"
 #include "Filter.hpp"
-#include "Packet.hpp"
-#include "Switch.hpp"
-#include "Topology.hpp"
-#include "Node.hpp"
+#include "InetAddress.hpp"
 
 // Returns true if packet appears valid; pos and proto will be set
 static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsigned int &pos,unsigned int &proto)
@@ -61,8 +54,8 @@ static bool _ipv6GetPayload(const uint8_t *frameData,unsigned int frameLen,unsig
 namespace ZeroTier {
 
 bool Filter::run(
-	const RuntimeEnvironment *RR,
 	const uint64_t nwid,
+	const bool receiving,
 	const Address &ztSource,
 	const Address &ztDest,
 	const MAC &macSource,
@@ -72,8 +65,13 @@ bool Filter::run(
 	const unsigned int etherType,
 	const unsigned int vlanId,
 	const ZT_VirtualNetworkRule *rules,
-	const unsigned int ruleCount)
+	const unsigned int ruleCount,
+	const Tag *tags,
+	const unsigned int tagCount,
+	Address &sendCopyOfPacketTo)
 {
+	sendCopyOfPacketTo.zero();
+
 	// For each set of rules we start by assuming that they match (since no constraints
 	// yields a 'match all' rule).
 	uint8_t thisSetMatches = 1;
@@ -92,6 +90,8 @@ bool Filter::run(
 					// This set did match, so perform action!
 					if (rt != ZT_NETWORK_RULE_ACTION_DROP) {
 						if ((rt == ZT_NETWORK_RULE_ACTION_TEE)||(rt == ZT_NETWORK_RULE_ACTION_REDIRECT)) {
+							sendCopyOfPacketTo = rules[rn].v.zt;
+							/*
 							// Tee and redirect both want this frame copied to somewhere else.
 							Packet outp(Address(rules[rn].v.zt),RR->identity.address(),Packet::VERB_EXT_FRAME);
 							outp.append(nwid);
@@ -102,6 +102,7 @@ bool Filter::run(
 							outp.append(frameData,frameLen);
 							outp.compress();
 							RR->sw->send(outp,true,nwid);
+							*/
 						}
 						// For REDIRECT we will want to DROP at this node. For TEE we ACCEPT at this node but
 						// also forward it along as we just did.
@@ -244,9 +245,20 @@ bool Filter::run(
 				thisRuleMatches = (uint8_t)((frameLen >= (unsigned int)rules[rn].v.frameSize[0])&&(frameLen <= (unsigned int)rules[rn].v.frameSize[1]));
 				break;
 			case ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE:
-				break;
 			case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL:
 			case ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY:
+				for(unsigned int i=0;i<tagCount;++i) { // sequential scan is probably fastest since this is going to be <64 entries (usually only one or two)
+					if (tags[i].id() == rules[rn].v.tag.id) {
+						if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_RANGE) {
+							thisRuleMatches = (uint8_t)((tags[i].value() >= rules[rn].v.tag.value[0])&&(tags[i].value() <= rules[rn].v.tag.value[1]));
+						} else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ALL) {
+							thisRuleMatches = (uint8_t)((tags[i].value() & rules[rn].v.tag.value[0]) == rules[rn].v.tag.value[0]);
+						} else if (rt == ZT_NETWORK_RULE_MATCH_TAG_VALUE_BITS_ANY) {
+							thisRuleMatches = (uint8_t)((tags[i].value() & rules[rn].v.tag.value[0]) != 0);
+						}
+						break;
+					}
+				}
 				break;
 		}
 

node/Filter.hpp

@@ -21,15 +21,16 @@
 
 #include <stdint.h>
 
+#include <vector>
+
 #include "Constants.hpp"
 #include "../include/ZeroTierOne.h"
+#include "Address.hpp"
+#include "MAC.hpp"
+#include "Tag.hpp"
 
 namespace ZeroTier {
 
-class Address;
-class RuntimeEnvironment;
-class MAC;
-
 /**
  * Network packet filter for rules engine
  */
@@ -42,8 +43,8 @@ public:
 	 * This returns whether or not the packet should be accepted and may also
 	 * take other actions for e.g. the TEE and REDIRECT targets.
 	 *
-	 * @param RR ZeroTier runtime environment (context)
 	 * @param nwid ZeroTier network ID
+	 * @param receiving True if on receiving side, false on sending side
 	 * @param ztSource Source ZeroTier address
 	 * @param ztDest Destination ZeroTier address
 	 * @param macSource Ethernet layer source address
@@ -54,10 +55,14 @@ public:
 	 * @param vlanId 16-bit VLAN ID
 	 * @param rules Pointer to array of rules
 	 * @param ruleCount Number of rules
+	 * @param tags Tags associated with this node on this network
+	 * @param tagCount Number of tags
+	 * @param sendCopyOfPacketTo Result parameter: if non-NULL send a copy of this packet to another node
+	 * @return True if packet should be accepted for send or receive
 	 */
 	static bool run(
-		const RuntimeEnvironment *RR,
 		const uint64_t nwid,
+		const bool receiving,
 		const Address &ztSource,
 		const Address &ztDest,
 		const MAC &macSource,
@@ -67,7 +72,10 @@ public:
 		const unsigned int etherType,
 		const unsigned int vlanId,
 		const ZT_VirtualNetworkRule *rules,
-		const unsigned int ruleCount);
+		const unsigned int ruleCount,
+		const Tag *tags,
+		const unsigned int tagCount,
+		Address &sendCopyOfPacketTo);
 };
 
 } // namespace ZeroTier

node/IncomingPacket.cpp

@@ -38,6 +38,9 @@
 #include "Node.hpp"
 #include "DeferredPackets.hpp"
 #include "Filter.hpp"
+#include "CertificateOfMembership.hpp"
+#include "Capability.hpp"
+#include "Tag.hpp"
 
 namespace ZeroTier {
 
@@ -106,7 +109,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,bool deferred)
 				case Packet::VERB_EXT_FRAME:                      return _doEXT_FRAME(RR,peer);
 				case Packet::VERB_ECHO:                           return _doECHO(RR,peer);
 				case Packet::VERB_MULTICAST_LIKE:                 return _doMULTICAST_LIKE(RR,peer);
-				case Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return _doNETWORK_MEMBERSHIP_CERTIFICATE(RR,peer);
+				case Packet::VERB_NETWORK_CREDENTIALS:            return _doNETWORK_CREDENTIALS(RR,peer);
 				case Packet::VERB_NETWORK_CONFIG_REQUEST:         return _doNETWORK_CONFIG_REQUEST(RR,peer);
 				case Packet::VERB_MULTICAST_GATHER:               return _doMULTICAST_GATHER(RR,peer);
 				case Packet::VERB_MULTICAST_FRAME:                return _doMULTICAST_FRAME(RR,peer);
@@ -155,22 +158,10 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				break;
 
 			case Packet::ERROR_IDENTITY_COLLISION:
-				if (RR->topology->isRoot(peer->identity()))
+				if (RR->topology->isUpstream(peer->identity()))
 					RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
 				break;
 
-			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
-				/* Note: certificates are public so it's safe to push them to anyone
-				 * who asks. */
-				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
-				if ((network)&&(network->hasConfig())&&(network->config().com)) {
-					Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
-					network->config().com.serialize(outp);
-					outp.armor(peer->key(),true);
-					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
-				}
-			}	break;
-
 			case Packet::ERROR_NETWORK_ACCESS_DENIED_: {
 				SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
 				if ((network)&&(network->controller() == peer->address()))
@@ -218,9 +209,13 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		uint64_t worldTimestamp = 0;
 		{
 			unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
-			if (ptr < size()) // ZeroTier One < 1.0.3 did not include physical destination address info
+
+			// Get external surface address if present (was not in old versions)
+			if (ptr < size())
 				ptr += externalSurfaceAddress.deserialize(*this,ptr);
-			if ((ptr + 16) <= size()) { // older versions also did not include World IDs or timestamps
+
+			// Get world ID and world timestamp if present (was not in old versions)
+			if ((ptr + 16) <= size()) {
 				worldId = at<uint64_t>(ptr); ptr += 8;
 				worldTimestamp = at<uint64_t>(ptr);
 			}
@@ -295,7 +290,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		}
 
 		if (externalSurfaceAddress)
-			RR->sa->iam(id.address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isRoot(id),RR->node->now());
+			RR->sa->iam(id.address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(id),RR->node->now());
 
 		Packet outp(id.address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_HELLO);
@@ -379,13 +374,15 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					return true;
 				}
 
-				const bool trusted = RR->topology->isRoot(peer->identity());
-
 				InetAddress externalSurfaceAddress;
 				unsigned int ptr = ZT_PROTO_VERB_HELLO__OK__IDX_REVISION + 2;
-				if (ptr < size()) // ZeroTier One < 1.0.3 did not include this field
+
+				// Get reported external surface address if present (was not on old versions)
+				if (ptr < size())
 					ptr += externalSurfaceAddress.deserialize(*this,ptr);
-				if ((trusted)&&((ptr + 2) <= size())) { // older versions also did not include this field, and right now we only use if from a root
+
+				// Handle world updates from root servers if present (was not on old versions)
+				if (((ptr + 2) <= size())&&(RR->topology->isRoot(peer->identity()))) {
 					World worldUpdate;
 					const unsigned int worldLen = at<uint16_t>(ptr); ptr += 2;
 					if (worldLen > 0) {
@@ -401,17 +398,13 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				peer->setRemoteVersion(vProto,vMajor,vMinor,vRevision);
 
 				if (externalSurfaceAddress)
-					RR->sa->iam(peer->address(),_localAddress,_remoteAddress,externalSurfaceAddress,trusted,RR->node->now());
+					RR->sa->iam(peer->address(),_localAddress,_remoteAddress,externalSurfaceAddress,RR->topology->isUpstream(peer->identity()),RR->node->now());
 			}	break;
 
 			case Packet::VERB_WHOIS: {
-				if (RR->topology->isRoot(peer->identity())) {
+				if (RR->topology->isUpstream(peer->identity())) {
 					const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY);
-					// Right now we can skip this since OK(WHOIS) is only accepted from
-					// roots. In the future it should be done if we query less trusted
-					// sources.
-					//if (id.locallyValidate())
-						RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr<Peer>(new Peer(RR,RR->identity,id))));
+					RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr<Peer>(new Peer(RR,RR->identity,id))));
 				}
 			} break;
 
@@ -544,7 +537,6 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 				if (!network->isAllowed(peer)) {
 					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
-					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
 				}
 
@@ -599,7 +591,6 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 
 				if (!network->isAllowed(peer)) {
 					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
-					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
 				}
 
@@ -704,20 +695,34 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 	return true;
 }
 
-bool IncomingPacket::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
+bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
 {
 	try {
 		CertificateOfMembership com;
+		Capability cap;
+		Tag tag;
 
-		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
-		while (ptr < size()) {
-			ptr += com.deserialize(*this,ptr);
+		unsigned int p = ZT_PACKET_IDX_PAYLOAD;
+		while ((p < size())&&((*this)[p])) {
+			p += com.deserialize(*this,p);
 			peer->validateAndSetNetworkMembershipCertificate(com.networkId(),com);
 		}
+		++p; // skip trailing 0 after COMs if present
+
+		if (p < size()) { // check if new capabilities and tags fields are present
+			const unsigned int numCapabilities = at<uint16_t>(p); p += 2;
+			for(unsigned int i=0;i<numCapabilities;++i) {
+				p += cap.deserialize(*this,p);
+			}
+			const unsigned int numTags = at<uint16_t>(p); p += 2;
+			for(unsigned int i=0;i<numTags;++i) {
+				p += tag.deserialize(*this,p);
+			}
+		}
 
-		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP);
 	} catch ( ... ) {
-		TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
+		TRACE("dropped NETWORK_CREDENTIALS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
 	return true;
 }
@@ -859,7 +864,6 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			// that cert might be what we needed.
 			if (!network->isAllowed(peer)) {
 				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
-				_sendErrorNeedCertificate(RR,peer,network->id());
 				return true;
 			}
 
@@ -1069,22 +1073,8 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		// into the one we send along to next hops.
 		const unsigned int lengthOfSignedPortionAndSignature = 29 + vlf;
 
-		// Get previous hop's credential, if any
-		const unsigned int previousHopCredentialLength = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 29 + vlf);
-		CertificateOfMembership previousHopCom;
-		if (previousHopCredentialLength >= 1) {
-			switch((*this)[ZT_PACKET_IDX_PAYLOAD + 31 + vlf]) {
-				case 0x01: { // network certificate of membership for previous hop
-					const unsigned int phcl = previousHopCom.deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 32 + vlf);
-					if (phcl != (previousHopCredentialLength - 1)) {
-						TRACE("dropped CIRCUIT_TEST from %s(%s): previous hop COM invalid (%u != %u)",source().toString().c_str(),_remoteAddress.toString().c_str(),phcl,(previousHopCredentialLength - 1));
-						return true;
-					}
-				}	break;
-				default: break;
-			}
-		}
-		vlf += previousHopCredentialLength;
+		// Add length of second "additional fields" section.
+		vlf += at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 29 + vlf);
 
 		// Check credentials (signature already verified)
 		NetworkConfig originatorCredentialNetworkConfig;
@@ -1166,13 +1156,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 		if (breadth > 0) {
 			Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST);
 			outp.append(field(ZT_PACKET_IDX_PAYLOAD,lengthOfSignedPortionAndSignature),lengthOfSignedPortionAndSignature);
-			const unsigned int previousHopCredentialPos = outp.size();
-			outp.append((uint16_t)0); // no previous hop credentials: default
-			if ((originatorCredentialNetworkConfig)&&(!originatorCredentialNetworkConfig.isPublic())&&(originatorCredentialNetworkConfig.com)) {
-				outp.append((uint8_t)0x01); // COM
-				originatorCredentialNetworkConfig.com.serialize(outp);
-				outp.setAt<uint16_t>(previousHopCredentialPos,(uint16_t)(outp.size() - (previousHopCredentialPos + 2)));
-			}
+			outp.append((uint16_t)0); // no additional fields
 			if (remainingHopsPtr < size())
 				outp.append(field(remainingHopsPtr,size() - remainingHopsPtr),size() - remainingHopsPtr);
 
@@ -1241,7 +1225,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 	try {
 		// If this were allowed from anyone, it would itself be a DOS vector. Right
 		// now we only allow it from roots and controllers of networks you have joined.
-		bool allowed = RR->topology->isRoot(peer->identity());
+		bool allowed = RR->topology->isUpstream(peer->identity());
 		if (!allowed) {
 			std::vector< SharedPtr<Network> > allNetworks(RR->node->allNetworks());
 			for(std::vector< SharedPtr<Network> >::const_iterator n(allNetworks.begin());n!=allNetworks.end();++n) {
@@ -1300,16 +1284,6 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 	return true;
 }
 
-bool IncomingPacket::_doREQUEST_OBJECT(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
-{
-	return true;
-}
-
-bool IncomingPacket::_doOBJECT_UPDATED(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
-{
-	return true;
-}
-
 void IncomingPacket::computeSalsa2012Sha512ProofOfWork(unsigned int difficulty,const void *challenge,unsigned int challengeLength,unsigned char result[16])
 {
 	unsigned char salsabuf[131072]; // 131072 == protocol constant, size of memory buffer for this proof of work function
@@ -1388,15 +1362,4 @@ bool IncomingPacket::testSalsa2012Sha512ProofOfWorkResult(unsigned int difficult
 	return true;
 }
 
-void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,uint64_t nwid)
-{
-	Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR);
-	outp.append((unsigned char)verb());
-	outp.append(packetId());
-	outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
-	outp.append(nwid);
-	outp.armor(peer->key(),true);
-	RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
-}
-
 } // namespace ZeroTier

node/IncomingPacket.hpp

@@ -172,7 +172,7 @@ private:
 	bool _doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
-	bool _doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
+	bool _doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
@@ -180,11 +180,6 @@ private:
 	bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
-	bool _doREQUEST_OBJECT(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
-	bool _doOBJECT_UPDATED(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
-
-	// Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to communicate
-	void _sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,uint64_t nwid);
 
 	uint64_t _receiveTime;
 	InetAddress _localAddress;

node/Membership.hpp

@@ -0,0 +1,154 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2016  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#ifndef ZT_MEMBERSHIP_HPP
+#define ZT_MEMBERSHIP_HPP
+
+#include <stdint.h>
+
+#include <utility>
+#include <algorithm>
+
+#include "Constants.hpp"
+#include "../include/ZeroTierOne.h"
+#include "CertificateOfMembership.hpp"
+#include "Capability.hpp"
+#include "Tag.hpp"
+#include "Hashtable.hpp"
+#include "NetworkConfig.hpp"
+
+namespace ZeroTier {
+
+class Peer;
+
+/**
+ * Information related to a peer's participation on a network
+ *
+ * This structure is not thread-safe and must be locked during use.
+ */
+class Membership
+{
+private:
+	struct TState
+	{
+		TState() : lastPushed(0),lastReceived(0) {}
+		// Last time we pushed this tag to this peer
+		uint64_t lastPushed;
+		// Last time we received this tag from this peer
+		uint64_t lastReceived;
+		// Tag from peer
+		Tag tag;
+	};
+
+	struct CState
+	{
+		CState() : lastPushed(0),lastReceived(0) {}
+		// Last time we pushed this capability to this peer
+		uint64_t lastPushed;
+		// Last time we received this capability from this peer
+		uint64_t lastReceived;
+		// Capability from peer
+		Capability cap;
+	};
+
+public:
+	Membership() :
+		_lastPushedCom(0),
+		_com(),
+		_caps(8),
+		_tags(8)
+	{
+	}
+
+	/**
+	 * Send COM and other credentials to this peer if needed
+	 *
+	 * This checks last pushed times for our COM and for other credentials and
+	 * sends VERB_NETWORK_CREDENTIALS if the recipient might need them.
+	 *
+	 * @param peer Peer that "owns" this membership
+	 * @param nconf Network configuration
+	 * @param now Current time
+	 * @param capIds Capability IDs that this peer might need
+	 * @param capCount Number of capability IDs
+	 * @param tagIds Tag IDs that this peer might need
+	 * @param tagCount Number of tag IDs
+	 */
+	void sendCredentialsIfNeeded(const Peer &peer,const NetworkConfig &nconf,const uint64_t now,const uint32_t *capIds,const unsigned int capCount,const uint32_t *tagIds,const unsigned int tagCount) const;
+
+	/**
+	 * @param nconf Network configuration
+	 * @param id Tag ID
+	 * @return Pointer to tag or NULL if not found
+	 */
+	inline const Tag *getTag(const NetworkConfig &nconf,const uint32_t id) const
+	{
+		const TState *t = _tags.get(id);
+		return ((t) ? (((t->lastReceived != 0)&&(t->tag.expiration() < nconf.timestamp)) ? &(t->tag) : (const Tag *)0) : (const Tag *)0);
+	}
+
+	/**
+	 * @param nconf Network configuration
+	 * @param id Capablity ID
+	 * @return Pointer to capability or NULL if not found
+	 */
+	inline const Capability *getCapability(const NetworkConfig &nconf,const uint32_t id) const
+	{
+		const CState *c = _caps.get(id);
+		return ((c) ? (((c->lastReceived != 0)&&(c->cap.expiration() < nconf.timestamp)) ? &(c->cap) : (const Capability *)0) : (const Capability *)0);
+	}
+
+	/**
+	 * Clean up old or stale entries
+	 */
+	inline void clean(const uint64_t now)
+	{
+		uint32_t *i = (uint32_t *)0;
+		CState *cs = (CState *)0;
+		Hashtable<uint32_t,CState>::Iterator csi(_caps);
+		while (csi.next(i,cs)) {
+			if ((now - std::max(cs->lastPushed,cs->lastReceived)) > (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 3))
+				_caps.erase(*i);
+		}
+
+		i = (uint32_t *)0;
+		TState *ts = (TState *)0;
+		Hashtable<uint32_t,TState>::Iterator tsi(_tags);
+		while (tsi.next(i,ts)) {
+			if ((now - std::max(ts->lastPushed,ts->lastReceived)) > (ZT_NETWORK_COM_DEFAULT_REVISION_MAX_DELTA * 3))
+				_tags.erase(*i);
+		}
+	}
+
+private:
+	// Last time we pushed our COM to this peer
+	uint64_t _lastPushedCom;
+
+	// COM from this peer
+	CertificateOfMembership _com;
+
+	// Capability-related state
+	Hashtable<uint32_t,CState> _caps;
+
+	// Tag-related state
+	Hashtable<uint32_t,TState> _tags;
+};
+
+} // namespace ZeroTier
+
+#endif

node/Network.cpp

@@ -216,6 +216,8 @@ void Network::requestConfiguration()
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MAJOR);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION,(uint64_t)ZEROTIER_ONE_VERSION_MINOR);
 	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION,(uint64_t)ZEROTIER_ONE_VERSION_REVISION);
+	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES,(uint64_t)ZT_MAX_NETWORK_RULES);
+	rmd.add(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES,(uint64_t)ZT_MAX_CAPABILITY_RULES);
 
 	if (controller() == RR->identity.address()) {
 		if (RR->localNetworkController) {

node/NetworkConfig.hpp

@@ -35,6 +35,8 @@
 #include "MulticastGroup.hpp"
 #include "Address.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Capability.hpp"
+#include "Tag.hpp"
 #include "Dictionary.hpp"
 
 /**
@@ -76,6 +78,8 @@ namespace ZeroTier {
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION "majv"
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MINOR_VERSION "minv"
 #define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_REVISION "revv"
+#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_NETWORK_RULES "Mr"
+#define ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_MAX_CAPABILITY_RULES "Mcr"
 
 // These dictionary keys are short so they don't take up much room.
 
@@ -288,6 +292,32 @@ public:
 	inline bool operator==(const NetworkConfig &nc) const { return (memcmp(this,&nc,sizeof(NetworkConfig)) == 0); }
 	inline bool operator!=(const NetworkConfig &nc) const { return (!(*this == nc)); }
 
+	/**
+	 * Add a specialist or mask flags if already present
+	 *
+	 * This masks the existing flags if the specialist is already here or adds
+	 * it otherwise.
+	 *
+	 * @param a Address of specialist
+	 * @param f Flags (OR of specialist role/type flags)
+	 * @return True if successfully masked or added
+	 */
+	inline bool addSpecialist(const Address &a,const uint64_t f)
+	{
+		const uint64_t aint = a.toInt();
+		for(unsigned int i=0;i<specialistCount;++i) {
+			if ((specialists[i] & 0xffffffffffULL) == aint) {
+				specialists[i] |= f;
+				return true;
+			}
+		}
+		if (specialistCount < ZT_MAX_NETWORK_SPECIALISTS) {
+			specialists[specialistCount++] = f | aint;
+			return true;
+		}
+		return false;
+	}
+
 	/*
 	inline void dump() const
 	{
@@ -316,32 +346,6 @@ public:
 	}
 	*/
 
-	/**
-	 * Add a specialist or mask flags if already present
-	 *
-	 * This masks the existing flags if the specialist is already here or adds
-	 * it otherwise.
-	 *
-	 * @param a Address of specialist
-	 * @param f Flags (OR of specialist role/type flags)
-	 * @return True if successfully masked or added
-	 */
-	inline bool addSpecialist(const Address &a,const uint64_t f)
-	{
-		const uint64_t aint = a.toInt();
-		for(unsigned int i=0;i<specialistCount;++i) {
-			if ((specialists[i] & 0xffffffffffULL) == aint) {
-				specialists[i] |= f;
-				return true;
-			}
-		}
-		if (specialistCount < ZT_MAX_NETWORK_SPECIALISTS) {
-			specialists[specialistCount++] = f | aint;
-			return true;
-		}
-		return false;
-	}
-
 	/**
 	 * Network ID that this configuration applies to
 	 */
@@ -397,6 +401,16 @@ public:
 	 */
 	unsigned int ruleCount;
 
+	/**
+	 * Number of capabilities
+	 */
+	unsigned int capabilityCount;
+
+	/**
+	 * Number of tags
+	 */
+	unsigned int tagCount;
+
 	/**
 	 * Specialist devices
 	 *
@@ -416,10 +430,20 @@ public:
 	InetAddress staticIps[ZT_MAX_ZT_ASSIGNED_ADDRESSES];
 
 	/**
-	 * Rules table
+	 * Base network rules
 	 */
 	ZT_VirtualNetworkRule rules[ZT_MAX_NETWORK_RULES];
 
+	/**
+	 * Capabilities for this node on this network
+	 */
+	Capability capabilities[ZT_MAX_NETWORK_CAPABILITIES];
+
+	/**
+	 * Tags for this node on this network
+	 */
+	Tag tags[ZT_MAX_NETWORK_TAGS];
+
 	/**
 	 * Network type (currently just public or private)
 	 */

node/Packet.cpp

@@ -38,7 +38,7 @@ const char *Packet::verbString(Verb v)
 		case VERB_EXT_FRAME: return "EXT_FRAME";
 		case VERB_ECHO: return "ECHO";
 		case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE";
-		case VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return "NETWORK_MEMBERSHIP_CERTIFICATE";
+		case VERB_NETWORK_CREDENTIALS: return "NETWORK_CREDENTIALS";
 		case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST";
 		case VERB_MULTICAST_GATHER: return "MULTICAST_GATHER";
 		case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME";
@@ -46,8 +46,6 @@ const char *Packet::verbString(Verb v)
 		case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST";
 		case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT";
 		case VERB_REQUEST_PROOF_OF_WORK: return "REQUEST_PROOF_OF_WORK";
-		case VERB_REQUEST_OBJECT: return "REQUEST_OBJECT";
-		case VERB_OBJECT_UPDATED: return "OBJECT_UPDATED";
 	}
 	return "(unknown)";
 }

node/Packet.hpp

@@ -51,19 +51,23 @@
  *   + Yet another multicast redesign
  *   + New crypto completely changes key agreement cipher
  * 4 - 0.6.0 ... 1.0.6
- *   + New identity format based on hashcash design
+ *   + BREAKING CHANGE: New identity format based on hashcash design
  * 5 - 1.1.0 ... 1.1.5
  *   + Supports circuit test, proof of work, and echo
  *   + Supports in-band world (root server definition) updates
  *   + Clustering! (Though this will work with protocol v4 clients.)
  *   + Otherwise backward compatible with protocol v4
  * 6 - 1.1.5 ... 1.1.10
- *   + Deprecate old dictionary-based network config format
- *   + Introduce new binary serialized network config and meta-data
- * 7 - 1.1.10 -- CURRENT
+ *   + Network configuration format revisions including binary values
+ * 7 - 1.1.10 -- 1.2.0
  *   + Introduce trusted paths for local SDN use
+ * 8 - 1.2.0  -- CURRENT
+ *   + Multipart network configurations for large network configs
+ *   + Tags and Capabilities
+ *   + Inline push of CertificateOfMembership deprecated
+ *   + Certificates of representation for federation and mesh
  */
-#define ZT_PROTO_VERSION 7
+#define ZT_PROTO_VERSION 8
 
 /**
  * Minimum supported protocol version
@@ -523,7 +527,7 @@ public:
 		/**
 		 * No operation (ignored, no reply)
 		 */
-		VERB_NOP = 0,
+		VERB_NOP = 0x00,
 
 		/**
 		 * Announcement of a node's existence:
@@ -566,7 +570,7 @@ public:
 		 *
 		 * ERROR has no payload.
 		 */
-		VERB_HELLO = 1,
+		VERB_HELLO = 0x01,
 
 		/**
 		 * Error response:
@@ -575,7 +579,7 @@ public:
 		 *   <[1] error code>
 		 *   <[...] error-dependent payload>
 		 */
-		VERB_ERROR = 2,
+		VERB_ERROR = 0x02,
 
 		/**
 		 * Success response:
@@ -583,7 +587,7 @@ public:
 		 *   <[8] in-re packet ID>
 		 *   <[...] request-specific payload>
 		 */
-		VERB_OK = 3,
+		VERB_OK = 0x03,
 
 		/**
 		 * Query an identity by address:
@@ -598,7 +602,7 @@ public:
 		 * If the address is not found, no response is generated. WHOIS requests
 		 * will time out much like ARP requests and similar do in L2.
 		 */
-		VERB_WHOIS = 4,
+		VERB_WHOIS = 0x04,
 
 		/**
 		 * Meet another node at a given protocol address:
@@ -626,7 +630,7 @@ public:
 		 *
 		 * No OK or ERROR is generated.
 		 */
-		VERB_RENDEZVOUS = 5,
+		VERB_RENDEZVOUS = 0x05,
 
 		/**
 		 * ZT-to-ZT unicast ethernet frame (shortened EXT_FRAME):
@@ -642,31 +646,29 @@ public:
 		 * ERROR may be generated if a membership certificate is needed for a
 		 * closed network. Payload will be network ID.
 		 */
-		VERB_FRAME = 6,
+		VERB_FRAME = 0x06,
 
 		/**
 		 * Full Ethernet frame with MAC addressing and optional fields:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
-		 *  [<[...] certificate of network membership>]
+		 *  [<[...] certificate of network membership (DEPRECATED)>]
 		 *   <[6] destination MAC or all zero for destination node>
 		 *   <[6] source MAC or all zero for node of origin>
 		 *   <[2] 16-bit ethertype>
 		 *   <[...] ethernet payload>
 		 *
 		 * Flags:
-		 *   0x01 - Certificate of network membership is attached
+		 *   0x01 - Certificate of network membership attached (DEPRECATED)
 		 *
 		 * An extended frame carries full MAC addressing, making them a
 		 * superset of VERB_FRAME. They're used for bridging or when we
 		 * want to attach a certificate since FRAME does not support that.
 		 *
-		 * Multicast frames may not be sent as EXT_FRAME.
-		 *
 		 * ERROR may be generated if a membership certificate is needed for a
 		 * closed network. Payload will be network ID.
 		 */
-		VERB_EXT_FRAME = 7,
+		VERB_EXT_FRAME = 0x07,
 
 		/**
 		 * ECHO request (a.k.a. ping):
@@ -676,7 +678,7 @@ public:
 		 * is generated. Response to ECHO requests is optional and ECHO may be
 		 * ignored if a node detects a possible flood.
 		 */
-		VERB_ECHO = 8,
+		VERB_ECHO = 0x08,
 
 		/**
 		 * Announce interest in multicast group(s):
@@ -690,45 +692,76 @@ public:
 		 * controllers and root servers. In the current network, root servers
 		 * will provide the service of final multicast cache.
 		 *
-		 * If sending LIKEs to root servers for backward compatibility reasons,
-		 * VERB_NETWORK_MEMBERSHIP_CERTIFICATE must be sent as well ahead of
-		 * time so that roots can authenticate GATHER requests.
+		 * VERB_NETWORK_CREDENTIALS should be pushed along with this, especially
+		 * if using upstream (e.g. root) nodes as multicast databases. This allows
+		 * GATHERs to be authenticated.
 		 *
 		 * OK/ERROR are not generated.
 		 */
-		VERB_MULTICAST_LIKE = 9,
+		VERB_MULTICAST_LIKE = 0x09,
 
 		/**
-		 * Network member certificate replication/push:
+		 * Network membership credential push:
 		 *   <[...] serialized certificate of membership>
-		 *   [ ... additional certificates may follow ...]
+		 *   [<[...] additional certificates of membership>]
+		 *   <[1] null byte for backward compatibility (see below)>
+		 *   <[2] 16-bit number of capabilities>
+		 *   <[...] one or more serialized Capability>
+		 *   <[2] 16-bit number of tags>
+		 *   <[...] one or more serialized Tags>
 		 *
 		 * This is sent in response to ERROR_NEED_MEMBERSHIP_CERTIFICATE and may
 		 * be pushed at any other time to keep exchanged certificates up to date.
 		 *
+		 * Protocol versions prior to 8 do not support capabilities or tags and
+		 * just expect an array of COMs. Adding a single NULL byte after the COM
+		 * array causes these older versions to harmlessly abort parsing and
+		 * ignore the newer fields. The new version checks for this null byte to
+		 * indicate the end of the COM array, since all serialized COMs begin with
+		 * non-zero bytes (see CertificateOfMembership).
+		 *
 		 * OK/ERROR are not generated.
 		 */
-		VERB_NETWORK_MEMBERSHIP_CERTIFICATE = 10,
+		VERB_NETWORK_CREDENTIALS = 0x0a,
 
 		/**
-		 * DEPRECATED but still supported, interpreted as an object request:
-		 *
-		 * /controller/network/<network ID>/member/<requester address>
+		 * Network configuration request:
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit length of request meta-data dictionary>
+		 *   <[...] string-serialized request meta-data>
+		 *  [<[8] 64-bit timestamp of netconf we currently have>]
 		 *
-		 * When received in this manner the response is sent via the old
-		 * OK(NETWORK_CONFIG_REQUEST) instead of OK(REQUEST_OBJECT). If the
-		 * response is too large, a dictionary is sent with the single key
-		 * OVF set to 1. In this case REQUEST_OBJECT must be used.
+		 * This message requests network configuration from a node capable of
+		 * providing it. If the optional revision is included, a response is
+		 * only generated if there is a newer network configuration available.
 		 *
 		 * OK response payload:
 		 *   <[8] 64-bit network ID>
-		 *   <[2] 16-bit length of network configuration dictionary>
-		 *   <[...] network configuration dictionary>
+		 *   <[2] 16-bit length of network configuration dictionary field>
+		 *   <[...] network configuration dictionary (or fragment)>
+		 *   [<[4] 32-bit total length of assembled dictionary>]
+		 *   [<[4] 32-bit index of fragment in this reply>]
+		 *
+		 * Fields after the dictionary are extensions to support multipart
+		 * sending of large network configs. If they are not present the
+		 * sent config must be assumed to be whole.
 		 *
 		 * ERROR response payload:
 		 *   <[8] 64-bit network ID>
 		 */
-		VERB_NETWORK_CONFIG_REQUEST = 11,
+		VERB_NETWORK_CONFIG_REQUEST = 0x0b,
+
+		/**
+		 * Network configuration refresh request:
+		 *   <[...] array of 64-bit network IDs>
+		 *
+		 * This can be sent by the network controller to inform a node that it
+		 * should now make a NETWORK_CONFIG_REQUEST.
+		 *
+		 * It does not generate an OK or ERROR message, and is treated only as
+		 * a hint to refresh now.
+		 */
+		VERB_NETWORK_CONFIG_REFRESH = 0x0c,
 
 		/**
 		 * Request endpoints for multicast distribution:
@@ -737,10 +770,10 @@ public:
 		 *   <[6] MAC address of multicast group being queried>
 		 *   <[4] 32-bit ADI for multicast group being queried>
 		 *   <[4] 32-bit requested max number of multicast peers>
-		 *  [<[...] network certificate of membership>]
+		 *  [<[...] network certificate of membership (DEPRECATED)>]
 		 *
 		 * Flags:
-		 *   0x01 - Network certificate of membership is attached
+		 *   0x01 - COM is attached (DEPRECATED)
 		 *
 		 * This message asks a peer for additional known endpoints that have
 		 * LIKEd a given multicast group. It's sent when the sender wishes
@@ -750,6 +783,9 @@ public:
 		 * More than one OK response can occur if the response is broken up across
 		 * multiple packets or if querying a clustered node.
 		 *
+		 * Send VERB_NETWORK_CREDENTIALS prior to GATHERing if doing so from
+		 * upstream nodes like root servers that are not involved in our network.
+		 *
 		 * OK response payload:
 		 *   <[8] 64-bit network ID>
 		 *   <[6] MAC address of multicast group being queried>
@@ -761,13 +797,13 @@ public:
 		 *
 		 * ERROR is not generated; queries that return no response are dropped.
 		 */
-		VERB_MULTICAST_GATHER = 13,
+		VERB_MULTICAST_GATHER = 0x0d,
 
 		/**
 		 * Multicast frame:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
-		 *  [<[...] network certificate of membership>]
+		 *  [<[...] network certificate of membership (DEPRECATED)>]
 		 *  [<[4] 32-bit implicit gather limit>]
 		 *  [<[6] source MAC>]
 		 *   <[6] destination MAC (multicast address)>
@@ -776,7 +812,7 @@ public:
 		 *   <[...] ethernet payload>
 		 *
 		 * Flags:
-		 *   0x01 - Network certificate of membership is attached
+		 *   0x01 - Network certificate of membership attached (DEPRECATED)
 		 *   0x02 - Implicit gather limit field is present
 		 *   0x04 - Source MAC is specified -- otherwise it's computed from sender
 		 *
@@ -791,11 +827,11 @@ public:
 		 *   <[6] MAC address of multicast group>
 		 *   <[4] 32-bit ADI for multicast group>
 		 *   <[1] flags>
-		 *  [<[...] network certficate of membership>]
+		 *  [<[...] network certficate of membership (DEPRECATED)>]
 		 *  [<[...] implicit gather results if flag 0x01 is set>]
 		 *
 		 * OK flags (same bits as request flags):
-		 *   0x01 - OK includes certificate of network membership
+		 *   0x01 - OK includes certificate of network membership (DEPRECATED)
 		 *   0x02 - OK includes implicit gather results
 		 *
 		 * ERROR response payload:
@@ -803,7 +839,9 @@ public:
 		 *   <[6] multicast group MAC>
 		 *   <[4] 32-bit multicast group ADI>
 		 */
-		VERB_MULTICAST_FRAME = 14,
+		VERB_MULTICAST_FRAME = 0x0e,
+
+		// 0x0f is reserved for an old deprecated message
 
 		/**
 		 * Push of potential endpoints for direct communication:
@@ -839,7 +877,7 @@ public:
 		 *
 		 * OK and ERROR are not generated.
 		 */
-		VERB_PUSH_DIRECT_PATHS = 16,
+		VERB_PUSH_DIRECT_PATHS = 0x10,
 
 		/**
 		 * Source-routed circuit test message:
@@ -855,9 +893,8 @@ public:
 		 *   [ ... end of signed portion of request ... ]
 		 *   <[2] 16-bit length of signature of request>
 		 *   <[...] signature of request by originator>
-		 *   <[2] 16-bit previous hop credential length (including type)>
-		 *   [[1] previous hop credential type]
-		 *   [[...] previous hop credential]
+		 *   <[2] 16-bit length of additional fields>
+		 *   [[...] additional fields]
 		 *   <[...] next hop(s) in path>
 		 *
 		 * Flags:
@@ -867,9 +904,6 @@ public:
 		 * Originator credential types:
 		 *   0x01 - 64-bit network ID for which originator is controller
 		 *
-		 * Previous hop credential types:
-		 *   0x01 - Certificate of network membership
-		 *
 		 * Path record format:
 		 *   <[1] 8-bit flags (unused, must be zero)>
 		 *   <[1] 8-bit breadth (number of next hops)>
@@ -918,7 +952,7 @@ public:
 		 *   <[8] 64-bit timestamp (echoed from original>
 		 *   <[8] 64-bit test ID (echoed from original)>
 		 */
-		VERB_CIRCUIT_TEST = 17,
+		VERB_CIRCUIT_TEST = 0x11,
 
 		/**
 		 * Circuit test hop report:
@@ -955,7 +989,7 @@ public:
 		 * If a test report is received and no circuit test was sent, it should be
 		 * ignored. This message generates no OK or ERROR response.
 		 */
-		VERB_CIRCUIT_TEST_REPORT = 18,
+		VERB_CIRCUIT_TEST_REPORT = 0x12,
 
 		/**
 		 * Request proof of work:
@@ -998,63 +1032,7 @@ public:
 		 *
 		 * ERROR has no payload.
 		 */
-		VERB_REQUEST_PROOF_OF_WORK = 19,
-
-		/**
-		 * Request an object or a chunk of an object with optional meta-data:
-		 *   <[8] 64-bit chunk offset>
-		 *   <[2] 16-bit chunk length or 0 for any / sender-preferred>
-		 *   <[2] 16-bit object path length in bytes>
-		 *   <[...] object path>
-		 *   <[2] 16-bit length of request meta-data dictionary>
-		 *   <[...] request meta-data dictionary>
-		 *
-		 * This is used to request an object. Objects can be things like network
-		 * configs, software updates, etc. This provides an in-band way to
-		 * distribute such things and obsoletes the network config specific
-		 * messages. (They are still supported for backward compatibility.)
-		 *
-		 * The use of path and request/response meta-data makes the semantics of
-		 * this analogous to HTTP POST, and it could therefore be mapped to
-		 * HTTP POST requests to permit plugins that leverage the ZT protocol
-		 * to do out-of-band things like special authentication, etc.
-		 *
-		 * Large objects can be transferred via repeated calls with higher and
-		 * higher chunk offsets and then SHA-512 verified on receipt, but this is
-		 * not efficient. It should not be used heavily as an alternative to
-		 * TCP. It's a bit more like X-Modem and other old-school SEND/ACK
-		 * protocols. It is potentially a good idea for software updates since
-		 * it means that ZT can update itself even on networks with no "vanilla"
-		 * Internet access.
-		 *
-		 * OK and ERROR responses are optional but recommended. ERROR responses
-		 * can include OBJECT_NOT_FOUND.
-		 *
-		 * OK response payload:
-		 *   <[16] first 16 bytes of SHA-512 of complete object>
-		 *   <[8] 64-bit total object size>
-		 *   <[8] 64-bit chunk offset>
-		 *   <[2] 16-bit length of chunk payload>
-		 *   <[...] chunk payload>
-		 */
-		VERB_REQUEST_OBJECT = 20,
-
-		/**
-		 * Notification of a remote object update:
-		 *   <[8] 64-bit total object size or 0 if unspecified here>
-		 *   <[16] first 16 bytes of SHA-512 of object (if size specified)>
-		 *   <[2] 16-bit length of object path>
-		 *   <[...] object path>
-		 *   <[2] 16-bit length of meta-data dictionary>
-		 *   <[...] meta-data dictionary>
-		 *
-		 * This can be sent to notify another peer that an object has updated and
-		 * should be re-requested. The receiving peer is not required to do anything
-		 * or send anything in response to this. If the first size field is zero, the
-		 * SHA-512 hash is also unspecified and should be zero. This means that the
-		 * object was updated but must be re-requested.
-		 */
-		VERB_OBJECT_UPDATED = 21
+		VERB_REQUEST_PROOF_OF_WORK = 0x13
 	};
 
 	/**
@@ -1063,31 +1041,28 @@ public:
 	enum ErrorCode
 	{
 		/* No error, not actually used in transit */
-		ERROR_NONE = 0,
+		ERROR_NONE = 0x00,
 
 		/* Invalid request */
-		ERROR_INVALID_REQUEST = 1,
+		ERROR_INVALID_REQUEST = 0x01,
 
 		/* Bad/unsupported protocol version */
-		ERROR_BAD_PROTOCOL_VERSION = 2,
+		ERROR_BAD_PROTOCOL_VERSION = 0x02,
 
 		/* Unknown object queried */
-		ERROR_OBJ_NOT_FOUND = 3,
+		ERROR_OBJ_NOT_FOUND = 0x03,
 
 		/* HELLO pushed an identity whose address is already claimed */
-		ERROR_IDENTITY_COLLISION = 4,
+		ERROR_IDENTITY_COLLISION = 0x04,
 
 		/* Verb or use case not supported/enabled by this node */
-		ERROR_UNSUPPORTED_OPERATION = 5,
-
-		/* Message to private network rejected -- no unexpired certificate on file */
-		ERROR_NEED_MEMBERSHIP_CERTIFICATE = 6,
+		ERROR_UNSUPPORTED_OPERATION = 0x05,
 
 		/* Tried to join network, but you're not a member */
-		ERROR_NETWORK_ACCESS_DENIED_ = 7, /* extra _ to avoid Windows name conflict */
+		ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */
 
 		/* Multicasts to this group are not wanted */
-		ERROR_UNWANTED_MULTICAST = 8
+		ERROR_UNWANTED_MULTICAST = 0x08
 	};
 
 //#ifdef ZT_TRACE

node/Peer.hpp

@@ -31,7 +31,6 @@
 #include "../include/ZeroTierOne.h"
 
 #include "RuntimeEnvironment.hpp"
-#include "CertificateOfMembership.hpp"
 #include "Path.hpp"
 #include "Address.hpp"
 #include "Utils.hpp"
@@ -44,10 +43,6 @@
 #include "Mutex.hpp"
 #include "NonCopyable.hpp"
 
-// Very rough computed estimate: (8 + 256 + 80 + (16 * 64) + (128 * 256) + (128 * 16))
-// 1048576 provides tons of headroom -- overflow would just cause peer not to be persisted
-#define ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE 1048576
-
 namespace ZeroTier {
 
 /**
@@ -362,31 +357,6 @@ public:
 	 */
 	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
 
-	/**
-	 * Check network COM agreement with this peer
-	 *
-	 * @param nwid Network ID
-	 * @param com Another certificate of membership
-	 * @return True if supplied COM agrees with ours, false if not or if we don't have one
-	 */
-	bool networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const;
-
-	/**
-	 * Check the validity of the COM and add/update if valid and new
-	 *
-	 * @param nwid Network ID
-	 * @param com Externally supplied COM
-	 */
-	bool validateAndSetNetworkMembershipCertificate(uint64_t nwid,const CertificateOfMembership &com);
-
-	/**
-	 * @param nwid Network ID
-	 * @param now Current time
-	 * @param updateLastPushedTime If true, go ahead and update the last pushed time regardless of return value
-	 * @return Whether or not this peer needs another COM push from us
-	 */
-	bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime);
-
 	/**
 	 * Perform periodic cleaning operations
 	 *
@@ -434,138 +404,12 @@ public:
 		else return std::pair<InetAddress,InetAddress>();
 	}
 
-	template<unsigned int C>
-	inline void serialize(Buffer<C> &b) const
-	{
-		Mutex::Lock _l(_networkComs_m);
-
-		const unsigned int recSizePos = b.size();
-		b.addSize(4); // space for uint32_t field length
-
-		b.append((uint16_t)1); // version of serialized Peer data
-
-		_id.serialize(b,false);
-
-		b.append((uint64_t)_lastUsed);
-		b.append((uint64_t)_lastReceive);
-		b.append((uint64_t)_lastUnicastFrame);
-		b.append((uint64_t)_lastMulticastFrame);
-		b.append((uint64_t)_lastAnnouncedTo);
-		b.append((uint64_t)_lastDirectPathPushSent);
-		b.append((uint64_t)_lastDirectPathPushReceive);
-		b.append((uint64_t)_lastPathSort);
-		b.append((uint16_t)_vProto);
-		b.append((uint16_t)_vMajor);
-		b.append((uint16_t)_vMinor);
-		b.append((uint16_t)_vRevision);
-		b.append((uint32_t)_latency);
-		b.append((uint16_t)_directPathPushCutoffCount);
-
-		b.append((uint16_t)_numPaths);
-		for(unsigned int i=0;i<_numPaths;++i)
-			_paths[i].serialize(b);
-
-		b.append((uint32_t)_networkComs.size());
-		{
-			uint64_t *k = (uint64_t *)0;
-			_NetworkCom *v = (_NetworkCom *)0;
-			Hashtable<uint64_t,_NetworkCom>::Iterator i(const_cast<Peer *>(this)->_networkComs);
-			while (i.next(k,v)) {
-				b.append((uint64_t)*k);
-				b.append((uint64_t)v->ts);
-				v->com.serialize(b);
-			}
-		}
-
-		b.append((uint32_t)_lastPushedComs.size());
-		{
-			uint64_t *k = (uint64_t *)0;
-			uint64_t *v = (uint64_t *)0;
-			Hashtable<uint64_t,uint64_t>::Iterator i(const_cast<Peer *>(this)->_lastPushedComs);
-			while (i.next(k,v)) {
-				b.append((uint64_t)*k);
-				b.append((uint64_t)*v);
-			}
-		}
-
-		b.template setAt<uint32_t>(recSizePos,(uint32_t)(b.size() - (recSizePos + 4))); // set size
-	}
-
-	/**
-	 * Create a new Peer from a serialized instance
-	 *
-	 * @param renv Runtime environment
-	 * @param myIdentity This node's identity
-	 * @param b Buffer containing serialized Peer data
-	 * @param p Pointer to current position in buffer, will be updated in place as buffer is read (value/result)
-	 * @return New instance of Peer or NULL if serialized data was corrupt or otherwise invalid (may also throw an exception via Buffer)
-	 */
-	template<unsigned int C>
-	static inline SharedPtr<Peer> deserializeNew(const RuntimeEnvironment *renv,const Identity &myIdentity,const Buffer<C> &b,unsigned int &p)
-	{
-		const unsigned int recSize = b.template at<uint32_t>(p); p += 4;
-		if ((p + recSize) > b.size())
-			return SharedPtr<Peer>(); // size invalid
-		if (b.template at<uint16_t>(p) != 1)
-			return SharedPtr<Peer>(); // version mismatch
-		p += 2;
-
-		Identity npid;
-		p += npid.deserialize(b,p);
-		if (!npid)
-			return SharedPtr<Peer>();
-
-		SharedPtr<Peer> np(new Peer(renv,myIdentity,npid));
-
-		np->_lastUsed = b.template at<uint64_t>(p); p += 8;
-		np->_lastReceive = b.template at<uint64_t>(p); p += 8;
-		np->_lastUnicastFrame = b.template at<uint64_t>(p); p += 8;
-		np->_lastMulticastFrame = b.template at<uint64_t>(p); p += 8;
-		np->_lastAnnouncedTo = b.template at<uint64_t>(p); p += 8;
-		np->_lastDirectPathPushSent = b.template at<uint64_t>(p); p += 8;
-		np->_lastDirectPathPushReceive = b.template at<uint64_t>(p); p += 8;
-		np->_lastPathSort = b.template at<uint64_t>(p); p += 8;
-		np->_vProto = b.template at<uint16_t>(p); p += 2;
-		np->_vMajor = b.template at<uint16_t>(p); p += 2;
-		np->_vMinor = b.template at<uint16_t>(p); p += 2;
-		np->_vRevision = b.template at<uint16_t>(p); p += 2;
-		np->_latency = b.template at<uint32_t>(p); p += 4;
-		np->_directPathPushCutoffCount = b.template at<uint16_t>(p); p += 2;
-
-		const unsigned int numPaths = b.template at<uint16_t>(p); p += 2;
-		for(unsigned int i=0;i<numPaths;++i) {
-			if (i < ZT_MAX_PEER_NETWORK_PATHS) {
-				p += np->_paths[np->_numPaths++].deserialize(b,p);
-			} else {
-				// Skip any paths beyond max, but still read stream
-				Path foo;
-				p += foo.deserialize(b,p);
-			}
-		}
-
-		const unsigned int numNetworkComs = b.template at<uint32_t>(p); p += 4;
-		for(unsigned int i=0;i<numNetworkComs;++i) {
-			_NetworkCom &c = np->_networkComs[b.template at<uint64_t>(p)]; p += 8;
-			c.ts = b.template at<uint64_t>(p); p += 8;
-			p += c.com.deserialize(b,p);
-		}
-
-		const unsigned int numLastPushed = b.template at<uint32_t>(p); p += 4;
-		for(unsigned int i=0;i<numLastPushed;++i) {
-			const uint64_t nwid = b.template at<uint64_t>(p); p += 8;
-			const uint64_t ts = b.template at<uint64_t>(p); p += 8;
-			np->_lastPushedComs.set(nwid,ts);
-		}
-
-		return np;
-	}
-
 private:
 	void _doDeadPathDetection(Path &p,const uint64_t now);
 	Path *_getBestPath(const uint64_t now);
 	Path *_getBestPath(const uint64_t now,int inetAddressFamily);
 
-	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized
+	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 
 	const RuntimeEnvironment *RR;
 	uint64_t _lastUsed;
@@ -586,17 +430,6 @@ private:
 	unsigned int _latency;
 	unsigned int _directPathPushCutoffCount;
 
-	struct _NetworkCom
-	{
-		_NetworkCom() {}
-		_NetworkCom(uint64_t t,const CertificateOfMembership &c) : ts(t),com(c) {}
-		uint64_t ts;
-		CertificateOfMembership com;
-	};
-	Hashtable<uint64_t,_NetworkCom> _networkComs;
-	Hashtable<uint64_t,uint64_t> _lastPushedComs;
-	Mutex _networkComs_m;
-
 	AtomicCounter __refCount;
 };
 

node/Topology.cpp

@@ -47,36 +47,7 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 	_trustedPathCount(0),
 	_amRoot(false)
 {
-	std::string alls(RR->node->dataStoreGet("peers.save"));
-	const uint8_t *all = reinterpret_cast<const uint8_t *>(alls.data());
-	RR->node->dataStoreDelete("peers.save");
-
-	Buffer<ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE> *deserializeBuf = new Buffer<ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE>();
-	unsigned int ptr = 0;
-	while ((ptr + 4) < alls.size()) {
-		try {
-			const unsigned int reclen = ( // each Peer serialized record is prefixed by a record length
-					((((unsigned int)all[ptr]) & 0xff) << 24) |
-					((((unsigned int)all[ptr + 1]) & 0xff) << 16) |
-					((((unsigned int)all[ptr + 2]) & 0xff) << 8) |
-					(((unsigned int)all[ptr + 3]) & 0xff)
-				);
-			unsigned int pos = 0;
-			deserializeBuf->copyFrom(all + ptr,reclen + 4);
-			SharedPtr<Peer> p(Peer::deserializeNew(RR,RR->identity,*deserializeBuf,pos));
-			ptr += pos;
-			if (!p)
-				break; // stop if invalid records
-			if (p->address() != RR->identity.address())
-				_peers.set(p->address(),p);
-		} catch ( ... ) {
-			break; // stop if invalid records
-		}
-	}
-	delete deserializeBuf;
-
-	clean(RR->node->now());
-
+	// Get cached world if present
 	std::string dsWorld(RR->node->dataStoreGet("world"));
 	World cachedWorld;
 	if (dsWorld.length() > 0) {
@@ -87,6 +58,8 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 			cachedWorld = World(); // clear if cached world is invalid
 		}
 	}
+
+	// Use default or cached world depending on which is shinier
 	World defaultWorld;
 	{
 		Buffer<ZT_DEFAULT_WORLD_LENGTH> wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH);

service/OneService.cpp

@@ -705,6 +705,9 @@ public:
 			}
 			authToken = _trimString(authToken);
 
+			// Clean up any legacy files if present
+			OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + "peers.save").c_str());
+
 			_node = new Node(
 				OSUtils::now(),
 				this,