Bunch more work on DNS, cleanup, removal of obsolete cruft.

controller/EmbeddedNetworkController.cpp

@@ -1126,56 +1126,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
 	return 404;
 }
 
-void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt)
-{
-	static volatile unsigned long idCounter = 0;
-	char id[128],tmp[128];
-	std::string k,v;
-
-	try {
-		// Convert Dictionary into JSON object
-		json d;
-		char *saveptr = (char *)0;
-		for(char *l=Utils::stok(rt.data,"\n",&saveptr);(l);l=Utils::stok((char *)0,"\n",&saveptr)) {
-			char *eq = strchr(l,'=');
-			if (eq > l) {
-				k.assign(l,(unsigned long)(eq - l));
-				v.clear();
-				++eq;
-				while (*eq) {
-					if (*eq == '\\') {
-						++eq;
-						if (*eq) {
-							switch(*eq) {
-								case 'r': v.push_back('\r'); break;
-								case 'n': v.push_back('\n'); break;
-								case '0': v.push_back((char)0); break;
-								case 'e': v.push_back('='); break;
-								default: v.push_back(*eq); break;
-							}
-							++eq;
-						}
-					} else {
-						v.push_back(*(eq++));
-					}
-				}
-				if ((k.length() > 0)&&(v.length() > 0))
-					d[k] = v;
-			}
-		}
-
-		const int64_t now = OSUtils::now();
-		OSUtils::ztsnprintf(id,sizeof(id),"%.10llx-%.16llx-%.10llx-%.4x",_signingId.address().toInt(),now,rt.origin,(unsigned int)(idCounter++ & 0xffff));
-		d["id"] = id;
-		d["objtype"] = "trace";
-		d["ts"] = now;
-		d["nodeId"] = Utils::hex10(rt.origin,tmp);
-		_db.save(d,true);
-	} catch ( ... ) {
-		// drop invalid trace messages if an error occurs
-	}
-}
-
 void EmbeddedNetworkController::onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network)
 {
 	// Send an update to all members of the network that are online
@@ -1650,7 +1600,7 @@ void EmbeddedNetworkController::_request(
 					if ((ipRangeEnd < ipRangeStart)||(ipRangeStart == 0))
 						continue;
 					uint32_t ipRangeLen = ipRangeEnd - ipRangeStart;
-					
+
 					// Start with the LSB of the member's address
 					uint32_t ipTrialCounter = (uint32_t)(identity.address().toInt() & 0xffffffff);
 

controller/EmbeddedNetworkController.hpp

@@ -101,8 +101,6 @@ public:
 		std::string &responseBody,
 		std::string &responseContentType);
 
-	void handleRemoteTrace(const ZT_RemoteTrace &rt);
-
 	virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network);
 	virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member);
 	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);

include/ZeroTierOne.h

@@ -34,7 +34,7 @@
 
 #include <stdint.h>
 
-// For the struct sockaddr_storage structure
+/* For struct sockaddr_storage, which is referenced here. */
 #if defined(_WIN32) || defined(_WIN64)
 #include <WinSock2.h>
 #include <WS2tcpip.h>
@@ -46,6 +46,8 @@
 #include <sys/socket.h>
 #endif /* Windows or not */
 
+/* This symbol may be defined in a build environment or before including this
+ * header if you need to prepend something to function specifications. */
 #ifndef ZT_SDK_API
 #define ZT_SDK_API
 #endif
@@ -113,25 +115,20 @@ extern "C" {
 #define ZT_MAX_NETWORK_SHORT_NAME_LENGTH 127
 
 /**
- * Maximum number of pushed routes on a network
+ * Maximum number of pushed routes on a network (via ZT in-band mechanisms)
  */
-#define ZT_MAX_NETWORK_ROUTES 32
+#define ZT_MAX_NETWORK_ROUTES 64
 
 /**
- * Maximum number of statically assigned IP addresses per network endpoint using ZT address management (not DHCP)
+ * Maximum number of statically assigned IP addresses (via ZT in-band mechanisms)
  */
-#define ZT_MAX_ZT_ASSIGNED_ADDRESSES 16
+#define ZT_MAX_ZT_ASSIGNED_ADDRESSES 32
 
 /**
- * Maximum number of "specialists" on a network -- bridges, relays, etc.
+ * Maximum number of "specialists" on a network -- bridges, anchors, etc.
  */
 #define ZT_MAX_NETWORK_SPECIALISTS 256
 
-/**
- * Maximum number of multicast group subscriptions per network
- */
-#define ZT_MAX_NETWORK_MULTICAST_SUBSCRIPTIONS 2048
-
 /**
  * Rules engine revision ID, which specifies rules engine capabilities
  */
@@ -143,12 +140,12 @@ extern "C" {
 #define ZT_MAX_NETWORK_RULES 1024
 
 /**
- * Maximum number of per-member capabilities per network
+ * Maximum number of capabilities per network per member
  */
 #define ZT_MAX_NETWORK_CAPABILITIES 128
 
 /**
- * Maximum number of per-member tags per network
+ * Maximum number of tags per network per member
  */
 #define ZT_MAX_NETWORK_TAGS 128
 
@@ -163,7 +160,7 @@ extern "C" {
 #define ZT_MAX_CONFIGURABLE_PATHS 32
 
 /**
- * Maximum number of rules per capability
+ * Maximum number of rules per capability object
  */
 #define ZT_MAX_CAPABILITY_RULES 64
 
@@ -178,14 +175,16 @@ extern "C" {
 #define ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH 7
 
 /**
- * Maximum number of multicast groups a device / network interface can be subscribed to at once
+ * Maximum number of multicast group subscriptions on a local virtual network interface
  */
 #define ZT_MAX_MULTICAST_SUBSCRIPTIONS 1024
 
 /**
  * Maximum value for link quality (min is 0)
  */
-#define ZT_PATH_LINK_QUALITY_MAX 0xff
+#define ZT_PATH_LINK_QUALITY_MAX 255
+
+/* Rule specification contants **********************************************/
 
 /**
  * Packet characteristics flag: packet direction, 1 if inbound 0 if outbound
@@ -272,6 +271,8 @@ extern "C" {
  */
 #define ZT_RULE_PACKET_CHARACTERISTICS_TCP_FIN 0x0000000000000001ULL
 
+/****************************************************************************/
+
 // Fields in remote trace dictionaries
 #define ZT_REMOTE_TRACE_FIELD__EVENT "event"
 #define ZT_REMOTE_TRACE_FIELD__NODE_ID "nodeId"
@@ -402,6 +403,8 @@ enum ZT_ResultCode
 };
 
 /**
+ * Macro to check for a fatal error result code
+ *
  * @param x Result code
  * @return True if result code indicates a fatal error
  */
@@ -452,14 +455,21 @@ enum ZT_Event
 	ZT_EVENT_UP = 0,
 
 	/**
-	 * Node is offline -- network does not seem to be reachable by any available strategy
+	 * Node appears offline
+	 *
+	 * This indicates that the node doesn't seem to be able to reach anything,
+	 * or hasn't for a while. It's not a hard instantaneous thing.
 	 *
 	 * Meta-data: none
 	 */
 	ZT_EVENT_OFFLINE = 1,
 
 	/**
-	 * Node is online -- at least one upstream node appears reachable
+	 * Node appears online
+	 *
+	 * This indicates that the node was offline but now seems to be able to
+	 * reach something. Like OFFLINE it's not a hard instantaneous thing but
+	 * more of an indicator for UI reporting purposes.
 	 *
 	 * Meta-data: none
 	 */
@@ -469,49 +479,20 @@ enum ZT_Event
 	 * Node is shutting down
 	 *
 	 * This is generated within Node's destructor when it is being shut down.
-	 * It's done for convenience, since cleaning up other state in the event
-	 * handler may appear more idiomatic.
+	 * It's done for convenience in case you want to clean up anything during
+	 * node shutdown in your node event handler.
 	 *
 	 * Meta-data: none
 	 */
 	ZT_EVENT_DOWN = 3,
 
-	/**
-	 * Your identity has collided with another node's ZeroTier address
-	 *
-	 * This happens if two different public keys both hash (via the algorithm
-	 * in Identity::generate()) to the same 40-bit ZeroTier address.
-	 *
-	 * This is something you should "never" see, where "never" is defined as
-	 * once per 2^39 new node initializations / identity creations. If you do
-	 * see it, you're going to see it very soon after a node is first
-	 * initialized.
-	 *
-	 * This is reported as an event rather than a return code since it's
-	 * detected asynchronously via error messages from authoritative nodes.
-	 *
-	 * If this occurs, you must shut down and delete the node, delete the
-	 * identity.secret record/file from the data store, and restart to generate
-	 * a new identity. If you don't do this, you will not be able to communicate
-	 * with other nodes.
-	 *
-	 * We'd automate this process, but we don't think silently deleting
-	 * private keys or changing our address without telling the calling code
-	 * is good form. It violates the principle of least surprise.
-	 *
-	 * You can technically get away with not handling this, but we recommend
-	 * doing so in a mature reliable application. Besides, handling this
-	 * condition is a good way to make sure it never arises. It's like how
-	 * umbrellas prevent rain and smoke detectors prevent fires. They do, right?
-	 *
-	 * Meta-data: none
-	 */
-	ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION = 4,
+	// 4 once signaled identity collision but this is no longer an error
 
 	/**
 	 * Trace (debugging) message
 	 *
 	 * These events are only generated if this is a TRACE-enabled build.
+	 * This is for local debug traces, not remote trace diagnostics.
 	 *
 	 * Meta-data: C string, TRACE message
 	 */
@@ -521,7 +502,13 @@ enum ZT_Event
 	 * VERB_USER_MESSAGE received
 	 *
 	 * These are generated when a VERB_USER_MESSAGE packet is received via
-	 * ZeroTier VL1.
+	 * ZeroTier VL1. This can be used for below-VL2 in-band application
+	 * specific signaling over the ZeroTier protocol.
+	 *
+	 * It's up to you to ensure that you handle these in a way that does
+	 * not introduce a remote security vulnerability into your app! If
+	 * your USER_MESSAGE code has a buffer overflow or other vulnerability
+	 * then your app will be vulnerable and this is not ZT's fault. :)
 	 *
 	 * Meta-data: ZT_UserMessage structure
 	 */
@@ -530,12 +517,9 @@ enum ZT_Event
 	/**
 	 * Remote trace received
 	 *
-	 * These are generated when a VERB_REMOTE_TRACE is received. Note
-	 * that any node can fling one of these at us. It is your responsibility
-	 * to filter and determine if it's worth paying attention to. If it's
-	 * not just drop it. Most nodes that are not active controllers ignore
-	 * these, and controllers only save them if they pertain to networks
-	 * with remote tracing enabled.
+	 * NOTE: any node can fling a VERB_REMOTE_TRACE at you. It's up to you
+	 * to determine if you want to do anything with it or just silently
+	 * drop it on the floor. It's also up to you to handle these securely!
 	 *
 	 * Meta-data: ZT_RemoteTrace structure
 	 */
@@ -548,7 +532,7 @@ enum ZT_Event
 typedef struct
 {
 	/**
-	 * ZeroTier address of sender
+	 * ZeroTier address of sender (in least significant 40 bits only)
 	 */
 	uint64_t origin;
 
@@ -563,10 +547,10 @@ typedef struct
 	 *
 	 * The contents of data[] may be modified.
 	 */
-	char *data;
+	const char *data;
 
 	/**
-	 * Length of dict[] in bytes, including terminating null
+	 * Length of dict[] in bytes, INCLUDING terminating null
 	 */
 	unsigned int len;
 } ZT_RemoteTrace;
@@ -634,24 +618,6 @@ typedef struct
 	int online;
 } ZT_NodeStatus;
 
-/**
- * Internal node statistics
- * 
- * This structure is subject to change between versions.
- */
-typedef struct
-{
-	/**
-	 * Number of each protocol verb (possible verbs 0..31) received
-	 */
-	uint64_t inVerbCounts[32];
-
-	/**
-	 * Number of bytes for each protocol verb received
-	 */
-	uint64_t inVerbBytes[32];
-} ZT_NodeStatistics;
-
 /**
  * Virtual network status codes
  */
@@ -1084,6 +1050,29 @@ enum ZT_Architecture
 	ZT_ARCHITECTURE_S390X = 16
 };
 
+/**
+ * DNS record types for reporting DNS results
+ *
+ * These integer IDs (other than end of results) are the same as the DNS protocol's
+ * internal IDs. Not all of these are used by ZeroTier, and not all DNS record types
+ * are listed here. These are just common ones that are used now or may be used in
+ * the future for some purpose.
+ */
+enum ZT_DNSRecordType
+{
+	ZT_DNS_RECORD__END_OF_RESULTS = 0,
+	ZT_DNS_RECORD_A = 1,
+	ZT_DNS_RECORD_NS = 2,
+	ZT_DNS_RECORD_CNAME = 5,
+	ZT_DNS_RECORD_PTR = 12,
+	ZT_DNS_RECORD_MX = 15,
+	ZT_DNS_RECORD_TXT = 16,
+	ZT_DNS_RECORD_AAAA = 28,
+	ZT_DNS_RECORD_LOC = 29,
+	ZT_DNS_RECORD_SRV = 33,
+	ZT_DNS_RECORD_DNAME = 39
+};
+
 /**
  * Virtual network configuration
  */
@@ -1604,6 +1593,50 @@ typedef int (*ZT_PathLookupFunction)(
 	int,                              /* Desired ss_family or -1 for any */
 	struct sockaddr_storage *);       /* Result buffer */
 
+/**
+ * Function to request an asynchronous DNS TXT lookup
+ *
+ * Parameters:
+ *  (1) Node
+ *  (2) User pointer
+ *  (3) Thread pointer
+ *  (4) Array of DNS record types we want
+ *  (5) Number of DNS record types in array
+ *  (6) DNS name to fetch
+ *  (7) DNS request ID to supply to ZT_Node_processDNSResult()
+ *
+ * DNS is not handled in the core because every platform and runtime
+ * typically has its own DNS functions or libraries and these may need
+ * to interface with OS or network services in your local environment.
+ * Instead this function and its result submission counterpart are
+ * provided so you can provide a DNS implementation.
+ *
+ * If this callback is set in your callback struct to a NULL value,
+ * DNS will not be available. The ZeroTier protocol is designed to
+ * work in the absence of DNS but you may not get optimal results. For
+ * example you may default to root servers that are not geographically
+ * optimal or your node may cease to function if a root server's IP
+ * changes and there's no way to signal this.
+ *
+ * This function requests resolution of a DNS record. The result
+ * submission method ZT_Node_processDNSResult() must be called at
+ * least once in response. See its documentation.
+ *
+ * Right now ZeroTier only requests resolution of TXT records, but
+ * it's possible that this will change in the future.
+ *
+ * It's safe to call processDNSResult() from within your handler
+ * for this function.
+ */
+typedef void (*ZT_DNSResolver)(
+	ZT_Node *,                        /* Node */
+	void *,                           /* User ptr */
+	void *,                           /* Thread ptr */
+	enum ZT_DNSRecordType *,          /* DNS record type(s) to fetch */
+	unsigned int,                     /* Number of DNS record type(s) */
+	const char *,                     /* DNS name to fetch */
+	uintptr_t);                       /* Request ID for returning results */
+
 /****************************************************************************/
 /* C Node API                                                               */
 /****************************************************************************/
@@ -1613,11 +1646,6 @@ typedef int (*ZT_PathLookupFunction)(
  */
 struct ZT_Node_Callbacks
 {
-	/**
-	 * Struct version -- must currently be 0
-	 */
-	long version;
-
 	/**
 	 * REQUIRED: Function to store and/or replicate state objects
 	 */
@@ -1648,6 +1676,11 @@ struct ZT_Node_Callbacks
 	 */
 	ZT_EventCallback eventCallback;
 
+	/**
+	 * STRONGLY RECOMMENDED: Function to request a DNS lookup
+	 */
+	ZT_DNSResolver dnsResolver;
+
 	/**
 	 * OPTIONAL: Function to check whether a given physical path should be used
 	 */
@@ -1747,7 +1780,62 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame(
  * @param nextBackgroundTaskDeadline Value/result: set to deadline for next call to processBackgroundTasks()
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,void *tptr,int64_t now,volatile int64_t *nextBackgroundTaskDeadline);
+ZT_SDK_API enum ZT_ResultCode ZT_Node_processBackgroundTasks(
+	ZT_Node *node,
+	void *tptr,
+	int64_t now,
+	volatile int64_t *nextBackgroundTaskDeadline);
+
+/**
+ * Submit the result(s) of a requested DNS query
+ *
+ * This MUST be called at least once after the node requsts DNS resolution.
+ * If there are no results or DNS is not implemented or available, just
+ * send one ZT_DNS_RECORD__END_OF_RESULTS to signal that no results were
+ * obtained.
+ *
+ * If result is non-NULL but resultLength is zero then result is assumed to
+ * be a C string terminated by a zero. Passing an unterminated string with a
+ * zero resultLength will result in a crash.
+ *
+ * The results of A and AAAA records can be returned as either strings or
+ * binary IP address bytes (network byte order). If the result is a string,
+ * resultLength must be 0 to signal that result is a C string. Otherwise for
+ * A resultLength must be 4 and for AAAA it must be 16 if the result is
+ * in binary format.
+ *
+ * The Node implementation makes an effort to ignore obviously invalid
+ * submissions like an AAAA record in bianry form with length 25, but this
+ * is not guaranteed. It's possible to crash your program by calling this
+ * with garbage inputs.
+ *
+ * Results may be submitted in any order and order should not be assumed
+ * to have any meaning.
+ *
+ * The ZT_DNS_RECORD__END_OF_RESULTS pseudo-response must be sent after all
+ * results have been submitted. The result and resultLength paramters are
+ * ignored for this type ID.
+ *
+ * It is safe to call this function from inside the DNS request callback,
+ * such as to return a locally cached result or a result from some kind
+ * of local database. It's also safe to call this function from threads
+ * other than the one that received the DNS request.
+ *
+ * @param node Node instance that requested DNS resolution
+ * @param tptr Thread pointer to pass to functions/callbacks resulting from this call
+ * @param dnsRequestID Request ID supplied to DNS request callback
+ * @param recordType Record type of this result
+ * @param result Result (content depends on record type)
+ * @param resultLength Length of result
+ * @param resultIsString If non-zero, IP results for A and AAAA records are being given as C strings not binary IPs
+ */
+ZT_SDK_API void ZT_Node_processDNSResult(
+	ZT_Node *node,
+	void *tptr,
+	uintptr_t dnsRequestID,
+	enum ZT_DNSRecordType recordType,
+	const void *result,
+	unsigned int resultLength);
 
 /**
  * Join a network

node/Constants.hpp

@@ -470,7 +470,7 @@
 
 /**
  * Delay between full-fledge pings of directly connected peers.
- * 
+ *
  * With multipath bonding enabled ping peers more often to measure
  * packet loss and latency. This uses more bandwidth so is disabled
  * by default to avoid increasing idle bandwidth use for regular
@@ -621,23 +621,17 @@
 
 /**
  * Size of a buffer to store either a C25519 or an ECC P-384 signature
+ *
+ * This must be large enough to hold all signature types.
  */
 #define ZT_SIGNATURE_BUFFER_SIZE 96
 
-/**
- * Desired buffer size for UDP sockets (used in service and osdep but defined here)
- */
-#if (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__))
-#define ZT_UDP_DESIRED_BUF_SIZE 1048576
-#else
-#define ZT_UDP_DESIRED_BUF_SIZE 131072
-#endif
-
 /**
  * Desired / recommended min stack size for threads (used on some platforms to reset thread stack size)
  */
 #define ZT_THREAD_MIN_STACK_SIZE 1048576
 
+// Internal cryptographic algorithm IDs
 #define ZT_CRYPTO_ALG_C25519 0
 #define ZT_CRYPTO_ALG_P384 1
 

node/Identity.hpp

@@ -121,7 +121,7 @@ public:
 	 * @param sha Buffer to receive SHA512 (MUST be ZT_SHA512_DIGEST_LEN (64) bytes in length)
 	 * @return True on success, false if no private key
 	 */
-	inline bool sha512PrivateKey(void *sha) const
+	inline bool sha512PrivateKey(void *const sha) const
 	{
 		if (_hasPrivate) {
 			switch(_type) {
@@ -136,6 +136,29 @@ public:
 		return false;
 	}
 
+	/**
+	 * Compute a 128-bit short hash of this identity's public key
+	 *
+	 * This is the first 128 bits of a SHA384 hash and is the hash used
+	 * in VERB_WILL_RELAY to report reachability.
+	 *
+	 * @param h 128-bit buffer to receive hash (must be 16 bytes in size)
+	 */
+	inline void publicKeyHash128(void *const h) const
+	{
+		uint8_t tmp[48];
+		switch(_type) {
+			case C25519:
+				SHA384(tmp,_pub.c25519,ZT_C25519_PUBLIC_KEY_LEN);
+				break;
+			case P384:
+				SHA384(tmp,&_pub,ZT_C25519_PUBLIC_KEY_LEN + ZT_ECC384_PUBLIC_KEY_SIZE);
+				break;
+		}
+		for(int i=0;i<16;++i)
+			((uint8_t *)h)[i] = tmp[i];
+	}
+
 	/**
 	 * Sign a message with this identity (private key required)
 	 *
@@ -173,6 +196,7 @@ public:
 				SHA384(h,h,48 + ZT_C25519_PUBLIC_KEY_LEN);
 				ECC384ECDSASign(_priv.p384,h,(uint8_t *)sig);
 				return ZT_ECC384_SIGNATURE_SIZE;
+
 		}
 		return 0;
 	}
@@ -236,10 +260,7 @@ public:
 					C25519::agree(_priv.c25519,id._pub.c25519,rawkey);
 					ECC384ECDH(id._pub.p384,_priv.p384,rawkey + ZT_C25519_SHARED_KEY_LEN);
 					SHA384(h,rawkey,ZT_C25519_SHARED_KEY_LEN + ZT_ECC384_SHARED_SECRET_SIZE);
-					for(unsigned int i=0;i<32;++i)
-						key[i] = h[i];
-					for(unsigned int i=0;i<16;++i)
-						key[i] ^= h[32+i];
+					memcpy(key,h,ZT_PEER_SECRET_KEY_LENGTH);
 					return true;
 				} else if (id._type == C25519) {
 					// If the other identity is a C25519 identity we can agree using only that type.

node/IncomingPacket.cpp

@@ -94,7 +94,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr)
 			switch(v) {
 				//case Packet::VERB_NOP:
 				default: // ignore unknown verbs, but if they pass auth check they are "received"
-					peer->received(tPtr,_path,hops(),packetId(),payloadLength(),v,0,Packet::VERB_NOP,false,0);
+					peer->received(tPtr,_path,hops(),packetId(),payloadLength(),v,0,Packet::VERB_NOP,0);
 					break;
 				case Packet::VERB_HELLO:                      r = _doHELLO(RR,tPtr,true); break;
 				case Packet::VERB_ACK:                        r = _doACK(RR,tPtr,peer); break;
@@ -116,11 +116,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr)
 				case Packet::VERB_USER_MESSAGE:               r = _doUSER_MESSAGE(RR,tPtr,peer); break;
 				case Packet::VERB_REMOTE_TRACE:               r = _doREMOTE_TRACE(RR,tPtr,peer); break;
 			}
-			if (r) {
-				RR->node->statsLogVerb((unsigned int)v,(unsigned int)size());
-				return true;
-			}
-			return false;
+			return r;
 		} else {
 			RR->sw->requestWhois(tPtr,RR->node->now(),sourceAddress);
 			return false;
@@ -149,7 +145,8 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 		case Packet::ERROR_OBJ_NOT_FOUND:
 			// Object not found, currently only meaningful from network controllers.
 			if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) {
-				const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+				networkId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD);
+				const SharedPtr<Network> network(RR->node->network(networkId));
 				if ((network)&&(network->controller() == peer->address()))
 					network->setNotFound();
 			}
@@ -160,19 +157,13 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 			// consider it meaningful from network controllers. This would indicate
 			// that the queried node does not support acting as a controller.
 			if (inReVerb == Packet::VERB_NETWORK_CONFIG_REQUEST) {
-				const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+				networkId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD);
+				const SharedPtr<Network> network(RR->node->network(networkId));
 				if ((network)&&(network->controller() == peer->address()))
 					network->setNotFound();
 			}
 			break;
 
-		case Packet::ERROR_IDENTITY_COLLISION:
-			// This is a trusted upstream telling us our 5-digit ID is taken. This
-			// causes the node to generate another.
-			if (RR->topology->isRoot(peer->identity()))
-				RR->node->postEvent(tPtr,ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
-			break;
-
 		case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
 			// Peers can send this to ask for a cert for a network.
 			networkId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD);
@@ -184,7 +175,8 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 
 		case Packet::ERROR_NETWORK_ACCESS_DENIED_: {
 			// Network controller: network access denied.
-			const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
+			networkId = at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD);
+			const SharedPtr<Network> network(RR->node->network(networkId));
 			if ((network)&&(network->controller() == peer->address()))
 				network->setAccessDenied();
 		}	break;
@@ -203,7 +195,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 		default: break;
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_ERROR,inRePacketId,inReVerb,false,networkId);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_ERROR,inRePacketId,inReVerb,networkId);
 
 	return true;
 }
@@ -381,7 +373,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
 	_path->send(RR,tPtr,outp.data(),outp.size(),now);
 
 	peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version
-	peer->received(tPtr,_path,hops(),pid,payloadLength(),Packet::VERB_HELLO,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),pid,payloadLength(),Packet::VERB_HELLO,0,Packet::VERB_NOP,0);
 
 	return true;
 }
@@ -424,8 +416,17 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP
 
 		case Packet::VERB_WHOIS:
 			if (RR->topology->isRoot(peer->identity())) {
-				const Identity id(*this,ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY);
-				RR->sw->doAnythingWaitingForPeer(tPtr,RR->topology->add(SharedPtr<Peer>(new Peer(RR,RR->identity,id))));
+				unsigned int p = ZT_PROTO_VERB_WHOIS__OK__IDX_IDENTITY;
+				while (p < size()) {
+					try {
+						Identity id;
+						p += id.deserialize(*this,p);
+						if (id)
+							RR->sw->doAnythingWaitingForPeer(tPtr,RR->topology->add(SharedPtr<Peer>(new Peer(RR,RR->identity,id))));
+					} catch ( ... ) {
+						break;
+					}
+				}
 			}
 			break;
 
@@ -475,7 +476,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedP
 		default: break;
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_OK,inRePacketId,inReVerb,false,networkId);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_OK,inRePacketId,inReVerb,networkId);
 
 	return true;
 }
@@ -511,7 +512,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar
 		_path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now());
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_WHOIS,0,Packet::VERB_NOP,0);
 
 	return true;
 }
@@ -535,7 +536,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,void *tPtr,const
 		}
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP,0);
 
 	return true;
 }
@@ -544,10 +545,8 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 {
 	const uint64_t nwid = at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID);
 	const SharedPtr<Network> network(RR->node->network(nwid));
-	bool trustEstablished = false;
 	if (network) {
 		if (network->gate(tPtr,peer)) {
-			trustEstablished = true;
 			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
 				const unsigned int etherType = at<uint16_t>(ZT_PROTO_VERB_FRAME_IDX_ETHERTYPE);
 				const MAC sourceMac(peer->address(),nwid);
@@ -562,7 +561,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,void *tPtr,const Shar
 		}
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_FRAME,0,Packet::VERB_NOP,trustEstablished,nwid);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_FRAME,0,Packet::VERB_NOP,nwid);
 
 	return true;
 }
@@ -596,7 +595,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
 			const uint8_t *const frameData = (const uint8_t *)field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,frameLen);
 
 			if ((!from)||(from == network->mac())) {
-				peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay
+				peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,nwid);
 				return true;
 			}
 
@@ -607,19 +606,19 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
 							network->learnBridgeRoute(from,peer->address());
 						} else {
 							RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (remote)");
-							peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay
+							peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,nwid);
 							return true;
 						}
 					} else if (to != network->mac()) {
 						if (to.isMulticast()) {
 							if (network->config().multicastLimit == 0) {
 								RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"multicast disabled");
-								peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay
+								peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,nwid);
 								return true;
 							}
 						} else if (!network->config().permitsBridging(RR->identity.address())) {
 							RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_EXT_FRAME,from,to,"bridging not allowed (local)");
-							peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid); // trustEstablished because COM is okay
+							peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,nwid);
 							return true;
 						}
 					}
@@ -639,11 +638,9 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,void *tPtr,const
 			_path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now());
 		}
 
-		peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,true,nwid);
-	} else {
-		peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,false,nwid);
 	}
 
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP,nwid);
 	return true;
 }
 
@@ -661,7 +658,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,void *tPtr,const Share
 	outp.armor(peer->key(),true);
 	_path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now());
 
-	peer->received(tPtr,_path,hops(),pid,payloadLength(),Packet::VERB_ECHO,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),pid,payloadLength(),Packet::VERB_ECHO,0,Packet::VERB_NOP,0);
 
 	return true;
 }
@@ -687,7 +684,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,void *tPtr,c
 			RR->mc->add(tPtr,now,nwid,MulticastGroup(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)),peer->address());
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP,0);
 	return true;
 }
 
@@ -701,7 +698,6 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t
 	Tag tag;
 	Revocation revocation;
 	CertificateOfOwnership coo;
-	bool trustEstablished = false;
 	SharedPtr<Network> network;
 
 	unsigned int p = ZT_PACKET_IDX_PAYLOAD;
@@ -710,16 +706,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t
 		if (com) {
 			network = RR->node->network(com.networkId());
 			if (network) {
-				switch (network->addCredential(tPtr,com)) {
-					case Membership::ADD_REJECTED:
-						break;
-					case Membership::ADD_ACCEPTED_NEW:
-					case Membership::ADD_ACCEPTED_REDUNDANT:
-						trustEstablished = true;
-						break;
-					case Membership::ADD_DEFERRED_FOR_WHOIS:
-						return false;
-				}
+				if (network->addCredential(tPtr,com) == Membership::ADD_DEFERRED_FOR_WHOIS)
+					return false;
 			}
 		}
 	}
@@ -732,16 +720,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t
 			if ((!network)||(network->id() != cap.networkId()))
 				network = RR->node->network(cap.networkId());
 			if (network) {
-				switch (network->addCredential(tPtr,cap)) {
-					case Membership::ADD_REJECTED:
-						break;
-					case Membership::ADD_ACCEPTED_NEW:
-					case Membership::ADD_ACCEPTED_REDUNDANT:
-						trustEstablished = true;
-						break;
-					case Membership::ADD_DEFERRED_FOR_WHOIS:
-						return false;
-				}
+				if (network->addCredential(tPtr,cap) == Membership::ADD_DEFERRED_FOR_WHOIS)
+					return false;
 			}
 		}
 
@@ -753,16 +733,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t
 			if ((!network)||(network->id() != tag.networkId()))
 				network = RR->node->network(tag.networkId());
 			if (network) {
-				switch (network->addCredential(tPtr,tag)) {
-					case Membership::ADD_REJECTED:
-						break;
-					case Membership::ADD_ACCEPTED_NEW:
-					case Membership::ADD_ACCEPTED_REDUNDANT:
-						trustEstablished = true;
-						break;
-					case Membership::ADD_DEFERRED_FOR_WHOIS:
-						return false;
-				}
+				if (network->addCredential(tPtr,tag) == Membership::ADD_DEFERRED_FOR_WHOIS)
+					return false;
 			}
 		}
 
@@ -774,16 +746,8 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t
 			if ((!network)||(network->id() != revocation.networkId()))
 				network = RR->node->network(revocation.networkId());
 			if (network) {
-				switch(network->addCredential(tPtr,peer->address(),revocation)) {
-					case Membership::ADD_REJECTED:
-						break;
-					case Membership::ADD_ACCEPTED_NEW:
-					case Membership::ADD_ACCEPTED_REDUNDANT:
-						trustEstablished = true;
-						break;
-					case Membership::ADD_DEFERRED_FOR_WHOIS:
-						return false;
-				}
+				if (network->addCredential(tPtr,peer->address(),revocation) == Membership::ADD_DEFERRED_FOR_WHOIS)
+					return false;
 			}
 		}
 
@@ -795,21 +759,13 @@ bool IncomingPacket::_doNETWORK_CREDENTIALS(const RuntimeEnvironment *RR,void *t
 			if ((!network)||(network->id() != coo.networkId()))
 				network = RR->node->network(coo.networkId());
 			if (network) {
-				switch(network->addCredential(tPtr,coo)) {
-					case Membership::ADD_REJECTED:
-						break;
-					case Membership::ADD_ACCEPTED_NEW:
-					case Membership::ADD_ACCEPTED_REDUNDANT:
-						trustEstablished = true;
-						break;
-					case Membership::ADD_DEFERRED_FOR_WHOIS:
-						return false;
-				}
+				if (network->addCredential(tPtr,coo) == Membership::ADD_DEFERRED_FOR_WHOIS)
+					return false;
 			}
 		}
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,trustEstablished,(network) ? network->id() : 0);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_NETWORK_CREDENTIALS,0,Packet::VERB_NOP,(network) ? network->id() : 0);
 
 	return true;
 }
@@ -835,7 +791,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,void
 		_path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now());
 	}
 
-	peer->received(tPtr,_path,hopCount,requestPacketId,payloadLength(),Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,false,nwid);
+	peer->received(tPtr,_path,hopCount,requestPacketId,payloadLength(),Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP,nwid);
 
 	return true;
 }
@@ -855,9 +811,7 @@ bool IncomingPacket::_doNETWORK_CONFIG(const RuntimeEnvironment *RR,void *tPtr,c
 			_path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now());
 		}
 	}
-
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,false,(network) ? network->id() : 0);
-
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_NETWORK_CONFIG,0,Packet::VERB_NOP,(network) ? network->id() : 0);
 	return true;
 }
 
@@ -879,18 +833,14 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr
 		} catch ( ... ) {} // discard invalid COMs
 	}
 
-	bool trustEstablished = false;
 	if (network) {
-		if (network->gate(tPtr,peer)) {
-			trustEstablished = true;
-		} else {
+		if (!network->gate(tPtr,peer)) {
 			_sendErrorNeedCredentials(RR,tPtr,peer,nwid);
 			return false;
 		}
 	}
 
 	const int64_t now = RR->node->now();
-	//if ((gatherLimit > 0)&&((trustEstablished)||(RR->topology->amUpstream())||(RR->node->localControllerHasAuthorized(now,nwid,peer->address())))) {
 	if (gatherLimit) {
 		Packet outp(peer->address(),RR->identity.address(),Packet::VERB_OK);
 		outp.append((unsigned char)Packet::VERB_MULTICAST_GATHER);
@@ -905,7 +855,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,void *tPtr
 		}
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,trustEstablished,nwid);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP,nwid);
 
 	return true;
 }
@@ -963,17 +913,17 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,
 
 		if (network->config().multicastLimit == 0) {
 			RR->t->incomingNetworkFrameDropped(tPtr,network,_path,packetId(),size(),peer->address(),Packet::VERB_MULTICAST_FRAME,from,to.mac(),"multicast disabled");
-			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
+			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,nwid);
 			return true;
 		}
 		if (!to.mac().isMulticast()) {
 			RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"destination not multicast");
-			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
+			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,nwid);
 			return true;
 		}
 		if ((!from)||(from.isMulticast())||(from == network->mac())) {
 			RR->t->incomingPacketInvalid(tPtr,_path,packetId(),source(),hops(),Packet::VERB_MULTICAST_FRAME,"invalid source MAC");
-			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
+			peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,nwid);
 			return true;
 		}
 
@@ -1027,7 +977,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,void *tPtr,
 			}
 		}
 
-		peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,true,nwid);
+		peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP,nwid);
 		return true;
 	} else {
 		_sendErrorNeedCredentials(RR,tPtr,peer,nwid);
@@ -1041,7 +991,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
 
 	// First, subject this to a rate limit
 	if (!peer->rateGatePushDirectPaths(now)) {
-		peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0);
+		peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,0);
 		return true;
 	}
 
@@ -1090,7 +1040,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,void *tPt
 		ptr += addrLen;
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP,0);
 
 	return true;
 }
@@ -1106,7 +1056,7 @@ bool IncomingPacket::_doUSER_MESSAGE(const RuntimeEnvironment *RR,void *tPtr,con
 		RR->node->postEvent(tPtr,ZT_EVENT_USER_MESSAGE,reinterpret_cast<const void *>(&um));
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_USER_MESSAGE,0,Packet::VERB_NOP,0);
 
 	return true;
 }
@@ -1130,7 +1080,7 @@ bool IncomingPacket::_doREMOTE_TRACE(const RuntimeEnvironment *RR,void *tPtr,con
 		}
 	}
 
-	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,false,0);
+	peer->received(tPtr,_path,hops(),packetId(),payloadLength(),Packet::VERB_REMOTE_TRACE,0,Packet::VERB_NOP,0);
 
 	return true;
 }

node/Node.cpp

@@ -62,8 +62,6 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 	_lastHousekeepingRun(0),
 	_lastMemoizedTraceSettings(0)
 {
-	if (callbacks->version != 0)
-		throw ZT_EXCEPTION_INVALID_ARGUMENT;
 	memcpy(&_cb,callbacks,sizeof(ZT_Node_Callbacks));
 
 	// Initialize non-cryptographic PRNG from a good random source
@@ -74,7 +72,6 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 	memset(_expectingRepliesToBucketPtr,0,sizeof(_expectingRepliesToBucketPtr));
 	memset(_expectingRepliesTo,0,sizeof(_expectingRepliesTo));
 	memset(_lastIdentityVerification,0,sizeof(_lastIdentityVerification));
-	memset((void *)(&_stats),0,sizeof(_stats));
 
 	uint64_t idtmp[2];
 	idtmp[0] = 0; idtmp[1] = 0;
@@ -472,7 +469,7 @@ ZT_PeerList *Node::peers() const
 			p->paths[p->pathCount].expired = 0;
 			p->paths[p->pathCount].preferred = ((*path) == bestp) ? 1 : 0;
 			p->paths[p->pathCount].latency = (float)(*path)->latency();
-			p->paths[p->pathCount].packetDelayVariance = (*path)->packetDelayVariance(); 
+			p->paths[p->pathCount].packetDelayVariance = (*path)->packetDelayVariance();
 			p->paths[p->pathCount].throughputDisturbCoeff = (*path)->throughputDisturbanceCoefficient();
 			p->paths[p->pathCount].packetErrorRatio = (*path)->packetErrorRatio();
 			p->paths[p->pathCount].packetLossRatio = (*path)->packetLossRatio();

node/Node.hpp

@@ -266,12 +266,6 @@ public:
 		return false;
 	}
 
-	inline void statsLogVerb(const unsigned int v,const unsigned int bytes)
-	{
-		++_stats.inVerbCounts[v];
-		_stats.inVerbBytes[v] += (uint64_t)bytes;
-	}
-
 private:
 	RuntimeEnvironment _RR;
 	RuntimeEnvironment *RR;
@@ -285,9 +279,6 @@ private:
 	// Time of last identity verification indexed by InetAddress.rateGateHash() -- used in IncomingPacket::_doHELLO() via rateGateIdentityVerification()
 	int64_t _lastIdentityVerification[16384];
 
-	// Statistics about stuff happening
-	volatile ZT_NodeStatistics _stats;
-
 	// Map that remembers if we have recently sent a network config to someone
 	// querying us as a controller.
 	struct _LocalControllerAuth

node/Packet.hpp

@@ -74,8 +74,10 @@
  *    + Peer-to-peer multicast replication (optional)
  *    + Old planet/moon stuff is DEAD!
  *    + AES256-GCM encryption is now the default
- *    + NIST P-384 type identities now supported (25519 still default)
- *    + Min proto version is now 8 (1.1.17 and newer)
+ *    + NIST P-384 (type 1) identities now supported
+ *    + Minimum proto version is now 8 (1.1.17 and newer)
+ *    + WILL_RELAY allows mesh-like operation
+ *    + Ephemeral keys are now negotiated opportunistically
  */
 #define ZT_PROTO_VERSION 11
 
@@ -887,7 +889,81 @@ public:
 		 * pertain directly to activity on a network, or to global observers if
 		 * locally configured.
 		 */
-		VERB_REMOTE_TRACE = 0x15
+		VERB_REMOTE_TRACE = 0x15,
+
+		/**
+		 * A signed locator for this node:
+		 *   <[8] 64-bit flags>
+		 *   <[2] 16-bit length of locator>
+		 *   <[...] serialized locator>
+		 *
+		 * This message is sent in response to OK(HELLO) and can be pushed
+		 * opportunitistically. Its payload is a signed Locator object that
+		 * attests to where and how this Node may be reached. A locator can
+		 * contain static IPs/ports or other ZeroTier nodes that can be used
+		 * to reach this one.
+		 *
+		 * These Locator objects can be stored e.g. by roots in LF to publish
+		 * node reachability. Since they're signed any node can verify that
+		 * the originating node approves of their content.
+		 */
+		VERB_LOCATOR = 0x16,
+
+		/**
+		 * A list of peers this node will relay traffic to/from:
+		 *   <[2] 16-bit number of peers>
+		 *   <[16] 128-bit hash of node public key>
+		 *   <[2] 16-bit latency to node or 0 if unspecified>
+		 *   <[1] 8-bit number of network hops to node or 0 if unspecified>
+		 *   <[4] 32-bit max bandwidth in megabits or 0 if unspecified>
+		 *  [<[...] additional hash,latency,hops,bandwidth tuples>]
+		 *
+		 * This messages can be pushed to indicate that this peer is willing
+		 * to relay traffic to other peers. It contains a list of 128-bit
+		 * hashes (the first 128 bits of a SHA512) of identity public keys
+		 * of currently reachable and willing-to-relay-for nodes.
+		 *
+		 * This can be used to initiate mesh-like behavior in ZeroTier. The
+		 * peers for which this node is willing to relay are reported as
+		 * hashes of their identity public keys. This prevents this message
+		 * from revealing explicit information about linked peers. The
+		 * receiving peer can only "see" a will-relay entry if it knows the
+		 * identity of the peer it is trying to reach.
+		 */
+  	VERB_WILL_RELAY = 0x17,
+
+		/**
+		 * A push of one or more ephemeral key pairs:
+		 *   <[2] 8-bit length of random padding>
+		 *   <[...] random padding>
+		 *   <[1] 8-bit number of keys in message>
+		 *   <[1] 8-bit key type>
+		 *   <[4] 32-bit max key ttl in seconds or 0 for unspecified>
+		 *   <[4] 32-bit reserved field (currently always 0)>
+		 *   <[...] public key (length determined by type)>
+		 *  [<[...] additional keys as type, ttl, flags, key>]
+		 *
+		 * This verb is used to push ephemeral keys. A node replies to each
+		 * ephemeral key push with an OK message containing its own current
+		 * ephemeral keys that it wants to use for p2p communication.
+		 *
+		 * These are ephemeral public keys. Currently keys of type C25519
+		 * and P-384 are supported and both will be pushed.
+		 *
+		 * If more than one key is pushed, key agreement is performed using
+		 * all keys for which both peers pushed the same key type. The raw
+		 * results of these keys are then hashed together in order of key
+		 * type ID with SHA384 to yield a session key. If the desired session
+		 * key is shorter than 384 bits the first N bits are used.
+		 *
+		 * The random padding component can be used to ranomize the length
+		 * of these packets so adversaries can't easily selectively block
+		 * ephemeral key exchange by exploiting a fixed packet length.
+		 *
+		 * OK response payload:
+		 *   <[...] responder's keys, same format as verb payload>
+		 */
+		VERB_EPHEMERAL_KEY = 0x18
 	};
 
 	/**

node/Path.hpp

@@ -97,7 +97,6 @@ public:
 	inline Path() :
 		_lastOut(0),
 		_lastIn(0),
-		_lastTrustEstablishedPacketReceived(0),
 		_lastPathQualityComputeTime(0),
 		_localSocket(-1),
 		_latency(0xffff),
@@ -130,7 +129,6 @@ public:
 	inline Path(const int64_t localSocket,const InetAddress &addr) :
 		_lastOut(0),
 		_lastIn(0),
-		_lastTrustEstablishedPacketReceived(0),
 		_lastPathQualityComputeTime(0),
 		_localSocket(localSocket),
 		_latency(0xffff),
@@ -170,11 +168,6 @@ public:
 	 */
 	inline void received(const uint64_t t) { _lastIn = t; }
 
-	/**
-	 * Set time last trusted packet was received (done in Peer::received())
-	 */
-	inline void trustedPacketReceived(const uint64_t t) { _lastTrustEstablishedPacketReceived = t; }
-
 	/**
 	 * Send a packet via this path (last out time is also updated)
 	 *
@@ -226,11 +219,6 @@ public:
 	 */
 	inline InetAddress::IpScope ipScope() const { return _ipScope; }
 
-	/**
-	 * @return True if path has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms
-	 */
-	inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); }
-
 	/**
 	 * @return Preference rank, higher == better
 	 */
@@ -642,17 +630,11 @@ public:
 	 */
 	inline int64_t lastIn() const { return _lastIn; }
 
-	/**
-	 * @return Time last trust-established packet was received
-	 */
-	inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }
-
 private:
 	Mutex _statistics_m;
 
 	volatile int64_t _lastOut;
 	volatile int64_t _lastIn;
-	volatile int64_t _lastTrustEstablishedPacketReceived;
 	volatile int64_t _lastPathQualityComputeTime;
 	int64_t _localSocket;
 	volatile unsigned int _latency;

node/Peer.cpp

@@ -51,7 +51,6 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_lastWhoisRequestReceived(0),
 	_lastEchoRequestReceived(0),
 	_lastCredentialsReceived(0),
-	_lastTrustEstablishedPacketReceived(0),
 	_lastSentFullHello(0),
 	_lastACKWindowReset(0),
 	_lastQoSWindowReset(0),
@@ -87,7 +86,6 @@ void Peer::received(
 	const Packet::Verb verb,
 	const uint64_t inRePacketId,
 	const Packet::Verb inReVerb,
-	const bool trustEstablished,
 	const uint64_t networkId)
 {
 	const int64_t now = RR->node->now();
@@ -105,11 +103,6 @@ void Peer::received(
 			break;
 	}
 
-	if (trustEstablished) {
-		_lastTrustEstablishedPacketReceived = now;
-		path->trustedPacketReceived(now);
-	}
-
 	{
 		Mutex::Lock _l(_paths_m);
 
@@ -202,51 +195,48 @@ void Peer::received(
 		}
 	}
 
-	// If we have a trust relationship periodically push a message enumerating
-	// all known external addresses for ourselves. If we already have a path this
-	// is done less frequently.
-	if (this->trustEstablished(now)) {
-		const int64_t sinceLastPush = now - _lastDirectPathPushSent;
-		if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) {
-			_lastDirectPathPushSent = now;
-			std::vector<InetAddress> pathsToPush(RR->node->directPaths());
-			if (pathsToPush.size() > 0) {
-				std::vector<InetAddress>::const_iterator p(pathsToPush.begin());
-				while (p != pathsToPush.end()) {
-					Packet *const outp = new Packet(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
-					outp->addSize(2); // leave room for count
-					unsigned int count = 0;
-					while ((p != pathsToPush.end())&&((outp->size() + 24) < 1200)) {
-						uint8_t addressType = 4;
-						switch(p->ss_family) {
-							case AF_INET:
-								break;
-							case AF_INET6:
-								addressType = 6;
-								break;
-							default: // we currently only push IP addresses
-								++p;
-								continue;
-						}
+	// Periodically push direct paths to the peer, doing so more often if we do not
+	// currently have a direct path.
+	const int64_t sinceLastPush = now - _lastDirectPathPushSent;
+	if (sinceLastPush >= ((hops == 0) ? ZT_DIRECT_PATH_PUSH_INTERVAL_HAVEPATH : ZT_DIRECT_PATH_PUSH_INTERVAL)) {
+		_lastDirectPathPushSent = now;
+		std::vector<InetAddress> pathsToPush(RR->node->directPaths());
+		if (pathsToPush.size() > 0) {
+			std::vector<InetAddress>::const_iterator p(pathsToPush.begin());
+			while (p != pathsToPush.end()) {
+				Packet *const outp = new Packet(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
+				outp->addSize(2); // leave room for count
+				unsigned int count = 0;
+				while ((p != pathsToPush.end())&&((outp->size() + 24) < 1200)) {
+					uint8_t addressType = 4;
+					switch(p->ss_family) {
+						case AF_INET:
+							break;
+						case AF_INET6:
+							addressType = 6;
+							break;
+						default: // we currently only push IP addresses
+							++p;
+							continue;
+					}
 
-						outp->append((uint8_t)0); // no flags
-						outp->append((uint16_t)0); // no extensions
-						outp->append(addressType);
-						outp->append((uint8_t)((addressType == 4) ? 6 : 18));
-						outp->append(p->rawIpData(),((addressType == 4) ? 4 : 16));
-						outp->append((uint16_t)p->port());
+					outp->append((uint8_t)0); // no flags
+					outp->append((uint16_t)0); // no extensions
+					outp->append(addressType);
+					outp->append((uint8_t)((addressType == 4) ? 6 : 18));
+					outp->append(p->rawIpData(),((addressType == 4) ? 4 : 16));
+					outp->append((uint16_t)p->port());
 
-						++count;
-						++p;
-					}
-					if (count) {
-						outp->setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count);
-						outp->compress();
-						outp->armor(_key,true);
-						path->send(RR,tPtr,outp->data(),outp->size(),now);
-					}
-					delete outp;
+					++count;
+					++p;
+				}
+				if (count) {
+					outp->setAt(ZT_PACKET_IDX_PAYLOAD,(uint16_t)count);
+					outp->compress();
+					outp->armor(_key,true);
+					path->send(RR,tPtr,outp->data(),outp->size(),now);
 				}
+				delete outp;
 			}
 		}
 	}

node/Peer.hpp

@@ -95,7 +95,6 @@ public:
 	 * @param verb Packet verb
 	 * @param inRePacketId Packet ID in reply to (default: none)
 	 * @param inReVerb Verb in reply to (for OK/ERROR, default: VERB_NOP)
-	 * @param trustEstablished If true, some form of non-trivial trust (like allowed in network) has been established
 	 * @param networkId Network ID if this pertains to a network, or 0 otherwise
 	 */
 	void received(
@@ -107,7 +106,6 @@ public:
 		const Packet::Verb verb,
 		const uint64_t inRePacketId,
 		const Packet::Verb inReVerb,
-		const bool trustEstablished,
 		const uint64_t networkId);
 
 	/**
@@ -444,11 +442,6 @@ public:
 	 */
 	inline bool canUseMultipath() { return _canUseMultipath; }
 
-	/**
-	 * @return True if peer has received a trust established packet (e.g. common network membership) in the past ZT_TRUST_EXPIRATION ms
-	 */
-	inline bool trustEstablished(const int64_t now) const { return ((now - _lastTrustEstablishedPacketReceived) < ZT_TRUST_EXPIRATION); }
-
 	/**
 	 * Rate limit gate for VERB_PUSH_DIRECT_PATHS
 	 */
@@ -567,7 +560,6 @@ private:
 	int64_t _lastWhoisRequestReceived;
 	int64_t _lastEchoRequestReceived;
 	int64_t _lastCredentialsReceived;
-	int64_t _lastTrustEstablishedPacketReceived;
 	int64_t _lastSentFullHello;
 	int64_t _lastPathPrune;
 	int64_t _lastACKWindowReset;

osdep/Binder.hpp

@@ -67,6 +67,12 @@
 #include "Phy.hpp"
 #include "OSUtils.hpp"
 
+#if (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__))
+#define ZT_UDP_DESIRED_BUF_SIZE 1048576
+#else
+#define ZT_UDP_DESIRED_BUF_SIZE 131072
+#endif
+
 // Period between refreshes of bindings
 #define ZT_BINDER_REFRESH_PERIOD 30000
 

osdep/MacEthernetTap.cpp

@@ -89,7 +89,6 @@ MacEthernetTap::MacEthernetTap(
 	_nwid(nwid),
 	_homePath(homePath),
 	_mtu(mtu),
-	_metric(metric),
 	_agentStdin(-1),
 	_agentStdout(-1),
 	_agentStderr(-1),

osdep/MacEthernetTap.hpp

@@ -83,7 +83,6 @@ private:
 	std::vector<MulticastGroup> _multicastGroups;
 	Mutex _putLock;
 	unsigned int _mtu;
-	unsigned int _metric;
 	int _shutdownSignalPipe[2];
 	int _agentStdin,_agentStdout,_agentStderr,_agentStdin2,_agentStdout2,_agentStderr2;
 	long _agentPid;

service/OneService.cpp

@@ -495,7 +495,6 @@ public:
 
 			{
 				struct ZT_Node_Callbacks cb;
-				cb.version = 0;
 				cb.stateGetFunction = SnodeStateGetFunction;
 				cb.statePutFunction = SnodeStatePutFunction;
 				cb.wirePacketSendFunction = SnodeWirePacketSendFunction;
@@ -764,7 +763,7 @@ public:
 	}
 
 	void readLocalSettings()
-	{		
+	{
 		// Read local configuration
 		std::map<InetAddress,ZT_PhysicalPathConfiguration> ppc;
 
@@ -1810,12 +1809,6 @@ public:
 	inline void nodeEventCallback(enum ZT_Event event,const void *metaData)
 	{
 		switch(event) {
-			case ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION: {
-				Mutex::Lock _l(_termReason_m);
-				_termReason = ONE_IDENTITY_COLLISION;
-				_fatalErrorMessage = "identity/address collision";
-				this->terminate();
-			}	break;
 
 			case ZT_EVENT_TRACE: {
 				if (metaData) {
@@ -1830,6 +1823,7 @@ public:
 
 			default:
 				break;
+
 		}
 	}