A ton more work... almost there

include/ZeroTierCore.h

@@ -419,183 +419,44 @@ enum ZT_TraceCredentialRejectionReason
 	ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID = 4
 };
 
-#if 0
-
-/**
- * Physical path address from a trace event
- *
- * This is a special packed address format that roughly mirrors Endpoint in the core
- * and is designed to support both present and future address types.
- */
-ZT_PACKED_STRUCT(struct ZT_TraceEventPathAddress
-{
-	uint8_t type;        /* ZT_TraceEventPathAddressType */
-	uint8_t address[63]; /* Type-dependent address: 4-byte IPv4, 16-byte IPV6, etc. */
-	uint16_t port;       /* UDP/TCP port for address types for which this is meaningful */
-});
-
-/**
- * Header for all trace events
- *
- * All packet types begin with these fields in this order.
- */
-ZT_PACKED_STRUCT(struct ZT_TraceEvent
-{
-	uint16_t evSize;                                  /* sizeof(ZT_TraceEvent_XX structure) (inclusive) */
-	uint16_t evType;                                  /* ZT_TraceEventType */
-	uint32_t codeLocation;                            /* arbitrary identifier of location in source code */
-});
-
-/* Temporary macros to make it easier to declare all ZT_TraceEvent's sub-types */
-#define _ZT_TRACE_EVENT_STRUCT_START(e) ZT_PACKED_STRUCT_START struct ZT_TraceEvent_##e { \
-	uint16_t evSize; \
-	uint16_t evType; \
-	uint32_t codeLocation;
-#define _ZT_TRACE_EVENT_STRUCT_END() } ZT_PACKED_STRUCT_END;
-
-/**
- * An unexpected or internal error occurred
- */
-_ZT_TRACE_EVENT_STRUCT_START(UNEXPECTED_ERROR)
-	char message[256];                               /* arbitrary human-readable message */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node is resetting all paths in a given address scope
- *
- * This happens when the node detects and external surface IP addressing change
- * via a trusted (usually root) peer. It's used to renegotiate links when nodes
- * move around on the network.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_RESETTING_PATHS_IN_SCOPE)
-	ZT_Fingerprint reporter;                         /* node that triggered the reset */
-	struct ZT_TraceEventPathAddress from;            /* physical origin of triggering packet */
-	struct ZT_TraceEventPathAddress oldExternal;     /* previous detected external address */
-	struct ZT_TraceEventPathAddress newExternal;     /* new detected external address */
-	uint8_t scope;                                   /* IP scope being reset */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node is trying a new path
- *
- * Paths are tried in response to PUSH_DIRECT_PATHS, RENDEZVOUS, and other places
- * we might hear of them. A node tries a path by sending a trial message to it.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_TRYING_NEW_PATH)
-	ZT_Fingerprint peer;                             /* node we're trying to reach */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical address being tried */
-	struct ZT_TraceEventPathAddress triggerAddress;  /* physical origin of triggering packet */
-	uint64_t triggeringPacketId;                     /* packet ID of triggering packet */
-	uint8_t triggeringPacketVerb;                    /* packet verb of triggering packet */
-	ZT_Fingerprint triggeringPeer;                   /* peer that triggered attempt */
-	uint8_t reason;                                  /* ZT_TraceTryingNewPathReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node has learned a new path to another node
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_LEARNED_NEW_PATH)
-	uint64_t packetId;                               /* packet ID of confirming packet */
-	ZT_Fingerprint peer;                             /* peer on other side of new path */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical address learned */
-	struct ZT_TraceEventPathAddress replaced;        /* if non-empty, an older address that was replaced */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * An incoming packet was dropped at the VL1 level
- *
- * This indicates a packet that passed MAC check but was dropped for some other
- * reason such as rate limits, being malformed, etc.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL1_INCOMING_PACKET_DROPPED)
-	uint64_t packetId;                               /* packet ID of failed packet */
-	uint64_t networkId;                              /* VL2 network ID or 0 if unrelated to a network or unknown */
-	ZT_Fingerprint peer;                             /* peer that sent packet */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical origin address of packet */
-	uint8_t hops;                                    /* hop count of packet */
-	uint8_t verb;                                    /* packet verb */
-	uint8_t reason;                                  /* ZT_TracePacketDropReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node declined to send a packet read from a local network port/tap
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_OUTGOING_FRAME_DROPPED)
-	uint64_t networkId;                              /* network ID */
-	uint64_t sourceMac;                              /* source MAC address */
-	uint64_t destMac;                                /* destination MAC address */
-	uint16_t etherType;                              /* Ethernet type of frame */
-	uint16_t frameLength;                            /* length of dropped frame */
-	uint8_t frameHead[64];                           /* first up to 64 bytes of dropped frame */
-	uint8_t reason;                                  /* ZT_TraceFrameDropReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * An incoming frame was dropped
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_INCOMING_FRAME_DROPPED)
-	uint64_t packetId;                               /* VL1 packet ID */
-	uint64_t networkId;                              /* VL2 network ID */
-	uint64_t sourceMac;                              /* 48-bit source MAC */
-	uint64_t destMac;                                /* 48-bit destination MAC */
-	ZT_Fingerprint sender;                           /* sending peer */
-	struct ZT_TraceEventPathAddress physicalAddress; /* physical source address of packet */
-	uint8_t hops;                                    /* hop count of packet */
-	uint16_t frameLength;                            /* length of frame in bytes */
-	uint8_t frameHead[64];                           /* first up to 64 bytes of dropped frame */
-	uint8_t verb;                                    /* packet verb indicating how frame was sent */
-	uint8_t credentialRequestSent;                   /* if non-zero a request for credentials was sent */
-	uint8_t reason;                                  /* ZT_TraceFrameDropReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Node is requesting a new network config and certificate from a network controller
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_NETWORK_CONFIG_REQUESTED)
-	uint64_t networkId;                              /* VL2 network ID */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * Network filter trace results
- *
- * These are generated when filter tracing is enabled to allow filters to be debugged.
- * Format for rule set logs is documented elsewhere.
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_NETWORK_FILTER)
-	uint64_t networkId;                              /* VL2 network ID */
-	uint8_t primaryRuleSetLog[512];                  /* primary rule set log */
-	uint8_t matchingCapabilityRuleSetLog[512];       /* capability rule set log (if any) */
-	uint32_t matchingCapabilityId;                   /* capability ID or 0 if none */
-	int64_t matchingCapabilityTimestamp;             /* capability timestamp or 0 if none */
-	uint64_t source;                                 /* source ZeroTier address */
-	uint64_t dest;                                   /* destination ZeroTier address */
-	uint64_t sourceMac;                              /* packet source MAC */
-	uint64_t destMac;                                /* packet destination MAC */
-	uint16_t frameLength;                            /* length of filtered frame */
-	uint8_t frameHead[64];                           /* first up to 64 bytes of filtered frame */
-	uint16_t etherType;                              /* frame Ethernet type */
-	uint16_t vlanId;                                 /* frame VLAN ID (currently unused, always 0) */
-	uint8_t noTee;                                   /* if true noTee flag was set in filter */
-	uint8_t inbound;                                 /* direction: 1 for inbound, 0 for outbound */
-	int8_t accept;                                   /* 0: drop, 1: accept, 2: "super-accept" */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-/**
- * An incoming credential from a peer was rejected
- */
-_ZT_TRACE_EVENT_STRUCT_START(VL2_CREDENTIAL_REJECTED)
-	uint64_t networkId;                              /* VL2 network ID */
-	ZT_Fingerprint peer;                             /* sending peer */
-	uint32_t credentialId;                           /* credential ID */
-	int64_t credentialTimestamp;                     /* credential timestamp */
-	uint8_t credentialType;                          /* credential type */
-	uint8_t reason;                                  /* ZT_TraceCredentialRejectionReason */
-_ZT_TRACE_EVENT_STRUCT_END()
-
-#undef _ZT_TRACE_EVENT_STRUCT_START
-#undef _ZT_TRACE_EVENT_STRUCT_END
-
-#endif
+// Fields used in trace output dictionaries. Which fields are present depends on
+// the trace event type. All trace dictionaries contain TYPE and CODE_LOCATION.
+#define ZT_TRACE_FIELD_TYPE "t"
+#define ZT_TRACE_FIELD_CODE_LOCATION "@"
+#define ZT_TRACE_FIELD_ENDPOINT "e"
+#define ZT_TRACE_FIELD_OLD_ENDPOINT "oe"
+#define ZT_TRACE_FIELD_NEW_ENDPOINT "ne"
+#define ZT_TRACE_FIELD_TRIGGER_FROM_ENDPOINT "te"
+#define ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_ID "ti"
+#define ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_VERB "tv"
+#define ZT_TRACE_FIELD_TRIGGER_FROM_PEER_FINGERPRINT_HASH "tp"
+#define ZT_TRACE_FIELD_MESSAGE "m"
+#define ZT_TRACE_FIELD_RESET_ADDRESS_SCOPE "rs"
+#define ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH "f"
+#define ZT_TRACE_FIELD_PACKET_ID "p"
+#define ZT_TRACE_FIELD_PACKET_VERB "v"
+#define ZT_TRACE_FIELD_PACKET_HOPS "h"
+#define ZT_TRACE_FIELD_NETWORK_ID "n"
+#define ZT_TRACE_FIELD_REASON "r"
+#define ZT_TRACE_FIELD_SOURCE_MAC "sm"
+#define ZT_TRACE_FIELD_DEST_MAC "dm"
+#define ZT_TRACE_FIELD_ETHERTYPE "et"
+#define ZT_TRACE_FIELD_VLAN_ID "vlid"
+#define ZT_TRACE_FIELD_FRAME_LENGTH "fl"
+#define ZT_TRACE_FIELD_FRAME_DATA "fd"
+#define ZT_TRACE_FIELD_FLAG_CREDENTIAL_REQUEST_SENT "crs"
+#define ZT_TRACE_FIELD_PRIMARY_RULE_SET_LOG "rL"
+#define ZT_TRACE_FIELD_MATCHING_CAPABILITY_RULE_SET_LOG "caRL"
+#define ZT_TRACE_FIELD_MATCHING_CAPABILITY_ID "caID"
+#define ZT_TRACE_FIELD_MATCHING_CAPABILITY_TIMESTAMP "caTS"
+#define ZT_TRACE_FIELD_SOURCE_ZT_ADDRESS "sz"
+#define ZT_TRACE_FIELD_DEST_ZT_ADDRESS "dz"
+#define ZT_TRACE_FIELD_RULE_FLAG_NOTEE "rNT"
+#define ZT_TRACE_FIELD_RULE_FLAG_INBOUND "rIN"
+#define ZT_TRACE_FIELD_RULE_FLAG_ACCEPT "rACC"
+#define ZT_TRACE_FIELD_CREDENTIAL_ID "crID"
+#define ZT_TRACE_FIELD_CREDENTIAL_TYPE "crT"
+#define ZT_TRACE_FIELD_CREDENTIAL_TIMESTAMP "crTS"
 
 /****************************************************************************/
 

node/AES.hpp

@@ -56,10 +56,7 @@ public:
 	/**
 	 * Create an un-initialized AES instance (must call init() before use)
 	 */
-	ZT_INLINE AES() noexcept
-	{
-		Utils::memoryLock(this,sizeof(AES));
-	}
+	ZT_INLINE AES() noexcept {}
 
 	/**
 	 * Create an AES instance with the given key
@@ -68,14 +65,12 @@ public:
 	 */
 	explicit ZT_INLINE AES(const void *const key) noexcept
 	{
-		Utils::memoryLock(this,sizeof(AES));
 		this->init(key);
 	}
 
 	ZT_INLINE ~AES()
 	{
 		Utils::burn(&_k,sizeof(_k));
-		Utils::memoryUnlock(this,sizeof(AES));
 	}
 
 	/**

node/Address.hpp

@@ -19,7 +19,9 @@
 #include "TriviallyCopyable.hpp"
 #include "Containers.hpp"
 
-#define ZT_ADDRESS_STRING_SIZE_MAX 11
+#define ZT_ADDRESS_STRING_SIZE_MAX (ZT_ADDRESS_LENGTH_HEX + 1)
+
+static_assert(ZT_ADDRESS_LENGTH == 5,"parts of Address will need modification for any change in ZT_ADDRESS_LENGTH");
 
 namespace ZeroTier {
 

node/Buf.hpp

@@ -86,6 +86,15 @@ public:
 	static void *operator new(std::size_t sz);
 	static void operator delete(void *ptr);
 
+	/**
+	 * Raw data held in buffer
+	 *
+	 * The additional eight bytes should not be used and should be considered undefined.
+	 * They exist to allow reads and writes of integer types to silently overflow if a
+	 * read or write is performed at the end of the buffer.
+	 */
+	uint8_t unsafeData[ZT_BUF_MEM_SIZE + 8];
+
 	/**
 	 * Free all instances of Buf in shared pool.
 	 *
@@ -129,47 +138,82 @@ public:
 	};
 
 	/**
-	 * Assemble all slices in a vector into a single slice starting at position 0
-	 *
-	 * The returned slice will start at 0 and contain the entire vector unless the
-	 * vector is too large to fit in a single buffer. If that or any other error
-	 * occurs the returned slice will be empty and contain a NULL Buf.
-	 *
-	 * The vector may be modified by this function and should be considered
-	 * undefined after it is called.
-	 *
-	 * @tparam FCVC Capacity of FCV (generally inferred automatically)
-	 * @param fcv FCV containing one or more slices
-	 * @return Single slice containing fully assembled buffer (empty on error)
+	 * A vector of slices making up a packet that might span more than one buffer.
 	 */
-	template<unsigned int FCVC>
-	static ZT_INLINE Buf::Slice assembleSliceVector(FCV<Buf::Slice,FCVC> &fcv) noexcept
+	class PacketVector : public ZeroTier::FCV<Slice,ZT_MAX_PACKET_FRAGMENTS>
 	{
-		Buf::Slice r;
-
-		typename FCV<Buf::Slice,FCVC>::iterator s(fcv.begin());
-		unsigned int l = s->e - s->s;
-		if (l <= ZT_BUF_MEM_SIZE) {
-			r.b.move(s->b);
-			if (s->s > 0)
-				memmove(r.b->unsafeData,r.b->unsafeData + s->s,l);
-			r.e = l;
+	public:
+		ZT_INLINE PacketVector() : ZeroTier::FCV<Slice,ZT_MAX_PACKET_FRAGMENTS>() {}
+
+		ZT_INLINE unsigned int totalSize() const noexcept
+		{
+			unsigned int size = 0;
+			for(PacketVector::const_iterator s(begin());s!=end();++s)
+				size += s->e - s->s;
+			return size;
+		}
 
-			while (++s != fcv.end()) {
-				l = s->e - s->s;
-				if (l > (ZT_BUF_MEM_SIZE - r.e)) {
-					r.b.zero();
-					r.e = 0;
-					break;
+		/**
+		 * Merge this packet vector into a single destination buffer
+		 * 
+		 * @param b Destination buffer
+		 * @return Size of data in destination or -1 on error
+		 */
+		ZT_INLINE int mergeCopy(Buf &b) const noexcept
+		{
+			unsigned int size = 0;
+			for(PacketVector::const_iterator s(begin());s!=end();++s) {
+				const unsigned int start = s->s;
+				const unsigned int rem = s->e - start;
+				if (likely((size + rem) <= ZT_BUF_MEM_SIZE)) {
+					Utils::copy(b.unsafeData + size,s->b->unsafeData + start,rem);
+					size += rem;
+				} else {
+					return -1;
 				}
-				Utils::copy(r.b->unsafeData + r.e,s->b->unsafeData + s->s,l);
-				s->b.zero(); // let go of buffer in vector as soon as possible
-				r.e += l;
 			}
+			return (int)size;
 		}
 
-		return r;
-	}
+		/**
+		 * Merge this packet vector into a single destination buffer with an arbitrary copy function
+		 * 
+		 * This can be used to e.g. simultaneously merge and decrypt a packet.
+		 * 
+		 * @param b Destination buffer
+		 * @param simpleCopyBefore Don't start using copyFunction until this index (0 to always use)
+		 * @param copyFunction Function to invoke with memcpy-like arguments: (dest, source, size)
+		 * @tparam F Type of copyFunction (typically inferred)
+		 * @return Size of data in destination or -1 on error
+		 */
+		template<typename F>
+		ZT_INLINE int mergeMap(Buf &b,const unsigned int simpleCopyBefore,F copyFunction) const noexcept
+		{
+			unsigned int size = 0;
+			for(PacketVector::const_iterator s(begin());s!=end();++s) {
+				unsigned int start = s->s;
+				unsigned int rem = s->e - start;
+				if (likely((size + rem) <= ZT_BUF_MEM_SIZE)) {
+					if (size < simpleCopyBefore) {
+						unsigned int sc = simpleCopyBefore - size;
+						if (unlikely(sc > rem))
+							sc = rem;
+						Utils::copy(b.unsafeData + size,s->b->unsafeData + start,sc);
+						start += sc;
+						rem -= sc;
+					}
+
+					if (likely(rem > 0)) {
+						copyFunction(b.unsafeData + size,s->b->unsafeData + start,rem);
+						size += rem;
+					}
+				} else {
+					return -1;
+				}
+			}
+			return (int)size;
+		}
+	};
 
 	/**
 	 * Create a new uninitialized buffer with undefined contents (use clear() to zero if needed)
@@ -421,6 +465,84 @@ public:
 		return ((ii += (int)len) <= ZT_BUF_MEM_SIZE) ? b : nullptr;
 	}
 
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI8() const noexcept
+	{
+		static_assert(I < ZT_BUF_MEM_SIZE,"overflow");
+		return unsafeData[I];
+	}
+
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI16() const noexcept
+	{
+		static_assert((I + 1) < ZT_BUF_MEM_SIZE,"overflow");
+#ifdef ZT_NO_UNALIGNED_ACCESS
+		return (
+			((uint16_t)unsafeData[I] << 8U) |
+			(uint16_t)unsafeData[I + 1]);
+#else
+		return Utils::ntoh(*reinterpret_cast<const uint16_t *>(unsafeData + I));
+#endif
+	}
+
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI32() const noexcept
+	{
+		static_assert((I + 3) < ZT_BUF_MEM_SIZE,"overflow");
+#ifdef ZT_NO_UNALIGNED_ACCESS
+		return (
+			((uint32_t)unsafeData[I] << 24U) |
+			((uint32_t)unsafeData[I + 1] << 16U) |
+			((uint32_t)unsafeData[I + 2] << 8U) |
+			(uint32_t)unsafeData[I + 3]);
+#else
+		return Utils::ntoh(*reinterpret_cast<const uint32_t *>(unsafeData + I));
+#endif
+	}
+
+	/**
+	 * Load a value at an index that is compile time checked against the maximum buffer size
+	 * 
+	 * @tparam I Static index
+	 * @return Value
+	 */
+	template<unsigned int I>
+	ZT_INLINE uint8_t lI64() const noexcept
+	{
+		static_assert((I + 7) < ZT_BUF_MEM_SIZE,"overflow");
+#ifdef ZT_NO_UNALIGNED_ACCESS
+		return (
+			((uint64_t)unsafeData[I] << 56U) |
+			((uint64_t)unsafeData[I + 1] << 48U) |
+			((uint64_t)unsafeData[I + 2] << 40U) |
+			((uint64_t)unsafeData[I + 3] << 32U) |
+			((uint64_t)unsafeData[I + 4] << 24U) |
+			((uint64_t)unsafeData[I + 5] << 16U) |
+			((uint64_t)unsafeData[I + 6] << 8U) |
+			(uint64_t)unsafeData[I + 7]);
+#else
+		return Utils::ntoh(*reinterpret_cast<const uint64_t *>(unsafeData + I));
+#endif
+	}
+
 	/**
 	 * Load a value at an index without advancing the index
 	 *
@@ -688,43 +810,6 @@ public:
 	 */
 	static constexpr unsigned int capacity() noexcept { return ZT_BUF_MEM_SIZE; }
 
-	/**
-	 * Cast data in 'b' to a (usually packed) structure type
-	 *
-	 * Warning: this does no bounds checking. It should only be used with packed
-	 * struct types designed for use in packet decoding such as those in
-	 * Protocol.hpp, and if 'i' is non-zero the caller must check bounds.
-	 *
-	 * @tparam T Structure type to cast 'b' to
-	 * @param i Index of start of structure (default: 0)
-	 * @return Reference to 'b' cast to type T
-	 */
-	template<typename T>
-	ZT_INLINE T &as(const unsigned int i = 0) noexcept { return *reinterpret_cast<T *>(unsafeData + i); }
-
-	/**
-	 * Cast data in 'b' to a (usually packed) structure type (const)
-	 *
-	 * Warning: this does no bounds checking. It should only be used with packed
-	 * struct types designed for use in packet decoding such as those in
-	 * Protocol.hpp, and if 'i' is non-zero the caller must check bounds.
-	 *
-	 * @tparam T Structure type to cast 'b' to
-	 * @param i Index of start of structure (default: 0)
-	 * @return Reference to 'b' cast to type T
-	 */
-	template<typename T>
-	ZT_INLINE const T &as(const unsigned int i = 0) const noexcept { return *reinterpret_cast<const T *>(unsafeData + i); }
-
-	/**
-	 * Raw data held in buffer
-	 *
-	 * The additional eight bytes should not be used and should be considered undefined.
-	 * They exist to allow reads and writes of integer types to silently overflow if a
-	 * read or write is performed at the end of the buffer.
-	 */
-	uint8_t unsafeData[ZT_BUF_MEM_SIZE + 8];
-
 private:
 	// Next item in free buffer pool linked list if Buf is placed in pool, undefined and unused otherwise
 	std::atomic<uintptr_t> __nextInPool;

node/CMakeLists.txt

@@ -74,7 +74,6 @@ set(core_src
 	Path.cpp
 	Peer.cpp
 	Poly1305.cpp
-	Protocol.cpp
 	Revocation.cpp
 	Salsa20.cpp
 	SelfAwareness.cpp

node/Constants.hpp

@@ -36,6 +36,11 @@
  */
 #define ZT_ADDRESS_LENGTH 5
 
+/**
+ * Length of a ZeroTier address in digits
+ */
+#define ZT_ADDRESS_LENGTH_HEX 10
+
 /**
  * Addresses beginning with this byte are reserved for the joy of in-band signaling
  */
@@ -72,9 +77,9 @@
 #define ZT_MAX_NETWORK_CONFIG_BYTES 131072
 
 /**
- * Length of symmetric keys (currently all symmetric crypto is 256 bit).
+ * Length of symmetric keys
  */
-#define ZT_SYMMETRIC_KEY_SIZE 32
+#define ZT_SYMMETRIC_KEY_SIZE 48
 
 /**
  * Time limit for ephemeral keys: 30 minutes.
@@ -202,6 +207,11 @@
  */
 #define ZT_PEER_GENERAL_RATE_LIMIT 500
 
+/**
+ * Rate limit for responses to short probes to prevent amplification attacks
+ */
+#define ZT_PEER_PROBE_RESPONSE_RATE_LIMIT 5000
+
 /**
  * Don't do expensive identity validation more often than this
  *

node/Dictionary.cpp

@@ -19,70 +19,46 @@ Dictionary::Dictionary()
 {
 }
 
-std::vector<uint8_t> &Dictionary::operator[](const char *k)
+Vector<uint8_t> &Dictionary::operator[](const char *k)
 {
 	return m_entries[s_toKey(k)];
 }
 
-const std::vector<uint8_t> &Dictionary::operator[](const char *k) const
+const Vector<uint8_t> &Dictionary::operator[](const char *k) const
 {
-	static const std::vector<uint8_t> emptyEntry;
-	Map< uint64_t,std::vector<uint8_t> >::const_iterator e(m_entries.find(s_toKey(k)));
-	return (e == m_entries.end()) ? emptyEntry : e->second;
+	static const Vector<uint8_t> s_emptyEntry;
+	Map< uint64_t,Vector<uint8_t> >::const_iterator e(m_entries.find(s_toKey(k)));
+	return (e == m_entries.end()) ? s_emptyEntry : e->second;
 }
 
 void Dictionary::add(const char *k,bool v)
 {
-	std::vector<uint8_t> &e = (*this)[k];
+	Vector<uint8_t> &e = (*this)[k];
 	e.resize(2);
 	e[0] = (uint8_t)(v ? '1' : '0');
 	e[1] = 0;
 }
 
-void Dictionary::add(const char *k,uint16_t v)
-{
-	std::vector<uint8_t> &e = (*this)[k];
-	e.resize(5);
-	Utils::hex(v,(char *)e.data());
-}
-
-void Dictionary::add(const char *k,uint32_t v)
-{
-	std::vector<uint8_t> &e = (*this)[k];
-	e.resize(9);
-	Utils::hex(v,(char *)e.data());
-}
-
-void Dictionary::add(const char *k,uint64_t v)
-{
-	std::vector<uint8_t> &e = (*this)[k];
-	e.resize(17);
-	Utils::hex(v,(char *)e.data());
-}
-
 void Dictionary::add(const char *k,const Address &v)
 {
-	std::vector<uint8_t> &e = (*this)[k];
+	Vector<uint8_t> &e = (*this)[k];
 	e.resize(ZT_ADDRESS_STRING_SIZE_MAX);
 	v.toString((char *)e.data());
 }
 
 void Dictionary::add(const char *k,const char *v)
 {
-	std::vector<uint8_t> &e = (*this)[k];
-	e.clear();
-	if (v) {
-		for(;;) {
-			const uint8_t c = (uint8_t)*(v++);
-			e.push_back(c);
-			if (!c) break;
-		}
+	if ((v)&&(*v)) {
+		Vector<uint8_t> &e = (*this)[k];
+		e.clear();
+		while (*v)
+			e.push_back((uint8_t)*(v++));
 	}
 }
 
 void Dictionary::add(const char *k,const void *data,unsigned int len)
 {
-	std::vector<uint8_t> &e = (*this)[k];
+	Vector<uint8_t> &e = (*this)[k];
 	if (len != 0) {
 		e.assign((const uint8_t *)data,(const uint8_t *)data + len);
 	} else {
@@ -92,7 +68,7 @@ void Dictionary::add(const char *k,const void *data,unsigned int len)
 
 bool Dictionary::getB(const char *k,bool dfl) const
 {
-	const std::vector<uint8_t> &e = (*this)[k];
+	const Vector<uint8_t> &e = (*this)[k];
 	if (!e.empty()) {
 		switch ((char)e[0]) {
 			case '1':
@@ -112,7 +88,7 @@ uint64_t Dictionary::getUI(const char *k,uint64_t dfl) const
 {
 	uint8_t tmp[18];
 	uint64_t v = dfl;
-	const std::vector<uint8_t> &e = (*this)[k];
+	const Vector<uint8_t> &e = (*this)[k];
 	if (!e.empty()) {
 		if (e.back() != 0) {
 			const unsigned long sl = e.size();
@@ -125,11 +101,11 @@ uint64_t Dictionary::getUI(const char *k,uint64_t dfl) const
 	return v;
 }
 
-void Dictionary::getS(const char *k,char *v,unsigned int cap) const
+void Dictionary::getS(const char *k,char *v,const unsigned int cap) const
 {
 	if (cap == 0) // sanity check
 		return;
-	const std::vector<uint8_t> &e = (*this)[k];
+	const Vector<uint8_t> &e = (*this)[k];
 	unsigned int i = 0;
 	const unsigned int last = cap - 1;
 	for(;;) {
@@ -148,52 +124,14 @@ void Dictionary::clear()
 
 void Dictionary::encode(Vector<uint8_t> &out) const
 {
-	uint64_t str[2] = { 0,0 }; // second entry causes all strings to be null-terminated even if 8 chars in length
-
+	uint64_t str[2] = { 0,0 }; // second uint64_t being 0 means all strings always 0-terminated
 	out.clear();
-
-	for(Map< uint64_t,std::vector<uint8_t> >::const_iterator ti(m_entries.begin());ti != m_entries.end();++ti) {
+	for(Map< uint64_t,Vector<uint8_t> >::const_iterator ti(m_entries.begin());ti != m_entries.end();++ti) {
 		str[0] = ti->first;
-		const char *k = (const char *)str;
-		for(;;) {
-			char kc = *(k++);
-			if (!kc) break;
-			if ((kc >= 33)&&(kc <= 126)&&(kc != 61)&&(kc != 92)) // printable ASCII with no spaces, equals, or backslash
-				out.push_back((uint8_t)kc);
-		}
-
-		out.push_back(61); // =
-
-		for(std::vector<uint8_t>::const_iterator i(ti->second.begin());i!=ti->second.end();++i) {
-			uint8_t c = *i;
-			switch(c) {
-				case 0:
-					out.push_back(92);
-					out.push_back(48);
-					break;
-				case 10:
-					out.push_back(92);
-					out.push_back(110);
-					break;
-				case 13:
-					out.push_back(92);
-					out.push_back(114);
-					break;
-				case 61:
-					out.push_back(92);
-					out.push_back(101);
-					break;
-				case 92:
-					out.push_back(92);
-					out.push_back(92);
-					break;
-				default:
-					out.push_back(c);
-					break;
-			}
-		}
-
-		out.push_back(10);
+		s_appendKey(out,reinterpret_cast<const char *>(str));
+		for(std::vector<uint8_t>::const_iterator i(ti->second.begin());i!=ti->second.end();++i)
+			s_appendValueByte(out,*i);
+		out.push_back((uint8_t)'\n');
 	}
 }
 
@@ -203,7 +141,7 @@ bool Dictionary::decode(const void *data,unsigned int len)
 
 	uint64_t k = 0;
 	unsigned int ki = 0;
-	std::vector<uint8_t> *v = nullptr;
+	Vector<uint8_t> *v = nullptr;
 	bool escape = false;
 	for(unsigned int di=0;di<len;++di) {
 		uint8_t c = reinterpret_cast<const uint8_t *>(data)[di];
@@ -229,11 +167,11 @@ bool Dictionary::decode(const void *data,unsigned int len)
 						break;
 				}
 			} else {
-				if (c == 10) {
+				if (c == (uint8_t)'\n') {
 					k = 0;
 					ki = 0;
 					v = nullptr;
-				} else if (c == 92) {
+				} else if (c == 92) { // backslash
 					escape = true;
 				} else {
 					v->push_back(c);
@@ -242,7 +180,7 @@ bool Dictionary::decode(const void *data,unsigned int len)
 		} else {
 			if ((c < 33)||(c > 126)||(c == 92)) {
 				return false;
-			} else if (c == 61) {
+			} else if (c == (uint8_t)'=') {
 				v = &m_entries[k];
 			} else {
 				reinterpret_cast<uint8_t *>(&k)[ki & 7U] ^= c;

node/Dictionary.hpp

@@ -35,6 +35,9 @@ namespace ZeroTier {
  * of simple or standardized binary encoding. Nevertheless it is efficient
  * and it works so there is no need to change it and break backward
  * compatibility.
+ * 
+ * Use of the append functions is faster than building and then encoding a
+ * dictionary.
  */
 class Dictionary
 {
@@ -47,7 +50,7 @@ public:
 	 * @param k Key to look up
 	 * @return Reference to value
 	 */
-	std::vector<uint8_t> &operator[](const char *k);
+	Vector<uint8_t> &operator[](const char *k);
 
 	/**
 	 * Get a const reference to a value
@@ -55,7 +58,7 @@ public:
 	 * @param k Key to look up
 	 * @return Reference to value or to empty vector if not found
 	 */
-	const std::vector<uint8_t> &operator[](const char *k) const;
+	const Vector<uint8_t> &operator[](const char *k) const;
 
 	/**
 	 * Add a boolean as '1' or '0'
@@ -65,21 +68,12 @@ public:
 	/**
 	 * Add an integer as a hexadecimal string value
 	 */
-	void add(const char *k,uint16_t v);
-
-	/**
-	 * Add an integer as a hexadecimal string value
-	 */
-	void add(const char *k,uint32_t v);
-
-	/**
-	 * Add an integer as a hexadecimal string value
-	 */
-	void add(const char *k,uint64_t v);
-
-	ZT_INLINE void add(const char *k,int16_t v) { add(k,(uint16_t)v); }
-	ZT_INLINE void add(const char *k,int32_t v) { add(k,(uint32_t)v); }
-	ZT_INLINE void add(const char *k,int64_t v) { add(k,(uint64_t)v); }
+	ZT_INLINE void add(const char *const k,const uint64_t v) { char buf[17]; add(k,Utils::hex(v,buf)); }
+	ZT_INLINE void add(const char *const k,const int64_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
+	ZT_INLINE void add(const char *const k,const uint32_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
+	ZT_INLINE void add(const char *const k,const int32_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
+	ZT_INLINE void add(const char *const k,const uint16_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
+	ZT_INLINE void add(const char *const k,const int16_t v) { char buf[17]; add(k,Utils::hex((uint64_t)v,buf)); }
 
 	/**
 	 * Add an address in 10-digit hex string format
@@ -126,6 +120,24 @@ public:
 	 */
 	void getS(const char *k,char *v,unsigned int cap) const;
 
+	/**
+	 * Get an object supporting the marshal/unmarshal interface pattern
+	 * 
+	 * @param k Key to look up
+	 * @param obj Object to unmarshal() into
+	 * @return True if unmarshal was successful
+	 */
+	template<typename T>
+	ZT_INLINE bool getO(const char *k,T &obj) const
+	{
+		const Vector<uint8_t> &d = (*this)[k];
+		if (d.empty())
+			return false;
+		if (obj.unmarshal(d.data(),(unsigned int)d.size()) <= 0)
+			return false;
+		return true;
+	}
+
 	/**
 	 * Erase all entries in dictionary
 	 */
@@ -163,7 +175,190 @@ public:
 	 */
 	bool decode(const void *data,unsigned int len);
 
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const bool v)
+	{
+		s_appendKey(out,k);
+		out.push_back((uint8_t)(v ? '1' : '0'));
+		out.push_back((uint8_t)'\n');
+	}
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const Address v)
+	{
+		s_appendKey(out,k);
+		const uint64_t a = v.toInt();
+		static_assert(ZT_ADDRESS_LENGTH_HEX == 10,"this must be rewritten for any change in address length");
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 36U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 32U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 28U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 24U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 20U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 16U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 12U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 8U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[(a >> 4U) & 0xfU]);
+		out.push_back((uint8_t)Utils::HEXCHARS[a & 0xfU]);
+		out.push_back((uint8_t)'\n');
+	}
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint64_t v)
+	{
+		char buf[17];
+		Utils::hex(v,buf);
+		unsigned int i = 0;
+		while (buf[i])
+			out.push_back((uint8_t)buf[i++]);
+		out.push_back((uint8_t)'\n');
+	}
+
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int64_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint32_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int32_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint16_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int16_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const uint8_t v) { append(out,k,(uint64_t)v); }
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const int8_t v) { append(out,k,(uint64_t)v); }
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const char *v)
+	{
+		if ((v)&&(*v)) {
+			s_appendKey(out,k);
+			while (*v)
+				s_appendValueByte(out,(uint8_t)*(v++));
+			out.push_back((uint8_t)'\n');
+		}
+	}
+
+	/**
+	 * Append a key=value pair to a buffer (vector or FCV)
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Value
+	 * @param vlen Value length in bytes
+	 */
+	template<typename V>
+	ZT_INLINE static void append(V &out,const char *const k,const void *const v,const unsigned int vlen)
+	{
+		s_appendKey(out,k);
+		for(unsigned int i=0;i<vlen;++i)
+			s_appendValueByte(out,reinterpret_cast<const uint8_t *>(v)[i]);
+		out.push_back((uint8_t)'\n');
+	}
+
+	/**
+	 * Append a packet ID as raw bytes in the provided byte order
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param pid Packet ID
+	 */
+	template<typename V>
+	static ZT_INLINE void appendPacketId(V &out,const char *const k,const uint64_t pid)
+	{
+		append(out,k,&pid,8);
+	}
+
+	/**
+	 * Append key=value with any object implementing the correct marshal interface
+	 * 
+	 * @param out Buffer
+	 * @param k Key (must be <= 8 characters)
+	 * @param v Marshal-able object
+	 * @return Bytes appended or negative on error (return value of marshal())
+	 */
+	template<typename V,typename T>
+	static ZT_INLINE int appendObject(V &out,const char *const k,const T &v)
+	{
+		uint8_t tmp[4096]; // large enough for any current object
+		if (T::marshalSizeMax() > sizeof(tmp))
+			return -1;
+		const int mlen = v.marshal(tmp);
+		if (mlen > 0)
+			append(out,k,tmp,(unsigned int)mlen);
+		return mlen;
+	}
+
 private:
+	template<typename V>
+	ZT_INLINE static void s_appendValueByte(V &out,const uint8_t c)
+	{
+		switch(c) {
+			case 0:
+				out.push_back(92); // backslash
+				out.push_back(48);
+				break;
+			case 10:
+				out.push_back(92);
+				out.push_back(110);
+				break;
+			case 13:
+				out.push_back(92);
+				out.push_back(114);
+				break;
+			case 61:
+				out.push_back(92);
+				out.push_back(101);
+				break;
+			case 92:
+				out.push_back(92);
+				out.push_back(92);
+				break;
+			default:
+				out.push_back(c);
+				break;
+		}
+	}
+	template<typename V>
+	ZT_INLINE static void s_appendKey(V &out,const char *const k)
+	{
+		for(unsigned int i=0;i<8;++i) {
+			const char kc = k[i];
+			if (!kc) break;
+			if ((kc >= 33)&&(kc <= 126)&&(kc != 61)&&(kc != 92)) // printable ASCII with no spaces, equals, or backslash
+				out.push_back((uint8_t)kc);
+		}
+		out.push_back((uint8_t)'=');
+	}
+
 	// This just packs up to 8 character bytes into a 64-bit word. There is no need
 	// for this to be portable in terms of endian-ness. It's just for fast key lookup.
 	static ZT_INLINE uint64_t s_toKey(const char *k)

node/FCV.hpp

@@ -128,6 +128,8 @@ public:
 
 	ZT_INLINE unsigned int size() const noexcept { return _s; }
 	ZT_INLINE bool empty() const noexcept { return (_s == 0); }
+	ZT_INLINE T *data() noexcept { return reinterpret_cast<T *>(_m); }
+	ZT_INLINE const T *data() const noexcept { return reinterpret_cast<const T *>(_m); }
 	static constexpr unsigned int capacity() noexcept { return C; }
 
 	/**
@@ -144,7 +146,7 @@ public:
 	}
 
 	/**
-	 * Push a new value onto the vector and return it, or return last item if capacity is reached
+	 * Push new default value or return last in vector if full.
 	 *
 	 * @return Reference to new item
 	 */
@@ -158,7 +160,7 @@ public:
 	}
 
 	/**
-	 * Push a new value onto the vector and return it, or return last item if capacity is reached
+	 * Push new default value or replace and return last in vector if full.
 	 *
 	 * @return Reference to new item
 	 */
@@ -218,7 +220,7 @@ public:
 	 * @param i Index to obtain as a reference, resizing if needed
 	 * @return Reference to value at this index
 	 */
-	ZT_INLINE T &at(unsigned int i)
+	ZT_INLINE T &at(const unsigned int i)
 	{
 		if (i >= _s) {
 			if (unlikely(i >= C))

node/Fingerprint.hpp

@@ -84,7 +84,7 @@ public:
 	}
 
 	ZT_INLINE void zero() noexcept { memoryZero(this); }
-	ZT_INLINE unsigned long hashCode() const noexcept { return m_cfp.address; }
+	ZT_INLINE unsigned long hashCode() const noexcept { return (unsigned long)m_cfp.address; }
 
 	ZT_INLINE operator bool() const noexcept { return (m_cfp.address != 0); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
 

node/Identity.cpp

@@ -75,7 +75,7 @@ struct identityV0ProofOfWorkCriteria
 };
 
 // This is a simpler memory-intensive hash function for V1 identity generation.
-// It's not quite as intensive as the V0 frankenhash, is a little more orderly in
+// It's not quite as heavy as the V0 frankenhash, is a little more orderly in
 // its design, but remains relatively resistant to GPU acceleration due to memory
 // requirements for efficient computation.
 #define ZT_IDENTITY_V1_POW_MEMORY_SIZE 98304
@@ -180,17 +180,20 @@ bool Identity::generate(const Type t)
 	m_hasPrivate = true;
 
 	switch(t) {
+
 		case C25519: {
 			// Generate C25519/Ed25519 key pair whose hash satisfies a "hashcash" criterion and generate the
 			// address from the last 40 bits of this hash. This is different from the fingerprint hash for V0.
 			uint8_t digest[64];
 			char *const genmem = new char[ZT_V0_IDENTITY_GEN_MEMORY];
+			Address address;
 			do {
-				C25519::generateSatisfying(identityV0ProofOfWorkCriteria(digest,genmem), m_pub.c25519, m_priv.c25519);
-				m_address.setTo(digest + 59);
-			} while (m_address.isReserved());
+				C25519::generateSatisfying(identityV0ProofOfWorkCriteria(digest,genmem),m_pub,m_priv);
+				address.setTo(digest + 59);
+			} while (address.isReserved());
 			delete[] genmem;
-			_computeHash();
+			m_fp.m_cfp.address = address.toInt();
+			m_computeHash();
 		} break;
 
 		case P384: {
@@ -201,21 +204,20 @@ bool Identity::generate(const Type t)
 				// Loop until we pass the PoW criteria. The nonce is only 8 bits, so generate
 				// some new key material every time it wraps. The ECC384 generator is slightly
 				// faster so use that one.
-				m_pub.nonce = 0;
-				C25519::generateCombined(m_pub.c25519, m_priv.c25519);
-				ECC384GenerateKey(m_pub.p384, m_priv.p384);
+				m_pub[0] = 0; // zero nonce
+				C25519::generateCombined(m_pub + 1,m_priv + 1);
+				ECC384GenerateKey(m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
 				for(;;) {
-					if (identityV1ProofOfWorkCriteria(&m_pub, sizeof(m_pub), b))
+					if (identityV1ProofOfWorkCriteria(&m_pub,sizeof(m_pub),b))
 						break;
-					if (++m_pub.nonce == 0)
-						ECC384GenerateKey(m_pub.p384, m_priv.p384);
+					if (++m_pub[0] == 0)
+						ECC384GenerateKey(m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
 				}
 
 				// If we passed PoW then check that the address is valid, otherwise loop
 				// back around and run the whole process again.
-				_computeHash();
-				m_address.setTo(m_fp.hash());
-				if (!m_address.isReserved())
+				m_computeHash();
+				if (!m_fp.address().isReserved())
 					break;
 			}
 			free(b);
@@ -231,28 +233,27 @@ bool Identity::generate(const Type t)
 bool Identity::locallyValidate() const noexcept
 {
 	try {
-		if ((!m_address.isReserved()) && (m_address)) {
+		if ((m_fp)&&((!m_fp.address().isReserved()))) {
 			switch (m_type) {
-
 				case C25519: {
 					uint8_t digest[64];
-					char *genmem = new char[ZT_V0_IDENTITY_GEN_MEMORY];
-					identityV0ProofOfWorkFrankenhash(m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE, digest, genmem);
-					delete[] genmem;
-					return ((m_address == Address(digest + 59)) && (digest[0] < 17));
+					char *const genmem = (char *)malloc(ZT_V0_IDENTITY_GEN_MEMORY);
+					if (!genmem)
+						return false;
+					identityV0ProofOfWorkFrankenhash(m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,digest,genmem);
+					free(genmem);
+					return ((m_fp.address() == Address(digest + 59)) && (digest[0] < 17));
 				}
-
 				case P384: {
-					if (m_address != Address(m_fp.hash()))
+					if (m_fp.address() != Address(m_fp.hash()))
 						return false;
-					uint64_t *const b = (uint64_t *)malloc(ZT_IDENTITY_V1_POW_MEMORY_SIZE * 8); // NOLINT(hicpp-use-auto,modernize-use-auto)
-					if (!b)
+					uint64_t *const genmem = (uint64_t *)malloc(ZT_IDENTITY_V1_POW_MEMORY_SIZE * 8);
+					if (!genmem)
 						return false;
-					const bool ok = identityV1ProofOfWorkCriteria(&m_pub, sizeof(m_pub), b);
-					free(b);
+					const bool ok = identityV1ProofOfWorkCriteria(m_pub,sizeof(m_pub),genmem);
+					free(genmem);
 					return ok;
 				}
-
 			}
 		}
 	} catch ( ... ) {}
@@ -263,15 +264,12 @@ void Identity::hashWithPrivate(uint8_t h[ZT_FINGERPRINT_HASH_SIZE]) const
 {
 	if (m_hasPrivate) {
 		switch (m_type) {
-
 			case C25519:
-				SHA384(h, m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE, m_priv.c25519, ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
+				SHA384(h,m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv,ZT_C25519_COMBINED_PRIVATE_KEY_SIZE);
 				break;
-
 			case P384:
-				SHA384(h, &m_pub, sizeof(m_pub), &m_priv, sizeof(m_priv));
+				SHA384(h,m_pub,sizeof(m_pub),m_priv,sizeof(m_priv));
 				break;
-
 		}
 		return;
 	}
@@ -282,21 +280,19 @@ unsigned int Identity::sign(const void *data,unsigned int len,void *sig,unsigned
 {
 	if (m_hasPrivate) {
 		switch(m_type) {
-
 			case C25519:
 				if (siglen >= ZT_C25519_SIGNATURE_LEN) {
-					C25519::sign(m_priv.c25519, m_pub.c25519, data, len, sig);
+					C25519::sign(m_priv,m_pub,data,len,sig);
 					return ZT_C25519_SIGNATURE_LEN;
 				}
-
 			case P384:
 				if (siglen >= ZT_ECC384_SIGNATURE_SIZE) {
+					// SECURITY: signatures also include the public keys to further enforce their coupling.
 					uint8_t h[48];
-					SHA384(h, data, len, &m_pub, ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE); // include C25519 public key in hash
-					ECC384ECDSASign(m_priv.p384, h, (uint8_t *)sig);
+					SHA384(h,data,len,m_pub,sizeof(m_pub));
+					ECC384ECDSASign(m_priv + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,h,(uint8_t *)sig);
 					return ZT_ECC384_SIGNATURE_SIZE;
 				}
-
 		}
 	}
 	return 0;
@@ -305,18 +301,15 @@ unsigned int Identity::sign(const void *data,unsigned int len,void *sig,unsigned
 bool Identity::verify(const void *data,unsigned int len,const void *sig,unsigned int siglen) const
 {
 	switch(m_type) {
-
 		case C25519:
-			return C25519::verify(m_pub.c25519, data, len, sig, siglen);
-
+			return C25519::verify(m_pub,data,len,sig,siglen);
 		case P384:
 			if (siglen == ZT_ECC384_SIGNATURE_SIZE) {
 				uint8_t h[48];
-				SHA384(h, data, len, &m_pub, ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE);
-				return ECC384ECDSAVerify(m_pub.p384, h, (const uint8_t *)sig);
+				SHA384(h,data,len,m_pub,sizeof(m_pub));
+				return ECC384ECDSAVerify(m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,h,(const uint8_t *)sig);
 			}
 			break;
-
 	}
 	return false;
 }
@@ -327,37 +320,33 @@ bool Identity::agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) cons
 	uint8_t h[64];
 	if (m_hasPrivate) {
 		if (m_type == C25519) {
-
 			if ((id.m_type == C25519) || (id.m_type == P384)) {
 				// If we are a C25519 key we can agree with another C25519 key or with only the
 				// C25519 portion of a type 1 P-384 key.
-				C25519::agree(m_priv.c25519, id.m_pub.c25519, rawkey);
+				C25519::agree(m_priv,id.m_pub,rawkey);
 				SHA512(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE);
 				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			}
-
 		} else if (m_type == P384) {
-
 			if (id.m_type == P384) {
 				// For another P384 identity we execute DH agreement with BOTH keys and then
 				// hash the results together. For those (cough FIPS cough) who only consider
 				// P384 to be kosher, the C25519 secret can be considered a "salt"
 				// or something. For those who don't trust P384 this means the privacy of
 				// your traffic is also protected by C25519.
-				C25519::agree(m_priv.c25519, id.m_pub.c25519, rawkey);
-				ECC384ECDH(id.m_pub.p384, m_priv.p384, rawkey + ZT_C25519_ECDH_SHARED_SECRET_SIZE);
+				C25519::agree(m_priv,id.m_pub,rawkey);
+				ECC384ECDH(id.m_pub + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,m_priv + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE,rawkey + ZT_C25519_ECDH_SHARED_SECRET_SIZE);
 				SHA384(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE + ZT_ECC384_SHARED_SECRET_SIZE);
 				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			} else if (id.m_type == C25519) {
 				// If the other identity is a C25519 identity we can agree using only that type.
-				C25519::agree(m_priv.c25519, id.m_pub.c25519, rawkey);
+				C25519::agree(m_priv,id.m_pub,rawkey);
 				SHA512(h,rawkey,ZT_C25519_ECDH_SHARED_SECRET_SIZE);
 				Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,h);
 				return true;
 			}
-
 		}
 	}
 	return false;
@@ -366,42 +355,39 @@ bool Identity::agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) cons
 char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_LENGTH]) const
 {
 	char *p = buf;
-	m_address.toString(p);
+	m_fp.address().toString(p);
 	p += 10;
 	*(p++) = ':';
 
 	switch(m_type) {
-
 		case C25519: {
 			*(p++) = '0';
 			*(p++) = ':';
-			Utils::hex(m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE, p);
+			Utils::hex(m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE,p);
 			p += ZT_C25519_COMBINED_PUBLIC_KEY_SIZE * 2;
-			if ((m_hasPrivate) && (includePrivate)) {
+			if ((m_hasPrivate)&&(includePrivate)) {
 				*(p++) = ':';
-				Utils::hex(m_priv.c25519, ZT_C25519_COMBINED_PRIVATE_KEY_SIZE, p);
+				Utils::hex(m_priv,ZT_C25519_COMBINED_PRIVATE_KEY_SIZE,p);
 				p += ZT_C25519_COMBINED_PRIVATE_KEY_SIZE * 2;
 			}
 			*p = (char)0;
 			return buf;
 		}
-
 		case P384: {
 			*(p++) = '1';
 			*(p++) = ':';
-			int el = Utils::b32e((const uint8_t *)(&m_pub), sizeof(m_pub), p, (int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
+			int el = Utils::b32e(m_pub,sizeof(m_pub),p,(int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
 			if (el <= 0) return nullptr;
 			p += el;
-			if ((m_hasPrivate) && (includePrivate)) {
+			if ((m_hasPrivate)&&(includePrivate)) {
 				*(p++) = ':';
-				el = Utils::b32e((const uint8_t *)(&m_priv), sizeof(m_priv), p, (int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
+				el = Utils::b32e(m_priv,sizeof(m_priv),p,(int)(ZT_IDENTITY_STRING_BUFFER_LENGTH - (uintptr_t)(p - buf)));
 				if (el <= 0) return nullptr;
 				p += el;
 			}
 			*p = (char)0;
 			return buf;
 		}
-
 	}
 
 	return nullptr;
@@ -409,19 +395,10 @@ char *Identity::toString(bool includePrivate,char buf[ZT_IDENTITY_STRING_BUFFER_
 
 bool Identity::fromString(const char *str)
 {
-	m_fp.zero();
-	m_hasPrivate = false;
-
-	if (!str) {
-		m_address.zero();
-		return false;
-	}
-
 	char tmp[ZT_IDENTITY_STRING_BUFFER_LENGTH];
-	if (!Utils::scopy(tmp,sizeof(tmp),str)) {
-		m_address.zero();
+	memoryZero(this);
+	if ((!str)||(!Utils::scopy(tmp,sizeof(tmp),str)))
 		return false;
-	}
 
 	int fno = 0;
 	char *saveptr = nullptr;
@@ -429,9 +406,9 @@ bool Identity::fromString(const char *str)
 		switch(fno++) {
 
 			case 0:
-				m_address = Address(Utils::hexStrToU64(f));
-				if (m_address.isReserved()) {
-					m_address.zero();
+				m_fp.m_cfp.address = Utils::hexStrToU64(f) & ZT_ADDRESS_MASK;
+				if (m_fp.address().isReserved()) {
+					memoryZero(this);
 					return false;
 				}
 				break;
@@ -442,7 +419,7 @@ bool Identity::fromString(const char *str)
 				} else if ((f[0] == '1')&&(!f[1])) {
 					m_type = P384;
 				} else {
-					m_address.zero();
+					memoryZero(this);
 					return false;
 				}
 				break;
@@ -451,15 +428,15 @@ bool Identity::fromString(const char *str)
 				switch(m_type) {
 
 					case C25519:
-						if (Utils::unhex(f, strlen(f), m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) != ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) {
-							m_address.zero();
+						if (Utils::unhex(f,strlen(f),m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) != ZT_C25519_COMBINED_PUBLIC_KEY_SIZE) {
+							memoryZero(this);
 							return false;
 						}
 						break;
 
 					case P384:
-						if (Utils::b32d(f, (uint8_t *)(&m_pub), sizeof(m_pub)) != sizeof(m_pub)) {
-							m_address.zero();
+						if (Utils::b32d(f,m_pub,sizeof(m_pub)) != sizeof(m_pub)) {
+							memoryZero(this);
 							return false;
 						}
 						break;
@@ -472,8 +449,8 @@ bool Identity::fromString(const char *str)
 					switch(m_type) {
 
 						case C25519:
-							if (Utils::unhex(f, strlen(f), m_priv.c25519, ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) != ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) {
-								m_address.zero();
+							if (Utils::unhex(f,strlen(f),m_priv,ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) != ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) {
+								memoryZero(this);
 								return false;
 							} else {
 								m_hasPrivate = true;
@@ -481,8 +458,8 @@ bool Identity::fromString(const char *str)
 							break;
 
 						case P384:
-							if (Utils::b32d(f, (uint8_t *)(&m_priv), sizeof(m_priv)) != sizeof(m_priv)) {
-								m_address.zero();
+							if (Utils::b32d(f,m_priv,sizeof(m_priv)) != sizeof(m_priv)) {
+								memoryZero(this);
 								return false;
 							} else {
 								m_hasPrivate = true;
@@ -497,13 +474,13 @@ bool Identity::fromString(const char *str)
 	}
 
 	if (fno < 3) {
-		m_address.zero();
+		memoryZero(this);
 		return false;
 	}
 
-	_computeHash();
-	if ((m_type == P384) && (m_address != Address(m_fp.hash()))) {
-		m_address.zero();
+	m_computeHash();
+	if ((m_type == P384)&&(m_fp.address() != Address(m_fp.hash()))) {
+		memoryZero(this);
 		return false;
 	}
 
@@ -512,14 +489,15 @@ bool Identity::fromString(const char *str)
 
 int Identity::marshal(uint8_t data[ZT_IDENTITY_MARSHAL_SIZE_MAX],const bool includePrivate) const noexcept
 {
-	m_address.copyTo(data);
+	m_fp.address().copyTo(data);
 	switch(m_type) {
+
 		case C25519:
 			data[ZT_ADDRESS_LENGTH] = (uint8_t)C25519;
-			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1, m_pub.c25519);
+			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1,m_pub);
 			if ((includePrivate)&&(m_hasPrivate)) {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE] = ZT_C25519_COMBINED_PRIVATE_KEY_SIZE;
-				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1, m_priv.c25519);
+				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1,m_priv);
 				return ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1 + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE;
 			} else {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE] = 0;
@@ -528,10 +506,10 @@ int Identity::marshal(uint8_t data[ZT_IDENTITY_MARSHAL_SIZE_MAX],const bool incl
 
 		case P384:
 			data[ZT_ADDRESS_LENGTH] = (uint8_t)P384;
-			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1,&m_pub);
+			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1,m_pub);
 			if ((includePrivate)&&(m_hasPrivate)) {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE] = ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE;
-				Utils::copy<ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1,&m_priv);
+				Utils::copy<ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE>(data + ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1,m_priv);
 				return ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1 + ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE;
 			} else {
 				data[ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE] = 0;
@@ -544,12 +522,11 @@ int Identity::marshal(uint8_t data[ZT_IDENTITY_MARSHAL_SIZE_MAX],const bool incl
 
 int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 {
-	m_fp.zero();
-	m_hasPrivate = false;
+	memoryZero(this);
 
 	if (len < (1 + ZT_ADDRESS_LENGTH))
 		return -1;
-	m_address.setTo(data);
+	m_fp.m_cfp.address = Address(data).toInt();
 
 	unsigned int privlen;
 	switch((m_type = (Type)data[ZT_ADDRESS_LENGTH])) {
@@ -558,15 +535,15 @@ int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 			if (len < (ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1))
 				return -1;
 
-			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(m_pub.c25519, data + ZT_ADDRESS_LENGTH + 1);
-			_computeHash();
+			Utils::copy<ZT_C25519_COMBINED_PUBLIC_KEY_SIZE>(m_pub,data + ZT_ADDRESS_LENGTH + 1);
+			m_computeHash();
 
 			privlen = data[ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE];
 			if (privlen == ZT_C25519_COMBINED_PRIVATE_KEY_SIZE) {
 				if (len < (ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1 + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE))
 					return -1;
 				m_hasPrivate = true;
-				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(m_priv.c25519, data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1);
+				Utils::copy<ZT_C25519_COMBINED_PRIVATE_KEY_SIZE>(m_priv,data + ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1);
 				return ZT_ADDRESS_LENGTH + 1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + 1 + ZT_C25519_COMBINED_PRIVATE_KEY_SIZE;
 			} else if (privlen == 0) {
 				m_hasPrivate = false;
@@ -578,9 +555,9 @@ int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 			if (len < (ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE + 1))
 				return -1;
 
-			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(&m_pub, data + ZT_ADDRESS_LENGTH + 1);
-			_computeHash(); // this sets the address for P384
-			if (m_address != Address(m_fp.hash())) // this sanity check is possible with V1 identities
+			Utils::copy<ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE>(m_pub,data + ZT_ADDRESS_LENGTH + 1);
+			m_computeHash(); // this sets the address for P384
+			if (m_fp.address() != Address(m_fp.hash())) // this sanity check is possible with V1 identities
 				return -1;
 
 			privlen = data[ZT_ADDRESS_LENGTH + 1 + ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE];
@@ -601,21 +578,18 @@ int Identity::unmarshal(const uint8_t *data,const int len) noexcept
 	return -1;
 }
 
-void Identity::_computeHash()
+void Identity::m_computeHash()
 {
 	switch(m_type) {
 		default:
 			m_fp.zero();
 			break;
-
 		case C25519:
-			m_fp.m_cfp.address = m_address.toInt();
-			SHA384(m_fp.m_cfp.hash, m_pub.c25519, ZT_C25519_COMBINED_PUBLIC_KEY_SIZE);
+			SHA384(m_fp.m_cfp.hash,m_pub,ZT_C25519_COMBINED_PUBLIC_KEY_SIZE);
 			break;
-
 		case P384:
-			SHA384(m_fp.m_cfp.hash, &m_pub, sizeof(m_pub));
-			m_fp.m_cfp.address = m_address.toInt();
+			SHA384(m_fp.m_cfp.hash,m_pub,sizeof(m_pub));
+			m_fp.m_cfp.address = Address(m_fp.m_cfp.hash).toInt();
 			break;
 	}
 }

node/Identity.hpp

@@ -24,10 +24,6 @@
 #include "Fingerprint.hpp"
 #include "Containers.hpp"
 
-#include <cstdio>
-#include <cstdlib>
-#include <cstring>
-
 #define ZT_IDENTITY_STRING_BUFFER_LENGTH 1024
 #define ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE (1 + ZT_C25519_COMBINED_PUBLIC_KEY_SIZE + ZT_ECC384_PUBLIC_KEY_SIZE)
 #define ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE (ZT_C25519_COMBINED_PRIVATE_KEY_SIZE + ZT_ECC384_PRIVATE_KEY_SIZE)
@@ -127,16 +123,12 @@ public:
 	ZT_INLINE bool hasPrivate() const noexcept { return m_hasPrivate; }
 
 	/**
-	 * Get a 384-bit hash of this identity's public key(s)
-	 *
-	 * The hash returned by this function differs by identity type. For C25519 (type 0)
-	 * identities this returns a simple SHA384 of the public key, which is NOT the same
-	 * as the hash used to generate the address. For type 1 C25519+P384 identities this
-	 * returns the same compoound SHA384 hash that is used for purposes of hashcash
-	 * and address computation. This difference is because the v0 hash is expensive while
-	 * the v1 hash is fast.
-	 *
-	 * @return Hash of public key(s)
+	 * @return This identity's address
+	 */
+	ZT_INLINE Address address() const noexcept { return Address(m_fp.m_cfp.address); }
+
+	/**
+	 * @return Full fingerprint of this identity (address plus SHA384 of keys)
 	 */
 	ZT_INLINE const Fingerprint &fingerprint() const noexcept { return m_fp; }
 
@@ -185,11 +177,6 @@ public:
 	 */
 	bool agree(const Identity &id,uint8_t key[ZT_SYMMETRIC_KEY_SIZE]) const;
 
-	/**
-	 * @return This identity's address
-	 */
-	ZT_INLINE Address address() const noexcept { return m_address; }
-
 	/**
 	 * Serialize to a more human-friendly string
 	 *
@@ -214,7 +201,7 @@ public:
 	/**
 	 * @return True if this identity contains something
 	 */
-	explicit ZT_INLINE operator bool() const noexcept { return (m_address); }
+	explicit ZT_INLINE operator bool() const noexcept { return (m_fp); }
 
 	ZT_INLINE unsigned long hashCode() const noexcept { return m_fp.hashCode(); }
 
@@ -230,19 +217,11 @@ public:
 	int unmarshal(const uint8_t *data,int len) noexcept;
 
 private:
-	void _computeHash();
+	void m_computeHash();
 
-	Address m_address;
 	Fingerprint m_fp;
-	ZT_PACKED_STRUCT(struct { // do not re-order these fields
-		uint8_t c25519[ZT_C25519_COMBINED_PRIVATE_KEY_SIZE];
-		uint8_t p384[ZT_ECC384_PRIVATE_KEY_SIZE];
-	}) m_priv;
-	ZT_PACKED_STRUCT(struct { // do not re-order these fields
-		uint8_t nonce;                            // nonce for PoW generate/verify
-		uint8_t c25519[ZT_C25519_COMBINED_PUBLIC_KEY_SIZE]; // Curve25519 and Ed25519 public keys
-		uint8_t p384[ZT_ECC384_PUBLIC_KEY_SIZE];  // NIST P-384 public key
-	}) m_pub;
+	uint8_t m_priv[ZT_IDENTITY_P384_COMPOUND_PRIVATE_KEY_SIZE];
+	uint8_t m_pub[ZT_IDENTITY_P384_COMPOUND_PUBLIC_KEY_SIZE];
 	Type m_type; // _type determines which fields in _priv and _pub are used
 	bool m_hasPrivate;
 };

node/Membership.cpp

@@ -124,13 +124,13 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 {
 	const int64_t newts = com.timestamp();
 	if (newts <= m_comRevocationThreshold) {
-		RR->t->credentialRejected(tPtr,0xd9992121,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
+		RR->t->credentialRejected(tPtr,0xd9992121,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
 		return ADD_REJECTED;
 	}
 
 	const int64_t oldts = m_com.timestamp();
 	if (newts < oldts) {
-		RR->t->credentialRejected(tPtr,0xd9928192,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
+		RR->t->credentialRejected(tPtr,0xd9928192,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
 		return ADD_REJECTED;
 	}
 	if ((newts == oldts)&&(m_com == com))
@@ -138,13 +138,13 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 
 	switch(com.verify(RR,tPtr)) {
 		default:
-			RR->t->credentialRejected(tPtr,0x0f198241,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+			RR->t->credentialRejected(tPtr,0x0f198241,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 			return Membership::ADD_REJECTED;
 		case Credential::VERIFY_OK:
 			m_com = com;
 			return ADD_ACCEPTED_NEW;
 		case Credential::VERIFY_BAD_SIGNATURE:
-			RR->t->credentialRejected(tPtr,0xbaf0aaaa,com.networkId(),sourcePeerIdentity.address(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_SIGNATURE_VERIFICATION_FAILED);
+			RR->t->credentialRejected(tPtr,0xbaf0aaaa,com.networkId(),sourcePeerIdentity,com.id(),com.timestamp(),ZT_CREDENTIAL_TYPE_COM,ZT_TRACE_CREDENTIAL_REJECTION_REASON_SIGNATURE_VERIFICATION_FAILED);
 			return ADD_REJECTED;
 		case Credential::VERIFY_NEED_IDENTITY:
 			return ADD_DEFERRED_FOR_WHOIS;
@@ -165,7 +165,7 @@ static ZT_INLINE Membership::AddCredentialResult _addCredImpl(
 	C *rc = remoteCreds.get(cred.id());
 	if (rc) {
 		if (rc->timestamp() > cred.timestamp()) {
-			RR->t->credentialRejected(tPtr,0x40000001,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
+			RR->t->credentialRejected(tPtr,0x40000001,nconf.networkId,sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_OLDER_THAN_LATEST);
 			return Membership::ADD_REJECTED;
 		}
 		if (*rc == cred)
@@ -174,13 +174,13 @@ static ZT_INLINE Membership::AddCredentialResult _addCredImpl(
 
 	const int64_t *const rt = revocations.get(Membership::credentialKey(C::credentialType(),cred.id()));
 	if ((rt)&&(*rt >= cred.timestamp())) {
-		RR->t->credentialRejected(tPtr,0x24248124,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
+		RR->t->credentialRejected(tPtr,0x24248124,nconf.networkId,sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_REVOKED);
 		return Membership::ADD_REJECTED;
 	}
 
 	switch(cred.verify(RR,tPtr)) {
 		default:
-			RR->t->credentialRejected(tPtr,0x01feba012,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+			RR->t->credentialRejected(tPtr,0x01feba012,nconf.networkId,sourcePeerIdentity,cred.id(),cred.timestamp(),C::credentialType(),ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 			return Membership::ADD_REJECTED;
 		case 0:
 			if (!rc)
@@ -200,7 +200,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 	int64_t *rt;
 	switch(rev.verify(RR,tPtr)) {
 		default:
-			RR->t->credentialRejected(tPtr,0x938fffff,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+			RR->t->credentialRejected(tPtr,0x938fffff,nconf.networkId,sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 			return ADD_REJECTED;
 		case 0: {
 			const ZT_CredentialType ct = rev.typeBeingRevoked();
@@ -222,7 +222,7 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 					}
 					return ADD_ACCEPTED_REDUNDANT;
 				default:
-					RR->t->credentialRejected(tPtr,0x0bbbb1a4,nconf.networkId,sourcePeerIdentity.address(),sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
+					RR->t->credentialRejected(tPtr,0x0bbbb1a4,nconf.networkId,sourcePeerIdentity,rev.id(),0,ZT_CREDENTIAL_TYPE_REVOCATION,ZT_TRACE_CREDENTIAL_REJECTION_REASON_INVALID);
 					return ADD_REJECTED;
 			}
 		}

node/Network.hpp

@@ -27,12 +27,6 @@
 #include "CertificateOfMembership.hpp"
 #include "Containers.hpp"
 
-#include <cstdint>
-#include <string>
-#include <map>
-#include <vector>
-#include <algorithm>
-
 #define ZT_NETWORK_MAX_INCOMING_UPDATES 3
 
 namespace ZeroTier {

node/NetworkConfig.cpp

@@ -21,7 +21,7 @@
 
 namespace ZeroTier {
 
-bool NetworkConfig::toDictionary(Dictionary &d,bool includeLegacy) const
+bool NetworkConfig::toDictionary(Dictionary &d) const
 {
 	uint8_t tmp[ZT_BUF_MEM_SIZE];
 	try {

node/NetworkConfig.hpp

@@ -165,10 +165,9 @@ struct NetworkConfig : TriviallyCopyable
 	 * Write this network config to a dictionary for transport
 	 *
 	 * @param d Dictionary
-	 * @param includeLegacy If true, include legacy fields for old node versions
 	 * @return True if dictionary was successfully created, false if e.g. overflow
 	 */
-	bool toDictionary(Dictionary &d,bool includeLegacy) const;
+	bool toDictionary(Dictionary &d) const;
 
 	/**
 	 * Read this network config from a dictionary

node/NetworkController.hpp

@@ -77,8 +77,8 @@ public:
 		virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) = 0;
 	};
 
-	NetworkController() {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
-	virtual ~NetworkController() {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	NetworkController() {}
+	virtual ~NetworkController() {}
 
 	/**
 	 * Called when this is added to a Node to initialize and supply info

node/OS.hpp

@@ -17,10 +17,10 @@
 #ifndef ZT_OS_HPP
 #define ZT_OS_HPP
 
-#include <stdint.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
-#include <stdlib.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
-#include <string.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
-#include <stdio.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
 
 #if defined(_WIN32) || defined(_WIN64)
 #ifdef _MSC_VER

node/Peer.cpp

@@ -21,10 +21,11 @@
 #include "InetAddress.hpp"
 #include "Protocol.hpp"
 #include "Endpoint.hpp"
+#include "Expect.hpp"
 
 namespace ZeroTier {
 
-Peer::Peer(const RuntimeEnvironment *renv) : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+Peer::Peer(const RuntimeEnvironment *renv) :
 	RR(renv),
 	m_lastReceive(0),
 	m_lastSend(0),
@@ -32,6 +33,7 @@ Peer::Peer(const RuntimeEnvironment *renv) : // NOLINT(cppcoreguidelines-pro-typ
 	m_lastWhoisRequestReceived(0),
 	m_lastEchoRequestReceived(0),
 	m_lastPrioritizedPaths(0),
+	m_lastProbeReceived(0),
 	m_alivePathCount(0),
 	m_tryQueue(),
 	m_tryQueuePtr(m_tryQueue.end()),
@@ -43,8 +45,9 @@ Peer::Peer(const RuntimeEnvironment *renv) : // NOLINT(cppcoreguidelines-pro-typ
 {
 }
 
-Peer::~Peer() // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+Peer::~Peer()
 {
+	Utils::burn(m_helloMacKey,sizeof(m_helloMacKey));
 }
 
 bool Peer::init(const Identity &peerIdentity)
@@ -55,11 +58,13 @@ bool Peer::init(const Identity &peerIdentity)
 		return false;
 	m_id = peerIdentity;
 
-	uint8_t ktmp[ZT_SYMMETRIC_KEY_SIZE];
-	if (!RR->identity.agree(peerIdentity,ktmp))
+	uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
+	if (!RR->identity.agree(peerIdentity,k))
 		return false;
-	m_identityKey.init(RR->node->now(), ktmp);
-	Utils::burn(ktmp,sizeof(ktmp));
+	m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
+	Utils::burn(k,sizeof(k));
+
+	m_deriveSecondaryIdentityKeys();
 
 	return true;
 }
@@ -76,7 +81,7 @@ void Peer::received(
 	const int64_t now = RR->node->now();
 
 	m_lastReceive = now;
-	m_inMeter.log(now, payloadLength);
+	m_inMeter.log(now,payloadLength);
 
 	if (hops == 0) {
 		RWMutex::RMaybeWLock l(m_lock);
@@ -98,7 +103,7 @@ void Peer::received(
 
 				// If the path list is full, replace the least recently active path. Otherwise append new path.
 				unsigned int newPathIdx = 0;
-				if (m_alivePathCount >= ZT_MAX_PEER_NETWORK_PATHS) {
+				if (m_alivePathCount == ZT_MAX_PEER_NETWORK_PATHS) {
 					int64_t lastReceiveTimeMax = 0;
 					for (unsigned int i=0;i < m_alivePathCount;++i) {
 						if ((m_paths[i]->address().family() == path->address().family()) &&
@@ -132,18 +137,12 @@ void Peer::received(
 				RR->t->learnedNewPath(tPtr, 0x582fabdd, packetId, m_id, path->address(), old);
 			} else {
 				path->sent(now,hello(tPtr,path->localSocket(),path->address(),now));
-				RR->t->tryingNewPath(tPtr, 0xb7747ddd, m_id, path->address(), path->address(), packetId, (uint8_t)verb, m_id, ZT_TRACE_TRYING_NEW_PATH_REASON_PACKET_RECEIVED_FROM_UNKNOWN_PATH);
+				RR->t->tryingNewPath(tPtr, 0xb7747ddd, m_id, path->address(), path->address(), packetId, (uint8_t)verb, m_id);
 			}
 		}
 	}
 }
 
-void Peer::send(void *const tPtr,const int64_t now,const void *const data,const unsigned int len,const SharedPtr<Path> &via) noexcept
-{
-	via->send(RR,tPtr,data,len,now);
-	sent(now,len);
-}
-
 void Peer::send(void *const tPtr,const int64_t now,const void *const data,const unsigned int len) noexcept
 {
 	SharedPtr<Path> via(this->path(now));
@@ -190,25 +189,6 @@ unsigned int Peer::hello(void *tPtr,int64_t localSocket,const InetAddress &atAdd
 #endif
 }
 
-unsigned int Peer::probe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now)
-{
-	if (m_vProto < 11) {
-		Buf outp;
-		Protocol::Header &ph = outp.as<Protocol::Header>(); // NOLINT(hicpp-use-auto,modernize-use-auto)
-		//ph.packetId = Protocol::getPacketId();
-		m_id.address().copyTo(ph.destination);
-		RR->identity.address().copyTo(ph.source);
-		ph.flags = 0;
-		ph.verb = Protocol::VERB_NOP;
-		Protocol::armor(outp, sizeof(Protocol::Header), m_identityKey.key(), this->cipher());
-		RR->node->putPacket(tPtr,localSocket,atAddress,outp.unsafeData,sizeof(Protocol::Header));
-		return sizeof(Protocol::Header);
-	} else {
-		RR->node->putPacket(tPtr, -1, atAddress, &m_probe, 4);
-		return 4;
-	}
-}
-
 void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 {
 	RWMutex::Lock l(m_lock);
@@ -219,90 +199,104 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 		needHello = true;
 	}
 
-	m_prioritizePaths(now);
-
-	if (m_alivePathCount == 0) {
-		// If there are no direct paths, attempt to make one. If there are queued addresses
-		// to try, attempt one of those. Otherwise try a path we can fetch via API callbacks
-		// and/or a remembered bootstrap path.
-		if (m_tryQueue.empty()) {
-			InetAddress addr;
-			if (RR->node->externalPathLookup(tPtr, m_id, -1, addr)) {
-				if ((addr)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr, m_id, -1, addr))) {
-					RR->t->tryingNewPath(tPtr, 0x84a10000, m_id, addr, InetAddress::NIL, 0, 0, Identity::NIL, ZT_TRACE_TRYING_NEW_PATH_REASON_EXPLICITLY_SUGGESTED_ADDRESS);
-					sent(now,probe(tPtr,-1,addr,now));
-				}
+	// If we have no active paths and none queued to try, attempt any
+	// old paths we have cached in m_bootstrap or that external code
+	// supplies to the core via the optional API callback.
+	if (m_tryQueue.empty()&&(m_alivePathCount == 0)) {
+		InetAddress addr;
+		if (RR->node->externalPathLookup(tPtr, m_id, -1, addr)) {
+			if ((addr)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr, m_id, -1, addr))) {
+				RR->t->tryingNewPath(tPtr, 0x84a10000, m_id, addr, InetAddress::NIL, 0, 0, Identity::NIL);
+				sent(now,m_sendProbe(tPtr,-1,addr,now));
 			}
-			if (!m_bootstrap.empty()) {
-				unsigned int tryAtIndex = (unsigned int)Utils::random() % (unsigned int)m_bootstrap.size();
-				for(SortedMap< Endpoint::Type,Endpoint >::const_iterator i(m_bootstrap.begin());i != m_bootstrap.end();++i) {
-					if (tryAtIndex > 0) {
-						--tryAtIndex;
-					} else {
-						if ((i->second.isInetAddr())&&(!i->second.inetAddr().ipsEqual(addr))) {
-							RR->t->tryingNewPath(tPtr, 0x0a009444, m_id, i->second.inetAddr(), InetAddress::NIL, 0, 0, Identity::NIL, ZT_TRACE_TRYING_NEW_PATH_REASON_BOOTSTRAP_ADDRESS);
-							sent(now,probe(tPtr,-1,i->second.inetAddr(),now));
-							break;
-						}
+		}
+
+		if (!m_bootstrap.empty()) {
+			unsigned int tryAtIndex = (unsigned int)Utils::random() % (unsigned int)m_bootstrap.size();
+			for(SortedMap< Endpoint::Type,Endpoint >::const_iterator i(m_bootstrap.begin());i != m_bootstrap.end();++i) {
+				if (tryAtIndex > 0) {
+					--tryAtIndex;
+				} else {
+					if ((i->second.isInetAddr())&&(!i->second.inetAddr().ipsEqual(addr))) {
+						RR->t->tryingNewPath(tPtr, 0x0a009444, m_id, i->second.inetAddr(), InetAddress::NIL, 0, 0, Identity::NIL);
+						sent(now,m_sendProbe(tPtr,-1,i->second.inetAddr(),now));
+						break;
 					}
 				}
 			}
-		} else {
-			for(int k=0;(k<ZT_NAT_T_MAX_QUEUED_ATTEMPTS_PER_PULSE)&&(!m_tryQueue.empty());++k) {
-				if (m_tryQueuePtr == m_tryQueue.end())
-					m_tryQueuePtr = m_tryQueue.begin();
+		}
+	}
+
+	m_prioritizePaths(now);
+
+	// Attempt queued paths to try.
+	for(int k=0;(k<ZT_NAT_T_MAX_QUEUED_ATTEMPTS_PER_PULSE)&&(!m_tryQueue.empty());++k) {
+		if (m_tryQueuePtr == m_tryQueue.end())
+			m_tryQueuePtr = m_tryQueue.begin();
 
-				if ((now - m_tryQueuePtr->ts) > ZT_PATH_ALIVE_TIMEOUT) {
-					m_tryQueue.erase(m_tryQueuePtr++);
-					continue;
+		if ((now - m_tryQueuePtr->ts) > ZT_PATH_ALIVE_TIMEOUT) {
+			m_tryQueue.erase(m_tryQueuePtr++);
+			continue;
+		}
+
+		if (m_tryQueuePtr->target.isInetAddr()) {
+			// Make sure target does not overlap with any existing path.
+			bool duplicate = false;
+			for(unsigned int i=0;i<m_alivePathCount;++i) {
+				if (m_paths[i]->address() == m_tryQueuePtr->target.inetAddr()) {
+					duplicate = true;
+					break;
 				}
+			}
+			if (duplicate) {
+				m_tryQueue.erase(m_tryQueuePtr++);
+				continue;
+			}
 
-				if (m_tryQueuePtr->target.isInetAddr()) {
-					if ((m_tryQueuePtr->breakSymmetricBFG1024) && (RR->node->natMustDie())) {
-						// Attempt aggressive NAT traversal if both requested and enabled.
-						uint16_t ports[1023];
-						for (unsigned int i=0;i<1023;++i)
-							ports[i] = (uint64_t)(i + 1);
-						for (unsigned int i=0;i<512;++i) {
-							const uint64_t rn = Utils::random();
-							const unsigned int a = (unsigned int)rn % 1023;
-							const unsigned int b = (unsigned int)(rn >> 32U) % 1023;
-							if (a != b) {
-								uint16_t tmp = ports[a];
-								ports[a] = ports[b];
-								ports[b] = tmp;
-							}
-						}
-						InetAddress addr(m_tryQueuePtr->target.inetAddr());
-						for (unsigned int i = 0;i < ZT_NAT_T_BFG1024_PORTS_PER_ATTEMPT;++i) {
-							addr.setPort(ports[i]);
-							sent(now,probe(tPtr,-1,addr,now));
-						}
-					} else {
-						// Otherwise send a normal probe.
-						sent(now,probe(tPtr, -1, m_tryQueuePtr->target.inetAddr(), now));
+			if (m_tryQueuePtr->breakSymmetricBFG1024 && RR->node->natMustDie()) {
+				// Attempt aggressive NAT traversal if both requested and enabled.
+				uint16_t ports[1023];
+				for (unsigned int i=0;i<1023;++i)
+					ports[i] = (uint64_t)(i + 1);
+				for (unsigned int i=0;i<512;++i) {
+					const uint64_t rn = Utils::random();
+					const unsigned int a = (unsigned int)rn % 1023;
+					const unsigned int b = (unsigned int)(rn >> 32U) % 1023;
+					if (a != b) {
+						uint16_t tmp = ports[a];
+						ports[a] = ports[b];
+						ports[b] = tmp;
 					}
 				}
-
-				++m_tryQueuePtr;
+				InetAddress addr(m_tryQueuePtr->target.inetAddr());
+				for (unsigned int i=0;i<ZT_NAT_T_BFG1024_PORTS_PER_ATTEMPT;++i) {
+					addr.setPort(ports[i]);
+					sent(now,m_sendProbe(tPtr,-1,addr,now));
+				}
+			} else {
+				// Otherwise send a normal probe.
+				sent(now,m_sendProbe(tPtr, -1, m_tryQueuePtr->target.inetAddr(), now));
 			}
 		}
-	} else {
-		// Keep direct paths alive, sending a HELLO if we need one or else just a simple byte.
-		for(unsigned int i=0;i < m_alivePathCount;++i) {
-			if (needHello) {
-				needHello = false;
-				const unsigned int bytes = hello(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
-				m_paths[i]->sent(now, bytes);
-				sent(now,bytes);
-			} else if ((now - m_paths[i]->lastOut()) >= ZT_PATH_KEEPALIVE_PERIOD) {
-				m_paths[i]->send(RR, tPtr, &now, 1, now);
-				sent(now,1);
-			}
+
+		++m_tryQueuePtr;
+	}
+
+	// Do keepalive on all currently active paths.
+	for(unsigned int i=0;i<m_alivePathCount;++i) {
+		if (needHello) {
+			needHello = false;
+			const unsigned int bytes = hello(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
+			m_paths[i]->sent(now, bytes);
+			sent(now,bytes);
+		} else if ((now - m_paths[i]->lastOut()) >= ZT_PATH_KEEPALIVE_PERIOD) {
+			m_paths[i]->send(RR, tPtr, &now, 1, now);
+			sent(now,1);
 		}
 	}
 
-	// If we could not reliably send a HELLO via a direct path, send it by way of a root.
+	// If we need a HELLO and were not able to send one via any other path,
+	// send one indirectly.
 	if (needHello) {
 		const SharedPtr<Peer> root(RR->topology->root());
 		if (root) {
@@ -321,7 +315,7 @@ void Peer::tryDirectPath(const int64_t now,const Endpoint &ep,const bool breakSy
 {
 	RWMutex::Lock l(m_lock);
 
-	for(List<p_TryQueueItem>::iterator i(m_tryQueue.begin());i != m_tryQueue.end();++i) { // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+	for(List<p_TryQueueItem>::iterator i(m_tryQueue.begin());i != m_tryQueue.end();++i) {
 		if (i->target == ep) {
 			i->ts = now;
 			i->breakSymmetricBFG1024 = breakSymmetricBFG1024;
@@ -338,14 +332,20 @@ void Peer::tryDirectPath(const int64_t now,const Endpoint &ep,const bool breakSy
 
 void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now)
 {
-	RWMutex::RLock l(m_lock);
-	for(unsigned int i=0;i < m_alivePathCount;++i) {
+	RWMutex::Lock l(m_lock);
+	unsigned int pc = 0;
+	for(unsigned int i=0;i<m_alivePathCount;++i) {
 		if ((m_paths[i]) && ((m_paths[i]->address().family() == inetAddressFamily) && (m_paths[i]->address().ipScope() == scope))) {
-			const unsigned int bytes = probe(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
+			const unsigned int bytes = m_sendProbe(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
 			m_paths[i]->sent(now, bytes);
 			sent(now,bytes);
+		} else if (pc != i) {
+			m_paths[pc++] = m_paths[i];
 		}
 	}
+	m_alivePathCount = pc;
+	while (pc < ZT_MAX_PEER_NETWORK_PATHS)
+		m_paths[pc].zero();
 }
 
 bool Peer::directlyConnected(int64_t now)
@@ -360,10 +360,11 @@ bool Peer::directlyConnected(int64_t now)
 	}
 }
 
-void Peer::getAllPaths(std::vector< SharedPtr<Path> > &paths)
+void Peer::getAllPaths(Vector< SharedPtr<Path> > &paths)
 {
 	RWMutex::RLock l(m_lock);
 	paths.clear();
+	paths.reserve(m_alivePathCount);
 	paths.assign(m_paths, m_paths + m_alivePathCount);
 }
 
@@ -385,16 +386,27 @@ void Peer::save(void *tPtr) const
 
 int Peer::marshal(uint8_t data[ZT_PEER_MARSHAL_SIZE_MAX]) const noexcept
 {
-	data[0] = 0; // serialized peer version
-
 	RWMutex::RLock l(m_lock);
 
-	int s = m_identityKey.marshal(RR->localCacheSymmetric, data + 1);
-	if (s < 0)
+	if (!m_identityKey)
 		return -1;
-	int p = 1 + s;
 
-	s = m_id.marshal(data + p, false);
+	data[0] = 0; // serialized peer version
+
+	// Include our identity's address to detect if this changes and require
+	// recomputation of m_identityKey.
+	RR->identity.address().copyTo(data + 1);
+
+	// SECURITY: encryption in place is only to protect secrets if they are
+	// cached to local storage. It's not used over the wire. Dumb ECB is fine
+	// because secret keys are random and have no structure to reveal.
+	RR->localCacheSymmetric.encrypt(m_identityKey->secret,data + 6);
+	RR->localCacheSymmetric.encrypt(m_identityKey->secret + 22,data + 17);
+	RR->localCacheSymmetric.encrypt(m_identityKey->secret + 38,data + 33);
+
+	int p = 54;
+
+	int s = m_id.marshal(data + p, false);
 	if (s < 0)
 		return -1;
 	p += s;
@@ -431,35 +443,38 @@ int Peer::unmarshal(const uint8_t *restrict data,const int len) noexcept
 {
 	RWMutex::Lock l(m_lock);
 
-	if ((len <= 1) || (data[0] != 0))
+	if ((len <= 54) || (data[0] != 0))
 		return -1;
 
-	int s = m_identityKey.unmarshal(RR->localCacheSymmetric, data + 1, len);
-	if (s < 0)
-		return -1;
-	int p = 1 + s;
-
-	// If the identity key did not pass verification, it may mean that our local
-	// identity has changed. In this case we do not have to forget everything about
-	// the peer but we must generate a new identity key by key agreement with our
-	// new identity.
-	if (!m_identityKey) {
-		uint8_t tmp[ZT_SYMMETRIC_KEY_SIZE];
-		if (!RR->identity.agree(m_id, tmp))
-			return -1;
-		m_identityKey.init(RR->node->now(), tmp);
-		Utils::burn(tmp,sizeof(tmp));
+	m_identityKey.zero();
+	m_ephemeralKeys[0].zero();
+	m_ephemeralKeys[1].zero();
+
+	if (Address(data + 1) == RR->identity.address()) {
+		uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
+		static_assert(ZT_SYMMETRIC_KEY_SIZE == 48,"marshal() and unmarshal() must be revisited if ZT_SYMMETRIC_KEY_SIZE is changed");
+		RR->localCacheSymmetric.decrypt(data + 1,k);
+		RR->localCacheSymmetric.decrypt(data + 17,k + 16);
+		RR->localCacheSymmetric.decrypt(data + 33,k + 32);
+		m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
+		Utils::burn(k,sizeof(k));
 	}
 
-	// These are ephemeral and start out as NIL after unmarshal.
-	m_ephemeralKeys[0].clear();
-	m_ephemeralKeys[1].clear();
+	int p = 49;
 
-	s = m_id.unmarshal(data + 38, len - 38);
+	int s = m_id.unmarshal(data + 38, len - 38);
 	if (s < 0)
 		return s;
 	p += s;
 
+	if (!m_identityKey) {
+		uint8_t k[ZT_SYMMETRIC_KEY_SIZE];
+		if (!RR->identity.agree(m_id,k))
+			return -1;
+		m_identityKey.set(new SymmetricKey(RR->node->now(),k,true));
+		Utils::burn(k,sizeof(k));
+	}
+
 	s = m_locator.unmarshal(data + p, len - p);
 	if (s < 0)
 		return s;
@@ -523,4 +538,40 @@ void Peer::m_prioritizePaths(int64_t now)
 	}
 }
 
+unsigned int Peer::m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now)
+{
+	// Assumes m_lock is locked
+	if ((m_vProto < 11)||(m_probe == 0)) {
+		const SharedPtr<SymmetricKey> k(m_key());
+		const uint64_t packetId = k->nextMessage(RR->identity.address(),m_id.address());
+
+		uint8_t p[ZT_PROTO_MIN_PACKET_LENGTH + 1];
+		Utils::storeAsIsEndian<uint64_t>(p + ZT_PROTO_PACKET_ID_INDEX,packetId);
+		m_id.address().copyTo(p + ZT_PROTO_PACKET_DESTINATION_INDEX);
+		RR->identity.address().copyTo(p + ZT_PROTO_PACKET_SOURCE_INDEX);
+		p[ZT_PROTO_PACKET_FLAGS_INDEX] = 0;
+		p[ZT_PROTO_PACKET_VERB_INDEX] = Protocol::VERB_ECHO;
+		p[ZT_PROTO_PACKET_VERB_INDEX + 1] = (uint8_t)now; // arbitrary byte
+
+		Protocol::armor(p,ZT_PROTO_MIN_PACKET_LENGTH,k,cipher());
+
+		RR->expect->sending(packetId,now);
+		RR->node->putPacket(tPtr,-1,atAddress,p,ZT_PROTO_MIN_PACKET_LENGTH);
+
+		return ZT_PROTO_MIN_PACKET_LENGTH;
+	} else {
+		RR->node->putPacket(tPtr,-1,atAddress,&m_probe,4);
+		return 4;
+	}
+}
+
+void Peer::m_deriveSecondaryIdentityKeys() noexcept
+{
+	uint8_t hk[ZT_SYMMETRIC_KEY_SIZE];
+	KBKDFHMACSHA384(m_identityKey->secret,ZT_KBKDF_LABEL_HELLO_DICTIONARY_ENCRYPT,0,0,hk);
+	m_helloCipher.init(hk);
+	Utils::burn(hk,sizeof(hk));
+	KBKDFHMACSHA384(m_identityKey->secret,ZT_KBKDF_LABEL_PACKET_HMAC,0,0,m_helloMacKey);
+}
+
 } // namespace ZeroTier

node/Peer.hpp

@@ -32,7 +32,7 @@
 #include "Containers.hpp"
 
 // version, identity, locator, bootstrap, version info, length of any additional fields
-#define ZT_PEER_MARSHAL_SIZE_MAX (1 + ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX + ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 1 + (ZT_MAX_PEER_NETWORK_PATHS * ZT_ENDPOINT_MARSHAL_SIZE_MAX) + (2*4) + 2)
+#define ZT_PEER_MARSHAL_SIZE_MAX (1 + ZT_ADDRESS_LENGTH + ZT_SYMMETRIC_KEY_SIZE + ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 1 + (ZT_MAX_PEER_NETWORK_PATHS * ZT_ENDPOINT_MARSHAL_SIZE_MAX) + (2*4) + 2)
 
 namespace ZeroTier {
 
@@ -50,8 +50,8 @@ public:
 	/**
 	 * Create an uninitialized peer
 	 *
-	 * The peer will need to be initialized with init() or unmarshal() before
-	 * it can be used.
+	 * New peers must be initialized via either init() or unmarshal() prior to
+	 * use or null pointer dereference may occur.
 	 *
 	 * @param renv Runtime environment
 	 */
@@ -95,7 +95,7 @@ public:
 	 * @param t New probe token
 	 * @return Old probe token
 	 */
-	ZT_INLINE uint32_t setProbeToken(const uint32_t t) const noexcept
+	ZT_INLINE uint32_t setProbeToken(const uint32_t t) noexcept
 	{
 		RWMutex::Lock l(m_lock);
 		const uint32_t pt = m_probe;
@@ -186,7 +186,11 @@ public:
 	 * @param len Length in bytes
 	 * @param via Path over which to send data (may or may not be an already-learned path for this peer)
 	 */
-	void send(void *tPtr,int64_t now,const void *data,unsigned int len,const SharedPtr<Path> &via) noexcept;
+	ZT_INLINE void send(void *tPtr,int64_t now,const void *data,unsigned int len,const SharedPtr<Path> &via) noexcept
+	{
+		via->send(RR,tPtr,data,len,now);
+		sent(now,len);
+	}
 
 	/**
 	 * Send data to this peer over the best available path
@@ -212,17 +216,6 @@ public:
 	 */
 	unsigned int hello(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
 
-	/**
-	 * Send a NOP message to e.g. probe a new link
-	 *
-	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
-	 * @param localSocket Local source socket
-	 * @param atAddress Destination address
-	 * @param now Current time
-	 * @return Number of bytes sent
-	 */
-	unsigned int probe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
-
 	/**
 	 * Ping this peer if needed and/or perform other periodic tasks.
 	 *
@@ -303,13 +296,71 @@ public:
 	}
 
 	/**
-	 * @return Preferred cipher suite for normal encrypted P2P communication
+	 * @return Cipher suite that should be used to communicate with this peer
 	 */
 	ZT_INLINE uint8_t cipher() const noexcept
 	{
+		//if (m_vProto >= 11)
+		//	return ZT_PROTO_CIPHER_SUITE__AES_GMAC_SIV;
 		return ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012;
 	}
 
+	/**
+	 * @return The permanent shared key for this peer computed by simple identity agreement
+	 */
+	ZT_INLINE SharedPtr<SymmetricKey> identityKey() noexcept
+	{
+		return m_identityKey;
+	}
+
+	/**
+	 * @return AES instance for HELLO dictionary / encrypted section encryption/decryption
+	 */
+	ZT_INLINE const AES &identityHelloDictionaryEncryptionCipher() noexcept
+	{
+		return m_helloCipher;
+	}
+
+	/**
+	 * @return Key for HMAC on HELLOs
+	 */
+	ZT_INLINE const uint8_t *identityHelloHmacKey() noexcept
+	{
+		return m_helloMacKey;
+	}
+
+	/**
+	 * @return Raw identity key bytes
+	 */
+	ZT_INLINE const uint8_t *rawIdentityKey() noexcept
+	{
+		RWMutex::RLock l(m_lock);
+		return m_identityKey->secret;
+	}
+
+	/**
+	 * @return Current best key: either the latest ephemeral or the identity key
+	 */
+	ZT_INLINE SharedPtr<SymmetricKey> key() noexcept
+	{
+		RWMutex::RLock l(m_lock);
+		return m_key();
+	}
+
+	/**
+	 * Check whether a key is ephemeral
+	 * 
+	 * This is used to check whether a packet is received with forward secrecy enabled
+	 * or not.
+	 * 
+	 * @param k Key to check
+	 * @return True if this key is ephemeral, false if it's the long-lived identity key
+	 */
+	ZT_INLINE bool isEphemeral(const SharedPtr<SymmetricKey> &k) const noexcept
+	{
+		return (m_identityKey != k);
+	}
+
 	/**
 	 * Set the currently known remote version of this peer's client
 	 *
@@ -342,7 +393,7 @@ public:
 	 *
 	 * @param paths Vector of paths with the first path being the current preferred path
 	 */
-	void getAllPaths(std::vector< SharedPtr<Path> > &paths);
+	void getAllPaths(Vector< SharedPtr<Path> > &paths);
 
 	/**
 	 * Save the latest version of this peer to the data store
@@ -379,19 +430,45 @@ public:
 		return false;
 	}
 
+	/**
+	 * Rate limit gate for inbound probes
+	 */
+	ZT_INLINE bool rateGateProbeRequest(const int64_t now) noexcept
+	{
+		if((now - m_lastProbeReceived) > ZT_PEER_PROBE_RESPONSE_RATE_LIMIT) {
+			m_lastProbeReceived = now;
+			return true;
+		}
+		return false;
+	}
+
 private:
 	void m_prioritizePaths(int64_t now);
+	unsigned int m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
+	void m_deriveSecondaryIdentityKeys() noexcept;
+
+	ZT_INLINE SharedPtr<SymmetricKey> m_key() noexcept
+	{
+		// assumes m_lock is locked (for read at least)
+		return (m_ephemeralKeys[0]) ? m_ephemeralKeys[0] : m_identityKey;
+	}
 
 	const RuntimeEnvironment *RR;
 
 	// Read/write mutex for non-atomic non-const fields.
 	RWMutex m_lock;
 
-	// The permanent identity key resulting from agreement between our identity and this peer's identity.
-	SymmetricKey< AES,0,0 > m_identityKey;
+	// Static identity key
+	SharedPtr<SymmetricKey> m_identityKey;
 
-	// Most recently successful (for decrypt) ephemeral key and one previous key.
-	SymmetricKey< AES,ZT_SYMMETRIC_KEY_TTL,ZT_SYMMETRIC_KEY_TTL_MESSAGES > m_ephemeralKeys[2];
+	// Cipher for encrypting or decrypting the encrypted section of HELLO packets.
+	AES m_helloCipher;
+
+	// Key for HELLO HMAC-SHA384
+	uint8_t m_helloMacKey[ZT_SYMMETRIC_KEY_SIZE];
+
+	// Current and previous ephemeral key
+	SharedPtr<SymmetricKey> m_ephemeralKeys[2];
 
 	Identity m_id;
 	Locator m_locator;
@@ -412,6 +489,9 @@ private:
 	// The last time we sorted paths in order of preference. (This happens pretty often.)
 	std::atomic<int64_t> m_lastPrioritizedPaths;
 
+	// The last time we got a probe from this peer.
+	std::atomic<int64_t> m_lastProbeReceived;
+
 	// Meters measuring actual bandwidth in, out, and relayed via this peer (mostly if this is a root).
 	Meter<> m_inMeter;
 	Meter<> m_outMeter;

node/Poly1305.cpp

@@ -425,12 +425,20 @@ ZT_INLINE void poly1305_update(poly1305_context *ctx,const unsigned char *m,size
 
 } // anonymous namespace
 
-void poly1305(void *auth,const void *data,unsigned int len,const void *key) noexcept
+void Poly1305::init(const void *key) noexcept
 {
-  poly1305_context ctx;
-  poly1305_init(&ctx,reinterpret_cast<const unsigned char *>(key));
-  poly1305_update(&ctx,reinterpret_cast<const unsigned char *>(data),(size_t)len);
-  poly1305_finish(&ctx,reinterpret_cast<unsigned char *>(auth));
+  static_assert(sizeof(ctx) >= sizeof(poly1305_context),"buffer in class smaller than required structure size");
+  poly1305_init(reinterpret_cast<poly1305_context *>(&ctx),reinterpret_cast<const unsigned char *>(key));
+}
+
+void Poly1305::update(const void *data,unsigned int len) noexcept
+{
+  poly1305_update(reinterpret_cast<poly1305_context *>(&ctx),reinterpret_cast<const unsigned char *>(data),(size_t)len);
+}
+
+void Poly1305::finish(void *auth) noexcept
+{
+  poly1305_finish(reinterpret_cast<poly1305_context *>(&ctx),reinterpret_cast<unsigned char *>(auth));
 }
 
 } // namespace ZeroTier

node/Poly1305.hpp

@@ -20,14 +20,24 @@ namespace ZeroTier {
 #define ZT_POLY1305_MAC_SIZE 16
 
 /**
- * Compute a one-time authentication code
- *
- * @param auth Buffer to receive code -- MUST be 16 bytes in length
- * @param data Data to authenticate
- * @param len Length of data to authenticate in bytes
- * @param key 32-byte one-time use key to authenticate data (must not be reused)
+ * Poly1305 one-time MAC calculator
  */
-void poly1305(void *auth,const void *data,unsigned int len,const void *key) noexcept;
+class Poly1305
+{
+public:
+	ZT_INLINE Poly1305() {}
+	ZT_INLINE Poly1305(const void *key) { this->init(key); }
+
+	void init(const void *key) noexcept;
+	void update(const void *data,unsigned int len) noexcept;
+	void finish(void *auth) noexcept;
+
+private:
+	struct {
+	  size_t aligner;
+  	unsigned char opaque[136];
+	} ctx;
+};
 
 } // namespace ZeroTier
 

node/Protocol.cpp

@@ -1,93 +0,0 @@
-/*
- * Copyright (c)2013-2020 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2024-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#include "Protocol.hpp"
-#include "Buf.hpp"
-#include "Utils.hpp"
-
-#include <cstdlib>
-#include <ctime>
-
-#ifdef __WINDOWS__
-#include <process.h>
-#else
-#include <unistd.h>
-#endif
-
-namespace ZeroTier {
-namespace Protocol {
-
-void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint8_t cipherSuite) noexcept
-{
-	Protocol::Header &ph = pkt.as<Protocol::Header>(); // NOLINT(hicpp-use-auto,modernize-use-auto)
-	ph.flags = (ph.flags & 0xc7U) | ((cipherSuite << 3U) & 0x38U); // flags: FFCCCHHH where CCC is cipher
-
-	switch(cipherSuite) {
-		case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE: {
-			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
-			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
-			Salsa20 s20(perPacketKey,&ph.packetId);
-
-			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
-			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
-
-			// only difference here is that we don't encrypt the payload
-
-			uint64_t mac[2];
-			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
-			ph.mac = mac[0];
-		} break;
-
-		case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012: {
-			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
-			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
-			Salsa20 s20(perPacketKey,&ph.packetId);
-
-			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
-			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
-
-			const unsigned int encLen = packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START;
-			s20.crypt12(pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen);
-
-			uint64_t mac[2];
-			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen,macKey);
-			ph.mac = mac[0];
-		} break;
-
-		case ZT_PROTO_CIPHER_SUITE__AES_GMAC_SIV: {
-		} break;
-	}
-}
-
-int compress(SharedPtr<Buf> &pkt,int packetSize) noexcept
-{
-	if (packetSize <= 128)
-		return packetSize;
-
-	SharedPtr<Buf> pkt2(new Buf());
-	if (!pkt2) return packetSize;
-
-	const int uncompressedLen = packetSize - ZT_PROTO_PACKET_PAYLOAD_START;
-	const int compressedLen = LZ4_compress_fast(reinterpret_cast<const char *>(pkt->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),reinterpret_cast<char *>(pkt2->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),uncompressedLen,ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START);
-	if ((compressedLen > 0)&&(compressedLen < uncompressedLen)) {
-		Utils::copy<ZT_PROTO_PACKET_PAYLOAD_START>(pkt2->unsafeData,pkt->unsafeData);
-		pkt.swap(pkt2);
-		pkt->as<Protocol::Header>().verb |= ZT_PROTO_VERB_FLAG_COMPRESSED;
-		return compressedLen + ZT_PROTO_PACKET_PAYLOAD_START;
-	}
-
-	return packetSize;
-}
-
-} // namespace Protocol
-} // namespace ZeroTier

node/Protocol.hpp

@@ -22,6 +22,7 @@
 #include "Buf.hpp"
 #include "Address.hpp"
 #include "Identity.hpp"
+#include "SymmetricKey.hpp"
 
 /*
  * Packet format:
@@ -188,16 +189,6 @@
  */
 #define ZT_PROTO_PACKET_FRAGMENT_INDICATOR 0xff
 
-/**
- * Index at which fragment indicator is found in fragments
- */
-#define ZT_PROTO_PACKET_FRAGMENT_INDICATOR_INDEX 13
-
-/**
- * Index of flags field in regular packet headers
- */
-#define ZT_PROTO_PACKET_FLAGS_INDEX 18
-
 /**
  * Length of a probe packet
  */
@@ -248,6 +239,16 @@
  */
 #define ZT_KBKDF_LABEL_PACKET_HMAC 'M'
 
+#define ZT_PROTO_PACKET_FRAGMENT_INDICATOR_INDEX 13
+#define ZT_PROTO_PACKET_FRAGMENT_COUNTS          14
+
+#define ZT_PROTO_PACKET_ID_INDEX          0
+#define ZT_PROTO_PACKET_DESTINATION_INDEX 8
+#define ZT_PROTO_PACKET_SOURCE_INDEX      13
+#define ZT_PROTO_PACKET_FLAGS_INDEX       18
+#define ZT_PROTO_PACKET_MAC_INDEX         19
+#define ZT_PROTO_PACKET_VERB_INDEX        27
+
 #define ZT_PROTO_HELLO_NODE_META_INSTANCE_ID      "i"
 #define ZT_PROTO_HELLO_NODE_META_LOCATOR          "l"
 #define ZT_PROTO_HELLO_NODE_META_PROBE_TOKEN      "p"
@@ -288,8 +289,7 @@ enum Verb
 	 *   <[8] timestamp>
 	 *   <[...] binary serialized full sender identity>
 	 *   <[...] physical destination address of packet (LEGACY)>
-	 *   <[2] 16-bit reserved "encrypted zero" field (LEGACY)>
-	 *   <[4] 32 additional random nonce bits>
+	 *   <[12] 96-bit CTR IV>
 	 *   [... start of encrypted section ...]
 	 *   <[2] 16-bit length of encrypted dictionary>
 	 *   <[...] encrypted dictionary>
@@ -306,9 +306,8 @@ enum Verb
 	 * good to proactively limit exposed information.
 	 *
 	 * Inner encryption is AES-CTR with a key derived using KBKDF and a
-	 * label indicating this specific usage. The 96-bit CTR nonce is the
-	 * packet ID followed by the additional 32 random bits provided before
-	 * the encrypted section.
+	 * label indicating this specific usage. A 96-bit CTR IV precedes this
+	 * encrypted section.
 	 *
 	 * Authentication and encryption in HELLO and OK(HELLO) are always done
 	 * with the long-lived identity key, not ephemeral shared keys. This
@@ -319,7 +318,7 @@ enum Verb
 	 * HELLO and OK(HELLO) include an extra HMAC at the end of the packet.
 	 * This authenticates them to a level of certainty beyond that afforded
 	 * by regular AEAD. HMAC is computed over the whole packet prior to
-	 * encryption/MAC and with the 3-bit hop count field masked as it is
+	 * packet MAC and with the 3-bit hop count field masked as it is
 	 * with regular packet AEAD, and it is then included in the regular
 	 * packet MAC.
 	 *
@@ -811,15 +810,79 @@ static ZT_INLINE void salsa2012DeriveKey(const uint8_t *const in,uint8_t *const
 #endif
 }
 
+/**
+ * Fill out packet header fields (except for mac, which is filled out by armor())
+ * 
+ * @param pkt Start of packet buffer
+ * @param packetId Packet IV / cryptographic MAC
+ * @param destination Destination ZT address
+ * @param source Source (sending) ZT address
+ * @param verb Protocol verb
+ * @return Index of packet start
+ */
+static ZT_INLINE int newPacket(uint8_t pkt[28],const uint64_t packetId,const Address destination,const Address source,const Verb verb) noexcept
+{
+	Utils::storeAsIsEndian<uint64_t>(pkt + ZT_PROTO_PACKET_ID_INDEX,packetId);
+	destination.copyTo(pkt + ZT_PROTO_PACKET_DESTINATION_INDEX);
+	source.copyTo(pkt + ZT_PROTO_PACKET_SOURCE_INDEX);
+	pkt[ZT_PROTO_PACKET_FLAGS_INDEX] = 0;
+	// mac is left undefined as it's filled out by armor()
+	pkt[ZT_PROTO_PACKET_VERB_INDEX] = (uint8_t)verb;
+	return ZT_PROTO_PACKET_VERB_INDEX + 1;
+}
+static ZT_INLINE int newPacket(Buf &pkt,const uint64_t packetId,const Address destination,const Address source,const Verb verb) noexcept { return newPacket(pkt.unsafeData,packetId,destination,source,verb); }
+
 /**
  * Encrypt and compute packet MAC
  *
  * @param pkt Packet data to encrypt (in place)
  * @param packetSize Packet size, must be at least ZT_PROTO_MIN_PACKET_LENGTH or crash will occur
- * @param key Key to use for encryption (not per-packet key)
+ * @param key Key to use for encryption
  * @param cipherSuite Cipher suite to use for AEAD encryption or just MAC
  */
-void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint8_t cipherSuite) noexcept;
+static ZT_INLINE void armor(uint8_t *const pkt,const int packetSize,const SharedPtr<SymmetricKey> &key,const uint8_t cipherSuite) noexcept
+{
+#if 0
+	Protocol::Header &ph = pkt.as<Protocol::Header>(); // NOLINT(hicpp-use-auto,modernize-use-auto)
+	ph.flags = (ph.flags & 0xc7U) | ((cipherSuite << 3U) & 0x38U); // flags: FFCCCHHH where CCC is cipher
+
+	switch(cipherSuite) {
+		case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE: {
+			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
+			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
+			Salsa20 s20(perPacketKey,&ph.packetId);
+
+			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
+			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
+
+			// only difference here is that we don't encrypt the payload
+
+			uint64_t mac[2];
+			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
+			ph.mac = mac[0];
+		} break;
+
+		case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012: {
+			uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
+			salsa2012DeriveKey(key,perPacketKey,pkt,packetSize);
+			Salsa20 s20(perPacketKey,&ph.packetId);
+
+			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
+			s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
+
+			const unsigned int encLen = packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START;
+			s20.crypt12(pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen);
+
+			uint64_t mac[2];
+			poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,encLen,macKey);
+			ph.mac = mac[0];
+		} break;
+
+		case ZT_PROTO_CIPHER_SUITE__AES_GMAC_SIV: {
+		} break;
+	}
+#endif
+}
 
 /**
  * Attempt to compress packet payload
@@ -833,7 +896,11 @@ void armor(Buf &pkt,int packetSize,const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],uint
  * @param packetSize Total size of packet in bytes (including headers)
  * @return New size of packet after compression or original size of compression wasn't helpful
  */
-int compress(SharedPtr<Buf> &pkt,int packetSize) noexcept;
+static ZT_INLINE int compress(SharedPtr<Buf> &pkt,int packetSize) noexcept
+{
+	// TODO
+	return packetSize;
+}
 
 } // namespace Protocol
 } // namespace ZeroTier

node/RuntimeEnvironment.hpp

@@ -40,7 +40,7 @@ class Expect;
 class RuntimeEnvironment
 {
 public:
-	ZT_INLINE RuntimeEnvironment(Node *n) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,google-explicit-constructor,hicpp-explicit-conversions)
+	ZT_INLINE RuntimeEnvironment(Node *n) noexcept :
 		node(n),
 		localNetworkController(nullptr),
 		rtmem(nullptr),
@@ -51,8 +51,8 @@ public:
 		topology(nullptr),
 		sa(nullptr)
 	{
-		publicIdentityStr[0] = (char)0;
-		secretIdentityStr[0] = (char)0;
+		publicIdentityStr[0] = nullptr;
+		secretIdentityStr[0] = nullptr;
 	}
 
 	ZT_INLINE ~RuntimeEnvironment() noexcept

node/SHA512.cpp

@@ -199,7 +199,7 @@ void SHA384(void *digest,const void *data0,unsigned int len0,const void *data1,u
 
 #endif // !ZT_HAVE_NATIVE_SHA512
 
-void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,uint8_t mac[48])
+void HMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],const void *msg,const unsigned int msglen,uint8_t mac[48])
 {
 	uint64_t kInPadded[16]; // input padded key
 	uint64_t outer[22]; // output padded key | H(input padded key | msg)
@@ -208,14 +208,16 @@ void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,
 	const uint64_t k1 = Utils::loadAsIsEndian<uint64_t>(key + 8);
 	const uint64_t k2 = Utils::loadAsIsEndian<uint64_t>(key + 16);
 	const uint64_t k3 = Utils::loadAsIsEndian<uint64_t>(key + 24);
+	const uint64_t k4 = Utils::loadAsIsEndian<uint64_t>(key + 32);
+	const uint64_t k5 = Utils::loadAsIsEndian<uint64_t>(key + 40);
 
 	const uint64_t ipad = 0x3636363636363636ULL;
 	kInPadded[0] = k0 ^ ipad;
 	kInPadded[1] = k1 ^ ipad;
 	kInPadded[2] = k2 ^ ipad;
 	kInPadded[3] = k3 ^ ipad;
-	kInPadded[4] = ipad;
-	kInPadded[5] = ipad;
+	kInPadded[4] = k4 ^ ipad;
+	kInPadded[5] = k5 ^ ipad;
 	kInPadded[6] = ipad;
 	kInPadded[7] = ipad;
 	kInPadded[8] = ipad;
@@ -232,8 +234,8 @@ void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,
 	outer[1] = k1 ^ opad;
 	outer[2] = k2 ^ opad;
 	outer[3] = k3 ^ opad;
-	outer[4] = opad;
-	outer[5] = opad;
+	outer[4] = k4 ^ opad;
+	outer[5] = k5 ^ opad;
 	outer[6] = opad;
 	outer[7] = opad;
 	outer[8] = opad;
@@ -245,12 +247,12 @@ void HMACSHA384(const uint8_t key[32],const void *msg,const unsigned int msglen,
 	outer[14] = opad;
 	outer[15] = opad;
 
-	SHA384(reinterpret_cast<uint8_t *>(outer) + 128,kInPadded,128,msg,msglen); // H(input padded key | msg)
-
-	SHA384(mac,outer,176); // H(output padded key | H(input padded key | msg))
+	// H(output padded key | H(input padded key | msg))
+	SHA384(reinterpret_cast<uint8_t *>(outer) + 128,kInPadded,128,msg,msglen);
+	SHA384(mac,outer,176);
 }
 
-void KBKDFHMACSHA384(const uint8_t key[32],const char label,const char context,const uint32_t iter,uint8_t out[32])
+void KBKDFHMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],const char label,const char context,const uint32_t iter,uint8_t out[ZT_SYMMETRIC_KEY_SIZE])
 {
 	uint8_t kbkdfMsg[13];
 	uint8_t kbuf[48];
@@ -265,7 +267,7 @@ void KBKDFHMACSHA384(const uint8_t key[32],const char label,const char context,c
 	kbkdfMsg[11] = 1;
 	kbkdfMsg[12] = 0; // key length: 256 bits as big-endian 32-bit value
 	HMACSHA384(key,&kbkdfMsg,sizeof(kbkdfMsg),kbuf);
-	Utils::copy<32>(out,kbuf);
+	Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(out,kbuf);
 }
 
 } // namespace ZeroTier

node/SHA512.hpp

@@ -70,7 +70,7 @@ void SHA384(void *digest,const void *data0,unsigned int len0,const void *data1,u
  * @param msglen Length of message
  * @param mac Buffer to fill with result
  */
-void HMACSHA384(const uint8_t key[32],const void *msg,unsigned int msglen,uint8_t mac[48]);
+void HMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],const void *msg,unsigned int msglen,uint8_t mac[48]);
 
 /**
  * Compute KBKDF (key-based key derivation function) using HMAC-SHA-384 as a PRF
@@ -81,7 +81,7 @@ void HMACSHA384(const uint8_t key[32],const void *msg,unsigned int msglen,uint8_
  * @param iter Key iteration for generation of multiple keys for the same label/context
  * @param out Output to receive derived key
  */
-void KBKDFHMACSHA384(const uint8_t key[32],char label,char context,uint32_t iter,uint8_t out[32]);
+void KBKDFHMACSHA384(const uint8_t key[ZT_SYMMETRIC_KEY_SIZE],char label,char context,uint32_t iter,uint8_t out[ZT_SYMMETRIC_KEY_SIZE]);
 
 } // namespace ZeroTier
 

node/Salsa20.hpp

@@ -29,6 +29,8 @@
 #define ZT_SALSA20_SSE 1
 #endif
 
+#define ZT_SALSA20_KEY_SIZE 32
+
 namespace ZeroTier {
 
 /**
@@ -43,14 +45,14 @@ public:
 	static constexpr bool accelerated() noexcept { return false; }
 #endif
 
-	ZT_INLINE Salsa20() noexcept {} // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Salsa20() noexcept {}
 	ZT_INLINE ~Salsa20() { Utils::burn(&_state,sizeof(_state)); }
 
 	/**
 	 * @param key 256-bit (32 byte) key
 	 * @param iv 64-bit initialization vector
 	 */
-	ZT_INLINE Salsa20(const void *key,const void *iv) noexcept { init(key,iv); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	ZT_INLINE Salsa20(const void *key,const void *iv) noexcept { init(key,iv); }
 
 	/**
 	 * Initialize cipher

node/ScopedPtr.hpp

@@ -49,7 +49,7 @@ public:
 	ZT_INLINE bool operator!=(T *const p) const noexcept { return (m_ptr != p); }
 
 private:
-	ZT_INLINE ScopedPtr() noexcept {} // NOLINT(hicpp-use-equals-default,hicpp-use-equals-delete,modernize-use-equals-default)
+	ZT_INLINE ScopedPtr() noexcept {}
 	ZT_INLINE ScopedPtr(const ScopedPtr &p) noexcept : m_ptr(nullptr) {}
 	ZT_INLINE ScopedPtr &operator=(const ScopedPtr &p) noexcept { return *this; }
 

node/SelfAwareness.hpp

@@ -70,7 +70,7 @@ private:
 		InetAddress reporterPhysicalAddress;
 		InetAddress::IpScope scope;
 
-		ZT_INLINE p_PhySurfaceKey() noexcept {} // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
+		ZT_INLINE p_PhySurfaceKey() noexcept {}
 		ZT_INLINE p_PhySurfaceKey(const Address &r, const int64_t rol, const InetAddress &ra, InetAddress::IpScope s) noexcept : reporter(r), receivedOnLocalSocket(rol), reporterPhysicalAddress(ra), scope(s) {}
 
 		ZT_INLINE unsigned long hashCode() const noexcept { return ((unsigned long)reporter.toInt() + (unsigned long)receivedOnLocalSocket + (unsigned long)scope); }

node/SharedPtr.hpp

@@ -109,7 +109,7 @@ public:
 		from.m_ptr = nullptr;
 	}
 
-	ZT_INLINE operator bool() const noexcept { return (m_ptr != nullptr); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
+	ZT_INLINE operator bool() const noexcept { return (m_ptr != nullptr); }
 
 	ZT_INLINE T &operator*() const noexcept { return *m_ptr; }
 	ZT_INLINE T *operator->() const noexcept { return m_ptr; }

node/Speck128.hpp

@@ -39,14 +39,14 @@ public:
 	/**
 	 * Create an uninitialized instance, init() must be called to set up.
 	 */
-	ZT_INLINE Speck128() noexcept {} // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Speck128() noexcept {}
 
 	/**
 	 * Initialize Speck from a 128-bit key
 	 *
 	 * @param k 128-bit / 16 byte key
 	 */
-	ZT_INLINE Speck128(const void *k) noexcept { this->init(k); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,google-explicit-constructor,hicpp-explicit-conversions)
+	ZT_INLINE Speck128(const void *k) noexcept { this->init(k); }
 
 	ZT_INLINE ~Speck128() { Utils::burn(m_expandedKey, sizeof(m_expandedKey)); }
 
@@ -162,8 +162,8 @@ public:
 	 */
 	ZT_INLINE void encrypt(const void *const in,void *const out) const noexcept
 	{
-		uint64_t x = Utils::loadLittleEndian<uint64_t>(in); // NOLINT(hicpp-use-auto,modernize-use-auto)
-		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		uint64_t x = Utils::loadLittleEndian<uint64_t>(in);
+		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8);
 		encryptXY(x,y);
 		Utils::storeLittleEndian<uint64_t>(out,x);
 		Utils::storeLittleEndian<uint64_t>(reinterpret_cast<uint8_t *>(out) + 8,y);
@@ -177,8 +177,8 @@ public:
 	 */
 	ZT_INLINE void decrypt(const void *const in,void *const out) const noexcept
 	{
-		uint64_t x = Utils::loadLittleEndian<uint64_t>(in); // NOLINT(hicpp-use-auto,modernize-use-auto)
-		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		uint64_t x = Utils::loadLittleEndian<uint64_t>(in);
+		uint64_t y = Utils::loadLittleEndian<uint64_t>(reinterpret_cast<const uint8_t *>(in) + 8);
 		decryptXY(x,y);
 		Utils::storeLittleEndian<uint64_t>(out,x);
 		Utils::storeLittleEndian<uint64_t>(reinterpret_cast<uint8_t *>(out) + 8,y);

node/SymmetricKey.hpp

@@ -17,119 +17,70 @@
 #include "Constants.hpp"
 #include "Utils.hpp"
 #include "InetAddress.hpp"
+#include "AES.hpp"
+#include "SharedPtr.hpp"
+#include "Address.hpp"
 
 namespace ZeroTier {
 
-#define ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX 52
-
 /**
  * Container for symmetric keys and ciphers initialized with them
  *
  * This container is responsible for tracking key TTL to maintain it
  * below our security bounds and tell us when it's time to re-key.
- *
- * Set TTL and TTLM to 0 for permanent keys. These still track uses
- * but do not signal expiration.
- *
- * @tparam C Cipher to embed (must accept key in constructor and/or init() method)
- * @tparam TTL Maximum time to live in milliseconds or 0 for a permanent key with unlimited TTL
- * @tparam TTLM Maximum time to live in messages or 0 for a permanent key with unlimited TTL
  */
-template<typename C,int64_t TTL,uint64_t TTLM>
 class SymmetricKey
 {
+	friend class SharedPtr<SymmetricKey>;
+
 public:
 	/**
-	 * Symmetric cipher keyed with this key
+	 * Secret key
 	 */
-	const C cipher;
+	const uint8_t secret[ZT_SYMMETRIC_KEY_SIZE];
 
 	/**
-	 * Construct an uninitialized symmetric key container
+	 * Symmetric cipher keyed with this key
 	 */
-	ZT_INLINE SymmetricKey() noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init,hicpp-use-equals-default,modernize-use-equals-default)
-		cipher(),
-		m_ts(0),
-		m_nonceBase(0),
-		m_odometer(0)
-	{
-		Utils::memoryLock(m_secret, sizeof(m_secret));
-	}
+	const AES cipher;
 
 	/**
 	 * Construct a new symmetric key
 	 *
-	 * @param ts Current time (must still be given for permanent keys even though there is no expiry checking)
-	 * @param key 32-byte / 256-bit key
+	 * SECURITY: we use a best effort method to construct the nonce's starting point so as
+	 * to avoid nonce duplication across invocations. The most significant bits are the
+	 * number of seconds since epoch but with the most significant bit masked to zero.
+	 * The least significant bits are random. Key life time is limited to 2^31 messages
+	 * per key as per the AES-GMAC-SIV spec, and this is a SIV mode anyway so nonce repetition
+	 * is non-catastrophic.
+	 * 
+	 * The masking of the most significant bit is because we bisect the nonce space by
+	 * which direction the message is going. If the sender's ZeroTier address is
+	 * numerically greater than the receiver, this bit is flipped. This means that
+	 * two sides of a conversation that have created their key instances at the same
+	 * time are much less likely to duplicate nonces when sending pacekts from either
+	 * end.
+	 * 
+	 * @param ts Current time
+	 * @param key 48-bit / 384-byte key
+	 * @param perm If true this is a permanent key
 	 */
-	explicit ZT_INLINE SymmetricKey(const int64_t ts,const void *const key) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-		cipher(key),
+	explicit ZT_INLINE SymmetricKey(const int64_t ts,const void *const key,const bool perm) noexcept :
+		secret(),
+		cipher(key), // uses first 256 bits of 384-bit key
 		m_ts(ts),
-		m_nonceBase((uint64_t)ts << 22U), // << 22 to shift approximately the seconds since epoch into the most significant 32 bits
-		m_odometer(0)
-	{
-		Utils::memoryLock(m_secret, sizeof(m_secret));
-		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, key);
-	}
-
-	ZT_INLINE SymmetricKey(const SymmetricKey &k) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-		cipher(k.m_secret),
-		m_ts(k.ts),
-		m_nonceBase(k.m_nonceBase),
-		m_odometer(k.m_odometer)
+		m_nonceBase(((((uint64_t)ts / 1000ULL) << 32U) & 0x7fffffff00000000ULL) | (Utils::random() & 0x00000000ffffffffULL)),
+		m_odometer(0),
+		m_permanent(perm)
 	{
-		Utils::memoryLock(m_secret, sizeof(m_secret));
-		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, k.m_secret);
+		Utils::memoryLock(this,sizeof(SymmetricKey));
+		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(const_cast<uint8_t *>(secret), key);
 	}
 
 	ZT_INLINE ~SymmetricKey() noexcept
 	{
-		Utils::burn(m_secret, sizeof(m_secret));
-		Utils::memoryUnlock(m_secret, sizeof(m_secret));
-	}
-
-	ZT_INLINE SymmetricKey &operator=(const SymmetricKey &k) noexcept
-	{
-		if (&k != this) {
-			cipher.init(k.m_secret);
-			m_ts = k.m_ts;
-			m_nonceBase = k.m_nonceBase;
-			m_odometer = k.m_odometer;
-			Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, k.m_secret);
-		}
-		return *this;
-	}
-
-	/**
-	 * Initialize or change the key wrapped by this SymmetricKey object
-	 *
-	 * If the supplied key is identical to the current key, no change occurs and false is returned.
-	 *
-	 * @param ts Current time
-	 * @param key 32-byte / 256-bit key
-	 * @return True if the symmetric key was changed
-	 */
-	ZT_INLINE bool init(const int64_t ts,const void *const key) noexcept
-	{
-		if ((m_ts > 0) && (memcmp(m_secret, key, ZT_SYMMETRIC_KEY_SIZE) == 0))
-			return false;
-		cipher.init(key);
-		m_ts = ts;
-		m_nonceBase = (uint64_t)ts << 22U; // << 22 to shift approximately the seconds since epoch into the most significant 32 bits;
-		m_odometer = 0;
-		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(m_secret, key);
-		return true;
-	}
-
-	/**
-	 * Clear key and set to NIL value (boolean evaluates to false)
-	 */
-	ZT_INLINE void clear() noexcept
-	{
-		m_ts = 0;
-		m_nonceBase = 0;
-		m_odometer = 0;
-		Utils::zero<ZT_SYMMETRIC_KEY_SIZE>(m_secret);
+		Utils::burn(const_cast<uint8_t *>(secret),ZT_SYMMETRIC_KEY_SIZE);
+		Utils::memoryUnlock(this,sizeof(SymmetricKey));
 	}
 
 	/**
@@ -140,7 +91,7 @@ public:
 	 */
 	ZT_INLINE bool expiringSoon(const int64_t now) const noexcept
 	{
-		return (TTL > 0) && (((now - m_ts) >= (TTL / 2)) || (m_odometer >= (TTLM / 2)) );
+		return (!m_permanent) && (((now - m_ts) >= (ZT_SYMMETRIC_KEY_TTL / 2)) || (m_odometer >= (ZT_SYMMETRIC_KEY_TTL_MESSAGES / 2)) );
 	}
 
 	/**
@@ -151,116 +102,27 @@ public:
 	 */
 	ZT_INLINE bool expired(const int64_t now) const noexcept
 	{
-		return (TTL > 0) && (((now - m_ts) >= TTL) || (m_odometer >= TTLM) );
-	}
-
-	/**
-	 * @return True if this is a never-expiring key, such as the identity key created by identity key agreement
-	 */
-	constexpr bool permanent() const noexcept
-	{
-		return TTL == 0;
-	}
-
-	/**
-	 * Get the raw key that was used to initialize the cipher.
-	 *
-	 * @return 32-byte / 256-bit symmetric key
-	 */
-	ZT_INLINE const uint8_t *key() const noexcept
-	{
-		return m_secret;
+		return (!m_permanent) && (((now - m_ts) >= ZT_SYMMETRIC_KEY_TTL) || (m_odometer >= ZT_SYMMETRIC_KEY_TTL_MESSAGES) );
 	}
 
 	/**
-	 * Advance usage counter by one and return the next unique initialization vector for a new message.
+	 * Advance usage counter by one and return the next IV / packet ID.
 	 *
+	 * @param sender Sending ZeroTier address
+	 * @param receiver Receiving ZeroTier address
 	 * @return Next unique IV for next message
 	 */
-	ZT_INLINE uint64_t nextMessageIv() noexcept
+	ZT_INLINE uint64_t nextMessage(const Address sender,const Address receiver) noexcept
 	{
-		return m_nonceBase + m_odometer++;
-	}
-
-	/**
-	 * @return True if this object is not NIL
-	 */
-	ZT_INLINE operator bool() const noexcept { return (m_ts > 0); } // NOLINT(google-explicit-constructor,hicpp-explicit-conversions)
-
-	static constexpr int marshalSizeMax() noexcept { return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX; }
-
-	/**
-	 * Marshal with encryption at rest
-	 *
-	 * @tparam MC Cipher type (AES in our code) to use for encryption at rest
-	 * @param keyEncCipher Initialized cipher
-	 * @param data Destination for marshaled key
-	 * @return Bytes written or -1 on error
-	 */
-	template<typename MC>
-	ZT_INLINE int marshal(const MC &keyEncCipher,uint8_t data[ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX]) const noexcept
-	{
-		Utils::storeBigEndian<uint64_t>(data,(uint64_t)m_ts);
-		Utils::storeBigEndian<uint64_t>(data + 8, m_odometer.load());
-		Utils::storeBigEndian<uint32_t>(data + 16,Utils::fnv1a32(m_secret, sizeof(m_secret)));
-
-		// Key encryption at rest is CBC using the last 32 bits of the timestamp, the odometer,
-		// and the FNV1a checksum as a 128-bit IV. A duplicate IV wouldn't matter much anyway since
-		// keys should be unique. Simple ECB would be fine as they also have no structure, but this
-		// looks better.
-		uint8_t tmp[16];
-		for(int i=0;i<16;++i)
-			tmp[i] = data[i + 4] ^ m_secret[i];
-		keyEncCipher.encrypt(tmp,data + 20);
-		for(int i=0;i<16;++i)
-			tmp[i] = data[i + 20] ^ m_secret[i + 16];
-		keyEncCipher.encrypt(tmp,data + 36);
-
-		return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX;
-	}
-
-	/**
-	 * Unmarshal, decrypt, and verify key checksum
-	 *
-	 * Key checksum verification failure results in the SymmetricKey being zeroed out to its
-	 * nil value, but the bytes read are still returned. The caller must check this if it
-	 * requires the key to be present and verified.
-	 *
-	 * @tparam MC Cipher type (AES in our code) to use for encryption at rest
-	 * @param keyDecCipher Initialized cipher for decryption
-	 * @param data Source to read
-	 * @param len Bytes remaining at source
-	 * @return Bytes read from source
-	 */
-	template<typename MC>
-	ZT_INLINE int unmarshal(const MC &keyDecCipher,const uint8_t *restrict data,int len) noexcept
-	{
-		if (len < ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX)
-			return -1;
-
-		m_ts = (int64_t)Utils::loadBigEndian<uint64_t>(data);
-		m_odometer = (uint64_t)Utils::loadBigEndian<uint64_t>(data + 8);
-		const uint32_t fnv = Utils::loadBigEndian<uint32_t>(data + 16); // NOLINT(hicpp-use-auto,modernize-use-auto)
-
-		uint8_t tmp[16];
-		keyDecCipher.decrypt(data + 20,tmp);
-		for(int i=0;i<16;++i)
-			m_secret[i] = data[i + 4] ^ tmp[i];
-		keyDecCipher.decrypt(data + 36,tmp);
-		for(int i=0;i<16;++i)
-			m_secret[i + 16] = data[i + 20] ^ tmp[i];
-
-		if (Utils::fnv1a32(m_secret, sizeof(m_secret)) != fnv)
-			clear();
-
-		return ZT_SYMMETRICKEY_MARSHAL_SIZE_MAX;
+		return (m_nonceBase + m_odometer++) ^ (((uint64_t)(sender > receiver)) << 63U);
 	}
 
 private:
-	int64_t m_ts;
-	uint64_t m_nonceBase;
+	const int64_t m_ts;
+	const uint64_t m_nonceBase;
 	std::atomic<uint64_t> m_odometer;
-	uint8_t m_secret[ZT_SYMMETRIC_KEY_SIZE];
+	std::atomic<int> __refCount;
+	const bool m_permanent;
 };
 
 } // namespace ZeroTier

node/Tag.hpp

@@ -55,7 +55,7 @@ class Tag : public Credential
 public:
 	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_TAG; }
 
-	ZT_INLINE Tag() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	ZT_INLINE Tag() noexcept { memoryZero(this); }
 
 	/**
 	 * @param nwid Network ID
@@ -64,7 +64,7 @@ public:
 	 * @param id Tag ID
 	 * @param value Tag value
 	 */
-	ZT_INLINE Tag(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) noexcept : // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	ZT_INLINE Tag(const uint64_t nwid,const int64_t ts,const Address &issuedTo,const uint32_t id,const uint32_t value) noexcept :
 		m_id(id),
 		m_value(value),
 		m_networkId(nwid),

node/Tests.cpp

@@ -48,7 +48,7 @@
 
 #ifdef __UNIX_LIKE__
 #include <unistd.h>
-#include <sys/time.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <sys/time.h>
 #include <sys/types.h>
 #endif
 

node/Tests.h

@@ -43,8 +43,8 @@
 
 #ifdef ZT_ENABLE_TESTS
 
-#include <stdint.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
-#include <stdio.h> // NOLINT(modernize-deprecated-headers,hicpp-deprecated-headers)
+#include <stdint.h>
+#include <stdio.h>
 
 #ifndef ZT_T_PRINTF
 #define ZT_T_PRINTF(fmt,...) printf((fmt),##__VA_ARGS__),fflush(stdout)

node/Topology.hpp

@@ -160,7 +160,7 @@ public:
 	ZT_INLINE void eachPeer(F f) const
 	{
 		RWMutex::RLock l(m_peers_l);
-		for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i) // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+		for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i)
 			f(i->second);
 	}
 
@@ -178,14 +178,14 @@ public:
 	{
 		RWMutex::RLock l(m_peers_l);
 
-		std::vector<uintptr_t> rootPeerPtrs;
+		Vector<uintptr_t> rootPeerPtrs;
 		rootPeerPtrs.reserve(m_rootPeers.size());
-		for(std::vector< SharedPtr<Peer> >::const_iterator rp(m_rootPeers.begin());rp != m_rootPeers.end();++rp) // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+		for(Vector< SharedPtr<Peer> >::const_iterator rp(m_rootPeers.begin());rp != m_rootPeers.end();++rp)
 			rootPeerPtrs.push_back((uintptr_t)rp->ptr());
 		std::sort(rootPeerPtrs.begin(),rootPeerPtrs.end());
 
 		try {
-			for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i) // NOLINT(modernize-loop-convert,hicpp-use-auto,modernize-use-auto)
+			for(Map< Address,SharedPtr<Peer> >::const_iterator i(m_peers.begin());i != m_peers.end();++i)
 				f(i->second,std::binary_search(rootPeerPtrs.begin(),rootPeerPtrs.end(),(uintptr_t)i->second.ptr()));
 		} catch ( ... ) {} // should not throw
 	}

node/Trace.cpp

@@ -17,6 +17,7 @@
 #include "Peer.hpp"
 #include "Path.hpp"
 #include "InetAddress.hpp"
+#include "FCV.hpp"
 
 // NOTE: packet IDs are always handled in network byte order, so no need to convert them.
 
@@ -34,16 +35,12 @@ void Trace::unexpectedError(
 	const char *message,
 	...)
 {
-	va_list ap;
-	ZT_TraceEvent_UNEXPECTED_ERROR ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_UNEXPECTED_ERROR);
-	ev.codeLocation = codeLocation;
-	Utils::zero<sizeof(ev.message)>(ev.message);
-	va_start(ap,message);
-	vsnprintf(ev.message,sizeof(ev.message),message,ap);
-	va_end(ap);
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_UNEXPECTED_ERROR);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_MESSAGE,message);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_resettingPathsInScope(
@@ -55,15 +52,18 @@ void Trace::_resettingPathsInScope(
 	const InetAddress &newExternal,
 	const InetAddress::IpScope scope)
 {
-	ZT_TraceEvent_VL1_RESETTING_PATHS_IN_SCOPE ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE);
-	ev.codeLocation = Utils::hton(codeLocation);
-	from.forTrace(ev.from);
-	oldExternal.forTrace(ev.oldExternal);
-	newExternal.forTrace(ev.newExternal);
-	ev.scope = (uint8_t)scope;
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	if (from)
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(from));
+	if (oldExternal)
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_OLD_ENDPOINT,Endpoint(oldExternal));
+	if (newExternal)
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_NEW_ENDPOINT,Endpoint(newExternal));
+	Dictionary::append(buf,ZT_TRACE_FIELD_RESET_ADDRESS_SCOPE,scope);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_tryingNewPath(
@@ -74,21 +74,20 @@ void Trace::_tryingNewPath(
 	const InetAddress &triggerAddress,
 	const uint64_t triggeringPacketId,
 	const uint8_t triggeringPacketVerb,
-	const Identity &triggeringPeer,
-	const ZT_TraceTryingNewPathReason reason)
+	const Identity &triggeringPeer)
 {
-	ZT_TraceEvent_VL1_TRYING_NEW_PATH ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_TRYING_NEW_PATH);
-	ev.codeLocation = Utils::hton(codeLocation);
-	Utils::copy<sizeof(ev.peer)>(&ev.peer,trying.fingerprint().apiFingerprint());
-	physicalAddress.forTrace(ev.physicalAddress);
-	triggerAddress.forTrace(ev.triggerAddress);
-	ev.triggeringPacketId = triggeringPacketId;
-	ev.triggeringPacketVerb = triggeringPacketVerb;
-	Utils::copy<sizeof(ev.triggeringPeer)>(&ev.triggeringPeer,triggeringPeer.fingerprint().apiFingerprint());
-	ev.reason = (uint8_t)reason;
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_TRYING_NEW_PATH);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,trying.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
+	if (triggerAddress)
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_TRIGGER_FROM_ENDPOINT,Endpoint(triggerAddress));
+	Dictionary::appendPacketId(buf,ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_ID,triggeringPacketId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_TRIGGER_FROM_PACKET_VERB,triggeringPacketVerb);
+	if (triggeringPeer)
+		Dictionary::append(buf,ZT_TRACE_FIELD_TRIGGER_FROM_PEER_FINGERPRINT_HASH,triggeringPeer.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_learnedNewPath(
@@ -99,16 +98,17 @@ void Trace::_learnedNewPath(
 	const InetAddress &physicalAddress,
 	const InetAddress &replaced)
 {
-	ZT_TraceEvent_VL1_LEARNED_NEW_PATH ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_LEARNED_NEW_PATH);
-	ev.codeLocation = Utils::hton(codeLocation);
-	ev.packetId = packetId; // packet IDs are kept in big-endian
-	Utils::copy<sizeof(ev.peer)>(&ev.peer,peerIdentity.fingerprint().apiFingerprint());
-	physicalAddress.forTrace(ev.physicalAddress);
-	replaced.forTrace(ev.replaced);
-
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_LEARNED_NEW_PATH);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::appendPacketId(buf,ZT_TRACE_FIELD_PACKET_ID,packetId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,peerIdentity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
+	if (physicalAddress)
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(physicalAddress));
+	if (replaced)
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_OLD_ENDPOINT,Endpoint(replaced));
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_incomingPacketDropped(
@@ -122,19 +122,19 @@ void Trace::_incomingPacketDropped(
 	const uint8_t verb,
 	const ZT_TracePacketDropReason reason)
 {
-	ZT_TraceEvent_VL1_INCOMING_PACKET_DROPPED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
-	ev.codeLocation = Utils::hton(codeLocation);
-	ev.packetId = packetId; // packet IDs are kept in big-endian
-	ev.networkId = Utils::hton(networkId);
-	Utils::copy<sizeof(ev.peer)>(&ev.peer,peerIdentity.fingerprint().apiFingerprint());
-	physicalAddress.forTrace(ev.physicalAddress);
-	ev.hops = hops;
-	ev.verb = verb;
-	ev.reason = (uint8_t)reason;
-
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::appendPacketId(buf,ZT_TRACE_FIELD_PACKET_ID,packetId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,peerIdentity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
+	if (physicalAddress)
+		Dictionary::append(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(physicalAddress));
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_HOPS,hops);
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_VERB,verb);
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_outgoingNetworkFrameDropped(
@@ -148,25 +148,18 @@ void Trace::_outgoingNetworkFrameDropped(
 	const uint8_t *frameData,
 	const ZT_TraceFrameDropReason reason)
 {
-	ZT_TraceEvent_VL2_OUTGOING_FRAME_DROPPED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_OUTGOING_FRAME_DROPPED);
-	ev.codeLocation = Utils::hton(codeLocation);
-	ev.networkId = Utils::hton(networkId);
-	ev.sourceMac = Utils::hton(sourceMac.toInt());
-	ev.destMac = Utils::hton(destMac.toInt());
-	ev.etherType = Utils::hton(etherType);
-	ev.frameLength = Utils::hton(frameLength);
-	if (frameData) {
-		unsigned int l = frameLength;
-		if (l > sizeof(ev.frameHead))
-			l = sizeof(ev.frameHead);
-		Utils::copy(ev.frameHead,frameData,l);
-		Utils::zero(ev.frameHead + l,sizeof(ev.frameHead) - l);
-	}
-	ev.reason = (uint8_t)reason;
-
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_MAC,sourceMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_MAC,destMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_ETHERTYPE,etherType);
+	Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_LENGTH,frameLength);
+	if (frameData)
+		Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_DATA,frameData,std::min((unsigned int)64,(unsigned int)frameLength));
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_incomingNetworkFrameDropped(
@@ -184,29 +177,23 @@ void Trace::_incomingNetworkFrameDropped(
 	const bool credentialRequestSent,
 	const ZT_TraceFrameDropReason reason)
 {
-	ZT_TraceEvent_VL2_INCOMING_FRAME_DROPPED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_INCOMING_FRAME_DROPPED);
-	ev.codeLocation = Utils::hton(codeLocation);
-	ev.networkId = Utils::hton(networkId);
-	ev.sourceMac = Utils::hton(sourceMac.toInt());
-	ev.destMac = Utils::hton(destMac.toInt());
-	Utils::copy<sizeof(ev.sender)>(&ev.sender,peerIdentity.fingerprint().apiFingerprint());
-	physicalAddress.forTrace(ev.physicalAddress);
-	ev.hops = hops;
-	ev.frameLength = Utils::hton(frameLength);
-	if (frameData) {
-		unsigned int l = frameLength;
-		if (l > sizeof(ev.frameHead))
-			l = sizeof(ev.frameHead);
-		Utils::copy(ev.frameHead,frameData,l);
-		Utils::zero(ev.frameHead + l,sizeof(ev.frameHead) - l);
-	}
-	ev.verb = verb;
-	ev.credentialRequestSent = (uint8_t)credentialRequestSent;
-	ev.reason = (uint8_t)reason;
-
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_INCOMING_FRAME_DROPPED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_MAC,sourceMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_MAC,destMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,peerIdentity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
+	if (physicalAddress)
+		Dictionary::appendObject(buf,ZT_TRACE_FIELD_ENDPOINT,Endpoint(physicalAddress));
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_HOPS,hops);
+	Dictionary::append(buf,ZT_TRACE_FIELD_PACKET_VERB,verb);
+	Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_LENGTH,frameLength);
+	if (frameData)
+		Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_DATA,frameData,std::min((unsigned int)64,(unsigned int)frameLength));
+	Dictionary::append(buf,ZT_TRACE_FIELD_FLAG_CREDENTIAL_REQUEST_SENT,credentialRequestSent);
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_networkConfigRequestSent(
@@ -214,12 +201,12 @@ void Trace::_networkConfigRequestSent(
 	const uint32_t codeLocation,
 	const uint64_t networkId)
 {
-	ZT_TraceEvent_VL2_NETWORK_CONFIG_REQUESTED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_CONFIG_REQUESTED);
-	ev.codeLocation = Utils::hton(codeLocation);
-	ev.networkId = Utils::hton(networkId);
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_NETWORK_CONFIG_REQUESTED);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_networkFilter(
@@ -242,64 +229,53 @@ void Trace::_networkFilter(
 	const bool inbound,
 	const int accept)
 {
-	ZT_TraceEvent_VL2_NETWORK_FILTER ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
-	ev.codeLocation = Utils::hton(codeLocation);
-	ev.networkId = Utils::hton(networkId);
-	Utils::copy<sizeof(ev.primaryRuleSetLog)>(ev.primaryRuleSetLog,primaryRuleSetLog);
-	if (matchingCapabilityRuleSetLog)
-		Utils::copy<sizeof(ev.matchingCapabilityRuleSetLog)>(ev.matchingCapabilityRuleSetLog,matchingCapabilityRuleSetLog);
-	else Utils::zero<sizeof(ev.matchingCapabilityRuleSetLog)>(ev.matchingCapabilityRuleSetLog);
-	ev.matchingCapabilityId = Utils::hton(matchingCapabilityId);
-	ev.matchingCapabilityTimestamp = Utils::hton(matchingCapabilityTimestamp);
-	ev.source = Utils::hton(source.toInt());
-	ev.dest = Utils::hton(dest.toInt());
-	ev.sourceMac = Utils::hton(sourceMac.toInt());
-	ev.destMac = Utils::hton(destMac.toInt());
-	ev.frameLength = Utils::hton(frameLength);
-	if (frameData) {
-		unsigned int l = frameLength;
-		if (l > sizeof(ev.frameHead))
-			l = sizeof(ev.frameHead);
-		Utils::copy(ev.frameHead,frameData,l);
-		Utils::zero(ev.frameHead + l,sizeof(ev.frameHead) - l);
-	}
-	ev.etherType = Utils::hton(etherType);
-	ev.vlanId = Utils::hton(vlanId);
-	ev.noTee = (uint8_t)noTee;
-	ev.inbound = (uint8_t)inbound;
-	ev.accept = (int8_t)accept;
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_NETWORK_FILTER);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
+	if ((primaryRuleSetLog)&&(!Utils::allZero(primaryRuleSetLog,512)))
+		Dictionary::append(buf,ZT_TRACE_FIELD_PRIMARY_RULE_SET_LOG,primaryRuleSetLog,512);
+	if ((matchingCapabilityRuleSetLog)&&(!Utils::allZero(matchingCapabilityRuleSetLog,512)))
+		Dictionary::append(buf,ZT_TRACE_FIELD_MATCHING_CAPABILITY_RULE_SET_LOG,matchingCapabilityRuleSetLog,512);
+	Dictionary::append(buf,ZT_TRACE_FIELD_MATCHING_CAPABILITY_ID,matchingCapabilityId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_MATCHING_CAPABILITY_TIMESTAMP,matchingCapabilityTimestamp);
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_ZT_ADDRESS,source);
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_ZT_ADDRESS,dest);
+	Dictionary::append(buf,ZT_TRACE_FIELD_SOURCE_MAC,sourceMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_DEST_MAC,destMac.toInt());
+	Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_LENGTH,frameLength);
+	if (frameData)
+		Dictionary::append(buf,ZT_TRACE_FIELD_FRAME_DATA,frameData,std::min((unsigned int)64,(unsigned int)frameLength));
+	Dictionary::append(buf,ZT_TRACE_FIELD_ETHERTYPE,etherType);
+	Dictionary::append(buf,ZT_TRACE_FIELD_VLAN_ID,vlanId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_RULE_FLAG_NOTEE,noTee);
+	Dictionary::append(buf,ZT_TRACE_FIELD_RULE_FLAG_INBOUND,inbound);
+	Dictionary::append(buf,ZT_TRACE_FIELD_RULE_FLAG_ACCEPT,(int32_t)accept);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 void Trace::_credentialRejected(
 	void *const tPtr,
 	const uint32_t codeLocation,
 	const uint64_t networkId,
-	const Address &address,
 	const Identity &identity,
 	const uint32_t credentialId,
 	const int64_t credentialTimestamp,
 	const uint8_t credentialType,
 	const ZT_TraceCredentialRejectionReason reason)
 {
-	ZT_TraceEvent_VL2_CREDENTIAL_REJECTED ev; // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
-	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
-	ev.codeLocation = Utils::hton(codeLocation);
-	ev.networkId = Utils::hton(networkId);
-	if (identity) {
-		Utils::copy<sizeof(ev.peer)>(&ev.peer,identity.fingerprint().apiFingerprint());
-	} else {
-		ev.peer.address = address.toInt();
-		Utils::zero<sizeof(ev.peer.hash)>(ev.peer.hash);
-	}
-	ev.credentialId = Utils::hton(credentialId);
-	ev.credentialTimestamp = Utils::hton(credentialTimestamp);
-	ev.credentialType = credentialType;
-	ev.reason = (uint8_t)reason;
-	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+	FCV<uint8_t,4096> buf;
+	Dictionary::append(buf,ZT_TRACE_FIELD_TYPE,ZT_TRACE_VL2_NETWORK_FILTER);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CODE_LOCATION,codeLocation);
+	Dictionary::append(buf,ZT_TRACE_FIELD_NETWORK_ID,networkId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_IDENTITY_FINGERPRINT_HASH,identity.fingerprint().hash(),ZT_FINGERPRINT_HASH_SIZE);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CREDENTIAL_ID,credentialId);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CREDENTIAL_TIMESTAMP,credentialTimestamp);
+	Dictionary::append(buf,ZT_TRACE_FIELD_CREDENTIAL_TYPE,credentialType);
+	Dictionary::append(buf,ZT_TRACE_FIELD_REASON,reason);
+	buf.push_back(0);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,buf.data());
 }
 
 } // namespace ZeroTier

node/Trace.hpp

@@ -104,11 +104,10 @@ public:
 		const InetAddress &triggerAddress,
 		uint64_t triggeringPacketId,
 		uint8_t triggeringPacketVerb,
-		const Identity &triggeringPeer,
-		ZT_TraceTryingNewPathReason reason)
+		const Identity &triggeringPeer)
 	{
 		if ((_f & ZT_TRACE_F_VL1) != 0)
-			_tryingNewPath(tPtr,codeLocation,trying,physicalAddress,triggerAddress,triggeringPacketId,triggeringPacketVerb,triggeringPeer,reason);
+			_tryingNewPath(tPtr,codeLocation,trying,physicalAddress,triggerAddress,triggeringPacketId,triggeringPacketVerb,triggeringPeer);
 	}
 
 	ZT_INLINE void learnedNewPath(
@@ -228,7 +227,6 @@ public:
 		void *const tPtr,
 		const uint32_t codeLocation,
 		uint64_t networkId,
-		const Address &address,
 		const Identity &identity,
 		uint32_t credentialId,
 		int64_t credentialTimestamp,
@@ -236,7 +234,7 @@ public:
 		ZT_TraceCredentialRejectionReason reason)
 	{
 		if ((_f & ZT_TRACE_F_VL2) != 0)
-			_credentialRejected(tPtr,codeLocation,networkId,address,identity,credentialId,credentialTimestamp,credentialType,reason);
+			_credentialRejected(tPtr,codeLocation,networkId,identity,credentialId,credentialTimestamp,credentialType,reason);
 	}
 
 private:
@@ -256,8 +254,7 @@ private:
 		const InetAddress &triggerAddress,
 		uint64_t triggeringPacketId,
 		uint8_t triggeringPacketVerb,
-		const Identity &triggeringPeer,
-		ZT_TraceTryingNewPathReason reason);
+		const Identity &triggeringPeer);
 	void _learnedNewPath(
 		void *tPtr,
 		uint32_t codeLocation,
@@ -326,7 +323,6 @@ private:
 		void *tPtr,
 		uint32_t codeLocation,
 		uint64_t networkId,
-		const Address &address,
 		const Identity &identity,
 		uint32_t credentialId,
 		int64_t credentialTimestamp,

node/Utils.cpp

@@ -36,7 +36,7 @@ namespace ZeroTier {
 namespace Utils {
 
 #ifdef ZT_ARCH_X64
-CPUIDRegisters::CPUIDRegisters() noexcept // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+CPUIDRegisters::CPUIDRegisters() noexcept
 {
 #ifdef __WINDOWS__
 	int regs[4];
@@ -100,58 +100,21 @@ char *decimal(unsigned long n,char s[24]) noexcept
 	return s;
 }
 
-char *hex(uint8_t i,char s[3]) noexcept
+char *hex(uint64_t i,char buf[17]) noexcept
 {
-	s[0] = HEXCHARS[(i >> 4U) & 0xfU];
-	s[1] = HEXCHARS[i & 0xfU];
-	s[2] = 0;
-	return s;
-}
-
-char *hex(uint16_t i,char s[5]) noexcept
-{
-	s[0] = HEXCHARS[(i >> 12U) & 0xfU];
-	s[1] = HEXCHARS[(i >> 8U) & 0xfU];
-	s[2] = HEXCHARS[(i >> 4U) & 0xfU];
-	s[3] = HEXCHARS[i & 0xfU];
-	s[4] = 0;
-	return s;
-}
-
-char *hex(uint32_t i,char s[9]) noexcept
-{
-	s[0] = HEXCHARS[(i >> 28U) & 0xfU];
-	s[1] = HEXCHARS[(i >> 24U) & 0xfU];
-	s[2] = HEXCHARS[(i >> 20U) & 0xfU];
-	s[3] = HEXCHARS[(i >> 16U) & 0xfU];
-	s[4] = HEXCHARS[(i >> 12U) & 0xfU];
-	s[5] = HEXCHARS[(i >> 8U) & 0xfU];
-	s[6] = HEXCHARS[(i >> 4U) & 0xfU];
-	s[7] = HEXCHARS[i & 0xfU];
-	s[8] = 0;
-	return s;
-}
-
-char *hex(uint64_t i,char s[17]) noexcept
-{
-	s[0] = HEXCHARS[(i >> 60U) & 0xfU];
-	s[1] = HEXCHARS[(i >> 56U) & 0xfU];
-	s[2] = HEXCHARS[(i >> 52U) & 0xfU];
-	s[3] = HEXCHARS[(i >> 48U) & 0xfU];
-	s[4] = HEXCHARS[(i >> 44U) & 0xfU];
-	s[5] = HEXCHARS[(i >> 40U) & 0xfU];
-	s[6] = HEXCHARS[(i >> 36U) & 0xfU];
-	s[7] = HEXCHARS[(i >> 32U) & 0xfU];
-	s[8] = HEXCHARS[(i >> 28U) & 0xfU];
-	s[9] = HEXCHARS[(i >> 24U) & 0xfU];
-	s[10] = HEXCHARS[(i >> 20U) & 0xfU];
-	s[11] = HEXCHARS[(i >> 16U) & 0xfU];
-	s[12] = HEXCHARS[(i >> 12U) & 0xfU];
-	s[13] = HEXCHARS[(i >> 8U) & 0xfU];
-	s[14] = HEXCHARS[(i >> 4U) & 0xfU];
-	s[15] = HEXCHARS[i & 0xfU];
-	s[16] = 0;
-	return s;
+	if (i) {
+		char *p = buf + 16;
+		*p = 0;
+		while (i) {
+			*(--p) = HEXCHARS[i & 0xfU];
+			i >>= 4;
+		}
+		return p;
+	} else {
+		buf[0] = '0';
+		buf[1] = 0;
+		return buf;
+	}
 }
 
 uint64_t unhex(const char *s) noexcept
@@ -165,11 +128,11 @@ uint64_t unhex(const char *s) noexcept
 
 			uint8_t c = 0;
 			if ((hc >= 48)&&(hc <= 57))
-				c = hc - 48;
+				c = (uint8_t)hc - 48;
 			else if ((hc >= 97)&&(hc <= 102))
-				c = hc - 87;
+				c = (uint8_t)hc - 87;
 			else if ((hc >= 65)&&(hc <= 70))
-				c = hc - 55;
+				c = (uint8_t)hc - 55;
 
 			n <<= 4U;
 			n |= (uint64_t)c;
@@ -294,7 +257,7 @@ void getSecureRandom(void *const buf,const unsigned int bytes) noexcept
 #ifdef ZT_ARCH_X64
 				if (CPUID.rdrand) {
 					uint64_t tmp = 0;
-					for(int k=0;k<ZT_GETSECURERANDOM_STATE_SIZE;++k) { // NOLINT(modernize-loop-convert)
+					for(int k=0;k<ZT_GETSECURERANDOM_STATE_SIZE;++k) {
 						_rdrand64_step((unsigned long long *)&tmp);
 						randomState[k] ^= tmp;
 					}

node/Utils.hpp

@@ -137,38 +137,14 @@ char *decimal(unsigned long n,char s[24]) noexcept;
 /**
  * Convert an unsigned integer into hex
  *
+ * The returned pointer won't point to the start of 'buf', since
+ * hex writing is done in reverse order.
+ * 
  * @param i Any unsigned integer
  * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
  * @return Pointer to s containing hex string with trailing zero byte
  */
-char *hex(uint8_t i,char s[3]) noexcept;
-
-/**
- * Convert an unsigned integer into hex
- *
- * @param i Any unsigned integer
- * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
- * @return Pointer to s containing hex string with trailing zero byte
- */
-char *hex(uint16_t i,char s[5]) noexcept;
-
-/**
- * Convert an unsigned integer into hex
- *
- * @param i Any unsigned integer
- * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
- * @return Pointer to s containing hex string with trailing zero byte
- */
-char *hex(uint32_t i,char s[9]) noexcept;
-
-/**
- * Convert an unsigned integer into hex
- *
- * @param i Any unsigned integer
- * @param s Buffer to receive hex, must be at least (2*sizeof(i))+1 in size or overflow will occur.
- * @return Pointer to s containing hex string with trailing zero byte
- */
-char *hex(uint64_t i,char s[17]) noexcept;
+char *hex(uint64_t i,char buf[17]) noexcept;
 
 /**
  * Decode an unsigned integer in hex format
@@ -625,8 +601,8 @@ template<unsigned int L>
 static ZT_INLINE void copy(void *const dest,const void *const src) noexcept
 {
 #ifdef ZT_ARCH_X64
-	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest); // NOLINT(hicpp-use-auto,modernize-use-auto)
-	const uint8_t *s = reinterpret_cast<const uint8_t *>(src); // NOLINT(hicpp-use-auto,modernize-use-auto)
+	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest);
+	const uint8_t *s = reinterpret_cast<const uint8_t *>(src);
 	for(unsigned int i=0;i<(L >> 6U);++i) {
 		__m128i x0 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(s));
 		__m128i x1 = _mm_loadu_si128(reinterpret_cast<const __m128i *>(s + 16));
@@ -698,7 +674,7 @@ template<unsigned int L>
 static ZT_INLINE void zero(void *const dest) noexcept
 {
 #ifdef ZT_ARCH_X64
-	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest); // NOLINT(hicpp-use-auto,modernize-use-auto)
+	uint8_t *volatile d = reinterpret_cast<uint8_t *>(dest);
 	__m128i z = _mm_setzero_si128();
 	for(unsigned int i=0;i<(L >> 6U);++i) {
 		_mm_storeu_si128(reinterpret_cast<__m128i *>(d),z);
@@ -764,16 +740,16 @@ struct Mallocator
 	typedef T value_type;
 
 	template <class U> struct rebind { typedef Mallocator<U> other; };
-	ZT_INLINE Mallocator() noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
-	ZT_INLINE Mallocator(const Mallocator&) noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
-	template <class U> ZT_INLINE Mallocator(const Mallocator<U>&) noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default,google-explicit-constructor,hicpp-explicit-conversions)
-	ZT_INLINE ~Mallocator() noexcept {} // NOLINT(hicpp-use-equals-default,modernize-use-equals-default)
+	ZT_INLINE Mallocator() noexcept {}
+	ZT_INLINE Mallocator(const Mallocator&) noexcept {}
+	template <class U> ZT_INLINE Mallocator(const Mallocator<U>&) noexcept {}
+	ZT_INLINE ~Mallocator() noexcept {}
 
 	ZT_INLINE pointer allocate(size_type s,void const * = nullptr)
 	{
 		if (0 == s)
 			return nullptr;
-		pointer temp = (pointer)malloc(s * sizeof(T)); // NOLINT(hicpp-use-auto,modernize-use-auto)
+		pointer temp = (pointer)malloc(s * sizeof(T));
 		if (temp == nullptr)
 			throw std::bad_alloc();
 		return temp;

node/VL1.cpp

@@ -32,11 +32,45 @@ namespace {
 
 ZT_INLINE const Identity &identityFromPeerPtr(const SharedPtr<Peer> &p)
 {
-	if (p)
-		return p->identity();
-	return Identity::NIL;
+	return (p) ? p->identity() : Identity::NIL;
 }
 
+struct p_SalsaPolyCopyFunction
+{
+	Salsa20 s20;
+	Poly1305 poly1305;
+	ZT_INLINE p_SalsaPolyCopyFunction(const void *salsaKey,const void *salsaIv) :
+		s20(salsaKey,salsaIv),
+		poly1305()
+	{
+		uint8_t macKey[ZT_POLY1305_KEY_SIZE];
+		s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
+		poly1305.init(macKey);
+	}
+	ZT_INLINE void operator()(void *dest,const void *src,const unsigned int len) noexcept
+	{
+		poly1305.update(src,len);
+		s20.crypt12(src,dest,len);
+	}
+};
+
+struct p_PolyCopyFunction
+{
+	Poly1305 poly1305;
+	ZT_INLINE p_PolyCopyFunction(const void *salsaKey,const void *salsaIv) :
+		poly1305()
+	{
+		uint8_t macKey[ZT_POLY1305_KEY_SIZE];
+		Salsa20(salsaKey,salsaIv).crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
+		poly1305.init(macKey);
+	}
+	ZT_INLINE void operator()(void *dest,const void *src,const unsigned int len) noexcept
+	{
+		poly1305.update(src,len);
+		Utils::copy(dest,src,len);
+	}
+};
+
 } // anonymous namespace
 
 VL1::VL1(const RuntimeEnvironment *renv) :
@@ -46,59 +80,75 @@ VL1::VL1(const RuntimeEnvironment *renv) :
 
 void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAddress &fromAddr,SharedPtr<Buf> &data,const unsigned int len)
 {
-	// Get canonical Path object for this originating address and local socket pair.
 	const SharedPtr<Path> path(RR->topology->path(localSocket,fromAddr));
-
 	const int64_t now = RR->node->now();
 
 	// Update path's last receive time (this is updated when anything is received at all, even if invalid or a keepalive)
 	path->received(now,len);
 
 	try {
-		// Handle 8-byte short probes, which are used as a low-bandwidth way to initiate a real handshake.
-		if (len == ZT_PROTO_PROBE_LENGTH) {
-			const SharedPtr<Peer> peer(RR->topology->peerByProbe(data->lI64(0)));
-			if ((peer)&&(peer->rateGateInboundProbe(now)))
-				path->sent(now,peer->nop(tPtr,path->localSocket(),path->address(),now));
+		// Handle short probes, which are used as a low-bandwidth way to initiate a real handshake.
+		// These are subjected to a significant rate limit to prevent DOS or amplification attacks.
+		// The probe itself is a token passed via HELLO, so these are only used with peers we've
+		// already started communicating with.
+		if (unlikely(len == ZT_PROTO_PROBE_LENGTH)) {
+			PeerList peers(RR->topology->peersByProbeToken(data->lI32(0)));
+			for(unsigned int pi=0;pi<peers.size();++pi) {
+				if (peers[pi]->rateGateProbeRequest(now))
+					peers[pi]->hello(tPtr,localSocket,fromAddr,now);
+			}
 			return;
 		}
 
-		// Discard any other runt packets that aren't probes. These are likely to be keepalives.
-		// No reason to bother even logging them. Note that the last receive time for the path
-		// was still updated, so tiny keepalives do keep the path alive.
-		if (len < ZT_PROTO_MIN_FRAGMENT_LENGTH)
+		// Any other "runt" packets are discarded, though they still count toward a path's
+		// last receive time as they may be keepalives.
+		if (unlikely(len < ZT_PROTO_MIN_FRAGMENT_LENGTH))
 			return;
 
-		// A vector of slices of buffers that aspires to eventually hold an assembled packet.
-		// These are reassembled into a single contiguous buffer at the same time as decryption
-		// and authentication.
-		FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS> pktv;
+		/*
+		 * Packet format:
+		 *   <[8] 64-bit packet ID / crypto IV>
+		 *   <[5] destination ZT address>
+		 *   <[5] source ZT address>
+		 *   <[1] outer visible flags, cipher, and hop count (bits: FFCCHHH)>
+		 *   <[8] 64-bit MAC (or trusted path ID in trusted path mode)>
+		 *   [... -- begin encryption envelope -- ...]
+		 *   <[1] inner envelope flags (MS 3 bits) and verb (LS 5 bits)>
+		 *   [... verb-specific payload ...]
+		 */
 
-		// Destination address of packet (filled below)
-		Address destination;
+		static_assert((ZT_PROTO_PACKET_DESTINATION_INDEX + ZT_ADDRESS_LENGTH) < ZT_PROTO_MIN_FRAGMENT_LENGTH,"overflow");
+		Address destination(data->unsafeData + ZT_PROTO_PACKET_DESTINATION_INDEX);
+		if (destination != RR->identity.address()) {
+			m_relay(tPtr,path,destination,data,len);
+			return;
+		}
 
-		if (data->lI8(ZT_PROTO_PACKET_FRAGMENT_INDICATOR_INDEX) == ZT_PROTO_PACKET_FRAGMENT_INDICATOR) {
-			// Fragment -----------------------------------------------------------------------------------------------------
+		// ----------------------------------------------------------------------------------------------------------------
+		// If we made it this far, the packet is at least MIN_FRAGMENT_LENGTH and is addressed to this node's ZT address
+		// ----------------------------------------------------------------------------------------------------------------
 
-			const Protocol::FragmentHeader &fragmentHeader = data->as<Protocol::FragmentHeader>();
-			destination.setTo(fragmentHeader.destination);
+		static_assert((ZT_PROTO_PACKET_ID_INDEX + sizeof(uint64_t)) < ZT_PROTO_MIN_FRAGMENT_LENGTH,"overflow");
+		const uint64_t packetId = Utils::loadAsIsEndian<uint64_t>(data->unsafeData + ZT_PROTO_PACKET_ID_INDEX);
 
-			if (destination != RR->identity.address()) {
-				m_relay(tPtr, path, destination, data, len);
-				return;
-			}
+		Buf::PacketVector pktv;
 
+		static_assert(ZT_PROTO_PACKET_FRAGMENT_INDICATOR_INDEX <= ZT_PROTO_MIN_FRAGMENT_LENGTH,"overflow");
+		if (data->unsafeData[ZT_PROTO_PACKET_FRAGMENT_INDICATOR_INDEX] == ZT_PROTO_PACKET_FRAGMENT_INDICATOR) {
+			// This looks like a fragment (excluding the head) of a larger packet.
+			static_assert(ZT_PROTO_PACKET_FRAGMENT_COUNTS < ZT_PROTO_MIN_FRAGMENT_LENGTH,"overflow");
+			const unsigned int totalFragments = (data->unsafeData[ZT_PROTO_PACKET_FRAGMENT_COUNTS] >> 4U) & 0x0fU;
+			const unsigned int fragmentNo = data->unsafeData[ZT_PROTO_PACKET_FRAGMENT_COUNTS] & 0x0fU;
 			switch (m_inputPacketAssembler.assemble(
-				fragmentHeader.packetId,
+				packetId,
 				pktv,
 				data,
 				ZT_PROTO_PACKET_FRAGMENT_PAYLOAD_START_AT,
-				(unsigned int)(len - ZT_PROTO_PACKET_FRAGMENT_PAYLOAD_START_AT),
-				fragmentHeader.counts & 0xfU, // fragment number
-				fragmentHeader.counts >> 4U,  // total number of fragments in message is specified in each fragment
+				len - ZT_PROTO_PACKET_FRAGMENT_PAYLOAD_START_AT,
+				fragmentNo,
+				totalFragments,
 				now,
-				path,
-				ZT_MAX_INCOMING_FRAGMENTS_PER_PATH)) {
+				path)) {
 				case Defragmenter<ZT_MAX_PACKET_FRAGMENTS>::COMPLETE:
 					break;
 				default:
@@ -110,30 +160,21 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					return;
 			}
 		} else {
-			// Not fragment, meaning whole packet or head of series with fragments ------------------------------------------
-
-			if (len < ZT_PROTO_MIN_PACKET_LENGTH)
-				return;
-			const Protocol::Header &packetHeader = data->as<Protocol::Header>();
-			destination.setTo(packetHeader.destination);
-
-			if (destination != RR->identity.address()) {
-				m_relay(tPtr, path, destination, data, len);
+			if (unlikely(len < ZT_PROTO_MIN_PACKET_LENGTH))
 				return;
-			}
-
-			if ((packetHeader.flags & ZT_PROTO_FLAG_FRAGMENTED) != 0) {
+			static_assert(ZT_PROTO_PACKET_FLAGS_INDEX < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
+			if ((data->unsafeData[ZT_PROTO_PACKET_FLAGS_INDEX] & ZT_PROTO_FLAG_FRAGMENTED) != 0) {
+				// This is the head of a series of fragments that we may or may not already have.
 				switch (m_inputPacketAssembler.assemble(
-					packetHeader.packetId,
+					packetId,
 					pktv,
 					data,
-					0,
+					0, // fragment index is 0 since this is the head
 					len,
 					0, // always the zero'eth fragment
 					0, // this is specified in fragments, not in the head
 					now,
-					path,
-					ZT_MAX_INCOMING_FRAGMENTS_PER_PATH)) {
+					path)) {
 					case Defragmenter<ZT_MAX_PACKET_FRAGMENTS>::COMPLETE:
 						break;
 					default:
@@ -144,7 +185,8 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 						//case Defragmenter<ZT_MAX_PACKET_FRAGMENTS>::ERR_OUT_OF_MEMORY:
 						return;
 				}
-			} else { // packet isn't fragmented, so skip the Defragmenter logic completely.
+			} else {
+				// This is a single whole packet with no fragments.
 				Buf::Slice &s = pktv.push();
 				s.b.swap(data);
 				s.s = 0;
@@ -152,284 +194,221 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 			}
 		}
 
-		// Packet defragmented and apparently addressed to this node ------------------------------------------------------
+		// ----------------------------------------------------------------------------------------------------------------
+		// If we made it this far without returning, a packet is fully assembled and ready to process.
+		// ----------------------------------------------------------------------------------------------------------------
 
-		// Subject pktv to a few sanity checks just to make sure Defragmenter worked correctly and
-		// there is enough room in each slice to shift their contents to sizes that are multiples
-		// of 64 if needed for crypto.
-		if ((pktv.empty()) || (((int)pktv[0].e - (int)pktv[0].s) < (int)sizeof(Protocol::Header))) {
-			RR->t->unexpectedError(tPtr,0x3df19990,"empty or undersized packet vector after parsing packet from %s of length %d",Trace::str(path->address()).s,(int)len);
-			return;
-		}
-		for(FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS>::const_iterator s(pktv.begin());s!=pktv.end();++s) {
-			if ((s->e > (ZT_BUF_MEM_SIZE - 64))||(s->s > s->e))
-				return;
-		}
+		const uint8_t *const hdr = pktv[0].b->unsafeData + pktv[0].s;
 
-		Protocol::Header *ph = &(pktv[0].b->as<Protocol::Header>(pktv[0].s));
-		const Address source(ph->source);
+		static_assert((ZT_PROTO_PACKET_SOURCE_INDEX + ZT_ADDRESS_LENGTH) < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
+		const Address source(hdr + ZT_PROTO_PACKET_SOURCE_INDEX);
+		static_assert(ZT_PROTO_PACKET_FLAGS_INDEX < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
+		const uint8_t hops = hdr[ZT_PROTO_PACKET_FLAGS_INDEX] & ZT_PROTO_FLAG_FIELD_HOPS_MASK;
+		const uint8_t cipher = (hdr[ZT_PROTO_PACKET_FLAGS_INDEX] >> 3U) & 3U;
 
-		if (source == RR->identity.address())
-			return;
-		SharedPtr<Peer> peer(RR->topology->peer(tPtr,source));
-
-		Buf::Slice pkt;
+		const SharedPtr<Buf> pkt(new Buf());
+		int pktSize = 0;
 		bool authenticated = false;
 
-		const uint8_t hops = Protocol::packetHops(*ph);
-		const uint8_t cipher = Protocol::packetCipher(*ph);
-
-		unsigned int packetSize = pktv[0].e - pktv[0].s;
-		for(FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS>::const_iterator s(pktv.begin()+1);s!=pktv.end();++s)
-			packetSize += s->e - s->s;
-		if (packetSize > ZT_PROTO_MAX_PACKET_LENGTH) {
-			RR->t->incomingPacketDropped(tPtr,0x010348da,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
-			return;
-		}
-
-		// If we don't know this peer and this is not a HELLO, issue a WHOIS and enqueue this packet to try again.
-		if ((!peer)&&(!(((cipher == ZT_PROTO_CIPHER_SUITE__POLY1305_NONE)||(cipher == ZT_PROTO_CIPHER_SUITE__NONE))&&((ph->verb & 0x1fU) == Protocol::VERB_HELLO)))) {
-			pkt = Buf::assembleSliceVector(pktv);
-			if (pkt.e < ZT_PROTO_MIN_PACKET_LENGTH) {
-				RR->t->incomingPacketDropped(tPtr,0xbada9366,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+		static_assert(ZT_PROTO_PACKET_VERB_INDEX < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
+		if ( ((cipher == ZT_PROTO_CIPHER_SUITE__POLY1305_NONE)||(cipher == ZT_PROTO_CIPHER_SUITE__NONE)) && ((hdr[ZT_PROTO_PACKET_VERB_INDEX] & ZT_PROTO_VERB_MASK) == Protocol::VERB_HELLO) ) {
+			// Handle unencrypted HELLO packets.
+			pktSize = pktv.mergeCopy(*pkt);
+			if (unlikely(pktSize < ZT_PROTO_MIN_PACKET_LENGTH)) {
+				ZT_SPEW("discarding packet %.16llx from %s: assembled packet size: %d",packetId,fromAddr.toString().c_str(),pktSize);
 				return;
 			}
-			{
-				Mutex::Lock wl(m_whoisQueue_l);
-				p_WhoisQueueItem &wq = m_whoisQueue[source];
-				wq.inboundPackets.push_back(pkt);
-			}
-			m_sendPendingWhois(tPtr, now);
+			const SharedPtr<Peer> peer(m_HELLO(tPtr, path, *pkt, pktSize));
+			if (peer)
+				peer->received(tPtr,path,hops,packetId,pktSize - ZT_PROTO_PACKET_PAYLOAD_START,Protocol::VERB_HELLO,Protocol::VERB_NOP);
 			return;
 		}
 
-		switch(cipher) {
-			case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE:
-				if (peer) {
-					pkt = Buf::assembleSliceVector(pktv);
-					if (pkt.e < ZT_PROTO_MIN_PACKET_LENGTH) {
-						RR->t->incomingPacketDropped(tPtr,0x432aa9da,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+		// ----------------------------------------------------------------------------------------------------------------
+		// Making it this far means the packet is not a plaintext HELLO, so do normal AEAD decrypt and packet handling.
+		// ----------------------------------------------------------------------------------------------------------------
+
+		SharedPtr<Peer> peer(RR->topology->peer(tPtr,source));
+		if (peer) {
+			switch(cipher) {
+
+				case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE: {
+					uint8_t perPacketKey[ZT_SALSA20_KEY_SIZE];
+					Protocol::salsa2012DeriveKey(peer->rawIdentityKey(),perPacketKey,*pktv[0].b,pktv.totalSize());
+					p_PolyCopyFunction s20cf(perPacketKey,&packetId);
+
+					pktSize = pktv.mergeMap<p_PolyCopyFunction &>(*pkt,ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,s20cf);
+					if (unlikely(pktSize < ZT_PROTO_MIN_PACKET_LENGTH)) {
+						ZT_SPEW("discarding packet %.16llx from %s: assembled packet size: %d",packetId,fromAddr.toString().c_str(),pktSize);
 						return;
 					}
-					ph = &(pkt.b->as<Protocol::Header>());
-
-					// Generate one-time-use MAC key using Salsa20.
-					uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
-					uint8_t macKey[ZT_POLY1305_KEY_SIZE];
-					Protocol::salsa2012DeriveKey(peer->key(),perPacketKey,*pktv[0].b,packetSize);
-					Salsa20(perPacketKey,&ph->packetId).crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
 
-					// Verify packet MAC.
 					uint64_t mac[2];
-					poly1305(mac,pkt.b->unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
-					if (ph->mac != mac[0]) {
-						RR->t->incomingPacketDropped(tPtr,0xcc89c812,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+					s20cf.poly1305.finish(mac);
+					static_assert((ZT_PROTO_PACKET_MAC_INDEX + 8) < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
+					if (unlikely(Utils::loadAsIsEndian<uint64_t>(hdr + ZT_PROTO_PACKET_MAC_INDEX) != mac[0])) {
+						RR->t->incomingPacketDropped(tPtr,0xcc89c812,packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 						return;
 					}
+
 					authenticated = true;
-				}
-				break;
+				}	break;
 
-			case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012:
-				if (peer) {
-					// Derive per-packet key using symmetric key plus some data from the packet header.
-					uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
-					Protocol::salsa2012DeriveKey(peer->key(),perPacketKey,*pktv[0].b,packetSize);
-					Salsa20 s20(perPacketKey,&ph->packetId);
-
-					// Do one Salsa20 block to generate the one-time-use Poly1305 key.
-					uint8_t macKey[ZT_POLY1305_KEY_SIZE];
-					s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
-
-					// Get a buffer to store the decrypted and fully contiguous packet.
-					pkt.b.set(new Buf());
-
-					// Salsa20 is a stream cipher but it's only seekable to multiples of 64 bytes.
-					// This moves data in slices around so that all slices have sizes that are
-					// multiples of 64 except the last slice. Note that this does not corrupt
-					// the assembled slice vector, just moves data around.
-					if (pktv.size() > 1) {
-						unsigned int prevOverflow,i;
-						for (typename FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS>::iterator ps(pktv.begin()),s(ps + 1);s!=pktv.end();) {
-							prevOverflow = (ps->e - ps->s) & 63U; // amount by which previous exceeds a multiple of 64
-							for(i=0;i<prevOverflow;++i) {
-								if (s->s >= s->e)
-									goto next_slice;
-								ps->b->unsafeData[ps->e++] = s->b->unsafeData[s->s++]; // move from head of current to end of previous
-							}
-							next_slice: ps = s++;
-						}
-					}
+				case ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012: {
+					uint8_t perPacketKey[ZT_SALSA20_KEY_SIZE];
+					Protocol::salsa2012DeriveKey(peer->rawIdentityKey(),perPacketKey,*pktv[0].b,pktv.totalSize());
+					p_SalsaPolyCopyFunction s20cf(perPacketKey,&packetId);
 
-					// Simultaneously decrypt and assemble packet into a contiguous buffer.
-					// Since we moved data around above all slices will have sizes that are
-					// multiples of 64.
-					Utils::copy<sizeof(Protocol::Header)>(pkt.b->unsafeData,ph);
-					pkt.e = sizeof(Protocol::Header);
-					for(FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS>::iterator s(pktv.begin());s!=pktv.end();++s) {
-						const unsigned int sliceSize = s->e - s->s;
-						s20.crypt12(s->b->unsafeData + s->s,pkt.b->unsafeData + pkt.e,sliceSize);
-						pkt.e += sliceSize;
+					pktSize = pktv.mergeMap<p_SalsaPolyCopyFunction &>(*pkt,ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,s20cf);
+					if (unlikely(pktSize < ZT_PROTO_MIN_PACKET_LENGTH)) {
+						ZT_SPEW("discarding packet %.16llx from %s: assembled packet size: %d",packetId,fromAddr.toString().c_str(),pktSize);
+						return;
 					}
-					ph = &(pkt.b->as<Protocol::Header>());
 
-					// Verify packet MAC.
 					uint64_t mac[2];
-					poly1305(mac,pkt.b->unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
-					if (ph->mac != mac[0]) {
-						RR->t->incomingPacketDropped(tPtr,0xbc881231,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+					s20cf.poly1305.finish(mac);
+					static_assert((ZT_PROTO_PACKET_MAC_INDEX + 8) < ZT_PROTO_MIN_PACKET_LENGTH,"overflow");
+					if (unlikely(Utils::loadAsIsEndian<uint64_t>(hdr + ZT_PROTO_PACKET_MAC_INDEX) != mac[0])) {
+						RR->t->incomingPacketDropped(tPtr,0xcc89c812,packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 						return;
 					}
+
 					authenticated = true;
-				} else {
-					RR->t->incomingPacketDropped(tPtr,0xb0b01999,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-					return;
-				}
-				break;
+				}	break;
 
-			case ZT_PROTO_CIPHER_SUITE__NONE: {
-				// CIPHER_SUITE__NONE is only used with trusted paths. Verification is performed by
-				// checking the address and the presence of its corresponding trusted path ID in the
-				// packet header's MAC field.
+				case ZT_PROTO_CIPHER_SUITE__NONE: {
+					// TODO
+				} break;
 
-				pkt = Buf::assembleSliceVector(pktv);
-				if (pkt.e < ZT_PROTO_MIN_PACKET_LENGTH)
-					RR->t->incomingPacketDropped(tPtr,0x3d3337df,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
-				ph = &(pkt.b->as<Protocol::Header>());
+				case ZT_PROTO_CIPHER_SUITE__AES_GMAC_SIV: {
+					// TODO
+				}	break;
 
-				if (RR->topology->shouldInboundPathBeTrusted(path->address(),Utils::ntoh(ph->mac))) {
-					authenticated = true;
-				} else {
-					RR->t->incomingPacketDropped(tPtr,0x2dfa910b,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_NOT_TRUSTED_PATH);
+				default:
+					RR->t->incomingPacketDropped(tPtr,0x5b001099,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
 					return;
-				}
-			} break;
-
-			//case ZT_PROTO_CIPHER_SUITE__AES_GCM_NRH:
-			//	if (peer) {
-			//	}
-			//	break;
-
-			default:
-				RR->t->incomingPacketDropped(tPtr,0x5b001099,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
-				return;
+			}
 		}
 
-		// Packet fully assembled, authenticated 'true' if already authenticated via MAC ----------------------------------
-
-		// Return any still held buffers in pktv to the buffer pool.
-		pktv.clear();
+		if (likely(authenticated)) {
+			// If authentication was successful go on and process the packet.
+#if 0
+			const Protocol::Verb verb = (Protocol::Verb)(ph->verb & ZT_PROTO_VERB_MASK);
 
-		const Protocol::Verb verb = (Protocol::Verb)(ph->verb & ZT_PROTO_VERB_MASK);
-
-		// All verbs except HELLO require authentication before being handled. The HELLO
-		// handler does its own authentication.
-		if (((!authenticated)||(!peer))&&(verb != Protocol::VERB_HELLO)) {
-			RR->t->incomingPacketDropped(tPtr,0x5b001099,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-			return;
-		}
-
-		// Decompress packet payload if compressed. For additional safety decompression is
-		// only performed on packets whose MACs have already been validated. (Only HELLO is
-		// sent without this, and HELLO doesn't benefit from compression.)
-		if ((ph->verb & ZT_PROTO_VERB_FLAG_COMPRESSED) != 0) {
-			if (!authenticated) {
-				RR->t->incomingPacketDropped(tPtr,0x390bcd0a,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+			// All verbs except HELLO require authentication before being handled. The HELLO
+			// handler does its own authentication.
+			if (((!authenticated)||(!peer))&&(verb != Protocol::VERB_HELLO)) {
+				RR->t->incomingPacketDropped(tPtr,0x5b001099,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 				return;
 			}
 
-			SharedPtr<Buf> nb(new Buf());
-			const int uncompressedLen = LZ4_decompress_safe(
-				reinterpret_cast<const char *>(pkt.b->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),
-				reinterpret_cast<char *>(nb->unsafeData),
-				(int)(packetSize - ZT_PROTO_PACKET_PAYLOAD_START),
-				ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START);
+			// Decompress packet payload if compressed. For additional safety decompression is
+			// only performed on packets whose MACs have already been validated. (Only HELLO is
+			// sent without this, and HELLO doesn't benefit from compression.)
+			if ((ph->verb & ZT_PROTO_VERB_FLAG_COMPRESSED) != 0) {
+				if (!authenticated) {
+					RR->t->incomingPacketDropped(tPtr,0x390bcd0a,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+					return;
+				}
 
-			if ((uncompressedLen > 0)&&(uncompressedLen <= (ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START))) {
-				pkt.b.swap(nb);
-				pkt.e = packetSize = (unsigned int)uncompressedLen;
-			} else {
-				RR->t->incomingPacketDropped(tPtr,0xee9e4392,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_INVALID_COMPRESSED_DATA);
-				return;
-			}
-		}
+				SharedPtr<Buf> nb(new Buf());
+				const int uncompressedLen = LZ4_decompress_safe(
+					reinterpret_cast<const char *>(pkt.b->unsafeData + ZT_PROTO_PACKET_PAYLOAD_START),
+					reinterpret_cast<char *>(nb->unsafeData),
+					(int)(packetSize - ZT_PROTO_PACKET_PAYLOAD_START),
+					ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START);
 
-		/*
-		 * Important notes:
-		 *
-		 * All verbs except HELLO assume that authenticated is true and peer is non-NULL.
-		 * This is checked above. HELLO will accept either case and always performs its
-		 * own secondary validation. The path argument is never NULL.
-		 *
-		 * VL1 and VL2 are conceptually separate layers of the ZeroTier protocol. In the
-		 * code they are almost entirely logically separate. To make the code easier to
-		 * understand the handlers for VL2 data paths have been moved to a VL2 class.
-		 */
+				if ((uncompressedLen > 0)&&(uncompressedLen <= (ZT_BUF_MEM_SIZE - ZT_PROTO_PACKET_PAYLOAD_START))) {
+					pkt.b.swap(nb);
+					pkt.e = packetSize = (unsigned int)uncompressedLen;
+				} else {
+					RR->t->incomingPacketDropped(tPtr,0xee9e4392,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_INVALID_COMPRESSED_DATA);
+					return;
+				}
+			}
 
-		bool ok = true; // set to false if a packet turns out to be invalid
-		Protocol::Verb inReVerb = Protocol::VERB_NOP; // set via result parameter to _ERROR and _OK
-		switch(verb) {
-			case Protocol::VERB_NOP:                        break;
-			case Protocol::VERB_HELLO:                      ok = m_HELLO(tPtr, path, peer, *pkt.b, (int) packetSize, authenticated); break;
-			case Protocol::VERB_ERROR:                      ok = m_ERROR(tPtr, path, peer, *pkt.b, (int) packetSize, inReVerb); break;
-			case Protocol::VERB_OK:                         ok = m_OK(tPtr, path, peer, *pkt.b, (int) packetSize, inReVerb); break;
-			case Protocol::VERB_WHOIS:                      ok = m_WHOIS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_RENDEZVOUS:                 ok = m_RENDEZVOUS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_FRAME:                      ok = RR->vl2->m_FRAME(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_EXT_FRAME:                  ok = RR->vl2->m_EXT_FRAME(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_ECHO:                       ok = m_ECHO(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_MULTICAST_LIKE:             ok = RR->vl2->m_MULTICAST_LIKE(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_NETWORK_CREDENTIALS:        ok = RR->vl2->m_NETWORK_CREDENTIALS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_NETWORK_CONFIG_REQUEST:     ok = RR->vl2->m_NETWORK_CONFIG_REQUEST(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_NETWORK_CONFIG:             ok = RR->vl2->m_NETWORK_CONFIG(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_MULTICAST_GATHER:           ok = RR->vl2->m_MULTICAST_GATHER(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_MULTICAST_FRAME_deprecated: ok = RR->vl2->m_MULTICAST_FRAME_deprecated(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_PUSH_DIRECT_PATHS:          ok = m_PUSH_DIRECT_PATHS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_USER_MESSAGE:               ok = m_USER_MESSAGE(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_MULTICAST:                  ok = RR->vl2->m_MULTICAST(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			case Protocol::VERB_ENCAP:                      ok = m_ENCAP(tPtr, path, peer, *pkt.b, (int) packetSize); break;
-			default:
-				RR->t->incomingPacketDropped(tPtr,0xeeeeeff0,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_UNRECOGNIZED_VERB);
-				break;
+			/*
+			* Important notes:
+			*
+			* All verbs except HELLO assume that authenticated is true and peer is non-NULL.
+			* This is checked above. HELLO will accept either case and always performs its
+			* own secondary validation. The path argument is never NULL.
+			*
+			* VL1 and VL2 are conceptually separate layers of the ZeroTier protocol. In the
+			* code they are almost entirely logically separate. To make the code easier to
+			* understand the handlers for VL2 data paths have been moved to a VL2 class.
+			*/
+
+			bool ok = true; // set to false if a packet turns out to be invalid
+			Protocol::Verb inReVerb = Protocol::VERB_NOP; // set via result parameter to _ERROR and _OK
+			switch(verb) {
+				case Protocol::VERB_NOP:                        break;
+				case Protocol::VERB_HELLO:                      ok = (bool)(m_HELLO(tPtr, path, *pkt.b, (int) packetSize)); break;
+				case Protocol::VERB_ERROR:                      ok = m_ERROR(tPtr, path, peer, *pkt.b, (int) packetSize, inReVerb); break;
+				case Protocol::VERB_OK:                         ok = m_OK(tPtr, path, peer, *pkt.b, (int) packetSize, inReVerb); break;
+				case Protocol::VERB_WHOIS:                      ok = m_WHOIS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_RENDEZVOUS:                 ok = m_RENDEZVOUS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_FRAME:                      ok = RR->vl2->m_FRAME(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_EXT_FRAME:                  ok = RR->vl2->m_EXT_FRAME(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_ECHO:                       ok = m_ECHO(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST_LIKE:             ok = RR->vl2->m_MULTICAST_LIKE(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_NETWORK_CREDENTIALS:        ok = RR->vl2->m_NETWORK_CREDENTIALS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_NETWORK_CONFIG_REQUEST:     ok = RR->vl2->m_NETWORK_CONFIG_REQUEST(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_NETWORK_CONFIG:             ok = RR->vl2->m_NETWORK_CONFIG(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST_GATHER:           ok = RR->vl2->m_MULTICAST_GATHER(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST_FRAME_deprecated: ok = RR->vl2->m_MULTICAST_FRAME_deprecated(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_PUSH_DIRECT_PATHS:          ok = m_PUSH_DIRECT_PATHS(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_USER_MESSAGE:               ok = m_USER_MESSAGE(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_MULTICAST:                  ok = RR->vl2->m_MULTICAST(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				case Protocol::VERB_ENCAP:                      ok = m_ENCAP(tPtr, path, peer, *pkt.b, (int) packetSize); break;
+				default:
+					RR->t->incomingPacketDropped(tPtr,0xeeeeeff0,ph->packetId,0,identityFromPeerPtr(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_UNRECOGNIZED_VERB);
+					break;
+			}
+			if (ok)
+				peer->received(tPtr,path,hops,ph->packetId,packetSize - ZT_PROTO_PACKET_PAYLOAD_START,verb,inReVerb);
+#endif
+		} else {
+			// If decryption and authentication were not successful, try to look up identities.
+			// This is rate limited by virtue of the retry rate limit timer.
+			if (pktSize <= 0)
+				pktSize = pktv.mergeCopy(*pkt);
+			if (pktSize >= ZT_PROTO_MIN_PACKET_LENGTH) {
+				bool sendPending;
+				{
+					Mutex::Lock wl(m_whoisQueue_l);
+					p_WhoisQueueItem &wq = m_whoisQueue[source];
+					const unsigned int wpidx = wq.waitingPacketCount++ % ZT_VL1_MAX_WHOIS_WAITING_PACKETS;
+					wq.waitingPacketSize[wpidx] = (unsigned int)pktSize;
+					wq.waitingPacket[wpidx] = pkt;
+					sendPending = (now - wq.lastRetry) >= ZT_WHOIS_RETRY_DELAY;
+				}
+				if (sendPending)
+					m_sendPendingWhois(tPtr,now);
+			}
 		}
-		if (ok)
-			peer->received(tPtr,path,hops,ph->packetId,packetSize - ZT_PROTO_PACKET_PAYLOAD_START,verb,inReVerb);
 	} catch ( ... ) {
-		RR->t->unexpectedError(tPtr,0xea1b6dea,"unexpected exception in onRemotePacket() parsing packet from %s",Trace::str(path->address()).s);
+		RR->t->unexpectedError(tPtr,0xea1b6dea,"unexpected exception in onRemotePacket() parsing packet from %s",path->address().toString().c_str());
 	}
 }
 
 void VL1::m_relay(void *tPtr, const SharedPtr<Path> &path, const Address &destination, SharedPtr<Buf> &data, unsigned int len)
 {
-	const uint8_t newHopCount = (data->lI8(ZT_PROTO_PACKET_FLAGS_INDEX) & 7U) + 1;
-	if (newHopCount >= ZT_RELAY_MAX_HOPS)
-		return;
-	data->sI8(ZT_PROTO_PACKET_FLAGS_INDEX,(data->lI8(ZT_PROTO_PACKET_FLAGS_INDEX) & 0xf8U) | newHopCount);
-
-	const SharedPtr<Peer> toPeer(RR->topology->peer(tPtr,destination,false));
-	if (!toPeer)
-		return;
-	const uint64_t now = RR->node->now();
-	const SharedPtr<Path> toPath(toPeer->path(now));
-	if (!toPath)
-		return;
-
-	toPath->send(RR,tPtr,data->unsafeData,len,now);
 }
 
 void VL1::m_sendPendingWhois(void *tPtr, int64_t now)
 {
-	SharedPtr<Peer> root(RR->topology->root());
-	if (!root)
+	const SharedPtr<Peer> root(RR->topology->root());
+	if (unlikely(!root))
 		return;
-	SharedPtr<Path> rootPath(root->path(now));
-	if (!rootPath)
+	const SharedPtr<Path> rootPath(root->path(now));
+	if (unlikely(!rootPath))
 		return;
 
 	std::vector<Address> toSend;
 	{
 		Mutex::Lock wl(m_whoisQueue_l);
-		for(Map<Address,p_WhoisQueueItem>::iterator wi(m_whoisQueue.begin());wi != m_whoisQueue.end();++wi) {
+		for(Map<Address,p_WhoisQueueItem>::iterator wi(m_whoisQueue.begin());wi!=m_whoisQueue.end();++wi) {
 			if ((now - wi->second.lastRetry) >= ZT_WHOIS_RETRY_DELAY) {
 				wi->second.lastRetry = now;
 				++wi->second.retries;
@@ -438,225 +417,164 @@ void VL1::m_sendPendingWhois(void *tPtr, int64_t now)
 		}
 	}
 
-	Buf outp;
-	Protocol::Header &ph = outp.as<Protocol::Header>();
+	if (toSend.empty())
+		return;
 
+	const SharedPtr<SymmetricKey> key(root->key());
+	uint8_t outp[ZT_DEFAULT_UDP_MTU - ZT_PROTO_MIN_PACKET_LENGTH];
 	std::vector<Address>::iterator a(toSend.begin());
 	while (a != toSend.end()) {
-		ph.packetId = Protocol::getPacketId();
-		root->address().copyTo(ph.destination);
-		RR->identity.address().copyTo(ph.source);
-		ph.flags = 0;
-		ph.verb = Protocol::VERB_OK;
-
-		int outl = sizeof(Protocol::Header);
-		while ((a != toSend.end())&&(outl < ZT_PROTO_MAX_PACKET_LENGTH)) {
-			a->copyTo(outp.unsafeData + outl);
+		const uint64_t packetId = key->nextMessage(RR->identity.address(),root->address());
+		int p = Protocol::newPacket(outp,packetId,root->address(),RR->identity.address(),Protocol::VERB_WHOIS);
+		while ((a != toSend.end())&&(p < (sizeof(outp) - ZT_ADDRESS_LENGTH))) {
+			a->copyTo(outp + p);
 			++a;
-			outl += ZT_ADDRESS_LENGTH;
-		}
-
-		if (outl > (int)sizeof(Protocol::Header)) {
-			Protocol::armor(outp,outl,root->key(),root->cipher());
-			RR->expect->sending(ph.packetId,now);
-			rootPath->send(RR,tPtr,outp.unsafeData,outl,now);
+			p += ZT_ADDRESS_LENGTH;
 		}
+		Protocol::armor(outp,p,key,root->cipher());
+		RR->expect->sending(packetId,now);
+		root->send(tPtr,now,outp,p,rootPath);
 	}
 }
 
-bool VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize, bool authenticated)
+SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt, int packetSize)
 {
-	if (packetSize < (int)sizeof(Protocol::HELLO)) {
-		RR->t->incomingPacketDropped(tPtr,0x2bdb0001,0,0,identityFromPeerPtr(peer),path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
-		return false;
-	}
-	Protocol::HELLO &p = pkt.as<Protocol::HELLO>();
-	const uint8_t hops = Protocol::packetHops(p.h);
-	p.h.flags &= (uint8_t)~ZT_PROTO_FLAG_FIELD_HOPS_MASK; // mask off hops for MAC calculation
-	int ptr = sizeof(Protocol::HELLO);
+	// SECURITY: we know if we made it this far the packet's header is valid and
+	// packetSize is at least the size of a header.
 
-	if (p.versionProtocol < ZT_PROTO_VERSION_MIN) {
-		RR->t->incomingPacketDropped(tPtr,0xe8d12bad,p.h.packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_PEER_TOO_OLD);
-		return false;
-	}
+	const uint64_t packetId = Utils::loadAsIsEndian<uint64_t>(pkt.unsafeData + ZT_PROTO_PACKET_ID_INDEX);
+	const uint64_t mac = Utils::loadAsIsEndian<uint64_t>(pkt.unsafeData + ZT_PROTO_PACKET_MAC_INDEX);
+
+	// Get hops field and then mask hops to zero for MAC checking.
+	const uint8_t hops = pkt.unsafeData[ZT_PROTO_PACKET_FLAGS_INDEX] & ZT_PROTO_FLAG_FIELD_HOPS_MASK;
+	pkt.unsafeData[ZT_PROTO_PACKET_FLAGS_INDEX] &= ~ZT_PROTO_FLAG_FIELD_HOPS_MASK;
+
+	const uint8_t protoVersion = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START>;
+	unsigned int versionMajor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 1>(); // LEGACY
+	unsigned int versionMinor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 2>(); // LEGACY
+	unsigned int versionRev = pkt.lI16<ZT_PROTO_PACKET_PAYLOAD_START + 3>(); // LEGACY
+	const uint64_t timestamp = pkt.lI64<ZT_PROTO_PACKET_PAYLOAD_START + 5>();
+
+	int p = ZT_PROTO_PACKET_PAYLOAD_START + 13;
 
 	Identity id;
-	if (pkt.rO(ptr,id) < 0) {
-		RR->t->incomingPacketDropped(tPtr,0x707a9810,p.h.packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
-		return false;
+	if (unlikely(pkt.rO(p,id) < 0)) {
+		RR->t->incomingPacketDropped(tPtr,0x707a9810,packetId,0,Identity::NIL,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+		return SharedPtr<Peer>();
 	}
-	if (Address(p.h.source) != id.address()) {
-		RR->t->incomingPacketDropped(tPtr,0x06aa9ff1,p.h.packetId,0,Identity::NIL,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-		return false;
+	if (unlikely(id.address() != Address(pkt.unsafeData + ZT_PROTO_PACKET_SOURCE_INDEX))) {
+		RR->t->incomingPacketDropped(tPtr,0x707a9010,packetId,0,Identity::NIL,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+		return SharedPtr<Peer>();
 	}
 
-	// Packet is basically valid and identity unmarshaled successfully --------------------------------------------------
-
-	uint8_t key[ZT_SYMMETRIC_KEY_SIZE];
-	if ((peer) && (id == peer->identity())) {
-		Utils::copy<ZT_SYMMETRIC_KEY_SIZE>(key,peer->key());
+	SharedPtr<Peer> peer(RR->topology->peer(tPtr,id.address(),true));
+	if (peer) {
+		if (peer->identity() != id) {
+			RR->t->incomingPacketDropped(tPtr,0x707a9891,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			return SharedPtr<Peer>();
+		}
 	} else {
-		peer.zero();
-		if (!RR->identity.agree(id,key)) {
-			RR->t->incomingPacketDropped(tPtr,0x46db8010,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-			return false;
+		if (unlikely(!id.locallyValidate())) {
+			RR->t->incomingPacketDropped(tPtr,0x707a9892,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+			return SharedPtr<Peer>();
 		}
-	}
-
-	if ((!peer)||(!authenticated)) {
-		uint8_t perPacketKey[ZT_SYMMETRIC_KEY_SIZE];
-		uint8_t macKey[ZT_POLY1305_KEY_SIZE];
-		Protocol::salsa2012DeriveKey(peer->key(),perPacketKey,pkt,packetSize);
-		Salsa20(perPacketKey,&p.h.packetId).crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
-		uint64_t mac[2];
-		poly1305(mac,pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
-		if (p.h.mac != mac[0]) {
-			RR->t->incomingPacketDropped(tPtr,0x11bfff81,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-			return false;
+		peer.set(new Peer(RR));
+		if (unlikely(!peer->init(id))) {
+			RR->t->incomingPacketDropped(tPtr,0x707a9893,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_UNSPECIFIED);
+			return SharedPtr<Peer>();
 		}
+		peer = RR->topology->add(tPtr,peer);
 	}
 
-	// Packet has passed Poly1305 MAC authentication --------------------------------------------------------------------
-
-	uint8_t hmacKey[ZT_SYMMETRIC_KEY_SIZE],hmac[ZT_HMACSHA384_LEN];
-	if (peer->remoteVersionProtocol() >= 11) {
-		if (packetSize <= ZT_HMACSHA384_LEN) { // sanity check, should be impossible
-			RR->t->incomingPacketDropped(tPtr,0x1000662a,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-			return false;
+	// ------------------------------------------------------------------------------------------------------------------
+	// If we made it this far, peer is non-NULL and the identity is valid and matches it.
+	// ------------------------------------------------------------------------------------------------------------------
+
+	if (protoVersion >= 11) {
+		// V2.x and newer use HMAC-SHA384 for HELLO, which offers a larger security margin
+		// to guard key exchange and connection setup than typical AEAD.
+		uint8_t hmac[ZT_HMACSHA384_LEN];
+		HMACSHA384(peer->identityHelloHmacKey(),pkt.unsafeData,packetSize,hmac);
+		if (unlikely((packetSize < ZT_HMACSHA384_LEN)||(!Utils::secureEq(hmac,(pkt.unsafeData + packetSize) - ZT_HMACSHA384_LEN,ZT_HMACSHA384_LEN)))) {
+			RR->t->incomingPacketDropped(tPtr,0x707a9891,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			return SharedPtr<Peer>();
 		}
-		packetSize -= ZT_HMACSHA384_LEN;
-		KBKDFHMACSHA384(key,ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC,0,0,hmacKey); // iter == 0 for HELLO, 1 for OK(HELLO)
-		HMACSHA384(hmacKey,pkt.unsafeData,packetSize,hmac);
-		if (!Utils::secureEq(pkt.unsafeData + packetSize,hmac,ZT_HMACSHA384_LEN)) {
-			RR->t->incomingPacketDropped(tPtr,0x1000662a,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
-			return false;
+		packetSize -= ZT_HMACSHA384_LEN; // trim this off the end since we're done with it
+	} else {
+		// Older versions use Poly1305 MAC (but no whole packet encryption) for HELLO.
+		if (likely(packetSize > ZT_PROTO_PACKET_ENCRYPTED_SECTION_START)) {
+			uint8_t perPacketKey[ZT_SALSA20_KEY_SIZE];
+			Protocol::salsa2012DeriveKey(peer->rawIdentityKey(),perPacketKey,pkt,packetSize);
+			uint8_t macKey[ZT_POLY1305_KEY_SIZE];
+			Salsa20(perPacketKey,&packetId).crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
+			Poly1305 poly1305(macKey);
+			poly1305.update(pkt.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START);
+			uint64_t polyMac[2];
+			poly1305.finish(polyMac);
+			if (unlikely(mac != polyMac[0])) {
+				RR->t->incomingPacketDropped(tPtr,0x11bfff82,packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+				return SharedPtr<Peer>();
+			}
+		} else {
+			RR->t->incomingPacketDropped(tPtr,0x11bfff81,packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			return SharedPtr<Peer>();
 		}
 	}
 
-	// Packet has passed HMAC-SHA384 (if present) -----------------------------------------------------------------------
-
-	InetAddress externalSurfaceAddress;
-	Dictionary nodeMetaData;
+	// ------------------------------------------------------------------------------------------------------------------
+	// This far means we passed MAC (Poly1305 or HMAC-SHA384 for newer peers)
+	// ------------------------------------------------------------------------------------------------------------------
 
-	// Get external surface address if present.
-	if (ptr < packetSize) {
-		if (pkt.rO(ptr,externalSurfaceAddress) < 0) {
-			RR->t->incomingPacketDropped(tPtr,0x10001003,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
-			return false;
-		}
+	// LEGACY: this is superseded by the sent-to field in the meta-data dictionary if present.
+	InetAddress sentTo;
+	if (unlikely(pkt.rO(p,sentTo) < 0)) {
+		RR->t->incomingPacketDropped(tPtr,0x707a9811,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+		return SharedPtr<Peer>();
 	}
 
-	if ((ptr < packetSize)&&(peer->remoteVersionProtocol() >= 11)) {
-		// Everything after this point is encrypted with Salsa20/12. This is only a privacy measure
-		// since there's nothing truly secret in a HELLO packet. It also means that an observer
-		// can't even get ephemeral public keys without first knowing the long term secret key,
-		// adding a little defense in depth.
-		uint8_t iv[8];
-		for (int i = 0; i < 8; ++i) iv[i] = pkt.unsafeData[i];
-		iv[7] &= 0xf8U; // this exists for pure legacy reasons, meh...
-		Salsa20 s20(key,iv);
-		s20.crypt12(pkt.unsafeData + ptr,pkt.unsafeData + ptr,packetSize - ptr);
-
-		ptr += pkt.rI16(ptr); // skip length field which currently is always zero in v2.0+
-
-		if (ptr < packetSize) {
-			const unsigned int dictionarySize = pkt.rI16(ptr);
-			const void *const dictionaryBytes = pkt.rBnc(ptr,dictionarySize);
-			if (Buf::readOverflow(ptr,packetSize)) {
-				RR->t->incomingPacketDropped(tPtr,0x0d0f0112,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
-				return false;
-			}
-			if (dictionarySize) {
-				if (!nodeMetaData.decode(dictionaryBytes,dictionarySize)) {
-					RR->t->incomingPacketDropped(tPtr,0x67192344,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
-					return false;
-				}
-			}
-
-			ptr += pkt.rI16(ptr); // skip any additional fields, currently always 0
+	if ((protoVersion >= 11)&&((p + 12) < packetSize)) {
+		uint64_t ctrNonce[2];
+		ctrNonce[0] = Utils::loadAsIsEndian<uint64_t>(pkt.unsafeData + p);
+	#if __BYTE_ORDER == __BIG_ENDIAN
+		ctrNonce[1] = ((uint64_t)Utils::loadAsIsEndian<uint32_t>(pkt.unsafeData + p + 8)) << 32U;
+	#else
+		ctrNonce[1] = Utils::loadAsIsEndian<uint32_t>(pkt.unsafeData + p + 8);
+	#endif
+		p += 12;
+
+		AES::CTR ctr(peer->identityHelloDictionaryEncryptionCipher());
+		ctr.init(reinterpret_cast<uint8_t *>(ctrNonce),pkt.unsafeData + p);
+		ctr.crypt(pkt.unsafeData + p,(packetSize - p) - ZT_HMACSHA384_LEN);
+		ctr.finish();
+
+		const unsigned int dictSize = pkt.rI16(p);
+		if (unlikely((p + dictSize) > packetSize)) {
+			RR->t->incomingPacketDropped(tPtr,0x707a9815,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+			return peer;
 		}
-	}
-
-	if (Buf::readOverflow(ptr,packetSize)) { // sanity check, should be impossible
-		RR->t->incomingPacketDropped(tPtr,0x50003470,0,p.h.packetId,id,path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
-		return false;
-	}
-
-	// Packet is fully decoded and has passed all tests -----------------------------------------------------------------
-
-	const int64_t now = RR->node->now();
-
-	if (!peer) {
-		if (!id.locallyValidate()) {
-			RR->t->incomingPacketDropped(tPtr,0x2ff7a909,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
-			return false;
+		Dictionary md;
+		if (!md.decode(pkt.unsafeData + p,dictSize)) {
+			RR->t->incomingPacketDropped(tPtr,0x707a9816,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+			return peer;
 		}
-		peer.set(new Peer(RR));
-		if (!peer)
-			return false;
-		peer->init(id);
-		peer = RR->topology->add(tPtr,peer);
-	}
-
-	// All validation steps complete, peer learned if not yet known -----------------------------------------------------
-
-	if ((hops == 0) && (externalSurfaceAddress))
-		RR->sa->iam(tPtr,id,path->localSocket(),path->address(),externalSurfaceAddress,RR->topology->isRoot(id),now);
 
-	peer->setRemoteVersion(p.versionProtocol,p.versionMajor,p.versionMinor,Utils::ntoh(p.versionRev));
-
-	// Compose and send OK(HELLO) ---------------------------------------------------------------------------------------
-
-	std::vector<uint8_t> myNodeMetaDataBin;
-	{
-		Dictionary myNodeMetaData;
-		myNodeMetaData.encode(myNodeMetaDataBin);
-	}
-	if (myNodeMetaDataBin.size() > ZT_PROTO_MAX_PACKET_LENGTH) {
-		RR->t->unexpectedError(tPtr,0xbc8861e0,"node meta-data dictionary exceeds maximum packet length while composing OK(HELLO) to %s",Trace::str(id.address(),path).s);
-		return false;
-	}
-
-	Buf outp;
-	Protocol::OK::HELLO &ok = outp.as<Protocol::OK::HELLO>();
-
-	ok.h.h.packetId = Protocol::getPacketId();
-	id.address().copyTo(ok.h.h.destination);
-	RR->identity.address().copyTo(ok.h.h.source);
-	ok.h.h.flags = 0;
-	ok.h.h.verb = Protocol::VERB_OK;
-
-	ok.h.inReVerb = Protocol::VERB_HELLO;
-	ok.h.inRePacketId = p.h.packetId;
-
-	ok.timestampEcho = p.timestamp;
-	ok.versionProtocol = ZT_PROTO_VERSION;
-	ok.versionMajor = ZEROTIER_VERSION_MAJOR;
-	ok.versionMinor = ZEROTIER_VERSION_MINOR;
-	ok.versionRev = ZT_CONST_TO_BE_UINT16(ZEROTIER_VERSION_REVISION);
-
-	int outl = sizeof(Protocol::OK::HELLO);
-	outp.wO(outl,path->address());
-
-	outp.wI16(outl,0); // legacy field, always 0
-
-	if (p.versionProtocol >= 11) {
-		outp.wI16(outl,(uint16_t)myNodeMetaDataBin.size());
-		outp.wB(outl,myNodeMetaDataBin.data(),(unsigned int)myNodeMetaDataBin.size());
-		outp.wI16(outl,0); // length of additional fields, currently 0
-
-		if ((outl + ZT_HMACSHA384_LEN) > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check, shouldn't be possible
-			return false;
-
-		KBKDFHMACSHA384(key,ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC,0,1,hmacKey); // iter == 1 for OK
-		HMACSHA384(hmacKey,outp.unsafeData + sizeof(ok.h),outl - sizeof(ok.h),outp.unsafeData + outl);
-		outl += ZT_HMACSHA384_LEN;
+		if (!md.empty()) {
+			InetAddress sentTo2;
+			if (md.getO(ZT_PROTO_HELLO_NODE_META_PHYSICAL_DEST,sentTo2))
+				sentTo = sentTo2;
+			const uint64_t packedVer = md.getUI(ZT_PROTO_HELLO_NODE_META_SOFTWARE_VERSION);
+			if (packedVer != 0) {
+				versionMajor = (unsigned int)(packedVer >> 48U) & 0xffffU;
+				versionMinor = (unsigned int)(packedVer >> 32U) & 0xffffU;
+				versionRev = (unsigned int)(packedVer >> 16U) & 0xffffU;
+			}
+			const uint32_t probeToken = (uint32_t)md.getUI(ZT_PROTO_HELLO_NODE_META_PROBE_TOKEN);
+			if (probeToken != 0)
+				peer->setProbeToken(probeToken);
+		}
 	}
 
-	Protocol::armor(outp,outl,peer->key(),peer->cipher());
-	path->send(RR,tPtr,outp.unsafeData,outl,now);
-
-	return true;
+	peer->setRemoteVersion(protoVersion,versionMajor,versionMinor,versionRev);
 }
 
 bool VL1::m_ERROR(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb)

node/VL1.hpp

@@ -23,6 +23,8 @@
 #include "FCV.hpp"
 #include "Containers.hpp"
 
+#define ZT_VL1_MAX_WHOIS_WAITING_PACKETS 32
+
 namespace ZeroTier {
 
 class RuntimeEnvironment;
@@ -66,7 +68,7 @@ private:
 	void m_sendPendingWhois(void *tPtr, int64_t now);
 
 	// Handlers for VL1 verbs -- for clarity's sake VL2 verbs are in the VL2 class.
-	bool m_HELLO(void *tPtr, const SharedPtr<Path> &path, SharedPtr<Peer> &peer, Buf &pkt, int packetSize, bool authenticated);
+	SharedPtr<Peer> m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt, int packetSize);
 	bool m_ERROR(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
 	bool m_OK(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb);
 	bool m_WHOIS(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
@@ -76,16 +78,17 @@ private:
 	bool m_USER_MESSAGE(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
 	bool m_ENCAP(void *tPtr, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize);
 
+	Defragmenter<ZT_MAX_PACKET_FRAGMENTS> m_inputPacketAssembler;
+
 	struct p_WhoisQueueItem
 	{
-		ZT_INLINE p_WhoisQueueItem() : lastRetry(0), inboundPackets(), retries(0) {}
+		ZT_INLINE p_WhoisQueueItem() : lastRetry(0),retries(0),waitingPacketCount(0) {}
 		int64_t lastRetry;
-		FCV<Buf::Slice,32> inboundPackets; // capacity can be changed but this should be plenty
 		unsigned int retries;
+		unsigned int waitingPacketCount;
+		unsigned int waitingPacketSize[ZT_VL1_MAX_WHOIS_WAITING_PACKETS];
+		SharedPtr<Buf> waitingPacket[ZT_VL1_MAX_WHOIS_WAITING_PACKETS];
 	};
-
-	Defragmenter<ZT_MAX_PACKET_FRAGMENTS> m_inputPacketAssembler;
-
 	Map<Address,p_WhoisQueueItem> m_whoisQueue;
 	Mutex m_whoisQueue_l;
 };