It almost compiles again

include/ZeroTierCore.h

@@ -328,6 +328,9 @@ enum ZT_CredentialType
  */
 enum ZT_TraceEventType
 {
+	/* An unexpected error is an internal assertion / sanity check failure, out of memory, etc. */
+	ZT_TRACE_UNEXPECTED_ERROR = 0,
+
 	/* VL1 events related to the peer-to-peer layer */
 	ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE = 1,
 	ZT_TRACE_VL1_TRYING_NEW_PATH = 2,
@@ -433,16 +436,25 @@ ZT_PACKED_STRUCT(struct ZT_TraceEventPathAddress
  */
 ZT_PACKED_STRUCT(struct ZT_TraceEvent
 {
-	uint16_t evSize;     /* sizeof(ZT_TraceEvent_XX structure) (inclusive size in bytes) */
-	uint16_t evType;     /* ZT_TraceEventType */
+	uint16_t evSize;                                  /* sizeof(ZT_TraceEvent_XX structure) (inclusive) */
+	uint16_t evType;                                  /* ZT_TraceEventType */
+	uint32_t codeLocation;                            /* arbitrary identifier of location in source code */
 });
 
 /* Temporary macros to make it easier to declare all ZT_TraceEvent's sub-types */
 #define _ZT_TRACE_EVENT_STRUCT_START(e) ZT_PACKED_STRUCT_START struct ZT_TraceEvent_##e { \
 	uint16_t evSize; \
-	uint16_t evType;
+	uint16_t evType; \
+	uint32_t codeLocation;
 #define _ZT_TRACE_EVENT_STRUCT_END() } ZT_PACKED_STRUCT_END;
 
+/**
+ * An unexpected or internal error occurred
+ */
+_ZT_TRACE_EVENT_STRUCT_START(UNEXPECTED_ERROR)
+	char message[256];                               /* arbitrary human-readable message */
+_ZT_TRACE_EVENT_STRUCT_END()
+
 /**
  * Node is resetting all paths in a given address scope
  *

node/Buf.cpp

@@ -66,7 +66,7 @@ void *_Buf_get()
 #else
 		s_pool.store(bb);
 #endif
-		b = (Buf *)malloc(sizeof(Buf<>));
+		b = (Buf *)malloc(sizeof(Buf));
 		if (!b)
 			throw std::bad_alloc();
 	} else {

node/Credential.cpp

@@ -19,8 +19,8 @@
 #include "CertificateOfMembership.hpp"
 #include "CertificateOfOwnership.hpp"
 #include "Revocation.hpp"
-#include "Switch.hpp"
 #include "Network.hpp"
+#include "Topology.hpp"
 
 // These are compile-time asserts to make sure temporary marshal buffers here and
 // also in NtworkConfig.cpp are always large enough to marshal all credential types.

node/Endpoint.hpp

@@ -14,17 +14,17 @@
 #ifndef ZT_ENDPOINT_HPP
 #define ZT_ENDPOINT_HPP
 
-#include <cstdio>
-#include <cstdlib>
-#include <cstdint>
-#include <cstring>
-
 #include "Constants.hpp"
 #include "InetAddress.hpp"
 #include "Address.hpp"
 #include "Utils.hpp"
 #include "TriviallyCopyable.hpp"
 
+#include <cstdio>
+#include <cstdlib>
+#include <cstdint>
+#include <cstring>
+
 // max name size + type byte + port (for DNS name/port) + 3x 16-bit coordinate for location
 #define ZT_ENDPOINT_MARSHAL_SIZE_MAX (ZT_ENDPOINT_MAX_NAME_SIZE+1+2+2+2+2)
 

node/Identity.cpp

@@ -83,6 +83,8 @@ struct _Identity_generate_cond
 
 } // anonymous namespace
 
+const Identity Identity::NIL;
+
 void Identity::generate(const Type t)
 {
 	uint8_t digest[64];

node/Identity.hpp

@@ -56,6 +56,11 @@ public:
 		P384 = ZT_CRYPTO_ALG_P384      // Type 1 -- NIST P-384 with linked Curve25519/Ed25519 secondaries (2.x+)
 	};
 
+	/**
+	 * A nil/empty identity instance
+	 */
+	static const Identity NIL;
+
 	ZT_ALWAYS_INLINE Identity() { memoryZero(this); }
 	ZT_ALWAYS_INLINE ~Identity() { Utils::burn(reinterpret_cast<void *>(&this->_priv),sizeof(this->_priv)); }
 

node/InetAddress.hpp

@@ -87,57 +87,57 @@ public:
 
 	ZT_ALWAYS_INLINE InetAddress() { memoryZero(this); }
 	ZT_ALWAYS_INLINE InetAddress(const InetAddress &a) { memoryCopy(this,&a); }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr_storage &ss) { *this = ss; }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr_storage *ss) { *this = ss; }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr &sa) { *this = sa; }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr *sa) { *this = sa; }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr_in &sa) { *this = sa; }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr_in *sa) { *this = sa; }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr_in6 &sa) { *this = sa; }
-	explicit ZT_ALWAYS_INLINE InetAddress(const struct sockaddr_in6 *sa) { *this = sa; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr_storage &ss) { *this = ss; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr_storage *ss) { *this = ss; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr &sa) { *this = sa; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr *sa) { *this = sa; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr_in &sa) { *this = sa; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr_in *sa) { *this = sa; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr_in6 &sa) { *this = sa; }
+	explicit ZT_ALWAYS_INLINE InetAddress(const sockaddr_in6 *sa) { *this = sa; }
 	ZT_ALWAYS_INLINE InetAddress(const void *ipBytes,unsigned int ipLen,unsigned int port) { this->set(ipBytes,ipLen,port); }
 	ZT_ALWAYS_INLINE InetAddress(const uint32_t ipv4,unsigned int port) { this->set(&ipv4,4,port); }
 	explicit ZT_ALWAYS_INLINE InetAddress(const char *ipSlashPort) { this->fromString(ipSlashPort); }
 
 	ZT_ALWAYS_INLINE void clear() { memoryZero(this); }
 
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr_storage &ss)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr_storage &ss)
 	{
 		memoryCopyUnsafe(this,&ss);
 		return *this;
 	}
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr_storage *ss)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr_storage *ss)
 	{
 		if (ss)
 			memoryCopyUnsafe(this,ss);
 		else memoryZero(this);
 		return *this;
 	}
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr_in &sa)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr_in &sa)
 	{
 		copySockaddrToThis(&sa);
 		return *this;
 	}
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr_in *sa)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr_in *sa)
 	{
 		if (sa)
 			copySockaddrToThis(sa);
 		else memset(reinterpret_cast<void *>(this),0,sizeof(InetAddress));
 		return *this;
 	}
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr_in6 &sa)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr_in6 &sa)
 	{
 		copySockaddrToThis(&sa);
 		return *this;
 	}
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr_in6 *sa)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr_in6 *sa)
 	{
 		if (sa)
 			copySockaddrToThis(sa);
 		else memset(reinterpret_cast<void *>(this),0,sizeof(InetAddress));
 		return *this;
 	}
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr &sa)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr &sa)
 	{
 		if (sa.sa_family == AF_INET)
 			copySockaddrToThis(reinterpret_cast<const sockaddr_in *>(&sa));
@@ -146,7 +146,7 @@ public:
 		else memset(reinterpret_cast<void *>(this),0,sizeof(InetAddress));
 		return *this;
 	}
-	ZT_ALWAYS_INLINE InetAddress &operator=(const struct sockaddr *sa)
+	ZT_ALWAYS_INLINE InetAddress &operator=(const sockaddr *sa)
 	{
 		if (sa) {
 			if (sa->sa_family == AF_INET)

node/MulticastGroup.hpp

@@ -14,14 +14,14 @@
 #ifndef ZT_MULTICASTGROUP_HPP
 #define ZT_MULTICASTGROUP_HPP
 
-#include <cstdint>
-
 #include "Constants.hpp"
 #include "MAC.hpp"
 #include "InetAddress.hpp"
 #include "Utils.hpp"
 #include "TriviallyCopyable.hpp"
 
+#include <cstdint>
+
 namespace ZeroTier {
 
 /**
@@ -64,7 +64,7 @@ public:
 			// multicast address directly from the IP address, and it gives us
 			// 24 bits of uniqueness. Collisions aren't likely to be common enough
 			// to care about.
-			const uint8_t *a = (const uint8_t *)ip.rawIpData();
+			const uint8_t *const a = reinterpret_cast<const uint8_t *>(ip.rawIpData());
 			return MulticastGroup(MAC(0x33,0x33,0xff,a[13],a[14],a[15]),0);
 		}
 		return MulticastGroup();

node/Network.hpp

@@ -196,7 +196,7 @@ public:
 	 * @param size Size of data in chunk buffer (total, not relative to ptr)
 	 * @return Update ID if update was fully assembled and accepted or 0 otherwise
 	 */
-	uint64_t handleConfigChunk(void *tPtr,uint64_t packetId,const SharedPtr<Peer> &source,const Buf<> &chunk,int ptr,int size);
+	uint64_t handleConfigChunk(void *tPtr,uint64_t packetId,const SharedPtr<Peer> &source,const Buf &chunk,int ptr,int size);
 
 	/**
 	 * Set network configuration

node/Node.hpp

@@ -65,7 +65,7 @@ public:
 	void operator delete(void* p) { _mm_free(p); }
 #endif
 
-	// Public API Functions ----------------------------------------------------
+	// Public API Functions ---------------------------------------------------------------------------------------------
 
 	ZT_ResultCode processWirePacket(
 		void *tptr,
@@ -104,7 +104,7 @@ public:
 	int sendUserMessage(void *tptr,uint64_t dest,uint64_t typeId,const void *data,unsigned int len);
 	void setController(void *networkControllerInstance);
 
-	// Internal functions ------------------------------------------------------
+	// Internal functions -----------------------------------------------------------------------------------------------
 
 	/**
 	 * @return Most recent time value supplied to core via API

node/Peer.cpp

@@ -12,14 +12,15 @@
 /****/
 
 #include "Constants.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Trace.hpp"
 #include "Peer.hpp"
+#include "Topology.hpp"
 #include "Node.hpp"
-#include "Switch.hpp"
 #include "Network.hpp"
 #include "SelfAwareness.hpp"
-#include "Packet.hpp"
-#include "Trace.hpp"
 #include "InetAddress.hpp"
+#include "Protocol.hpp"
 
 #include <set>
 
@@ -43,7 +44,11 @@ Peer::Peer(const RuntimeEnvironment *renv) :
 	_lastTriedStaticPath(0),
 	_lastPrioritizedPaths(0),
 	_latency(0xffff),
-	_alivePathCount(0)
+	_alivePathCount(0),
+	_vProto(0),
+	_vMajor(0),
+	_vMinor(0),
+	_vRevision(0)
 {
 }
 
@@ -52,10 +57,6 @@ bool Peer::init(const Identity &myIdentity,const Identity &peerIdentity)
 	if (_id == peerIdentity)
 		return true;
 	_id = peerIdentity;
-	_vProto = 0;
-	_vMajor = 0;
-	_vMinor = 0;
-	_vRevision = 0;
 	return myIdentity.agree(peerIdentity,_key);
 }
 
@@ -65,9 +66,9 @@ void Peer::received(
 	const unsigned int hops,
 	const uint64_t packetId,
 	const unsigned int payloadLength,
-	const Packet::Verb verb,
+	const Protocol::Verb verb,
 	const uint64_t inRePacketId,
-	const Packet::Verb inReVerb,
+	const Protocol::Verb inReVerb,
 	const uint64_t networkId)
 {
 	const int64_t now = RR->node->now();
@@ -83,7 +84,7 @@ void Peer::received(
 		}
 		_lock.runlock();
 
-		if (verb == Packet::VERB_OK) {
+		if (verb == Protocol::VERB_OK) {
 			RWMutex::Lock l(_lock);
 
 			int64_t lastReceiveTimeMax = 0;
@@ -115,10 +116,10 @@ void Peer::received(
 			_paths[lastReceiveTimeMaxAt] = path;
 			_bootstrap = path->address();
 			_prioritizePaths(now);
-			RR->t->learnedNewPath(tPtr,packetId,_id,path->address(),old);
+			RR->t->learnedNewPath(tPtr,0x582fabdd,packetId,_id,path->address(),old);
 		} else {
 			if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id,path->localSocket(),path->address())) {
-				RR->t->tryingNewPath(tPtr,_id,path->address(),path->address(),packetId,(uint8_t)verb,_id.address(),_id.hash(),ZT_TRACE_TRYING_NEW_PATH_REASON_PACKET_RECEIVED_FROM_UNKNOWN_PATH);
+				RR->t->tryingNewPath(tPtr,0xb7747ddd,_id,path->address(),path->address(),packetId,(uint8_t)verb,_id.address(),_id.hash(),ZT_TRACE_TRYING_NEW_PATH_REASON_PACKET_RECEIVED_FROM_UNKNOWN_PATH);
 				sendHELLO(tPtr,path->localSocket(),path->address(),now);
 				path->sent(now);
 			}
@@ -131,11 +132,11 @@ path_check_done:
 
 		InetAddress addr;
 		if ((_bootstrap.type() == Endpoint::INETADDR_V4)||(_bootstrap.type() == Endpoint::INETADDR_V6)) {
-			RR->t->tryingNewPath(tPtr,_id,_bootstrap.inetAddr(),InetAddress::NIL,0,0,0,nullptr,ZT_TRACE_TRYING_NEW_PATH_REASON_BOOTSTRAP_ADDRESS);
+			RR->t->tryingNewPath(tPtr,0x0a009444,_id,_bootstrap.inetAddr(),InetAddress::NIL,0,0,0,nullptr,ZT_TRACE_TRYING_NEW_PATH_REASON_BOOTSTRAP_ADDRESS);
 			sendHELLO(tPtr,-1,_bootstrap.inetAddr(),now);
 		} if (RR->node->externalPathLookup(tPtr,_id,-1,addr)) {
 			if (RR->node->shouldUsePathForZeroTierTraffic(tPtr,_id,-1,addr)) {
-				RR->t->tryingNewPath(tPtr,_id,_bootstrap.inetAddr(),InetAddress::NIL,0,0,0,nullptr,ZT_TRACE_TRYING_NEW_PATH_REASON_EXPLICITLY_SUGGESTED_ADDRESS);
+				RR->t->tryingNewPath(tPtr,0x84a10000,_id,_bootstrap.inetAddr(),InetAddress::NIL,0,0,0,nullptr,ZT_TRACE_TRYING_NEW_PATH_REASON_EXPLICITLY_SUGGESTED_ADDRESS);
 				sendHELLO(tPtr,-1,addr,now);
 			}
 		}
@@ -155,6 +156,7 @@ path_check_done:
 		}
 
 		if (!addrs.empty()) {
+#if 0
 			ScopedPtr<Packet> outp(new Packet(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS));
 			outp->addSize(2); // leave room for count
 			unsigned int count = 0;
@@ -197,6 +199,7 @@ path_check_done:
 				outp->armor(_key,true);
 				path->send(RR,tPtr,outp->data(),outp->size(),now);
 			}
+#endif
 		}
 	}
 }
@@ -222,6 +225,7 @@ bool Peer::shouldTryPath(void *tPtr,int64_t now,const SharedPtr<Peer> &suggested
 
 void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atAddress,int64_t now)
 {
+#if 0
 	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
 
 	outp.append((unsigned char)ZT_PROTO_VERSION);
@@ -240,6 +244,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA
 	} else {
 		RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC
 	}
+#endif
 }
 
 void Peer::ping(void *tPtr,int64_t now,const bool pingAllAddressTypes)

node/Protocol.cpp

@@ -44,8 +44,12 @@ unsigned long long _packetIdCtr = _initPacketID();
 static std::atomic<unsigned long long> _packetIdCtr(_initPacketID());
 #endif
 
-uintptr_t _checkStructureSizing()
+uintptr_t _checkSizes()
 {
+	// These are compiled time checked assertions that make sure our platform/compiler is sane
+	// and that packed structures are working properly.
+	if (ZT_PROTO_MAX_PACKET_LENGTH > ZT_BUF_MEM_SIZE)
+		throw std::runtime_error("ZT_PROTO_MAX_PACKET_LENGTH > ZT_BUF_MEM_SIZE");
 	if (sizeof(Header) != ZT_PROTO_MIN_PACKET_LENGTH)
 		throw std::runtime_error("sizeof(Header) != ZT_PROTO_MIN_PACKET_LENGTH");
 	if (sizeof(FragmentHeader) != ZT_PROTO_MIN_FRAGMENT_LENGTH)
@@ -55,7 +59,8 @@ uintptr_t _checkStructureSizing()
 
 } // anonymous namespace
 
-volatile uintptr_t _compileTimeStructCheckHappened = _checkStructureSizing();
+// Make compiler compile and "run" _checkSizes()
+volatile uintptr_t _checkSizesIMeanIt = _checkSizes();
 
 uint64_t getPacketId()
 {

node/Protocol.hpp

@@ -50,6 +50,7 @@
  * 10 - 1.4.0 ... 1.4.6
  * 11 - 2.0.0 ... CURRENT
  *    + Peer-to-peer multicast replication
+ *    + HELLO and OK(HELLO) include an extra HMAC to further harden auth
  *    + Old planet/moon stuff is DEAD!
  *    + AES encryption support
  *    + NIST P-384 (type 1) identities
@@ -152,7 +153,10 @@
 #define ZT_PROTO_VERB_MASK 0x1fU
 
 /**
- * Key derivation function label for the key used with HMAC-384 in HELLO
+ * Key derivation function label for the keys used with HMAC-384 in HELLO
+ *
+ * With the KDF the 'iter' parameter is 0 for the key used for
+ * HMAC in HELLO and 1 for the one used in OK(HELLO).
  */
 #define ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC 'H'
 
@@ -284,6 +288,7 @@ enum Verb
 	 * ordinary packet authentication.
 	 *
 	 * OK payload:
+	 *   [... HMAC-384 starts here ...]
 	 *   <[8] HELLO timestamp field echo>
 	 *   <[1] protocol version>
 	 *   <[1] software major version>
@@ -293,6 +298,7 @@ enum Verb
 	 *   <[2] 16-bit reserved (legacy) field, always 0>
 	 *   <[2] 16-bit length of meta-data dictionary>
 	 *   <[...] meta-data dictionary>
+	 *   <[2] 16-bit length of any additional fields>
 	 *   <[48] HMAC-SHA384 of all fields to this point (as plaintext)>
 	 *
 	 * With the exception of the timestamp, the other fields pertain to the
@@ -824,26 +830,24 @@ namespace OK {
  */
 ZT_PACKED_STRUCT(struct Header
 {
+	Protocol::Header h;
 	uint8_t inReVerb;
 	uint64_t inRePacketId;
 });
 
 ZT_PACKED_STRUCT(struct WHOIS
 {
-	Protocol::Header h;
-	OK::Header oh;
+	OK::Header h;
 });
 
 ZT_PACKED_STRUCT(struct ECHO
 {
-	Protocol::Header h;
-	OK::Header oh;
+	OK::Header h;
 });
 
 ZT_PACKED_STRUCT(struct HELLO
 {
-	Protocol::Header h;
-	OK::Header oh;
+	OK::Header h;
 	uint64_t timestampEcho;
 	uint8_t versionProtocol;
 	uint8_t versionMajor;
@@ -853,8 +857,7 @@ ZT_PACKED_STRUCT(struct HELLO
 
 ZT_PACKED_STRUCT(struct EXT_FRAME
 {
-	Protocol::Header h;
-	OK::Header oh;
+	OK::Header h;
 	uint64_t networkId;
 	uint8_t flags;
 	uint8_t destMac[6];
@@ -864,8 +867,7 @@ ZT_PACKED_STRUCT(struct EXT_FRAME
 
 ZT_PACKED_STRUCT(struct NETWORK_CONFIG
 {
-	Protocol::Header h;
-	OK::Header oh;
+	OK::Header h;
 	uint64_t networkId;
 	uint64_t configUpdateId;
 });
@@ -883,6 +885,7 @@ namespace ERROR {
  */
 ZT_PACKED_STRUCT(struct Header
 {
+	Protocol::Header h;
 	int8_t inReVerb;
 	uint64_t inRePacketId;
 	uint8_t error;
@@ -890,15 +893,13 @@ ZT_PACKED_STRUCT(struct Header
 
 ZT_PACKED_STRUCT(struct NEED_MEMBERSHIP_CERTIFICATE
 {
-	Protocol::Header h;
-	ERROR::Header eh;
+	ERROR::Header h;
 	uint64_t networkId;
 });
 
 ZT_PACKED_STRUCT(struct UNSUPPORTED_OPERATION__NETWORK_CONFIG_REQUEST
 {
-	Protocol::Header h;
-	ERROR::Header eh;
+	ERROR::Header h;
 	uint64_t networkId;
 });
 
@@ -906,19 +907,6 @@ ZT_PACKED_STRUCT(struct UNSUPPORTED_OPERATION__NETWORK_CONFIG_REQUEST
 
 /****************************************************************************/
 
-/**
- * Increment the 3-bit hops field embedded in the packet flags field
- */
-ZT_ALWAYS_INLINE unsigned int incrementPacketHops(Header &h)
-{
-	uint8_t flags = h.flags;
-	uint8_t hops = flags;
-	flags &= 0xf8U;
-	++hops;
-	h.flags = flags | (hops & 0x07U);
-	return (unsigned int)hops;
-}
-
 /**
  * @return 3-bit hops field embedded in packet flags field
  */

node/RuntimeEnvironment.hpp

@@ -22,8 +22,8 @@
 
 namespace ZeroTier {
 
-class NodeConfig;
-class Switch;
+class VL1;
+class VL2;
 class Topology;
 class Node;
 class NetworkController;
@@ -36,19 +36,24 @@ class Trace;
 class RuntimeEnvironment
 {
 public:
-	inline RuntimeEnvironment(Node *n) :
-		node(n)
-		,localNetworkController((NetworkController *)0)
-		,rtmem((void *)0)
-		,sw((Switch *)0)
-		,topology((Topology *)0)
-		,sa((SelfAwareness *)0)
+	ZT_ALWAYS_INLINE RuntimeEnvironment(Node *n) :
+		node(n),
+		localNetworkController(nullptr),
+		rtmem(nullptr),
+		t(nullptr),
+		vl2(nullptr),
+		vl1(nullptr),
+		topology(nullptr),
+		sa(nullptr)
 	{
 		publicIdentityStr[0] = (char)0;
 		secretIdentityStr[0] = (char)0;
 	}
 
-	inline ~RuntimeEnvironment() { Utils::burn(secretIdentityStr,sizeof(secretIdentityStr)); }
+	ZT_ALWAYS_INLINE ~RuntimeEnvironment()
+	{
+		Utils::burn(secretIdentityStr,sizeof(secretIdentityStr));
+	}
 
 	// Node instance that owns this RuntimeEnvironment
 	Node *const node;
@@ -66,7 +71,8 @@ public:
 	 * These are constant and never null after startup unless indicated. */
 
 	Trace *t;
-	Switch *sw;
+	VL2 *vl2;
+	VL1 *vl1;
 	Topology *topology;
 	SelfAwareness *sa;
 

node/SHA512.cpp

@@ -1,14 +1,13 @@
 // This code is public domain, taken from a PD crypto source file on GitHub.
 
-#include <cstdint>
-#include <cstdlib>
-#include <cstring>
-
 #include "SHA512.hpp"
 #include "Utils.hpp"
 
 #include <utility>
 #include <algorithm>
+#include <cstdint>
+#include <cstdlib>
+#include <cstring>
 
 namespace ZeroTier {
 

node/SelfAwareness.cpp

@@ -21,7 +21,6 @@
 #include "RuntimeEnvironment.hpp"
 #include "Topology.hpp"
 #include "Peer.hpp"
-#include "Switch.hpp"
 #include "Trace.hpp"
 
 // Entry timeout -- make it fairly long since this is just to prevent stale buildup
@@ -86,7 +85,7 @@ void SelfAwareness::iam(void *tPtr,const Identity &reporter,const int64_t receiv
 		_ResetWithinScope rset(tPtr,now,myPhysicalAddress.ss_family,(InetAddress::IpScope)scope);
 		RR->topology->eachPeer<_ResetWithinScope &>(rset);
 
-		RR->t->resettingPathsInScope(tPtr,reporter,reporterPhysicalAddress,entry.mySurface,myPhysicalAddress,scope);
+		RR->t->resettingPathsInScope(tPtr,0x9afff100,reporter,reporterPhysicalAddress,entry.mySurface,myPhysicalAddress,scope);
 	} else {
 		// Otherwise just update DB to use to determine external surface info
 		entry.mySurface = myPhysicalAddress;

node/SharedPtr.hpp

@@ -14,6 +14,7 @@
 #ifndef ZT_SHAREDPTR_HPP
 #define ZT_SHAREDPTR_HPP
 
+#include "Constants.hpp"
 #include "AtomicCounter.hpp"
 #include "TriviallyCopyable.hpp"
 

node/Trace.cpp

@@ -16,6 +16,10 @@
 #include "Node.hpp"
 #include "Peer.hpp"
 
+#include <cstdio>
+#include <cstdlib>
+#include <cstdarg>
+
 // NOTE: packet IDs are always handled in network byte order, so no need to convert them.
 
 namespace ZeroTier {
@@ -25,13 +29,31 @@ Trace::Trace(const RuntimeEnvironment *renv) :
 	_vl1(false),
 	_vl2(false),
 	_vl2Filter(false),
-	_vl2Multicast(false),
-	_eventBufSize(0)
+	_vl2Multicast(false)
 {
 }
 
+void Trace::unexpectedError(
+	void *tPtr,
+	uint32_t codeLocation,
+	const char *message,
+	...)
+{
+	va_list ap;
+	ZT_TraceEvent_UNEXPECTED_ERROR ev;
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_UNEXPECTED_ERROR);
+	ev.codeLocation = codeLocation;
+	memset(ev.message,0,sizeof(ev.message));
+	va_start(ap,message);
+	vsnprintf(ev.message,sizeof(ev.message),message,ap);
+	va_end(ap);
+	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
+}
+
 void Trace::_resettingPathsInScope(
 	void *const tPtr,
+	const uint32_t codeLocation,
 	const Identity &reporter,
 	const InetAddress &from,
 	const InetAddress &oldExternal,
@@ -39,30 +61,32 @@ void Trace::_resettingPathsInScope(
 	const InetAddress::IpScope scope)
 {
 	ZT_TraceEvent_VL1_RESETTING_PATHS_IN_SCOPE ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE);
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_RESETTING_PATHS_IN_SCOPE);
+	ev.codeLocation = Utils::hton(codeLocation);
 	from.forTrace(ev.from);
 	oldExternal.forTrace(ev.oldExternal);
 	newExternal.forTrace(ev.newExternal);
 	ev.scope = (uint8_t)scope;
-
 	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_tryingNewPath(
 	void *const tPtr,
+	const uint32_t codeLocation,
 	const Identity &trying,
 	const InetAddress &physicalAddress,
 	const InetAddress &triggerAddress,
-	uint64_t triggeringPacketId,
-	uint8_t triggeringPacketVerb,
-	uint64_t triggeredByAddress,
+	const uint64_t triggeringPacketId,
+	const uint8_t triggeringPacketVerb,
+	const uint64_t triggeredByAddress,
 	const uint8_t *triggeredByIdentityHash,
-	ZT_TraceTryingNewPathReason reason)
+	const ZT_TraceTryingNewPathReason reason)
 {
 	ZT_TraceEvent_VL1_TRYING_NEW_PATH ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL1_TRYING_NEW_PATH);
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_TRYING_NEW_PATH);
+	ev.codeLocation = Utils::hton(codeLocation);
 	ev.address = Utils::hton(trying.address().toInt());
 	memcpy(ev.identityHash,trying.hash(),48);
 	physicalAddress.forTrace(ev.physicalAddress);
@@ -74,21 +98,22 @@ void Trace::_tryingNewPath(
 		memcpy(ev.triggeredByIdentityHash,triggeredByIdentityHash,48);
 	else memset(ev.triggeredByIdentityHash,0,48);
 	ev.reason = (uint8_t)reason;
-
 	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_learnedNewPath(
 	void *const tPtr,
-	uint64_t packetId,
+	const uint32_t codeLocation,
+	const uint64_t packetId,
 	const Identity &peerIdentity,
 	const InetAddress &physicalAddress,
 	const InetAddress &replaced)
 {
 	ZT_TraceEvent_VL1_LEARNED_NEW_PATH ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL1_LEARNED_NEW_PATH);
-	ev.packetId = packetId;
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_LEARNED_NEW_PATH);
+	ev.codeLocation = Utils::hton(codeLocation);
+	ev.packetId = packetId; // packet IDs are kept in big-endian
 	ev.address = Utils::hton(peerIdentity.address().toInt());
 	memcpy(ev.identityHash,peerIdentity.hash(),48);
 	physicalAddress.forTrace(ev.physicalAddress);
@@ -99,18 +124,20 @@ void Trace::_learnedNewPath(
 
 void Trace::_incomingPacketDropped(
 	void *const tPtr,
-	uint64_t packetId,
-	uint64_t networkId,
+	const uint32_t codeLocation,
+	const uint64_t packetId,
+	const uint64_t networkId,
 	const Identity &peerIdentity,
 	const InetAddress &physicalAddress,
-	uint8_t hops,
-	uint8_t verb,
-	ZT_TracePacketDropReason reason)
+	const uint8_t hops,
+	const uint8_t verb,
+	const ZT_TracePacketDropReason reason)
 {
 	ZT_TraceEvent_VL1_INCOMING_PACKET_DROPPED ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
-	ev.packetId = packetId;
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL1_INCOMING_PACKET_DROPPED);
+	ev.codeLocation = Utils::hton(codeLocation);
+	ev.packetId = packetId; // packet IDs are kept in big-endian
 	ev.networkId = Utils::hton(networkId);
 	if (peerIdentity) {
 		ev.address = Utils::hton(peerIdentity.address().toInt());
@@ -129,17 +156,19 @@ void Trace::_incomingPacketDropped(
 
 void Trace::_outgoingNetworkFrameDropped(
 	void *const tPtr,
-	uint64_t networkId,
+	const uint32_t codeLocation,
+	const uint64_t networkId,
 	const MAC &sourceMac,
 	const MAC &destMac,
-	uint16_t etherType,
-	uint16_t frameLength,
+	const uint16_t etherType,
+	const uint16_t frameLength,
 	const uint8_t *frameData,
-	ZT_TraceFrameDropReason reason)
+	const ZT_TraceFrameDropReason reason)
 {
 	ZT_TraceEvent_VL2_OUTGOING_FRAME_DROPPED ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL2_OUTGOING_FRAME_DROPPED);
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_OUTGOING_FRAME_DROPPED);
+	ev.codeLocation = Utils::hton(codeLocation);
 	ev.networkId = Utils::hton(networkId);
 	ev.sourceMac = Utils::hton(sourceMac.toInt());
 	ev.destMac = Utils::hton(destMac.toInt());
@@ -159,21 +188,23 @@ void Trace::_outgoingNetworkFrameDropped(
 
 void Trace::_incomingNetworkFrameDropped(
 	void *const tPtr,
-	uint64_t networkId,
+	const uint32_t codeLocation,
+	const uint64_t networkId,
 	const MAC &sourceMac,
 	const MAC &destMac,
 	const Identity &peerIdentity,
 	const InetAddress &physicalAddress,
-	uint8_t hops,
-	uint16_t frameLength,
+	const uint8_t hops,
+	const uint16_t frameLength,
 	const uint8_t *frameData,
-	uint8_t verb,
-	bool credentialRequestSent,
-	ZT_TraceFrameDropReason reason)
+	const uint8_t verb,
+	const bool credentialRequestSent,
+	const ZT_TraceFrameDropReason reason)
 {
 	ZT_TraceEvent_VL2_INCOMING_FRAME_DROPPED ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL2_INCOMING_FRAME_DROPPED);
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_INCOMING_FRAME_DROPPED);
+	ev.codeLocation = Utils::hton(codeLocation);
 	ev.networkId = Utils::hton(networkId);
 	ev.sourceMac = Utils::hton(sourceMac.toInt());
 	ev.destMac = Utils::hton(destMac.toInt());
@@ -197,38 +228,41 @@ void Trace::_incomingNetworkFrameDropped(
 
 void Trace::_networkConfigRequestSent(
 	void *const tPtr,
-	uint64_t networkId)
+	const uint32_t codeLocation,
+	const uint64_t networkId)
 {
 	ZT_TraceEvent_VL2_NETWORK_CONFIG_REQUESTED ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_CONFIG_REQUESTED);
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_CONFIG_REQUESTED);
+	ev.codeLocation = Utils::hton(codeLocation);
 	ev.networkId = Utils::hton(networkId);
-
 	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_networkFilter(
 	void *const tPtr,
-	uint64_t networkId,
+	const uint32_t codeLocation,
+	const uint64_t networkId,
 	const uint8_t primaryRuleSetLog[512],
 	const uint8_t matchingCapabilityRuleSetLog[512],
-	uint32_t matchingCapabilityId,
-	int64_t matchingCapabilityTimestamp,
+	const uint32_t matchingCapabilityId,
+	const int64_t matchingCapabilityTimestamp,
 	const Address &source,
 	const Address &dest,
 	const MAC &sourceMac,
 	const MAC &destMac,
-	uint16_t frameLength,
+	const uint16_t frameLength,
 	const uint8_t *frameData,
-	uint16_t etherType,
-	uint16_t vlanId,
-	bool noTee,
-	bool inbound,
-	int accept)
+	const uint16_t etherType,
+	const uint16_t vlanId,
+	const bool noTee,
+	const bool inbound,
+	const int accept)
 {
 	ZT_TraceEvent_VL2_NETWORK_FILTER ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
+	ev.codeLocation = Utils::hton(codeLocation);
 	ev.networkId = Utils::hton(networkId);
 	memcpy(ev.primaryRuleSetLog,primaryRuleSetLog,sizeof(ev.primaryRuleSetLog));
 	if (matchingCapabilityRuleSetLog)
@@ -253,29 +287,29 @@ void Trace::_networkFilter(
 	ev.noTee = (uint8_t)noTee;
 	ev.inbound = (uint8_t)inbound;
 	ev.accept = (int8_t)accept;
-
 	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 
 void Trace::_credentialRejected(
 	void *const tPtr,
-	uint64_t networkId,
+	const uint32_t codeLocation,
+	const uint64_t networkId,
 	const Address &address,
-	uint32_t credentialId,
-	int64_t credentialTimestamp,
-	uint8_t credentialType,
-	ZT_TraceCredentialRejectionReason reason)
+	const uint32_t credentialId,
+	const int64_t credentialTimestamp,
+	const uint8_t credentialType,
+	const ZT_TraceCredentialRejectionReason reason)
 {
 	ZT_TraceEvent_VL2_CREDENTIAL_REJECTED ev;
-	ev.evSize = CONST_TO_BE_UINT16(sizeof(ev));
-	ev.evType = CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
+	ev.evSize = ZT_CONST_TO_BE_UINT16(sizeof(ev));
+	ev.evType = ZT_CONST_TO_BE_UINT16(ZT_TRACE_VL2_NETWORK_FILTER);
+	ev.codeLocation = Utils::hton(codeLocation);
 	ev.networkId = Utils::hton(networkId);
 	ev.address = Utils::hton(address.toInt());
 	ev.credentialId = Utils::hton(credentialId);
 	ev.credentialTimestamp = Utils::hton(credentialTimestamp);
 	ev.credentialType = credentialType;
 	ev.reason = (uint8_t)reason;
-
 	RR->node->postEvent(tPtr,ZT_EVENT_TRACE,&ev);
 }
 

node/Trace.hpp

@@ -66,19 +66,27 @@ public:
 
 	explicit Trace(const RuntimeEnvironment *renv);
 
+	void unexpectedError(
+		void *tPtr,
+		uint32_t codeLocation,
+		const char *message,
+		...);
+
 	ZT_ALWAYS_INLINE void resettingPathsInScope(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		const Identity &reporter,
 		const InetAddress &from,
 		const InetAddress &oldExternal,
 		const InetAddress &newExternal,
 		const InetAddress::IpScope scope)
 	{
-		if (_vl1) _resettingPathsInScope(tPtr,reporter,from,oldExternal,newExternal,scope);
+		if (_vl1) _resettingPathsInScope(tPtr,codeLocation,reporter,from,oldExternal,newExternal,scope);
 	}
 
 	ZT_ALWAYS_INLINE void tryingNewPath(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		const Identity &trying,
 		const InetAddress &physicalAddress,
 		const InetAddress &triggerAddress,
@@ -88,21 +96,23 @@ public:
 		const uint8_t *triggeredByIdentityHash,
 		ZT_TraceTryingNewPathReason reason)
 	{
-		if (_vl1) _tryingNewPath(tPtr,trying,physicalAddress,triggerAddress,triggeringPacketId,triggeringPacketVerb,triggeredByAddress,triggeredByIdentityHash,reason);
+		if (_vl1) _tryingNewPath(tPtr,codeLocation,trying,physicalAddress,triggerAddress,triggeringPacketId,triggeringPacketVerb,triggeredByAddress,triggeredByIdentityHash,reason);
 	}
 
 	ZT_ALWAYS_INLINE void learnedNewPath(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		uint64_t packetId,
 		const Identity &peerIdentity,
 		const InetAddress &physicalAddress,
 		const InetAddress &replaced)
 	{
-		if (_vl1) _learnedNewPath(tPtr,packetId,peerIdentity,physicalAddress,replaced);
+		if (_vl1) _learnedNewPath(tPtr,codeLocation,packetId,peerIdentity,physicalAddress,replaced);
 	}
 
 	ZT_ALWAYS_INLINE void incomingPacketDropped(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		uint64_t packetId,
 		uint64_t networkId,
 		const Identity &peerIdentity,
@@ -111,11 +121,12 @@ public:
 		uint8_t verb,
 		const ZT_TracePacketDropReason reason)
 	{
-		if (_vl1) _incomingPacketDropped(tPtr,packetId,networkId,peerIdentity,physicalAddress,hops,verb,reason);
+		if (_vl1) _incomingPacketDropped(tPtr,codeLocation,packetId,networkId,peerIdentity,physicalAddress,hops,verb,reason);
 	}
 
 	ZT_ALWAYS_INLINE void outgoingNetworkFrameDropped(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		uint64_t networkId,
 		const MAC &sourceMac,
 		const MAC &destMac,
@@ -124,11 +135,12 @@ public:
 		const uint8_t *frameData,
 		ZT_TraceFrameDropReason reason)
 	{
-		if (_vl2) _outgoingNetworkFrameDropped(tPtr,networkId,sourceMac,destMac,etherType,frameLength,frameData,reason);
+		if (_vl2) _outgoingNetworkFrameDropped(tPtr,codeLocation,networkId,sourceMac,destMac,etherType,frameLength,frameData,reason);
 	}
 
 	ZT_ALWAYS_INLINE void incomingNetworkFrameDropped(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		uint64_t networkId,
 		const MAC &sourceMac,
 		const MAC &destMac,
@@ -141,18 +153,20 @@ public:
 		bool credentialRequestSent,
 		ZT_TraceFrameDropReason reason)
 	{
-		if (_vl2) _incomingNetworkFrameDropped(tPtr,networkId,sourceMac,destMac,peerIdentity,physicalAddress,hops,frameLength,frameData,verb,credentialRequestSent,reason);
+		if (_vl2) _incomingNetworkFrameDropped(tPtr,codeLocation,networkId,sourceMac,destMac,peerIdentity,physicalAddress,hops,frameLength,frameData,verb,credentialRequestSent,reason);
 	}
 
 	ZT_ALWAYS_INLINE void networkConfigRequestSent(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		uint64_t networkId)
 	{
-		if (_vl2) _networkConfigRequestSent(tPtr,networkId);
+		if (_vl2) _networkConfigRequestSent(tPtr,codeLocation,networkId);
 	}
 
 	ZT_ALWAYS_INLINE void networkFilter(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		uint64_t networkId,
 		const uint8_t primaryRuleSetLog[512],
 		const uint8_t matchingCapabilityRuleSetLog[512],
@@ -173,6 +187,7 @@ public:
 		if (_vl2Filter) {
 			_networkFilter(
 				tPtr,
+				codeLocation,
 				networkId,
 				primaryRuleSetLog,
 				matchingCapabilityRuleSetLog,
@@ -194,6 +209,7 @@ public:
 
 	ZT_ALWAYS_INLINE void credentialRejected(
 		void *const tPtr,
+		const uint32_t codeLocation,
 		uint64_t networkId,
 		const Address &address,
 		uint32_t credentialId,
@@ -201,12 +217,13 @@ public:
 		uint8_t credentialType,
 		ZT_TraceCredentialRejectionReason reason)
 	{
-		if (_vl2) _credentialRejected(tPtr,networkId,address,credentialId,credentialTimestamp,credentialType,reason);
+		if (_vl2) _credentialRejected(tPtr,codeLocation,networkId,address,credentialId,credentialTimestamp,credentialType,reason);
 	}
 
 private:
 	void _resettingPathsInScope(
 		void *tPtr,
+		uint32_t codeLocation,
 		const Identity &reporter,
 		const InetAddress &from,
 		const InetAddress &oldExternal,
@@ -214,6 +231,7 @@ private:
 		InetAddress::IpScope scope);
 	void _tryingNewPath(
 		void *tPtr,
+		uint32_t codeLocation,
 		const Identity &trying,
 		const InetAddress &physicalAddress,
 		const InetAddress &triggerAddress,
@@ -224,12 +242,14 @@ private:
 		ZT_TraceTryingNewPathReason reason);
 	void _learnedNewPath(
 		void *tPtr,
+		uint32_t codeLocation,
 		uint64_t packetId,
 		const Identity &peerIdentity,
 		const InetAddress &physicalAddress,
 		const InetAddress &replaced);
 	void _incomingPacketDropped(
 		void *tPtr,
+		uint32_t codeLocation,
 		uint64_t packetId,
 		uint64_t networkId,
 		const Identity &peerIdentity,
@@ -239,6 +259,7 @@ private:
 		ZT_TracePacketDropReason reason);
 	void _outgoingNetworkFrameDropped(
 		void *tPtr,
+		uint32_t codeLocation,
 		uint64_t networkId,
 		const MAC &sourceMac,
 		const MAC &destMac,
@@ -247,7 +268,8 @@ private:
 		const uint8_t *frameData,
 		ZT_TraceFrameDropReason reason);
 	void _incomingNetworkFrameDropped(
-		void *const tPtr,
+		void *tPtr,
+		uint32_t codeLocation,
 		uint64_t networkId,
 		const MAC &sourceMac,
 		const MAC &destMac,
@@ -261,9 +283,11 @@ private:
 		ZT_TraceFrameDropReason reason);
 	void _networkConfigRequestSent(
 		void *tPtr,
+		uint32_t codeLocation,
 		uint64_t networkId);
 	void _networkFilter(
 		void *tPtr,
+		uint32_t codeLocation,
 		uint64_t networkId,
 		const uint8_t primaryRuleSetLog[512],
 		const uint8_t matchingCapabilityRuleSetLog[512],
@@ -282,6 +306,7 @@ private:
 		int accept);
 	void _credentialRejected(
 		void *tPtr,
+		uint32_t codeLocation,
 		uint64_t networkId,
 		const Address &address,
 		uint32_t credentialId,
@@ -300,9 +325,6 @@ private:
 		Mutex lock;
 	};
 
-	uint8_t _eventBuf[8192]; // must be less than ZT_PROTO_MAX_PACKET_LENGTH
-	unsigned int _eventBufSize;
-
 	std::vector<_MonitoringPeer> _monitoringPeers;
 	RWMutex _monitoringPeers_l;
 };

node/Utils.cpp

@@ -409,22 +409,27 @@ int b32d(const char *encoded,uint8_t *result,int bufSize)
 uint64_t random()
 {
 	// https://en.wikipedia.org/wiki/Xorshift#xoshiro256**
-	static Mutex l;
-	static uint64_t s0 = getSecureRandomU64();
-	static uint64_t s1 = getSecureRandomU64();
-	static uint64_t s2 = getSecureRandomU64();
-	static uint64_t s3 = getSecureRandomU64();
-
-	l.lock();
-	const uint64_t result = ROL64(s1 * 5,7) * 9;
+	static volatile uint64_t s_s0 = getSecureRandomU64();
+	static volatile uint64_t s_s1 = getSecureRandomU64();
+	static volatile uint64_t s_s2 = getSecureRandomU64();
+	static volatile uint64_t s_s3 = getSecureRandomU64();
+
+	uint64_t s0 = s_s0;
+	uint64_t s1 = s_s1;
+	uint64_t s2 = s_s2;
+	uint64_t s3 = s_s3;
+	const uint64_t result = ROL64(s1 * 5,7U) * 9;
 	const uint64_t t = s1 << 17U;
 	s2 ^= s0;
 	s3 ^= s1;
 	s1 ^= s2;
 	s0 ^= s3;
 	s2 ^= t;
-	s3 = ROL64(s3,45);
-	l.unlock();
+	s3 = ROL64(s3,45U);
+	s_s0 = s0;
+	s_s1 = s1;
+	s_s2 = s2;
+	s_s3 = s3;
 
 	return result;
 }

node/VL1.cpp

@@ -13,6 +13,7 @@
 
 #include "VL1.hpp"
 #include "RuntimeEnvironment.hpp"
+#include "Node.hpp"
 #include "Topology.hpp"
 #include "VL2.hpp"
 #include "Salsa20.hpp"
@@ -21,12 +22,24 @@
 #include "Identity.hpp"
 #include "SelfAwareness.hpp"
 #include "SHA512.hpp"
+#include "Peer.hpp"
+#include "Path.hpp"
 
 namespace ZeroTier {
 
+namespace {
+
+ZT_ALWAYS_INLINE const Identity &ifPeerNonNull(const SharedPtr<Peer> &p)
+{
+	if (p)
+		return p->identity();
+	return Identity::NIL;
+}
+
+} // anonymous namespace
+
 VL1::VL1(const RuntimeEnvironment *renv) :
-	RR(renv),
-	_vl2(nullptr)
+	RR(renv)
 {
 }
 
@@ -40,6 +53,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 	const SharedPtr<Path> path(RR->topology->getPath(localSocket,fromAddr));
 	path->received(now);
 
+	// Really short packets are keepalives and other junk.
 	if (len < ZT_PROTO_MIN_FRAGMENT_LENGTH)
 		return;
 
@@ -131,8 +145,10 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 		// Subject pktv to a few sanity checks just to make sure Defragmenter worked correctly and
 		// there is enough room in each slice to shift their contents to sizes that are multiples
 		// of 64 if needed for crypto.
-		if ((pktv.empty()) || (((int)pktv[0].e - (int)pktv[0].s) < sizeof(Protocol::Header)))
+		if ((pktv.empty()) || (((int)pktv[0].e - (int)pktv[0].s) < sizeof(Protocol::Header))) {
+			RR->t->unexpectedError(tPtr,0x3df19990,"empty or undersized packet vector");
 			return;
+		}
 		for(FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS>::const_iterator s(pktv.begin());s!=pktv.end();++s) {
 			if ((s->e > (ZT_BUF_MEM_SIZE - 64))||(s->s > s->e))
 				return;
@@ -155,7 +171,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 		for(FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS>::const_iterator s(pktv.begin()+1);s!=pktv.end();++s)
 			packetSize += s->e - s->s;
 		if (packetSize > ZT_PROTO_MAX_PACKET_LENGTH) {
-			RR->t->incomingPacketDropped(tPtr,ph->packetId,0,Identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+			RR->t->incomingPacketDropped(tPtr,0x010348da,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 			return;
 		}
 
@@ -163,18 +179,15 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 		if ((!peer)&&(!(((cipher == ZT_PROTO_CIPHER_SUITE__POLY1305_NONE)||(cipher == ZT_PROTO_CIPHER_SUITE__NONE))&&((ph->verb & 0x1fU) == Protocol::VERB_HELLO)))) {
 			pkt = Buf::assembleSliceVector(pktv);
 			if (pkt.e < ZT_PROTO_MIN_PACKET_LENGTH) {
-				RR->t->incomingPacketDropped(tPtr,ph->packetId,0,Identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+				RR->t->incomingPacketDropped(tPtr,0xbada9366,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 				return;
 			}
 			{
 				Mutex::Lock wl(_whoisQueue_l);
 				_WhoisQueueItem &wq = _whoisQueue[source];
 				wq.inboundPackets.push_back(pkt);
-				if (wq.retries == 0) {
-					wq.retries = 1;
-					_sendPendingWhois();
-				}
 			}
+			_sendPendingWhois(tPtr,now);
 			return;
 		}
 
@@ -182,8 +195,10 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 			case ZT_PROTO_CIPHER_SUITE__POLY1305_NONE:
 				if (peer) {
 					pkt = Buf::assembleSliceVector(pktv);
-					if (pkt.e < ZT_PROTO_MIN_PACKET_LENGTH)
-						RR->t->incomingPacketDropped(tPtr,ph->packetId,0,Identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+					if (pkt.e < ZT_PROTO_MIN_PACKET_LENGTH) {
+						RR->t->incomingPacketDropped(tPtr,0x432aa9da,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+						return;
+					}
 					ph = &(pkt.b->as<Protocol::Header>());
 
 					// Generate one-time-use MAC key using Salsa20.
@@ -196,7 +211,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					uint64_t mac[2];
 					poly1305(mac,pkt.b->b + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
 					if (ph->mac != mac[0]) {
-						RR->t->incomingPacketDropped(tPtr,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+						RR->t->incomingPacketDropped(tPtr,0xcc89c812,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 						return;
 					}
 					authenticated = true;
@@ -216,8 +231,10 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 
 					// Get a buffer to store the decrypted and fully contiguous packet.
 					pkt.b = Buf::get();
-					if (!pkt.b) // only possible on out of memory condition
+					if (!pkt.b) {
+						RR->t->unexpectedError(tPtr,0x1de16991,"Buf::get() failed (out of memory?)");
 						return;
+					}
 
 					// Salsa20 is a stream cipher but it's only seekable to multiples of 64 bytes.
 					// This moves data in slices around so that all slices have sizes that are
@@ -237,6 +254,8 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					}
 
 					// Simultaneously decrypt and assemble packet into a contiguous buffer.
+					// Since we moved data around above all slices will have sizes that are
+					// multiples of 64.
 					memcpy(pkt.b->b,ph,sizeof(Protocol::Header));
 					pkt.e = sizeof(Protocol::Header);
 					for(FCV<Buf::Slice,ZT_MAX_PACKET_FRAGMENTS>::iterator s(pktv.begin());s!=pktv.end();++s) {
@@ -250,12 +269,12 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					uint64_t mac[2];
 					poly1305(mac,pkt.b->b + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
 					if (ph->mac != mac[0]) {
-						RR->t->incomingPacketDropped(tPtr,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+						RR->t->incomingPacketDropped(tPtr,0xbc881231,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 						return;
 					}
 					authenticated = true;
 				} else {
-					RR->t->incomingPacketDropped(tPtr,ph->packetId,0,Identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+					RR->t->incomingPacketDropped(tPtr,0xb0b01999,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 					return;
 				}
 				break;
@@ -267,14 +286,13 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 
 				pkt = Buf::assembleSliceVector(pktv);
 				if (pkt.e < ZT_PROTO_MIN_PACKET_LENGTH)
-					RR->t->incomingPacketDropped(tPtr,ph->packetId,0,Identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+					RR->t->incomingPacketDropped(tPtr,0x3d3337df,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 				ph = &(pkt.b->as<Protocol::Header>());
 
 				if (RR->topology->shouldInboundPathBeTrusted(path->address(),Utils::ntoh(ph->mac))) {
 					authenticated = true;
 				} else {
-					if (peer)
-						RR->t->incomingPacketDropped(tPtr,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_NOT_TRUSTED_PATH);
+					RR->t->incomingPacketDropped(tPtr,0x2dfa910b,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_NOT_TRUSTED_PATH);
 					return;
 				}
 			} break;
@@ -285,8 +303,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 			//	break;
 
 			default:
-				if (peer)
-					RR->t->incomingPacketDropped(tPtr,ph->packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+				RR->t->incomingPacketDropped(tPtr,0x5b001099,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
 				return;
 		}
 
@@ -295,16 +312,20 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 		// Return any still held buffers in pktv to the buffer pool.
 		pktv.clear();
 
+		const Protocol::Verb verb = (Protocol::Verb)(ph->verb & ZT_PROTO_VERB_MASK);
+
 		// Decompress packet payload if compressed.
 		if ((ph->verb & ZT_PROTO_VERB_FLAG_COMPRESSED) != 0) {
 			if (!authenticated) {
-				RR->t->incomingPacketDropped(tPtr,ph->packetId,0,Identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+				RR->t->incomingPacketDropped(tPtr,0x390bcd0a,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 				return;
 			}
 
 			SharedPtr<Buf> nb(Buf::get());
-			if (!nb) // can only happen if we're out of memory
+			if (!nb) {
+				RR->t->unexpectedError(tPtr,0xffe169fa,"Buf::get() failed (out of memory?)");
 				return;
+			}
 
 			const int uncompressedLen = LZ4_decompress_safe(
 				reinterpret_cast<const char *>(pkt.b->b + ZT_PROTO_PACKET_PAYLOAD_START),
@@ -316,13 +337,14 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 				pkt.b.swap(nb);
 				pkt.e = packetSize = (unsigned int)uncompressedLen;
 			} else {
-				if (peer)
-					RR->t->incomingPacketDropped(tPtr,ph->packetId,0,peer->identity(),path->address(),hops,(Protocol::Verb)(ph->verb & ZT_PROTO_VERB_MASK),ZT_TRACE_PACKET_DROP_REASON_INVALID_COMPRESSED_DATA);
+				RR->t->incomingPacketDropped(tPtr,0xee9e4392,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_INVALID_COMPRESSED_DATA);
 				return;
 			}
 		}
 
-		const Protocol::Verb verb = (Protocol::Verb)(ph->verb & ZT_PROTO_VERB_MASK);
+		// VL1 and VL2 are conceptually and (mostly) logically separate layers.
+		// Verbs that relate to VL1 (P2P transport) are handled in this class.
+		// VL2 (virtual Ethernet / SDN) verbs are handled in the VL2 class.
 		switch(verb) {
 			case Protocol::VERB_NOP:
 				peer->received(tPtr,path,hops,ph->packetId,packetSize - ZT_PROTO_PACKET_PAYLOAD_START,Protocol::VERB_NOP,0,Protocol::VERB_NOP,0);
@@ -333,32 +355,26 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 			case Protocol::VERB_OK:                         _OK(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
 			case Protocol::VERB_WHOIS:                      _WHOIS(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
 			case Protocol::VERB_RENDEZVOUS:                 _RENDEZVOUS(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_FRAME:                      _vl2->_FRAME(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_EXT_FRAME:                  _vl2->_EXT_FRAME(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_FRAME:                      RR->vl2->_FRAME(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_EXT_FRAME:                  RR->vl2->_EXT_FRAME(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
 			case Protocol::VERB_ECHO:                       _ECHO(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated);
-			case Protocol::VERB_MULTICAST_LIKE:             _vl2->_MULTICAST_LIKE(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_NETWORK_CREDENTIALS:        _vl2->_NETWORK_CREDENTIALS(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_NETWORK_CONFIG_REQUEST:     _vl2->_NETWORK_CONFIG_REQUEST(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_NETWORK_CONFIG:             _vl2->_NETWORK_CONFIG(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_MULTICAST_GATHER:           _vl2->_MULTICAST_GATHER(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_MULTICAST_FRAME_deprecated: _vl2->_MULTICAST_FRAME_deprecated(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_MULTICAST_LIKE:             RR->vl2->_MULTICAST_LIKE(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_NETWORK_CREDENTIALS:        RR->vl2->_NETWORK_CREDENTIALS(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_NETWORK_CONFIG_REQUEST:     RR->vl2->_NETWORK_CONFIG_REQUEST(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_NETWORK_CONFIG:             RR->vl2->_NETWORK_CONFIG(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_MULTICAST_GATHER:           RR->vl2->_MULTICAST_GATHER(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_MULTICAST_FRAME_deprecated: RR->vl2->_MULTICAST_FRAME_deprecated(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
 			case Protocol::VERB_PUSH_DIRECT_PATHS:          _PUSH_DIRECT_PATHS(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
 			case Protocol::VERB_USER_MESSAGE:               _USER_MESSAGE(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
-			case Protocol::VERB_MULTICAST:                  _vl2->_MULTICAST(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
+			case Protocol::VERB_MULTICAST:                  RR->vl2->_MULTICAST(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
 			case Protocol::VERB_ENCAP:                      _ENCAP(tPtr,path,peer,*pkt.b,(int)packetSize,authenticated); break;
 
 			default:
-				if (peer)
-					RR->t->incomingPacketDropped(tPtr,ph->packetId,0,peer->identity(),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_UNRECOGNIZED_VERB);
+				RR->t->incomingPacketDropped(tPtr,0xdeadeff0,ph->packetId,0,ifPeerNonNull(peer),path->address(),hops,verb,ZT_TRACE_PACKET_DROP_REASON_UNRECOGNIZED_VERB);
 				break;
 		}
 	} catch ( ... ) {
-		uint64_t packetId = 0;
-		if (len >= 8) {
-			for(int i=0;i<8;++i)
-				reinterpret_cast<uint8_t *>(&packetId)[i] = data->b[i];
-		}
-		RR->t->incomingPacketDropped(tPtr,packetId,0,Identity(),path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+		RR->t->unexpectedError(tPtr,0xea1b6dea,"unexpected exception in onRemotePacket()");
 	}
 }
 
@@ -366,15 +382,59 @@ void VL1::_relay(void *tPtr,const SharedPtr<Path> &path,const Address &destinati
 {
 }
 
-void VL1::_sendPendingWhois()
+void VL1::_sendPendingWhois(void *const tPtr,const int64_t now)
 {
-	// assume _whoisQueue_l locked
+	SharedPtr<Peer> root(RR->topology->root());
+	if (!root)
+		return;
+	SharedPtr<Path> rootPath(root->path(now));
+	if (!rootPath)
+		return;
+
+	std::vector<Address> toSend;
+	{
+		Mutex::Lock wl(_whoisQueue_l);
+		Hashtable<Address,_WhoisQueueItem>::Iterator wi(_whoisQueue);
+		Address *a = nullptr;
+		_WhoisQueueItem *wq = nullptr;
+		while (wi.next(a,wq)) {
+			if ((now - wq->lastRetry) >= ZT_WHOIS_RETRY_DELAY) {
+				wq->lastRetry = now;
+				++wq->retries;
+				toSend.push_back(*a);
+			}
+		}
+	}
+
+	Buf outp;
+	Protocol::Header &ph = outp.as<Protocol::Header>();
+
+	std::vector<Address>::iterator a(toSend.begin());
+	while (a != toSend.end()) {
+		ph.packetId = Protocol::getPacketId();
+		root->address().copyTo(ph.destination);
+		RR->identity.address().copyTo(ph.source);
+		ph.flags = 0;
+		ph.verb = Protocol::VERB_OK;
+
+		int ptr = sizeof(Protocol::Header);
+		while ((a != toSend.end())&&(ptr < (ZT_PROTO_MAX_PACKET_LENGTH - 1))) {
+			a->copyTo(outp.b + ptr);
+			++a;
+			ptr += ZT_ADDRESS_LENGTH;
+		}
+
+		if (ptr > sizeof(Protocol::Header)) {
+			Protocol::armor(outp,ptr,root->key(),ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012);
+			rootPath->send(RR,tPtr,outp.b,ptr,now);
+		}
+	}
 }
 
 void VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize,bool authenticated)
 {
 	if (packetSize < sizeof(Protocol::HELLO)) {
-		RR->t->incomingPacketDropped(tPtr,0,0,Identity(),path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+		RR->t->incomingPacketDropped(tPtr,0x2bdb0001,0,0,ifPeerNonNull(peer),path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 		return;
 	}
 
@@ -383,20 +443,22 @@ void VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 	int ptr = sizeof(Protocol::HELLO);
 
 	if (p.versionProtocol < ZT_PROTO_VERSION_MIN) {
-		RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,Identity(),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_PEER_TOO_OLD);
+		RR->t->incomingPacketDropped(tPtr,0xe8d12bad,p.h.packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_PEER_TOO_OLD);
 		return;
 	}
 
 	Identity id;
 	if (pkt.rO(ptr,id) < 0) {
-		RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,Identity(),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+		RR->t->incomingPacketDropped(tPtr,0x707a9810,p.h.packetId,0,ifPeerNonNull(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
 		return;
 	}
 	if (Address(p.h.source) != id.address()) {
-		RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+		RR->t->incomingPacketDropped(tPtr,0x06aa9ff1,p.h.packetId,0,Identity::NIL,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 		return;
 	}
 
+	// Packet is basically valid and identity unmarshaled ---------------------------------------------------------------
+
 	// Get long-term static key for this node.
 	uint8_t key[ZT_PEER_SECRET_KEY_LENGTH];
 	if ((peer)&&(id == peer->identity())) {
@@ -404,12 +466,12 @@ void VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 	} else {
 		peer.zero();
 		if (!RR->identity.agree(id,key)) {
-			RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			RR->t->incomingPacketDropped(tPtr,0x46db8010,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 			return;
 		}
 	}
 
-	// Verify packet using Poly1305. For v2.x+ HELLOs this will be the first of two MACs for HELLO.
+	// Verify packet using Poly1305, which for v2.x
 	{
 		uint8_t perPacketKey[ZT_PEER_SECRET_KEY_LENGTH];
 		uint8_t macKey[ZT_POLY1305_KEY_LEN];
@@ -418,31 +480,31 @@ void VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 		uint64_t mac[2];
 		poly1305(mac,pkt.b + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,macKey);
 		if (p.h.mac != mac[0]) {
-			RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+			RR->t->incomingPacketDropped(tPtr,0x11bfff81,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 			return;
 		}
 	}
 
+	// Packet has passed Poly1305 verification --------------------------------------------------------------------------
+
 	InetAddress externalSurfaceAddress;
 	Dictionary nodeMetaData;
+	uint8_t hmacKey[ZT_PEER_SECRET_KEY_LENGTH],hmac[ZT_HMACSHA384_LEN];
+	bool hmacAuthenticated = false;
 
 	// Get external surface address if present.
 	if (ptr < packetSize) {
 		if (pkt.rO(ptr,externalSurfaceAddress) < 0) {
-			RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+			RR->t->incomingPacketDropped(tPtr,0xf1000023,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
 			return;
 		}
 	}
 
 	if (ptr < packetSize) {
-		// NOTE to auditors:
-		// Anything after the external surface address is encrypted using Salsa20/12.
-		// The key for this is un-mangled static peer key. This is a bit of a legacy
-		// thing and isn't absolutely necessary, but does help conceal information
-		// that does not need to be in plaintext. This should not be considered
-		// a required part of the security/authentication model since leakage of
-		// this information would have no impact on either authentication or actual
-		// payload encryption.
+		// Everything after this point is encrypted with Salsa20/12. This is only a privacy measure
+		// since there's nothing truly secret in a HELLO packet. It also means that an observer
+		// can't even get ephemeral public keys without first knowing the long term secret key,
+		// adding a little defense in depth.
 		uint8_t iv[8];
 		for(int i=0;i<8;++i) iv[i] = pkt.b[i];
 		iv[7] &= 0xf8U;
@@ -454,44 +516,52 @@ void VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 			const unsigned int dictionarySize = pkt.rI16(ptr);
 			const void *const dictionaryBytes = pkt.b + ptr;
 			if ((ptr += (int)dictionarySize) > packetSize) {
-				RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+				RR->t->incomingPacketDropped(tPtr,0x0d0f0112,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
 				return;
 			}
 
 			ptr += pkt.rI16(ptr); // skip any additional fields, currently always 0
 			if (ptr > packetSize) {
-				RR->t->incomingPacketDropped(tPtr,0,0,Identity(),path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
+				RR->t->incomingPacketDropped(tPtr,0x451f2341,0,p.h.packetId,id,path->address(),0,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_MALFORMED_PACKET);
 				return;
 			}
 
 			if ((ptr + ZT_SHA384_DIGEST_LEN) <= packetSize) {
-				uint8_t hmacKey[ZT_PEER_SECRET_KEY_LENGTH],mac[ZT_HMACSHA384_LEN];
-				KBKDFHMACSHA384(key,ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC,0,0,hmacKey);
-				HMACSHA384(hmacKey,pkt.b + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,mac);
-				if (!Utils::secureEq(pkt.b + ptr,mac,ZT_HMACSHA384_LEN)) {
-					RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,peer->identity(),path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+				KBKDFHMACSHA384(key,ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC,0,0,hmacKey); // iter == 0 for HELLO
+				HMACSHA384(hmacKey,pkt.b + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,packetSize - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,hmac);
+				if (!Utils::secureEq(pkt.b + ptr,hmac,ZT_HMACSHA384_LEN)) {
+					RR->t->incomingPacketDropped(tPtr,0x1000662a,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
 					return;
 				}
+				hmacAuthenticated = true;
 			}
 
 			if (dictionarySize) {
 				if (!nodeMetaData.decode(dictionaryBytes,dictionarySize)) {
-					RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+					RR->t->incomingPacketDropped(tPtr,0x67192344,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
 					return;
 				}
 			}
 		}
 	}
 
+	// v2.x+ peers must include HMAC, older peers don't
+	if ((!hmacAuthenticated)&&(p.versionProtocol >= 11)) {
+		RR->t->incomingPacketDropped(tPtr,0x571feeea,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_NOP,ZT_TRACE_PACKET_DROP_REASON_MAC_FAILED);
+		return;
+	}
+
+	// Packet is fully decoded and has passed full HMAC (if present) ----------------------------------------------------
+
 	const int64_t now = RR->node->now();
 
 	if (!peer) {
 		if (!RR->node->rateGateIdentityVerification(now,path->address())) {
-			RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_RATE_LIMIT_EXCEEDED);
+			RR->t->incomingPacketDropped(tPtr,0xaffa9ff7,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_RATE_LIMIT_EXCEEDED);
 			return;
 		}
 		if (!id.locallyValidate()) {
-			RR->t->incomingPacketDropped(tPtr,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
+			RR->t->incomingPacketDropped(tPtr,0x2ff7a909,p.h.packetId,0,id,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
 			return;
 		}
 		peer.set(new Peer(RR));
@@ -501,24 +571,30 @@ void VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 		peer = RR->topology->add(tPtr,peer);
 	}
 
+	// All validation steps complete, peer learned if not yet known -----------------------------------------------------
+
 	if ((hops == 0)&&(externalSurfaceAddress))
 		RR->sa->iam(tPtr,id,path->localSocket(),path->address(),externalSurfaceAddress,RR->topology->isRoot(id),now);
 
-	// Send OK(HELLO) with an echo of the packet's timestamp and some of the same
-	// information about us: version, sent-to address, etc.
+	std::vector<uint8_t> myNodeMetaDataBin;
+	{
+		Dictionary myNodeMetaData;
+		myNodeMetaData.encode(myNodeMetaDataBin);
+	}
+	if (myNodeMetaDataBin.size() > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check
+		return;
 
-	SharedPtr<Buf> outp(Buf::get());
-	if (!outp) return;
-	Protocol::OK::HELLO &ok = outp->as<Protocol::OK::HELLO>();
+	Buf outp;
+	Protocol::OK::HELLO &ok = outp.as<Protocol::OK::HELLO>();
 
-	ok.h.packetId = Protocol::getPacketId();
-	id.address().copyTo(ok.h.destination);
-	RR->identity.address().copyTo(ok.h.source);
-	ok.h.flags = 0;
-	ok.h.verb = Protocol::VERB_OK;
+	ok.h.h.packetId = Protocol::getPacketId();
+	id.address().copyTo(ok.h.h.destination);
+	RR->identity.address().copyTo(ok.h.h.source);
+	ok.h.h.flags = 0;
+	ok.h.h.verb = Protocol::VERB_OK;
 
-	ok.oh.inReVerb = Protocol::VERB_HELLO;
-	ok.oh.inRePacketId = p.h.packetId;
+	ok.h.inReVerb = Protocol::VERB_HELLO;
+	ok.h.inRePacketId = p.h.packetId;
 
 	ok.timestampEcho = p.timestamp;
 	ok.versionProtocol = ZT_PROTO_VERSION;
@@ -527,33 +603,24 @@ void VL1::_HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Bu
 	ok.versionRev = ZT_CONST_TO_BE_UINT16(ZEROTIER_ONE_VERSION_REVISION);
 
 	int outl = sizeof(Protocol::OK::HELLO);
-	outp->wO(outl,path->address());
+	outp.wO(outl,path->address());
 
-#if 0
-	ZT_GET_NEW_BUF(outp,Protocol::OK::HELLO);
+	if (p.versionProtocol >= 11) {
+		outp.wI(outl,(uint16_t)0); // legacy field, always 0
+		outp.wI(outl,(uint16_t)myNodeMetaDataBin.size());
+		outp.wB(outl,myNodeMetaDataBin.data(),(unsigned int)myNodeMetaDataBin.size());
+		outp.wI(outl,(uint16_t)0); // length of additional fields, currently 0
 
-	outp->data.fields.h.packetId = Protocol::getPacketId();
-	peer->address().copyTo(outp->data.fields.h.destination);
-	RR->identity.address().copyTo(outp->data.fields.h.source);
-	outp->data.fields.h.flags = 0;
-	outp->data.fields.h.verb = Protocol::VERB_OK;
-
-	outp->data.fields.oh.inReVerb = Protocol::VERB_HELLO;
-	outp->data.fields.oh.inRePacketId = p.idBE;
-
-	outp->data.fields.timestampEcho = pkt.data.fields.timestamp;
-	outp->data.fields.versionProtocol = ZT_PROTO_VERSION;
-	outp->data.fields.versionMajor = ZEROTIER_ONE_VERSION_MAJOR;
-	outp->data.fields.versionMinor = ZEROTIER_ONE_VERSION_MINOR;
-	outp->data.fields.versionRev = CONST_TO_BE_UINT16(ZEROTIER_ONE_VERSION_REVISION);
+		if ((outl + ZT_HMACSHA384_LEN) > ZT_PROTO_MAX_PACKET_LENGTH) // sanity check, shouldn't be possible
+			return;
 
-	int outl = sizeof(Protocol::OK::HELLO);
-	outp->wO(outl,p.path->address());
-	if (!Buf<>::writeOverflow(outl)) {
-		Protocol::armor(*outp,outl,peer->key(),ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012);
-		p.path->send(RR,tPtr,outp->data.bytes,outl,RR->node->now());
+		KBKDFHMACSHA384(key,ZT_PROTO_KDF_KEY_LABEL_HELLO_HMAC,0,1,hmacKey); // iter == 1 for OK
+		HMACSHA384(hmacKey,outp.b + sizeof(ok.h),outl - sizeof(ok.h),outp.b + outl);
+		outl += ZT_HMACSHA384_LEN;
 	}
-#endif
+
+	Protocol::armor(outp,outl,peer->key(),ZT_PROTO_CIPHER_SUITE__POLY1305_SALSA2012);
+	path->send(RR,tPtr,outp.b,outl,now);
 
 	peer->setRemoteVersion(p.versionProtocol,p.versionMajor,p.versionMinor,Utils::ntoh(p.versionRev));
 	peer->received(tPtr,path,hops,p.h.packetId,packetSize - ZT_PROTO_PACKET_PAYLOAD_START,Protocol::VERB_HELLO,0,Protocol::VERB_NOP,0);

node/VL1.hpp

@@ -53,7 +53,7 @@ public:
 
 private:
 	void _relay(void *tPtr,const SharedPtr<Path> &path,const Address &destination,SharedPtr<Buf> &data,unsigned int len);
-	void _sendPendingWhois();
+	void _sendPendingWhois(void *tPtr,int64_t now);
 
 	// Handlers for VL1 verbs
 	void _HELLO(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize,bool authenticated);
@@ -67,7 +67,6 @@ private:
 	void _ENCAP(void *tPtr,const SharedPtr<Path> &path,const SharedPtr<Peer> &peer,Buf &pkt,int packetSize,bool authenticated);
 
 	const RuntimeEnvironment *RR;
-	VL2 *const _vl2;
 
 	struct _WhoisQueueItem
 	{

node/VL2.cpp

@@ -13,10 +13,13 @@
 
 #include "VL2.hpp"
 #include "RuntimeEnvironment.hpp"
+#include "Node.hpp"
 #include "VL1.hpp"
 #include "Topology.hpp"
 #include "Peer.hpp"
 #include "Path.hpp"
+#include "Network.hpp"
+#include "MAC.hpp"
 
 namespace ZeroTier {
 
@@ -28,6 +31,10 @@ VL2::~VL2()
 {
 }
 
+void VL2::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
+{
+}
+
 void VL2::_FRAME(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize,bool authenticated)
 {
 }

node/VL2.hpp

@@ -28,6 +28,8 @@ class Path;
 class Peer;
 class RuntimeEnvironment;
 class VL1;
+class Network;
+class MAC;
 
 class VL2
 {
@@ -37,6 +39,20 @@ public:
 	VL2(const RuntimeEnvironment *renv);
 	~VL2();
 
+	/**
+	 * Called when a packet comes from a local Ethernet tap
+	 *
+	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
+	 * @param network Which network's TAP did this packet come from?
+	 * @param from Originating MAC address
+	 * @param to Destination MAC address
+	 * @param etherType Ethernet packet type
+	 * @param vlanId VLAN ID or 0 if none
+	 * @param data Ethernet payload
+	 * @param len Frame length
+	 */
+	void onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
+
 protected:
 	void _FRAME(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize,bool authenticated);
 	void _EXT_FRAME(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize,bool authenticated);