Merge branch 'adamierymenko-dev' into netcon

controller/SqliteNetworkController.cpp

@@ -44,12 +44,15 @@
 #include "../ext/json-parser/json.h"
 
 #include "SqliteNetworkController.hpp"
+
+#include "../node/Node.hpp"
 #include "../node/Utils.hpp"
 #include "../node/CertificateOfMembership.hpp"
 #include "../node/NetworkConfig.hpp"
 #include "../node/InetAddress.hpp"
 #include "../node/MAC.hpp"
 #include "../node/Address.hpp"
+
 #include "../osdep/OSUtils.hpp"
 
 // Include ZT_NETCONF_SCHEMA_SQL constant to init database
@@ -117,8 +120,10 @@ struct NetworkRecord {
 
 } // anonymous namespace
 
-SqliteNetworkController::SqliteNetworkController(const char *dbPath) :
+SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath) :
+	_node(node),
 	_dbPath(dbPath),
+	_circuitTestPath(circuitTestPath),
 	_db((sqlite3 *)0)
 {
 	if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK)
@@ -1742,6 +1747,8 @@ NetworkController::ResultCode SqliteNetworkController::_doNetworkConfigRequest(c
 					for(uint32_t k=ipRangeStart,l=0;(k<=ipRangeEnd)&&(l < 1000000);++k,++l) {
 						uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart;
 						++ipTrialCounter;
+						if ((ip & 0x000000ff) == 0x000000ff)
+							continue; // don't allow addresses that end in .255
 
 						for(std::vector< std::pair<uint32_t,int> >::const_iterator r(routedNetworks.begin());r!=routedNetworks.end();++r) {
 							if ((ip & (0xffffffff << (32 - r->second))) == r->first) {

controller/SqliteNetworkController.hpp

@@ -45,10 +45,12 @@
 
 namespace ZeroTier {
 
+class Node;
+
 class SqliteNetworkController : public NetworkController
 {
 public:
-	SqliteNetworkController(const char *dbPath);
+	SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath);
 	virtual ~SqliteNetworkController();
 
 	virtual NetworkController::ResultCode doNetworkConfigRequest(
@@ -104,7 +106,9 @@ private:
 		const Dictionary &metaData,
 		Dictionary &netconf);
 
+	Node *_node;
 	std::string _dbPath;
+	std::string _circuitTestPath;
 	std::string _instanceId;
 
 	// A circular buffer last log

java/CMakeLists.txt

@@ -53,8 +53,8 @@ set(src_files
     ../osdep/Http.cpp
     ../osdep/OSUtils.cpp
     jni/com_zerotierone_sdk_Node.cpp
-    jni/ZT1_jniutils.cpp
-    jni/ZT1_jnicache.cpp
+    jni/ZT_jniutils.cpp
+    jni/ZT_jnicache.cpp
     )
 
 set(include_dirs

java/jni/Android.mk

@@ -38,7 +38,7 @@ LOCAL_SRC_FILES := \
 # JNI Files
 LOCAL_SRC_FILES += \
 	com_zerotierone_sdk_Node.cpp \
-	ZT1_jniutils.cpp \
-	ZT1_jnicache.cpp
+	ZT_jniutils.cpp \
+	ZT_jnicache.cpp
 
 include $(BUILD_SHARED_LIBRARY)

java/jni/ZT1_jnicache.cpp

@@ -25,8 +25,8 @@
  * LLC. Start here: http://www.zerotier.com/
  */
 
-#include "ZT1_jnicache.h"
-#include "ZT1_jniutils.h"
+#include "ZT_jnicache.h"
+#include "ZT_jniutils.h"
 
 JniCache::JniCache()
     : m_jvm(NULL)

java/jni/ZT1_jnicache.h

@@ -25,8 +25,8 @@
  * LLC. Start here: http://www.zerotier.com/
  */
 
-#ifndef ZT1_JNICACHE_H_
-#define ZT1_JNICACHE_H_
+#ifndef ZT_JNICACHE_H_
+#define ZT_JNICACHE_H_
 
 #include <jni.h>
 #include <map>

java/jni/ZT1_jniutils.cpp

@@ -1,5 +1,5 @@
-#include "ZT1_jniutils.h"
-#include "ZT1_jnicache.h"
+#include "ZT_jniutils.h"
+#include "ZT_jnicache.h"
 #include <string>
 #include <assert.h>
 
@@ -9,7 +9,7 @@ extern JniCache cache;
 extern "C" {
 #endif
 
-jobject createResultObject(JNIEnv *env, ZT1_ResultCode code)
+jobject createResultObject(JNIEnv *env, ZT_ResultCode code)
 {
     jclass resultClass = NULL;
     
@@ -25,23 +25,23 @@ jobject createResultObject(JNIEnv *env, ZT1_ResultCode code)
     std::string fieldName;
     switch(code)
     {
-    case ZT1_RESULT_OK:
-        LOGV("ZT1_RESULT_OK");
+    case ZT_RESULT_OK:
+        LOGV("ZT_RESULT_OK");
         fieldName = "RESULT_OK";
         break;
-    case ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY:
-        LOGV("ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY");
+    case ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY:
+        LOGV("ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY");
         fieldName = "RESULT_FATAL_ERROR_OUT_OF_MEMORY";
         break;
-    case ZT1_RESULT_FATAL_ERROR_DATA_STORE_FAILED:
+    case ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED:
         LOGV("RESULT_FATAL_ERROR_DATA_STORE_FAILED");
         fieldName = "RESULT_FATAL_ERROR_DATA_STORE_FAILED";
         break;
-    case ZT1_RESULT_ERROR_NETWORK_NOT_FOUND:
+    case ZT_RESULT_ERROR_NETWORK_NOT_FOUND:
         LOGV("RESULT_FATAL_ERROR_DATA_STORE_FAILED");
         fieldName = "RESULT_ERROR_NETWORK_NOT_FOUND";
         break;
-    case ZT1_RESULT_FATAL_ERROR_INTERNAL:
+    case ZT_RESULT_FATAL_ERROR_INTERNAL:
     default:
         LOGV("RESULT_FATAL_ERROR_DATA_STORE_FAILED");
         fieldName = "RESULT_FATAL_ERROR_INTERNAL";
@@ -64,7 +64,7 @@ jobject createResultObject(JNIEnv *env, ZT1_ResultCode code)
 }
 
 
-jobject createVirtualNetworkStatus(JNIEnv *env, ZT1_VirtualNetworkStatus status)
+jobject createVirtualNetworkStatus(JNIEnv *env, ZT_VirtualNetworkStatus status)
 {
     jobject statusObject = NULL;
 
@@ -77,22 +77,22 @@ jobject createVirtualNetworkStatus(JNIEnv *env, ZT1_VirtualNetworkStatus status)
     std::string fieldName;
     switch(status)
     {
-    case ZT1_NETWORK_STATUS_REQUESTING_CONFIGURATION:
+    case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION:
         fieldName = "NETWORK_STATUS_REQUESTING_CONFIGURATION";
         break;
-    case ZT1_NETWORK_STATUS_OK:
+    case ZT_NETWORK_STATUS_OK:
         fieldName = "NETWORK_STATUS_OK";
         break;
-    case ZT1_NETWORK_STATUS_ACCESS_DENIED:
+    case ZT_NETWORK_STATUS_ACCESS_DENIED:
         fieldName = "NETWORK_STATUS_ACCESS_DENIED";
         break;
-    case ZT1_NETWORK_STATUS_NOT_FOUND:
+    case ZT_NETWORK_STATUS_NOT_FOUND:
         fieldName = "NETWORK_STATUS_NOT_FOUND";
         break;
-    case ZT1_NETWORK_STATUS_PORT_ERROR:
+    case ZT_NETWORK_STATUS_PORT_ERROR:
         fieldName = "NETWORK_STATUS_PORT_ERROR";
         break;
-    case ZT1_NETWORK_STATUS_CLIENT_TOO_OLD:
+    case ZT_NETWORK_STATUS_CLIENT_TOO_OLD:
         fieldName = "NETWORK_STATUS_CLIENT_TOO_OLD";
         break;
     }
@@ -104,7 +104,7 @@ jobject createVirtualNetworkStatus(JNIEnv *env, ZT1_VirtualNetworkStatus status)
     return statusObject;
 }
 
-jobject createEvent(JNIEnv *env, ZT1_Event event)
+jobject createEvent(JNIEnv *env, ZT_Event event)
 {
     jclass eventClass = NULL;
     jobject eventObject = NULL;
@@ -118,31 +118,31 @@ jobject createEvent(JNIEnv *env, ZT1_Event event)
     std::string fieldName;
     switch(event)
     {
-    case ZT1_EVENT_UP:
+    case ZT_EVENT_UP:
         fieldName = "EVENT_UP";
         break;
-    case ZT1_EVENT_OFFLINE:
+    case ZT_EVENT_OFFLINE:
         fieldName = "EVENT_OFFLINE";
         break;
-    case ZT1_EVENT_ONLINE:
+    case ZT_EVENT_ONLINE:
         fieldName = "EVENT_ONLINE";
         break;
-    case ZT1_EVENT_DOWN:
+    case ZT_EVENT_DOWN:
         fieldName = "EVENT_DOWN";
         break;
-    case ZT1_EVENT_FATAL_ERROR_IDENTITY_COLLISION:
+    case ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION:
         fieldName = "EVENT_FATAL_ERROR_IDENTITY_COLLISION";
         break;
-    case ZT1_EVENT_SAW_MORE_RECENT_VERSION:
+    case ZT_EVENT_SAW_MORE_RECENT_VERSION:
         fieldName = "EVENT_SAW_MORE_RECENT_VERSION";
         break;
-    case ZT1_EVENT_AUTHENTICATION_FAILURE:
+    case ZT_EVENT_AUTHENTICATION_FAILURE:
         fieldName = "EVENT_AUTHENTICATION_FAILURE";
         break;
-    case ZT1_EVENT_INVALID_PACKET:
+    case ZT_EVENT_INVALID_PACKET:
         fieldName = "EVENT_INVALID_PACKET";
         break;
-    case ZT1_EVENT_TRACE:
+    case ZT_EVENT_TRACE:
         fieldName = "EVENT_TRACE";
         break;
     }
@@ -154,7 +154,7 @@ jobject createEvent(JNIEnv *env, ZT1_Event event)
     return eventObject;
 }
 
-jobject createPeerRole(JNIEnv *env, ZT1_PeerRole role)
+jobject createPeerRole(JNIEnv *env, ZT_PeerRole role)
 {
     jclass peerRoleClass = NULL;
     jobject peerRoleObject = NULL;
@@ -168,13 +168,13 @@ jobject createPeerRole(JNIEnv *env, ZT1_PeerRole role)
     std::string fieldName;
     switch(role)
     {
-    case ZT1_PEER_ROLE_LEAF:
+    case ZT_PEER_ROLE_LEAF:
         fieldName = "PEER_ROLE_LEAF";
         break;
-    case ZT1_PEER_ROLE_HUB:
+    case ZT_PEER_ROLE_HUB:
         fieldName = "PEER_ROLE_HUB";
         break;
-    case ZT1_PEER_ROLE_ROOTSERVER:
+    case ZT_PEER_ROLE_ROOTSERVER:
         fieldName = "PEER_ROLE_ROOTSERVER";
         break;
     }
@@ -186,7 +186,7 @@ jobject createPeerRole(JNIEnv *env, ZT1_PeerRole role)
     return peerRoleObject;
 }
 
-jobject createVirtualNetworkType(JNIEnv *env, ZT1_VirtualNetworkType type)
+jobject createVirtualNetworkType(JNIEnv *env, ZT_VirtualNetworkType type)
 {
     jclass vntypeClass = NULL;
     jobject vntypeObject = NULL;
@@ -200,10 +200,10 @@ jobject createVirtualNetworkType(JNIEnv *env, ZT1_VirtualNetworkType type)
     std::string fieldName;
     switch(type)
     {
-    case ZT1_NETWORK_TYPE_PRIVATE:
+    case ZT_NETWORK_TYPE_PRIVATE:
         fieldName = "NETWORK_TYPE_PRIVATE";
         break;
-    case ZT1_NETWORK_TYPE_PUBLIC:
+    case ZT_NETWORK_TYPE_PUBLIC:
         fieldName = "NETWORK_TYPE_PUBLIC";
         break;
     }
@@ -213,7 +213,7 @@ jobject createVirtualNetworkType(JNIEnv *env, ZT1_VirtualNetworkType type)
     return vntypeObject;
 }
 
-jobject createVirtualNetworkConfigOperation(JNIEnv *env, ZT1_VirtualNetworkConfigOperation op)
+jobject createVirtualNetworkConfigOperation(JNIEnv *env, ZT_VirtualNetworkConfigOperation op)
 {
     jclass vnetConfigOpClass = NULL;
     jobject vnetConfigOpObject = NULL;
@@ -227,16 +227,16 @@ jobject createVirtualNetworkConfigOperation(JNIEnv *env, ZT1_VirtualNetworkConfi
     std::string fieldName;
     switch(op)
     {
-    case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_UP:
+    case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP:
         fieldName = "VIRTUAL_NETWORK_CONFIG_OPERATION_UP";
         break;
-    case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE:
+    case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE:
         fieldName = "VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE";
         break;
-    case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN:
+    case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN:
         fieldName = "VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN";
         break;
-    case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY:
+    case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY:
         fieldName = "VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY";
         break;
     }
@@ -372,7 +372,7 @@ jobject newInetSocketAddress(JNIEnv *env, const sockaddr_storage &addr)
     return inetSocketAddressObject;
 }
 
-jobject newMulticastGroup(JNIEnv *env, const ZT1_MulticastGroup &mc)
+jobject newMulticastGroup(JNIEnv *env, const ZT_MulticastGroup &mc)
 {
     jclass multicastGroupClass = NULL;
     jmethodID multicastGroup_constructor = NULL;
@@ -417,7 +417,7 @@ jobject newMulticastGroup(JNIEnv *env, const ZT1_MulticastGroup &mc)
     return multicastGroupObj;
 }
 
-jobject newPeerPhysicalPath(JNIEnv *env, const ZT1_PeerPhysicalPath &ppp)
+jobject newPeerPhysicalPath(JNIEnv *env, const ZT_PeerPhysicalPath &ppp)
 {
     LOGV("newPeerPhysicalPath Called");
     jclass pppClass = NULL;
@@ -514,7 +514,7 @@ jobject newPeerPhysicalPath(JNIEnv *env, const ZT1_PeerPhysicalPath &ppp)
     return pppObject;
 }
 
-jobject newPeer(JNIEnv *env, const ZT1_Peer &peer) 
+jobject newPeer(JNIEnv *env, const ZT_Peer &peer) 
 {
     LOGV("newPeer called");
 
@@ -656,7 +656,7 @@ jobject newPeer(JNIEnv *env, const ZT1_Peer &peer)
     return peerObject;
 }
 
-jobject newNetworkConfig(JNIEnv *env, const ZT1_VirtualNetworkConfig &vnetConfig)
+jobject newNetworkConfig(JNIEnv *env, const ZT_VirtualNetworkConfig &vnetConfig)
 {
     jclass vnetConfigClass = NULL;
     jmethodID vnetConfig_constructor = NULL;

java/jni/ZT1_jniutils.h

@@ -1,5 +1,5 @@
-#ifndef ZT1_jniutils_h_
-#define ZT1_jniutils_h_
+#ifndef ZT_jniutils_h_
+#define ZT_jniutils_h_
 #include <stdio.h>
 #include <jni.h>
 #include <ZeroTierOne.h>
@@ -23,22 +23,22 @@ extern "C" {
 #define LOGE(...) fprintf(stdout, __VA_ARGS__)
 #endif
 
-jobject createResultObject(JNIEnv *env, ZT1_ResultCode code);
-jobject createVirtualNetworkStatus(JNIEnv *env, ZT1_VirtualNetworkStatus status);
-jobject createVirtualNetworkType(JNIEnv *env, ZT1_VirtualNetworkType type);
-jobject createEvent(JNIEnv *env, ZT1_Event event);
-jobject createPeerRole(JNIEnv *env, ZT1_PeerRole role);
-jobject createVirtualNetworkConfigOperation(JNIEnv *env, ZT1_VirtualNetworkConfigOperation op);
+jobject createResultObject(JNIEnv *env, ZT_ResultCode code);
+jobject createVirtualNetworkStatus(JNIEnv *env, ZT_VirtualNetworkStatus status);
+jobject createVirtualNetworkType(JNIEnv *env, ZT_VirtualNetworkType type);
+jobject createEvent(JNIEnv *env, ZT_Event event);
+jobject createPeerRole(JNIEnv *env, ZT_PeerRole role);
+jobject createVirtualNetworkConfigOperation(JNIEnv *env, ZT_VirtualNetworkConfigOperation op);
 
 jobject newInetSocketAddress(JNIEnv *env, const sockaddr_storage &addr);
 jobject newInetAddress(JNIEnv *env, const sockaddr_storage &addr);
 
-jobject newMulticastGroup(JNIEnv *env, const ZT1_MulticastGroup &mc);
+jobject newMulticastGroup(JNIEnv *env, const ZT_MulticastGroup &mc);
 
-jobject newPeer(JNIEnv *env, const ZT1_Peer &peer);
-jobject newPeerPhysicalPath(JNIEnv *env, const ZT1_PeerPhysicalPath &ppp);
+jobject newPeer(JNIEnv *env, const ZT_Peer &peer);
+jobject newPeerPhysicalPath(JNIEnv *env, const ZT_PeerPhysicalPath &ppp);
 
-jobject newNetworkConfig(JNIEnv *env, const ZT1_VirtualNetworkConfig &config);
+jobject newNetworkConfig(JNIEnv *env, const ZT_VirtualNetworkConfig &config);
 
 jobject newVersion(JNIEnv *env, int major, int minor, int rev, long featureFlags);
 

java/jni/com_zerotierone_sdk_Node.cpp

@@ -26,8 +26,8 @@
  */
 
 #include "com_zerotierone_sdk_Node.h"
-#include "ZT1_jniutils.h"
-#include "ZT1_jnicache.h"
+#include "ZT_jniutils.h"
+#include "ZT_jnicache.h"
 
 #include <ZeroTierOne.h>
 
@@ -74,7 +74,7 @@ namespace {
 
         JavaVM *jvm;
 
-        ZT1_Node *node;
+        ZT_Node *node;
 
         jobject dataStoreGetListener;
         jobject dataStorePutListener;
@@ -86,11 +86,11 @@ namespace {
 
 
     int VirtualNetworkConfigFunctionCallback(
-        ZT1_Node *node,
+        ZT_Node *node,
         void *userData,
         uint64_t nwid,
-        enum ZT1_VirtualNetworkConfigOperation operation,
-        const ZT1_VirtualNetworkConfig *config)
+        enum ZT_VirtualNetworkConfigOperation operation,
+        const ZT_VirtualNetworkConfig *config)
     {
         LOGD("VritualNetworkConfigFunctionCallback");
         JniRef *ref = (JniRef*)userData;
@@ -133,7 +133,7 @@ namespace {
             (jlong)nwid, operationObject, networkConfigObject);
     }
 
-    void VirtualNetworkFrameFunctionCallback(ZT1_Node *node,void *userData,
+    void VirtualNetworkFrameFunctionCallback(ZT_Node *node,void *userData,
         uint64_t nwid,
         uint64_t sourceMac,
         uint64_t destMac,
@@ -186,7 +186,7 @@ namespace {
     }
 
 
-    void EventCallback(ZT1_Node *node,void *userData,enum ZT1_Event event, const void *data)
+    void EventCallback(ZT_Node *node,void *userData,enum ZT_Event event, const void *data)
     {
         LOGD("EventCallback");
         JniRef *ref = (JniRef*)userData;
@@ -245,18 +245,18 @@ namespace {
 
         switch(event)
         {
-        case ZT1_EVENT_UP:
-        case ZT1_EVENT_OFFLINE:
-        case ZT1_EVENT_ONLINE:
-        case ZT1_EVENT_DOWN:
-        case ZT1_EVENT_FATAL_ERROR_IDENTITY_COLLISION:
+        case ZT_EVENT_UP:
+        case ZT_EVENT_OFFLINE:
+        case ZT_EVENT_ONLINE:
+        case ZT_EVENT_DOWN:
+        case ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION:
         {
             LOGV("Regular Event");
             // call onEvent()
             env->CallVoidMethod(ref->eventListener, onEventMethod, eventObject);
         }
         break;
-        case ZT1_EVENT_SAW_MORE_RECENT_VERSION:
+        case ZT_EVENT_SAW_MORE_RECENT_VERSION:
         {
             LOGV("Version Event");
             // call onOutOfDate()
@@ -268,8 +268,8 @@ namespace {
             }
         }
         break;
-        case ZT1_EVENT_AUTHENTICATION_FAILURE:
-        case ZT1_EVENT_INVALID_PACKET:
+        case ZT_EVENT_AUTHENTICATION_FAILURE:
+        case ZT_EVENT_INVALID_PACKET:
         {
             LOGV("Network Error Event");
             // call onNetworkError()
@@ -281,7 +281,7 @@ namespace {
             }
         }
         break;
-        case ZT1_EVENT_TRACE:
+        case ZT_EVENT_TRACE:
         {
             LOGV("Trace Event");
             // call onTrace()
@@ -296,7 +296,7 @@ namespace {
         }
     }
 
-    long DataStoreGetFunction(ZT1_Node *node,void *userData,
+    long DataStoreGetFunction(ZT_Node *node,void *userData,
         const char *objectName,
         void *buffer,
         unsigned long bufferSize,
@@ -368,7 +368,7 @@ namespace {
         return retval;
     }
 
-    int DataStorePutFunction(ZT1_Node *node,void *userData,
+    int DataStorePutFunction(ZT_Node *node,void *userData,
         const char *objectName,
         const void *buffer,
         unsigned long bufferSize,
@@ -426,7 +426,7 @@ namespace {
         }
     }
 
-    int WirePacketSendFunction(ZT1_Node *node,void *userData,\
+    int WirePacketSendFunction(ZT_Node *node,void *userData,\
         const struct sockaddr_storage *address,
         const void *buffer,
         unsigned int bufferSize)
@@ -466,7 +466,7 @@ namespace {
     typedef std::map<uint64_t, JniRef*> NodeMap;
     static NodeMap nodeMap;
 
-    ZT1_Node* findNode(uint64_t nodeId)
+    ZT_Node* findNode(uint64_t nodeId)
     {
         NodeMap::iterator found = nodeMap.find(nodeId);
         if(found != nodeMap.end())
@@ -498,10 +498,10 @@ JNIEXPORT void JNICALL JNI_OnUnload(JavaVM *vm, void *reserved)
 JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_node_1init(
     JNIEnv *env, jobject obj, jlong now)
 {
-    LOGV("Creating ZT1_Node struct");
-    jobject resultObject = createResultObject(env, ZT1_RESULT_OK);
+    LOGV("Creating ZT_Node struct");
+    jobject resultObject = createResultObject(env, ZT_RESULT_OK);
 
-    ZT1_Node *node;
+    ZT_Node *node;
     JniRef *ref = new JniRef;
     ref->id = (uint64_t)now;
     env->GetJavaVM(&ref->jvm);
@@ -593,7 +593,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_node_1init(
     }
     ref->eventListener = env->NewGlobalRef(tmp);
 
-    ZT1_ResultCode rc = ZT1_Node_new(
+    ZT_ResultCode rc = ZT_Node_new(
         &node,
         ref,
         (uint64_t)now,
@@ -604,13 +604,13 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_node_1init(
         &VirtualNetworkConfigFunctionCallback,
         &EventCallback);
 
-    if(rc != ZT1_RESULT_OK)
+    if(rc != ZT_RESULT_OK)
     {
         LOGE("Error creating Node: %d", rc);
         resultObject = createResultObject(env, rc);
         if(node)
         {
-            ZT1_Node_delete(node);
+            ZT_Node_delete(node);
             node = NULL;
         }
         delete ref;
@@ -632,7 +632,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_node_1init(
 JNIEXPORT void JNICALL Java_com_zerotier_sdk_Node_node_1delete(
     JNIEnv *env, jobject obj, jlong id)
 {
-    LOGV("Destroying ZT1_Node struct");
+    LOGV("Destroying ZT_Node struct");
     uint64_t nodeId = (uint64_t)id;
 
     NodeMap::iterator found = nodeMap.find(nodeId);
@@ -641,7 +641,7 @@ JNIEXPORT void JNICALL Java_com_zerotier_sdk_Node_node_1delete(
         JniRef *ref = found->second;
         nodeMap.erase(found);
 
-        ZT1_Node_delete(ref->node);
+        ZT_Node_delete(ref->node);
 
         delete ref;
         ref = NULL;
@@ -671,18 +671,18 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processVirtualNetworkFrame(
 {
     uint64_t nodeId = (uint64_t) id;
     
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     unsigned int nbtd_len = env->GetArrayLength(out_nextBackgroundTaskDeadline);
     if(nbtd_len < 1)
     {
         // array for next background task length has 0 elements!
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     uint64_t now = (uint64_t)in_now;
@@ -697,7 +697,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processVirtualNetworkFrame(
 
     uint64_t nextBackgroundTaskDeadline = 0;
 
-    ZT1_ResultCode rc = ZT1_Node_processVirtualNetworkFrame(
+    ZT_ResultCode rc = ZT_Node_processVirtualNetworkFrame(
         node,
         now,
         nwid,
@@ -732,17 +732,17 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
     jlongArray out_nextBackgroundTaskDeadline)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     unsigned int nbtd_len = env->GetArrayLength(out_nextBackgroundTaskDeadline);
     if(nbtd_len < 1)
     {
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     uint64_t now = (uint64_t)in_now;
@@ -752,7 +752,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
     if(inetAddressClass == NULL)
     {
         // can't find java.net.InetAddress
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     jmethodID getAddressMethod = cache.findMethod(
@@ -760,13 +760,13 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
     if(getAddressMethod == NULL)
     {
         // cant find InetAddress.getAddres()
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     jclass InetSocketAddressClass = cache.findClass("java/net/InetSocketAddress");
     if(InetSocketAddressClass == NULL)
     {
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     jmethodID inetSockGetAddressMethod = cache.findMethod(
@@ -776,7 +776,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
 
     if(addrObject == NULL)
     {
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     jmethodID inetSock_getPort = cache.findMethod(
@@ -785,7 +785,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
     if(env->ExceptionCheck() || inetSock_getPort == NULL) 
     {
         LOGE("Couldn't find getPort method on InetSocketAddress");
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     // call InetSocketAddress.getPort()
@@ -793,7 +793,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
     if(env->ExceptionCheck())
     {
         LOGE("Exception calling InetSocketAddress.getPort()");
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     // Call InetAddress.getAddress()
@@ -801,7 +801,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
     if(addressArray == NULL)
     {
         // unable to call getAddress()
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     unsigned int addrSize = env->GetArrayLength(addressArray);
@@ -833,7 +833,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
     {
         // unknown address type
         env->ReleaseByteArrayElements(addressArray, addr, 0);
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
 
@@ -842,7 +842,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processWirePacket(
 
     uint64_t nextBackgroundTaskDeadline = 0;
 
-    ZT1_ResultCode rc = ZT1_Node_processWirePacket(
+    ZT_ResultCode rc = ZT_Node_processWirePacket(
         node,
         now,
         &remoteAddress,
@@ -872,23 +872,23 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_processBackgroundTasks(
     jlongArray out_nextBackgroundTaskDeadline)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     unsigned int nbtd_len = env->GetArrayLength(out_nextBackgroundTaskDeadline);
     if(nbtd_len < 1)
     {
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     uint64_t now = (uint64_t)in_now;
     uint64_t nextBackgroundTaskDeadline = 0;
 
-    ZT1_ResultCode rc = ZT1_Node_processBackgroundTasks(node, now, &nextBackgroundTaskDeadline);
+    ZT_ResultCode rc = ZT_Node_processBackgroundTasks(node, now, &nextBackgroundTaskDeadline);
 
     jlong *outDeadline = env->GetLongArrayElements(out_nextBackgroundTaskDeadline, NULL);
     outDeadline[0] = (jlong)nextBackgroundTaskDeadline;
@@ -906,16 +906,16 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_join(
     JNIEnv *env, jobject obj, jlong id, jlong in_nwid)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     uint64_t nwid = (uint64_t)in_nwid;
 
-    ZT1_ResultCode rc = ZT1_Node_join(node, nwid);
+    ZT_ResultCode rc = ZT_Node_join(node, nwid);
 
     return createResultObject(env, rc);
 }
@@ -929,16 +929,16 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_leave(
     JNIEnv *env, jobject obj, jlong id, jlong in_nwid)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     uint64_t nwid = (uint64_t)in_nwid;
 
-    ZT1_ResultCode rc = ZT1_Node_leave(node, nwid);
+    ZT_ResultCode rc = ZT_Node_leave(node, nwid);
 
     return createResultObject(env, rc);
 }
@@ -956,18 +956,18 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_multicastSubscribe(
     jlong in_multicastAdi)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     uint64_t nwid = (uint64_t)in_nwid;
     uint64_t multicastGroup = (uint64_t)in_multicastGroup;
     unsigned long multicastAdi = (unsigned long)in_multicastAdi;
 
-    ZT1_ResultCode rc = ZT1_Node_multicastSubscribe(
+    ZT_ResultCode rc = ZT_Node_multicastSubscribe(
         node, nwid, multicastGroup, multicastAdi);
 
     return createResultObject(env, rc);
@@ -986,18 +986,18 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_multicastUnsubscribe(
     jlong in_multicastAdi)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
-        return createResultObject(env, ZT1_RESULT_FATAL_ERROR_INTERNAL);
+        return createResultObject(env, ZT_RESULT_FATAL_ERROR_INTERNAL);
     }
 
     uint64_t nwid = (uint64_t)in_nwid;
     uint64_t multicastGroup = (uint64_t)in_multicastGroup;
     unsigned long multicastAdi = (unsigned long)in_multicastAdi;
 
-    ZT1_ResultCode rc = ZT1_Node_multicastUnsubscribe(
+    ZT_ResultCode rc = ZT_Node_multicastUnsubscribe(
         node, nwid, multicastGroup, multicastAdi);
 
     return createResultObject(env, rc);
@@ -1012,14 +1012,14 @@ JNIEXPORT jlong JNICALL Java_com_zerotier_sdk_Node_address(
     JNIEnv *env , jobject obj, jlong id)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
         return 0;
     }
 
-    uint64_t address = ZT1_Node_address(node);
+    uint64_t address = ZT_Node_address(node);
     return (jlong)address;
 }
 
@@ -1032,7 +1032,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_status
    (JNIEnv *env, jobject obj, jlong id)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
@@ -1062,8 +1062,8 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_status
         return NULL;
     }
 
-    ZT1_NodeStatus nodeStatus;
-    ZT1_Node_status(node, &nodeStatus);
+    ZT_NodeStatus nodeStatus;
+    ZT_Node_status(node, &nodeStatus);
 
     jfieldID addressField = NULL;
     jfieldID publicIdentityField = NULL;
@@ -1124,18 +1124,18 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_networkConfig(
     JNIEnv *env, jobject obj, jlong id, jlong nwid)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
         return 0;
     }
 
-    ZT1_VirtualNetworkConfig *vnetConfig = ZT1_Node_networkConfig(node, nwid);
+    ZT_VirtualNetworkConfig *vnetConfig = ZT_Node_networkConfig(node, nwid);
     
     jobject vnetConfigObject = newNetworkConfig(env, *vnetConfig);
 
-    ZT1_Node_freeQueryResult(node, vnetConfig);
+    ZT_Node_freeQueryResult(node, vnetConfig);
 
     return vnetConfigObject;
 }
@@ -1153,7 +1153,7 @@ JNIEXPORT jobject JNICALL Java_com_zerotier_sdk_Node_version(
     int revision = 0;
     unsigned long featureFlags = 0;
 
-    ZT1_version(&major, &minor, &revision, &featureFlags);
+    ZT_version(&major, &minor, &revision, &featureFlags);
 
     return newVersion(env, major, minor, revision, featureFlags);
 }
@@ -1167,18 +1167,18 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_peers(
     JNIEnv *env, jobject obj, jlong id)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
         return 0;
     }
 
-    ZT1_PeerList *peerList = ZT1_Node_peers(node);
+    ZT_PeerList *peerList = ZT_Node_peers(node);
     
     if(peerList == NULL)
     {
-        LOGE("ZT1_Node_peers returned NULL");
+        LOGE("ZT_Node_peers returned NULL");
         return NULL;
     }
 
@@ -1187,7 +1187,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_peers(
     if(env->EnsureLocalCapacity(peerCount))
     {
         LOGE("EnsureLocalCapacity failed!!");
-        ZT1_Node_freeQueryResult(node, peerList);
+        ZT_Node_freeQueryResult(node, peerList);
         return NULL;
     }
 
@@ -1195,7 +1195,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_peers(
     if(env->ExceptionCheck() || peerClass == NULL)
     {
         LOGE("Error finding Peer class");
-        ZT1_Node_freeQueryResult(node, peerList);
+        ZT_Node_freeQueryResult(node, peerList);
         return NULL;
     }
 
@@ -1205,7 +1205,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_peers(
     if(env->ExceptionCheck() || peerArrayObj == NULL)
     {
         LOGE("Error creating Peer[] array");
-        ZT1_Node_freeQueryResult(node, peerList);
+        ZT_Node_freeQueryResult(node, peerList);
         return NULL;
     }
 
@@ -1221,7 +1221,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_peers(
         }
     }
 
-    ZT1_Node_freeQueryResult(node, peerList);
+    ZT_Node_freeQueryResult(node, peerList);
     peerList = NULL;
 
     return peerArrayObj;
@@ -1236,14 +1236,14 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_networks(
     JNIEnv *env, jobject obj, jlong id)
 {
     uint64_t nodeId = (uint64_t) id;
-    ZT1_Node *node = findNode(nodeId);
+    ZT_Node *node = findNode(nodeId);
     if(node == NULL)
     {
         // cannot find valid node.  We should  never get here.
         return 0;
     }
 
-    ZT1_VirtualNetworkList *networkList = ZT1_Node_networks(node);
+    ZT_VirtualNetworkList *networkList = ZT_Node_networks(node);
     if(networkList == NULL)
     {
         return NULL;
@@ -1253,7 +1253,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_networks(
     if(env->ExceptionCheck() || vnetConfigClass == NULL)
     {
         LOGE("Error finding VirtualNetworkConfig class");
-        ZT1_Node_freeQueryResult(node, networkList);
+        ZT_Node_freeQueryResult(node, networkList);
         return NULL;
     }
 
@@ -1262,7 +1262,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_networks(
     if(env->ExceptionCheck() || networkListObject == NULL)
     {
         LOGE("Error creating VirtualNetworkConfig[] array");
-        ZT1_Node_freeQueryResult(node, networkList);
+        ZT_Node_freeQueryResult(node, networkList);
         return NULL;
     }
 
@@ -1277,7 +1277,7 @@ JNIEXPORT jobjectArray JNICALL Java_com_zerotier_sdk_Node_networks(
         }
     }
 
-    ZT1_Node_freeQueryResult(node, networkList);
+    ZT_Node_freeQueryResult(node, networkList);
 
     return networkListObject;
 }

java/src/com/zerotier/sdk/VirtualNetworkConfig.java

@@ -33,7 +33,7 @@ import java.net.InetSocketAddress;
 
 public final class VirtualNetworkConfig {
     public static final int MAX_MULTICAST_SUBSCRIPTIONS = 4096;
-    public static final int ZT1_MAX_ZT_ASSIGNED_ADDRESSES = 16;
+    public static final int ZT_MAX_ZT_ASSIGNED_ADDRESSES = 16;
 
     private long nwid;
     private long mac;

make-mac.mk

@@ -58,8 +58,8 @@ ifeq ($(ZT_DEBUG),1)
 	# C25519 in particular is almost UNUSABLE in heavy testing without it.
 ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS)
 else
-	CFLAGS?=-O3 -fstack-protector
-	CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -fvectorize -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS)
+	CFLAGS?=-Ofast -fstack-protector
+	CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS)
 	STRIP=strip
 endif
 

node/Buffer.hpp

@@ -391,6 +391,23 @@ public:
 		::memmove(_b,_b + at,_l -= at);
 	}
 
+	/**
+	 * Erase something from the middle of the buffer
+	 *
+	 * @param start Starting position
+	 * @param length Length of block to erase
+	 * @throw std::out_of_range Position plus length is beyond size of buffer
+	 */
+	inline void erase(const unsigned int at,const unsigned int length)
+		throw(std::out_of_range)
+	{
+		const unsigned int endr = at + length;
+		if (endr > _l)
+			throw std::out_of_range("Buffer: erase() range beyond end of buffer");
+		::memmove(_b + at,_b + endr,_l - endr);
+		_l -= length;
+	}
+
 	/**
 	 * Set buffer data length to zero
 	 */

node/CertificateOfMembership.hpp

@@ -315,78 +315,6 @@ public:
 	 */
 	inline const Address &signedBy() const throw() { return _signedBy; }
 
-	/**
-	 * Serialize to std::string or compatible class
-	 *
-	 * @param b String or other class supporting push_back() and append() like std::string
-	 */
-	template<typename T>
-	inline void serialize2(T &b) const
-	{
-		uint64_t tmp[3];
-		char tmp2[ZT_ADDRESS_LENGTH];
-		b.push_back((char)COM_UINT64_ED25519);
-		b.push_back((char)((_qualifiers.size() >> 8) & 0xff));
-		b.push_back((char)(_qualifiers.size() & 0xff));
-		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
-			tmp[0] = Utils::hton(q->id);
-			tmp[1] = Utils::hton(q->value);
-			tmp[2] = Utils::hton(q->maxDelta);
-			b.append(reinterpret_cast<const char *>(reinterpret_cast<void *>(tmp)),sizeof(tmp));
-		}
-		_signedBy.copyTo(tmp2,ZT_ADDRESS_LENGTH);
-		b.append(tmp2,ZT_ADDRESS_LENGTH);
-		if (_signedBy)
-			b.append((const char *)_signature.data,_signature.size());
-	}
-
-	/**
-	 * Deserialize from std::string::iterator or compatible iterator or char* pointer
-	 *
-	 * @param p Iterator
-	 * @param end End of buffer
-	 */
-	template<typename T>
-	inline void deserialize2(T &p,const T &end)
-	{
-		uint64_t tmp[3];
-		char tmp2[ZT_ADDRESS_LENGTH];
-		unsigned int qcount;
-
-		_qualifiers.clear();
-		_signedBy.zero();
-
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		if (*(p++) != (char)COM_UINT64_ED25519) throw std::invalid_argument("unknown certificate of membership type");
-
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		qcount = (unsigned int)*(p++) << 8;
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		qcount |= (unsigned int)*(p++);
-
-		for(unsigned int i=0;i<qcount;++i) {
-			char *p2 = reinterpret_cast<char *>(reinterpret_cast<void *>(tmp));
-			for(unsigned int j=0;j<sizeof(tmp);++j) {
-				if (p == end) throw std::out_of_range("incomplete certificate of membership");
-				*(p2++) = *(p++);
-			}
-			_qualifiers.push_back(_Qualifier(Utils::ntoh(tmp[0]),Utils::ntoh(tmp[1]),Utils::ntoh(tmp[2])));
-		}
-
-		for(unsigned int j=0;j<ZT_ADDRESS_LENGTH;++j) {
-			if (p == end) throw std::out_of_range("incomplete certificate of membership");
-			tmp2[j] = *(p++);
-		}
-		_signedBy.setTo(tmp2,ZT_ADDRESS_LENGTH);
-
-		if (_signedBy) {
-			for(unsigned int j=0;j<_signature.size();++j) {
-				if (p == end) throw std::out_of_range("incomplete certificate of membership");
-				_signature.data[j] = (unsigned char)*(p++);
-			}
-		}
-	}
-
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b) const
 	{

node/Constants.hpp

@@ -161,7 +161,7 @@
 /**
  * Default MTU used for Ethernet tap device
  */
-#define ZT_IF_MTU ZT1_MAX_MTU
+#define ZT_IF_MTU ZT_MAX_MTU
 
 /**
  * Maximum number of packet fragments we'll support
@@ -324,6 +324,13 @@
  */
 #define ZT_DIRECT_PATH_PUSH_INTERVAL 300000
 
+/**
+ * How long (max) to remember network certificates of membership?
+ *
+ * This only applies to networks we don't belong to.
+ */
+#define ZT_PEER_NETWORK_COM_EXPIRATION 3600000
+
 /**
  * Sanity limit on maximum bridge routes
  *

node/Defaults.cpp

@@ -75,7 +75,7 @@ static inline std::map< Address,Identity > _mkRootTopologyAuth()
 Defaults::Defaults() :
 	defaultRootTopology((const char *)ZT_DEFAULT_ROOT_TOPOLOGY,ZT_DEFAULT_ROOT_TOPOLOGY_LEN),
 	rootTopologyAuthorities(_mkRootTopologyAuth()),
-	v4Broadcast(((uint32_t)0xffffffff),ZT1_DEFAULT_PORT)
+	v4Broadcast(((uint32_t)0xffffffff),ZT_DEFAULT_PORT)
 {
 }
 

node/Dictionary.cpp

@@ -32,6 +32,68 @@
 
 namespace ZeroTier {
 
+Dictionary::iterator Dictionary::find(const std::string &key)
+{
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i;
+	}
+	return end();
+}
+Dictionary::const_iterator Dictionary::find(const std::string &key) const
+{
+	for(const_iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i;
+	}
+	return end();
+}
+
+bool Dictionary::getBoolean(const std::string &key,bool dfl) const
+{
+	const_iterator e(find(key));
+	if (e == end())
+		return dfl;
+	if (e->second.length() < 1)
+		return dfl;
+	switch(e->second[0]) {
+		case '1':
+		case 't':
+		case 'T':
+		case 'y':
+		case 'Y':
+			return true;
+	}
+	return false;
+}
+
+std::string &Dictionary::operator[](const std::string &key)
+{
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i->second;
+	}
+	push_back(std::pair<std::string,std::string>(key,std::string()));
+	std::sort(begin(),end());
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i->second;
+	}
+	return front().second; // should be unreachable!
+}
+
+std::string Dictionary::toString() const
+{
+	std::string s;
+	for(const_iterator kv(begin());kv!=end();++kv) {
+		_appendEsc(kv->first.data(),(unsigned int)kv->first.length(),s);
+		s.push_back('=');
+		_appendEsc(kv->second.data(),(unsigned int)kv->second.length(),s);
+		s.append(ZT_EOL_S);
+	}
+	return s;
+}
+
 void Dictionary::updateFromString(const char *s,unsigned int maxlen)
 {
 	bool escapeState = false;
@@ -80,6 +142,16 @@ void Dictionary::fromString(const char *s,unsigned int maxlen)
 	updateFromString(s,maxlen);
 }
 
+void Dictionary::eraseKey(const std::string &key)
+{
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key) {
+			this->erase(i);
+			return;
+		}
+	}
+}
+
 bool Dictionary::sign(const Identity &id,uint64_t now)
 {
 	try {

node/Dictionary.hpp

@@ -31,8 +31,9 @@
 #include <stdint.h>
 
 #include <string>
-#include <map>
+#include <vector>
 #include <stdexcept>
+#include <algorithm>
 
 #include "Constants.hpp"
 #include "Utils.hpp"
@@ -56,12 +57,12 @@ class Identity;
  *
  * Keys beginning with "~!" are reserved for signature data fields.
  *
- * Note: the signature code depends on std::map<> being sorted, but no
- * other code does. So if the underlying data structure is ever swapped
- * out for an unsorted one, the signature code will have to be updated
- * to sort before composing the string to sign.
+ * It's stored as a simple vector and can be linearly scanned or
+ * binary searched. Dictionaries are only used for very small things
+ * outside the core loop, so this is not a significant performance
+ * issue and it reduces memory use and code footprint.
  */
-class Dictionary : public std::map<std::string,std::string>
+class Dictionary : public std::vector< std::pair<std::string,std::string> >
 {
 public:
 	Dictionary() {}
@@ -77,21 +78,8 @@ public:
 	 */
 	Dictionary(const std::string &s) { fromString(s.c_str(),(unsigned int)s.length()); }
 
-	/**
-	 * Get a key, throwing an exception if it is not present
-	 *
-	 * @param key Key to look up
-	 * @return Reference to value
-	 * @throws std::invalid_argument Key not found
-	 */
-	inline const std::string &get(const std::string &key) const
-		throw(std::invalid_argument)
-	{
-		const_iterator e(find(key));
-		if (e == end())
-			throw std::invalid_argument(std::string("missing required field: ")+key);
-		return e->second;
-	}
+	iterator find(const std::string &key);
+	const_iterator find(const std::string &key) const;
 
 	/**
 	 * Get a key, returning a default if not present
@@ -113,23 +101,7 @@ public:
 	 * @param dfl Default boolean result if key not found or empty (default: false)
 	 * @return Boolean value of key
 	 */
-	inline bool getBoolean(const std::string &key,bool dfl = false) const
-	{
-		const_iterator e(find(key));
-		if (e == end())
-			return dfl;
-		if (e->second.length() < 1)
-			return dfl;
-		switch(e->second[0]) {
-			case '1':
-			case 't':
-			case 'T':
-			case 'y':
-			case 'Y':
-				return true;
-		}
-		return false;
-	}
+	bool getBoolean(const std::string &key,bool dfl = false) const;
 
 	/**
 	 * @param key Key to get
@@ -170,6 +142,8 @@ public:
 		return Utils::strTo64(e->second.c_str());
 	}
 
+	std::string &operator[](const std::string &key);
+
 	/**
 	 * @param key Key to set
 	 * @param value String value
@@ -239,17 +213,7 @@ public:
 	/**
 	 * @return String-serialized dictionary
 	 */
-	inline std::string toString() const
-	{
-		std::string s;
-		for(const_iterator kv(begin());kv!=end();++kv) {
-			_appendEsc(kv->first.data(),(unsigned int)kv->first.length(),s);
-			s.push_back('=');
-			_appendEsc(kv->second.data(),(unsigned int)kv->second.length(),s);
-			s.append(ZT_EOL_S);
-		}
-		return s;
-	}
+	std::string toString() const;
 
 	/**
 	 * Clear and initialize from a string
@@ -278,14 +242,19 @@ public:
 	 */
 	uint64_t signatureTimestamp() const;
 
+	/**
+	 * @param key Key to erase
+	 */
+	void eraseKey(const std::string &key);
+
 	/**
 	 * Remove any signature from this dictionary
 	 */
 	inline void removeSignature()
 	{
-		erase(ZT_DICTIONARY_SIGNATURE);
-		erase(ZT_DICTIONARY_SIGNATURE_IDENTITY);
-		erase(ZT_DICTIONARY_SIGNATURE_TIMESTAMP);
+		eraseKey(ZT_DICTIONARY_SIGNATURE);
+		eraseKey(ZT_DICTIONARY_SIGNATURE_IDENTITY);
+		eraseKey(ZT_DICTIONARY_SIGNATURE_TIMESTAMP);
 	}
 
 	/**
@@ -305,21 +274,6 @@ public:
 	 */
 	bool verify(const Identity &id) const;
 
-  inline bool operator==(const Dictionary &d) const
-  {
-    // std::map::operator== is broken on uclibc++
-    if (size() != d.size())
-      return false;
-    const_iterator a(begin());
-    const_iterator b(d.begin());
-    while (a != end()) {
-      if (*(a++) != *(b++))
-        return false;
-    }
-    return true;
-  }
-  inline bool operator!=(const Dictionary &d) const { return (!(*this == d)); }
-
 private:
 	void _mkSigBuf(std::string &buf) const;
 	static void _appendEsc(const char *data,unsigned int len,std::string &to);

node/Identity.hpp

@@ -220,7 +220,6 @@ public:
 	 */
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b,bool includePrivate = false) const
-		throw(std::out_of_range)
 	{
 		_address.appendTo(b);
 		b.append((unsigned char)IDENTITY_TYPE_C25519);
@@ -245,7 +244,6 @@ public:
 	 */
 	template<unsigned int C>
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
-		throw(std::out_of_range,std::invalid_argument)
 	{
 		delete _privateKey;
 		_privateKey = (C25519::Private *)0;

node/IncomingPacket.cpp

@@ -30,6 +30,7 @@
 #include <stdlib.h>
 
 #include "../version.h"
+#include "../include/ZeroTierOne.h"
 
 #include "Constants.hpp"
 #include "Defaults.hpp"
@@ -69,7 +70,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 			switch(verb()) {
 				//case Packet::VERB_NOP:
 				default: // ignore unknown verbs, but if they pass auth check they are "received"
-					peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),verb(),0,Packet::VERB_NOP);
+					peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),verb(),0,Packet::VERB_NOP);
 					return true;
 				case Packet::VERB_HELLO:                          return _doHELLO(RR);
 				case Packet::VERB_ERROR:                          return _doERROR(RR,peer);
@@ -85,6 +86,8 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 				case Packet::VERB_MULTICAST_GATHER:               return _doMULTICAST_GATHER(RR,peer);
 				case Packet::VERB_MULTICAST_FRAME:                return _doMULTICAST_FRAME(RR,peer);
 				case Packet::VERB_PUSH_DIRECT_PATHS:              return _doPUSH_DIRECT_PATHS(RR,peer);
+				case Packet::VERB_CIRCUIT_TEST:                   return _doCIRCUIT_TEST(RR,peer);
+				case Packet::VERB_CIRCUIT_TEST_REPORT:            return _doCIRCUIT_TEST_REPORT(RR,peer);
 			}
 		} else {
 			RR->sw->requestWhois(source());
@@ -130,7 +133,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 
 			case Packet::ERROR_IDENTITY_COLLISION:
 				if (RR->topology->isRoot(peer->identity()))
-					RR->node->postEvent(ZT1_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
+					RR->node->postEvent(ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION);
 				break;
 
 			case Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE: {
@@ -144,7 +147,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
 						nconf->com().serialize(outp);
 						outp.armor(peer->key(),true);
-						RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					}
 				}
 			}	break;
@@ -165,7 +168,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 			default: break;
 		}
 
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb);
 	} catch (std::exception &ex) {
 		TRACE("dropped ERROR from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -224,20 +227,20 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 				unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
 				if (RR->identity.agree(id,key,ZT_PEER_SECRET_KEY_LENGTH)) {
 					if (dearmor(key)) { // ensure packet is authentic, otherwise drop
-						RR->node->postEvent(ZT1_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
+						RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 						TRACE("rejected HELLO from %s(%s): address already claimed",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 						Packet outp(id.address(),RR->identity.address(),Packet::VERB_ERROR);
 						outp.append((unsigned char)Packet::VERB_HELLO);
 						outp.append(packetId());
 						outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
 						outp.armor(key,true);
-						RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+						RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 					} else {
-						RR->node->postEvent(ZT1_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
+						RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 						TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 					}
 				} else {
-					RR->node->postEvent(ZT1_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
+					RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 					TRACE("rejected HELLO from %s(%s): key agreement failed",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 				}
 
@@ -246,7 +249,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 				// Identity is the same as the one we already have -- check packet integrity
 
 				if (!dearmor(peer->key())) {
-					RR->node->postEvent(ZT1_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
+					RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 					TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 					return true;
 				}
@@ -258,7 +261,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 
 			// Check identity proof of work
 			if (!id.locallyValidate()) {
-				RR->node->postEvent(ZT1_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
+				RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 				TRACE("dropped HELLO from %s(%s): identity invalid",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 				return true;
 			}
@@ -266,7 +269,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 			// Check packet integrity and authentication
 			SharedPtr<Peer> newPeer(new Peer(RR->identity,id));
 			if (!dearmor(newPeer->key())) {
-				RR->node->postEvent(ZT1_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
+				RR->node->postEvent(ZT_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 				TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 				return true;
 			}
@@ -278,7 +281,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 
 		// VALID -- continues here
 
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP);
 		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
 
 		bool trusted = false;
@@ -316,7 +319,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		}
 
 		outp.armor(peer->key(),true);
-		RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 	} catch (std::exception &ex) {
 		TRACE("dropped HELLO from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -419,9 +422,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					// OK(MULTICAST_FRAME) includes certificate of membership update
 					CertificateOfMembership com;
 					offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS);
-					SharedPtr<Network> network(RR->node->network(nwid));
-					if ((network)&&(com.hasRequiredFields()))
-						network->validateAndAddMembershipCertificate(com);
+					peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
 				}
 
 				if ((flags & 0x02) != 0) {
@@ -436,7 +437,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 			default: break;
 		}
 
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb);
 	} catch (std::exception &ex) {
 		TRACE("dropped OK from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -456,7 +457,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append(packetId());
 				queried->identity().serialize(outp,false);
 				outp.armor(peer->key(),true);
-				RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			} else {
 				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
 				outp.append((unsigned char)Packet::VERB_WHOIS);
@@ -464,12 +465,12 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 				outp.append(payload(),ZT_ADDRESS_LENGTH);
 				outp.armor(peer->key(),true);
-				RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			}
 		} else {
 			TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str());
 		}
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -487,8 +488,8 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<
 			if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
 				InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
 				TRACE("RENDEZVOUS from %s says %s might be at %s, starting NAT-t",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
-				peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP);
-				RR->sw->rendezvous(withPeer,_localInterfaceId,atAddr);
+				peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP);
+				RR->sw->rendezvous(withPeer,_localAddress,atAddr);
 			} else {
 				TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
 			}
@@ -509,7 +510,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID)));
 		if (network) {
 			if (size() > ZT_PROTO_VERB_FRAME_IDX_PAYLOAD) {
-				if (!network->isAllowed(peer->address())) {
+				if (!network->isAllowed(peer)) {
 					TRACE("dropped FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
 					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
@@ -525,7 +526,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				RR->node->putFrame(network->id(),MAC(peer->address(),network->id()),network->mac(),etherType,0,field(ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,payloadLen),payloadLen);
 			}
 
-			peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP);
+			peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP);
 		} else {
 			TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
@@ -550,13 +551,11 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				if ((flags & 0x01) != 0) {
 					CertificateOfMembership com;
 					comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
-					if (com.hasRequiredFields()) {
-						if (!network->validateAndAddMembershipCertificate(com))
-							comFailed = true; // technically this check is redundant to isAllowed(), but do it anyway for thoroughness
-					}
+					if (!peer->validateAndSetNetworkMembershipCertificate(RR,network->id(),com))
+						comFailed = true;
 				}
 
-				if ((comFailed)||(!network->isAllowed(peer->address()))) {
+				if ((comFailed)||(!network->isAllowed(peer))) {
 					TRACE("dropped EXT_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),network->id());
 					_sendErrorNeedCertificate(RR,peer,network->id());
 					return true;
@@ -602,7 +601,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				RR->node->putFrame(network->id(),from,to,etherType,0,field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,payloadLen),payloadLen);
 			}
 
-			peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP);
+			peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP);
 		} else {
 			TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
@@ -623,7 +622,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18)
 			RR->mc->add(now,at<uint64_t>(ptr),MulticastGroup(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)),peer->address());
 
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP);
 	} catch (std::exception &ex) {
 		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -640,14 +639,10 @@ bool IncomingPacket::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
 		while (ptr < size()) {
 			ptr += com.deserialize(*this,ptr);
-			if (com.hasRequiredFields()) {
-				SharedPtr<Network> network(RR->node->network(com.networkId()));
-				if (network)
-					network->validateAndAddMembershipCertificate(com);
-			}
+			peer->validateAndSetNetworkMembershipCertificate(RR,com.networkId(),com);
 		}
 
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
 	} catch (std::exception &ex) {
 		TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -666,7 +661,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 
 		const unsigned int h = hops();
 		const uint64_t pid = packetId();
-		peer->received(RR,_localInterfaceId,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP);
 
 		if (RR->localNetworkController) {
 			Dictionary netconf;
@@ -688,7 +683,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 						if (outp.size() > ZT_PROTO_MAX_PACKET_LENGTH) {
 							TRACE("NETWORK_CONFIG_REQUEST failed: internal error: netconf size %u is too large",(unsigned int)netconfStr.length());
 						} else {
-							RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+							RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 						}
 					}
 				}	break;
@@ -700,7 +695,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
-					RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				}	break;
 
 				case NetworkController::NETCONF_QUERY_ACCESS_DENIED: {
@@ -710,7 +705,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
-					RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				} break;
 
 				case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR:
@@ -732,7 +727,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
 			outp.append(nwid);
 			outp.armor(peer->key(),true);
-			RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+			RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 		}
 	} catch (std::exception &exc) {
 		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
@@ -753,7 +748,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 				nw->requestConfiguration();
 			ptr += 8;
 		}
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP);
 	} catch (std::exception &exc) {
 		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -780,11 +775,11 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 			outp.append((uint32_t)mg.adi());
 			if (RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit)) {
 				outp.armor(peer->key(),true);
-				RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+				RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 			}
 		}
 
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP);
 	} catch (std::exception &exc) {
 		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -807,13 +802,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			if ((flags & 0x01) != 0) {
 				CertificateOfMembership com;
 				offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
-				if (com.hasRequiredFields())
-					network->validateAndAddMembershipCertificate(com);
+				peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
 			}
 
 			// Check membership after we've read any included COM, since
 			// that cert might be what we needed.
-			if (!network->isAllowed(peer->address())) {
+			if (!network->isAllowed(peer)) {
 				TRACE("dropped MULTICAST_FRAME from %s(%s): not a member of private network %.16llx",peer->address().toString().c_str(),_remoteAddress.toString().c_str(),(unsigned long long)network->id());
 				_sendErrorNeedCertificate(RR,peer,network->id());
 				return true;
@@ -871,12 +865,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 				outp.append((unsigned char)0x02); // flag 0x02 = contains gather results
 				if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) {
 					outp.armor(peer->key(),true);
-					RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+					RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 				}
 			}
 		} // else ignore -- not a member of this network
 
-		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP);
+		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP);
 	} catch (std::exception &exc) {
 		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -905,14 +899,14 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 					InetAddress a(field(ptr,4),4,at<uint16_t>(ptr + 4));
 					if ( ((flags & (0x01 | 0x02)) == 0) && (Path::isAddressValidForPath(a)) ) {
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						peer->attemptToContactAt(RR,_localInterfaceId,a,RR->node->now());
+						peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
 					}
 				}	break;
 				case 6: {
 					InetAddress a(field(ptr,16),16,at<uint16_t>(ptr + 16));
 					if ( ((flags & (0x01 | 0x02)) == 0) && (Path::isAddressValidForPath(a)) ) {
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						peer->attemptToContactAt(RR,_localInterfaceId,a,RR->node->now());
+						peer->attemptToContactAt(RR,_localAddress,a,RR->node->now());
 					}
 				}	break;
 			}
@@ -926,6 +920,175 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 	return true;
 }
 
+bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
+{
+	try {
+		const Address originatorAddress(field(ZT_PACKET_IDX_PAYLOAD,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+		SharedPtr<Peer> originator(RR->topology->getPeer(originatorAddress));
+		if (!originator) {
+			RR->sw->requestWhois(originatorAddress);
+			return false;
+		}
+
+		const unsigned int flags = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 5);
+		const uint64_t timestamp = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 7);
+		const uint64_t testId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 15);
+
+		// Tracks total length of variable length fields, initialized to originator credential length below
+		unsigned int vlf;
+
+		// Originator credentials
+		const unsigned int originatorCredentialLength = vlf = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 23);
+		uint64_t originatorCredentialNetworkId = 0;
+		if (originatorCredentialLength >= 1) {
+			switch((*this)[ZT_PACKET_IDX_PAYLOAD + 25]) {
+				case 0x01: { // 64-bit network ID, originator must be controller
+					if (originatorCredentialLength >= 9)
+						originatorCredentialNetworkId = at<uint64_t>(ZT_PACKET_IDX_PAYLOAD + 26);
+				}	break;
+				default: break;
+			}
+		}
+
+		// Add length of "additional fields," which are currently unused
+		vlf += at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 25 + vlf);
+
+		// Verify signature -- only tests signed by their originators are allowed
+		const unsigned int signatureLength = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 27 + vlf);
+		if (!originator->identity().verify(field(ZT_PACKET_IDX_PAYLOAD,27 + vlf),27 + vlf,field(ZT_PACKET_IDX_PAYLOAD + 29 + vlf,signatureLength),signatureLength)) {
+			TRACE("dropped CIRCUIT_TEST from %s(%s): signature by originator %s invalid",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str());
+			return true;
+		}
+		vlf += signatureLength;
+
+		// Save this length so we can copy the immutable parts of this test
+		// into the one we send along to next hops.
+		const unsigned int lengthOfSignedPortionAndSignature = 29 + vlf;
+
+		// Get previous hop's credential, if any
+		const unsigned int previousHopCredentialLength = at<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 29 + vlf);
+		CertificateOfMembership previousHopCom;
+		if (previousHopCredentialLength >= 1) {
+			switch((*this)[ZT_PACKET_IDX_PAYLOAD + 31 + vlf]) {
+				case 0x01: { // network certificate of membership for previous hop
+					if (previousHopCom.deserialize(*this,ZT_PACKET_IDX_PAYLOAD + 32 + vlf) != (previousHopCredentialLength - 1)) {
+						TRACE("dropped CIRCUIT_TEST from %s(%s): previous hop COM invalid",source().toString().c_str(),_remoteAddress.toString().c_str());
+						return true;
+					}
+				}	break;
+				default: break;
+			}
+		}
+		vlf += previousHopCredentialLength;
+
+		// Check credentials (signature already verified)
+		SharedPtr<NetworkConfig> originatorCredentialNetworkConfig;
+		if (originatorCredentialNetworkId) {
+			if (Network::controllerFor(originatorCredentialNetworkId) == originatorAddress) {
+				SharedPtr<Network> nw(RR->node->network(originatorCredentialNetworkId));
+				if (nw) {
+					originatorCredentialNetworkConfig = nw->config2();
+					if ( (originatorCredentialNetworkConfig) && ((originatorCredentialNetworkConfig->isPublic())||(peer->address() == originatorAddress)||((originatorCredentialNetworkConfig->com())&&(previousHopCom)&&(originatorCredentialNetworkConfig->com().agreesWith(previousHopCom)))) ) {
+						TRACE("CIRCUIT_TEST %.16llx received from hop %s(%s) and originator %s with valid network ID credential %.16llx (verified from originator and next hop)",testId,source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
+					} else {
+						TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and previous hop %s did not supply a valid COM",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId,peer->address().toString().c_str());
+						return true;
+					}
+				} else {
+					TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID %.16llx as credential, and we are not a member",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
+					return true;
+				}
+			} else {
+				TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s specified network ID as credential, is not controller for %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str(),originatorCredentialNetworkId);
+				return true;
+			}
+		} else {
+			TRACE("dropped CIRCUIT_TEST from %s(%s): originator %s did not specify a credential or credential type",source().toString().c_str(),_remoteAddress.toString().c_str(),originatorAddress.toString().c_str());
+			return true;
+		}
+
+		const uint64_t now = RR->node->now();
+
+		unsigned int breadth = 0;
+		Address nextHop[256]; // breadth is a uin8_t, so this is the max
+		InetAddress nextHopBestPathAddress[256];
+		unsigned int remainingHopsPtr = ZT_PACKET_IDX_PAYLOAD + 33 + vlf;
+		if ((ZT_PACKET_IDX_PAYLOAD + 31 + vlf) < size()) {
+			// unsigned int nextHopFlags = (*this)[ZT_PACKET_IDX_PAYLOAD + 31 + vlf]
+			breadth = (*this)[ZT_PACKET_IDX_PAYLOAD + 32 + vlf];
+			for(unsigned int h=0;h<breadth;++h) {
+				nextHop[h].setTo(field(remainingHopsPtr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH);
+				remainingHopsPtr += ZT_ADDRESS_LENGTH;
+				SharedPtr<Peer> nhp(RR->topology->getPeer(nextHop[h]));
+				if (nhp) {
+					RemotePath *const rp = nhp->getBestPath(now);
+					if (rp)
+						nextHopBestPathAddress[h] = rp->address();
+				}
+			}
+		}
+
+		// Report back to originator, depending on flags and whether we are last hop
+		if ( ((flags & 0x01) != 0) || ((breadth == 0)&&((flags & 0x02) != 0)) ) {
+			Packet outp(originatorAddress,RR->identity.address(),Packet::VERB_CIRCUIT_TEST_REPORT);
+			outp.append((uint64_t)timestamp);
+			outp.append((uint64_t)testId);
+			outp.append((uint64_t)now);
+			outp.append((uint8_t)ZT_VENDOR_ZEROTIER);
+			outp.append((uint8_t)ZT_PROTO_VERSION);
+			outp.append((uint8_t)ZEROTIER_ONE_VERSION_MAJOR);
+			outp.append((uint8_t)ZEROTIER_ONE_VERSION_MINOR);
+			outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
+			outp.append((uint16_t)ZT_PLATFORM_UNSPECIFIED);
+			outp.append((uint16_t)ZT_ARCHITECTURE_UNSPECIFIED);
+			outp.append((uint16_t)0); // error code, currently unused
+			outp.append((uint64_t)0); // flags, currently unused
+			outp.append((uint64_t)packetId());
+			outp.append((uint8_t)hops());
+			_localAddress.serialize(outp);
+			_remoteAddress.serialize(outp);
+			outp.append((uint16_t)0); // no additional fields
+			outp.append((uint8_t)breadth);
+			for(unsigned int h=0;h<breadth;++h) {
+				nextHop[h].appendTo(outp);
+				nextHopBestPathAddress[h].serialize(outp); // appends 0 if null InetAddress
+			}
+			RR->sw->send(outp,true,0);
+		}
+
+		// If there are next hops, forward the test along through the graph
+		if (breadth > 0) {
+			Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST);
+			outp.append(field(ZT_PACKET_IDX_PAYLOAD,lengthOfSignedPortionAndSignature),lengthOfSignedPortionAndSignature);
+			const unsigned int previousHopCredentialPos = outp.size();
+			outp.append((uint16_t)0); // no previous hop credentials: default
+			if ((originatorCredentialNetworkConfig)&&(!originatorCredentialNetworkConfig->isPublic())&&(originatorCredentialNetworkConfig->com())) {
+				outp.append((uint8_t)0x01); // COM
+				originatorCredentialNetworkConfig->com().serialize(outp);
+				outp.setAt<uint16_t>(previousHopCredentialPos,(uint16_t)(size() - previousHopCredentialPos));
+			}
+			if (remainingHopsPtr < size())
+				outp.append(field(remainingHopsPtr,size() - remainingHopsPtr),size() - remainingHopsPtr);
+
+			for(unsigned int h=0;h<breadth;++h) {
+				outp.newInitializationVector();
+				outp.setDestination(nextHop[h]);
+				RR->sw->send(outp,true,originatorCredentialNetworkId);
+			}
+		}
+	} catch (std::exception &exc) {
+		TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
+	} catch ( ... ) {
+		TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception: (unknown)",source().toString().c_str(),_remoteAddress.toString().c_str());
+	}
+	return true;
+}
+
+bool IncomingPacket::_doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer)
+{
+	return true;
+}
+
 void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,uint64_t nwid)
 {
 	Packet outp(source(),RR->identity.address(),Packet::VERB_ERROR);
@@ -934,7 +1097,7 @@ void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,cons
 	outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
 	outp.append(nwid);
 	outp.armor(peer->key(),true);
-	RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
+	RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 }
 
 } // namespace ZeroTier

node/IncomingPacket.hpp

@@ -72,16 +72,16 @@ public:
 	 *
 	 * @param data Packet data
 	 * @param len Packet length
-	 * @param localInterfaceId Local interface ID
+	 * @param localAddress Local interface address
 	 * @param remoteAddress Address from which packet came
 	 * @param now Current time
 	 * @throws std::out_of_range Range error processing packet
 	 */
-	IncomingPacket(const void *data,unsigned int len,int localInterfaceId,const InetAddress &remoteAddress,uint64_t now) :
+	IncomingPacket(const void *data,unsigned int len,const InetAddress &localAddress,const InetAddress &remoteAddress,uint64_t now) :
  		Packet(data,len),
  		_receiveTime(now),
+ 		_localAddress(localAddress),
  		_remoteAddress(remoteAddress),
- 		_localInterfaceId(localInterfaceId),
  		__refCount()
 	{
 	}
@@ -124,13 +124,15 @@ private:
 	bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
+	bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
+	bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 
-	// Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to join
+	// Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to communicate
 	void _sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,uint64_t nwid);
 
 	uint64_t _receiveTime;
+	InetAddress _localAddress;
 	InetAddress _remoteAddress;
-	int _localInterfaceId;
 	AtomicCounter __refCount;
 };
 

node/InetAddress.hpp

@@ -38,6 +38,7 @@
 #include "../include/ZeroTierOne.h"
 #include "Utils.hpp"
 #include "MAC.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
@@ -362,6 +363,51 @@ struct InetAddress : public sockaddr_storage
 	 */
 	inline operator bool() const throw() { return (ss_family != 0); }
 
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		// Format is the same as in VERB_HELLO in Packet.hpp
+		switch(ss_family) {
+			case AF_INET:
+				b.append((uint8_t)0x04);
+				b.append(&(reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr),4);
+				b.append((uint16_t)port()); // just in case sin_port != uint16_t
+				return;
+			case AF_INET6:
+				b.append((uint8_t)0x06);
+				b.append(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,16);
+				b.append((uint16_t)port()); // just in case sin_port != uint16_t
+				return;
+			default:
+				b.append((uint8_t)0);
+				return;
+		}
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		memset(this,0,sizeof(InetAddress));
+		switch(b[p++]) {
+			case 0:
+				return 1;
+			case 0x04:
+				ss_family = AF_INET;
+				memcpy(&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr),b.field(p,4),4); p += 4;
+				reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2;
+				break;
+			case 0x06:
+				ss_family = AF_INET6;
+				memcpy(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,b.field(p,16),16); p += 16;
+				reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2;
+				break;
+			default:
+				throw std::invalid_argument("invalid serialized InetAddress");
+		}
+		return (p - startAt);
+	}
+
 	bool operator==(const InetAddress &a) const throw();
 	bool operator<(const InetAddress &a) const throw();
 	inline bool operator!=(const InetAddress &a) const throw() { return !(*this == a); }

node/Multicaster.cpp

@@ -237,12 +237,25 @@ void Multicaster::send(
 			if (sn) {
 				TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
 
+				const CertificateOfMembership *com = (CertificateOfMembership *)0;
+				SharedPtr<NetworkConfig> nconf;
+				if (sn->needsOurNetworkMembershipCertificate(nwid,now,true)) {
+					SharedPtr<Network> nw(RR->node->network(nwid));
+					if (nw) {
+						nconf = nw->config2();
+						if (nconf)
+							com = &(nconf->com());
+					}
+				}
+
 				Packet outp(sn->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
 				outp.append(nwid);
-				outp.append((uint8_t)0);
+				outp.append((uint8_t)(com ? 0x01 : 0x00));
 				mg.mac().appendTo(outp);
 				outp.append((uint32_t)mg.adi());
 				outp.append((uint32_t)gatherLimit);
+				if (com)
+					com->serialize(outp);
 				outp.armor(sn->key(),true);
 				sn->send(RR,outp.data(),outp.size(),now);
 			}

node/Network.cpp

@@ -59,6 +59,9 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 	Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id);
 	Utils::snprintf(mcdbn,sizeof(mcdbn),"networks.d/%.16llx.mcerts",_id);
 
+	// These files are no longer used, so clean them.
+	RR->node->dataStoreDelete(mcdbn);
+
 	if (_id == ZT_TEST_NETWORK_ID) {
 		applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address()));
 
@@ -79,68 +82,28 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 			// Save a one-byte CR to persist membership while we request a real netconf
 			RR->node->dataStorePut(confn,"\n",1,false);
 		}
-
-		try {
-			std::string mcdb(RR->node->dataStoreGet(mcdbn));
-			if (mcdb.length() > 6) {
-				const char *p = mcdb.data();
-				const char *e = p + mcdb.length();
-				if (!memcmp("ZTMCD0",p,6)) {
-					p += 6;
-					while (p != e) {
-						CertificateOfMembership com;
-						com.deserialize2(p,e);
-						if (!com)
-							break;
-						_certInfo[com.issuedTo()].com = com;
-					}
-				}
-			}
-		} catch ( ... ) {} // ignore invalid MCDB, we'll re-learn from peers
 	}
 
 	if (!_portInitialized) {
-		ZT1_VirtualNetworkConfig ctmp;
+		ZT_VirtualNetworkConfig ctmp;
 		_externalConfig(&ctmp);
-		_portError = RR->node->configureVirtualNetworkPort(_id,ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
+		_portError = RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
 		_portInitialized = true;
 	}
 }
 
 Network::~Network()
 {
-	ZT1_VirtualNetworkConfig ctmp;
+	ZT_VirtualNetworkConfig ctmp;
 	_externalConfig(&ctmp);
 
 	char n[128];
 	if (_destroyed) {
-		RR->node->configureVirtualNetworkPort(_id,ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
-
+		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
 		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
 		RR->node->dataStoreDelete(n);
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-		RR->node->dataStoreDelete(n);
 	} else {
-		RR->node->configureVirtualNetworkPort(_id,ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp);
-
-		clean();
-
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-
-		Mutex::Lock _l(_lock);
-		if ((!_config)||(_config->isPublic())||(_certInfo.empty())) {
-			RR->node->dataStoreDelete(n);
-		} else {
-			std::string buf("ZTMCD0");
-			Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-			Address *a = (Address *)0;
-			_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-			while (i.next(a,ci)) {
-				if (ci->com)
-					ci->com.serialize2(buf);
-			}
-			RR->node->dataStorePut(n,buf,true);
-		}
+		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp);
 	}
 }
 
@@ -178,13 +141,19 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg)
 		_myMulticastGroups.swap(nmg);
 }
 
+bool Network::tryAnnounceMulticastGroupsTo(const SharedPtr<Peer> &peer)
+{
+	Mutex::Lock _l(_lock);
+	return _tryAnnounceMulticastGroupsTo(RR->topology->rootAddresses(),_allMulticastGroups(),peer,RR->node->now());
+}
+
 bool Network::applyConfiguration(const SharedPtr<NetworkConfig> &conf)
 {
 	if (_destroyed) // sanity check
 		return false;
 	try {
 		if ((conf->networkId() == _id)&&(conf->issuedTo() == RR->identity.address())) {
-			ZT1_VirtualNetworkConfig ctmp;
+			ZT_VirtualNetworkConfig ctmp;
 			bool portInitialized;
 			{
 				Mutex::Lock _l(_lock);
@@ -195,7 +164,7 @@ bool Network::applyConfiguration(const SharedPtr<NetworkConfig> &conf)
 				portInitialized = _portInitialized;
 				_portInitialized = true;
 			}
-			_portError = RR->node->configureVirtualNetworkPort(_id,(portInitialized) ? ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
+			_portError = RR->node->configureVirtualNetworkPort(_id,(portInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
 			return true;
 		} else {
 			TRACE("ignored invalid configuration for network %.16llx (configuration contains mismatched network ID or issued-to address)",(unsigned long long)_id);
@@ -281,70 +250,6 @@ void Network::requestConfiguration()
 	RR->sw->send(outp,true,0);
 }
 
-bool Network::validateAndAddMembershipCertificate(const CertificateOfMembership &cert)
-{
-	if (!cert) // sanity check
-		return false;
-
-	Mutex::Lock _l(_lock);
-
-	{
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(cert.issuedTo());
-		if ((ci)&&(ci->com == cert))
-			return true; // we already have it
-	}
-
-	// Check signature, log and return if cert is invalid
-	if (cert.signedBy() != controller()) {
-		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,cert.signedBy().toString().c_str());
-		return false; // invalid signer
-	}
-
-	if (cert.signedBy() == RR->identity.address()) {
-
-		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
-		// So, verify that we signed th cert ourself
-		if (!cert.verify(RR->identity)) {
-			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-
-	} else {
-
-		SharedPtr<Peer> signer(RR->topology->getPeer(cert.signedBy()));
-
-		if (!signer) {
-			// This would be rather odd, since this is our controller... could happen
-			// if we get packets before we've gotten config.
-			RR->sw->requestWhois(cert.signedBy());
-			return false; // signer unknown
-		}
-
-		if (!cert.verify(signer->identity())) {
-			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-	}
-
-	// If we made it past authentication, add or update cert in our cert info store
-	_certInfo[cert.issuedTo()].com = cert;
-
-	return true;
-}
-
-bool Network::peerNeedsOurMembershipCertificate(const Address &to,uint64_t now)
-{
-	Mutex::Lock _l(_lock);
-	if ((_config)&&(!_config->isPublic())&&(_config->com())) {
-		_RemoteMemberCertificateInfo &ci = _certInfo[to];
-		if ((now - ci.lastPushed) > (ZT_NETWORK_AUTOCONF_DELAY / 2)) {
-			ci.lastPushed = now;
-			return true;
-		}
-	}
-	return false;
-}
-
 void Network::clean()
 {
 	const uint64_t now = RR->node->now();
@@ -353,22 +258,6 @@ void Network::clean()
 	if (_destroyed)
 		return;
 
-	if ((_config)&&(_config->isPublic())) {
-		// Open (public) networks do not track certs or cert pushes at all.
-		_certInfo.clear();
-	} else if (_config) {
-		// Clean obsolete entries from private network cert info table
-		Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-		Address *a = (Address *)0;
-		_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-		const uint64_t forgetIfBefore = now - (ZT_PEER_ACTIVITY_TIMEOUT * 16); // arbitrary reasonable cutoff
-		while (i.next(a,ci)) {
-			if ((ci->lastPushed < forgetIfBefore)&&(!ci->com.agreesWith(_config->com())))
-				_certInfo.erase(*a);
-		}
-	}
-
-	// Clean learned multicast groups if we haven't heard from them in a while
 	{
 		Hashtable< MulticastGroup,uint64_t >::Iterator i(_multicastGroupsBehindMe);
 		MulticastGroup *mg = (MulticastGroup *)0;
@@ -431,9 +320,9 @@ void Network::setEnabled(bool enabled)
 	Mutex::Lock _l(_lock);
 	if (_enabled != enabled) {
 		_enabled = enabled;
-		ZT1_VirtualNetworkConfig ctmp;
+		ZT_VirtualNetworkConfig ctmp;
 		_externalConfig(&ctmp);
-		_portError = RR->node->configureVirtualNetworkPort(_id,ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE,&ctmp);
+		_portError = RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE,&ctmp);
 	}
 }
 
@@ -444,24 +333,24 @@ void Network::destroy()
 	_destroyed = true;
 }
 
-ZT1_VirtualNetworkStatus Network::_status() const
+ZT_VirtualNetworkStatus Network::_status() const
 {
 	// assumes _lock is locked
 	if (_portError)
-		return ZT1_NETWORK_STATUS_PORT_ERROR;
+		return ZT_NETWORK_STATUS_PORT_ERROR;
 	switch(_netconfFailure) {
 		case NETCONF_FAILURE_ACCESS_DENIED:
-			return ZT1_NETWORK_STATUS_ACCESS_DENIED;
+			return ZT_NETWORK_STATUS_ACCESS_DENIED;
 		case NETCONF_FAILURE_NOT_FOUND:
-			return ZT1_NETWORK_STATUS_NOT_FOUND;
+			return ZT_NETWORK_STATUS_NOT_FOUND;
 		case NETCONF_FAILURE_NONE:
-			return ((_config) ? ZT1_NETWORK_STATUS_OK : ZT1_NETWORK_STATUS_REQUESTING_CONFIGURATION);
+			return ((_config) ? ZT_NETWORK_STATUS_OK : ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION);
 		default:
-			return ZT1_NETWORK_STATUS_PORT_ERROR;
+			return ZT_NETWORK_STATUS_PORT_ERROR;
 	}
 }
 
-void Network::_externalConfig(ZT1_VirtualNetworkConfig *ec) const
+void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 {
 	// assumes _lock is locked
 	ec->nwid = _id;
@@ -470,7 +359,7 @@ void Network::_externalConfig(ZT1_VirtualNetworkConfig *ec) const
 		Utils::scopy(ec->name,sizeof(ec->name),_config->name().c_str());
 	else ec->name[0] = (char)0;
 	ec->status = _status();
-	ec->type = (_config) ? (_config->isPrivate() ? ZT1_NETWORK_TYPE_PRIVATE : ZT1_NETWORK_TYPE_PUBLIC) : ZT1_NETWORK_TYPE_PRIVATE;
+	ec->type = (_config) ? (_config->isPrivate() ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC) : ZT_NETWORK_TYPE_PRIVATE;
 	ec->mtu = ZT_IF_MTU;
 	ec->dhcp = 0;
 	ec->bridge = (_config) ? ((_config->allowPassiveBridging() || (std::find(_config->activeBridges().begin(),_config->activeBridges().end(),RR->identity.address()) != _config->activeBridges().end())) ? 1 : 0) : 0;
@@ -479,7 +368,7 @@ void Network::_externalConfig(ZT1_VirtualNetworkConfig *ec) const
 	ec->enabled = (_enabled) ? 1 : 0;
 	ec->netconfRevision = (_config) ? (unsigned long)_config->revision() : 0;
 
-	ec->multicastSubscriptionCount = std::min((unsigned int)_myMulticastGroups.size(),(unsigned int)ZT1_MAX_NETWORK_MULTICAST_SUBSCRIPTIONS);
+	ec->multicastSubscriptionCount = std::min((unsigned int)_myMulticastGroups.size(),(unsigned int)ZT_MAX_NETWORK_MULTICAST_SUBSCRIPTIONS);
 	for(unsigned int i=0;i<ec->multicastSubscriptionCount;++i) {
 		ec->multicastSubscriptions[i].mac = _myMulticastGroups[i].mac().toInt();
 		ec->multicastSubscriptions[i].adi = _myMulticastGroups[i].adi();
@@ -487,14 +376,14 @@ void Network::_externalConfig(ZT1_VirtualNetworkConfig *ec) const
 
 	if (_config) {
 		ec->assignedAddressCount = (unsigned int)_config->staticIps().size();
-		for(unsigned long i=0;i<ZT1_MAX_ZT_ASSIGNED_ADDRESSES;++i) {
+		for(unsigned long i=0;i<ZT_MAX_ZT_ASSIGNED_ADDRESSES;++i) {
 			if (i < _config->staticIps().size())
 				memcpy(&(ec->assignedAddresses[i]),&(_config->staticIps()[i]),sizeof(struct sockaddr_storage));
 		}
 	} else ec->assignedAddressCount = 0;
 }
 
-bool Network::_isAllowed(const Address &peer) const
+bool Network::_isAllowed(const SharedPtr<Peer> &peer) const
 {
 	// Assumes _lock is locked
 	try {
@@ -502,82 +391,97 @@ bool Network::_isAllowed(const Address &peer) const
 			return false;
 		if (_config->isPublic())
 			return true;
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(peer);
-		if (!ci)
-			return false;
-		return _config->com().agreesWith(ci->com);
+		return ((_config->com())&&(peer->networkMembershipCertificatesAgree(_id,_config->com())));
 	} catch (std::exception &exc) {
-		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer.toString().c_str(),exc.what());
+		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer->address().toString().c_str(),exc.what());
 	} catch ( ... ) {
-		TRACE("isAllowed() check failed for peer %s: unexpected exception: unknown exception",peer.toString().c_str());
+		TRACE("isAllowed() check failed for peer %s: unexpected exception: unknown exception",peer->address().toString().c_str());
 	}
 	return false; // default position on any failure
 }
 
-std::vector<MulticastGroup> Network::_allMulticastGroups() const
+bool Network::_tryAnnounceMulticastGroupsTo(const std::vector<Address> &alwaysAddresses,const std::vector<MulticastGroup> &allMulticastGroups,const SharedPtr<Peer> &peer,uint64_t now) const
 {
-	// Assumes _lock is locked
-	std::vector<MulticastGroup> mgs;
-	mgs.reserve(_myMulticastGroups.size() + _multicastGroupsBehindMe.size() + 1);
-	mgs.insert(mgs.end(),_myMulticastGroups.begin(),_myMulticastGroups.end());
-	_multicastGroupsBehindMe.appendKeys(mgs);
-	if ((_config)&&(_config->enableBroadcast()))
-		mgs.push_back(Network::BROADCAST);
-	std::sort(mgs.begin(),mgs.end());
-	mgs.erase(std::unique(mgs.begin(),mgs.end()),mgs.end());
-	return mgs;
-}
-
-// Used in Network::_announceMulticastGroups()
-class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths
-{
-public:
-	_AnnounceMulticastGroupsToPeersWithActiveDirectPaths(const RuntimeEnvironment *renv,Network *nw) :
-		RR(renv),
-		_now(renv->node->now()),
-		_network(nw),
-		_rootAddresses(renv->topology->rootAddresses()),
-		_allMulticastGroups(nw->_allMulticastGroups())
-	{}
+	// assumes _lock is locked
+	if (
+	    (_isAllowed(peer)) ||
+	    (peer->address() == this->controller()) ||
+	    (std::find(alwaysAddresses.begin(),alwaysAddresses.end(),peer->address()) != alwaysAddresses.end())
+	   ) {
+
+		if ((_config)&&(_config->com())&&(!_config->isPublic())&&(peer->needsOurNetworkMembershipCertificate(_id,now,true))) {
+			Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
+			_config->com().serialize(outp);
+			outp.armor(peer->key(),true);
+			peer->send(RR,outp.data(),outp.size(),now);
+		}
 
-	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
-	{
-		if ( ( (p->hasActiveDirectPath(_now)) && ( (_network->_isAllowed(p->address())) || (p->address() == _network->controller()) ) ) || (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end()) ) {
-			Packet outp(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
+		{
+			Packet outp(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
 
-			for(std::vector<MulticastGroup>::iterator mg(_allMulticastGroups.begin());mg!=_allMulticastGroups.end();++mg) {
+			for(std::vector<MulticastGroup>::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) {
 				if ((outp.size() + 18) >= ZT_UDP_DEFAULT_PAYLOAD_MTU) {
-					outp.armor(p->key(),true);
-					p->send(RR,outp.data(),outp.size(),_now);
-					outp.reset(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
+					outp.armor(peer->key(),true);
+					peer->send(RR,outp.data(),outp.size(),now);
+					outp.reset(peer->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
 				}
 
 				// network ID, MAC, ADI
-				outp.append((uint64_t)_network->id());
+				outp.append((uint64_t)_id);
 				mg->mac().appendTo(outp);
 				outp.append((uint32_t)mg->adi());
 			}
 
 			if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) {
-				outp.armor(p->key(),true);
-				p->send(RR,outp.data(),outp.size(),_now);
+				outp.armor(peer->key(),true);
+				peer->send(RR,outp.data(),outp.size(),now);
 			}
 		}
+
+		return true;
 	}
+	return false;
+}
+
+class _AnnounceMulticastGroupsToAll
+{
+public:
+	_AnnounceMulticastGroupsToAll(const RuntimeEnvironment *renv,Network *nw) :
+		_now(renv->node->now()),
+		RR(renv),
+		_network(nw),
+		_rootAddresses(renv->topology->rootAddresses()),
+		_allMulticastGroups(nw->_allMulticastGroups())
+	{}
+
+	inline void operator()(Topology &t,const SharedPtr<Peer> &p) { _network->_tryAnnounceMulticastGroupsTo(_rootAddresses,_allMulticastGroups,p,_now); }
 
 private:
-	const RuntimeEnvironment *RR;
 	uint64_t _now;
+	const RuntimeEnvironment *RR;
 	Network *_network;
 	std::vector<Address> _rootAddresses;
 	std::vector<MulticastGroup> _allMulticastGroups;
 };
-
 void Network::_announceMulticastGroups()
 {
 	// Assumes _lock is locked
-	_AnnounceMulticastGroupsToPeersWithActiveDirectPaths afunc(RR,this);
-	RR->topology->eachPeer<_AnnounceMulticastGroupsToPeersWithActiveDirectPaths &>(afunc);
+	_AnnounceMulticastGroupsToAll afunc(RR,this);
+	RR->topology->eachPeer<_AnnounceMulticastGroupsToAll &>(afunc);
+}
+
+std::vector<MulticastGroup> Network::_allMulticastGroups() const
+{
+	// Assumes _lock is locked
+	std::vector<MulticastGroup> mgs;
+	mgs.reserve(_myMulticastGroups.size() + _multicastGroupsBehindMe.size() + 1);
+	mgs.insert(mgs.end(),_myMulticastGroups.begin(),_myMulticastGroups.end());
+	_multicastGroupsBehindMe.appendKeys(mgs);
+	if ((_config)&&(_config->enableBroadcast()))
+		mgs.push_back(Network::BROADCAST);
+	std::sort(mgs.begin(),mgs.end());
+	mgs.erase(std::unique(mgs.begin(),mgs.end()),mgs.end());
+	return mgs;
 }
 
 } // namespace ZeroTier

node/Network.hpp

@@ -55,7 +55,8 @@
 namespace ZeroTier {
 
 class RuntimeEnvironment;
-class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
+class Peer;
+class _AnnounceMulticastGroupsToAll; // internal function object in Network.cpp
 
 /**
  * A virtual LAN
@@ -63,7 +64,7 @@ class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
 class Network : NonCopyable
 {
 	friend class SharedPtr<Network>;
-	friend class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
+	friend class _AnnounceMulticastGroupsToAll;
 
 public:
 	/**
@@ -92,7 +93,13 @@ public:
 	/**
 	 * @return Address of network's controller (most significant 40 bits of ID)
 	 */
-	inline Address controller() throw() { return Address(_id >> 24); }
+	inline Address controller() const throw() { return Address(_id >> 24); }
+
+	/**
+	 * @param nwid Network ID
+	 * @return Address of network's controller
+	 */
+	static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); }
 
 	/**
 	 * @return Multicast group memberships for this network's port (local, not learned via bridging)
@@ -133,6 +140,14 @@ public:
 	 */
 	void multicastUnsubscribe(const MulticastGroup &mg);
 
+	/**
+	 * Announce multicast groups to a peer if that peer is authorized on this network
+	 *
+	 * @param peer Peer to try to announce multicast groups to
+	 * @return True if peer was authorized and groups were announced
+	 */
+	bool tryAnnounceMulticastGroupsTo(const SharedPtr<Peer> &peer);
+
 	/**
 	 * Apply a NetworkConfig to this network
 	 *
@@ -177,33 +192,10 @@ public:
 	void requestConfiguration();
 
 	/**
-	 * Add or update a membership certificate
-	 *
-	 * @param cert Certificate of membership
-	 * @return True if certificate was accepted as valid
-	 */
-	bool validateAndAddMembershipCertificate(const CertificateOfMembership &cert);
-
-	/**
-	 * Check if we should push membership certificate to a peer, AND update last pushed
-	 *
-	 * If we haven't pushed a cert to this peer in a long enough time, this returns
-	 * true and updates the last pushed time. Otherwise it returns false.
-	 *
-	 * This doesn't actually send anything, since COMs can hitch a ride with several
-	 * different kinds of packets.
-	 *
-	 * @param to Destination peer
-	 * @param now Current time
-	 * @return True if we should include a COM with whatever we're currently sending
-	 */
-	bool peerNeedsOurMembershipCertificate(const Address &to,uint64_t now);
-
-	/**
-	 * @param peer Peer address to check
+	 * @param peer Peer to check
 	 * @return True if peer is allowed to communicate on this network
 	 */
-	inline bool isAllowed(const Address &peer) const
+	inline bool isAllowed(const SharedPtr<Peer> &peer) const
 	{
 		Mutex::Lock _l(_lock);
 		return _isAllowed(peer);
@@ -222,7 +214,7 @@ public:
 	/**
 	 * @return Status of this network
 	 */
-	inline ZT1_VirtualNetworkStatus status() const
+	inline ZT_VirtualNetworkStatus status() const
 	{
 		Mutex::Lock _l(_lock);
 		return _status();
@@ -231,7 +223,7 @@ public:
 	/**
 	 * @param ec Buffer to fill with externally-visible network configuration
 	 */
-	inline void externalConfig(ZT1_VirtualNetworkConfig *ec) const
+	inline void externalConfig(ZT_VirtualNetworkConfig *ec) const
 	{
 		Mutex::Lock _l(_lock);
 		_externalConfig(ec);
@@ -347,16 +339,10 @@ public:
 	inline bool operator>=(const Network &n) const throw() { return (_id >= n._id); }
 
 private:
-	struct _RemoteMemberCertificateInfo
-	{
-		_RemoteMemberCertificateInfo() : com(),lastPushed(0) {}
-		CertificateOfMembership com; // remote member's COM
-		uint64_t lastPushed; // when did we last push ours to them?
-	};
-
-	ZT1_VirtualNetworkStatus _status() const;
-	void _externalConfig(ZT1_VirtualNetworkConfig *ec) const; // assumes _lock is locked
-	bool _isAllowed(const Address &peer) const;
+	ZT_VirtualNetworkStatus _status() const;
+	void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked
+	bool _isAllowed(const SharedPtr<Peer> &peer) const;
+	bool _tryAnnounceMulticastGroupsTo(const std::vector<Address> &rootAddresses,const std::vector<MulticastGroup> &allMulticastGroups,const SharedPtr<Peer> &peer,uint64_t now) const;
 	void _announceMulticastGroups();
 	std::vector<MulticastGroup> _allMulticastGroups() const;
 
@@ -370,8 +356,6 @@ private:
 	Hashtable< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups that seem to be behind us and when we last saw them (if we are a bridge)
 	Hashtable< MAC,Address > _remoteBridgeRoutes; // remote addresses where given MACs are reachable (for tracking devices behind remote bridges)
 
-	Hashtable< Address,_RemoteMemberCertificateInfo > _certInfo;
-
 	SharedPtr<NetworkConfig> _config; // Most recent network configuration, which is an immutable value-object
 	volatile uint64_t _lastConfigUpdate;
 

node/NetworkConfig.cpp

@@ -87,28 +87,28 @@ void NetworkConfig::_fromDictionary(const Dictionary &d)
 
 	// NOTE: d.get(name) throws if not found, d.get(name,default) returns default
 
-	_nwid = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID).c_str());
+	_nwid = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,"0").c_str());
 	if (!_nwid)
 		throw std::invalid_argument("configuration contains zero network ID");
 
-	_timestamp = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP).c_str());
+	_timestamp = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,"0").c_str());
 	_revision = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_REVISION,"1").c_str()); // older controllers don't send this, so default to 1
 
 	memset(_etWhitelist,0,sizeof(_etWhitelist));
-	std::vector<std::string> ets(Utils::split(d.get(ZT_NETWORKCONFIG_DICT_KEY_ALLOWED_ETHERNET_TYPES).c_str(),",","",""));
+	std::vector<std::string> ets(Utils::split(d.get(ZT_NETWORKCONFIG_DICT_KEY_ALLOWED_ETHERNET_TYPES,"").c_str(),",","",""));
 	for(std::vector<std::string>::const_iterator et(ets.begin());et!=ets.end();++et) {
 		unsigned int tmp = Utils::hexStrToUInt(et->c_str()) & 0xffff;
 		_etWhitelist[tmp >> 3] |= (1 << (tmp & 7));
 	}
 
-	_issuedTo = Address(d.get(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO));
+	_issuedTo = Address(d.get(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,"0"));
 	_multicastLimit = Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,zero).c_str());
 	if (_multicastLimit == 0) _multicastLimit = ZT_MULTICAST_DEFAULT_LIMIT;
 	_allowPassiveBridging = (Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_ALLOW_PASSIVE_BRIDGING,zero).c_str()) != 0);
 	_private = (Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_PRIVATE,one).c_str()) != 0);
 	_enableBroadcast = (Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_ENABLE_BROADCAST,one).c_str()) != 0);
-	_name = d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME);
-	if (_name.length() > ZT1_MAX_NETWORK_SHORT_NAME_LENGTH)
+	_name = d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME,"");
+	if (_name.length() > ZT_MAX_NETWORK_SHORT_NAME_LENGTH)
 		throw std::invalid_argument("network short name too long (max: 255 characters)");
 
 	// In dictionary IPs are split into V4 and V6 addresses, but we don't really
@@ -142,8 +142,8 @@ void NetworkConfig::_fromDictionary(const Dictionary &d)
 			_localRoutes.push_back(addr);
 		else _staticIps.push_back(addr);
 	}
-	if (_localRoutes.size() > ZT1_MAX_ZT_ASSIGNED_ADDRESSES) throw std::invalid_argument("too many ZT-assigned routes");
-	if (_staticIps.size() > ZT1_MAX_ZT_ASSIGNED_ADDRESSES) throw std::invalid_argument("too many ZT-assigned IP addresses");
+	if (_localRoutes.size() > ZT_MAX_ZT_ASSIGNED_ADDRESSES) throw std::invalid_argument("too many ZT-assigned routes");
+	if (_staticIps.size() > ZT_MAX_ZT_ASSIGNED_ADDRESSES) throw std::invalid_argument("too many ZT-assigned IP addresses");
 	std::sort(_localRoutes.begin(),_localRoutes.end());
 	_localRoutes.erase(std::unique(_localRoutes.begin(),_localRoutes.end()),_localRoutes.end());
 	std::sort(_staticIps.begin(),_staticIps.end());

node/Node.cpp

@@ -48,6 +48,8 @@
 #include "SelfAwareness.hpp"
 #include "Defaults.hpp"
 
+const struct sockaddr_storage ZT_SOCKADDR_NULL = {0};
+
 namespace ZeroTier {
 
 /****************************************************************************/
@@ -57,12 +59,12 @@ namespace ZeroTier {
 Node::Node(
 	uint64_t now,
 	void *uptr,
-	ZT1_DataStoreGetFunction dataStoreGetFunction,
-	ZT1_DataStorePutFunction dataStorePutFunction,
-	ZT1_WirePacketSendFunction wirePacketSendFunction,
-	ZT1_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
-	ZT1_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT1_EventCallback eventCallback,
+	ZT_DataStoreGetFunction dataStoreGetFunction,
+	ZT_DataStorePutFunction dataStorePutFunction,
+	ZT_WirePacketSendFunction wirePacketSendFunction,
+	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
+	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
+	ZT_EventCallback eventCallback,
 	const char *overrideRootTopology) :
 	_RR(this),
 	RR(&_RR),
@@ -141,7 +143,7 @@ Node::Node(
 	}
 	RR->topology->setRootServers(Dictionary(rt.get("rootservers","")));
 
-	postEvent(ZT1_EVENT_UP);
+	postEvent(ZT_EVENT_UP);
 }
 
 Node::~Node()
@@ -155,20 +157,20 @@ Node::~Node()
 	delete RR->sw;
 }
 
-ZT1_ResultCode Node::processWirePacket(
+ZT_ResultCode Node::processWirePacket(
 	uint64_t now,
-	int localInterfaceId,
+	const struct sockaddr_storage *localAddress,
 	const struct sockaddr_storage *remoteAddress,
 	const void *packetData,
 	unsigned int packetLength,
 	volatile uint64_t *nextBackgroundTaskDeadline)
 {
 	_now = now;
-	RR->sw->onRemotePacket(localInterfaceId,*(reinterpret_cast<const InetAddress *>(remoteAddress)),packetData,packetLength);
-	return ZT1_RESULT_OK;
+	RR->sw->onRemotePacket(*(reinterpret_cast<const InetAddress *>(localAddress)),*(reinterpret_cast<const InetAddress *>(remoteAddress)),packetData,packetLength);
+	return ZT_RESULT_OK;
 }
 
-ZT1_ResultCode Node::processVirtualNetworkFrame(
+ZT_ResultCode Node::processVirtualNetworkFrame(
 	uint64_t now,
 	uint64_t nwid,
 	uint64_t sourceMac,
@@ -183,8 +185,8 @@ ZT1_ResultCode Node::processVirtualNetworkFrame(
 	SharedPtr<Network> nw(this->network(nwid));
 	if (nw) {
 		RR->sw->onLocalEthernet(nw,MAC(sourceMac),MAC(destMac),etherType,vlanId,frameData,frameLength);
-		return ZT1_RESULT_OK;
-	} else return ZT1_RESULT_ERROR_NETWORK_NOT_FOUND;
+		return ZT_RESULT_OK;
+	} else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 }
 
 class _PingPeersThatNeedPing
@@ -228,7 +230,7 @@ private:
 	std::vector<Address> _rootAddresses;
 };
 
-ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
+ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
 {
 	_now = now;
 	Mutex::Lock bl(_backgroundTasksLock);
@@ -264,7 +266,7 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 				if (nr->second) {
 					SharedPtr<Peer> rp(RR->topology->getPeer(nr->first));
 					if ((rp)&&(!rp->hasActiveDirectPath(now)))
-						rp->attemptToContactAt(RR,-1,nr->second,now);
+						rp->attemptToContactAt(RR,InetAddress(),nr->second,now);
 				}
 			}
 
@@ -276,9 +278,9 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 			bool oldOnline = _online;
 			_online = ((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT);
 			if (oldOnline != _online)
-				postEvent(_online ? ZT1_EVENT_ONLINE : ZT1_EVENT_OFFLINE);
+				postEvent(_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE);
 		} catch ( ... ) {
-			return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+			return ZT_RESULT_FATAL_ERROR_INTERNAL;
 		}
 	} else {
 		timeUntilNextPingCheck -= (unsigned long)timeSinceLastPingCheck;
@@ -291,30 +293,30 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 			RR->sa->clean(now);
 			RR->mc->clean(now);
 		} catch ( ... ) {
-			return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+			return ZT_RESULT_FATAL_ERROR_INTERNAL;
 		}
 	}
 
 	try {
 		*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 
-	return ZT1_RESULT_OK;
+	return ZT_RESULT_OK;
 }
 
-ZT1_ResultCode Node::join(uint64_t nwid)
+ZT_ResultCode Node::join(uint64_t nwid)
 {
 	Mutex::Lock _l(_networks_m);
 	SharedPtr<Network> nw = _network(nwid);
 	if(!nw)
 		_networks.push_back(std::pair< uint64_t,SharedPtr<Network> >(nwid,SharedPtr<Network>(new Network(RR,nwid))));
 	std::sort(_networks.begin(),_networks.end()); // will sort by nwid since it's the first in a pair<>
-	return ZT1_RESULT_OK;
+	return ZT_RESULT_OK;
 }
 
-ZT1_ResultCode Node::leave(uint64_t nwid)
+ZT_ResultCode Node::leave(uint64_t nwid)
 {
 	std::vector< std::pair< uint64_t,SharedPtr<Network> > > newn;
 	Mutex::Lock _l(_networks_m);
@@ -324,25 +326,25 @@ ZT1_ResultCode Node::leave(uint64_t nwid)
 		else n->second->destroy();
 	}
 	_networks.swap(newn);
-	return ZT1_RESULT_OK;
+	return ZT_RESULT_OK;
 }
 
-ZT1_ResultCode Node::multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
+ZT_ResultCode Node::multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
 {
 	SharedPtr<Network> nw(this->network(nwid));
 	if (nw) {
 		nw->multicastSubscribe(MulticastGroup(MAC(multicastGroup),(uint32_t)(multicastAdi & 0xffffffff)));
-		return ZT1_RESULT_OK;
-	} else return ZT1_RESULT_ERROR_NETWORK_NOT_FOUND;
+		return ZT_RESULT_OK;
+	} else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 }
 
-ZT1_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
+ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
 {
 	SharedPtr<Network> nw(this->network(nwid));
 	if (nw) {
 		nw->multicastUnsubscribe(MulticastGroup(MAC(multicastGroup),(uint32_t)(multicastAdi & 0xffffffff)));
-		return ZT1_RESULT_OK;
-	} else return ZT1_RESULT_ERROR_NETWORK_NOT_FOUND;
+		return ZT_RESULT_OK;
+	} else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 }
 
 uint64_t Node::address() const
@@ -350,7 +352,7 @@ uint64_t Node::address() const
 	return RR->identity.address().toInt();
 }
 
-void Node::status(ZT1_NodeStatus *status) const
+void Node::status(ZT_NodeStatus *status) const
 {
 	status->address = RR->identity.address().toInt();
 	status->publicIdentity = RR->publicIdentityStr.c_str();
@@ -358,20 +360,20 @@ void Node::status(ZT1_NodeStatus *status) const
 	status->online = _online ? 1 : 0;
 }
 
-ZT1_PeerList *Node::peers() const
+ZT_PeerList *Node::peers() const
 {
 	std::vector< std::pair< Address,SharedPtr<Peer> > > peers(RR->topology->allPeers());
 	std::sort(peers.begin(),peers.end());
 
-	char *buf = (char *)::malloc(sizeof(ZT1_PeerList) + (sizeof(ZT1_Peer) * peers.size()));
+	char *buf = (char *)::malloc(sizeof(ZT_PeerList) + (sizeof(ZT_Peer) * peers.size()));
 	if (!buf)
-		return (ZT1_PeerList *)0;
-	ZT1_PeerList *pl = (ZT1_PeerList *)buf;
-	pl->peers = (ZT1_Peer *)(buf + sizeof(ZT1_PeerList));
+		return (ZT_PeerList *)0;
+	ZT_PeerList *pl = (ZT_PeerList *)buf;
+	pl->peers = (ZT_Peer *)(buf + sizeof(ZT_PeerList));
 
 	pl->peerCount = 0;
 	for(std::vector< std::pair< Address,SharedPtr<Peer> > >::iterator pi(peers.begin());pi!=peers.end();++pi) {
-		ZT1_Peer *p = &(pl->peers[pl->peerCount++]);
+		ZT_Peer *p = &(pl->peers[pl->peerCount++]);
 		p->address = pi->second->address().toInt();
 		p->lastUnicastFrame = pi->second->lastUnicastFrame();
 		p->lastMulticastFrame = pi->second->lastMulticastFrame();
@@ -385,7 +387,7 @@ ZT1_PeerList *Node::peers() const
 			p->versionRev = -1;
 		}
 		p->latency = pi->second->latency();
-		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT1_PEER_ROLE_ROOT : ZT1_PEER_ROLE_LEAF;
+		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
 
 		std::vector<RemotePath> paths(pi->second->paths());
 		RemotePath *bestPath = pi->second->getBestPath(_now);
@@ -404,27 +406,27 @@ ZT1_PeerList *Node::peers() const
 	return pl;
 }
 
-ZT1_VirtualNetworkConfig *Node::networkConfig(uint64_t nwid) const
+ZT_VirtualNetworkConfig *Node::networkConfig(uint64_t nwid) const
 {
 	Mutex::Lock _l(_networks_m);
 	SharedPtr<Network> nw = _network(nwid);
 	if(nw) {
-		ZT1_VirtualNetworkConfig *nc = (ZT1_VirtualNetworkConfig *)::malloc(sizeof(ZT1_VirtualNetworkConfig));
+		ZT_VirtualNetworkConfig *nc = (ZT_VirtualNetworkConfig *)::malloc(sizeof(ZT_VirtualNetworkConfig));
 		nw->externalConfig(nc);
 		return nc;
 	}
-	return (ZT1_VirtualNetworkConfig *)0;
+	return (ZT_VirtualNetworkConfig *)0;
 }
 
-ZT1_VirtualNetworkList *Node::networks() const
+ZT_VirtualNetworkList *Node::networks() const
 {
 	Mutex::Lock _l(_networks_m);
 
-	char *buf = (char *)::malloc(sizeof(ZT1_VirtualNetworkList) + (sizeof(ZT1_VirtualNetworkConfig) * _networks.size()));
+	char *buf = (char *)::malloc(sizeof(ZT_VirtualNetworkList) + (sizeof(ZT_VirtualNetworkConfig) * _networks.size()));
 	if (!buf)
-		return (ZT1_VirtualNetworkList *)0;
-	ZT1_VirtualNetworkList *nl = (ZT1_VirtualNetworkList *)buf;
-	nl->networks = (ZT1_VirtualNetworkConfig *)(buf + sizeof(ZT1_VirtualNetworkList));
+		return (ZT_VirtualNetworkList *)0;
+	ZT_VirtualNetworkList *nl = (ZT_VirtualNetworkList *)buf;
+	nl->networks = (ZT_VirtualNetworkConfig *)(buf + sizeof(ZT_VirtualNetworkList));
 
 	nl->networkCount = 0;
 	for(std::vector< std::pair< uint64_t,SharedPtr<Network> > >::const_iterator n(_networks.begin());n!=_networks.end();++n)
@@ -439,7 +441,7 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT1_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
@@ -462,6 +464,64 @@ void Node::setNetconfMaster(void *networkControllerInstance)
 	RR->localNetworkController = reinterpret_cast<NetworkController *>(networkControllerInstance);
 }
 
+ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
+{
+	if (test->hopCount > 0) {
+		try {
+			Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST);
+			RR->identity.address().appendTo(outp);
+			outp.append((uint16_t)((test->reportAtEveryHop != 0) ? 0x03 : 0x02));
+			outp.append((uint64_t)test->timestamp);
+			outp.append((uint64_t)test->testId);
+			outp.append((uint16_t)0); // originator credential length, updated later
+			if (test->credentialNetworkId) {
+				outp.append((uint8_t)0x01);
+				outp.append((uint64_t)test->credentialNetworkId);
+				outp.setAt<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 23,(uint16_t)9);
+			}
+			outp.append((uint16_t)0);
+			C25519::Signature sig(RR->identity.sign(reinterpret_cast<const char *>(outp.data()) + ZT_PACKET_IDX_PAYLOAD,outp.size() - ZT_PACKET_IDX_PAYLOAD));
+			outp.append((uint16_t)sig.size());
+			outp.append(sig.data,sig.size());
+			outp.append((uint16_t)0); // originator doesn't need an extra credential, since it's the originator
+			for(unsigned int h=1;h<test->hopCount;++h) {
+				outp.append((uint8_t)0);
+				outp.append((uint8_t)(test->hops[h].breadth & 0xff));
+				for(unsigned int a=0;a<test->hops[h].breadth;++a)
+					Address(test->hops[h].addresses[a]).appendTo(outp);
+			}
+
+			for(unsigned int a=0;a<test->hops[0].breadth;++a) {
+				outp.newInitializationVector();
+				outp.setDestination(Address(test->hops[0].addresses[a]));
+				RR->sw->send(outp,true,test->credentialNetworkId);
+			}
+		} catch ( ... ) {
+			return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet
+		}
+	}
+
+	{
+		test->_internalPtr = reinterpret_cast<void *>(reportCallback);
+		Mutex::Lock _l(_circuitTests_m);
+		if (std::find(_circuitTests.begin(),_circuitTests.end(),test) == _circuitTests.end())
+			_circuitTests.push_back(test);
+	}
+
+	return ZT_RESULT_OK;
+}
+
+void Node::circuitTestEnd(ZT_CircuitTest *test)
+{
+	Mutex::Lock _l(_circuitTests_m);
+	for(;;) {
+		std::vector< ZT_CircuitTest * >::iterator ct(std::find(_circuitTests.begin(),_circuitTests.end(),test));
+		if (ct == _circuitTests.end())
+			break;
+		else _circuitTests.erase(ct);
+	}
+}
+
 /****************************************************************************/
 /* Node methods used only within node/                                      */
 /****************************************************************************/
@@ -472,7 +532,7 @@ std::string Node::dataStoreGet(const char *name)
 	std::string r;
 	unsigned long olen = 0;
 	do {
-		long n = _dataStoreGetFunction(reinterpret_cast<ZT1_Node *>(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen);
+		long n = _dataStoreGetFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,buf,sizeof(buf),(unsigned long)r.length(),&olen);
 		if (n <= 0)
 			return std::string();
 		r.append(buf,n);
@@ -486,7 +546,7 @@ void Node::postNewerVersionIfNewer(unsigned int major,unsigned int minor,unsigne
 		_newestVersionSeen[0] = major;
 		_newestVersionSeen[1] = minor;
 		_newestVersionSeen[2] = rev;
-		this->postEvent(ZT1_EVENT_SAW_MORE_RECENT_VERSION,(const void *)_newestVersionSeen);
+		this->postEvent(ZT_EVENT_SAW_MORE_RECENT_VERSION,(const void *)_newestVersionSeen);
 	}
 }
 
@@ -519,7 +579,7 @@ void Node::postTrace(const char *module,unsigned int line,const char *fmt,...)
 	tmp2[sizeof(tmp2)-1] = (char)0;
 
 	Utils::snprintf(tmp1,sizeof(tmp1),"[%s] %s:%u %s",nowstr,module,line,tmp2);
-	postEvent(ZT1_EVENT_TRACE,tmp1);
+	postEvent(ZT_EVENT_TRACE,tmp1);
 }
 #endif // ZT_TRACE
 
@@ -531,6 +591,20 @@ uint64_t Node::prng()
 	return _prngStream[p];
 }
 
+void Node::postCircuitTestReport(const ZT_CircuitTestReport *report)
+{
+	std::vector< ZT_CircuitTest * > toNotify;
+	{
+		Mutex::Lock _l(_circuitTests_m);
+		for(std::vector< ZT_CircuitTest * >::iterator i(_circuitTests.begin());i!=_circuitTests.end();++i) {
+			if ((*i)->testId == report->testId)
+				toNotify.push_back(*i);
+		}
+	}
+	for(std::vector< ZT_CircuitTest * >::iterator i(toNotify.begin());i!=toNotify.end();++i)
+		(reinterpret_cast<void (*)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)>((*i)->_internalPtr))(reinterpret_cast<ZT_Node *>(this),*i,report);
+}
+
 } // namespace ZeroTier
 
 /****************************************************************************/
@@ -539,59 +613,59 @@ uint64_t Node::prng()
 
 extern "C" {
 
-enum ZT1_ResultCode ZT1_Node_new(
-	ZT1_Node **node,
+enum ZT_ResultCode ZT_Node_new(
+	ZT_Node **node,
 	void *uptr,
 	uint64_t now,
-	ZT1_DataStoreGetFunction dataStoreGetFunction,
-	ZT1_DataStorePutFunction dataStorePutFunction,
-	ZT1_WirePacketSendFunction wirePacketSendFunction,
-	ZT1_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
-	ZT1_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT1_EventCallback eventCallback,
+	ZT_DataStoreGetFunction dataStoreGetFunction,
+	ZT_DataStorePutFunction dataStorePutFunction,
+	ZT_WirePacketSendFunction wirePacketSendFunction,
+	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
+	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
+	ZT_EventCallback eventCallback,
 	const char *overrideRootTopology)
 {
-	*node = (ZT1_Node *)0;
+	*node = (ZT_Node *)0;
 	try {
-		*node = reinterpret_cast<ZT1_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback,overrideRootTopology));
-		return ZT1_RESULT_OK;
+		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback,overrideRootTopology));
+		return ZT_RESULT_OK;
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch (std::runtime_error &exc) {
-		return ZT1_RESULT_FATAL_ERROR_DATA_STORE_FAILED;
+		return ZT_RESULT_FATAL_ERROR_DATA_STORE_FAILED;
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-void ZT1_Node_delete(ZT1_Node *node)
+void ZT_Node_delete(ZT_Node *node)
 {
 	try {
 		delete (reinterpret_cast<ZeroTier::Node *>(node));
 	} catch ( ... ) {}
 }
 
-enum ZT1_ResultCode ZT1_Node_processWirePacket(
-	ZT1_Node *node,
+enum ZT_ResultCode ZT_Node_processWirePacket(
+	ZT_Node *node,
 	uint64_t now,
-	int localInterfaceId,
+	const struct sockaddr_storage *localAddress,
 	const struct sockaddr_storage *remoteAddress,
 	const void *packetData,
 	unsigned int packetLength,
 	volatile uint64_t *nextBackgroundTaskDeadline)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->processWirePacket(now,localInterfaceId,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline);
+		return reinterpret_cast<ZeroTier::Node *>(node)->processWirePacket(now,localAddress,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline);
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		reinterpret_cast<ZeroTier::Node *>(node)->postEvent(ZT1_EVENT_INVALID_PACKET,(const void *)remoteAddress);
-		return ZT1_RESULT_OK;
+		reinterpret_cast<ZeroTier::Node *>(node)->postEvent(ZT_EVENT_INVALID_PACKET,(const void *)remoteAddress);
+		return ZT_RESULT_OK;
 	}
 }
 
-enum ZT1_ResultCode ZT1_Node_processVirtualNetworkFrame(
-	ZT1_Node *node,
+enum ZT_ResultCode ZT_Node_processVirtualNetworkFrame(
+	ZT_Node *node,
 	uint64_t now,
 	uint64_t nwid,
 	uint64_t sourceMac,
@@ -605,121 +679,137 @@ enum ZT1_ResultCode ZT1_Node_processVirtualNetworkFrame(
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->processVirtualNetworkFrame(now,nwid,sourceMac,destMac,etherType,vlanId,frameData,frameLength,nextBackgroundTaskDeadline);
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-enum ZT1_ResultCode ZT1_Node_processBackgroundTasks(ZT1_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
+enum ZT_ResultCode ZT_Node_processBackgroundTasks(ZT_Node *node,uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->processBackgroundTasks(now,nextBackgroundTaskDeadline);
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-enum ZT1_ResultCode ZT1_Node_join(ZT1_Node *node,uint64_t nwid)
+enum ZT_ResultCode ZT_Node_join(ZT_Node *node,uint64_t nwid)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->join(nwid);
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-enum ZT1_ResultCode ZT1_Node_leave(ZT1_Node *node,uint64_t nwid)
+enum ZT_ResultCode ZT_Node_leave(ZT_Node *node,uint64_t nwid)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->leave(nwid);
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-enum ZT1_ResultCode ZT1_Node_multicastSubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
+enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->multicastSubscribe(nwid,multicastGroup,multicastAdi);
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-enum ZT1_ResultCode ZT1_Node_multicastUnsubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
+enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->multicastUnsubscribe(nwid,multicastGroup,multicastAdi);
 	} catch (std::bad_alloc &exc) {
-		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
+		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-uint64_t ZT1_Node_address(ZT1_Node *node)
+uint64_t ZT_Node_address(ZT_Node *node)
 {
 	return reinterpret_cast<ZeroTier::Node *>(node)->address();
 }
 
-void ZT1_Node_status(ZT1_Node *node,ZT1_NodeStatus *status)
+void ZT_Node_status(ZT_Node *node,ZT_NodeStatus *status)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node *>(node)->status(status);
 	} catch ( ... ) {}
 }
 
-ZT1_PeerList *ZT1_Node_peers(ZT1_Node *node)
+ZT_PeerList *ZT_Node_peers(ZT_Node *node)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->peers();
 	} catch ( ... ) {
-		return (ZT1_PeerList *)0;
+		return (ZT_PeerList *)0;
 	}
 }
 
-ZT1_VirtualNetworkConfig *ZT1_Node_networkConfig(ZT1_Node *node,uint64_t nwid)
+ZT_VirtualNetworkConfig *ZT_Node_networkConfig(ZT_Node *node,uint64_t nwid)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->networkConfig(nwid);
 	} catch ( ... ) {
-		return (ZT1_VirtualNetworkConfig *)0;
+		return (ZT_VirtualNetworkConfig *)0;
 	}
 }
 
-ZT1_VirtualNetworkList *ZT1_Node_networks(ZT1_Node *node)
+ZT_VirtualNetworkList *ZT_Node_networks(ZT_Node *node)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->networks();
 	} catch ( ... ) {
-		return (ZT1_VirtualNetworkList *)0;
+		return (ZT_VirtualNetworkList *)0;
 	}
 }
 
-void ZT1_Node_freeQueryResult(ZT1_Node *node,void *qr)
+void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node *>(node)->freeQueryResult(qr);
 	} catch ( ... ) {}
 }
 
-void ZT1_Node_setNetconfMaster(ZT1_Node *node,void *networkControllerInstance)
+void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node *>(node)->setNetconfMaster(networkControllerInstance);
 	} catch ( ... ) {}
 }
 
-int ZT1_Node_addLocalInterfaceAddress(ZT1_Node *node,const struct sockaddr_storage *addr,int metric,ZT1_LocalInterfaceAddressTrust trust)
+ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->circuitTestBegin(test,reportCallback);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
+void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->circuitTestEnd(test);
+	} catch ( ... ) {}
+}
+
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
@@ -728,21 +818,21 @@ int ZT1_Node_addLocalInterfaceAddress(ZT1_Node *node,const struct sockaddr_stora
 	}
 }
 
-void ZT1_Node_clearLocalInterfaceAddresses(ZT1_Node *node)
+void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
 {
 	try {
 		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
 	} catch ( ... ) {}
 }
 
-void ZT1_version(int *major,int *minor,int *revision,unsigned long *featureFlags)
+void ZT_version(int *major,int *minor,int *revision,unsigned long *featureFlags)
 {
 	if (major) *major = ZEROTIER_ONE_VERSION_MAJOR;
 	if (minor) *minor = ZEROTIER_ONE_VERSION_MINOR;
 	if (revision) *revision = ZEROTIER_ONE_VERSION_REVISION;
 	if (featureFlags) {
 		*featureFlags = (
-			ZT1_FEATURE_FLAG_THREAD_SAFE
+			ZT_FEATURE_FLAG_THREAD_SAFE
 		);
 	}
 }

node/Node.hpp

@@ -58,7 +58,7 @@ namespace ZeroTier {
 /**
  * Implementation of Node object as defined in CAPI
  *
- * The pointer returned by ZT1_Node_new() is an instance of this class.
+ * The pointer returned by ZT_Node_new() is an instance of this class.
  */
 class Node
 {
@@ -66,26 +66,26 @@ public:
 	Node(
 		uint64_t now,
 		void *uptr,
-		ZT1_DataStoreGetFunction dataStoreGetFunction,
-		ZT1_DataStorePutFunction dataStorePutFunction,
-		ZT1_WirePacketSendFunction wirePacketSendFunction,
-		ZT1_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
-		ZT1_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-		ZT1_EventCallback eventCallback,
+		ZT_DataStoreGetFunction dataStoreGetFunction,
+		ZT_DataStorePutFunction dataStorePutFunction,
+		ZT_WirePacketSendFunction wirePacketSendFunction,
+		ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
+		ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
+		ZT_EventCallback eventCallback,
 		const char *overrideRootTopology);
 
 	~Node();
 
 	// Public API Functions ----------------------------------------------------
 
-	ZT1_ResultCode processWirePacket(
+	ZT_ResultCode processWirePacket(
 		uint64_t now,
-		int localInterfaceId,
+		const struct sockaddr_storage *localAddress,
 		const struct sockaddr_storage *remoteAddress,
 		const void *packetData,
 		unsigned int packetLength,
 		volatile uint64_t *nextBackgroundTaskDeadline);
-	ZT1_ResultCode processVirtualNetworkFrame(
+	ZT_ResultCode processVirtualNetworkFrame(
 		uint64_t now,
 		uint64_t nwid,
 		uint64_t sourceMac,
@@ -95,20 +95,22 @@ public:
 		const void *frameData,
 		unsigned int frameLength,
 		volatile uint64_t *nextBackgroundTaskDeadline);
-	ZT1_ResultCode processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline);
-	ZT1_ResultCode join(uint64_t nwid);
-	ZT1_ResultCode leave(uint64_t nwid);
-	ZT1_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
-	ZT1_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
+	ZT_ResultCode processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline);
+	ZT_ResultCode join(uint64_t nwid);
+	ZT_ResultCode leave(uint64_t nwid);
+	ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
+	ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 	uint64_t address() const;
-	void status(ZT1_NodeStatus *status) const;
-	ZT1_PeerList *peers() const;
-	ZT1_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
-	ZT1_VirtualNetworkList *networks() const;
+	void status(ZT_NodeStatus *status) const;
+	ZT_PeerList *peers() const;
+	ZT_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
+	ZT_VirtualNetworkList *networks() const;
 	void freeQueryResult(void *qr);
-	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT1_LocalInterfaceAddressTrust trust);
+	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
+	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
+	void circuitTestEnd(ZT_CircuitTest *test);
 
 	// Internal functions ------------------------------------------------------
 
@@ -120,18 +122,18 @@ public:
 	/**
 	 * Enqueue a ZeroTier message to be sent
 	 *
-	 * @param localInterfaceId Local interface ID, -1 for unspecified/random
+	 * @param localAddress Local address
 	 * @param addr Destination address
 	 * @param data Packet data
 	 * @param len Packet length
 	 * @return True if packet appears to have been sent
 	 */
-	inline bool putPacket(int localInterfaceId,const InetAddress &addr,const void *data,unsigned int len)
+	inline bool putPacket(const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len)
 	{
 		return (_wirePacketSendFunction(
-			reinterpret_cast<ZT1_Node *>(this),
+			reinterpret_cast<ZT_Node *>(this),
 			_uPtr,
-			localInterfaceId,
+			reinterpret_cast<const struct sockaddr_storage *>(&localAddress),
 			reinterpret_cast<const struct sockaddr_storage *>(&addr),
 			data,
 			len) == 0);
@@ -151,7 +153,7 @@ public:
 	inline void putFrame(uint64_t nwid,const MAC &source,const MAC &dest,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 	{
 		_virtualNetworkFrameFunction(
-			reinterpret_cast<ZT1_Node *>(this),
+			reinterpret_cast<ZT_Node *>(this),
 			_uPtr,
 			nwid,
 			source.toInt(),
@@ -168,6 +170,16 @@ public:
 		return _network(nwid);
 	}
 
+	inline bool belongsToNetwork(uint64_t nwid) const
+	{
+		Mutex::Lock _l(_networks_m);
+		for(std::vector< std::pair< uint64_t, SharedPtr<Network> > >::const_iterator i=_networks.begin();i!=_networks.end();++i) {
+			if (i->first == nwid)
+				return true;
+		}
+		return false;
+	}
+
 	inline std::vector< SharedPtr<Network> > allNetworks() const
 	{
 		std::vector< SharedPtr<Network> > nw;
@@ -187,9 +199,9 @@ public:
 		return _directPaths;
 	}
 
-	inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_dataStorePutFunction(reinterpret_cast<ZT1_Node *>(this),_uPtr,name,data,len,(int)secure) == 0); }
+	inline bool dataStorePut(const char *name,const void *data,unsigned int len,bool secure) { return (_dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,data,len,(int)secure) == 0); }
 	inline bool dataStorePut(const char *name,const std::string &data,bool secure) { return dataStorePut(name,(const void *)data.data(),(unsigned int)data.length(),secure); }
-	inline void dataStoreDelete(const char *name) { _dataStorePutFunction(reinterpret_cast<ZT1_Node *>(this),_uPtr,name,(const void *)0,0,0); }
+	inline void dataStoreDelete(const char *name) { _dataStorePutFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,name,(const void *)0,0,0); }
 	std::string dataStoreGet(const char *name);
 
 	/**
@@ -198,7 +210,7 @@ public:
 	 * @param ev Event type
 	 * @param md Meta-data (default: NULL/none)
 	 */
-	inline void postEvent(ZT1_Event ev,const void *md = (const void *)0) { _eventCallback(reinterpret_cast<ZT1_Node *>(this),_uPtr,ev,md); }
+	inline void postEvent(ZT_Event ev,const void *md = (const void *)0) { _eventCallback(reinterpret_cast<ZT_Node *>(this),_uPtr,ev,md); }
 
 	/**
 	 * Update virtual network port configuration
@@ -207,7 +219,7 @@ public:
 	 * @param op Configuration operation
 	 * @param nc Network configuration
 	 */
-	inline int configureVirtualNetworkPort(uint64_t nwid,ZT1_VirtualNetworkConfigOperation op,const ZT1_VirtualNetworkConfig *nc) { return _virtualNetworkConfigFunction(reinterpret_cast<ZT1_Node *>(this),_uPtr,nwid,op,nc); }
+	inline int configureVirtualNetworkPort(uint64_t nwid,ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nc) { return _virtualNetworkConfigFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,nwid,op,nc); }
 
 	/**
 	 * @return True if we appear to be online
@@ -228,6 +240,13 @@ public:
 	 */
 	uint64_t prng();
 
+	/**
+	 * Post a circuit test report to any listeners for a given test ID
+	 *
+	 * @param report Report (includes test ID)
+	 */
+	void postCircuitTestReport(const ZT_CircuitTestReport *report);
+
 private:
 	inline SharedPtr<Network> _network(uint64_t nwid) const
 	{
@@ -244,16 +263,19 @@ private:
 
 	void *_uPtr; // _uptr (lower case) is reserved in Visual Studio :P
 
-	ZT1_DataStoreGetFunction _dataStoreGetFunction;
-	ZT1_DataStorePutFunction _dataStorePutFunction;
-	ZT1_WirePacketSendFunction _wirePacketSendFunction;
-	ZT1_VirtualNetworkFrameFunction _virtualNetworkFrameFunction;
-	ZT1_VirtualNetworkConfigFunction _virtualNetworkConfigFunction;
-	ZT1_EventCallback _eventCallback;
+	ZT_DataStoreGetFunction _dataStoreGetFunction;
+	ZT_DataStorePutFunction _dataStorePutFunction;
+	ZT_WirePacketSendFunction _wirePacketSendFunction;
+	ZT_VirtualNetworkFrameFunction _virtualNetworkFrameFunction;
+	ZT_VirtualNetworkConfigFunction _virtualNetworkConfigFunction;
+	ZT_EventCallback _eventCallback;
 
 	std::vector< std::pair< uint64_t, SharedPtr<Network> > > _networks;
 	Mutex _networks_m;
 
+	std::vector< ZT_CircuitTest * > _circuitTests;
+	Mutex _circuitTests_m;
+
 	std::vector<Path> _directPaths;
 	Mutex _directPaths_m;
 

node/OutboundMulticast.cpp

@@ -103,11 +103,11 @@ void OutboundMulticast::init(
 void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr)
 {
 	if (_haveCom) {
-		SharedPtr<Network> network(RR->node->network(_nwid));
-		if ((network)&&(network->peerNeedsOurMembershipCertificate(toAddr,RR->node->now()))) {
+		SharedPtr<Peer> peer(RR->topology->getPeer(toAddr));
+		if ( (!peer) || (peer->needsOurNetworkMembershipCertificate(_nwid,RR->node->now(),true)) ) {
+			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			_packetWithCom.newInitializationVector();
 			_packetWithCom.setDestination(toAddr);
-			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			RR->sw->send(_packetWithCom,true,_nwid);
 			return;
 		}

node/Packet.cpp

@@ -31,6 +31,8 @@ namespace ZeroTier {
 
 const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
 
+#ifdef ZT_TRACE
+
 const char *Packet::verbString(Verb v)
 	throw()
 {
@@ -52,6 +54,8 @@ const char *Packet::verbString(Verb v)
 		case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME";
 		case VERB_SET_EPHEMERAL_KEY: return "SET_EPHEMERAL_KEY";
 		case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS";
+		case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST";
+		case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT";
 	}
 	return "(unknown)";
 }
@@ -73,6 +77,8 @@ const char *Packet::errorString(ErrorCode e)
 	return "(unknown)";
 }
 
+#endif // ZT_TRACE
+
 void Packet::armor(const void *key,bool encryptPayload)
 {
 	unsigned char mangledKey[32];

node/Packet.hpp

@@ -553,10 +553,10 @@ public:
 		 * address that require re-establishing connectivity.
 		 *
 		 * Destination address types and formats (not all of these are used now):
-		 *   0 - None -- no destination address data present
-		 *   1 - Ethernet address -- format: <[6] Ethernet MAC>
-		 *   4 - 6-byte IPv4 UDP address/port -- format: <[4] IP>, <[2] port>
-		 *   6 - 18-byte IPv6 UDP address/port -- format: <[16] IP>, <[2] port>
+		 *   0x00 - None -- no destination address data present
+		 *   0x01 - Ethernet address -- format: <[6] Ethernet MAC>
+		 *   0x04 - 6-byte IPv4 UDP address/port -- format: <[4] IP>, <[2] port>
+		 *   0x06 - 18-byte IPv6 UDP address/port -- format: <[16] IP>, <[2] port>
 		 *
 		 * OK payload:
 		 *   <[8] timestamp (echoed from original HELLO)>
@@ -904,7 +904,120 @@ public:
 		 *
 		 * OK and ERROR are not generated.
 		 */
-		VERB_PUSH_DIRECT_PATHS = 16
+		VERB_PUSH_DIRECT_PATHS = 16,
+
+		/* Source-routed circuit test message:
+		 *   <[5] address of originator of circuit test>
+		 *   <[2] 16-bit flags>
+		 *   <[8] 64-bit timestamp>
+		 *   <[8] 64-bit test ID (arbitrary, set by tester)>
+		 *   <[2] 16-bit originator credential length (includes type)>
+		 *   [[1] originator credential type (for authorizing test)]
+		 *   [[...] originator credential]
+		 *   <[2] 16-bit length of additional fields>
+		 *   [[...] additional fields]
+		 *   [ ... end of signed portion of request ... ]
+		 *   <[2] 16-bit length of signature of request>
+		 *   <[...] signature of request by originator>
+		 *   <[2] 16-bit previous hop credential length (including type)>
+		 *   [[1] previous hop credential type]
+		 *   [[...] previous hop credential]
+		 *   <[...] next hop(s) in path>
+		 *
+		 * Flags:
+		 *   0x01 - Report back to originator at middle hops
+		 *   0x02 - Report back to originator at last hop
+		 *
+		 * Originator credential types:
+		 *   0x01 - 64-bit network ID for which originator is controller
+		 *
+		 * Previous hop credential types:
+		 *   0x01 - Certificate of network membership
+		 *
+		 * Path record format:
+		 *   <[1] 8-bit flags (unused, must be zero)>
+		 *   <[1] 8-bit breadth (number of next hops)>
+		 *   <[...] one or more ZeroTier addresses of next hops>
+		 *
+		 * The circuit test allows a device to send a message that will traverse
+		 * the network along a specified path, with each hop optionally reporting
+		 * back to the tester via VERB_CIRCUIT_TEST_REPORT.
+		 *
+		 * Each circuit test packet includes a digital signature by the originator
+		 * of the request, as well as a credential by which that originator claims
+		 * authorization to perform the test. Currently this signature is ed25519,
+		 * but in the future flags might be used to indicate an alternative
+		 * algorithm. For example, the originator might be a network controller.
+		 * In this case the test might be authorized if the recipient is a member
+		 * of a network controlled by it, and if the previous hop(s) are also
+		 * members. Each hop may include its certificate of network membership.
+		 *
+		 * Circuit test paths consist of a series of records. When a node receives
+		 * an authorized circuit test, it:
+		 *
+		 * (1) Reports back to circuit tester as flags indicate
+		 * (2) Reads and removes the next hop from the packet's path
+		 * (3) Sends the packet along to next hop(s), if any.
+		 *
+		 * It is perfectly legal for a path to contain the same hop more than
+		 * once. In fact, this can be a very useful test to determine if a hop
+		 * can be reached bidirectionally and if so what that connectivity looks
+		 * like.
+		 *
+		 * The breadth field in source-routed path records allows a hop to forward
+		 * to more than one recipient, allowing the tester to specify different
+		 * forms of graph traversal in a test.
+		 *
+		 * There is no hard limit to the number of hops in a test, but it is
+		 * practically limited by the maximum size of a (possibly fragmented)
+		 * ZeroTier packet.
+		 *
+		 * Support for circuit tests is optional. If they are not supported, the
+		 * node should respond with an UNSUPPORTED_OPERATION error. If a circuit
+		 * test request is not authorized, it may be ignored or reported as
+		 * an INVALID_REQUEST. No OK messages are generated, but TEST_REPORT
+		 * messages may be sent (see below).
+		 *
+		 * ERROR packet format:
+		 *   <[8] 64-bit timestamp (echoed from original>
+		 *   <[8] 64-bit test ID (echoed from original)>
+		 */
+		VERB_CIRCUIT_TEST = 17,
+
+		/* Circuit test hop report:
+		 *   <[8] 64-bit timestamp (from original test)>
+		 *   <[8] 64-bit test ID (from original test)>
+		 *   <[8] 64-bit reporter timestamp (reporter's clock, 0 if unspec)>
+		 *   <[1] 8-bit vendor ID (set to 0, currently unused)>
+		 *   <[1] 8-bit reporter protocol version>
+		 *   <[1] 8-bit reporter major version>
+		 *   <[1] 8-bit reporter minor version>
+		 *   <[2] 16-bit reporter revision>
+		 *   <[2] 16-bit reporter OS/platform>
+		 *   <[2] 16-bit reporter architecture>
+		 *   <[2] 16-bit error code (set to 0, currently unused)>
+		 *   <[8] 64-bit report flags (set to 0, currently unused)>
+		 *   <[8] 64-bit source packet ID>
+		 *   <[1] 8-bit source packet hop count (ZeroTier hop count)>
+		 *   <[...] local wire address on which packet was received>
+		 *   <[...] remote wire address from which packet was received>
+		 *   <[2] 16-bit length of additional fields>
+		 *   <[...] additional fields>
+		 *   <[1] 8-bit number of next hops (breadth)>
+		 *   <[...] next hop information>
+		 *
+		 * Next hop information record format:
+		 *   <[5] ZeroTier address of next hop>
+		 *   <[...] current best direct path address, if any, 0 if none>
+		 *
+		 * Circuit test reports can be sent by hops in a circuit test to report
+		 * back results. They should include information about the sender as well
+		 * as about the paths to which next hops are being sent.
+		 *
+		 * If a test report is received and no circuit test was sent, it should be
+		 * ignored. This message generates no OK or ERROR response.
+		 */
+		VERB_CIRCUIT_TEST_REPORT = 18
 	};
 
 	/**
@@ -940,19 +1053,12 @@ public:
 		ERROR_UNWANTED_MULTICAST = 8
 	};
 
-	/**
-	 * @param v Verb
-	 * @return String representation (e.g. HELLO, OK)
-	 */
+#ifdef ZT_TRACE
 	static const char *verbString(Verb v)
 		throw();
-
-	/**
-	 * @param e Error code
-	 * @return String error name
-	 */
 	static const char *errorString(ErrorCode e)
 		throw();
+#endif
 
 	template<unsigned int C2>
 	Packet(const Buffer<C2> &b) :

node/Path.hpp

@@ -57,13 +57,13 @@ public:
 	 * Nearly all paths will be normal trust. The other levels are for high
 	 * performance local SDN use only.
 	 *
-	 * These values MUST match ZT1_LocalInterfaceAddressTrust in ZeroTierOne.h
+	 * These values MUST match ZT_LocalInterfaceAddressTrust in ZeroTierOne.h
 	 */
-	enum Trust
+	enum Trust // NOTE: max 255
 	{
 		TRUST_NORMAL = 0,
-		TRUST_PRIVACY = 1,
-		TRUST_ULTIMATE = 2
+		TRUST_PRIVACY = 10,
+		TRUST_ULTIMATE = 20
 	};
 
 	Path() :
@@ -114,7 +114,7 @@ public:
 	 */
 	inline bool reliable() const throw()
 	{
-		return ((_ipScope != InetAddress::IP_SCOPE_GLOBAL)&&(_ipScope != InetAddress::IP_SCOPE_PSEUDOPRIVATE));
+		return ( (_addr.ss_family == AF_INET6) || ((_ipScope != InetAddress::IP_SCOPE_GLOBAL)&&(_ipScope != InetAddress::IP_SCOPE_PSEUDOPRIVATE)) );
 	}
 
 	/**
@@ -122,14 +122,6 @@ public:
 	 */
 	inline operator bool() const throw() { return (_addr); }
 
-	// Comparisons are by address only
-	inline bool operator==(const Path &p) const throw() { return (_addr == p._addr); }
-	inline bool operator!=(const Path &p) const throw() { return (_addr != p._addr); }
-	inline bool operator<(const Path &p) const throw() { return (_addr < p._addr); }
-	inline bool operator>(const Path &p) const throw() { return (_addr > p._addr); }
-	inline bool operator<=(const Path &p) const throw() { return (_addr <= p._addr); }
-	inline bool operator>=(const Path &p) const throw() { return (_addr >= p._addr); }
-
 	/**
 	 * Check whether this address is valid for a ZeroTier path
 	 *
@@ -163,7 +155,7 @@ public:
 		return false;
 	}
 
-private:
+protected:
 	InetAddress _addr;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 	Trust _trust;

node/Peer.cpp

@@ -37,6 +37,8 @@
 
 #include <algorithm>
 
+#define ZT_PEER_PATH_SORT_INTERVAL 5000
+
 namespace ZeroTier {
 
 // Used to send varying values for NAT keepalive
@@ -51,12 +53,15 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastAnnouncedTo(0),
 	_lastPathConfirmationSent(0),
 	_lastDirectPathPush(0),
+	_lastPathSort(0),
 	_vMajor(0),
 	_vMinor(0),
 	_vRevision(0),
 	_id(peerIdentity),
 	_numPaths(0),
-	_latency(0)
+	_latency(0),
+	_networkComs(4),
+	_lastPushedComs(4)
 {
 	if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
 		throw std::runtime_error("new peer identity key agreement failed");
@@ -64,7 +69,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 
 void Peer::received(
 	const RuntimeEnvironment *RR,
-	int localInterfaceId,
+	const InetAddress &localAddr,
 	const InetAddress &remoteAddr,
 	unsigned int hops,
 	uint64_t packetId,
@@ -73,116 +78,92 @@ void Peer::received(
 	Packet::Verb inReVerb)
 {
 	const uint64_t now = RR->node->now();
-	_lastReceive = now;
+	bool needMulticastGroupAnnounce = false;
 
-	if (!hops) {
-		bool pathIsConfirmed = false;
+	{
+		Mutex::Lock _l(_lock);
 
-		/* Learn new paths from direct (hops == 0) packets */
-		{
-			unsigned int np = _numPaths;
-			for(unsigned int p=0;p<np;++p) {
-				if ((_paths[p].address() == remoteAddr)&&(_paths[p].localInterfaceId() == localInterfaceId)) {
-					_paths[p].received(now);
-					pathIsConfirmed = true;
-					break;
-				}
-			}
+		_lastReceive = now;
 
-			if (!pathIsConfirmed) {
-				if ((verb == Packet::VERB_OK)&&(inReVerb == Packet::VERB_HELLO)) {
-					// Learn paths if they've been confirmed via a HELLO
-					RemotePath *slot = (RemotePath *)0;
-					if (np < ZT1_MAX_PEER_NETWORK_PATHS) {
-						// Add new path
-						slot = &(_paths[np++]);
-					} else {
-						// Replace oldest non-fixed path
-						uint64_t slotLRmin = 0xffffffffffffffffULL;
-						for(unsigned int p=0;p<ZT1_MAX_PEER_NETWORK_PATHS;++p) {
-							if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
-								slotLRmin = _paths[p].lastReceived();
-								slot = &(_paths[p]);
-							}
-						}
-					}
-					if (slot) {
-						*slot = RemotePath(localInterfaceId,remoteAddr,false);
-						slot->received(now);
-						_numPaths = np;
+		if (!hops) {
+			bool pathIsConfirmed = false;
+
+			/* Learn new paths from direct (hops == 0) packets */
+			{
+				unsigned int np = _numPaths;
+				for(unsigned int p=0;p<np;++p) {
+					if ((_paths[p].address() == remoteAddr)&&(_paths[p].localAddress() == localAddr)) {
+						_paths[p].received(now);
 						pathIsConfirmed = true;
-					}
-				} else {
-					/* If this path is not known, send a HELLO. We don't learn
-					 * paths without confirming that a bidirectional link is in
-					 * fact present, but any packet that decodes and authenticates
-					 * correctly is considered valid. */
-					if ((now - _lastPathConfirmationSent) >= ZT_MIN_PATH_CONFIRMATION_INTERVAL) {
-						_lastPathConfirmationSent = now;
-						TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
-						attemptToContactAt(RR,localInterfaceId,remoteAddr,now);
+						break;
 					}
 				}
-			}
-		}
 
-		/* Announce multicast groups of interest to direct peers if they are
-		 * considered authorized members of a given network. Also announce to
-		 * root servers and network controllers. */
-		if ((pathIsConfirmed)&&((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000))) {
-			_lastAnnouncedTo = now;
+				if (!pathIsConfirmed) {
+					if ((verb == Packet::VERB_OK)&&(inReVerb == Packet::VERB_HELLO)) {
+
+						// Learn paths if they've been confirmed via a HELLO
+						RemotePath *slot = (RemotePath *)0;
+						if (np < ZT_MAX_PEER_NETWORK_PATHS) {
+							// Add new path
+							slot = &(_paths[np++]);
+						} else {
+							// Replace oldest non-fixed path
+							uint64_t slotLRmin = 0xffffffffffffffffULL;
+							for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
+								if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
+									slotLRmin = _paths[p].lastReceived();
+									slot = &(_paths[p]);
+								}
+							}
+						}
+						if (slot) {
+							*slot = RemotePath(localAddr,remoteAddr,false);
+							slot->received(now);
+							_numPaths = np;
+							pathIsConfirmed = true;
+							_sortPaths(now);
+						}
 
-			const bool isRoot = RR->topology->isRoot(_id);
-
-			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
-			const std::vector< SharedPtr<Network> > networks(RR->node->allNetworks());
-			for(std::vector< SharedPtr<Network> >::const_iterator n(networks.begin());n!=networks.end();++n) {
-				if ( (isRoot) || ((*n)->isAllowed(_id.address())) || (_id.address() == (*n)->controller()) ) {
-					const std::vector<MulticastGroup> mgs((*n)->allMulticastGroups());
-					for(std::vector<MulticastGroup>::const_iterator mg(mgs.begin());mg!=mgs.end();++mg) {
-						if ((outp.size() + 18) > ZT_UDP_DEFAULT_PAYLOAD_MTU) {
-							outp.armor(_key,true);
-							RR->node->putPacket(localInterfaceId,remoteAddr,outp.data(),outp.size());
-							outp.reset(_id.address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
+					} else {
+
+						/* If this path is not known, send a HELLO. We don't learn
+						 * paths without confirming that a bidirectional link is in
+						 * fact present, but any packet that decodes and authenticates
+						 * correctly is considered valid. */
+						if ((now - _lastPathConfirmationSent) >= ZT_MIN_PATH_CONFIRMATION_INTERVAL) {
+							_lastPathConfirmationSent = now;
+							TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
+							attemptToContactAt(RR,localAddr,remoteAddr,now);
 						}
 
-						// network ID, MAC, ADI
-						outp.append((uint64_t)(*n)->id());
-						mg->mac().appendTo(outp);
-						outp.append((uint32_t)mg->adi());
 					}
 				}
 			}
-			if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) {
-				outp.armor(_key,true);
-				RR->node->putPacket(localInterfaceId,remoteAddr,outp.data(),outp.size());
-			}
 		}
-	}
 
-	if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
-		_lastUnicastFrame = now;
-	else if (verb == Packet::VERB_MULTICAST_FRAME)
-		_lastMulticastFrame = now;
-}
-
-RemotePath *Peer::getBestPath(uint64_t now)
-{
-	RemotePath *bestPath = (RemotePath *)0;
-	uint64_t lrMax = 0;
-	int rank = 0;
-	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		if ( (_paths[p].active(now)) && ((_paths[p].lastReceived() >= lrMax)||(_paths[p].preferenceRank() >= rank)) ) {
-			lrMax = _paths[p].lastReceived();
-			rank = _paths[p].preferenceRank();
-			bestPath = &(_paths[p]);
+		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
+			_lastAnnouncedTo = now;
+			needMulticastGroupAnnounce = true;
 		}
+
+		if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
+			_lastUnicastFrame = now;
+		else if (verb == Packet::VERB_MULTICAST_FRAME)
+			_lastMulticastFrame = now;
+	}
+
+	if (needMulticastGroupAnnounce) {
+		const std::vector< SharedPtr<Network> > networks(RR->node->allNetworks());
+		for(std::vector< SharedPtr<Network> >::const_iterator n(networks.begin());n!=networks.end();++n)
+			(*n)->tryAnnounceMulticastGroupsTo(SharedPtr<Peer>(this));
 	}
-	return bestPath;
 }
 
-void Peer::attemptToContactAt(const RuntimeEnvironment *RR,int localInterfaceId,const InetAddress &atAddress,uint64_t now)
+void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now)
 {
+	// _lock not required here since _id is immutable and nothing else is accessed
+
 	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
 	outp.append((unsigned char)ZT_PROTO_VERSION);
 	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
@@ -209,21 +190,22 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,int localInterfaceId,
 	}
 
 	outp.armor(_key,false); // HELLO is sent in the clear
-	RR->node->putPacket(localInterfaceId,atAddress,outp.data(),outp.size());
+	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 }
 
 void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 {
-	RemotePath *const bestPath = getBestPath(now);
+	Mutex::Lock _l(_lock);
+	RemotePath *const bestPath = _getBestPath(now);
 	if (bestPath) {
 		if ((now - bestPath->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
 			TRACE("PING %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
-			attemptToContactAt(RR,bestPath->localInterfaceId(),bestPath->address(),now);
+			attemptToContactAt(RR,bestPath->localAddress(),bestPath->address(),now);
 			bestPath->sent(now);
 		} else if (((now - bestPath->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!bestPath->reliable())) {
 			_natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads
 			TRACE("NAT keepalive %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
-			RR->node->putPacket(bestPath->localInterfaceId(),bestPath->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
+			RR->node->putPacket(bestPath->localAddress(),bestPath->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
 			bestPath->sent(now);
 		}
 	}
@@ -231,6 +213,8 @@ void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 
 void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
 {
+	Mutex::Lock _l(_lock);
+
 	if (((now - _lastDirectPathPush) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
 		_lastDirectPathPush = now;
 
@@ -299,25 +283,28 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	}
 }
 
-void Peer::addPath(const RemotePath &newp)
+void Peer::addPath(const RemotePath &newp,uint64_t now)
 {
+	Mutex::Lock _l(_lock);
+
 	unsigned int np = _numPaths;
 
 	for(unsigned int p=0;p<np;++p) {
 		if (_paths[p].address() == newp.address()) {
 			_paths[p].setFixed(newp.fixed());
+			_sortPaths(now);
 			return;
 		}
 	}
 
 	RemotePath *slot = (RemotePath *)0;
-	if (np < ZT1_MAX_PEER_NETWORK_PATHS) {
+	if (np < ZT_MAX_PEER_NETWORK_PATHS) {
 		// Add new path
 		slot = &(_paths[np++]);
 	} else {
 		// Replace oldest non-fixed path
 		uint64_t slotLRmin = 0xffffffffffffffffULL;
-		for(unsigned int p=0;p<ZT1_MAX_PEER_NETWORK_PATHS;++p) {
+		for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
 			if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
 				slotLRmin = _paths[p].lastReceived();
 				slot = &(_paths[p]);
@@ -328,6 +315,8 @@ void Peer::addPath(const RemotePath &newp)
 		*slot = newp;
 		_numPaths = np;
 	}
+
+	_sortPaths(now);
 }
 
 void Peer::clearPaths(bool fixedToo)
@@ -349,13 +338,14 @@ void Peer::clearPaths(bool fixedToo)
 
 bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope scope,uint64_t now)
 {
+	Mutex::Lock _l(_lock);
 	unsigned int np = _numPaths;
 	unsigned int x = 0;
 	unsigned int y = 0;
 	while (x < np) {
 		if (_paths[x].address().ipScope() == scope) {
 			if (_paths[x].fixed()) {
-				attemptToContactAt(RR,_paths[x].localInterfaceId(),_paths[x].address(),now);
+				attemptToContactAt(RR,_paths[x].localAddress(),_paths[x].address(),now);
 				_paths[y++] = _paths[x]; // keep fixed paths
 			}
 		} else {
@@ -364,11 +354,13 @@ bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope sc
 		++x;
 	}
 	_numPaths = y;
+	_sortPaths(now);
 	return (y < np);
 }
 
 void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const
 {
+	Mutex::Lock _l(_lock);
 	uint64_t bestV4 = 0,bestV6 = 0;
 	for(unsigned int p=0,np=_numPaths;p<np;++p) {
 		if (_paths[p].active(now)) {
@@ -390,4 +382,154 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6)
 	}
 }
 
+bool Peer::networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const
+{
+	Mutex::Lock _l(_lock);
+	const _NetworkCom *ourCom = _networkComs.get(nwid);
+	if (ourCom)
+		return ourCom->com.agreesWith(com);
+	return false;
+}
+
+bool Peer::validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com)
+{
+	// Sanity checks
+	if ((!com)||(com.issuedTo() != _id.address()))
+		return false;
+
+	// Return true if we already have this *exact* COM
+	{
+		Mutex::Lock _l(_lock);
+		_NetworkCom *ourCom = _networkComs.get(nwid);
+		if ((ourCom)&&(ourCom->com == com))
+			return true;
+	}
+
+	// Check signature, log and return if cert is invalid
+	if (com.signedBy() != Network::controllerFor(nwid)) {
+		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,com.signedBy().toString().c_str());
+		return false; // invalid signer
+	}
+
+	if (com.signedBy() == RR->identity.address()) {
+
+		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
+		// So, verify that we signed th cert ourself
+		if (!com.verify(RR->identity)) {
+			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,com.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+
+	} else {
+
+		SharedPtr<Peer> signer(RR->topology->getPeer(com.signedBy()));
+
+		if (!signer) {
+			// This would be rather odd, since this is our controller... could happen
+			// if we get packets before we've gotten config.
+			RR->sw->requestWhois(com.signedBy());
+			return false; // signer unknown
+		}
+
+		if (!com.verify(signer->identity())) {
+			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,com.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+	}
+
+	// If we made it past all those checks, add or update cert in our cert info store
+	{
+		Mutex::Lock _l(_lock);
+		_networkComs.set(nwid,_NetworkCom(RR->node->now(),com));
+	}
+
+	return true;
+}
+
+bool Peer::needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime)
+{
+	Mutex::Lock _l(_lock);
+	uint64_t &lastPushed = _lastPushedComs[nwid];
+	const uint64_t tmp = lastPushed;
+	if (updateLastPushedTime)
+		lastPushed = now;
+	return ((now - tmp) >= (ZT_NETWORK_AUTOCONF_DELAY / 2));
+}
+
+void Peer::clean(const RuntimeEnvironment *RR,uint64_t now)
+{
+	Mutex::Lock _l(_lock);
+
+	{
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if (_paths[x].active(now))
+				_paths[y++] = _paths[x];
+			++x;
+		}
+		_numPaths = y;
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		_NetworkCom *v = (_NetworkCom *)0;
+		Hashtable< uint64_t,_NetworkCom >::Iterator i(_networkComs);
+		while (i.next(k,v)) {
+			if ( (!RR->node->belongsToNetwork(*k)) && ((now - v->ts) >= ZT_PEER_NETWORK_COM_EXPIRATION) )
+				_networkComs.erase(*k);
+		}
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		uint64_t *v = (uint64_t *)0;
+		Hashtable< uint64_t,uint64_t >::Iterator i(_lastPushedComs);
+		while (i.next(k,v)) {
+			if ((now - *v) > (ZT_NETWORK_AUTOCONF_DELAY * 2))
+				_lastPushedComs.erase(*k);
+		}
+	}
+}
+
+struct _SortPathsByQuality
+{
+	uint64_t _now;
+	_SortPathsByQuality(const uint64_t now) : _now(now) {}
+	inline bool operator()(const RemotePath &a,const RemotePath &b) const
+	{
+		const uint64_t qa = (
+			((uint64_t)a.active(_now) << 63) |
+			(((uint64_t)(a.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)a.lastReceived() & 0x7ffffffffffffULL) );
+		const uint64_t qb = (
+			((uint64_t)b.active(_now) << 63) |
+			(((uint64_t)(b.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)b.lastReceived() & 0x7ffffffffffffULL) );
+		return (qb < qa); // invert sense to sort in descending order
+	}
+};
+void Peer::_sortPaths(const uint64_t now)
+{
+	// assumes _lock is locked
+	_lastPathSort = now;
+	std::sort(&(_paths[0]),&(_paths[_numPaths]),_SortPathsByQuality(now));
+}
+
+RemotePath *Peer::_getBestPath(const uint64_t now)
+{
+	// assumes _lock is locked
+	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
+		_sortPaths(now);
+	if (_paths[0].active(now)) {
+		return &(_paths[0]);
+	} else {
+		_sortPaths(now);
+		if (_paths[0].active(now))
+			return &(_paths[0]);
+	}
+	return (RemotePath *)0;
+}
+
 } // namespace ZeroTier

node/Peer.hpp

@@ -49,8 +49,14 @@
 #include "Packet.hpp"
 #include "SharedPtr.hpp"
 #include "AtomicCounter.hpp"
+#include "Hashtable.hpp"
+#include "Mutex.hpp"
 #include "NonCopyable.hpp"
 
+// Very rough computed estimate: (8 + 256 + 80 + (16 * 64) + (128 * 256) + (128 * 16))
+// 1048576 provides tons of headroom -- overflow would just cause peer not to be persisted
+#define ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE 1048576
+
 namespace ZeroTier {
 
 /**
@@ -105,7 +111,7 @@ public:
 	 * and appears to be valid.
 	 *
 	 * @param RR Runtime environment
-	 * @param localInterfaceId Local interface ID or -1 if unspecified
+	 * @param localAddr Local address
 	 * @param remoteAddr Internet address of sender
 	 * @param hops ZeroTier (not IP) hops
 	 * @param packetId Packet ID
@@ -115,7 +121,7 @@ public:
 	 */
 	void received(
 		const RuntimeEnvironment *RR,
-		int localInterfaceId,
+		const InetAddress &localAddr,
 		const InetAddress &remoteAddr,
 		unsigned int hops,
 		uint64_t packetId,
@@ -129,7 +135,11 @@ public:
 	 * @param now Current time
 	 * @return Best path or NULL if there are no active (or fixed) direct paths
 	 */
-	RemotePath *getBestPath(uint64_t now);
+	inline RemotePath *getBestPath(uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		return _getBestPath(now);
+	}
 
 	/**
 	 * Send via best path
@@ -157,11 +167,11 @@ public:
 	 * for NAT traversal and path verification.
 	 *
 	 * @param RR Runtime environment
-	 * @param localInterfaceId Local interface ID or -1 for unspecified
+	 * @param localAddr Local address
 	 * @param atAddress Destination address
 	 * @param now Current time
 	 */
-	void attemptToContactAt(const RuntimeEnvironment *RR,int localInterfaceId,const InetAddress &atAddress,uint64_t now);
+	void attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now);
 
 	/**
 	 * Send pings or keepalives depending on configured timeouts
@@ -187,6 +197,7 @@ public:
 	inline std::vector<RemotePath> paths() const
 	{
 		std::vector<RemotePath> pp;
+		Mutex::Lock _l(_lock);
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
 			pp.push_back(_paths[p]);
 		return pp;
@@ -198,6 +209,7 @@ public:
 	inline uint64_t lastDirectReceive() const
 		throw()
 	{
+		Mutex::Lock _l(_lock);
 		uint64_t x = 0;
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
 			x = std::max(x,_paths[p].lastReceived());
@@ -210,6 +222,7 @@ public:
 	inline uint64_t lastDirectSend() const
 		throw()
 	{
+		Mutex::Lock _l(_lock);
 		uint64_t x = 0;
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
 			x = std::max(x,_paths[p].lastSend());
@@ -282,6 +295,7 @@ public:
 	inline bool hasActiveDirectPath(uint64_t now) const
 		throw()
 	{
+		Mutex::Lock _l(_lock);
 		for(unsigned int p=0,np=_numPaths;p<np;++p) {
 			if (_paths[p].active(now))
 				return true;
@@ -293,8 +307,9 @@ public:
 	 * Add a path (if we don't already have it)
 	 *
 	 * @param p New path to add
+	 * @param now Current time
 	 */
-	void addPath(const RemotePath &newp);
+	void addPath(const RemotePath &newp,uint64_t now);
 
 	/**
 	 * Clear paths
@@ -331,6 +346,7 @@ public:
 	 */
 	inline void setRemoteVersion(unsigned int vproto,unsigned int vmaj,unsigned int vmin,unsigned int vrev)
 	{
+		Mutex::Lock _l(_lock);
 		_vProto = (uint16_t)vproto;
 		_vMajor = (uint16_t)vmaj;
 		_vMinor = (uint16_t)vmin;
@@ -354,6 +370,7 @@ public:
 	inline bool atLeastVersion(unsigned int major,unsigned int minor,unsigned int rev)
 		throw()
 	{
+		Mutex::Lock _l(_lock);
 		if ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)) {
 			if (_vMajor > major)
 				return true;
@@ -381,6 +398,37 @@ public:
 	 */
 	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
 
+	/**
+	 * Check network COM agreement with this peer
+	 *
+	 * @param nwid Network ID
+	 * @param com Another certificate of membership
+	 * @return True if supplied COM agrees with ours, false if not or if we don't have one
+	 */
+	bool networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const;
+
+	/**
+	 * Check the validity of the COM and add/update if valid and new
+	 *
+	 * @param RR Runtime Environment
+	 * @param nwid Network ID
+	 * @param com Externally supplied COM
+	 */
+	bool validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com);
+
+	/**
+	 * @param nwid Network ID
+	 * @param now Current time
+	 * @param updateLastPushedTime If true, go ahead and update the last pushed time regardless of return value
+	 * @return Whether or not this peer needs another COM push from us
+	 */
+	bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime);
+
+	/**
+	 * Perform periodic cleaning operations
+	 */
+	void clean(const RuntimeEnvironment *RR,uint64_t now);
+
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
 	 *
@@ -401,8 +449,133 @@ public:
 		else return std::pair<InetAddress,InetAddress>();
 	}
 
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		Mutex::Lock _l(_lock);
+
+		const unsigned int atPos = b.size();
+		b.addSize(4); // space for uint32_t field length
+
+		b.append((uint32_t)1); // version of serialized Peer data
+
+		_id.serialize(b,false);
+
+		b.append((uint64_t)_lastUsed);
+		b.append((uint64_t)_lastReceive);
+		b.append((uint64_t)_lastUnicastFrame);
+		b.append((uint64_t)_lastMulticastFrame);
+		b.append((uint64_t)_lastAnnouncedTo);
+		b.append((uint64_t)_lastPathConfirmationSent);
+		b.append((uint64_t)_lastDirectPathPush);
+		b.append((uint64_t)_lastPathSort);
+		b.append((uint16_t)_vProto);
+		b.append((uint16_t)_vMajor);
+		b.append((uint16_t)_vMinor);
+		b.append((uint16_t)_vRevision);
+		b.append((uint32_t)_latency);
+
+		b.append((uint32_t)_numPaths);
+		for(unsigned int i=0;i<_numPaths;++i)
+			_paths[i].serialize(b);
+
+		b.append((uint32_t)_networkComs.size());
+		{
+			uint64_t *k = (uint64_t *)0;
+			_NetworkCom *v = (_NetworkCom *)0;
+			Hashtable<uint64_t,_NetworkCom>::Iterator i(const_cast<Peer *>(this)->_networkComs);
+			while (i.next(k,v)) {
+				b.append((uint64_t)*k);
+				b.append((uint64_t)v->ts);
+				v->com.serialize(b);
+			}
+		}
+
+		b.append((uint32_t)_lastPushedComs.size());
+		{
+			uint64_t *k = (uint64_t *)0;
+			uint64_t *v = (uint64_t *)0;
+			Hashtable<uint64_t,uint64_t>::Iterator i(const_cast<Peer *>(this)->_lastPushedComs);
+			while (i.next(k,v)) {
+				b.append((uint64_t)*k);
+				b.append((uint64_t)*v);
+			}
+		}
+
+		b.setAt(atPos,(uint32_t)(b.size() - atPos)); // set size
+	}
+
+	/**
+	 * Create a new Peer from a serialized instance
+	 *
+	 * @param myIdentity This node's identity
+	 * @param b Buffer containing serialized Peer data
+	 * @param p Pointer to current position in buffer, will be updated in place as buffer is read (value/result)
+	 * @return New instance of Peer or NULL if serialized data was corrupt or otherwise invalid (may also throw an exception via Buffer)
+	 */
+	template<unsigned int C>
+	static inline SharedPtr<Peer> deserializeNew(const Identity &myIdentity,const Buffer<C> &b,unsigned int &p)
+	{
+		const uint32_t recSize = b.template at<uint32_t>(p);
+		if ((p + recSize) > b.size())
+			return SharedPtr<Peer>(); // size invalid
+		p += 4;
+		if (b.template at<uint32_t>(p) != 1)
+			return SharedPtr<Peer>(); // version mismatch
+		p += 4;
+
+		Identity npid;
+		p += npid.deserialize(b,p);
+		if (!npid)
+			return SharedPtr<Peer>();
+
+		SharedPtr<Peer> np(new Peer(myIdentity,npid));
+
+		np->_lastUsed = b.template at<uint64_t>(p); p += 8;
+		np->_lastReceive = b.template at<uint64_t>(p); p += 8;
+		np->_lastUnicastFrame = b.template at<uint64_t>(p); p += 8;
+		np->_lastMulticastFrame = b.template at<uint64_t>(p); p += 8;
+		np->_lastAnnouncedTo = b.template at<uint64_t>(p); p += 8;
+		np->_lastPathConfirmationSent = b.template at<uint64_t>(p); p += 8;
+		np->_lastDirectPathPush = b.template at<uint64_t>(p); p += 8;
+		np->_lastPathSort = b.template at<uint64_t>(p); p += 8;
+		np->_vProto = b.template at<uint16_t>(p); p += 2;
+		np->_vMajor = b.template at<uint16_t>(p); p += 2;
+		np->_vMinor = b.template at<uint16_t>(p); p += 2;
+		np->_vRevision = b.template at<uint16_t>(p); p += 2;
+		np->_latency = b.template at<uint32_t>(p); p += 4;
+
+		const unsigned int numPaths = b.template at<uint32_t>(p); p += 4;
+		for(unsigned int i=0;i<numPaths;++i) {
+			if (i < ZT_MAX_PEER_NETWORK_PATHS) {
+				p += np->_paths[np->_numPaths++].deserialize(b,p);
+			} else {
+				// Skip any paths beyond max, but still read stream
+				RemotePath foo;
+				p += foo.deserialize(b,p);
+			}
+		}
+
+		const unsigned int numNetworkComs = b.template at<uint32_t>(p); p += 4;
+		for(unsigned int i=0;i<numNetworkComs;++i) {
+			_NetworkCom &c = np->_networkComs[b.template at<uint64_t>(p)]; p += 8;
+			c.ts = b.template at<uint64_t>(p); p += 8;
+			p += c.com.deserialize(b,p);
+		}
+
+		const unsigned int numLastPushed = b.template at<uint32_t>(p); p += 4;
+		for(unsigned int i=0;i<numLastPushed;++i) {
+			const uint64_t nwid = b.template at<uint64_t>(p); p += 8;
+			const uint64_t ts = b.template at<uint64_t>(p); p += 8;
+			np->_lastPushedComs.set(nwid,ts);
+		}
+
+		return np;
+	}
+
 private:
-	void _announceMulticastGroups(const RuntimeEnvironment *RR,uint64_t now);
+	void _sortPaths(const uint64_t now);
+	RemotePath *_getBestPath(const uint64_t now);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	uint64_t _lastUsed;
@@ -412,15 +585,27 @@ private:
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastPathConfirmationSent;
 	uint64_t _lastDirectPathPush;
+	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 	Identity _id;
-	RemotePath _paths[ZT1_MAX_PEER_NETWORK_PATHS];
+	RemotePath _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	unsigned int _numPaths;
 	unsigned int _latency;
 
+	struct _NetworkCom
+	{
+		_NetworkCom() {}
+		_NetworkCom(uint64_t t,const CertificateOfMembership &c) : ts(t),com(c) {}
+		uint64_t ts;
+		CertificateOfMembership com;
+	};
+	Hashtable<uint64_t,_NetworkCom> _networkComs;
+	Hashtable<uint64_t,uint64_t> _lastPushedComs;
+
+	Mutex _lock;
 	AtomicCounter __refCount;
 };
 

node/Poly1305.cpp

@@ -7,14 +7,18 @@ Public domain.
 #include "Constants.hpp"
 #include "Poly1305.hpp"
 
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
 #ifdef __WINDOWS__
 #pragma warning(disable: 4146)
 #endif
 
 namespace ZeroTier {
 
-//////////////////////////////////////////////////////////////////////////////
-//////////////////////////////////////////////////////////////////////////////
+#if 0
 
 static inline void add(unsigned int h[17],const unsigned int c[17])
 {
@@ -113,13 +117,278 @@ static inline int crypto_onetimeauth(unsigned char *out,const unsigned char *in,
   return 0;
 }
 
-//////////////////////////////////////////////////////////////////////////////
-//////////////////////////////////////////////////////////////////////////////
-
 void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key)
 	throw()
 {
 	crypto_onetimeauth((unsigned char *)auth,(const unsigned char *)data,len,(const unsigned char *)key);
 }
 
+#endif
+
+namespace {
+
+typedef struct poly1305_context {
+  size_t aligner;
+  unsigned char opaque[136];
+} poly1305_context;
+
+/*
+  poly1305 implementation using 32 bit * 32 bit = 64 bit multiplication and 64 bit addition
+*/
+
+#define poly1305_block_size 16
+
+/* 17 + sizeof(size_t) + 14*sizeof(unsigned long) */
+typedef struct poly1305_state_internal_t {
+  unsigned long r[5];
+  unsigned long h[5];
+  unsigned long pad[4];
+  size_t leftover;
+  unsigned char buffer[poly1305_block_size];
+  unsigned char final;
+} poly1305_state_internal_t;
+
+/* interpret four 8 bit unsigned integers as a 32 bit unsigned integer in little endian */
+static unsigned long
+U8TO32(const unsigned char *p) {
+  return
+    (((unsigned long)(p[0] & 0xff)      ) |
+       ((unsigned long)(p[1] & 0xff) <<  8) |
+         ((unsigned long)(p[2] & 0xff) << 16) |
+         ((unsigned long)(p[3] & 0xff) << 24));
+}
+
+/* store a 32 bit unsigned integer as four 8 bit unsigned integers in little endian */
+static void
+U32TO8(unsigned char *p, unsigned long v) {
+  p[0] = (v      ) & 0xff;
+  p[1] = (v >>  8) & 0xff;
+  p[2] = (v >> 16) & 0xff;
+  p[3] = (v >> 24) & 0xff;
+}
+
+static inline void
+poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+
+  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
+  st->r[0] = (U8TO32(&key[ 0])     ) & 0x3ffffff;
+  st->r[1] = (U8TO32(&key[ 3]) >> 2) & 0x3ffff03;
+  st->r[2] = (U8TO32(&key[ 6]) >> 4) & 0x3ffc0ff;
+  st->r[3] = (U8TO32(&key[ 9]) >> 6) & 0x3f03fff;
+  st->r[4] = (U8TO32(&key[12]) >> 8) & 0x00fffff;
+
+  /* h = 0 */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+  st->h[3] = 0;
+  st->h[4] = 0;
+
+  /* save pad for later */
+  st->pad[0] = U8TO32(&key[16]);
+  st->pad[1] = U8TO32(&key[20]);
+  st->pad[2] = U8TO32(&key[24]);
+  st->pad[3] = U8TO32(&key[28]);
+
+  st->leftover = 0;
+  st->final = 0;
+}
+
+static inline void
+poly1305_blocks(poly1305_state_internal_t *st, const unsigned char *m, size_t bytes) {
+  const unsigned long hibit = (st->final) ? 0 : (1 << 24); /* 1 << 128 */
+  unsigned long r0,r1,r2,r3,r4;
+  unsigned long s1,s2,s3,s4;
+  unsigned long h0,h1,h2,h3,h4;
+  unsigned long long d0,d1,d2,d3,d4;
+  unsigned long c;
+
+  r0 = st->r[0];
+  r1 = st->r[1];
+  r2 = st->r[2];
+  r3 = st->r[3];
+  r4 = st->r[4];
+
+  s1 = r1 * 5;
+  s2 = r2 * 5;
+  s3 = r3 * 5;
+  s4 = r4 * 5;
+
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+  h3 = st->h[3];
+  h4 = st->h[4];
+
+  while (bytes >= poly1305_block_size) {
+    /* h += m[i] */
+    h0 += (U8TO32(m+ 0)     ) & 0x3ffffff;
+    h1 += (U8TO32(m+ 3) >> 2) & 0x3ffffff;
+    h2 += (U8TO32(m+ 6) >> 4) & 0x3ffffff;
+    h3 += (U8TO32(m+ 9) >> 6) & 0x3ffffff;
+    h4 += (U8TO32(m+12) >> 8) | hibit;
+
+    /* h *= r */
+    d0 = ((unsigned long long)h0 * r0) + ((unsigned long long)h1 * s4) + ((unsigned long long)h2 * s3) + ((unsigned long long)h3 * s2) + ((unsigned long long)h4 * s1);
+    d1 = ((unsigned long long)h0 * r1) + ((unsigned long long)h1 * r0) + ((unsigned long long)h2 * s4) + ((unsigned long long)h3 * s3) + ((unsigned long long)h4 * s2);
+    d2 = ((unsigned long long)h0 * r2) + ((unsigned long long)h1 * r1) + ((unsigned long long)h2 * r0) + ((unsigned long long)h3 * s4) + ((unsigned long long)h4 * s3);
+    d3 = ((unsigned long long)h0 * r3) + ((unsigned long long)h1 * r2) + ((unsigned long long)h2 * r1) + ((unsigned long long)h3 * r0) + ((unsigned long long)h4 * s4);
+    d4 = ((unsigned long long)h0 * r4) + ((unsigned long long)h1 * r3) + ((unsigned long long)h2 * r2) + ((unsigned long long)h3 * r1) + ((unsigned long long)h4 * r0);
+
+    /* (partial) h %= p */
+                  c = (unsigned long)(d0 >> 26); h0 = (unsigned long)d0 & 0x3ffffff;
+    d1 += c;      c = (unsigned long)(d1 >> 26); h1 = (unsigned long)d1 & 0x3ffffff;
+    d2 += c;      c = (unsigned long)(d2 >> 26); h2 = (unsigned long)d2 & 0x3ffffff;
+    d3 += c;      c = (unsigned long)(d3 >> 26); h3 = (unsigned long)d3 & 0x3ffffff;
+    d4 += c;      c = (unsigned long)(d4 >> 26); h4 = (unsigned long)d4 & 0x3ffffff;
+    h0 += c * 5;  c =                (h0 >> 26); h0 =                h0 & 0x3ffffff;
+    h1 += c;
+
+    m += poly1305_block_size;
+    bytes -= poly1305_block_size;
+  }
+
+  st->h[0] = h0;
+  st->h[1] = h1;
+  st->h[2] = h2;
+  st->h[3] = h3;
+  st->h[4] = h4;
+}
+
+static inline void
+poly1305_update(poly1305_context *ctx, const unsigned char *m, size_t bytes) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  size_t i;
+
+  /* handle leftover */
+  if (st->leftover) {
+    size_t want = (poly1305_block_size - st->leftover);
+    if (want > bytes)
+      want = bytes;
+    for (i = 0; i < want; i++)
+      st->buffer[st->leftover + i] = m[i];
+    bytes -= want;
+    m += want;
+    st->leftover += want;
+    if (st->leftover < poly1305_block_size)
+      return;
+    poly1305_blocks(st, st->buffer, poly1305_block_size);
+    st->leftover = 0;
+  }
+
+  /* process full blocks */
+  if (bytes >= poly1305_block_size) {
+    size_t want = (bytes & ~(poly1305_block_size - 1));
+    poly1305_blocks(st, m, want);
+    m += want;
+    bytes -= want;
+  }
+
+  /* store leftover */
+  if (bytes) {
+    for (i = 0; i < bytes; i++)
+      st->buffer[st->leftover + i] = m[i];
+    st->leftover += bytes;
+  }
+}
+
+static inline void
+poly1305_finish(poly1305_context *ctx, unsigned char mac[16]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  unsigned long h0,h1,h2,h3,h4,c;
+  unsigned long g0,g1,g2,g3,g4;
+  unsigned long long f;
+  unsigned long mask;
+
+  /* process the remaining block */
+  if (st->leftover) {
+    size_t i = st->leftover;
+    st->buffer[i++] = 1;
+    for (; i < poly1305_block_size; i++)
+      st->buffer[i] = 0;
+    st->final = 1;
+    poly1305_blocks(st, st->buffer, poly1305_block_size);
+  }
+
+  /* fully carry h */
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+  h3 = st->h[3];
+  h4 = st->h[4];
+
+               c = h1 >> 26; h1 = h1 & 0x3ffffff;
+  h2 +=     c; c = h2 >> 26; h2 = h2 & 0x3ffffff;
+  h3 +=     c; c = h3 >> 26; h3 = h3 & 0x3ffffff;
+  h4 +=     c; c = h4 >> 26; h4 = h4 & 0x3ffffff;
+  h0 += c * 5; c = h0 >> 26; h0 = h0 & 0x3ffffff;
+  h1 +=     c;
+
+  /* compute h + -p */
+  g0 = h0 + 5; c = g0 >> 26; g0 &= 0x3ffffff;
+  g1 = h1 + c; c = g1 >> 26; g1 &= 0x3ffffff;
+  g2 = h2 + c; c = g2 >> 26; g2 &= 0x3ffffff;
+  g3 = h3 + c; c = g3 >> 26; g3 &= 0x3ffffff;
+  g4 = h4 + c - (1 << 26);
+
+  /* select h if h < p, or h + -p if h >= p */
+  mask = (g4 >> ((sizeof(unsigned long) * 8) - 1)) - 1;
+  g0 &= mask;
+  g1 &= mask;
+  g2 &= mask;
+  g3 &= mask;
+  g4 &= mask;
+  mask = ~mask;
+  h0 = (h0 & mask) | g0;
+  h1 = (h1 & mask) | g1;
+  h2 = (h2 & mask) | g2;
+  h3 = (h3 & mask) | g3;
+  h4 = (h4 & mask) | g4;
+
+  /* h = h % (2^128) */
+  h0 = ((h0      ) | (h1 << 26)) & 0xffffffff;
+  h1 = ((h1 >>  6) | (h2 << 20)) & 0xffffffff;
+  h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
+  h3 = ((h3 >> 18) | (h4 <<  8)) & 0xffffffff;
+
+  /* mac = (h + pad) % (2^128) */
+  f = (unsigned long long)h0 + st->pad[0]            ; h0 = (unsigned long)f;
+  f = (unsigned long long)h1 + st->pad[1] + (f >> 32); h1 = (unsigned long)f;
+  f = (unsigned long long)h2 + st->pad[2] + (f >> 32); h2 = (unsigned long)f;
+  f = (unsigned long long)h3 + st->pad[3] + (f >> 32); h3 = (unsigned long)f;
+
+  U32TO8(mac +  0, h0);
+  U32TO8(mac +  4, h1);
+  U32TO8(mac +  8, h2);
+  U32TO8(mac + 12, h3);
+
+  /* zero out the state */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+  st->h[3] = 0;
+  st->h[4] = 0;
+  st->r[0] = 0;
+  st->r[1] = 0;
+  st->r[2] = 0;
+  st->r[3] = 0;
+  st->r[4] = 0;
+  st->pad[0] = 0;
+  st->pad[1] = 0;
+  st->pad[2] = 0;
+  st->pad[3] = 0;
+}
+
+} // anonymous namespace
+
+void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key)
+  throw()
+{
+  poly1305_context ctx;
+  poly1305_init(&ctx,reinterpret_cast<const unsigned char *>(key));
+  poly1305_update(&ctx,reinterpret_cast<const unsigned char *>(data),(size_t)len);
+  poly1305_finish(&ctx,reinterpret_cast<unsigned char *>(auth));
+}
+
 } // namespace ZeroTier

node/RemotePath.hpp

@@ -39,6 +39,8 @@
 #include "AntiRecursion.hpp"
 #include "RuntimeEnvironment.hpp"
 
+#define ZT_REMOTEPATH_FLAG_FIXED 0x0001
+
 namespace ZeroTier {
 
 /**
@@ -53,17 +55,17 @@ public:
 		Path(),
 		_lastSend(0),
 		_lastReceived(0),
-		_localInterfaceId(-1),
-		_fixed(false) {}
+		_localAddress(),
+		_flags(0) {}
 
-	RemotePath(int localInterfaceId,const InetAddress &addr,bool fixed) :
+	RemotePath(const InetAddress &localAddress,const InetAddress &addr,bool fixed) :
 		Path(addr,0,TRUST_NORMAL),
 		_lastSend(0),
 		_lastReceived(0),
-		_localInterfaceId(localInterfaceId),
-		_fixed(fixed) {}
+		_localAddress(localAddress),
+		_flags(fixed ? ZT_REMOTEPATH_FLAG_FIXED : 0) {}
 
-	inline int localInterfaceId() const throw() { return _localInterfaceId; }
+	inline const InetAddress &localAddress() const throw() { return _localAddress; }
 
 	inline uint64_t lastSend() const throw() { return _lastSend; }
 	inline uint64_t lastReceived() const throw() { return _lastReceived; }
@@ -71,7 +73,7 @@ public:
 	/**
 	 * @return Is this a fixed path?
 	 */
-	inline bool fixed() const throw() { return _fixed; }
+	inline bool fixed() const throw() { return ((_flags & ZT_REMOTEPATH_FLAG_FIXED) != 0); }
 
 	/**
 	 * @param f New value of fixed flag
@@ -79,7 +81,9 @@ public:
 	inline void setFixed(const bool f)
 		throw()
 	{
-		_fixed = f;
+		if (f)
+			_flags |= ZT_REMOTEPATH_FLAG_FIXED;
+		else _flags &= ~ZT_REMOTEPATH_FLAG_FIXED;
 	}
 
 	/**
@@ -113,7 +117,7 @@ public:
 	inline bool active(uint64_t now) const
 		throw()
 	{
-		return ( (_fixed) || ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT) );
+		return ( ((_flags & ZT_REMOTEPATH_FLAG_FIXED) != 0) || ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT) );
 	}
 
 	/**
@@ -127,7 +131,7 @@ public:
 	 */
 	inline bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 	{
-		if (RR->node->putPacket(_localInterfaceId,address(),data,len)) {
+		if (RR->node->putPacket(_localAddress,address(),data,len)) {
 			sent(now);
 			RR->antiRec->logOutgoingZT(data,len);
 			return true;
@@ -135,11 +139,39 @@ public:
 		return false;
 	}
 
-private:
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append((uint8_t)1); // version
+		_addr.serialize(b);
+		b.append((uint8_t)_trust);
+		b.append((uint64_t)_lastSend);
+		b.append((uint64_t)_lastReceived);
+		_localAddress.serialize(b);
+		b.append((uint16_t)_flags);
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		if (b[p++] != 1)
+			throw std::invalid_argument("invalid serialized RemotePath");
+		p += _addr.deserialize(b,p);
+		_ipScope = _addr.ipScope();
+		_trust = (Path::Trust)b[p++];
+		_lastSend = b.template at<uint64_t>(p); p += 8;
+		_lastReceived = b.template at<uint64_t>(p); p += 8;
+		p += _localAddress.deserialize(b,p);
+		_flags = b.template at<uint16_t>(p); p += 2;
+		return (p - startAt);
+	}
+
+protected:
 	uint64_t _lastSend;
 	uint64_t _lastReceived;
-	int _localInterfaceId;
-	bool _fixed;
+	InetAddress _localAddress;
+	uint16_t _flags;
 };
 
 } // namespace ZeroTier

node/Salsa20.cpp

@@ -122,7 +122,7 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv,unsigned in
 	_state.i[0] = U8TO32_LITTLE(constants + 0);
 #endif
 
-	_roundsDiv2 = rounds / 2;
+	_roundsDiv4 = rounds / 4;
 }
 
 void Salsa20::encrypt(const void *in,void *out,unsigned int bytes)
@@ -180,7 +180,7 @@ void Salsa20::encrypt(const void *in,void *out,unsigned int bytes)
 		__m128i X2s = X2;
 		__m128i X3s = X3;
 
-		for (i=0;i<_roundsDiv2;++i) {
+		for (i=0;i<_roundsDiv4;++i) {
 			__m128i T = _mm_add_epi32(X0, X3);
 			X1 = _mm_xor_si128(X1, _mm_slli_epi32(T, 7));
 			X1 = _mm_xor_si128(X1, _mm_srli_epi32(T, 25));
@@ -214,6 +214,42 @@ void Salsa20::encrypt(const void *in,void *out,unsigned int bytes)
 			X1 = _mm_shuffle_epi32(X1, 0x39);
 			X2 = _mm_shuffle_epi32(X2, 0x4E);
 			X3 = _mm_shuffle_epi32(X3, 0x93);
+
+			// --
+
+			T = _mm_add_epi32(X0, X3);
+			X1 = _mm_xor_si128(X1, _mm_slli_epi32(T, 7));
+			X1 = _mm_xor_si128(X1, _mm_srli_epi32(T, 25));
+			T = _mm_add_epi32(X1, X0);
+			X2 = _mm_xor_si128(X2, _mm_slli_epi32(T, 9));
+			X2 = _mm_xor_si128(X2, _mm_srli_epi32(T, 23));
+			T = _mm_add_epi32(X2, X1);
+			X3 = _mm_xor_si128(X3, _mm_slli_epi32(T, 13));
+			X3 = _mm_xor_si128(X3, _mm_srli_epi32(T, 19));
+			T = _mm_add_epi32(X3, X2);
+			X0 = _mm_xor_si128(X0, _mm_slli_epi32(T, 18));
+			X0 = _mm_xor_si128(X0, _mm_srli_epi32(T, 14));
+
+			X1 = _mm_shuffle_epi32(X1, 0x93);
+			X2 = _mm_shuffle_epi32(X2, 0x4E);
+			X3 = _mm_shuffle_epi32(X3, 0x39);
+
+			T = _mm_add_epi32(X0, X1);
+			X3 = _mm_xor_si128(X3, _mm_slli_epi32(T, 7));
+			X3 = _mm_xor_si128(X3, _mm_srli_epi32(T, 25));
+			T = _mm_add_epi32(X3, X0);
+			X2 = _mm_xor_si128(X2, _mm_slli_epi32(T, 9));
+			X2 = _mm_xor_si128(X2, _mm_srli_epi32(T, 23));
+			T = _mm_add_epi32(X2, X3);
+			X1 = _mm_xor_si128(X1, _mm_slli_epi32(T, 13));
+			X1 = _mm_xor_si128(X1, _mm_srli_epi32(T, 19));
+			T = _mm_add_epi32(X1, X2);
+			X0 = _mm_xor_si128(X0, _mm_slli_epi32(T, 18));
+			X0 = _mm_xor_si128(X0, _mm_srli_epi32(T, 14));
+
+			X1 = _mm_shuffle_epi32(X1, 0x39);
+			X2 = _mm_shuffle_epi32(X2, 0x4E);
+			X3 = _mm_shuffle_epi32(X3, 0x93);
 		}
 
 		X0 = _mm_add_epi32(X0s,X0);
@@ -260,7 +296,42 @@ void Salsa20::encrypt(const void *in,void *out,unsigned int bytes)
 		x14 = j14;
 		x15 = j15;
 
-		for(i=0;i<_roundsDiv2;++i) {
+		for(i=0;i<_roundsDiv4;++i) {
+			 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+			 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+			x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+			 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+			 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+			x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+			 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+			 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+			x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+			 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+			 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+			x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+			 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+			 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+			x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+			x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+			 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+			 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+			 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+			 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+			 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+			 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+			 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+			 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+			x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+			 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+			 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+			x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+			x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+			x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+			x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+			x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+			// --
+
 			 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
 			 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
 			x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));

node/Salsa20.hpp

@@ -84,7 +84,7 @@ private:
 #endif // ZT_SALSA20_SSE
 		uint32_t i[16];
 	} _state;
-	unsigned int _roundsDiv2;
+	unsigned int _roundsDiv4;
 };
 
 } // namespace ZeroTier

node/Switch.cpp

@@ -78,11 +78,8 @@ Switch::~Switch()
 {
 }
 
-void Switch::onRemotePacket(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len)
+void Switch::onRemotePacket(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
-	if (localInterfaceId < 0)
-		localInterfaceId = 0;
-
 	try {
 		if (len == 13) {
 			/* LEGACY: before VERB_PUSH_DIRECT_PATHS, peers used broadcast
@@ -99,14 +96,14 @@ void Switch::onRemotePacket(int localInterfaceId,const InetAddress &fromAddr,con
 					_lastBeaconResponse = now;
 					Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NOP);
 					outp.armor(peer->key(),false);
-					RR->node->putPacket(localInterfaceId,fromAddr,outp.data(),outp.size());
+					RR->node->putPacket(localAddr,fromAddr,outp.data(),outp.size());
 				}
 			}
 		} else if (len > ZT_PROTO_MIN_FRAGMENT_LENGTH) {
 			if (((const unsigned char *)data)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR) {
-				_handleRemotePacketFragment(localInterfaceId,fromAddr,data,len);
+				_handleRemotePacketFragment(localAddr,fromAddr,data,len);
 			} else if (len >= ZT_PROTO_MIN_PACKET_LENGTH) {
-				_handleRemotePacketHead(localInterfaceId,fromAddr,data,len);
+				_handleRemotePacketHead(localAddr,fromAddr,data,len);
 			}
 		}
 	} catch (std::exception &ex) {
@@ -205,7 +202,8 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		// Destination is another ZeroTier peer on the same network
 
 		Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this
-		const bool includeCom = network->peerNeedsOurMembershipCertificate(toZT,RR->node->now());
+		SharedPtr<Peer> toPeer(RR->topology->getPeer(toZT));
+		const bool includeCom = ((!toPeer)||(toPeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true)));;
 		if ((fromBridged)||(includeCom)) {
 			Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
@@ -270,9 +268,10 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		}
 
 		for(unsigned int b=0;b<numBridges;++b) {
+			SharedPtr<Peer> bridgePeer(RR->topology->getPeer(bridges[b]));
 			Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
-			if (network->peerNeedsOurMembershipCertificate(bridges[b],RR->node->now())) {
+			if ((!bridgePeer)||(bridgePeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) {
 				outp.append((unsigned char)0x01); // 0x01 -- COM included
 				nconf->com().serialize(outp);
 			} else {
@@ -379,14 +378,14 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 	return true;
 }
 
-void Switch::rendezvous(const SharedPtr<Peer> &peer,int localInterfaceId,const InetAddress &atAddr)
+void Switch::rendezvous(const SharedPtr<Peer> &peer,const InetAddress &localAddr,const InetAddress &atAddr)
 {
 	TRACE("sending NAT-t message to %s(%s)",peer->address().toString().c_str(),atAddr.toString().c_str());
 	const uint64_t now = RR->node->now();
-	peer->attemptToContactAt(RR,localInterfaceId,atAddr,now);
+	peer->attemptToContactAt(RR,localAddr,atAddr,now);
 	{
 		Mutex::Lock _l(_contactQueue_m);
-		_contactQueue.push_back(ContactQueueEntry(peer,now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY,localInterfaceId,atAddr));
+		_contactQueue.push_back(ContactQueueEntry(peer,now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY,localAddr,atAddr));
 	}
 }
 
@@ -456,14 +455,14 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 				} else {
 					if (qi->strategyIteration == 0) {
 						// First strategy: send packet directly to destination
-						qi->peer->attemptToContactAt(RR,qi->localInterfaceId,qi->inaddr,now);
+						qi->peer->attemptToContactAt(RR,qi->localAddr,qi->inaddr,now);
 					} else if (qi->strategyIteration <= 4) {
 						// Strategies 1-4: try escalating ports for symmetric NATs that remap sequentially
 						InetAddress tmpaddr(qi->inaddr);
 						int p = (int)qi->inaddr.port() + qi->strategyIteration;
 						if (p < 0xffff) {
 							tmpaddr.setPort((unsigned int)p);
-							qi->peer->attemptToContactAt(RR,qi->localInterfaceId,tmpaddr,now);
+							qi->peer->attemptToContactAt(RR,qi->localAddr,tmpaddr,now);
 						} else qi->strategyIteration = 5;
 					} else {
 						// All strategies tried, expire entry
@@ -554,7 +553,7 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 	return nextDelay;
 }
 
-void Switch::_handleRemotePacketFragment(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len)
+void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
 	Packet::Fragment fragment(data,len);
 	Address destination(fragment.destination());
@@ -625,9 +624,9 @@ void Switch::_handleRemotePacketFragment(int localInterfaceId,const InetAddress
 	}
 }
 
-void Switch::_handleRemotePacketHead(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len)
+void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
-	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localInterfaceId,fromAddr,RR->node->now()));
+	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,RR->node->now()));
 
 	Address source(packet->source());
 	Address destination(packet->destination());
@@ -750,7 +749,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // no paths, no root servers?
 		}
 
-		if ((network)&&(relay)&&(network->isAllowed(peer->address()))) {
+		if ((network)&&(relay)&&(network->isAllowed(peer))) {
 			// Push hints for direct connectivity to this peer if we are relaying
 			peer->pushDirectPaths(RR,viaPath,now,false);
 		}

node/Switch.hpp

@@ -69,12 +69,12 @@ public:
 	/**
 	 * Called when a packet is received from the real network
 	 *
-	 * @param localInterfaceId Local interface ID or -1 for unspecified
+	 * @param localAddr Local interface address
 	 * @param fromAddr Internet IP address of origin
 	 * @param data Packet data
 	 * @param len Packet length
 	 */
-	void onRemotePacket(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len);
+	void onRemotePacket(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len);
 
 	/**
 	 * Called when a packet comes from a local Ethernet tap
@@ -131,10 +131,10 @@ public:
 	 * Attempt NAT traversal to peer at a given physical address
 	 *
 	 * @param peer Peer to contact
-	 * @param localInterfaceId Local interface ID or -1 if unspecified
+	 * @param localAddr Local interface address
 	 * @param atAddr Address of peer
 	 */
-	void rendezvous(const SharedPtr<Peer> &peer,int localInterfaceId,const InetAddress &atAddr);
+	void rendezvous(const SharedPtr<Peer> &peer,const InetAddress &localAddr,const InetAddress &atAddr);
 
 	/**
 	 * Request WHOIS on a given address
@@ -171,8 +171,8 @@ public:
 	unsigned long doTimerTasks(uint64_t now);
 
 private:
-	void _handleRemotePacketFragment(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len);
-	void _handleRemotePacketHead(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len);
+	void _handleRemotePacketFragment(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len);
+	void _handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len);
 	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(const Packet &packet,bool encrypt,uint64_t nwid);
 
@@ -252,17 +252,17 @@ private:
 	struct ContactQueueEntry
 	{
 		ContactQueueEntry() {}
-		ContactQueueEntry(const SharedPtr<Peer> &p,uint64_t ft,int liid,const InetAddress &a) :
+		ContactQueueEntry(const SharedPtr<Peer> &p,uint64_t ft,const InetAddress &laddr,const InetAddress &a) :
 			peer(p),
 			fireAtTime(ft),
 			inaddr(a),
-			localInterfaceId(liid),
+			localAddr(laddr),
 			strategyIteration(0) {}
 
 		SharedPtr<Peer> peer;
 		uint64_t fireAtTime;
 		InetAddress inaddr;
-		int localInterfaceId;
+		InetAddress localAddr;
 		unsigned int strategyIteration;
 	};
 	std::list<ContactQueueEntry> _contactQueue;

node/Topology.cpp

@@ -31,6 +31,7 @@
 #include "Defaults.hpp"
 #include "Dictionary.hpp"
 #include "Node.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
@@ -38,10 +39,68 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 	RR(renv),
 	_amRoot(false)
 {
+	std::string alls(RR->node->dataStoreGet("peers.save"));
+	const uint8_t *all = reinterpret_cast<const uint8_t *>(alls.data());
+	RR->node->dataStoreDelete("peers.save");
+
+	unsigned int ptr = 0;
+	while ((ptr + 4) < alls.size()) {
+		// Each Peer serializes itself prefixed by a record length (not including the size of the length itself)
+		unsigned int reclen = (unsigned int)all[ptr] & 0xff;
+		reclen <<= 8;
+		reclen |= (unsigned int)all[ptr + 1] & 0xff;
+		reclen <<= 8;
+		reclen |= (unsigned int)all[ptr + 2] & 0xff;
+		reclen <<= 8;
+		reclen |= (unsigned int)all[ptr + 3] & 0xff;
+
+		if (((ptr + reclen) > alls.size())||(reclen > ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE))
+			break;
+
+		try {
+			unsigned int pos = 0;
+			SharedPtr<Peer> p(Peer::deserializeNew(RR->identity,Buffer<ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE>(all + ptr,reclen),pos));
+			if (pos != reclen)
+				break;
+			ptr += pos;
+			if ((p)&&(p->address() != RR->identity.address())) {
+				_peers[p->address()] = p;
+			} else {
+				break; // stop if invalid records
+			}
+		} catch (std::exception &exc) {
+			break;
+		} catch ( ... ) {
+			break; // stop if invalid records
+		}
+	}
+
+	clean(RR->node->now());
 }
 
 Topology::~Topology()
 {
+	Buffer<ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE> pbuf;
+	std::string all;
+
+	Address *a = (Address *)0;
+	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
+	Hashtable< Address,SharedPtr<Peer> >::Iterator i(_peers);
+	while (i.next(a,p)) {
+		if (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end()) {
+			pbuf.clear();
+			try {
+				(*p)->serialize(pbuf);
+				try {
+					all.append((const char *)pbuf.data(),pbuf.size());
+				} catch ( ... ) {
+					return; // out of memory? just skip
+				}
+			} catch ( ... ) {} // peer too big? shouldn't happen, but it so skip
+		}
+	}
+
+	RR->node->dataStorePut("peers.save",all,true);
 }
 
 void Topology::setRootServers(const std::map< Identity,std::vector<InetAddress> > &sn)
@@ -58,11 +117,11 @@ void Topology::setRootServers(const std::map< Identity,std::vector<InetAddress>
 
 	for(std::map< Identity,std::vector<InetAddress> >::const_iterator i(sn.begin());i!=sn.end();++i) {
 		if (i->first != RR->identity) { // do not add self as a peer
-			SharedPtr<Peer> &p = _activePeers[i->first.address()];
+			SharedPtr<Peer> &p = _peers[i->first.address()];
 			if (!p)
 				p = SharedPtr<Peer>(new Peer(RR->identity,i->first));
 			for(std::vector<InetAddress>::const_iterator j(i->second.begin());j!=i->second.end();++j)
-				p->addPath(RemotePath(0,*j,true));
+				p->addPath(RemotePath(InetAddress(),*j,true),now);
 			p->use(now);
 			_rootPeers.push_back(p);
 		}
@@ -81,7 +140,7 @@ void Topology::setRootServers(const Dictionary &sn)
 		if ((d->first.length() == ZT_ADDRESS_LENGTH_HEX)&&(d->second.length() > 0)) {
 			try {
 				Dictionary snspec(d->second);
-				std::vector<InetAddress> &a = m[Identity(snspec.get("id"))];
+				std::vector<InetAddress> &a = m[Identity(snspec.get("id",""))];
 				std::string udp(snspec.get("udp",std::string()));
 				if (udp.length() > 0)
 					a.push_back(InetAddress(udp));
@@ -103,7 +162,7 @@ SharedPtr<Peer> Topology::addPeer(const SharedPtr<Peer> &peer)
 	const uint64_t now = RR->node->now();
 	Mutex::Lock _l(_lock);
 
-	SharedPtr<Peer> &p = _activePeers.set(peer->address(),peer);
+	SharedPtr<Peer> &p = _peers.set(peer->address(),peer);
 	p->use(now);
 	_saveIdentity(p->identity());
 
@@ -120,7 +179,7 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 	const uint64_t now = RR->node->now();
 	Mutex::Lock _l(_lock);
 
-	SharedPtr<Peer> &ap = _activePeers[zta];
+	SharedPtr<Peer> &ap = _peers[zta];
 
 	if (ap) {
 		ap->use(now);
@@ -136,7 +195,7 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 		} catch ( ... ) {} // invalid identity?
 	}
 
-	_activePeers.erase(zta);
+	_peers.erase(zta);
 
 	return SharedPtr<Peer>();
 }
@@ -160,7 +219,7 @@ SharedPtr<Peer> Topology::getBestRoot(const Address *avoid,unsigned int avoidCou
 					if (++sna == _rootAddresses.end())
 						sna = _rootAddresses.begin(); // wrap around at end
 					if (*sna != RR->identity.address()) { // pick one other than us -- starting from me+1 in sorted set order
-						SharedPtr<Peer> *p = _activePeers.get(*sna);
+						SharedPtr<Peer> *p = _peers.get(*sna);
 						if ((p)&&((*p)->hasActiveDirectPath(now))) {
 							bestRoot = *p;
 							break;
@@ -249,12 +308,15 @@ bool Topology::isRoot(const Identity &id) const
 void Topology::clean(uint64_t now)
 {
 	Mutex::Lock _l(_lock);
-	Hashtable< Address,SharedPtr<Peer> >::Iterator i(_activePeers);
+	Hashtable< Address,SharedPtr<Peer> >::Iterator i(_peers);
 	Address *a = (Address *)0;
 	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
-	while (i.next(a,p))
+	while (i.next(a,p)) {
 		if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) {
-			_activePeers.erase(*a);
+			_peers.erase(*a);
+		} else {
+			(*p)->clean(RR,now);
+		}
 	}
 }
 

node/Topology.hpp

@@ -164,7 +164,7 @@ public:
 	inline void eachPeer(F f)
 	{
 		Mutex::Lock _l(_lock);
-		Hashtable< Address,SharedPtr<Peer> >::Iterator i(_activePeers);
+		Hashtable< Address,SharedPtr<Peer> >::Iterator i(_peers);
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
 		while (i.next(a,p))
@@ -177,7 +177,7 @@ public:
 	inline std::vector< std::pair< Address,SharedPtr<Peer> > > allPeers() const
 	{
 		Mutex::Lock _l(_lock);
-		return _activePeers.entries();
+		return _peers.entries();
 	}
 
 	/**
@@ -194,7 +194,7 @@ private:
 
 	const RuntimeEnvironment *RR;
 
-	Hashtable< Address,SharedPtr<Peer> > _activePeers;
+	Hashtable< Address,SharedPtr<Peer> > _peers;
 	std::map< Identity,std::vector<InetAddress> > _roots;
 	std::vector< Address > _rootAddresses;
 	std::vector< SharedPtr<Peer> > _rootPeers;

node/Utils.cpp

@@ -261,25 +261,6 @@ std::vector<std::string> Utils::split(const char *s,const char *const sep,const
 	return fields;
 }
 
-std::string Utils::trim(const std::string &s)
-{
-	unsigned long end = (unsigned long)s.length();
-	while (end) {
-		char c = s[end - 1];
-		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
-			--end;
-		else break;
-	}
-	unsigned long start = 0;
-	while (start < end) {
-		char c = s[start];
-		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
-			++start;
-		else break;
-	}
-	return s.substr(start,end - start);
-}
-
 unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...)
 	throw(std::length_error)
 {

node/Utils.hpp

@@ -256,14 +256,6 @@ public:
 		return true;
 	}
 
-	/**
-	 * Trim whitespace from the start and end of a string
-	 *
-	 * @param s String to trim
-	 * @return Trimmed string
-	 */
-	static std::string trim(const std::string &s);
-
 	/**
 	 * Variant of snprintf that is portable and throws an exception
 	 *

one.cpp

@@ -73,7 +73,7 @@
 
 #include "service/OneService.hpp"
 
-#define ZT1_PID_PATH "zerotier-one.pid"
+#define ZT_PID_PATH "zerotier-one.pid"
 
 using namespace ZeroTier;
 
@@ -976,7 +976,7 @@ int main(int argc,char **argv)
 
 	std::string overrideRootTopology;
 	std::string homeDir;
-	unsigned int port = ZT1_DEFAULT_PORT;
+	unsigned int port = ZT_DEFAULT_PORT;
 	bool skipRootCheck = false;
 
 	for(int i=1;i<argc;++i) {
@@ -1154,7 +1154,7 @@ int main(int argc,char **argv)
 #endif // __WINDOWS__
 
 #ifdef __UNIX_LIKE__
-	std::string pidPath(homeDir + ZT_PATH_SEPARATOR_S + ZT1_PID_PATH);
+	std::string pidPath(homeDir + ZT_PATH_SEPARATOR_S + ZT_PID_PATH);
 	{
 		// Write .pid file to home folder
 		FILE *pf = fopen(pidPath.c_str(),"w");

selftest.cpp

@@ -269,6 +269,22 @@ static int testCrypto()
 	}
 	std::cout << "PASS" << std::endl;
 
+	std::cout << "[crypto] Benchmarking Poly1305... "; std::cout.flush();
+	{
+		unsigned char *bb = (unsigned char *)::malloc(1234567);
+		for(unsigned int i=0;i<1234567;++i)
+			bb[i] = (unsigned char)i;
+		double bytes = 0.0;
+		uint64_t start = OSUtils::now();
+		for(unsigned int i=0;i<200;++i) {
+			Poly1305::compute(buf1,bb,1234567,poly1305TV0Key);
+			bytes += 1234567.0;
+		}
+		uint64_t end = OSUtils::now();
+		std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl;
+		::free((void *)bb);
+	}
+
 	std::cout << "[crypto] Testing C25519 and Ed25519 against test vectors... "; std::cout.flush();
 	for(int k=0;k<ZT_NUM_C25519_TEST_VECTORS;++k) {
 		C25519::Pair p1,p2;

service/ControlPlane.cpp

@@ -64,7 +64,7 @@ static std::string _jsonEscape(const char *s)
 }
 static std::string _jsonEscape(const std::string &s) { return _jsonEscape(s.c_str()); }
 
-static std::string _jsonEnumerate(const ZT1_MulticastGroup *mg,unsigned int count)
+static std::string _jsonEnumerate(const ZT_MulticastGroup *mg,unsigned int count)
 {
 	std::string buf;
 	char tmp[128];
@@ -101,7 +101,7 @@ static std::string _jsonEnumerate(const struct sockaddr_storage *ss,unsigned int
 	return buf;
 }
 
-static void _jsonAppend(unsigned int depth,std::string &buf,const ZT1_VirtualNetworkConfig *nc,const std::string &portDeviceName)
+static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_VirtualNetworkConfig *nc,const std::string &portDeviceName)
 {
 	char json[4096];
 	char prefix[32];
@@ -114,16 +114,16 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT1_VirtualNet
 
 	const char *nstatus = "",*ntype = "";
 	switch(nc->status) {
-		case ZT1_NETWORK_STATUS_REQUESTING_CONFIGURATION: nstatus = "REQUESTING_CONFIGURATION"; break;
-		case ZT1_NETWORK_STATUS_OK:                       nstatus = "OK"; break;
-		case ZT1_NETWORK_STATUS_ACCESS_DENIED:            nstatus = "ACCESS_DENIED"; break;
-		case ZT1_NETWORK_STATUS_NOT_FOUND:                nstatus = "NOT_FOUND"; break;
-		case ZT1_NETWORK_STATUS_PORT_ERROR:               nstatus = "PORT_ERROR"; break;
-		case ZT1_NETWORK_STATUS_CLIENT_TOO_OLD:           nstatus = "CLIENT_TOO_OLD"; break;
+		case ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION: nstatus = "REQUESTING_CONFIGURATION"; break;
+		case ZT_NETWORK_STATUS_OK:                       nstatus = "OK"; break;
+		case ZT_NETWORK_STATUS_ACCESS_DENIED:            nstatus = "ACCESS_DENIED"; break;
+		case ZT_NETWORK_STATUS_NOT_FOUND:                nstatus = "NOT_FOUND"; break;
+		case ZT_NETWORK_STATUS_PORT_ERROR:               nstatus = "PORT_ERROR"; break;
+		case ZT_NETWORK_STATUS_CLIENT_TOO_OLD:           nstatus = "CLIENT_TOO_OLD"; break;
 	}
 	switch(nc->type) {
-		case ZT1_NETWORK_TYPE_PRIVATE:                    ntype = "PRIVATE"; break;
-		case ZT1_NETWORK_TYPE_PUBLIC:                     ntype = "PUBLIC"; break;
+		case ZT_NETWORK_TYPE_PRIVATE:                    ntype = "PRIVATE"; break;
+		case ZT_NETWORK_TYPE_PUBLIC:                     ntype = "PUBLIC"; break;
 	}
 
 	Utils::snprintf(json,sizeof(json),
@@ -162,7 +162,7 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT1_VirtualNet
 	buf.append(json);
 }
 
-static std::string _jsonEnumerate(unsigned int depth,const ZT1_PeerPhysicalPath *pp,unsigned int count)
+static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath *pp,unsigned int count)
 {
 	char json[1024];
 	char prefix[32];
@@ -198,7 +198,7 @@ static std::string _jsonEnumerate(unsigned int depth,const ZT1_PeerPhysicalPath
 	return buf;
 }
 
-static void _jsonAppend(unsigned int depth,std::string &buf,const ZT1_Peer *peer)
+static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer)
 {
 	char json[1024];
 	char prefix[32];
@@ -211,9 +211,9 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT1_Peer *peer
 
 	const char *prole = "";
 	switch(peer->role) {
-		case ZT1_PEER_ROLE_LEAF:  prole = "LEAF"; break;
-		case ZT1_PEER_ROLE_RELAY: prole = "RELAY"; break;
-		case ZT1_PEER_ROLE_ROOT:  prole = "ROOT"; break;
+		case ZT_PEER_ROLE_LEAF:  prole = "LEAF"; break;
+		case ZT_PEER_ROLE_RELAY: prole = "RELAY"; break;
+		case ZT_PEER_ROLE_ROOT:  prole = "ROOT"; break;
 	}
 
 	Utils::snprintf(json,sizeof(json),
@@ -356,7 +356,7 @@ unsigned int ControlPlane::handleRequest(
 
 			if (ps[0] == "status") {
 				responseContentType = "application/json";
-				ZT1_NodeStatus status;
+				ZT_NodeStatus status;
 				_node->status(&status);
 				Utils::snprintf(json,sizeof(json),
 					"{\n"
@@ -386,7 +386,7 @@ unsigned int ControlPlane::handleRequest(
 				responseBody = "{}"; // TODO
 				scode = 200;
 			} else if (ps[0] == "network") {
-				ZT1_VirtualNetworkList *nws = _node->networks();
+				ZT_VirtualNetworkList *nws = _node->networks();
 				if (nws) {
 					if (ps.size() == 1) {
 						// Return [array] of all networks
@@ -415,7 +415,7 @@ unsigned int ControlPlane::handleRequest(
 					_node->freeQueryResult((void *)nws);
 				} else scode = 500;
 			} else if (ps[0] == "peer") {
-				ZT1_PeerList *pl = _node->peers();
+				ZT_PeerList *pl = _node->peers();
 				if (pl) {
 					if (ps.size() == 1) {
 						// Return [array] of all peers
@@ -473,7 +473,7 @@ unsigned int ControlPlane::handleRequest(
 				if (ps.size() == 2) {
 					uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str());
 					_node->join(wantnw); // does nothing if we are a member
-					ZT1_VirtualNetworkList *nws = _node->networks();
+					ZT_VirtualNetworkList *nws = _node->networks();
 					if (nws) {
 						for(unsigned long i=0;i<nws->networkCount;++i) {
 							if (nws->networks[i].nwid == wantnw) {
@@ -506,7 +506,7 @@ unsigned int ControlPlane::handleRequest(
 			if (ps[0] == "config") {
 				// TODO
 			} else if (ps[0] == "network") {
-				ZT1_VirtualNetworkList *nws = _node->networks();
+				ZT_VirtualNetworkList *nws = _node->networks();
 				if (nws) {
 					if (ps.size() == 2) {
 						uint64_t wantnw = Utils::hexStrToU64(ps[1].c_str());

service/OneService.cpp

@@ -133,20 +133,20 @@ namespace ZeroTier { typedef BSDEthernetTap EthernetTap; }
 #define ZT_TAP_CHECK_MULTICAST_INTERVAL 5000
 
 // Path under ZT1 home for controller database if controller is enabled
-#define ZT1_CONTROLLER_DB_PATH "controller.db"
+#define ZT_CONTROLLER_DB_PATH "controller.db"
 
 // TCP fallback relay host -- geo-distributed using Amazon Route53 geo-aware DNS
-#define ZT1_TCP_FALLBACK_RELAY "tcp-fallback.zerotier.com"
-#define ZT1_TCP_FALLBACK_RELAY_PORT 443
+#define ZT_TCP_FALLBACK_RELAY "tcp-fallback.zerotier.com"
+#define ZT_TCP_FALLBACK_RELAY_PORT 443
 
 // Frequency at which we re-resolve the TCP fallback relay
-#define ZT1_TCP_FALLBACK_RERESOLVE_DELAY 86400000
+#define ZT_TCP_FALLBACK_RERESOLVE_DELAY 86400000
 
 // Attempt to engage TCP fallback after this many ms of no reply to packets sent to global-scope IPs
-#define ZT1_TCP_FALLBACK_AFTER 60000
+#define ZT_TCP_FALLBACK_AFTER 60000
 
 // How often to check for local interface addresses
-#define ZT1_LOCAL_INTERFACE_CHECK_INTERVAL 300000
+#define ZT_LOCAL_INTERFACE_CHECK_INTERVAL 300000
 
 namespace ZeroTier {
 
@@ -353,14 +353,33 @@ public:
 static BackgroundSoftwareUpdateChecker backgroundSoftwareUpdateChecker;
 #endif // ZT_AUTO_UPDATE
 
+static std::string _trimString(const std::string &s)
+{
+	unsigned long end = (unsigned long)s.length();
+	while (end) {
+		char c = s[end - 1];
+		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
+			--end;
+		else break;
+	}
+	unsigned long start = 0;
+	while (start < end) {
+		char c = s[start];
+		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
+			++start;
+		else break;
+	}
+	return s.substr(start,end - start);
+}
+
 class OneServiceImpl;
 
-static int SnodeVirtualNetworkConfigFunction(ZT1_Node *node,void *uptr,uint64_t nwid,enum ZT1_VirtualNetworkConfigOperation op,const ZT1_VirtualNetworkConfig *nwconf);
-static void SnodeEventCallback(ZT1_Node *node,void *uptr,enum ZT1_Event event,const void *metaData);
-static long SnodeDataStoreGetFunction(ZT1_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize);
-static int SnodeDataStorePutFunction(ZT1_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure);
-static int SnodeWirePacketSendFunction(ZT1_Node *node,void *uptr,int localInterfaceId,const struct sockaddr_storage *addr,const void *data,unsigned int len);
-static void SnodeVirtualNetworkFrameFunction(ZT1_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
+static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t nwid,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf);
+static void SnodeEventCallback(ZT_Node *node,void *uptr,enum ZT_Event event,const void *metaData);
+static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize);
+static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure);
+static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len);
+static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
 static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
@@ -411,18 +430,14 @@ struct TcpConnection
 	Mutex writeBuf_m;
 };
 
-// Interface IDs -- the uptr for UDP sockets is set to point to one of these
-static const int ZT1_INTERFACE_ID_DEFAULT = 0; // default, usually port 9993
-static const int ZT1_INTERFACE_ID_UPNP = 1; // a randomly chosen UDP socket used with uPnP mappings, if enabled
-
 class OneServiceImpl : public OneService
 {
 public:
 	OneServiceImpl(const char *hp,unsigned int port,const char *overrideRootTopology) :
 		_homePath((hp) ? hp : "."),
-		_tcpFallbackResolver(ZT1_TCP_FALLBACK_RELAY),
+		_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY),
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-		_controller((_homePath + ZT_PATH_SEPARATOR_S + ZT1_CONTROLLER_DB_PATH).c_str()),
+		_controller((SqliteNetworkController *)0),
 #endif
 		_phy(this,false,true),
 		_overrideRootTopology((overrideRootTopology) ? overrideRootTopology : ""),
@@ -441,9 +456,6 @@ public:
 #endif
 		_run(true)
 	{
-		struct sockaddr_in in4;
-		struct sockaddr_in6 in6;
-
 		const int portTrials = (port == 0) ? 256 : 1; // if port is 0, pick random
 		for(int k=0;k<portTrials;++k) {
 			if (port == 0) {
@@ -452,24 +464,26 @@ public:
 				port = 40000 + (randp % 25500);
 			}
 
-			memset((void *)&in4,0,sizeof(in4));
-			in4.sin_family = AF_INET;
-			in4.sin_port = Utils::hton((uint16_t)port);
-
-			_v4UdpSocket = _phy.udpBind((const struct sockaddr *)&in4,reinterpret_cast<void *>(const_cast<int *>(&ZT1_INTERFACE_ID_DEFAULT)),131072);
+			_v4LocalAddress = InetAddress((uint32_t)0,port);
+			_v4UdpSocket = _phy.udpBind((const struct sockaddr *)&_v4LocalAddress,reinterpret_cast<void *>(&_v4LocalAddress),131072);
 
 			if (_v4UdpSocket) {
+				struct sockaddr_in in4;
+				memset(&in4,0,sizeof(in4));
+				in4.sin_family = AF_INET;
 				in4.sin_addr.s_addr = Utils::hton((uint32_t)0x7f000001); // right now we just listen for TCP @localhost
+				in4.sin_port = Utils::hton((uint16_t)port);
 				_v4TcpListenSocket = _phy.tcpListen((const struct sockaddr *)&in4,this);
 
 				if (_v4TcpListenSocket) {
+					_v6LocalAddress = InetAddress("\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0",16,port);
+					_v6UdpSocket = _phy.udpBind((const struct sockaddr *)&_v6LocalAddress,reinterpret_cast<void *>(&_v6LocalAddress),131072);
+
+					struct sockaddr_in6 in6;
 					memset((void *)&in6,0,sizeof(in6));
 					in6.sin6_family = AF_INET6;
 					in6.sin6_port = in4.sin_port;
-
-					_v6UdpSocket = _phy.udpBind((const struct sockaddr *)&in6,reinterpret_cast<void *>(const_cast<int *>(&ZT1_INTERFACE_ID_DEFAULT)),131072);
-
-					in6.sin6_addr.s6_addr[15] = 1; // listen for TCP only at localhost
+					in6.sin6_addr.s6_addr[15] = 1; // IPv6 localhost == ::1
 					_v6TcpListenSocket = _phy.tcpListen((const struct sockaddr *)&in6,this);
 
 					_port = port;
@@ -490,20 +504,14 @@ public:
 		OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "zerotier-one.port").c_str(),std::string(portstr));
 
 #ifdef ZT_USE_MINIUPNPC
-		// Bind a random secondary port for use with uPnP, since some NAT routers
+		// Bind a secondary port for use with uPnP, since some NAT routers
 		// (cough Ubiquity Edge cough) barf up a lung if you do both conventional
 		// NAT-t and uPnP from behind the same port. I think this is a bug, but
 		// everyone else's router bugs are our problem. :P
-		for(int k=0;k<256;++k) {
-			unsigned int randp = 0;
-			Utils::getSecureRandom(&randp,sizeof(randp));
-			unsigned int upnport = 40000 + (randp % 25500);
-
-			memset((void *)&in4,0,sizeof(in4));
-			in4.sin_family = AF_INET;
-			in4.sin_port = Utils::hton((uint16_t)upnport);
-
-			_v4UpnpUdpSocket = _phy.udpBind((const struct sockaddr *)&in4,reinterpret_cast<void *>(const_cast<int *>(&ZT1_INTERFACE_ID_UPNP)),131072);
+		for(int k=0;k<512;++k) {
+			const unsigned int upnport = 40000 + (((port + 1) * (k + 1)) % 25500);
+			_v4UpnpLocalAddress = InetAddress(0,upnport);
+			_v4UpnpUdpSocket = _phy.udpBind((const struct sockaddr *)&_v4UpnpLocalAddress,reinterpret_cast<void *>(&_v4UpnpLocalAddress),131072);
 			if (_v4UpnpUdpSocket) {
 				_upnpClient = new UPNPClient(upnport);
 				break;
@@ -521,6 +529,9 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 		_phy.close(_v4UpnpUdpSocket);
 		delete _upnpClient;
+#endif
+#ifdef ZT_ENABLE_NETWORK_CONTROLLER
+		delete _controller;
 #endif
 	}
 
@@ -544,7 +555,7 @@ public:
 					} else OSUtils::lockDownFile(authTokenPath.c_str(),false);
 				}
 			}
-			authToken = Utils::trim(authToken);
+			authToken = _trimString(authToken);
 
 			_node = new Node(
 				OSUtils::now(),
@@ -558,14 +569,15 @@ public:
 				((_overrideRootTopology.length() > 0) ? _overrideRootTopology.c_str() : (const char *)0));
 
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-			_node->setNetconfMaster((void *)&_controller);
+			_controller = new SqliteNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(_homePath + ZT_PATH_SEPARATOR_S + "circuitTestResults.d").c_str());
+			_node->setNetconfMaster((void *)_controller);
 #endif
 
 			_controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S + "ui").c_str());
 			_controlPlane->addAuthToken(authToken.c_str());
 
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-			_controlPlane->setController(&_controller);
+			_controlPlane->setController(_controller);
 #endif
 
 			{	// Remember networks from previous session
@@ -582,7 +594,7 @@ public:
 			_lastRestart = clockShouldBe;
 			uint64_t lastTapMulticastGroupCheck = 0;
 			uint64_t lastTcpFallbackResolve = 0;
-			uint64_t lastLocalInterfaceAddressCheck = (OSUtils::now() - ZT1_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give UPnP time to configure and other things time to settle
+			uint64_t lastLocalInterfaceAddressCheck = (OSUtils::now() - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give UPnP time to configure and other things time to settle
 #ifdef ZT_AUTO_UPDATE
 			uint64_t lastSoftwareUpdateCheck = 0;
 #endif // ZT_AUTO_UPDATE
@@ -615,12 +627,12 @@ public:
 				}
 #endif // ZT_AUTO_UPDATE
 
-				if ((now - lastTcpFallbackResolve) >= ZT1_TCP_FALLBACK_RERESOLVE_DELAY) {
+				if ((now - lastTcpFallbackResolve) >= ZT_TCP_FALLBACK_RERESOLVE_DELAY) {
 					lastTcpFallbackResolve = now;
 					_tcpFallbackResolver.resolveNow();
 				}
 
-				if ((_tcpFallbackTunnel)&&((now - _lastDirectReceiveFromGlobal) < (ZT1_TCP_FALLBACK_AFTER / 2)))
+				if ((_tcpFallbackTunnel)&&((now - _lastDirectReceiveFromGlobal) < (ZT_TCP_FALLBACK_AFTER / 2)))
 					_phy.close(_tcpFallbackTunnel->sock);
 
 				if ((now - lastTapMulticastGroupCheck) >= ZT_TAP_CHECK_MULTICAST_INTERVAL) {
@@ -636,7 +648,7 @@ public:
 					}
 				}
 
-				if ((now - lastLocalInterfaceAddressCheck) >= ZT1_LOCAL_INTERFACE_CHECK_INTERVAL) {
+				if ((now - lastLocalInterfaceAddressCheck) >= ZT_LOCAL_INTERFACE_CHECK_INTERVAL) {
 					lastLocalInterfaceAddressCheck = now;
 
 #ifdef __UNIX_LIKE__
@@ -652,7 +664,7 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
-						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),0,ZT1_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 #endif
 
 					struct ifaddrs *ifatbl = (struct ifaddrs *)0;
@@ -670,7 +682,7 @@ public:
 								if (!isZT) {
 									InetAddress ip(ifa->ifa_addr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT1_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 								}
 							}
 							ifa = ifa->ifa_next;
@@ -704,7 +716,7 @@ public:
 								while (ua) {
 									InetAddress ip(ua->Address.lpSockaddr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT1_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 									ua = ua->Next;
 								}
 							}
@@ -792,14 +804,14 @@ public:
 #endif
 		if ((len >= 16)&&(reinterpret_cast<const InetAddress *>(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL))
 			_lastDirectReceiveFromGlobal = OSUtils::now();
-		ZT1_ResultCode rc = _node->processWirePacket(
+		ZT_ResultCode rc = _node->processWirePacket(
 			OSUtils::now(),
-			*(reinterpret_cast<const int *>(*uptr)), // for UDP sockets, we set uptr to point to their interface ID
+			reinterpret_cast<const struct sockaddr_storage *>(*uptr),
 			(const struct sockaddr_storage *)from, // Phy<> uses sockaddr_storage, so it'll always be that big
 			data,
 			len,
 			&_nextBackgroundTaskDeadline);
-		if (ZT1_ResultCode_isFatal(rc)) {
+		if (ZT_ResultCode_isFatal(rc)) {
 			char tmp[256];
 			Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc);
 			Mutex::Lock _l(_termReason_m);
@@ -941,14 +953,14 @@ public:
 							}
 
 							if (from) {
-								ZT1_ResultCode rc = _node->processWirePacket(
+								ZT_ResultCode rc = _node->processWirePacket(
 									OSUtils::now(),
 									0,
 									reinterpret_cast<struct sockaddr_storage *>(&from),
 									data,
 									plen,
 									&_nextBackgroundTaskDeadline);
-								if (ZT1_ResultCode_isFatal(rc)) {
+								if (ZT_ResultCode_isFatal(rc)) {
 									char tmp[256];
 									Utils::snprintf(tmp,sizeof(tmp),"fatal error code from processWirePacket: %d",(int)rc);
 									Mutex::Lock _l(_termReason_m);
@@ -999,12 +1011,12 @@ public:
 	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
 
-	inline int nodeVirtualNetworkConfigFunction(uint64_t nwid,enum ZT1_VirtualNetworkConfigOperation op,const ZT1_VirtualNetworkConfig *nwc)
+	inline int nodeVirtualNetworkConfigFunction(uint64_t nwid,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwc)
 	{
 		Mutex::Lock _l(_taps_m);
 		std::map< uint64_t,EthernetTap * >::iterator t(_taps.find(nwid));
 		switch(op) {
-			case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_UP:
+			case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP:
 				if (t == _taps.end()) {
 					try {
 						char friendlyName[1024];
@@ -1034,7 +1046,7 @@ public:
 					}
 				}
 				// fall through...
-			case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE:
+			case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE:
 				if (t != _taps.end()) {
 					t->second->setEnabled(nwc->enabled != 0);
 
@@ -1057,8 +1069,8 @@ public:
 					return -999; // tap init failed
 				}
 				break;
-			case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN:
-			case ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY:
+			case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN:
+			case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY:
 				if (t != _taps.end()) {
 #ifdef __WINDOWS__
 					std::string winInstanceId(t->second->instanceId());
@@ -1067,7 +1079,7 @@ public:
 					_taps.erase(t);
 					_tapAssignedIps.erase(nwid);
 #ifdef __WINDOWS__
-					if ((op == ZT1_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY)&&(winInstanceId.length() > 0))
+					if ((op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY)&&(winInstanceId.length() > 0))
 						WindowsEthernetTap::deletePersistentTapDevice(winInstanceId.c_str());
 #endif
 				}
@@ -1076,17 +1088,17 @@ public:
 		return 0;
 	}
 
-	inline void nodeEventCallback(enum ZT1_Event event,const void *metaData)
+	inline void nodeEventCallback(enum ZT_Event event,const void *metaData)
 	{
 		switch(event) {
-			case ZT1_EVENT_FATAL_ERROR_IDENTITY_COLLISION: {
+			case ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION: {
 				Mutex::Lock _l(_termReason_m);
 				_termReason = ONE_IDENTITY_COLLISION;
 				_fatalErrorMessage = "identity/address collision";
 				this->terminate();
 			}	break;
 
-			case ZT1_EVENT_TRACE: {
+			case ZT_EVENT_TRACE: {
 				if (metaData) {
 					::fprintf(stderr,"%s"ZT_EOL_S,(const char *)metaData);
 					::fflush(stderr);
@@ -1152,10 +1164,10 @@ public:
 		}
 	}
 
-	inline int nodeWirePacketSendFunction(int localInterfaceId,const struct sockaddr_storage *addr,const void *data,unsigned int len)
+	inline int nodeWirePacketSendFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len)
 	{
 #ifdef ZT_USE_MINIUPNPC
-		if (localInterfaceId == ZT1_INTERFACE_ID_UPNP) {
+		if ((localAddr->ss_family == AF_INET)&&(reinterpret_cast<const struct sockaddr_in *>(localAddr)->sin_port == reinterpret_cast<const struct sockaddr_in *>(&_v4UpnpLocalAddress)->sin_port)) {
 #ifdef ZT_BREAK_UDP
 			if (!OSUtils::fileExists("/tmp/ZT_BREAK_UDP")) {
 #endif
@@ -1180,15 +1192,15 @@ public:
 				}
 #endif
 
-#ifdef ZT1_TCP_FALLBACK_RELAY
+#ifdef ZT_TCP_FALLBACK_RELAY
 				// TCP fallback tunnel support
 				if ((len >= 16)&&(reinterpret_cast<const InetAddress *>(addr)->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) {
 					uint64_t now = OSUtils::now();
 
 					// Engage TCP tunnel fallback if we haven't received anything valid from a global
-					// IP address in ZT1_TCP_FALLBACK_AFTER milliseconds. If we do start getting
+					// IP address in ZT_TCP_FALLBACK_AFTER milliseconds. If we do start getting
 					// valid direct traffic we'll stop using it and close the socket after a while.
-					if (((now - _lastDirectReceiveFromGlobal) > ZT1_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT1_TCP_FALLBACK_AFTER)) {
+					if (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER)) {
 						if (_tcpFallbackTunnel) {
 							Mutex::Lock _l(_tcpFallbackTunnel->writeBuf_m);
 							if (!_tcpFallbackTunnel->writeBuf.length())
@@ -1204,7 +1216,7 @@ public:
 							_tcpFallbackTunnel->writeBuf.append(reinterpret_cast<const char *>(reinterpret_cast<const void *>(&(reinterpret_cast<const struct sockaddr_in *>(addr)->sin_port))),2);
 							_tcpFallbackTunnel->writeBuf.append((const char *)data,len);
 							result = 0;
-						} else if (((now - _lastSendToGlobal) < ZT1_TCP_FALLBACK_AFTER)&&((now - _lastSendToGlobal) > (ZT_PING_CHECK_INVERVAL / 2))) {
+						} else if (((now - _lastSendToGlobal) < ZT_TCP_FALLBACK_AFTER)&&((now - _lastSendToGlobal) > (ZT_PING_CHECK_INVERVAL / 2))) {
 							std::vector<InetAddress> tunnelIps(_tcpFallbackResolver.get());
 							if (tunnelIps.empty()) {
 								if (!_tcpFallbackResolver.running())
@@ -1212,7 +1224,7 @@ public:
 							} else {
 								bool connected = false;
 								InetAddress addr(tunnelIps[(unsigned long)now % tunnelIps.size()]);
-								addr.setPort(ZT1_TCP_FALLBACK_RELAY_PORT);
+								addr.setPort(ZT_TCP_FALLBACK_RELAY_PORT);
 								_phy.tcpConnect(reinterpret_cast<const struct sockaddr *>(&addr),connected);
 							}
 						}
@@ -1220,7 +1232,7 @@ public:
 
 					_lastSendToGlobal = now;
 				}
-#endif // ZT1_TCP_FALLBACK_RELAY
+#endif // ZT_TCP_FALLBACK_RELAY
 
 				break;
 
@@ -1327,11 +1339,12 @@ private:
 	const std::string _homePath;
 	BackgroundResolver _tcpFallbackResolver;
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-	SqliteNetworkController _controller;
+	SqliteNetworkController *_controller;
 #endif
 	Phy<OneServiceImpl *> _phy;
 	std::string _overrideRootTopology;
 	Node *_node;
+	InetAddress _v4LocalAddress,_v6LocalAddress;
 	PhySocket *_v4UdpSocket;
 	PhySocket *_v6UdpSocket;
 	PhySocket *_v4TcpListenSocket;
@@ -1356,6 +1369,7 @@ private:
 	unsigned int _port;
 
 #ifdef ZT_USE_MINIUPNPC
+	InetAddress _v4UpnpLocalAddress;
 	PhySocket *_v4UpnpUdpSocket;
 	UPNPClient *_upnpClient;
 #endif
@@ -1364,17 +1378,17 @@ private:
 	Mutex _run_m;
 };
 
-static int SnodeVirtualNetworkConfigFunction(ZT1_Node *node,void *uptr,uint64_t nwid,enum ZT1_VirtualNetworkConfigOperation op,const ZT1_VirtualNetworkConfig *nwconf)
+static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t nwid,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf)
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkConfigFunction(nwid,op,nwconf); }
-static void SnodeEventCallback(ZT1_Node *node,void *uptr,enum ZT1_Event event,const void *metaData)
+static void SnodeEventCallback(ZT_Node *node,void *uptr,enum ZT_Event event,const void *metaData)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeEventCallback(event,metaData); }
-static long SnodeDataStoreGetFunction(ZT1_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize)
+static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize)
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeDataStoreGetFunction(name,buf,bufSize,readIndex,totalSize); }
-static int SnodeDataStorePutFunction(ZT1_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure)
+static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure)
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeDataStorePutFunction(name,data,len,secure); }
-static int SnodeWirePacketSendFunction(ZT1_Node *node,void *uptr,int localInterfaceId,const struct sockaddr_storage *addr,const void *data,unsigned int len)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localInterfaceId,addr,data,len); }
-static void SnodeVirtualNetworkFrameFunction(ZT1_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
+static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len)
+{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len); }
+static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,sourceMac,destMac,etherType,vlanId,data,len); }
 
 static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)

windows/ZeroTierOne/ZeroTierOneService.cpp

@@ -90,7 +90,7 @@ restart_node:
 			_service = (ZeroTier::OneService *)0; // in case newInstance() fails
 			_service = ZeroTier::OneService::newInstance(
 				ZeroTier::OneService::platformDefaultHomePath().c_str(),
-				ZT1_DEFAULT_PORT);
+				ZT_DEFAULT_PORT);
 		}
 		switch(_service->run()) {
 			case ZeroTier::OneService::ONE_UNRECOVERABLE_ERROR: {