Merge branch 'dev' of http://git.int.zerotier.com/ZeroTier/ZeroTierOne into dev

controller/EmbeddedNetworkController.cpp

@@ -35,6 +35,7 @@
 #include <memory>
 
 #include "../include/ZeroTierOne.h"
+#include "../version.h"
 #include "../node/Constants.hpp"
 
 #include "EmbeddedNetworkController.hpp"
@@ -430,9 +431,11 @@ EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPa
 	_startTime(OSUtils::now()),
 	_running(true),
 	_lastDumpedStatus(0),
-	_db(dbPath),
+	_db(dbPath,this),
 	_node(node)
 {
+	if ((dbPath[0] == '-')&&(dbPath[1] == 0))
+		_startThreads(); // start threads now in Central harnessed mode
 }
 
 EmbeddedNetworkController::~EmbeddedNetworkController()
@@ -638,26 +641,14 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 							if (newAuth != OSUtils::jsonBool(member["authorized"],false)) {
 								member["authorized"] = newAuth;
 								member[((newAuth) ? "lastAuthorizedTime" : "lastDeauthorizedTime")] = now;
-
-								json ah;
-								ah["a"] = newAuth;
-								ah["by"] = "api";
-								ah["ts"] = now;
-								ah["ct"] = json();
-								ah["c"] = json();
-								member["authHistory"].push_back(ah);
+								if (newAuth) {
+									member["lastAuthorizedCredentialType"] = "api";
+									member["lastAuthorizedCredential"] = json();
+								}
 
 								// Member is being de-authorized, so spray Revocation objects to all online members
-								if (!newAuth) {
-									Revocation rev((uint32_t)_node->prng(),nwid,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(address),Revocation::CREDENTIAL_TYPE_COM);
-									rev.sign(_signingId);
-
-									Mutex::Lock _l(_memberStatus_m);
-									for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) {
-										if ((i->first.networkId == nwid)&&(i->second.online(now)))
-											_node->ncSendRevocation(Address(i->first.nodeId),rev);
-									}
-								}
+								if (!newAuth)
+									onNetworkMemberDeauthorize(nwid,address);
 							}
 						}
 
@@ -724,14 +715,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 						json &revj = member["revision"];
 						member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
 						_db.saveNetworkMember(nwid,address,member);
-
-						// Push update to member if online
-						try {
-							Mutex::Lock _l(_memberStatus_m);
-							_MemberStatus &ms = _memberStatus[_MemberStatusKey(nwid,address)];
-							if ((ms.online(now))&&(ms.lastRequestMetaData))
-								request(nwid,InetAddress(),0,ms.identity,ms.lastRequestMetaData);
-						} catch ( ... ) {}
+						onNetworkMemberUpdate(nwid,address);
 					}
 
 					_addMemberNonPersistedFields(nwid,address,member,now);
@@ -896,22 +880,15 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 
 					if (b.count("authTokens")) {
 						json &authTokens = b["authTokens"];
-						if (authTokens.is_array()) {
-							json nat = json::array();
-							for(unsigned long i=0;i<authTokens.size();++i) {
-								json &token = authTokens[i];
-								if (token.is_object()) {
-									std::string tstr = token["token"];
-									if (tstr.length() > 0) {
-										json t = json::object();
-										t["token"] = tstr;
-										t["expires"] = OSUtils::jsonInt(token["expires"],0ULL);
-										t["maxUsesPerMember"] = OSUtils::jsonInt(token["maxUsesPerMember"],0ULL);
-										nat.push_back(t);
-									}
-								}
+						if (authTokens.is_object()) {
+							json nat;
+							for(json::iterator t(authTokens.begin());t!=authTokens.end();++t) {
+								if ((t.value().is_number())&&(t.value() >= 0))
+									nat[t.key()] = t.value();
 							}
 							network["authTokens"] = nat;
+						} else {
+							network["authTokens"] = {{}};
 						}
 					}
 
@@ -991,13 +968,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 					json &revj = network["revision"];
 					network["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
 					_db.saveNetwork(nwid,network);
-
-					// Send an update to all members of the network that are online
-					Mutex::Lock _l(_memberStatus_m);
-					for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) {
-						if ((i->first.networkId == nwid)&&(i->second.online(now))&&(i->second.lastRequestMetaData))
-							request(nwid,InetAddress(),0,i->second.identity,i->second.lastRequestMetaData);
-					}
+					onNetworkUpdate(nwid);
 				}
 
 				JSONDB::NetworkSummaryInfo ns;
@@ -1155,6 +1126,42 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt)
 	}
 }
 
+void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId)
+{
+	// Send an update to all members of the network that are online
+	const uint64_t now = OSUtils::now();
+	Mutex::Lock _l(_memberStatus_m);
+	for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) {
+		if ((i->first.networkId == networkId)&&(i->second.online(now))&&(i->second.lastRequestMetaData))
+			request(networkId,InetAddress(),0,i->second.identity,i->second.lastRequestMetaData);
+	}
+}
+
+void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId)
+{
+	// Push update to member if online
+	try {
+		Mutex::Lock _l(_memberStatus_m);
+		_MemberStatus &ms = _memberStatus[_MemberStatusKey(networkId,memberId)];
+		if ((ms.online(OSUtils::now()))&&(ms.lastRequestMetaData))
+			request(networkId,InetAddress(),0,ms.identity,ms.lastRequestMetaData);
+	} catch ( ... ) {}
+}
+
+void EmbeddedNetworkController::onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId)
+{
+	const uint64_t now = OSUtils::now();
+	Revocation rev((uint32_t)_node->prng(),networkId,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(memberId),Revocation::CREDENTIAL_TYPE_COM);
+	rev.sign(_signingId);
+	{
+		Mutex::Lock _l(_memberStatus_m);
+		for(auto i=_memberStatus.begin();i!=_memberStatus.end();++i) {
+			if ((i->first.networkId == networkId)&&(i->second.online(now)))
+				_node->ncSendRevocation(Address(i->first.nodeId),rev);
+		}
+	}
+}
+
 void EmbeddedNetworkController::threadMain()
 	throw()
 {
@@ -1195,7 +1202,7 @@ void EmbeddedNetworkController::threadMain()
 						first = false;
 					});
 				}
-				OSUtils::ztsnprintf(tmp,sizeof(tmp),"],\"clock\":%llu,\"startTime\":%llu,\"uptime\":%llu}",(unsigned long long)now,(unsigned long long)_startTime,(unsigned long long)(now - _startTime));
+				OSUtils::ztsnprintf(tmp,sizeof(tmp),"],\"clock\":%llu,\"startTime\":%llu,\"uptime\":%llu,\"vMajor\":%d,\"vMinor\":%d,\"vRev\":%d}",(unsigned long long)now,(unsigned long long)_startTime,(unsigned long long)(now - _startTime),ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION);
 				st.append(tmp);
 				_db.writeRaw("status",st);
 			}
@@ -1237,7 +1244,7 @@ void EmbeddedNetworkController::_request(
 	_initMember(member);
 
 	{
-		std::string haveIdStr(OSUtils::jsonString(member["identity"],""));
+		const std::string haveIdStr(OSUtils::jsonString(member["identity"],""));
 		if (haveIdStr.length() > 0) {
 			// If we already know this member's identity perform a full compare. This prevents
 			// a "collision" from being able to auth onto our network in place of an already
@@ -1268,56 +1275,29 @@ void EmbeddedNetworkController::_request(
 	}
 
 	// Determine whether and how member is authorized
-	const char *authorizedBy = (const char *)0;
+	bool authorized = false;
 	bool autoAuthorized = false;
 	json autoAuthCredentialType,autoAuthCredential;
 	if (OSUtils::jsonBool(member["authorized"],false)) {
-		authorizedBy = "memberIsAuthorized";
+		authorized = true;
 	} else if (!OSUtils::jsonBool(network["private"],true)) {
-		authorizedBy = "networkIsPublic";
-		json &ahist = member["authHistory"];
-		if ((!ahist.is_array())||(ahist.size() == 0))
-			autoAuthorized = true;
+		authorized = true;
+		autoAuthorized = true;
+		autoAuthCredentialType = "public";
 	} else {
 		char presentedAuth[512];
 		if (metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_AUTH,presentedAuth,sizeof(presentedAuth)) > 0) {
 			presentedAuth[511] = (char)0; // sanity check
-
-			// Check for bearer token presented by member
 			if ((strlen(presentedAuth) > 6)&&(!strncmp(presentedAuth,"token:",6))) {
 				const char *const presentedToken = presentedAuth + 6;
-
-				json &authTokens = network["authTokens"];
-				if (authTokens.is_array()) {
-					for(unsigned long i=0;i<authTokens.size();++i) {
-						json &token = authTokens[i];
-						if (token.is_object()) {
-							const uint64_t expires = OSUtils::jsonInt(token["expires"],0ULL);
-							const uint64_t maxUses = OSUtils::jsonInt(token["maxUsesPerMember"],0ULL);
-							std::string tstr = OSUtils::jsonString(token["token"],"");
-
-							if (((expires == 0ULL)||(expires > now))&&(tstr == presentedToken)) {
-								bool usable = (maxUses == 0);
-								if (!usable) {
-									uint64_t useCount = 0;
-									json &ahist = member["authHistory"];
-									if (ahist.is_array()) {
-										for(unsigned long j=0;j<ahist.size();++j) {
-											json &ah = ahist[j];
-											if ((OSUtils::jsonString(ah["ct"],"") == "token")&&(OSUtils::jsonString(ah["c"],"") == tstr)&&(OSUtils::jsonBool(ah["a"],false)))
-												++useCount;
-										}
-									}
-									usable = (useCount < maxUses);
-								}
-								if (usable) {
-									authorizedBy = "token";
-									autoAuthorized = true;
-									autoAuthCredentialType = "token";
-									autoAuthCredential = tstr;
-								}
-							}
-						}
+				json authTokens(network["authTokens"]);
+				json &tokenExpires = authTokens[presentedToken];
+				if (tokenExpires.is_number()) {
+					if ((tokenExpires == 0)||(tokenExpires > now)) {
+						authorized = true;
+						autoAuthorized = true;
+						autoAuthCredentialType = "token";
+						autoAuthCredential = presentedToken;
 					}
 				}
 			}
@@ -1325,23 +1305,14 @@ void EmbeddedNetworkController::_request(
 	}
 
 	// If we auto-authorized, update member record
-	if ((autoAuthorized)&&(authorizedBy)) {
+	if ((autoAuthorized)&&(authorized)) {
 		member["authorized"] = true;
 		member["lastAuthorizedTime"] = now;
-
-		json ah;
-		ah["a"] = true;
-		ah["by"] = authorizedBy;
-		ah["ts"] = now;
-		ah["ct"] = autoAuthCredentialType;
-		ah["c"] = autoAuthCredential;
-		member["authHistory"].push_back(ah);
-
-		json &revj = member["revision"];
-		member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
+		member["lastAuthorizedCredentialType"] = autoAuthCredentialType;
+		member["lastAuthorizedCredential"] = autoAuthCredential;
 	}
 
-	if (authorizedBy) {
+	if (authorized) {
 		// Update version info and meta-data if authorized and if this is a genuine request
 		if (requestPacketId) {
 			const uint64_t vMajor = metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_NODE_MAJOR_VERSION,0);
@@ -1367,6 +1338,7 @@ void EmbeddedNetworkController::_request(
 
 				if (fromAddr)
 					ms.physicalAddr = fromAddr;
+
 				char tmpip[64];
 				if (ms.physicalAddr)
 					member["physicalAddr"] = ms.physicalAddr.toString(tmpip);
@@ -1375,8 +1347,11 @@ void EmbeddedNetworkController::_request(
 	} else {
 		// If they are not authorized, STOP!
 		_removeMemberNonPersistedFields(member);
-		if (origMember != member)
+		if (origMember != member) {
+			json &revj = member["revision"];
+			member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
 			_db.saveNetworkMember(nwid,identity.address().toInt(),member);
+		}
 		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
 		return;
 	}
@@ -1420,7 +1395,7 @@ void EmbeddedNetworkController::_request(
 		if (rtt.length() == 10) {
 			nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str()));
 		} else {
-			nc->remoteTraceTarget = _signingId.address();
+			nc->remoteTraceTarget.zero();
 		}
 	}
 
@@ -1739,10 +1714,27 @@ void EmbeddedNetworkController::_request(
 	}
 
 	_removeMemberNonPersistedFields(member);
-	if (member != origMember)
+	if (member != origMember) {
+		json &revj = member["revision"];
+		member["revision"] = (revj.is_number() ? ((uint64_t)revj + 1ULL) : 1ULL);
 		_db.saveNetworkMember(nwid,identity.address().toInt(),member);
+	}
 
 	_sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6);
 }
 
+void EmbeddedNetworkController::_startThreads()
+{
+	Mutex::Lock _l(_threads_m);
+	if (_threads.size() == 0) {
+		long hwc = (long)std::thread::hardware_concurrency();
+		if (hwc < 1)
+			hwc = 1;
+		else if (hwc > 16)
+			hwc = 16;
+		for(long i=0;i<hwc;++i)
+			_threads.push_back(Thread::start(this));
+	}
+}
+
 } // namespace ZeroTier

controller/EmbeddedNetworkController.hpp

@@ -93,6 +93,11 @@ public:
 
 	void handleRemoteTrace(const ZT_RemoteTrace &rt);
 
+	// Called on update via POST or by JSONDB on external update of network or network member records
+	void onNetworkUpdate(const uint64_t networkId);
+	void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId);
+	void onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId);
+
 	void threadMain()
 		throw();
 
@@ -110,26 +115,12 @@ private:
 	};
 
 	void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData);
-
-	inline void _startThreads()
-	{
-		Mutex::Lock _l(_threads_m);
-		if (_threads.size() == 0) {
-			long hwc = (long)std::thread::hardware_concurrency();
-			if (hwc < 1)
-				hwc = 1;
-			else if (hwc > 16)
-				hwc = 16;
-			for(long i=0;i<hwc;++i)
-				_threads.push_back(Thread::start(this));
-		}
-	}
+	void _startThreads();
 
 	// These init objects with default and static/informational fields
 	inline void _initMember(nlohmann::json &member)
 	{
 		if (!member.count("authorized")) member["authorized"] = false;
-		if (!member.count("authHistory")) member["authHistory"] = nlohmann::json::array();
  		if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array();
 		if (!member.count("activeBridge")) member["activeBridge"] = false;
 		if (!member.count("tags")) member["tags"] = nlohmann::json::array();
@@ -139,6 +130,8 @@ private:
 		if (!member.count("revision")) member["revision"] = 0ULL;
 		if (!member.count("lastDeauthorizedTime")) member["lastDeauthorizedTime"] = 0ULL;
 		if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL;
+		if (!member.count("lastAuthorizedCredentialType")) member["lastAuthorizedCredentialType"] = nlohmann::json();
+		if (!member.count("lastAuthorizedCredential")) member["lastAuthorizedCredential"] = nlohmann::json();
 		if (!member.count("vMajor")) member["vMajor"] = -1;
 		if (!member.count("vMinor")) member["vMinor"] = -1;
 		if (!member.count("vRev")) member["vRev"] = -1;
@@ -156,7 +149,7 @@ private:
 		if (!network.count("enableBroadcast")) network["enableBroadcast"] = true;
 		if (!network.count("v4AssignMode")) network["v4AssignMode"] = {{"zt",false}};
 		if (!network.count("v6AssignMode")) network["v6AssignMode"] = {{"rfc4193",false},{"zt",false},{"6plane",false}};
-		if (!network.count("authTokens")) network["authTokens"] = nlohmann::json::array();
+		if (!network.count("authTokens")) network["authTokens"] = {{}};
 		if (!network.count("capabilities")) network["capabilities"] = nlohmann::json::array();
 		if (!network.count("tags")) network["tags"] = nlohmann::json::array();
 		if (!network.count("routes")) network["routes"] = nlohmann::json::array();

controller/JSONDB.cpp

@@ -29,75 +29,44 @@
 #endif
 
 #include "JSONDB.hpp"
-
-#define ZT_JSONDB_HTTP_TIMEOUT 60000
+#include "EmbeddedNetworkController.hpp"
 
 namespace ZeroTier {
 
 static const nlohmann::json _EMPTY_JSON(nlohmann::json::object());
-static const std::map<std::string,std::string> _ZT_JSONDB_GET_HEADERS;
 
-JSONDB::JSONDB(const std::string &basePath) :
+JSONDB::JSONDB(const std::string &basePath,EmbeddedNetworkController *parent) :
+	_parent(parent),
 	_basePath(basePath),
 	_rawInput(-1),
 	_rawOutput(-1),
 	_summaryThreadRun(true),
 	_dataReady(false)
 {
-	if ((_basePath.length() > 7)&&(_basePath.substr(0,7) == "http://")) {
-		// If base path is http:// we run in HTTP mode
-		// TODO: this doesn't yet support IPv6 since bracketed address notiation isn't supported.
-		// Typically it's just used with 127.0.0.1 anyway.
-		std::string hn = _basePath.substr(7);
-		std::size_t hnend = hn.find_first_of('/');
-		if (hnend != std::string::npos)
-			hn = hn.substr(0,hnend);
-		std::size_t hnsep = hn.find_last_of(':');
-		if (hnsep != std::string::npos)
-			hn[hnsep] = '/';
-		_httpAddr.fromString(hn.c_str());
-		if (hnend != std::string::npos)
-			_basePath = _basePath.substr(7 + hnend);
-		if (_basePath.length() == 0)
-			_basePath = "/";
-		if (_basePath[0] != '/')
-			_basePath = std::string("/") + _basePath;
 #ifndef __WINDOWS__
-	} else if (_basePath == "-") {
-		// If base path is "-" we run in stdin/stdout mode and expect our database to be populated on startup via stdin
-		// Not supported on Windows
+	if (_basePath == "-") {
+		// If base path is "-" we run in Central harnessed mode. We read pseudo-http-requests from stdin and write
+		// them to stdout.
 		_rawInput = STDIN_FILENO;
 		_rawOutput = STDOUT_FILENO;
 		fcntl(_rawInput,F_SETFL,O_NONBLOCK);
-#endif
 	} else {
+#endif
 		// Default mode of operation is to store files in the filesystem
 		OSUtils::mkdir(_basePath.c_str());
 		OSUtils::lockDownFile(_basePath.c_str(),true); // networks might contain auth tokens, etc., so restrict directory permissions
+#ifndef __WINDOWS__
 	}
+#endif
 
 	_networks_m.lock(); // locked until data is loaded, etc.
 
 	if (_rawInput < 0) {
-		unsigned int cnt = 0;
-		while (!_load(_basePath)) {
-			if ((++cnt & 7) == 0)
-				fprintf(stderr,"WARNING: controller still waiting to read '%s'..." ZT_EOL_S,_basePath.c_str());
-			Thread::sleep(250);
-		}
-
-		for(std::unordered_map<uint64_t,_NW>::iterator n(_networks.begin());n!=_networks.end();++n)
-			_summaryThreadToDo.push_back(n->first);
-
-		if (_summaryThreadToDo.size() > 0) {
-			_summaryThread = Thread::start(this);
-		} else {
-			_dataReady = true;
-			_networks_m.unlock();
-		}
+		_load(basePath);
+		_dataReady = true;
+		_networks_m.unlock();
 	} else {
-		// In IPC mode we wait for the first message to start, and we start
-		// this thread since this thread is responsible for reading from stdin.
+		// In harnessed mode we leave the lock locked and wait for our initial DB from Central.
 		_summaryThread = Thread::start(this);
 	}
 }
@@ -128,16 +97,6 @@ bool JSONDB::writeRaw(const std::string &n,const std::string &obj)
 		} else return true;
 #endif
 		return false;
-	} else if (_httpAddr) {
-		std::map<std::string,std::string> headers;
-		std::string body;
-		std::map<std::string,std::string> reqHeaders;
-		char tmp[64];
-		OSUtils::ztsnprintf(tmp,sizeof(tmp),"%lu",(unsigned long)obj.length());
-		reqHeaders["Content-Length"] = tmp;
-		reqHeaders["Content-Type"] = "application/json";
-		const unsigned int sc = Http::PUT(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),(_basePath+"/"+n).c_str(),reqHeaders,obj.data(),(unsigned long)obj.length(),headers,body);
-		return (sc == 200);
 	} else {
 		const std::string path(_genPath(n,true));
 		if (!path.length())
@@ -207,7 +166,8 @@ void JSONDB::saveNetwork(const uint64_t networkId,const nlohmann::json &networkC
 	writeRaw(n,OSUtils::jsonDump(networkConfig,-1));
 	{
 		Mutex::Lock _l(_networks_m);
-		_networks[networkId].config = nlohmann::json::to_msgpack(networkConfig);
+		_NW &nw = _networks[networkId];
+		nw.config = nlohmann::json::to_msgpack(networkConfig);
 	}
 	_recomputeSummaryInfo(networkId);
 }
@@ -219,7 +179,8 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co
 	writeRaw(n,OSUtils::jsonDump(memberConfig,-1));
 	{
 		Mutex::Lock _l(_networks_m);
-		_networks[networkId].members[nodeId] = nlohmann::json::to_msgpack(memberConfig);
+		std::vector<uint8_t> &m = _networks[networkId].members[nodeId];
+		m = nlohmann::json::to_msgpack(memberConfig);
 		_members[nodeId].insert(networkId);
 	}
 	_recomputeSummaryInfo(networkId);
@@ -227,7 +188,10 @@ void JSONDB::saveNetworkMember(const uint64_t networkId,const uint64_t nodeId,co
 
 nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId)
 {
-	if (!_httpAddr) { // Member deletion is done by Central in harnessed mode, and deleting the cache network entry also deletes all members
+	if (_rawOutput >= 0) {
+		// In harnessed mode, DB deletes occur in the Central database and we do
+		// not need to erase files.
+	} else {
 		std::vector<uint64_t> memberIds;
 		{
 			Mutex::Lock _l(_networks_m);
@@ -239,24 +203,15 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId)
 		}
 		for(std::vector<uint64_t>::iterator m(memberIds.begin());m!=memberIds.end();++m)
 			eraseNetworkMember(networkId,*m,false);
-	}
 
-	char n[256];
-	OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId);
-
-	if (_rawOutput >= 0) {
-		// In harnessed mode, deletes occur in Central or other management
-		// software and do not need to be executed this way.
-	} else if (_httpAddr) {
-		std::map<std::string,std::string> headers;
-		std::string body;
-		Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body);
-	} else {
+		char n[256];
+		OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx",(unsigned long long)networkId);
 		const std::string path(_genPath(n,false));
 		if (path.length())
 			OSUtils::rm(path.c_str());
 	}
 
+	// This also erases all members from the memory cache
 	{
 		Mutex::Lock _l(_networks_m);
 		std::unordered_map<uint64_t,_NW>::iterator i(_networks.find(networkId));
@@ -270,17 +225,11 @@ nlohmann::json JSONDB::eraseNetwork(const uint64_t networkId)
 
 nlohmann::json JSONDB::eraseNetworkMember(const uint64_t networkId,const uint64_t nodeId,bool recomputeSummaryInfo)
 {
-	char n[256];
-	OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId);
-
 	if (_rawOutput >= 0) {
-		// In harnessed mode, deletes occur in Central or other management
-		// software and do not need to be executed this way.
-	} else if (_httpAddr) {
-		std::map<std::string,std::string> headers;
-		std::string body;
-		Http::DEL(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),(_basePath+"/"+n).c_str(),_ZT_JSONDB_GET_HEADERS,headers,body);
+		// In harnessed mode, DB deletes occur in Central and we do not remove files.
 	} else {
+		char n[256];
+		OSUtils::ztsnprintf(n,sizeof(n),"network/%.16llx/member/%.10llx",(unsigned long long)networkId,(unsigned long long)nodeId);
 		const std::string path(_genPath(n,false));
 		if (path.length())
 			OSUtils::rm(path.c_str());
@@ -320,7 +269,6 @@ void JSONDB::threadMain()
 	while (_summaryThreadRun) {
 #ifndef __WINDOWS__
 		if (_rawInput < 0) {
-			// In HTTP and filesystem mode we just wait for summary to-do items
 			Thread::sleep(25);
 		} else {
 			// In IPC mode we wait but also select() on STDIN to read database updates
@@ -337,8 +285,8 @@ void JSONDB::threadMain()
 					} else if (rawInputBuf.length() > 0) {
 						try {
 							const nlohmann::json obj(OSUtils::jsonParse(rawInputBuf));
-
 							gotMessage = true;
+
 							if (!_dataReady) {
 								_dataReady = true;
 								_networks_m.unlock();
@@ -346,11 +294,12 @@ void JSONDB::threadMain()
 
 							if (obj.is_array()) {
 								for(unsigned long i=0;i<obj.size();++i)
-									_add(obj[i]);
+									_addOrUpdate(obj[i]);
 							} else if (obj.is_object()) {
-								_add(obj);
+								_addOrUpdate(obj);
 							}
 						} catch ( ... ) {} // ignore malformed JSON
+
 						rawInputBuf.clear();
 					}
 				}
@@ -369,7 +318,7 @@ void JSONDB::threadMain()
 			else _summaryThreadToDo.swap(todo);
 		}
 
-		if (!_dataReady) {
+		if (!_dataReady) { // sanity check
 			_dataReady = true;
 			_networks_m.unlock();
 		}
@@ -450,29 +399,71 @@ void JSONDB::threadMain()
 #endif
 }
 
-bool JSONDB::_add(const nlohmann::json &j)
+bool JSONDB::_addOrUpdate(const nlohmann::json &j)
 {
 	try {
 		if (j.is_object()) {
 			std::string id(OSUtils::jsonString(j["id"],"0"));
-			std::string objtype(OSUtils::jsonString(j["objtype"],""));
-
+			const std::string objtype(OSUtils::jsonString(j["objtype"],""));
 			if ((id.length() == 16)&&(objtype == "network")) {
+
 				const uint64_t nwid = Utils::hexStrToU64(id.c_str());
 				if (nwid) {
-					Mutex::Lock _l(_networks_m);
-					_networks[nwid].config = nlohmann::json::to_msgpack(j);
+					bool update;
+					{
+						Mutex::Lock _l(_networks_m);
+						_NW &nw = _networks[nwid];
+						update = !nw.config.empty();
+						nw.config = nlohmann::json::to_msgpack(j);
+					}
+					if (update)
+						_parent->onNetworkUpdate(nwid);
+					_recomputeSummaryInfo(nwid);
 					return true;
 				}
+
 			} else if ((id.length() == 10)&&(objtype == "member")) {
+
 				const uint64_t mid = Utils::hexStrToU64(id.c_str());
 				const uint64_t nwid = Utils::hexStrToU64(OSUtils::jsonString(j["nwid"],"0").c_str());
 				if ((mid)&&(nwid)) {
-					Mutex::Lock _l(_networks_m);
-					_networks[nwid].members[mid] = nlohmann::json::to_msgpack(j);
-					_members[mid].insert(nwid);
+					bool update = false;
+					bool deauth = false;
+					{
+						Mutex::Lock _l(_networks_m);
+						std::vector<uint8_t> &m = _networks[nwid].members[mid];
+						if (!m.empty()) {
+							update = true;
+							nlohmann::json oldm(nlohmann::json::from_msgpack(m));
+							deauth = ((OSUtils::jsonBool(oldm["authorized"],false))&&(!OSUtils::jsonBool(j["authorized"],false)));
+						}
+						m = nlohmann::json::to_msgpack(j);
+						_members[mid].insert(nwid);
+					}
+					if (update) {
+						_parent->onNetworkMemberUpdate(nwid,mid);
+						if (deauth)
+							_parent->onNetworkMemberDeauthorize(nwid,mid);
+					}
+					_recomputeSummaryInfo(nwid);
 					return true;
 				}
+
+			} else if (objtype == "_delete") { // pseudo-object-type, only used in Central harnessed mode
+
+				const std::string deleteType(OSUtils::jsonString(j["deleteType"],""));
+				id = OSUtils::jsonString(j["deleteId"],"");
+				if ((deleteType == "network")&&(id.length() == 16)) {
+					eraseNetwork(Utils::hexStrToU64(id.c_str()));
+				} else if ((deleteType == "member")&&(id.length() == 10)) {
+					const std::string networkId(OSUtils::jsonString(j["deleteNetworkId"],""));
+					const uint64_t nwid = Utils::hexStrToU64(networkId.c_str());
+					const uint64_t mid = Utils::hexStrToU64(id.c_str());
+					if (networkId.length() == 16)
+						eraseNetworkMember(nwid,mid,true);
+					_parent->onNetworkMemberDeauthorize(nwid,mid);
+				}
+
 			}
 		}
 	} catch ( ... ) {}
@@ -484,48 +475,21 @@ bool JSONDB::_load(const std::string &p)
 	// This is not used in stdin/stdout mode. Instead data is populated by
 	// sending it all to stdin.
 
-	if (_httpAddr) {
-		// In HTTP harnessed mode we download our entire working data set on startup.
-
-		std::string body;
-		std::map<std::string,std::string> headers;
-		const unsigned int sc = Http::GET(0,ZT_JSONDB_HTTP_TIMEOUT,reinterpret_cast<const struct sockaddr *>(&_httpAddr),_basePath.c_str(),_ZT_JSONDB_GET_HEADERS,headers,body);
-		if (sc == 200) {
-			try {
-				nlohmann::json dbImg(OSUtils::jsonParse(body));
-				std::string tmp;
-				if (dbImg.is_object()) {
-					Mutex::Lock _l(_networks_m);
-					for(nlohmann::json::iterator i(dbImg.begin());i!=dbImg.end();++i) {
-						try {
-							_add(i.value());
-						} catch ( ... ) {}
-					}
-					return true;
-				}
-			} catch ( ... ) {} // invalid JSON, so maybe incomplete request
-		}
-		return false;
-
-	} else {
-		// In regular mode we recursively read it from controller.d/ on disk
-
-		std::vector<std::string> dl(OSUtils::listDirectory(p.c_str(),true));
-		for(std::vector<std::string>::const_iterator di(dl.begin());di!=dl.end();++di) {
-			if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) {
-				std::string buf;
-				if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) {
-					try {
-						_add(OSUtils::jsonParse(buf));
-					} catch ( ... ) {}
-				}
-			} else {
-				this->_load((p + ZT_PATH_SEPARATOR_S + *di));
+	std::vector<std::string> dl(OSUtils::listDirectory(p.c_str(),true));
+	for(std::vector<std::string>::const_iterator di(dl.begin());di!=dl.end();++di) {
+		if ((di->length() > 5)&&(di->substr(di->length() - 5) == ".json")) {
+			std::string buf;
+			if (OSUtils::readFile((p + ZT_PATH_SEPARATOR_S + *di).c_str(),buf)) {
+				try {
+					_addOrUpdate(OSUtils::jsonParse(buf));
+				} catch ( ... ) {}
 			}
+		} else {
+			this->_load((p + ZT_PATH_SEPARATOR_S + *di));
 		}
-		return true;
-
 	}
+
+	return true;
 }
 
 void JSONDB::_recomputeSummaryInfo(const uint64_t networkId)
@@ -543,23 +507,15 @@ std::string JSONDB::_genPath(const std::string &n,bool create)
 	if (pt.size() == 0)
 		return std::string();
 
-	char sep;
-	if (_httpAddr) {
-		sep = '/';
-		create = false;
-	} else {
-		sep = ZT_PATH_SEPARATOR;
-	}
-
 	std::string p(_basePath);
 	if (create) OSUtils::mkdir(p.c_str());
 	for(unsigned long i=0,j=(unsigned long)(pt.size()-1);i<j;++i) {
-		p.push_back(sep);
+		p.push_back(ZT_PATH_SEPARATOR);
 		p.append(pt[i]);
 		if (create) OSUtils::mkdir(p.c_str());
 	}
 
-	p.push_back(sep);
+	p.push_back(ZT_PATH_SEPARATOR);
 	p.append(pt[pt.size()-1]);
 	p.append(".json");
 

controller/JSONDB.hpp

@@ -37,11 +37,12 @@
 #include "../node/Mutex.hpp"
 #include "../ext/json/json.hpp"
 #include "../osdep/OSUtils.hpp"
-#include "../osdep/Http.hpp"
 #include "../osdep/Thread.hpp"
 
 namespace ZeroTier {
 
+class EmbeddedNetworkController;
+
 /**
  * Hierarchical JSON store that persists into the filesystem or via HTTP
  */
@@ -59,7 +60,7 @@ public:
 		uint64_t mostRecentDeauthTime;
 	};
 
-	JSONDB(const std::string &basePath);
+	JSONDB(const std::string &basePath,EmbeddedNetworkController *parent);
 	~JSONDB();
 
 	/**
@@ -156,13 +157,13 @@ public:
 		throw();
 
 private:
-	bool _add(const nlohmann::json &j);
+	bool _addOrUpdate(const nlohmann::json &j);
 	bool _load(const std::string &p);
 	void _recomputeSummaryInfo(const uint64_t networkId);
 	std::string _genPath(const std::string &n,bool create);
 
+	EmbeddedNetworkController *const _parent;
 	std::string _basePath;
-	InetAddress _httpAddr;
 	int _rawInput,_rawOutput;
 	Mutex _rawLock;
 

include/ZeroTierOne.h

@@ -92,6 +92,34 @@ extern "C" {
  */
 #define ZT_MAX_MTU 10000
 
+/**
+ * Minimum UDP payload size allowed
+ */
+#define ZT_MIN_PHYSMTU 1400
+
+/**
+ * Default UDP payload size (physical path MTU) not including UDP and IP overhead
+ *
+ * This is 1500 - IPv6 UDP overhead - PPPoE overhead and is safe for 99.9% of
+ * all Internet links.
+ */
+#define ZT_DEFAULT_PHYSMTU 1444
+
+/**
+ * Maximum physical UDP payload
+ */
+#define ZT_MAX_PHYSPAYLOAD 10100
+
+/**
+ * Headroom for max physical MTU
+ */
+#define ZT_MAX_HEADROOM 224
+
+/**
+ * Maximum payload MTU for UDP packets
+ */
+#define ZT_MAX_PHYSMTU (ZT_MAX_PHYSPAYLOAD + ZT_MAX_HEADROOM)
+
 /**
  * Maximum size of a remote trace message's serialized Dictionary
  */
@@ -148,9 +176,9 @@ extern "C" {
 #define ZT_MAX_PEER_NETWORK_PATHS 4
 
 /**
- * Maximum number of trusted physical network paths
+ * Maximum number of path configurations that can be set
  */
-#define ZT_MAX_TRUSTED_PATHS 16
+#define ZT_MAX_CONFIGURABLE_PATHS 32
 
 /**
  * Maximum number of rules per capability
@@ -1034,11 +1062,6 @@ typedef struct
 	 */
 	unsigned int mtu;
 
-	/**
-	 * Recommended MTU to avoid fragmentation at the physical layer (hint)
-	 */
-	unsigned int physicalMtu;
-
 	/**
 	 * If nonzero, the network this port belongs to indicates DHCP availability
 	 *
@@ -1108,6 +1131,21 @@ typedef struct
 	unsigned long networkCount;
 } ZT_VirtualNetworkList;
 
+/**
+ * Physical path configuration
+ */
+typedef struct {
+	/**
+	 * If non-zero set this physical network path to be trusted to disable encryption and authentication
+	 */
+	uint64_t trustedPathId;
+
+	/**
+	 * Physical path MTU from ZT_MIN_PHYSMTU and ZT_MAX_PHYSMTU or <= 0 to use default
+	 */
+	int mtu;
+} ZT_PhysicalPathConfiguration;
+
 /**
  * Physical network path to a peer
  */
@@ -1832,27 +1870,14 @@ ZT_SDK_API int ZT_Node_sendUserMessage(ZT_Node *node,void *tptr,uint64_t dest,ui
 ZT_SDK_API void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance);
 
 /**
- * Set trusted paths
- *
- * A trusted path is a physical network (network/bits) over which both
- * encryption and authentication can be skipped to improve performance.
- * Each trusted path must have a non-zero unique ID that is the same across
- * all participating nodes.
- *
- * We don't recommend using trusted paths at all unless you really *need*
- * near-bare-metal performance. Even on a LAN authentication and encryption
- * are never a bad thing, and anything that introduces an "escape hatch"
- * for encryption should be treated with the utmost care.
- *
- * Calling with NULL pointers for networks and ids and a count of zero clears
- * all trusted paths.
+ * Set configuration for a given physical path
  *
  * @param node Node instance
- * @param networks Array of [count] networks
- * @param ids Array of [count] corresponding non-zero path IDs (zero path IDs are ignored)
- * @param count Number of trusted paths-- values greater than ZT_MAX_TRUSTED_PATHS are clipped
+ * @param pathNetwork Network/CIDR of path or NULL to clear the cache and reset all paths to default
+ * @param pathConfig Path configuration or NULL to erase this entry and therefore reset it to NULL
+ * @return OK or error code
  */
-ZT_SDK_API void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count);
+ZT_SDK_API enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node *node,const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig);
 
 /**
  * Get ZeroTier One version

make-linux.mk

@@ -99,6 +99,15 @@ ifeq ($(CC_MACH),amd64)
 	ZT_ARCHITECTURE=2
 	ZT_USE_X64_ASM_SALSA2012=1
 endif
+ifeq ($(CC_MACH),powerpc64le)
+	ZT_ARCHITECTURE=8
+endif
+ifeq ($(CC_MACH),ppc64le)
+	ZT_ARCHITECTURE=8
+endif
+ifeq ($(CC_MACH),ppc64el)
+	ZT_ARCHITECTURE=8
+endif
 ifeq ($(CC_MACH),i386)
 	ZT_ARCHITECTURE=1
 endif
@@ -210,12 +219,6 @@ endif
 
 all:	one
 
-#ext/x64-salsa2012-asm/salsa2012.o:
-#	$(CC) -c ext/x64-salsa2012-asm/salsa2012.s -o ext/x64-salsa2012-asm/salsa2012.o
-
-#ext/arm32-neon-salsa2012-asm/salsa2012.o:
-#	$(CC) -c ext/arm32-neon-salsa2012-asm/salsa2012.s -o ext/arm32-neon-salsa2012-asm/salsa2012.o
-
 one:	$(CORE_OBJS) $(ONE_OBJS) one.o
 	$(CXX) $(CXXFLAGS) $(LDFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LDLIBS)
 	$(STRIP) zerotier-one
@@ -252,6 +255,9 @@ distclean:	clean
 
 realclean:	distclean
 
+official-static:	FORCE
+	make -j4 ZT_STATIC=1 LDLIBS=/usr/lib/libjemalloc.a all selftest
+
 debug:	FORCE
 	make ZT_DEBUG=1 one
 	make ZT_DEBUG=1 selftest

node/AtomicCounter.hpp

@@ -50,7 +50,7 @@ public:
 	inline int load() const
 	{
 #ifdef __GNUC__
-		return __sync_or_and_fetch(&_v,0);
+		return __sync_or_and_fetch(const_cast<int *>(&_v),0);
 #else
 		return _v.load();
 #endif

node/Capability.cpp

@@ -30,6 +30,7 @@
 #include "Topology.hpp"
 #include "Switch.hpp"
 #include "Network.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -59,7 +60,7 @@ int Capability::verify(const RuntimeEnvironment *RR,void *tPtr) const
 				if (!id.verify(tmp.data(),tmp.size(),_custody[c].signature))
 					return -1;
 			} else {
-				RR->sw->requestWhois(tPtr,_custody[c].from);
+				RR->sw->requestWhois(tPtr,RR->node->now(),_custody[c].from);
 				return 1;
 			}
 		}

node/CertificateOfMembership.cpp

@@ -29,6 +29,7 @@
 #include "Topology.hpp"
 #include "Switch.hpp"
 #include "Network.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -223,7 +224,7 @@ int CertificateOfMembership::verify(const RuntimeEnvironment *RR,void *tPtr) con
 
 	const Identity id(RR->topology->getIdentity(tPtr,_signedBy));
 	if (!id) {
-		RR->sw->requestWhois(tPtr,_signedBy);
+		RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy);
 		return 1;
 	}
 

node/CertificateOfOwnership.cpp

@@ -30,6 +30,7 @@
 #include "Topology.hpp"
 #include "Switch.hpp"
 #include "Network.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -39,7 +40,7 @@ int CertificateOfOwnership::verify(const RuntimeEnvironment *RR,void *tPtr) cons
 		return -1;
 	const Identity id(RR->topology->getIdentity(tPtr,_signedBy));
 	if (!id) {
-		RR->sw->requestWhois(tPtr,_signedBy);
+		RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy);
 		return 1;
 	}
 	try {

node/Constants.hpp

@@ -171,15 +171,6 @@
  */
 #define ZT_ADDRESS_RESERVED_PREFIX 0xff
 
-/**
- * Default payload MTU for UDP packets
- *
- * In the future we might support UDP path MTU discovery, but for now we
- * set a maximum that is equal to 1500 minus 8 (for PPPoE overhead, common
- * in some markets) minus 48 (IPv6 UDP overhead).
- */
-#define ZT_UDP_DEFAULT_PAYLOAD_MTU 1444
-
 /**
  * Default MTU used for Ethernet tap device
  */
@@ -226,22 +217,17 @@
 /**
  * Delay between WHOIS retries in ms
  */
-#define ZT_WHOIS_RETRY_DELAY 1000
-
-/**
- * Maximum identity WHOIS retries (each attempt tries consulting a different peer)
- */
-#define ZT_MAX_WHOIS_RETRIES 4
+#define ZT_WHOIS_RETRY_DELAY 500
 
 /**
  * Transmit queue entry timeout
  */
-#define ZT_TRANSMIT_QUEUE_TIMEOUT (ZT_WHOIS_RETRY_DELAY * (ZT_MAX_WHOIS_RETRIES + 1))
+#define ZT_TRANSMIT_QUEUE_TIMEOUT 5000
 
 /**
  * Receive queue entry timeout
  */
-#define ZT_RECEIVE_QUEUE_TIMEOUT (ZT_WHOIS_RETRY_DELAY * (ZT_MAX_WHOIS_RETRIES + 1))
+#define ZT_RECEIVE_QUEUE_TIMEOUT 5000
 
 /**
  * Maximum latency to allow for OK(HELLO) before packet is discarded

node/IncomingPacket.cpp

@@ -66,10 +66,9 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr)
 			// packets are dropped on the floor.
 			const uint64_t tpid = trustedPathId();
 			if (RR->topology->shouldInboundPathBeTrusted(_path->address(),tpid)) {
-				RR->t->incomingPacketTrustedPath(tPtr,_path,packetId(),sourceAddress,tpid,true);
 				trusted = true;
 			} else {
-				RR->t->incomingPacketTrustedPath(tPtr,_path,packetId(),sourceAddress,tpid,false);
+				RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops(),"path not trusted");
 				return true;
 			}
 		} else if ((c == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)&&(verb() == Packet::VERB_HELLO)) {
@@ -81,7 +80,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr)
 		if (peer) {
 			if (!trusted) {
 				if (!dearmor(peer->key())) {
-					RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops());
+					RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,packetId(),sourceAddress,hops(),"invalid MAC");
 					return true;
 				}
 			}
@@ -116,7 +115,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr)
 				case Packet::VERB_REMOTE_TRACE:               return _doREMOTE_TRACE(RR,tPtr,peer);
 			}
 		} else {
-			RR->sw->requestWhois(tPtr,sourceAddress);
+			RR->sw->requestWhois(tPtr,RR->node->now(),sourceAddress);
 			return false;
 		}
 	} catch ( ... ) {
@@ -246,10 +245,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
 						outp.armor(key,true,_path->nextOutgoingCounter());
 						_path->send(RR,tPtr,outp.data(),outp.size(),RR->node->now());
 					} else {
-						RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops());
+						RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid MAC");
 					}
 				} else {
-					RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops());
+					RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid identity");
 				}
 
 				return true;
@@ -257,7 +256,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
 				// Identity is the same as the one we already have -- check packet integrity
 
 				if (!dearmor(peer->key())) {
-					RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops());
+					RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid MAC");
 					return true;
 				}
 
@@ -282,7 +281,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool
 		// Check packet integrity and MAC (this is faster than locallyValidate() so do it first to filter out total crap)
 		SharedPtr<Peer> newPeer(new Peer(RR,RR->identity,id));
 		if (!dearmor(newPeer->key())) {
-			RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops());
+			RR->t->incomingPacketMessageAuthenticationFailure(tPtr,_path,pid,fromAddress,hops(),"invalid MAC");
 			return true;
 		}
 
@@ -557,7 +556,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const Shar
 			++count;
 		} else {
 			// Request unknown WHOIS from upstream from us (if we have one)
-			RR->sw->requestWhois(tPtr,addr);
+			RR->sw->requestWhois(tPtr,RR->node->now(),addr);
 		}
 	}
 

node/Membership.cpp

@@ -147,7 +147,6 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 			return ADD_REJECTED;
 		case 0:
 			_com = com;
-			RR->t->credentialAccepted(tPtr,com);
 			return ADD_ACCEPTED_NEW;
 		case 1:
 			return ADD_DEFERRED_FOR_WHOIS;
@@ -179,7 +178,6 @@ static Membership::AddCredentialResult _addCredImpl(Hashtable<uint32_t,C> &remot
 			RR->t->credentialRejected(tPtr,cred,"invalid");
 			return Membership::ADD_REJECTED;
 		case 0:
-			RR->t->credentialAccepted(tPtr,cred);
 			if (!rc)
 				rc = &(remoteCreds[cred.id()]);
 			*rc = cred;
@@ -205,7 +203,6 @@ Membership::AddCredentialResult Membership::addCredential(const RuntimeEnvironme
 			switch(ct) {
 				case Credential::CREDENTIAL_TYPE_COM:
 					if (rev.threshold() > _comRevocationThreshold) {
-						RR->t->credentialAccepted(tPtr,rev);
 						_comRevocationThreshold = rev.threshold();
 						return ADD_ACCEPTED_NEW;
 					}

node/Multicaster.cpp

@@ -111,7 +111,7 @@ unsigned int Multicaster::gather(const Address &queryingPeer,uint64_t nwid,const
 		// Members are returned in random order so that repeated gather queries
 		// will return different subsets of a large multicast group.
 		k = 0;
-		while ((added < limit)&&(k < s->members.size())&&((appendTo.size() + ZT_ADDRESS_LENGTH) <= ZT_UDP_DEFAULT_PAYLOAD_MTU)) {
+		while ((added < limit)&&(k < s->members.size())&&((appendTo.size() + ZT_ADDRESS_LENGTH) <= ZT_PROTO_MAX_PACKET_LENGTH)) {
 			rptr = (unsigned int)RR->node->prng();
 
 restart_member_scan:

node/Mutex.hpp

@@ -32,47 +32,96 @@
 
 #ifdef __UNIX_LIKE__
 
+#include <stdint.h>
 #include <stdlib.h>
 #include <pthread.h>
 
 namespace ZeroTier {
 
+#if defined(__GNUC__) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64))
+
+// Inline ticket lock on x64 systems with GCC and CLANG (Mac, Linux) -- this is really fast as long as locking durations are very short
 class Mutex : NonCopyable
 {
 public:
-	Mutex()
+	Mutex() :
+		nextTicket(0),
+		nowServing(0)
 	{
-		pthread_mutex_init(&_mh,(const pthread_mutexattr_t *)0);
 	}
 
-	~Mutex()
+	inline void lock() const
 	{
-		pthread_mutex_destroy(&_mh);
+		const uint16_t myTicket = __sync_fetch_and_add(&(const_cast<Mutex *>(this)->nextTicket),1); 
+		while (nowServing != myTicket) {
+			__asm__ __volatile__("rep;nop"::);
+			__asm__ __volatile__("":::"memory");
+		}   
 	}
 
-	inline void lock()
+	inline void unlock() const
 	{
-		pthread_mutex_lock(&_mh);
+		++(const_cast<Mutex *>(this)->nowServing);
 	}
 
-	inline void unlock()
+	/**
+	 * Uses C++ contexts and constructor/destructor to lock/unlock automatically
+	 */
+	class Lock : NonCopyable
 	{
-		pthread_mutex_unlock(&_mh);
+	public:
+		Lock(Mutex &m) :
+			_m(&m)
+		{
+			m.lock();
+		}
+
+		Lock(const Mutex &m) :
+			_m(const_cast<Mutex *>(&m))
+		{
+			_m->lock();
+		}
+
+		~Lock()
+		{
+			_m->unlock();
+		}
+
+	private:
+		Mutex *const _m;
+	};
+
+private:
+	uint16_t nextTicket;
+	uint16_t nowServing;
+};
+
+#else
+
+// libpthread based mutex lock
+class Mutex : NonCopyable
+{
+public:
+	Mutex()
+	{
+		pthread_mutex_init(&_mh,(const pthread_mutexattr_t *)0);
+	}
+
+	~Mutex()
+	{
+		pthread_mutex_destroy(&_mh);
 	}
 
 	inline void lock() const
 	{
-		(const_cast <Mutex *> (this))->lock();
+		pthread_mutex_lock(&((const_cast <Mutex *> (this))->_mh));
 	}
 
 	inline void unlock() const
 	{
-		(const_cast <Mutex *> (this))->unlock();
+		pthread_mutex_unlock(&((const_cast <Mutex *> (this))->_mh));
 	}
 
-	/**
-	 * Uses C++ contexts and constructor/destructor to lock/unlock automatically
-	 */
 	class Lock : NonCopyable
 	{
 	public:
@@ -101,6 +150,8 @@ private:
 	pthread_mutex_t _mh;
 };
 
+#endif
+
 } // namespace ZeroTier
 
 #endif // Apple / Linux
@@ -112,6 +163,7 @@ private:
 
 namespace ZeroTier {
 
+// Windows critical section based lock
 class Mutex : NonCopyable
 {
 public:
@@ -125,16 +177,6 @@ public:
 		DeleteCriticalSection(&_cs);
 	}
 
-	inline void lock()
-	{
-		EnterCriticalSection(&_cs);
-	}
-
-	inline void unlock()
-	{
-		LeaveCriticalSection(&_cs);
-	}
-
 	inline void lock() const
 	{
 		(const_cast <Mutex *> (this))->lock();

node/Network.cpp

@@ -1346,7 +1346,6 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	ec->status = _status();
 	ec->type = (_config) ? (_config.isPrivate() ? ZT_NETWORK_TYPE_PRIVATE : ZT_NETWORK_TYPE_PUBLIC) : ZT_NETWORK_TYPE_PRIVATE;
 	ec->mtu = (_config) ? _config.mtu : ZT_DEFAULT_MTU;
-	ec->physicalMtu = ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 16);
 	ec->dhcp = 0;
 	std::vector<Address> ab(_config.activeBridges());
 	ec->bridge = ((_config.allowPassiveBridging())||(std::find(ab.begin(),ab.end(),RR->identity.address()) != ab.end())) ? 1 : 0;

node/NetworkConfig.cpp

@@ -35,6 +35,7 @@ namespace ZeroTier {
 bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,bool includeLegacy) const
 {
 	Buffer<ZT_NETWORKCONFIG_DICT_CAPACITY> *tmp = new Buffer<ZT_NETWORKCONFIG_DICT_CAPACITY>();
+	char tmp2[128];
 
 	try {
 		d.clear();
@@ -46,8 +47,8 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,this->timestamp)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_CREDENTIAL_TIME_MAX_DELTA,this->credentialTimeMaxDelta)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REVISION,this->revision)) return false;
-		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo)) return false;
-		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET,this->remoteTraceTarget)) return false;
+		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,this->issuedTo.toString(tmp2))) return false;
+		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_REMOTE_TRACE_TARGET,this->remoteTraceTarget.toString(tmp2))) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_FLAGS,this->flags)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,(uint64_t)this->multicastLimit)) return false;
 		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_TYPE,(uint64_t)this->type)) return false;

node/Node.cpp

@@ -100,7 +100,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,uint6
 	} else {
 		idtmp[0] = RR->identity.address().toInt(); idtmp[1] = 0;
 		n = stateObjectGet(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,tmp,sizeof(tmp) - 1);
-		if ((n > 0)&&(n < sizeof(RR->publicIdentityStr))&&(n < sizeof(tmp))) {
+		if ((n > 0)&&(n < (int)sizeof(RR->publicIdentityStr))&&(n < (int)sizeof(tmp))) {
 			if (memcmp(tmp,RR->publicIdentityStr,n))
 				stateObjectPut(tptr,ZT_STATE_OBJECT_IDENTITY_PUBLIC,idtmp,RR->publicIdentityStr,(unsigned int)strlen(RR->publicIdentityStr));
 		}
@@ -249,6 +249,19 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint
 		try {
 			_lastPingCheck = now;
 
+			// Do pings and keepalives
+			Hashtable< Address,std::vector<InetAddress> > upstreamsToContact;
+			RR->topology->getUpstreamsToContact(upstreamsToContact);
+			_PingPeersThatNeedPing pfunc(RR,tptr,upstreamsToContact,now);
+			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
+
+			// Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds)
+			Hashtable< Address,std::vector<InetAddress> >::Iterator i(upstreamsToContact);
+			Address *upstreamAddress = (Address *)0;
+			std::vector<InetAddress> *upstreamStableEndpoints = (std::vector<InetAddress> *)0;
+			while (i.next(upstreamAddress,upstreamStableEndpoints))
+				RR->sw->requestWhois(tptr,now,*upstreamAddress);
+
 			// Get networks that need config without leaving mutex locked
 			{
 				std::vector< std::pair< SharedPtr<Network>,bool > > nwl;
@@ -268,19 +281,6 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,uint64_t now,volatile uint
 				}
 			}
 
-			// Do pings and keepalives
-			Hashtable< Address,std::vector<InetAddress> > upstreamsToContact;
-			RR->topology->getUpstreamsToContact(upstreamsToContact);
-			_PingPeersThatNeedPing pfunc(RR,tptr,upstreamsToContact,now);
-			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
-
-			// Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds)
-			Hashtable< Address,std::vector<InetAddress> >::Iterator i(upstreamsToContact);
-			Address *upstreamAddress = (Address *)0;
-			std::vector<InetAddress> *upstreamStableEndpoints = (std::vector<InetAddress> *)0;
-			while (i.next(upstreamAddress,upstreamStableEndpoints))
-				RR->sw->requestWhois(tptr,*upstreamAddress);
-
 			// Update online status, post status change as event
 			const bool oldOnline = _online;
 			_online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot()));
@@ -561,9 +561,9 @@ uint64_t Node::prng()
 	return z + y;
 }
 
-void Node::setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count)
+ZT_ResultCode Node::setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork, const ZT_PhysicalPathConfiguration *pathConfig)
 {
-	RR->topology->setTrustedPaths(reinterpret_cast<const InetAddress *>(networks),ids,count);
+	return ZT_RESULT_OK;
 }
 
 World Node::planet() const
@@ -592,7 +592,7 @@ void Node::ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &de
 				const unsigned int totalSize = dconf->sizeBytes();
 				unsigned int chunkIndex = 0;
 				while (chunkIndex < totalSize) {
-					const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - (ZT_PACKET_IDX_PAYLOAD + 256)));
+					const unsigned int chunkLen = std::min(totalSize - chunkIndex,(unsigned int)(ZT_PROTO_MAX_PACKET_LENGTH - (ZT_PACKET_IDX_PAYLOAD + 256)));
 					Packet outp(destination,RR->identity.address(),(requestPacketId) ? Packet::VERB_OK : Packet::VERB_NETWORK_CONFIG);
 					if (requestPacketId) {
 						outp.append((unsigned char)Packet::VERB_NETWORK_CONFIG_REQUEST);
@@ -815,7 +815,7 @@ enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,void *tptr,uint64_t moonWorldId,u
 	}
 }
 
-ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId)
+enum ZT_ResultCode ZT_Node_deorbit(ZT_Node *node,void *tptr,uint64_t moonWorldId)
 {
 	try {
 		return reinterpret_cast<ZeroTier::Node *>(node)->deorbit(tptr,moonWorldId);
@@ -902,11 +902,13 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 	} catch ( ... ) {}
 }
 
-void ZT_Node_setTrustedPaths(ZT_Node *node,const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count)
+enum ZT_ResultCode ZT_Node_setPhysicalPathConfiguration(ZT_Node *node,const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig)
 {
 	try {
-		reinterpret_cast<ZeroTier::Node *>(node)->setTrustedPaths(networks,ids,count);
-	} catch ( ... ) {}
+		return reinterpret_cast<ZeroTier::Node *>(node)->setPhysicalPathConfiguration(pathNetwork,pathConfig);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
 }
 
 void ZT_version(int *major,int *minor,int *revision)

node/Node.hpp

@@ -192,7 +192,7 @@ public:
 	inline bool externalPathLookup(void *tPtr,const Address &ztaddr,int family,InetAddress &addr) { return ( (_cb.pathLookupFunction) ? (_cb.pathLookupFunction(reinterpret_cast<ZT_Node *>(this),_uPtr,tPtr,ztaddr.toInt(),family,reinterpret_cast<struct sockaddr_storage *>(&addr)) != 0) : false ); }
 
 	uint64_t prng();
-	void setTrustedPaths(const struct sockaddr_storage *networks,const uint64_t *ids,unsigned int count);
+	ZT_ResultCode setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig);
 
 	World planet() const;
 	std::vector<World> moons() const;

node/Packet.hpp

@@ -225,12 +225,8 @@
 
 /**
  * Packet buffer size (can be changed)
- *
- * The current value is big enough for ZT_MAX_PACKET_FRAGMENTS, the pragmatic
- * packet fragment limit, times the default UDP MTU. Most packets won't be
- * this big.
  */
-#define ZT_PROTO_MAX_PACKET_LENGTH (ZT_MAX_PACKET_FRAGMENTS * ZT_UDP_DEFAULT_PAYLOAD_MTU)
+#define ZT_PROTO_MAX_PACKET_LENGTH (ZT_MAX_PACKET_FRAGMENTS * ZT_DEFAULT_PHYSMTU)
 
 /**
  * Minimum viable packet length (a.k.a. header length)

node/Peer.cpp

@@ -194,8 +194,12 @@ void Peer::received(
 				}
 			}
 		}
-	} else if (this->trustEstablished(now)) {
-		// Send PUSH_DIRECT_PATHS if hops>0 (relayed) and we have a trust relationship (common network membership)
+	}
+
+	// If we are being relayed or if we're using a global address, send PUSH_DIRECT_PATHS.
+	// In the global address case we push only configured direct paths to accomplish
+	// fall-forward to local backplane networks over e.g. LAN or Amazon VPC.
+	if ( ((hops > 0)||(path->ipScope() == InetAddress::IP_SCOPE_GLOBAL)) && (this->trustEstablished(now)) ) {
 		if ((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL) {
 			_lastDirectPathPushSent = now;
 
@@ -205,13 +209,15 @@ void Peer::received(
 			for(std::vector<InetAddress>::const_iterator i(dps.begin());i!=dps.end();++i)
 				pathsToPush.push_back(*i);
 
-			std::vector<InetAddress> sym(RR->sa->getSymmetricNatPredictions());
-			for(unsigned long i=0,added=0;i<sym.size();++i) {
-				InetAddress tmp(sym[(unsigned long)RR->node->prng() % sym.size()]);
-				if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) {
-					pathsToPush.push_back(tmp);
-					if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY)
-						break;
+			if (hops > 0) {
+				std::vector<InetAddress> sym(RR->sa->getSymmetricNatPredictions());
+				for(unsigned long i=0,added=0;i<sym.size();++i) {
+					InetAddress tmp(sym[(unsigned long)RR->node->prng() % sym.size()]);
+					if (std::find(pathsToPush.begin(),pathsToPush.end(),tmp) == pathsToPush.end()) {
+						pathsToPush.push_back(tmp);
+						if (++added >= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY)
+							break;
+					}
 				}
 			}
 
@@ -369,7 +375,7 @@ void Peer::tryMemorizedPath(void *tPtr,uint64_t now)
 		_lastTriedMemorizedPath = now;
 		InetAddress mp;
 		if (RR->node->externalPathLookup(tPtr,_id.address(),-1,mp))
-			attemptToContactAt(tPtr,InetAddress(),mp,now,true,0);
+			attemptToContactAt(tPtr,-1,mp,now,true,0);
 	}
 }
 
@@ -424,18 +430,21 @@ void Peer::redirect(void *tPtr,const int64_t localSocket,const InetAddress &remo
 
 	SharedPtr<Path> op;
 	SharedPtr<Path> np(RR->topology->getPath(localSocket,remoteAddress));
+	np->received(now);
 	attemptToContactAt(tPtr,localSocket,remoteAddress,now,true,np->nextOutgoingCounter());
 
 	{
 		Mutex::Lock _l(_paths_m);
 		if (remoteAddress.ss_family == AF_INET) {
 			op = _v4Path.p;
-			_v4Path.p = np;
+			_v4Path.lr = now;
 			_v4Path.sticky = now;
+			_v4Path.p = np;
 		} else if (remoteAddress.ss_family == AF_INET6) {
 			op = _v6Path.p;
-			_v6Path.p = np;
+			_v6Path.lr = now;
 			_v6Path.sticky = now;
+			_v6Path.p = np;
 		}
 	}
 

node/Peer.hpp

@@ -439,6 +439,90 @@ public:
 		return false;
 	}
 
+	/**
+	 * Serialize a peer for storage in local cache
+	 *
+	 * This does not serialize everything, just identity and addresses where the peer
+	 * may be reached.
+	 */
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append((uint8_t)0);
+
+		_id.serialize(b);
+
+		b.append(_lastReceive);
+		b.append(_lastNontrivialReceive);
+		b.append(_lastTriedMemorizedPath);
+		b.append(_lastDirectPathPushSent);
+		b.append(_lastDirectPathPushReceive);
+		b.append(_lastCredentialRequestSent);
+		b.append(_lastWhoisRequestReceived);
+		b.append(_lastEchoRequestReceived);
+		b.append(_lastComRequestReceived);
+		b.append(_lastComRequestSent);
+		b.append(_lastCredentialsReceived);
+		b.append(_lastTrustEstablishedPacketReceived);
+
+		b.append((uint16_t)_vProto);
+		b.append((uint16_t)_vMajor);
+		b.append((uint16_t)_vMinor);
+		b.append((uint16_t)_vRevision);
+
+		{
+			Mutex::Lock _l(_paths_m);
+			unsigned int pcount = 0;
+			if (_v4Path.p) ++pcount;
+			if (_v6Path.p) ++pcount;
+			b.append((uint8_t)pcount);
+			if (_v4Path.p) _v4Path.p->address().serialize(b);
+			if (_v6Path.p) _v6Path.p->address().serialize(b);
+		}
+
+		b.append((uint16_t)0);
+	}
+
+	template<unsigned int C>
+	inline static SharedPtr<Peer> deserializeFromCache(uint64_t now,void *tPtr,Buffer<C> &b,const RuntimeEnvironment *renv)
+	{
+		try {
+			unsigned int ptr = 0;
+			if (b[ptr++] != 0)
+				return SharedPtr<Peer>();
+
+			Identity id;
+			ptr += id.deserialize(b,ptr);
+			if (!id)
+				return SharedPtr<Peer>();
+
+			SharedPtr<Peer> p(new Peer(renv,renv->identity,id));
+
+			ptr += 12 * 8; // skip deserializing ephemeral state in this case
+
+			p->_vProto = b.template at<uint16_t>(ptr); ptr += 2;
+			p->_vMajor = b.template at<uint16_t>(ptr); ptr += 2;
+			p->_vMinor = b.template at<uint16_t>(ptr); ptr += 2;
+			p->_vRevision = b.template at<uint16_t>(ptr); ptr += 2;
+
+			const unsigned int pcount = (unsigned int)b[ptr++];
+			for(unsigned int i=0;i<pcount;++i) {
+				InetAddress inaddr;
+				try {
+					ptr += inaddr.deserialize(b,ptr);
+					if (inaddr)
+						p->attemptToContactAt(tPtr,-1,inaddr,now,true,0);
+				} catch ( ... ) {
+					break;
+				}
+			}
+
+			return p;
+		} catch ( ... ) {
+			return SharedPtr<Peer>();
+		}
+	}
+
 private:
 	struct _PeerPath
 	{

node/Revocation.cpp

@@ -30,6 +30,7 @@
 #include "Topology.hpp"
 #include "Switch.hpp"
 #include "Network.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -39,7 +40,7 @@ int Revocation::verify(const RuntimeEnvironment *RR,void *tPtr) const
 		return -1;
 	const Identity id(RR->topology->getIdentity(tPtr,_signedBy));
 	if (!id) {
-		RR->sw->requestWhois(tPtr,_signedBy);
+		RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy);
 		return 1;
 	}
 	try {

node/SelfAwareness.cpp

@@ -147,13 +147,14 @@ std::vector<InetAddress> SelfAwareness::getSymmetricNatPredictions()
 	 * read or modify traffic, but they could gather meta-data for forensics
 	 * purpsoes or use this as a DOS attack vector. */
 
-	std::map< uint32_t,std::pair<uint64_t,unsigned int> > maxPortByIp;
+	std::map< uint32_t,unsigned int > maxPortByIp;
 	InetAddress theOneTrueSurface;
-	bool symmetric = false;
 	{
 		Mutex::Lock _l(_phy_m);
 
-		{	// First get IPs from only trusted peers, and perform basic NAT type characterization
+		// First check to see if this is a symmetric NAT and enumerate external IPs learned from trusted peers
+		bool symmetric = false;
+		{
 			Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy);
 			PhySurfaceKey *k = (PhySurfaceKey *)0;
 			PhySurfaceEntry *e = (PhySurfaceEntry *)0;
@@ -163,42 +164,47 @@ std::vector<InetAddress> SelfAwareness::getSymmetricNatPredictions()
 						theOneTrueSurface = e->mySurface;
 					else if (theOneTrueSurface != e->mySurface)
 						symmetric = true;
-					maxPortByIp[reinterpret_cast<const struct sockaddr_in *>(&(e->mySurface))->sin_addr.s_addr] = std::pair<uint64_t,unsigned int>(e->ts,e->mySurface.port());
+					maxPortByIp[reinterpret_cast<const struct sockaddr_in *>(&(e->mySurface))->sin_addr.s_addr] = e->mySurface.port();
 				}
 			}
 		}
+		if (!symmetric)
+			return std::vector<InetAddress>();
 
-		{	// Then find max port per IP from a trusted peer
+		{	// Then find the highest issued port per IP
 			Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy);
 			PhySurfaceKey *k = (PhySurfaceKey *)0;
 			PhySurfaceEntry *e = (PhySurfaceEntry *)0;
 			while (i.next(k,e)) {
 				if ((e->mySurface.ss_family == AF_INET)&&(e->mySurface.ipScope() == InetAddress::IP_SCOPE_GLOBAL)) {
-					std::map< uint32_t,std::pair<uint64_t,unsigned int> >::iterator mp(maxPortByIp.find(reinterpret_cast<const struct sockaddr_in *>(&(e->mySurface))->sin_addr.s_addr));
-					if ((mp != maxPortByIp.end())&&(mp->second.first < e->ts)) {
-						mp->second.first = e->ts;
-						mp->second.second = e->mySurface.port();
-					}
+					const unsigned int port = e->mySurface.port();
+					std::map< uint32_t,unsigned int >::iterator mp(maxPortByIp.find(reinterpret_cast<const struct sockaddr_in *>(&(e->mySurface))->sin_addr.s_addr));
+					if ((mp != maxPortByIp.end())&&(mp->second < port))
+						mp->second = port;
 				}
 			}
 		}
 	}
 
-	if (symmetric) {
-		std::vector<InetAddress> r;
-		for(unsigned int k=1;k<=3;++k) {
-			for(std::map< uint32_t,std::pair<uint64_t,unsigned int> >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) {
-				unsigned int p = i->second.second + k;
-				if (p > 65535) p -= 64511;
-				InetAddress pred(&(i->first),4,p);
-				if (std::find(r.begin(),r.end(),pred) == r.end())
-					r.push_back(pred);
-			}
-		}
-		return r;
+	std::vector<InetAddress> r;
+
+	// Try next port up from max for each
+	for(std::map< uint32_t,unsigned int >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) {
+		unsigned int p = i->second + 1;
+		if (p > 65535) p -= 64511;
+		const InetAddress pred(&(i->first),4,p);
+		if (std::find(r.begin(),r.end(),pred) == r.end())
+			r.push_back(pred);
+	}
+
+	// Try a random port for each -- there are only 65535 so eventually it should work
+	for(std::map< uint32_t,unsigned int >::iterator i(maxPortByIp.begin());i!=maxPortByIp.end();++i) {
+		const InetAddress pred(&(i->first),4,1024 + ((unsigned int)RR->node->prng() % 64511));
+		if (std::find(r.begin(),r.end(),pred) == r.end())
+			r.push_back(pred);
 	}
 
-	return std::vector<InetAddress>();
+	return r;
 }
 
 } // namespace ZeroTier

node/Switch.cpp

@@ -50,7 +50,6 @@ namespace ZeroTier {
 Switch::Switch(const RuntimeEnvironment *renv) :
 	RR(renv),
 	_lastBeaconResponse(0),
-	_outstandingWhoisRequests(32),
 	_lastUniteAttempt(8) // only really used on root servers and upstreams, and it'll grow there just fine
 {
 }
@@ -229,8 +228,8 @@ void Switch::onRemotePacket(void *tPtr,const int64_t localSocket,const InetAddre
 								}
 							}
 						} else {
-							relayTo = RR->topology->getUpstreamPeer(&source,1,true);
-							if (relayTo)
+							relayTo = RR->topology->getUpstreamPeer();
+							if ((relayTo)&&(relayTo->address() != source))
 								relayTo->sendDirect(tPtr,packet.data(),packet.size(),now,true);
 						}
 					}
@@ -545,41 +544,48 @@ void Switch::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const
 
 void Switch::send(void *tPtr,Packet &packet,bool encrypt)
 {
-	if (packet.destination() == RR->identity.address())
+	const Address dest(packet.destination());
+	if (dest == RR->identity.address())
 		return;
 	if (!_trySend(tPtr,packet,encrypt)) {
-		Mutex::Lock _l(_txQueue_m);
-		_txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt));
+		{
+			Mutex::Lock _l(_txQueue_m);
+			_txQueue.push_back(TXQueueEntry(dest,RR->node->now(),packet,encrypt));
+		}
+		if (!RR->topology->getPeer(tPtr,dest))
+			requestWhois(tPtr,RR->node->now(),dest);
 	}
 }
 
-void Switch::requestWhois(void *tPtr,const Address &addr)
+void Switch::requestWhois(void *tPtr,const uint64_t now,const Address &addr)
 {
 	if (addr == RR->identity.address())
 		return;
-	bool inserted = false;
+
 	{
-		Mutex::Lock _l(_outstandingWhoisRequests_m);
-		WhoisRequest &r = _outstandingWhoisRequests[addr];
-		if (r.lastSent) {
-			r.retries = 0; // reset retry count if entry already existed, but keep waiting and retry again after normal timeout
-		} else {
-			r.lastSent = RR->node->now();
-			inserted = true;
-		}
+		Mutex::Lock _l(_lastSentWhoisRequest_m);
+		uint64_t &last = _lastSentWhoisRequest[addr];
+		if ((now - last) < ZT_WHOIS_RETRY_DELAY)
+			return;
+		else last = now;
+	}
+
+	const SharedPtr<Peer> upstream(RR->topology->getUpstreamPeer());
+	if (upstream) {
+		Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS);
+		addr.appendTo(outp);
+		RR->node->expectReplyTo(outp.packetId());
+		send(tPtr,outp,true);
 	}
-	if (inserted)
-		_sendWhoisRequest(tPtr,addr,(const Address *)0,0);
 }
 
 void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr<Peer> &peer)
 {
-	{	// cancel pending WHOIS since we now know this peer
-		Mutex::Lock _l(_outstandingWhoisRequests_m);
-		_outstandingWhoisRequests.erase(peer->address());
+	{
+		Mutex::Lock _l(_lastSentWhoisRequest_m);
+		_lastSentWhoisRequest.erase(peer->address());
 	}
 
-	// finish processing any packets waiting on peer's public key / identity
 	const uint64_t now = RR->node->now();
 	for(unsigned int ptr=0;ptr<ZT_RX_QUEUE_SIZE;++ptr) {
 		RXQueueEntry *const rq = &(_rxQueue[ptr]);
@@ -589,57 +595,62 @@ void Switch::doAnythingWaitingForPeer(void *tPtr,const SharedPtr<Peer> &peer)
 		}
 	}
 
-	{	// finish sending any packets waiting on peer's public key / identity
+	{
 		Mutex::Lock _l(_txQueue_m);
 		for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) {
 			if (txi->dest == peer->address()) {
-				if (_trySend(tPtr,txi->packet,txi->encrypt))
+				if (_trySend(tPtr,txi->packet,txi->encrypt)) {
 					_txQueue.erase(txi++);
-				else ++txi;
-			} else ++txi;
-		}
-	}
-}
-
-unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now)
-{
-	unsigned long nextDelay = 0xffffffff; // ceiling delay, caller will cap to minimum
-
-	{	// Retry outstanding WHOIS requests
-		Mutex::Lock _l(_outstandingWhoisRequests_m);
-		Hashtable< Address,WhoisRequest >::Iterator i(_outstandingWhoisRequests);
-		Address *a = (Address *)0;
-		WhoisRequest *r = (WhoisRequest *)0;
-		while (i.next(a,r)) {
-			const unsigned long since = (unsigned long)(now - r->lastSent);
-			if (since >= ZT_WHOIS_RETRY_DELAY) {
-				if (r->retries >= ZT_MAX_WHOIS_RETRIES) {
-					_outstandingWhoisRequests.erase(*a);
 				} else {
-					r->lastSent = now;
-					r->peersConsulted[r->retries] = _sendWhoisRequest(tPtr,*a,r->peersConsulted,(r->retries > 1) ? r->retries : 0);
-					++r->retries;
-					nextDelay = std::min(nextDelay,(unsigned long)ZT_WHOIS_RETRY_DELAY);
+					++txi;
 				}
 			} else {
-				nextDelay = std::min(nextDelay,ZT_WHOIS_RETRY_DELAY - since);
+				++txi;
 			}
 		}
 	}
+}
+
+unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now)
+{
+	const uint64_t timeSinceLastCheck = now - _lastCheckedQueues;
+	if (timeSinceLastCheck < ZT_WHOIS_RETRY_DELAY)
+		return (unsigned long)(ZT_WHOIS_RETRY_DELAY - timeSinceLastCheck);
+	_lastCheckedQueues = now;
 
-	{	// Time out TX queue packets that never got WHOIS lookups or other info.
+	std::vector<Address> needWhois;
+	{
 		Mutex::Lock _l(_txQueue_m);
 		for(std::list< TXQueueEntry >::iterator txi(_txQueue.begin());txi!=_txQueue.end();) {
-			if (_trySend(tPtr,txi->packet,txi->encrypt))
+			if (_trySend(tPtr,txi->packet,txi->encrypt)) {
 				_txQueue.erase(txi++);
-			else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) {
+			} else if ((now - txi->creationTime) > ZT_TRANSMIT_QUEUE_TIMEOUT) {
 				RR->t->txTimedOut(tPtr,txi->dest);
 				_txQueue.erase(txi++);
-			} else ++txi;
+			} else {
+				if (!RR->topology->getPeer(tPtr,txi->dest))
+					needWhois.push_back(txi->dest);
+				++txi;
+			}
+		}
+	}
+	for(std::vector<Address>::const_iterator i(needWhois.begin());i!=needWhois.end();++i)
+		requestWhois(tPtr,now,*i);
+
+	for(unsigned int ptr=0;ptr<ZT_RX_QUEUE_SIZE;++ptr) {
+		RXQueueEntry *const rq = &(_rxQueue[ptr]);
+		if ((rq->timestamp)&&(rq->complete)) {
+			if ((rq->frag0.tryDecode(RR,tPtr))||((now - rq->timestamp) > ZT_RECEIVE_QUEUE_TIMEOUT)) {
+				rq->timestamp = 0;
+			} else {
+				const Address src(rq->frag0.source());
+				if (!RR->topology->getPeer(tPtr,src))
+					requestWhois(tPtr,now,src);
+			}
 		}
 	}
 
-	{	// Remove really old last unite attempt entries to keep table size controlled
+	{
 		Mutex::Lock _l(_lastUniteAttempt_m);
 		Hashtable< _LastUniteKey,uint64_t >::Iterator i(_lastUniteAttempt);
 		_LastUniteKey *k = (_LastUniteKey *)0;
@@ -650,7 +661,18 @@ unsigned long Switch::doTimerTasks(void *tPtr,uint64_t now)
 		}
 	}
 
-	return nextDelay;
+	{
+		Mutex::Lock _l(_lastSentWhoisRequest_m);
+		Hashtable< Address,uint64_t >::Iterator i(_lastSentWhoisRequest);
+		Address *a = (Address *)0;
+		uint64_t *ts = (uint64_t *)0;
+		while (i.next(a,ts)) {
+			if ((now - *ts) > (ZT_WHOIS_RETRY_DELAY * 2))
+				_lastSentWhoisRequest.erase(*a);
+		}
+	}
+
+	return ZT_WHOIS_RETRY_DELAY;
 }
 
 bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address &destination)
@@ -664,18 +686,6 @@ bool Switch::_shouldUnite(const uint64_t now,const Address &source,const Address
 	return false;
 }
 
-Address Switch::_sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted)
-{
-	SharedPtr<Peer> upstream(RR->topology->getUpstreamPeer(peersAlreadyConsulted,numPeersAlreadyConsulted,false));
-	if (upstream) {
-		Packet outp(upstream->address(),RR->identity.address(),Packet::VERB_WHOIS);
-		addr.appendTo(outp);
-		RR->node->expectReplyTo(outp.packetId());
-		send(tPtr,outp,true);
-	}
-	return Address();
-}
-
 bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt)
 {
 	SharedPtr<Path> viaPath;
@@ -709,14 +719,16 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt)
 			}
 		}
 	} else {
-		requestWhois(tPtr,destination);
-		return false; // if we are not in cluster mode, there is no way we can send without knowing the peer directly
+		return false;
 	}
 
-	unsigned int chunkSize = std::min(packet.size(),(unsigned int)ZT_UDP_DEFAULT_PAYLOAD_MTU);
+	unsigned int mtu = ZT_DEFAULT_PHYSMTU;
+	uint64_t trustedPathId = 0;
+	RR->topology->getOutboundPathInfo(viaPath->address(),mtu,trustedPathId);
+
+	unsigned int chunkSize = std::min(packet.size(),mtu);
 	packet.setFragmented(chunkSize < packet.size());
 
-	const uint64_t trustedPathId = RR->topology->getOutboundPathTrust(viaPath->address());
 	if (trustedPathId) {
 		packet.setTrusted(trustedPathId);
 	} else {
@@ -728,13 +740,13 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt)
 			// Too big for one packet, fragment the rest
 			unsigned int fragStart = chunkSize;
 			unsigned int remaining = packet.size() - chunkSize;
-			unsigned int fragsRemaining = (remaining / (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH));
-			if ((fragsRemaining * (ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining)
+			unsigned int fragsRemaining = (remaining / (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH));
+			if ((fragsRemaining * (mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH)) < remaining)
 				++fragsRemaining;
 			const unsigned int totalFragments = fragsRemaining + 1;
 
 			for(unsigned int fno=1;fno<totalFragments;++fno) {
-				chunkSize = std::min(remaining,(unsigned int)(ZT_UDP_DEFAULT_PAYLOAD_MTU - ZT_PROTO_MIN_FRAGMENT_LENGTH));
+				chunkSize = std::min(remaining,(unsigned int)(mtu - ZT_PROTO_MIN_FRAGMENT_LENGTH));
 				Packet::Fragment frag(packet,fragStart,chunkSize,fno,totalFragments);
 				viaPath->send(RR,tPtr,frag.data(),frag.size(),now);
 				fragStart += chunkSize;

node/Switch.hpp

@@ -111,9 +111,10 @@ public:
 	 * Request WHOIS on a given address
 	 *
 	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
+	 * @param now Current time
 	 * @param addr Address to look up
 	 */
-	void requestWhois(void *tPtr,const Address &addr);
+	void requestWhois(void *tPtr,const uint64_t now,const Address &addr);
 
 	/**
 	 * Run any processes that are waiting for this peer's identity
@@ -139,34 +140,27 @@ public:
 
 private:
 	bool _shouldUnite(const uint64_t now,const Address &source,const Address &destination);
-	Address _sendWhoisRequest(void *tPtr,const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(void *tPtr,Packet &packet,bool encrypt); // packet is modified if return is true
 
 	const RuntimeEnvironment *const RR;
 	uint64_t _lastBeaconResponse;
+	volatile uint64_t _lastCheckedQueues;
 
-	// Outstanding WHOIS requests and how many retries they've undergone
-	struct WhoisRequest
-	{
-		WhoisRequest() : lastSent(0),retries(0) {}
-		uint64_t lastSent;
-		Address peersConsulted[ZT_MAX_WHOIS_RETRIES]; // by retry
-		unsigned int retries; // 0..ZT_MAX_WHOIS_RETRIES
-	};
-	Hashtable< Address,WhoisRequest > _outstandingWhoisRequests;
-	Mutex _outstandingWhoisRequests_m;
+	// Time we last sent a WHOIS request for each address
+	Hashtable< Address,uint64_t > _lastSentWhoisRequest;
+	Mutex _lastSentWhoisRequest_m;
 
 	// Packets waiting for WHOIS replies or other decode info or missing fragments
 	struct RXQueueEntry
 	{
 		RXQueueEntry() : timestamp(0) {}
-		uint64_t timestamp; // 0 if entry is not in use
-		uint64_t packetId;
+		volatile uint64_t timestamp; // 0 if entry is not in use
+		volatile uint64_t packetId;
 		IncomingPacket frag0; // head of packet
 		Packet::Fragment frags[ZT_MAX_PACKET_FRAGMENTS - 1]; // later fragments (if any)
 		unsigned int totalFragments; // 0 if only frag0 received, waiting for frags
 		uint32_t haveFragments; // bit mask, LSB to MSB
-		bool complete; // if true, packet is complete
+		volatile bool complete; // if true, packet is complete
 	};
 	RXQueueEntry _rxQueue[ZT_RX_QUEUE_SIZE];
 	AtomicCounter _rxQueuePtr;
@@ -174,19 +168,20 @@ private:
 	// Returns matching or next available RX queue entry
 	inline RXQueueEntry *_findRXQueueEntry(uint64_t packetId)
 	{
-		unsigned int ptr = static_cast<unsigned int>(_rxQueuePtr.load());
-		for(unsigned int k=0;k<ZT_RX_QUEUE_SIZE;++k) {
-			RXQueueEntry *rq = &(_rxQueue[--ptr % ZT_RX_QUEUE_SIZE]);
+		const unsigned int current = static_cast<unsigned int>(_rxQueuePtr.load());
+		for(unsigned int k=1;k<=ZT_RX_QUEUE_SIZE;++k) {
+			RXQueueEntry *rq = &(_rxQueue[(current - k) % ZT_RX_QUEUE_SIZE]);
 			if ((rq->packetId == packetId)&&(rq->timestamp))
 				return rq;
 		}
-		return &(_rxQueue[static_cast<unsigned int>(++_rxQueuePtr) % ZT_RX_QUEUE_SIZE]);
+		++_rxQueuePtr;
+		return &(_rxQueue[static_cast<unsigned int>(current) % ZT_RX_QUEUE_SIZE]);
 	}
 
-	// Returns next RX queue entry in ring buffer and increments ring counter
+	// Returns current entry in rx queue ring buffer and increments ring pointer
 	inline RXQueueEntry *_nextRXQueueEntry()
 	{
-		return &(_rxQueue[static_cast<unsigned int>(++_rxQueuePtr) % ZT_RX_QUEUE_SIZE]);
+		return &(_rxQueue[static_cast<unsigned int>((++_rxQueuePtr) - 1) % ZT_RX_QUEUE_SIZE]);
 	}
 
 	// ZeroTier-layer TX queue entry

node/Tag.cpp

@@ -30,6 +30,7 @@
 #include "Topology.hpp"
 #include "Switch.hpp"
 #include "Network.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -39,7 +40,7 @@ int Tag::verify(const RuntimeEnvironment *RR,void *tPtr) const
 		return -1;
 	const Identity id(RR->topology->getIdentity(tPtr,_signedBy));
 	if (!id) {
-		RR->sw->requestWhois(tPtr,_signedBy);
+		RR->sw->requestWhois(tPtr,RR->node->now(),_signedBy);
 		return 1;
 	}
 	try {

node/Topology.cpp

@@ -65,7 +65,7 @@ static const unsigned char ZT_DEFAULT_WORLD[ZT_DEFAULT_WORLD_LENGTH] = {0x01,0x0
 
 Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) :
 	RR(renv),
-	_trustedPathCount(0),
+	_numConfiguredPhysicalPaths(0),
 	_amRoot(false)
 {
 	uint8_t tmp[ZT_WORLD_MAX_SERIALIZED_LENGTH];
@@ -88,6 +88,15 @@ Topology::Topology(const RuntimeEnvironment *renv,void *tPtr) :
 	addWorld(tPtr,defaultPlanet,false);
 }
 
+Topology::~Topology()
+{
+	Hashtable< Address,SharedPtr<Peer> >::Iterator i(_peers);
+	Address *a = (Address *)0;
+	SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
+	while (i.next(a,p))
+		_savePeer((void *)0,*p);
+}
+
 SharedPtr<Peer> Topology::addPeer(void *tPtr,const SharedPtr<Peer> &peer)
 {
 	SharedPtr<Peer> np;
@@ -113,23 +122,21 @@ SharedPtr<Peer> Topology::getPeer(void *tPtr,const Address &zta)
 			return *ap;
 	}
 
-	/*
 	try {
-		char buf[ZT_PEER_MAX_SERIALIZED_STATE_SIZE];
+		Buffer<ZT_PEER_MAX_SERIALIZED_STATE_SIZE> buf;
 		uint64_t idbuf[2]; idbuf[0] = zta.toInt(); idbuf[1] = 0;
-		int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER,idbuf,buf,(unsigned int)sizeof(buf));
+		int len = RR->node->stateObjectGet(tPtr,ZT_STATE_OBJECT_PEER,idbuf,buf.unsafeData(),ZT_PEER_MAX_SERIALIZED_STATE_SIZE);
 		if (len > 0) {
 			Mutex::Lock _l(_peers_m);
 			SharedPtr<Peer> &ap = _peers[zta];
 			if (ap)
 				return ap;
-			ap = Peer::createFromStateUpdate(RR,tPtr,buf,len);
+			ap = Peer::deserializeFromCache(RR->node->now(),tPtr,buf,RR);
 			if (!ap)
 				_peers.erase(zta);
 			return ap;
 		}
 	} catch ( ... ) {} // ignore invalid identities or other strage failures
-	*/
 
 	return SharedPtr<Peer>();
 }
@@ -147,13 +154,11 @@ Identity Topology::getIdentity(void *tPtr,const Address &zta)
 	return Identity();
 }
 
-SharedPtr<Peer> Topology::getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid)
+SharedPtr<Peer> Topology::getUpstreamPeer()
 {
 	const uint64_t now = RR->node->now();
-	unsigned int bestQualityOverall = ~((unsigned int)0);
-	unsigned int bestQualityNotAvoid = ~((unsigned int)0);
-	const SharedPtr<Peer> *bestOverall = (const SharedPtr<Peer> *)0;
-	const SharedPtr<Peer> *bestNotAvoid = (const SharedPtr<Peer> *)0;
+	unsigned int bestq = ~((unsigned int)0);
+	const SharedPtr<Peer> *best = (const SharedPtr<Peer> *)0;
 
 	Mutex::Lock _l1(_peers_m);
 	Mutex::Lock _l2(_upstreams_m);
@@ -161,32 +166,17 @@ SharedPtr<Peer> Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi
 	for(std::vector<Address>::const_iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) {
 		const SharedPtr<Peer> *p = _peers.get(*a);
 		if (p) {
-			bool avoiding = false;
-			for(unsigned int i=0;i<avoidCount;++i) {
-				if (avoid[i] == (*p)->address()) {
-					avoiding = true;
-					break;
-				}
-			}
 			const unsigned int q = (*p)->relayQuality(now);
-			if (q <= bestQualityOverall) {
-				bestQualityOverall = q;
-				bestOverall = &(*p);
-			}
-			if ((!avoiding)&&(q <= bestQualityNotAvoid)) {
-				bestQualityNotAvoid = q;
-				bestNotAvoid = &(*p);
+			if (q <= bestq) {
+				bestq = q;
+				best = p;
 			}
 		}
 	}
 
-	if (bestNotAvoid) {
-		return *bestNotAvoid;
-	} else if ((!strictAvoid)&&(bestOverall)) {
-		return *bestOverall;
-	}
-
-	return SharedPtr<Peer>();
+	if (!best)
+		return SharedPtr<Peer>();
+	return *best;
 }
 
 bool Topology::isUpstream(const Identity &id) const
@@ -383,8 +373,10 @@ void Topology::doPeriodicTasks(void *tPtr,uint64_t now)
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
 		while (i.next(a,p)) {
-			if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) )
+			if ( (!(*p)->isAlive(now)) && (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),*a) == _upstreamAddresses.end()) ) {
+				_savePeer(tPtr,*p);
 				_peers.erase(*a);
+			}
 		}
 	}
 
@@ -440,4 +432,14 @@ void Topology::_memoizeUpstreams(void *tPtr)
 	_cor.sign(RR->identity,RR->node->now());
 }
 
+void Topology::_savePeer(void *tPtr,const SharedPtr<Peer> &peer)
+{
+	try {
+		Buffer<ZT_PEER_MAX_SERIALIZED_STATE_SIZE> buf;
+		peer->serialize(buf);
+		uint64_t tmpid[2]; tmpid[0] = peer->address().toInt(); tmpid[1] = 0;
+		RR->node->stateObjectPut(tPtr,ZT_STATE_OBJECT_PEER,tmpid,buf.data(),buf.size());
+	} catch ( ... ) {} // sanity check, discard invalid entries
+}
+
 } // namespace ZeroTier

node/Topology.hpp

@@ -59,6 +59,7 @@ class Topology
 {
 public:
 	Topology(const RuntimeEnvironment *renv,void *tPtr);
+	~Topology();
 
 	/**
 	 * Add a peer to database
@@ -126,19 +127,9 @@ public:
 	/**
 	 * Get the current best upstream peer
 	 *
-	 * @return Root server with lowest latency or NULL if none
+	 * @return Upstream or NULL if none available
 	 */
-	inline SharedPtr<Peer> getUpstreamPeer() { return getUpstreamPeer((const Address *)0,0,false); }
-
-	/**
-	 * Get the current best upstream peer, avoiding those in the supplied avoid list
-	 *
-	 * @param avoid Nodes to avoid
-	 * @param avoidCount Number of nodes to avoid
-	 * @param strictAvoid If false, consider avoided root servers anyway if no non-avoid root servers are available
-	 * @return Root server or NULL if none available
-	 */
-	SharedPtr<Peer> getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid);
+	SharedPtr<Peer> getUpstreamPeer();
 
 	/**
 	 * @param id Identity to check
@@ -348,6 +339,41 @@ public:
 	 */
 	inline bool amRoot() const { return _amRoot; }
 
+	/**
+	 * Get info about a path
+	 *
+	 * The supplied result variables are not modified if no special config info is found.
+	 *
+	 * @param physicalAddress Physical endpoint address
+	 * @param mtu Variable set to MTU
+	 * @param trustedPathId Variable set to trusted path ID
+	 */
+	inline void getOutboundPathInfo(const InetAddress &physicalAddress,unsigned int &mtu,uint64_t &trustedPathId)
+	{
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
+			if (_physicalPathConfig[i].first.containsAddress(physicalAddress)) {
+				trustedPathId = _physicalPathConfig[i].second.trustedPathId;
+				mtu = _physicalPathConfig[i].second.mtu;
+				return;
+			}
+		}
+	}
+
+	/**
+	 * Get the payload MTU for an outbound physical path (returns default if not configured)
+	 *
+	 * @param physicalAddress Physical endpoint address
+	 * @return MTU
+	 */
+	inline unsigned int getOutboundPathMtu(const InetAddress &physicalAddress)
+	{
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
+			if (_physicalPathConfig[i].first.containsAddress(physicalAddress))
+				return _physicalPathConfig[i].second.mtu;
+		}
+		return ZT_DEFAULT_PHYSMTU;
+	}
+
 	/**
 	 * Get the outbound trusted path ID for a physical address, or 0 if none
 	 *
@@ -356,9 +382,9 @@ public:
 	 */
 	inline uint64_t getOutboundPathTrust(const InetAddress &physicalAddress)
 	{
-		for(unsigned int i=0;i<_trustedPathCount;++i) {
-			if (_trustedPathNetworks[i].containsAddress(physicalAddress))
-				return _trustedPathIds[i];
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
+			if (_physicalPathConfig[i].first.containsAddress(physicalAddress))
+				return _physicalPathConfig[i].second.trustedPathId;
 		}
 		return 0;
 	}
@@ -371,30 +397,48 @@ public:
 	 */
 	inline bool shouldInboundPathBeTrusted(const InetAddress &physicalAddress,const uint64_t trustedPathId)
 	{
-		for(unsigned int i=0;i<_trustedPathCount;++i) {
-			if ((_trustedPathIds[i] == trustedPathId)&&(_trustedPathNetworks[i].containsAddress(physicalAddress)))
+		for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i) {
+			if ((_physicalPathConfig[i].second.trustedPathId == trustedPathId)&&(_physicalPathConfig[i].first.containsAddress(physicalAddress)))
 				return true;
 		}
 		return false;
 	}
 
 	/**
-	 * Set trusted paths in this topology
-	 *
-	 * @param networks Array of networks (prefix/netmask bits)
-	 * @param ids Array of trusted path IDs
-	 * @param count Number of trusted paths (if larger than ZT_MAX_TRUSTED_PATHS overflow is ignored)
+	 * Set or clear physical path configuration (called via Node::setPhysicalPathConfiguration)
 	 */
-	inline void setTrustedPaths(const InetAddress *networks,const uint64_t *ids,unsigned int count)
+	inline void setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig)
 	{
-		if (count > ZT_MAX_TRUSTED_PATHS)
-			count = ZT_MAX_TRUSTED_PATHS;
-		Mutex::Lock _l(_trustedPaths_m);
-		for(unsigned int i=0;i<count;++i) {
-			_trustedPathIds[i] = ids[i];
-			_trustedPathNetworks[i] = networks[i];
+		if (!pathNetwork) {
+			_numConfiguredPhysicalPaths = 0;
+		} else {
+			std::map<InetAddress,ZT_PhysicalPathConfiguration> cpaths;
+			for(unsigned int i=0,j=_numConfiguredPhysicalPaths;i<j;++i)
+				cpaths[_physicalPathConfig[i].first] = _physicalPathConfig[i].second;
+
+			if (pathConfig) {
+				ZT_PhysicalPathConfiguration pc(*pathConfig);
+
+				if (pc.mtu <= 0)
+					pc.mtu = ZT_DEFAULT_PHYSMTU;
+				else if (pc.mtu < ZT_MIN_PHYSMTU)
+					pc.mtu = ZT_MIN_PHYSMTU;
+				else if (pc.mtu > ZT_MAX_PHYSMTU)
+					pc.mtu = ZT_MAX_PHYSMTU;
+
+				cpaths[*(reinterpret_cast<const InetAddress *>(pathNetwork))] = pc;
+			} else {
+				cpaths.erase(*(reinterpret_cast<const InetAddress *>(pathNetwork)));
+			}
+
+			unsigned int cnt = 0;
+			for(std::map<InetAddress,ZT_PhysicalPathConfiguration>::const_iterator i(cpaths.begin());((i!=cpaths.end())&&(cnt<ZT_MAX_CONFIGURABLE_PATHS));++i) {
+				_physicalPathConfig[cnt].first = i->first;
+				_physicalPathConfig[cnt].second = i->second;
+				++cnt;
+			}
+			_numConfiguredPhysicalPaths = cnt;
 		}
-		_trustedPathCount = count;
 	}
 
 	/**
@@ -419,13 +463,12 @@ public:
 private:
 	Identity _getIdentity(void *tPtr,const Address &zta);
 	void _memoizeUpstreams(void *tPtr);
+	void _savePeer(void *tPtr,const SharedPtr<Peer> &peer);
 
 	const RuntimeEnvironment *const RR;
 
-	uint64_t _trustedPathIds[ZT_MAX_TRUSTED_PATHS];
-	InetAddress _trustedPathNetworks[ZT_MAX_TRUSTED_PATHS];
-	unsigned int _trustedPathCount;
-	Mutex _trustedPaths_m;
+	std::pair<InetAddress,ZT_PhysicalPathConfiguration> _physicalPathConfig[ZT_MAX_CONFIGURABLE_PATHS];
+	volatile unsigned int _numConfiguredPhysicalPaths;
 
 	Hashtable< Address,SharedPtr<Peer> > _peers;
 	Mutex _peers_m;

node/Trace.cpp

@@ -164,12 +164,7 @@ void Trace::incomingNetworkFrameDropped(void *const tPtr,const SharedPtr<Network
 	_send(tPtr,d,*network);
 }
 
-void Trace::incomingPacketTrustedPath(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved)
-{
-	// TODO
-}
-
-void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops)
+void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops,const char *reason)
 {
 	char tmp[128];
 	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
@@ -179,6 +174,8 @@ void Trace::incomingPacketMessageAuthenticationFailure(void *const tPtr,const Sh
 	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_ZTADDR,source);
 	d.add(ZT_REMOTE_TRACE_FIELD__REMOTE_PHYADDR,path->address().toString(tmp));
 	d.add(ZT_REMOTE_TRACE_FIELD__LOCAL_SOCKET,path->localSocket());
+	if (reason)
+		d.add(ZT_REMOTE_TRACE_FIELD__REASON,reason);
 	_send(tPtr,d,0);
 }
 
@@ -344,76 +341,6 @@ void Trace::credentialRejected(void *const tPtr,const Revocation &c,const char *
 	_send(tPtr,d,c.networkId());
 }
 
-void Trace::credentialAccepted(void *const tPtr,const CertificateOfMembership &c)
-{
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
-	_send(tPtr,d,c.networkId());
-}
-
-void Trace::credentialAccepted(void *const tPtr,const CertificateOfOwnership &c)
-{
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
-	_send(tPtr,d,c.networkId());
-}
-
-void Trace::credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c)
-{
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
-	_send(tPtr,d,0);
-}
-
-void Trace::credentialAccepted(void *const tPtr,const Capability &c)
-{
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
-	_send(tPtr,d,c.networkId());
-}
-
-void Trace::credentialAccepted(void *const tPtr,const Tag &c)
-{
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TIMESTAMP,c.timestamp());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ISSUED_TO,c.issuedTo());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_INFO,(uint64_t)c.value());
-	_send(tPtr,d,c.networkId());
-}
-
-void Trace::credentialAccepted(void *const tPtr,const Revocation &c)
-{
-	Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> d;
-	d.add(ZT_REMOTE_TRACE_FIELD__EVENT,ZT_REMOTE_TRACE_EVENT__CREDENTIAL_ACCEPTED_S);
-	d.add(ZT_REMOTE_TRACE_FIELD__NETWORK_ID,c.networkId());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_TYPE,(uint64_t)c.credentialType());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_ID,(uint64_t)c.id());
-	d.add(ZT_REMOTE_TRACE_FIELD__CREDENTIAL_REVOCATION_TARGET,c.target());
-	_send(tPtr,d,c.networkId());
-}
-
 void Trace::_send(void *const tPtr,const Dictionary<ZT_MAX_REMOTE_TRACE_SIZE> &d)
 {
 #ifdef ZT_TRACE

node/Trace.hpp

@@ -108,8 +108,7 @@ public:
 	void peerLearnedNewPath(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &oldPath,const SharedPtr<Path> &newPath,const uint64_t packetId);
 	void peerRedirected(void *const tPtr,const uint64_t networkId,Peer &peer,const SharedPtr<Path> &oldPath,const SharedPtr<Path> &newPath);
 
-	void incomingPacketTrustedPath(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const uint64_t trustedPathId,bool approved);
-	void incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops);
+	void incomingPacketMessageAuthenticationFailure(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops,const char *reason);
 	void incomingPacketInvalid(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const unsigned int hops,const Packet::Verb verb,const char *reason);
 	void incomingPacketDroppedHELLO(void *const tPtr,const SharedPtr<Path> &path,const uint64_t packetId,const Address &source,const char *reason);
 
@@ -142,12 +141,6 @@ public:
 	void credentialRejected(void *const tPtr,const Capability &c,const char *reason);
 	void credentialRejected(void *const tPtr,const Tag &c,const char *reason);
 	void credentialRejected(void *const tPtr,const Revocation &c,const char *reason);
-	void credentialAccepted(void *const tPtr,const CertificateOfMembership &c);
-	void credentialAccepted(void *const tPtr,const CertificateOfOwnership &c);
-	void credentialAccepted(void *const tPtr,const CertificateOfRepresentation &c);
-	void credentialAccepted(void *const tPtr,const Capability &c);
-	void credentialAccepted(void *const tPtr,const Tag &c);
-	void credentialAccepted(void *const tPtr,const Revocation &c);
 
 private:
 	const RuntimeEnvironment *const RR;

osdep/Binder.hpp

@@ -227,7 +227,7 @@ public:
 							case InetAddress::IP_SCOPE_GLOBAL:
 							case InetAddress::IP_SCOPE_SHARED:
 							case InetAddress::IP_SCOPE_PRIVATE:
-								for(int x=0;x<portCount;++x) {
+								for(int x=0;x<(int)portCount;++x) {
 									ip.setPort(ports[x]);
 									localIfAddrs.insert(std::pair<InetAddress,std::string>(ip,std::string(devname)));
 								}
@@ -268,7 +268,7 @@ public:
 					case InetAddress::IP_SCOPE_GLOBAL:
 					case InetAddress::IP_SCOPE_SHARED:
 					case InetAddress::IP_SCOPE_PRIVATE:
-						for(int x=0;x<portCount;++x) {
+						for(int x=0;x<(int)portCount;++x) {
 							ip.setPort(ports[x]);
 							localIfAddrs.insert(std::pair<InetAddress,std::string>(ip,ifname));
 						}
@@ -302,7 +302,7 @@ public:
 								case InetAddress::IP_SCOPE_GLOBAL:
 								case InetAddress::IP_SCOPE_SHARED:
 								case InetAddress::IP_SCOPE_PRIVATE:
-									for(int x=0;x<portCount;++x) {
+									for(int x=0;x<(int)portCount;++x) {
 										ip.setPort(ports[x]);
 										localIfAddrs.insert(std::pair<InetAddress,std::string>(ip,std::string(ifa->ifa_name)));
 									}

osdep/LinuxEthernetTap.cpp

@@ -314,7 +314,7 @@ bool LinuxEthernetTap::addIpSyn(std::vector<InetAddress> ips)
 		OSUtils::writeFile(filepath.c_str(), cfg_contents.c_str(), cfg_contents.length());
 		// Finaly, add IPs
 		for(int i=0; i<(int)ips.size(); i++){
-			char iptmp[128],iptmp2[128[;
+			char iptmp[128],iptmp2[128];
 			if (ips[i].isV4())
 				::execlp("ip","ip","addr","add",ips[i].toString(iptmp),"broadcast",ips[i].broadcast().toIpString(iptmp2),"dev",_dev.c_str(),(const char *)0);
 			else

service/OneService.cpp

@@ -99,8 +99,8 @@ namespace ZeroTier { typedef TestEthernetTap EthernetTap; }
 #include "../controller/EmbeddedNetworkController.hpp"
 #include "../node/Node.hpp"
 // Use the virtual netcon endpoint instead of a tun/tap port driver
-#include "../src/SocketTap.hpp"
-namespace ZeroTier { typedef SocketTap EthernetTap; }
+#include "../src/VirtualTap.hpp"
+namespace ZeroTier { typedef VirtualTap EthernetTap; }
 
 #else
 
@@ -564,16 +564,14 @@ public:
 
 			// Read local configuration
 			{
-				uint64_t trustedPathIds[ZT_MAX_TRUSTED_PATHS];
-				InetAddress trustedPathNetworks[ZT_MAX_TRUSTED_PATHS];
-				unsigned int trustedPathCount = 0;
+				std::map<InetAddress,ZT_PhysicalPathConfiguration> ppc;
 
 				// LEGACY: support old "trustedpaths" flat file
 				FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r");
 				if (trustpaths) {
 					fprintf(stderr,"WARNING: 'trustedpaths' flat file format is deprecated in favor of path definitions in local.conf" ZT_EOL_S);
 					char buf[1024];
-					while ((fgets(buf,sizeof(buf),trustpaths))&&(trustedPathCount < ZT_MAX_TRUSTED_PATHS)) {
+					while (fgets(buf,sizeof(buf),trustpaths)) {
 						int fno = 0;
 						char *saveptr = (char *)0;
 						uint64_t trustedPathId = 0;
@@ -587,9 +585,8 @@ public:
 							++fno;
 						}
 						if ( (trustedPathId != 0) && ((trustedPathNetwork.ss_family == AF_INET)||(trustedPathNetwork.ss_family == AF_INET6)) && (trustedPathNetwork.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (trustedPathNetwork.netmaskBits() > 0) ) {
-							trustedPathIds[trustedPathCount] = trustedPathId;
-							trustedPathNetworks[trustedPathCount] = trustedPathNetwork;
-							++trustedPathCount;
+							ppc[trustedPathNetwork].trustedPathId = trustedPathId;
+							ppc[trustedPathNetwork].mtu = 0; // use default
 						}
 					}
 					fclose(trustpaths);
@@ -618,12 +615,10 @@ public:
 							if (phy.value().is_object()) {
 								uint64_t tpid;
 								if ((tpid = OSUtils::jsonInt(phy.value()["trustedPathId"],0ULL)) != 0ULL) {
-									if ( ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6)) && (trustedPathCount < ZT_MAX_TRUSTED_PATHS) && (net.ipScope() != InetAddress::IP_SCOPE_GLOBAL) && (net.netmaskBits() > 0) ) {
-										trustedPathIds[trustedPathCount] = tpid;
-										trustedPathNetworks[trustedPathCount] = net;
-										++trustedPathCount;
-									}
+									if ((net.ss_family == AF_INET)||(net.ss_family == AF_INET6))
+										ppc[net].trustedPathId = tpid;
 								}
+								ppc[net].mtu = (int)OSUtils::jsonInt(phy.value()["mtu"],0ULL); // 0 means use default
 							}
 						}
 					}
@@ -638,8 +633,10 @@ public:
 				}
 
 				// Set trusted paths if there are any
-				if (trustedPathCount)
-					_node->setTrustedPaths(reinterpret_cast<const struct sockaddr_storage *>(trustedPathNetworks),trustedPathIds,trustedPathCount);
+				if (ppc.size() > 0) {
+					for(std::map<InetAddress,ZT_PhysicalPathConfiguration>::iterator i(ppc.begin());i!=ppc.end();++i)
+						_node->setPhysicalPathConfiguration(reinterpret_cast<const struct sockaddr_storage *>(&(i->first)),&(i->second));
+				}
 			}
 
 			// Apply other runtime configuration from local.conf
@@ -925,29 +922,15 @@ public:
 		return _homePath;
 	}
 
-	virtual EthernetTap * getTap(uint64_t nwid)
-	{
-		Mutex::Lock _l(_nets_m);
-		std::map<uint64_t,NetworkState>::const_iterator n(_nets.find(nwid));
-		if (n == _nets.end())
-		return NULL;
-		return n->second.tap;
-	}
-
-	virtual EthernetTap *getTap(InetAddress &addr)
+	std::vector<ZT_VirtualNetworkRoute> *getRoutes(uint64_t nwid)
 	{
 		Mutex::Lock _l(_nets_m);
-		std::map<uint64_t,NetworkState>::iterator it;
-		for(it = _nets.begin(); it != _nets.end(); it++) {
-			if(it->second.tap) {
-				for(int j=0; j<it->second.tap->_ips.size(); j++) {
-					if(it->second.tap->_ips[j].isEqualPrefix(addr) || it->second.tap->_ips[j].ipsEqual(addr) || it->second.tap->_ips[j].containsAddress(addr)) {
-						return it->second.tap;
-					}
-				}
-			}
+		NetworkState &n = _nets[nwid];
+		std::vector<ZT_VirtualNetworkRoute> *routes = new std::vector<ZT_VirtualNetworkRoute>();
+		for(int i=0; i<ZT_MAX_NETWORK_ROUTES; i++) {
+			routes->push_back(n.config.routes[i]);
 		}
-		return NULL;
+		return routes;
 	}
 
 	virtual Node *getNode()
@@ -1742,9 +1725,10 @@ public:
 
 				case TcpConnection::TCP_UNCATEGORIZED_INCOMING:
 					switch(reinterpret_cast<uint8_t *>(data)[0]) {
-						// HTTP: GET, PUT, POST, HEAD
+						// HTTP: GET, PUT, POST, HEAD, DELETE
 						case 'G':
 						case 'P':
+						case 'D':
 						case 'H': {
 							// This is only allowed from IPs permitted to access the management
 							// backplane, which is just 127.0.0.1/::1 unless otherwise configured.
@@ -2061,6 +2045,8 @@ public:
 		char p[1024];
 		FILE *f;
 		bool secure = false;
+		char dirname[1024];
+		dirname[0] = 0;
 
 		switch(type) {
 			case ZT_STATE_OBJECT_IDENTITY_PUBLIC:
@@ -2074,12 +2060,18 @@ public:
 				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str());
 				break;
 			case ZT_STATE_OBJECT_MOON:
-				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id[0]);
+				OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "moons.d",_homePath.c_str());
+				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.moon",dirname,(unsigned long long)id[0]);
 				break;
 			case ZT_STATE_OBJECT_NETWORK_CONFIG:
-				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id[0]);
+				OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "networks.d",_homePath.c_str());
+				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.conf",dirname,(unsigned long long)id[0]);
 				secure = true;
 				break;
+			case ZT_STATE_OBJECT_PEER:
+				OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "peers.d",_homePath.c_str());
+				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.10llx.peer",dirname,(unsigned long long)id[0]);
+				break;
 			default:
 				return;
 		}
@@ -2098,6 +2090,10 @@ public:
 			}
 
 			f = fopen(p,"w");
+			if ((!f)&&(dirname[0])) { // create subdirectory if it does not exist
+				OSUtils::mkdir(dirname);
+				f = fopen(p,"w");
+			}
 			if (f) {
 				if (fwrite(data,len,1,f) != 1)
 					fprintf(stderr,"WARNING: unable to write to file: %s (I/O error)" ZT_EOL_S,p);
@@ -2122,14 +2118,17 @@ public:
 			case ZT_STATE_OBJECT_IDENTITY_SECRET:
 				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "identity.secret",_homePath.c_str());
 				break;
-			case ZT_STATE_OBJECT_NETWORK_CONFIG:
-				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d/%.16llx.conf",_homePath.c_str(),(unsigned long long)id);
-				break;
 			case ZT_STATE_OBJECT_PLANET:
 				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "planet",_homePath.c_str());
 				break;
 			case ZT_STATE_OBJECT_MOON:
-				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d/%.16llx.moon",_homePath.c_str(),(unsigned long long)id);
+				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "moons.d" ZT_PATH_SEPARATOR_S "%.16llx.moon",_homePath.c_str(),(unsigned long long)id);
+				break;
+			case ZT_STATE_OBJECT_NETWORK_CONFIG:
+				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "networks.d" ZT_PATH_SEPARATOR_S "%.16llx.conf",_homePath.c_str(),(unsigned long long)id);
+				break;
+			case ZT_STATE_OBJECT_PEER:
+				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "peers.d" ZT_PATH_SEPARATOR_S "%.10llx.peer",_homePath.c_str(),(unsigned long long)id[0]);
 				break;
 			default:
 				return -1;
@@ -2156,19 +2155,30 @@ public:
 				const uint64_t now = OSUtils::now();
 				if (((now - _lastDirectReceiveFromGlobal) > ZT_TCP_FALLBACK_AFTER)&&((now - _lastRestart) > ZT_TCP_FALLBACK_AFTER)) {
 					if (_tcpFallbackTunnel) {
-						Mutex::Lock _l(_tcpFallbackTunnel->writeq_m);
-						if (_tcpFallbackTunnel->writeq.length() == 0)
-							_phy.setNotifyWritable(_tcpFallbackTunnel->sock,true);
-						const unsigned long mlen = len + 7;
-						_tcpFallbackTunnel->writeq.push_back((char)0x17);
-						_tcpFallbackTunnel->writeq.push_back((char)0x03);
-						_tcpFallbackTunnel->writeq.push_back((char)0x03); // fake TLS 1.2 header
-						_tcpFallbackTunnel->writeq.push_back((char)((mlen >> 8) & 0xff));
-						_tcpFallbackTunnel->writeq.push_back((char)(mlen & 0xff));
-						_tcpFallbackTunnel->writeq.push_back((char)4); // IPv4
-						_tcpFallbackTunnel->writeq.append(reinterpret_cast<const char *>(reinterpret_cast<const void *>(&(reinterpret_cast<const struct sockaddr_in *>(addr)->sin_addr.s_addr))),4);
-						_tcpFallbackTunnel->writeq.append(reinterpret_cast<const char *>(reinterpret_cast<const void *>(&(reinterpret_cast<const struct sockaddr_in *>(addr)->sin_port))),2);
-						_tcpFallbackTunnel->writeq.append((const char *)data,len);
+						bool flushNow = false;
+						{
+							Mutex::Lock _l(_tcpFallbackTunnel->writeq_m);
+							if (_tcpFallbackTunnel->writeq.size() < (1024 * 64)) {
+								if (_tcpFallbackTunnel->writeq.length() == 0) {
+									_phy.setNotifyWritable(_tcpFallbackTunnel->sock,true);
+									flushNow = true;
+								}
+								const unsigned long mlen = len + 7;
+								_tcpFallbackTunnel->writeq.push_back((char)0x17);
+								_tcpFallbackTunnel->writeq.push_back((char)0x03);
+								_tcpFallbackTunnel->writeq.push_back((char)0x03); // fake TLS 1.2 header
+								_tcpFallbackTunnel->writeq.push_back((char)((mlen >> 8) & 0xff));
+								_tcpFallbackTunnel->writeq.push_back((char)(mlen & 0xff));
+								_tcpFallbackTunnel->writeq.push_back((char)4); // IPv4
+								_tcpFallbackTunnel->writeq.append(reinterpret_cast<const char *>(reinterpret_cast<const void *>(&(reinterpret_cast<const struct sockaddr_in *>(addr)->sin_addr.s_addr))),4);
+								_tcpFallbackTunnel->writeq.append(reinterpret_cast<const char *>(reinterpret_cast<const void *>(&(reinterpret_cast<const struct sockaddr_in *>(addr)->sin_port))),2);
+								_tcpFallbackTunnel->writeq.append((const char *)data,len);
+							}
+						}
+						if (flushNow) {
+							void *tmpptr = (void *)_tcpFallbackTunnel;
+							phyOnTcpWritable(_tcpFallbackTunnel->sock,&tmpptr);
+						}
 					} else if (((now - _lastSendToGlobalV4) < ZT_TCP_FALLBACK_AFTER)&&((now - _lastSendToGlobalV4) > (ZT_PING_CHECK_INVERVAL / 2))) {
 						const InetAddress addr(ZT_TCP_FALLBACK_RELAY);
 						TcpConnection *tc = new TcpConnection();
@@ -2301,7 +2311,7 @@ public:
 			fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: %s" ZT_EOL_S,exc.what());
 			scode = 500;
 		} catch ( ... ) {
-			fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: unknown exceptino" ZT_EOL_S);
+			fprintf(stderr,"WARNING: unexpected exception processing control HTTP request: unknown exception" ZT_EOL_S);
 			scode = 500;
 		}
 

service/OneService.hpp

@@ -35,8 +35,8 @@
 #ifdef ZT_SDK
 #include "../node/Node.hpp"
 // Use the virtual netcon endpoint instead of a tun/tap port driver
-#include "../src/SocketTap.hpp"
-namespace ZeroTier { typedef SocketTap EthernetTap; }
+#include "../src/VirtualTap.hpp"
+namespace ZeroTier { typedef VirtualTap EthernetTap; }
 #endif
 
 namespace ZeroTier {
@@ -150,10 +150,9 @@ public:
 	virtual void leave(const char *hp) = 0;
 	virtual void join(const char *hp) = 0;
 	virtual std::string givenHomePath() = 0;
-	virtual EthernetTap * getTap(uint64_t nwid) = 0;
-	virtual EthernetTap * getTap(InetAddress &addr) = 0;
 	virtual Node * getNode() = 0;
 	virtual void removeNets() = 0;
+	virtual std::vector<ZT_VirtualNetworkRoute> *getRoutes(uint64_t nwid) = 0;
 #endif
 
 	/**

service/README.md

@@ -14,7 +14,8 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al
 	"physical": { /* Settings that apply to physical L2/L3 network paths. */
 		"NETWORK/bits": { /* Network e.g. 10.0.0.0/24 or fd00::/32 */
 			"blacklist": true|false, /* If true, blacklist this path for all ZeroTier traffic */
-			"trustedPathId": 0|!0 /* If present and nonzero, define this as a trusted path (see below) */
+			"trustedPathId": 0|!0, /* If present and nonzero, define this as a trusted path (see below) */
+			"mtu": 0|!0 /* if present and non-zero, set UDP maximum payload MTU for this path */
 		} /* ,... additional networks */
 	},
 	"virtual": { /* Settings applied to ZeroTier virtual network devices (VL1) */