Revert "Backport guts of 1.8 to 1.6 tree so we can point release without waiting for UI quirks to be fixed."

This reverts commit 48ce7632fa39faedd7ac46c267e0d333ab5f7ffd.

controller/ConnectionPool.hpp

@@ -1,161 +0,0 @@
-/*
- * Copyright (c)2021 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2025-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#ifndef ZT_CONNECTION_POOL_H_
-#define ZT_CONNECTION_POOL_H_
-
-
-#ifndef _DEBUG
-	#define _DEBUG(x)
-#endif
-
-#include <deque>
-#include <set>
-#include <memory>
-#include <mutex>
-#include <exception>
-#include <string>
-
-namespace ZeroTier {
-
-struct ConnectionUnavailable : std::exception { 
-    char const* what() const throw() {
-        return "Unable to allocate connection";
-    }; 
-};
-
-
-class Connection {
-public:
-    virtual ~Connection() {};
-};
-
-class ConnectionFactory {
-public:
-    virtual ~ConnectionFactory() {};
-    virtual std::shared_ptr<Connection> create()=0;
-};
-
-struct ConnectionPoolStats {
-    size_t pool_size;
-    size_t borrowed_size;
-};
-
-template<class T>
-class ConnectionPool {
-public:
-    ConnectionPool(size_t max_pool_size, size_t min_pool_size, std::shared_ptr<ConnectionFactory> factory)
-        : m_maxPoolSize(max_pool_size)
-        , m_minPoolSize(min_pool_size)
-        , m_factory(factory)
-    {
-        while(m_pool.size() < m_minPoolSize){
-            m_pool.push_back(m_factory->create());
-        }
-    };
-
-    ConnectionPoolStats get_stats() {
-        std::unique_lock<std::mutex> lock(m_poolMutex);
-
-        ConnectionPoolStats stats;
-        stats.pool_size = m_pool.size();
-        stats.borrowed_size = m_borrowed.size();			
-
-        return stats;
-    };
-
-    ~ConnectionPool() {
-    };
-
-    /**
-     * Borrow
-     *
-     * Borrow a connection for temporary use
-     *
-     * When done, either (a) call unborrow() to return it, or (b) (if it's bad) just let it go out of scope.  This will cause it to automatically be replaced.
-     * @retval a shared_ptr to the connection object
-     */
-    std::shared_ptr<T> borrow() {
-        std::unique_lock<std::mutex> l(m_poolMutex);
-        
-        while((m_pool.size() + m_borrowed.size()) < m_minPoolSize) {
-            std::shared_ptr<Connection> conn = m_factory->create();
-            m_pool.push_back(conn);
-        }
-
-        if(m_pool.size()==0){
-            
-            if ((m_pool.size() + m_borrowed.size()) <= m_maxPoolSize) {
-                try {
-                    std::shared_ptr<Connection> conn = m_factory->create();
-                    m_borrowed.insert(conn);
-                    return std::static_pointer_cast<T>(conn);
-                } catch (std::exception &e) {
-                    throw ConnectionUnavailable();
-                }
-            } else {
-                for(auto it = m_borrowed.begin(); it != m_borrowed.end(); ++it){
-                    if((*it).unique()) {
-                        // This connection has been abandoned! Destroy it and create a new connection
-                        try {
-                            // If we are able to create a new connection, return it
-                            _DEBUG("Creating new connection to replace discarded connection");
-                            std::shared_ptr<Connection> conn = m_factory->create();
-                            m_borrowed.erase(it);
-                            m_borrowed.insert(conn);
-                            return std::static_pointer_cast<T>(conn);
-                        } catch(std::exception& e) {
-                            // Error creating a replacement connection
-                            throw ConnectionUnavailable();
-                        }
-                    }
-                }
-                // Nothing available
-                throw ConnectionUnavailable();
-            }
-        }
-
-        // Take one off the front
-        std::shared_ptr<Connection> conn = m_pool.front();
-        m_pool.pop_front();
-        // Add it to the borrowed list
-        m_borrowed.insert(conn);
-        return std::static_pointer_cast<T>(conn);
-    };
-
-    /**
-     * Unborrow a connection
-     *
-     * Only call this if you are returning a working connection.  If the connection was bad, just let it go out of scope (so the connection manager can replace it).
-     * @param the connection
-     */
-    void unborrow(std::shared_ptr<T> conn) {
-        // Lock
-        std::unique_lock<std::mutex> lock(m_poolMutex);
-        m_borrowed.erase(conn);
-        if ((m_pool.size() + m_borrowed.size()) < m_maxPoolSize) {
-            m_pool.push_back(conn);
-        }
-    };
-protected:
-    size_t m_maxPoolSize;
-    size_t m_minPoolSize;
-    std::shared_ptr<ConnectionFactory> m_factory;
-    std::deque<std::shared_ptr<Connection> > m_pool;
-    std::set<std::shared_ptr<Connection> > m_borrowed;
-    std::mutex m_poolMutex;
-};
-
-}
-
-#endif

controller/DB.cpp

@@ -49,9 +49,6 @@ void DB::initNetwork(nlohmann::json &network)
 		}};
 	}
 	if (!network.count("dns")) network["dns"] = nlohmann::json::array();
-	if (!network.count("ssoEnabled")) network["ssoEnabled"] = false;
-	if (!network.count("clientId")) network["clientId"] = "";
-	if (!network.count("authorizationEndpoint")) network["authorizationEndpoint"] = "";
 
 	network["objtype"] = "network";
 }
@@ -59,7 +56,6 @@ void DB::initNetwork(nlohmann::json &network)
 void DB::initMember(nlohmann::json &member)
 {
 	if (!member.count("authorized")) member["authorized"] = false;
-	if (!member.count("ssoExempt")) member["ssoExempt"] = false;
 	if (!member.count("ipAssignments")) member["ipAssignments"] = nlohmann::json::array();
 	if (!member.count("activeBridge")) member["activeBridge"] = false;
 	if (!member.count("tags")) member["tags"] = nlohmann::json::array();
@@ -71,7 +67,6 @@ void DB::initMember(nlohmann::json &member)
 	if (!member.count("lastAuthorizedTime")) member["lastAuthorizedTime"] = 0ULL;
 	if (!member.count("lastAuthorizedCredentialType")) member["lastAuthorizedCredentialType"] = nlohmann::json();
 	if (!member.count("lastAuthorizedCredential")) member["lastAuthorizedCredential"] = nlohmann::json();
-	if (!member.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = 0LL;
 	if (!member.count("vMajor")) member["vMajor"] = -1;
 	if (!member.count("vMinor")) member["vMinor"] = -1;
 	if (!member.count("vRev")) member["vRev"] = -1;
@@ -97,8 +92,6 @@ void DB::cleanMember(nlohmann::json &member)
 	member.erase("recentLog");
 	member.erase("lastModified");
 	member.erase("lastRequestMetaData");
-	member.erase("authenticationURL"); // computed
-	member.erase("authenticationClientID"); // computed
 }
 
 DB::DB() {}
@@ -181,9 +174,8 @@ bool DB::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohma
 	{
 		std::lock_guard<std::mutex> l2(nw->lock);
 		network = nw->config;
-		for(auto m=nw->members.begin();m!=nw->members.end();++m) {
+		for(auto m=nw->members.begin();m!=nw->members.end();++m)
 			members.push_back(m->second);
-		}
 	}
 	return true;
 }
@@ -196,14 +188,6 @@ void DB::networks(std::set<uint64_t> &networks)
 		networks.insert(n->first);
 }
 
-void DB::networkMemberSSOHasExpired(uint64_t nwid, int64_t now) {
-	std::lock_guard<std::mutex> l(_networks_l);
-	auto nw = _networks.find(nwid);
-	if (nw != _networks.end()) {
-		nw->second->mostRecentDeauthTime = now;
-	}
-}
-
 void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners)
 {
 	uint64_t memberId = 0;

controller/DB.hpp

@@ -31,12 +31,9 @@
 #include <atomic>
 #include <mutex>
 #include <set>
-#include <map>
 
 #include "../ext/json/json.hpp"
 
-#define ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE 25000
-
 namespace ZeroTier
 {
 
@@ -104,12 +101,11 @@ public:
 	}
 
 	virtual bool save(nlohmann::json &record,bool notifyListeners) = 0;
+
 	virtual void eraseNetwork(const uint64_t networkId) = 0;
 	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0;
-	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0;
 
-	virtual std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL) { return ""; }
-	virtual void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts);
+	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0;
 
 	inline void addListener(DB::ChangeListener *const listener)
 	{
@@ -152,8 +148,8 @@ protected:
 		std::mutex lock;
 	};
 
-	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners);
-	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners);
+	void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners);
+	void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners);
 	void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info);
 
 	std::vector<DB::ChangeListener *> _changeListeners;

controller/DBMirrorSet.cpp

@@ -36,7 +36,7 @@ DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener) :
 			}
 
 			for(auto db=dbs.begin();db!=dbs.end();++db) {
-				(*db)->each([&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) {
+				(*db)->each([this,&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) {
 					try {
 						if (network.is_object()) {
 							if (memberId == 0) {
@@ -125,26 +125,6 @@ bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,std::vect
 	return false;
 }
 
-std::string DBMirrorSet::getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL) 
-{
-	std::lock_guard<std::mutex> l(_dbs_l);
-	for(auto d=_dbs.begin();d!=_dbs.end();++d) { 
-		std::string url = (*d)->getSSOAuthURL(member, redirectURL);
-		if (!url.empty()) {
-			return url;
-		}
-	}
-	return "";
-}
-
-void DBMirrorSet::networkMemberSSOHasExpired(uint64_t nwid, int64_t ts)
-{
-	std::lock_guard<std::mutex> l(_dbs_l);
-	for(auto d=_dbs.begin();d!=_dbs.end();++d) { 
-		(*d)->networkMemberSSOHasExpired(nwid, ts);
-	}
-}
-
 void DBMirrorSet::networks(std::set<uint64_t> &networks)
 {
 	std::lock_guard<std::mutex> l(_dbs_l);
@@ -248,47 +228,4 @@ void DBMirrorSet::onNetworkMemberDeauthorize(const void *db,uint64_t networkId,u
 	_listener->onNetworkMemberDeauthorize(this,networkId,memberId);
 }
 
-void DBMirrorSet::membersExpiring(std::set< std::pair<uint64_t, uint64_t> > &soon, std::set< std::pair<uint64_t, uint64_t> > &expired)
-{
-	std::unique_lock<std::mutex> l(_membersExpiringSoon_l);
-	int64_t now = OSUtils::now();
-	for(auto next=_membersExpiringSoon.begin();next!=_membersExpiringSoon.end();) {
-		if (next->first > now) {
-			const uint64_t nwid = next->second.first;
-			const uint64_t memberId = next->second.second;
-			nlohmann::json network, member;
-			if (this->get(nwid, network, memberId, member)) {
-				try {
-					const bool authorized = member["authorized"];
-					const bool ssoExempt = member["ssoExempt"];
-					const int64_t authenticationExpiryTime = member["authenticationExpiryTime"];
-					if ((authenticationExpiryTime == next->first)&&(authorized)&&(!ssoExempt)) {
-						if ((authenticationExpiryTime - now) > ZT_MEMBER_AUTH_TIMEOUT_NOTIFY_BEFORE) {
-							// Stop when we get to entries too far in the future.
-							break;
-						} else {
-							const bool ssoEnabled = network["ssoEnabled"];
-							if (ssoEnabled)
-								soon.insert(std::pair<uint64_t, uint64_t>(nwid, memberId));
-						}
-					} else {
-						// Obsolete entry, no longer authorized, or SSO exempt.
-					}
-				} catch ( ... ) {
-					// Invalid member object, erase.
-				}
-			} else {
-				// Not found.
-			}
-		}
-		_membersExpiringSoon.erase(next++);
-	}
-}
-
-void DBMirrorSet::memberWillExpire(int64_t expTime, uint64_t nwid, uint64_t memberId)
-{
-	std::unique_lock<std::mutex> l(_membersExpiringSoon_l);
-	_membersExpiringSoon.insert(std::pair< int64_t, std::pair< uint64_t, uint64_t > >(expTime, std::pair< uint64_t, uint64_t >(nwid, memberId)));
-}
-
 } // namespace ZeroTier

controller/DBMirrorSet.hpp

@@ -51,9 +51,6 @@ public:
 	virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member);
 	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);
 
-	std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL);
-	void networkMemberSSOHasExpired(uint64_t nwid, int64_t ts);
-
 	inline void addDB(const std::shared_ptr<DB> &db)
 	{
 		db->addListener(this);
@@ -61,17 +58,12 @@ public:
 		_dbs.push_back(db);
 	}
 
-	void membersExpiring(std::set< std::pair<uint64_t, uint64_t> > &soon, std::set< std::pair<uint64_t, uint64_t> > &expired);
-	void memberWillExpire(int64_t expTime, uint64_t nwid, uint64_t memberId);
-
 private:
 	DB::ChangeListener *const _listener;
 	std::atomic_bool _running;
 	std::thread _syncCheckerThread;
 	std::vector< std::shared_ptr< DB > > _dbs;
 	mutable std::mutex _dbs_l;
-	std::set< std::pair< int64_t, std::pair<uint64_t, uint64_t> > > _membersExpiringSoon;
-	mutable std::mutex _membersExpiringSoon_l;
 };
 
 } // namespace ZeroTier

controller/EmbeddedNetworkController.cpp

@@ -28,9 +28,6 @@
 #include <map>
 #include <thread>
 #include <memory>
-#include <iomanip>
-#include <sstream>
-#include <cctype>
 
 #include "../include/ZeroTierOne.h"
 #include "../version.h"
@@ -63,29 +60,6 @@ namespace ZeroTier {
 
 namespace {
 
-std::string url_encode(const std::string &value) {
-    std::ostringstream escaped;
-    escaped.fill('0');
-    escaped << std::hex;
-
-    for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
-        std::string::value_type c = (*i);
-
-        // Keep alphanumeric and other accepted characters intact
-        if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
-            escaped << c;
-            continue;
-        }
-
-        // Any other characters are percent-encoded
-        escaped << std::uppercase;
-        escaped << '%' << std::setw(2) << int((unsigned char) c);
-        escaped << std::nouppercase;
-    }
-
-    return escaped.str();
-}
-
 static json _renderRule(ZT_VirtualNetworkRule &rule)
 {
 	char tmp[128];
@@ -502,10 +476,6 @@ EmbeddedNetworkController::~EmbeddedNetworkController()
 		t->join();
 }
 
-void EmbeddedNetworkController::setSSORedirectURL(const std::string &url) {
-	_ssoRedirectURL = url_encode(url);
-}
-
 void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 {
 	char tmp[64];
@@ -718,10 +688,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 					DB::initMember(member);
 
 					try {
-						if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"], false);
-						if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"], false);
-						if (b.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = (uint64_t)OSUtils::jsonInt(b["authenticationExpiryTime"], 0ULL);
-						if (b.count("authenticationURL")) member["authenticationURL"] = OSUtils::jsonString(b["authenticationURL"], "");
+						if (b.count("activeBridge")) member["activeBridge"] = OSUtils::jsonBool(b["activeBridge"],false);
+						if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"],false);
 
 						if (b.count("remoteTraceTarget")) {
 							const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],""));
@@ -1280,7 +1248,7 @@ void EmbeddedNetworkController::_request(
 	Utils::hex(nwid,nwids);
 	_db.get(nwid,network,identity.address().toInt(),member,ns);
 	if ((!network.is_object())||(network.empty())) {
-		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND, nullptr, 0);
+		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND);
 		return;
 	}
 	const bool newMember = ((!member.is_object())||(member.empty()));
@@ -1294,11 +1262,11 @@ void EmbeddedNetworkController::_request(
 			// known member.
 			try {
 				if (Identity(haveIdStr.c_str()) != identity) {
-					_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
+					_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
 					return;
 				}
 			} catch ( ... ) {
-				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
+				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
 				return;
 			}
 		} else {
@@ -1355,38 +1323,6 @@ void EmbeddedNetworkController::_request(
 		member["lastAuthorizedCredential"] = autoAuthCredential;
 	}
 
-	// Should we check SSO Stuff?
-	// If network is configured with SSO, and the member is not marked exempt: yes
-	// Otherwise no, we use standard auth logic.
-	bool networkSSOEnabled = OSUtils::jsonBool(network["ssoEnabled"], false);
-	bool memberSSOExempt = OSUtils::jsonBool(member["ssoExempt"], false);
-	std::string authenticationURL;
-	if (networkSSOEnabled && !memberSSOExempt) {
-		authenticationURL = _db.getSSOAuthURL(member, _ssoRedirectURL);
-		std::string memberId = member["id"];
-		//fprintf(stderr, "ssoEnabled && !ssoExempt %s-%s\n", nwids, memberId.c_str());
-		uint64_t authenticationExpiryTime = (int64_t)OSUtils::jsonInt(member["authenticationExpiryTime"], 0);
-		//fprintf(stderr, "authExpiryTime: %lld\n", authenticationExpiryTime);
-		if (authenticationExpiryTime < now) {
-			if (!authenticationURL.empty()) {
-				_db.networkMemberSSOHasExpired(nwid, now);
-				onNetworkMemberDeauthorize(&_db, nwid, identity.address().toInt());
-
-				Dictionary<3072> authInfo;
-				authInfo.add("aU", authenticationURL.c_str());
-				//fprintf(stderr, "sending auth URL: %s\n", authenticationURL.c_str());
-
-				DB::cleanMember(member);
-				_db.save(member,true);
-
-				_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
-				return;
-			}
-		} else if (authorized) {
-			_db.memberWillExpire(authenticationExpiryTime, nwid, identity.address().toInt());
-		}
-	}
-
 	if (authorized) {
 		// Update version info and meta-data if authorized and if this is a genuine request
 		if (requestPacketId) {
@@ -1411,18 +1347,17 @@ void EmbeddedNetworkController::_request(
 				ms.lastRequestMetaData = metaData;
 				ms.identity = identity;
 			}
-		}		
+		}
 	} else {
-		
 		// If they are not authorized, STOP!
 		DB::cleanMember(member);
 		_db.save(member,true);
-		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED, nullptr, 0);
+		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
 		return;
 	}
 
 	// -------------------------------------------------------------------------
-	// If we made it this far, they are authorized (and authenticated).
+	// If we made it this far, they are authorized.
 	// -------------------------------------------------------------------------
 
 	int64_t credentialtmd = ZT_NETWORKCONFIG_DEFAULT_CREDENTIAL_TIME_MAX_MAX_DELTA;
@@ -1451,11 +1386,7 @@ void EmbeddedNetworkController::_request(
 	nc->mtu = std::max(std::min((unsigned int)OSUtils::jsonInt(network["mtu"],ZT_DEFAULT_MTU),(unsigned int)ZT_MAX_MTU),(unsigned int)ZT_MIN_MTU);
 	nc->multicastLimit = (unsigned int)OSUtils::jsonInt(network["multicastLimit"],32ULL);
 
-	nc->ssoEnabled = OSUtils::jsonBool(network["ssoEnabled"], false);
-	nc->authenticationExpiryTime = OSUtils::jsonInt(member["authenticationExpiryTime"], 0LL);
-	if (!authenticationURL.empty())
-		Utils::scopy(nc->authenticationURL, sizeof(nc->authenticationURL), authenticationURL.c_str());
-
+	
 	std::string rtt(OSUtils::jsonString(member["remoteTraceTarget"],""));
 	if (rtt.length() == 10) {
 		nc->remoteTraceTarget = Address(Utils::hexStrToU64(rtt.c_str()));
@@ -1484,8 +1415,6 @@ void EmbeddedNetworkController::_request(
 	json &memberTags = member["tags"];
 	json &dns = network["dns"];
 
-	//fprintf(stderr, "IP Assignment Pools for Network %s: %s\n", nwids, OSUtils::jsonDump(ipAssignmentPools, 2).c_str());
-
 	if (metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_RULES_ENGINE_REV,0) <= 0) {
 		// Old versions with no rules engine support get an allow everything rule.
 		// Since rules are enforced bidirectionally, newer versions *will* still
@@ -1801,11 +1730,11 @@ void EmbeddedNetworkController::_request(
 		nc->certificateOfOwnershipCount = 1;
 	}
 
-	CertificateOfMembership com(now,credentialtmd,nwid,identity);
+	CertificateOfMembership com(now,credentialtmd,nwid,identity.address());
 	if (com.sign(_signingId)) {
 		nc->com = com;
 	} else {
-		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR, nullptr, 0);
+		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_INTERNAL_SERVER_ERROR);
 		return;
 	}
 
@@ -1824,44 +1753,17 @@ void EmbeddedNetworkController::_startThreads()
 		_threads.emplace_back([this]() {
 			for(;;) {
 				_RQEntry *qe = (_RQEntry *)0;
-				auto timedWaitResult = _queue.get(qe, 1000);
-				if (timedWaitResult == BlockingQueue<_RQEntry *>::STOP) {
+				if (!_queue.get(qe))
 					break;
-				} else if (timedWaitResult == BlockingQueue<_RQEntry *>::OK) {
+				try {
 					if (qe) {
-						try {
-							_request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData);
-						} catch (std::exception &e) {
-							fprintf(stderr,"ERROR: exception in controller request handling thread: %s" ZT_EOL_S,e.what());
-						} catch ( ... ) {
-							fprintf(stderr,"ERROR: exception in controller request handling thread: unknown exception" ZT_EOL_S);
-						}
+						_request(qe->nwid,qe->fromAddr,qe->requestPacketId,qe->identity,qe->metaData);
 						delete qe;
 					}
-				}
-
-				std::set< std::pair<uint64_t, uint64_t> > soon;
-				std::set< std::pair<uint64_t, uint64_t> > expired;
-				_db.membersExpiring(soon, expired);
-
-				for(auto s=soon.begin();s!=soon.end();++s) {
-					Identity identity;
-					Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> lastMetaData;
-					{
-						std::unique_lock<std::mutex> ll(_memberStatus_l);
-						auto ms = _memberStatus.find(_MemberStatusKey(s->first, s->second));
-						if (ms != _memberStatus.end()) {
-							lastMetaData = ms->second.lastRequestMetaData;
-							identity = ms->second.identity;
-						}
-					}
-					if (identity) {
-						request(s->first,InetAddress(),0,identity,lastMetaData);
-					}
-				}
-
-				for(auto e=expired.begin();e!=expired.end();++e) {
-					onNetworkMemberDeauthorize(nullptr, e->first, e->second);
+				} catch (std::exception &e) {
+					fprintf(stderr,"ERROR: exception in controller request handling thread: %s" ZT_EOL_S,e.what());
+				} catch ( ... ) {
+					fprintf(stderr,"ERROR: exception in controller request handling thread: unknown exception" ZT_EOL_S);
 				}
 			}
 		});

controller/EmbeddedNetworkController.hpp

@@ -57,8 +57,6 @@ public:
 
 	virtual void init(const Identity &signingId,Sender *sender);
 
-	void setSSORedirectURL(const std::string &url);
-
 	virtual void request(
 		uint64_t nwid,
 		const InetAddress &fromAddr,
@@ -153,7 +151,6 @@ private:
 	std::mutex _memberStatus_l;
 
 	RedisConfig *_rc;
-	std::string _ssoRedirectURL;
 };
 
 } // namespace ZeroTier

controller/FileDB.cpp

@@ -140,7 +140,8 @@ void FileDB::eraseNetwork(const uint64_t networkId)
 void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
 {
 	nlohmann::json network,member,nullJson;
-	get(networkId,network,memberId,member);
+	get(networkId,network);
+	get(memberId,member);
 	char p[4096];
 	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member" ZT_PATH_SEPARATOR_S "%.10llx.json",_networksPath.c_str(),networkId,memberId);
 	OSUtils::rm(p);

controller/PostgreSQL.hpp

@@ -20,9 +20,6 @@
 
 #define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
 
-#include "ConnectionPool.hpp"
-#include <pqxx/pqxx>
-
 #include <memory>
 #include <redis++/redis++.h>
 
@@ -34,69 +31,14 @@ namespace ZeroTier {
 
 struct RedisConfig;
 
-
-class PostgresConnection : public Connection {
-public:
-	virtual ~PostgresConnection() {
-	}
-
-	std::shared_ptr<pqxx::connection> c;
-	int a;
-};
-
-
-class PostgresConnFactory : public ConnectionFactory {
-public:
-	PostgresConnFactory(std::string &connString) 
-		: m_connString(connString)
-	{
-	}
-
-	virtual std::shared_ptr<Connection> create() {
-		auto c = std::shared_ptr<PostgresConnection>(new PostgresConnection());
-		c->c = std::make_shared<pqxx::connection>(m_connString);
-		return std::static_pointer_cast<Connection>(c);
-	}
-private:
-	std::string m_connString;
-};
-
-class PostgreSQL;
-
-class MemberNotificationReceiver : public pqxx::notification_receiver {
-public: 
-	MemberNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
-	virtual ~MemberNotificationReceiver() {
-		fprintf(stderr, "MemberNotificationReceiver destroyed\n");
-	}
-
-	virtual void operator() (const std::string &payload, int backendPid);
-private:
-	PostgreSQL *_psql;
-};
-
-class NetworkNotificationReceiver : public pqxx::notification_receiver {
-public:
-	NetworkNotificationReceiver(PostgreSQL *p, pqxx::connection &c, const std::string &channel);
-	virtual ~NetworkNotificationReceiver() {
-		fprintf(stderr, "NetworkNotificationReceiver destroyed\n");
-	};
-
-	virtual void operator() (const std::string &payload, int packend_pid);
-private:
-	PostgreSQL *_psql;
-};
-
 /**
  * A controller database driver that talks to PostgreSQL
  *
  * This is for use with ZeroTier Central.  Others are free to build and use it
- * but be aware that we might change it at any time.
+ * but be aware taht we might change it at any time.
  */
 class PostgreSQL : public DB
 {
-	friend class MemberNotificationReceiver;
-	friend class NetworkNotificationReceiver;
 public:
 	PostgreSQL(const Identity &myId, const char *path, int listenPort, RedisConfig *rc);
 	virtual ~PostgreSQL();
@@ -107,29 +49,21 @@ public:
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
-	virtual std::string getSSOAuthURL(const nlohmann::json &member, const std::string &redirectURL);
 
 protected:
 	struct _PairHasher
 	{
 		inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); }
 	};
-	virtual void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners) {
-		DB::_memberChanged(old, memberConfig, notifyListeners);
-	}
-
-	virtual void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners) {
-		DB::_networkChanged(old, networkConfig, notifyListeners);
-	}
 
 private:
-	void initializeNetworks();
-	void initializeMembers();
+	void initializeNetworks(PGconn *conn);
+	void initializeMembers(PGconn *conn);
 	void heartbeat();
 	void membersDbWatcher();
-	void _membersWatcher_Postgres();
+	void _membersWatcher_Postgres(PGconn *conn);
 	void networksDbWatcher();
-	void _networksWatcher_Postgres();
+	void _networksWatcher_Postgres(PGconn *conn);
 
 	void _membersWatcher_Redis();
 	void _networksWatcher_Redis();
@@ -146,7 +80,7 @@ private:
 		NO_OVERRIDE = 1
 	};
 
-	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+	PGconn * getPgConn( OverrideMode m = ALLOW_PGBOUNCER_OVERRIDE );
 
 	const Identity _myId;
 	const Address _myAddress;
@@ -169,7 +103,6 @@ private:
 	mutable volatile bool _waitNoticePrinted;
 
 	int _listenPort;
-	uint8_t _ssoPsk[48];
 
 	RedisConfig *_rc;
 	std::shared_ptr<sw::redis::Redis> _redis;

include/ZeroTierOne.h

@@ -420,6 +420,157 @@ enum ZT_ResultCode
  */
 #define ZT_ResultCode_isFatal(x) ((((int)(x)) >= 100)&&(((int)(x)) < 1000))
 
+
+/**
+ *  Multipath bonding policy
+ */
+enum ZT_MultipathBondingPolicy
+{
+	/**
+	 * Normal operation. No fault tolerance, no load balancing
+	 */
+	ZT_BONDING_POLICY_NONE = 0,
+
+	/**
+	 * Sends traffic out on only one path at a time. Configurable immediate
+	 * fail-over.
+	 */
+	ZT_BONDING_POLICY_ACTIVE_BACKUP = 1,
+
+	/**
+	 * Sends traffic out on all paths
+	 */
+	ZT_BONDING_POLICY_BROADCAST = 2,
+
+	/**
+	 * Stripes packets across all paths
+	 */
+	ZT_BONDING_POLICY_BALANCE_RR = 3,
+
+	/**
+	 * Packets destined for specific peers will always be sent over the same
+	 * path.
+	 */
+	ZT_BONDING_POLICY_BALANCE_XOR = 4,
+
+	/**
+	 * Balances flows among all paths according to path performance
+	 */
+	ZT_BONDING_POLICY_BALANCE_AWARE = 5
+};
+
+/**
+ * Multipath active re-selection policy (linkSelectMethod)
+ */
+enum ZT_MultipathLinkSelectMethod
+{
+	/**
+	 * Primary link regains status as active link whenever it comes back up
+	 * (default when links are explicitly specified)
+	 */
+	ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS = 0,
+
+	/**
+	 * Primary link regains status as active link when it comes back up and
+	 * (if) it is better than the currently-active link.
+	 */
+	ZT_MULTIPATH_RESELECTION_POLICY_BETTER = 1,
+
+	/**
+	 * Primary link regains status as active link only if the currently-active
+	 * link fails.
+	 */
+	ZT_MULTIPATH_RESELECTION_POLICY_FAILURE = 2,
+
+	/**
+	 * The primary link can change if a superior path is detected.
+	 * (default if user provides no fail-over guidance)
+	 */
+	ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE = 3
+};
+
+/**
+ * Mode of multipath link interface
+ */
+enum ZT_MultipathLinkMode
+{
+	ZT_MULTIPATH_SLAVE_MODE_PRIMARY = 0,
+	ZT_MULTIPATH_SLAVE_MODE_SPARE = 1
+};
+
+/**
+ * Strategy for path monitoring
+ */
+enum ZT_MultipathMonitorStrategy
+{
+	/**
+	 * Use bonding policy's default strategy
+	 */
+	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DEFAULT = 0,
+
+	/**
+	 * Does not actively send probes to judge aliveness, will rely
+	 * on conventional traffic and summary statistics.
+	 */
+	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE = 1,
+
+	/**
+	 * Sends probes at a constant rate to judge aliveness.
+	 */
+	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_ACTIVE = 2,
+
+	/**
+	 * Sends probes at varying rates which correlate to native
+	 * traffic loads to judge aliveness.
+	 */
+	ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC = 3
+};
+
+/**
+ * Strategy for re-balancing protocol flows
+ */
+enum ZT_MultipathFlowRebalanceStrategy
+{
+	/**
+	 * Flows will only be re-balanced among links during
+	 * assignment or failover. This minimizes the possibility
+	 * of sequence reordering and is thus the default setting.
+	 */
+	ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_PASSIVE = 0,
+
+	/**
+	 * Flows that are active may be re-assigned to a new more
+	 * suitable link if it can be done without disrupting the flow.
+	 * This setting can sometimes cause sequence re-ordering.
+	 */
+	ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_OPPORTUNISTIC = 0,
+
+	/**
+	 * Flows will be continuously re-assigned the most suitable link
+	 * in order to maximize "balance". This can often cause sequence
+	 * reordering and is thus only reccomended for protocols like UDP.
+	 */
+	ZT_MULTIPATH_FLOW_REBALANCE_STRATEGY_AGGRESSIVE = 2
+};
+
+/**
+ * Indices for the path quality weight vector
+ */
+enum ZT_MultipathQualityWeightIndex
+{
+	ZT_QOS_LAT_IDX,
+	ZT_QOS_LTM_IDX,
+	ZT_QOS_PDV_IDX,
+	ZT_QOS_PLR_IDX,
+	ZT_QOS_PER_IDX,
+	ZT_QOS_THR_IDX,
+	ZT_QOS_THM_IDX,
+	ZT_QOS_THV_IDX,
+	ZT_QOS_AGE_IDX,
+	ZT_QOS_SCP_IDX,
+	ZT_QOS_WEIGHT_SIZE
+};
+
 /**
  * Status codes sent to status update callback when things happen
  */
@@ -669,12 +820,7 @@ enum ZT_VirtualNetworkStatus
 	/**
 	 * ZeroTier core version too old
 	 */
-	ZT_NETWORK_STATUS_CLIENT_TOO_OLD = 5,
-
-	/**
-	 * External authentication is required (e.g. SSO)
-	 */
-	ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED = 6
+	ZT_NETWORK_STATUS_CLIENT_TOO_OLD = 5
 };
 
 /**
@@ -1193,21 +1339,6 @@ typedef struct
 	 * Network specific DNS configuration
 	 */
 	ZT_VirtualNetworkDNS dns;
-
-	/**
-	 * sso enabled
-	 */
-	bool ssoEnabled;
-
-	/**
-	 * If the status us AUTHENTICATION_REQUIRED, this may contain a URL for authentication.
-	 */
-	char authenticationURL[2048];
-
-	/**
-	 * Time that current authentication expires. only valid if ssoEnabled is true
-	 */
-	uint64_t authenticationExpiryTime;
 } ZT_VirtualNetworkConfig;
 
 /**

node/BondController.cpp

@@ -0,0 +1,212 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2025-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "../osdep/OSUtils.hpp"
+
+#include "Constants.hpp"
+#include "BondController.hpp"
+#include "Peer.hpp"
+
+namespace ZeroTier {
+
+int BondController::_minReqPathMonitorInterval;
+uint8_t BondController::_defaultBondingPolicy;
+
+BondController::BondController(const RuntimeEnvironment *renv) :
+	RR(renv)
+{
+	bondStartTime = RR->node->now();
+	_defaultBondingPolicy = ZT_BONDING_POLICY_NONE;
+}
+
+bool BondController::linkAllowed(std::string &policyAlias, SharedPtr<Link> link)
+{
+	bool foundInDefinitions = false;
+	if (_linkDefinitions.count(policyAlias)) {
+		auto it = _linkDefinitions[policyAlias].begin();
+		while (it != _linkDefinitions[policyAlias].end()) {
+			if (link->ifname() == (*it)->ifname()) {
+				foundInDefinitions = true;
+				break;
+			}
+			++it;
+		}
+	}
+	return _linkDefinitions[policyAlias].empty() || foundInDefinitions;
+}
+
+void BondController::addCustomLink(std::string& policyAlias, SharedPtr<Link> link)
+{
+	Mutex::Lock _l(_links_m);
+	_linkDefinitions[policyAlias].push_back(link);
+	auto search = _interfaceToLinkMap[policyAlias].find(link->ifname());
+	if (search == _interfaceToLinkMap[policyAlias].end()) {
+		link->setAsUserSpecified(true);
+		_interfaceToLinkMap[policyAlias].insert(std::pair<std::string, SharedPtr<Link>>(link->ifname(), link));
+	}
+}
+
+bool BondController::addCustomPolicy(const SharedPtr<Bond>& newBond)
+{
+	Mutex::Lock _l(_bonds_m);
+	if (!_bondPolicyTemplates.count(newBond->policyAlias())) {
+		_bondPolicyTemplates[newBond->policyAlias()] = newBond;
+		return true;
+	}
+	return false;
+}
+
+bool BondController::assignBondingPolicyToPeer(int64_t identity, const std::string& policyAlias)
+{
+	Mutex::Lock _l(_bonds_m);
+	if (!_policyTemplateAssignments.count(identity)) {
+		_policyTemplateAssignments[identity] = policyAlias;
+		return true;
+	}
+	return false;
+}
+
+SharedPtr<Bond> BondController::getBondByPeerId(int64_t identity)
+{
+	Mutex::Lock _l(_bonds_m);
+	return _bonds.count(identity) ? _bonds[identity] : SharedPtr<Bond>();
+}
+
+SharedPtr<Bond> BondController::createTransportTriggeredBond(const RuntimeEnvironment *renv, const SharedPtr<Peer>& peer)
+{
+	Mutex::Lock _l(_bonds_m);
+	int64_t identity = peer->identity().address().toInt();
+	Bond *bond = nullptr;
+	char traceMsg[128];
+	if (!_bonds.count(identity)) {
+		std::string policyAlias;
+		if (!_policyTemplateAssignments.count(identity)) {
+			if (_defaultBondingPolicy) {
+				sprintf(traceMsg, "%s (bond) Creating new default %s bond to peer %llx",
+					OSUtils::humanReadableTimestamp().c_str(), getPolicyStrByCode(_defaultBondingPolicy).c_str(), identity); RR->t->bondStateMessage(NULL, traceMsg);
+				bond = new Bond(renv, _defaultBondingPolicy, peer);
+			}
+			if (!_defaultBondingPolicy && _defaultBondingPolicyStr.length()) {
+				sprintf(traceMsg, "%s (bond) Creating new default custom %s bond to peer %llx",
+					OSUtils::humanReadableTimestamp().c_str(), _defaultBondingPolicyStr.c_str(), identity);
+				RR->t->bondStateMessage(NULL, traceMsg);
+				bond = new Bond(renv, _bondPolicyTemplates[_defaultBondingPolicyStr].ptr(), peer);
+			}
+		}
+		else {
+			if (!_bondPolicyTemplates[_policyTemplateAssignments[identity]]) {
+				sprintf(traceMsg, "%s (bond) Creating new bond. Assignment for peer %llx was specified as %s but the bond definition was not found. Using default %s",
+					OSUtils::humanReadableTimestamp().c_str(), identity, _policyTemplateAssignments[identity].c_str(), getPolicyStrByCode(_defaultBondingPolicy).c_str());
+				RR->t->bondStateMessage(NULL, traceMsg);
+				bond = new Bond(renv, _defaultBondingPolicy, peer);
+			}
+			else {
+				sprintf(traceMsg, "%s (bond) Creating new default bond %s to peer %llx",
+					OSUtils::humanReadableTimestamp().c_str(), _defaultBondingPolicyStr.c_str(), identity);
+				RR->t->bondStateMessage(NULL, traceMsg);
+				bond = new Bond(renv, _bondPolicyTemplates[_policyTemplateAssignments[identity]].ptr(), peer);
+			}
+		}
+	}
+	if (bond) {
+		_bonds[identity] = bond;
+		/**
+		 * Determine if user has specified anything that could affect the bonding policy's decisions
+		 */
+		if (_interfaceToLinkMap.count(bond->policyAlias())) {
+			std::map<std::string, SharedPtr<Link> >::iterator it = _interfaceToLinkMap[bond->policyAlias()].begin();
+			while (it != _interfaceToLinkMap[bond->policyAlias()].end()) {
+				if (it->second->isUserSpecified()) {
+					bond->_userHasSpecifiedLinks = true;
+				}
+				if (it->second->isUserSpecified() && it->second->primary()) {
+					bond->_userHasSpecifiedPrimaryLink = true;
+				}
+				if (it->second->isUserSpecified() && it->second->userHasSpecifiedFailoverInstructions()) {
+					bond->_userHasSpecifiedFailoverInstructions = true;
+				}
+				if (it->second->isUserSpecified() && (it->second->speed() > 0)) {
+					bond->_userHasSpecifiedLinkSpeeds = true;
+				}
+				++it;
+			}
+		}
+		return bond;
+	}
+	return SharedPtr<Bond>();
+}
+
+SharedPtr<Link> BondController::getLinkBySocket(const std::string& policyAlias, uint64_t localSocket)
+{
+	Mutex::Lock _l(_links_m);
+	char ifname[16];
+	_phy->getIfName((PhySocket *) ((uintptr_t)localSocket), ifname, 16);
+	std::string ifnameStr(ifname);
+	auto search = _interfaceToLinkMap[policyAlias].find(ifnameStr);
+	if (search == _interfaceToLinkMap[policyAlias].end()) {
+		SharedPtr<Link> s = new Link(ifnameStr, 0, 0, 0, 0, 0, true, ZT_MULTIPATH_SLAVE_MODE_SPARE, "", 0.0);
+		_interfaceToLinkMap[policyAlias].insert(std::pair<std::string,SharedPtr<Link> >(ifnameStr, s));
+		return s;
+	}
+	else {
+		return search->second;
+	}
+}
+
+SharedPtr<Link> BondController::getLinkByName(const std::string& policyAlias, const std::string& ifname)
+{
+	Mutex::Lock _l(_links_m);
+	auto search = _interfaceToLinkMap[policyAlias].find(ifname);
+	if (search != _interfaceToLinkMap[policyAlias].end()) {
+		return search->second;
+	}
+	return SharedPtr<Link>();
+}
+
+bool BondController::allowedToBind(const std::string& ifname)
+{
+	return true;
+	/*
+	if (!_defaultBondingPolicy) {
+		return true; // no restrictions
+	}
+	Mutex::Lock _l(_links_m);
+	if (_interfaceToLinkMap.empty()) {
+		return true; // no restrictions
+	}
+	std::map<std::string, std::map<std::string, SharedPtr<Link> > >::iterator policyItr = _interfaceToLinkMap.begin();
+	while (policyItr != _interfaceToLinkMap.end()) {
+		std::map<std::string, SharedPtr<Link> >::iterator linkItr = policyItr->second.begin();
+		while (linkItr != policyItr->second.end()) {
+			if (linkItr->first == ifname) {
+				return true;
+			}
+			++linkItr;
+		}
+		++policyItr;
+	}
+	return false;
+	*/
+}
+
+void BondController::processBackgroundTasks(void *tPtr, const int64_t now)
+{
+	Mutex::Lock _l(_bonds_m);
+	std::map<int64_t,SharedPtr<Bond> >::iterator bondItr = _bonds.begin();
+	while (bondItr != _bonds.end()) {
+		bondItr->second->processBackgroundTasks(tPtr, now);
+		++bondItr;
+	}
+}
+
+} // namespace ZeroTier

node/BondController.hpp

@@ -0,0 +1,239 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2025-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_BONDCONTROLLER_HPP
+#define ZT_BONDCONTROLLER_HPP
+
+#include <map>
+#include <vector>
+
+#include "SharedPtr.hpp"
+#include "../osdep/Phy.hpp"
+#include "../osdep/Link.hpp"
+
+namespace ZeroTier {
+
+class RuntimeEnvironment;
+class Bond;
+class Peer;
+
+class BondController
+{
+	friend class Bond;
+
+public:
+
+	BondController(const RuntimeEnvironment *renv);
+
+	/**
+	 * @return Whether this link is permitted to become a member of a bond.
+	 */
+	bool linkAllowed(std::string &policyAlias, SharedPtr<Link> link);
+
+	/**
+	 * @return The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements.
+	 */
+	int minReqPathMonitorInterval() { return _minReqPathMonitorInterval; }
+
+	/**
+	 * @param minReqPathMonitorInterval The minimum interval required to poll the active bonds to fulfill all active monitoring timing requirements.
+	 */
+	static void setMinReqPathMonitorInterval(int minReqPathMonitorInterval) { _minReqPathMonitorInterval = minReqPathMonitorInterval; }
+
+	/**
+	 * @return Whether the bonding layer is currently set up to be used.
+	 */
+	bool inUse() { return !_bondPolicyTemplates.empty() || _defaultBondingPolicy; }
+
+	/**
+	 * @param basePolicyName Bonding policy name (See ZeroTierOne.h)
+	 * @return The bonding policy code for a given human-readable bonding policy name
+	 */
+	static int getPolicyCodeByStr(const std::string& basePolicyName)
+	{
+		if (basePolicyName == "active-backup") { return 1; }
+		if (basePolicyName == "broadcast") { return 2; }
+		if (basePolicyName == "balance-rr") { return 3; }
+		if (basePolicyName == "balance-xor") { return 4; }
+		if (basePolicyName == "balance-aware") { return 5; }
+		return 0; // "none"
+	}
+
+	/**
+	 * @param policy Bonding policy code (See ZeroTierOne.h)
+	 * @return The human-readable name for the given bonding policy code
+	 */
+	static std::string getPolicyStrByCode(int policy)
+	{
+		if (policy == 1) { return "active-backup"; }
+		if (policy == 2) { return "broadcast"; }
+		if (policy == 3) { return "balance-rr"; }
+		if (policy == 4) { return "balance-xor"; }
+		if (policy == 5) { return "balance-aware"; }
+		return "none";
+	}
+
+	/**
+	 * Sets the default bonding policy for new or undefined bonds.
+	 *
+	 * @param bp Bonding policy
+	 */
+	void setBondingLayerDefaultPolicy(uint8_t bp) { _defaultBondingPolicy = bp; }
+
+	/**
+	 * Sets the default (custom) bonding policy for new or undefined bonds.
+	 *
+	 * @param alias Human-readable string alias for bonding policy
+	 */
+	void setBondingLayerDefaultPolicyStr(std::string alias) { _defaultBondingPolicyStr = alias; }
+
+	/**
+	 * @return The default bonding policy
+	 */
+	static int defaultBondingPolicy() { return _defaultBondingPolicy; }
+
+	/**
+	 * Add a user-defined link to a given bonding policy.
+	 *
+	 * @param policyAlias User-defined custom name for variant of bonding policy
+	 * @param link Pointer to new link definition
+	 */
+	void addCustomLink(std::string& policyAlias, SharedPtr<Link> link);
+
+	/**
+	 * Add a user-defined bonding policy that is based on one of the standard types.
+	 *
+	 * @param newBond Pointer to custom Bond object
+	 * @return Whether a uniquely-named custom policy was successfully added
+	 */
+	bool addCustomPolicy(const SharedPtr<Bond>& newBond);
+
+	/**
+	 * Assigns a specific bonding policy
+	 *
+	 * @param identity
+	 * @param policyAlias
+	 * @return
+	 */
+	bool assignBondingPolicyToPeer(int64_t identity, const std::string& policyAlias);
+
+	/**
+	 * Get pointer to bond by a given peer ID
+	 *
+	 * @param peer Remote peer ID
+	 * @return A pointer to the Bond
+	 */
+	SharedPtr<Bond> getBondByPeerId(int64_t identity);
+
+	/**
+	 * Add a new bond to the bond controller.
+	 *
+	 * @param renv Runtime environment
+	 * @param peer Remote peer that this bond services
+	 * @return A pointer to the newly created Bond
+	 */
+	SharedPtr<Bond> createTransportTriggeredBond(const RuntimeEnvironment *renv, const SharedPtr<Peer>& peer);
+
+	/**
+	 * Periodically perform maintenance tasks for the bonding layer.
+	 *
+	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
+	 * @param now Current time
+	 */
+	void processBackgroundTasks(void *tPtr, int64_t now);
+
+	/**
+	 * Gets a reference to a physical link definition given a policy alias and a local socket.
+	 *
+	 * @param policyAlias Policy in use
+	 * @param localSocket Local source socket
+	 * @return Physical link definition
+	 */
+	SharedPtr<Link> getLinkBySocket(const std::string& policyAlias, uint64_t localSocket);
+
+	/**
+	 * Gets a reference to a physical link definition given its human-readable system name.
+	 *
+	 * @param policyAlias Policy in use
+	 * @param ifname Alphanumeric human-readable name
+	 * @return Physical link definition
+	 */
+	SharedPtr<Link> getLinkByName(const std::string& policyAlias, const std::string& ifname);
+
+	/**
+	 * @param ifname Name of interface that we want to know if we can bind to
+	 */
+	bool allowedToBind(const std::string& ifname);
+
+	uint64_t getBondStartTime() { return bondStartTime; }
+
+private:
+
+	Phy<BondController *> *_phy;
+	const RuntimeEnvironment *RR;
+
+	Mutex _bonds_m;
+	Mutex _links_m;
+
+	/**
+	 * The last time that the bond controller updated the set of bonds.
+	 */
+	uint64_t _lastBackgroundBondControlTaskCheck;
+
+	/**
+	 * The minimum monitoring interval among all paths in this bond.
+	 */
+	static int _minReqPathMonitorInterval;
+
+	/**
+	 * The default bonding policy used for new bonds unless otherwise specified.
+	 */
+	static uint8_t _defaultBondingPolicy;
+
+	/**
+	 * The default bonding policy used for new bonds unless otherwise specified.
+	 */
+	std::string _defaultBondingPolicyStr;
+
+	/**
+	 * All currently active bonds.
+	 */
+	std::map<int64_t,SharedPtr<Bond> > _bonds;
+
+	/**
+	 * Map of peers to custom bonding policies
+	 */
+	std::map<int64_t,std::string> _policyTemplateAssignments;
+
+	/**
+	 * User-defined bonding policies (can be assigned to a peer)
+	 */
+	std::map<std::string,SharedPtr<Bond> > _bondPolicyTemplates;
+
+	/**
+	 * Set of links defined for a given bonding policy
+	 */
+	std::map<std::string,std::vector<SharedPtr<Link> > > _linkDefinitions;
+
+	/**
+	 * Set of link objects mapped to their physical interfaces
+	 */
+	std::map<std::string, std::map<std::string, SharedPtr<Link> > > _interfaceToLinkMap;
+
+	// TODO: Remove
+	uint64_t bondStartTime;
+};
+
+} // namespace ZeroTier
+
+#endif

node/CertificateOfMembership.cpp

@@ -20,67 +20,165 @@
 
 namespace ZeroTier {
 
-CertificateOfMembership::CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Identity &issuedTo)
+void CertificateOfMembership::setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta)
 {
-	_qualifiers[0].id = COM_RESERVED_ID_TIMESTAMP;
-	_qualifiers[0].value = timestamp;
-	_qualifiers[0].maxDelta = timestampMaxDelta;
-	_qualifiers[1].id = COM_RESERVED_ID_NETWORK_ID;
-	_qualifiers[1].value = nwid;
-	_qualifiers[1].maxDelta = 0;
-	_qualifiers[2].id = COM_RESERVED_ID_ISSUED_TO;
-	_qualifiers[2].value = issuedTo.address().toInt();
-	_qualifiers[2].maxDelta = 0xffffffffffffffffULL;
-
-	// Include hash of full identity public key in COM for hardening purposes. Pack it in
-	// using the original COM format. Format may be revised in the future to make this cleaner.
-	uint64_t idHash[6];
-	issuedTo.publicKeyHash(idHash);
-	for(unsigned long i=0;i<4;++i) {
-		_qualifiers[i + 3].id = (uint64_t)(i + 3);
-		_qualifiers[i + 3].value = Utils::ntoh(idHash[i]);
-		_qualifiers[i + 3].maxDelta = 0xffffffffffffffffULL;
+	_signedBy.zero();
+
+	for(unsigned int i=0;i<_qualifierCount;++i) {
+		if (_qualifiers[i].id == id) {
+			_qualifiers[i].value = value;
+			_qualifiers[i].maxDelta = maxDelta;
+			return;
+		}
+	}
+
+	if (_qualifierCount < ZT_NETWORK_COM_MAX_QUALIFIERS) {
+		_qualifiers[_qualifierCount].id = id;
+		_qualifiers[_qualifierCount].value = value;
+		_qualifiers[_qualifierCount].maxDelta = maxDelta;
+		++_qualifierCount;
+		std::sort(&(_qualifiers[0]),&(_qualifiers[_qualifierCount]));
+	}
+}
+
+#ifdef ZT_SUPPORT_OLD_STYLE_NETCONF
+
+std::string CertificateOfMembership::toString() const
+{
+	char tmp[ZT_NETWORK_COM_MAX_QUALIFIERS * 32];
+	std::string s;
+
+	s.append("1:"); // COM_UINT64_ED25519
+
+	uint64_t *const buf = new uint64_t[_qualifierCount * 3];
+	try {
+		unsigned int ptr = 0;
+		for(unsigned int i=0;i<_qualifierCount;++i) {
+			buf[ptr++] = Utils::hton(_qualifiers[i].id);
+			buf[ptr++] = Utils::hton(_qualifiers[i].value);
+			buf[ptr++] = Utils::hton(_qualifiers[i].maxDelta);
+		}
+		s.append(Utils::hex(buf,ptr * sizeof(uint64_t),tmp));
+		delete [] buf;
+	} catch ( ... ) {
+		delete [] buf;
+		throw;
+	}
+
+	s.push_back(':');
+
+	s.append(_signedBy.toString(tmp));
+
+	if (_signedBy) {
+		s.push_back(':');
+		s.append(Utils::hex(_signature.data,ZT_C25519_SIGNATURE_LEN,tmp));
 	}
 
-	_qualifierCount = 7;
+	return s;
+}
+
+void CertificateOfMembership::fromString(const char *s)
+{
+	_qualifierCount = 0;
+	_signedBy.zero();
 	memset(_signature.data,0,ZT_C25519_SIGNATURE_LEN);
+
+	if (!*s)
+		return;
+
+	unsigned int colonAt = 0;
+	while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
+
+	if (!((colonAt == 1)&&(s[0] == '1'))) // COM_UINT64_ED25519?
+		return;
+
+	s += colonAt + 1;
+	colonAt = 0;
+	while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
+
+	if (colonAt) {
+		const unsigned int buflen = colonAt / 2;
+		char *const buf = new char[buflen];
+		unsigned int bufactual = Utils::unhex(s,colonAt,buf,buflen);
+		char *bufptr = buf;
+		try {
+			while (bufactual >= 24) {
+				if (_qualifierCount < ZT_NETWORK_COM_MAX_QUALIFIERS) {
+					_qualifiers[_qualifierCount].id = Utils::ntoh(*((uint64_t *)bufptr)); bufptr += 8;
+					_qualifiers[_qualifierCount].value = Utils::ntoh(*((uint64_t *)bufptr)); bufptr += 8;
+					_qualifiers[_qualifierCount].maxDelta = Utils::ntoh(*((uint64_t *)bufptr)); bufptr += 8;
+					++_qualifierCount;
+				} else {
+					bufptr += 24;
+				}
+				bufactual -= 24;
+			}
+		} catch ( ... ) {}
+		delete [] buf;
+	}
+
+	if (s[colonAt]) {
+		s += colonAt + 1;
+		colonAt = 0;
+		while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
+
+		if (colonAt) {
+			char addrbuf[ZT_ADDRESS_LENGTH];
+			if (Utils::unhex(s,colonAt,addrbuf,sizeof(addrbuf)) == ZT_ADDRESS_LENGTH)
+				_signedBy.setTo(addrbuf,ZT_ADDRESS_LENGTH);
+
+			if ((_signedBy)&&(s[colonAt])) {
+				s += colonAt + 1;
+				colonAt = 0;
+				while ((s[colonAt])&&(s[colonAt] != ':')) ++colonAt;
+				if (colonAt) {
+					if (Utils::unhex(s,colonAt,_signature.data,ZT_C25519_SIGNATURE_LEN) != ZT_C25519_SIGNATURE_LEN)
+						_signedBy.zero();
+				} else {
+					_signedBy.zero();
+				}
+			} else {
+				_signedBy.zero();
+			}
+		}
+	}
+
+	std::sort(&(_qualifiers[0]),&(_qualifiers[_qualifierCount]));
 }
 
-bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other, const Identity &otherIdentity) const
+#endif // ZT_SUPPORT_OLD_STYLE_NETCONF
+
+bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other) const
 {
+	unsigned int myidx = 0;
+	unsigned int otheridx = 0;
+
 	if ((_qualifierCount == 0)||(other._qualifierCount == 0))
 		return false;
 
-	std::map< uint64_t, uint64_t > otherFields;
-	for(unsigned int i=0;i<other._qualifierCount;++i)
-		otherFields[other._qualifiers[i].id] = other._qualifiers[i].value;
-
-	bool fullIdentityVerification = false;
-	for(unsigned int i=0;i<_qualifierCount;++i) {
-		const uint64_t qid = _qualifiers[i].id;
-		if ((qid >= 3)&&(qid <= 6))
-			fullIdentityVerification = true;
-		std::map< uint64_t, uint64_t >::iterator otherQ(otherFields.find(qid));
-		if (otherQ == otherFields.end())
+	while (myidx < _qualifierCount) {
+		// Fail if we're at the end of other, since this means the field is
+		// missing.
+		if (otheridx >= other._qualifierCount)
 			return false;
-		const uint64_t a = _qualifiers[i].value;
-		const uint64_t b = otherQ->second;
-		if (((a >= b) ? (a - b) : (b - a)) > _qualifiers[i].maxDelta)
-			return false;
-	}
 
-	// If this COM has a full hash of its identity, assume the other must have this as well.
-	// Otherwise we are on a controller that does not incorporate these.
-	if (fullIdentityVerification) {
-		uint64_t idHash[6];
-		otherIdentity.publicKeyHash(idHash);
-		for(unsigned long i=0;i<4;++i) {
-			std::map< uint64_t, uint64_t >::iterator otherQ(otherFields.find((uint64_t)(i + 3)));
-			if (otherQ == otherFields.end())
-				return false;
-			if (otherQ->second != Utils::ntoh(idHash[i]))
+		// Seek to corresponding tuple in other, ignoring tuples that
+		// we may not have. If we run off the end of other, the tuple is
+		// missing. This works because tuples are sorted by ID.
+		while (other._qualifiers[otheridx].id != _qualifiers[myidx].id) {
+			++otheridx;
+			if (otheridx >= other._qualifierCount)
 				return false;
 		}
+
+		// Compare to determine if the absolute value of the difference
+		// between these two parameters is within our maxDelta.
+		const uint64_t a = _qualifiers[myidx].value;
+		const uint64_t b = other._qualifiers[myidx].value;
+		if (((a >= b) ? (a - b) : (b - a)) > _qualifiers[myidx].maxDelta)
+			return false;
+
+		++myidx;
 	}
 
 	return true;

node/CertificateOfMembership.hpp

@@ -94,8 +94,6 @@ public:
 		 * ZeroTier address to whom certificate was issued
 		 */
 		COM_RESERVED_ID_ISSUED_TO = 2
-
-		// IDs 3-6 reserved for full hash of identity to which this COM was issued.
 	};
 
 	/**
@@ -112,7 +110,20 @@ public:
 	 * @param nwid Network ID
 	 * @param issuedTo Certificate recipient
 	 */
-	CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Identity &issuedTo);
+	CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Address &issuedTo)
+	{
+		_qualifiers[0].id = COM_RESERVED_ID_TIMESTAMP;
+		_qualifiers[0].value = timestamp;
+		_qualifiers[0].maxDelta = timestampMaxDelta;
+		_qualifiers[1].id = COM_RESERVED_ID_NETWORK_ID;
+		_qualifiers[1].value = nwid;
+		_qualifiers[1].maxDelta = 0;
+		_qualifiers[2].id = COM_RESERVED_ID_ISSUED_TO;
+		_qualifiers[2].value = issuedTo.toInt();
+		_qualifiers[2].maxDelta = 0xffffffffffffffffULL;
+		_qualifierCount = 3;
+		memset(_signature.data,0,ZT_C25519_SIGNATURE_LEN);
+	}
 
 	/**
 	 * Create from binary-serialized COM in buffer
@@ -172,6 +183,36 @@ public:
 		return 0ULL;
 	}
 
+	/**
+	 * Add or update a qualifier in this certificate
+	 *
+	 * Any signature is invalidated and signedBy is set to null.
+	 *
+	 * @param id Qualifier ID
+	 * @param value Qualifier value
+	 * @param maxDelta Qualifier maximum allowed difference (absolute value of difference)
+	 */
+	void setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta);
+	inline void setQualifier(ReservedId id,uint64_t value,uint64_t maxDelta) { setQualifier((uint64_t)id,value,maxDelta); }
+
+#ifdef ZT_SUPPORT_OLD_STYLE_NETCONF
+	/**
+	 * @return String-serialized representation of this certificate
+	 */
+	std::string toString() const;
+
+	/**
+	 * Set this certificate equal to the hex-serialized string
+	 *
+	 * Invalid strings will result in invalid or undefined certificate
+	 * contents. These will subsequently fail validation and comparison.
+	 * Empty strings will result in an empty certificate.
+	 *
+	 * @param s String to deserialize
+	 */
+	void fromString(const char *s);
+#endif // ZT_SUPPORT_OLD_STYLE_NETCONF
+
 	/**
 	 * Compare two certificates for parameter agreement
 	 *
@@ -183,10 +224,9 @@ public:
 	 * tuples present in this cert but not in other result in 'false'.
 	 *
 	 * @param other Cert to compare with
-	 * @param otherIdentity Identity of other node
 	 * @return True if certs agree and 'other' may be communicated with
 	 */
-	bool agreesWith(const CertificateOfMembership &other, const Identity &otherIdentity) const;
+	bool agreesWith(const CertificateOfMembership &other) const;
 
 	/**
 	 * Sign this certificate

node/Constants.hpp

@@ -361,7 +361,7 @@
 /**
  * Maximum number of outgoing packets we monitor for QoS information
  */
-#define ZT_QOS_MAX_OUTSTANDING_RECORDS (1024 * 16)
+#define ZT_QOS_MAX_OUTSTANDING_RECORDS (1024*16)
 
 /**
  * Interval used for rate-limiting the computation of path quality estimates.
@@ -403,11 +403,117 @@
 
 /**
  * All unspecified traffic is put in this bucket. Anything in a bucket with a
- * smaller value is de-prioritized. Anything in a bucket with a higher value is
+ * smaller value is deprioritized. Anything in a bucket with a higher value is
  prioritized over other traffic.
  */
 #define ZT_AQM_DEFAULT_BUCKET 0
 
+/**
+ * How often we emit a one-liner bond summary for each peer
+ */
+#define ZT_MULTIPATH_BOND_STATUS_INTERVAL 60000
+
+/**
+ * How long before we consider a path to be dead in the general sense. This is
+ * used while searching for default or alternative paths to try in the absence
+ * of direct guidance from the user or a selection policy.
+ */
+#define ZT_MULTIPATH_DEFAULT_FAILOVER_INTERVAL 10000
+
+/**
+ * How often flows are evaluated
+ */
+#define ZT_MULTIPATH_FLOW_CHECK_INTERVAL 10000
+
+/**
+ * How long before we consider a flow to be dead and remove it from the
+ * policy's list.
+ */
+#define ZT_MULTIPATH_FLOW_EXPIRATION_INTERVAL (60000 * 5)
+
+/**
+ * How often a flow's statistical counters are reset
+ */
+#define ZT_FLOW_STATS_RESET_INTERVAL ZT_MULTIPATH_FLOW_EXPIRATION_INTERVAL
+
+/**
+ * Maximum number of flows allowed before we start forcibly forgetting old ones
+ */
+#define ZT_FLOW_MAX_COUNT (1024*64)
+
+/**
+ * How often flows are rebalanced across link (if at all)
+ */
+#define ZT_FLOW_MIN_REBALANCE_INTERVAL 5000
+
+/**
+ * How often flows are rebalanced across link (if at all)
+ */
+#define ZT_FLOW_REBALANCE_INTERVAL 5000
+
+/**
+ * A defensive timer to prevent path quality metrics from being
+ * processed too often.
+ */
+#define ZT_BOND_BACKGROUND_TASK_MIN_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
+
+/**
+ * How often a bonding policy's background tasks are processed,
+ * some need more frequent attention than others.
+ */
+#define ZT_MULTIPATH_ACTIVE_BACKUP_CHECK_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
+
+/**
+ * Minimum amount of time (since a previous transition) before the active-backup bonding
+ * policy is allowed to transition to a different link. Only valid for active-backup.
+ */
+#define ZT_MULTIPATH_MIN_ACTIVE_BACKUP_AUTOFLOP_INTERVAL 10000
+
+/**
+ * How often a peer checks that incoming (and outgoing) traffic on a bonded link is
+ * appropriately paired.
+ */
+#define ZT_PATH_NEGOTIATION_CHECK_INTERVAL 15000
+
+/**
+ * Time horizon for path negotiation paths cutoff
+ */
+#define ZT_PATH_NEGOTIATION_CUTOFF_TIME 60000
+
+/**
+ * Maximum number of path negotiations within cutoff time
+ *
+ * This limits response to PATH_NEGOTIATION to CUTOFF_LIMIT responses
+ * per CUTOFF_TIME milliseconds per peer to prevent this from being
+ * useful for DOS amplification attacks.
+ */
+#define ZT_PATH_NEGOTIATION_CUTOFF_LIMIT 8
+
+/**
+ * How many times a peer will attempt to petition another peer to synchronize its
+ * traffic to the same path before giving up and surrendering to the other peer's preference.
+ */
+#define ZT_PATH_NEGOTIATION_TRY_COUNT 3
+
+/**
+ * How much greater the quality of a path should be before an
+ * optimization procedure triggers a switch.
+ */
+#define ZT_MULTIPATH_ACTIVE_BACKUP_OPTIMIZE_MIN_THRESHOLD 0.10
+
+/**
+ * Artificially inflates the failover score for paths which meet
+ * certain non-performance-related policy ranking criteria.
+ */
+#define ZT_MULTIPATH_FAILOVER_HANDICAP_PREFERRED 500
+#define ZT_MULTIPATH_FAILOVER_HANDICAP_PRIMARY 1000
+#define ZT_MULTIPATH_FAILOVER_HANDICAP_NEGOTIATED 5000
+
+/**
+ * An indicator that no flow is to be associated with the given packet
+ */
+#define ZT_QOS_NO_FLOW -1
+
 /**
  * Timeout for overall peer activity (measured from last receive)
  */
@@ -498,8 +604,8 @@
 #define ZT_ACK_CUTOFF_LIMIT 128
 #define ZT_ACK_DRAINAGE_DIVISOR (1000 / ZT_ACK_CUTOFF_LIMIT)
 
-#define ZT_BOND_DEFAULT_REFRCTORY_PERIOD 8000
-#define ZT_BOND_MAX_REFRACTORY_PERIOD 600000
+#define ZT_MULTIPATH_DEFAULT_REFRCTORY_PERIOD 8000
+#define ZT_MULTIPATH_MAX_REFRACTORY_PERIOD 600000
 
 /**
  * Maximum number of direct path pushes within cutoff time
@@ -535,92 +641,6 @@
  */
 #define ZT_PEER_GENERAL_RATE_LIMIT 1000
 
-
-/**
- * Minimum allowed amount of time between flow/path optimizations (anti-flapping)
- */
-#define ZT_BOND_OPTIMIZE_INTERVAL 15000
-
-/**
- * Maximum number of flows allowed before we start forcibly forgetting old ones
- */
-#define ZT_FLOW_MAX_COUNT (1024 * 64)
-
-/**
- * How often we emit a bond summary for each bond
- */
-#define ZT_BOND_STATUS_INTERVAL 30000
-
-/**
- * How long before we consider a path to be dead in the general sense. This is
- * used while searching for default or alternative paths to try in the absence
- * of direct guidance from the user or a selection policy.
- */
-#define ZT_BOND_FAILOVER_DEFAULT_INTERVAL 5000
-
-/**
- * Anything below this value gets into thrashing territory since we divide
- * this value by ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL to send ECHOs often.
- */
-#define ZT_BOND_FAILOVER_MIN_INTERVAL 250
-
-/**
- * How many times per failover interval that an ECHO is sent. This should be
- * at least 2. Anything more then 4 starts to increase overhead significantly.
- */
-#define ZT_BOND_ECHOS_PER_FAILOVER_INTERVAL 4
-
-/**
- * A defensive timer to prevent path quality metrics from being
- * processed too often.
- */
-#define ZT_BOND_BACKGROUND_TASK_MIN_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
-
-/**
- * How often a bonding policy's background tasks are processed,
- * some need more frequent attention than others.
- */
-#define ZT_BOND_ACTIVE_BACKUP_CHECK_INTERVAL ZT_CORE_TIMER_TASK_GRANULARITY
-
-/**
- * Time horizon for path negotiation paths cutoff
- */
-#define ZT_PATH_NEGOTIATION_CUTOFF_TIME 60000
-
-/**
- * Maximum number of path negotiations within cutoff time
- *
- * This limits response to PATH_NEGOTIATION to CUTOFF_LIMIT responses
- * per CUTOFF_TIME milliseconds per peer to prevent this from being
- * useful for DOS amplification attacks.
- */
-#define ZT_PATH_NEGOTIATION_CUTOFF_LIMIT 8
-
-/**
- * How many times a peer will attempt to petition another peer to synchronize its
- * traffic to the same path before giving up and surrendering to the other peer's preference.
- */
-#define ZT_PATH_NEGOTIATION_TRY_COUNT 3
-
-/**
- * How much greater the quality of a path should be before an
- * optimization procedure triggers a switch.
- */
-#define ZT_BOND_ACTIVE_BACKUP_OPTIMIZE_MIN_THRESHOLD 0.10
-
-/**
- * Artificially inflates the failover score for paths which meet
- * certain non-performance-related policy ranking criteria.
- */
-#define ZT_BOND_FAILOVER_HANDICAP_PREFERRED  500
-#define ZT_BOND_FAILOVER_HANDICAP_PRIMARY    1000
-#define ZT_BOND_FAILOVER_HANDICAP_NEGOTIATED 5000
-
-/**
- * An indicator that no flow is to be associated with the given packet
- */
-#define ZT_QOS_NO_FLOW -1
-
 /**
  * Don't do expensive identity validation more often than this
  *
@@ -646,6 +666,11 @@
  */
 #define ZT_TRUST_EXPIRATION 600000
 
+/**
+ * Enable support for older network configurations from older (pre-1.1.6) controllers
+ */
+#define ZT_SUPPORT_OLD_STYLE_NETCONF 1
+
 /**
  * Desired buffer size for UDP sockets (used in service and osdep but defined here)
  */

node/Flow.hpp

@@ -0,0 +1,124 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2025-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#ifndef ZT_FLOW_HPP
+#define ZT_FLOW_HPP
+
+#include "Path.hpp"
+#include "SharedPtr.hpp"
+
+namespace ZeroTier {
+
+/**
+ * A protocol flow that is identified by the origin and destination port.
+ */
+struct Flow
+{
+	/**
+	 * @param flowId Given flow ID
+	 * @param now Current time
+	 */
+	Flow(int32_t flowId, int64_t now) :
+		_flowId(flowId),
+		_bytesInPerUnitTime(0),
+		_bytesOutPerUnitTime(0),
+		_lastActivity(now),
+		_lastPathReassignment(0),
+		_assignedPath(SharedPtr<Path>())
+	{}
+
+	/**
+	 * Reset flow statistics
+	 */
+	void resetByteCounts()
+	{
+		_bytesInPerUnitTime = 0;
+		_bytesOutPerUnitTime = 0;
+	}
+
+	/**
+	 * @return The Flow's ID
+	 */
+	int32_t id() { return _flowId; }
+
+	/**
+	 * @return Number of incoming bytes processed on this flow per unit time
+	 */
+	int64_t bytesInPerUnitTime() { return _bytesInPerUnitTime; }
+
+	/**
+	 * Record number of incoming bytes on this flow
+	 *
+	 * @param bytes Number of incoming bytes
+	 */
+	void recordIncomingBytes(uint64_t bytes) { _bytesInPerUnitTime += bytes; }
+
+	/**
+	 * @return Number of outgoing bytes processed on this flow per unit time
+	 */
+	int64_t bytesOutPerUnitTime() { return _bytesOutPerUnitTime; }
+
+	/**
+	 * Record number of outgoing bytes on this flow
+	 *
+	 * @param bytes
+	 */
+	void recordOutgoingBytes(uint64_t bytes) { _bytesOutPerUnitTime += bytes; }
+
+	/**
+	 * @return The total number of bytes processed on this flow
+	 */
+	uint64_t totalBytes() { return _bytesInPerUnitTime + _bytesOutPerUnitTime; }
+
+	/**
+	 * How long since a packet was sent or received in this flow
+	 *
+	 * @param now Current time
+	 * @return The age of the flow in terms of last recorded activity
+	 */
+	int64_t age(int64_t now) { return now - _lastActivity; }
+
+	/**
+	 * Record that traffic was processed on this flow at the given time.
+	 *
+	 * @param now Current time
+	 */
+	void updateActivity(int64_t now) { _lastActivity = now; }
+
+	/**
+	 * @return Path assigned to this flow
+	 */
+	SharedPtr<Path> assignedPath() { return _assignedPath; }
+
+	/**
+	 * @param path Assigned path over which this flow should be handled
+	 */
+	void assignPath(const SharedPtr<Path> &path, int64_t now) {
+		_assignedPath = path;
+		_lastPathReassignment = now;
+	}
+
+	AtomicCounter __refCount;
+
+	int32_t _flowId;
+	uint64_t _bytesInPerUnitTime;
+	uint64_t _bytesOutPerUnitTime;
+	int64_t _lastActivity;
+	int64_t _lastPathReassignment;
+	SharedPtr<Path> _assignedPath;
+	SharedPtr<Path> _previouslyAssignedPath;
+};
+
+} // namespace ZeroTier
+
+#endif

node/Identity.hpp

@@ -109,18 +109,6 @@ public:
 	 */
 	inline bool hasPrivate() const { return (_privateKey != (C25519::Private *)0); }
 
-	/**
-	 * Compute a SHA384 hash of this identity's address and public key(s).
-	 * 
-	 * @param sha384buf Buffer with 48 bytes of space to receive hash
-	 */
-	inline void publicKeyHash(void *sha384buf) const
-	{
-		uint8_t address[ZT_ADDRESS_LENGTH];
-		_address.copyTo(address, ZT_ADDRESS_LENGTH);
-		SHA384(sha384buf, address, ZT_ADDRESS_LENGTH, _publicKey.data, ZT_C25519_PUBLIC_KEY_LEN);
-	}
-
 	/**
 	 * Compute the SHA512 hash of our private key (if we have one)
 	 *

node/IncomingPacket.cpp

@@ -88,6 +88,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,void *tPtr,int32_t f
 					peer->received(tPtr,_path,hops(),packetId(),payloadLength(),v,0,Packet::VERB_NOP,false,0,ZT_QOS_NO_FLOW);
 					break;
 				case Packet::VERB_HELLO:                      r = _doHELLO(RR,tPtr,true); break;
+				case Packet::VERB_ACK:                        r = _doACK(RR,tPtr,peer); break;
 				case Packet::VERB_QOS_MEASUREMENT:            r = _doQOS_MEASUREMENT(RR,tPtr,peer); break;
 				case Packet::VERB_ERROR:                      r = _doERROR(RR,tPtr,peer); break;
 				case Packet::VERB_OK:                         r = _doOK(RR,tPtr,peer); break;
@@ -190,29 +191,6 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 			}
 		}	break;
 
-		case Packet::ERROR_NETWORK_AUTHENTICATION_REQUIRED: {
-			const SharedPtr<Network> network(RR->node->network(at<uint64_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD)));
-			if ((network)&&(network->controller() == peer->address())) {
-				bool noUrl = true;
-				int s = (int)size() - (ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8);
-				if (s > 2) {
-					const uint16_t errorDataSize = at<uint16_t>(ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 8);
-					s -= 2;
-					if (s >= (int)errorDataSize) {
-						Dictionary<3072> authInfo(((const char *)this->data()) + (ZT_PROTO_VERB_ERROR_IDX_PAYLOAD + 10), errorDataSize);
-						char authenticationURL[2048];
-						if (authInfo.get("aU", authenticationURL, sizeof(authenticationURL)) > 0) {
-							authenticationURL[sizeof(authenticationURL) - 1] = 0; // ensure always zero terminated
-							network->setAuthenticationRequired(authenticationURL);
-							noUrl = false;
-						}
-					}
-				}
-				if (noUrl)
-					network->setAuthenticationRequired("");
-			}
-		}	break;
-
 		default: break;
 	}
 
@@ -221,12 +199,35 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,void *tPtr,const Shar
 	return true;
 }
 
+bool IncomingPacket::_doACK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
+{
+	SharedPtr<Bond> bond = peer->bond();
+	if (!bond || !bond->rateGateACK(RR->node->now())) {
+		return true;
+	}
+	/* Dissect incoming ACK packet. From this we can estimate current throughput of the path, establish known
+	 * maximums and detect packet loss. */
+	int32_t ackedBytes;
+	if (payloadLength() != sizeof(ackedBytes)) {
+		return true; // ignore
+	}
+	memcpy(&ackedBytes, payload(), sizeof(ackedBytes));
+	if (bond) {
+		bond->receivedAck(_path, RR->node->now(), Utils::ntoh(ackedBytes));
+	}
+	return true;
+}
+
 bool IncomingPacket::_doQOS_MEASUREMENT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer)
 {
 	SharedPtr<Bond> bond = peer->bond();
-	if (!bond || !bond->rateGateQoS(RR->node->now(), _path)) {
+	/* TODO: Fix rate gate issue
+	if (!bond || !bond->rateGateQoS(RR->node->now())) {
 		return true;
 	}
+	*/
+	/* Dissect incoming QoS packet. From this we can compute latency values and their variance.
+	 * The latency variance is used as a measure of "jitter". */
 	if (payloadLength() > ZT_QOS_MAX_PACKET_SIZE || payloadLength() < ZT_QOS_MIN_PACKET_SIZE) {
 		return true; // ignore
 	}
@@ -1305,7 +1306,7 @@ bool IncomingPacket::_doPATH_NEGOTIATION_REQUEST(const RuntimeEnvironment *RR,vo
 {
 	uint64_t now = RR->node->now();
 	SharedPtr<Bond> bond = peer->bond();
-	if (!bond || !bond->rateGatePathNegotiation(now, _path)) {
+	if (!bond || !bond->rateGatePathNegotiation(now)) {
 		return true;
 	}
 	if (payloadLength() != sizeof(int16_t)) {

node/IncomingPacket.hpp

@@ -112,6 +112,7 @@ private:
 	// been authenticated, decrypted, decompressed, and classified.
 	bool _doERROR(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doHELLO(const RuntimeEnvironment *RR,void *tPtr,const bool alreadyAuthenticated);
+	bool _doACK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doQOS_MEASUREMENT(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doOK(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);
 	bool _doWHOIS(const RuntimeEnvironment *RR,void *tPtr,const SharedPtr<Peer> &peer);

node/Membership.hpp

@@ -91,14 +91,13 @@ public:
 	 * Check whether the peer represented by this Membership should be allowed on this network at all
 	 *
 	 * @param nconf Our network config
-	 * @param otherNodeIdentity Identity of remote node
 	 * @return True if this peer is allowed on this network at all
 	 */
-	inline bool isAllowedOnNetwork(const NetworkConfig &thisNodeNetworkConfig, const Identity &otherNodeIdentity) const
+	inline bool isAllowedOnNetwork(const NetworkConfig &nconf) const
 	{
-		if (thisNodeNetworkConfig.isPublic()) return true;
+		if (nconf.isPublic()) return true;
 		if (_com.timestamp() <= _comRevocationThreshold) return false;
-		return thisNodeNetworkConfig.com.agreesWith(_com, otherNodeIdentity);
+		return nconf.com.agreesWith(_com);
 	}
 
 	inline bool recentlyAssociated(const int64_t now) const

node/Network.cpp

@@ -1022,7 +1022,6 @@ int Network::setConfiguration(void *tPtr,const NetworkConfig &nconf,bool saveToD
 		}
 
 		_portError = RR->node->configureVirtualNetworkPort(tPtr,_id,&_uPtr,(oldPortInitialized) ? ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_CONFIG_UPDATE : ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_UP,&ctmp);
-		_authenticationURL = nconf.authenticationURL;
 
 		if (saveToDisk) {
 			Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> *const d = new Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY>();
@@ -1227,7 +1226,7 @@ bool Network::gate(void *tPtr,const SharedPtr<Peer> &peer)
 	try {
 		if (_config) {
 			Membership *m = _memberships.get(peer->address());
-			if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config, peer->identity()))) ) {
+			if ( (_config.isPublic()) || ((m)&&(m->isAllowedOnNetwork(_config))) ) {
 				if (!m)
 					m = &(_membership(peer->address()));
 				if (m->multicastLikeGate(now)) {
@@ -1380,8 +1379,6 @@ ZT_VirtualNetworkStatus Network::_status() const
 			return ZT_NETWORK_STATUS_NOT_FOUND;
 		case NETCONF_FAILURE_NONE:
 			return ((_config) ? ZT_NETWORK_STATUS_OK : ZT_NETWORK_STATUS_REQUESTING_CONFIGURATION);
-		case NETCONF_FAILURE_AUTHENTICATION_REQUIRED:
-			return ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED;
 		default:
 			return ZT_NETWORK_STATUS_PORT_ERROR;
 	}
@@ -1432,10 +1429,6 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	}
 
 	memcpy(&ec->dns, &_config.dns, sizeof(ZT_VirtualNetworkDNS));
-
-	Utils::scopy(ec->authenticationURL, sizeof(ec->authenticationURL), _authenticationURL.c_str());
-	ec->authenticationExpiryTime = _config.authenticationExpiryTime;
-	ec->ssoEnabled = _config.ssoEnabled;
 }
 
 void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup)
@@ -1487,11 +1480,8 @@ void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMu
 		Membership *m = (Membership *)0;
 		Hashtable<Address,Membership>::Iterator i(_memberships);
 		while (i.next(a,m)) {
-			const Identity remoteIdentity(RR->topology->getIdentity(tPtr, *a));
-			if (remoteIdentity) {
-				if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config, remoteIdentity)) && (!std::binary_search(alwaysAnnounceTo.begin(),alwaysAnnounceTo.end(),*a)) )
-					_announceMulticastGroupsTo(tPtr,*a,groups);
-			}
+			if ( ( m->multicastLikeGate(now) || (newMulticastGroup) ) && (m->isAllowedOnNetwork(_config)) && (!std::binary_search(alwaysAnnounceTo.begin(),alwaysAnnounceTo.end(),*a)) )
+				_announceMulticastGroupsTo(tPtr,*a,groups);
 		}
 	}
 }

node/Network.hpp

@@ -220,17 +220,6 @@ public:
 		_netconfFailure = NETCONF_FAILURE_NOT_FOUND;
 	}
 
-	/**
-	 * Set netconf failure to 'authentication required' possibly with an authorization URL
-	 */
-	inline void setAuthenticationRequired(const char *url)
-	{
-		Mutex::Lock _l(_lock);
-		_netconfFailure = NETCONF_FAILURE_AUTHENTICATION_REQUIRED;
-		_authenticationURL = (url) ? url : "";
-		_config.ssoEnabled = true;
-	}	
-
 	/**
 	 * Causes this network to request an updated configuration from its master node now
 	 *
@@ -446,11 +435,9 @@ private:
 		NETCONF_FAILURE_NONE,
 		NETCONF_FAILURE_ACCESS_DENIED,
 		NETCONF_FAILURE_NOT_FOUND,
-		NETCONF_FAILURE_INIT_FAILED,
-		NETCONF_FAILURE_AUTHENTICATION_REQUIRED
+		NETCONF_FAILURE_INIT_FAILED
 	} _netconfFailure;
 	int _portError; // return value from port config callback
-	std::string _authenticationURL;
 
 	Hashtable<Address,Membership> _memberships;
 

node/NetworkConfig.cpp

@@ -182,14 +182,6 @@ bool NetworkConfig::toDictionary(Dictionary<ZT_NETWORKCONFIG_DICT_CAPACITY> &d,b
 			if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_DNS,*tmp)) return false;
 		}
 
-		if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_SSO_ENABLED, this->ssoEnabled)) return false;
-		if (this->ssoEnabled) {
-			if (this->authenticationURL[0]) {
-				if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL, this->authenticationURL)) return false;
-			}
-			if (!d.add(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_EXPIRY_TIME, this->authenticationExpiryTime)) return false;
-		}
-
 		delete tmp;
 	} catch ( ... ) {
 		delete tmp;
@@ -373,20 +365,6 @@ bool NetworkConfig::fromDictionary(const Dictionary<ZT_NETWORKCONFIG_DICT_CAPACI
 				unsigned int p = 0;
 				DNS::deserializeDNS(*tmp, p, &dns);
 			}
-
-
-			this->ssoEnabled = d.getB(ZT_NETWORKCONFIG_DICT_KEY_SSO_ENABLED, false);
-			if (this->ssoEnabled) {
-				if (d.get(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL, this->authenticationURL, (unsigned int)sizeof(this->authenticationURL)) > 0) {
-					this->authenticationURL[sizeof(this->authenticationURL) - 1] = 0; // ensure null terminated
-				} else {
-					this->authenticationURL[0] = 0;
-				}
-				this->authenticationExpiryTime = d.getI(ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_EXPIRY_TIME, 0);
-			} else {
-				this->authenticationURL[0] = 0;
-				this->authenticationExpiryTime = 0;
-			}
 		}
 
 		//printf("~~~\n%s\n~~~\n",d.data());

node/NetworkConfig.hpp

@@ -94,7 +94,7 @@
 namespace ZeroTier {
 
 // Dictionary capacity needed for max size network config
-#define ZT_NETWORKCONFIG_DICT_CAPACITY (4096 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
+#define ZT_NETWORKCONFIG_DICT_CAPACITY (1024 + (sizeof(ZT_VirtualNetworkRule) * ZT_MAX_NETWORK_RULES) + (sizeof(Capability) * ZT_MAX_NETWORK_CAPABILITIES) + (sizeof(Tag) * ZT_MAX_NETWORK_TAGS) + (sizeof(CertificateOfOwnership) * ZT_MAX_CERTIFICATES_OF_OWNERSHIP))
 
 // Dictionary capacity needed for max size network meta-data
 #define ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY 1024
@@ -178,12 +178,6 @@ namespace ZeroTier {
 #define ZT_NETWORKCONFIG_DICT_KEY_CERTIFICATES_OF_OWNERSHIP "COO"
 // dns (binary blobs)
 #define ZT_NETWORKCONFIG_DICT_KEY_DNS "DNS"
-// sso enabld
-#define ZT_NETWORKCONFIG_DICT_KEY_SSO_ENABLED "ssoe"
-// authentication URL
-#define ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_URL "aurl"
-// authentication expiry
-#define ZT_NETWORKCONFIG_DICT_KEY_AUTHENTICATION_EXPIRY_TIME "aexpt"
 
 // Legacy fields -- these are obsoleted but are included when older clients query
 
@@ -239,10 +233,7 @@ public:
 		tags(),
 		certificatesOfOwnership(),
 		type(ZT_NETWORK_TYPE_PRIVATE),
-		dnsCount(0),
-		ssoEnabled(false),
-		authenticationURL(),
-		authenticationExpiryTime(0)
+		dnsCount(0)
 	{
 		name[0] = 0;
 		memset(specialists, 0, sizeof(uint64_t)*ZT_MAX_NETWORK_SPECIALISTS);
@@ -613,21 +604,6 @@ public:
 	 * ZT pushed DNS configuration
 	 */
 	ZT_VirtualNetworkDNS dns;
-
-	/**
-	 * SSO enabled flag.
-	 */
-	bool ssoEnabled;
-
-	/**
-	 * Authentication URL if authentication is required
-	 */
-	char authenticationURL[2048];
-
-	/**
-	 * Time current authentication expires or 0 if external authentication is disabled
-	 */
-	uint64_t authenticationExpiryTime;
 };
 
 } // namespace ZeroTier

node/NetworkController.hpp

@@ -38,8 +38,7 @@ public:
 		NC_ERROR_NONE = 0,
 		NC_ERROR_OBJECT_NOT_FOUND = 1,
 		NC_ERROR_ACCESS_DENIED = 2,
-		NC_ERROR_INTERNAL_SERVER_ERROR = 3,
-		NC_ERROR_AUTHENTICATION_REQUIRED = 4
+		NC_ERROR_INTERNAL_SERVER_ERROR = 3
 	};
 
 	/**
@@ -70,17 +69,12 @@ public:
 		/**
 		 * Send a network configuration request error
 		 *
-		 * If errorData/errorDataSize are provided they must point to a valid serialized
-		 * Dictionary containing error data. They can be null/zero if not specified.
-		 * 
 		 * @param nwid Network ID
 		 * @param requestPacketId Request packet ID or 0 if none
 		 * @param destination Destination peer Address
 		 * @param errorCode Error code
-		 * @param errorData Data associated with error or NULL if none
-		 * @param errorDataSize Size of errorData in bytes
 		 */
-		virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode, const void *errorData, unsigned int errorDataSize) = 0;
+		virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode) = 0;
 	};
 
 	NetworkController() {}

node/Node.cpp

@@ -103,7 +103,7 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		const unsigned long mcs = sizeof(Multicaster) + (((sizeof(Multicaster) & 0xf) != 0) ? (16 - (sizeof(Multicaster) & 0xf)) : 0);
 		const unsigned long topologys = sizeof(Topology) + (((sizeof(Topology) & 0xf) != 0) ? (16 - (sizeof(Topology) & 0xf)) : 0);
 		const unsigned long sas = sizeof(SelfAwareness) + (((sizeof(SelfAwareness) & 0xf) != 0) ? (16 - (sizeof(SelfAwareness) & 0xf)) : 0);
-		const unsigned long bc = sizeof(Bond) + (((sizeof(Bond) & 0xf) != 0) ? (16 - (sizeof(Bond) & 0xf)) : 0);
+		const unsigned long bc = sizeof(BondController) + (((sizeof(BondController) & 0xf) != 0) ? (16 - (sizeof(BondController) & 0xf)) : 0);
 
 		m = reinterpret_cast<char *>(::malloc(16 + ts + sws + mcs + topologys + sas + bc));
 		if (!m)
@@ -121,14 +121,14 @@ Node::Node(void *uptr,void *tptr,const struct ZT_Node_Callbacks *callbacks,int64
 		m += topologys;
 		RR->sa = new (m) SelfAwareness(RR);
 		m += sas;
-		RR->bc = new (m) Bond(RR);
+		RR->bc = new (m) BondController(RR);
 	} catch ( ... ) {
 		if (RR->sa) RR->sa->~SelfAwareness();
 		if (RR->topology) RR->topology->~Topology();
 		if (RR->mc) RR->mc->~Multicaster();
 		if (RR->sw) RR->sw->~Switch();
 		if (RR->t) RR->t->~Trace();
-		if (RR->bc) RR->bc->~Bond();
+		if (RR->bc) RR->bc->~BondController();
 		::free(m);
 		throw;
 	}
@@ -147,7 +147,7 @@ Node::~Node()
 	if (RR->mc) RR->mc->~Multicaster();
 	if (RR->sw) RR->sw->~Switch();
 	if (RR->t) RR->t->~Trace();
-	if (RR->bc) RR->bc->~Bond();
+	if (RR->bc) RR->bc->~BondController();
 	::free(RR->rtmem);
 }
 
@@ -252,14 +252,18 @@ ZT_ResultCode Node::processBackgroundTasks(void *tptr,int64_t now,volatile int64
 	_now = now;
 	Mutex::Lock bl(_backgroundTasksLock);
 
-	// Process background bond tasks
-	unsigned long bondCheckInterval = ZT_PING_CHECK_INVERVAL;
+
+	unsigned long bondCheckInterval = ZT_CORE_TIMER_TASK_GRANULARITY;
 	if (RR->bc->inUse()) {
-		bondCheckInterval = std::max(RR->bc->minReqMonitorInterval(), ZT_CORE_TIMER_TASK_GRANULARITY);
-		if ((now - _lastGratuitousPingCheck) >= ZT_CORE_TIMER_TASK_GRANULARITY) {
+		// Gratuitously ping active peers so that QoS metrics have enough data to work with (if active path monitoring is enabled)
+		bondCheckInterval = std::min(std::max(RR->bc->minReqPathMonitorInterval(), ZT_CORE_TIMER_TASK_GRANULARITY), ZT_PING_CHECK_INVERVAL);
+		if ((now - _lastGratuitousPingCheck) >= bondCheckInterval) {
+			Hashtable< Address,std::vector<InetAddress> > alwaysContact;
+			_PingPeersThatNeedPing pfunc(RR,tptr,alwaysContact,now);
+			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 			_lastGratuitousPingCheck = now;
-			RR->bc->processBackgroundTasks(tptr, now);
 		}
+		RR->bc->processBackgroundTasks(tptr, now);
 	}
 
 	unsigned long timeUntilNextPingCheck = ZT_PING_CHECK_INVERVAL;
@@ -508,7 +512,7 @@ ZT_PeerList *Node::peers() const
 		}
 		if (pi->second->bond()) {
 			p->isBonded = pi->second->bond();
-			p->bondingPolicy = pi->second->bond()->policy();
+			p->bondingPolicy = pi->second->bond()->getPolicy();
 			p->isHealthy = pi->second->bond()->isHealthy();
 			p->numAliveLinks = pi->second->bond()->getNumAliveLinks();
 			p->numTotalLinks = pi->second->bond()->getNumTotalLinks();
@@ -727,7 +731,7 @@ void Node::ncSendRevocation(const Address &destination,const Revocation &rev)
 	}
 }
 
-void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode, const void *errorData, unsigned int errorDataSize)
+void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode)
 {
 	if (destination == RR->identity.address()) {
 		SharedPtr<Network> n(network(nwid));
@@ -740,9 +744,6 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 			case NetworkController::NC_ERROR_ACCESS_DENIED:
 				n->setAccessDenied();
 				break;
-			case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED: {
-			}
-				break;
 
 			default: break;
 		}
@@ -759,18 +760,8 @@ void Node::ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &des
 			case NetworkController::NC_ERROR_ACCESS_DENIED:
 				outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 				break;
-			case NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED:
-				outp.append((unsigned char)Packet::ERROR_NETWORK_AUTHENTICATION_REQUIRED);
-				break;
 		}
-
 		outp.append(nwid);
-
-		if ((errorData)&&(errorDataSize > 0)&&(errorDataSize <= 0xffff)) {
-			outp.append((uint16_t)errorDataSize);
-			outp.append(errorData, errorDataSize);
-		}
-
 		RR->sw->send((void *)0,outp,true);
 	} // else we can't send an ERROR() in response to nothing, so discard
 }

node/Node.hpp

@@ -34,7 +34,7 @@
 #include "Salsa20.hpp"
 #include "NetworkController.hpp"
 #include "Hashtable.hpp"
-#include "Bond.hpp"
+#include "BondController.hpp"
 
 // Bit mask for "expecting reply" hash
 #define ZT_EXPECTING_REPLIES_BUCKET_MASK1 255
@@ -187,7 +187,7 @@ public:
 
 	inline const Identity &identity() const { return _RR.identity; }
 
-	inline Bond *bondController() const { return _RR.bc; }
+	inline BondController *bondController() const { return _RR.bc; }
 
 	/**
 	 * Register that we are expecting a reply to a packet ID
@@ -245,7 +245,7 @@ public:
 
 	virtual void ncSendConfig(uint64_t nwid,uint64_t requestPacketId,const Address &destination,const NetworkConfig &nc,bool sendLegacyFormatConfig);
 	virtual void ncSendRevocation(const Address &destination,const Revocation &rev);
-	virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode, const void *errorData, unsigned int errorDataSize);
+	virtual void ncSendError(uint64_t nwid,uint64_t requestPacketId,const Address &destination,NetworkController::ErrorCode errorCode);
 
 	inline const Address &remoteTraceTarget() const { return _remoteTraceTarget; }
 	inline Trace::Level remoteTraceLevel() const { return _remoteTraceLevel; }

node/Packet.hpp

@@ -792,12 +792,6 @@ public:
 		 *
 		 * ERROR response payload:
 		 *   <[8] 64-bit network ID>
-     *   <[2] 16-bit length of error-related data (optional)>
-     *   <[...] error-related data (optional)>
-     * 
-     * Error related data is a Dictionary containing things like a URL
-     * for authentication or a human-readable error message, and is
-     * optional and may be absent or empty.
 		 */
 		VERB_NETWORK_CONFIG_REQUEST = 0x0b,
 
@@ -1082,10 +1076,7 @@ public:
 		ERROR_NETWORK_ACCESS_DENIED_ = 0x07, /* extra _ at end to avoid Windows name conflict */
 
 		/* Multicasts to this group are not wanted */
-		ERROR_UNWANTED_MULTICAST = 0x08,
-
-    /* Network requires external or 2FA authentication (e.g. SSO). */
-    ERROR_NETWORK_AUTHENTICATION_REQUIRED = 0x09
+		ERROR_UNWANTED_MULTICAST = 0x08
 	};
 
 	template<unsigned int C2>

node/Path.hpp

@@ -29,6 +29,8 @@
 #include "Packet.hpp"
 #include "RingBuffer.hpp"
 
+#include "../osdep/Link.hpp"
+
 /**
  * Maximum return value of preferenceRank()
  */
@@ -86,7 +88,46 @@ public:
 		_localSocket(-1),
 		_latency(0xffff),
 		_addr(),
-		_ipScope(InetAddress::IP_SCOPE_NONE)
+		_ipScope(InetAddress::IP_SCOPE_NONE),
+		_lastAckReceived(0),
+		_lastAckSent(0),
+		_lastQoSMeasurement(0),
+		_lastThroughputEstimation(0),
+		_lastRefractoryUpdate(0),
+		_lastAliveToggle(0),
+		_lastEligibilityState(false),
+		_lastTrialBegin(0),
+		_refractoryPeriod(0),
+		_monitorInterval(0),
+		_upDelay(0),
+		_downDelay(0),
+		_ipvPref(0),
+		_mode(0),
+		_onlyPathOnLink(false),
+		_enabled(false),
+		_bonded(false),
+		_negotiated(false),
+		_deprecated(false),
+		_shouldReallocateFlows(false),
+		_assignedFlowCount(0),
+		_latencyMean(0),
+		_latencyVariance(0),
+		_packetLossRatio(0),
+		_packetErrorRatio(0),
+		_throughputMean(0),
+		_throughputMax(0),
+		_throughputVariance(0),
+		_allocation(0),
+		_byteLoad(0),
+		_relativeByteLoad(0),
+		_affinity(0),
+		_failoverScore(0),
+		_unackedBytes(0),
+		_packetsReceivedSinceLastAck(0),
+		_packetsReceivedSinceLastQoS(0),
+		_bytesAckedSinceLastThroughputEstimation(0),
+		_packetsIn(0),
+		_packetsOut(0)
 		{}
 
 	Path(const int64_t localSocket,const InetAddress &addr) :
@@ -96,7 +137,46 @@ public:
 		_localSocket(localSocket),
 		_latency(0xffff),
 		_addr(addr),
-		_ipScope(addr.ipScope())
+		_ipScope(addr.ipScope()),
+		_lastAckReceived(0),
+		_lastAckSent(0),
+		_lastQoSMeasurement(0),
+		_lastThroughputEstimation(0),
+		_lastRefractoryUpdate(0),
+		_lastAliveToggle(0),
+		_lastEligibilityState(false),
+		_lastTrialBegin(0),
+		_refractoryPeriod(0),
+		_monitorInterval(0),
+		_upDelay(0),
+		_downDelay(0),
+		_ipvPref(0),
+		_mode(0),
+		_onlyPathOnLink(false),
+		_enabled(false),
+		_bonded(false),
+		_negotiated(false),
+		_deprecated(false),
+		_shouldReallocateFlows(false),
+		_assignedFlowCount(0),
+		_latencyMean(0),
+		_latencyVariance(0),
+		_packetLossRatio(0),
+		_packetErrorRatio(0),
+		_throughputMean(0),
+		_throughputMax(0),
+		_throughputVariance(0),
+		_allocation(0),
+		_byteLoad(0),
+		_relativeByteLoad(0),
+		_affinity(0),
+		_failoverScore(0),
+		_unackedBytes(0),
+		_packetsReceivedSinceLastAck(0),
+		_packetsReceivedSinceLastQoS(0),
+		_bytesAckedSinceLastThroughputEstimation(0),
+		_packetsIn(0),
+		_packetsOut(0)
 	{}
 
 	/**
@@ -106,6 +186,9 @@ public:
 	 */
 	inline void received(const uint64_t t)
 	{
+		if (!alive(t,_bonded)) {
+			_lastAliveToggle = _lastIn;
+		}
 		_lastIn = t;
 	}
 
@@ -234,11 +317,21 @@ public:
 		return (((age < (ZT_PATH_HEARTBEAT_PERIOD + 5000)) ? l : (l + 0xffff + age)) * (long)((ZT_INETADDRESS_MAX_SCOPE - _ipScope) + 1));
 	}
 
+	/**
+	 * @param bonded Whether this path is part of a bond.
+	 */
+	inline void setBonded(bool bonded) { _bonded = bonded; }
+
+	/**
+	 * @return True if this path is currently part of a bond.
+	 */
+	inline bool bonded() { return _bonded; }
+
 	/**
 	 * @return True if this path is alive (receiving heartbeats)
 	 */
-	inline bool alive(const int64_t now) const {
-		return (now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000);
+	inline bool alive(const int64_t now, bool bondingEnabled = false) const {
+		return (bondingEnabled && _monitorInterval) ? ((now - _lastIn) < (_monitorInterval * 3)) : ((now - _lastIn) < (ZT_PATH_HEARTBEAT_PERIOD + 5000));
 	}
 
 	/**
@@ -246,6 +339,11 @@ public:
 	 */
 	inline bool needsHeartbeat(const int64_t now) const { return ((now - _lastOut) >= ZT_PATH_HEARTBEAT_PERIOD); }
 
+	/**
+	 * @return True if this path needs a heartbeat in accordance to the user-specified path monitor frequency
+	 */
+	inline bool needsGratuitousHeartbeat(const int64_t now) { return allowed() && (_monitorInterval > 0) && ((now - _lastOut) >= _monitorInterval); }
+
 	/**
 	 * @return Last time we sent something
 	 */
@@ -266,7 +364,134 @@ public:
 	 */
 	inline int64_t lastTrustEstablishedPacketReceived() const { return _lastTrustEstablishedPacketReceived; }
 
-	void *_bondingMetricPtr;
+	/**
+	 * @return Time since last VERB_ACK was received
+	 */
+	inline int64_t ackAge(int64_t now) { return _lastAckReceived ? now - _lastAckReceived : 0; }
+
+	/**
+	 * Set or update a refractory period for the path.
+	 *
+	 * @param punishment How much a path should be punished
+	 * @param pathFailure Whether this call is the result of a recent path failure
+	 */
+	inline void adjustRefractoryPeriod(int64_t now, uint32_t punishment, bool pathFailure) {
+		if (pathFailure) {
+			unsigned int suggestedRefractoryPeriod = _refractoryPeriod ? punishment + (_refractoryPeriod * 2) : punishment;
+			_refractoryPeriod = std::min(suggestedRefractoryPeriod, (unsigned int)ZT_MULTIPATH_MAX_REFRACTORY_PERIOD);
+			_lastRefractoryUpdate = 0;
+		} else {
+			uint32_t drainRefractory = 0;
+			if (_lastRefractoryUpdate) {
+				drainRefractory = (now - _lastRefractoryUpdate);
+			} else {
+				drainRefractory = (now - _lastAliveToggle);
+			}
+			_lastRefractoryUpdate = now;
+			if (_refractoryPeriod > drainRefractory) {
+				_refractoryPeriod -= drainRefractory;
+			} else {
+				_refractoryPeriod = 0;
+				_lastRefractoryUpdate = 0;
+			}
+		}
+	}
+
+	/**
+	 * Determine the current state of eligibility of the path.
+	 *
+	 * @param includeRefractoryPeriod Whether current punishment should be taken into consideration
+	 * @return True if this path can be used in a bond at the current time
+	 */
+	inline bool eligible(uint64_t now, int ackSendInterval, bool includeRefractoryPeriod = false) {
+		if (includeRefractoryPeriod && _refractoryPeriod) {
+			return false;
+		}
+		bool acceptableAge    = age(now) < ((_monitorInterval * 4) + _downDelay); // Simple RX age (driven by packets of any type and gratuitous VERB_HELLOs)
+		bool acceptableAckAge = ackAge(now) < (ackSendInterval); // Whether the remote peer is actually responding to our outgoing traffic or simply sending stuff to us
+		bool notTooEarly      = (now - _lastAliveToggle) >= _upDelay; // Whether we've waited long enough since the link last came online
+		bool inTrial          = (now - _lastTrialBegin) < _upDelay; // Whether this path is still in its trial period
+		bool currEligibility  = allowed() && (((acceptableAge || acceptableAckAge) && notTooEarly) || inTrial);
+		return currEligibility;
+	}
+
+	/**
+	 * Record when this path first entered the bond. Each path is given a trial period where it is admitted
+	 * to the bond without requiring observations to prove its performance or reliability.
+	 */
+	inline void startTrial(uint64_t now) { _lastTrialBegin = now; }
+
+	/**
+	 * @return True if a path is permitted to be used in a bond (according to user pref.)
+	 */
+	inline bool allowed() {
+		return _enabled
+			&& (!_ipvPref
+				|| ((_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46 || _ipvPref == 64))
+				|| ((_addr.isV6() && (_ipvPref == 6 || _ipvPref == 46 || _ipvPref == 64)))));
+	}
+
+	/**
+	 * @return True if a path is preferred over another on the same physical link (according to user pref.)
+	 */
+	inline bool preferred() {
+		return _onlyPathOnLink
+			|| (_addr.isV4() && (_ipvPref == 4 || _ipvPref == 46))
+			|| (_addr.isV6() && (_ipvPref == 6 || _ipvPref == 64));
+	}
+
+	/**
+	 * @param now Current time
+	 * @return Whether an ACK (VERB_ACK) packet needs to be emitted at this time
+	 */
+	inline bool needsToSendAck(int64_t now, int ackSendInterval) {
+		return ((now - _lastAckSent) >= ackSendInterval ||
+			(_packetsReceivedSinceLastAck == ZT_QOS_TABLE_SIZE)) && _packetsReceivedSinceLastAck;
+	}
+
+	/**
+	 * @param now Current time
+	 * @return Whether a QoS (VERB_QOS_MEASUREMENT) packet needs to be emitted at this time
+	 */
+	inline bool needsToSendQoS(int64_t now, int qosSendInterval) {
+		return ((_packetsReceivedSinceLastQoS >= ZT_QOS_TABLE_SIZE) ||
+			((now - _lastQoSMeasurement) > qosSendInterval)) && _packetsReceivedSinceLastQoS;
+	}
+
+	/**
+	 * Reset packet counters
+	 */
+	inline void resetPacketCounts()
+	{
+		_packetsIn = 0;
+		_packetsOut = 0;
+	}
+
+
+	/**
+	 * The mean latency (computed from a sliding window.)
+	 */
+	float latencyMean() { return _latencyMean; }
+
+	/**
+	 * Packet delay variance (computed from a sliding window.)
+	 */
+	float latencyVariance() { return _latencyVariance; }
+
+	/**
+	 * The ratio of lost packets to received packets.
+	 */
+	float packetLossRatio() { return _packetLossRatio; }
+
+	/**
+	 * The ratio of packets that failed their MAC/CRC checks to those that did not.
+	 */
+	float packetErrorRatio() { return _packetErrorRatio; }
+
+	/**
+	*
+	*/
+	uint8_t allocation() { return _allocation; }
 
 private:
 
@@ -278,6 +503,212 @@ private:
 	InetAddress _addr;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 	AtomicCounter __refCount;
+
+	std::map<uint64_t,uint64_t> qosStatsOut; // id:egress_time
+	std::map<uint64_t,uint64_t> qosStatsIn; // id:now
+	std::map<uint64_t,uint16_t> ackStatsIn; // id:len
+
+	RingBuffer<int,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordSize;
+	RingBuffer<float,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> qosRecordLossSamples;
+	RingBuffer<uint64_t,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> throughputSamples;
+	RingBuffer<bool,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> packetValiditySamples;
+	RingBuffer<float,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> _throughputVarianceSamples;
+	RingBuffer<uint16_t,ZT_QOS_SHORTTERM_SAMPLE_WIN_SIZE> latencySamples;
+
+	/**
+	 * Last time that a VERB_ACK was received on this path.
+	 */
+	uint64_t _lastAckReceived;
+
+	/**
+	 * Last time that a VERB_ACK was sent out on this path.
+	 */
+	uint64_t _lastAckSent;
+
+	/**
+	 * Last time that a VERB_QOS_MEASUREMENT was sent out on this path.
+	 */
+	uint64_t _lastQoSMeasurement;
+
+	/**
+	 * Last time that the path's throughput was estimated.
+	 */
+	uint64_t _lastThroughputEstimation;
+
+	/**
+	 * The last time that the refractory period was updated.
+	 */
+	uint64_t _lastRefractoryUpdate;
+
+	/**
+	 * The last time that the path was marked as "alive".
+	 */
+	uint64_t _lastAliveToggle;
+
+	/**
+	 * State of eligibility at last check. Used for determining state changes.
+	 */
+	bool _lastEligibilityState;
+
+	/**
+	 * Timestamp indicating when this path's trial period began.
+	 */
+	uint64_t _lastTrialBegin;
+
+	/**
+	 * Amount of time that this path will be prevented from becoming a member of a bond.
+	 */
+	uint32_t _refractoryPeriod;
+
+	/**
+	 * Monitor interval specific to this path or that was inherited from the bond controller.
+	 */
+	int32_t _monitorInterval;
+
+	/**
+	 * Up delay interval specific to this path or that was inherited from the bond controller.
+	 */
+	uint32_t _upDelay;
+
+	/**
+	 * Down delay interval specific to this path or that was inherited from the bond controller.
+	 */
+	uint32_t _downDelay;
+
+	/**
+	 * IP version preference inherited from the physical link.
+	 */
+	uint8_t _ipvPref;
+
+	/**
+	 * Mode inherited from the physical link.
+	 */
+	uint8_t _mode;
+
+	/**
+	 * IP version preference inherited from the physical link.
+	 */
+	bool _onlyPathOnLink;
+
+	/**
+	 * Enabled state inherited from the physical link.
+	 */
+	bool _enabled;
+
+	/**
+	 * Whether this path is currently part of a bond.
+	 */
+	bool _bonded;
+
+	/**
+	 * Whether this path was intentionally negotiated by either peer.
+	 */
+	bool _negotiated;
+
+	/**
+	 * Whether this path has been deprecated due to performance issues. Current traffic flows
+	 * will be re-allocated to other paths in the most non-disruptive manner (if possible),
+	 * and new traffic will not be allocated to this path.
+	 */
+	bool _deprecated;
+
+	/**
+	 * Whether flows should be moved from this path. Current traffic flows will be re-allocated
+	 * immediately.
+	 */
+	bool _shouldReallocateFlows;
+
+	/**
+	 * The number of flows currently assigned to this path.
+	 */
+	uint16_t _assignedFlowCount;
+
+	/**
+	 * The mean latency (computed from a sliding window.)
+	 */
+	float _latencyMean;
+
+	/**
+	 * Packet delay variance (computed from a sliding window.)
+	 */
+	float _latencyVariance;
+
+	/**
+	 * The ratio of lost packets to received packets.
+	 */
+	float _packetLossRatio;
+
+	/**
+	 * The ratio of packets that failed their MAC/CRC checks to those that did not.
+	 */
+	float _packetErrorRatio;
+
+	/**
+	 * The estimated mean throughput of this path.
+	 */
+	uint64_t _throughputMean;
+
+	/**
+	 * The maximum observed throughput of this path.
+	 */
+	uint64_t _throughputMax;
+
+	/**
+	 * The variance in the estimated throughput of this path.
+	 */
+	float _throughputVariance;
+
+	/**
+	 * The relative quality of this path to all others in the bond, [0-255].
+	 */
+	uint8_t _allocation;
+
+	/**
+	 * How much load this path is under.
+	 */
+	uint64_t _byteLoad;
+
+	/**
+	 * How much load this path is under (relative to other paths in the bond.)
+	 */
+	uint8_t _relativeByteLoad;
+
+	/**
+	 * Relative value expressing how "deserving" this path is of new traffic.
+	 */
+	uint8_t _affinity;
+
+	/**
+	 * Score that indicates to what degree this path is preferred over others that
+	 * are available to the bonding policy. (specifically for active-backup)
+	 */
+	uint32_t _failoverScore;
+
+	/**
+	 * Number of bytes thus far sent that have not been acknowledged by the remote peer.
+	 */
+	int64_t _unackedBytes;
+
+	/**
+	 * Number of packets received since the last VERB_ACK was sent to the remote peer.
+	 */
+	int32_t _packetsReceivedSinceLastAck;
+
+	/**
+	 * Number of packets received since the last VERB_QOS_MEASUREMENT was sent to the remote peer.
+	 */
+	int32_t _packetsReceivedSinceLastQoS;
+
+	/**
+	 * Bytes acknowledged via incoming VERB_ACK since the last estimation of throughput.
+	 */
+	uint64_t _bytesAckedSinceLastThroughputEstimation;
+
+	/**
+	 * Counters used for tracking path load.
+	 */
+	int _packetsIn;
+	int _packetsOut;
 };
 
 } // namespace ZeroTier

node/Peer.cpp

@@ -50,7 +50,12 @@ Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Ident
 	_directPathPushCutoffCount(0),
 	_credentialsCutoffCount(0),
 	_echoRequestCutoffCount(0),
+	_uniqueAlivePathCount(0),
 	_localMultipathSupported(false),
+	_remoteMultipathSupported(false),
+	_canUseMultipath(false),
+	_shouldCollectPathStatistics(0),
+	_bondingPolicy(0),
 	_lastComputedAggregateMeanLatency(0)
 {
 	if (!myIdentity.agree(peerIdentity,_key))
@@ -146,10 +151,6 @@ void Peer::received(
 					_paths[replacePath].lr = now;
 					_paths[replacePath].p = path;
 					_paths[replacePath].priority = 1;
-					Mutex::Lock _l(_bond_m);
-					if(_bond) {
-						_bond->nominatePathToBond(_paths[replacePath].p, now);
-					}
 				}
 			} else {
 				Mutex::Lock ltl(_lastTriedPath_m);
@@ -228,8 +229,7 @@ void Peer::received(
 
 SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int32_t flowId)
 {
-	Mutex::Lock _l(_bond_m);
-	if (!_bond) {
+	if (!_bondToPeer) {
 		Mutex::Lock _l(_paths_m);
 		unsigned int bestPath = ZT_MAX_PEER_NETWORK_PATHS;
 		/**
@@ -253,7 +253,7 @@ SharedPtr<Path> Peer::getAppropriatePath(int64_t now, bool includeExpired, int32
 		}
 		return SharedPtr<Path>();
 	}
-	return _bond->getAppropriatePath(now, flowId);
+	return _bondToPeer->getAppropriatePath(now, flowId);
 }
 
 void Peer::introduce(void *const tPtr,const int64_t now,const SharedPtr<Peer> &other) const
@@ -444,32 +444,39 @@ void Peer::tryMemorizedPath(void *tPtr,int64_t now)
 
 void Peer::performMultipathStateCheck(void *tPtr, int64_t now)
 {
-	Mutex::Lock _l(_bond_m);
-	if (_bond) {
-		// Once enabled the Bond object persists, no need to update state
-		return;
-	}
 	/**
 	 * Check for conditions required for multipath bonding and create a bond
 	 * if allowed.
 	 */
-	int numAlivePaths = 0;
-	for(unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
-		if (_paths[i].p && _paths[i].p->alive(now)) {
-			numAlivePaths++;
+	_localMultipathSupported = ((RR->bc->inUse()) && (ZT_PROTO_VERSION > 9));
+	if (_localMultipathSupported) {
+		int currAlivePathCount = 0;
+		int duplicatePathsFound = 0;
+		for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
+			if (_paths[i].p) {
+				currAlivePathCount++;
+				for (unsigned int j=0;j<ZT_MAX_PEER_NETWORK_PATHS;++j) {
+					if (_paths[i].p && _paths[j].p && _paths[i].p->address().ipsEqual2(_paths[j].p->address()) && i != j) {
+						duplicatePathsFound+=1;
+						break;
+					}
+				}
+			}
 		}
+		_uniqueAlivePathCount = (currAlivePathCount - (duplicatePathsFound / 2));
+		_remoteMultipathSupported = _vProto > 9;
+		_canUseMultipath = _localMultipathSupported && _remoteMultipathSupported && (_uniqueAlivePathCount > 1);
 	}
-	_localMultipathSupported = ((numAlivePaths >= 1) && (RR->bc->inUse()) && (ZT_PROTO_VERSION > 9));
-	if (_localMultipathSupported && !_bond) {
+	if (_canUseMultipath && !_bondToPeer) {
 		if (RR->bc) {
-			_bond = RR->bc->createTransportTriggeredBond(RR, this);
+			_bondToPeer = RR->bc->createTransportTriggeredBond(RR, this);
 			/**
 			 * Allow new bond to retroactively learn all paths known to this peer
 			 */
-			if (_bond) {
+			if (_bondToPeer) {
 				for (unsigned int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
 					if (_paths[i].p) {
-						_bond->nominatePathToBond(_paths[i].p, now);
+						_bondToPeer->nominatePath(_paths[i].p, now);
 					}
 				}
 			}
@@ -503,7 +510,8 @@ unsigned int Peer::doPingAndKeepalive(void *tPtr,int64_t now)
 		if (_paths[i].p) {
 			// Clean expired and reduced priority paths
 			if ( ((now - _paths[i].lr) < ZT_PEER_PATH_EXPIRATION) && (_paths[i].priority == maxPriority) ) {
-				if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))) {
+				if ((sendFullHello)||(_paths[i].p->needsHeartbeat(now))
+					|| (_canUseMultipath && _paths[i].p->needsGratuitousHeartbeat(now))) {
 					attemptToContactAt(tPtr,_paths[i].p->localSocket(),_paths[i].p->address(),now,sendFullHello);
 					_paths[i].p->sent(now);
 					sent |= (_paths[i].p->address().ss_family == AF_INET) ? 0x1 : 0x2;
@@ -583,24 +591,27 @@ void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddres
 void Peer::recordOutgoingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
 	uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
 {
-	if (_localMultipathSupported && _bond) {
-		_bond->recordOutgoingPacket(path, packetId, payloadLength, verb, flowId, now);
+	if (!_shouldCollectPathStatistics || !_bondToPeer) {
+		return;
 	}
+	_bondToPeer->recordOutgoingPacket(path, packetId, payloadLength, verb, flowId, now);
 }
 
 void Peer::recordIncomingInvalidPacket(const SharedPtr<Path>& path)
 {
-	if (_localMultipathSupported && _bond) {
-		_bond->recordIncomingInvalidPacket(path);
+	if (!_shouldCollectPathStatistics || !_bondToPeer) {
+		return;
 	}
+	_bondToPeer->recordIncomingInvalidPacket(path);
 }
 
 void Peer::recordIncomingPacket(const SharedPtr<Path> &path, const uint64_t packetId,
 	uint16_t payloadLength, const Packet::Verb verb, const int32_t flowId, int64_t now)
 {
-	if (_localMultipathSupported && _bond) {
-		_bond->recordIncomingPacket(path, packetId, payloadLength, verb, flowId, now);
+	if (!_shouldCollectPathStatistics || !_bondToPeer) {
+		return;
 	}
+	_bondToPeer->recordIncomingPacket(path, packetId, payloadLength, verb, flowId, now);
 }
 
 } // namespace ZeroTier

node/Peer.hpp

@@ -33,6 +33,7 @@
 #include "Hashtable.hpp"
 #include "Mutex.hpp"
 #include "Bond.hpp"
+#include "BondController.hpp"
 #include "AES.hpp"
 
 #define ZT_PEER_MAX_SERIALIZED_STATE_SIZE (sizeof(Peer) + 32 + (sizeof(Path) * 2))
@@ -304,13 +305,12 @@ public:
 	 */
 	inline unsigned int latency(const int64_t now)
 	{
-		if (_localMultipathSupported) {
+		if (_canUseMultipath) {
 			return (int)_lastComputedAggregateMeanLatency;
 		} else {
 			SharedPtr<Path> bp(getAppropriatePath(now,false));
-			if (bp) {
+			if (bp)
 				return bp->latency();
-			}
 			return 0xffff;
 		}
 	}
@@ -419,15 +419,35 @@ public:
 	}
 
 	/**
-	 * Rate limit gate for inbound ECHO requests
+	 * Rate limit gate for inbound ECHO requests. This rate limiter works
+	 * by draining a certain number of requests per unit time. Each peer may
+	 * theoretically receive up to ZT_ECHO_CUTOFF_LIMIT requests per second.
 	 */
 	inline bool rateGateEchoRequest(const int64_t now)
 	{
-		if ((now - _lastEchoRequestReceived) >= ZT_PEER_GENERAL_RATE_LIMIT) {
-			_lastEchoRequestReceived = now;
-			return true;
+		/*
+		// TODO: Rethink this
+		if (_canUseMultipath) {
+			_echoRequestCutoffCount++;
+			int numToDrain = (now - _lastEchoCheck) / ZT_ECHO_DRAINAGE_DIVISOR;
+			_lastEchoCheck = now;
+			fprintf(stderr, "ZT_ECHO_CUTOFF_LIMIT=%d, (now - _lastEchoCheck)=%d, numToDrain=%d, ZT_ECHO_DRAINAGE_DIVISOR=%d\n", ZT_ECHO_CUTOFF_LIMIT, (now - _lastEchoCheck), numToDrain, ZT_ECHO_DRAINAGE_DIVISOR);
+			if (_echoRequestCutoffCount > numToDrain) {
+				_echoRequestCutoffCount-=numToDrain;
+			}
+			else {
+				_echoRequestCutoffCount = 0;
+			}
+			return (_echoRequestCutoffCount < ZT_ECHO_CUTOFF_LIMIT);
+		} else {
+			if ((now - _lastEchoRequestReceived) >= (ZT_PEER_GENERAL_RATE_LIMIT)) {
+				_lastEchoRequestReceived = now;
+				return true;
+			}
+			return false;
 		}
-		return false;
+		*/
+		return true;
 	}
 
 	/**
@@ -503,20 +523,16 @@ public:
 	}
 
 	/**
-	 * @return The bonding policy used to reach this peer
+	 *
+	 * @return
 	 */
-	SharedPtr<Bond> bond() { return _bond; }
+	SharedPtr<Bond> bond() { return _bondToPeer; }
 
 	/**
-	 * @return The bonding policy used to reach this peer
+	 *
+	 * @return
 	 */
-	inline int8_t bondingPolicy() {
-		Mutex::Lock _l(_paths_m);
-		if (_bond) {
-			return _bond->policy();
-		}
-		return ZT_BOND_POLICY_NONE;
-	}
+	inline int8_t bondingPolicy() { return _bondingPolicy; }
 
 	//inline const AES *aesKeysIfSupported() const
 	//{ return (const AES *)0; }
@@ -566,7 +582,6 @@ private:
 
 	_PeerPath _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	Mutex _paths_m;
-	Mutex _bond_m;
 
 	Identity _id;
 
@@ -576,13 +591,18 @@ private:
 
 	AtomicCounter __refCount;
 
+	bool _remotePeerMultipathEnabled;
+	int _uniqueAlivePathCount;
 	bool _localMultipathSupported;
+	bool _remoteMultipathSupported;
+	bool _canUseMultipath;
 
 	volatile bool _shouldCollectPathStatistics;
+	volatile int8_t _bondingPolicy;
 
 	int32_t _lastComputedAggregateMeanLatency;
 
-	SharedPtr<Bond> _bond;
+	SharedPtr<Bond> _bondToPeer;
 };
 
 } // namespace ZeroTier

node/RuntimeEnvironment.hpp

@@ -30,7 +30,7 @@ class Multicaster;
 class NetworkController;
 class SelfAwareness;
 class Trace;
-class Bond;
+class BondController;
 
 /**
  * Holds global state for an instance of ZeroTier::Node
@@ -76,7 +76,7 @@ public:
 	Multicaster *mc;
 	Topology *topology;
 	SelfAwareness *sa;
-	Bond *bc;
+	BondController *bc;
 
 	// This node's identity and string representations thereof
 	Identity identity;

node/Switch.cpp

@@ -1003,12 +1003,14 @@ bool Switch::_trySend(void *tPtr,Packet &packet,bool encrypt,int32_t flowId)
 
 	const SharedPtr<Peer> peer(RR->topology->getPeer(tPtr,destination));
 	if (peer) {
-		if ((peer->bondingPolicy() == ZT_BOND_POLICY_BROADCAST)
+		if ((peer->bondingPolicy() == ZT_BONDING_POLICY_BROADCAST)
 			&& (packet.verb() == Packet::VERB_FRAME || packet.verb() == Packet::VERB_EXT_FRAME)) {
 			const SharedPtr<Peer> relay(RR->topology->getUpstreamPeer());
 			Mutex::Lock _l(peer->_paths_m);
 			for(int i=0;i<ZT_MAX_PEER_NETWORK_PATHS;++i) {
 				if (peer->_paths[i].p && peer->_paths[i].p->alive(now)) {
+					char pathStr[128];
+					peer->_paths[i].p->address().toString(pathStr);
 					_sendViaSpecificPath(tPtr,peer,peer->_paths[i].p,now,packet,encrypt,flowId);
 				}
 			}
@@ -1045,6 +1047,7 @@ void Switch::_sendViaSpecificPath(void *tPtr,SharedPtr<Peer> peer,SharedPtr<Path
 	if (trustedPathId) {
 		packet.setTrusted(trustedPathId);
 	} else {
+		Packet::Verb v = packet.verb();
 		packet.armor(peer->key(),encrypt,peer->aesKeysIfSupported());
 		RR->node->expectReplyTo(packet.packetId());
 	}

node/Trace.cpp

@@ -69,7 +69,7 @@ void Trace::peerConfirmingUnknownPath(void *const tPtr,const uint64_t networkId,
 	char tmp[128];
 	if (!path) return; // sanity check
 
-	ZT_LOCAL_TRACE(tPtr,RR,"trying unknown path %s to %.10llx (packet %.16llx verb %d local socket %lld network %.16llx)",path->address().toString(tmp),peer.address().toInt(),packetId,verb,path->localSocket(),networkId);
+	ZT_LOCAL_TRACE(tPtr,RR,"trying unknown path %s to %.10llx (packet %.16llx verb %d local socket %lld network %.16llx)",path->address().toString(tmp),peer.address().toInt(),packetId,(double)verb,path->localSocket(),networkId);
 
 	std::pair<Address,Trace::Level> byn;
 	if (networkId) { Mutex::Lock l(_byNet_m); _byNet.get(networkId,byn); }

node/Utils.cpp

@@ -28,12 +28,10 @@
 #include <sys/stat.h>
 #include <sys/uio.h>
 #include <dirent.h>
-#ifdef ZT_ARCH_ARM_HAS_NEON
 #ifdef __LINUX__
 #include <sys/auxv.h>
 #endif
 #endif
-#endif
 
 #ifdef __WINDOWS__
 #include <wincrypt.h>

objects.mk

@@ -28,7 +28,8 @@ CORE_OBJS=\
 	node/Topology.o \
 	node/Trace.o \
 	node/Utils.o \
-	node/Bond.o
+	node/Bond.o \
+	node/BondController.o
 
 ONE_OBJS=\
 	controller/EmbeddedNetworkController.o \

one.cpp

@@ -84,7 +84,7 @@
 #include "osdep/Http.hpp"
 #include "osdep/Thread.hpp"
 
-#include "node/Bond.hpp"
+#include "node/BondController.hpp"
 
 #include "service/OneService.hpp"
 
@@ -393,9 +393,7 @@ static int cli(int argc,char **argv)
 									char tmp[256];
 									std::string addr = path["address"];
 									const int64_t now = OSUtils::now();
-									int64_t lastSendDiff = (uint64_t)path["lastSend"] ? now - (uint64_t)path["lastSend"] : -1;
-									int64_t lastReceiveDiff = (uint64_t)path["lastReceive"] ? now - (uint64_t)path["lastReceive"] : -1;
-									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%lld;%lld",addr.c_str(),lastSendDiff,lastReceiveDiff);
+									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%s;%lld;%lld",addr.c_str(),now - (int64_t)path["lastSend"],now - (int64_t)path["lastReceive"]);
 									bestPath = tmp;
 									break;
 								}
@@ -462,9 +460,7 @@ static int cli(int argc,char **argv)
 									char tmp[256];
 									std::string addr = path["address"];
 									const int64_t now = OSUtils::now();
-									int64_t lastSendDiff = (uint64_t)path["lastSend"] ? now - (uint64_t)path["lastSend"] : -1;
-									int64_t lastReceiveDiff = (uint64_t)path["lastReceive"] ? now - (uint64_t)path["lastReceive"] : -1;
-									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%-8lld %-8lld %s",lastSendDiff,lastReceiveDiff,addr.c_str());
+									OSUtils::ztsnprintf(tmp,sizeof(tmp),"%-8lld %-8lld %s",now - (int64_t)path["lastSend"],now - (int64_t)path["lastReceive"],addr.c_str());
 									bestPath = std::string("DIRECT ") + tmp;
 									break;
 								}
@@ -496,13 +492,14 @@ static int cli(int argc,char **argv)
 			return 1;
 		}
 	} else if (command == "bond") {
-		/* zerotier-cli bond <cmd> */
+		/* zerotier-cli bond */
 		if (arg1.empty()) {
-			printf("(bond) command is missing required arguments" ZT_EOL_S);
+			printf("(bond) command is missing required arugments" ZT_EOL_S);
 			return 2;
 		}
 		/* zerotier-cli bond list */
 		if (arg1 == "list") {
+			fprintf(stderr, "zerotier-cli bond list\n");
 			const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/bonds",requestHeaders,responseHeaders,responseBody);
 			if (scode == 0) {
 				printf("Error connecting to the ZeroTier service: %s\n\nPlease check that the service is running and that TCP port 9993 can be contacted via 127.0.0.1." ZT_EOL_S, responseBody.c_str());
@@ -528,11 +525,11 @@ static int cli(int argc,char **argv)
 						for(unsigned long k=0;k<j.size();++k) {
 							nlohmann::json &p = j[k];
 							bool isBonded = p["isBonded"];
+							int8_t bondingPolicy = p["bondingPolicy"];
+							bool isHealthy = p["isHealthy"];
+							int8_t numAliveLinks = p["numAliveLinks"];
+							int8_t numTotalLinks = p["numTotalLinks"];
 							if (isBonded) {
-								int8_t bondingPolicy = p["bondingPolicy"];
-								bool isHealthy = p["isHealthy"];
-								int8_t numAliveLinks = p["numAliveLinks"];
-								int8_t numTotalLinks = p["numTotalLinks"];
 								bFoundBond = true;
 								std::string healthStr;
 								if (isHealthy) {
@@ -541,8 +538,8 @@ static int cli(int argc,char **argv)
 									healthStr = "DEGRADED";
 								}
 								std::string policyStr = "none";
-								if (bondingPolicy >= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) {
-									policyStr = Bond::getPolicyStrByCode(bondingPolicy);
+								if (bondingPolicy >= ZT_BONDING_POLICY_NONE && bondingPolicy <= ZT_BONDING_POLICY_BALANCE_AWARE) {
+									policyStr = BondController::getPolicyStrByCode(bondingPolicy);
 								}
 								printf("%10s  %32s    %8s        %d/%d" ZT_EOL_S,
 									OSUtils::jsonString(p ["address"],"-").c_str(),
@@ -563,7 +560,11 @@ static int cli(int argc,char **argv)
 				return 1;
 			}
 		}
-		else if (arg1.length() == 10) {
+		else if (arg1.length() == 10) { /* zerotier-cli bond <peerId> enable */
+			if (arg2 == "enable") {
+				fprintf(stderr, "zerotier-cli bond <peerId> enable\n");
+				return 0;
+			}
 			if (arg2 == "rotate") { /* zerotier-cli bond <peerId> rotate */
 				fprintf(stderr, "zerotier-cli bond <peerId> rotate\n");
 				requestHeaders["Content-Type"] = "application/json";
@@ -617,6 +618,7 @@ static int cli(int argc,char **argv)
 					if (json) {
 						printf("%s" ZT_EOL_S,OSUtils::jsonDump(j).c_str());
 					} else {
+						bool bFoundBond = false;
 						std::string healthStr;
 						if (OSUtils::jsonInt(j["isHealthy"],0)) {
 							healthStr = "Healthy";
@@ -627,15 +629,15 @@ static int cli(int argc,char **argv)
 						int numTotalLinks = OSUtils::jsonInt(j["numTotalLinks"],0);
 						printf("Peer               : %s\n", arg1.c_str());
 						printf("Bond               : %s\n", OSUtils::jsonString(j["bondingPolicy"],"-").c_str());
-						//if (bondingPolicy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
-						printf("Link Select Method : %d\n", (int)OSUtils::jsonInt(j["linkSelectMethod"],0));
+						//if (bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) {
+						printf("Link Select Method : %d\n", OSUtils::jsonInt(j["linkSelectMethod"],0));
 						//}
 						printf("Status             : %s\n", healthStr.c_str());
 						printf("Links              : %d/%d\n", numAliveLinks, numTotalLinks);
-						printf("Failover Interval  : %d (ms)\n", (int)OSUtils::jsonInt(j["failoverInterval"],0));
-						printf("Up Delay           : %d (ms)\n", (int)OSUtils::jsonInt(j["upDelay"],0));
-						printf("Down Delay         : %d (ms)\n", (int)OSUtils::jsonInt(j["downDelay"],0));
-						printf("Packets Per Link   : %d (ms)\n", (int)OSUtils::jsonInt(j["packetsPerLink"],0));
+						printf("Failover Interval  : %d (ms)\n", OSUtils::jsonInt(j["failoverInterval"],0));
+						printf("Up Delay           : %d (ms)\n", OSUtils::jsonInt(j["upDelay"],0));
+						printf("Down Delay         : %d (ms)\n", OSUtils::jsonInt(j["downDelay"],0));
+						printf("Packets Per Link   : %d (ms)\n", OSUtils::jsonInt(j["packetsPerLink"],0));
 						nlohmann::json &p = j["links"];
 						if (p.is_array()) {
 							printf("\n     Interface Name\t\t\t\t\t     Path\t Alive\n");
@@ -647,7 +649,7 @@ static int cli(int argc,char **argv)
 									i,
 									OSUtils::jsonString(p[i]["ifname"],"-").c_str(),
 									OSUtils::jsonString(p[i]["path"],"-").c_str(),
-									(int)OSUtils::jsonInt(p[i]["alive"],0));
+									OSUtils::jsonInt(p[i]["alive"],0));
 							}
 							printf("\n        Latency     Jitter     Loss     Error        Speed   Alloc\n");
 							for(int i=0; i<80; i++) { printf("-"); }
@@ -660,8 +662,8 @@ static int cli(int argc,char **argv)
 									OSUtils::jsonDouble(p[i]["latencyVariance"], 0),
 									OSUtils::jsonDouble(p[i]["packetLossRatio"], 0),
 									OSUtils::jsonDouble(p[i]["packetErrorRatio"], 0),
-									(int)OSUtils::jsonInt(p[i]["givenLinkSpeed"], 0),
-									(int)OSUtils::jsonInt(p[i]["allocation"], 0));
+									OSUtils::jsonInt(p[i]["givenLinkSpeed"], 0),
+									OSUtils::jsonInt(p[i]["allocation"], 0));
 							}
 						}
 					}
@@ -674,7 +676,7 @@ static int cli(int argc,char **argv)
 			}
 		}
 		/* zerotier-cli bond command was malformed in some way */
-		printf("(bond) command is missing required arguments" ZT_EOL_S);
+		printf("(bond) command is missing required arugments" ZT_EOL_S);
 		return 2;
 		const unsigned int scode = Http::GET(1024 * 1024 * 16,60000,(const struct sockaddr *)&addr,"/bonds",requestHeaders,responseHeaders,responseBody);
 		if (scode == 0) {
@@ -709,13 +711,14 @@ static int cli(int argc,char **argv)
 				if (j.is_array()) {
 					for(unsigned long k=0;k<j.size();++k) {
 						nlohmann::json &p = j[k];
+
 						bool isBonded = p["isBonded"];
-						if (isBonded) {
-							int8_t bondingPolicy = p["bondingPolicy"];
-							bool isHealthy = p["isHealthy"];
-							int8_t numAliveLinks = p["numAliveLinks"];
-							int8_t numTotalLinks = p["numTotalLinks"];
+						int8_t bondingPolicy = p["bondingPolicy"];
+						bool isHealthy = p["isHealthy"];
+						int8_t numAliveLinks = p["numAliveLinks"];
+						int8_t numTotalLinks = p["numTotalLinks"];
 
+						if (isBonded) {
 							bFoundBond = true;
 							std::string healthStr;
 							if (isHealthy) {
@@ -724,8 +727,8 @@ static int cli(int argc,char **argv)
 								healthStr = "Degraded";
 							}
 							std::string policyStr = "none";
-							if (bondingPolicy >= ZT_BOND_POLICY_NONE && bondingPolicy <= ZT_BOND_POLICY_BALANCE_AWARE) {
-								policyStr = Bond::getPolicyStrByCode(bondingPolicy);
+							if (bondingPolicy >= ZT_BONDING_POLICY_NONE && bondingPolicy <= ZT_BONDING_POLICY_BALANCE_AWARE) {
+								policyStr = BondController::getPolicyStrByCode(bondingPolicy);
 							}
 
 							printf("%10s  %32s    %8s        %d/%d" ZT_EOL_S,
@@ -786,23 +789,14 @@ static int cli(int argc,char **argv)
 								}
 							}
 							if (aa.length() == 0) aa = "-";
-							const std::string status = OSUtils::jsonString(n["status"],"-");
 							printf("200 listnetworks %s %s %s %s %s %s %s" ZT_EOL_S,
 								OSUtils::jsonString(n["nwid"],"-").c_str(),
 								OSUtils::jsonString(n["name"],"-").c_str(),
 								OSUtils::jsonString(n["mac"],"-").c_str(),
-								status.c_str(),
+								OSUtils::jsonString(n["status"],"-").c_str(),
 								OSUtils::jsonString(n["type"],"-").c_str(),
 								OSUtils::jsonString(n["portDeviceName"],"-").c_str(),
 								aa.c_str());
-							if (OSUtils::jsonBool(n["ssoEnabled"], false)) {
-								uint64_t authenticationExpiryTime = n["authenticationExpiryTime"];
-								if (status == "AUTHENTICATION_REQUIRED") {
-									printf("    AUTH EXPIRED, URL: %s" ZT_EOL_S, OSUtils::jsonString(n["authenticationURL"], "(null)").c_str());
-								} else if (status == "OK") {
-									printf("    AUTH OK, expires in: %lld seconds" ZT_EOL_S, ((int64_t)authenticationExpiryTime - OSUtils::now()) / 1000LL);
-								}
-							}
 						}
 					}
 				}
@@ -1316,6 +1310,7 @@ static int cli(int argc,char **argv)
 		struct ifconf ifc;
 		char buf[1024];
 		char stringBuffer[128];
+		int success = 0;
 
 		int sock = socket(AF_INET, SOCK_DGRAM, IPPROTO_IP);
 

selftest.cpp

@@ -561,8 +561,8 @@ static int testCertificate()
 	std::cout << idA.address().toString(buf) << ", " << idB.address().toString(buf) << std::endl;
 
 	std::cout << "[certificate] Generating certificates A and B...";
-	CertificateOfMembership cA(10000,100,1,idA);
-	CertificateOfMembership cB(10099,100,1,idB);
+	CertificateOfMembership cA(10000,100,1,idA.address());
+	CertificateOfMembership cB(10099,100,1,idB.address());
 	std::cout << std::endl;
 
 	std::cout << "[certificate] Signing certificates A and B with authority...";
@@ -574,13 +574,13 @@ static int testCertificate()
 	//std::cout << "[certificate] B: " << cB.toString() << std::endl;
 
 	std::cout << "[certificate] A agrees with B and B with A... ";
-	if (cA.agreesWith(cB, idB))
+	if (cA.agreesWith(cB))
 		std::cout << "yes, ";
 	else {
 		std::cout << "FAIL" << std::endl;
 		return -1;
 	}
-	if (cB.agreesWith(cA, idA))
+	if (cB.agreesWith(cA))
 		std::cout << "yes." << std::endl;
 	else {
 		std::cout << "FAIL" << std::endl;
@@ -588,18 +588,18 @@ static int testCertificate()
 	}
 
 	std::cout << "[certificate] Generating two certificates that should not agree...";
-	cA = CertificateOfMembership(10000,100,1,idA);
-	cB = CertificateOfMembership(10101,100,1,idB);
+	cA = CertificateOfMembership(10000,100,1,idA.address());
+	cB = CertificateOfMembership(10101,100,1,idB.address());
 	std::cout << std::endl;
 
 	std::cout << "[certificate] A agrees with B and B with A... ";
-	if (!cA.agreesWith(cB, idB))
+	if (!cA.agreesWith(cB))
 		std::cout << "no, ";
 	else {
 		std::cout << "FAIL" << std::endl;
 		return -1;
 	}
-	if (!cB.agreesWith(cA, idA))
+	if (!cB.agreesWith(cA))
 		std::cout << "no." << std::endl;
 	else {
 		std::cout << "FAIL" << std::endl;

service/OneService.cpp

@@ -49,6 +49,7 @@
 #include "../osdep/Binder.hpp"
 #include "../osdep/ManagedRoute.hpp"
 #include "../osdep/BlockingQueue.hpp"
+#include "../osdep/Link.hpp"
 
 #include "OneService.hpp"
 #include "SoftwareUpdater.hpp"
@@ -183,7 +184,6 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc,
 		case ZT_NETWORK_STATUS_NOT_FOUND:                nstatus = "NOT_FOUND"; break;
 		case ZT_NETWORK_STATUS_PORT_ERROR:               nstatus = "PORT_ERROR"; break;
 		case ZT_NETWORK_STATUS_CLIENT_TOO_OLD:           nstatus = "CLIENT_TOO_OLD"; break;
-		case ZT_NETWORK_STATUS_AUTHENTICATION_REQUIRED:  nstatus = "AUTHENTICATION_REQUIRED"; break;
 	}
 	switch(nc->type) {
 		case ZT_NETWORK_TYPE_PRIVATE:                    ntype = "PRIVATE"; break;
@@ -251,9 +251,6 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc,
 	}
 	nj["dns"] = m;
 
-	nj["authenticationURL"] = nc->authenticationURL;
-	nj["authenticationExpiryTime"] = nc->authenticationExpiryTime;
-	nj["ssoEnabled"] = nc->ssoEnabled;
 }
 
 static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer)
@@ -303,11 +300,12 @@ static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer)
 
 static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 {
+	char tmp[256];
 	uint64_t now = OSUtils::now();
 
-	int bondingPolicy = bond->policy();
-	pj["bondingPolicy"] = Bond::getPolicyStrByCode(bondingPolicy);
-	if (bondingPolicy == ZT_BOND_POLICY_NONE) {
+	int bondingPolicy = bond->getPolicy();
+	pj["bondingPolicy"] = BondController::getPolicyStrByCode(bondingPolicy);
+	if (bondingPolicy == ZT_BONDING_POLICY_NONE) {
 		return;
 	}
 
@@ -317,15 +315,15 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 	pj["failoverInterval"] = bond->getFailoverInterval();
 	pj["downDelay"] = bond->getDownDelay();
 	pj["upDelay"] = bond->getUpDelay();
-	if (bondingPolicy == ZT_BOND_POLICY_BALANCE_RR) {
+	if (bondingPolicy == ZT_BONDING_POLICY_BALANCE_RR) {
 		pj["packetsPerLink"] = bond->getPacketsPerLink();
 	}
-	if (bondingPolicy == ZT_BOND_POLICY_ACTIVE_BACKUP) {
+	if (bondingPolicy == ZT_BONDING_POLICY_ACTIVE_BACKUP) {
 		pj["linkSelectMethod"] = bond->getLinkSelectMethod();
 	}
 
 	nlohmann::json pa = nlohmann::json::array();
-	std::vector< SharedPtr<Path> > paths = bond->paths(now);
+	std::vector< SharedPtr<Path> > paths = bond->getPeer()->paths(now);
 
 	for(unsigned int i=0;i<paths.size();++i) {
 		char pathStr[128];
@@ -334,7 +332,6 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 		nlohmann::json j;
 		j["ifname"] = bond->getLink(paths[i])->ifname();
 		j["path"] = pathStr;
-		/*
 		j["alive"] = paths[i]->alive(now,true);
 		j["bonded"] = paths[i]->bonded();
 		j["latencyMean"] = paths[i]->latencyMean();
@@ -343,7 +340,6 @@ static void _bondToJson(nlohmann::json &pj, SharedPtr<Bond> &bond)
 		j["packetErrorRatio"] = paths[i]->packetErrorRatio();
 		j["givenLinkSpeed"] = 1000;
 		j["allocation"] = paths[i]->allocation();
-		*/
 		pa.push_back(j);
 	}
 	pj["links"] = pa;
@@ -539,12 +535,6 @@ public:
 			memset(&config, 0, sizeof(ZT_VirtualNetworkConfig));
 		}
 
-		~NetworkState()
-		{
-			this->managedRoutes.clear();
-			this->tap.reset();
-		}
-
 		std::shared_ptr<EthernetTap> tap;
 		ZT_VirtualNetworkConfig config; // memcpy() of raw config from core
 		std::vector<InetAddress> managedIps;
@@ -583,7 +573,6 @@ public:
 	Mutex _run_m;
 
 	RedisConfig *_rc;
-	std::string _ssoRedirectURL;
 
 	// end member variables ----------------------------------------------------
 
@@ -621,7 +610,6 @@ public:
 #endif
 		,_run(true)
 		,_rc(NULL)
-		,_ssoRedirectURL()
 	{
 		_ports[0] = 0;
 		_ports[1] = 0;
@@ -735,22 +723,25 @@ public:
 			OSUtils::ztsnprintf(portstr,sizeof(portstr),"%u",_ports[0]);
 			OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr));
 
-			// Attempt to bind to a secondary port.
+			// Attempt to bind to a secondary port chosen from our ZeroTier address.
 			// This exists because there are buggy NATs out there that fail if more
 			// than one device behind the same NAT tries to use the same internal
 			// private address port number. Buggy NATs are a running theme.
-			//
-			// This used to pick the secondary port based on the node ID until we
-			// discovered another problem: buggy routers and malicious traffic 
-			// "detection".  A lot of routers have such things built in these days
-			// and mis-detect ZeroTier traffic as malicious and block it resulting
-			// in a node that appears to be in a coma.  Secondary ports are now 
-			// randomized on startup.
 			if (_allowSecondaryPort) {
 				if (_secondaryPort) {
 					_ports[1] = _secondaryPort;
 				} else {
-					_ports[1] = _getRandomPort();
+					_ports[1] = 20000 + ((unsigned int)_node->address() % 45500);
+					for(int i=0;;++i) {
+						if (i > 1000) {
+							_ports[1] = 0;
+							break;
+						} else if (++_ports[1] >= 65536) {
+							_ports[1] = 20000;
+						}
+						if (_trialBind(_ports[1]))
+							break;
+					}
 				}
 			}
 #ifdef ZT_USE_MINIUPNPC
@@ -762,7 +753,7 @@ public:
 					if (_tertiaryPort) {
 						_ports[2] = _tertiaryPort;
 					} else {
-						_ports[2] = 20000 + (_ports[0] % 40000);
+						_ports[2] = _ports[1];
 						for(int i=0;;++i) {
 							if (i > 1000) {
 								_ports[2] = 0;
@@ -788,9 +779,6 @@ public:
 
 			// Network controller is now enabled by default for desktop and server
 			_controller = new EmbeddedNetworkController(_node,_homePath.c_str(),_controllerDbPath.c_str(),_ports[0], _rc);
-			if (!_ssoRedirectURL.empty()) {
-				_controller->setSSORedirectURL(_ssoRedirectURL);
-			}
 			_node->setNetconfMaster((void *)_controller);
 
 			// Join existing networks in networks.d
@@ -823,7 +811,6 @@ public:
 			int64_t lastCleanedPeersDb = 0;
 			int64_t lastLocalInterfaceAddressCheck = (clockShouldBe - ZT_LOCAL_INTERFACE_CHECK_INTERVAL) + 15000; // do this in 15s to give portmapper time to configure and other things time to settle
 			int64_t lastLocalConfFileCheck = OSUtils::now();
-			int64_t lastOnline = lastLocalConfFileCheck;
 			for(;;) {
 				_run_m.lock();
 				if (!_run) {
@@ -865,16 +852,6 @@ public:
 					}
 				}
 
-				// If secondary port is not configured to a constant value and we've been offline for a while,
-				// bind a new secondary port. This is a workaround for a "coma" issue caused by buggy NATs that stop
-				// working on one port after a while.
-				if (_node->online()) {
-					lastOnline = now;
-				} else if ((_secondaryPort == 0)&&((now - lastOnline) > ZT_PATH_HEARTBEAT_PERIOD)) {
-					_secondaryPort = _getRandomPort();
-					lastBindRefresh = 0;
-				}
-
 				// Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow routes (e.g. shadow default)
 				if (((now - lastBindRefresh) >= (_node->bondController()->inUse() ? ZT_BINDER_REFRESH_PERIOD / 4 : ZT_BINDER_REFRESH_PERIOD))||(restarted)) {
 					lastBindRefresh = now;
@@ -1035,11 +1012,8 @@ public:
 			}
 		}
 
-		// Make a copy so lookups don't modify in place;
-		json lc(_localConfig);
-
 		// Get any trusted paths in local.conf (we'll parse the rest of physical[] elsewhere)
-		json &physical = lc["physical"];
+		json &physical = _localConfig["physical"];
 		if (physical.is_object()) {
 			for(json::iterator phy(physical.begin());phy!=physical.end();++phy) {
 				InetAddress net(OSUtils::jsonString(phy.key(),"").c_str());
@@ -1056,22 +1030,20 @@ public:
 			}
 		}
 
-		json &settings = lc["settings"];
+		json &settings = _localConfig["settings"];
 		if (settings.is_object()) {
 			// Allow controller DB path to be put somewhere else
 			const std::string cdbp(OSUtils::jsonString(settings["controllerDbPath"],""));
 			if (cdbp.length() > 0)
 				_controllerDbPath = cdbp;
 
-			_ssoRedirectURL = OSUtils::jsonString(settings["ssoRedirectURL"], "");
-
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 			// TODO:  Redis config
 			json &redis = settings["redis"];
 			if (redis.is_object() && _rc == NULL) {
 				_rc = new RedisConfig;
 				_rc->hostname = OSUtils::jsonString(redis["hostname"],"");
-				_rc->port = OSUtils::jsonInt(redis["port"],0);
+				_rc->port = redis["port"];
 				_rc->password = OSUtils::jsonString(redis["password"],"");
 				_rc->clusterMode = OSUtils::jsonBool(redis["clusterMode"], false);
 			}
@@ -1287,7 +1259,7 @@ public:
 										_bondToJson(res,bond);
 										scode = 200;
 									} else {
-										fprintf(stderr, "unable to find bond to peer %llx\n", (unsigned long long)id);
+										fprintf(stderr, "unable to find bond to peer %llx\n", id);
 										scode = 400;
 									}
 								}
@@ -1299,11 +1271,8 @@ public:
 					} else {
 						scode = 400; /* bond controller is not enabled */
 					}
-				} else if (ps[0] == "config") {
-					Mutex::Lock lc(_localConfig_m);
-					res = _localConfig;
-					scode = 200;
-				} else if (ps[0] == "status") {
+				}
+				if (ps[0] == "status") {
 					ZT_NodeStatus status;
 					_node->status(&status);
 
@@ -1497,7 +1466,7 @@ public:
 									if (bond) {
 										scode = bond->abForciblyRotateLink() ? 200 : 400;
 									} else {
-										fprintf(stderr, "unable to find bond to peer %llx\n", (unsigned long long)id);
+										fprintf(stderr, "unable to find bond to peer %llx\n", id);
 										scode = 400;
 									}
 								}
@@ -1509,35 +1478,8 @@ public:
 					} else {
 						scode = 400; /* bond controller is not enabled */
 					}
-				} else if (ps[0] == "config") {
-					// Right now we only support writing the things the UI supports changing.
-					if (ps.size() == 2) {
-						if (ps[1] == "settings") {
-							try {
-								json j(OSUtils::jsonParse(body));
-								if (j.is_object()) {
-									Mutex::Lock lcl(_localConfig_m);
-									json lc(_localConfig);
-									for(json::const_iterator s(j.begin());s!=j.end();++s) {
-										lc["settings"][s.key()] = s.value();
-									}
-									std::string lcStr = OSUtils::jsonDump(lc, 4);
-									if (OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(), lcStr)) {
-										_localConfig = lc;
-									}
-								} else {
-									scode = 400;
-								}
-							} catch ( ... ) {
-								scode = 400;
-							}
-						} else {
-							scode = 404;
-						}
-					} else {
-						scode = 404;
-					}
-				} else if (ps[0] == "moon") {
+				}
+				if (ps[0] == "moon") {
 					if (ps.size() == 2) {
 
 						uint64_t seed = 0;
@@ -1769,11 +1711,11 @@ public:
 				if (basePolicyStr.empty()) {
 					fprintf(stderr, "error: no base policy was specified for custom policy (%s)\n", customPolicyStr.c_str());
 				}
-				if (_node->bondController()->getPolicyCodeByStr(basePolicyStr) == ZT_BOND_POLICY_NONE) {
+				if (_node->bondController()->getPolicyCodeByStr(basePolicyStr) == ZT_BONDING_POLICY_NONE) {
 					fprintf(stderr, "error: custom policy (%s) is invalid, unknown base policy (%s).\n",
 						customPolicyStr.c_str(), basePolicyStr.c_str());
 					continue;
-				} if (_node->bondController()->getPolicyCodeByStr(customPolicyStr) != ZT_BOND_POLICY_NONE) {
+				} if (_node->bondController()->getPolicyCodeByStr(customPolicyStr) != ZT_BONDING_POLICY_NONE) {
 					fprintf(stderr, "error: custom policy (%s) will be ignored, cannot use standard policy names for custom policies.\n",
 						customPolicyStr.c_str());
 					continue;
@@ -1802,12 +1744,20 @@ public:
 					newTemplateBond->setUserQualityWeights(weights,ZT_QOS_WEIGHT_SIZE);
 				}
 				// Bond-specific properties
+				newTemplateBond->setOverflowMode(OSUtils::jsonInt(customPolicy["overflow"],false));
 				newTemplateBond->setUpDelay(OSUtils::jsonInt(customPolicy["upDelay"],-1));
 				newTemplateBond->setDownDelay(OSUtils::jsonInt(customPolicy["downDelay"],-1));
 				newTemplateBond->setFlowRebalanceStrategy(OSUtils::jsonInt(customPolicy["flowRebalanceStrategy"],(uint64_t)0));
 				newTemplateBond->setFailoverInterval(OSUtils::jsonInt(customPolicy["failoverInterval"],(uint64_t)0));
 				newTemplateBond->setPacketsPerLink(OSUtils::jsonInt(customPolicy["packetsPerLink"],-1));
 
+				std::string linkMonitorStrategyStr(OSUtils::jsonString(customPolicy["linkMonitorStrategy"],""));
+				uint8_t linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DEFAULT;
+				if (linkMonitorStrategyStr == "passive") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_PASSIVE; }
+				if (linkMonitorStrategyStr == "active") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_ACTIVE; }
+				if (linkMonitorStrategyStr == "dynamic") { linkMonitorStrategy = ZT_MULTIPATH_SLAVE_MONITOR_STRATEGY_DYNAMIC; }
+				newTemplateBond->setLinkMonitorStrategy(linkMonitorStrategy);
+
 				// Policy-Specific link set
 				json &links = customPolicy["links"];
 				for (json::iterator linkItr = links.begin(); linkItr != links.end();++linkItr) {
@@ -1823,40 +1773,40 @@ public:
 							speed, alloc, linkNameStr.c_str());
 						enabled = false;
 					}
-					//uint32_t upDelay = OSUtils::jsonInt(link["upDelay"],-1);
-					//uint32_t downDelay = OSUtils::jsonInt(link["downDelay"],-1);
+					uint32_t upDelay = OSUtils::jsonInt(link["upDelay"],-1);
+					uint32_t downDelay = OSUtils::jsonInt(link["downDelay"],-1);
 					uint8_t ipvPref = OSUtils::jsonInt(link["ipvPref"],0);
-					//uint32_t linkMonitorInterval = OSUtils::jsonInt(link["monitorInterval"],(uint64_t)0);
+					uint32_t linkMonitorInterval = OSUtils::jsonInt(link["monitorInterval"],(uint64_t)0);
 					std::string failoverToStr(OSUtils::jsonString(link["failoverTo"],""));
 					// Mode
 					std::string linkModeStr(OSUtils::jsonString(link["mode"],"spare"));
-					uint8_t linkMode = ZT_BOND_SLAVE_MODE_SPARE;
-					if (linkModeStr == "primary") { linkMode = ZT_BOND_SLAVE_MODE_PRIMARY; }
-					if (linkModeStr == "spare") { linkMode = ZT_BOND_SLAVE_MODE_SPARE; }
+					uint8_t linkMode = ZT_MULTIPATH_SLAVE_MODE_SPARE;
+					if (linkModeStr == "primary") { linkMode = ZT_MULTIPATH_SLAVE_MODE_PRIMARY; }
+					if (linkModeStr == "spare") { linkMode = ZT_MULTIPATH_SLAVE_MODE_SPARE; }
 					// ipvPref
 					if ((ipvPref != 0) && (ipvPref != 4) && (ipvPref != 6) && (ipvPref != 46) && (ipvPref != 64)) {
 						fprintf(stderr, "error: invalid ipvPref value (%d), link disabled.\n", ipvPref);
 						enabled = false;
 					}
-					if (linkMode == ZT_BOND_SLAVE_MODE_SPARE && failoverToStr.length()) {
+					if (linkMode == ZT_MULTIPATH_SLAVE_MODE_SPARE && failoverToStr.length()) {
 						fprintf(stderr, "error: cannot specify failover links for spares, link disabled.\n");
 						failoverToStr = "";
 						enabled = false;
 					}
-					_node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,speed,enabled,linkMode,failoverToStr,alloc));
+					_node->bondController()->addCustomLink(customPolicyStr, new Link(linkNameStr,ipvPref,speed,linkMonitorInterval,upDelay,downDelay,enabled,linkMode,failoverToStr,alloc));
 				}
 				std::string linkSelectMethodStr(OSUtils::jsonString(customPolicy["activeReselect"],"optimize"));
 				if (linkSelectMethodStr == "always") {
-					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_ALWAYS);
+					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_ALWAYS);
 				}
 				if (linkSelectMethodStr == "better") {
-					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_BETTER);
+					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_BETTER);
 				}
 				if (linkSelectMethodStr == "failure") {
-					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_FAILURE);
+					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_FAILURE);
 				}
 				if (linkSelectMethodStr == "optimize") {
-					newTemplateBond->setLinkSelectMethod(ZT_BOND_RESELECTION_POLICY_OPTIMIZE);
+					newTemplateBond->setLinkSelectMethod(ZT_MULTIPATH_RESELECTION_POLICY_OPTIMIZE);
 				}
 				if (newTemplateBond->getLinkSelectMethod() < 0 || newTemplateBond->getLinkSelectMethod() > 3) {
 					fprintf(stderr, "warning: invalid value (%s) for linkSelectMethod, assuming mode: always\n", linkSelectMethodStr.c_str());
@@ -1889,7 +1839,7 @@ public:
 		_secondaryPort = (unsigned int)OSUtils::jsonInt(settings["secondaryPort"],0);
 		_tertiaryPort = (unsigned int)OSUtils::jsonInt(settings["tertiaryPort"],0);
 		if (_secondaryPort != 0 || _tertiaryPort != 0) {
-			fprintf(stderr,"WARNING: using manually-specified secondary and/or tertiary ports. This can cause NAT issues." ZT_EOL_S);
+			fprintf(stderr,"WARNING: using manually-specified ports. This can cause NAT issues." ZT_EOL_S);
 		}
 		_portMappingEnabled = OSUtils::jsonBool(settings["portMappingEnabled"],true);
 
@@ -2078,7 +2028,7 @@ public:
 				unsigned int mostMatchingPrefixBits = 0;
 				for(std::set<InetAddress>::const_iterator i(myIps.begin());i!=myIps.end();++i) {
 					const unsigned int matchingPrefixBits = i->matchingPrefixBits(*target);
-					if (matchingPrefixBits >= mostMatchingPrefixBits && ((target->isV4() && i->isV4()) || (target->isV6() && i->isV6()))) {
+					if (matchingPrefixBits >= mostMatchingPrefixBits) {
 						mostMatchingPrefixBits = matchingPrefixBits;
 						src = &(*i);
 					}
@@ -2441,7 +2391,7 @@ public:
 							Dictionary<4096> nc;
 							nc.load(nlcbuf.c_str());
 							Buffer<1024> allowManaged;
-							if (nc.get("allowManaged", allowManaged) && allowManaged.size() > 0) {
+							if (nc.get("allowManaged", allowManaged) && !allowManaged.size() == 0) {
 								std::string addresses (allowManaged.begin(), allowManaged.size());
 								if (allowManaged.size() <= 5) { // untidy parsing for backward compatibility
 									if (allowManaged[0] == '1' || allowManaged[0] == 't' || allowManaged[0] == 'T') {
@@ -2675,6 +2625,7 @@ public:
 			case ZT_STATE_OBJECT_NETWORK_CONFIG:
 				OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "networks.d",_homePath.c_str());
 				OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.conf",dirname,(unsigned long long)id[0]);
+				secure = true;
 				break;
 			case ZT_STATE_OBJECT_PEER:
 				OSUtils::ztsnprintf(dirname,sizeof(dirname),"%s" ZT_PATH_SEPARATOR_S "peers.d",_homePath.c_str());
@@ -3093,6 +3044,9 @@ public:
 				if (!strncmp(p->c_str(),ifname,p->length()))
 					return false;
 			}
+			if (!_node->bondController()->allowedToBind(std::string(ifname))) {
+				return false;
+			}
 		}
 		{
 			// Check global blacklists
@@ -3130,23 +3084,6 @@ public:
 		return true;
 	}
 
-	unsigned int _getRandomPort()
-	{
-		unsigned int randp = 0;
-		Utils::getSecureRandom(&randp,sizeof(randp));
-		randp = 20000 + (randp % 45500);
-		for(int i=0;;++i) {
-			if (i > 1000) {
-				return 0;
-			} else if (++randp >= 65536) {
-				randp = 20000;
-			}
-			if (_trialBind(randp))
-				break;
-		}
-		return randp;
-	}
-
 	bool _trialBind(unsigned int port)
 	{
 		struct sockaddr_in in4;