Merge remote-tracking branch 'origin/dev' into multipath

RELEASE-NOTES.md

@@ -1,6 +1,12 @@
 ZeroTier Release Notes
 ======
 
+# 2019-08-12 -- Version 1.4.2
+
+ * Fix high CPU use bug on some platforms
+ * Fix issues with PostgreSQL controller DB (only affects Central)
+ * Restore backward compatibility with MacOS versions prior to 10.13
+
 # 2019-07-29 -- Version 1.4.0
 
 ### Major Changes

controller/DB.cpp

@@ -52,6 +52,7 @@ void DB::initNetwork(nlohmann::json &network)
 	if (!network.count("mtu")) network["mtu"] = ZT_DEFAULT_MTU;
 	if (!network.count("remoteTraceTarget")) network["remoteTraceTarget"] = nlohmann::json();
 	if (!network.count("removeTraceLevel")) network["remoteTraceLevel"] = 0;
+	if (!network.count("rulesSource")) network["rulesSource"] = "";
 	if (!network.count("rules")) {
 		// If unspecified, rules are set to allow anything and behave like a flat L2 segment
 		network["rules"] = {{
@@ -104,16 +105,7 @@ void DB::cleanMember(nlohmann::json &member)
 	member.erase("lastRequestMetaData");
 }
 
-DB::DB(const Identity &myId,const char *path) :
-	_myId(myId),
-	_myAddress(myId.address()),
-	_path((path) ? path : "")
-{
-	char tmp[32];
-	_myAddress.toString(tmp);
-	_myAddressStr = tmp;
-}
-
+DB::DB() {}
 DB::~DB() {}
 
 bool DB::get(const uint64_t networkId,nlohmann::json &network)
@@ -199,34 +191,15 @@ bool DB::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohma
 	return true;
 }
 
-bool DB::summary(const uint64_t networkId,NetworkSummaryInfo &info)
-{
-	waitForReady();
-	std::shared_ptr<_Network> nw;
-	{
-		std::lock_guard<std::mutex> l(_networks_l);
-		auto nwi = _networks.find(networkId);
-		if (nwi == _networks.end())
-			return false;
-		nw = nwi->second;
-	}
-	{
-		std::lock_guard<std::mutex> l2(nw->lock);
-		_fillSummaryInfo(nw,info);
-	}
-	return true;
-}
-
-void DB::networks(std::vector<uint64_t> &networks)
+void DB::networks(std::set<uint64_t> &networks)
 {
 	waitForReady();
 	std::lock_guard<std::mutex> l(_networks_l);
-	networks.reserve(_networks.size() + 1);
 	for(auto n=_networks.begin();n!=_networks.end();++n)
-		networks.push_back(n->first);
+		networks.insert(n->first);
 }
 
-void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool initialized)
+void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners)
 {
 	uint64_t memberId = 0;
 	uint64_t networkId = 0;
@@ -310,10 +283,10 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool in
 			}
 		}
 
-		if (initialized) {
+		if (notifyListeners) {
 			std::lock_guard<std::mutex> ll(_changeListeners_l);
 			for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) {
-				(*i)->onNetworkMemberUpdate(networkId,memberId,memberConfig);
+				(*i)->onNetworkMemberUpdate(this,networkId,memberId,memberConfig);
 			}
 		}
 	} else if (memberId) {
@@ -333,15 +306,15 @@ void DB::_memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool in
 		}
 	}
 
-	if ((initialized)&&((wasAuth)&&(!isAuth)&&(networkId)&&(memberId))) {
+	if ((notifyListeners)&&((wasAuth)&&(!isAuth)&&(networkId)&&(memberId))) {
 		std::lock_guard<std::mutex> ll(_changeListeners_l);
 		for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) {
-			(*i)->onNetworkMemberDeauthorize(networkId,memberId);
+			(*i)->onNetworkMemberDeauthorize(this,networkId,memberId);
 		}
 	}
 }
 
-void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool initialized)
+void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners)
 {
 	if (networkConfig.is_object()) {
 		const std::string ids = networkConfig["id"];
@@ -359,10 +332,10 @@ void DB::_networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool
 				std::lock_guard<std::mutex> l2(nw->lock);
 				nw->config = networkConfig;
 			}
-			if (initialized) {
+			if (notifyListeners) {
 				std::lock_guard<std::mutex> ll(_changeListeners_l);
 				for(auto i=_changeListeners.begin();i!=_changeListeners.end();++i) {
-					(*i)->onNetworkUpdate(networkId,networkConfig);
+					(*i)->onNetworkUpdate(this,networkId,networkConfig);
 				}
 			}
 		}

controller/DB.hpp

@@ -27,6 +27,8 @@
 #ifndef ZT_CONTROLLER_DB_HPP
 #define ZT_CONTROLLER_DB_HPP
 
+//#define ZT_CONTROLLER_USE_LIBPQ
+
 #include "../node/Constants.hpp"
 #include "../node/Identity.hpp"
 #include "../node/InetAddress.hpp"
@@ -41,6 +43,7 @@
 #include <vector>
 #include <atomic>
 #include <mutex>
+#include <set>
 
 #include "../ext/json/json.hpp"
 
@@ -58,9 +61,9 @@ public:
 	public:
 		ChangeListener() {}
 		virtual ~ChangeListener() {}
-		virtual void onNetworkUpdate(uint64_t networkId,const nlohmann::json &network) {}
-		virtual void onNetworkMemberUpdate(uint64_t networkId,uint64_t memberId,const nlohmann::json &member) {}
-		virtual void onNetworkMemberDeauthorize(uint64_t networkId,uint64_t memberId) {}
+		virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network) {}
+		virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member) {}
+		virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId) {}
 	};
 
 	struct NetworkSummaryInfo
@@ -78,7 +81,7 @@ public:
 	static void cleanNetwork(nlohmann::json &network);
 	static void cleanMember(nlohmann::json &member);
 
-	DB(const Identity &myId,const char *path);
+	DB();
 	virtual ~DB();
 
 	virtual bool waitForReady() = 0;
@@ -94,12 +97,27 @@ public:
 	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member);
 	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,NetworkSummaryInfo &info);
 	bool get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members);
-	bool summary(const uint64_t networkId,NetworkSummaryInfo &info);
-	void networks(std::vector<uint64_t> &networks);
 
-	virtual void save(nlohmann::json *orig,nlohmann::json &record) = 0;
+	void networks(std::set<uint64_t> &networks);
+
+	template<typename F>
+	inline void each(F f)
+	{
+		nlohmann::json nullJson;
+		std::lock_guard<std::mutex> lck(_networks_l);
+		for(auto nw=_networks.begin();nw!=_networks.end();++nw) {
+			f(nw->first,nw->second->config,0,nullJson); // first provide network with 0 for member ID
+			for(auto m=nw->second->members.begin();m!=nw->second->members.end();++m) {
+				f(nw->first,nw->second->config,m->first,m->second);
+			}
+		}
+	}
+
+	virtual bool save(nlohmann::json &record,bool notifyListeners) = 0;
+
 	virtual void eraseNetwork(const uint64_t networkId) = 0;
 	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId) = 0;
+
 	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress) = 0;
 
 	inline void addListener(DB::ChangeListener *const listener)
@@ -109,6 +127,28 @@ public:
 	}
 
 protected:
+	static inline bool _compareRecords(const nlohmann::json &a,const nlohmann::json &b)
+	{
+		if (a.is_object() == b.is_object()) {
+			if (a.is_object()) {
+				if (a.size() != b.size())
+					return false;
+				auto amap = a.get<nlohmann::json::object_t>();
+				auto bmap = b.get<nlohmann::json::object_t>();
+				for(auto ai=amap.begin();ai!=amap.end();++ai) {
+					if (ai->first != "revision") { // ignore revision, compare only non-revision-counter fields
+						auto bi = bmap.find(ai->first);
+						if ((bi == bmap.end())||(bi->second != ai->second))
+							return false;
+					}
+				}
+				return true;
+			}
+			return (a == b);
+		}
+		return false;
+	}
+
 	struct _Network
 	{
 		_Network() : mostRecentDeauthTime(0) {}
@@ -121,15 +161,10 @@ protected:
 		std::mutex lock;
 	};
 
-	void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool initialized);
-	void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool initialized);
+	void _memberChanged(nlohmann::json &old,nlohmann::json &memberConfig,bool notifyListeners);
+	void _networkChanged(nlohmann::json &old,nlohmann::json &networkConfig,bool notifyListeners);
 	void _fillSummaryInfo(const std::shared_ptr<_Network> &nw,NetworkSummaryInfo &info);
 
-	const Identity _myId;
-	const Address _myAddress;
-	const std::string _path;
-	std::string _myAddressStr;
-
 	std::vector<DB::ChangeListener *> _changeListeners;
 	std::unordered_map< uint64_t,std::shared_ptr<_Network> > _networks;
 	std::unordered_multimap< uint64_t,uint64_t > _networkByMember;

controller/DBMirrorSet.cpp

@@ -0,0 +1,244 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#include "DBMirrorSet.hpp"
+
+namespace ZeroTier {
+
+DBMirrorSet::DBMirrorSet(DB::ChangeListener *listener) :
+	_listener(listener),
+	_running(true)
+{
+	_syncCheckerThread = std::thread([this]() {
+		for(;;) {
+			for(int i=0;i<120;++i) { // 1 minute delay between checks
+				if (!_running)
+					return;
+				std::this_thread::sleep_for(std::chrono::milliseconds(500));
+			}
+
+			std::vector< std::shared_ptr<DB> > dbs;
+			{
+				std::lock_guard<std::mutex> l(_dbs_l);
+				if (_dbs.size() <= 1)
+					continue; // no need to do this if there's only one DB, so skip the iteration
+				dbs = _dbs;
+			}
+
+			for(auto db=dbs.begin();db!=dbs.end();++db) {
+				(*db)->each([this,&dbs,&db](uint64_t networkId,const nlohmann::json &network,uint64_t memberId,const nlohmann::json &member) {
+					try {
+						if (network.is_object()) {
+							if (memberId == 0) {
+								for(auto db2=dbs.begin();db2!=dbs.end();++db2) {
+									if (db->get() != db2->get()) {
+										nlohmann::json nw2;
+										if ((!(*db2)->get(networkId,nw2))||((nw2.is_object())&&(OSUtils::jsonInt(nw2["revision"],0) < OSUtils::jsonInt(network["revision"],0)))) {
+											nw2 = network;
+											(*db2)->save(nw2,false);
+										}
+									}
+								}
+							} else if (member.is_object()) {
+								for(auto db2=dbs.begin();db2!=dbs.end();++db2) {
+									if (db->get() != db2->get()) {
+										nlohmann::json nw2,m2;
+										if ((!(*db2)->get(networkId,nw2,memberId,m2))||((m2.is_object())&&(OSUtils::jsonInt(m2["revision"],0) < OSUtils::jsonInt(member["revision"],0)))) {
+											m2 = member;
+											(*db2)->save(m2,false);
+										}
+									}
+								}
+							}
+						}
+					} catch ( ... ) {} // skip entries that generate JSON errors
+				});
+			}
+		}
+	});
+}
+
+DBMirrorSet::~DBMirrorSet()
+{
+	_running = false;
+	_syncCheckerThread.join();
+}
+
+bool DBMirrorSet::hasNetwork(const uint64_t networkId) const
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->hasNetwork(networkId))
+			return true;
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network)) {
+			return true;
+		}
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network,memberId,member))
+			return true;
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,DB::NetworkSummaryInfo &info)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network,memberId,member,info))
+			return true;
+	}
+	return false;
+}
+
+bool DBMirrorSet::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if ((*d)->get(networkId,network,members))
+			return true;
+	}
+	return false;
+}
+
+void DBMirrorSet::networks(std::set<uint64_t> &networks)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->networks(networks);
+	}
+}
+
+bool DBMirrorSet::waitForReady()
+{
+	bool r = false;
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		r |= (*d)->waitForReady();
+	}
+	return r;
+}
+
+bool DBMirrorSet::isReady()
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if (!(*d)->isReady())
+			return false;
+	}
+	return true;
+}
+
+bool DBMirrorSet::save(nlohmann::json &record,bool notifyListeners)
+{
+	std::vector< std::shared_ptr<DB> > dbs;
+	{
+		std::lock_guard<std::mutex> l(_dbs_l);
+		dbs = _dbs;
+	}
+	if (notifyListeners) {
+		for(auto d=dbs.begin();d!=dbs.end();++d) {
+			if ((*d)->save(record,true))
+				return true;
+		}
+		return false;
+	} else {
+		bool modified = false;
+		for(auto d=dbs.begin();d!=dbs.end();++d) {
+			modified |= (*d)->save(record,false);
+		}
+		return modified;
+	}
+}
+
+void DBMirrorSet::eraseNetwork(const uint64_t networkId)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->eraseNetwork(networkId);
+	}
+}
+
+void DBMirrorSet::eraseMember(const uint64_t networkId,const uint64_t memberId)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->eraseMember(networkId,memberId);
+	}
+}
+
+void DBMirrorSet::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
+{
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		(*d)->nodeIsOnline(networkId,memberId,physicalAddress);
+	}
+}
+
+void DBMirrorSet::onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network)
+{
+	nlohmann::json record(network);
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if (d->get() != db) {
+			(*d)->save(record,false);
+		}
+	}
+	_listener->onNetworkUpdate(this,networkId,network);
+}
+
+void DBMirrorSet::onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member)
+{
+	nlohmann::json record(member);
+	std::lock_guard<std::mutex> l(_dbs_l);
+	for(auto d=_dbs.begin();d!=_dbs.end();++d) {
+		if (d->get() != db) {
+			(*d)->save(record,false);
+		}
+	}
+	_listener->onNetworkMemberUpdate(this,networkId,memberId,member);
+}
+
+void DBMirrorSet::onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId)
+{
+	_listener->onNetworkMemberDeauthorize(this,networkId,memberId);
+}
+
+} // namespace ZeroTier

controller/DBMirrorSet.hpp

@@ -0,0 +1,84 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#ifndef ZT_DBMIRRORSET_HPP
+#define ZT_DBMIRRORSET_HPP
+
+#include "DB.hpp"
+
+#include <vector>
+#include <memory>
+#include <mutex>
+#include <set>
+#include <thread>
+
+namespace ZeroTier {
+
+class DBMirrorSet : public DB::ChangeListener
+{
+public:
+	DBMirrorSet(DB::ChangeListener *listener);
+	virtual ~DBMirrorSet();
+
+	bool hasNetwork(const uint64_t networkId) const;
+
+	bool get(const uint64_t networkId,nlohmann::json &network);
+	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member);
+	bool get(const uint64_t networkId,nlohmann::json &network,const uint64_t memberId,nlohmann::json &member,DB::NetworkSummaryInfo &info);
+	bool get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohmann::json> &members);
+
+	void networks(std::set<uint64_t> &networks);
+
+	bool waitForReady();
+	bool isReady();
+	bool save(nlohmann::json &record,bool notifyListeners);
+	void eraseNetwork(const uint64_t networkId);
+	void eraseMember(const uint64_t networkId,const uint64_t memberId);
+	void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress);
+
+	// These are called by various DB instances when changes occur.
+	virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network);
+	virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member);
+	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);
+
+	inline void addDB(const std::shared_ptr<DB> &db)
+	{
+		db->addListener(this);
+		std::lock_guard<std::mutex> l(_dbs_l);
+		_dbs.push_back(db);
+	}
+
+private:
+	DB::ChangeListener *const _listener;
+	std::atomic_bool _running;
+	std::thread _syncCheckerThread;
+	std::vector< std::shared_ptr< DB > > _dbs;
+	mutable std::mutex _dbs_l;
+};
+
+} // namespace ZeroTier
+
+#endif

controller/EmbeddedNetworkController.cpp

@@ -469,12 +469,14 @@ static bool _parseRule(json &r,ZT_VirtualNetworkRule &rule)
 
 } // anonymous namespace
 
-EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *dbPath, int listenPort, MQConfig *mqc) :
+EmbeddedNetworkController::EmbeddedNetworkController(Node *node,const char *ztPath,const char *dbPath, int listenPort, MQConfig *mqc) :
 	_startTime(OSUtils::now()),
 	_listenPort(listenPort),
 	_node(node),
+	_ztPath(ztPath),
 	_path(dbPath),
 	_sender((NetworkController::Sender *)0),
+	_db(this),
 	_mqc(mqc)
 {
 }
@@ -496,12 +498,16 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 	if ((_path.length() > 9)&&(_path.substr(0,9) == "postgres:")) {
-		_db.reset(new PostgreSQL(_signingId,_path.substr(9).c_str(), _listenPort, _mqc));
+		_db.addDB(std::shared_ptr<DB>(new PostgreSQL(_signingId,_path.substr(9).c_str(), _listenPort, _mqc)));
 	} else {
 #endif
+		_db.addDB(std::shared_ptr<DB>(new FileDB(_path.c_str())));
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+	}
+#endif
 
 	std::string lfJSON;
-	OSUtils::readFile((_path + ZT_PATH_SEPARATOR_S ".." ZT_PATH_SEPARATOR_S "local.conf").c_str(),lfJSON);
+	OSUtils::readFile((_ztPath + ZT_PATH_SEPARATOR_S "local.conf").c_str(),lfJSON);
 	if (lfJSON.length() > 0) {
 		nlohmann::json lfConfig(OSUtils::jsonParse(lfJSON));
 		nlohmann::json &settings = lfConfig["settings"];
@@ -521,7 +527,7 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 							std::size_t pubHdrEnd = lfOwnerPublic.find_first_of("\n\r\t ");
 							if (pubHdrEnd != std::string::npos) {
 								lfOwnerPublic = lfOwnerPublic.substr(0,pubHdrEnd);
-								_db.reset(new LFDB(_signingId,_path.c_str(),lfOwner.c_str(),lfOwnerPublic.c_str(),lfHost.c_str(),lfPort,storeOnlineState));
+								_db.addDB(std::shared_ptr<DB>(new LFDB(_signingId,_path.c_str(),lfOwner.c_str(),lfOwnerPublic.c_str(),lfHost.c_str(),lfPort,storeOnlineState)));
 							}
 						}
 					}
@@ -529,16 +535,8 @@ void EmbeddedNetworkController::init(const Identity &signingId,Sender *sender)
 			}
 		}
 	}
-	if (!_db)
-		_db.reset(new FileDB(_signingId,_path.c_str()));
-
-	_db->addListener(this);
 
-#ifdef ZT_CONTROLLER_USE_LIBPQ
-	}
-#endif
-
-	_db->waitForReady();
+	_db.waitForReady();
 }
 
 void EmbeddedNetworkController::request(
@@ -569,15 +567,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 	std::string &responseBody,
 	std::string &responseContentType)
 {
-	if (!_db)
-		return 500;
-
 	if ((path.size() > 0)&&(path[0] == "network")) {
 
 		if ((path.size() >= 2)&&(path[1].length() == 16)) {
 			const uint64_t nwid = Utils::hexStrToU64(path[1].c_str());
 			json network;
-			if (!_db->get(nwid,network))
+			if (!_db.get(nwid,network))
 				return 404;
 
 			if (path.size() >= 3) {
@@ -589,7 +584,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 
 						const uint64_t address = Utils::hexStrToU64(path[3].c_str());
 						json member;
-						if (!_db->get(nwid,network,address,member))
+						if (!_db.get(nwid,network,address,member))
 							return 404;
 						responseBody = OSUtils::jsonDump(member);
 						responseContentType = "application/json";
@@ -599,7 +594,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 
 						responseBody = "{";
 						std::vector<json> members;
-						if (_db->get(nwid,network,members)) {
+						if (_db.get(nwid,network,members)) {
 							responseBody.reserve((members.size() + 2) * 32);
 							std::string mid;
 							for(auto member=members.begin();member!=members.end();++member) {
@@ -628,12 +623,12 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 		} else if (path.size() == 1) {
 			// List networks
 
-			std::vector<uint64_t> networkIds;
-			_db->networks(networkIds);
+			std::set<uint64_t> networkIds;
+			_db.networks(networkIds);
 			char tmp[64];
 			responseBody = "[";
 			responseBody.reserve((networkIds.size() + 1) * 24);
-			for(std::vector<uint64_t>::const_iterator i(networkIds.begin());i!=networkIds.end();++i) {
+			for(std::set<uint64_t>::const_iterator i(networkIds.begin());i!=networkIds.end();++i) {
 				if (responseBody.length() > 1)
 					responseBody.push_back(',');
 				OSUtils::ztsnprintf(tmp,sizeof(tmp),"\"%.16llx\"",(unsigned long long)*i);
@@ -650,7 +645,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpGET(
 		// Controller status
 
 		char tmp[4096];
-		const bool dbOk = _db->isReady();
+		const bool dbOk = _db.isReady();
 		OSUtils::ztsnprintf(tmp,sizeof(tmp),"{\n\t\"controller\": true,\n\t\"apiVersion\": %d,\n\t\"clock\": %llu,\n\t\"databaseReady\": %s\n}\n",ZT_NETCONF_CONTROLLER_API_VERSION,(unsigned long long)OSUtils::now(),dbOk ? "true" : "false");
 		responseBody = tmp;
 		responseContentType = "application/json";
@@ -669,8 +664,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 	std::string &responseBody,
 	std::string &responseContentType)
 {
-	if (!_db)
-		return 500;
 	if (path.empty())
 		return 404;
 
@@ -704,8 +697,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 					OSUtils::ztsnprintf(addrs,sizeof(addrs),"%.10llx",(unsigned long long)address);
 
 					json member,network;
-					_db->get(nwid,network,address,member);
-					json origMember(member); // for detecting changes
+					_db.get(nwid,network,address,member);
 					DB::initMember(member);
 
 					try {
@@ -799,7 +791,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 					member["nwid"] = nwids;
 
 					DB::cleanMember(member);
-					_db->save(&origMember,member);
+					_db.save(member,true);
 					responseBody = OSUtils::jsonDump(member);
 					responseContentType = "application/json";
 
@@ -818,7 +810,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 						Utils::getSecureRandom(&nwidPostfix,sizeof(nwidPostfix));
 						uint64_t tryNwid = nwidPrefix | (nwidPostfix & 0xffffffULL);
 						if ((tryNwid & 0xffffffULL) == 0ULL) tryNwid |= 1ULL;
-						if (!_db->hasNetwork(tryNwid)) {
+						if (!_db.hasNetwork(tryNwid)) {
 							nwid = tryNwid;
 							break;
 						}
@@ -829,8 +821,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 				OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",(unsigned long long)nwid);
 
 				json network;
-				_db->get(nwid,network);
-				json origNetwork(network); // for detecting changes
+				_db.get(nwid,network);
 				DB::initNetwork(network);
 
 				try {
@@ -1061,7 +1052,7 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpPOST(
 				network["nwid"] = nwids; // legacy
 
 				DB::cleanNetwork(network);
-				_db->save(&origNetwork,network);
+				_db.save(network,true);
 
 				responseBody = OSUtils::jsonDump(network);
 				responseContentType = "application/json";
@@ -1083,8 +1074,6 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
 	std::string &responseBody,
 	std::string &responseContentType)
 {
-	if (!_db)
-		return 500;
 	if (path.empty())
 		return 404;
 
@@ -1096,8 +1085,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
 					const uint64_t address = Utils::hexStrToU64(path[3].c_str());
 
 					json network,member;
-					_db->get(nwid,network,address,member);
-					_db->eraseMember(nwid, address);
+					_db.get(nwid,network,address,member);
+					_db.eraseMember(nwid, address);
 
 					{
 						std::lock_guard<std::mutex> l(_memberStatus_l);
@@ -1112,8 +1101,8 @@ unsigned int EmbeddedNetworkController::handleControlPlaneHttpDELETE(
 				}
 			} else {
 				json network;
-				_db->get(nwid,network);
-				_db->eraseNetwork(nwid);
+				_db.get(nwid,network);
+				_db.eraseNetwork(nwid);
 
 				{
 					std::lock_guard<std::mutex> l(_memberStatus_l);
@@ -1143,9 +1132,6 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt)
 	char id[128],tmp[128];
 	std::string k,v;
 
-	if (!_db)
-		return;
-
 	try {
 		// Convert Dictionary into JSON object
 		json d;
@@ -1184,13 +1170,13 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace &rt)
 		d["objtype"] = "trace";
 		d["ts"] = now;
 		d["nodeId"] = Utils::hex10(rt.origin,tmp);
-		_db->save((nlohmann::json *)0,d);
+		_db.save(d,true);
 	} catch ( ... ) {
 		// drop invalid trace messages if an error occurs
 	}
 }
 
-void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId,const nlohmann::json &network)
+void EmbeddedNetworkController::onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network)
 {
 	// Send an update to all members of the network that are online
 	const int64_t now = OSUtils::now();
@@ -1201,7 +1187,7 @@ void EmbeddedNetworkController::onNetworkUpdate(const uint64_t networkId,const n
 	}
 }
 
-void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId,const nlohmann::json &member)
+void EmbeddedNetworkController::onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member)
 {
 	// Push update to member if online
 	try {
@@ -1212,7 +1198,7 @@ void EmbeddedNetworkController::onNetworkMemberUpdate(const uint64_t networkId,c
 	} catch ( ... ) {}
 }
 
-void EmbeddedNetworkController::onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId)
+void EmbeddedNetworkController::onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId)
 {
 	const int64_t now = OSUtils::now();
 	Revocation rev((uint32_t)_node->prng(),networkId,0,now,ZT_REVOCATION_FLAG_FAST_PROPAGATE,Address(memberId),Revocation::CREDENTIAL_TYPE_COM);
@@ -1235,10 +1221,7 @@ void EmbeddedNetworkController::_request(
 {
 	char nwids[24];
 	DB::NetworkSummaryInfo ns;
-	json network,member,origMember;
-
-	if (!_db)
-		return;
+	json network,member;
 
 	if (((!_signingId)||(!_signingId.hasPrivate()))||(_signingId.address().toInt() != (nwid >> 24))||(!_sender))
 		return;
@@ -1253,15 +1236,14 @@ void EmbeddedNetworkController::_request(
 		ms.lastRequestTime = now;
 	}
 
-	_db->nodeIsOnline(nwid,identity.address().toInt(),fromAddr);
+	_db.nodeIsOnline(nwid,identity.address().toInt(),fromAddr);
 
 	Utils::hex(nwid,nwids);
-	_db->get(nwid,network,identity.address().toInt(),member,ns);
+	_db.get(nwid,network,identity.address().toInt(),member,ns);
 	if ((!network.is_object())||(network.size() == 0)) {
 		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_OBJECT_NOT_FOUND);
 		return;
 	}
-	origMember = member;
 	const bool newMember = ((!member.is_object())||(member.size() == 0));
 	DB::initMember(member);
 
@@ -1362,7 +1344,7 @@ void EmbeddedNetworkController::_request(
 	} else {
 		// If they are not authorized, STOP!
 		DB::cleanMember(member);
-		_db->save(&origMember,member);
+		_db.save(member,true);
 		_sender->ncSendError(nwid,requestPacketId,identity.address(),NetworkController::NC_ERROR_ACCESS_DENIED);
 		return;
 	}
@@ -1734,7 +1716,7 @@ void EmbeddedNetworkController::_request(
 	}
 
 	DB::cleanMember(member);
-	_db->save(&origMember,member);
+	_db.save(member,true);
 	_sender->ncSendConfig(nwid,requestPacketId,identity.address(),*(nc.get()),metaData.getUI(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_VERSION,0) < 6);
 }
 

controller/EmbeddedNetworkController.hpp

@@ -51,6 +51,7 @@
 #include "../ext/json/json.hpp"
 
 #include "DB.hpp"
+#include "DBMirrorSet.hpp"
 
 namespace ZeroTier {
 
@@ -65,7 +66,7 @@ public:
 	 * @param node Parent node
 	 * @param dbPath Database path (file path or database credentials)
 	 */
-	EmbeddedNetworkController(Node *node,const char *dbPath, int listenPort, MQConfig *mqc = NULL);
+	EmbeddedNetworkController(Node *node,const char *ztPath,const char *dbPath, int listenPort, MQConfig *mqc = NULL);
 	virtual ~EmbeddedNetworkController();
 
 	virtual void init(const Identity &signingId,Sender *sender);
@@ -101,9 +102,9 @@ public:
 
 	void handleRemoteTrace(const ZT_RemoteTrace &rt);
 
-	virtual void onNetworkUpdate(const uint64_t networkId,const nlohmann::json &network);
-	virtual void onNetworkMemberUpdate(const uint64_t networkId,const uint64_t memberId,const nlohmann::json &member);
-	virtual void onNetworkMemberDeauthorize(const uint64_t networkId,const uint64_t memberId);
+	virtual void onNetworkUpdate(const void *db,uint64_t networkId,const nlohmann::json &network);
+	virtual void onNetworkMemberUpdate(const void *db,uint64_t networkId,uint64_t memberId,const nlohmann::json &member);
+	virtual void onNetworkMemberDeauthorize(const void *db,uint64_t networkId,uint64_t memberId);
 
 private:
 	void _request(uint64_t nwid,const InetAddress &fromAddr,uint64_t requestPacketId,const Identity &identity,const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY> &metaData);
@@ -148,12 +149,13 @@ private:
 	const int64_t _startTime;
 	int _listenPort;
 	Node *const _node;
+	std::string _ztPath;
 	std::string _path;
 	Identity _signingId;
 	std::string _signingIdAddressString;
 	NetworkController::Sender *_sender;
 
-	std::unique_ptr<DB> _db;
+	DBMirrorSet _db;
 	BlockingQueue< _RQEntry * > _queue;
 
 	std::vector<std::thread> _threads;

controller/FileDB.cpp

@@ -29,11 +29,11 @@
 namespace ZeroTier
 {
 
-FileDB::FileDB(const Identity &myId,const char *path) :
-	DB(myId,path),
+FileDB::FileDB(const char *path) :
+	DB(),
+	_path(path),
 	_networksPath(_path + ZT_PATH_SEPARATOR_S + "network"),
 	_tracePath(_path + ZT_PATH_SEPARATOR_S + "trace"),
-	_onlineChanged(false),
 	_running(true)
 {
 	OSUtils::mkdir(_path.c_str());
@@ -71,54 +71,6 @@ FileDB::FileDB(const Identity &myId,const char *path) :
 			} catch ( ... ) {}
 		}
 	}
-
-	_onlineUpdateThread = std::thread([this]() {
-		unsigned int cnt = 0;
-		while (this->_running) {
-			std::this_thread::sleep_for(std::chrono::microseconds(100));
-			if ((++cnt % 20) == 0) { // 5 seconds
-				std::lock_guard<std::mutex> l(this->_online_l);
-				if (!this->_running) return;
-				if (this->_onlineChanged) {
-					char p[4096],atmp[64];
-					for(auto nw=this->_online.begin();nw!=this->_online.end();++nw) {
-						OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx-online.json",_networksPath.c_str(),(unsigned long long)nw->first);
-						FILE *f = fopen(p,"wb");
-						if (f) {
-							fprintf(f,"{");
-							const char *memberPrefix = "";
-							for(auto m=nw->second.begin();m!=nw->second.end();++m) {
-								fprintf(f,"%s\"%.10llx\":{" ZT_EOL_S,memberPrefix,(unsigned long long)m->first);
-								memberPrefix = ",";
-								InetAddress lastAddr;
-								const char *timestampPrefix = " ";
-								int cnt = 0;
-								for(auto ts=m->second.rbegin();ts!=m->second.rend();) {
-									if (cnt < 25) {
-										if (lastAddr != ts->second) {
-											lastAddr = ts->second;
-											fprintf(f,"%s\"%lld\":\"%s\"" ZT_EOL_S,timestampPrefix,(long long)ts->first,ts->second.toString(atmp));
-											timestampPrefix = ",";
-											++cnt;
-											++ts;
-										} else {
-											ts = std::map<int64_t,InetAddress>::reverse_iterator(m->second.erase(std::next(ts).base()));
-										}
-									} else {
-										ts = std::map<int64_t,InetAddress>::reverse_iterator(m->second.erase(std::next(ts).base()));
-									}
-								}
-								fprintf(f,"}");
-							}
-							fprintf(f,"}" ZT_EOL_S);
-							fclose(f);
-						}
-					}
-					this->_onlineChanged = false;
-				}
-			}
-		}
-	});
 }
 
 FileDB::~FileDB()
@@ -134,38 +86,37 @@ FileDB::~FileDB()
 bool FileDB::waitForReady() { return true; }
 bool FileDB::isReady() { return true; }
 
-void FileDB::save(nlohmann::json *orig,nlohmann::json &record)
+bool FileDB::save(nlohmann::json &record,bool notifyListeners)
 {
 	char p1[4096],p2[4096],pb[4096];
+	bool modified = false;
 	try {
-		if (orig) {
-			if (*orig != record) {
-				record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1;
-			}
-		} else {
-			record["revision"] = 1;
-		}
-
 		const std::string objtype = record["objtype"];
 		if (objtype == "network") {
+
 			const uint64_t nwid = OSUtils::jsonIntHex(record["id"],0ULL);
 			if (nwid) {
 				nlohmann::json old;
 				get(nwid,old);
-				if ((!old.is_object())||(old != record)) {
+				if ((!old.is_object())||(!_compareRecords(old,record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
 					OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),nwid);
 					if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1)))
 						fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1);
-					_networkChanged(old,record,true);
+					_networkChanged(old,record,notifyListeners);
+					modified = true;
 				}
 			}
+
 		} else if (objtype == "member") {
+
 			const uint64_t id = OSUtils::jsonIntHex(record["id"],0ULL);
 			const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"],0ULL);
 			if ((id)&&(nwid)) {
 				nlohmann::json network,old;
 				get(nwid,network,id,old);
-				if ((!old.is_object())||(old != record)) {
+				if ((!old.is_object())||(!_compareRecords(old,record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
 					OSUtils::ztsnprintf(pb,sizeof(pb),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member",_networksPath.c_str(),(unsigned long long)nwid);
 					OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%.10llx.json",pb,(unsigned long long)id);
 					if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1))) {
@@ -175,17 +126,14 @@ void FileDB::save(nlohmann::json *orig,nlohmann::json &record)
 						if (!OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1)))
 							fprintf(stderr,"WARNING: controller unable to write to path: %s" ZT_EOL_S,p1);
 					}
-					_memberChanged(old,record,true);
+					_memberChanged(old,record,notifyListeners);
+					modified = true;
 				}
 			}
-		} else if (objtype == "trace") {
-			const std::string id = record["id"];
-			if (id.length() > 0) {
-				OSUtils::ztsnprintf(p1,sizeof(p1),"%s" ZT_PATH_SEPARATOR_S "%s.json",_tracePath.c_str(),id.c_str());
-				OSUtils::writeFile(p1,OSUtils::jsonDump(record,-1));
-			}
+
 		}
 	} catch ( ... ) {} // drop invalid records missing fields
+	return modified;
 }
 
 void FileDB::eraseNetwork(const uint64_t networkId)
@@ -195,14 +143,11 @@ void FileDB::eraseNetwork(const uint64_t networkId)
 	char p[16384];
 	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),networkId);
 	OSUtils::rm(p);
-	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx-online.json",_networksPath.c_str(),networkId);
-	OSUtils::rm(p);
 	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member",_networksPath.c_str(),(unsigned long long)networkId);
 	OSUtils::rmDashRf(p);
 	_networkChanged(network,nullJson,true);
 	std::lock_guard<std::mutex> l(this->_online_l);
 	this->_online.erase(networkId);
-	this->_onlineChanged = true;
 }
 
 void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
@@ -216,7 +161,6 @@ void FileDB::eraseMember(const uint64_t networkId,const uint64_t memberId)
 	_memberChanged(member,nullJson,true);
 	std::lock_guard<std::mutex> l(this->_online_l);
 	this->_online[networkId].erase(memberId);
-	this->_onlineChanged = true;
 }
 
 void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress)
@@ -226,7 +170,6 @@ void FileDB::nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const
 	physicalAddress.toString(atmp);
 	std::lock_guard<std::mutex> l(this->_online_l);
 	this->_online[networkId][memberId][OSUtils::now()] = physicalAddress;
-	this->_onlineChanged = true;
 }
 
 } // namespace ZeroTier

controller/FileDB.hpp

@@ -35,23 +35,23 @@ namespace ZeroTier
 class FileDB : public DB
 {
 public:
-	FileDB(const Identity &myId,const char *path);
+	FileDB(const char *path);
 	virtual ~FileDB();
 
 	virtual bool waitForReady();
 	virtual bool isReady();
-	virtual void save(nlohmann::json *orig,nlohmann::json &record);
+	virtual bool save(nlohmann::json &record,bool notifyListeners);
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress);
 
 protected:
+	std::string _path;
 	std::string _networksPath;
 	std::string _tracePath;
 	std::thread _onlineUpdateThread;
 	std::map< uint64_t,std::map<uint64_t,std::map<int64_t,InetAddress> > > _online;
 	std::mutex _online_l;
-	bool _onlineChanged;
 	bool _running;
 };
 

controller/LFDB.cpp

@@ -38,7 +38,7 @@ namespace ZeroTier
 {
 
 LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,const char *lfOwnerPublic,const char *lfNodeHost,int lfNodePort,bool storeOnlineState) :
-	DB(myId,path),
+	DB(),
 	_myId(myId),
 	_lfOwnerPrivate((lfOwnerPrivate) ? lfOwnerPrivate : ""),
 	_lfOwnerPublic((lfOwnerPublic) ? lfOwnerPublic : ""),
@@ -53,7 +53,6 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 		const uint64_t controllerAddressInt = _myId.address().toInt();
 		_myId.address().toString(controllerAddress);
 		std::string networksSelectorName("com.zerotier.controller.lfdb:"); networksSelectorName.append(controllerAddress); networksSelectorName.append("/network");
-		std::string membersSelectorName("com.zerotier.controller.lfdb:"); membersSelectorName.append(controllerAddress); membersSelectorName.append("/member");
 
 		// LF record masking key is the first 32 bytes of SHA512(controller private key) in hex,
 		// hiding record values from anything but the controller or someone who has its key.
@@ -64,7 +63,7 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 
 		httplib::Client htcli(_lfNodeHost.c_str(),_lfNodePort,600);
 		int64_t timeRangeStart = 0;
-		while (_running) {
+		while (_running.load()) {
 			{
 				std::lock_guard<std::mutex> sl(_state_l);
 				for(auto ns=_state.begin();ns!=_state.end();++ns) {
@@ -79,16 +78,22 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 							newrec["OwnerPrivate"] = _lfOwnerPrivate;
 							newrec["MaskingKey"] = maskingKey;
 							newrec["PulseIfUnchanged"] = true;
-							auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
-							if (resp) {
-								if (resp->status == 200) {
-									ns->second.dirty = false;
-									printf("SET network %.16llx %s\n",ns->first,resp->body.c_str());
+							try {
+								auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
+								if (resp) {
+									if (resp->status == 200) {
+										ns->second.dirty = false;
+										//printf("SET network %.16llx %s\n",ns->first,resp->body.c_str());
+									} else {
+										fprintf(stderr,"ERROR: LFDB: %d from node (create/update network): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+									}
 								} else {
-									fprintf(stderr,"ERROR: LFDB: %d from node (create/update network): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+									fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
 								}
-							} else {
-								fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+							} catch (std::exception &e) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update network): %s" ZT_EOL_S,e.what());
+							} catch ( ... ) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update network): unknown exception" ZT_EOL_S);
 							}
 						}
 					}
@@ -125,16 +130,22 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 							newrec["MaskingKey"] = maskingKey;
 							newrec["Timestamp"] = ms->second.lastOnlineTime;
 							newrec["PulseIfUnchanged"] = true;
-							auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
-							if (resp) {
-								if (resp->status == 200) {
-									ms->second.lastOnlineDirty = false;
-									printf("SET member online %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
+							try {
+								auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
+								if (resp) {
+									if (resp->status == 200) {
+										ms->second.lastOnlineDirty = false;
+										//printf("SET member online %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
+									} else {
+										fprintf(stderr,"ERROR: LFDB: %d from node (create/update member online status): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+									}
 								} else {
-									fprintf(stderr,"ERROR: LFDB: %d from node (create/update member online status): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+									fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
 								}
-							} else {
-								fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+							} catch (std::exception &e) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member online status): %s" ZT_EOL_S,e.what());
+							} catch ( ... ) {
+								fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member online status): unknown exception" ZT_EOL_S);
 							}
 						}
 
@@ -144,7 +155,7 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 								nlohmann::json newrec,selector0,selector1,selectors;
 								selector0["Name"] = networksSelectorName;
 								selector0["Ordinal"] = ns->first;
-								selector1["Name"] = membersSelectorName;
+								selector1["Name"] = "member";
 								selector1["Ordinal"] = ms->first;
 								selectors.push_back(selector0);
 								selectors.push_back(selector1);
@@ -153,16 +164,22 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 								newrec["OwnerPrivate"] = _lfOwnerPrivate;
 								newrec["MaskingKey"] = maskingKey;
 								newrec["PulseIfUnchanged"] = true;
-								auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
-								if (resp) {
-									if (resp->status == 200) {
-										ms->second.dirty = false;
-										printf("SET member %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
+								try {
+									auto resp = htcli.Post("/makerecord",newrec.dump(),"application/json");
+									if (resp) {
+										if (resp->status == 200) {
+											ms->second.dirty = false;
+											//printf("SET member %.16llx %.10llx %s\n",ns->first,ms->first,resp->body.c_str());
+										} else {
+											fprintf(stderr,"ERROR: LFDB: %d from node (create/update member): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+										}
 									} else {
-										fprintf(stderr,"ERROR: LFDB: %d from node (create/update member): %s" ZT_EOL_S,resp->status,resp->body.c_str());
+										fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
 									}
-								} else {
-									fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
+								} catch (std::exception &e) {
+									fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member): %s" ZT_EOL_S,e.what());
+								} catch ( ... ) {
+									fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (create/update member): unknown exception" ZT_EOL_S);
 								}
 							}
 						}
@@ -170,18 +187,18 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 				}
 			}
 
-			{
+			try {
 				std::ostringstream query;
-				query
-					<< '{'
-						<< "\"Ranges\":[{"
-							<< "\"Name\":\"" << networksSelectorName << "\","
-							<< "\"Range\":[0,18446744073709551615]"
-						<< "}],"
-						<< "\"TimeRange\":[" << timeRangeStart << ",18446744073709551615],"
-						<< "\"MaskingKey\":\"" << maskingKey << "\","
-						<< "\"Owners\":[\"" << _lfOwnerPublic << "\"]"
-					<< '}';
+				query <<
+					"{"
+						"\"Ranges\":[{"
+							"\"Name\":\"" << networksSelectorName << "\","
+							"\"Range\":[0,18446744073709551615]"
+						"}],"
+						"\"TimeRange\":[" << timeRangeStart << ",9223372036854775807],"
+						"\"MaskingKey\":\"" << maskingKey << "\","
+						"\"Owners\":[\"" << _lfOwnerPublic << "\"]"
+					"}";
 				auto resp = htcli.Post("/query",query.str(),"application/json");
 				if (resp) {
 					if (resp->status == 200) {
@@ -190,64 +207,66 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 							for(std::size_t ri=0;ri<results.size();++ri) {
 								nlohmann::json &rset = results[ri];
 								if ((rset.is_array())&&(rset.size() > 0)) {
+
 									nlohmann::json &result = rset[0];
 									if (result.is_object()) {
 										nlohmann::json &record = result["Record"];
 										if (record.is_object()) {
 											const std::string recordValue = result["Value"];
-											printf("GET network %s\n",recordValue.c_str());
+											//printf("GET network %s\n",recordValue.c_str());
 											nlohmann::json network(OSUtils::jsonParse(recordValue));
 											if (network.is_object()) {
 												const std::string idstr = network["id"];
 												const uint64_t id = Utils::hexStrToU64(idstr.c_str());
 												if ((id >> 24) == controllerAddressInt) { // sanity check
 
-													std::lock_guard<std::mutex> sl(_state_l);
-													_NetworkState &ns = _state[id];
-													if (!ns.dirty) {
-														nlohmann::json oldNetwork;
-														if (get(id,oldNetwork)) {
-															const uint64_t revision = network["revision"];
-															const uint64_t prevRevision = oldNetwork["revision"];
-															if (prevRevision < revision) {
-																_networkChanged(oldNetwork,network,timeRangeStart > 0);
-															}
-														} else {
-															nlohmann::json nullJson;
-															_networkChanged(nullJson,network,timeRangeStart > 0);
+													nlohmann::json oldNetwork;
+													if ((timeRangeStart > 0)&&(get(id,oldNetwork))) {
+														const uint64_t revision = network["revision"];
+														const uint64_t prevRevision = oldNetwork["revision"];
+														if (prevRevision < revision) {
+															_networkChanged(oldNetwork,network,timeRangeStart > 0);
 														}
+													} else {
+														nlohmann::json nullJson;
+														_networkChanged(nullJson,network,timeRangeStart > 0);
 													}
 
 												}
 											}
 										}
 									}
+
 								}
 							}
 						}
 					} else {
-						fprintf(stderr,"ERROR: LFDB: %d from node: %s" ZT_EOL_S,resp->status,resp->body.c_str());
+						fprintf(stderr,"ERROR: LFDB: %d from node (check for network updates): %s" ZT_EOL_S,resp->status,resp->body.c_str());
 					}
 				} else {
 					fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
 				}
+			} catch (std::exception &e) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for network updates): %s" ZT_EOL_S,e.what());
+			} catch ( ... ) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for network updates): unknown exception" ZT_EOL_S);
 			}
 
-			{
+			try {
 				std::ostringstream query;
-				query
-					<< '{'
-						<< "\"Ranges\":[{"
-							<< "\"Name\":\"" << networksSelectorName << "\","
-							<< "\"Range\":[0,18446744073709551615]"
-						<< "},{"
-							<< "\"Name\":\"" << membersSelectorName << "\","
-							<< "\"Range\":[0,18446744073709551615]"
-						<< "}],"
-						<< "\"TimeRange\":[" << timeRangeStart << ",18446744073709551615],"
-						<< "\"MaskingKey\":\"" << maskingKey << "\","
-						<< "\"Owners\":[\"" << _lfOwnerPublic << "\"]"
-					<< '}';
+				query <<
+					"{"
+						"\"Ranges\":[{"
+							"\"Name\":\"" << networksSelectorName << "\","
+							"\"Range\":[0,18446744073709551615]"
+						"},{"
+							"\"Name\":\"member\","
+							"\"Range\":[0,18446744073709551615]"
+						"}],"
+						"\"TimeRange\":[" << timeRangeStart << ",9223372036854775807],"
+						"\"MaskingKey\":\"" << maskingKey << "\","
+						"\"Owners\":[\"" << _lfOwnerPublic << "\"]"
+					"}";
 				auto resp = htcli.Post("/query",query.str(),"application/json");
 				if (resp) {
 					if (resp->status == 200) {
@@ -256,12 +275,13 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 							for(std::size_t ri=0;ri<results.size();++ri) {
 								nlohmann::json &rset = results[ri];
 								if ((rset.is_array())&&(rset.size() > 0)) {
+
 									nlohmann::json &result = rset[0];
 									if (result.is_object()) {
 										nlohmann::json &record = result["Record"];
 										if (record.is_object()) {
 											const std::string recordValue = result["Value"];
-											printf("GET member %s\n",recordValue.c_str());
+											//printf("GET member %s\n",recordValue.c_str());
 											nlohmann::json member(OSUtils::jsonParse(recordValue));
 											if (member.is_object()) {
 												const std::string nwidstr = member["nwid"];
@@ -270,17 +290,13 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 												const uint64_t id = Utils::hexStrToU64(idstr.c_str());
 												if ((id)&&((nwid >> 24) == controllerAddressInt)) { // sanity check
 
-													std::lock_guard<std::mutex> sl(_state_l);
-													auto ns = _state.find(nwid);
-													if ((ns == _state.end())||(!ns->second.members[id].dirty)) {
-														nlohmann::json network,oldMember;
-														if (get(nwid,network,id,oldMember)) {
-															const uint64_t revision = member["revision"];
-															const uint64_t prevRevision = oldMember["revision"];
-															if (prevRevision < revision)
-																_memberChanged(oldMember,member,timeRangeStart > 0);
-														}
-													} else {
+													nlohmann::json network,oldMember;
+													if ((timeRangeStart > 0)&&(get(nwid,network,id,oldMember))) {
+														const uint64_t revision = member["revision"];
+														const uint64_t prevRevision = oldMember["revision"];
+														if (prevRevision < revision)
+															_memberChanged(oldMember,member,timeRangeStart > 0);
+													} else if (hasNetwork(nwid)) {
 														nlohmann::json nullJson;
 														_memberChanged(nullJson,member,timeRangeStart > 0);
 													}
@@ -289,24 +305,29 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 											}
 										}
 									}
+
 								}
 							}
 						}
 					} else {
-						fprintf(stderr,"ERROR: LFDB: %d from node: %s" ZT_EOL_S,resp->status,resp->body.c_str());
+						fprintf(stderr,"ERROR: LFDB: %d from node (check for member updates): %s" ZT_EOL_S,resp->status,resp->body.c_str());
 					}
 				} else {
 					fprintf(stderr,"ERROR: LFDB: node is offline" ZT_EOL_S);
 				}
+			} catch (std::exception &e) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for member updates): %s" ZT_EOL_S,e.what());
+			} catch ( ... ) {
+				fprintf(stderr,"ERROR: LFDB: unexpected exception querying node (check for member updates): unknown exception" ZT_EOL_S);
 			}
 
 			timeRangeStart = time(nullptr) - 120; // start next query 2m before now to avoid losing updates
-			_ready = true;
+			_ready.store(true);
 
-			for(int k=0;k<20;++k) { // 2s delay between queries for remotely modified networks or members
-				if (!_running)
+			for(int k=0;k<4;++k) { // 2s delay between queries for remotely modified networks or members
+				if (!_running.load())
 					return;
-				std::this_thread::sleep_for(std::chrono::milliseconds(100));
+				std::this_thread::sleep_for(std::chrono::milliseconds(500));
 			}
 		}
 	});
@@ -314,45 +335,40 @@ LFDB::LFDB(const Identity &myId,const char *path,const char *lfOwnerPrivate,cons
 
 LFDB::~LFDB()
 {
-	_running = false;
+	_running.store(false);
 	_syncThread.join();
 }
 
 bool LFDB::waitForReady()
 {
-	while (!_ready) {
-		std::this_thread::sleep_for(std::chrono::milliseconds(100));
+	while (!_ready.load()) {
+		std::this_thread::sleep_for(std::chrono::milliseconds(500));
 	}
 	return true;
 }
 
 bool LFDB::isReady()
 {
-	return (_ready);
+	return (_ready.load());
 }
 
-void LFDB::save(nlohmann::json *orig,nlohmann::json &record)
+bool LFDB::save(nlohmann::json &record,bool notifyListeners)
 {
-	if (orig) {
-		if (*orig != record) {
-			record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1;
-		}
-	} else {
-		record["revision"] = 1;
-	}
-
+	bool modified = false;
 	const std::string objtype = record["objtype"];
 	if (objtype == "network") {
 		const uint64_t nwid = OSUtils::jsonIntHex(record["id"],0ULL);
 		if (nwid) {
 			nlohmann::json old;
 			get(nwid,old);
-			if ((!old.is_object())||(old != record)) {
-				_networkChanged(old,record,true);
+			if ((!old.is_object())||(!_compareRecords(old,record))) {
+				record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+				_networkChanged(old,record,notifyListeners);
 				{
 					std::lock_guard<std::mutex> l(_state_l);
 					_state[nwid].dirty = true;
 				}
+				modified = true;
 			}
 		}
 	} else if (objtype == "member") {
@@ -361,15 +377,18 @@ void LFDB::save(nlohmann::json *orig,nlohmann::json &record)
 		if ((id)&&(nwid)) {
 			nlohmann::json network,old;
 			get(nwid,network,id,old);
-			if ((!old.is_object())||(old != record)) {
-				_memberChanged(old,record,true);
+			if ((!old.is_object())||(!_compareRecords(old,record))) {
+				record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+				_memberChanged(old,record,notifyListeners);
 				{
 					std::lock_guard<std::mutex> l(_state_l);
 					_state[nwid].members[id].dirty = true;
 				}
+				modified = true;
 			}
 		}
 	}
+	return modified;
 }
 
 void LFDB::eraseNetwork(const uint64_t networkId)

controller/LFDB.hpp

@@ -43,7 +43,7 @@ class LFDB : public DB
 {
 public:
 	/**
-	 * @param myId Identity of controller node (with secret)
+	 * @param myId This controller's identity
 	 * @param path Base path for ZeroTier node itself
 	 * @param lfOwnerPrivate LF owner private in PEM format
 	 * @param lfOwnerPublic LF owner public in @base62 format
@@ -56,7 +56,7 @@ public:
 
 	virtual bool waitForReady();
 	virtual bool isReady();
-	virtual void save(nlohmann::json *orig,nlohmann::json &record);
+	virtual bool save(nlohmann::json &record,bool notifyListeners);
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId,const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId,const uint64_t memberId,const InetAddress &physicalAddress);

controller/PostgreSQL.cpp

@@ -24,9 +24,11 @@
  * of your own application.
  */
 
+#include "PostgreSQL.hpp"
+
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 
-#include "PostgreSQL.hpp"
+#include "../node/Constants.hpp"
 #include "EmbeddedNetworkController.hpp"
 #include "RabbitMQ.hpp"
 #include "../version.h"
@@ -37,6 +39,7 @@
 #include <amqp_tcp_socket.h>
 
 using json = nlohmann::json;
+
 namespace {
 
 static const int DB_MINIMUM_VERSION = 5;
@@ -58,6 +61,7 @@ static const char *_timestr()
 	return ts;
 }
 
+/*
 std::string join(const std::vector<std::string> &elements, const char * const separator)
 {
 	switch(elements.size()) {
@@ -72,21 +76,26 @@ std::string join(const std::vector<std::string> &elements, const char * const se
 		return os.str();
 	}
 }
+*/
 
-}
+} // anonymous namespace
 
 using namespace ZeroTier;
 
 PostgreSQL::PostgreSQL(const Identity &myId, const char *path, int listenPort, MQConfig *mqc)
-    : DB(myId, path)
-    , _ready(0)
+	: DB()
+	, _myId(myId)
+	, _myAddress(myId.address())
+	, _ready(0)
 	, _connected(1)
-    , _run(1)
-    , _waitNoticePrinted(false)
+	, _run(1)
+	, _waitNoticePrinted(false)
 	, _listenPort(listenPort)
 	, _mqc(mqc)
 {
-	_connString = std::string(path) + " application_name=controller_" +_myAddressStr;
+	char myAddress[64];
+	_myAddressStr = myId.address().toString(myAddress);
+	_connString = std::string(path) + " application_name=controller_" + _myAddressStr;
 
 	// Database Schema Version Check
 	PGconn *conn = getPgConn();
@@ -162,27 +171,43 @@ bool PostgreSQL::isReady()
 	return ((_ready == 2)&&(_connected));
 }
 
-void PostgreSQL::save(nlohmann::json *orig, nlohmann::json &record)
+bool PostgreSQL::save(nlohmann::json &record,bool notifyListeners)
 {
+	bool modified = false;
 	try {
-		if (!record.is_object()) {
-			return;
-		}
-		waitForReady();
-		if (orig) {
-			if (*orig != record) {
-				record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1;
-				_commitQueue.post(new nlohmann::json(record));
+		if (!record.is_object())
+			return false;
+		const std::string objtype = record["objtype"];
+		if (objtype == "network") {
+			const uint64_t nwid = OSUtils::jsonIntHex(record["id"],0ULL);
+			if (nwid) {
+				nlohmann::json old;
+				get(nwid,old);
+				if ((!old.is_object())||(!_compareRecords(old,record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+					_commitQueue.post(std::pair<nlohmann::json,bool>(record,notifyListeners));
+					modified = true;
+				}
+			}
+		} else if (objtype == "member") {
+			const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"],0ULL);
+			const uint64_t id = OSUtils::jsonIntHex(record["id"],0ULL);
+			if ((id)&&(nwid)) {
+				nlohmann::json network,old;
+				get(nwid,network,id,old);
+				if ((!old.is_object())||(!_compareRecords(old,record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"],0ULL) + 1ULL;
+					_commitQueue.post(std::pair<nlohmann::json,bool>(record,notifyListeners));
+					modified = true;
+				}
 			}
-		} else {
-			record["revision"] = 1;
-			_commitQueue.post(new nlohmann::json(record));
 		}
 	} catch (std::exception &e) {
 		fprintf(stderr, "Error on PostgreSQL::save: %s\n", e.what());
 	} catch (...) {
 		fprintf(stderr, "Unknown error on PostgreSQL::save\n");
 	}
+	return modified;
 }
 
 void PostgreSQL::eraseNetwork(const uint64_t networkId)
@@ -190,21 +215,23 @@ void PostgreSQL::eraseNetwork(const uint64_t networkId)
 	char tmp2[24];
 	waitForReady();
 	Utils::hex(networkId, tmp2);
-	json *tmp = new json();
-	(*tmp)["id"] = tmp2;
-	(*tmp)["objtype"] = "_delete_network";
+	std::pair<nlohmann::json,bool> tmp;
+	tmp.first["id"] = tmp2;
+	tmp.first["objtype"] = "_delete_network";
+	tmp.second = true;
 	_commitQueue.post(tmp);
 }
 
 void PostgreSQL::eraseMember(const uint64_t networkId, const uint64_t memberId) 
 {
 	char tmp2[24];
-	json *tmp = new json();
+	std::pair<nlohmann::json,bool> tmp;
 	Utils::hex(networkId, tmp2);
-	(*tmp)["nwid"] = tmp2;
+	tmp.first["nwid"] = tmp2;
 	Utils::hex(memberId, tmp2);
-	(*tmp)["id"] = tmp2;
-	(*tmp)["objtype"] = "_delete_member";
+	tmp.first["id"] = tmp2;
+	tmp.first["objtype"] = "_delete_member";
+	tmp.second = true;
 	_commitQueue.post(tmp);
 }
 
@@ -544,7 +571,7 @@ void PostgreSQL::heartbeat()
 	if (gethostname(hostnameTmp, sizeof(hostnameTmp))!= 0) {
 		hostnameTmp[0] = (char)0;
 	} else {
-		for (int i = 0; i < sizeof(hostnameTmp); ++i) {
+		for (int i = 0; i < (int)sizeof(hostnameTmp); ++i) {
 			if ((hostnameTmp[i] == '.')||(hostnameTmp[i] == 0)) {
 				hostnameTmp[i] = (char)0;
 				break;
@@ -595,8 +622,8 @@ void PostgreSQL::heartbeat()
 				"public_identity = EXCLUDED.public_identity, v_major = EXCLUDED.v_major, v_minor = EXCLUDED.v_minor, "
 				"v_rev = EXCLUDED.v_rev, v_build = EXCLUDED.v_rev, host_port = EXCLUDED.host_port, "
 				"use_rabbitmq = EXCLUDED.use_rabbitmq",
-				10,       // number of parameters
-				NULL,    // oid field.   ignore
+				10,	   // number of parameters
+				NULL,	// oid field.   ignore
 				values,  // values for substitution
 				NULL, // lengths in bytes of each value
 				NULL,  // binary?
@@ -717,7 +744,7 @@ void PostgreSQL::_membersWatcher_RabbitMQ() {
 			fprintf(stderr, "RABBITMQ ERROR member change: %s\n", e.what());
 		} catch(...) {
 			fprintf(stderr, "RABBITMQ ERROR member change: unknown error\n");
-        }
+		}
 	}
 }
 
@@ -834,18 +861,18 @@ void PostgreSQL::commitThread()
 		exit(1);
 	}
 
-	json *config = nullptr;
-	while(_commitQueue.get(config)&(_run == 1)) {
-		if (!config) {
+	std::pair<nlohmann::json,bool> qitem;
+	while(_commitQueue.get(qitem)&(_run == 1)) {
+		if (!qitem.first.is_object()) {
 			continue;
 		}
 		if (PQstatus(conn) == CONNECTION_BAD) {
 			fprintf(stderr, "ERROR: Connection to database failed: %s\n", PQerrorMessage(conn));
 			PQfinish(conn);
-			delete config;
 			exit(1);
 		}
-		try { 
+		try {
+			nlohmann::json *config = &(qitem.first);
 			const std::string objtype = (*config)["objtype"];
 			if (objtype == "member") {
 				try {
@@ -1000,12 +1027,12 @@ void PostgreSQL::commitThread()
 						nlohmann::json memOrig;
 
 						nlohmann::json memNew(*config);
-						
+
 						get(nwidInt, nwOrig, memberidInt, memOrig);
 				
-						_memberChanged(memOrig, memNew, (this->_ready>=2));
+						_memberChanged(memOrig, memNew, qitem.second);
 					} else {
-						fprintf(stderr, "Can't notify of change.  Error parsing nwid or memberid: %lu-%lu\n", nwidInt, memberidInt);
+						fprintf(stderr, "Can't notify of change.  Error parsing nwid or memberid: %llu-%llu\n", (unsigned long long)nwidInt, (unsigned long long)memberidInt);
 					}
 
 				} catch (std::exception &e) {
@@ -1020,7 +1047,10 @@ void PostgreSQL::commitThread()
 					if (!(*config)["remoteTraceTarget"].is_null()) {
 						remoteTraceTarget = (*config)["remoteTraceTarget"];
 					}
-					std::string rulesSource = (*config)["rulesSource"];
+					std::string rulesSource;
+					if ((*config)["rulesSource"].is_string()) {
+						rulesSource = (*config)["rulesSource"];
+					}
 					std::string caps = OSUtils::jsonDump((*config)["capabilitles"], -1);
 					std::string now = std::to_string(OSUtils::now());
 					std::string mtu = std::to_string((int)(*config)["mtu"]);
@@ -1052,12 +1082,29 @@ void PostgreSQL::commitThread()
 						v6mode.c_str(),
 					};
 
+					// This ugly query exists because when we want to mirror networks to/from
+					// another data store (e.g. FileDB or LFDB) it is possible to get a network
+					// that doesn't exist in Central's database. This does an upsert and sets
+					// the owner_id to the "first" global admin in the user DB if the record
+					// did not previously exist. If the record already exists owner_id is left
+					// unchanged, so owner_id should be left out of the update clause.
 					PGresult *res = PQexecParams(conn,
-						"UPDATE ztc_network SET controller_id = $2, capabilities = $3, enable_broadcast = $4, "
-						"last_updated = $5, mtu = $6, multicast_limit = $7, name = $8, private = $9, "
-						"remote_trace_level = $10, remote_trace_target = $11, rules = $12, rules_source = $13, "
-						"tags = $14, v4_assign_mode = $15, v6_assign_mode = $16 "
-						"WHERE id = $1",
+						"INSERT INTO ztc_network (id, creation_time, owner_id, controller_id, capabilities, enable_broadcast, "
+						"last_modified, mtu, multicast_limit, name, private, "
+						"remote_trace_level, remote_trace_target, rules, rules_source, "
+						"tags, v4_assign_mode, v6_assign_mode) VALUES ("
+						"$1, TO_TIMESTAMP($5::double precision/1000), "
+						"(SELECT user_id AS owner_id FROM ztc_global_permissions WHERE authorize = true AND del = true AND modify = true AND read = true LIMIT 1),"
+						"$2, $3, $4, TO_TIMESTAMP($5::double precision/1000), "
+						"$6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16) "
+						"ON CONFLICT (id) DO UPDATE set controller_id = EXCLUDED.controller_id, "
+						"capabilities = EXCLUDED.capabilities, enable_broadcast = EXCLUDED.enable_broadcast, "
+						"last_modified = EXCLUDED.last_modified, mtu = EXCLUDED.mtu, "
+						"multicast_limit = EXCLUDED.multicast_limit, name = EXCLUDED.name, "
+						"private = EXCLUDED.private, remote_trace_level = EXCLUDED.remote_trace_level, "
+						"remote_trace_target = EXCLUDED.remote_trace_target, rules = EXCLUDED.rules, "
+						"rules_source = EXCLUDED.rules_source, tags = EXCLUDED.tags, "
+						"v4_assign_mode = EXCLUDED.v4_assign_mode, v6_assign_mode = EXCLUDED.v6_assign_mode",
 						16,
 						NULL,
 						values,
@@ -1226,16 +1273,14 @@ void PostgreSQL::commitThread()
 
 						get(nwidInt, nwOrig);
 
-						_networkChanged(nwOrig, nwNew, true);
+						_networkChanged(nwOrig, nwNew, qitem.second);
 					} else {
-						fprintf(stderr, "Can't notify network changed: %lu\n", nwidInt);
+						fprintf(stderr, "Can't notify network changed: %llu\n", (unsigned long long)nwidInt);
 					}
 
 				} catch (std::exception &e) {
 					fprintf(stderr, "ERROR: Error updating member: %s\n", e.what());
 				}
-			} else if (objtype == "trace") {
-				fprintf(stderr, "ERROR: Trace not yet implemented");
 			} else if (objtype == "_delete_network") {
 				try {
 					std::string networkId = (*config)["nwid"];
@@ -1292,8 +1337,6 @@ void PostgreSQL::commitThread()
 		} catch (std::exception &e) {
 			fprintf(stderr, "ERROR: Error getting objtype: %s\n", e.what());
 		}
-		delete config;
-		config = nullptr;
 
 		std::this_thread::sleep_for(std::chrono::milliseconds(10));
 	}
@@ -1315,9 +1358,9 @@ void PostgreSQL::onlineNotificationThread()
 	}
 	_connected = 1;
 
-	int64_t	lastUpdatedNetworkStatus = 0;
+	//int64_t	lastUpdatedNetworkStatus = 0;
 	std::unordered_map< std::pair<uint64_t,uint64_t>,int64_t,_PairHasher > lastOnlineCumulative;
-	
+
 	while (_run == 1) {
 		if (PQstatus(conn) != CONNECTION_OK) {
 			fprintf(stderr, "ERROR: Online Notification thread lost connection to Postgres.");
@@ -1421,129 +1464,6 @@ void PostgreSQL::onlineNotificationThread()
 			PQclear(res);
 		}
 
-		const int64_t now = OSUtils::now();
-		if ((now - lastUpdatedNetworkStatus) > 10000) {
-			lastUpdatedNetworkStatus = now;
-
-			std::vector<std::pair<uint64_t, std::shared_ptr<_Network>>> networks;
-			{
-				std::lock_guard<std::mutex> l(_networks_l);
-				for (auto i = _networks.begin(); i != _networks.end(); ++i) {
-					networks.push_back(*i);
-				}
-			}
-
-			std::stringstream networkUpdate;
-			networkUpdate << "INSERT INTO ztc_network_status (network_id, bridge_count, authorized_member_count, online_member_count, total_member_count, last_modified) VALUES ";
-			bool nwFirstRun = true;
-			bool networkAdded = false;
-			for (auto i = networks.begin(); i != networks.end(); ++i) {
-				char tmp[64];
-				Utils::hex(i->first, tmp);
-
-				std::string networkId(tmp);
-
-				std::vector<std::string> &_notUsed = updateMap[networkId];
-				(void)_notUsed;
-
-				uint64_t authMemberCount = 0;
-				uint64_t totalMemberCount = 0;
-				uint64_t onlineMemberCount = 0;
-				uint64_t bridgeCount = 0;
-				uint64_t ts = now;
-				{
-					std::lock_guard<std::mutex> l2(i->second->lock);
-					authMemberCount = i->second->authorizedMembers.size();
-					totalMemberCount = i->second->members.size();
-					bridgeCount = i->second->activeBridgeMembers.size();
-					for (auto m=i->second->members.begin(); m != i->second->members.end(); ++m) {
-						auto lo = lastOnlineCumulative.find(std::pair<uint64_t,uint64_t>(i->first, m->first));
-						if (lo != lastOnlineCumulative.end()) {
-							if ((now - lo->second) <= (ZT_NETWORK_AUTOCONF_DELAY * 2)) {
-								++onlineMemberCount;
-							} else {
-								lastOnlineCumulative.erase(lo);
-							}
-						}
-					}
-				}
-
-				const char *nvals[1] = {
-					networkId.c_str()
-				};
-
-				res = PQexecParams(conn,
-					"SELECT id FROM ztc_network WHERE id = $1",
-					1,
-					NULL,
-					nvals,
-					NULL,
-					NULL,
-					0);
-
-				if (PQresultStatus(res) != PGRES_TUPLES_OK) {
-					fprintf(stderr, "Network lookup failed: %s", PQerrorMessage(conn));
-					PQclear(res);
-					continue;
-				}
-
-				int nrows = PQntuples(res);
-				PQclear(res);
-
-				if (nrows == 1) {
-					std::string bc = std::to_string(bridgeCount);
-					std::string amc = std::to_string(authMemberCount);
-					std::string omc = std::to_string(onlineMemberCount);
-					std::string tmc = std::to_string(totalMemberCount);
-					std::string timestamp = std::to_string(ts);
-
-					if (nwFirstRun) {
-						nwFirstRun = false;
-					} else {
-						networkUpdate << ", ";
-					}
-
-					networkUpdate << "('" << networkId << "', " << bc << ", " << amc << ", " << omc << ", " << tmc << ", "
-							<< "TO_TIMESTAMP(" << timestamp << "::double precision/1000))";
-
-					networkAdded = true;
-
-				} else if (nrows > 1) {
-					fprintf(stderr, "Number of networks > 1?!?!?");
-					continue;
-				} else {
-					continue;
-				}
-			}
-			networkUpdate << " ON CONFLICT (network_id) DO UPDATE SET bridge_count = EXCLUDED.bridge_count, "
-					<< "authorized_member_count = EXCLUDED.authorized_member_count, online_member_count = EXCLUDED.online_member_count, "
-					<< "total_member_count = EXCLUDED.total_member_count, last_modified = EXCLUDED.last_modified";
-			if (networkAdded) {
-				res = PQexec(conn, networkUpdate.str().c_str());
-				if (PQresultStatus(res) != PGRES_COMMAND_OK) {
-					fprintf(stderr, "Error during multiple network upsert: %s", PQresultErrorMessage(res));
-				}
-				PQclear(res);
-			}
-		}
-
-		// for (auto it = updateMap.begin(); it != updateMap.end(); ++it) {
-		// 	std::string networkId = it->first;
-		// 	std::vector<std::string> members = it->second;
-		// 	std::stringstream queryBuilder;
-
-		// 	std::string membersStr = ::join(members, ",");
-
-		// 	queryBuilder << "NOTIFY controller, '" << networkId << ":" << membersStr << "'";
-		// 	std::string query = queryBuilder.str();
-
-		// 	PGresult *res = PQexec(conn,query.c_str());
-		// 	if (PQresultStatus(res) != PGRES_COMMAND_OK) {
-		// 		fprintf(stderr, "ERROR: Error sending NOTIFY: %s\n", PQresultErrorMessage(res));
-		// 	}
-		// 	PQclear(res);
-		// }
-
 		std::this_thread::sleep_for(std::chrono::milliseconds(10));
 	}
 	fprintf(stderr, "%s: Fell out of run loop in onlineNotificationThread\n", _myAddressStr.c_str());
@@ -1554,7 +1474,8 @@ void PostgreSQL::onlineNotificationThread()
 	}
 }
 
-PGconn *PostgreSQL::getPgConn(OverrideMode m) {
+PGconn *PostgreSQL::getPgConn(OverrideMode m)
+{
 	if (m == ALLOW_PGBOUNCER_OVERRIDE) {
 		char *connStr = getenv("PGBOUNCER_CONNSTR");
 		if (connStr != NULL) {
@@ -1568,4 +1489,5 @@ PGconn *PostgreSQL::getPgConn(OverrideMode m) {
 
 	return PQconnectdb(_connString.c_str());
 }
+
 #endif //ZT_CONTROLLER_USE_LIBPQ

controller/PostgreSQL.hpp

@@ -23,22 +23,21 @@
  * directly against ZeroTier software without disclosing the source code
  * of your own application.
  */
- 
+
+#include "DB.hpp"
+
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 
 #ifndef ZT_CONTROLLER_LIBPQ_HPP
 #define ZT_CONTROLLER_LIBPQ_HPP
 
-#include "DB.hpp"
-
 #define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
 
 extern "C" {
-    typedef struct pg_conn PGconn;
+typedef struct pg_conn PGconn;
 }
 
-namespace ZeroTier
-{
+namespace ZeroTier {
 
 struct MQConfig;
 
@@ -51,66 +50,69 @@ struct MQConfig;
 class PostgreSQL : public DB
 {
 public:
-    PostgreSQL(const Identity &myId, const char *path, int listenPort, MQConfig *mqc = NULL);
-    virtual ~PostgreSQL();
+	PostgreSQL(const Identity &myId, const char *path, int listenPort, MQConfig *mqc = NULL);
+	virtual ~PostgreSQL();
 
-    virtual bool waitForReady();
-    virtual bool isReady();
-    virtual void save(nlohmann::json *orig, nlohmann::json &record);
-    virtual void eraseNetwork(const uint64_t networkId);
-    virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
-    virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json &record,bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress &physicalAddress);
 
 protected:
-    struct _PairHasher
+	struct _PairHasher
 	{
 		inline std::size_t operator()(const std::pair<uint64_t,uint64_t> &p) const { return (std::size_t)(p.first ^ p.second); }
 	};
 
 private:
-    void initializeNetworks(PGconn *conn);
-    void initializeMembers(PGconn *conn);
-    void heartbeat();
-    void membersDbWatcher();
-    void _membersWatcher_Postgres(PGconn *conn);
-    void _membersWatcher_RabbitMQ();
-    void networksDbWatcher();
-    void _networksWatcher_Postgres(PGconn *conn);
-    void _networksWatcher_RabbitMQ();
-
-    void commitThread();
-    void onlineNotificationThread();
-
-    enum OverrideMode {
-        ALLOW_PGBOUNCER_OVERRIDE = 0,
-        NO_OVERRIDE = 1
-    };
+	void initializeNetworks(PGconn *conn);
+	void initializeMembers(PGconn *conn);
+	void heartbeat();
+	void membersDbWatcher();
+	void _membersWatcher_Postgres(PGconn *conn);
+	void _membersWatcher_RabbitMQ();
+	void networksDbWatcher();
+	void _networksWatcher_Postgres(PGconn *conn);
+	void _networksWatcher_RabbitMQ();
+
+	void commitThread();
+	void onlineNotificationThread();
+
+	enum OverrideMode {
+		ALLOW_PGBOUNCER_OVERRIDE = 0,
+		NO_OVERRIDE = 1
+	};
 
-    PGconn * getPgConn( OverrideMode m = ALLOW_PGBOUNCER_OVERRIDE );
+	PGconn * getPgConn( OverrideMode m = ALLOW_PGBOUNCER_OVERRIDE );
 
-    std::string _connString;
+	const Identity _myId;
+	const Address _myAddress;
+	std::string _myAddressStr;
+	std::string _connString;
 
-    BlockingQueue<nlohmann::json *> _commitQueue;
+	BlockingQueue< std::pair<nlohmann::json,bool> > _commitQueue;
 
-    std::thread _heartbeatThread;
-    std::thread _membersDbWatcher;
-    std::thread _networksDbWatcher;
-    std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
-    std::thread _onlineNotificationThread;
+	std::thread _heartbeatThread;
+	std::thread _membersDbWatcher;
+	std::thread _networksDbWatcher;
+	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
+	std::thread _onlineNotificationThread;
 
 	std::unordered_map< std::pair<uint64_t,uint64_t>,std::pair<int64_t,InetAddress>,_PairHasher > _lastOnline;
 
-    mutable std::mutex _lastOnline_l;
-    mutable std::mutex _readyLock;
-    std::atomic<int> _ready, _connected, _run;
-    mutable volatile bool _waitNoticePrinted;
+	mutable std::mutex _lastOnline_l;
+	mutable std::mutex _readyLock;
+	std::atomic<int> _ready, _connected, _run;
+	mutable volatile bool _waitNoticePrinted;
 
-    int _listenPort;
+	int _listenPort;
 
-    MQConfig *_mqc;
+	MQConfig *_mqc;
 };
 
-}
+} // namespace ZeroTier
 
 #endif // ZT_CONTROLLER_LIBPQ_HPP
 

controller/RabbitMQ.cpp

@@ -1,3 +1,30 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+
 #include "RabbitMQ.hpp"
 
 #ifdef ZT_CONTROLLER_USE_LIBPQ
@@ -11,95 +38,95 @@ namespace ZeroTier
 {
 
 RabbitMQ::RabbitMQ(MQConfig *cfg, const char *queueName)
-    : _mqc(cfg)
-    , _qName(queueName)
-    , _socket(NULL)
-    , _status(0)
+	: _mqc(cfg)
+	, _qName(queueName)
+	, _socket(NULL)
+	, _status(0)
 {   
 }
 
 RabbitMQ::~RabbitMQ()
 {
-    amqp_channel_close(_conn, _channel, AMQP_REPLY_SUCCESS);
-    amqp_connection_close(_conn, AMQP_REPLY_SUCCESS);
-    amqp_destroy_connection(_conn);
+	amqp_channel_close(_conn, _channel, AMQP_REPLY_SUCCESS);
+	amqp_connection_close(_conn, AMQP_REPLY_SUCCESS);
+	amqp_destroy_connection(_conn);
 }
 
 void RabbitMQ::init()
 {
-    struct timeval tval;
-    memset(&tval, 0, sizeof(struct timeval));
-    tval.tv_sec = 5;
-
-    fprintf(stderr, "Initializing RabbitMQ %s\n", _qName);
-    _conn = amqp_new_connection();
-    _socket = amqp_tcp_socket_new(_conn);
-    if (!_socket) {
-        throw std::runtime_error("Can't create socket for RabbitMQ");
-    }
-    
-    _status = amqp_socket_open_noblock(_socket, _mqc->host, _mqc->port, &tval);
-    if (_status) {
-        throw std::runtime_error("Can't connect to RabbitMQ");
-    }
-    
-    amqp_rpc_reply_t r = amqp_login(_conn, "/", 0, 131072, 0, AMQP_SASL_METHOD_PLAIN,
-        _mqc->username, _mqc->password);
-    if (r.reply_type != AMQP_RESPONSE_NORMAL) {
-        throw std::runtime_error("RabbitMQ Login Error");
-    }
-
-    static int chan = 0;
+	struct timeval tval;
+	memset(&tval, 0, sizeof(struct timeval));
+	tval.tv_sec = 5;
+
+	fprintf(stderr, "Initializing RabbitMQ %s\n", _qName);
+	_conn = amqp_new_connection();
+	_socket = amqp_tcp_socket_new(_conn);
+	if (!_socket) {
+		throw std::runtime_error("Can't create socket for RabbitMQ");
+	}
+	
+	_status = amqp_socket_open_noblock(_socket, _mqc->host, _mqc->port, &tval);
+	if (_status) {
+		throw std::runtime_error("Can't connect to RabbitMQ");
+	}
+	
+	amqp_rpc_reply_t r = amqp_login(_conn, "/", 0, 131072, 0, AMQP_SASL_METHOD_PLAIN,
+		_mqc->username, _mqc->password);
+	if (r.reply_type != AMQP_RESPONSE_NORMAL) {
+		throw std::runtime_error("RabbitMQ Login Error");
+	}
+
+	static int chan = 0;
 	{
 		Mutex::Lock l(_chan_m);
-    	_channel = ++chan;
+		_channel = ++chan;
+	}
+	amqp_channel_open(_conn, _channel);
+	r = amqp_get_rpc_reply(_conn);
+	if(r.reply_type != AMQP_RESPONSE_NORMAL) {
+		throw std::runtime_error("Error opening communication channel");
 	}
-    amqp_channel_open(_conn, _channel);
-    r = amqp_get_rpc_reply(_conn);
-    if(r.reply_type != AMQP_RESPONSE_NORMAL) {
-        throw std::runtime_error("Error opening communication channel");
-    }
-    
-    _q = amqp_queue_declare(_conn, _channel, amqp_cstring_bytes(_qName), 0, 0, 0, 0, amqp_empty_table);
-    r = amqp_get_rpc_reply(_conn);
-    if (r.reply_type != AMQP_RESPONSE_NORMAL) {
-        throw std::runtime_error("Error declaring queue " + std::string(_qName));
-    }
-
-    amqp_basic_consume(_conn, _channel, amqp_cstring_bytes(_qName), amqp_empty_bytes, 0, 1, 0, amqp_empty_table);
-    r = amqp_get_rpc_reply(_conn);
-    if (r.reply_type != AMQP_RESPONSE_NORMAL) {
-        throw std::runtime_error("Error consuming queue " + std::string(_qName));
-    }
-    fprintf(stderr, "RabbitMQ Init OK %s\n", _qName);
+	
+	_q = amqp_queue_declare(_conn, _channel, amqp_cstring_bytes(_qName), 0, 0, 0, 0, amqp_empty_table);
+	r = amqp_get_rpc_reply(_conn);
+	if (r.reply_type != AMQP_RESPONSE_NORMAL) {
+		throw std::runtime_error("Error declaring queue " + std::string(_qName));
+	}
+
+	amqp_basic_consume(_conn, _channel, amqp_cstring_bytes(_qName), amqp_empty_bytes, 0, 1, 0, amqp_empty_table);
+	r = amqp_get_rpc_reply(_conn);
+	if (r.reply_type != AMQP_RESPONSE_NORMAL) {
+		throw std::runtime_error("Error consuming queue " + std::string(_qName));
+	}
+	fprintf(stderr, "RabbitMQ Init OK %s\n", _qName);
 }
 
 std::string RabbitMQ::consume()
 {
-    amqp_rpc_reply_t res;
-    amqp_envelope_t envelope;
-    amqp_maybe_release_buffers(_conn);
-
-    struct timeval timeout;
-    timeout.tv_sec = 1;
-    timeout.tv_usec = 0;
-
-    res = amqp_consume_message(_conn, &envelope, &timeout, 0);
-    if (res.reply_type != AMQP_RESPONSE_NORMAL) {
-        if (res.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION && res.library_error == AMQP_STATUS_TIMEOUT) {
-            // timeout waiting for message.  Return empty string
-            return "";
-        } else {
-            throw std::runtime_error("Error getting message");
-        }
-    }
-
-    std::string msg(
-        (const char*)envelope.message.body.bytes,
-        envelope.message.body.len
-    );
-    amqp_destroy_envelope(&envelope);
-    return msg;
+	amqp_rpc_reply_t res;
+	amqp_envelope_t envelope;
+	amqp_maybe_release_buffers(_conn);
+
+	struct timeval timeout;
+	timeout.tv_sec = 1;
+	timeout.tv_usec = 0;
+
+	res = amqp_consume_message(_conn, &envelope, &timeout, 0);
+	if (res.reply_type != AMQP_RESPONSE_NORMAL) {
+		if (res.reply_type == AMQP_RESPONSE_LIBRARY_EXCEPTION && res.library_error == AMQP_STATUS_TIMEOUT) {
+			// timeout waiting for message.  Return empty string
+			return "";
+		} else {
+			throw std::runtime_error("Error getting message");
+		}
+	}
+
+	std::string msg(
+		(const char*)envelope.message.body.bytes,
+		envelope.message.body.len
+	);
+	amqp_destroy_envelope(&envelope);
+	return msg;
 }
 
 }

controller/RabbitMQ.hpp

@@ -23,16 +23,19 @@
  * directly against ZeroTier software without disclosing the source code
  * of your own application.
  */
+
 #ifndef ZT_CONTROLLER_RABBITMQ_HPP
 #define ZT_CONTROLLER_RABBITMQ_HPP
 
+#include "DB.hpp"
+
 namespace ZeroTier
 {
 struct MQConfig {
-    const char *host;
-    int port;
-    const char *username;
-    const char *password;
+	const char *host;
+	int port;
+	const char *username;
+	const char *password;
 };
 }
 
@@ -49,26 +52,25 @@ namespace ZeroTier
 
 class RabbitMQ {
 public:
-    RabbitMQ(MQConfig *cfg, const char *queueName);
-    ~RabbitMQ();
+	RabbitMQ(MQConfig *cfg, const char *queueName);
+	~RabbitMQ();
 
-    void init();
+	void init();
 
-    std::string consume();
+	std::string consume();
 
 private:
-    MQConfig *_mqc;
-    const char *_qName;
+	MQConfig *_mqc;
+	const char *_qName;
 
-    amqp_socket_t *_socket;
-    amqp_connection_state_t _conn;
-    amqp_queue_declare_ok_t *_q;
-    int _status;
+	amqp_socket_t *_socket;
+	amqp_connection_state_t _conn;
+	amqp_queue_declare_ok_t *_q;
+	int _status;
 
-    int _channel;
+	int _channel;
 
 	Mutex _chan_m;
-
 };
 
 }

debian/changelog

@@ -1,3 +1,16 @@
+zerotier-one (1.4.2-2) unstable; urgency=medium
+
+  * See https://github.com/zerotier/ZeroTierOne for release notes.
+  * This is a new build that fixes a binary build issue with containers and SELinux
+
+ -- Adam Ierymenko <[email protected]>  Thu, 04 Aug 2019 01:00:00 -0700
+
+zerotier-one (1.4.2) unstable; urgency=medium
+
+  * See https://github.com/zerotier/ZeroTierOne for release notes.
+
+ -- Adam Ierymenko <[email protected]>  Thu, 04 Aug 2019 01:00:00 -0700
+
 zerotier-one (1.4.0) unstable; urgency=medium
 
   * See https://github.com/zerotier/ZeroTierOne for release notes.

ext/bin/tap-mac/tap.kext/Contents/Info.plist

@@ -0,0 +1,36 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>CFBundleDevelopmentRegion</key>
+	<string>English</string>
+	<key>CFBundleExecutable</key>
+	<string>tap</string>
+	<key>CFBundleIdentifier</key>
+	<string>com.zerotier.tap</string>
+	<key>CFBundleInfoDictionaryVersion</key>
+	<string>6.0</string>
+	<key>CFBundleName</key>
+	<string>tap</string>
+	<key>CFBundlePackageType</key>
+	<string>KEXT</string>
+	<key>CFBundleShortVersionString</key>
+	<string>20150118</string>
+	<key>CFBundleSignature</key>
+	<string>????</string>
+	<key>CFBundleVersion</key>
+	<string>1.0</string>
+	<key>OSBundleLibraries</key>
+	<dict>
+		<key>com.apple.kpi.mach</key>
+		<string>8.0</string>
+		<key>com.apple.kpi.bsd</key>
+		<string>8.0</string>
+		<key>com.apple.kpi.libkern</key>
+		<string>8.0</string>
+		<key>com.apple.kpi.unsupported</key>
+		<string>8.0</string>
+	</dict>
+</dict>
+</plist>
+

ext/bin/tap-mac/tap.kext/Contents/_CodeSignature/CodeResources

@@ -0,0 +1,105 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+	<key>files</key>
+	<dict/>
+	<key>files2</key>
+	<dict/>
+	<key>rules</key>
+	<dict>
+		<key>^Resources/</key>
+		<true/>
+		<key>^Resources/.*\.lproj/</key>
+		<dict>
+			<key>optional</key>
+			<true/>
+			<key>weight</key>
+			<real>1000</real>
+		</dict>
+		<key>^Resources/.*\.lproj/locversion.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>1100</real>
+		</dict>
+		<key>^version.plist$</key>
+		<true/>
+	</dict>
+	<key>rules2</key>
+	<dict>
+		<key>.*\.dSYM($|/)</key>
+		<dict>
+			<key>weight</key>
+			<real>11</real>
+		</dict>
+		<key>^(.*/)?\.DS_Store$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>2000</real>
+		</dict>
+		<key>^(Frameworks|SharedFrameworks|PlugIns|Plug-ins|XPCServices|Helpers|MacOS|Library/(Automator|Spotlight|LoginItems))/</key>
+		<dict>
+			<key>nested</key>
+			<true/>
+			<key>weight</key>
+			<real>10</real>
+		</dict>
+		<key>^.*</key>
+		<true/>
+		<key>^Info\.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^PkgInfo$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^Resources/</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^Resources/.*\.lproj/</key>
+		<dict>
+			<key>optional</key>
+			<true/>
+			<key>weight</key>
+			<real>1000</real>
+		</dict>
+		<key>^Resources/.*\.lproj/locversion.plist$</key>
+		<dict>
+			<key>omit</key>
+			<true/>
+			<key>weight</key>
+			<real>1100</real>
+		</dict>
+		<key>^[^/]+$</key>
+		<dict>
+			<key>nested</key>
+			<true/>
+			<key>weight</key>
+			<real>10</real>
+		</dict>
+		<key>^embedded\.provisionprofile$</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+		<key>^version\.plist$</key>
+		<dict>
+			<key>weight</key>
+			<real>20</real>
+		</dict>
+	</dict>
+</dict>
+</plist>

ext/installfiles/mac/postinst.sh

@@ -3,6 +3,7 @@
 export PATH=/bin:/usr/bin:/sbin:/usr/sbin:/usr/local/bin
 
 OSX_RELEASE=`sw_vers -productVersion | cut -d . -f 1,2`
+DARWIN_MAJOR=`uname -r | cut -d . -f 1`
 
 launchctl unload /Library/LaunchDaemons/com.zerotier.one.plist >>/dev/null 2>&1
 sleep 0.5
@@ -43,9 +44,11 @@ rm -f zerotier-cli zerotier-idtool
 ln -sf "/Library/Application Support/ZeroTier/One/zerotier-one" zerotier-cli
 ln -sf "/Library/Application Support/ZeroTier/One/zerotier-one" zerotier-idtool
 
-cd "/Library/Application Support/ZeroTier/One"
-kextload -r . tap.kext >>/dev/null 2>&1 &
-disown %1
+if [ $DARWIN_MAJOR -le 16 ]; then
+	cd "/Library/Application Support/ZeroTier/One"
+	kextload -r . tap.kext >>/dev/null 2>&1 &
+	disown %1
+fi
 
 launchctl load /Library/LaunchDaemons/com.zerotier.one.plist >>/dev/null 2>&1
 

ext/installfiles/windows/ZeroTier One.aip

@@ -27,10 +27,10 @@
     <ROW Property="CTRLS" Value="2"/>
     <ROW Property="MSIFASTINSTALL" MultiBuildValue="DefaultBuild:2"/>
     <ROW Property="Manufacturer" Value="ZeroTier, Inc."/>
-    <ROW Property="ProductCode" Value="1033:{FF7D9C9B-E9F3-460A-8EA9-9074AF186E1E} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{E5E38B77-644B-48BA-A120-3CA86EECA9B1} " Type="16"/>
     <ROW Property="ProductLanguage" Value="1033"/>
     <ROW Property="ProductName" Value="ZeroTier One"/>
-    <ROW Property="ProductVersion" Value="1.4.0" Type="32"/>
+    <ROW Property="ProductVersion" Value="1.4.2" Type="32"/>
     <ROW Property="REBOOT" MultiBuildValue="DefaultBuild:ReallySuppress"/>
     <ROW Property="RUNAPPLICATION" Value="1" Type="4"/>
     <ROW Property="SecureCustomProperties" Value="OLDPRODUCTS;AI_NEWERPRODUCTFOUND;AI_SETUPEXEPATH;SETUPEXEDIR"/>
@@ -64,7 +64,7 @@
     <ROW Directory="x86_Dir" Directory_Parent="tapwindows_Dir" DefaultDir="x86"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiCompsComponent">
-    <ROW Component="AI_CustomARPName" ComponentId="{3771DD83-BFE1-467A-88C3-DE78086A2B44}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{A660DE2F-81EF-45AB-AC9B-BAD4AB0E806D}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_DisableModify" ComponentId="{020DCABD-5D56-49B9-AF48-F07F0B55E590}" Directory_="APPDIR" Attributes="4" KeyPath="NoModify" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{8E02B36C-7A19-429B-A93E-77A9261AC918}" Directory_="APPDIR" Attributes="4" KeyPath="AI_ExePath"/>
     <ROW Component="Hardcodet.Wpf.TaskbarNotification.dll" ComponentId="{BEA825AF-2555-44AF-BE40-47FFC16DCBA6}" Directory_="APPDIR" Attributes="0" KeyPath="Hardcodet.Wpf.TaskbarNotification.dll"/>
@@ -454,7 +454,7 @@
     <ROW XmlAttribute="xsischemaLocation" XmlElement="swidsoftware_identification_tag" Name="xsi:schemaLocation" Flags="14" Order="3" Value="http://standards.iso.org/iso/19770/-2/2008/schema.xsd software_identification_tag.xsd"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.XmlElementComponent">
-    <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="0"/>
+    <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="2"/>
     <ROW XmlElement="swidentitlement_required_indicator" ParentElement="swidsoftware_identification_tag" Name="swid:entitlement_required_indicator" Condition="1" Order="0" Flags="14" Text="false"/>
     <ROW XmlElement="swidmajor" ParentElement="swidnumeric" Name="swid:major" Condition="1" Order="0" Flags="14" Text="1"/>
     <ROW XmlElement="swidminor" ParentElement="swidnumeric" Name="swid:minor" Condition="1" Order="1" Flags="14" Text="4"/>

include/ZeroTierOne.h

@@ -198,6 +198,11 @@ extern "C" {
  */
 #define ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH 7
 
+/**
+ * Maximum number of multicast groups a device / network interface can be subscribed to at once
+ */
+#define ZT_MAX_MULTICAST_SUBSCRIPTIONS 1024
+
 /**
  * Maximum value for link quality (min is 0)
  */
@@ -1193,6 +1198,19 @@ typedef struct
 	 * Routes (excluding those implied by assigned addresses and their masks)
 	 */
 	ZT_VirtualNetworkRoute routes[ZT_MAX_NETWORK_ROUTES];
+
+	/**
+	 * Number of multicast groups subscribed
+	 */
+	unsigned int multicastSubscriptionCount;
+
+	/**
+	 * Multicast groups to which this network's device is subscribed
+	 */
+	struct {
+		uint64_t mac; /* MAC in lower 48 bits */
+		uint32_t adi; /* Additional distinguishing information, usually zero except for IPv4 ARP groups */
+	} multicastSubscriptions[ZT_MAX_MULTICAST_SUBSCRIPTIONS];
 } ZT_VirtualNetworkConfig;
 
 /**

make-linux.mk

@@ -1,12 +1,12 @@
 # Automagically pick clang or gcc, with preference for clang
 # This is only done if we have not overridden these with an environment or CLI variable
 ifeq ($(origin CC),default)
-	CC:=$(shell if [ -e /usr/bin/clang ]; then echo clang; else echo gcc; fi)
-	CC:=$(shell if [ -e /opt/intel/bin/icc ]; then echo /opt/intel/bin/icc -ipo -ansi-alias; else echo $(CC); fi)
+        CC:=$(shell if [ -e /usr/bin/clang ]; then echo clang; else echo gcc; fi)
+        CC:=$(shell if [ -e /opt/rh/devtoolset-8/root/usr/bin/gcc ]; then echo /opt/rh/devtoolset-8/root/usr/bin/gcc; else echo $(CC); fi)
 endif
 ifeq ($(origin CXX),default)
-	CXX:=$(shell if [ -e /usr/bin/clang++ ]; then echo clang++; else echo g++; fi)
-	CXX:=$(shell if [ -e /opt/intel/bin/icc ]; then echo /opt/intel/bin/icc -ipo -ansi-alias; else echo $(CXX); fi)
+        CXX:=$(shell if [ -e /usr/bin/clang++ ]; then echo clang++; else echo g++; fi)
+        CXX:=$(shell if [ -e /opt/rh/devtoolset-8/root/usr/bin/g++ ]; then echo /opt/rh/devtoolset-8/root/usr/bin/g++; else echo $(CXX); fi)
 endif
 
 INCLUDES?=
@@ -14,7 +14,6 @@ DEFS?=
 LDLIBS?=
 DESTDIR?=
 
-
 include objects.mk
 ONE_OBJS+=osdep/LinuxEthernetTap.o
 ONE_OBJS+=osdep/LinuxNetLink.o
@@ -402,4 +401,11 @@ debian-clean: FORCE
 redhat:	FORCE
 	rpmbuild -ba zerotier-one.spec
 
+# This installs the packages needed to build ZT locally on CentOS 7 and
+# is here largely for documentation purposes.
+centos-7-setup: FORCE
+	yum install -y gcc gcc-c++ make epel-release git
+	yum install -y centos-release-scl
+	yum install -y devtoolset-8-gcc devtoolset-8-gcc-c++
+
 FORCE:

make-mac.mk

@@ -19,7 +19,7 @@ ZT_VERSION_BUILD=$(shell cat version.h | grep -F VERSION_BUILD | cut -d ' ' -f 3
 DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_BUILD_ARCHITECTURE)
 
 include objects.mk
-ONE_OBJS+=osdep/MacEthernetTap.o ext/http-parser/http_parser.o
+ONE_OBJS+=osdep/MacEthernetTap.o osdep/MacKextEthernetTap.o ext/http-parser/http_parser.o
 
 ifeq ($(ZT_CONTROLLER),1)
 	LIBS+=-lpq -lrabbitmq

node/C25519.cpp

@@ -22,6 +22,10 @@ Derived from public domain code by D. J. Bernstein.
 #pragma warning(disable: 4146)
 #endif
 
+#ifdef __GNUC__
+#pragma GCC diagnostic ignored "-Wunused-function"
+#endif
+
 namespace {
 
 #define crypto_int32 int32_t
@@ -739,263 +743,6 @@ static void crypto_scalarmult(u8 *mypublic, const u8 *secret, const u8 *basepoin
   fcontract(mypublic, z);
 }
 
-#if 0
-void add(unsigned int out[32],const unsigned int a[32],const unsigned int b[32])
-{
-	unsigned int j;
-	unsigned int u;
-	u = 0;
-	for (j = 0;j < 31;++j) { u += a[j] + b[j]; out[j] = u & 255; u >>= 8; }
-	u += a[31] + b[31]; out[31] = u;
-}
-
-void sub(unsigned int out[32],const unsigned int a[32],const unsigned int b[32])
-{
-	unsigned int j;
-	unsigned int u;
-	u = 218;
-	for (j = 0;j < 31;++j) {
-		u += a[j] + 65280 - b[j];
-		out[j] = u & 255;
-		u >>= 8;
-	}
-	u += a[31] - b[31];
-	out[31] = u;
-}
-
-void squeeze(unsigned int a[32])
-{
-	unsigned int j;
-	unsigned int u;
-	u = 0;
-	for (j = 0;j < 31;++j) { u += a[j]; a[j] = u & 255; u >>= 8; }
-	u += a[31]; a[31] = u & 127;
-	u = 19 * (u >> 7);
-	for (j = 0;j < 31;++j) { u += a[j]; a[j] = u & 255; u >>= 8; }
-	u += a[31]; a[31] = u;
-}
-
-static const unsigned int minusp[32] = {
- 19, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 128
-} ;
-
-void freeze(unsigned int a[32])
-{
-	unsigned int aorig[32];
-	unsigned int j;
-	unsigned int negative;
-
-	for (j = 0;j < 32;++j) aorig[j] = a[j];
-	add(a,a,minusp);
-	negative = -((a[31] >> 7) & 1);
-	for (j = 0;j < 32;++j) a[j] ^= negative & (aorig[j] ^ a[j]);
-}
-
-void mult(unsigned int out[32],const unsigned int a[32],const unsigned int b[32])
-{
-	unsigned int i;
-	unsigned int j;
-	unsigned int u;
-
-	for (i = 0;i < 32;++i) {
-		u = 0;
-		for (j = 0;j <= i;++j) u += a[j] * b[i - j];
-		for (j = i + 1;j < 32;++j) u += 38 * a[j] * b[i + 32 - j];
-		out[i] = u;
-	}
-	squeeze(out);
-}
-
-void mult121665(unsigned int out[32],const unsigned int a[32])
-{
-	unsigned int j;
-	unsigned int u;
-
-	u = 0;
-	for (j = 0;j < 31;++j) { u += 121665 * a[j]; out[j] = u & 255; u >>= 8; }
-	u += 121665 * a[31]; out[31] = u & 127;
-	u = 19 * (u >> 7);
-	for (j = 0;j < 31;++j) { u += out[j]; out[j] = u & 255; u >>= 8; }
-	u += out[j]; out[j] = u;
-}
-
-void square(unsigned int out[32],const unsigned int a[32])
-{
-	unsigned int i;
-	unsigned int j;
-	unsigned int u;
-
-	for (i = 0;i < 32;++i) {
-		u = 0;
-		for (j = 0;j < i - j;++j) u += a[j] * a[i - j];
-		for (j = i + 1;j < i + 32 - j;++j) u += 38 * a[j] * a[i + 32 - j];
-		u *= 2;
-		if ((i & 1) == 0) {
-			u += a[i / 2] * a[i / 2];
-			u += 38 * a[i / 2 + 16] * a[i / 2 + 16];
-		}
-		out[i] = u;
-	}
-	squeeze(out);
-}
-
-void select(unsigned int p[64],unsigned int q[64],const unsigned int r[64],const unsigned int s[64],unsigned int b)
-{
-	unsigned int j;
-	unsigned int t;
-	unsigned int bminus1;
-
-	bminus1 = b - 1;
-	for (j = 0;j < 64;++j) {
-		t = bminus1 & (r[j] ^ s[j]);
-		p[j] = s[j] ^ t;
-		q[j] = r[j] ^ t;
-	}
-}
-
-static void mainloop(unsigned int work[64],const unsigned char e[32])
-{
-	unsigned int xzm1[64];
-	unsigned int xzm[64];
-	unsigned int xzmb[64];
-	unsigned int xzm1b[64];
-	unsigned int xznb[64];
-	unsigned int xzn1b[64];
-	unsigned int a0[64];
-	unsigned int a1[64];
-	unsigned int b0[64];
-	unsigned int b1[64];
-	unsigned int c1[64];
-	unsigned int r[32];
-	unsigned int s[32];
-	unsigned int t[32];
-	unsigned int u[32];
-	//unsigned int i;
-	unsigned int j;
-	unsigned int b;
-	int pos;
-
-	for (j = 0;j < 32;++j) xzm1[j] = work[j];
-	xzm1[32] = 1;
-	for (j = 33;j < 64;++j) xzm1[j] = 0;
-
-	xzm[0] = 1;
-	for (j = 1;j < 64;++j) xzm[j] = 0;
-
-	for (pos = 254;pos >= 0;--pos) {
-		b = e[pos / 8] >> (pos & 7);
-		b &= 1;
-		select(xzmb,xzm1b,xzm,xzm1,b);
-		add(a0,xzmb,xzmb + 32);
-		sub(a0 + 32,xzmb,xzmb + 32);
-		add(a1,xzm1b,xzm1b + 32);
-		sub(a1 + 32,xzm1b,xzm1b + 32);
-		square(b0,a0);
-		square(b0 + 32,a0 + 32);
-		mult(b1,a1,a0 + 32);
-		mult(b1 + 32,a1 + 32,a0);
-		add(c1,b1,b1 + 32);
-		sub(c1 + 32,b1,b1 + 32);
-		square(r,c1 + 32);
-		sub(s,b0,b0 + 32);
-		mult121665(t,s);
-		add(u,t,b0);
-		mult(xznb,b0,b0 + 32);
-		mult(xznb + 32,s,u);
-		square(xzn1b,c1);
-		mult(xzn1b + 32,r,work);
-		select(xzm,xzm1,xznb,xzn1b,b);
-	}
-
-	for (j = 0;j < 64;++j) work[j] = xzm[j];
-}
-
-static void recip(unsigned int out[32],const unsigned int z[32])
-{
-	unsigned int z2[32];
-	unsigned int z9[32];
-	unsigned int z11[32];
-	unsigned int z2_5_0[32];
-	unsigned int z2_10_0[32];
-	unsigned int z2_20_0[32];
-	unsigned int z2_50_0[32];
-	unsigned int z2_100_0[32];
-	unsigned int t0[32];
-	unsigned int t1[32];
-	int i;
-
-	/* 2 */ square(z2,z);
-	/* 4 */ square(t1,z2);
-	/* 8 */ square(t0,t1);
-	/* 9 */ mult(z9,t0,z);
-	/* 11 */ mult(z11,z9,z2);
-	/* 22 */ square(t0,z11);
-	/* 2^5 - 2^0 = 31 */ mult(z2_5_0,t0,z9);
-
-	/* 2^6 - 2^1 */ square(t0,z2_5_0);
-	/* 2^7 - 2^2 */ square(t1,t0);
-	/* 2^8 - 2^3 */ square(t0,t1);
-	/* 2^9 - 2^4 */ square(t1,t0);
-	/* 2^10 - 2^5 */ square(t0,t1);
-	/* 2^10 - 2^0 */ mult(z2_10_0,t0,z2_5_0);
-
-	/* 2^11 - 2^1 */ square(t0,z2_10_0);
-	/* 2^12 - 2^2 */ square(t1,t0);
-	/* 2^20 - 2^10 */ for (i = 2;i < 10;i += 2) { square(t0,t1); square(t1,t0); }
-	/* 2^20 - 2^0 */ mult(z2_20_0,t1,z2_10_0);
-
-	/* 2^21 - 2^1 */ square(t0,z2_20_0);
-	/* 2^22 - 2^2 */ square(t1,t0);
-	/* 2^40 - 2^20 */ for (i = 2;i < 20;i += 2) { square(t0,t1); square(t1,t0); }
-	/* 2^40 - 2^0 */ mult(t0,t1,z2_20_0);
-
-	/* 2^41 - 2^1 */ square(t1,t0);
-	/* 2^42 - 2^2 */ square(t0,t1);
-	/* 2^50 - 2^10 */ for (i = 2;i < 10;i += 2) { square(t1,t0); square(t0,t1); }
-	/* 2^50 - 2^0 */ mult(z2_50_0,t0,z2_10_0);
-
-	/* 2^51 - 2^1 */ square(t0,z2_50_0);
-	/* 2^52 - 2^2 */ square(t1,t0);
-	/* 2^100 - 2^50 */ for (i = 2;i < 50;i += 2) { square(t0,t1); square(t1,t0); }
-	/* 2^100 - 2^0 */ mult(z2_100_0,t1,z2_50_0);
-
-	/* 2^101 - 2^1 */ square(t1,z2_100_0);
-	/* 2^102 - 2^2 */ square(t0,t1);
-	/* 2^200 - 2^100 */ for (i = 2;i < 100;i += 2) { square(t1,t0); square(t0,t1); }
-	/* 2^200 - 2^0 */ mult(t1,t0,z2_100_0);
-
-	/* 2^201 - 2^1 */ square(t0,t1);
-	/* 2^202 - 2^2 */ square(t1,t0);
-	/* 2^250 - 2^50 */ for (i = 2;i < 50;i += 2) { square(t0,t1); square(t1,t0); }
-	/* 2^250 - 2^0 */ mult(t0,t1,z2_50_0);
-
-	/* 2^251 - 2^1 */ square(t1,t0);
-	/* 2^252 - 2^2 */ square(t0,t1);
-	/* 2^253 - 2^3 */ square(t1,t0);
-	/* 2^254 - 2^4 */ square(t0,t1);
-	/* 2^255 - 2^5 */ square(t1,t0);
-	/* 2^255 - 21 */ mult(out,t1,z11);
-}
-
-int crypto_scalarmult(unsigned char *q,const unsigned char *n,const unsigned char *p)
-{
-	unsigned int work[96];
-	unsigned char e[32];
-	unsigned int i;
-	for (i = 0;i < 32;++i) e[i] = n[i];
-	e[0] &= 248;
-	e[31] &= 127;
-	e[31] |= 64;
-	for (i = 0;i < 32;++i) work[i] = p[i];
-	mainloop(work,e);
-	recip(work + 32,work + 32);
-	mult(work + 64,work,work + 32);
-	freeze(work + 64);
-	for (i = 0;i < 32;++i) q[i] = work[64 + i];
-	return 0;
-}
-#endif
-
 static const unsigned char base[32] = {9};
 static inline void crypto_scalarmult_base(unsigned char *q,const unsigned char *n)
 {
@@ -1056,9 +803,9 @@ typedef struct
 	fe25519 y;
 } ge25519_aff;
 
-static void fe25519_sub(fe25519 *r, const fe25519 *x, const fe25519 *y);
+static inline void fe25519_sub(fe25519 *r, const fe25519 *x, const fe25519 *y);
 
-crypto_uint32 equal(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
+static inline crypto_uint32 equal(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
 {
 	crypto_uint32 x = a ^ b; /* 0: yes; 1..65535: no */
 	x -= 1; /* 4294967295: yes; 0..65534: no */
@@ -1066,7 +813,7 @@ crypto_uint32 equal(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
 	return x;
 }
 
-crypto_uint32 ge(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
+static inline crypto_uint32 ge(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
 {
 	unsigned int x = a;
 	x -= (unsigned int) b; /* 0..65535: yes; 4294901761..4294967295: no */
@@ -1075,17 +822,17 @@ crypto_uint32 ge(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
 	return x;
 }
 
-crypto_uint32 times19(crypto_uint32 a)
+static inline crypto_uint32 times19(crypto_uint32 a)
 {
 	return (a << 4) + (a << 1) + a;
 }
 
-crypto_uint32 times38(crypto_uint32 a)
+static inline crypto_uint32 times38(crypto_uint32 a)
 {
 	return (a << 5) + (a << 2) + (a << 1);
 }
 
-void reduce_add_sub(fe25519 *r)
+static inline void reduce_add_sub(fe25519 *r)
 {
 	crypto_uint32 t;
 	int i,rep;
@@ -1105,7 +852,7 @@ void reduce_add_sub(fe25519 *r)
 	}
 }
 
-void reduce_mul(fe25519 *r)
+static inline void reduce_mul(fe25519 *r)
 {
 	crypto_uint32 t;
 	int i,rep;
@@ -1126,7 +873,7 @@ void reduce_mul(fe25519 *r)
 }
 
 /* reduction modulo 2^255-19 */
-void fe25519_freeze(fe25519 *r)
+static inline void fe25519_freeze(fe25519 *r)
 {
 	int i;
 	crypto_uint32 m = equal(r->v[31],127);
@@ -1142,7 +889,7 @@ void fe25519_freeze(fe25519 *r)
 	r->v[0] -= m&237;
 }
 
-void fe25519_unpack(fe25519 *r, const unsigned char x[32])
+static inline void fe25519_unpack(fe25519 *r, const unsigned char x[32])
 {
 	int i;
 	for(i=0;i<32;i++) r->v[i] = x[i];
@@ -1150,7 +897,7 @@ void fe25519_unpack(fe25519 *r, const unsigned char x[32])
 }
 
 /* Assumes input x being reduced below 2^255 */
-void fe25519_pack(unsigned char r[32], const fe25519 *x)
+static inline void fe25519_pack(unsigned char r[32], const fe25519 *x)
 {
 	int i;
 	fe25519 y = *x;
@@ -1159,7 +906,7 @@ void fe25519_pack(unsigned char r[32], const fe25519 *x)
 		r[i] = y.v[i];
 }
 
-int fe25519_iseq_vartime(const fe25519 *x, const fe25519 *y)
+static inline int fe25519_iseq_vartime(const fe25519 *x, const fe25519 *y)
 {
 	int i;
 	fe25519 t1 = *x;
@@ -1171,7 +918,7 @@ int fe25519_iseq_vartime(const fe25519 *x, const fe25519 *y)
 	return 1;
 }
 
-void fe25519_cmov(fe25519 *r, const fe25519 *x, unsigned char b)
+static inline void fe25519_cmov(fe25519 *r, const fe25519 *x, unsigned char b)
 {
 	int i;
 	crypto_uint32 mask = b;
@@ -1179,27 +926,27 @@ void fe25519_cmov(fe25519 *r, const fe25519 *x, unsigned char b)
 	for(i=0;i<32;i++) r->v[i] ^= mask & (x->v[i] ^ r->v[i]);
 }
 
-unsigned char fe25519_getparity(const fe25519 *x)
+static inline unsigned char fe25519_getparity(const fe25519 *x)
 {
 	fe25519 t = *x;
 	fe25519_freeze(&t);
 	return t.v[0] & 1;
 }
 
-void fe25519_setone(fe25519 *r)
+static inline void fe25519_setone(fe25519 *r)
 {
 	int i;
 	r->v[0] = 1;
 	for(i=1;i<32;i++) r->v[i]=0;
 }
 
-void fe25519_setzero(fe25519 *r)
+static inline void fe25519_setzero(fe25519 *r)
 {
 	int i;
 	for(i=0;i<32;i++) r->v[i]=0;
 }
 
-void fe25519_neg(fe25519 *r, const fe25519 *x)
+static inline void fe25519_neg(fe25519 *r, const fe25519 *x)
 {
 	fe25519 t;
 	int i;
@@ -1208,14 +955,14 @@ void fe25519_neg(fe25519 *r, const fe25519 *x)
 	fe25519_sub(r, r, &t);
 }
 
-void fe25519_add(fe25519 *r, const fe25519 *x, const fe25519 *y)
+static inline void fe25519_add(fe25519 *r, const fe25519 *x, const fe25519 *y)
 {
 	int i;
 	for(i=0;i<32;i++) r->v[i] = x->v[i] + y->v[i];
 	reduce_add_sub(r);
 }
 
-void fe25519_sub(fe25519 *r, const fe25519 *x, const fe25519 *y)
+static inline void fe25519_sub(fe25519 *r, const fe25519 *x, const fe25519 *y)
 {
 	int i;
 	crypto_uint32 t[32];
@@ -1226,7 +973,7 @@ void fe25519_sub(fe25519 *r, const fe25519 *x, const fe25519 *y)
 	reduce_add_sub(r);
 }
 
-void fe25519_mul(fe25519 *r, const fe25519 *x, const fe25519 *y)
+static inline void fe25519_mul(fe25519 *r, const fe25519 *x, const fe25519 *y)
 {
 	int i,j;
 	crypto_uint32 t[63];
@@ -1243,12 +990,12 @@ void fe25519_mul(fe25519 *r, const fe25519 *x, const fe25519 *y)
 	reduce_mul(r);
 }
 
-void fe25519_square(fe25519 *r, const fe25519 *x)
+static inline void fe25519_square(fe25519 *r, const fe25519 *x)
 {
 	fe25519_mul(r, x, x);
 }
 
-void fe25519_invert(fe25519 *r, const fe25519 *x)
+static inline void fe25519_invert(fe25519 *r, const fe25519 *x)
 {
 	fe25519 z2;
 	fe25519 z9;
@@ -1315,7 +1062,7 @@ void fe25519_invert(fe25519 *r, const fe25519 *x)
 	/* 2^255 - 21 */ fe25519_mul(r,&t1,&z11);
 }
 
-void fe25519_pow2523(fe25519 *r, const fe25519 *x)
+static inline void fe25519_pow2523(fe25519 *r, const fe25519 *x)
 {
 	fe25519 z2;
 	fe25519 z9;
@@ -1369,13 +1116,10 @@ void fe25519_pow2523(fe25519 *r, const fe25519 *x)
 	/* 2^252 - 3 */ fe25519_mul(r,&t,x);
 }
 
-static const crypto_uint32 m[32] = {0xED, 0xD3, 0xF5, 0x5C, 0x1A, 0x63, 0x12, 0x58, 0xD6, 0x9C, 0xF7, 0xA2, 0xDE, 0xF9, 0xDE, 0x14,
-																		0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10};
-
-static const crypto_uint32 mu[33] = {0x1B, 0x13, 0x2C, 0x0A, 0xA3, 0xE5, 0x9C, 0xED, 0xA7, 0x29, 0x63, 0x08, 0x5D, 0x21, 0x06, 0x21,
-																		 0xEB, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F};
+static const crypto_uint32 m[32] = {0xED, 0xD3, 0xF5, 0x5C, 0x1A, 0x63, 0x12, 0x58, 0xD6, 0x9C, 0xF7, 0xA2, 0xDE, 0xF9, 0xDE, 0x14, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10};
+static const crypto_uint32 mu[33] = {0x1B, 0x13, 0x2C, 0x0A, 0xA3, 0xE5, 0x9C, 0xED, 0xA7, 0x29, 0x63, 0x08, 0x5D, 0x21, 0x06, 0x21, 0xEB, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0x0F};
 
-crypto_uint32 lt(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
+static inline crypto_uint32 lt(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
 {
 	unsigned int x = a;
 	x -= (unsigned int) b; /* 0..65535: no; 4294901761..4294967295: yes */
@@ -1384,7 +1128,7 @@ crypto_uint32 lt(crypto_uint32 a,crypto_uint32 b) /* 16-bit inputs */
 }
 
 /* Reduce coefficients of r before calling reduce_add_sub */
-void reduce_add_sub(sc25519 *r)
+static inline void reduce_add_sub(sc25519 *r)
 {
 	crypto_uint32 pb = 0;
 	crypto_uint32 b;
@@ -1405,7 +1149,7 @@ void reduce_add_sub(sc25519 *r)
 }
 
 /* Reduce coefficients of x before calling barrett_reduce */
-void barrett_reduce(sc25519 *r, const crypto_uint32 x[64])
+static inline void barrett_reduce(sc25519 *r, const crypto_uint32 x[64])
 {
 	/* See HAC, Alg. 14.42 */
 	int i,j;
@@ -1456,7 +1200,7 @@ void barrett_reduce(sc25519 *r, const crypto_uint32 x[64])
 	reduce_add_sub(r);
 }
 
-void sc25519_from32bytes(sc25519 *r, const unsigned char x[32])
+static inline void sc25519_from32bytes(sc25519 *r, const unsigned char x[32])
 {
 	int i;
 	crypto_uint32 t[64];
@@ -1465,7 +1209,7 @@ void sc25519_from32bytes(sc25519 *r, const unsigned char x[32])
 	barrett_reduce(r, t);
 }
 
-void sc25519_from64bytes(sc25519 *r, const unsigned char x[64])
+static inline void sc25519_from64bytes(sc25519 *r, const unsigned char x[64])
 {
 	int i;
 	crypto_uint32 t[64];
@@ -1473,13 +1217,13 @@ void sc25519_from64bytes(sc25519 *r, const unsigned char x[64])
 	barrett_reduce(r, t);
 }
 
-void sc25519_to32bytes(unsigned char r[32], const sc25519 *x)
+static inline void sc25519_to32bytes(unsigned char r[32], const sc25519 *x)
 {
 	int i;
 	for(i=0;i<32;i++) r[i] = x->v[i];
 }
 
-void sc25519_add(sc25519 *r, const sc25519 *x, const sc25519 *y)
+static inline void sc25519_add(sc25519 *r, const sc25519 *x, const sc25519 *y)
 {
 	int i, carry;
 	for(i=0;i<32;i++) r->v[i] = x->v[i] + y->v[i];
@@ -1492,7 +1236,7 @@ void sc25519_add(sc25519 *r, const sc25519 *x, const sc25519 *y)
 	reduce_add_sub(r);
 }
 
-void sc25519_mul(sc25519 *r, const sc25519 *x, const sc25519 *y)
+static inline void sc25519_mul(sc25519 *r, const sc25519 *x, const sc25519 *y)
 {
 	int i,j,carry;
 	crypto_uint32 t[64];
@@ -1512,7 +1256,7 @@ void sc25519_mul(sc25519 *r, const sc25519 *x, const sc25519 *y)
 	barrett_reduce(r, t);
 }
 
-void sc25519_window3(signed char r[85], const sc25519 *s)
+static inline void sc25519_window3(signed char r[85], const sc25519 *s)
 {
 	char carry;
 	int i;
@@ -1549,7 +1293,7 @@ void sc25519_window3(signed char r[85], const sc25519 *s)
 	r[84] += carry;
 }
 
-void sc25519_2interleave2(unsigned char r[127], const sc25519 *s1, const sc25519 *s2)
+static inline void sc25519_2interleave2(unsigned char r[127], const sc25519 *s1, const sc25519 *s2)
 {
 	int i;
 	for(i=0;i<31;i++)
@@ -2438,27 +2182,27 @@ static const ge25519_aff ge25519_base_multiples_affine[425] = {
  {{0x69, 0x3e, 0x47, 0x97, 0x2c, 0xaf, 0x52, 0x7c, 0x78, 0x83, 0xad, 0x1b, 0x39, 0x82, 0x2f, 0x02, 0x6f, 0x47, 0xdb, 0x2a, 0xb0, 0xe1, 0x91, 0x99, 0x55, 0xb8, 0x99, 0x3a, 0xa0, 0x44, 0x11, 0x51}}}
 };
 
-void p1p1_to_p2(ge25519_p2 *r, const ge25519_p1p1 *p)
+static inline void p1p1_to_p2(ge25519_p2 *r, const ge25519_p1p1 *p)
 {
 	fe25519_mul(&r->x, &p->x, &p->t);
 	fe25519_mul(&r->y, &p->y, &p->z);
 	fe25519_mul(&r->z, &p->z, &p->t);
 }
 
-void p1p1_to_p2_2(ge25519_p3 *r, const ge25519_p1p1 *p)
+static inline void p1p1_to_p2_2(ge25519_p3 *r, const ge25519_p1p1 *p)
 {
 	fe25519_mul(&r->x, &p->x, &p->t);
 	fe25519_mul(&r->y, &p->y, &p->z);
 	fe25519_mul(&r->z, &p->z, &p->t);
 }
 
-void p1p1_to_p3(ge25519_p3 *r, const ge25519_p1p1 *p)
+static inline void p1p1_to_p3(ge25519_p3 *r, const ge25519_p1p1 *p)
 {
 	p1p1_to_p2_2(r, p);
 	fe25519_mul(&r->t, &p->x, &p->y);
 }
 
-void ge25519_mixadd2(ge25519_p3 *r, const ge25519_aff *q)
+static inline void ge25519_mixadd2(ge25519_p3 *r, const ge25519_aff *q)
 {
 	fe25519 a,b,t1,t2,c,d,e,f,g,h,qt;
 	fe25519_mul(&qt, &q->x, &q->y);
@@ -2481,7 +2225,7 @@ void ge25519_mixadd2(ge25519_p3 *r, const ge25519_aff *q)
 	fe25519_mul(&r->t, &e, &h);
 }
 
-void add_p1p1(ge25519_p1p1 *r, const ge25519_p3 *p, const ge25519_p3 *q)
+static inline void add_p1p1(ge25519_p1p1 *r, const ge25519_p3 *p, const ge25519_p3 *q)
 {
 	fe25519 a, b, c, d, t;
 
@@ -2502,7 +2246,7 @@ void add_p1p1(ge25519_p1p1 *r, const ge25519_p3 *p, const ge25519_p3 *q)
 }
 
 /* See http://www.hyperelliptic.org/EFD/g1p/auto-twisted-extended-1.html#doubling-dbl-2008-hwcd */
-void dbl_p1p1(ge25519_p1p1 *r, const ge25519_p2 *p)
+static inline void dbl_p1p1(ge25519_p1p1 *r, const ge25519_p2 *p)
 {
 	fe25519 a,b,c,d;
 	fe25519_square(&a, &p->x);
@@ -2521,13 +2265,13 @@ void dbl_p1p1(ge25519_p1p1 *r, const ge25519_p2 *p)
 }
 
 /* Constant-time version of: if(b) r = p */
-void cmov_aff(ge25519_aff *r, const ge25519_aff *p, unsigned char b)
+static inline void cmov_aff(ge25519_aff *r, const ge25519_aff *p, unsigned char b)
 {
 	fe25519_cmov(&r->x, &p->x, b);
 	fe25519_cmov(&r->y, &p->y, b);
 }
 
-unsigned char equal(signed char b,signed char c)
+static inline unsigned char equal(signed char b,signed char c)
 {
 	unsigned char ub = b;
 	unsigned char uc = c;
@@ -2538,14 +2282,14 @@ unsigned char equal(signed char b,signed char c)
 	return (unsigned char)y;
 }
 
-unsigned char negative(signed char b)
+static inline unsigned char negative(signed char b)
 {
 	unsigned long long x = b; /* 18446744073709551361..18446744073709551615: yes; 0..255: no */
 	x >>= 63; /* 1: yes; 0: no */
 	return (unsigned char)x;
 }
 
-void choose_t(ge25519_aff *t, unsigned long long pos, signed char b)
+static inline void choose_t(ge25519_aff *t, unsigned long long pos, signed char b)
 {
 	/* constant time */
 	fe25519 v;
@@ -2558,7 +2302,7 @@ void choose_t(ge25519_aff *t, unsigned long long pos, signed char b)
 	fe25519_cmov(&t->x, &v, negative(b));
 }
 
-void setneutral(ge25519 *r)
+static inline void setneutral(ge25519 *r)
 {
 	fe25519_setzero(&r->x);
 	fe25519_setone(&r->y);
@@ -2567,7 +2311,7 @@ void setneutral(ge25519 *r)
 }
 
 /* return 0 on success, -1 otherwise */
-int ge25519_unpackneg_vartime(ge25519_p3 *r, const unsigned char p[32])
+static inline int ge25519_unpackneg_vartime(ge25519_p3 *r, const unsigned char p[32])
 {
 	unsigned char par;
 	fe25519 t, chk, num, den, den2, den4, den6;
@@ -2614,7 +2358,7 @@ int ge25519_unpackneg_vartime(ge25519_p3 *r, const unsigned char p[32])
 	return 0;
 }
 
-void ge25519_pack(unsigned char r[32], const ge25519_p3 *p)
+static inline void ge25519_pack(unsigned char r[32], const ge25519_p3 *p)
 {
 	fe25519 tx, ty, zi;
 	fe25519_invert(&zi, &p->z);
@@ -2625,7 +2369,7 @@ void ge25519_pack(unsigned char r[32], const ge25519_p3 *p)
 }
 
 /* computes [s1]p1 + [s2]p2 */
-void ge25519_double_scalarmult_vartime(ge25519_p3 *r, const ge25519_p3 *p1, const sc25519 *s1, const ge25519_p3 *p2, const sc25519 *s2)
+static inline void ge25519_double_scalarmult_vartime(ge25519_p3 *r, const ge25519_p3 *p1, const sc25519 *s1, const ge25519_p3 *p2, const sc25519 *s2)
 {
 	ge25519_p1p1 tp1p1;
 	ge25519_p3 pre[16];
@@ -2670,7 +2414,7 @@ void ge25519_double_scalarmult_vartime(ge25519_p3 *r, const ge25519_p3 *p1, cons
 	}
 }
 
-void ge25519_scalarmult_base(ge25519_p3 *r, const sc25519 *s)
+static inline void ge25519_scalarmult_base(ge25519_p3 *r, const sc25519 *s)
 {
 	signed char b[85];
 	int i;
@@ -2687,7 +2431,7 @@ void ge25519_scalarmult_base(ge25519_p3 *r, const sc25519 *s)
 	}
 }
 
-void get_hram(unsigned char *hram, const unsigned char *sm, const unsigned char *pk, unsigned char *playground, unsigned long long smlen)
+static inline void get_hram(unsigned char *hram, const unsigned char *sm, const unsigned char *pk, unsigned char *playground, unsigned long long smlen)
 {
 	unsigned long long i;
 
@@ -2695,7 +2439,6 @@ void get_hram(unsigned char *hram, const unsigned char *sm, const unsigned char
 	for (i = 32;i < 64;++i)    playground[i] = pk[i-32];
 	for (i = 64;i < smlen;++i) playground[i] = sm[i];
 
-	//crypto_hash_sha512(hram,playground,smlen);
 	ZeroTier::SHA512::hash(hram,playground,(unsigned int)smlen);
 }
 

node/Membership.cpp

@@ -48,9 +48,12 @@ Membership::Membership() :
 {
 }
 
-void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex)
+void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf)
 {
-	const Capability *sendCap = (localCapabilityIndex >= 0) ? &(nconf.capabilities[localCapabilityIndex]) : (const Capability *)0;
+	const Capability *sendCaps[ZT_MAX_NETWORK_CAPABILITIES];
+	unsigned int sendCapCount = 0;
+	for(unsigned int c=0;c<nconf.capabilityCount;++c)
+		sendCaps[sendCapCount++] = &(nconf.capabilities[c]);
 
 	const Tag *sendTags[ZT_MAX_NETWORK_TAGS];
 	unsigned int sendTagCount = 0;
@@ -62,10 +65,11 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const i
 	for(unsigned int c=0;c<nconf.certificateOfOwnershipCount;++c)
 		sendCoos[sendCooCount++] = &(nconf.certificatesOfOwnership[c]);
 
+	unsigned int capPtr = 0;
 	unsigned int tagPtr = 0;
 	unsigned int cooPtr = 0;
 	bool sendCom = (bool)(nconf.com);
-	while ((tagPtr < sendTagCount)||(cooPtr < sendCooCount)||(sendCom)||(sendCap)) {
+	while ((capPtr < sendCapCount)||(tagPtr < sendTagCount)||(cooPtr < sendCooCount)||(sendCom)) {
 		Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_CREDENTIALS);
 
 		if (sendCom) {
@@ -74,11 +78,14 @@ void Membership::pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const i
 		}
 		outp.append((uint8_t)0x00);
 
-		if (sendCap) {
-			outp.append((uint16_t)1);
-			sendCap->serialize(outp);
-			sendCap = (const Capability *)0;
-		} else outp.append((uint16_t)0);
+		const unsigned int capCountAt = outp.size();
+		outp.addSize(2);
+		unsigned int thisPacketCapCount = 0;
+		while ((capPtr < sendCapCount)&&((outp.size() + sizeof(Capability) + 16) < ZT_PROTO_MAX_PACKET_LENGTH)) {
+			sendCaps[capPtr++]->serialize(outp);
+			++thisPacketCapCount;
+		}
+		outp.setAt(capCountAt,(uint16_t)thisPacketCapCount);
 
 		const unsigned int tagCountAt = outp.size();
 		outp.addSize(2);

node/Membership.hpp

@@ -74,9 +74,8 @@ public:
 	 * @param now Current time
 	 * @param peerAddress Address of member peer (the one that this Membership describes)
 	 * @param nconf My network config
-	 * @param localCapabilityIndex Index of local capability to include (in nconf.capabilities[]) or -1 if none
 	 */
-	void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf,int localCapabilityIndex);
+	void pushCredentials(const RuntimeEnvironment *RR,void *tPtr,const int64_t now,const Address &peerAddress,const NetworkConfig &nconf);
 
 	/**
 	 * @return True if we haven't pushed credentials in a long time (to cause proactive credential push)
@@ -137,7 +136,7 @@ public:
 			if (_isCredentialTimestampValid(nconf,*v)&&(v->owns(r)))
 				return true;
 		}
-		return false;
+		return _isV6NDPEmulated(nconf,r);
 	}
 
 	/**
@@ -192,6 +191,15 @@ public:
 	static uint64_t credentialKey(const Credential::Type &t,const uint32_t i) { return (((uint64_t)t << 32) | (uint64_t)i); }
 
 private:
+	inline bool _isV6NDPEmulated(const NetworkConfig &nconf,const MAC &m) const { return false; }
+	inline bool _isV6NDPEmulated(const NetworkConfig &nconf,const InetAddress &ip) const
+	{
+		if ((ip.isV6())&&(nconf.ndpEmulation())&&((InetAddress::makeIpv66plane(nconf.networkId,nconf.issuedTo.toInt()).ipsEqual(ip))||(InetAddress::makeIpv6rfc4193(nconf.networkId,nconf.issuedTo.toInt()).ipsEqual(ip)))) {
+			return true;
+		}
+		return false;
+	}
+
 	template<typename C>
 	inline bool _isCredentialTimestampValid(const NetworkConfig &nconf,const C &remoteCredential) const
 	{

node/Multicaster.hpp

@@ -168,6 +168,7 @@ private:
 		MulticastGroup mg;
 
 		inline bool operator==(const Key &k) const { return ((nwid == k.nwid)&&(mg == k.mg)); }
+		inline bool operator!=(const Key &k) const { return ((nwid != k.nwid)||(mg != k.mg)); }
 		inline unsigned long hashCode() const { return (mg.hashCode() ^ (unsigned long)(nwid ^ (nwid >> 32))); }
 	};
 
@@ -176,6 +177,9 @@ private:
 		MulticastGroupMember() {}
 		MulticastGroupMember(const Address &a,uint64_t ts) : address(a),timestamp(ts) {}
 
+		inline bool operator<(const MulticastGroupMember &a) const { return (address < a.address); }
+		inline bool operator==(const MulticastGroupMember &a) const { return (address == a.address); }
+		inline bool operator!=(const MulticastGroupMember &a) const { return (address != a.address); }
 		inline bool operator<(const Address &a) const { return (address < a); }
 		inline bool operator==(const Address &a) const { return (address == a); }
 		inline bool operator!=(const Address &a) const { return (address != a); }

node/Network.cpp

@@ -1433,6 +1433,12 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 			memset(&(ec->routes[i]),0,sizeof(ZT_VirtualNetworkRoute));
 		}
 	}
+
+	ec->multicastSubscriptionCount = (unsigned int)_myMulticastGroups.size();
+	for(unsigned long i=0;i<(unsigned long)_myMulticastGroups.size();++i) {
+		ec->multicastSubscriptions[i].mac = _myMulticastGroups[i].mac().toInt();
+		ec->multicastSubscriptions[i].adi = _myMulticastGroups[i].adi();
+	}
 }
 
 void Network::_sendUpdatesToMembers(void *tPtr,const MulticastGroup *const newMulticastGroup)

node/Network.hpp

@@ -365,7 +365,7 @@ public:
 	inline void pushCredentialsNow(void *tPtr,const Address &to,const int64_t now)
 	{
 		Mutex::Lock _l(_lock);
-		_membership(to).pushCredentials(RR,tPtr,now,to,_config,-1);
+		_membership(to).pushCredentials(RR,tPtr,now,to,_config);
 	}
 
 	/**
@@ -380,7 +380,7 @@ public:
 		Mutex::Lock _l(_lock);
 		Membership &m = _membership(to);
 		if (m.shouldPushCredentials(now))
-			m.pushCredentials(RR,tPtr,now,to,_config,-1);
+			m.pushCredentials(RR,tPtr,now,to,_config);
 	}
 
 	/**

objects.mk

@@ -28,11 +28,13 @@ CORE_OBJS=\
 
 ONE_OBJS=\
 	controller/EmbeddedNetworkController.o \
+	controller/DBMirrorSet.o \
 	controller/DB.o \
 	controller/FileDB.o \
 	controller/LFDB.o \
 	controller/PostgreSQL.o \
 	controller/RabbitMQ.o \
+	osdep/EthernetTap.o \
 	osdep/ManagedRoute.o \
 	osdep/Http.o \
 	osdep/OSUtils.o \

osdep/BSDEthernetTap.hpp

@@ -38,10 +38,11 @@
 #include "../node/MulticastGroup.hpp"
 #include "../node/MAC.hpp"
 #include "Thread.hpp"
+#include "EthernetTap.hpp"
 
 namespace ZeroTier {
 
-class BSDEthernetTap
+class BSDEthernetTap : public EthernetTap
 {
 public:
 	BSDEthernetTap(
@@ -54,18 +55,18 @@ public:
 		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
 		void *arg);
 
-	~BSDEthernetTap();
+	virtual ~BSDEthernetTap();
 
-	void setEnabled(bool en);
-	bool enabled() const;
-	bool addIp(const InetAddress &ip);
-	bool removeIp(const InetAddress &ip);
-	std::vector<InetAddress> ips() const;
-	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
-	std::string deviceName() const;
-	void setFriendlyName(const char *friendlyName);
-	void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
-	void setMtu(unsigned int mtu);
+	virtual void setEnabled(bool en);
+	virtual bool enabled() const;
+	virtual bool addIp(const InetAddress &ip);
+	virtual bool removeIp(const InetAddress &ip);
+	virtual std::vector<InetAddress> ips() const;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
+	virtual std::string deviceName() const;
+	virtual void setFriendlyName(const char *friendlyName);
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
+	virtual void setMtu(unsigned int mtu);
 
 	void threadMain()
 		throw();

osdep/EthernetTap.cpp

@@ -0,0 +1,135 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#include "EthernetTap.hpp"
+#include "OSUtils.hpp"
+
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef ZT_SDK
+
+#include "../controller/EmbeddedNetworkController.hpp"
+#include "../node/Node.hpp"
+#include "../include/VirtualTap.hpp"
+
+#else
+
+#ifdef __APPLE__
+#include <sys/sysctl.h>
+#include "MacEthernetTap.hpp"
+#include "MacKextEthernetTap.hpp"
+#endif // __APPLE__
+
+#ifdef __LINUX__
+#include "LinuxEthernetTap.hpp"
+#endif // __LINUX__
+
+#ifdef __WINDOWS__
+#include "WindowsEthernetTap.hpp"
+#endif // __WINDOWS__
+
+#ifdef __FreeBSD__
+#include "BSDEthernetTap.hpp"
+#endif // __FreeBSD__
+
+#ifdef __NetBSD__
+#include "NetBSDEthernetTap.hpp"
+#endif // __NetBSD__
+
+#ifdef __OpenBSD__
+#include "BSDEthernetTap.hpp"
+#endif // __OpenBSD__
+
+#endif
+
+namespace ZeroTier {
+
+std::shared_ptr<EthernetTap> EthernetTap::newInstance(
+	const char *tapDeviceType, // OS-specific, NULL for default
+	const char *homePath,
+	const MAC &mac,
+	unsigned int mtu,
+	unsigned int metric,
+	uint64_t nwid,
+	const char *friendlyName,
+	void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
+	void *arg)
+{
+
+#ifdef ZT_SDK
+
+	return std::shared_ptr<EthernetTap>(new VirtualTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+
+#else // not ZT_SDK
+
+#ifdef __APPLE__
+	char osrelease[256];
+	size_t size = sizeof(osrelease);
+	if (sysctlbyname("kern.osrelease",osrelease,&size,nullptr,0) == 0) {
+		char *dotAt = strchr(osrelease,'.');
+		if (dotAt) {
+			*dotAt = (char)0;
+			// The "feth" virtual Ethernet device type appeared in Darwin 17.x.x. Older versions
+			// (Sierra and earlier) must use the a kernel extension.
+			if (strtol(osrelease,(char **)0,10) < 17) {
+				return std::shared_ptr<EthernetTap>(new MacKextEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+			} else {
+				return std::shared_ptr<EthernetTap>(new MacEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+			}
+		}
+	}
+#endif // __APPLE__
+
+#ifdef __LINUX__
+	return std::shared_ptr<EthernetTap>(new LinuxEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+#endif // __LINUX__
+
+#ifdef __WINDOWS__
+	return std::shared_ptr<EthernetTap>(new WindowsEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+#endif // __WINDOWS__
+
+#ifdef __FreeBSD__
+	return std::shared_ptr<EthernetTap>(new BSDEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+#endif // __FreeBSD__
+
+#ifdef __NetBSD__
+	return std::shared_ptr<EthernetTap>(new NetBSDEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+#endif // __NetBSD__
+
+#ifdef __OpenBSD__
+	return std::shared_ptr<EthernetTap>(new BSDEthernetTap(homePath,mac,mtu,metric,nwid,friendlyName,handler,arg));
+#endif // __OpenBSD__
+
+#endif // ZT_SDK?
+
+	return std::shared_ptr<EthernetTap>();
+}
+
+EthernetTap::EthernetTap() {}
+EthernetTap::~EthernetTap() {}
+
+} // namespace ZeroTier

osdep/EthernetTap.hpp

@@ -0,0 +1,72 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#ifndef ZT_ETHERNETTAP_HPP
+#define ZT_ETHERNETTAP_HPP
+
+#include "../node/Constants.hpp"
+#include "../node/MAC.hpp"
+#include "../node/InetAddress.hpp"
+#include "../node/MulticastGroup.hpp"
+
+#include <string>
+#include <memory>
+#include <vector>
+
+namespace ZeroTier {
+
+class EthernetTap
+{
+public:
+	static std::shared_ptr<EthernetTap> newInstance(
+		const char *tapDeviceType, // OS-specific, NULL for default
+		const char *homePath,
+		const MAC &mac,
+		unsigned int mtu,
+		unsigned int metric,
+		uint64_t nwid,
+		const char *friendlyName,
+		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
+		void *arg);
+
+	EthernetTap();
+	virtual ~EthernetTap();
+
+	virtual void setEnabled(bool en) = 0;
+	virtual bool enabled() const = 0;
+	virtual bool addIp(const InetAddress &ip) = 0;
+	virtual bool removeIp(const InetAddress &ip) = 0;
+	virtual std::vector<InetAddress> ips() const = 0;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len) = 0;
+	virtual std::string deviceName() const = 0;
+	virtual void setFriendlyName(const char *friendlyName) = 0;
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed) = 0;
+	virtual void setMtu(unsigned int mtu) = 0;
+};
+
+} // namespace ZeroTier
+
+#endif

osdep/LinuxEthernetTap.cpp

@@ -24,6 +24,17 @@
  * of your own application.
  */
 
+#include "../node/Constants.hpp"
+
+#ifdef __LINUX__
+
+#include "../node/Utils.hpp"
+#include "../node/Mutex.hpp"
+#include "../node/Dictionary.hpp"
+#include "OSUtils.hpp"
+#include "LinuxEthernetTap.hpp"
+#include "LinuxNetLink.hpp"
+
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -50,14 +61,6 @@
 #include <utility>
 #include <string>
 
-#include "../node/Constants.hpp"
-#include "../node/Utils.hpp"
-#include "../node/Mutex.hpp"
-#include "../node/Dictionary.hpp"
-#include "OSUtils.hpp"
-#include "LinuxEthernetTap.hpp"
-#include "LinuxNetLink.hpp"
-
 // ff:ff:ff:ff:ff:ff with no ADI
 static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
 
@@ -519,3 +522,5 @@ void LinuxEthernetTap::threadMain()
 }
 
 } // namespace ZeroTier
+
+#endif // __LINUX__

osdep/LinuxEthernetTap.hpp

@@ -33,16 +33,15 @@
 #include <string>
 #include <vector>
 #include <stdexcept>
+#include <atomic>
 
 #include "../node/MulticastGroup.hpp"
 #include "Thread.hpp"
+#include "EthernetTap.hpp"
 
 namespace ZeroTier {
 
-/**
- * Linux Ethernet tap using kernel tun/tap driver
- */
-class LinuxEthernetTap
+class LinuxEthernetTap : public EthernetTap
 {
 public:
 	LinuxEthernetTap(
@@ -55,21 +54,21 @@ public:
 		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
 		void *arg);
 
-	~LinuxEthernetTap();
+	virtual ~LinuxEthernetTap();
 
-	void setEnabled(bool en);
-	bool enabled() const;
-	bool addIp(const InetAddress &ip);
+	virtual void setEnabled(bool en);
+	virtual bool enabled() const;
+	virtual bool addIp(const InetAddress &ip);
 #ifdef __SYNOLOGY__
 	bool addIpSyn(std::vector<InetAddress> ips);
 #endif
-	bool removeIp(const InetAddress &ip);
-	std::vector<InetAddress> ips() const;
-	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
-	std::string deviceName() const;
-	void setFriendlyName(const char *friendlyName);
-	void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
-	void setMtu(unsigned int mtu);
+	virtual bool removeIp(const InetAddress &ip);
+	virtual std::vector<InetAddress> ips() const;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
+	virtual std::string deviceName() const;
+	virtual void setFriendlyName(const char *friendlyName);
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
+	virtual void setMtu(unsigned int mtu);
 
 	void threadMain()
 		throw();
@@ -85,7 +84,7 @@ private:
 	unsigned int _mtu;
 	int _fd;
 	int _shutdownSignalPipe[2];
-	volatile bool _enabled;
+	std::atomic_bool _enabled;
 };
 
 } // namespace ZeroTier

osdep/MacEthernetTap.cpp

@@ -24,6 +24,17 @@
  * of your own application.
  */
 
+#include "../node/Constants.hpp"
+
+#ifdef __APPLE__
+
+#include "../node/Utils.hpp"
+#include "../node/Mutex.hpp"
+#include "../node/Dictionary.hpp"
+#include "OSUtils.hpp"
+#include "MacEthernetTap.hpp"
+#include "MacEthernetTapAgent.h"
+
 #include <stdint.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -57,14 +68,6 @@
 #include <set>
 #include <algorithm>
 
-#include "../node/Constants.hpp"
-#include "../node/Utils.hpp"
-#include "../node/Mutex.hpp"
-#include "../node/Dictionary.hpp"
-#include "OSUtils.hpp"
-#include "MacEthernetTap.hpp"
-#include "MacEthernetTapAgent.h"
-
 static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
 
 namespace ZeroTier {
@@ -463,3 +466,5 @@ void MacEthernetTap::threadMain()
 }
 
 } // namespace ZeroTier
+
+#endif // __APPLE__

osdep/MacEthernetTap.hpp

@@ -27,24 +27,24 @@
 #ifndef ZT_OSXETHERNETTAP_HPP
 #define ZT_OSXETHERNETTAP_HPP
 
-#include <stdio.h>
-#include <stdlib.h>
-
-#include <stdexcept>
-#include <string>
-#include <vector>
-
 #include "../node/Constants.hpp"
 #include "../node/MAC.hpp"
 #include "../node/InetAddress.hpp"
 #include "../node/MulticastGroup.hpp"
 #include "../node/Mutex.hpp"
-
 #include "Thread.hpp"
+#include "EthernetTap.hpp"
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <stdexcept>
+#include <string>
+#include <vector>
 
 namespace ZeroTier {
 
-class MacEthernetTap
+class MacEthernetTap : public EthernetTap
 {
 public:
 	MacEthernetTap(
@@ -57,18 +57,18 @@ public:
 		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
 		void *arg);
 
-	~MacEthernetTap();
+	virtual ~MacEthernetTap();
 
-	void setEnabled(bool en);
-	bool enabled() const;
-	bool addIp(const InetAddress &ip);
-	bool removeIp(const InetAddress &ip);
-	std::vector<InetAddress> ips() const;
-	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
-	std::string deviceName() const;
-	void setFriendlyName(const char *friendlyName);
-	void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
-	void setMtu(unsigned int mtu);
+	virtual void setEnabled(bool en);
+	virtual bool enabled() const;
+	virtual bool addIp(const InetAddress &ip);
+	virtual bool removeIp(const InetAddress &ip);
+	virtual std::vector<InetAddress> ips() const;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
+	virtual std::string deviceName() const;
+	virtual void setFriendlyName(const char *friendlyName);
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
+	virtual void setMtu(unsigned int mtu);
 
 	void threadMain()
 		throw();

osdep/MacKextEthernetTap.cpp

@@ -0,0 +1,703 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <errno.h>
+
+#include <unistd.h>
+#include <signal.h>
+
+#include <fcntl.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/ioctl.h>
+#include <sys/wait.h>
+#include <sys/select.h>
+#include <sys/cdefs.h>
+#include <sys/uio.h>
+#include <sys/param.h>
+#include <sys/ioctl.h>
+#include <sys/socket.h>
+#include <netinet/in.h>
+#include <arpa/inet.h>
+#include <net/route.h>
+#include <net/if.h>
+#include <net/if_arp.h>
+#include <net/if_dl.h>
+#include <net/if_media.h>
+#include <sys/sysctl.h>
+#include <netinet6/in6_var.h>
+#include <netinet/in_var.h>
+#include <netinet/icmp6.h>
+
+// OSX compile fix... in6_var defines this in a struct which namespaces it for C++ ... why?!?
+struct prf_ra {
+	u_char onlink : 1;
+	u_char autonomous : 1;
+	u_char reserved : 6;
+} prf_ra;
+
+#include <netinet6/nd6.h>
+#include <ifaddrs.h>
+
+// These are KERNEL_PRIVATE... why?
+#ifndef SIOCAUTOCONF_START
+#define SIOCAUTOCONF_START _IOWR('i', 132, struct in6_ifreq)    /* accept rtadvd on this interface */
+#endif
+#ifndef SIOCAUTOCONF_STOP
+#define SIOCAUTOCONF_STOP _IOWR('i', 133, struct in6_ifreq)    /* stop accepting rtadv for this interface */
+#endif
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+// This source is from:
+// http://www.opensource.apple.com/source/Libinfo/Libinfo-406.17/gen.subproj/getifmaddrs.c?txt
+// It's here because OSX 10.6 does not have this convenience function.
+
+#define	SALIGN	(sizeof(uint32_t) - 1)
+#define	SA_RLEN(sa)	((sa)->sa_len ? (((sa)->sa_len + SALIGN) & ~SALIGN) : \
+(SALIGN + 1))
+#define	MAX_SYSCTL_TRY	5
+#define	RTA_MASKS	(RTA_GATEWAY | RTA_IFP | RTA_IFA)
+
+/* FreeBSD uses NET_RT_IFMALIST and RTM_NEWMADDR from <sys/socket.h> */
+/* We can use NET_RT_IFLIST2 and RTM_NEWMADDR2 on Darwin */
+//#define DARWIN_COMPAT
+
+//#ifdef DARWIN_COMPAT
+#define GIM_SYSCTL_MIB NET_RT_IFLIST2
+#define GIM_RTM_ADDR RTM_NEWMADDR2
+//#else
+//#define GIM_SYSCTL_MIB NET_RT_IFMALIST
+//#define GIM_RTM_ADDR RTM_NEWMADDR
+//#endif
+
+// Not in 10.6 includes so use our own
+struct _intl_ifmaddrs {
+	struct _intl_ifmaddrs *ifma_next;
+	struct sockaddr *ifma_name;
+	struct sockaddr *ifma_addr;
+	struct sockaddr *ifma_lladdr;
+};
+
+static inline int _intl_getifmaddrs(struct _intl_ifmaddrs **pif)
+{
+	int icnt = 1;
+	int dcnt = 0;
+	int ntry = 0;
+	size_t len;
+	size_t needed;
+	int mib[6];
+	int i;
+	char *buf;
+	char *data;
+	char *next;
+	char *p;
+	struct ifma_msghdr2 *ifmam;
+	struct _intl_ifmaddrs *ifa, *ift;
+	struct rt_msghdr *rtm;
+	struct sockaddr *sa;
+
+	mib[0] = CTL_NET;
+	mib[1] = PF_ROUTE;
+	mib[2] = 0;             /* protocol */
+	mib[3] = 0;             /* wildcard address family */
+	mib[4] = GIM_SYSCTL_MIB;
+	mib[5] = 0;             /* no flags */
+	do {
+		if (sysctl(mib, 6, NULL, &needed, NULL, 0) < 0)
+			return (-1);
+		if ((buf = (char *)malloc(needed)) == NULL)
+			return (-1);
+		if (sysctl(mib, 6, buf, &needed, NULL, 0) < 0) {
+			if (errno != ENOMEM || ++ntry >= MAX_SYSCTL_TRY) {
+				free(buf);
+				return (-1);
+			}
+			free(buf);
+			buf = NULL;
+		}
+	} while (buf == NULL);
+
+	for (next = buf; next < buf + needed; next += rtm->rtm_msglen) {
+		rtm = (struct rt_msghdr *)(void *)next;
+		if (rtm->rtm_version != RTM_VERSION)
+			continue;
+		switch (rtm->rtm_type) {
+			case GIM_RTM_ADDR:
+				ifmam = (struct ifma_msghdr2 *)(void *)rtm;
+				if ((ifmam->ifmam_addrs & RTA_IFA) == 0)
+					break;
+				icnt++;
+				p = (char *)(ifmam + 1);
+				for (i = 0; i < RTAX_MAX; i++) {
+					if ((RTA_MASKS & ifmam->ifmam_addrs &
+						 (1 << i)) == 0)
+						continue;
+					sa = (struct sockaddr *)(void *)p;
+					len = SA_RLEN(sa);
+					dcnt += len;
+					p += len;
+				}
+				break;
+		}
+	}
+
+	data = (char *)malloc(sizeof(struct _intl_ifmaddrs) * icnt + dcnt);
+	if (data == NULL) {
+		free(buf);
+		return (-1);
+	}
+
+	ifa = (struct _intl_ifmaddrs *)(void *)data;
+	data += sizeof(struct _intl_ifmaddrs) * icnt;
+
+	memset(ifa, 0, sizeof(struct _intl_ifmaddrs) * icnt);
+	ift = ifa;
+
+	for (next = buf; next < buf + needed; next += rtm->rtm_msglen) {
+		rtm = (struct rt_msghdr *)(void *)next;
+		if (rtm->rtm_version != RTM_VERSION)
+			continue;
+
+		switch (rtm->rtm_type) {
+			case GIM_RTM_ADDR:
+				ifmam = (struct ifma_msghdr2 *)(void *)rtm;
+				if ((ifmam->ifmam_addrs & RTA_IFA) == 0)
+					break;
+
+				p = (char *)(ifmam + 1);
+				for (i = 0; i < RTAX_MAX; i++) {
+					if ((RTA_MASKS & ifmam->ifmam_addrs &
+						 (1 << i)) == 0)
+						continue;
+					sa = (struct sockaddr *)(void *)p;
+					len = SA_RLEN(sa);
+					switch (i) {
+						case RTAX_GATEWAY:
+							ift->ifma_lladdr =
+							(struct sockaddr *)(void *)data;
+							memcpy(data, p, len);
+							data += len;
+							break;
+
+						case RTAX_IFP:
+							ift->ifma_name =
+							(struct sockaddr *)(void *)data;
+							memcpy(data, p, len);
+							data += len;
+							break;
+
+						case RTAX_IFA:
+							ift->ifma_addr =
+							(struct sockaddr *)(void *)data;
+							memcpy(data, p, len);
+							data += len;
+							break;
+
+						default:
+							data += len;
+							break;
+					}
+					p += len;
+				}
+				ift->ifma_next = ift + 1;
+				ift = ift->ifma_next;
+				break;
+		}
+	}
+
+	free(buf);
+
+	if (ift > ifa) {
+		ift--;
+		ift->ifma_next = NULL;
+		*pif = ifa;
+	} else {
+		*pif = NULL;
+		free(ifa);
+	}
+	return (0);
+}
+
+static inline void _intl_freeifmaddrs(struct _intl_ifmaddrs *ifmp)
+{
+	free(ifmp);
+}
+
+// --------------------------------------------------------------------------
+// --------------------------------------------------------------------------
+
+#include <string>
+#include <map>
+#include <set>
+#include <algorithm>
+
+#include "../node/Constants.hpp"
+#include "../node/Utils.hpp"
+#include "../node/Mutex.hpp"
+#include "../node/Dictionary.hpp"
+#include "OSUtils.hpp"
+#include "MacKextEthernetTap.hpp"
+
+// ff:ff:ff:ff:ff:ff with no ADI
+static const ZeroTier::MulticastGroup _blindWildcardMulticastGroup(ZeroTier::MAC(0xff),0);
+
+static inline bool _setIpv6Stuff(const char *ifname,bool performNUD,bool acceptRouterAdverts)
+{
+	struct in6_ndireq nd;
+	struct in6_ifreq ifr;
+
+	int s = socket(AF_INET6,SOCK_DGRAM,0);
+	if (s <= 0)
+		return false;
+
+	memset(&nd,0,sizeof(nd));
+	strncpy(nd.ifname,ifname,sizeof(nd.ifname));
+
+	if (ioctl(s,SIOCGIFINFO_IN6,&nd)) {
+		close(s);
+		return false;
+	}
+
+	unsigned long oldFlags = (unsigned long)nd.ndi.flags;
+
+	if (performNUD)
+		nd.ndi.flags |= ND6_IFF_PERFORMNUD;
+	else nd.ndi.flags &= ~ND6_IFF_PERFORMNUD;
+
+	if (oldFlags != (unsigned long)nd.ndi.flags) {
+		if (ioctl(s,SIOCSIFINFO_FLAGS,&nd)) {
+			close(s);
+			return false;
+		}
+	}
+
+	memset(&ifr,0,sizeof(ifr));
+	strncpy(ifr.ifr_name,ifname,sizeof(ifr.ifr_name));
+	if (ioctl(s,acceptRouterAdverts ? SIOCAUTOCONF_START : SIOCAUTOCONF_STOP,&ifr)) {
+		close(s);
+		return false;
+	}
+
+	close(s);
+	return true;
+}
+
+namespace ZeroTier {
+
+static long globalTapsRunning = 0;
+static Mutex globalTapCreateLock;
+
+MacKextEthernetTap::MacKextEthernetTap(
+	const char *homePath,
+	const MAC &mac,
+	unsigned int mtu,
+	unsigned int metric,
+	uint64_t nwid,
+	const char *friendlyName,
+	void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *data,unsigned int len),
+	void *arg) :
+	_handler(handler),
+	_arg(arg),
+	_nwid(nwid),
+	_homePath(homePath),
+	_mtu(mtu),
+	_metric(metric),
+	_fd(0),
+	_enabled(true)
+{
+	char devpath[64],ethaddr[64],mtustr[32],metstr[32],nwids[32];
+	struct stat stattmp;
+
+	OSUtils::ztsnprintf(nwids,sizeof(nwids),"%.16llx",nwid);
+
+	Mutex::Lock _gl(globalTapCreateLock);
+
+	if (::stat("/dev/zt0",&stattmp)) {
+		long kextpid = (long)vfork();
+		if (kextpid == 0) {
+			::chdir(homePath);
+			OSUtils::redirectUnixOutputs("/dev/null",(const char *)0);
+			::execl("/sbin/kextload","/sbin/kextload","-q","-repository",homePath,"tap.kext",(const char *)0);
+			::_exit(-1);
+		} else if (kextpid > 0) {
+			int exitcode = -1;
+			::waitpid(kextpid,&exitcode,0);
+		}
+		::usleep(500); // give tap device driver time to start up and try again
+		if (::stat("/dev/zt0",&stattmp))
+			throw std::runtime_error("/dev/zt# tap devices do not exist and cannot load tap.kext");
+	}
+
+	// Try to reopen the last device we had, if we had one and it's still unused.
+	std::map<std::string,std::string> globalDeviceMap;
+	FILE *devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"r");
+	if (devmapf) {
+		char buf[256];
+		while (fgets(buf,sizeof(buf),devmapf)) {
+			char *x = (char *)0;
+			char *y = (char *)0;
+			char *saveptr = (char *)0;
+			for(char *f=Utils::stok(buf,"\r\n=",&saveptr);(f);f=Utils::stok((char *)0,"\r\n=",&saveptr)) {
+				if (!x) x = f;
+				else if (!y) y = f;
+				else break;
+			}
+			if ((x)&&(y)&&(x[0])&&(y[0]))
+				globalDeviceMap[x] = y;
+		}
+		fclose(devmapf);
+	}
+	bool recalledDevice = false;
+	std::map<std::string,std::string>::const_iterator gdmEntry = globalDeviceMap.find(nwids);
+	if (gdmEntry != globalDeviceMap.end()) {
+		std::string devpath("/dev/"); devpath.append(gdmEntry->second);
+		if (stat(devpath.c_str(),&stattmp) == 0) {
+			_fd = ::open(devpath.c_str(),O_RDWR);
+			if (_fd > 0) {
+				_dev = gdmEntry->second;
+				recalledDevice = true;
+			}
+		}
+	}
+
+	// Open the first unused tap device if we didn't recall a previous one.
+	if (!recalledDevice) {
+		for(int i=0;i<64;++i) {
+			OSUtils::ztsnprintf(devpath,sizeof(devpath),"/dev/zt%d",i);
+			if (stat(devpath,&stattmp))
+				throw std::runtime_error("no more TAP devices available");
+			_fd = ::open(devpath,O_RDWR);
+			if (_fd > 0) {
+				char foo[16];
+				OSUtils::ztsnprintf(foo,sizeof(foo),"zt%d",i);
+				_dev = foo;
+				break;
+			}
+		}
+	}
+
+	if (_fd <= 0)
+		throw std::runtime_error("unable to open TAP device or no more devices available");
+
+	if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) {
+		::close(_fd);
+		throw std::runtime_error("unable to set flags on file descriptor for TAP device");
+	}
+
+	// Configure MAC address and MTU, bring interface up
+	OSUtils::ztsnprintf(ethaddr,sizeof(ethaddr),"%.2x:%.2x:%.2x:%.2x:%.2x:%.2x",(int)mac[0],(int)mac[1],(int)mac[2],(int)mac[3],(int)mac[4],(int)mac[5]);
+	OSUtils::ztsnprintf(mtustr,sizeof(mtustr),"%u",_mtu);
+	OSUtils::ztsnprintf(metstr,sizeof(metstr),"%u",_metric);
+	long cpid = (long)vfork();
+	if (cpid == 0) {
+		::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"lladdr",ethaddr,"mtu",mtustr,"metric",metstr,"up",(const char *)0);
+		::_exit(-1);
+	} else if (cpid > 0) {
+		int exitcode = -1;
+		::waitpid(cpid,&exitcode,0);
+		if (exitcode) {
+			::close(_fd);
+			throw std::runtime_error("ifconfig failure setting link-layer address and activating tap interface");
+		}
+	}
+
+	_setIpv6Stuff(_dev.c_str(),true,false);
+
+	// Set close-on-exec so that devices cannot persist if we fork/exec for update
+	fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC);
+
+	::pipe(_shutdownSignalPipe);
+
+	++globalTapsRunning;
+
+	globalDeviceMap[nwids] = _dev;
+	devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"w");
+	if (devmapf) {
+		gdmEntry = globalDeviceMap.begin();
+		while (gdmEntry != globalDeviceMap.end()) {
+			fprintf(devmapf,"%s=%s\n",gdmEntry->first.c_str(),gdmEntry->second.c_str());
+			++gdmEntry;
+		}
+		fclose(devmapf);
+	}
+
+	_thread = Thread::start(this);
+}
+
+MacKextEthernetTap::~MacKextEthernetTap()
+{
+	::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit
+	Thread::join(_thread);
+
+	::close(_fd);
+	::close(_shutdownSignalPipe[0]);
+	::close(_shutdownSignalPipe[1]);
+
+	{
+		Mutex::Lock _gl(globalTapCreateLock);
+		if (--globalTapsRunning <= 0) {
+			globalTapsRunning = 0; // sanity check -- should not be possible
+
+			char tmp[16384];
+			sprintf(tmp,"%s/%s",_homePath.c_str(),"tap.kext");
+			long kextpid = (long)vfork();
+			if (kextpid == 0) {
+				OSUtils::redirectUnixOutputs("/dev/null",(const char *)0);
+				::execl("/sbin/kextunload","/sbin/kextunload",tmp,(const char *)0);
+				::_exit(-1);
+			} else if (kextpid > 0) {
+				int exitcode = -1;
+				::waitpid(kextpid,&exitcode,0);
+			}
+		}
+	}
+}
+
+void MacKextEthernetTap::setEnabled(bool en)
+{
+	_enabled = en;
+	// TODO: interface status change
+}
+
+bool MacKextEthernetTap::enabled() const
+{
+	return _enabled;
+}
+
+bool MacKextEthernetTap::addIp(const InetAddress &ip)
+{
+	if (!ip)
+		return false;
+
+	long cpid = (long)vfork();
+	if (cpid == 0) {
+		char tmp[128];
+		::execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toString(tmp),"alias",(const char *)0);
+		::_exit(-1);
+	} else if (cpid > 0) {
+		int exitcode = -1;
+		::waitpid(cpid,&exitcode,0);
+		return (exitcode == 0);
+	} // else return false...
+
+	return false;
+}
+
+bool MacKextEthernetTap::removeIp(const InetAddress &ip)
+{
+	if (!ip)
+		return true;
+	std::vector<InetAddress> allIps(ips());
+	for(std::vector<InetAddress>::iterator i(allIps.begin());i!=allIps.end();++i) {
+		if (*i == ip) {
+			long cpid = (long)vfork();
+			if (cpid == 0) {
+				char tmp[128];
+				execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),(ip.ss_family == AF_INET6) ? "inet6" : "inet",ip.toIpString(tmp),"-alias",(const char *)0);
+				_exit(-1);
+			} else if (cpid > 0) {
+				int exitcode = -1;
+				waitpid(cpid,&exitcode,0);
+				return (exitcode == 0);
+			}
+		}
+	}
+	return false;
+}
+
+std::vector<InetAddress> MacKextEthernetTap::ips() const
+{
+	struct ifaddrs *ifa = (struct ifaddrs *)0;
+	if (getifaddrs(&ifa))
+		return std::vector<InetAddress>();
+
+	std::vector<InetAddress> r;
+
+	struct ifaddrs *p = ifa;
+	while (p) {
+		if ((!strcmp(p->ifa_name,_dev.c_str()))&&(p->ifa_addr)&&(p->ifa_netmask)&&(p->ifa_addr->sa_family == p->ifa_netmask->sa_family)) {
+			switch(p->ifa_addr->sa_family) {
+				case AF_INET: {
+					struct sockaddr_in *sin = (struct sockaddr_in *)p->ifa_addr;
+					struct sockaddr_in *nm = (struct sockaddr_in *)p->ifa_netmask;
+					r.push_back(InetAddress(&(sin->sin_addr.s_addr),4,Utils::countBits((uint32_t)nm->sin_addr.s_addr)));
+				}	break;
+				case AF_INET6: {
+					struct sockaddr_in6 *sin = (struct sockaddr_in6 *)p->ifa_addr;
+					struct sockaddr_in6 *nm = (struct sockaddr_in6 *)p->ifa_netmask;
+					uint32_t b[4];
+					memcpy(b,nm->sin6_addr.s6_addr,sizeof(b));
+					r.push_back(InetAddress(sin->sin6_addr.s6_addr,16,Utils::countBits(b[0]) + Utils::countBits(b[1]) + Utils::countBits(b[2]) + Utils::countBits(b[3])));
+				}	break;
+			}
+		}
+		p = p->ifa_next;
+	}
+
+	if (ifa)
+		freeifaddrs(ifa);
+
+	std::sort(r.begin(),r.end());
+	r.erase(std::unique(r.begin(),r.end()),r.end());
+
+	return r;
+}
+
+void MacKextEthernetTap::put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len)
+{
+	char putBuf[ZT_MAX_MTU + 64];
+	if ((_fd > 0)&&(len <= _mtu)&&(_enabled)) {
+		to.copyTo(putBuf,6);
+		from.copyTo(putBuf + 6,6);
+		*((uint16_t *)(putBuf + 12)) = htons((uint16_t)etherType);
+		memcpy(putBuf + 14,data,len);
+		len += 14;
+		::write(_fd,putBuf,len);
+	}
+}
+
+std::string MacKextEthernetTap::deviceName() const
+{
+	return _dev;
+}
+
+void MacKextEthernetTap::setFriendlyName(const char *friendlyName)
+{
+}
+
+void MacKextEthernetTap::scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed)
+{
+	std::vector<MulticastGroup> newGroups;
+
+	struct _intl_ifmaddrs *ifmap = (struct _intl_ifmaddrs *)0;
+	if (!_intl_getifmaddrs(&ifmap)) {
+		struct _intl_ifmaddrs *p = ifmap;
+		while (p) {
+			if (p->ifma_addr->sa_family == AF_LINK) {
+				struct sockaddr_dl *in = (struct sockaddr_dl *)p->ifma_name;
+				struct sockaddr_dl *la = (struct sockaddr_dl *)p->ifma_addr;
+				if ((la->sdl_alen == 6)&&(in->sdl_nlen <= _dev.length())&&(!memcmp(_dev.data(),in->sdl_data,in->sdl_nlen)))
+					newGroups.push_back(MulticastGroup(MAC(la->sdl_data + la->sdl_nlen,6),0));
+			}
+			p = p->ifma_next;
+		}
+		_intl_freeifmaddrs(ifmap);
+	}
+
+	std::vector<InetAddress> allIps(ips());
+	for(std::vector<InetAddress>::iterator ip(allIps.begin());ip!=allIps.end();++ip)
+		newGroups.push_back(MulticastGroup::deriveMulticastGroupForAddressResolution(*ip));
+
+	std::sort(newGroups.begin(),newGroups.end());
+	std::unique(newGroups.begin(),newGroups.end());
+
+	for(std::vector<MulticastGroup>::iterator m(newGroups.begin());m!=newGroups.end();++m) {
+		if (!std::binary_search(_multicastGroups.begin(),_multicastGroups.end(),*m))
+			added.push_back(*m);
+	}
+	for(std::vector<MulticastGroup>::iterator m(_multicastGroups.begin());m!=_multicastGroups.end();++m) {
+		if (!std::binary_search(newGroups.begin(),newGroups.end(),*m))
+			removed.push_back(*m);
+	}
+
+	_multicastGroups.swap(newGroups);
+}
+
+void MacKextEthernetTap::setMtu(unsigned int mtu)
+{
+	if (mtu != _mtu) {
+		_mtu = mtu;
+		long cpid = (long)vfork();
+		if (cpid == 0) {
+			char tmp[64];
+			OSUtils::ztsnprintf(tmp,sizeof(tmp),"%u",mtu);
+			execl("/sbin/ifconfig","/sbin/ifconfig",_dev.c_str(),"mtu",tmp,(const char *)0);
+			_exit(-1);
+		} else if (cpid > 0) {
+			int exitcode = -1;
+			waitpid(cpid,&exitcode,0);
+		}
+	}
+}
+
+void MacKextEthernetTap::threadMain()
+	throw()
+{
+	fd_set readfds,nullfds;
+	MAC to,from;
+	int n,nfds,r;
+	char getBuf[ZT_MAX_MTU + 64];
+
+	Thread::sleep(500);
+
+	FD_ZERO(&readfds);
+	FD_ZERO(&nullfds);
+	nfds = (int)std::max(_shutdownSignalPipe[0],_fd) + 1;
+
+	r = 0;
+	for(;;) {
+		FD_SET(_shutdownSignalPipe[0],&readfds);
+		FD_SET(_fd,&readfds);
+		select(nfds,&readfds,&nullfds,&nullfds,(struct timeval *)0);
+
+		if (FD_ISSET(_shutdownSignalPipe[0],&readfds)) // writes to shutdown pipe terminate thread
+			break;
+
+		if (FD_ISSET(_fd,&readfds)) {
+			n = (int)::read(_fd,getBuf + r,sizeof(getBuf) - r);
+			if (n < 0) {
+				if ((errno != EINTR)&&(errno != ETIMEDOUT))
+					break;
+			} else {
+				// Some tap drivers like to send the ethernet frame and the
+				// payload in two chunks, so handle that by accumulating
+				// data until we have at least a frame.
+				r += n;
+				if (r > 14) {
+					if (r > ((int)_mtu + 14)) // sanity check for weird TAP behavior on some platforms
+						r = _mtu + 14;
+
+					if (_enabled) {
+						to.setTo(getBuf,6);
+						from.setTo(getBuf + 6,6);
+						unsigned int etherType = ntohs(((const uint16_t *)getBuf)[6]);
+						// TODO: VLAN support
+						_handler(_arg,(void *)0,_nwid,from,to,etherType,0,(const void *)(getBuf + 14),r - 14);
+					}
+
+					r = 0;
+				}
+			}
+		}
+	}
+}
+
+} // namespace ZeroTier

osdep/MacKextEthernetTap.hpp

@@ -0,0 +1,93 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2018  ZeroTier, Inc.  https://www.zerotier.com/
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * You can be released from the requirements of the license by purchasing
+ * a commercial license. Buying such a license is mandatory as soon as you
+ * develop commercial closed-source software that incorporates or links
+ * directly against ZeroTier software without disclosing the source code
+ * of your own application.
+ */
+
+#ifndef ZT_MacKextEthernetTap_HPP
+#define ZT_MacKextEthernetTap_HPP
+
+#include <stdio.h>
+#include <stdlib.h>
+
+#include <stdexcept>
+#include <string>
+#include <vector>
+
+#include "../node/Constants.hpp"
+#include "../node/MAC.hpp"
+#include "../node/InetAddress.hpp"
+#include "../node/MulticastGroup.hpp"
+
+#include "Thread.hpp"
+#include "EthernetTap.hpp"
+
+namespace ZeroTier {
+
+class MacKextEthernetTap : public EthernetTap
+{
+public:
+	MacKextEthernetTap(
+		const char *homePath,
+		const MAC &mac,
+		unsigned int mtu,
+		unsigned int metric,
+		uint64_t nwid,
+		const char *friendlyName,
+		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
+		void *arg);
+
+	virtual ~MacKextEthernetTap();
+
+	virtual void setEnabled(bool en);
+	virtual bool enabled() const;
+	virtual bool addIp(const InetAddress &ip);
+	virtual bool removeIp(const InetAddress &ip);
+	virtual std::vector<InetAddress> ips() const;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
+	virtual std::string deviceName() const;
+	virtual void setFriendlyName(const char *friendlyName);
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
+	virtual void setMtu(unsigned int mtu);
+
+	void threadMain()
+		throw();
+
+private:
+	void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int);
+	void *_arg;
+	uint64_t _nwid;
+	Thread _thread;
+	std::string _homePath;
+	std::string _dev;
+	std::vector<MulticastGroup> _multicastGroups;
+	unsigned int _mtu;
+	unsigned int _metric;
+	int _fd;
+	int _shutdownSignalPipe[2];
+	volatile bool _enabled;
+};
+
+} // namespace ZeroTier
+
+#endif

osdep/NetBSDEthernetTap.hpp

@@ -38,10 +38,11 @@
 #include "../node/MulticastGroup.hpp"
 #include "../node/MAC.hpp"
 #include "Thread.hpp"
+#include "EthernetTap.hpp"
 
 namespace ZeroTier {
 
-class NetBSDEthernetTap
+class NetBSDEthernetTap : public EthernetTap
 {
 public:
 	NetBSDEthernetTap(
@@ -54,17 +55,17 @@ public:
 		void (*handler)(void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
 		void *arg);
 
-	~NetBSDEthernetTap();
+	virtual ~NetBSDEthernetTap();
 
-	void setEnabled(bool en);
-	bool enabled() const;
-	bool addIp(const InetAddress &ip);
-	bool removeIp(const InetAddress &ip);
-	std::vector<InetAddress> ips() const;
-	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
-	std::string deviceName() const;
-	void setFriendlyName(const char *friendlyName);
-	void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
+	virtual void setEnabled(bool en);
+	virtual bool enabled() const;
+	virtual bool addIp(const InetAddress &ip);
+	virtual bool removeIp(const InetAddress &ip);
+	virtual std::vector<InetAddress> ips() const;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
+	virtual std::string deviceName() const;
+	virtual void setFriendlyName(const char *friendlyName);
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
 
 	void threadMain()
 		throw();

osdep/TestEthernetTap.hpp

@@ -1,161 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2019  ZeroTier, Inc.  https://www.zerotier.com/
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program. If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * You can be released from the requirements of the license by purchasing
- * a commercial license. Buying such a license is mandatory as soon as you
- * develop commercial closed-source software that incorporates or links
- * directly against ZeroTier software without disclosing the source code
- * of your own application.
- */
-
-#ifndef ZT_TESTETHERNETTAP_HPP
-#define ZT_TESTETHERNETTAP_HPP
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <stdint.h>
-#include <sys/types.h>
-#include <sys/socket.h>
-
-#include <string>
-#include <vector>
-#include <stdexcept>
-#include <set>
-
-#include "../node/Constants.hpp"
-#include "../node/InetAddress.hpp"
-#include "../node/MulticastGroup.hpp"
-#include "../node/Mutex.hpp"
-#include "../node/Utils.hpp"
-#include "../osdep/OSUtils.hpp"
-
-namespace ZeroTier {
-
-/**
- * Dummy test Ethernet tap that does not actually open a device on the system
- */
-class TestEthernetTap
-{
-public:
-	TestEthernetTap(
-		const char *homePath,
-		const MAC &mac,
-		unsigned int mtu,
-		unsigned int metric,
-		uint64_t nwid,
-		const char *friendlyName,
-		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
-		void *arg) :
-		_nwid(nwid),
-		_dev("zt_test_"),
-		_enabled(true)
-	{
-		char tmp[32];
-		OSUtils::ztsnprintf(tmp,sizeof(tmp),"%.16llx",(unsigned long long)_nwid);
-		_dev.append(tmp);
-#ifdef ZT_TEST_TAP_REPORT_TO
-		_reportTo.fromString(ZT_TEST_TAP_REPORT_TO);
-		if (_reportTo.ss_family == AF_INET)
-			_reportsock = socket(AF_INET,SOCK_DGRAM,0);
-		else if (_reportTo.ss_family == AF_INET6)
-			_reportsock = socket(AF_INET6,SOCK_DGRAM,0);
-		else _reportsock = -1;
-#endif
-	}
-
-	~TestEthernetTap()
-	{
-#ifdef ZT_TEST_TAP_REPORT_TO
-		if (_reportsock >= 0)
-			close(_reportsock);
-#endif
-	}
-
-	inline void setEnabled(bool en) { _enabled = en; }
-	inline bool enabled() const { return _enabled; }
-
-	inline bool addIp(const InetAddress &ip)
-	{
-		Mutex::Lock _l(_lock);
-		_ips.insert(ip);
-		return true;
-	}
-
-	inline bool removeIp(const InetAddress &ip)
-	{
-		Mutex::Lock _l(_lock);
-		_ips.erase(ip);
-		return true;
-	}
-
-	inline std::vector<InetAddress> ips() const
-	{
-		Mutex::Lock _l(_lock);
-		return std::vector<InetAddress>(_ips.begin(),_ips.end());
-	}
-
-	inline void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len)
-	{
-#ifdef ZT_TEST_TAP_REPORT_TO
-		char tmp[10000];
-		if ((_reportsock >= 0)&&(len < (sizeof(tmp) - 22))) {
-			const uint64_t nwid2 = Utils::hton(_nwid);
-			memcpy(tmp,&nwid2,8);
-			from.copyTo(tmp + 8,6);
-			to.copyTo(tmp + 14,6);
-			const uint16_t etherType2 = Utils::hton((uint16_t)etherType);
-			memcpy(tmp + 20,&etherType2,2);
-			memcpy(tmp + 22,data,len);
-			sendto(_reportsock,tmp,len + 22,0,reinterpret_cast<const struct sockaddr *>(&_reportTo),(_reportTo.ss_family == AF_INET) ? sizeof(struct sockaddr_in) : sizeof(struct sockaddr_in6));
-		}
-#endif
-	}
-
-	inline std::string deviceName() const
-	{
-		return _dev;
-	}
-
-	inline void setFriendlyName(const char *friendlyName)
-	{
-	}
-
-	inline void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed)
-	{
-	}
-
-	inline void setMtu(unsigned int mtu)
-	{
-	}
-
-private:
-	uint64_t _nwid;
-	std::string _dev;
-	std::set<InetAddress> _ips;
-	InetAddress _reportTo;
-#ifdef ZT_TEST_TAP_REPORT_TO
-	int _reportsock;
-#endif
-	bool _enabled;
-	Mutex _lock;
-};
-
-} // namespace ZeroTier
-
-#endif

osdep/WindowsEthernetTap.hpp

@@ -41,10 +41,11 @@
 #include "../node/MulticastGroup.hpp"
 #include "../node/InetAddress.hpp"
 #include "../osdep/Thread.hpp"
+#include "EthernetTap.hpp"
 
 namespace ZeroTier {
 
-class WindowsEthernetTap
+class WindowsEthernetTap : public EthernetTap
 {
 public:
 	/**
@@ -97,18 +98,18 @@ public:
 		void (*handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int),
 		void *arg);
 
-	~WindowsEthernetTap();
+	virtual ~WindowsEthernetTap();
 
-	void setEnabled(bool en);
-	bool enabled() const;
-	bool addIp(const InetAddress &ip);
-	bool removeIp(const InetAddress &ip);
-	std::vector<InetAddress> ips() const;
-	void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
-	std::string deviceName() const;
-	void setFriendlyName(const char *friendlyName);
-	void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
-	void setMtu(unsigned int mtu);
+	virtual void setEnabled(bool en);
+	virtual bool enabled() const;
+	virtual bool addIp(const InetAddress &ip);
+	virtual bool removeIp(const InetAddress &ip);
+	virtual std::vector<InetAddress> ips() const;
+	virtual void put(const MAC &from,const MAC &to,unsigned int etherType,const void *data,unsigned int len);
+	virtual std::string deviceName() const;
+	virtual void setFriendlyName(const char *friendlyName);
+	virtual void scanMulticastGroups(std::vector<MulticastGroup> &added,std::vector<MulticastGroup> &removed);
+	virtual void setMtu(unsigned int mtu);
 
 	inline const NET_LUID &luid() const { return _deviceLuid; }
 	inline const GUID &guid() const { return _deviceGuid; }
@@ -118,7 +119,7 @@ public:
 	void threadMain()
 		throw();
 
-    bool isInitialized() const { return _initialized; };
+	bool isInitialized() const { return _initialized; };
 
 private:
 	NET_IFINDEX _getDeviceIndex(); // throws on failure

service/OneService.cpp

@@ -100,52 +100,10 @@ using json = nlohmann::json;
 
 #include "../controller/EmbeddedNetworkController.hpp"
 #include "../controller/RabbitMQ.hpp"
-
-#ifdef ZT_USE_TEST_TAP
-
-#include "../osdep/TestEthernetTap.hpp"
-namespace ZeroTier { typedef TestEthernetTap EthernetTap; }
-
-#else
-
-#ifdef ZT_SDK
-
-#include "../controller/EmbeddedNetworkController.hpp"
-#include "../node/Node.hpp"
-// Use the virtual netcon endpoint instead of a tun/tap port driver
-#include "../include/VirtualTap.hpp"
-namespace ZeroTier { typedef VirtualTap EthernetTap; }
-
-#else
-
-#ifdef __APPLE__
-#include "../osdep/MacEthernetTap.hpp"
-namespace ZeroTier { typedef MacEthernetTap EthernetTap; }
-#endif // __APPLE__
-#ifdef __LINUX__
-#include "../osdep/LinuxEthernetTap.hpp"
-namespace ZeroTier { typedef LinuxEthernetTap EthernetTap; }
-#endif // __LINUX__
+#include "../osdep/EthernetTap.hpp"
 #ifdef __WINDOWS__
 #include "../osdep/WindowsEthernetTap.hpp"
-namespace ZeroTier { typedef WindowsEthernetTap EthernetTap; }
-#endif // __WINDOWS__
-#ifdef __FreeBSD__
-#include "../osdep/BSDEthernetTap.hpp"
-namespace ZeroTier { typedef BSDEthernetTap EthernetTap; }
-#endif // __FreeBSD__
-#ifdef __NetBSD__
-#include "../osdep/NetBSDEthernetTap.hpp"
-namespace ZeroTier { typedef NetBSDEthernetTap EthernetTap; }
-#endif // __NetBSD__
-#ifdef __OpenBSD__
-#include "../osdep/BSDEthernetTap.hpp"
-namespace ZeroTier { typedef BSDEthernetTap EthernetTap; }
-#endif // __OpenBSD__
-
-#endif // ZT_SDK
-
-#endif // ZT_USE_TEST_TAP
+#endif
 
 #ifndef ZT_SOFTWARE_UPDATE_DEFAULT
 #define ZT_SOFTWARE_UPDATE_DEFAULT "disable"
@@ -273,6 +231,15 @@ static void _networkToJson(nlohmann::json &nj,const ZT_VirtualNetworkConfig *nc,
 		ra.push_back(rj);
 	}
 	nj["routes"] = ra;
+
+	nlohmann::json mca = nlohmann::json::array();
+	for(unsigned int i=0;i<nc->multicastSubscriptionCount;++i) {
+		nlohmann::json m;
+		m["mac"] = MAC(nc->multicastSubscriptions[i].mac).toString(tmp);
+		m["adi"] = nc->multicastSubscriptions[i].adi;
+		mca.push_back(m);
+	}
+	nj["multicastSubscriptions"] = mca;
 }
 
 static void _peerToJson(nlohmann::json &pj,const ZT_Peer *peer)
@@ -534,7 +501,7 @@ public:
 			settings.allowDefault = false;
 		}
 
-		EthernetTap *tap;
+		std::shared_ptr<EthernetTap> tap;
 		ZT_VirtualNetworkConfig config; // memcpy() of raw config from core
 		std::vector<InetAddress> managedIps;
 		std::list< SharedPtr<ManagedRoute> > managedRoutes;
@@ -767,7 +734,7 @@ public:
 			OSUtils::rmDashRf((_homePath + ZT_PATH_SEPARATOR_S "iddb.d").c_str());
 
 			// Network controller is now enabled by default for desktop and server
-			_controller = new EmbeddedNetworkController(_node,_controllerDbPath.c_str(),_ports[0], _mqc);
+			_controller = new EmbeddedNetworkController(_node,_homePath.c_str(),_controllerDbPath.c_str(),_ports[0], _mqc);
 			_node->setNetconfMaster((void *)_controller);
 
 			// Join existing networks in networks.d
@@ -946,8 +913,6 @@ public:
 
 		{
 			Mutex::Lock _l(_nets_m);
-			for(std::map<uint64_t,NetworkState>::iterator n(_nets.begin());n!=_nets.end();++n)
-				delete n->second.tap;
 			_nets.clear();
 		}
 
@@ -994,15 +959,17 @@ public:
 		Mutex::Lock _l2(_localConfig_m);
 		std::string lcbuf;
 		if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(),lcbuf)) {
-			try {
-				_localConfig = OSUtils::jsonParse(lcbuf);
-				if (!_localConfig.is_object()) {
-					fprintf(stderr,"ERROR: unable to parse local.conf (root element is not a JSON object)" ZT_EOL_S);
+			if (lcbuf.length() > 0) {
+				try {
+					_localConfig = OSUtils::jsonParse(lcbuf);
+					if (!_localConfig.is_object()) {
+						fprintf(stderr,"ERROR: unable to parse local.conf (root element is not a JSON object)" ZT_EOL_S);
+						exit(1);
+					}
+				} catch ( ... ) {
+					fprintf(stderr,"ERROR: unable to parse local.conf (invalid JSON)" ZT_EOL_S);
 					exit(1);
 				}
-			} catch ( ... ) {
-				fprintf(stderr,"ERROR: unable to parse local.conf (invalid JSON)" ZT_EOL_S);
-				exit(1);
 			}
 		}
 
@@ -1777,7 +1744,7 @@ public:
 		if (syncRoutes) {
 			char tapdev[64];
 #if defined(__WINDOWS__) && !defined(ZT_SDK)
-			OSUtils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)n.tap->luid().Value);
+			OSUtils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)((WindowsEthernetTap *)(n.tap.get()))->luid().Value);
 #else
 			Utils::scopy(tapdev,sizeof(tapdev),n.tap->deviceName().c_str());
 #endif
@@ -2131,7 +2098,8 @@ public:
 						char friendlyName[128];
 						OSUtils::ztsnprintf(friendlyName,sizeof(friendlyName),"ZeroTier One [%.16llx]",nwid);
 
-						n.tap = new EthernetTap(
+						n.tap = EthernetTap::newInstance(
+							nullptr,
 							_homePath.c_str(),
 							MAC(nwc->mac),
 							nwc->mtu,
@@ -2202,7 +2170,7 @@ public:
 					// without WindowsEthernetTap::isInitialized() returning true, the won't actually
 					// be online yet and setting managed routes on it will fail.
 					const int MAX_SLEEP_COUNT = 500;
-					for (int i = 0; !n.tap->isInitialized() && i < MAX_SLEEP_COUNT; i++) {
+					for (int i = 0; !((WindowsEthernetTap *)(n.tap.get()))->isInitialized() && i < MAX_SLEEP_COUNT; i++) {
 						Sleep(10);
 					}
 #endif
@@ -2218,10 +2186,10 @@ public:
 			case ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY:
 				if (n.tap) { // sanity check
 #if defined(__WINDOWS__) && !defined(ZT_SDK)
-					std::string winInstanceId(n.tap->instanceId());
+					std::string winInstanceId(((WindowsEthernetTap *)(n.tap.get()))->instanceId());
 #endif
 					*nuptr = (void *)0;
-					delete n.tap;
+					n.tap.reset();
 					_nets.erase(nwid);
 #if defined(__WINDOWS__) && !defined(ZT_SDK)
 					if ((op == ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY)&&(winInstanceId.length() > 0))

version.h

@@ -40,7 +40,7 @@
 /**
  * Revision
  */
-#define ZEROTIER_ONE_VERSION_REVISION 0
+#define ZEROTIER_ONE_VERSION_REVISION 2
 
 /**
  * Build version
@@ -49,7 +49,7 @@
  * to force a minor update without an actual version number change. It's
  * not part of the actual release version number.
  */
-#define ZEROTIER_ONE_VERSION_BUILD 1
+#define ZEROTIER_ONE_VERSION_BUILD 0
 
 #ifndef ZT_BUILD_ARCHITECTURE
 #define ZT_BUILD_ARCHITECTURE 0

windows-clean.bat

@@ -0,0 +1,9 @@
+DEL "ZeroTier One.msi"
+DEL zt1_update*.exe
+RMDIR /Q /S windows\Build
+RMDIR /Q /S windows\copyutil\bin
+RMDIR /Q /S windows\copyutil\obj
+RMDIR /Q /S windows\WinUI\bin
+RMDIR /Q /S windows\WinUI\obj
+RMDIR /Q /S windows\ZeroTierOne\Release
+RMDIR /Q /S windows\ZeroTierOne\x64

windows/WinUI/AboutView.xaml

@@ -19,7 +19,7 @@
                     <Run Text="ZeroTier One"/>
                 </Paragraph>
                 <Paragraph TextAlignment="Center">
-                    <Run FontSize="14" Text="Version 1.4.0"/>
+                    <Run FontSize="14" Text="Version 1.4.2"/>
                     <LineBreak/>
                     <Run FontSize="14" Text="(c) 2011-2019 ZeroTier, Inc."/>
                     <LineBreak/>

windows/ZeroTierOne/ZeroTierOne.vcxproj

@@ -28,6 +28,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClCompile Include="..\..\controller\DB.cpp" />
+    <ClCompile Include="..\..\controller\DBMirrorSet.cpp" />
     <ClCompile Include="..\..\controller\EmbeddedNetworkController.cpp" />
     <ClCompile Include="..\..\controller\FileDB.cpp" />
     <ClCompile Include="..\..\controller\LFDB.cpp" />
@@ -94,6 +95,7 @@
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Debug|x64'">false</ExcludedFromBuild>
       <ExcludedFromBuild Condition="'$(Configuration)|$(Platform)'=='Profile|x64'">false</ExcludedFromBuild>
     </ClCompile>
+    <ClCompile Include="..\..\osdep\EthernetTap.cpp" />
     <ClCompile Include="..\..\osdep\Http.cpp" />
     <ClCompile Include="..\..\osdep\ManagedRoute.cpp" />
     <ClCompile Include="..\..\osdep\OSUtils.cpp" />
@@ -115,6 +117,7 @@
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="..\..\controller\DB.hpp" />
+    <ClInclude Include="..\..\controller\DBMirrorSet.hpp" />
     <ClInclude Include="..\..\controller\EmbeddedNetworkController.hpp" />
     <ClInclude Include="..\..\controller\FileDB.hpp" />
     <ClInclude Include="..\..\controller\LFDB.hpp" />
@@ -182,6 +185,7 @@
     <ClInclude Include="..\..\node\Utils.hpp" />
     <ClInclude Include="..\..\node\World.hpp" />
     <ClInclude Include="..\..\osdep\Binder.hpp" />
+    <ClInclude Include="..\..\osdep\EthernetTap.hpp" />
     <ClInclude Include="..\..\osdep\Http.hpp" />
     <ClInclude Include="..\..\osdep\ManagedRoute.hpp" />
     <ClInclude Include="..\..\osdep\OSUtils.hpp" />

windows/ZeroTierOne/ZeroTierOne.vcxproj.filters

@@ -270,6 +270,12 @@
     <ClCompile Include="..\..\controller\LFDB.cpp">
       <Filter>Source Files\controller</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\controller\DBMirrorSet.cpp">
+      <Filter>Source Files\controller</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\osdep\EthernetTap.cpp">
+      <Filter>Source Files\osdep</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="resource.h">
@@ -518,6 +524,12 @@
     <ClInclude Include="..\..\ext\cpp-httplib\httplib.h">
       <Filter>Header Files\ext\cpp-httplib</Filter>
     </ClInclude>
+    <ClInclude Include="..\..\controller\DBMirrorSet.hpp">
+      <Filter>Header Files\controller</Filter>
+    </ClInclude>
+    <ClInclude Include="..\..\osdep\EthernetTap.hpp">
+      <Filter>Header Files\osdep</Filter>
+    </ClInclude>
   </ItemGroup>
   <ItemGroup>
     <ResourceCompile Include="ZeroTierOne.rc">

zerotier-one.spec

@@ -1,6 +1,6 @@
 Name:           zerotier-one
-Version:        1.4.0
-Release:        1%{?dist}
+Version:        1.4.2
+Release:        2%{?dist}
 Summary:        ZeroTier One network virtualization service
 
 License:        GPLv3
@@ -145,6 +145,9 @@ esac
 %endif
 
 %changelog
+* Mon Aug 04 2019 Adam Ierymenko <[email protected]> - 1.4.2-0.1
+- see https://github.com/zerotier/ZeroTierOne for release notes
+
 * Mon Jul 29 2019 Adam Ierymenko <[email protected]> - 1.4.0-0.1
 - see https://github.com/zerotier/ZeroTierOne for release notes