Merge pull request #2237 from zerotier/main-into-dev

Main into dev

README.md

@@ -44,6 +44,12 @@ The base path contains the ZeroTier One service main entry point (`one.cpp`), se
  - `windows/`: Visual Studio solution files, Windows service code, and the Windows task bar app UI.
  - `zeroidc/`: OIDC implementation used by ZeroTier service to log into SSO-enabled networks. (This part is written in Rust, and more Rust will be appearing in this repository in the future.)
 
+### Contributing
+
+Please do pull requests off of the `dev` branch.
+
+Releases are done by merging `dev` into `main` and then tagging and doing builds. 
+
 ### Build and Platform Notes
 
 To build on Mac and Linux just type `make`. On FreeBSD and OpenBSD `gmake` (GNU make) is required and can be installed from packages or ports. For Windows there is a Visual Studio solution in `windows/`.
@@ -81,7 +87,7 @@ On most distributions, macOS, and Windows, the installer will start the service
 
 A home folder for your system will automatically be created.
 
-The service is controlled via the JSON API, which by default is available at 127.0.0.1 port 9993. We include a *zerotier-cli* command line utility to make API calls for standard things like joining and leaving networks. The *authtoken.secret* file in the home folder contains the secret token for accessing this API. See [service/README.md](service/README.md) for API documentation.
+The service is controlled via the JSON API, which by default is available at `127.0.0.1:9993`. It also listens on `0.0.0.0:9993` which is only usable if `allowManagementFrom` is properly configured in `local.conf`. We include a *zerotier-cli* command line utility to make API calls for standard things like joining and leaving networks. The *authtoken.secret* file in the home folder contains the secret token for accessing this API. See [service/README.md](service/README.md) for API documentation.
 
 Here's where home folders live (by default) on each OS:
 

controller/DB.cpp

@@ -196,7 +196,7 @@ bool DB::get(const uint64_t networkId,nlohmann::json &network,std::vector<nlohma
 void DB::networks(std::set<uint64_t> &networks)
 {
 	waitForReady();
-	Metrics::db_get_member_list++;
+	Metrics::db_get_network_list++;
 	std::shared_lock<std::shared_mutex> l(_networks_l);
 	for(auto n=_networks.begin();n!=_networks.end();++n)
 		networks.insert(n->first);

controller/EmbeddedNetworkController.cpp

@@ -869,9 +869,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	// Control plane Endpoints
 	std::string controllerPath = "/controller";
 	std::string networkListPath = "/controller/network";
+	std::string networkListPath2 = "/unstable/controller/network";
 	std::string networkPath = "/controller/network/([0-9a-fA-F]{16})";
 	std::string oldAndBustedNetworkCreatePath = "/controller/network/([0-9a-fA-F]{10})______";
 	std::string memberListPath = "/controller/network/([0-9a-fA-F]{16})/member";
+	std::string memberListPath2 = "/unstable/controller/network/([0-9a-fA-F]{16})/member";
 	std::string memberPath = "/controller/network/([0-9a-fA-F]{16})/member/([0-9a-fA-F]{10})";
 
 	auto controllerGet = [&, setContent](const httplib::Request &req, httplib::Response &res) {
@@ -910,6 +912,48 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	s.Get(networkListPath, networkListGet);
 	sv6.Get(networkListPath, networkListGet);
 
+	auto networkListGet2 = [&, setContent](const httplib::Request &req, httplib::Response &res) {
+		std::set<uint64_t> networkIds;
+		_db.networks(networkIds);
+
+		auto meta = json::object();
+		auto data = json::array();
+
+		for(std::set<uint64_t>::const_iterator nwid(networkIds.begin()); nwid != networkIds.end(); ++nwid) {
+			json network;
+			if (!_db.get(*nwid, network)) {
+				continue;
+			}
+
+			std::vector<json> memTmp;
+			if (_db.get(*nwid, network, memTmp)) {
+				uint64_t authorizedCount = 0;
+				uint64_t totalCount = memTmp.size();
+
+				for (auto m = memTmp.begin(); m != memTmp.end(); ++m) {
+					bool a = OSUtils::jsonBool((*m)["authorized"], 0);
+					if (a) { authorizedCount++; }
+				}
+
+				auto nwMeta = json::object();
+				nwMeta["totalMemberCount"] = totalCount;
+				nwMeta["authorizedMemberCount"] = authorizedCount;
+				network["meta"] = nwMeta;
+			}
+
+			data.push_back(network);
+		}
+		meta["networkCount"] = networkIds.size();
+
+		auto out = json::object();
+		out["data"] = data;
+		out["meta"] = meta;
+
+		setContent(req, res, out.dump());
+	};
+	s.Get(networkListPath2, networkListGet2);
+	sv6.Get(networkListPath2, networkListGet2);
+
 	auto networkGet = [&, setContent](const httplib::Request &req, httplib::Response &res) {
 		auto networkID = req.matches[1];
 		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
@@ -925,7 +969,7 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	sv6.Get(networkPath, networkGet);
 
 	auto createNewNetwork = [&, setContent](const httplib::Request &req, httplib::Response &res) {
-		fprintf(stderr, "creating new network (new style)\n");
+		// fprintf(stderr, "creating new network (new style)\n");
 		uint64_t nwid = 0;
 		uint64_t nwidPrefix = (Utils::hexStrToU64(_signingIdAddressString.c_str()) << 24) & 0xffffffffff000000ULL;
 		uint64_t nwidPostfix = 0;
@@ -1035,6 +1079,41 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 	s.Get(memberListPath, memberListGet);
 	sv6.Get(memberListPath, memberListGet);
 
+	auto memberListGet2 = [&, setContent](const httplib::Request &req, httplib::Response &res) {
+		auto networkID = req.matches[1];
+		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
+		json network;
+		if (!_db.get(nwid, network)) {
+			res.status = 404;
+			return;
+		}
+
+		auto out = nlohmann::json::object();
+		auto meta = nlohmann::json::object();
+		auto members = nlohmann::json::array();
+		std::vector<json> memTmp;
+		if (_db.get(nwid, network, memTmp)) {
+			members.push_back(memTmp);
+		}
+
+		uint64_t authorizedCount = 0;
+		uint64_t totalCount = memTmp.size();
+		for (auto m = memTmp.begin(); m != memTmp.end(); ++m) {
+			bool a = OSUtils::jsonBool((*m)["authorized"], 0);
+			if (a) { authorizedCount++; }
+		}
+
+		meta["totalCount"] = totalCount;
+		meta["authorizedCount"] = authorizedCount;
+
+		out["data"] = members;
+		out["meta"] = meta;
+
+		setContent(req, res, out.dump());
+	};
+	s.Get(memberListPath2, memberListGet2);
+	sv6.Get(memberListPath2, memberListGet2);
+
 	auto memberGet = [&, setContent](const httplib::Request &req, httplib::Response &res) {
 		auto networkID = req.matches[1];
 		auto memberID = req.matches[2];
@@ -1057,6 +1136,12 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 		auto memberID = req.matches[2].str();
 		uint64_t nwid = Utils::hexStrToU64(networkID.c_str());
 		uint64_t memid = Utils::hexStrToU64(memberID.c_str());
+
+		if (!_db.hasNetwork(nwid)) {
+			res.status = 404;
+			return;
+		}
+
 		json network;
 		json member;
 		_db.get(nwid, network, memid, member);
@@ -1068,6 +1153,7 @@ void EmbeddedNetworkController::configureHTTPControlPlane(
 		if (b.count("noAutoAssignIps")) member["noAutoAssignIps"] = OSUtils::jsonBool(b["noAutoAssignIps"], false);
 		if (b.count("authenticationExpiryTime")) member["authenticationExpiryTime"] = (uint64_t)OSUtils::jsonInt(b["authenticationExpiryTime"], 0ULL);
 		if (b.count("authenticationURL")) member["authenticationURL"] = OSUtils::jsonString(b["authenticationURL"], "");
+		if (b.count("name")) member["name"] = OSUtils::jsonString(b["name"], "");
 
 		if (b.count("remoteTraceTarget")) {
 			const std::string rtt(OSUtils::jsonString(b["remoteTraceTarget"],""));

controller/FileDB.cpp

@@ -136,7 +136,7 @@ void FileDB::eraseNetwork(const uint64_t networkId)
 	char p[16384];
 	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx.json",_networksPath.c_str(),networkId);
 	OSUtils::rm(p);
-	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx" ZT_PATH_SEPARATOR_S "member",_networksPath.c_str(),(unsigned long long)networkId);
+	OSUtils::ztsnprintf(p,sizeof(p),"%s" ZT_PATH_SEPARATOR_S "%.16llx",_networksPath.c_str(),(unsigned long long)networkId);
 	OSUtils::rmDashRf(p);
 	_networkChanged(network,nullJson,true);
 	std::lock_guard<std::mutex> l(this->_online_l);

service/OneService.cpp

@@ -1172,25 +1172,25 @@ public:
 					}
 				}
 
-				// If secondary port is not configured to a constant value and we've been offline for a while,
-				// bind a new secondary port. This is a workaround for a "coma" issue caused by buggy NATs that stop
-				// working on one port after a while.
-				if (_secondaryPort == 0) {
-					if (_node->online()) {
-						lastOnline = now;
-					}
-					if ((now - lastOnline) > ZT_PATH_HEARTBEAT_PERIOD  || restarted) {
-						_ports[1] = _getRandomPort();
+				// Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow routes (e.g. shadow default)
+				if (((now - lastBindRefresh) >= (_node->bondController()->inUse() ? ZT_BINDER_REFRESH_PERIOD / 4 : ZT_BINDER_REFRESH_PERIOD))||restarted) {
+					// If secondary port is not configured to a constant value and we've been offline for a while,
+					// bind a new secondary port. This is a workaround for a "coma" issue caused by buggy NATs that stop
+					// working on one port after a while.
+					if (_secondaryPort == 0) {
+						if (_node->online()) {
+							lastOnline = now;
+						}
+						else if (now - lastOnline > (ZT_PEER_PING_PERIOD * 2) || restarted) {
+							lastOnline = now;	// don't keep changing the port before we have a chance to connect
+							_ports[1] = _getRandomPort();
+
 #if ZT_DEBUG==1
-						fprintf(stderr, "randomized secondary port. Now it's %d\n", _ports[1]);
+							fprintf(stderr, "Randomized secondary port. Now it's %d\n", _ports[1]);
 #endif
-						lastOnline = now; // don't keep spamming this branch. online() will be false for a few seconds
+						}
 					}
-				}
-
 
-				// Refresh bindings in case device's interfaces have changed, and also sync routes to update any shadow routes (e.g. shadow default)
-				if (((now - lastBindRefresh) >= (_node->bondController()->inUse() ? ZT_BINDER_REFRESH_PERIOD / 4 : ZT_BINDER_REFRESH_PERIOD))||restarted) {
 					unsigned int p[3];
 					unsigned int pc = 0;
 					for(int i=0;i<3;++i) {