Merge branch 'dev' into dev-extosdep

# Conflicts:
#	controller/DB.hpp
#	controller/DBMirrorSet.cpp
#	controller/DBMirrorSet.hpp
#	controller/EmbeddedNetworkController.cpp
#	controller/FileDB.cpp
#	controller/FileDB.hpp
#	controller/LFDB.cpp
#	controller/LFDB.hpp
#	controller/PostgreSQL.cpp
#	controller/PostgreSQL.hpp
#	node/Metrics.cpp
#	node/Metrics.hpp
#	osdep/EthernetTap.cpp
#	osdep/Http.hpp
#	osdep/ManagedRoute.cpp
#	service/OneService.cpp

.github/workflows/build.yml

@@ -1,4 +1,7 @@
-on: [ push ]
+on:
+  pull_request:
+  push:
+  workflow_dispatch:
 
 jobs:
   build_ubuntu:

.github/workflows/validate.yml

@@ -1,4 +1,5 @@
 on:
+  pull_request:
   push:
   workflow_dispatch:
 

Makefile

@@ -31,3 +31,6 @@ drone:
 	@echo "rendering .drone.yaml from .drone.jsonnet"
 	drone jsonnet --format --stream
 	drone sign zerotier/ZeroTierOne --save
+
+clang-format:
+	find node osdep service tcp-proxy controller -iname '*.cpp' -o -iname '*.hpp' | xargs clang-format -i

README.docker.md

@@ -64,6 +64,7 @@ You can control a few settings including the identity used and the authtoken use
 - `ZEROTIER_API_SECRET`: replaces the `authtoken.secret` before booting and allows you to manage the control socket's authentication key.
 - `ZEROTIER_IDENTITY_PUBLIC`: the `identity.public` file for zerotier-one. Use `zerotier-idtool` to generate one of these for you.
 - `ZEROTIER_IDENTITY_SECRET`: the `identity.secret` file for zerotier-one. Use `zerotier-idtool` to generate one of these for you.
+- `ZEROTIER_LOCAL_CONF`: Sets the the `local.conf` file content for zerotier-one
 
 ### Tips
 

controller/CV1.cpp

@@ -0,0 +1,1952 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "CV1.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#include "../node/Constants.hpp"
+#include "../node/SHA512.hpp"
+#include "../version.h"
+#include "CtlUtil.hpp"
+#include "EmbeddedNetworkController.hpp"
+#include "Redis.hpp"
+
+#include <chrono>
+#include <climits>
+#include <iomanip>
+#include <libpq-fe.h>
+#include <smeeclient.h>
+#include <sstream>
+
+// #define REDIS_TRACE 1
+
+using json = nlohmann::json;
+
+namespace {
+
+static const int DB_MINIMUM_VERSION = 38;
+
+}	// anonymous namespace
+
+using namespace ZeroTier;
+
+using Attrs = std::vector<std::pair<std::string, std::string> >;
+using Item = std::pair<std::string, Attrs>;
+using ItemStream = std::vector<Item>;
+
+CV1::CV1(const Identity& myId, const char* path, int listenPort, RedisConfig* rc)
+	: DB()
+	, _pool()
+	, _myId(myId)
+	, _myAddress(myId.address())
+	, _ready(0)
+	, _connected(1)
+	, _run(1)
+	, _waitNoticePrinted(false)
+	, _listenPort(listenPort)
+	, _rc(rc)
+	, _redis(NULL)
+	, _cluster(NULL)
+	, _redisMemberStatus(false)
+	, _smee(NULL)
+{
+	char myAddress[64];
+	_myAddressStr = myId.address().toString(myAddress);
+	_connString = std::string(path);
+	auto f = std::make_shared<PostgresConnFactory>(_connString);
+	_pool = std::make_shared<ConnectionPool<PostgresConnection> >(15, 5, std::static_pointer_cast<ConnectionFactory>(f));
+
+	memset(_ssoPsk, 0, sizeof(_ssoPsk));
+	char* const ssoPskHex = getenv("ZT_SSO_PSK");
+#ifdef ZT_TRACE
+	fprintf(stderr, "ZT_SSO_PSK: %s\n", ssoPskHex);
+#endif
+	if (ssoPskHex) {
+		// SECURITY: note that ssoPskHex will always be null-terminated if libc actually
+		// returns something non-NULL. If the hex encodes something shorter than 48 bytes,
+		// it will be padded at the end with zeroes. If longer, it'll be truncated.
+		Utils::unhex(ssoPskHex, _ssoPsk, sizeof(_ssoPsk));
+	}
+	const char* redisMemberStatus = getenv("ZT_REDIS_MEMBER_STATUS");
+	if (redisMemberStatus && (strcmp(redisMemberStatus, "true") == 0)) {
+		_redisMemberStatus = true;
+		fprintf(stderr, "Using redis for member status\n");
+	}
+
+	auto c = _pool->borrow();
+	pqxx::work txn { *c->c };
+
+	pqxx::row r { txn.exec1("SELECT version FROM ztc_database") };
+	int dbVersion = r[0].as<int>();
+	txn.commit();
+
+	if (dbVersion < DB_MINIMUM_VERSION) {
+		fprintf(stderr, "Central database schema version too low.  This controller version requires a minimum schema version of %d. Please upgrade your Central instance", DB_MINIMUM_VERSION);
+		exit(1);
+	}
+	_pool->unborrow(c);
+
+	if (_rc != NULL) {
+		sw::redis::ConnectionOptions opts;
+		sw::redis::ConnectionPoolOptions poolOpts;
+		opts.host = _rc->hostname;
+		opts.port = _rc->port;
+		opts.password = _rc->password;
+		opts.db = 0;
+		opts.keep_alive = true;
+		opts.connect_timeout = std::chrono::seconds(3);
+		poolOpts.size = 25;
+		poolOpts.wait_timeout = std::chrono::seconds(5);
+		poolOpts.connection_lifetime = std::chrono::minutes(3);
+		poolOpts.connection_idle_time = std::chrono::minutes(1);
+		if (_rc->clusterMode) {
+			fprintf(stderr, "Using Redis in Cluster Mode\n");
+			_cluster = std::make_shared<sw::redis::RedisCluster>(opts, poolOpts);
+		}
+		else {
+			fprintf(stderr, "Using Redis in Standalone Mode\n");
+			_redis = std::make_shared<sw::redis::Redis>(opts, poolOpts);
+		}
+	}
+
+	_readyLock.lock();
+
+	fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, ::_timestr(), (unsigned long long)_myAddress.toInt());
+	_waitNoticePrinted = true;
+
+	initializeNetworks();
+	initializeMembers();
+
+	_heartbeatThread = std::thread(&CV1::heartbeat, this);
+	_membersDbWatcher = std::thread(&CV1::membersDbWatcher, this);
+	_networksDbWatcher = std::thread(&CV1::networksDbWatcher, this);
+	for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) {
+		_commitThread[i] = std::thread(&CV1::commitThread, this);
+	}
+	_onlineNotificationThread = std::thread(&CV1::onlineNotificationThread, this);
+
+	configureSmee();
+}
+
+CV1::~CV1()
+{
+	if (_smee != NULL) {
+		smeeclient::smee_client_delete(_smee);
+		_smee = NULL;
+	}
+
+	_run = 0;
+	std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+	_heartbeatThread.join();
+	_membersDbWatcher.join();
+	_networksDbWatcher.join();
+	_commitQueue.stop();
+	for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) {
+		_commitThread[i].join();
+	}
+	_onlineNotificationThread.join();
+}
+
+void CV1::configureSmee()
+{
+	const char* TEMPORAL_SCHEME = "ZT_TEMPORAL_SCHEME";
+	const char* TEMPORAL_HOST = "ZT_TEMPORAL_HOST";
+	const char* TEMPORAL_PORT = "ZT_TEMPORAL_PORT";
+	const char* TEMPORAL_NAMESPACE = "ZT_TEMPORAL_NAMESPACE";
+	const char* SMEE_TASK_QUEUE = "ZT_SMEE_TASK_QUEUE";
+
+	const char* scheme = getenv(TEMPORAL_SCHEME);
+	if (scheme == NULL) {
+		scheme = "http";
+	}
+	const char* host = getenv(TEMPORAL_HOST);
+	const char* port = getenv(TEMPORAL_PORT);
+	const char* ns = getenv(TEMPORAL_NAMESPACE);
+	const char* task_queue = getenv(SMEE_TASK_QUEUE);
+
+	if (scheme != NULL && host != NULL && port != NULL && ns != NULL && task_queue != NULL) {
+		fprintf(stderr, "creating smee client\n");
+		std::string hostPort = std::string(scheme) + std::string("://") + std::string(host) + std::string(":") + std::string(port);
+		this->_smee = smeeclient::smee_client_new(hostPort.c_str(), ns, task_queue);
+	}
+	else {
+		fprintf(stderr, "Smee client not configured\n");
+	}
+}
+
+bool CV1::waitForReady()
+{
+	while (_ready < 2) {
+		_readyLock.lock();
+		_readyLock.unlock();
+	}
+	return true;
+}
+
+bool CV1::isReady()
+{
+	return ((_ready == 2) && (_connected));
+}
+
+bool CV1::save(nlohmann::json& record, bool notifyListeners)
+{
+	bool modified = false;
+	try {
+		if (! record.is_object()) {
+			fprintf(stderr, "record is not an object?!?\n");
+			return false;
+		}
+		const std::string objtype = record["objtype"];
+		if (objtype == "network") {
+			// fprintf(stderr, "network save\n");
+			const uint64_t nwid = OSUtils::jsonIntHex(record["id"], 0ULL);
+			if (nwid) {
+				nlohmann::json old;
+				get(nwid, old);
+				if ((! old.is_object()) || (! _compareRecords(old, record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL;
+					_commitQueue.post(std::pair<nlohmann::json, bool>(record, notifyListeners));
+					modified = true;
+				}
+			}
+		}
+		else if (objtype == "member") {
+			std::string networkId = record["nwid"];
+			std::string memberId = record["id"];
+			const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"], 0ULL);
+			const uint64_t id = OSUtils::jsonIntHex(record["id"], 0ULL);
+			// fprintf(stderr, "member save %s-%s\n", networkId.c_str(), memberId.c_str());
+			if ((id) && (nwid)) {
+				nlohmann::json network, old;
+				get(nwid, network, id, old);
+				if ((! old.is_object()) || (! _compareRecords(old, record))) {
+					// fprintf(stderr, "commit queue post\n");
+					record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL;
+					_commitQueue.post(std::pair<nlohmann::json, bool>(record, notifyListeners));
+					modified = true;
+				}
+				else {
+					// fprintf(stderr, "no change\n");
+				}
+			}
+		}
+		else {
+			fprintf(stderr, "uhh waaat\n");
+		}
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "Error on PostgreSQL::save: %s\n", e.what());
+	}
+	catch (...) {
+		fprintf(stderr, "Unknown error on PostgreSQL::save\n");
+	}
+	return modified;
+}
+
+void CV1::eraseNetwork(const uint64_t networkId)
+{
+	fprintf(stderr, "PostgreSQL::eraseNetwork\n");
+	char tmp2[24];
+	waitForReady();
+	Utils::hex(networkId, tmp2);
+	std::pair<nlohmann::json, bool> tmp;
+	tmp.first["id"] = tmp2;
+	tmp.first["objtype"] = "_delete_network";
+	tmp.second = true;
+	_commitQueue.post(tmp);
+	nlohmann::json nullJson;
+	_networkChanged(tmp.first, nullJson, true);
+}
+
+void CV1::eraseMember(const uint64_t networkId, const uint64_t memberId)
+{
+	fprintf(stderr, "PostgreSQL::eraseMember\n");
+	char tmp2[24];
+	waitForReady();
+	std::pair<nlohmann::json, bool> tmp, nw;
+	Utils::hex(networkId, tmp2);
+	tmp.first["nwid"] = tmp2;
+	Utils::hex(memberId, tmp2);
+	tmp.first["id"] = tmp2;
+	tmp.first["objtype"] = "_delete_member";
+	tmp.second = true;
+	_commitQueue.post(tmp);
+	nlohmann::json nullJson;
+	_memberChanged(tmp.first, nullJson, true);
+}
+
+void CV1::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
+{
+	std::lock_guard<std::mutex> l(_lastOnline_l);
+	NodeOnlineRecord& i = _lastOnline[std::pair<uint64_t, uint64_t>(networkId, memberId)];
+	i.lastSeen = OSUtils::now();
+	if (physicalAddress) {
+		i.physicalAddress = physicalAddress;
+	}
+	i.osArch = std::string(osArch);
+}
+
+void CV1::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
+AuthInfo CV1::getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL)
+{
+	Metrics::db_get_sso_info++;
+	// NONCE is just a random character string.  no semantic meaning
+	// state = HMAC SHA384 of Nonce based on shared sso key
+	//
+	// need nonce timeout in database? make sure it's used within X time
+	// X is 5 minutes for now.  Make configurable later?
+	//
+	// how do we tell when a nonce is used? if auth_expiration_time is set
+	std::string networkId = member["nwid"];
+	std::string memberId = member["id"];
+
+	char authenticationURL[4096] = { 0 };
+	AuthInfo info;
+	info.enabled = true;
+
+	// if (memberId == "a10dccea52" && networkId == "8056c2e21c24673d") {
+	//	fprintf(stderr, "invalid authinfo for grant's machine\n");
+	//	info.version=1;
+	//	return info;
+	// }
+	//  fprintf(stderr, "PostgreSQL::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str());
+	std::shared_ptr<PostgresConnection> c;
+	try {
+		c = _pool->borrow();
+		pqxx::work w(*c->c);
+
+		char nonceBytes[16] = { 0 };
+		std::string nonce = "";
+
+		// check if the member exists first.
+		pqxx::row count = w.exec_params1("SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND deleted = false", memberId, networkId);
+		if (count[0].as<int>() == 1) {
+			// get active nonce, if exists.
+			pqxx::result r = w.exec_params(
+				"SELECT nonce FROM ztc_sso_expiry "
+				"WHERE network_id = $1 AND member_id = $2 "
+				"AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)",
+				networkId,
+				memberId);
+
+			if (r.size() == 0) {
+				// no active nonce.
+				// find an unused nonce, if one exists.
+				pqxx::result r = w.exec_params(
+					"SELECT nonce FROM ztc_sso_expiry "
+					"WHERE network_id = $1 AND member_id = $2 "
+					"AND authentication_expiry_time IS NULL AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)",
+					networkId,
+					memberId);
+
+				if (r.size() == 1) {
+					// we have an existing nonce.  Use it
+					nonce = r.at(0)[0].as<std::string>();
+					Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes));
+				}
+				else if (r.empty()) {
+					// create a nonce
+					Utils::getSecureRandom(nonceBytes, 16);
+					char nonceBuf[64] = { 0 };
+					Utils::hex(nonceBytes, sizeof(nonceBytes), nonceBuf);
+					nonce = std::string(nonceBuf);
+
+					pqxx::result ir = w.exec_params0(
+						"INSERT INTO ztc_sso_expiry "
+						"(nonce, nonce_expiration, network_id, member_id) VALUES "
+						"($1, TO_TIMESTAMP($2::double precision/1000), $3, $4)",
+						nonce,
+						OSUtils::now() + 300000,
+						networkId,
+						memberId);
+
+					w.commit();
+				}
+				else {
+					// > 1 ?!?  Thats an error!
+					fprintf(stderr, "> 1 unused nonce!\n");
+					exit(6);
+				}
+			}
+			else if (r.size() == 1) {
+				nonce = r.at(0)[0].as<std::string>();
+				Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes));
+			}
+			else {
+				// more than 1 nonce in use?  Uhhh...
+				fprintf(stderr, "> 1 nonce in use for network member?!?\n");
+				exit(7);
+			}
+
+			r = w.exec_params(
+				"SELECT oc.client_id, oc.authorization_endpoint, oc.issuer, oc.provider, oc.sso_impl_version "
+				"FROM ztc_network AS n "
+				"INNER JOIN ztc_org o "
+				"  ON o.owner_id = n.owner_id "
+				"LEFT OUTER JOIN ztc_network_oidc_config noc "
+				"  ON noc.network_id = n.id "
+				"LEFT OUTER JOIN ztc_oidc_config oc "
+				"  ON noc.client_id = oc.client_id AND oc.org_id = o.org_id "
+				"WHERE n.id = $1 AND n.sso_enabled = true",
+				networkId);
+
+			std::string client_id = "";
+			std::string authorization_endpoint = "";
+			std::string issuer = "";
+			std::string provider = "";
+			uint64_t sso_version = 0;
+
+			if (r.size() == 1) {
+				client_id = r.at(0)[0].as<std::optional<std::string> >().value_or("");
+				authorization_endpoint = r.at(0)[1].as<std::optional<std::string> >().value_or("");
+				issuer = r.at(0)[2].as<std::optional<std::string> >().value_or("");
+				provider = r.at(0)[3].as<std::optional<std::string> >().value_or("");
+				sso_version = r.at(0)[4].as<std::optional<uint64_t> >().value_or(1);
+			}
+			else if (r.size() > 1) {
+				fprintf(stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", networkId.c_str());
+			}
+			else {
+				fprintf(stderr, "No client or auth endpoint?!?\n");
+			}
+
+			info.version = sso_version;
+
+			// no catch all else because we don't actually care if no records exist here. just continue as normal.
+			if ((! client_id.empty()) && (! authorization_endpoint.empty())) {
+				uint8_t state[48];
+				HMACSHA384(_ssoPsk, nonceBytes, sizeof(nonceBytes), state);
+				char state_hex[256];
+				Utils::hex(state, 48, state_hex);
+
+				if (info.version == 0) {
+					char url[2048] = { 0 };
+					OSUtils::ztsnprintf(
+						url,
+						sizeof(authenticationURL),
+						"%s?response_type=id_token&response_mode=form_post&scope=openid+email+profile&redirect_uri=%s&nonce=%s&state=%s&client_id=%s",
+						authorization_endpoint.c_str(),
+						url_encode(redirectURL).c_str(),
+						nonce.c_str(),
+						state_hex,
+						client_id.c_str());
+					info.authenticationURL = std::string(url);
+				}
+				else if (info.version == 1) {
+					info.ssoClientID = client_id;
+					info.issuerURL = issuer;
+					info.ssoProvider = provider;
+					info.ssoNonce = nonce;
+					info.ssoState = std::string(state_hex) + "_" + networkId;
+					info.centralAuthURL = redirectURL;
+#ifdef ZT_DEBUG
+					fprintf(
+						stderr,
+						"ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: %s\nprovider: %s\n",
+						info.ssoClientID.c_str(),
+						info.issuerURL.c_str(),
+						info.ssoNonce.c_str(),
+						info.ssoState.c_str(),
+						info.centralAuthURL.c_str(),
+						provider.c_str());
+#endif
+				}
+			}
+			else {
+				fprintf(stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), authorization_endpoint.c_str());
+			}
+		}
+
+		_pool->unborrow(c);
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "ERROR: Error updating member on load for network %s: %s\n", networkId.c_str(), e.what());
+	}
+
+	return info;   // std::string(authenticationURL);
+}
+
+void CV1::initializeNetworks()
+{
+	try {
+		std::string setKey = "networks:{" + _myAddressStr + "}";
+
+		fprintf(stderr, "Initializing Networks...\n");
+
+		if (_redisMemberStatus) {
+			fprintf(stderr, "Init Redis for networks...\n");
+			try {
+				if (_rc->clusterMode) {
+					_cluster->del(setKey);
+				}
+				else {
+					_redis->del(setKey);
+				}
+			}
+			catch (sw::redis::Error& e) {
+				// ignore. if this key doesn't exist, there's no reason to delete it
+			}
+		}
+
+		std::unordered_set<std::string> networkSet;
+
+		char qbuf[2048] = { 0 };
+		sprintf(
+			qbuf,
+			"SELECT n.id, (EXTRACT(EPOCH FROM n.creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, n.capabilities, "
+			"n.enable_broadcast, (EXTRACT(EPOCH FROM n.last_modified AT TIME ZONE 'UTC')*1000)::bigint AS last_modified, n.mtu, n.multicast_limit, n.name, n.private, n.remote_trace_level, "
+			"n.remote_trace_target, n.revision, n.rules, n.tags, n.v4_assign_mode, n.v6_assign_mode, n.sso_enabled, (CASE WHEN n.sso_enabled THEN noc.client_id ELSE NULL END) as client_id, "
+			"(CASE WHEN n.sso_enabled THEN oc.authorization_endpoint ELSE NULL END) as authorization_endpoint, "
+			"(CASE WHEN n.sso_enabled THEN oc.provider ELSE NULL END) as provider, d.domain, d.servers, "
+			"ARRAY(SELECT CONCAT(host(ip_range_start),'|', host(ip_range_end)) FROM ztc_network_assignment_pool WHERE network_id = n.id) AS assignment_pool, "
+			"ARRAY(SELECT CONCAT(host(address),'/',bits::text,'|',COALESCE(host(via), 'NULL'))FROM ztc_network_route WHERE network_id = n.id) AS routes "
+			"FROM ztc_network n "
+			"LEFT OUTER JOIN ztc_org o "
+			" ON o.owner_id = n.owner_id "
+			"LEFT OUTER JOIN ztc_network_oidc_config noc "
+			"	ON noc.network_id = n.id "
+			"LEFT OUTER JOIN ztc_oidc_config oc "
+			"	ON noc.client_id = oc.client_id AND oc.org_id = o.org_id "
+			"LEFT OUTER JOIN ztc_network_dns d "
+			"	ON d.network_id = n.id "
+			"WHERE deleted = false AND controller_id = '%s'",
+			_myAddressStr.c_str());
+		auto c = _pool->borrow();
+		auto c2 = _pool->borrow();
+		pqxx::work w { *c->c };
+
+		fprintf(stderr, "Load networks from psql...\n");
+		auto stream = pqxx::stream_from::query(w, qbuf);
+
+		std::tuple<
+			std::string	  // network ID
+			,
+			std::optional<int64_t>	 // creationTime
+			,
+			std::optional<std::string>	 // capabilities
+			,
+			std::optional<bool>	  // enableBroadcast
+			,
+			std::optional<uint64_t>	  // lastModified
+			,
+			std::optional<int>	 // mtu
+			,
+			std::optional<int>	 // multicastLimit
+			,
+			std::optional<std::string>	 // name
+			,
+			bool   // private
+			,
+			std::optional<int>	 // remoteTraceLevel
+			,
+			std::optional<std::string>	 // remoteTraceTarget
+			,
+			std::optional<uint64_t>	  // revision
+			,
+			std::optional<std::string>	 // rules
+			,
+			std::optional<std::string>	 // tags
+			,
+			std::optional<std::string>	 // v4AssignMode
+			,
+			std::optional<std::string>	 // v6AssignMode
+			,
+			std::optional<bool>	  // ssoEnabled
+			,
+			std::optional<std::string>	 // clientId
+			,
+			std::optional<std::string>	 // authorizationEndpoint
+			,
+			std::optional<std::string>	 // ssoProvider
+			,
+			std::optional<std::string>	 // domain
+			,
+			std::optional<std::string>	 // servers
+			,
+			std::string	  // assignmentPoolString
+			,
+			std::string	  // routeString
+			>
+			row;
+
+		uint64_t count = 0;
+		auto tmp = std::chrono::high_resolution_clock::now();
+		uint64_t total = 0;
+		while (stream >> row) {
+			auto start = std::chrono::high_resolution_clock::now();
+
+			json empty;
+			json config;
+
+			initNetwork(config);
+
+			std::string nwid = std::get<0>(row);
+			std::optional<int64_t> creationTime = std::get<1>(row);
+			std::optional<std::string> capabilities = std::get<2>(row);
+			std::optional<bool> enableBroadcast = std::get<3>(row);
+			std::optional<uint64_t> lastModified = std::get<4>(row);
+			std::optional<int> mtu = std::get<5>(row);
+			std::optional<int> multicastLimit = std::get<6>(row);
+			std::optional<std::string> name = std::get<7>(row);
+			bool isPrivate = std::get<8>(row);
+			std::optional<int> remoteTraceLevel = std::get<9>(row);
+			std::optional<std::string> remoteTraceTarget = std::get<10>(row);
+			std::optional<uint64_t> revision = std::get<11>(row);
+			std::optional<std::string> rules = std::get<12>(row);
+			std::optional<std::string> tags = std::get<13>(row);
+			std::optional<std::string> v4AssignMode = std::get<14>(row);
+			std::optional<std::string> v6AssignMode = std::get<15>(row);
+			std::optional<bool> ssoEnabled = std::get<16>(row);
+			std::optional<std::string> clientId = std::get<17>(row);
+			std::optional<std::string> authorizationEndpoint = std::get<18>(row);
+			std::optional<std::string> ssoProvider = std::get<19>(row);
+			std::optional<std::string> dnsDomain = std::get<20>(row);
+			std::optional<std::string> dnsServers = std::get<21>(row);
+			std::string assignmentPoolString = std::get<22>(row);
+			std::string routesString = std::get<23>(row);
+
+			config["id"] = nwid;
+			config["nwid"] = nwid;
+			config["creationTime"] = creationTime.value_or(0);
+			config["capabilities"] = json::parse(capabilities.value_or("[]"));
+			config["enableBroadcast"] = enableBroadcast.value_or(false);
+			config["lastModified"] = lastModified.value_or(0);
+			config["mtu"] = mtu.value_or(2800);
+			config["multicastLimit"] = multicastLimit.value_or(64);
+			config["name"] = name.value_or("");
+			config["private"] = isPrivate;
+			config["remoteTraceLevel"] = remoteTraceLevel.value_or(0);
+			config["remoteTraceTarget"] = remoteTraceTarget.value_or("");
+			config["revision"] = revision.value_or(0);
+			config["rules"] = json::parse(rules.value_or("[]"));
+			config["tags"] = json::parse(tags.value_or("[]"));
+			config["v4AssignMode"] = json::parse(v4AssignMode.value_or("{}"));
+			config["v6AssignMode"] = json::parse(v6AssignMode.value_or("{}"));
+			config["ssoEnabled"] = ssoEnabled.value_or(false);
+			config["objtype"] = "network";
+			config["ipAssignmentPools"] = json::array();
+			config["routes"] = json::array();
+			config["clientId"] = clientId.value_or("");
+			config["authorizationEndpoint"] = authorizationEndpoint.value_or("");
+			config["provider"] = ssoProvider.value_or("");
+
+			networkSet.insert(nwid);
+
+			if (dnsDomain.has_value()) {
+				std::string serverList = dnsServers.value();
+				json obj;
+				auto servers = json::array();
+				if (serverList.rfind("{", 0) != std::string::npos) {
+					serverList = serverList.substr(1, serverList.size() - 2);
+					std::stringstream ss(serverList);
+					while (ss.good()) {
+						std::string server;
+						std::getline(ss, server, ',');
+						servers.push_back(server);
+					}
+				}
+				obj["domain"] = dnsDomain.value();
+				obj["servers"] = servers;
+				config["dns"] = obj;
+			}
+
+			config["ipAssignmentPools"] = json::array();
+			if (assignmentPoolString != "{}") {
+				std::string tmp = assignmentPoolString.substr(1, assignmentPoolString.size() - 2);
+				std::vector<std::string> assignmentPools = split(tmp, ',');
+				for (auto it = assignmentPools.begin(); it != assignmentPools.end(); ++it) {
+					std::vector<std::string> r = split(*it, '|');
+					json ip;
+					ip["ipRangeStart"] = r[0];
+					ip["ipRangeEnd"] = r[1];
+					config["ipAssignmentPools"].push_back(ip);
+				}
+			}
+
+			config["routes"] = json::array();
+			if (routesString != "{}") {
+				std::string tmp = routesString.substr(1, routesString.size() - 2);
+				std::vector<std::string> routes = split(tmp, ',');
+				for (auto it = routes.begin(); it != routes.end(); ++it) {
+					std::vector<std::string> r = split(*it, '|');
+					json route;
+					route["target"] = r[0];
+					route["via"] = ((route["via"] == "NULL") ? nullptr : r[1]);
+					config["routes"].push_back(route);
+				}
+			}
+
+			Metrics::network_count++;
+
+			_networkChanged(empty, config, false);
+
+			auto end = std::chrono::high_resolution_clock::now();
+			auto dur = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+			;
+			total += dur.count();
+			++count;
+			if (count > 0 && count % 10000 == 0) {
+				fprintf(stderr, "Averaging %llu us per network\n", (total / count));
+			}
+		}
+
+		if (count > 0) {
+			fprintf(stderr, "Took %llu us per network to load\n", (total / count));
+		}
+		stream.complete();
+
+		w.commit();
+		_pool->unborrow(c2);
+		_pool->unborrow(c);
+		fprintf(stderr, "done.\n");
+
+		if (! networkSet.empty()) {
+			if (_redisMemberStatus) {
+				fprintf(stderr, "adding networks to redis...\n");
+				if (_rc->clusterMode) {
+					auto tx = _cluster->transaction(_myAddressStr, true, false);
+					uint64_t count = 0;
+					for (std::string nwid : networkSet) {
+						tx.sadd(setKey, nwid);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _cluster->transaction(_myAddressStr, true, false);
+						}
+					}
+					tx.exec();
+				}
+				else {
+					auto tx = _redis->transaction(true, false);
+					uint64_t count = 0;
+					for (std::string nwid : networkSet) {
+						tx.sadd(setKey, nwid);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _redis->transaction(true, false);
+						}
+					}
+					tx.exec();
+				}
+				fprintf(stderr, "done.\n");
+			}
+		}
+
+		if (++this->_ready == 2) {
+			if (_waitNoticePrinted) {
+				fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt());
+			}
+			_readyLock.unlock();
+		}
+		fprintf(stderr, "network init done.\n");
+	}
+	catch (sw::redis::Error& e) {
+		fprintf(stderr, "ERROR: Error initializing networks in Redis: %s\n", e.what());
+		std::this_thread::sleep_for(std::chrono::milliseconds(5000));
+		exit(-1);
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "ERROR: Error initializing networks: %s\n", e.what());
+		std::this_thread::sleep_for(std::chrono::milliseconds(5000));
+		exit(-1);
+	}
+}
+
+void CV1::initializeMembers()
+{
+	std::string memberId;
+	std::string networkId;
+	try {
+		std::unordered_map<std::string, std::string> networkMembers;
+		fprintf(stderr, "Initializing Members...\n");
+
+		std::string setKeyBase = "network-nodes-all:{" + _myAddressStr + "}:";
+
+		if (_redisMemberStatus) {
+			fprintf(stderr, "Initialize Redis for members...\n");
+			std::unique_lock<std::shared_mutex> l(_networks_l);
+			std::unordered_set<std::string> deletes;
+			for (auto it : _networks) {
+				uint64_t nwid_i = it.first;
+				char nwidTmp[64] = { 0 };
+				OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
+				std::string nwid(nwidTmp);
+				std::string key = setKeyBase + nwid;
+				deletes.insert(key);
+			}
+
+			if (! deletes.empty()) {
+				try {
+					if (_rc->clusterMode) {
+						auto tx = _cluster->transaction(_myAddressStr, true, false);
+						for (std::string k : deletes) {
+							tx.del(k);
+						}
+						tx.exec();
+					}
+					else {
+						auto tx = _redis->transaction(true, false);
+						for (std::string k : deletes) {
+							tx.del(k);
+						}
+						tx.exec();
+					}
+				}
+				catch (sw::redis::Error& e) {
+					// ignore
+				}
+			}
+		}
+
+		char qbuf[2048];
+		sprintf(
+			qbuf,
+			"SELECT m.id, m.network_id, m.active_bridge, m.authorized, m.capabilities, "
+			"(EXTRACT(EPOCH FROM m.creation_time AT TIME ZONE 'UTC')*1000)::bigint, m.identity, "
+			"(EXTRACT(EPOCH FROM m.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, "
+			"(EXTRACT(EPOCH FROM m.last_deauthorized_time AT TIME ZONE 'UTC')*1000)::bigint, "
+			"m.remote_trace_level, m.remote_trace_target, m.tags, m.v_major, m.v_minor, m.v_rev, m.v_proto, "
+			"m.no_auto_assign_ips, m.revision, m.sso_exempt, "
+			"(CASE WHEN n.sso_enabled = TRUE AND m.sso_exempt = FALSE THEN "
+			" ( "
+			"	SELECT (EXTRACT(EPOCH FROM e.authentication_expiry_time)*1000)::bigint "
+			"	FROM ztc_sso_expiry e "
+			"	INNER JOIN ztc_network n1 "
+			"	ON n1.id = e.network_id  AND n1.deleted = TRUE "
+			"	WHERE e.network_id = m.network_id AND e.member_id = m.id AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL "
+			"	ORDER BY e.authentication_expiry_time DESC LIMIT 1 "
+			" ) "
+			" ELSE NULL "
+			" END) AS authentication_expiry_time, "
+			"ARRAY(SELECT DISTINCT address FROM ztc_member_ip_assignment WHERE member_id = m.id AND network_id = m.network_id) AS assigned_addresses "
+			"FROM ztc_member m "
+			"INNER JOIN ztc_network n "
+			"	ON n.id = m.network_id "
+			"WHERE n.controller_id = '%s' AND n.deleted = FALSE AND m.deleted = FALSE",
+			_myAddressStr.c_str());
+		auto c = _pool->borrow();
+		auto c2 = _pool->borrow();
+		pqxx::work w { *c->c };
+
+		fprintf(stderr, "Load members from psql...\n");
+		auto stream = pqxx::stream_from::query(w, qbuf);
+
+		std::tuple<
+			std::string	  // memberId
+			,
+			std::string	  // memberId
+			,
+			std::optional<bool>	  // activeBridge
+			,
+			std::optional<bool>	  // authorized
+			,
+			std::optional<std::string>	 // capabilities
+			,
+			std::optional<uint64_t>	  // creationTime
+			,
+			std::optional<std::string>	 // identity
+			,
+			std::optional<uint64_t>	  // lastAuthorizedTime
+			,
+			std::optional<uint64_t>	  // lastDeauthorizedTime
+			,
+			std::optional<int>	 // remoteTraceLevel
+			,
+			std::optional<std::string>	 // remoteTraceTarget
+			,
+			std::optional<std::string>	 // tags
+			,
+			std::optional<int>	 // vMajor
+			,
+			std::optional<int>	 // vMinor
+			,
+			std::optional<int>	 // vRev
+			,
+			std::optional<int>	 // vProto
+			,
+			std::optional<bool>	  // noAutoAssignIps
+			,
+			std::optional<uint64_t>	  // revision
+			,
+			std::optional<bool>	  // ssoExempt
+			,
+			std::optional<uint64_t>	  // authenticationExpiryTime
+			,
+			std::string	  // assignedAddresses
+			>
+			row;
+
+		uint64_t count = 0;
+		auto tmp = std::chrono::high_resolution_clock::now();
+		uint64_t total = 0;
+		while (stream >> row) {
+			auto start = std::chrono::high_resolution_clock::now();
+			json empty;
+			json config;
+
+			initMember(config);
+
+			memberId = std::get<0>(row);
+			networkId = std::get<1>(row);
+			std::optional<bool> activeBridge = std::get<2>(row);
+			std::optional<bool> authorized = std::get<3>(row);
+			std::optional<std::string> capabilities = std::get<4>(row);
+			std::optional<uint64_t> creationTime = std::get<5>(row);
+			std::optional<std::string> identity = std::get<6>(row);
+			std::optional<uint64_t> lastAuthorizedTime = std::get<7>(row);
+			std::optional<uint64_t> lastDeauthorizedTime = std::get<8>(row);
+			std::optional<int> remoteTraceLevel = std::get<9>(row);
+			std::optional<std::string> remoteTraceTarget = std::get<10>(row);
+			std::optional<std::string> tags = std::get<11>(row);
+			std::optional<int> vMajor = std::get<12>(row);
+			std::optional<int> vMinor = std::get<13>(row);
+			std::optional<int> vRev = std::get<14>(row);
+			std::optional<int> vProto = std::get<15>(row);
+			std::optional<bool> noAutoAssignIps = std::get<16>(row);
+			std::optional<uint64_t> revision = std::get<17>(row);
+			std::optional<bool> ssoExempt = std::get<18>(row);
+			std::optional<uint64_t> authenticationExpiryTime = std::get<19>(row);
+			std::string assignedAddresses = std::get<20>(row);
+
+			networkMembers.insert(std::pair<std::string, std::string>(setKeyBase + networkId, memberId));
+
+			config["id"] = memberId;
+			config["address"] = memberId;
+			config["nwid"] = networkId;
+			config["activeBridge"] = activeBridge.value_or(false);
+			config["authorized"] = authorized.value_or(false);
+			config["capabilities"] = json::parse(capabilities.value_or("[]"));
+			config["creationTime"] = creationTime.value_or(0);
+			config["identity"] = identity.value_or("");
+			config["lastAuthorizedTime"] = lastAuthorizedTime.value_or(0);
+			config["lastDeauthorizedTime"] = lastDeauthorizedTime.value_or(0);
+			config["remoteTraceLevel"] = remoteTraceLevel.value_or(0);
+			config["remoteTraceTarget"] = remoteTraceTarget.value_or("");
+			config["tags"] = json::parse(tags.value_or("[]"));
+			config["vMajor"] = vMajor.value_or(-1);
+			config["vMinor"] = vMinor.value_or(-1);
+			config["vRev"] = vRev.value_or(-1);
+			config["vProto"] = vProto.value_or(-1);
+			config["noAutoAssignIps"] = noAutoAssignIps.value_or(false);
+			config["revision"] = revision.value_or(0);
+			config["ssoExempt"] = ssoExempt.value_or(false);
+			config["authenticationExpiryTime"] = authenticationExpiryTime.value_or(0);
+			config["objtype"] = "member";
+			config["ipAssignments"] = json::array();
+
+			if (assignedAddresses != "{}") {
+				std::string tmp = assignedAddresses.substr(1, assignedAddresses.size() - 2);
+				std::vector<std::string> addrs = split(tmp, ',');
+				for (auto it = addrs.begin(); it != addrs.end(); ++it) {
+					config["ipAssignments"].push_back(*it);
+				}
+			}
+
+			Metrics::member_count++;
+
+			_memberChanged(empty, config, false);
+
+			memberId = "";
+			networkId = "";
+
+			auto end = std::chrono::high_resolution_clock::now();
+			auto dur = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+			total += dur.count();
+			++count;
+			if (count > 0 && count % 10000 == 0) {
+				fprintf(stderr, "Averaging %llu us per member\n", (total / count));
+			}
+		}
+		if (count > 0) {
+			fprintf(stderr, "Took %llu us per member to load\n", (total / count));
+		}
+
+		stream.complete();
+
+		w.commit();
+		_pool->unborrow(c2);
+		_pool->unborrow(c);
+		fprintf(stderr, "done.\n");
+
+		if (! networkMembers.empty()) {
+			if (_redisMemberStatus) {
+				fprintf(stderr, "Load member data into redis...\n");
+				if (_rc->clusterMode) {
+					auto tx = _cluster->transaction(_myAddressStr, true, false);
+					uint64_t count = 0;
+					for (auto it : networkMembers) {
+						tx.sadd(it.first, it.second);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _cluster->transaction(_myAddressStr, true, false);
+						}
+					}
+					tx.exec();
+				}
+				else {
+					auto tx = _redis->transaction(true, false);
+					uint64_t count = 0;
+					for (auto it : networkMembers) {
+						tx.sadd(it.first, it.second);
+						if (++count % 30000 == 0) {
+							tx.exec();
+							tx = _redis->transaction(true, false);
+						}
+					}
+					tx.exec();
+				}
+				fprintf(stderr, "done.\n");
+			}
+		}
+
+		fprintf(stderr, "Done loading members...\n");
+
+		if (++this->_ready == 2) {
+			if (_waitNoticePrinted) {
+				fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt());
+			}
+			_readyLock.unlock();
+		}
+	}
+	catch (sw::redis::Error& e) {
+		fprintf(stderr, "ERROR: Error initializing members (redis): %s\n", e.what());
+		exit(-1);
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "ERROR: Error initializing member: %s-%s %s\n", networkId.c_str(), memberId.c_str(), e.what());
+		exit(-1);
+	}
+}
+
+void CV1::heartbeat()
+{
+	char publicId[1024];
+	char hostnameTmp[1024];
+	_myId.toString(false, publicId);
+	if (gethostname(hostnameTmp, sizeof(hostnameTmp)) != 0) {
+		hostnameTmp[0] = (char)0;
+	}
+	else {
+		for (int i = 0; i < (int)sizeof(hostnameTmp); ++i) {
+			if ((hostnameTmp[i] == '.') || (hostnameTmp[i] == 0)) {
+				hostnameTmp[i] = (char)0;
+				break;
+			}
+		}
+	}
+	const char* controllerId = _myAddressStr.c_str();
+	const char* publicIdentity = publicId;
+	const char* hostname = hostnameTmp;
+
+	while (_run == 1) {
+		// fprintf(stderr, "%s: heartbeat\n", controllerId);
+		auto c = _pool->borrow();
+		int64_t ts = OSUtils::now();
+
+		if (c->c) {
+			std::string major = std::to_string(ZEROTIER_ONE_VERSION_MAJOR);
+			std::string minor = std::to_string(ZEROTIER_ONE_VERSION_MINOR);
+			std::string rev = std::to_string(ZEROTIER_ONE_VERSION_REVISION);
+			std::string build = std::to_string(ZEROTIER_ONE_VERSION_BUILD);
+			std::string now = std::to_string(ts);
+			std::string host_port = std::to_string(_listenPort);
+			std::string use_redis = (_rc != NULL) ? "true" : "false";
+			std::string redis_mem_status = (_redisMemberStatus) ? "true" : "false";
+
+			try {
+				pqxx::work w { *c->c };
+
+				pqxx::result res = w.exec0(
+					"INSERT INTO ztc_controller (id, cluster_host, last_alive, public_identity, v_major, v_minor, v_rev, v_build, host_port, use_redis, redis_member_status) "
+					"VALUES ("
+					+ w.quote(controllerId) + ", " + w.quote(hostname) + ", TO_TIMESTAMP(" + now + "::double precision/1000), " + w.quote(publicIdentity) + ", " + major + ", " + minor + ", " + rev + ", " + build + ", " + host_port + ", "
+					+ use_redis + ", " + redis_mem_status
+					+ ") "
+					  "ON CONFLICT (id) DO UPDATE SET cluster_host = EXCLUDED.cluster_host, last_alive = EXCLUDED.last_alive, "
+					  "public_identity = EXCLUDED.public_identity, v_major = EXCLUDED.v_major, v_minor = EXCLUDED.v_minor, "
+					  "v_rev = EXCLUDED.v_rev, v_build = EXCLUDED.v_rev, host_port = EXCLUDED.host_port, "
+					  "use_redis = EXCLUDED.use_redis, redis_member_status = EXCLUDED.redis_member_status");
+				w.commit();
+			}
+			catch (std::exception& e) {
+				fprintf(stderr, "%s: Heartbeat update failed: %s\n", controllerId, e.what());
+				std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+				continue;
+			}
+		}
+		_pool->unborrow(c);
+
+		try {
+			if (_redisMemberStatus) {
+				if (_rc->clusterMode) {
+					_cluster->zadd("controllers", "controllerId", ts);
+				}
+				else {
+					_redis->zadd("controllers", "controllerId", ts);
+				}
+			}
+		}
+		catch (sw::redis::Error& e) {
+			fprintf(stderr, "ERROR: Redis error in heartbeat thread: %s\n", e.what());
+		}
+
+		std::this_thread::sleep_for(std::chrono::milliseconds(1000));
+	}
+	fprintf(stderr, "Exited heartbeat thread\n");
+}
+
+void CV1::membersDbWatcher()
+{
+	if (_rc) {
+		_membersWatcher_Redis();
+	}
+	else {
+		_membersWatcher_Postgres();
+	}
+
+	if (_run == 1) {
+		fprintf(stderr, "ERROR: %s membersDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str());
+		exit(9);
+	}
+	fprintf(stderr, "Exited membersDbWatcher\n");
+}
+
+void CV1::_membersWatcher_Postgres()
+{
+	auto c = _pool->borrow();
+
+	std::string stream = "member_" + _myAddressStr;
+
+	fprintf(stderr, "Listening to member stream: %s\n", stream.c_str());
+	MemberNotificationReceiver m(this, *c->c, stream);
+
+	while (_run == 1) {
+		c->c->await_notification(5, 0);
+	}
+
+	_pool->unborrow(c);
+}
+
+void CV1::_membersWatcher_Redis()
+{
+	char buf[11] = { 0 };
+	std::string key = "member-stream:{" + std::string(_myAddress.toString(buf)) + "}";
+	std::string lastID = "0";
+	fprintf(stderr, "Listening to member stream: %s\n", key.c_str());
+	while (_run == 1) {
+		try {
+			json tmp;
+			std::unordered_map<std::string, ItemStream> result;
+			if (_rc->clusterMode) {
+				_cluster->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
+			}
+			else {
+				_redis->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
+			}
+			if (! result.empty()) {
+				for (auto element : result) {
+#ifdef REDIS_TRACE
+					fprintf(stdout, "Received notification from: %s\n", element.first.c_str());
+#endif
+					for (auto rec : element.second) {
+						std::string id = rec.first;
+						auto attrs = rec.second;
+#ifdef REDIS_TRACE
+						fprintf(stdout, "Record ID: %s\n", id.c_str());
+						fprintf(stdout, "attrs len: %lu\n", attrs.size());
+#endif
+						for (auto a : attrs) {
+#ifdef REDIS_TRACE
+							fprintf(stdout, "key: %s\nvalue: %s\n", a.first.c_str(), a.second.c_str());
+#endif
+							try {
+								tmp = json::parse(a.second);
+								json& ov = tmp["old_val"];
+								json& nv = tmp["new_val"];
+								json oldConfig, newConfig;
+								if (ov.is_object())
+									oldConfig = ov;
+								if (nv.is_object())
+									newConfig = nv;
+								if (oldConfig.is_object() || newConfig.is_object()) {
+									_memberChanged(oldConfig, newConfig, (this->_ready >= 2));
+								}
+							}
+							catch (...) {
+								fprintf(stderr, "json parse error in _membersWatcher_Redis: %s\n", a.second.c_str());
+							}
+						}
+						if (_rc->clusterMode) {
+							_cluster->xdel(key, id);
+						}
+						else {
+							_redis->xdel(key, id);
+						}
+						lastID = id;
+						Metrics::redis_mem_notification++;
+					}
+				}
+			}
+		}
+		catch (sw::redis::Error& e) {
+			fprintf(stderr, "Error in Redis members watcher: %s\n", e.what());
+		}
+	}
+	fprintf(stderr, "membersWatcher ended\n");
+}
+
+void CV1::networksDbWatcher()
+{
+	if (_rc) {
+		_networksWatcher_Redis();
+	}
+	else {
+		_networksWatcher_Postgres();
+	}
+
+	if (_run == 1) {
+		fprintf(stderr, "ERROR: %s networksDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str());
+		exit(8);
+	}
+	fprintf(stderr, "Exited networksDbWatcher\n");
+}
+
+void CV1::_networksWatcher_Postgres()
+{
+	std::string stream = "network_" + _myAddressStr;
+
+	fprintf(stderr, "Listening to member stream: %s\n", stream.c_str());
+
+	auto c = _pool->borrow();
+
+	NetworkNotificationReceiver n(this, *c->c, stream);
+
+	while (_run == 1) {
+		c->c->await_notification(5, 0);
+	}
+}
+
+void CV1::_networksWatcher_Redis()
+{
+	char buf[11] = { 0 };
+	std::string key = "network-stream:{" + std::string(_myAddress.toString(buf)) + "}";
+	std::string lastID = "0";
+	while (_run == 1) {
+		try {
+			json tmp;
+			std::unordered_map<std::string, ItemStream> result;
+			if (_rc->clusterMode) {
+				_cluster->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
+			}
+			else {
+				_redis->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
+			}
+
+			if (! result.empty()) {
+				for (auto element : result) {
+#ifdef REDIS_TRACE
+					fprintf(stdout, "Received notification from: %s\n", element.first.c_str());
+#endif
+					for (auto rec : element.second) {
+						std::string id = rec.first;
+						auto attrs = rec.second;
+#ifdef REDIS_TRACE
+						fprintf(stdout, "Record ID: %s\n", id.c_str());
+						fprintf(stdout, "attrs len: %lu\n", attrs.size());
+#endif
+						for (auto a : attrs) {
+#ifdef REDIS_TRACE
+							fprintf(stdout, "key: %s\nvalue: %s\n", a.first.c_str(), a.second.c_str());
+#endif
+							try {
+								tmp = json::parse(a.second);
+								json& ov = tmp["old_val"];
+								json& nv = tmp["new_val"];
+								json oldConfig, newConfig;
+								if (ov.is_object())
+									oldConfig = ov;
+								if (nv.is_object())
+									newConfig = nv;
+								if (oldConfig.is_object() || newConfig.is_object()) {
+									_networkChanged(oldConfig, newConfig, (this->_ready >= 2));
+								}
+							}
+							catch (std::exception& e) {
+								fprintf(stderr, "json parse error in networkWatcher_Redis: what: %s json: %s\n", e.what(), a.second.c_str());
+							}
+						}
+						if (_rc->clusterMode) {
+							_cluster->xdel(key, id);
+						}
+						else {
+							_redis->xdel(key, id);
+						}
+						lastID = id;
+					}
+					Metrics::redis_net_notification++;
+				}
+			}
+		}
+		catch (sw::redis::Error& e) {
+			fprintf(stderr, "Error in Redis networks watcher: %s\n", e.what());
+		}
+	}
+	fprintf(stderr, "networksWatcher ended\n");
+}
+
+void CV1::commitThread()
+{
+	fprintf(stderr, "%s: commitThread start\n", _myAddressStr.c_str());
+	std::pair<nlohmann::json, bool> qitem;
+	while (_commitQueue.get(qitem) & (_run == 1)) {
+		// fprintf(stderr, "commitThread tick\n");
+		if (! qitem.first.is_object()) {
+			fprintf(stderr, "not an object\n");
+			continue;
+		}
+
+		std::shared_ptr<PostgresConnection> c;
+		try {
+			c = _pool->borrow();
+		}
+		catch (std::exception& e) {
+			fprintf(stderr, "ERROR: %s\n", e.what());
+			continue;
+		}
+
+		if (! c) {
+			fprintf(stderr, "Error getting database connection\n");
+			continue;
+		}
+
+		Metrics::pgsql_commit_ticks++;
+		try {
+			nlohmann::json& config = (qitem.first);
+			const std::string objtype = config["objtype"];
+			if (objtype == "member") {
+				// fprintf(stderr, "%s: commitThread: member\n", _myAddressStr.c_str());
+				std::string memberId;
+				std::string networkId;
+				try {
+					pqxx::work w(*c->c);
+
+					memberId = config["id"];
+					networkId = config["nwid"];
+
+					std::string target = "NULL";
+					if (! config["remoteTraceTarget"].is_null()) {
+						target = config["remoteTraceTarget"];
+					}
+
+					pqxx::row nwrow = w.exec_params1("SELECT COUNT(id) FROM ztc_network WHERE id = $1", networkId);
+					int nwcount = nwrow[0].as<int>();
+
+					if (nwcount != 1) {
+						fprintf(stderr, "network %s does not exist.  skipping member upsert\n", networkId.c_str());
+						w.abort();
+						_pool->unborrow(c);
+						continue;
+					}
+
+					pqxx::row mrow = w.exec_params1("SELECT COUNT(id) FROM ztc_member WHERE id = $1 AND network_id = $2", memberId, networkId);
+					int membercount = mrow[0].as<int>();
+
+					bool isNewMember = false;
+					if (membercount == 0) {
+						// new member
+						isNewMember = true;
+						pqxx::result res = w.exec_params0(
+							"INSERT INTO ztc_member (id, network_id, active_bridge, authorized, capabilities, "
+							"identity, last_authorized_time, last_deauthorized_time, no_auto_assign_ips, "
+							"remote_trace_level, remote_trace_target, revision, tags, v_major, v_minor, v_rev, v_proto) "
+							"VALUES ($1, $2, $3, $4, $5, $6, "
+							"TO_TIMESTAMP($7::double precision/1000), TO_TIMESTAMP($8::double precision/1000), "
+							"$9, $10, $11, $12, $13, $14, $15, $16, $17)",
+							memberId,
+							networkId,
+							(bool)config["activeBridge"],
+							(bool)config["authorized"],
+							OSUtils::jsonDump(config["capabilities"], -1),
+							OSUtils::jsonString(config["identity"], ""),
+							(uint64_t)config["lastAuthorizedTime"],
+							(uint64_t)config["lastDeauthorizedTime"],
+							(bool)config["noAutoAssignIps"],
+							(int)config["remoteTraceLevel"],
+							target,
+							(uint64_t)config["revision"],
+							OSUtils::jsonDump(config["tags"], -1),
+							(int)config["vMajor"],
+							(int)config["vMinor"],
+							(int)config["vRev"],
+							(int)config["vProto"]);
+					}
+					else {
+						// existing member
+						pqxx::result res = w.exec_params0(
+							"UPDATE ztc_member "
+							"SET active_bridge = $3, authorized = $4, capabilities = $5, identity = $6, "
+							"last_authorized_time = TO_TIMESTAMP($7::double precision/1000), "
+							"last_deauthorized_time = TO_TIMESTAMP($8::double precision/1000), "
+							"no_auto_assign_ips = $9, remote_trace_level = $10, remote_trace_target= $11, "
+							"revision = $12, tags = $13, v_major = $14, v_minor = $15, v_rev = $16, v_proto = $17 "
+							"WHERE id = $1 AND network_id = $2",
+							memberId,
+							networkId,
+							(bool)config["activeBridge"],
+							(bool)config["authorized"],
+							OSUtils::jsonDump(config["capabilities"], -1),
+							OSUtils::jsonString(config["identity"], ""),
+							(uint64_t)config["lastAuthorizedTime"],
+							(uint64_t)config["lastDeauthorizedTime"],
+							(bool)config["noAutoAssignIps"],
+							(int)config["remoteTraceLevel"],
+							target,
+							(uint64_t)config["revision"],
+							OSUtils::jsonDump(config["tags"], -1),
+							(int)config["vMajor"],
+							(int)config["vMinor"],
+							(int)config["vRev"],
+							(int)config["vProto"]);
+					}
+
+					if (! isNewMember) {
+						pqxx::result res = w.exec_params0("DELETE FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2", memberId, networkId);
+					}
+
+					std::vector<std::string> assignments;
+					bool ipAssignError = false;
+					for (auto i = config["ipAssignments"].begin(); i != config["ipAssignments"].end(); ++i) {
+						std::string addr = *i;
+
+						if (std::find(assignments.begin(), assignments.end(), addr) != assignments.end()) {
+							continue;
+						}
+
+						pqxx::result res = w.exec_params0("INSERT INTO ztc_member_ip_assignment (member_id, network_id, address) VALUES ($1, $2, $3) ON CONFLICT (network_id, member_id, address) DO NOTHING", memberId, networkId, addr);
+
+						assignments.push_back(addr);
+					}
+					if (ipAssignError) {
+						fprintf(stderr, "%s: ipAssignError\n", _myAddressStr.c_str());
+						w.abort();
+						_pool->unborrow(c);
+						c.reset();
+						continue;
+					}
+
+					w.commit();
+
+					if (_smee != NULL && isNewMember) {
+						pqxx::row row = w.exec_params1(
+							"SELECT "
+							"	count(h.hook_id) "
+							"FROM "
+							"	ztc_hook h "
+							"	INNER JOIN ztc_org o ON o.org_id = h.org_id "
+							"   INNER JOIN ztc_network n ON n.owner_id = o.owner_id "
+							" WHERE "
+							"n.id = $1 ",
+							networkId);
+						int64_t hookCount = row[0].as<int64_t>();
+						if (hookCount > 0) {
+							notifyNewMember(networkId, memberId);
+						}
+					}
+
+					const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL);
+					const uint64_t memberidInt = OSUtils::jsonIntHex(config["id"], 0ULL);
+					if (nwidInt && memberidInt) {
+						nlohmann::json nwOrig;
+						nlohmann::json memOrig;
+
+						nlohmann::json memNew(config);
+
+						get(nwidInt, nwOrig, memberidInt, memOrig);
+
+						_memberChanged(memOrig, memNew, qitem.second);
+					}
+					else {
+						fprintf(stderr, "%s: Can't notify of change.  Error parsing nwid or memberid: %llu-%llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt);
+					}
+				}
+				catch (std::exception& e) {
+					fprintf(stderr, "%s ERROR: Error updating member %s-%s: %s\n", _myAddressStr.c_str(), networkId.c_str(), memberId.c_str(), e.what());
+				}
+			}
+			else if (objtype == "network") {
+				try {
+					// fprintf(stderr, "%s: commitThread: network\n", _myAddressStr.c_str());
+					pqxx::work w(*c->c);
+
+					std::string id = config["id"];
+					std::string remoteTraceTarget = "";
+					if (! config["remoteTraceTarget"].is_null()) {
+						remoteTraceTarget = config["remoteTraceTarget"];
+					}
+					std::string rulesSource = "";
+					if (config["rulesSource"].is_string()) {
+						rulesSource = config["rulesSource"];
+					}
+
+					// This ugly query exists because when we want to mirror networks to/from
+					// another data store (e.g. FileDB or LFDB) it is possible to get a network
+					// that doesn't exist in Central's database. This does an upsert and sets
+					// the owner_id to the "first" global admin in the user DB if the record
+					// did not previously exist. If the record already exists owner_id is left
+					// unchanged, so owner_id should be left out of the update clause.
+					pqxx::result res = w.exec_params0(
+						"INSERT INTO ztc_network (id, creation_time, owner_id, controller_id, capabilities, enable_broadcast, "
+						"last_modified, mtu, multicast_limit, name, private, "
+						"remote_trace_level, remote_trace_target, rules, rules_source, "
+						"tags, v4_assign_mode, v6_assign_mode, sso_enabled) VALUES ("
+						"$1, TO_TIMESTAMP($5::double precision/1000), "
+						"(SELECT user_id AS owner_id FROM ztc_global_permissions WHERE authorize = true AND del = true AND modify = true AND read = true LIMIT 1),"
+						"$2, $3, $4, TO_TIMESTAMP($5::double precision/1000), "
+						"$6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) "
+						"ON CONFLICT (id) DO UPDATE set controller_id = EXCLUDED.controller_id, "
+						"capabilities = EXCLUDED.capabilities, enable_broadcast = EXCLUDED.enable_broadcast, "
+						"last_modified = EXCLUDED.last_modified, mtu = EXCLUDED.mtu, "
+						"multicast_limit = EXCLUDED.multicast_limit, name = EXCLUDED.name, "
+						"private = EXCLUDED.private, remote_trace_level = EXCLUDED.remote_trace_level, "
+						"remote_trace_target = EXCLUDED.remote_trace_target, rules = EXCLUDED.rules, "
+						"rules_source = EXCLUDED.rules_source, tags = EXCLUDED.tags, "
+						"v4_assign_mode = EXCLUDED.v4_assign_mode, v6_assign_mode = EXCLUDED.v6_assign_mode, "
+						"sso_enabled = EXCLUDED.sso_enabled",
+						id,
+						_myAddressStr,
+						OSUtils::jsonDump(config["capabilities"], -1),
+						(bool)config["enableBroadcast"],
+						OSUtils::now(),
+						(int)config["mtu"],
+						(int)config["multicastLimit"],
+						OSUtils::jsonString(config["name"], ""),
+						(bool)config["private"],
+						(int)config["remoteTraceLevel"],
+						remoteTraceTarget,
+						OSUtils::jsonDump(config["rules"], -1),
+						rulesSource,
+						OSUtils::jsonDump(config["tags"], -1),
+						OSUtils::jsonDump(config["v4AssignMode"], -1),
+						OSUtils::jsonDump(config["v6AssignMode"], -1),
+						OSUtils::jsonBool(config["ssoEnabled"], false));
+
+					res = w.exec_params0("DELETE FROM ztc_network_assignment_pool WHERE network_id = $1", 0);
+
+					auto pool = config["ipAssignmentPools"];
+					bool err = false;
+					for (auto i = pool.begin(); i != pool.end(); ++i) {
+						std::string start = (*i)["ipRangeStart"];
+						std::string end = (*i)["ipRangeEnd"];
+
+						res = w.exec_params0(
+							"INSERT INTO ztc_network_assignment_pool (network_id, ip_range_start, ip_range_end) "
+							"VALUES ($1, $2, $3)",
+							id,
+							start,
+							end);
+					}
+
+					res = w.exec_params0("DELETE FROM ztc_network_route WHERE network_id = $1", id);
+
+					auto routes = config["routes"];
+					err = false;
+					for (auto i = routes.begin(); i != routes.end(); ++i) {
+						std::string t = (*i)["target"];
+						std::vector<std::string> target;
+						std::istringstream f(t);
+						std::string s;
+						while (std::getline(f, s, '/')) {
+							target.push_back(s);
+						}
+						if (target.empty() || target.size() != 2) {
+							continue;
+						}
+						std::string targetAddr = target[0];
+						std::string targetBits = target[1];
+						std::string via = "NULL";
+						if (! (*i)["via"].is_null()) {
+							via = (*i)["via"];
+						}
+
+						res = w.exec_params0("INSERT INTO ztc_network_route (network_id, address, bits, via) VALUES ($1, $2, $3, $4)", id, targetAddr, targetBits, (via == "NULL" ? NULL : via.c_str()));
+					}
+					if (err) {
+						fprintf(stderr, "%s: route add error\n", _myAddressStr.c_str());
+						w.abort();
+						_pool->unborrow(c);
+						continue;
+					}
+
+					auto dns = config["dns"];
+					std::string domain = dns["domain"];
+					std::stringstream servers;
+					servers << "{";
+					for (auto j = dns["servers"].begin(); j < dns["servers"].end(); ++j) {
+						servers << *j;
+						if ((j + 1) != dns["servers"].end()) {
+							servers << ",";
+						}
+					}
+					servers << "}";
+
+					std::string s = servers.str();
+
+					res = w.exec_params0("INSERT INTO ztc_network_dns (network_id, domain, servers) VALUES ($1, $2, $3) ON CONFLICT (network_id) DO UPDATE SET domain = EXCLUDED.domain, servers = EXCLUDED.servers", id, domain, s);
+
+					w.commit();
+
+					const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL);
+					if (nwidInt) {
+						nlohmann::json nwOrig;
+						nlohmann::json nwNew(config);
+
+						get(nwidInt, nwOrig);
+
+						_networkChanged(nwOrig, nwNew, qitem.second);
+					}
+					else {
+						fprintf(stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt);
+					}
+				}
+				catch (std::exception& e) {
+					fprintf(stderr, "%s ERROR: Error updating network: %s\n", _myAddressStr.c_str(), e.what());
+				}
+				if (_redisMemberStatus) {
+					try {
+						std::string id = config["id"];
+						std::string controllerId = _myAddressStr.c_str();
+						std::string key = "networks:{" + controllerId + "}";
+						if (_rc->clusterMode) {
+							_cluster->sadd(key, id);
+						}
+						else {
+							_redis->sadd(key, id);
+						}
+					}
+					catch (sw::redis::Error& e) {
+						fprintf(stderr, "ERROR: Error adding network to Redis: %s\n", e.what());
+					}
+				}
+			}
+			else if (objtype == "_delete_network") {
+				// fprintf(stderr, "%s: commitThread: delete network\n", _myAddressStr.c_str());
+				try {
+					pqxx::work w(*c->c);
+
+					std::string networkId = config["nwid"];
+
+					pqxx::result res = w.exec_params0("UPDATE ztc_network SET deleted = true WHERE id = $1", networkId);
+
+					w.commit();
+				}
+				catch (std::exception& e) {
+					fprintf(stderr, "%s ERROR: Error deleting network: %s\n", _myAddressStr.c_str(), e.what());
+				}
+				if (_redisMemberStatus) {
+					try {
+						std::string id = config["id"];
+						std::string controllerId = _myAddressStr.c_str();
+						std::string key = "networks:{" + controllerId + "}";
+						if (_rc->clusterMode) {
+							_cluster->srem(key, id);
+							_cluster->del("network-nodes-online:{" + controllerId + "}:" + id);
+						}
+						else {
+							_redis->srem(key, id);
+							_redis->del("network-nodes-online:{" + controllerId + "}:" + id);
+						}
+					}
+					catch (sw::redis::Error& e) {
+						fprintf(stderr, "ERROR: Error adding network to Redis: %s\n", e.what());
+					}
+				}
+			}
+			else if (objtype == "_delete_member") {
+				// fprintf(stderr, "%s commitThread: delete member\n", _myAddressStr.c_str());
+				try {
+					pqxx::work w(*c->c);
+
+					std::string memberId = config["id"];
+					std::string networkId = config["nwid"];
+
+					pqxx::result res = w.exec_params0("UPDATE ztc_member SET hidden = true, deleted = true WHERE id = $1 AND network_id = $2", memberId, networkId);
+
+					w.commit();
+				}
+				catch (std::exception& e) {
+					fprintf(stderr, "%s ERROR: Error deleting member: %s\n", _myAddressStr.c_str(), e.what());
+				}
+				if (_redisMemberStatus) {
+					try {
+						std::string memberId = config["id"];
+						std::string networkId = config["nwid"];
+						std::string controllerId = _myAddressStr.c_str();
+						std::string key = "network-nodes-all:{" + controllerId + "}:" + networkId;
+						if (_rc->clusterMode) {
+							_cluster->srem(key, memberId);
+							_cluster->del("member:{" + controllerId + "}:" + networkId + ":" + memberId);
+						}
+						else {
+							_redis->srem(key, memberId);
+							_redis->del("member:{" + controllerId + "}:" + networkId + ":" + memberId);
+						}
+					}
+					catch (sw::redis::Error& e) {
+						fprintf(stderr, "ERROR: Error deleting member from Redis: %s\n", e.what());
+					}
+				}
+			}
+			else {
+				fprintf(stderr, "%s ERROR: unknown objtype\n", _myAddressStr.c_str());
+			}
+		}
+		catch (std::exception& e) {
+			fprintf(stderr, "%s ERROR: Error getting objtype: %s\n", _myAddressStr.c_str(), e.what());
+		}
+		_pool->unborrow(c);
+		c.reset();
+	}
+
+	fprintf(stderr, "%s commitThread finished\n", _myAddressStr.c_str());
+}
+
+void CV1::notifyNewMember(const std::string& networkID, const std::string& memberID)
+{
+	smeeclient::smee_client_notify_network_joined(_smee, networkID.c_str(), memberID.c_str());
+}
+
+void CV1::onlineNotificationThread()
+{
+	waitForReady();
+	if (_redisMemberStatus) {
+		onlineNotification_Redis();
+	}
+	else {
+		onlineNotification_Postgres();
+	}
+}
+
+/**
+ * ONLY UNCOMMENT FOR TEMPORARY DB MAINTENANCE
+ *
+ * This define temporarily turns off writing to the member status table
+ * so it can be reindexed when the indexes get too large.
+ */
+
+// #define DISABLE_MEMBER_STATUS 1
+
+void CV1::onlineNotification_Postgres()
+{
+	_connected = 1;
+
+	nlohmann::json jtmp1, jtmp2;
+	while (_run == 1) {
+		auto c = _pool->borrow();
+		auto c2 = _pool->borrow();
+		try {
+			fprintf(stderr, "%s onlineNotification_Postgres\n", _myAddressStr.c_str());
+			std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher> lastOnline;
+			{
+				std::lock_guard<std::mutex> l(_lastOnline_l);
+				lastOnline.swap(_lastOnline);
+			}
+
+#ifndef DISABLE_MEMBER_STATUS
+			pqxx::work w(*c->c);
+			pqxx::work w2(*c2->c);
+
+			fprintf(stderr, "online notification tick\n");
+
+			bool firstRun = true;
+			bool memberAdded = false;
+			int updateCount = 0;
+
+			pqxx::pipeline pipe(w);
+
+			for (auto i = lastOnline.begin(); i != lastOnline.end(); ++i) {
+				updateCount += 1;
+				uint64_t nwid_i = i->first.first;
+				char nwidTmp[64];
+				char memTmp[64];
+				char ipTmp[64];
+				OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
+				OSUtils::ztsnprintf(memTmp, sizeof(memTmp), "%.10llx", i->first.second);
+
+				if (! get(nwid_i, jtmp1, i->first.second, jtmp2)) {
+					continue;	// skip non existent networks/members
+				}
+
+				std::string networkId(nwidTmp);
+				std::string memberId(memTmp);
+
+				try {
+					pqxx::row r = w2.exec_params1("SELECT id, network_id FROM ztc_member WHERE network_id = $1 AND id = $2", networkId, memberId);
+				}
+				catch (pqxx::unexpected_rows& e) {
+					continue;
+				}
+
+				int64_t ts = i->second.lastSeen;
+				std::string ipAddr = i->second.physicalAddress.toIpString(ipTmp);
+				std::string timestamp = std::to_string(ts);
+				std::string osArch = i->second.osArch;
+
+				std::stringstream memberUpdate;
+				memberUpdate << "INSERT INTO ztc_member_status (network_id, member_id, address, last_updated) VALUES "
+							 << "('" << networkId << "', '" << memberId << "', ";
+				if (ipAddr.empty()) {
+					memberUpdate << "NULL, ";
+				}
+				else {
+					memberUpdate << "'" << ipAddr << "', ";
+				}
+				memberUpdate << "TO_TIMESTAMP(" << timestamp << "::double precision/1000)) "
+							 << " ON CONFLICT (network_id, member_id) DO UPDATE SET address = EXCLUDED.address, last_updated = EXCLUDED.last_updated";
+
+				pipe.insert(memberUpdate.str());
+				Metrics::pgsql_node_checkin++;
+			}
+			while (! pipe.empty()) {
+				pipe.retrieve();
+			}
+
+			pipe.complete();
+			w.commit();
+			fprintf(stderr, "%s: Updated online status of %d members\n", _myAddressStr.c_str(), updateCount);
+#endif
+		}
+		catch (std::exception& e) {
+			fprintf(stderr, "%s: error in onlinenotification thread: %s\n", _myAddressStr.c_str(), e.what());
+		}
+		_pool->unborrow(c2);
+		_pool->unborrow(c);
+
+		ConnectionPoolStats stats = _pool->get_stats();
+		fprintf(stderr, "%s pool stats: in use size: %llu, available size: %llu, total: %llu\n", _myAddressStr.c_str(), stats.borrowed_size, stats.pool_size, (stats.borrowed_size + stats.pool_size));
+
+		std::this_thread::sleep_for(std::chrono::seconds(10));
+	}
+	fprintf(stderr, "%s: Fell out of run loop in onlineNotificationThread\n", _myAddressStr.c_str());
+	if (_run == 1) {
+		fprintf(stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", _myAddressStr.c_str());
+		exit(6);
+	}
+}
+
+void CV1::onlineNotification_Redis()
+{
+	_connected = 1;
+
+	char buf[11] = { 0 };
+	std::string controllerId = std::string(_myAddress.toString(buf));
+
+	while (_run == 1) {
+		fprintf(stderr, "onlineNotification tick\n");
+		auto start = std::chrono::high_resolution_clock::now();
+		uint64_t count = 0;
+
+		std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher> lastOnline;
+		{
+			std::lock_guard<std::mutex> l(_lastOnline_l);
+			lastOnline.swap(_lastOnline);
+		}
+		try {
+			if (! lastOnline.empty()) {
+				if (_rc->clusterMode) {
+					auto tx = _cluster->transaction(controllerId, true, false);
+					count = _doRedisUpdate(tx, controllerId, lastOnline);
+				}
+				else {
+					auto tx = _redis->transaction(true, false);
+					count = _doRedisUpdate(tx, controllerId, lastOnline);
+				}
+			}
+		}
+		catch (sw::redis::Error& e) {
+			fprintf(stderr, "Error in online notification thread (redis): %s\n", e.what());
+		}
+
+		auto end = std::chrono::high_resolution_clock::now();
+		auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
+		auto total = dur.count();
+
+		fprintf(stderr, "onlineNotification ran in %llu ms\n", total);
+
+		std::this_thread::sleep_for(std::chrono::seconds(5));
+	}
+}
+
+uint64_t CV1::_doRedisUpdate(sw::redis::Transaction& tx, std::string& controllerId, std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher>& lastOnline)
+
+{
+	nlohmann::json jtmp1, jtmp2;
+	uint64_t count = 0;
+	for (auto i = lastOnline.begin(); i != lastOnline.end(); ++i) {
+		uint64_t nwid_i = i->first.first;
+		uint64_t memberid_i = i->first.second;
+		char nwidTmp[64];
+		char memTmp[64];
+		char ipTmp[64];
+		OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
+		OSUtils::ztsnprintf(memTmp, sizeof(memTmp), "%.10llx", memberid_i);
+
+		if (! get(nwid_i, jtmp1, memberid_i, jtmp2)) {
+			continue;	// skip non existent members/networks
+		}
+
+		std::string networkId(nwidTmp);
+		std::string memberId(memTmp);
+
+		int64_t ts = i->second.lastSeen;
+		std::string ipAddr = i->second.physicalAddress.toIpString(ipTmp);
+		std::string timestamp = std::to_string(ts);
+		std::string osArch = i->second.osArch;
+
+		std::unordered_map<std::string, std::string> record = { { "id", memberId }, { "address", ipAddr }, { "last_updated", std::to_string(ts) } };
+		tx.zadd("nodes-online:{" + controllerId + "}", memberId, ts)
+			.zadd("nodes-online2:{" + controllerId + "}", networkId + "-" + memberId, ts)
+			.zadd("network-nodes-online:{" + controllerId + "}:" + networkId, memberId, ts)
+			.zadd("active-networks:{" + controllerId + "}", networkId, ts)
+			.sadd("network-nodes-all:{" + controllerId + "}:" + networkId, memberId)
+			.hmset("member:{" + controllerId + "}:" + networkId + ":" + memberId, record.begin(), record.end());
+		++count;
+		Metrics::redis_node_checkin++;
+	}
+
+	// expire records from all-nodes and network-nodes member list
+	uint64_t expireOld = OSUtils::now() - 300000;
+
+	tx.zremrangebyscore("nodes-online:{" + controllerId + "}", sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
+	tx.zremrangebyscore("nodes-online2:{" + controllerId + "}", sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
+	tx.zremrangebyscore("active-networks:{" + controllerId + "}", sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
+	{
+		std::shared_lock<std::shared_mutex> l(_networks_l);
+		for (const auto& it : _networks) {
+			uint64_t nwid_i = it.first;
+			char nwidTmp[64];
+			OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
+			tx.zremrangebyscore("network-nodes-online:{" + controllerId + "}:" + nwidTmp, sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
+		}
+	}
+	tx.exec();
+	fprintf(stderr, "%s: Updated online status of %d members\n", _myAddressStr.c_str(), count);
+
+	return count;
+}
+
+#endif	 // ZT_CONTROLLER_USE_LIBPQ

controller/CV1.hpp

@@ -0,0 +1,141 @@
+/*
+ * Copyright (c)2019 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DB.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#ifndef ZT_CONTROLLER_CV1_HPP
+#define ZT_CONTROLLER_CV1_HPP
+
+#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
+
+#include "../node/Metrics.hpp"
+#include "ConnectionPool.hpp"
+#include "PostgreSQL.hpp"
+
+#include <memory>
+#include <pqxx/pqxx>
+#include <redis++/redis++.h>
+
+namespace smeeclient {
+struct SmeeClient;
+}
+
+namespace ZeroTier {
+
+struct RedisConfig;
+
+/**
+ * A controller database driver that talks to  PostgreSQL
+ *
+ * This is for use with ZeroTier Central.  Others are free to build and use it
+ * but be aware that we might change it at any time.
+ */
+class CV1 : public DB {
+  public:
+	CV1(const Identity& myId, const char* path, int listenPort, RedisConfig* rc);
+	virtual ~CV1();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json& record, bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL);
+
+	virtual bool ready()
+	{
+		return _ready == 2;
+	}
+
+  protected:
+	struct _PairHasher {
+		inline std::size_t operator()(const std::pair<uint64_t, uint64_t>& p) const
+		{
+			return (std::size_t)(p.first ^ p.second);
+		}
+	};
+	virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners)
+	{
+		DB::_memberChanged(old, memberConfig, notifyListeners);
+	}
+
+	virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners)
+	{
+		DB::_networkChanged(old, networkConfig, notifyListeners);
+	}
+
+  private:
+	void initializeNetworks();
+	void initializeMembers();
+	void heartbeat();
+	void membersDbWatcher();
+	void _membersWatcher_Postgres();
+	void networksDbWatcher();
+	void _networksWatcher_Postgres();
+
+	void _membersWatcher_Redis();
+	void _networksWatcher_Redis();
+
+	void commitThread();
+	void onlineNotificationThread();
+	void onlineNotification_Postgres();
+	void onlineNotification_Redis();
+	uint64_t _doRedisUpdate(sw::redis::Transaction& tx, std::string& controllerId, std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher>& lastOnline);
+
+	void configureSmee();
+	void notifyNewMember(const std::string& networkID, const std::string& memberID);
+
+	enum OverrideMode { ALLOW_PGBOUNCER_OVERRIDE = 0, NO_OVERRIDE = 1 };
+
+	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+
+	const Identity _myId;
+	const Address _myAddress;
+	std::string _myAddressStr;
+	std::string _connString;
+
+	BlockingQueue<std::pair<nlohmann::json, bool> > _commitQueue;
+
+	std::thread _heartbeatThread;
+	std::thread _membersDbWatcher;
+	std::thread _networksDbWatcher;
+	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
+	std::thread _onlineNotificationThread;
+
+	std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher> _lastOnline;
+
+	mutable std::mutex _lastOnline_l;
+	mutable std::mutex _readyLock;
+	std::atomic<int> _ready, _connected, _run;
+	mutable volatile bool _waitNoticePrinted;
+
+	int _listenPort;
+	uint8_t _ssoPsk[48];
+
+	RedisConfig* _rc;
+	std::shared_ptr<sw::redis::Redis> _redis;
+	std::shared_ptr<sw::redis::RedisCluster> _cluster;
+	bool _redisMemberStatus;
+
+	smeeclient::SmeeClient* _smee;
+};
+
+}	// namespace ZeroTier
+
+#endif	 // ZT_CONTROLLER_CV1_HPP
+
+#endif	 // ZT_CONTROLLER_USE_LIBPQ

controller/CV2.cpp

@@ -0,0 +1,1104 @@
+/*
+ * Copyright (c)2025 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "CV2.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#include "../node/Constants.hpp"
+#include "../node/SHA512.hpp"
+#include "../version.h"
+#include "CtlUtil.hpp"
+#include "EmbeddedNetworkController.hpp"
+
+#include <chrono>
+#include <climits>
+#include <iomanip>
+#include <libpq-fe.h>
+#include <sstream>
+
+using json = nlohmann::json;
+
+namespace {
+
+}
+
+using namespace ZeroTier;
+
+CV2::CV2(const Identity& myId, const char* path, int listenPort) : DB(), _pool(), _myId(myId), _myAddress(myId.address()), _ready(0), _connected(1), _run(1), _waitNoticePrinted(false), _listenPort(listenPort)
+{
+	fprintf(stderr, "CV2::CV2\n");
+	char myAddress[64];
+	_myAddressStr = myId.address().toString(myAddress);
+
+	_connString = std::string(path);
+
+	auto f = std::make_shared<PostgresConnFactory>(_connString);
+	_pool = std::make_shared<ConnectionPool<PostgresConnection> >(15, 5, std::static_pointer_cast<ConnectionFactory>(f));
+
+	memset(_ssoPsk, 0, sizeof(_ssoPsk));
+	char* const ssoPskHex = getenv("ZT_SSO_PSK");
+#ifdef ZT_TRACE
+	fprintf(stderr, "ZT_SSO_PSK: %s\n", ssoPskHex);
+#endif
+	if (ssoPskHex) {
+		// SECURITY: note that ssoPskHex will always be null-terminated if libc actually
+		// returns something non-NULL. If the hex encodes something shorter than 48 bytes,
+		// it will be padded at the end with zeroes. If longer, it'll be truncated.
+		Utils::unhex(ssoPskHex, _ssoPsk, sizeof(_ssoPsk));
+	}
+
+	_readyLock.lock();
+
+	fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, ::_timestr(), (unsigned long long)_myAddress.toInt());
+	_waitNoticePrinted = true;
+
+	initializeNetworks();
+	initializeMembers();
+
+	_heartbeatThread = std::thread(&CV2::heartbeat, this);
+	_membersDbWatcher = std::thread(&CV2::membersDbWatcher, this);
+	_networksDbWatcher = std::thread(&CV2::networksDbWatcher, this);
+	for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) {
+		_commitThread[i] = std::thread(&CV2::commitThread, this);
+	}
+	_onlineNotificationThread = std::thread(&CV2::onlineNotificationThread, this);
+}
+
+CV2::~CV2()
+{
+	_run = 0;
+	std::this_thread::sleep_for(std::chrono::milliseconds(100));
+
+	_heartbeatThread.join();
+	_membersDbWatcher.join();
+	_networksDbWatcher.join();
+	_commitQueue.stop();
+	for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) {
+		_commitThread[i].join();
+	}
+	_onlineNotificationThread.join();
+}
+
+bool CV2::waitForReady()
+{
+	while (_ready < 2) {
+		_readyLock.lock();
+		_readyLock.unlock();
+	}
+	return true;
+}
+
+bool CV2::isReady()
+{
+	return (_ready == 2) && _connected;
+}
+
+bool CV2::save(nlohmann::json& record, bool notifyListeners)
+{
+	bool modified = false;
+	try {
+		if (! record.is_object()) {
+			fprintf(stderr, "record is not an object?!?\n");
+			return false;
+		}
+		const std::string objtype = record["objtype"];
+		if (objtype == "network") {
+			// fprintf(stderr, "network save\n");
+			const uint64_t nwid = OSUtils::jsonIntHex(record["id"], 0ULL);
+			if (nwid) {
+				nlohmann::json old;
+				get(nwid, old);
+				if ((! old.is_object()) || (! _compareRecords(old, record))) {
+					record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL;
+					_commitQueue.post(std::pair<nlohmann::json, bool>(record, notifyListeners));
+					modified = true;
+				}
+			}
+		}
+		else if (objtype == "member") {
+			std::string networkId = record["nwid"];
+			std::string memberId = record["id"];
+			const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"], 0ULL);
+			const uint64_t id = OSUtils::jsonIntHex(record["id"], 0ULL);
+			// fprintf(stderr, "member save %s-%s\n", networkId.c_str(), memberId.c_str());
+			if ((id) && (nwid)) {
+				nlohmann::json network, old;
+				get(nwid, network, id, old);
+				if ((! old.is_object()) || (! _compareRecords(old, record))) {
+					// fprintf(stderr, "commit queue post\n");
+					record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL;
+					_commitQueue.post(std::pair<nlohmann::json, bool>(record, notifyListeners));
+					modified = true;
+				}
+				else {
+					// fprintf(stderr, "no change\n");
+				}
+			}
+		}
+		else {
+			fprintf(stderr, "uhh waaat\n");
+		}
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "Error on PostgreSQL::save: %s\n", e.what());
+	}
+	catch (...) {
+		fprintf(stderr, "Unknown error on PostgreSQL::save\n");
+	}
+	return modified;
+}
+
+void CV2::eraseNetwork(const uint64_t networkId)
+{
+	fprintf(stderr, "PostgreSQL::eraseNetwork\n");
+	char tmp2[24];
+	waitForReady();
+	Utils::hex(networkId, tmp2);
+	std::pair<nlohmann::json, bool> tmp;
+	tmp.first["id"] = tmp2;
+	tmp.first["objtype"] = "_delete_network";
+	tmp.second = true;
+	_commitQueue.post(tmp);
+	nlohmann::json nullJson;
+	_networkChanged(tmp.first, nullJson, true);
+}
+
+void CV2::eraseMember(const uint64_t networkId, const uint64_t memberId)
+{
+	fprintf(stderr, "PostgreSQL::eraseMember\n");
+	char tmp2[24];
+	waitForReady();
+	std::pair<nlohmann::json, bool> tmp, nw;
+	Utils::hex(networkId, tmp2);
+	tmp.first["nwid"] = tmp2;
+	Utils::hex(memberId, tmp2);
+	tmp.first["id"] = tmp2;
+	tmp.first["objtype"] = "_delete_member";
+	tmp.second = true;
+	_commitQueue.post(tmp);
+	nlohmann::json nullJson;
+	_memberChanged(tmp.first, nullJson, true);
+}
+
+void CV2::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
+{
+	std::lock_guard<std::mutex> l(_lastOnline_l);
+	NodeOnlineRecord& i = _lastOnline[std::pair<uint64_t, uint64_t>(networkId, memberId)];
+	i.lastSeen = OSUtils::now();
+	if (physicalAddress) {
+		i.physicalAddress = physicalAddress;
+	}
+	i.osArch = std::string(osArch);
+}
+
+void CV2::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
+AuthInfo CV2::getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL)
+{
+	// TODO: Redo this for CV2
+
+	Metrics::db_get_sso_info++;
+	// NONCE is just a random character string.  no semantic meaning
+	// state = HMAC SHA384 of Nonce based on shared sso key
+	//
+	// need nonce timeout in database? make sure it's used within X time
+	// X is 5 minutes for now.  Make configurable later?
+	//
+	// how do we tell when a nonce is used? if auth_expiration_time is set
+	std::string networkId = member["nwid"];
+	std::string memberId = member["id"];
+
+	char authenticationURL[4096] = { 0 };
+	AuthInfo info;
+	info.enabled = true;
+
+	// if (memberId == "a10dccea52" && networkId == "8056c2e21c24673d") {
+	//	fprintf(stderr, "invalid authinfo for grant's machine\n");
+	//	info.version=1;
+	//	return info;
+	// }
+	//  fprintf(stderr, "PostgreSQL::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str());
+	std::shared_ptr<PostgresConnection> c;
+	try {
+		// 		c = _pool->borrow();
+		// 		pqxx::work w(*c->c);
+
+		// 		char nonceBytes[16] = {0};
+		// 		std::string nonce = "";
+
+		// 		// check if the member exists first.
+		// 		pqxx::row count = w.exec_params1("SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND deleted = false", memberId, networkId);
+		// 		if (count[0].as<int>() == 1) {
+		// 			// get active nonce, if exists.
+		// 			pqxx::result r = w.exec_params("SELECT nonce FROM ztc_sso_expiry "
+		// 				"WHERE network_id = $1 AND member_id = $2 "
+		// 				"AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)",
+		// 				networkId, memberId);
+
+		// 			if (r.size() == 0) {
+		// 				// no active nonce.
+		// 				// find an unused nonce, if one exists.
+		// 				pqxx::result r = w.exec_params("SELECT nonce FROM ztc_sso_expiry "
+		// 					"WHERE network_id = $1 AND member_id = $2 "
+		// 					"AND authentication_expiry_time IS NULL AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)",
+		// 					networkId, memberId);
+
+		// 				if (r.size() == 1) {
+		// 					// we have an existing nonce.  Use it
+		// 					nonce = r.at(0)[0].as<std::string>();
+		// 					Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes));
+		// 				} else if (r.empty()) {
+		// 					// create a nonce
+		// 					Utils::getSecureRandom(nonceBytes, 16);
+		// 					char nonceBuf[64] = {0};
+		// 					Utils::hex(nonceBytes, sizeof(nonceBytes), nonceBuf);
+		// 					nonce = std::string(nonceBuf);
+
+		// 					pqxx::result ir = w.exec_params0("INSERT INTO ztc_sso_expiry "
+		// 						"(nonce, nonce_expiration, network_id, member_id) VALUES "
+		// 						"($1, TO_TIMESTAMP($2::double precision/1000), $3, $4)",
+		// 						nonce, OSUtils::now() + 300000, networkId, memberId);
+
+		// 					w.commit();
+		// 				}  else {
+		// 					// > 1 ?!?  Thats an error!
+		// 					fprintf(stderr, "> 1 unused nonce!\n");
+		// 					exit(6);
+		// 				}
+		// 			} else if (r.size() == 1) {
+		// 				nonce = r.at(0)[0].as<std::string>();
+		// 				Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes));
+		// 			} else {
+		// 				// more than 1 nonce in use?  Uhhh...
+		// 				fprintf(stderr, "> 1 nonce in use for network member?!?\n");
+		// 				exit(7);
+		// 			}
+
+		// 			r = w.exec_params(
+		// 				"SELECT oc.client_id, oc.authorization_endpoint, oc.issuer, oc.provider, oc.sso_impl_version "
+		// 				"FROM ztc_network AS n "
+		// 				"INNER JOIN ztc_org o "
+		// 				"  ON o.owner_id = n.owner_id "
+		// 			    "LEFT OUTER JOIN ztc_network_oidc_config noc "
+		// 				"  ON noc.network_id = n.id "
+		// 				"LEFT OUTER JOIN ztc_oidc_config oc "
+		// 				"  ON noc.client_id = oc.client_id AND oc.org_id = o.org_id "
+		// 				"WHERE n.id = $1 AND n.sso_enabled = true", networkId);
+
+		// 			std::string client_id = "";
+		// 			std::string authorization_endpoint = "";
+		// 			std::string issuer = "";
+		// 			std::string provider = "";
+		// 			uint64_t sso_version = 0;
+
+		// 			if (r.size() == 1) {
+		// 				client_id = r.at(0)[0].as<std::optional<std::string>>().value_or("");
+		// 				authorization_endpoint = r.at(0)[1].as<std::optional<std::string>>().value_or("");
+		// 				issuer = r.at(0)[2].as<std::optional<std::string>>().value_or("");
+		// 				provider = r.at(0)[3].as<std::optional<std::string>>().value_or("");
+		// 				sso_version = r.at(0)[4].as<std::optional<uint64_t>>().value_or(1);
+		// 			} else if (r.size() > 1) {
+		// 				fprintf(stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", networkId.c_str());
+		// 			} else {
+		// 				fprintf(stderr, "No client or auth endpoint?!?\n");
+		// 			}
+
+		// 			info.version = sso_version;
+
+		// 			// no catch all else because we don't actually care if no records exist here. just continue as normal.
+		// 			if ((!client_id.empty())&&(!authorization_endpoint.empty())) {
+
+		// 				uint8_t state[48];
+		// 				HMACSHA384(_ssoPsk, nonceBytes, sizeof(nonceBytes), state);
+		// 				char state_hex[256];
+		// 				Utils::hex(state, 48, state_hex);
+
+		// 				if (info.version == 0) {
+		// 					char url[2048] = {0};
+		// 					OSUtils::ztsnprintf(url, sizeof(authenticationURL),
+		// 						"%s?response_type=id_token&response_mode=form_post&scope=openid+email+profile&redirect_uri=%s&nonce=%s&state=%s&client_id=%s",
+		// 						authorization_endpoint.c_str(),
+		// 						url_encode(redirectURL).c_str(),
+		// 						nonce.c_str(),
+		// 						state_hex,
+		// 						client_id.c_str());
+		// 					info.authenticationURL = std::string(url);
+		// 				} else if (info.version == 1) {
+		// 					info.ssoClientID = client_id;
+		// 					info.issuerURL = issuer;
+		// 					info.ssoProvider = provider;
+		// 					info.ssoNonce = nonce;
+		// 					info.ssoState = std::string(state_hex) + "_" +networkId;
+		// 					info.centralAuthURL = redirectURL;
+		// #ifdef ZT_DEBUG
+		// 					fprintf(
+		// 						stderr,
+		// 						"ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: %s\nprovider: %s\n",
+		// 						info.ssoClientID.c_str(),
+		// 						info.issuerURL.c_str(),
+		// 						info.ssoNonce.c_str(),
+		// 						info.ssoState.c_str(),
+		// 						info.centralAuthURL.c_str(),
+		// 						provider.c_str());
+		// #endif
+		// 				}
+		// 			}  else {
+		// 				fprintf(stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), authorization_endpoint.c_str());
+		// 			}
+		// 		}
+
+		// 		_pool->unborrow(c);
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "ERROR: Error updating member on load for network %s: %s\n", networkId.c_str(), e.what());
+	}
+
+	return info;   // std::string(authenticationURL);
+}
+
+void CV2::initializeNetworks()
+{
+	fprintf(stderr, "Initializing networks...\n");
+
+	try {
+		char qbuf[2048];
+		sprintf(
+			qbuf,
+			"SELECT id, name, configuration , (EXTRACT(EPOCH FROM creation_time AT TIME ZONE 'UTC')*1000)::bigint, "
+			"(EXTRACT(EPOCH FROM last_modified AT TIME ZONE 'UTC')*1000)::bigint, revision "
+			"FROM networks_ctl WHERE controller_id = '%s'",
+			_myAddressStr.c_str());
+
+		auto c = _pool->borrow();
+		pqxx::work w(*c->c);
+
+		fprintf(stderr, "Load networks from psql...\n");
+		auto stream = pqxx::stream_from::query(w, qbuf);
+		std::tuple<
+			std::string	  // network ID
+			,
+			std::optional<std::string>	 // name
+			,
+			std::string	  // configuration
+			,
+			std::optional<uint64_t>	  // creation_time
+			,
+			std::optional<uint64_t>	  // last_modified
+			,
+			std::optional<uint64_t>	  // revision
+			>
+			row;
+		uint64_t count = 0;
+		uint64_t total = 0;
+		while (stream >> row) {
+			auto start = std::chrono::high_resolution_clock::now();
+
+			json empty;
+			json config;
+
+			initNetwork(config);
+
+			std::string nwid = std::get<0>(row);
+			std::string name = std::get<1>(row).value_or("");
+			json cfgtmp = json::parse(std::get<2>(row));
+			std::optional<uint64_t> created_at = std::get<3>(row);
+			std::optional<uint64_t> last_modified = std::get<4>(row);
+			std::optional<uint64_t> revision = std::get<5>(row);
+
+			config["id"] = nwid;
+			config["name"] = name;
+			config["creationTime"] = created_at.value_or(0);
+			config["lastModified"] = last_modified.value_or(0);
+			config["revision"] = revision.value_or(0);
+			config["capabilities"] = cfgtmp["capabilities"].is_array() ? cfgtmp["capabilities"] : json::array();
+			config["enableBroadcast"] = cfgtmp["enableBroadcast"].is_boolean() ? cfgtmp["enableBroadcast"].get<bool>() : false;
+			config["mtu"] = cfgtmp["mtu"].is_number() ? cfgtmp["mtu"].get<int32_t>() : 2800;
+			config["multicastLimit"] = cfgtmp["multicastLimit"].is_number() ? cfgtmp["multicastLimit"].get<int32_t>() : 64;
+			config["private"] = cfgtmp["private"].is_boolean() ? cfgtmp["private"].get<bool>() : true;
+			config["remoteTraceLevel"] = cfgtmp["remoteTraceLevel"].is_number() ? cfgtmp["remoteTraceLevel"].get<int32_t>() : 0;
+			config["remoteTraceTarget"] = cfgtmp["remoteTraceTarget"].is_string() ? cfgtmp["remoteTraceTarget"].get<std::string>() : "";
+			config["revision"] = revision.value_or(0);
+			config["rules"] = cfgtmp["rules"].is_array() ? cfgtmp["rules"] : json::array();
+			config["tags"] = cfgtmp["tags"].is_array() ? cfgtmp["tags"] : json::array();
+			if (cfgtmp["v4AssignMode"].is_object()) {
+				config["v4AssignMode"] = cfgtmp["v4AssignMode"];
+			}
+			else {
+				config["v4AssignMode"] = json::object();
+				config["v4AssignMode"]["zt"] = true;
+			}
+			if (cfgtmp["v6AssignMode"].is_object()) {
+				config["v6AssignMode"] = cfgtmp["v6AssignMode"];
+			}
+			else {
+				config["v6AssignMode"] = json::object();
+				config["v6AssignMode"]["zt"] = true;
+				config["v6AssignMode"]["6plane"] = true;
+				config["v6AssignMode"]["rfc4193"] = false;
+			}
+			config["ssoEnabled"] = cfgtmp["ssoEnabled"].is_boolean() ? cfgtmp["ssoEnabled"].get<bool>() : false;
+			config["objtype"] = "network";
+			config["routes"] = cfgtmp["routes"].is_array() ? cfgtmp["routes"] : json::array();
+			config["clientId"] = cfgtmp["clientId"].is_string() ? cfgtmp["clientId"].get<std::string>() : "";
+			config["authorizationEndpoint"] = cfgtmp["authorizationEndpoint"].is_string() ? cfgtmp["authorizationEndpoint"].get<std::string>() : nullptr;
+			config["provider"] = cfgtmp["ssoProvider"].is_string() ? cfgtmp["ssoProvider"].get<std::string>() : "";
+			if (! cfgtmp["dns"].is_object()) {
+				cfgtmp["dns"] = json::object();
+				cfgtmp["dns"]["domain"] = "";
+				cfgtmp["dns"]["servers"] = json::array();
+			}
+			else {
+				config["dns"] = cfgtmp["dns"];
+			}
+			config["ipAssignmentPools"] = cfgtmp["ipAssignmentPools"].is_array() ? cfgtmp["ipAssignmentPools"] : json::array();
+
+			Metrics::network_count++;
+
+			_networkChanged(empty, config, false);
+
+			auto end = std::chrono::high_resolution_clock::now();
+			auto dur = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+			;
+			total += dur.count();
+			++count;
+			if (count > 0 && count % 10000 == 0) {
+				fprintf(stderr, "Averaging %lu us per network\n", (total / count));
+			}
+		}
+
+		w.commit();
+		_pool->unborrow(c);
+		fprintf(stderr, "done.\n");
+
+		if (++this->_ready == 2) {
+			if (_waitNoticePrinted) {
+				fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt());
+			}
+			_readyLock.unlock();
+		}
+		fprintf(stderr, "network init done\n");
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "ERROR: Error initializing networks: %s\n", e.what());
+		std::this_thread::sleep_for(std::chrono::milliseconds(5000));
+		exit(-1);
+	}
+}
+
+void CV2::initializeMembers()
+{
+	std::string memberId;
+	std::string networkId;
+	try {
+		char qbuf[2048];
+		sprintf(
+			qbuf,
+			"SELECT nm.device_id, nm.network_id, nm.authorized, nm.active_bridge, nm.ip_assignments, nm.no_auto_assign_ips, "
+			"nm.sso_exempt, (EXTRACT(EPOCH FROM nm.authentication_expiry_time AT TIME ZONE 'UTC')*1000)::bigint, "
+			"(EXTRACT(EPOCH FROM nm.creation_time AT TIME ZONE 'UTC')*1000)::bigint, nm.identity, "
+			"(EXTRACT(EPOCH FROM nm.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, "
+			"(EXTRACT(EPOCH FROM nm.last_deauthorized_time AT TIME ZONE 'UTC')*1000)::bigint, "
+			"nm.remote_trace_level, nm.remote_trace_target, nm.revision, nm.capabilities, nm.tags "
+			"FROM network_memberships_ctl nm "
+			"INNER JOIN networks_ctl n "
+			"  ON nm.network_id = n.id "
+			"WHERE n.controller_id = '%s'",
+			_myAddressStr.c_str());
+
+		auto c = _pool->borrow();
+		pqxx::work w(*c->c);
+		fprintf(stderr, "Load members from psql...\n");
+		auto stream = pqxx::stream_from::query(w, qbuf);
+		std::tuple<
+			std::string	  // device ID
+			,
+			std::string	  // network ID
+			,
+			bool   // authorized
+			,
+			std::optional<bool>	  // active_bridge
+			,
+			std::optional<std::string>	 // ip_assignments
+			,
+			std::optional<bool>	  // no_auto_assign_ips
+			,
+			std::optional<bool>	  // sso_exempt
+			,
+			std::optional<uint64_t>	  // authentication_expiry_time
+			,
+			std::optional<uint64_t>	  // creation_time
+			,
+			std::optional<std::string>	 // identity
+			,
+			std::optional<uint64_t>	  // last_authorized_time
+			,
+			std::optional<uint64_t>	  // last_deauthorized_time
+			,
+			std::optional<int32_t>	 // remote_trace_level
+			,
+			std::optional<std::string>	 // remote_trace_target
+			,
+			std::optional<uint64_t>	  // revision
+			,
+			std::optional<std::string>	 // capabilities
+			,
+			std::optional<std::string>	 // tags
+			>
+			row;
+
+		uint64_t count = 0;
+		uint64_t total = 0;
+		while (stream >> row) {
+			auto start = std::chrono::high_resolution_clock::now();
+			json empty;
+			json config;
+
+			initMember(config);
+
+			memberId = std::get<0>(row);
+			networkId = std::get<1>(row);
+			bool authorized = std::get<2>(row);
+			std::optional<bool> active_bridge = std::get<3>(row);
+			std::string ip_assignments = std::get<4>(row).value_or("");
+			std::optional<bool> no_auto_assign_ips = std::get<5>(row);
+			std::optional<bool> sso_exempt = std::get<6>(row);
+			std::optional<uint64_t> authentication_expiry_time = std::get<7>(row);
+			std::optional<uint64_t> creation_time = std::get<8>(row);
+			std::optional<std::string> identity = std::get<9>(row);
+			std::optional<uint64_t> last_authorized_time = std::get<10>(row);
+			std::optional<uint64_t> last_deauthorized_time = std::get<11>(row);
+			std::optional<int32_t> remote_trace_level = std::get<12>(row);
+			std::optional<std::string> remote_trace_target = std::get<13>(row);
+			std::optional<uint64_t> revision = std::get<14>(row);
+			std::optional<std::string> capabilities = std::get<15>(row);
+			std::optional<std::string> tags = std::get<16>(row);
+
+			config["objtype"] = "member";
+			config["id"] = memberId;
+			config["address"] = identity.value_or("");
+			config["nwid"] = networkId;
+			config["authorized"] = authorized;
+			config["activeBridge"] = active_bridge.value_or(false);
+			config["ipAssignments"] = json::array();
+			if (ip_assignments != "{}") {
+				std::string tmp = ip_assignments.substr(1, ip_assignments.length() - 2);
+				std::vector<std::string> addrs = split(tmp, ',');
+				for (auto it = addrs.begin(); it != addrs.end(); ++it) {
+					config["ipAssignments"].push_back(*it);
+				}
+			}
+			config["capabilities"] = json::parse(capabilities.value_or("[]"));
+			config["creationTime"] = creation_time.value_or(0);
+			config["lastAuthorizedTime"] = last_authorized_time.value_or(0);
+			config["lastDeauthorizedTime"] = last_deauthorized_time.value_or(0);
+			config["noAutoAssignIPs"] = no_auto_assign_ips.value_or(false);
+			config["remoteTraceLevel"] = remote_trace_level.value_or(0);
+			config["remoteTraceTarget"] = remote_trace_target.value_or(nullptr);
+			config["revision"] = revision.value_or(0);
+			config["ssoExempt"] = sso_exempt.value_or(false);
+			config["authenticationExpiryTime"] = authentication_expiry_time.value_or(0);
+			config["tags"] = json::parse(tags.value_or("[]"));
+
+			Metrics::member_count++;
+
+			_memberChanged(empty, config, false);
+
+			memberId = "";
+			networkId = "";
+
+			auto end = std::chrono::high_resolution_clock::now();
+			auto dur = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
+			total += dur.count();
+			++count;
+			if (count > 0 && count % 10000 == 0) {
+				fprintf(stderr, "Averaging %lu us per member\n", (total / count));
+			}
+		}
+		if (count > 0) {
+			fprintf(stderr, "Took %lu us per member to load\n", (total / count));
+		}
+
+		stream.complete();
+		w.commit();
+		_pool->unborrow(c);
+		fprintf(stderr, "done.\n");
+
+		if (++this->_ready == 2) {
+			if (_waitNoticePrinted) {
+				fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt());
+			}
+			_readyLock.unlock();
+		}
+		fprintf(stderr, "member init done\n");
+	}
+	catch (std::exception& e) {
+		fprintf(stderr, "ERROR: Error initializing member: %s-%s %s\n", networkId.c_str(), memberId.c_str(), e.what());
+		exit(-1);
+	}
+}
+
+void CV2::heartbeat()
+{
+	char publicId[1024];
+	char hostnameTmp[1024];
+	_myId.toString(false, publicId);
+	if (gethostname(hostnameTmp, sizeof(hostnameTmp)) != 0) {
+		hostnameTmp[0] = (char)0;
+	}
+	else {
+		for (int i = 0; i < (int)sizeof(hostnameTmp); ++i) {
+			if ((hostnameTmp[i] == '.') || (hostnameTmp[i] == 0)) {
+				hostnameTmp[i] = (char)0;
+				break;
+			}
+		}
+	}
+	const char* controllerId = _myAddressStr.c_str();
+	const char* publicIdentity = publicId;
+	const char* hostname = hostnameTmp;
+
+	while (_run == 1) {
+		auto c = _pool->borrow();
+		int64_t ts = OSUtils::now();
+
+		if (c->c) {
+			std::string major = std::to_string(ZEROTIER_ONE_VERSION_MAJOR);
+			std::string minor = std::to_string(ZEROTIER_ONE_VERSION_MINOR);
+			std::string rev = std::to_string(ZEROTIER_ONE_VERSION_REVISION);
+			std::string version = major + "." + minor + "." + rev;
+			std::string versionStr = "v" + version;
+
+			try {
+				pqxx::work w { *c->c };
+				w.exec_params0(
+					"INSERT INTO controllers_ctl (id, hostname, last_heartbeat, public_identity, version) VALUES "
+					"($1, $2, TO_TIMESTAMP($3::double precision/1000), $4, $5) "
+					"ON CONFLICT (id) DO UPDATE SET hostname = EXCLUDED.hostname, last_heartbeat = EXCLUDED.last_heartbeat, "
+					"public_identity = EXCLUDED.public_identity, version = EXCLUDED.version",
+					controllerId,
+					hostname,
+					ts,
+					publicIdentity,
+					versionStr);
+				w.commit();
+			}
+			catch (std::exception& e) {
+				fprintf(stderr, "ERROR: Error in heartbeat: %s\n", e.what());
+				continue;
+			}
+			catch (...) {
+				fprintf(stderr, "ERROR: Unknown error in heartbeat\n");
+				continue;
+			}
+		}
+
+		_pool->unborrow(c);
+
+		std::this_thread::sleep_for(std::chrono::seconds(1));
+	}
+	fprintf(stderr, "Exited heartbeat thread\n");
+}
+
+void CV2::membersDbWatcher()
+{
+	auto c = _pool->borrow();
+
+	std::string stream = "member_" + _myAddressStr;
+
+	fprintf(stderr, "Listening to member stream: %s\n", stream.c_str());
+	MemberNotificationReceiver m(this, *c->c, stream);
+
+	while (_run == 1) {
+		c->c->await_notification(5, 0);
+	}
+
+	_pool->unborrow(c);
+
+	fprintf(stderr, "Exited membersDbWatcher\n");
+}
+
+void CV2::networksDbWatcher()
+{
+	std::string stream = "network_" + _myAddressStr;
+
+	fprintf(stderr, "Listening to member stream: %s\n", stream.c_str());
+
+	auto c = _pool->borrow();
+
+	NetworkNotificationReceiver n(this, *c->c, stream);
+
+	while (_run == 1) {
+		c->c->await_notification(5, 0);
+	}
+
+	_pool->unborrow(c);
+	fprintf(stderr, "Exited networksDbWatcher\n");
+}
+
+void CV2::commitThread()
+{
+	fprintf(stderr, "%s: commitThread start\n", _myAddressStr.c_str());
+	std::pair<nlohmann::json, bool> qitem;
+	while (_commitQueue.get(qitem) && (_run == 1)) {
+		// fprintf(stderr, "commitThread tick\n");
+		if (! qitem.first.is_object()) {
+			fprintf(stderr, "not an object\n");
+			continue;
+		}
+
+		std::shared_ptr<PostgresConnection> c;
+		try {
+			c = _pool->borrow();
+		}
+		catch (std::exception& e) {
+			fprintf(stderr, "ERROR: %s\n", e.what());
+			continue;
+		}
+
+		if (! c) {
+			fprintf(stderr, "Error getting database connection\n");
+			continue;
+		}
+
+		Metrics::pgsql_commit_ticks++;
+		try {
+			nlohmann::json& config = (qitem.first);
+			const std::string objtype = config["objtype"];
+			if (objtype == "member") {
+				// fprintf(stderr, "%s: commitThread: member\n", _myAddressStr.c_str());
+				std::string memberId;
+				std::string networkId;
+				try {
+					pqxx::work w(*c->c);
+
+					memberId = config["id"];
+					networkId = config["nwid"];
+
+					std::string target = "NULL";
+					if (! config["remoteTraceTarget"].is_null()) {
+						target = config["remoteTraceTarget"];
+					}
+
+					pqxx::row nwrow = w.exec_params1("SELECT COUNT(id) FROM networks WHERE id = $1", networkId);
+					int nwcount = nwrow[0].as<int>();
+
+					if (nwcount != 1) {
+						fprintf(stderr, "network %s does not exist.  skipping member upsert\n", networkId.c_str());
+						w.abort();
+						_pool->unborrow(c);
+						continue;
+					}
+
+					// only needed for hooks, and no hooks for now
+					// pqxx::row mrow = w.exec_params1("SELECT COUNT(id) FROM device_networks WHERE device_id = $1 AND network_id = $2", memberId, networkId);
+					// int membercount = mrow[0].as<int>();
+					// bool isNewMember = (membercount == 0);
+
+					pqxx::result res = w.exec_params0(
+						"INSERT INTO network_memberships_ctl (device_id, network_id, authorized, active_bridge, ip_assignments, "
+						"no_auto_assign_ips, sso_exempt, authentication_expiry_time, capabilities, creation_time, "
+						"identity, last_authorized_time, last_deauthorized_time, "
+						"remote_trace_level, remote_trace_target, revision, tags, version_major, version_minor, "
+						"version_revision, version_protocol) "
+						"VALUES ($1, $2, $3, $4, $5, $6, $7, TO_TIMESTAMP($8::double precision/1000), $9, "
+						"TO_TIMESTAMP($10::double precision/1000), $11, TO_TIMESTAMP($12::double precision/1000), "
+						"TO_TIMESTAMP($13::double precision/1000), $14, $15, $16, $17, $18, $19, $20, $21) "
+						"ON CONFLICT (device_id, network_id) DO UPDATE SET "
+						"authorized = EXCLUDED.authorized, active_bridge = EXCLUDED.active_bridge, "
+						"ip_assignments = EXCLUDED.ip_assignments, no_auto_assign_ips = EXCLUDED.no_auto_assign_ips, "
+						"sso_exempt = EXCLUDED.sso_exempt, authentication_expiry_time = EXCLUDED.authentication_expiry_time, "
+						"capabilities = EXCLUDED.capabilities, creation_time = EXCLUDED.creation_time, "
+						"identity = EXCLUDED.identity, last_authorized_time = EXCLUDED.last_authorized_time, "
+						"last_deauthorized_time = EXCLUDED.last_deauthorized_time, "
+						"remote_trace_level = EXCLUDED.remote_trace_level, remote_trace_target = EXCLUDED.remote_trace_target, "
+						"revision = EXCLUDED.revision, tags = EXCLUDED.tags, version_major = EXCLUDED.version_major, "
+						"version_minor = EXCLUDED.version_minor, version_revision = EXCLUDED.version_revision, "
+						"version_protocol = EXCLUDED.version_protocol",
+						memberId,
+						networkId,
+						(bool)config["authorized"],
+						(bool)config["activeBridge"],
+						config["ipAssignments"].get<std::vector<std::string> >(),
+						(bool)config["noAutoAssignIps"],
+						(bool)config["ssoExempt"],
+						(uint64_t)config["authenticationExpiryTime"],
+						OSUtils::jsonDump(config["capabilities"], -1),
+						(uint64_t)config["creationTime"],
+						OSUtils::jsonString(config["identity"], ""),
+						(uint64_t)config["lastAuthorizedTime"],
+						(uint64_t)config["lastDeauthorizedTime"],
+						(int)config["remoteTraceLevel"],
+						target,
+						(uint64_t)config["revision"],
+						OSUtils::jsonDump(config["tags"], -1),
+						(int)config["vMajor"],
+						(int)config["vMinor"],
+						(int)config["vRev"],
+						(int)config["vProto"]);
+
+					w.commit();
+
+					// No hooks for now
+					// if (_smee != NULL && isNewMember) {
+					// 	pqxx::row row = w.exec_params1(
+					// 		"SELECT "
+					// 		"	count(h.hook_id) "
+					// 		"FROM "
+					// 		"	ztc_hook h "
+					// 		"	INNER JOIN ztc_org o ON o.org_id = h.org_id "
+					// 		"   INNER JOIN ztc_network n ON n.owner_id = o.owner_id "
+					// 		" WHERE "
+					// 		"n.id = $1 ",
+					// 		networkId
+					// 	);
+					// 	int64_t hookCount = row[0].as<int64_t>();
+					// 	if (hookCount > 0) {
+					// 		notifyNewMember(networkId, memberId);
+					// 	}
+					// }
+
+					const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL);
+					const uint64_t memberidInt = OSUtils::jsonIntHex(config["id"], 0ULL);
+					if (nwidInt && memberidInt) {
+						nlohmann::json nwOrig;
+						nlohmann::json memOrig;
+
+						nlohmann::json memNew(config);
+
+						get(nwidInt, nwOrig, memberidInt, memOrig);
+
+						_memberChanged(memOrig, memNew, qitem.second);
+					}
+					else {
+						fprintf(stderr, "%s: Can't notify of change.  Error parsing nwid or memberid: %llu-%llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt);
+					}
+				}
+				catch (pqxx::data_exception& e) {
+					std::string cfgDump = OSUtils::jsonDump(config, 2);
+					fprintf(stderr, "Member save %s-%s: %s\n", networkId.c_str(), memberId.c_str(), cfgDump.c_str());
+
+					const pqxx::sql_error* s = dynamic_cast<const pqxx::sql_error*>(&e);
+					fprintf(stderr, "%s ERROR: Error updating member: %s\n", _myAddressStr.c_str(), e.what());
+					if (s) {
+						fprintf(stderr, "%s ERROR: SQL error: %s\n", _myAddressStr.c_str(), s->query().c_str());
+					}
+				}
+				catch (std::exception& e) {
+					std::string cfgDump = OSUtils::jsonDump(config, 2);
+					fprintf(stderr, "%s ERROR: Error updating member %s-%s: %s\njsonDump: %s\n", _myAddressStr.c_str(), networkId.c_str(), memberId.c_str(), e.what(), cfgDump.c_str());
+				}
+			}
+			else if (objtype == "network") {
+				try {
+					// fprintf(stderr, "%s: commitThread: network\n", _myAddressStr.c_str());
+					pqxx::work w(*c->c);
+
+					std::string id = config["id"];
+
+					// network must already exist
+					pqxx::result res = w.exec_params0(
+						"INSERT INTO networks_ctl (id, name, configuration, controller_id, revision) "
+						"VALUES ($1, $2, $3, $4, $5) "
+						"ON CONFLICT (id) DO UPDATE SET "
+						"name = EXCLUDED.name, configuration = EXCLUDED.configuration, revision = EXCLUDED.revision+1",
+						id,
+						OSUtils::jsonString(config["name"], ""),
+						OSUtils::jsonDump(config, -1),
+						_myAddressStr,
+						((uint64_t)config["revision"]));
+
+					w.commit();
+
+					const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL);
+					if (nwidInt) {
+						nlohmann::json nwOrig;
+						nlohmann::json nwNew(config);
+
+						get(nwidInt, nwOrig);
+
+						_networkChanged(nwOrig, nwNew, qitem.second);
+					}
+					else {
+						fprintf(stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt);
+					}
+				}
+				catch (pqxx::data_exception& e) {
+					const pqxx::sql_error* s = dynamic_cast<const pqxx::sql_error*>(&e);
+					fprintf(stderr, "%s ERROR: Error updating network: %s\n", _myAddressStr.c_str(), e.what());
+					if (s) {
+						fprintf(stderr, "%s ERROR: SQL error: %s\n", _myAddressStr.c_str(), s->query().c_str());
+					}
+				}
+				catch (std::exception& e) {
+					fprintf(stderr, "%s ERROR: Error updating network: %s\n", _myAddressStr.c_str(), e.what());
+				}
+			}
+			else if (objtype == "_delete_network") {
+				// fprintf(stderr, "%s: commitThread: delete network\n", _myAddressStr.c_str());
+				try {
+					// don't think we need this. Deletion handled by CV2 API
+
+					pqxx::work w(*c->c);
+					std::string networkId = config["id"];
+
+					w.exec_params0("DELETE FROM network_memberships_ctl WHERE network_id = $1", networkId);
+					w.exec_params0("DELETE FROM networks_ctl WHERE id = $1", networkId);
+
+					w.commit();
+				}
+				catch (std::exception& e) {
+					fprintf(stderr, "%s ERROR: Error deleting network: %s\n", _myAddressStr.c_str(), e.what());
+				}
+			}
+			else if (objtype == "_delete_member") {
+				// fprintf(stderr, "%s commitThread: delete member\n", _myAddressStr.c_str());
+				try {
+					pqxx::work w(*c->c);
+
+					std::string memberId = config["id"];
+					std::string networkId = config["nwid"];
+
+					pqxx::result res = w.exec_params0("DELETE FROM network_memberships_ctl WHERE device_id = $1 AND network_id = $2", memberId, networkId);
+
+					w.commit();
+				}
+				catch (std::exception& e) {
+					fprintf(stderr, "%s ERROR: Error deleting member: %s\n", _myAddressStr.c_str(), e.what());
+				}
+			}
+			else {
+				fprintf(stderr, "%s ERROR: unknown objtype\n", _myAddressStr.c_str());
+			}
+		}
+		catch (std::exception& e) {
+			fprintf(stderr, "%s ERROR: Error getting objtype: %s\n", _myAddressStr.c_str(), e.what());
+		}
+		_pool->unborrow(c);
+		c.reset();
+	}
+
+	fprintf(stderr, "%s commitThread finished\n", _myAddressStr.c_str());
+}
+
+void CV2::onlineNotificationThread()
+{
+	waitForReady();
+
+	_connected = 1;
+
+	nlohmann::json jtmp1, jtmp2;
+	while (_run == 1) {
+		auto c = _pool->borrow();
+		auto c2 = _pool->borrow();
+
+		try {
+			fprintf(stderr, "%s onlineNotificationThread\n", _myAddressStr.c_str());
+
+			std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher> lastOnline;
+			{
+				std::lock_guard<std::mutex> l(_lastOnline_l);
+				lastOnline.swap(_lastOnline);
+			}
+
+			pqxx::work w(*c->c);
+			pqxx::work w2(*c2->c);
+
+			bool firstRun = true;
+			bool memberAdded = false;
+			uint64_t updateCount = 0;
+
+			pqxx::pipeline pipe(w);
+
+			for (auto i = lastOnline.begin(); i != lastOnline.end(); ++i) {
+				updateCount++;
+
+				uint64_t nwid_i = i->first.first;
+				char nwidTmp[64];
+				char memTmp[64];
+				char ipTmp[64];
+
+				OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
+				OSUtils::ztsnprintf(memTmp, sizeof(memTmp), "%.10llx", i->first.second);
+
+				if (! get(nwid_i, jtmp1, i->first.second, jtmp2)) {
+					continue;	// skip non existent networks/members
+				}
+
+				std::string networkId(nwidTmp);
+				std::string memberId(memTmp);
+
+				try {
+					pqxx::row r = w2.exec_params1("SELECT device_id, network_id FROM network_memberships_ctl WHERE network_id = $1 AND device_id = $2", networkId, memberId);
+				}
+				catch (pqxx::unexpected_rows& e) {
+					continue;
+				}
+
+				int64_t ts = i->second.lastSeen;
+				std::string ipAddr = i->second.physicalAddress.toIpString(ipTmp);
+				std::string timestamp = std::to_string(ts);
+				std::string osArch = i->second.osArch;
+				std::vector<std::string> osArchSplit = split(osArch, '/');
+				std::string os = osArchSplit[0];
+				std::string arch = osArchSplit[1];
+
+				if (ipAddr.empty()) {
+					ipAddr = "relayed";
+				}
+
+				json record = {
+					{ ipAddr, ts },
+				};
+
+				std::string device_network_insert = "INSERT INTO network_memberships_ctl (device_id, network_id, last_seen, os, arch) "
+													"VALUES ('"
+													+ w2.esc(memberId) + "', '" + w2.esc(networkId) + "', '" + w2.esc(record.dump())
+													+ "'::JSONB, "
+													  "'"
+													+ w2.esc(os) + "', '" + w2.esc(arch)
+													+ "') "
+													  "ON CONFLICT (device_id, network_id) DO UPDATE SET os = EXCLUDED.os, arch = EXCLUDED.arch, "
+													  "last_seen = network_memberships_ctl.last_seen || EXCLUDED.last_seen";
+				pipe.insert(device_network_insert);
+
+				Metrics::pgsql_node_checkin++;
+			}
+
+			pipe.complete();
+			;
+			w2.commit();
+			w.commit();
+			fprintf(stderr, "%s: Updated online status of %lu members\n", _myAddressStr.c_str(), updateCount);
+		}
+		catch (std::exception& e) {
+			fprintf(stderr, "%s ERROR: Error in onlineNotificationThread: %s\n", _myAddressStr.c_str(), e.what());
+		}
+		catch (...) {
+			fprintf(stderr, "%s ERROR: Unknown error in onlineNotificationThread\n", _myAddressStr.c_str());
+		}
+		_pool->unborrow(c2);
+		_pool->unborrow(c);
+		std::this_thread::sleep_for(std::chrono::seconds(10));
+	}
+
+	fprintf(stderr, "%s: Fell out of run loop in onlineNotificationThread\n", _myAddressStr.c_str());
+	if (_run == 1) {
+		fprintf(stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", _myAddressStr.c_str());
+		exit(6);
+	}
+}
+#endif	 // ZT_CONTROLLER_USE_LIBPQ

controller/CV2.hpp

@@ -0,0 +1,112 @@
+/*
+ * Copyright (c)2025 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2026-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
+#include "DB.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#ifndef ZT_CONTROLLER_CV2_HPP
+#define ZT_CONTROLLER_CV2_HPP
+
+#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
+
+#include "../node/Metrics.hpp"
+#include "ConnectionPool.hpp"
+#include "PostgreSQL.hpp"
+
+#include <memory>
+#include <pqxx/pqxx>
+#include <redis++/redis++.h>
+
+namespace ZeroTier {
+
+class CV2 : public DB {
+  public:
+	CV2(const Identity& myId, const char* path, int listenPort);
+	virtual ~CV2();
+
+	virtual bool waitForReady();
+	virtual bool isReady();
+	virtual bool save(nlohmann::json& record, bool notifyListeners);
+	virtual void eraseNetwork(const uint64_t networkId);
+	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
+	virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL);
+
+	virtual bool ready()
+	{
+		return _ready == 2;
+	}
+
+  protected:
+	struct _PairHasher {
+		inline std::size_t operator()(const std::pair<uint64_t, uint64_t>& p) const
+		{
+			return (std::size_t)(p.first ^ p.second);
+		}
+	};
+	virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners)
+	{
+		DB::_memberChanged(old, memberConfig, notifyListeners);
+	}
+
+	virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners)
+	{
+		DB::_networkChanged(old, networkConfig, notifyListeners);
+	}
+
+  private:
+	void initializeNetworks();
+	void initializeMembers();
+	void heartbeat();
+	void membersDbWatcher();
+	void networksDbWatcher();
+
+	void commitThread();
+	void onlineNotificationThread();
+
+	// void notifyNewMember(const std::string &networkID, const std::string &memberID);
+
+	enum OverrideMode { ALLOW_PGBOUNCER_OVERRIDE = 0, NO_OVERRIDE = 1 };
+
+	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
+
+	const Identity _myId;
+	const Address _myAddress;
+	std::string _myAddressStr;
+	std::string _connString;
+
+	BlockingQueue<std::pair<nlohmann::json, bool> > _commitQueue;
+
+	std::thread _heartbeatThread;
+	std::thread _membersDbWatcher;
+	std::thread _networksDbWatcher;
+	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
+	std::thread _onlineNotificationThread;
+
+	std::unordered_map<std::pair<uint64_t, uint64_t>, NodeOnlineRecord, _PairHasher> _lastOnline;
+
+	mutable std::mutex _lastOnline_l;
+	mutable std::mutex _readyLock;
+	std::atomic<int> _ready, _connected, _run;
+	mutable volatile bool _waitNoticePrinted;
+
+	int _listenPort;
+	uint8_t _ssoPsk[48];
+};
+
+}	// namespace ZeroTier
+
+#endif	 // ZT_CONTROLLER_CV2_HPP
+#endif	 // ZT_CONTROLLER_USE_LIBPQ

controller/CtlUtil.cpp

@@ -0,0 +1,64 @@
+#include "CtlUtil.hpp"
+
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+
+#include <iomanip>
+#include <sstream>
+
+namespace ZeroTier {
+
+const char* _timestr()
+{
+	time_t t = time(0);
+	char* ts = ctime(&t);
+	char* p = ts;
+	if (! p)
+		return "";
+	while (*p) {
+		if (*p == '\n') {
+			*p = (char)0;
+			break;
+		}
+		++p;
+	}
+	return ts;
+}
+
+std::vector<std::string> split(std::string str, char delim)
+{
+	std::istringstream iss(str);
+	std::vector<std::string> tokens;
+	std::string item;
+	while (std::getline(iss, item, delim)) {
+		tokens.push_back(item);
+	}
+	return tokens;
+}
+
+std::string url_encode(const std::string& value)
+{
+	std::ostringstream escaped;
+	escaped.fill('0');
+	escaped << std::hex;
+
+	for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
+		std::string::value_type c = (*i);
+
+		// Keep alphanumeric and other accepted characters intact
+		if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
+			escaped << c;
+			continue;
+		}
+
+		// Any other characters are percent-encoded
+		escaped << std::uppercase;
+		escaped << '%' << std::setw(2) << int((unsigned char)c);
+		escaped << std::nouppercase;
+	}
+
+	return escaped.str();
+}
+
+}	// namespace ZeroTier
+
+#endif

controller/CtlUtil.hpp

@@ -0,0 +1,16 @@
+#ifndef ZT_CTLUTIL_HPP
+#define ZT_CTLUTIL_HPP
+
+#include <string>
+#include <vector>
+
+namespace ZeroTier {
+
+const char* _timestr();
+
+std::vector<std::string> split(std::string str, char delim);
+
+std::string url_encode(const std::string& value);
+}	// namespace ZeroTier
+
+#endif	 // namespace ZeroTier

controller/DB.hpp

@@ -61,6 +61,10 @@ struct AuthInfo {
  * Base class with common infrastructure for all controller DB implementations
  */
 class DB {
+#ifdef ZT_CONTROLLER_USE_LIBPQ
+	friend class MemberNotificationReceiver;
+	friend class NetworkNotificationReceiver;
+#endif
   public:
 	class ChangeListener {
 	  public:
@@ -132,6 +136,7 @@ class DB {
 	virtual void eraseNetwork(const uint64_t networkId) = 0;
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId) = 0;
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress) = 0;
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch) = 0;
 
 	virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL)
 	{

controller/DBMirrorSet.cpp

@@ -205,14 +205,19 @@ void DBMirrorSet::eraseMember(const uint64_t networkId, const uint64_t memberId)
 	}
 }
 
-void DBMirrorSet::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+void DBMirrorSet::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
 {
 	std::shared_lock<std::shared_mutex> l(_dbs_l);
 	for (auto d = _dbs.begin(); d != _dbs.end(); ++d) {
-		(*d)->nodeIsOnline(networkId, memberId, physicalAddress);
+		(*d)->nodeIsOnline(networkId, memberId, physicalAddress, osArch);
 	}
 }
 
+void DBMirrorSet::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
 void DBMirrorSet::onNetworkUpdate(const void* db, uint64_t networkId, const nlohmann::json& network)
 {
 	nlohmann::json record(network);

controller/DBMirrorSet.hpp

@@ -43,7 +43,8 @@ class DBMirrorSet : public DB::ChangeListener {
 	bool save(nlohmann::json& record, bool notifyListeners);
 	void eraseNetwork(const uint64_t networkId);
 	void eraseMember(const uint64_t networkId, const uint64_t memberId);
-	void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
 
 	// These are called by various DB instances when changes occur.
 	virtual void onNetworkUpdate(const void* db, uint64_t networkId, const nlohmann::json& network);

controller/EmbeddedNetworkController.cpp

@@ -38,7 +38,8 @@
 #include <thread>
 #include <utility>
 #ifdef ZT_CONTROLLER_USE_LIBPQ
-#include "PostgreSQL.hpp"
+#include "CV1.hpp"
+#include "CV2.hpp"
 #endif
 
 #include "../node/CertificateOfMembership.hpp"
@@ -594,9 +595,15 @@ void EmbeddedNetworkController::init(const Identity& signingId, Sender* sender)
 
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 	if ((_path.length() > 9) && (_path.substr(0, 9) == "postgres:")) {
-		_db.addDB(std::shared_ptr<DB>(new PostgreSQL(_signingId, _path.substr(9).c_str(), _listenPort, _rc)));
+		fprintf(stderr, "CV1\n");
+		_db.addDB(std::shared_ptr<DB>(new CV1(_signingId, _path.substr(9).c_str(), _listenPort, _rc)));
+	}
+	else if ((_path.length() > 4) && (_path.substr(0, 4) == "cv2:")) {
+		fprintf(stderr, "CV2\n");
+		_db.addDB(std::shared_ptr<DB>(new CV2(_signingId, _path.substr(4).c_str(), _listenPort)));
 	}
 	else {
+		fprintf(stderr, "FileDB\n");
 #endif
 		_db.addDB(std::shared_ptr<DB>(new FileDB(_path.c_str())));
 #ifdef ZT_CONTROLLER_USE_LIBPQ
@@ -1496,7 +1503,11 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA
 	c2++;
 	b2.start();
 #endif
-	_db.nodeIsOnline(nwid, identity.address().toInt(), fromAddr);
+	char osArch[256];
+	metaData.get(ZT_NETWORKCONFIG_REQUEST_METADATA_KEY_OS_ARCH, osArch, sizeof(osArch));
+	// fprintf(stderr, "Network Config Request: nwid=%.16llx, nodeid=%.10llx, osArch=%s\n",
+	// 	nwid, identity.address().toInt(), osArch);
+	_db.nodeIsOnline(nwid, identity.address().toInt(), fromAddr, osArch);
 #ifdef CENTRAL_CONTROLLER_REQUEST_BENCHMARK
 	b2.stop();
 
@@ -1650,6 +1661,7 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_NONCE, info.ssoNonce.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_STATE, info.ssoState.c_str());
 				authInfo.add(ZT_AUTHINFO_DICT_KEY_CLIENT_ID, info.ssoClientID.c_str());
+				authInfo.add(ZT_AUTHINFO_DICT_KEY_SSO_PROVIDER, info.ssoProvider.c_str());
 				_sender->ncSendError(nwid, requestPacketId, identity.address(), NetworkController::NC_ERROR_AUTHENTICATION_REQUIRED, authInfo.data(), authInfo.sizeBytes());
 			}
 			DB::cleanMember(member);

controller/FileDB.cpp

@@ -160,7 +160,7 @@ void FileDB::eraseMember(const uint64_t networkId, const uint64_t memberId)
 	this->_online[networkId].erase(memberId);
 }
 
-void FileDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+void FileDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
 {
 	char mid[32], atmp[64];
 	OSUtils::ztsnprintf(mid, sizeof(mid), "%.10llx", (unsigned long long)memberId);
@@ -169,4 +169,9 @@ void FileDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, con
 	this->_online[networkId][memberId][OSUtils::now()] = physicalAddress;
 }
 
+void FileDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
 }	// namespace ZeroTier

controller/FileDB.hpp

@@ -29,6 +29,7 @@ class FileDB : public DB {
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
 
   protected:
 	std::string _path;

controller/LFDB.cpp

@@ -420,7 +420,7 @@ void LFDB::eraseMember(const uint64_t networkId, const uint64_t memberId)
 	// TODO
 }
 
-void LFDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+void LFDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch)
 {
 	std::lock_guard<std::mutex> l(_state_l);
 	auto nw = _state.find(networkId);
@@ -435,4 +435,9 @@ void LFDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const
 	}
 }
 
+void LFDB::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
+{
+	this->nodeIsOnline(networkId, memberId, physicalAddress, "unknown/unknown");
+}
+
 }	// namespace ZeroTier

controller/LFDB.hpp

@@ -46,6 +46,7 @@ class LFDB : public DB {
 	virtual void eraseNetwork(const uint64_t networkId);
 	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
 	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
+	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress, const char* osArch);
 
   protected:
 	const Identity _myId;

controller/PostgreSQL.cpp

@@ -1,115 +1,14 @@
-/*
- * Copyright (c)2019 ZeroTier, Inc.
- *
- * Use of this software is governed by the Business Source License included
- * in the LICENSE.TXT file in the project's root directory.
- *
- * Change Date: 2026-01-01
- *
- * On the date above, in accordance with the Business Source License, use
- * of this software will be governed by version 2.0 of the Apache License.
- */
-/****/
-
-#include "PostgreSQL.hpp"
-
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 
-#include "../node/Constants.hpp"
-#include "../node/SHA512.hpp"
-#include "../version.h"
-#include "EmbeddedNetworkController.hpp"
-#include "Redis.hpp"
-
-#include <chrono>
-#include <climits>
-#include <iomanip>
-#include <libpq-fe.h>
-#include <smeeclient.h>
-#include <sstream>
-
-// #define REDIS_TRACE 1
-
-using json = nlohmann::json;
-
-namespace {
-
-static const int DB_MINIMUM_VERSION = 38;
-
-static const char* _timestr()
-{
-	time_t t = time(0);
-	char* ts = ctime(&t);
-	char* p = ts;
-	if (! p)
-		return "";
-	while (*p) {
-		if (*p == '\n') {
-			*p = (char)0;
-			break;
-		}
-		++p;
-	}
-	return ts;
-}
-
-/*
-std::string join(const std::vector<std::string> &elements, const char * const separator)
-{
-	switch(elements.size()) {
-	case 0:
-		return "";
-	case 1:
-		return elements[0];
-	default:
-		std::ostringstream os;
-		std::copy(elements.begin(), elements.end()-1, std::ostream_iterator<std::string>(os, separator));
-		os << *elements.rbegin();
-		return os.str();
-	}
-}
-*/
-
-std::vector<std::string> split(std::string str, char delim)
-{
-	std::istringstream iss(str);
-	std::vector<std::string> tokens;
-	std::string item;
-	while (std::getline(iss, item, delim)) {
-		tokens.push_back(item);
-	}
-	return tokens;
-}
-
-std::string url_encode(const std::string& value)
-{
-	std::ostringstream escaped;
-	escaped.fill('0');
-	escaped << std::hex;
-
-	for (std::string::const_iterator i = value.begin(), n = value.end(); i != n; ++i) {
-		std::string::value_type c = (*i);
-
-		// Keep alphanumeric and other accepted characters intact
-		if (isalnum(c) || c == '-' || c == '_' || c == '.' || c == '~') {
-			escaped << c;
-			continue;
-		}
-
-		// Any other characters are percent-encoded
-		escaped << std::uppercase;
-		escaped << '%' << std::setw(2) << int((unsigned char)c);
-		escaped << std::nouppercase;
-	}
+#include "PostgreSQL.hpp"
 
-	return escaped.str();
-}
+#include <nlohmann/json.hpp>
 
-}	// anonymous namespace
+using namespace nlohmann;
 
 using namespace ZeroTier;
 
-MemberNotificationReceiver::MemberNotificationReceiver(PostgreSQL* p, pqxx::connection& c, const std::string& channel) : pqxx::notification_receiver(c, channel), _psql(p)
+MemberNotificationReceiver::MemberNotificationReceiver(DB* p, pqxx::connection& c, const std::string& channel) : pqxx::notification_receiver(c, channel), _psql(p)
 {
 	fprintf(stderr, "initialize MemberNotificationReceiver\n");
 }
@@ -127,12 +26,12 @@ void MemberNotificationReceiver::operator()(const std::string& payload, int pack
 	if (nv.is_object())
 		newConfig = nv;
 	if (oldConfig.is_object() || newConfig.is_object()) {
-		_psql->_memberChanged(oldConfig, newConfig, (_psql->_ready >= 2));
+		_psql->_memberChanged(oldConfig, newConfig, _psql->isReady());
 		fprintf(stderr, "payload sent\n");
 	}
 }
 
-NetworkNotificationReceiver::NetworkNotificationReceiver(PostgreSQL* p, pqxx::connection& c, const std::string& channel) : pqxx::notification_receiver(c, channel), _psql(p)
+NetworkNotificationReceiver::NetworkNotificationReceiver(DB* p, pqxx::connection& c, const std::string& channel) : pqxx::notification_receiver(c, channel), _psql(p)
 {
 	fprintf(stderr, "initialize NetworkNotificationReceiver\n");
 }
@@ -150,1909 +49,9 @@ void NetworkNotificationReceiver::operator()(const std::string& payload, int pac
 	if (nv.is_object())
 		newConfig = nv;
 	if (oldConfig.is_object() || newConfig.is_object()) {
-		_psql->_networkChanged(oldConfig, newConfig, (_psql->_ready >= 2));
+		_psql->_networkChanged(oldConfig, newConfig, _psql->isReady());
 		fprintf(stderr, "payload sent\n");
 	}
 }
 
-using Attrs = std::vector<std::pair<std::string, std::string> >;
-using Item = std::pair<std::string, Attrs>;
-using ItemStream = std::vector<Item>;
-
-PostgreSQL::PostgreSQL(const Identity& myId, const char* path, int listenPort, RedisConfig* rc)
-	: DB()
-	, _pool()
-	, _myId(myId)
-	, _myAddress(myId.address())
-	, _ready(0)
-	, _connected(1)
-	, _run(1)
-	, _waitNoticePrinted(false)
-	, _listenPort(listenPort)
-	, _rc(rc)
-	, _redis(NULL)
-	, _cluster(NULL)
-	, _redisMemberStatus(false)
-	, _smee(NULL)
-{
-	char myAddress[64];
-	_myAddressStr = myId.address().toString(myAddress);
-	_connString = std::string(path);
-	auto f = std::make_shared<PostgresConnFactory>(_connString);
-	_pool = std::make_shared<ConnectionPool<PostgresConnection> >(15, 5, std::static_pointer_cast<ConnectionFactory>(f));
-
-	memset(_ssoPsk, 0, sizeof(_ssoPsk));
-	char* const ssoPskHex = getenv("ZT_SSO_PSK");
-#ifdef ZT_TRACE
-	fprintf(stderr, "ZT_SSO_PSK: %s\n", ssoPskHex);
-#endif
-	if (ssoPskHex) {
-		// SECURITY: note that ssoPskHex will always be null-terminated if libc actually
-		// returns something non-NULL. If the hex encodes something shorter than 48 bytes,
-		// it will be padded at the end with zeroes. If longer, it'll be truncated.
-		Utils::unhex(ssoPskHex, _ssoPsk, sizeof(_ssoPsk));
-	}
-	const char* redisMemberStatus = getenv("ZT_REDIS_MEMBER_STATUS");
-	if (redisMemberStatus && (strcmp(redisMemberStatus, "true") == 0)) {
-		_redisMemberStatus = true;
-		fprintf(stderr, "Using redis for member status\n");
-	}
-
-	auto c = _pool->borrow();
-	pqxx::work txn { *c->c };
-
-	pqxx::row r { txn.exec1("SELECT version FROM ztc_database") };
-	int dbVersion = r[0].as<int>();
-	txn.commit();
-
-	if (dbVersion < DB_MINIMUM_VERSION) {
-		fprintf(stderr, "Central database schema version too low.  This controller version requires a minimum schema version of %d. Please upgrade your Central instance", DB_MINIMUM_VERSION);
-		exit(1);
-	}
-	_pool->unborrow(c);
-
-	if (_rc != NULL) {
-		sw::redis::ConnectionOptions opts;
-		sw::redis::ConnectionPoolOptions poolOpts;
-		opts.host = _rc->hostname;
-		opts.port = _rc->port;
-		opts.password = _rc->password;
-		opts.db = 0;
-		opts.keep_alive = true;
-		opts.connect_timeout = std::chrono::seconds(3);
-		poolOpts.size = 25;
-		poolOpts.wait_timeout = std::chrono::seconds(5);
-		poolOpts.connection_lifetime = std::chrono::minutes(3);
-		poolOpts.connection_idle_time = std::chrono::minutes(1);
-		if (_rc->clusterMode) {
-			fprintf(stderr, "Using Redis in Cluster Mode\n");
-			_cluster = std::make_shared<sw::redis::RedisCluster>(opts, poolOpts);
-		}
-		else {
-			fprintf(stderr, "Using Redis in Standalone Mode\n");
-			_redis = std::make_shared<sw::redis::Redis>(opts, poolOpts);
-		}
-	}
-
-	_readyLock.lock();
-
-	fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL waiting for initial data download..." ZT_EOL_S, ::_timestr(), (unsigned long long)_myAddress.toInt());
-	_waitNoticePrinted = true;
-
-	initializeNetworks();
-	initializeMembers();
-
-	_heartbeatThread = std::thread(&PostgreSQL::heartbeat, this);
-	_membersDbWatcher = std::thread(&PostgreSQL::membersDbWatcher, this);
-	_networksDbWatcher = std::thread(&PostgreSQL::networksDbWatcher, this);
-	for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) {
-		_commitThread[i] = std::thread(&PostgreSQL::commitThread, this);
-	}
-	_onlineNotificationThread = std::thread(&PostgreSQL::onlineNotificationThread, this);
-
-	configureSmee();
-}
-
-PostgreSQL::~PostgreSQL()
-{
-	if (_smee != NULL) {
-		smeeclient::smee_client_delete(_smee);
-		_smee = NULL;
-	}
-
-	_run = 0;
-	std::this_thread::sleep_for(std::chrono::milliseconds(100));
-
-	_heartbeatThread.join();
-	_membersDbWatcher.join();
-	_networksDbWatcher.join();
-	_commitQueue.stop();
-	for (int i = 0; i < ZT_CENTRAL_CONTROLLER_COMMIT_THREADS; ++i) {
-		_commitThread[i].join();
-	}
-	_onlineNotificationThread.join();
-}
-
-void PostgreSQL::configureSmee()
-{
-	const char* TEMPORAL_SCHEME = "ZT_TEMPORAL_SCHEME";
-	const char* TEMPORAL_HOST = "ZT_TEMPORAL_HOST";
-	const char* TEMPORAL_PORT = "ZT_TEMPORAL_PORT";
-	const char* TEMPORAL_NAMESPACE = "ZT_TEMPORAL_NAMESPACE";
-	const char* SMEE_TASK_QUEUE = "ZT_SMEE_TASK_QUEUE";
-
-	const char* scheme = getenv(TEMPORAL_SCHEME);
-	if (scheme == NULL) {
-		scheme = "http";
-	}
-	const char* host = getenv(TEMPORAL_HOST);
-	const char* port = getenv(TEMPORAL_PORT);
-	const char* ns = getenv(TEMPORAL_NAMESPACE);
-	const char* task_queue = getenv(SMEE_TASK_QUEUE);
-
-	if (scheme != NULL && host != NULL && port != NULL && ns != NULL && task_queue != NULL) {
-		fprintf(stderr, "creating smee client\n");
-		std::string hostPort = std::string(scheme) + std::string("://") + std::string(host) + std::string(":") + std::string(port);
-		this->_smee = smeeclient::smee_client_new(hostPort.c_str(), ns, task_queue);
-	}
-	else {
-		fprintf(stderr, "Smee client not configured\n");
-	}
-}
-
-bool PostgreSQL::waitForReady()
-{
-	while (_ready < 2) {
-		_readyLock.lock();
-		_readyLock.unlock();
-	}
-	return true;
-}
-
-bool PostgreSQL::isReady()
-{
-	return ((_ready == 2) && (_connected));
-}
-
-bool PostgreSQL::save(nlohmann::json& record, bool notifyListeners)
-{
-	bool modified = false;
-	try {
-		if (! record.is_object()) {
-			fprintf(stderr, "record is not an object?!?\n");
-			return false;
-		}
-		const std::string objtype = record["objtype"];
-		if (objtype == "network") {
-			// fprintf(stderr, "network save\n");
-			const uint64_t nwid = OSUtils::jsonIntHex(record["id"], 0ULL);
-			if (nwid) {
-				nlohmann::json old;
-				get(nwid, old);
-				if ((! old.is_object()) || (! _compareRecords(old, record))) {
-					record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL;
-					_commitQueue.post(std::pair<nlohmann::json, bool>(record, notifyListeners));
-					modified = true;
-				}
-			}
-		}
-		else if (objtype == "member") {
-			std::string networkId = record["nwid"];
-			std::string memberId = record["id"];
-			const uint64_t nwid = OSUtils::jsonIntHex(record["nwid"], 0ULL);
-			const uint64_t id = OSUtils::jsonIntHex(record["id"], 0ULL);
-			// fprintf(stderr, "member save %s-%s\n", networkId.c_str(), memberId.c_str());
-			if ((id) && (nwid)) {
-				nlohmann::json network, old;
-				get(nwid, network, id, old);
-				if ((! old.is_object()) || (! _compareRecords(old, record))) {
-					// fprintf(stderr, "commit queue post\n");
-					record["revision"] = OSUtils::jsonInt(record["revision"], 0ULL) + 1ULL;
-					_commitQueue.post(std::pair<nlohmann::json, bool>(record, notifyListeners));
-					modified = true;
-				}
-				else {
-					// fprintf(stderr, "no change\n");
-				}
-			}
-		}
-		else {
-			fprintf(stderr, "uhh waaat\n");
-		}
-	}
-	catch (std::exception& e) {
-		fprintf(stderr, "Error on PostgreSQL::save: %s\n", e.what());
-	}
-	catch (...) {
-		fprintf(stderr, "Unknown error on PostgreSQL::save\n");
-	}
-	return modified;
-}
-
-void PostgreSQL::eraseNetwork(const uint64_t networkId)
-{
-	fprintf(stderr, "PostgreSQL::eraseNetwork\n");
-	char tmp2[24];
-	waitForReady();
-	Utils::hex(networkId, tmp2);
-	std::pair<nlohmann::json, bool> tmp;
-	tmp.first["id"] = tmp2;
-	tmp.first["objtype"] = "_delete_network";
-	tmp.second = true;
-	_commitQueue.post(tmp);
-	nlohmann::json nullJson;
-	_networkChanged(tmp.first, nullJson, true);
-}
-
-void PostgreSQL::eraseMember(const uint64_t networkId, const uint64_t memberId)
-{
-	fprintf(stderr, "PostgreSQL::eraseMember\n");
-	char tmp2[24];
-	waitForReady();
-	std::pair<nlohmann::json, bool> tmp, nw;
-	Utils::hex(networkId, tmp2);
-	tmp.first["nwid"] = tmp2;
-	Utils::hex(memberId, tmp2);
-	tmp.first["id"] = tmp2;
-	tmp.first["objtype"] = "_delete_member";
-	tmp.second = true;
-	_commitQueue.post(tmp);
-	nlohmann::json nullJson;
-	_memberChanged(tmp.first, nullJson, true);
-}
-
-void PostgreSQL::nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress)
-{
-	std::lock_guard<std::mutex> l(_lastOnline_l);
-	std::pair<int64_t, InetAddress>& i = _lastOnline[std::pair<uint64_t, uint64_t>(networkId, memberId)];
-	i.first = OSUtils::now();
-	if (physicalAddress) {
-		i.second = physicalAddress;
-	}
-}
-
-AuthInfo PostgreSQL::getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL)
-{
-	Metrics::db_get_sso_info++;
-	// NONCE is just a random character string.  no semantic meaning
-	// state = HMAC SHA384 of Nonce based on shared sso key
-	//
-	// need nonce timeout in database? make sure it's used within X time
-	// X is 5 minutes for now.  Make configurable later?
-	//
-	// how do we tell when a nonce is used? if auth_expiration_time is set
-	std::string networkId = member["nwid"];
-	std::string memberId = member["id"];
-
-	char authenticationURL[4096] = { 0 };
-	AuthInfo info;
-	info.enabled = true;
-
-	// if (memberId == "a10dccea52" && networkId == "8056c2e21c24673d") {
-	//	fprintf(stderr, "invalid authinfo for grant's machine\n");
-	//	info.version=1;
-	//	return info;
-	// }
-	//  fprintf(stderr, "PostgreSQL::updateMemberOnLoad: %s-%s\n", networkId.c_str(), memberId.c_str());
-	std::shared_ptr<PostgresConnection> c;
-	try {
-		c = _pool->borrow();
-		pqxx::work w(*c->c);
-
-		char nonceBytes[16] = { 0 };
-		std::string nonce = "";
-
-		// check if the member exists first.
-		pqxx::row count = w.exec_params1("SELECT count(id) FROM ztc_member WHERE id = $1 AND network_id = $2 AND deleted = false", memberId, networkId);
-		if (count[0].as<int>() == 1) {
-			// get active nonce, if exists.
-			pqxx::result r = w.exec_params(
-				"SELECT nonce FROM ztc_sso_expiry "
-				"WHERE network_id = $1 AND member_id = $2 "
-				"AND ((NOW() AT TIME ZONE 'UTC') <= authentication_expiry_time) AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)",
-				networkId,
-				memberId);
-
-			if (r.size() == 0) {
-				// no active nonce.
-				// find an unused nonce, if one exists.
-				pqxx::result r = w.exec_params(
-					"SELECT nonce FROM ztc_sso_expiry "
-					"WHERE network_id = $1 AND member_id = $2 "
-					"AND authentication_expiry_time IS NULL AND ((NOW() AT TIME ZONE 'UTC') <= nonce_expiration)",
-					networkId,
-					memberId);
-
-				if (r.size() == 1) {
-					// we have an existing nonce.  Use it
-					nonce = r.at(0)[0].as<std::string>();
-					Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes));
-				}
-				else if (r.empty()) {
-					// create a nonce
-					Utils::getSecureRandom(nonceBytes, 16);
-					char nonceBuf[64] = { 0 };
-					Utils::hex(nonceBytes, sizeof(nonceBytes), nonceBuf);
-					nonce = std::string(nonceBuf);
-
-					pqxx::result ir = w.exec_params0(
-						"INSERT INTO ztc_sso_expiry "
-						"(nonce, nonce_expiration, network_id, member_id) VALUES "
-						"($1, TO_TIMESTAMP($2::double precision/1000), $3, $4)",
-						nonce,
-						OSUtils::now() + 300000,
-						networkId,
-						memberId);
-
-					w.commit();
-				}
-				else {
-					// > 1 ?!?  Thats an error!
-					fprintf(stderr, "> 1 unused nonce!\n");
-					exit(6);
-				}
-			}
-			else if (r.size() == 1) {
-				nonce = r.at(0)[0].as<std::string>();
-				Utils::unhex(nonce.c_str(), nonceBytes, sizeof(nonceBytes));
-			}
-			else {
-				// more than 1 nonce in use?  Uhhh...
-				fprintf(stderr, "> 1 nonce in use for network member?!?\n");
-				exit(7);
-			}
-
-			r = w.exec_params(
-				"SELECT oc.client_id, oc.authorization_endpoint, oc.issuer, oc.provider, oc.sso_impl_version "
-				"FROM ztc_network AS n "
-				"INNER JOIN ztc_org o "
-				"  ON o.owner_id = n.owner_id "
-				"LEFT OUTER JOIN ztc_network_oidc_config noc "
-				"  ON noc.network_id = n.id "
-				"LEFT OUTER JOIN ztc_oidc_config oc "
-				"  ON noc.client_id = oc.client_id AND oc.org_id = o.org_id "
-				"WHERE n.id = $1 AND n.sso_enabled = true",
-				networkId);
-
-			std::string client_id = "";
-			std::string authorization_endpoint = "";
-			std::string issuer = "";
-			std::string provider = "";
-			uint64_t sso_version = 0;
-
-			if (r.size() == 1) {
-				client_id = r.at(0)[0].as<std::optional<std::string> >().value_or("");
-				authorization_endpoint = r.at(0)[1].as<std::optional<std::string> >().value_or("");
-				issuer = r.at(0)[2].as<std::optional<std::string> >().value_or("");
-				provider = r.at(0)[3].as<std::optional<std::string> >().value_or("");
-				sso_version = r.at(0)[4].as<std::optional<uint64_t> >().value_or(1);
-			}
-			else if (r.size() > 1) {
-				fprintf(stderr, "ERROR: More than one auth endpoint for an organization?!?!? NetworkID: %s\n", networkId.c_str());
-			}
-			else {
-				fprintf(stderr, "No client or auth endpoint?!?\n");
-			}
-
-			info.version = sso_version;
-
-			// no catch all else because we don't actually care if no records exist here. just continue as normal.
-			if ((! client_id.empty()) && (! authorization_endpoint.empty())) {
-				uint8_t state[48];
-				HMACSHA384(_ssoPsk, nonceBytes, sizeof(nonceBytes), state);
-				char state_hex[256];
-				Utils::hex(state, 48, state_hex);
-
-				if (info.version == 0) {
-					char url[2048] = { 0 };
-					OSUtils::ztsnprintf(
-						url,
-						sizeof(authenticationURL),
-						"%s?response_type=id_token&response_mode=form_post&scope=openid+email+profile&redirect_uri=%s&nonce=%s&state=%s&client_id=%s",
-						authorization_endpoint.c_str(),
-						url_encode(redirectURL).c_str(),
-						nonce.c_str(),
-						state_hex,
-						client_id.c_str());
-					info.authenticationURL = std::string(url);
-				}
-				else if (info.version == 1) {
-					info.ssoClientID = client_id;
-					info.issuerURL = issuer;
-					info.ssoProvider = provider;
-					info.ssoNonce = nonce;
-					info.ssoState = std::string(state_hex) + "_" + networkId;
-					info.centralAuthURL = redirectURL;
-#ifdef ZT_DEBUG
-					fprintf(
-						stderr,
-						"ssoClientID: %s\nissuerURL: %s\nssoNonce: %s\nssoState: %s\ncentralAuthURL: %s\nprovider: %s\n",
-						info.ssoClientID.c_str(),
-						info.issuerURL.c_str(),
-						info.ssoNonce.c_str(),
-						info.ssoState.c_str(),
-						info.centralAuthURL.c_str(),
-						provider.c_str());
-#endif
-				}
-			}
-			else {
-				fprintf(stderr, "client_id: %s\nauthorization_endpoint: %s\n", client_id.c_str(), authorization_endpoint.c_str());
-			}
-		}
-
-		_pool->unborrow(c);
-	}
-	catch (std::exception& e) {
-		fprintf(stderr, "ERROR: Error updating member on load for network %s: %s\n", networkId.c_str(), e.what());
-	}
-
-	return info;   // std::string(authenticationURL);
-}
-
-void PostgreSQL::initializeNetworks()
-{
-	try {
-		std::string setKey = "networks:{" + _myAddressStr + "}";
-
-		fprintf(stderr, "Initializing Networks...\n");
-
-		if (_redisMemberStatus) {
-			fprintf(stderr, "Init Redis for networks...\n");
-			try {
-				if (_rc->clusterMode) {
-					_cluster->del(setKey);
-				}
-				else {
-					_redis->del(setKey);
-				}
-			}
-			catch (sw::redis::Error& e) {
-				// ignore. if this key doesn't exist, there's no reason to delete it
-			}
-		}
-
-		std::unordered_set<std::string> networkSet;
-
-		char qbuf[2048] = { 0 };
-		sprintf(
-			qbuf,
-			"SELECT n.id, (EXTRACT(EPOCH FROM n.creation_time AT TIME ZONE 'UTC')*1000)::bigint as creation_time, n.capabilities, "
-			"n.enable_broadcast, (EXTRACT(EPOCH FROM n.last_modified AT TIME ZONE 'UTC')*1000)::bigint AS last_modified, n.mtu, n.multicast_limit, n.name, n.private, n.remote_trace_level, "
-			"n.remote_trace_target, n.revision, n.rules, n.tags, n.v4_assign_mode, n.v6_assign_mode, n.sso_enabled, (CASE WHEN n.sso_enabled THEN noc.client_id ELSE NULL END) as client_id, "
-			"(CASE WHEN n.sso_enabled THEN oc.authorization_endpoint ELSE NULL END) as authorization_endpoint, "
-			"(CASE WHEN n.sso_enabled THEN oc.provider ELSE NULL END) as provider, d.domain, d.servers, "
-			"ARRAY(SELECT CONCAT(host(ip_range_start),'|', host(ip_range_end)) FROM ztc_network_assignment_pool WHERE network_id = n.id) AS assignment_pool, "
-			"ARRAY(SELECT CONCAT(host(address),'/',bits::text,'|',COALESCE(host(via), 'NULL'))FROM ztc_network_route WHERE network_id = n.id) AS routes "
-			"FROM ztc_network n "
-			"LEFT OUTER JOIN ztc_org o "
-			" ON o.owner_id = n.owner_id "
-			"LEFT OUTER JOIN ztc_network_oidc_config noc "
-			"	ON noc.network_id = n.id "
-			"LEFT OUTER JOIN ztc_oidc_config oc "
-			"	ON noc.client_id = oc.client_id AND oc.org_id = o.org_id "
-			"LEFT OUTER JOIN ztc_network_dns d "
-			"	ON d.network_id = n.id "
-			"WHERE deleted = false AND controller_id = '%s'",
-			_myAddressStr.c_str());
-		auto c = _pool->borrow();
-		auto c2 = _pool->borrow();
-		pqxx::work w { *c->c };
-
-		fprintf(stderr, "Load networks from psql...\n");
-		auto stream = pqxx::stream_from::query(w, qbuf);
-
-		std::tuple<
-			std::string	  // network ID
-			,
-			std::optional<int64_t>	 // creationTime
-			,
-			std::optional<std::string>	 // capabilities
-			,
-			std::optional<bool>	  // enableBroadcast
-			,
-			std::optional<uint64_t>	  // lastModified
-			,
-			std::optional<int>	 // mtu
-			,
-			std::optional<int>	 // multicastLimit
-			,
-			std::optional<std::string>	 // name
-			,
-			bool   // private
-			,
-			std::optional<int>	 // remoteTraceLevel
-			,
-			std::optional<std::string>	 // remoteTraceTarget
-			,
-			std::optional<uint64_t>	  // revision
-			,
-			std::optional<std::string>	 // rules
-			,
-			std::optional<std::string>	 // tags
-			,
-			std::optional<std::string>	 // v4AssignMode
-			,
-			std::optional<std::string>	 // v6AssignMode
-			,
-			std::optional<bool>	  // ssoEnabled
-			,
-			std::optional<std::string>	 // clientId
-			,
-			std::optional<std::string>	 // authorizationEndpoint
-			,
-			std::optional<std::string>	 // ssoProvider
-			,
-			std::optional<std::string>	 // domain
-			,
-			std::optional<std::string>	 // servers
-			,
-			std::string	  // assignmentPoolString
-			,
-			std::string	  // routeString
-			>
-			row;
-
-		uint64_t count = 0;
-		auto tmp = std::chrono::high_resolution_clock::now();
-		uint64_t total = 0;
-		while (stream >> row) {
-			auto start = std::chrono::high_resolution_clock::now();
-
-			json empty;
-			json config;
-
-			initNetwork(config);
-
-			std::string nwid = std::get<0>(row);
-			std::optional<int64_t> creationTime = std::get<1>(row);
-			std::optional<std::string> capabilities = std::get<2>(row);
-			std::optional<bool> enableBroadcast = std::get<3>(row);
-			std::optional<uint64_t> lastModified = std::get<4>(row);
-			std::optional<int> mtu = std::get<5>(row);
-			std::optional<int> multicastLimit = std::get<6>(row);
-			std::optional<std::string> name = std::get<7>(row);
-			bool isPrivate = std::get<8>(row);
-			std::optional<int> remoteTraceLevel = std::get<9>(row);
-			std::optional<std::string> remoteTraceTarget = std::get<10>(row);
-			std::optional<uint64_t> revision = std::get<11>(row);
-			std::optional<std::string> rules = std::get<12>(row);
-			std::optional<std::string> tags = std::get<13>(row);
-			std::optional<std::string> v4AssignMode = std::get<14>(row);
-			std::optional<std::string> v6AssignMode = std::get<15>(row);
-			std::optional<bool> ssoEnabled = std::get<16>(row);
-			std::optional<std::string> clientId = std::get<17>(row);
-			std::optional<std::string> authorizationEndpoint = std::get<18>(row);
-			std::optional<std::string> ssoProvider = std::get<19>(row);
-			std::optional<std::string> dnsDomain = std::get<20>(row);
-			std::optional<std::string> dnsServers = std::get<21>(row);
-			std::string assignmentPoolString = std::get<22>(row);
-			std::string routesString = std::get<23>(row);
-
-			config["id"] = nwid;
-			config["nwid"] = nwid;
-			config["creationTime"] = creationTime.value_or(0);
-			config["capabilities"] = json::parse(capabilities.value_or("[]"));
-			config["enableBroadcast"] = enableBroadcast.value_or(false);
-			config["lastModified"] = lastModified.value_or(0);
-			config["mtu"] = mtu.value_or(2800);
-			config["multicastLimit"] = multicastLimit.value_or(64);
-			config["name"] = name.value_or("");
-			config["private"] = isPrivate;
-			config["remoteTraceLevel"] = remoteTraceLevel.value_or(0);
-			config["remoteTraceTarget"] = remoteTraceTarget.value_or("");
-			config["revision"] = revision.value_or(0);
-			config["rules"] = json::parse(rules.value_or("[]"));
-			config["tags"] = json::parse(tags.value_or("[]"));
-			config["v4AssignMode"] = json::parse(v4AssignMode.value_or("{}"));
-			config["v6AssignMode"] = json::parse(v6AssignMode.value_or("{}"));
-			config["ssoEnabled"] = ssoEnabled.value_or(false);
-			config["objtype"] = "network";
-			config["ipAssignmentPools"] = json::array();
-			config["routes"] = json::array();
-			config["clientId"] = clientId.value_or("");
-			config["authorizationEndpoint"] = authorizationEndpoint.value_or("");
-			config["provider"] = ssoProvider.value_or("");
-
-			networkSet.insert(nwid);
-
-			if (dnsDomain.has_value()) {
-				std::string serverList = dnsServers.value();
-				json obj;
-				auto servers = json::array();
-				if (serverList.rfind("{", 0) != std::string::npos) {
-					serverList = serverList.substr(1, serverList.size() - 2);
-					std::stringstream ss(serverList);
-					while (ss.good()) {
-						std::string server;
-						std::getline(ss, server, ',');
-						servers.push_back(server);
-					}
-				}
-				obj["domain"] = dnsDomain.value();
-				obj["servers"] = servers;
-				config["dns"] = obj;
-			}
-
-			config["ipAssignmentPools"] = json::array();
-			if (assignmentPoolString != "{}") {
-				std::string tmp = assignmentPoolString.substr(1, assignmentPoolString.size() - 2);
-				std::vector<std::string> assignmentPools = split(tmp, ',');
-				for (auto it = assignmentPools.begin(); it != assignmentPools.end(); ++it) {
-					std::vector<std::string> r = split(*it, '|');
-					json ip;
-					ip["ipRangeStart"] = r[0];
-					ip["ipRangeEnd"] = r[1];
-					config["ipAssignmentPools"].push_back(ip);
-				}
-			}
-
-			config["routes"] = json::array();
-			if (routesString != "{}") {
-				std::string tmp = routesString.substr(1, routesString.size() - 2);
-				std::vector<std::string> routes = split(tmp, ',');
-				for (auto it = routes.begin(); it != routes.end(); ++it) {
-					std::vector<std::string> r = split(*it, '|');
-					json route;
-					route["target"] = r[0];
-					route["via"] = ((route["via"] == "NULL") ? nullptr : r[1]);
-					config["routes"].push_back(route);
-				}
-			}
-
-			Metrics::network_count++;
-
-			_networkChanged(empty, config, false);
-
-			auto end = std::chrono::high_resolution_clock::now();
-			auto dur = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
-			;
-			total += dur.count();
-			++count;
-			if (count > 0 && count % 10000 == 0) {
-				fprintf(stderr, "Averaging %llu us per network\n", (total / count));
-			}
-		}
-
-		if (count > 0) {
-			fprintf(stderr, "Took %llu us per network to load\n", (total / count));
-		}
-		stream.complete();
-
-		w.commit();
-		_pool->unborrow(c2);
-		_pool->unborrow(c);
-		fprintf(stderr, "done.\n");
-
-		if (! networkSet.empty()) {
-			if (_redisMemberStatus) {
-				fprintf(stderr, "adding networks to redis...\n");
-				if (_rc->clusterMode) {
-					auto tx = _cluster->transaction(_myAddressStr, true, false);
-					uint64_t count = 0;
-					for (std::string nwid : networkSet) {
-						tx.sadd(setKey, nwid);
-						if (++count % 30000 == 0) {
-							tx.exec();
-							tx = _cluster->transaction(_myAddressStr, true, false);
-						}
-					}
-					tx.exec();
-				}
-				else {
-					auto tx = _redis->transaction(true, false);
-					uint64_t count = 0;
-					for (std::string nwid : networkSet) {
-						tx.sadd(setKey, nwid);
-						if (++count % 30000 == 0) {
-							tx.exec();
-							tx = _redis->transaction(true, false);
-						}
-					}
-					tx.exec();
-				}
-				fprintf(stderr, "done.\n");
-			}
-		}
-
-		if (++this->_ready == 2) {
-			if (_waitNoticePrinted) {
-				fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt());
-			}
-			_readyLock.unlock();
-		}
-		fprintf(stderr, "network init done.\n");
-	}
-	catch (sw::redis::Error& e) {
-		fprintf(stderr, "ERROR: Error initializing networks in Redis: %s\n", e.what());
-		std::this_thread::sleep_for(std::chrono::milliseconds(5000));
-		exit(-1);
-	}
-	catch (std::exception& e) {
-		fprintf(stderr, "ERROR: Error initializing networks: %s\n", e.what());
-		std::this_thread::sleep_for(std::chrono::milliseconds(5000));
-		exit(-1);
-	}
-}
-
-void PostgreSQL::initializeMembers()
-{
-	std::string memberId;
-	std::string networkId;
-	try {
-		std::unordered_map<std::string, std::string> networkMembers;
-		fprintf(stderr, "Initializing Members...\n");
-
-		std::string setKeyBase = "network-nodes-all:{" + _myAddressStr + "}:";
-
-		if (_redisMemberStatus) {
-			fprintf(stderr, "Initialize Redis for members...\n");
-			std::unique_lock<std::shared_mutex> l(_networks_l);
-			std::unordered_set<std::string> deletes;
-			for (auto it : _networks) {
-				uint64_t nwid_i = it.first;
-				char nwidTmp[64] = { 0 };
-				OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
-				std::string nwid(nwidTmp);
-				std::string key = setKeyBase + nwid;
-				deletes.insert(key);
-			}
-
-			if (! deletes.empty()) {
-				try {
-					if (_rc->clusterMode) {
-						auto tx = _cluster->transaction(_myAddressStr, true, false);
-						for (std::string k : deletes) {
-							tx.del(k);
-						}
-						tx.exec();
-					}
-					else {
-						auto tx = _redis->transaction(true, false);
-						for (std::string k : deletes) {
-							tx.del(k);
-						}
-						tx.exec();
-					}
-				}
-				catch (sw::redis::Error& e) {
-					// ignore
-				}
-			}
-		}
-
-		char qbuf[2048];
-		sprintf(
-			qbuf,
-			"SELECT m.id, m.network_id, m.active_bridge, m.authorized, m.capabilities, "
-			"(EXTRACT(EPOCH FROM m.creation_time AT TIME ZONE 'UTC')*1000)::bigint, m.identity, "
-			"(EXTRACT(EPOCH FROM m.last_authorized_time AT TIME ZONE 'UTC')*1000)::bigint, "
-			"(EXTRACT(EPOCH FROM m.last_deauthorized_time AT TIME ZONE 'UTC')*1000)::bigint, "
-			"m.remote_trace_level, m.remote_trace_target, m.tags, m.v_major, m.v_minor, m.v_rev, m.v_proto, "
-			"m.no_auto_assign_ips, m.revision, m.sso_exempt, "
-			"(CASE WHEN n.sso_enabled = TRUE AND m.sso_exempt = FALSE THEN "
-			" ( "
-			"	SELECT (EXTRACT(EPOCH FROM e.authentication_expiry_time)*1000)::bigint "
-			"	FROM ztc_sso_expiry e "
-			"	INNER JOIN ztc_network n1 "
-			"	ON n1.id = e.network_id  AND n1.deleted = TRUE "
-			"	WHERE e.network_id = m.network_id AND e.member_id = m.id AND n.sso_enabled = TRUE AND e.authentication_expiry_time IS NOT NULL "
-			"	ORDER BY e.authentication_expiry_time DESC LIMIT 1 "
-			" ) "
-			" ELSE NULL "
-			" END) AS authentication_expiry_time, "
-			"ARRAY(SELECT DISTINCT address FROM ztc_member_ip_assignment WHERE member_id = m.id AND network_id = m.network_id) AS assigned_addresses "
-			"FROM ztc_member m "
-			"INNER JOIN ztc_network n "
-			"	ON n.id = m.network_id "
-			"WHERE n.controller_id = '%s' AND n.deleted = FALSE AND m.deleted = FALSE",
-			_myAddressStr.c_str());
-		auto c = _pool->borrow();
-		auto c2 = _pool->borrow();
-		pqxx::work w { *c->c };
-
-		fprintf(stderr, "Load members from psql...\n");
-		auto stream = pqxx::stream_from::query(w, qbuf);
-
-		std::tuple<
-			std::string	  // memberId
-			,
-			std::string	  // memberId
-			,
-			std::optional<bool>	  // activeBridge
-			,
-			std::optional<bool>	  // authorized
-			,
-			std::optional<std::string>	 // capabilities
-			,
-			std::optional<uint64_t>	  // creationTime
-			,
-			std::optional<std::string>	 // identity
-			,
-			std::optional<uint64_t>	  // lastAuthorizedTime
-			,
-			std::optional<uint64_t>	  // lastDeauthorizedTime
-			,
-			std::optional<int>	 // remoteTraceLevel
-			,
-			std::optional<std::string>	 // remoteTraceTarget
-			,
-			std::optional<std::string>	 // tags
-			,
-			std::optional<int>	 // vMajor
-			,
-			std::optional<int>	 // vMinor
-			,
-			std::optional<int>	 // vRev
-			,
-			std::optional<int>	 // vProto
-			,
-			std::optional<bool>	  // noAutoAssignIps
-			,
-			std::optional<uint64_t>	  // revision
-			,
-			std::optional<bool>	  // ssoExempt
-			,
-			std::optional<uint64_t>	  // authenticationExpiryTime
-			,
-			std::string	  // assignedAddresses
-			>
-			row;
-
-		uint64_t count = 0;
-		auto tmp = std::chrono::high_resolution_clock::now();
-		uint64_t total = 0;
-		while (stream >> row) {
-			auto start = std::chrono::high_resolution_clock::now();
-			json empty;
-			json config;
-
-			initMember(config);
-
-			memberId = std::get<0>(row);
-			networkId = std::get<1>(row);
-			std::optional<bool> activeBridge = std::get<2>(row);
-			std::optional<bool> authorized = std::get<3>(row);
-			std::optional<std::string> capabilities = std::get<4>(row);
-			std::optional<uint64_t> creationTime = std::get<5>(row);
-			std::optional<std::string> identity = std::get<6>(row);
-			std::optional<uint64_t> lastAuthorizedTime = std::get<7>(row);
-			std::optional<uint64_t> lastDeauthorizedTime = std::get<8>(row);
-			std::optional<int> remoteTraceLevel = std::get<9>(row);
-			std::optional<std::string> remoteTraceTarget = std::get<10>(row);
-			std::optional<std::string> tags = std::get<11>(row);
-			std::optional<int> vMajor = std::get<12>(row);
-			std::optional<int> vMinor = std::get<13>(row);
-			std::optional<int> vRev = std::get<14>(row);
-			std::optional<int> vProto = std::get<15>(row);
-			std::optional<bool> noAutoAssignIps = std::get<16>(row);
-			std::optional<uint64_t> revision = std::get<17>(row);
-			std::optional<bool> ssoExempt = std::get<18>(row);
-			std::optional<uint64_t> authenticationExpiryTime = std::get<19>(row);
-			std::string assignedAddresses = std::get<20>(row);
-
-			networkMembers.insert(std::pair<std::string, std::string>(setKeyBase + networkId, memberId));
-
-			config["id"] = memberId;
-			config["address"] = memberId;
-			config["nwid"] = networkId;
-			config["activeBridge"] = activeBridge.value_or(false);
-			config["authorized"] = authorized.value_or(false);
-			config["capabilities"] = json::parse(capabilities.value_or("[]"));
-			config["creationTime"] = creationTime.value_or(0);
-			config["identity"] = identity.value_or("");
-			config["lastAuthorizedTime"] = lastAuthorizedTime.value_or(0);
-			config["lastDeauthorizedTime"] = lastDeauthorizedTime.value_or(0);
-			config["remoteTraceLevel"] = remoteTraceLevel.value_or(0);
-			config["remoteTraceTarget"] = remoteTraceTarget.value_or("");
-			config["tags"] = json::parse(tags.value_or("[]"));
-			config["vMajor"] = vMajor.value_or(-1);
-			config["vMinor"] = vMinor.value_or(-1);
-			config["vRev"] = vRev.value_or(-1);
-			config["vProto"] = vProto.value_or(-1);
-			config["noAutoAssignIps"] = noAutoAssignIps.value_or(false);
-			config["revision"] = revision.value_or(0);
-			config["ssoExempt"] = ssoExempt.value_or(false);
-			config["authenticationExpiryTime"] = authenticationExpiryTime.value_or(0);
-			config["objtype"] = "member";
-			config["ipAssignments"] = json::array();
-
-			if (assignedAddresses != "{}") {
-				std::string tmp = assignedAddresses.substr(1, assignedAddresses.size() - 2);
-				std::vector<std::string> addrs = split(tmp, ',');
-				for (auto it = addrs.begin(); it != addrs.end(); ++it) {
-					config["ipAssignments"].push_back(*it);
-				}
-			}
-
-			Metrics::member_count++;
-
-			_memberChanged(empty, config, false);
-
-			memberId = "";
-			networkId = "";
-
-			auto end = std::chrono::high_resolution_clock::now();
-			auto dur = std::chrono::duration_cast<std::chrono::microseconds>(end - start);
-			total += dur.count();
-			++count;
-			if (count > 0 && count % 10000 == 0) {
-				fprintf(stderr, "Averaging %llu us per member\n", (total / count));
-			}
-		}
-		if (count > 0) {
-			fprintf(stderr, "Took %llu us per member to load\n", (total / count));
-		}
-
-		stream.complete();
-
-		w.commit();
-		_pool->unborrow(c2);
-		_pool->unborrow(c);
-		fprintf(stderr, "done.\n");
-
-		if (! networkMembers.empty()) {
-			if (_redisMemberStatus) {
-				fprintf(stderr, "Load member data into redis...\n");
-				if (_rc->clusterMode) {
-					auto tx = _cluster->transaction(_myAddressStr, true, false);
-					uint64_t count = 0;
-					for (auto it : networkMembers) {
-						tx.sadd(it.first, it.second);
-						if (++count % 30000 == 0) {
-							tx.exec();
-							tx = _cluster->transaction(_myAddressStr, true, false);
-						}
-					}
-					tx.exec();
-				}
-				else {
-					auto tx = _redis->transaction(true, false);
-					uint64_t count = 0;
-					for (auto it : networkMembers) {
-						tx.sadd(it.first, it.second);
-						if (++count % 30000 == 0) {
-							tx.exec();
-							tx = _redis->transaction(true, false);
-						}
-					}
-					tx.exec();
-				}
-				fprintf(stderr, "done.\n");
-			}
-		}
-
-		fprintf(stderr, "Done loading members...\n");
-
-		if (++this->_ready == 2) {
-			if (_waitNoticePrinted) {
-				fprintf(stderr, "[%s] NOTICE: %.10llx controller PostgreSQL data download complete." ZT_EOL_S, _timestr(), (unsigned long long)_myAddress.toInt());
-			}
-			_readyLock.unlock();
-		}
-	}
-	catch (sw::redis::Error& e) {
-		fprintf(stderr, "ERROR: Error initializing members (redis): %s\n", e.what());
-		exit(-1);
-	}
-	catch (std::exception& e) {
-		fprintf(stderr, "ERROR: Error initializing member: %s-%s %s\n", networkId.c_str(), memberId.c_str(), e.what());
-		exit(-1);
-	}
-}
-
-void PostgreSQL::heartbeat()
-{
-	char publicId[1024];
-	char hostnameTmp[1024];
-	_myId.toString(false, publicId);
-	if (gethostname(hostnameTmp, sizeof(hostnameTmp)) != 0) {
-		hostnameTmp[0] = (char)0;
-	}
-	else {
-		for (int i = 0; i < (int)sizeof(hostnameTmp); ++i) {
-			if ((hostnameTmp[i] == '.') || (hostnameTmp[i] == 0)) {
-				hostnameTmp[i] = (char)0;
-				break;
-			}
-		}
-	}
-	const char* controllerId = _myAddressStr.c_str();
-	const char* publicIdentity = publicId;
-	const char* hostname = hostnameTmp;
-
-	while (_run == 1) {
-		// fprintf(stderr, "%s: heartbeat\n", controllerId);
-		auto c = _pool->borrow();
-		int64_t ts = OSUtils::now();
-
-		if (c->c) {
-			std::string major = std::to_string(ZEROTIER_ONE_VERSION_MAJOR);
-			std::string minor = std::to_string(ZEROTIER_ONE_VERSION_MINOR);
-			std::string rev = std::to_string(ZEROTIER_ONE_VERSION_REVISION);
-			std::string build = std::to_string(ZEROTIER_ONE_VERSION_BUILD);
-			std::string now = std::to_string(ts);
-			std::string host_port = std::to_string(_listenPort);
-			std::string use_redis = (_rc != NULL) ? "true" : "false";
-			std::string redis_mem_status = (_redisMemberStatus) ? "true" : "false";
-
-			try {
-				pqxx::work w { *c->c };
-
-				pqxx::result res = w.exec0(
-					"INSERT INTO ztc_controller (id, cluster_host, last_alive, public_identity, v_major, v_minor, v_rev, v_build, host_port, use_redis, redis_member_status) "
-					"VALUES ("
-					+ w.quote(controllerId) + ", " + w.quote(hostname) + ", TO_TIMESTAMP(" + now + "::double precision/1000), " + w.quote(publicIdentity) + ", " + major + ", " + minor + ", " + rev + ", " + build + ", " + host_port + ", "
-					+ use_redis + ", " + redis_mem_status
-					+ ") "
-					  "ON CONFLICT (id) DO UPDATE SET cluster_host = EXCLUDED.cluster_host, last_alive = EXCLUDED.last_alive, "
-					  "public_identity = EXCLUDED.public_identity, v_major = EXCLUDED.v_major, v_minor = EXCLUDED.v_minor, "
-					  "v_rev = EXCLUDED.v_rev, v_build = EXCLUDED.v_rev, host_port = EXCLUDED.host_port, "
-					  "use_redis = EXCLUDED.use_redis, redis_member_status = EXCLUDED.redis_member_status");
-				w.commit();
-			}
-			catch (std::exception& e) {
-				fprintf(stderr, "%s: Heartbeat update failed: %s\n", controllerId, e.what());
-				std::this_thread::sleep_for(std::chrono::milliseconds(1000));
-				continue;
-			}
-		}
-		_pool->unborrow(c);
-
-		try {
-			if (_redisMemberStatus) {
-				if (_rc->clusterMode) {
-					_cluster->zadd("controllers", "controllerId", ts);
-				}
-				else {
-					_redis->zadd("controllers", "controllerId", ts);
-				}
-			}
-		}
-		catch (sw::redis::Error& e) {
-			fprintf(stderr, "ERROR: Redis error in heartbeat thread: %s\n", e.what());
-		}
-
-		std::this_thread::sleep_for(std::chrono::milliseconds(1000));
-	}
-	fprintf(stderr, "Exited heartbeat thread\n");
-}
-
-void PostgreSQL::membersDbWatcher()
-{
-	if (_rc) {
-		_membersWatcher_Redis();
-	}
-	else {
-		_membersWatcher_Postgres();
-	}
-
-	if (_run == 1) {
-		fprintf(stderr, "ERROR: %s membersDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str());
-		exit(9);
-	}
-	fprintf(stderr, "Exited membersDbWatcher\n");
-}
-
-void PostgreSQL::_membersWatcher_Postgres()
-{
-	auto c = _pool->borrow();
-
-	std::string stream = "member_" + _myAddressStr;
-
-	fprintf(stderr, "Listening to member stream: %s\n", stream.c_str());
-	MemberNotificationReceiver m(this, *c->c, stream);
-
-	while (_run == 1) {
-		c->c->await_notification(5, 0);
-	}
-
-	_pool->unborrow(c);
-}
-
-void PostgreSQL::_membersWatcher_Redis()
-{
-	char buf[11] = { 0 };
-	std::string key = "member-stream:{" + std::string(_myAddress.toString(buf)) + "}";
-	std::string lastID = "0";
-	fprintf(stderr, "Listening to member stream: %s\n", key.c_str());
-	while (_run == 1) {
-		try {
-			json tmp;
-			std::unordered_map<std::string, ItemStream> result;
-			if (_rc->clusterMode) {
-				_cluster->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
-			}
-			else {
-				_redis->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
-			}
-			if (! result.empty()) {
-				for (auto element : result) {
-#ifdef REDIS_TRACE
-					fprintf(stdout, "Received notification from: %s\n", element.first.c_str());
-#endif
-					for (auto rec : element.second) {
-						std::string id = rec.first;
-						auto attrs = rec.second;
-#ifdef REDIS_TRACE
-						fprintf(stdout, "Record ID: %s\n", id.c_str());
-						fprintf(stdout, "attrs len: %lu\n", attrs.size());
-#endif
-						for (auto a : attrs) {
-#ifdef REDIS_TRACE
-							fprintf(stdout, "key: %s\nvalue: %s\n", a.first.c_str(), a.second.c_str());
-#endif
-							try {
-								tmp = json::parse(a.second);
-								json& ov = tmp["old_val"];
-								json& nv = tmp["new_val"];
-								json oldConfig, newConfig;
-								if (ov.is_object())
-									oldConfig = ov;
-								if (nv.is_object())
-									newConfig = nv;
-								if (oldConfig.is_object() || newConfig.is_object()) {
-									_memberChanged(oldConfig, newConfig, (this->_ready >= 2));
-								}
-							}
-							catch (...) {
-								fprintf(stderr, "json parse error in _membersWatcher_Redis: %s\n", a.second.c_str());
-							}
-						}
-						if (_rc->clusterMode) {
-							_cluster->xdel(key, id);
-						}
-						else {
-							_redis->xdel(key, id);
-						}
-						lastID = id;
-						Metrics::redis_mem_notification++;
-					}
-				}
-			}
-		}
-		catch (sw::redis::Error& e) {
-			fprintf(stderr, "Error in Redis members watcher: %s\n", e.what());
-		}
-	}
-	fprintf(stderr, "membersWatcher ended\n");
-}
-
-void PostgreSQL::networksDbWatcher()
-{
-	if (_rc) {
-		_networksWatcher_Redis();
-	}
-	else {
-		_networksWatcher_Postgres();
-	}
-
-	if (_run == 1) {
-		fprintf(stderr, "ERROR: %s networksDbWatcher should still be running! Exiting Controller.\n", _myAddressStr.c_str());
-		exit(8);
-	}
-	fprintf(stderr, "Exited networksDbWatcher\n");
-}
-
-void PostgreSQL::_networksWatcher_Postgres()
-{
-	std::string stream = "network_" + _myAddressStr;
-
-	fprintf(stderr, "Listening to member stream: %s\n", stream.c_str());
-
-	auto c = _pool->borrow();
-
-	NetworkNotificationReceiver n(this, *c->c, stream);
-
-	while (_run == 1) {
-		c->c->await_notification(5, 0);
-	}
-}
-
-void PostgreSQL::_networksWatcher_Redis()
-{
-	char buf[11] = { 0 };
-	std::string key = "network-stream:{" + std::string(_myAddress.toString(buf)) + "}";
-	std::string lastID = "0";
-	while (_run == 1) {
-		try {
-			json tmp;
-			std::unordered_map<std::string, ItemStream> result;
-			if (_rc->clusterMode) {
-				_cluster->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
-			}
-			else {
-				_redis->xread(key, lastID, std::chrono::seconds(1), 0, std::inserter(result, result.end()));
-			}
-
-			if (! result.empty()) {
-				for (auto element : result) {
-#ifdef REDIS_TRACE
-					fprintf(stdout, "Received notification from: %s\n", element.first.c_str());
-#endif
-					for (auto rec : element.second) {
-						std::string id = rec.first;
-						auto attrs = rec.second;
-#ifdef REDIS_TRACE
-						fprintf(stdout, "Record ID: %s\n", id.c_str());
-						fprintf(stdout, "attrs len: %lu\n", attrs.size());
-#endif
-						for (auto a : attrs) {
-#ifdef REDIS_TRACE
-							fprintf(stdout, "key: %s\nvalue: %s\n", a.first.c_str(), a.second.c_str());
-#endif
-							try {
-								tmp = json::parse(a.second);
-								json& ov = tmp["old_val"];
-								json& nv = tmp["new_val"];
-								json oldConfig, newConfig;
-								if (ov.is_object())
-									oldConfig = ov;
-								if (nv.is_object())
-									newConfig = nv;
-								if (oldConfig.is_object() || newConfig.is_object()) {
-									_networkChanged(oldConfig, newConfig, (this->_ready >= 2));
-								}
-							}
-							catch (std::exception& e) {
-								fprintf(stderr, "json parse error in networkWatcher_Redis: what: %s json: %s\n", e.what(), a.second.c_str());
-							}
-						}
-						if (_rc->clusterMode) {
-							_cluster->xdel(key, id);
-						}
-						else {
-							_redis->xdel(key, id);
-						}
-						lastID = id;
-					}
-					Metrics::redis_net_notification++;
-				}
-			}
-		}
-		catch (sw::redis::Error& e) {
-			fprintf(stderr, "Error in Redis networks watcher: %s\n", e.what());
-		}
-	}
-	fprintf(stderr, "networksWatcher ended\n");
-}
-
-void PostgreSQL::commitThread()
-{
-	fprintf(stderr, "%s: commitThread start\n", _myAddressStr.c_str());
-	std::pair<nlohmann::json, bool> qitem;
-	while (_commitQueue.get(qitem) & (_run == 1)) {
-		// fprintf(stderr, "commitThread tick\n");
-		if (! qitem.first.is_object()) {
-			fprintf(stderr, "not an object\n");
-			continue;
-		}
-
-		std::shared_ptr<PostgresConnection> c;
-		try {
-			c = _pool->borrow();
-		}
-		catch (std::exception& e) {
-			fprintf(stderr, "ERROR: %s\n", e.what());
-			continue;
-		}
-
-		if (! c) {
-			fprintf(stderr, "Error getting database connection\n");
-			continue;
-		}
-
-		Metrics::pgsql_commit_ticks++;
-		try {
-			nlohmann::json& config = (qitem.first);
-			const std::string objtype = config["objtype"];
-			if (objtype == "member") {
-				// fprintf(stderr, "%s: commitThread: member\n", _myAddressStr.c_str());
-				std::string memberId;
-				std::string networkId;
-				try {
-					pqxx::work w(*c->c);
-
-					memberId = config["id"];
-					networkId = config["nwid"];
-
-					std::string target = "NULL";
-					if (! config["remoteTraceTarget"].is_null()) {
-						target = config["remoteTraceTarget"];
-					}
-
-					pqxx::row nwrow = w.exec_params1("SELECT COUNT(id) FROM ztc_network WHERE id = $1", networkId);
-					int nwcount = nwrow[0].as<int>();
-
-					if (nwcount != 1) {
-						fprintf(stderr, "network %s does not exist.  skipping member upsert\n", networkId.c_str());
-						w.abort();
-						_pool->unborrow(c);
-						continue;
-					}
-
-					pqxx::row mrow = w.exec_params1("SELECT COUNT(id) FROM ztc_member WHERE id = $1 AND network_id = $2", memberId, networkId);
-					int membercount = mrow[0].as<int>();
-
-					bool isNewMember = false;
-					if (membercount == 0) {
-						// new member
-						isNewMember = true;
-						pqxx::result res = w.exec_params0(
-							"INSERT INTO ztc_member (id, network_id, active_bridge, authorized, capabilities, "
-							"identity, last_authorized_time, last_deauthorized_time, no_auto_assign_ips, "
-							"remote_trace_level, remote_trace_target, revision, tags, v_major, v_minor, v_rev, v_proto) "
-							"VALUES ($1, $2, $3, $4, $5, $6, "
-							"TO_TIMESTAMP($7::double precision/1000), TO_TIMESTAMP($8::double precision/1000), "
-							"$9, $10, $11, $12, $13, $14, $15, $16, $17)",
-							memberId,
-							networkId,
-							(bool)config["activeBridge"],
-							(bool)config["authorized"],
-							OSUtils::jsonDump(config["capabilities"], -1),
-							OSUtils::jsonString(config["identity"], ""),
-							(uint64_t)config["lastAuthorizedTime"],
-							(uint64_t)config["lastDeauthorizedTime"],
-							(bool)config["noAutoAssignIps"],
-							(int)config["remoteTraceLevel"],
-							target,
-							(uint64_t)config["revision"],
-							OSUtils::jsonDump(config["tags"], -1),
-							(int)config["vMajor"],
-							(int)config["vMinor"],
-							(int)config["vRev"],
-							(int)config["vProto"]);
-					}
-					else {
-						// existing member
-						pqxx::result res = w.exec_params0(
-							"UPDATE ztc_member "
-							"SET active_bridge = $3, authorized = $4, capabilities = $5, identity = $6, "
-							"last_authorized_time = TO_TIMESTAMP($7::double precision/1000), "
-							"last_deauthorized_time = TO_TIMESTAMP($8::double precision/1000), "
-							"no_auto_assign_ips = $9, remote_trace_level = $10, remote_trace_target= $11, "
-							"revision = $12, tags = $13, v_major = $14, v_minor = $15, v_rev = $16, v_proto = $17 "
-							"WHERE id = $1 AND network_id = $2",
-							memberId,
-							networkId,
-							(bool)config["activeBridge"],
-							(bool)config["authorized"],
-							OSUtils::jsonDump(config["capabilities"], -1),
-							OSUtils::jsonString(config["identity"], ""),
-							(uint64_t)config["lastAuthorizedTime"],
-							(uint64_t)config["lastDeauthorizedTime"],
-							(bool)config["noAutoAssignIps"],
-							(int)config["remoteTraceLevel"],
-							target,
-							(uint64_t)config["revision"],
-							OSUtils::jsonDump(config["tags"], -1),
-							(int)config["vMajor"],
-							(int)config["vMinor"],
-							(int)config["vRev"],
-							(int)config["vProto"]);
-					}
-
-					if (! isNewMember) {
-						pqxx::result res = w.exec_params0("DELETE FROM ztc_member_ip_assignment WHERE member_id = $1 AND network_id = $2", memberId, networkId);
-					}
-
-					std::vector<std::string> assignments;
-					bool ipAssignError = false;
-					for (auto i = config["ipAssignments"].begin(); i != config["ipAssignments"].end(); ++i) {
-						std::string addr = *i;
-
-						if (std::find(assignments.begin(), assignments.end(), addr) != assignments.end()) {
-							continue;
-						}
-
-						pqxx::result res = w.exec_params0("INSERT INTO ztc_member_ip_assignment (member_id, network_id, address) VALUES ($1, $2, $3) ON CONFLICT (network_id, member_id, address) DO NOTHING", memberId, networkId, addr);
-
-						assignments.push_back(addr);
-					}
-					if (ipAssignError) {
-						fprintf(stderr, "%s: ipAssignError\n", _myAddressStr.c_str());
-						w.abort();
-						_pool->unborrow(c);
-						c.reset();
-						continue;
-					}
-
-					w.commit();
-
-					if (_smee != NULL && isNewMember) {
-						pqxx::row row = w.exec_params1(
-							"SELECT "
-							"	count(h.hook_id) "
-							"FROM "
-							"	ztc_hook h "
-							"	INNER JOIN ztc_org o ON o.org_id = h.org_id "
-							"   INNER JOIN ztc_network n ON n.owner_id = o.owner_id "
-							" WHERE "
-							"n.id = $1 ",
-							networkId);
-						int64_t hookCount = row[0].as<int64_t>();
-						if (hookCount > 0) {
-							notifyNewMember(networkId, memberId);
-						}
-					}
-
-					const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL);
-					const uint64_t memberidInt = OSUtils::jsonIntHex(config["id"], 0ULL);
-					if (nwidInt && memberidInt) {
-						nlohmann::json nwOrig;
-						nlohmann::json memOrig;
-
-						nlohmann::json memNew(config);
-
-						get(nwidInt, nwOrig, memberidInt, memOrig);
-
-						_memberChanged(memOrig, memNew, qitem.second);
-					}
-					else {
-						fprintf(stderr, "%s: Can't notify of change.  Error parsing nwid or memberid: %llu-%llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt, (unsigned long long)memberidInt);
-					}
-				}
-				catch (std::exception& e) {
-					fprintf(stderr, "%s ERROR: Error updating member %s-%s: %s\n", _myAddressStr.c_str(), networkId.c_str(), memberId.c_str(), e.what());
-				}
-			}
-			else if (objtype == "network") {
-				try {
-					// fprintf(stderr, "%s: commitThread: network\n", _myAddressStr.c_str());
-					pqxx::work w(*c->c);
-
-					std::string id = config["id"];
-					std::string remoteTraceTarget = "";
-					if (! config["remoteTraceTarget"].is_null()) {
-						remoteTraceTarget = config["remoteTraceTarget"];
-					}
-					std::string rulesSource = "";
-					if (config["rulesSource"].is_string()) {
-						rulesSource = config["rulesSource"];
-					}
-
-					// This ugly query exists because when we want to mirror networks to/from
-					// another data store (e.g. FileDB or LFDB) it is possible to get a network
-					// that doesn't exist in Central's database. This does an upsert and sets
-					// the owner_id to the "first" global admin in the user DB if the record
-					// did not previously exist. If the record already exists owner_id is left
-					// unchanged, so owner_id should be left out of the update clause.
-					pqxx::result res = w.exec_params0(
-						"INSERT INTO ztc_network (id, creation_time, owner_id, controller_id, capabilities, enable_broadcast, "
-						"last_modified, mtu, multicast_limit, name, private, "
-						"remote_trace_level, remote_trace_target, rules, rules_source, "
-						"tags, v4_assign_mode, v6_assign_mode, sso_enabled) VALUES ("
-						"$1, TO_TIMESTAMP($5::double precision/1000), "
-						"(SELECT user_id AS owner_id FROM ztc_global_permissions WHERE authorize = true AND del = true AND modify = true AND read = true LIMIT 1),"
-						"$2, $3, $4, TO_TIMESTAMP($5::double precision/1000), "
-						"$6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17) "
-						"ON CONFLICT (id) DO UPDATE set controller_id = EXCLUDED.controller_id, "
-						"capabilities = EXCLUDED.capabilities, enable_broadcast = EXCLUDED.enable_broadcast, "
-						"last_modified = EXCLUDED.last_modified, mtu = EXCLUDED.mtu, "
-						"multicast_limit = EXCLUDED.multicast_limit, name = EXCLUDED.name, "
-						"private = EXCLUDED.private, remote_trace_level = EXCLUDED.remote_trace_level, "
-						"remote_trace_target = EXCLUDED.remote_trace_target, rules = EXCLUDED.rules, "
-						"rules_source = EXCLUDED.rules_source, tags = EXCLUDED.tags, "
-						"v4_assign_mode = EXCLUDED.v4_assign_mode, v6_assign_mode = EXCLUDED.v6_assign_mode, "
-						"sso_enabled = EXCLUDED.sso_enabled",
-						id,
-						_myAddressStr,
-						OSUtils::jsonDump(config["capabilities"], -1),
-						(bool)config["enableBroadcast"],
-						OSUtils::now(),
-						(int)config["mtu"],
-						(int)config["multicastLimit"],
-						OSUtils::jsonString(config["name"], ""),
-						(bool)config["private"],
-						(int)config["remoteTraceLevel"],
-						remoteTraceTarget,
-						OSUtils::jsonDump(config["rules"], -1),
-						rulesSource,
-						OSUtils::jsonDump(config["tags"], -1),
-						OSUtils::jsonDump(config["v4AssignMode"], -1),
-						OSUtils::jsonDump(config["v6AssignMode"], -1),
-						OSUtils::jsonBool(config["ssoEnabled"], false));
-
-					res = w.exec_params0("DELETE FROM ztc_network_assignment_pool WHERE network_id = $1", 0);
-
-					auto pool = config["ipAssignmentPools"];
-					bool err = false;
-					for (auto i = pool.begin(); i != pool.end(); ++i) {
-						std::string start = (*i)["ipRangeStart"];
-						std::string end = (*i)["ipRangeEnd"];
-
-						res = w.exec_params0(
-							"INSERT INTO ztc_network_assignment_pool (network_id, ip_range_start, ip_range_end) "
-							"VALUES ($1, $2, $3)",
-							id,
-							start,
-							end);
-					}
-
-					res = w.exec_params0("DELETE FROM ztc_network_route WHERE network_id = $1", id);
-
-					auto routes = config["routes"];
-					err = false;
-					for (auto i = routes.begin(); i != routes.end(); ++i) {
-						std::string t = (*i)["target"];
-						std::vector<std::string> target;
-						std::istringstream f(t);
-						std::string s;
-						while (std::getline(f, s, '/')) {
-							target.push_back(s);
-						}
-						if (target.empty() || target.size() != 2) {
-							continue;
-						}
-						std::string targetAddr = target[0];
-						std::string targetBits = target[1];
-						std::string via = "NULL";
-						if (! (*i)["via"].is_null()) {
-							via = (*i)["via"];
-						}
-
-						res = w.exec_params0("INSERT INTO ztc_network_route (network_id, address, bits, via) VALUES ($1, $2, $3, $4)", id, targetAddr, targetBits, (via == "NULL" ? NULL : via.c_str()));
-					}
-					if (err) {
-						fprintf(stderr, "%s: route add error\n", _myAddressStr.c_str());
-						w.abort();
-						_pool->unborrow(c);
-						continue;
-					}
-
-					auto dns = config["dns"];
-					std::string domain = dns["domain"];
-					std::stringstream servers;
-					servers << "{";
-					for (auto j = dns["servers"].begin(); j < dns["servers"].end(); ++j) {
-						servers << *j;
-						if ((j + 1) != dns["servers"].end()) {
-							servers << ",";
-						}
-					}
-					servers << "}";
-
-					std::string s = servers.str();
-
-					res = w.exec_params0("INSERT INTO ztc_network_dns (network_id, domain, servers) VALUES ($1, $2, $3) ON CONFLICT (network_id) DO UPDATE SET domain = EXCLUDED.domain, servers = EXCLUDED.servers", id, domain, s);
-
-					w.commit();
-
-					const uint64_t nwidInt = OSUtils::jsonIntHex(config["nwid"], 0ULL);
-					if (nwidInt) {
-						nlohmann::json nwOrig;
-						nlohmann::json nwNew(config);
-
-						get(nwidInt, nwOrig);
-
-						_networkChanged(nwOrig, nwNew, qitem.second);
-					}
-					else {
-						fprintf(stderr, "%s: Can't notify network changed: %llu\n", _myAddressStr.c_str(), (unsigned long long)nwidInt);
-					}
-				}
-				catch (std::exception& e) {
-					fprintf(stderr, "%s ERROR: Error updating network: %s\n", _myAddressStr.c_str(), e.what());
-				}
-				if (_redisMemberStatus) {
-					try {
-						std::string id = config["id"];
-						std::string controllerId = _myAddressStr.c_str();
-						std::string key = "networks:{" + controllerId + "}";
-						if (_rc->clusterMode) {
-							_cluster->sadd(key, id);
-						}
-						else {
-							_redis->sadd(key, id);
-						}
-					}
-					catch (sw::redis::Error& e) {
-						fprintf(stderr, "ERROR: Error adding network to Redis: %s\n", e.what());
-					}
-				}
-			}
-			else if (objtype == "_delete_network") {
-				// fprintf(stderr, "%s: commitThread: delete network\n", _myAddressStr.c_str());
-				try {
-					pqxx::work w(*c->c);
-
-					std::string networkId = config["nwid"];
-
-					pqxx::result res = w.exec_params0("UPDATE ztc_network SET deleted = true WHERE id = $1", networkId);
-
-					w.commit();
-				}
-				catch (std::exception& e) {
-					fprintf(stderr, "%s ERROR: Error deleting network: %s\n", _myAddressStr.c_str(), e.what());
-				}
-				if (_redisMemberStatus) {
-					try {
-						std::string id = config["id"];
-						std::string controllerId = _myAddressStr.c_str();
-						std::string key = "networks:{" + controllerId + "}";
-						if (_rc->clusterMode) {
-							_cluster->srem(key, id);
-							_cluster->del("network-nodes-online:{" + controllerId + "}:" + id);
-						}
-						else {
-							_redis->srem(key, id);
-							_redis->del("network-nodes-online:{" + controllerId + "}:" + id);
-						}
-					}
-					catch (sw::redis::Error& e) {
-						fprintf(stderr, "ERROR: Error adding network to Redis: %s\n", e.what());
-					}
-				}
-			}
-			else if (objtype == "_delete_member") {
-				// fprintf(stderr, "%s commitThread: delete member\n", _myAddressStr.c_str());
-				try {
-					pqxx::work w(*c->c);
-
-					std::string memberId = config["id"];
-					std::string networkId = config["nwid"];
-
-					pqxx::result res = w.exec_params0("UPDATE ztc_member SET hidden = true, deleted = true WHERE id = $1 AND network_id = $2", memberId, networkId);
-
-					w.commit();
-				}
-				catch (std::exception& e) {
-					fprintf(stderr, "%s ERROR: Error deleting member: %s\n", _myAddressStr.c_str(), e.what());
-				}
-				if (_redisMemberStatus) {
-					try {
-						std::string memberId = config["id"];
-						std::string networkId = config["nwid"];
-						std::string controllerId = _myAddressStr.c_str();
-						std::string key = "network-nodes-all:{" + controllerId + "}:" + networkId;
-						if (_rc->clusterMode) {
-							_cluster->srem(key, memberId);
-							_cluster->del("member:{" + controllerId + "}:" + networkId + ":" + memberId);
-						}
-						else {
-							_redis->srem(key, memberId);
-							_redis->del("member:{" + controllerId + "}:" + networkId + ":" + memberId);
-						}
-					}
-					catch (sw::redis::Error& e) {
-						fprintf(stderr, "ERROR: Error deleting member from Redis: %s\n", e.what());
-					}
-				}
-			}
-			else {
-				fprintf(stderr, "%s ERROR: unknown objtype\n", _myAddressStr.c_str());
-			}
-		}
-		catch (std::exception& e) {
-			fprintf(stderr, "%s ERROR: Error getting objtype: %s\n", _myAddressStr.c_str(), e.what());
-		}
-		_pool->unborrow(c);
-		c.reset();
-	}
-
-	fprintf(stderr, "%s commitThread finished\n", _myAddressStr.c_str());
-}
-
-void PostgreSQL::notifyNewMember(const std::string& networkID, const std::string& memberID)
-{
-	smeeclient::smee_client_notify_network_joined(_smee, networkID.c_str(), memberID.c_str());
-}
-
-void PostgreSQL::onlineNotificationThread()
-{
-	waitForReady();
-	if (_redisMemberStatus) {
-		onlineNotification_Redis();
-	}
-	else {
-		onlineNotification_Postgres();
-	}
-}
-
-/**
- * ONLY UNCOMMENT FOR TEMPORARY DB MAINTENANCE
- *
- * This define temporarily turns off writing to the member status table
- * so it can be reindexed when the indexes get too large.
- */
-
-// #define DISABLE_MEMBER_STATUS 1
-
-void PostgreSQL::onlineNotification_Postgres()
-{
-	_connected = 1;
-
-	nlohmann::json jtmp1, jtmp2;
-	while (_run == 1) {
-		auto c = _pool->borrow();
-		auto c2 = _pool->borrow();
-		try {
-			fprintf(stderr, "%s onlineNotification_Postgres\n", _myAddressStr.c_str());
-			std::unordered_map<std::pair<uint64_t, uint64_t>, std::pair<int64_t, InetAddress>, _PairHasher> lastOnline;
-			{
-				std::lock_guard<std::mutex> l(_lastOnline_l);
-				lastOnline.swap(_lastOnline);
-			}
-
-#ifndef DISABLE_MEMBER_STATUS
-			pqxx::work w(*c->c);
-			pqxx::work w2(*c2->c);
-
-			fprintf(stderr, "online notification tick\n");
-
-			bool firstRun = true;
-			bool memberAdded = false;
-			int updateCount = 0;
-
-			pqxx::pipeline pipe(w);
-
-			for (auto i = lastOnline.begin(); i != lastOnline.end(); ++i) {
-				updateCount += 1;
-				uint64_t nwid_i = i->first.first;
-				char nwidTmp[64];
-				char memTmp[64];
-				char ipTmp[64];
-				OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
-				OSUtils::ztsnprintf(memTmp, sizeof(memTmp), "%.10llx", i->first.second);
-
-				if (! get(nwid_i, jtmp1, i->first.second, jtmp2)) {
-					continue;	// skip non existent networks/members
-				}
-
-				std::string networkId(nwidTmp);
-				std::string memberId(memTmp);
-
-				try {
-					pqxx::row r = w2.exec_params1("SELECT id, network_id FROM ztc_member WHERE network_id = $1 AND id = $2", networkId, memberId);
-				}
-				catch (pqxx::unexpected_rows& e) {
-					continue;
-				}
-
-				int64_t ts = i->second.first;
-				std::string ipAddr = i->second.second.toIpString(ipTmp);
-				std::string timestamp = std::to_string(ts);
-
-				std::stringstream memberUpdate;
-				memberUpdate << "INSERT INTO ztc_member_status (network_id, member_id, address, last_updated) VALUES "
-							 << "('" << networkId << "', '" << memberId << "', ";
-				if (ipAddr.empty()) {
-					memberUpdate << "NULL, ";
-				}
-				else {
-					memberUpdate << "'" << ipAddr << "', ";
-				}
-				memberUpdate << "TO_TIMESTAMP(" << timestamp << "::double precision/1000)) "
-							 << " ON CONFLICT (network_id, member_id) DO UPDATE SET address = EXCLUDED.address, last_updated = EXCLUDED.last_updated";
-
-				pipe.insert(memberUpdate.str());
-				Metrics::pgsql_node_checkin++;
-			}
-			while (! pipe.empty()) {
-				pipe.retrieve();
-			}
-
-			pipe.complete();
-			w.commit();
-			fprintf(stderr, "%s: Updated online status of %d members\n", _myAddressStr.c_str(), updateCount);
-#endif
-		}
-		catch (std::exception& e) {
-			fprintf(stderr, "%s: error in onlinenotification thread: %s\n", _myAddressStr.c_str(), e.what());
-		}
-		_pool->unborrow(c2);
-		_pool->unborrow(c);
-
-		ConnectionPoolStats stats = _pool->get_stats();
-		fprintf(stderr, "%s pool stats: in use size: %llu, available size: %llu, total: %llu\n", _myAddressStr.c_str(), stats.borrowed_size, stats.pool_size, (stats.borrowed_size + stats.pool_size));
-
-		std::this_thread::sleep_for(std::chrono::seconds(10));
-	}
-	fprintf(stderr, "%s: Fell out of run loop in onlineNotificationThread\n", _myAddressStr.c_str());
-	if (_run == 1) {
-		fprintf(stderr, "ERROR: %s onlineNotificationThread should still be running! Exiting Controller.\n", _myAddressStr.c_str());
-		exit(6);
-	}
-}
-
-void PostgreSQL::onlineNotification_Redis()
-{
-	_connected = 1;
-
-	char buf[11] = { 0 };
-	std::string controllerId = std::string(_myAddress.toString(buf));
-
-	while (_run == 1) {
-		fprintf(stderr, "onlineNotification tick\n");
-		auto start = std::chrono::high_resolution_clock::now();
-		uint64_t count = 0;
-
-		std::unordered_map<std::pair<uint64_t, uint64_t>, std::pair<int64_t, InetAddress>, _PairHasher> lastOnline;
-		{
-			std::lock_guard<std::mutex> l(_lastOnline_l);
-			lastOnline.swap(_lastOnline);
-		}
-		try {
-			if (! lastOnline.empty()) {
-				if (_rc->clusterMode) {
-					auto tx = _cluster->transaction(controllerId, true, false);
-					count = _doRedisUpdate(tx, controllerId, lastOnline);
-				}
-				else {
-					auto tx = _redis->transaction(true, false);
-					count = _doRedisUpdate(tx, controllerId, lastOnline);
-				}
-			}
-		}
-		catch (sw::redis::Error& e) {
-			fprintf(stderr, "Error in online notification thread (redis): %s\n", e.what());
-		}
-
-		auto end = std::chrono::high_resolution_clock::now();
-		auto dur = std::chrono::duration_cast<std::chrono::milliseconds>(end - start);
-		auto total = dur.count();
-
-		fprintf(stderr, "onlineNotification ran in %llu ms\n", total);
-
-		std::this_thread::sleep_for(std::chrono::seconds(5));
-	}
-}
-
-uint64_t PostgreSQL::_doRedisUpdate(sw::redis::Transaction& tx, std::string& controllerId, std::unordered_map<std::pair<uint64_t, uint64_t>, std::pair<int64_t, InetAddress>, _PairHasher>& lastOnline)
-
-{
-	nlohmann::json jtmp1, jtmp2;
-	uint64_t count = 0;
-	for (auto i = lastOnline.begin(); i != lastOnline.end(); ++i) {
-		uint64_t nwid_i = i->first.first;
-		uint64_t memberid_i = i->first.second;
-		char nwidTmp[64];
-		char memTmp[64];
-		char ipTmp[64];
-		OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
-		OSUtils::ztsnprintf(memTmp, sizeof(memTmp), "%.10llx", memberid_i);
-
-		if (! get(nwid_i, jtmp1, memberid_i, jtmp2)) {
-			continue;	// skip non existent members/networks
-		}
-
-		std::string networkId(nwidTmp);
-		std::string memberId(memTmp);
-
-		int64_t ts = i->second.first;
-		std::string ipAddr = i->second.second.toIpString(ipTmp);
-		std::string timestamp = std::to_string(ts);
-
-		std::unordered_map<std::string, std::string> record = { { "id", memberId }, { "address", ipAddr }, { "last_updated", std::to_string(ts) } };
-		tx.zadd("nodes-online:{" + controllerId + "}", memberId, ts)
-			.zadd("nodes-online2:{" + controllerId + "}", networkId + "-" + memberId, ts)
-			.zadd("network-nodes-online:{" + controllerId + "}:" + networkId, memberId, ts)
-			.zadd("active-networks:{" + controllerId + "}", networkId, ts)
-			.sadd("network-nodes-all:{" + controllerId + "}:" + networkId, memberId)
-			.hmset("member:{" + controllerId + "}:" + networkId + ":" + memberId, record.begin(), record.end());
-		++count;
-		Metrics::redis_node_checkin++;
-	}
-
-	// expire records from all-nodes and network-nodes member list
-	uint64_t expireOld = OSUtils::now() - 300000;
-
-	tx.zremrangebyscore("nodes-online:{" + controllerId + "}", sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
-	tx.zremrangebyscore("nodes-online2:{" + controllerId + "}", sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
-	tx.zremrangebyscore("active-networks:{" + controllerId + "}", sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
-	{
-		std::shared_lock<std::shared_mutex> l(_networks_l);
-		for (const auto& it : _networks) {
-			uint64_t nwid_i = it.first;
-			char nwidTmp[64];
-			OSUtils::ztsnprintf(nwidTmp, sizeof(nwidTmp), "%.16llx", nwid_i);
-			tx.zremrangebyscore("network-nodes-online:{" + controllerId + "}:" + nwidTmp, sw::redis::RightBoundedInterval<double>(expireOld, sw::redis::BoundType::LEFT_OPEN));
-		}
-	}
-	tx.exec();
-	fprintf(stderr, "%s: Updated online status of %d members\n", _myAddressStr.c_str(), count);
-
-	return count;
-}
-
-#endif	 // ZT_CONTROLLER_USE_LIBPQ
+#endif

controller/PostgreSQL.hpp

@@ -1,5 +1,5 @@
 /*
- * Copyright (c)2019 ZeroTier, Inc.
+ * Copyright (c)2025 ZeroTier, Inc.
  *
  * Use of this software is governed by the Business Source License included
  * in the LICENSE.TXT file in the project's root directory.
@@ -11,34 +11,23 @@
  */
 /****/
 
-#include "DB.hpp"
-
 #ifdef ZT_CONTROLLER_USE_LIBPQ
 
-#ifndef ZT_CONTROLLER_LIBPQ_HPP
-#define ZT_CONTROLLER_LIBPQ_HPP
-
-#define ZT_CENTRAL_CONTROLLER_COMMIT_THREADS 4
+#ifndef ZT_CONTROLLER_POSTGRESQL_HPP
+#define ZT_CONTROLLER_POSTGRESQL_HPP
 
-#include "../node/Metrics.hpp"
 #include "ConnectionPool.hpp"
+#include "DB.hpp"
 
 #include <memory>
 #include <pqxx/pqxx>
-#include <redis++/redis++.h>
+
+namespace ZeroTier {
 
 extern "C" {
 typedef struct pg_conn PGconn;
 }
 
-namespace smeeclient {
-struct SmeeClient;
-}
-
-namespace ZeroTier {
-
-struct RedisConfig;
-
 class PostgresConnection : public Connection {
   public:
 	virtual ~PostgresConnection()
@@ -67,11 +56,9 @@ class PostgresConnFactory : public ConnectionFactory {
 	std::string m_connString;
 };
 
-class PostgreSQL;
-
 class MemberNotificationReceiver : public pqxx::notification_receiver {
   public:
-	MemberNotificationReceiver(PostgreSQL* p, pqxx::connection& c, const std::string& channel);
+	MemberNotificationReceiver(DB* p, pqxx::connection& c, const std::string& channel);
 	virtual ~MemberNotificationReceiver()
 	{
 		fprintf(stderr, "MemberNotificationReceiver destroyed\n");
@@ -80,12 +67,12 @@ class MemberNotificationReceiver : public pqxx::notification_receiver {
 	virtual void operator()(const std::string& payload, int backendPid);
 
   private:
-	PostgreSQL* _psql;
+	DB* _psql;
 };
 
 class NetworkNotificationReceiver : public pqxx::notification_receiver {
   public:
-	NetworkNotificationReceiver(PostgreSQL* p, pqxx::connection& c, const std::string& channel);
+	NetworkNotificationReceiver(DB* p, pqxx::connection& c, const std::string& channel);
 	virtual ~NetworkNotificationReceiver()
 	{
 		fprintf(stderr, "NetworkNotificationReceiver destroyed\n");
@@ -94,106 +81,17 @@ class NetworkNotificationReceiver : public pqxx::notification_receiver {
 	virtual void operator()(const std::string& payload, int packend_pid);
 
   private:
-	PostgreSQL* _psql;
+	DB* _psql;
 };
 
-/**
- * A controller database driver that talks to PostgreSQL
- *
- * This is for use with ZeroTier Central.  Others are free to build and use it
- * but be aware that we might change it at any time.
- */
-class PostgreSQL : public DB {
-	friend class MemberNotificationReceiver;
-	friend class NetworkNotificationReceiver;
-
-  public:
-	PostgreSQL(const Identity& myId, const char* path, int listenPort, RedisConfig* rc);
-	virtual ~PostgreSQL();
-
-	virtual bool waitForReady();
-	virtual bool isReady();
-	virtual bool save(nlohmann::json& record, bool notifyListeners);
-	virtual void eraseNetwork(const uint64_t networkId);
-	virtual void eraseMember(const uint64_t networkId, const uint64_t memberId);
-	virtual void nodeIsOnline(const uint64_t networkId, const uint64_t memberId, const InetAddress& physicalAddress);
-	virtual AuthInfo getSSOAuthInfo(const nlohmann::json& member, const std::string& redirectURL);
-
-  protected:
-	struct _PairHasher {
-		inline std::size_t operator()(const std::pair<uint64_t, uint64_t>& p) const
-		{
-			return (std::size_t)(p.first ^ p.second);
-		}
-	};
-	virtual void _memberChanged(nlohmann::json& old, nlohmann::json& memberConfig, bool notifyListeners)
-	{
-		DB::_memberChanged(old, memberConfig, notifyListeners);
-	}
-
-	virtual void _networkChanged(nlohmann::json& old, nlohmann::json& networkConfig, bool notifyListeners)
-	{
-		DB::_networkChanged(old, networkConfig, notifyListeners);
-	}
-
-  private:
-	void initializeNetworks();
-	void initializeMembers();
-	void heartbeat();
-	void membersDbWatcher();
-	void _membersWatcher_Postgres();
-	void networksDbWatcher();
-	void _networksWatcher_Postgres();
-
-	void _membersWatcher_Redis();
-	void _networksWatcher_Redis();
-
-	void commitThread();
-	void onlineNotificationThread();
-	void onlineNotification_Postgres();
-	void onlineNotification_Redis();
-	uint64_t _doRedisUpdate(sw::redis::Transaction& tx, std::string& controllerId, std::unordered_map<std::pair<uint64_t, uint64_t>, std::pair<int64_t, InetAddress>, _PairHasher>& lastOnline);
-
-	void configureSmee();
-	void notifyNewMember(const std::string& networkID, const std::string& memberID);
-
-	enum OverrideMode { ALLOW_PGBOUNCER_OVERRIDE = 0, NO_OVERRIDE = 1 };
-
-	std::shared_ptr<ConnectionPool<PostgresConnection> > _pool;
-
-	const Identity _myId;
-	const Address _myAddress;
-	std::string _myAddressStr;
-	std::string _connString;
-
-	BlockingQueue<std::pair<nlohmann::json, bool> > _commitQueue;
-
-	std::thread _heartbeatThread;
-	std::thread _membersDbWatcher;
-	std::thread _networksDbWatcher;
-	std::thread _commitThread[ZT_CENTRAL_CONTROLLER_COMMIT_THREADS];
-	std::thread _onlineNotificationThread;
-
-	std::unordered_map<std::pair<uint64_t, uint64_t>, std::pair<int64_t, InetAddress>, _PairHasher> _lastOnline;
-
-	mutable std::mutex _lastOnline_l;
-	mutable std::mutex _readyLock;
-	std::atomic<int> _ready, _connected, _run;
-	mutable volatile bool _waitNoticePrinted;
-
-	int _listenPort;
-	uint8_t _ssoPsk[48];
-
-	RedisConfig* _rc;
-	std::shared_ptr<sw::redis::Redis> _redis;
-	std::shared_ptr<sw::redis::RedisCluster> _cluster;
-	bool _redisMemberStatus;
-
-	smeeclient::SmeeClient* _smee;
+struct NodeOnlineRecord {
+	uint64_t lastSeen;
+	InetAddress physicalAddress;
+	std::string osArch;
 };
 
 }	// namespace ZeroTier
 
-#endif	 // ZT_CONTROLLER_LIBPQ_HPP
+#endif	 // ZT_CONTROLLER_POSTGRESQL_HPP
 
-#endif	 // ZT_CONTROLLER_USE_LIBPQ
+#endif	 // ZT_CONTROLLER_USE_LIBPQ

entrypoint.sh.release

@@ -9,15 +9,16 @@ mkztfile() {
   file=$1
   mode=$2
   content=$3
-
+  echo "creating $file"
   mkdir -p /var/lib/zerotier-one
-  echo "$content" > "/var/lib/zerotier-one/$file"
+  echo -n "$content" > "/var/lib/zerotier-one/$file"
   chmod "$mode" "/var/lib/zerotier-one/$file"
 }
 
 if [ "x$ZEROTIER_API_SECRET" != "x" ]
 then
   mkztfile authtoken.secret 0600 "$ZEROTIER_API_SECRET"
+  mkztfile metricstoken.secret 0600 "$ZEROTIER_API_SECRET"
 fi
 
 if [ "x$ZEROTIER_IDENTITY_PUBLIC" != "x" ]
@@ -30,6 +31,11 @@ then
   mkztfile identity.secret 0600 "$ZEROTIER_IDENTITY_SECRET"
 fi
 
+if [ "x$ZEROTIER_LOCAL_CONF" != "x" ]
+then
+  mkztfile local.conf 0644 "$ZEROTIER_LOCAL_CONF"
+fi
+
 mkztfile zerotier-one.port 0600 "9993"
 
 killzerotier() {

ext/central-controller-docker/Dockerfile

@@ -1,11 +1,16 @@
 # Dockerfile for ZeroTier Central Controllers
-FROM registry.zerotier.com/zerotier/ctlbuild:latest as builder
-MAINTAINER Adam Ierymekno <[email protected]>, Grant Limberg <[email protected]>
+FROM registry.zerotier.com/zerotier/ctlbuild:2025-05-13-01 AS builder
 ADD . /ZeroTierOne
 RUN export PATH=$PATH:~/.cargo/bin && cd ZeroTierOne && make clean && make central-controller -j8
 
-FROM registry.zerotier.com/zerotier/ctlrun:latest
+FROM golang:bookworm AS go_base
+RUN go install -tags 'postgres' github.com/golang-migrate/migrate/v4/cmd/migrate@latest
+
+FROM registry.zerotier.com/zerotier/ctlrun:2025-05-13-01
 COPY --from=builder /ZeroTierOne/zerotier-one /usr/local/bin/zerotier-one
+COPY --from=go_base /go/bin/migrate /usr/local/bin/migrate
+COPY ext/central-controller-docker/migrations /migrations
+
 RUN chmod a+x /usr/local/bin/zerotier-one
 RUN echo "/usr/local/lib64" > /etc/ld.so.conf.d/usr-local-lib64.conf && ldconfig
 

ext/central-controller-docker/Dockerfile.builder

@@ -1,8 +1,5 @@
 # Dockerfile for building ZeroTier Central Controllers
-FROM ubuntu:jammy as builder
-MAINTAINER Adam Ierymekno <[email protected]>, Grant Limberg <[email protected]>
-
-ARG git_branch=master
+FROM debian:bookworm
 
 RUN apt update && apt upgrade -y
 RUN apt -y install \

ext/central-controller-docker/Dockerfile.run_base

@@ -1,15 +1,17 @@
-FROM ubuntu:jammy
+FROM debian:bookworm
+
 
-RUN apt update && apt upgrade -y
 
+RUN apt update && apt upgrade -y
 RUN apt -y install \
-    netcat \
+    netcat-traditional \
     postgresql-client \
     postgresql-client-common \
     libjemalloc2 \
     libpq5 \
     curl \
     binutils \
-    linux-tools-gke \
     perf-tools-unstable \
-    google-perftools 
+    google-perftools \
+    gnupg
+

ext/central-controller-docker/main.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
 
-if [ -z "$ZT_IDENTITY_PATH" ]; then
-    echo '*** FAILED: ZT_IDENTITY_PATH environment variable is not defined'
-    exit 1
-fi
 if [ -z "$ZT_DB_HOST" ]; then
     echo '*** FAILED: ZT_DB_HOST environment variable not defined'
     exit 1
@@ -24,6 +20,9 @@ if [ -z "$ZT_DB_PASSWORD" ]; then
     echo '*** FAILED: ZT_DB_PASSWORD environment variable not defined'
     exit 1
 fi
+if [ -z "$ZT_DB_TYPE" ]; then
+    ZT_DB_TYPE="postgres"
+fi
 
 REDIS=""
 if [ "$ZT_USE_REDIS" == "true" ]; then
@@ -56,10 +55,14 @@ fi
 mkdir -p /var/lib/zerotier-one
 
 pushd /var/lib/zerotier-one
-ln -s $ZT_IDENTITY_PATH/identity.public identity.public
-ln -s $ZT_IDENTITY_PATH/identity.secret identity.secret
-if [ -f  "$ZT_IDENTITY_PATH/authtoken.secret" ]; then
-    ln -s $ZT_IDENTITY_PATH/authtoken.secret authtoken.secret
+if [ -d "$ZT_IDENTITY_PATH" ]; then
+    echo '*** Using existing ZT identity from path $ZT_IDENTITY_PATH'
+
+    ln -s $ZT_IDENTITY_PATH/identity.public identity.public
+    ln -s $ZT_IDENTITY_PATH/identity.secret identity.secret
+    if [ -f  "$ZT_IDENTITY_PATH/authtoken.secret" ]; then
+        ln -s $ZT_IDENTITY_PATH/authtoken.secret authtoken.secret
+    fi
 fi
 popd
 
@@ -70,7 +73,7 @@ APP_NAME="controller-$(cat /var/lib/zerotier-one/identity.public | cut -d ':' -f
 
 echo "{
     \"settings\": {
-        \"controllerDbPath\": \"postgres:host=${ZT_DB_HOST} port=${ZT_DB_PORT} dbname=${ZT_DB_NAME} user=${ZT_DB_USER} password=${ZT_DB_PASSWORD} application_name=${APP_NAME} sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}\",
+        \"controllerDbPath\": \"${ZT_DB_TYPE}:host=${ZT_DB_HOST} port=${ZT_DB_PORT} dbname=${ZT_DB_NAME} user=${ZT_DB_USER} password=${ZT_DB_PASSWORD} application_name=${APP_NAME} sslmode=prefer sslcert=${DB_CLIENT_CERT} sslkey=${DB_CLIENT_KEY} sslrootcert=${DB_SERVER_CA}\",
         \"portMappingEnabled\": true,
         \"softwareUpdate\": \"disable\",
         \"interfacePrefixBlacklist\": [
@@ -100,6 +103,15 @@ else
     done
 fi
 
+if [ "$ZT_DB_TYPE" == "cv2" ]; then
+    echo "Migrating database (if needed)..."
+    if [ -n "$DB_SERVER_CA" ]; then
+        /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=verify-full&sslrootcert=$DB_SERVER_CA&sslcert=$DB_CLIENT_CERT&sslkey=$DB_CLIENT_KEY" up  
+    else 
+        /usr/local/bin/migrate -source file:///migrations -database "postgres://$ZT_DB_USER:$ZT_DB_PASSWORD@$ZT_DB_HOST:$ZT_DB_PORT/$ZT_DB_NAME?x-migrations-table=controller_migrations&sslmode=disable" up
+    fi
+fi
+
 if [ -n "$ZT_TEMPORAL_HOST" ] && [ -n "$ZT_TEMPORAL_PORT" ]; then
     echo "waiting for temporal..."
     while ! nc -z ${ZT_TEMPORAL_HOST} ${ZT_TEMPORAL_PORT}; do

ext/central-controller-docker/migrations/0001_init.down.sql

@@ -0,0 +1,3 @@
+DROP TABLE IF EXISTS network_memberships_ctl;
+DROP TABLE IF EXISTS networks_ctl;
+DROP TABLE IF EXISTS controllers_ctl;

ext/central-controller-docker/migrations/0001_init.up.sql

@@ -0,0 +1,47 @@
+-- inits controller db schema
+
+CREATE TABLE IF NOT EXISTS controllers_ctl (
+    id text NOT NULL PRIMARY KEY,
+    hostname text,
+    last_heartbeat timestamp with time zone,
+    public_identity text NOT NULL,
+    version text
+);
+
+CREATE TABLE IF NOT EXISTS networks_ctl (
+    id character varying(22) NOT NULL PRIMARY KEY,
+    name text NOT NULL,
+    configuration jsonb DEFAULT '{}'::jsonb NOT NULL,
+    controller_id text REFERENCES controllers_ctl(id),
+    revision integer DEFAULT 0 NOT NULL,
+    last_modified timestamp with time zone DEFAULT now(),
+    creation_time timestamp with time zone DEFAULT now()
+);
+
+CREATE TABLE IF NOT EXISTS network_memberships_ctl (
+    device_id character varying(22) NOT NULL,
+    network_id character varying(22) NOT NULL REFERENCES networks_ctl(id),
+    authorized boolean,
+    active_bridge boolean,
+    ip_assignments text[],
+    no_auto_assign_ips boolean,
+    sso_exempt boolean,
+    authentication_expiry_time timestamp with time zone,
+    capabilities jsonb,
+    creation_time timestamp with time zone DEFAULT now(),
+    last_modified timestamp with time zone DEFAULT now(),
+    identity text DEFAULT ''::text,
+    last_authorized_credential text,
+    last_authorized_time timestamp with time zone,
+    last_deauthorized_time timestamp with time zone,
+    last_seen jsonb DEFAULT '{}'::jsonb NOT NULL, -- in the context of the network
+    remote_trace_level integer DEFAULT 0 NOT NULL,
+    remote_trace_target text DEFAULT ''::text NOT NULL,
+    revision integer DEFAULT 0 NOT NULL,
+    tags jsonb,
+    version_major integer DEFAULT 0 NOT NULL,
+    version_minor integer DEFAULT 0 NOT NULL,
+    version_revision integer DEFAULT 0 NOT NULL,
+    version_protocol integer DEFAULT 0 NOT NULL,
+    PRIMARY KEY (device_id, network_id)
+);

ext/central-controller-docker/migrations/0002_os_arch.down.sql

@@ -0,0 +1,3 @@
+ALTER TABLE network_memberships_ctl
+    DROP COLUMN os,
+    DROP COLUMN arch;

ext/central-controller-docker/migrations/0002_os_arch.up.sql

@@ -0,0 +1,3 @@
+ALTER TABLE network_memberships_ctl
+    ADD COLUMN os TEXT NOT NULL DEFAULT 'unknown',
+    ADD COLUMN arch TEXT NOT NULL DEFAULT 'unknown';

make-linux.mk

@@ -444,6 +444,10 @@ central-controller-docker: _buildx FORCE
 	docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP} -f ext/central-controller-docker/Dockerfile --build-arg git_branch=`git name-rev --name-only HEAD` . --push
 	@echo Image: registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP}
 
+centralv2-controller-docker: _buildx FORCE
+	docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:$(shell git rev-parse --short HEAD) -f ext/central-controller-docker/Dockerfile --build-arg git_branch=`git name-rev --name-only HEAD` . --push
+	@echo Image: us-central1-docker.pkg.dev/zerotier-421eb9/docker-images/ztcentral-controller:$(shell git rev-parse --short HEAD)
+
 debug:	FORCE
 	make ZT_DEBUG=1 one
 	make ZT_DEBUG=1 selftest

make-mac.mk

@@ -57,9 +57,9 @@ ONE_OBJS+=ext/libnatpmp/natpmp.o ext/libnatpmp/getgateway.o ext/miniupnpc/connec
 ifeq ($(ZT_CONTROLLER),1)
 	MACOS_VERSION_MIN=10.15
 	override CXXFLAGS=$(CFLAGS) -std=c++17 -stdlib=libc++
-	LIBS+=-L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a
+	LIBS+=-L/opt/homebrew/lib -L/usr/local/opt/libpqxx/lib -L/usr/local/opt/libpq/lib -L/usr/local/opt/openssl/lib/ -lpqxx -lpq -lssl -lcrypto -lgssapi_krb5 ext/redis-plus-plus-1.1.1/install/macos/lib/libredis++.a ext/hiredis-0.14.1/lib/macos/libhiredis.a rustybits/target/libsmeeclient.a
 	DEFS+=-DZT_CONTROLLER_USE_LIBPQ -DZT_CONTROLLER_USE_REDIS -DZT_CONTROLLER 
-	INCLUDES+=-I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/
+	INCLUDES+=-I/opt/homebrew/include -I/opt/homebrew/opt/libpq/include -I/usr/local/opt/libpq/include -I/usr/local/opt/libpqxx/include -Iext/hiredis-0.14.1/include/ -Iext/redis-plus-plus-1.1.1/install/macos/include/sw/ -Irustybits/target/
 else
 	MACOS_VERSION_MIN=10.13
 endif
@@ -115,7 +115,11 @@ mac-agent: FORCE
 osdep/MacDNSHelper.o: osdep/MacDNSHelper.mm
 	$(CXX) $(CXXFLAGS) -c osdep/MacDNSHelper.mm -o osdep/MacDNSHelper.o 
 
+ifeq ($(ZT_CONTROLLER),1)
+one:	zeroidc smeeclient $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent 
+else
 one:	zeroidc $(CORE_OBJS) $(ONE_OBJS) one.o mac-agent 
+endif
 	$(CXX) $(CXXFLAGS) -o zerotier-one $(CORE_OBJS) $(ONE_OBJS) one.o $(LIBS) rustybits/target/libzeroidc.a
 	# $(STRIP) zerotier-one
 	ln -sf zerotier-one zerotier-idtool
@@ -126,6 +130,15 @@ zerotier-one: one
 
 zeroidc: rustybits/target/libzeroidc.a
 
+ifeq ($(ZT_CONTROLLER),1)
+smeeclient: rustybits/target/libsmeeclient.a
+
+rustybits/target/libsmeeclient.a:	FORCE
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p smeeclient --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)
+	cd rustybits && lipo -create target/x86_64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a target/aarch64-apple-darwin/$(RUST_VARIANT)/libsmeeclient.a -output target/libsmeeclient.a
+endif
+
 rustybits/target/libzeroidc.a:	FORCE
 	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=x86_64-apple-darwin $(EXTRA_CARGO_FLAGS)
 	cd rustybits && MACOSX_DEPLOYMENT_TARGET=$(MACOS_VERSION_MIN) cargo build -p zeroidc --target=aarch64-apple-darwin $(EXTRA_CARGO_FLAGS)
@@ -195,6 +208,10 @@ central-controller-docker: _buildx FORCE
 	docker buildx build --platform linux/arm64,linux/amd64 --no-cache -t registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP} -f ext/central-controller-docker/Dockerfile --build-arg git_branch=$(shell git name-rev --name-only HEAD) . --push
 	@echo Image: registry.zerotier.com/zerotier-central/ztcentral-controller:${TIMESTAMP}
 
+centralv2-controller-docker: _buildx FORCE
+	docker buildx build --platform linux/amd64,linux/arm64 --no-cache -t us-central1-docker.pkg.dev/zerotier-d648c7/central-v2/ztcentral-controller:${TIMESTAMP} -f ext/central-controller-docker/Dockerfile --build-arg git_branch=`git name-rev --name-only HEAD` . --push
+	@echo Image: us-central1-docker.pkg.dev/zerotier-d648c7/central-v2/ztcentral-controller:${TIMESTAMP} 
+
 docker-release:	_buildx
 	docker buildx build --platform linux/386,linux/amd64,linux/arm/v7,linux/arm64,linux/mips64le,linux/ppc64le,linux/s390x -t zerotier/zerotier:${RELEASE_DOCKER_TAG} -t zerotier/zerotier:latest --build-arg VERSION=${RELEASE_VERSION} -f Dockerfile.release . --push
 	

node/Metrics.cpp

@@ -10,8 +10,10 @@
  * of this software will be governed by version 2.0 of the Apache License.
  */
 
-#include <prometheus/histogram.h>
+// clang-format off
 #include <prometheus/simpleapi.h>
+#include <prometheus/histogram.h>
+// clang-format on
 
 namespace prometheus {
 namespace simpleapi {

node/Metrics.hpp

@@ -12,8 +12,10 @@
 #ifndef METRICS_H_
 #define METRICS_H_
 
-#include <prometheus/histogram.h>
+// clang-format off
 #include <prometheus/simpleapi.h>
+#include <prometheus/histogram.h>
+// clang-format on
 
 namespace prometheus {
 namespace simpleapi {

objects.mk

@@ -39,7 +39,10 @@ ONE_OBJS=\
 	controller/DB.o \
 	controller/FileDB.o \
 	controller/LFDB.o \
+	controller/CtlUtil.o \
 	controller/PostgreSQL.o \
+	controller/CV1.o \
+	controller/CV2.o \
 	osdep/EthernetTap.o \
 	osdep/ManagedRoute.o \
 	osdep/Http.o \

osdep/Http.hpp

@@ -19,9 +19,11 @@
 #include <string>
 
 #if defined(_WIN32) || defined(_WIN64)
-#include <windows.h>
+// clang-format off
 #include <winsock2.h>
 #include <ws2tcpip.h>
+#include <windows.h>
+// clang-format on
 #else
 #include <arpa/inet.h>
 #include <netinet/in.h>

rustybits/Cargo.lock

@@ -1,6 +1,6 @@
 # This file is automatically @generated by Cargo.
 # It is not intended for manual editing.
-version = 3
+version = 4
 
 [[package]]
 name = "addr2line"
@@ -287,9 +287,9 @@ dependencies = [
 
 [[package]]
 name = "cc"
-version = "1.2.2"
+version = "1.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f34d93e62b03caf570cccc334cbc6c2fceca82f39211051345108adcba3eebdc"
+checksum = "be714c154be609ec7f5dad223a33bf1482fff90472de28f7362806e6d4832b8c"
 dependencies = [
  "shlex",
 ]
@@ -373,9 +373,9 @@ dependencies = [
 
 [[package]]
 name = "crossbeam-channel"
-version = "0.5.13"
+version = "0.5.15"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "33480d6946193aa8033910124896ca395333cae7e2d1113d1fef6c3272217df2"
+checksum = "82b8f8f868b36967f9606790d1903570de9ceaf870a7bf9fbbd3016d636a2cb2"
 dependencies = [
  "crossbeam-utils",
 ]
@@ -1508,9 +1508,9 @@ dependencies = [
 
 [[package]]
 name = "libc"
-version = "0.2.167"
+version = "0.2.171"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "09d6582e104315a817dff97f75133544b2e094ee22447d2acf4a74e189ba06fc"
+checksum = "c19937216e9d3aa9956d9bb8dfc0b0c8beb6058fc4f7a4dc4d850edf86a237d6"
 
 [[package]]
 name = "libm"
@@ -1804,9 +1804,9 @@ dependencies = [
 
 [[package]]
 name = "openssl"
-version = "0.10.70"
+version = "0.10.72"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "61cfb4e166a8bb8c9b55c500bc2308550148ece889be90f609377e58140f42c6"
+checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da"
 dependencies = [
  "bitflags 2.6.0",
  "cfg-if",
@@ -1836,9 +1836,9 @@ checksum = "ff011a302c396a5197692431fc1948019154afc178baf7d8e37367442a4601cf"
 
 [[package]]
 name = "openssl-sys"
-version = "0.9.105"
+version = "0.9.107"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b22d5b84be05a8d6947c7cb71f7c849aa0f112acd4bf51c2a7c1c988ac0a9dc"
+checksum = "8288979acd84749c744a9014b4382d42b8f7b2592847b5afb2ed29e5d16ede07"
 dependencies = [
  "cc",
  "libc",
@@ -2378,15 +2378,14 @@ dependencies = [
 
 [[package]]
 name = "ring"
-version = "0.17.8"
+version = "0.17.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee"
 dependencies = [
  "cc",
  "cfg-if",
  "getrandom",
  "libc",
- "spin",
  "untrusted",
  "windows-sys 0.52.0",
 ]
@@ -3218,9 +3217,9 @@ dependencies = [
 
 [[package]]
 name = "tokio"
-version = "1.42.0"
+version = "1.43.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5cec9b21b0450273377fc97bd4c33a8acffc8c996c987a7c5b319a0083707551"
+checksum = "492a604e2fd7f814268a378409e6c92b5525d747d10db9a229723f55a417958c"
 dependencies = [
  "backtrace",
  "bytes",
@@ -3236,9 +3235,9 @@ dependencies = [
 
 [[package]]
 name = "tokio-macros"
-version = "2.4.0"
+version = "2.5.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "693d596312e88961bc67d7f1f97af8a70227d9f90c31bba5806eec004978d752"
+checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8"
 dependencies = [
  "proc-macro2",
  "quote",

rustybits/smeeclient/Cargo.toml

@@ -13,7 +13,7 @@ serde = { version = "1", features = ["derive"] }
 temporal-sdk = { git = "https://github.com/temporalio/sdk-core", branch = "master" }
 temporal-client = { git = "https://github.com/temporalio/sdk-core", branch = "master", features = ["telemetry"] }
 temporal-sdk-core-protos = { git = "https://github.com/temporalio/sdk-core", branch = "master" }
-tokio = { version = "1.29", features = ["full"] }
+tokio = { version = "1.43", features = ["full"] }
 url = { version = "2" }
 uuid = { version = "1.4", features = ["v4"] }
 

rustybits/smeeclient/src/lib.rs

@@ -16,7 +16,10 @@ use serde::{Deserialize, Serialize};
 use std::str::FromStr;
 use std::time::Duration;
 use temporal_client::{Client, ClientOptionsBuilder, RetryClient, WorkflowClientTrait, WorkflowOptions};
-use temporal_sdk_core_protos::{coresdk::AsJsonPayloadExt, temporal::api::enums::v1::WorkflowIdReusePolicy};
+use temporal_sdk_core_protos::{
+    coresdk::AsJsonPayloadExt,
+    temporal::api::enums::v1::{WorkflowIdConflictPolicy, WorkflowIdReusePolicy},
+};
 use url::Url;
 use uuid::Uuid;
 
@@ -72,6 +75,7 @@ impl SmeeClient {
         println!("notifying network joined");
         let options = WorkflowOptions {
             id_reuse_policy: WorkflowIdReusePolicy::RejectDuplicate,
+            id_conflict_policy: WorkflowIdConflictPolicy::Fail,
             execution_timeout: None,
             run_timeout: None,
             task_timeout: None,

service/OneService.cpp

@@ -2217,6 +2217,7 @@ class OneServiceImpl : public OneService {
 		auto statusGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
 			ZT_NodeStatus status;
 			_node->status(&status);
+
 			auto out = json::object();
 			char tmp[256] = {};