Clean up and revise how roots are added/specced.

include/ZeroTierCore.h

@@ -175,6 +175,15 @@ extern "C" {
  */
 #define ZT_MAX_CERTIFICATES_OF_OWNERSHIP 4
 
+/**
+ * Maximum size in bytes for a root specification
+ *
+ * A root specification is just a serialized identity followed by a serialized
+ * locator. This provides the maximum size of those plus a lot of extra margin
+ * for any future expansions, but could change in future versions.
+ */
+#define ZT_ROOT_SPEC_MAX_SIZE 8192
+
 /**
  * Packet characteristics flag: packet direction, 1 if inbound 0 if outbound
  */
@@ -1761,28 +1770,29 @@ ZT_SDK_API enum ZT_ResultCode ZT_Node_multicastSubscribe(ZT_Node *node,void *tpt
 ZT_SDK_API enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 
 /**
- * Add a root server (has no effect if already added)
+ * Add a root node or update its locator
  *
  * @param node Node instance
  * @param tptr Thread pointer to pass to functions/callbacks resulting from this call
- * @param identity Identity of this root server
- * @param bootstrap Optional bootstrap address for initial contact
+ * @param rdef Root definition (serialized identity and locator)
+ * @param rdeflen Length of root definition in bytes
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-ZT_SDK_API enum ZT_ResultCode ZT_Node_addRoot(ZT_Node *node,void *tptr,const ZT_Identity *identity,const struct sockaddr_storage *bootstrap);
+ZT_SDK_API enum ZT_ResultCode ZT_Node_addRoot(ZT_Node *node,void *tptr,const void *rdef,unsigned int rdeflen);
 
 /**
- * Remove a root server
+ * Remove a root
  *
- * This removes this node's root designation but does not prevent this node
- * from communicating with it or close active paths to it.
+ * This doesn't fully remove the peer from the peer list. It just removes
+ * its root trust flag. If there is no longer any need to communicate with it
+ * it may gradually time out and be removed.
  *
  * @param node Node instance
  * @param tptr Thread pointer to pass to functions/callbacks resulting from this call
- * @param identity Identity to remove
+ * @param fp Fingerprint of root (will be looked up by address only if hash is all zeroes)
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-ZT_SDK_API enum ZT_ResultCode ZT_Node_removeRoot(ZT_Node *node,void *tptr,const ZT_Identity *identity);
+ZT_SDK_API enum ZT_ResultCode ZT_Node_removeRoot(ZT_Node *node,void *tptr,const ZT_Fingerprint *fp);
 
 /**
  * Get this node's 40-bit ZeroTier address
@@ -2015,6 +2025,19 @@ ZT_SDK_API uint64_t ZT_Identity_address(const ZT_Identity *id);
  */
 ZT_SDK_API const ZT_Fingerprint *ZT_Identity_fingerprint(const ZT_Identity *id);
 
+/**
+ * Make a root specification
+ *
+ * @param id Identity to sign root with (must have private key)
+ * @param ts Timestamp for root specification in milliseconds since epoch
+ * @param addrs Physical addresses for root
+ * @param addrcnt Number of physical addresses for root
+ * @param rootSpecBuf Buffer to receive result, should be at least ZT_ROOT_SPEC_MAX_SIZE bytes
+ * @param rootSpecBufSize Size of rootSpecBuf in bytes
+ * @return Bytes written to rootSpecBuf or -1 on error
+ */
+ZT_SDK_API int ZT_Identity_makeRootSpecification(ZT_Identity *id,int64_t ts,struct sockaddr_storage *addrs,unsigned int addrcnt,void *rootSpecBuf,unsigned int rootSpecBufSize);
+
 /**
  * Delete an identity and free associated memory
  *

node/Fingerprint.hpp

@@ -43,6 +43,13 @@ public:
 	 */
 	ZT_INLINE Fingerprint() noexcept { memoryZero(this); }
 
+	/**
+	 * Create a Fingerprint that is a copy of the external API companion structure
+	 *
+	 * @param apifp API fingerprint
+	 */
+	ZT_INLINE Fingerprint(const ZT_Fingerprint &apifp) noexcept { Utils::copy<sizeof(ZT_Fingerprint)>(&m_cfp,&apifp); }
+
 	ZT_INLINE Address address() const noexcept { return Address(m_cfp.address); }
 	ZT_INLINE const uint8_t *hash() const noexcept { return m_cfp.hash; }
 	ZT_INLINE ZT_Fingerprint *apiFingerprint() noexcept { return &m_cfp; }

node/Identity.cpp

@@ -17,6 +17,8 @@
 #include "Salsa20.hpp"
 #include "Poly1305.hpp"
 #include "Utils.hpp"
+#include "Endpoint.hpp"
+#include "Locator.hpp"
 
 #include <algorithm>
 
@@ -627,6 +629,15 @@ const ZT_Fingerprint *ZT_Identity_fingerprint(const ZT_Identity *id)
 	return reinterpret_cast<const ZeroTier::Identity *>(id)->fingerprint().apiFingerprint();
 }
 
+int ZT_Identity_makeRootSpecification(ZT_Identity *id,int64_t ts,struct sockaddr_storage *addrs,unsigned int addrcnt,void *rootSpecBuf,unsigned int rootSpecBufSize)
+{
+	ZeroTier::Vector<ZeroTier::Endpoint> endpoints;
+	endpoints.reserve(addrcnt);
+	for(unsigned int i=0;i<addrcnt;++i)
+		endpoints.push_back(ZeroTier::Endpoint(ZeroTier::asInetAddress(addrs[i]));
+	return ZeroTier::Locator::makeRootSpecification(reinterpret_cast<const ZeroTier::Identity *>(id),endpoints,rootSpecBuf,rootSpecBufSize);
+}
+
 ZT_SDK_API void ZT_Identity_delete(ZT_Identity *id)
 {
 	if (id)

node/Locator.cpp

@@ -100,10 +100,10 @@ int Locator::unmarshal(const uint8_t *restrict data, const int len) noexcept
 		if (sl > ZT_SIGNATURE_BUFFER_SIZE)
 			return -1;
 		m_signatureLength = sl;
-		if ((p + (int) sl) > len)
+		if ((p + (int)sl) > len)
 			return -1;
 		Utils::copy(m_signature, data + p, sl);
-		p += (int) sl;
+		p += (int)sl;
 
 		if ((p + 2) > len)
 			return -1;
@@ -116,4 +116,42 @@ int Locator::unmarshal(const uint8_t *restrict data, const int len) noexcept
 	return p;
 }
 
+int Locator::makeRootSpecification(const Identity &id,int64_t ts,const Vector<Endpoint> &endpoints,void *rootSpecBuf,unsigned int rootSpecBufSize)
+{
+	if (endpoints.size() > ZT_LOCATOR_MAX_ENDPOINTS)
+		return -1;
+	if (rootSpecBufSize < (ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 1))
+		return -1;
+
+	Locator loc;
+	for (Vector<Endpoint>::const_iterator e(endpoints.begin());e!=endpoints.end();++e)
+		loc.add(*e);
+	if (!loc.sign(ts,id))
+		return -1;
+
+	uint8_t *buf = reinterpret_cast<uint8_t *>(rootSpecBuf);
+	int idl = id.marshal(buf,false);
+	if (idl <= 0)
+		return -1;
+	buf += idl;
+	int locl = loc.marshal(buf);
+	if (locl <= 0)
+		return -1;
+	return idl + locl;
+}
+
+std::pair<Identity,Locator> Locator::parseRootSpecification(const void *rootSpec,unsigned int rootSpecSize)
+{
+	std::pair<Identity,Locator> rs;
+	int l = rs.first.unmarshal(reinterpret_cast<const uint8_t *>(rootSpec),(int)rootSpecSize);
+	if (l <= 0) {
+		rs.first.zero();
+		return rs;
+	}
+	l = rs.second.unmarshal(reinterpret_cast<const uint8_t *>(rootSpec) + l,(int)rootSpecSize - l);
+	if (l <= 0)
+		rs.first.zero();
+	return rs;
+}
+
 } // namespace ZeroTier

node/Locator.hpp

@@ -37,7 +37,7 @@ namespace ZeroTier {
 class Locator : public TriviallyCopyable
 {
 public:
-	ZT_INLINE Locator() noexcept { memoryZero(this); } // NOLINT(cppcoreguidelines-pro-type-member-init,hicpp-member-init)
+	ZT_INLINE Locator() noexcept { memoryZero(this); }
 
 	/**
 	 * Zero the Locator data structure
@@ -116,6 +116,27 @@ public:
 	int marshal(uint8_t data[ZT_LOCATOR_MARSHAL_SIZE_MAX],bool excludeSignature = false) const noexcept;
 	int unmarshal(const uint8_t *restrict data,int len) noexcept;
 
+	/**
+	 * Create a signed Locator and package it with the root's identity to make a root spec
+	 *
+	 * @param id Identity (must have secret)
+	 * @param ts Timestamp
+	 * @param endpoints Endpoints
+	 * @param rootSpecBuf Buffer to store identity and locator into
+	 * @param rootSpecBufSize Size of buffer
+	 * @return Bytes written to buffer or -1 on error
+	 */
+	static int makeRootSpecification(const Identity &id,int64_t ts,const Vector<Endpoint> &endpoints,void *rootSpecBuf,unsigned int rootSpecBufSize);
+
+	/**
+	 * Parse a root specification and decode the identity and locator
+	 *
+	 * @param rootSpec Root spec bytes
+	 * @param rootSpecSize Size in bytes
+	 * @return Identity and locator, with identity NULL if an error occurs
+	 */
+	static std::pair<Identity,Locator> parseRootSpecification(const void *rootSpec,unsigned int rootSpecSize);
+
 private:
 	int64_t m_ts;
 	unsigned int m_endpointCount;

node/Node.cpp

@@ -339,23 +339,23 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u
 	} else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 }
 
-ZT_ResultCode Node::addRoot(void *tPtr,const ZT_Identity *identity,const sockaddr_storage *bootstrap)
-{
-	if (!identity)
-		return ZT_RESULT_ERROR_BAD_PARAMETER;
-	InetAddress a;
-	if (bootstrap)
-		a = bootstrap;
-	RR->topology->addRoot(tPtr,*reinterpret_cast<const Identity *>(identity),a);
-	return ZT_RESULT_OK;
+ZT_ResultCode Node::addRoot(void *tPtr,const void *rdef,unsigned int rdeflen)
+{
+	std::pair<Identity,Locator> r(Locator::parseRootSpecification(rdef,rdeflen));
+	if (r.first) {
+		RR->topology->addRoot(tPtr,r.first,r.second);
+		return ZT_RESULT_OK;
+	}
+	return ZT_RESULT_ERROR_BAD_PARAMETER;
 }
 
-ZT_ResultCode Node::removeRoot(void *tPtr,const ZT_Identity *identity)
+ZT_ResultCode Node::removeRoot(void *tPtr,const ZT_Fingerprint *fp)
 {
-	if (!identity)
-		return ZT_RESULT_ERROR_BAD_PARAMETER;
-	RR->topology->removeRoot(tPtr, *reinterpret_cast<const Identity *>(identity));
-	return ZT_RESULT_OK;
+	if (fp) {
+		RR->topology->removeRoot(tPtr,Fingerprint(*fp));
+		return ZT_RESULT_OK;
+	}
+	return ZT_RESULT_ERROR_BAD_PARAMETER;
 }
 
 uint64_t Node::address() const
@@ -870,10 +870,10 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint
 	}
 }
 
-enum ZT_ResultCode ZT_Node_addRoot(ZT_Node *node,void *tptr,const ZT_Identity *identity,const struct sockaddr_storage *bootstrap)
+enum ZT_ResultCode ZT_Node_addRoot(ZT_Node *node,void *tptr,const void *rdef,unsigned int rdeflen)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addRoot(tptr,identity,bootstrap);
+		return reinterpret_cast<ZeroTier::Node *>(node)->addRoot(tptr,rdef,rdeflen);
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
@@ -881,10 +881,10 @@ enum ZT_ResultCode ZT_Node_addRoot(ZT_Node *node,void *tptr,const ZT_Identity *i
 	}
 }
 
-enum ZT_ResultCode ZT_Node_removeRoot(ZT_Node *node,void *tptr,const ZT_Identity *identity)
+enum ZT_ResultCode ZT_Node_removeRoot(ZT_Node *node,void *tptr,const ZT_Fingerprint *fp)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->removeRoot(tptr,identity);
+		return reinterpret_cast<ZeroTier::Node *>(node)->removeRoot(tptr,fp);
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {

node/Node.hpp

@@ -92,8 +92,8 @@ public:
 	ZT_ResultCode leave(uint64_t nwid,void **uptr,void *tptr);
 	ZT_ResultCode multicastSubscribe(void *tPtr,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 	ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
-	ZT_ResultCode addRoot(void *tPtr,const ZT_Identity *identity,const sockaddr_storage *bootstrap);
-	ZT_ResultCode removeRoot(void *tPtr,const ZT_Identity *identity);
+	ZT_ResultCode addRoot(void *tptr,const void *rdef,unsigned int rdeflen);
+	ZT_ResultCode removeRoot(void *tptr,const ZT_Fingerprint *fp);
 	uint64_t address() const;
 	void status(ZT_NodeStatus *status) const;
 	ZT_PeerList *peers() const;

node/Topology.cpp

@@ -16,8 +16,7 @@
 namespace ZeroTier {
 
 Topology::Topology(const RuntimeEnvironment *renv, void *tPtr) :
-	RR(renv),
-	m_numConfiguredPhysicalPaths(0)
+	RR(renv)
 {
 	uint64_t idtmp[2];
 	idtmp[0] = 0;
@@ -25,29 +24,26 @@ Topology::Topology(const RuntimeEnvironment *renv, void *tPtr) :
 	Vector<uint8_t> data(RR->node->stateObjectGet(tPtr, ZT_STATE_OBJECT_ROOTS, idtmp));
 	if (!data.empty()) {
 		uint8_t *dptr = data.data();
-		int drem = (int) data.size();
-		while (drem > 0) {
+		int drem = (int)data.size();
+		for (;;) {
 			Identity id;
 			int l = id.unmarshal(dptr, drem);
-			if (l > 0) {
-				m_roots.insert(id);
-				dptr += l;
-				drem -= l;
-				ZT_SPEW("loaded root %s", id.address().toString().c_str());
+			if ((l > 0)&&(id)) {
+				if ((drem -= l) <= 0)
+					break;
+				Locator loc;
+				l = loc.unmarshal(dptr, drem);
+				if ((l > 0)&&(loc)) {
+					m_roots[id] = loc;
+					dptr += l;
+					ZT_SPEW("loaded root %s", id.address().toString().c_str());
+					if ((drem -= l) <= 0)
+						break;
+				}
 			}
 		}
 	}
-
-	for (Set<Identity>::const_iterator r(m_roots.begin());r != m_roots.end();++r) {
-		SharedPtr<Peer> p;
-		m_loadCached(tPtr, r->address(), p);
-		if ((!p) || (p->identity() != *r)) {
-			p.set(new Peer(RR));
-			p->init(*r);
-		}
-		m_rootPeers.push_back(p);
-		m_peers[p->address()] = p;
-	}
+	m_updateRootPeers(tPtr);
 }
 
 SharedPtr<Peer> Topology::add(void *tPtr, const SharedPtr<Peer> &peer)
@@ -65,35 +61,13 @@ SharedPtr<Peer> Topology::add(void *tPtr, const SharedPtr<Peer> &peer)
 
 void Topology::setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork, const ZT_PhysicalPathConfiguration *pathConfig)
 {
+	RWMutex::Lock l(m_paths_l);
 	if (!pathNetwork) {
-		m_numConfiguredPhysicalPaths = 0;
+		m_physicalPathConfig.clear();
+	} else if (!pathConfig) {
+		m_physicalPathConfig.erase(asInetAddress(*pathNetwork));
 	} else {
-		std::map<InetAddress, ZT_PhysicalPathConfiguration> cpaths;
-		for (unsigned int i = 0, j = m_numConfiguredPhysicalPaths;i < j;++i)
-			cpaths[m_physicalPathConfig[i].first] = m_physicalPathConfig[i].second;
-
-		if (pathConfig) {
-			ZT_PhysicalPathConfiguration pc(*pathConfig);
-
-			if (pc.mtu <= 0)
-				pc.mtu = ZT_DEFAULT_UDP_MTU;
-			else if (pc.mtu < ZT_MIN_UDP_MTU)
-				pc.mtu = ZT_MIN_UDP_MTU;
-			else if (pc.mtu > ZT_MAX_UDP_MTU)
-				pc.mtu = ZT_MAX_UDP_MTU;
-
-			cpaths[*(reinterpret_cast<const InetAddress *>(pathNetwork))] = pc;
-		} else {
-			cpaths.erase(*(reinterpret_cast<const InetAddress *>(pathNetwork)));
-		}
-
-		unsigned int cnt = 0;
-		for (std::map<InetAddress, ZT_PhysicalPathConfiguration>::const_iterator i(cpaths.begin());((i != cpaths.end()) && (cnt < ZT_MAX_CONFIGURABLE_PATHS));++i) {
-			m_physicalPathConfig[cnt].first = i->first;
-			m_physicalPathConfig[cnt].second = i->second;
-			++cnt;
-		}
-		m_numConfiguredPhysicalPaths = cnt;
+		m_physicalPathConfig[asInetAddress(*pathNetwork)] = *pathConfig;
 	}
 }
 
@@ -109,41 +83,32 @@ struct p_RootSortComparisonOperator
 	}
 };
 
-void Topology::addRoot(void *const tPtr, const Identity &id, const InetAddress &bootstrap)
+void Topology::addRoot(void *const tPtr, const Identity &id, const Locator &loc)
 {
 	if (id == RR->identity)
 		return;
-
 	RWMutex::Lock l1(m_peers_l);
-	std::pair<Set<Identity>::iterator, bool> ir(m_roots.insert(id));
-	if (ir.second) {
-		SharedPtr<Peer> &p = m_peers[id.address()];
-		if (!p) {
-			p.set(new Peer(RR));
-			p->init(id);
-			if (bootstrap)
-				p->setBootstrap(Endpoint(bootstrap));
-		}
-		m_rootPeers.push_back(p);
-		std::sort(m_rootPeers.begin(), m_rootPeers.end(), p_RootSortComparisonOperator());
-		m_writeRootList(tPtr);
-	}
+	m_roots[id] = loc;
+	m_updateRootPeers(tPtr);
+	m_writeRootList(tPtr);
 }
 
-bool Topology::removeRoot(void *const tPtr, const Identity &id)
+bool Topology::removeRoot(void *const tPtr, const Fingerprint &fp)
 {
+	const bool hashIsZero = !fp.haveHash();
 	RWMutex::Lock l1(m_peers_l);
-	Set<Identity>::iterator r(m_roots.find(id));
-	if (r != m_roots.end()) {
-		for (Vector<SharedPtr<Peer> >::iterator p(m_rootPeers.begin());p != m_rootPeers.end();++p) {
-			if ((*p)->identity() == id) {
-				m_rootPeers.erase(p);
-				break;
+	for(Vector< SharedPtr<Peer> >::const_iterator r(m_rootPeers.begin());r!=m_rootPeers.end();++r) {
+		if ((*r)->address() == fp.address()) {
+			if ((hashIsZero)||(fp == (*r)->identity().fingerprint())) {
+				Map<Identity,Locator>::iterator rr(m_roots.find((*r)->identity()));
+				if (rr != m_roots.end()) {
+					m_roots.erase(rr);
+					m_updateRootPeers(tPtr);
+					m_writeRootList(tPtr);
+					return true;
+				}
 			}
 		}
-		m_roots.erase(r);
-		m_writeRootList(tPtr);
-		return true;
 	}
 	return false;
 }
@@ -216,21 +181,44 @@ void Topology::m_loadCached(void *tPtr, const Address &zta, SharedPtr<Peer> &pee
 
 void Topology::m_writeRootList(void *tPtr)
 {
-	// assumes m_peers_l is locked
-	uint8_t *const roots = (uint8_t *) malloc(ZT_IDENTITY_MARSHAL_SIZE_MAX * m_roots.size());
+	// assumes m_peers_l is locked for read or write
+	uint8_t *const roots = (uint8_t *)malloc((ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 2) * m_roots.size());
 	if (roots) { // sanity check
 		int p = 0;
-		for (Set<Identity>::const_iterator i(m_roots.begin());i != m_roots.end();++i) {
-			const int pp = i->marshal(roots + p, false);
-			if (pp > 0)
+		for (Map<Identity,Locator>::const_iterator r(m_roots.begin());r!=m_roots.end();++r) {
+			int pp = r->first.marshal(roots + p, false);
+			if (pp > 0) {
 				p += pp;
+				pp = r->second.marshal(roots + p);
+				if (pp > 0)
+					p += pp;
+			}
 		}
 		uint64_t id[2];
 		id[0] = 0;
 		id[1] = 0;
-		RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_ROOTS, id, roots, (unsigned int) p);
+		RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_ROOTS, id, roots, (unsigned int)p);
 		free(roots);
 	}
 }
 
+void Topology::m_updateRootPeers(void *tPtr)
+{
+	// assumes m_peers_l is locked for write
+	Vector< SharedPtr<Peer> > rp;
+	for (Map<Identity,Locator>::iterator r(m_roots.begin());r!=m_roots.end();++r) {
+		Map< Address,SharedPtr<Peer> >::iterator p(m_peers.find(r->first.address()));
+		if ((p == m_peers.end())||(p->second->identity() != r->first)) {
+			SharedPtr<Peer> np(new Peer(RR));
+			np->init(r->first);
+			m_peers[r->first.address()] = np;
+			rp.push_back(np);
+		} else {
+			rp.push_back(p->second);
+		}
+	}
+	m_rootPeers.swap(rp);
+	std::sort(m_rootPeers.begin(), m_rootPeers.end(), p_RootSortComparisonOperator());
+}
+
 } // namespace ZeroTier

node/Topology.hpp

@@ -188,22 +188,22 @@ public:
 	void setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig);
 
 	/**
-	 * Add a root server's identity to the root server set
+	 * Add or update a root server and its locator
 	 *
 	 * @param tPtr Thread pointer
-	 * @param id Root server identity
-	 * @param bootstrap If non-NULL, a bootstrap address to attempt to find this root
+	 * @param id Root identity
+	 * @param loc Root locator
 	 */
-	void addRoot(void *tPtr,const Identity &id,const InetAddress &bootstrap);
+	void addRoot(void *tPtr,const Identity &id,const Locator &loc);
 
 	/**
 	 * Remove a root server's identity from the root server set
 	 *
 	 * @param tPtr Thread pointer
-	 * @param id Root server identity
+	 * @param fp Root identity
 	 * @return True if root found and removed, false if not found
 	 */
-	bool removeRoot(void *tPtr,const Identity &id);
+	bool removeRoot(void *tPtr,const Fingerprint &fp);
 
 	/**
 	 * Sort roots in ascending order of apparent latency
@@ -225,6 +225,7 @@ public:
 private:
 	void m_loadCached(void *tPtr, const Address &zta, SharedPtr<Peer> &peer);
 	void m_writeRootList(void *tPtr);
+	void m_updateRootPeers(void *tPtr);
 
 	// This gets an integer key from an InetAddress for looking up paths.
 	static ZT_INLINE uint64_t s_getPathKey(const int64_t l,const InetAddress &r) noexcept
@@ -250,16 +251,14 @@ private:
 
 	const RuntimeEnvironment *const RR;
 
-	RWMutex m_paths_l;
-	RWMutex m_peers_l;
-
-	std::pair< InetAddress,ZT_PhysicalPathConfiguration > m_physicalPathConfig[ZT_MAX_CONFIGURABLE_PATHS];
-	unsigned int m_numConfiguredPhysicalPaths;
+	RWMutex m_paths_l; // locks m_physicalPathConfig and m_paths
+	RWMutex m_peers_l; // locks m_peers, m_roots, and m_rootPeers
 
+	Map< InetAddress,ZT_PhysicalPathConfiguration > m_physicalPathConfig;
 	Map< uint64_t,SharedPtr<Path> > m_paths;
 
 	Map< Address,SharedPtr<Peer> > m_peers;
-	Set< Identity > m_roots;
+	Map< Identity,Locator > m_roots;
 	Vector< SharedPtr<Peer> > m_rootPeers;
 };