Cleanup and make moons (federated roots) a little easier to deal with.

include/ZeroTierOne.h

@@ -835,9 +835,9 @@ enum ZT_VirtualNetworkConfigOperation
  */
 enum ZT_PeerRole
 {
-	ZT_PEER_ROLE_LEAF = 0,     // ordinary node
-	ZT_PEER_ROLE_UPSTREAM = 1, // moon root
-	ZT_PEER_ROLE_ROOT = 2      // planetary root
+	ZT_PEER_ROLE_LEAF = 0,       // ordinary node
+	ZT_PEER_ROLE_MOON = 1,       // moon root
+	ZT_PEER_ROLE_PLANET = 2      // planetary root
 };
 
 /**
@@ -1790,10 +1790,11 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint
  * called for each on startup.
  *
  * @param moonWorldId Moon's world ID
+ * @param moonSeed If non-zero, the ZeroTier address of any member of the moon to query for moon definition
  * @param len Length of moonWorld in bytes
  * @return Error if moon was invalid or failed to be added
  */
-enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId);
+enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed);
 
 /**
  * Remove a moon (does nothing if not present)

node/IncomingPacket.cpp

@@ -568,6 +568,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				id.serialize(outp,false);
 				++count;
 			} else {
+				// Request unknown WHOIS from upstream from us (if we have one)
 				RR->sw->requestWhois(addr);
 #ifdef ZT_ENABLE_CLUSTER
 				// Distribute WHOIS queries across a cluster if we do not know the ID.

node/Node.cpp

@@ -156,23 +156,26 @@ ZT_ResultCode Node::processVirtualNetworkFrame(
 class _PingPeersThatNeedPing
 {
 public:
-	_PingPeersThatNeedPing(const RuntimeEnvironment *renv,uint64_t now) :
+	_PingPeersThatNeedPing(const RuntimeEnvironment *renv,Hashtable< Address,std::vector<InetAddress> > &upstreamsToContact,uint64_t now) :
 		lastReceiveFromUpstream(0),
 		RR(renv),
+		_upstreamsToContact(upstreamsToContact),
 		_now(now),
 		_bestCurrentUpstream(RR->topology->getUpstreamPeer())
 	{
-		RR->topology->getUpstreamStableEndpoints(_upstreams);
 	}
 
 	uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		const std::vector<InetAddress> *const upstreamStableEndpoints = _upstreams.get(p->address());
+		const std::vector<InetAddress> *const upstreamStableEndpoints = _upstreamsToContact.get(p->address());
 		if (upstreamStableEndpoints) {
 			bool contacted = false;
 
+			// Upstreams must be pinged constantly over both IPv4 and IPv6 to allow
+			// them to perform three way handshake introductions for both stacks.
+
 			if (!p->doPingAndKeepalive(_now,AF_INET)) {
 				for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) {
 					const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()];
@@ -183,7 +186,6 @@ public:
 					}
 				}
 			} else contacted = true;
-
 			if (!p->doPingAndKeepalive(_now,AF_INET6)) {
 				for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) {
 					const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()];
@@ -202,6 +204,7 @@ public:
 			}
 
 			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
+			_upstreamsToContact.erase(p->address()); // erase from upstreams to contact so that we can WHOIS those that remain
 		} else if (p->isActive(_now)) {
 			p->doPingAndKeepalive(_now,-1);
 		}
@@ -209,9 +212,9 @@ public:
 
 private:
 	const RuntimeEnvironment *RR;
+	Hashtable< Address,std::vector<InetAddress> > &_upstreamsToContact;
 	const uint64_t _now;
 	const SharedPtr<Peer> _bestCurrentUpstream;
-	Hashtable< Address,std::vector<InetAddress> > _upstreams;
 };
 
 ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
@@ -238,17 +241,19 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
 				(*n)->requestConfiguration();
 
-			// Attempt to get identity for any unknown upstreams
-			const std::vector<Address> upstreams(RR->topology->upstreamAddresses());
-			for(std::vector<Address>::const_iterator a(upstreams.begin());a!=upstreams.end();++a) {
-				if (!RR->topology->getPeer(*a))
-					RR->sw->requestWhois(*a);
-			}
-
 			// Do pings and keepalives
-			_PingPeersThatNeedPing pfunc(RR,now);
+			Hashtable< Address,std::vector<InetAddress> > upstreamsToContact;
+			RR->topology->getUpstreamsToContact(upstreamsToContact);
+			_PingPeersThatNeedPing pfunc(RR,upstreamsToContact,now);
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
+			// Run WHOIS to create Peer for any upstreams we could not contact (including pending moon seeds)
+			Hashtable< Address,std::vector<InetAddress> >::Iterator i(upstreamsToContact);
+			Address *upstreamAddress = (Address *)0;
+			std::vector<InetAddress> *upstreamStableEndpoints = (std::vector<InetAddress> *)0;
+			while (i.next(upstreamAddress,upstreamStableEndpoints))
+				RR->sw->requestWhois(*upstreamAddress);
+
 			// Update online status, post status change as event
 			const bool oldOnline = _online;
 			_online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot()));
@@ -337,9 +342,9 @@ ZT_ResultCode Node::multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,u
 	} else return ZT_RESULT_ERROR_NETWORK_NOT_FOUND;
 }
 
-ZT_ResultCode Node::orbit(uint64_t moonWorldId)
+ZT_ResultCode Node::orbit(uint64_t moonWorldId,uint64_t moonSeed)
 {
-	RR->topology->addMoon(moonWorldId);
+	RR->topology->addMoon(moonWorldId,Address(moonSeed));
 	return ZT_RESULT_OK;
 }
 
@@ -919,10 +924,10 @@ enum ZT_ResultCode ZT_Node_multicastUnsubscribe(ZT_Node *node,uint64_t nwid,uint
 	}
 }
 
-enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId)
+enum ZT_ResultCode ZT_Node_orbit(ZT_Node *node,uint64_t moonWorldId,uint64_t moonSeed)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->orbit(moonWorldId);
+		return reinterpret_cast<ZeroTier::Node *>(node)->orbit(moonWorldId,moonSeed);
 	} catch ( ... ) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}

node/Node.hpp

@@ -95,7 +95,7 @@ public:
 	ZT_ResultCode leave(uint64_t nwid,void **uptr);
 	ZT_ResultCode multicastSubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 	ZT_ResultCode multicastUnsubscribe(uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
-	ZT_ResultCode orbit(uint64_t moonWorldId);
+	ZT_ResultCode orbit(uint64_t moonWorldId,uint64_t moonSeed);
 	ZT_ResultCode deorbit(uint64_t moonWorldId);
 	uint64_t address() const;
 	void status(ZT_NodeStatus *status) const;

node/Topology.hpp

@@ -170,14 +170,11 @@ public:
 	bool isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const;
 
 	/**
-	 * This gets the known stable endpoints for any upstream
-	 *
-	 * It also adds empty entries for any upstreams we are attempting to
-	 * contact.
+	 * Gets upstreams to contact and their stable endpoints (if known)
 	 *
 	 * @param eps Hash table to fill with addresses and their stable endpoints
 	 */
-	inline void getUpstreamStableEndpoints(Hashtable< Address,std::vector<InetAddress> > &eps) const
+	inline void getUpstreamsToContact(Hashtable< Address,std::vector<InetAddress> > &eps) const
 	{
 		Mutex::Lock _l(_upstreams_m);
 		for(std::vector<World::Root>::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) {
@@ -196,8 +193,8 @@ public:
 				}
 			}
 		}
-		for(std::vector<Address>::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m)
-			eps[*m];
+		for(std::vector< std::pair<uint64_t,Address> >::const_iterator m(_moonSeeds.begin());m!=_moonSeeds.end();++m)
+			eps[m->second];
 	}
 
 	/**
@@ -206,12 +203,7 @@ public:
 	inline std::vector<Address> upstreamAddresses() const
 	{
 		Mutex::Lock _l(_upstreams_m);
-		std::vector<Address> u(_upstreamAddresses);
-		for(std::vector<Address>::const_iterator m(_contactingMoons.begin());m!=_contactingMoons.end();++m) {
-			if (std::find(u.begin(),u.end(),*m) == u.end())
-				u.push_back(*m);
-		}
-		return u;
+		return _upstreamAddresses;
 	}
 
 	/**
@@ -260,13 +252,12 @@ public:
 	 * Add a moon
 	 *
 	 * This loads it from moons.d if present, and if not adds it to
-	 * a list of moons that we want to contact. It does not actually
-	 * send anything, though this will happen on the next background
-	 * task loop where pings etc. are checked.
+	 * a list of moons that we want to contact.
 	 *
 	 * @param id Moon ID
+	 * @param seed If non-NULL, an address of any member of the moon to contact
 	 */
-	void addMoon(const uint64_t id);
+	void addMoon(const uint64_t id,const Address &seed);
 
 	/**
 	 * Remove a moon
@@ -422,7 +413,7 @@ private:
 
 	World _planet;
 	std::vector<World> _moons;
-	std::vector<Address> _contactingMoons;
+	std::vector< std::pair<uint64_t,Address> > _moonSeeds;
 	std::vector<Address> _upstreamAddresses;
 	CertificateOfRepresentation _cor;
 	bool _amRoot;

node/Utils.cpp

@@ -47,21 +47,14 @@ namespace ZeroTier {
 
 const char Utils::HEXCHARS[16] = { '0','1','2','3','4','5','6','7','8','9','a','b','c','d','e','f' };
 
-static void _Utils_doBurn(char *ptr,unsigned int len)
+// Crazy hack to force memory to be securely zeroed in spite of the best efforts of optimizing compilers.
+static void _Utils_doBurn(volatile uint8_t *ptr,unsigned int len)
 {
-	for(unsigned int i=0;i<len;++i)
-		ptr[i] = (char)0;
-}
-void (*volatile _Utils_doBurn_ptr)(char *,unsigned int) = _Utils_doBurn;
-void Utils::burn(void *ptr,unsigned int len)
-	throw()
-{
-	// Ridiculous hack: call _doBurn() via a volatile function pointer to
-	// hold down compiler optimizers and beat them mercilessly until they
-	// cry and mumble something about never eliding secure memory zeroing
-	// again.
-	(_Utils_doBurn_ptr)((char *)ptr,len);
+	volatile uint8_t *const end = ptr + len;
+	while (ptr != end) *(ptr++) = (uint8_t)0;
 }
+static void (*volatile _Utils_doBurn_ptr)(volatile uint8_t *,unsigned int) = _Utils_doBurn;
+void Utils::burn(void *ptr,unsigned int len) { (_Utils_doBurn_ptr)((volatile uint8_t *)ptr,len); }
 
 std::string Utils::hex(const void *data,unsigned int len)
 {

node/Utils.hpp

@@ -59,8 +59,7 @@ public:
 	/**
 	 * Securely zero memory, avoiding compiler optimizations and such
 	 */
-	static void burn(void *ptr,unsigned int len)
-		throw();
+	static void burn(void *ptr,unsigned int len);
 
 	/**
 	 * Convert binary data to hexadecimal

service/ControlPlane.cpp

@@ -226,8 +226,8 @@ static void _jsonAppend(unsigned int depth,std::string &buf,const ZT_Peer *peer)
 	const char *prole = "";
 	switch(peer->role) {
 		case ZT_PEER_ROLE_LEAF: prole = "LEAF"; break;
-		case ZT_PEER_ROLE_UPSTREAM: prole = "UPSTREAM"; break;
-		case ZT_PEER_ROLE_ROOT: prole = "ROOT"; break;
+		case ZT_PEER_ROLE_MOON: prole = "MOON"; break;
+		case ZT_PEER_ROLE_PLANET: prole = "PLANET"; break;
 	}
 
 	Utils::snprintf(json,sizeof(json),

service/OneService.cpp

@@ -707,7 +707,7 @@ public:
 				for(std::vector<std::string>::iterator f(moonsDotD.begin());f!=moonsDotD.end();++f) {
 					std::size_t dot = f->find_last_of('.');
 					if ((dot == 16)&&(f->substr(16) == ".moon"))
-						_node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str()));
+						_node->orbit(Utils::hexStrToU64(f->substr(0,dot).c_str()),0);
 				}
 			}