Replace piecemeal designation of upstreams with the concept of moons, which is simpler and easier to use and inherits all the cool live update stuff of worlds (now called planets) and global roots.

include/ZeroTierOne.h

@@ -848,8 +848,8 @@ enum ZT_VirtualNetworkConfigOperation
 enum ZT_PeerRole
 {
 	ZT_PEER_ROLE_LEAF = 0,     // ordinary node
-	ZT_PEER_ROLE_UPSTREAM = 1, // upstream node
-	ZT_PEER_ROLE_ROOT = 2      // global root
+	ZT_PEER_ROLE_UPSTREAM = 1, // moon root
+	ZT_PEER_ROLE_ROOT = 2      // planetary root
 };
 
 /**
@@ -1903,17 +1903,6 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node);
  */
 int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const void *data,unsigned int len);
 
-/**
- * Set peer role
- *
- * Right now this can only be used to set a peer to either LEAF or
- * UPSTREAM, since roots are fixed and defined by the World.
- *
- * @param ztAddress ZeroTier address (least significant 40 bits)
- * @param role New peer role (LEAF or UPSTREAM)
- */
-void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,enum ZT_PeerRole role);
-
 /**
  * Set a network configuration master instance for this node
  *

node/IncomingPacket.cpp

@@ -214,8 +214,8 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut
 
 		Identity id;
 		InetAddress externalSurfaceAddress;
-		uint64_t worldId = ZT_WORLD_ID_NULL;
-		uint64_t worldTimestamp = 0;
+		uint64_t planetWorldId = 0;
+		uint64_t planetWorldTimestamp = 0;
 		{
 			unsigned int ptr = ZT_PROTO_VERB_HELLO_IDX_IDENTITY + id.deserialize(*this,ZT_PROTO_VERB_HELLO_IDX_IDENTITY);
 
@@ -223,10 +223,10 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut
 			if (ptr < size())
 				ptr += externalSurfaceAddress.deserialize(*this,ptr);
 
-			// Get world ID and world timestamp if present (was not in old versions)
+			// Get primary planet world ID and world timestamp if present
 			if ((ptr + 16) <= size()) {
-				worldId = at<uint64_t>(ptr); ptr += 8;
-				worldTimestamp = at<uint64_t>(ptr);
+				planetWorldId = at<uint64_t>(ptr); ptr += 8;
+				planetWorldTimestamp = at<uint64_t>(ptr);
 			}
 		}
 
@@ -356,14 +356,14 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,const bool alreadyAut
 			tmpa.serialize(outp);
 		}
 
-		if ((worldId != ZT_WORLD_ID_NULL)&&(RR->topology->worldTimestamp() > worldTimestamp)&&(worldId == RR->topology->worldId())) {
-			World w(RR->topology->world());
+		if ((planetWorldId)&&(RR->topology->planetWorldTimestamp() > planetWorldTimestamp)&&(planetWorldId == RR->topology->planetWorldId())) {
+			World w(RR->topology->planet());
 			const unsigned int sizeAt = outp.size();
 			outp.addSize(2); // make room for 16-bit size field
 			w.serialize(outp,false);
 			outp.setAt<uint16_t>(sizeAt,(uint16_t)(outp.size() - (sizeAt + 2)));
 		} else {
-			outp.append((uint16_t)0); // no world update needed
+			outp.append((uint16_t)0); // no planet update needed
 		}
 
 		outp.armor(peer->key(),true);
@@ -411,14 +411,14 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 				if (ptr < size())
 					ptr += externalSurfaceAddress.deserialize(*this,ptr);
 
-				// Handle world updates from root servers if present (was not on old versions)
-				if (((ptr + 2) <= size())&&(RR->topology->isRoot(peer->identity()))) {
+				// Handle planet or moon updates
+				if ((ptr + 2) <= size()) {
 					World worldUpdate;
 					const unsigned int worldLen = at<uint16_t>(ptr); ptr += 2;
 					if (worldLen > 0) {
 						World w;
 						w.deserialize(*this,ptr);
-						RR->topology->worldUpdateIfValid(w);
+						RR->topology->addWorld(w,true);
 					}
 				}
 

node/Node.cpp

@@ -160,75 +160,48 @@ ZT_ResultCode Node::processVirtualNetworkFrame(
 class _PingPeersThatNeedPing
 {
 public:
-	_PingPeersThatNeedPing(const RuntimeEnvironment *renv,const std::vector<Address> &upstreams,uint64_t now) :
+	_PingPeersThatNeedPing(const RuntimeEnvironment *renv,uint64_t now) :
 		lastReceiveFromUpstream(0),
 		RR(renv),
-		_upstreams(upstreams),
-		_now(now),
-		_world(RR->topology->world())
+		_now(now)
 	{
+		RR->topology->getUpstreamStableEndpoints(_upstreams);
 	}
 
 	uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		if (std::find(_upstreams.begin(),_upstreams.end(),p->address()) != _upstreams.end()) {
-			InetAddress stableEndpoint4,stableEndpoint6;
-			for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
-				if (r->identity == p->identity()) {
-					for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)r->stableEndpoints.size();++k) {
-						const InetAddress &addr = r->stableEndpoints[ptr++ % r->stableEndpoints.size()];
-						if (!stableEndpoint4) {
-							if (addr.ss_family == AF_INET)
-								stableEndpoint4 = addr;
-						}
-						if (!stableEndpoint6) {
-							if (addr.ss_family == AF_INET6)
-								stableEndpoint6 = addr;
-						}
+		const std::vector<InetAddress> *upstreamStableEndpoints = _upstreams.get(p->address());
+		if ((upstreamStableEndpoints)&&(upstreamStableEndpoints->size() > 0)) {
+			if (!p->doPingAndKeepalive(_now,AF_INET)) {
+				for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) {
+					const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()];
+					if (addr.ss_family == AF_INET) {
+						p->sendHELLO(InetAddress(),addr,_now);
+						break;
 					}
-					break;
 				}
 			}
-
-			// We keep connections to upstream peers alive forever.
-			bool needToContactIndirect = true;
-			if (p->doPingAndKeepalive(_now,AF_INET)) {
-				needToContactIndirect = false;
-			} else {
-				if (stableEndpoint4) {
-					needToContactIndirect = false;
-					p->sendHELLO(InetAddress(),stableEndpoint4,_now);
-				}
-			}
-			if (p->doPingAndKeepalive(_now,AF_INET6)) {
-				needToContactIndirect = false;
-			} else {
-				if (stableEndpoint6) {
-					needToContactIndirect = false;
-					p->sendHELLO(InetAddress(),stableEndpoint6,_now);
+			if (!p->doPingAndKeepalive(_now,AF_INET6)) {
+				for(unsigned long k=0,ptr=(unsigned long)RR->node->prng();k<(unsigned long)upstreamStableEndpoints->size();++k) {
+					const InetAddress &addr = (*upstreamStableEndpoints)[ptr++ % upstreamStableEndpoints->size()];
+					if (addr.ss_family == AF_INET6) {
+						p->sendHELLO(InetAddress(),addr,_now);
+						break;
+					}
 				}
 			}
-
-			// If we don't have a direct path or a static endpoint, send something indirectly to find one.
-			if (needToContactIndirect) {
-				Packet outp(p->address(),RR->identity.address(),Packet::VERB_NOP);
-				RR->sw->send(outp,true);
-			}
-
 			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
 		} else if (p->isActive(_now)) {
-			// Normal nodes get their preferred link kept alive if the node has generated frame traffic recently
 			p->doPingAndKeepalive(_now,-1);
 		}
 	}
 
 private:
 	const RuntimeEnvironment *RR;
-	const std::vector<Address> &_upstreams;
 	uint64_t _now;
-	World _world;
+	Hashtable< Address,std::vector<InetAddress> > _upstreams;
 };
 
 ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
@@ -263,7 +236,7 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			}
 
 			// Do pings and keepalives
-			_PingPeersThatNeedPing pfunc(RR,upstreams,now);
+			_PingPeersThatNeedPing pfunc(RR,now);
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
 			// Update online status, post status change as event
@@ -368,8 +341,8 @@ uint64_t Node::address() const
 void Node::status(ZT_NodeStatus *status) const
 {
 	status->address = RR->identity.address().toInt();
-	status->worldId = RR->topology->worldId();
-	status->worldTimestamp = RR->topology->worldTimestamp();
+	status->worldId = RR->topology->planetWorldId();
+	status->worldTimestamp = RR->topology->planetWorldTimestamp();
 	status->publicIdentity = RR->publicIdentityStr.c_str();
 	status->secretIdentity = RR->secretIdentityStr.c_str();
 	status->relayPolicy = _relayPolicy;
@@ -401,7 +374,7 @@ ZT_PeerList *Node::peers() const
 			p->versionRev = -1;
 		}
 		p->latency = pi->second->latency();
-		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : (RR->topology->isUpstream(pi->second->identity()) ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF);
+		p->role = RR->topology->role(pi->second->identity().address());
 
 		std::vector< std::pair< SharedPtr<Path>,bool > > paths(pi->second->paths(_now));
 		SharedPtr<Path> bestp(pi->second->getBestPath(_now,false));
@@ -488,11 +461,6 @@ int Node::sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigne
 	return 0;
 }
 
-void Node::setRole(uint64_t ztAddress,ZT_PeerRole role)
-{
-	RR->topology->setUpstream(Address(ztAddress),(role == ZT_PEER_ROLE_UPSTREAM));
-}
-
 void Node::setNetconfMaster(void *networkControllerInstance)
 {
 	RR->localNetworkController = reinterpret_cast<NetworkController *>(networkControllerInstance);
@@ -1016,13 +984,6 @@ int ZT_Node_sendUserMessage(ZT_Node *node,uint64_t dest,uint64_t typeId,const vo
 	}
 }
 
-void ZT_Node_setRole(ZT_Node *node,uint64_t ztAddress,ZT_PeerRole role)
-{
-	try {
-		reinterpret_cast<ZeroTier::Node *>(node)->setRole(ztAddress,role);
-	} catch ( ... ) {}
-}
-
 void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 {
 	try {

node/Node.hpp

@@ -105,7 +105,6 @@ public:
 	int addLocalInterfaceAddress(const struct sockaddr_storage *addr);
 	void clearLocalInterfaceAddresses();
 	int sendUserMessage(uint64_t dest,uint64_t typeId,const void *data,unsigned int len);
-	void setRole(uint64_t ztAddress,ZT_PeerRole role);
 	void setNetconfMaster(void *networkControllerInstance);
 	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
 	void circuitTestEnd(ZT_CircuitTest *test);

node/Peer.cpp

@@ -37,9 +37,6 @@
 
 namespace ZeroTier {
 
-// Used to send varying values for NAT keepalive
-static uint32_t _natKeepaliveBuf = 0;
-
 Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) :
 	_lastReceive(0),
 	_lastNontrivialReceive(0),
@@ -355,8 +352,8 @@ void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,u
 	outp.append(now);
 	RR->identity.serialize(outp,false);
 	atAddress.serialize(outp);
-	outp.append((uint64_t)RR->topology->worldId());
-	outp.append((uint64_t)RR->topology->worldTimestamp());
+	outp.append((uint64_t)RR->topology->planetWorldId());
+	outp.append((uint64_t)RR->topology->planetWorldTimestamp());
 	RR->node->expectReplyTo(outp.packetId());
 	outp.armor(_key,false); // HELLO is sent in the clear
 	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
@@ -401,12 +398,9 @@ bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily)
 	}
 
 	if (bestp >= 0) {
-		if ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) {
+		if ( ((now - _paths[bestp].lastReceive) >= ZT_PEER_PING_PERIOD) || (_paths[bestp].path->needsHeartbeat(now)) ) {
 			attemptToContactAt(_paths[bestp].path->localAddress(),_paths[bestp].path->address(),now);
 			_paths[bestp].path->sent(now);
-		} else if (_paths[bestp].path->needsHeartbeat(now)) {
-			_natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads
-			_paths[bestp].path->send(RR,&_natKeepaliveBuf,sizeof(_natKeepaliveBuf),now);
 		}
 		return true;
 	} else {

node/Switch.cpp

@@ -718,7 +718,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt)
 		 * go somewhere. */
 
 		SharedPtr<Path> viaPath(peer->getBestPath(now,false));
-		if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isRoot(peer->identity())) ) {
+		if ( (viaPath) && (!viaPath->alive(now)) && (!RR->topology->isUpstream(peer->identity())) ) {
 			if ((now - viaPath->lastOut()) > std::max((now - viaPath->lastIn()) * 4,(uint64_t)ZT_PATH_MIN_REACTIVATE_INTERVAL))
 				peer->attemptToContactAt(viaPath->localAddress(),viaPath->address(),now);
 			viaPath.zero();

node/Topology.cpp

@@ -48,33 +48,22 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 	_trustedPathCount(0),
 	_amRoot(false)
 {
-	// Get cached world if present
-	std::string dsWorld(RR->node->dataStoreGet("world"));
-	World cachedWorld;
-	if (dsWorld.length() > 0) {
-		try {
-			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp(dsWorld.data(),(unsigned int)dsWorld.length());
-			cachedWorld.deserialize(dswtmp,0);
-		} catch ( ... ) {
-			cachedWorld = World(); // clear if cached world is invalid
-		}
-	}
-
-	// Use default or cached world depending on which is shinier
-	World defaultWorld;
+	World defaultPlanet;
 	{
 		Buffer<ZT_DEFAULT_WORLD_LENGTH> wtmp(ZT_DEFAULT_WORLD,ZT_DEFAULT_WORLD_LENGTH);
-		defaultWorld.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top
+		defaultPlanet.deserialize(wtmp,0); // throws on error, which would indicate a bad static variable up top
 	}
-	if (cachedWorld.shouldBeReplacedBy(defaultWorld,false)) {
-		_setWorld(defaultWorld);
-		if (dsWorld.length() > 0)
-			RR->node->dataStoreDelete("world");
-	} else _setWorld(cachedWorld);
-}
+	addWorld(defaultPlanet,false);
 
-Topology::~Topology()
-{
+	try {
+		World cachedPlanet;
+		std::string buf(RR->node->dataStoreGet("planet"));
+		if (buf.length() > 0) {
+			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp(buf.data(),(unsigned int)buf.length());
+			cachedPlanet.deserialize(dswtmp,0);
+		}
+		addWorld(cachedPlanet,false);
+	} catch ( ... ) {}
 }
 
 SharedPtr<Peer> Topology::addPeer(const SharedPtr<Peer> &peer)
@@ -161,15 +150,14 @@ SharedPtr<Peer> Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi
 	Mutex::Lock _l(_lock);
 
 	if (_amRoot) {
-		/* If I am a root server, the "best" root server is the one whose address
-		 * is numerically greater than mine (with wrap at top of list). This
-		 * causes packets searching for a route to pretty much literally
-		 * circumnavigate the globe rather than bouncing between just two. */
-
-		for(unsigned long p=0;p<_rootAddresses.size();++p) {
-			if (_rootAddresses[p] == RR->identity.address()) {
-				for(unsigned long q=1;q<_rootAddresses.size();++q) {
-					const SharedPtr<Peer> *const nextsn = _peers.get(_rootAddresses[(p + q) % _rootAddresses.size()]);
+		/* If I am a root, pick another root that isn't mine and that
+		 * has a numerically greater ID. This causes packets to roam
+		 * around the top rather than bouncing between just two. */
+
+		for(unsigned long p=0;p<_upstreamAddresses.size();++p) {
+			if (_upstreamAddresses[p] == RR->identity.address()) {
+				for(unsigned long q=1;q<_upstreamAddresses.size();++q) {
+					const SharedPtr<Peer> *const nextsn = _peers.get(_upstreamAddresses[(p + q) % _upstreamAddresses.size()]);
 					if ((nextsn)&&((*nextsn)->hasActiveDirectPath(now)))
 						return *nextsn;
 				}
@@ -178,8 +166,7 @@ SharedPtr<Peer> Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi
 		}
 
 	} else {
-		/* Otherwise pick the best upstream from among roots and any other
-		 * designated upstreams that we trust. */
+		/* Otherwise pick the bestest looking upstream */
 
 		unsigned int bestQualityOverall = ~((unsigned int)0);
 		unsigned int bestQualityNotAvoid = ~((unsigned int)0);
@@ -219,82 +206,112 @@ SharedPtr<Peer> Topology::getUpstreamPeer(const Address *avoid,unsigned int avoi
 	return SharedPtr<Peer>();
 }
 
-bool Topology::isRoot(const Identity &id) const
-{
-	Mutex::Lock _l(_lock);
-	return (std::find(_rootAddresses.begin(),_rootAddresses.end(),id.address()) != _rootAddresses.end());
-}
-
 bool Topology::isUpstream(const Identity &id) const
 {
 	Mutex::Lock _l(_lock);
 	return (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),id.address()) != _upstreamAddresses.end());
 }
 
-void Topology::setUpstream(const Address &a,bool upstream)
+ZT_PeerRole Topology::role(const Address &ztaddr) const
 {
-	bool needWhois = false;
-	{
-		Mutex::Lock _l(_lock);
-		if (std::find(_rootAddresses.begin(),_rootAddresses.end(),a) == _rootAddresses.end()) {
-			if (upstream) {
-				if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),a) == _upstreamAddresses.end()) {
-					_upstreamAddresses.push_back(a);
-					const SharedPtr<Peer> *p = _peers.get(a);
-					if (!p) {
-						const Identity id(_getIdentity(a));
-						if (id) {
-							_peers.set(a,SharedPtr<Peer>(new Peer(RR,RR->identity,id)));
-						} else {
-							needWhois = true; // need to do this later due to _lock
-						}
-					}
-				}
-			} else {
-				std::vector<Address> ua;
-				for(std::vector<Address>::iterator i(_upstreamAddresses.begin());i!=_upstreamAddresses.end();++i) {
-					if (a != *i)
-						ua.push_back(*i);
-				}
-				_upstreamAddresses.swap(ua);
-			}
+	Mutex::Lock _l(_lock);
+	if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),ztaddr) != _upstreamAddresses.end()) {
+		for(std::vector<World::Root>::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) {
+			if (i->identity.address() == ztaddr)
+				return ZT_PEER_ROLE_ROOT;
 		}
+		return ZT_PEER_ROLE_UPSTREAM;
 	}
-	if (needWhois)
-		RR->sw->requestWhois(a);
+	return ZT_PEER_ROLE_LEAF;
 }
 
 bool Topology::isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const
 {
 	Mutex::Lock _l(_lock);
 
-	if (std::find(_rootAddresses.begin(),_rootAddresses.end(),ztaddr) != _rootAddresses.end()) {
-		for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
+	// For roots the only permitted addresses are those defined. This adds just a little
+	// bit of extra security against spoofing, replaying, etc.
+	if (std::find(_upstreamAddresses.begin(),_upstreamAddresses.end(),ztaddr) != _upstreamAddresses.end()) {
+		for(std::vector<World::Root>::const_iterator r(_planet.roots().begin());r!=_planet.roots().end();++r) {
 			for(std::vector<InetAddress>::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) {
 				if (ipaddr.ipsEqual(*e))
 					return false;
 			}
 		}
+		for(std::vector<World>::const_iterator m(_moons.begin());m!=_moons.end();++m) {
+			for(std::vector<World::Root>::const_iterator r(m->roots().begin());r!=m->roots().end();++r) {
+				for(std::vector<InetAddress>::const_iterator e(r->stableEndpoints.begin());e!=r->stableEndpoints.end();++e) {
+					if (ipaddr.ipsEqual(*e))
+						return false;
+				}
+			}
+		}
 		return true;
 	}
 
 	return false;
 }
 
-bool Topology::worldUpdateIfValid(const World &newWorld)
+bool Topology::addWorld(const World &newWorld,bool updateOnly)
 {
+	if ((newWorld.type() != World::TYPE_PLANET)&&(newWorld.type() != World::TYPE_MOON))
+		return false;
+
 	Mutex::Lock _l(_lock);
-	if (_world.shouldBeReplacedBy(newWorld,true)) {
-		_setWorld(newWorld);
-		try {
-			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp;
-			newWorld.serialize(dswtmp,false);
-			RR->node->dataStorePut("world",dswtmp.data(),dswtmp.size(),false);
-		} catch ( ... ) {
-			RR->node->dataStoreDelete("world");
+
+	World *existing = (World *)0;
+	switch(newWorld.type()) {
+		case World::TYPE_PLANET:
+			existing = &_planet;
+			break;
+		case World::TYPE_MOON:
+			for(std::vector< World >::iterator m(_moons.begin());m!=_moons.end();++m) {
+				if (m->id() == newWorld.id()) {
+					existing = &(*m);
+					break;
+				}
+			}
+			break;
+		default:
+			return false;
+	}
+
+	if (existing) {
+		if (existing->shouldBeReplacedBy(newWorld))
+			*existing = newWorld;
+		else return false;
+	} else if ((newWorld.type() == World::TYPE_MOON)&&(!updateOnly)) {
+		_moons.push_back(newWorld);
+		existing = &(_moons.back());
+	} else return false;
+
+	char savePath[64];
+	if (existing->type() == World::TYPE_MOON)
+		Utils::snprintf(savePath,sizeof(savePath),"moons.d/%.16llx",existing->id());
+	else Utils::scopy(savePath,sizeof(savePath),"planet");
+	try {
+		Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> dswtmp;
+		existing->serialize(dswtmp,false);
+		RR->node->dataStorePut(savePath,dswtmp.data(),dswtmp.size(),false);
+	} catch ( ... ) {
+		RR->node->dataStoreDelete(savePath);
+	}
+
+	_upstreamAddresses.clear();
+	_amRoot = false;
+	for(std::vector<World::Root>::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) {
+		if (i->identity == RR->identity)
+			_amRoot = true;
+		else _upstreamAddresses.push_back(i->identity.address());
+	}
+	for(std::vector<World>::const_iterator m(_moons.begin());m!=_moons.end();++m) {
+		for(std::vector<World::Root>::const_iterator i(m->roots().begin());i!=m->roots().end();++i) {
+			if (i->identity == RR->identity)
+				_amRoot = true;
+			else _upstreamAddresses.push_back(i->identity.address());
 		}
-		return true;
 	}
+
 	return false;
 }
 
@@ -334,34 +351,4 @@ Identity Topology::_getIdentity(const Address &zta)
 	return Identity();
 }
 
-void Topology::_setWorld(const World &newWorld)
-{
-	// assumed _lock is locked (or in constructor)
-
-	std::vector<Address> ua;
-	for(std::vector<Address>::iterator a(_upstreamAddresses.begin());a!=_upstreamAddresses.end();++a) {
-		if (std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())
-			ua.push_back(*a);
-	}
-
-	_world = newWorld;
-	_rootAddresses.clear();
-	_amRoot = false;
-
-	for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
-		_rootAddresses.push_back(r->identity.address());
-		if (std::find(ua.begin(),ua.end(),r->identity.address()) == ua.end())
-			ua.push_back(r->identity.address());
-		if (r->identity.address() == RR->identity.address()) {
-			_amRoot = true;
-		} else {
-			SharedPtr<Peer> *rp = _peers.get(r->identity.address());
-			if (!rp)
-				_peers.set(r->identity.address(),SharedPtr<Peer>(new Peer(RR,RR->identity,r->identity)));
-		}
-	}
-
-	_upstreamAddresses.swap(ua);
-}
-
 } // namespace ZeroTier

node/Topology.hpp

@@ -50,7 +50,6 @@ class Topology
 {
 public:
 	Topology(const RuntimeEnvironment *renv);
-	~Topology();
 
 	/**
 	 * Add a peer to database
@@ -141,12 +140,6 @@ public:
 	 */
 	SharedPtr<Peer> getUpstreamPeer(const Address *avoid,unsigned int avoidCount,bool strictAvoid);
 
-	/**
-	 * @param id Identity to check
-	 * @return True if this is a designated root server in this world
-	 */
-	bool isRoot(const Identity &id) const;
-
 	/**
 	 * @param id Identity to check
 	 * @return True if this is a root server or a network preferred relay from one of our networks
@@ -154,14 +147,10 @@ public:
 	bool isUpstream(const Identity &id) const;
 
 	/**
-	 * Set whether or not an address is upstream
-	 *
-	 * If the address is a root this does nothing, since roots are fixed.
-	 *
-	 * @param a Target address
-	 * @param upstream New upstream status
+	 * @param ztaddr ZeroTier address
+	 * @return Peer role for this device
 	 */
-	void setUpstream(const Address &a,bool upstream);
+	ZT_PeerRole role(const Address &ztaddr) const;
 
 	/**
 	 * Check for prohibited endpoints
@@ -179,6 +168,30 @@ public:
 	 */
 	bool isProhibitedEndpoint(const Address &ztaddr,const InetAddress &ipaddr) const;
 
+	/**
+	 * @param eps Hash table to fill with addresses and their stable endpoints
+	 */
+	inline void getUpstreamStableEndpoints(Hashtable< Address,std::vector<InetAddress> > &eps) const
+	{
+		Mutex::Lock _l(_lock);
+		for(std::vector<World::Root>::const_iterator i(_planet.roots().begin());i!=_planet.roots().end();++i) {
+			std::vector<InetAddress> &ips = eps[i->identity.address()];
+			for(std::vector<InetAddress>::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) {
+				if (std::find(ips.begin(),ips.end(),*j) == ips.end())
+					ips.push_back(*j);
+			}
+		}
+		for(std::vector<World>::const_iterator m(_moons.begin());m!=_moons.end();++m) {
+			for(std::vector<World::Root>::const_iterator i(m->roots().begin());i!=m->roots().end();++i) {
+				std::vector<InetAddress> &ips = eps[i->identity.address()];
+				for(std::vector<InetAddress>::const_iterator j(i->stableEndpoints.begin());j!=i->stableEndpoints.end();++j) {
+					if (std::find(ips.begin(),ips.end(),*j) == ips.end())
+						ips.push_back(*j);
+				}
+			}
+		}
+	}
+
 	/**
 	 * @return Vector of active upstream addresses (including roots)
 	 */
@@ -189,37 +202,38 @@ public:
 	}
 
 	/**
-	 * @return Current World (copy)
+	 * @return Current planet
 	 */
-	inline World world() const
+	inline World planet() const
 	{
 		Mutex::Lock _l(_lock);
-		return _world;
+		return _planet;
 	}
 
 	/**
-	 * @return Current world ID
+	 * @return Current planet's world ID
 	 */
-	inline uint64_t worldId() const
+	inline uint64_t planetWorldId() const
 	{
-		return _world.id(); // safe to read without lock, and used from within eachPeer() so don't lock
+		return _planet.id(); // safe to read without lock, and used from within eachPeer() so don't lock
 	}
 
 	/**
-	 * @return Current world timestamp
+	 * @return Current planet's world timestamp
 	 */
-	inline uint64_t worldTimestamp() const
+	inline uint64_t planetWorldTimestamp() const
 	{
-		return _world.timestamp(); // safe to read without lock, and used from within eachPeer() so don't lock
+		return _planet.timestamp(); // safe to read without lock, and used from within eachPeer() so don't lock
 	}
 
 	/**
 	 * Validate new world and update if newer and signature is okay
 	 *
-	 * @param newWorld Potential new world definition revision
-	 * @return True if an update actually occurred
+	 * @param newWorld A new or updated planet or moon to learn
+	 * @param updateOnly If true only update currently known worlds
+	 * @return True if it was valid and newer than current (or totally new for moons)
 	 */
-	bool worldUpdateIfValid(const World &newWorld);
+	bool addWorld(const World &newWorld,bool updateOnly);
 
 	/**
 	 * Clean and flush database
@@ -284,9 +298,9 @@ public:
 	}
 
 	/**
-	 * @return True if I am a root server in the current World
+	 * @return True if I am a root server in a planet or moon
 	 */
-	inline bool amRoot() const throw() { return _amRoot; }
+	inline bool amRoot() const { return _amRoot; }
 
 	/**
 	 * Get the outbound trusted path ID for a physical address, or 0 if none
@@ -339,7 +353,6 @@ public:
 
 private:
 	Identity _getIdentity(const Address &zta);
-	void _setWorld(const World &newWorld);
 
 	const RuntimeEnvironment *const RR;
 
@@ -347,14 +360,14 @@ private:
 	InetAddress _trustedPathNetworks[ZT_MAX_TRUSTED_PATHS];
 	unsigned int _trustedPathCount;
 
-	World _world;
+	World _planet;
+	std::vector< World > _moons;
 
 	Hashtable< Address,SharedPtr<Peer> > _peers;
 	Hashtable< Path::HashKey,SharedPtr<Path> > _paths;
 
-	std::vector< Address > _upstreamAddresses; // includes roots
-	std::vector< Address > _rootAddresses; // only roots
-	bool _amRoot; // am I a root?
+	std::vector< Address > _upstreamAddresses; // includes root addresses of both planets and moons
+	bool _amRoot; // am I a root in a planet or moon?
 
 	Mutex _lock;
 };

node/World.hpp

@@ -48,16 +48,6 @@
  */
 #define ZT_WORLD_MAX_SERIALIZED_LENGTH (((1024 + (32 * ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT)) * ZT_WORLD_MAX_ROOTS) + ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_SIGNATURE_LEN + 128)
 
-/**
- * World ID indicating null / empty World object
- */
-#define ZT_WORLD_ID_NULL 0
-
-/**
- * World ID for a test network with ephemeral or temporary roots
- */
-#define ZT_WORLD_ID_TESTNET 1
-
 /**
  * World ID for Earth
  *
@@ -90,15 +80,23 @@ namespace ZeroTier {
  * orbits, the Moon (about 1.3 light seconds), and nearby Lagrange points. A
  * world ID for Mars and nearby space is defined but not yet used, and a test
  * world ID is provided for testing purposes.
- *
- * If you absolutely must run your own "unofficial" ZeroTier network, please
- * define your world IDs above 0xffffffff (4294967295). Code to make a World
- * is in mkworld.cpp in the parent directory and must be edited to change
- * settings.
  */
 class World
 {
 public:
+	/**
+	 * World type -- do not change IDs
+	 */
+	enum Type
+	{
+		TYPE_NULL = 0,
+		TYPE_PLANET = 1, // Planets, of which there is currently one (Earth)
+		TYPE_MOON = 127  // Moons, which are user-created and many
+	};
+
+	/**
+	 * Upstream server definition in world/moon
+	 */
 	struct Root
 	{
 		Identity identity;
@@ -113,45 +111,44 @@ public:
 	 * Construct an empty / null World
 	 */
 	World() :
-		_id(ZT_WORLD_ID_NULL),
-		_ts(0) {}
+		_id(0),
+		_ts(0),
+		_type(TYPE_NULL) {}
 
 	/**
 	 * @return Root servers for this world and their stable endpoints
 	 */
-	inline const std::vector<World::Root> &roots() const throw() { return _roots; }
+	inline const std::vector<World::Root> &roots() const { return _roots; }
+
+	/**
+	 * @return World type: planet or moon
+	 */
+	inline Type type() const { return _type; }
 
 	/**
 	 * @return World unique identifier
 	 */
-	inline uint64_t id() const throw() { return _id; }
+	inline uint64_t id() const { return _id; }
 
 	/**
 	 * @return World definition timestamp
 	 */
-	inline uint64_t timestamp() const throw() { return _ts; }
+	inline uint64_t timestamp() const { return _ts; }
 
 	/**
 	 * Check whether a world update should replace this one
 	 *
-	 * A new world update is valid if it is for the same world ID, is newer,
-	 * and is signed by the current world's signing key. If this world object
-	 * is null, it can always be updated.
-	 *
 	 * @param update Candidate update
-	 * @param fullSignatureCheck Perform full cryptographic signature check (true == yes, false == skip)
-	 * @return True if update is newer than current and is properly signed
+	 * @return True if update is newer than current, matches its ID and type, and is properly signed (or if current is NULL)
 	 */
-	inline bool shouldBeReplacedBy(const World &update,bool fullSignatureCheck)
+	inline bool shouldBeReplacedBy(const World &update)
 	{
-		if (_id == ZT_WORLD_ID_NULL)
+		if ((_id == 0)||(_type == TYPE_NULL))
 			return true;
-		if ((_id == update._id)&&(_ts < update._ts)) {
-			if (fullSignatureCheck) {
-				Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
-				update.serialize(tmp,true);
-				return C25519::verify(_updatesMustBeSignedBy,tmp.data(),tmp.size(),update._signature);
-			} else return true;
+		if ((_id == update._id)&&(_ts < update._ts)&&(_type == update._type)) {
+			Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
+			update.serialize(tmp,true);
+			return C25519::verify(_updatesMustBeSignedBy,tmp.data(),tmp.size(),update._signature);
 		}
 		return false;
 	}
@@ -159,14 +156,14 @@ public:
 	/**
 	 * @return True if this World is non-empty
 	 */
-	inline operator bool() const throw() { return (_id != ZT_WORLD_ID_NULL); }
+	inline operator bool() const { return (_type != TYPE_NULL); }
 
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b,bool forSign = false) const
 	{
 		if (forSign) b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL);
 
-		b.append((uint8_t)0x01);
+		b.append((uint8_t)_type);
 		b.append((uint64_t)_id);
 		b.append((uint64_t)_ts);
 		b.append(_updatesMustBeSignedBy.data,ZT_C25519_PUBLIC_KEY_LEN);
@@ -190,14 +187,19 @@ public:
 
 		_roots.clear();
 
-		if (b[p++] != 0x01)
-			throw std::invalid_argument("invalid object type");
+		switch((Type)b[p++]) {
+			case TYPE_NULL: _type = TYPE_NULL; break; // shouldn't ever really happen in serialized data but it's not invalid
+			case TYPE_PLANET: _type = TYPE_PLANET; break;
+			case TYPE_MOON: _type = TYPE_MOON; break;
+			default:
+				throw std::invalid_argument("invalid world type");
+		}
 
 		_id = b.template at<uint64_t>(p); p += 8;
 		_ts = b.template at<uint64_t>(p); p += 8;
 		memcpy(_updatesMustBeSignedBy.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN;
 		memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN;
-		unsigned int numRoots = b[p++];
+		const unsigned int numRoots = (unsigned int)b[p++];
 		if (numRoots > ZT_WORLD_MAX_ROOTS)
 			throw std::invalid_argument("too many roots in World");
 		for(unsigned int k=0;k<numRoots;++k) {
@@ -216,12 +218,15 @@ public:
 		return (p - startAt);
 	}
 
-	inline bool operator==(const World &w) const throw() { return ((_id == w._id)&&(_ts == w._ts)&&(_updatesMustBeSignedBy == w._updatesMustBeSignedBy)&&(_signature == w._signature)&&(_roots == w._roots)); }
-	inline bool operator!=(const World &w) const throw() { return (!(*this == w)); }
+	inline bool operator==(const World &w) const { return ((_id == w._id)&&(_ts == w._ts)&&(_updatesMustBeSignedBy == w._updatesMustBeSignedBy)&&(_signature == w._signature)&&(_roots == w._roots)&&(_type == w._type)); }
+	inline bool operator!=(const World &w) const { return (!(*this == w)); }
+
+	inline bool operator<(const World &w) const { return (((int)_type < (int)w._type) ? true : ((_type == w._type) ? (_id < w._id) : false)); }
 
 protected:
 	uint64_t _id;
 	uint64_t _ts;
+	Type _type;
 	C25519::Public _updatesMustBeSignedBy;
 	C25519::Signature _signature;
 	std::vector<Root> _roots;

service/OneService.cpp

@@ -422,7 +422,7 @@ public:
 		try {
 			std::string authToken;
 			{
-				std::string authTokenPath(_homePath + ZT_PATH_SEPARATOR_S + "authtoken.secret");
+				std::string authTokenPath(_homePath + ZT_PATH_SEPARATOR_S "authtoken.secret");
 				if (!OSUtils::readFile(authTokenPath.c_str(),authToken)) {
 					unsigned char foo[24];
 					Utils::getSecureRandom(foo,sizeof(foo));
@@ -442,7 +442,8 @@ public:
 			authToken = _trimString(authToken);
 
 			// Clean up any legacy files if present
-			OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S + "peers.save").c_str());
+			OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S "peers.save").c_str());
+			OSUtils::rm((_homePath + ZT_PATH_SEPARATOR_S "world").c_str());
 
 			{
 				struct ZT_Node_Callbacks cb;
@@ -465,7 +466,7 @@ public:
 				unsigned int trustedPathCount = 0;
 
 				// Old style "trustedpaths" flat file -- will eventually go away
-				FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S + "trustedpaths").c_str(),"r");
+				FILE *trustpaths = fopen((_homePath + ZT_PATH_SEPARATOR_S "trustedpaths").c_str(),"r");
 				if (trustpaths) {
 					char buf[1024];
 					while ((fgets(buf,sizeof(buf),trustpaths))&&(trustedPathCount < ZT_MAX_TRUSTED_PATHS)) {
@@ -493,7 +494,7 @@ public:
 				// Read local config file
 				Mutex::Lock _l2(_localConfig_m);
 				std::string lcbuf;
-				if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S + "local.conf").c_str(),lcbuf)) {
+				if (OSUtils::readFile((_homePath + ZT_PATH_SEPARATOR_S "local.conf").c_str(),lcbuf)) {
 					try {
 						_localConfig = OSUtils::jsonParse(lcbuf);
 						if (!_localConfig.is_object()) {
@@ -581,7 +582,7 @@ public:
 			// Write file containing primary port to be read by CLIs, etc.
 			char portstr[64];
 			Utils::snprintf(portstr,sizeof(portstr),"%u",_ports[0]);
-			OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "zerotier-one.port").c_str(),std::string(portstr));
+			OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S "zerotier-one.port").c_str(),std::string(portstr));
 
 			// Attempt to bind to a secondary port chosen from our ZeroTier address.
 			// This exists because there are buggy NATs out there that fail if more
@@ -641,8 +642,8 @@ public:
 			_node->setNetconfMaster((void *)_controller);
 
 #ifdef ZT_ENABLE_CLUSTER
-			if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str())) {
-				_clusterDefinition = new ClusterDefinition(_node->address(),(_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str());
+			if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S "cluster").c_str())) {
+				_clusterDefinition = new ClusterDefinition(_node->address(),(_homePath + ZT_PATH_SEPARATOR_S "cluster").c_str());
 				if (_clusterDefinition->size() > 0) {
 					std::vector<ClusterDefinition::MemberDefinition> members(_clusterDefinition->members());
 					for(std::vector<ClusterDefinition::MemberDefinition>::iterator m(members.begin());m!=members.end();++m) {
@@ -689,12 +690,12 @@ public:
 			}
 #endif
 
-			_controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S + "ui").c_str());
+			_controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S "ui").c_str());
 			_controlPlane->addAuthToken(authToken.c_str());
 			_controlPlane->setController(_controller);
 
 			{	// Remember networks from previous session
-				std::vector<std::string> networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S + "networks.d").c_str()));
+				std::vector<std::string> networksDotD(OSUtils::listDirectory((_homePath + ZT_PATH_SEPARATOR_S "networks.d").c_str()));
 				for(std::vector<std::string>::iterator f(networksDotD.begin());f!=networksDotD.end();++f) {
 					std::size_t dot = f->find_last_of('.');
 					if ((dot == 16)&&(f->substr(16) == ".conf"))
@@ -919,9 +920,6 @@ public:
 				if ((nstr.length() == ZT_ADDRESS_LENGTH_HEX)&&(v.value().is_object())) {
 					const Address ztaddr(nstr.c_str());
 					if (ztaddr) {
-						const std::string rstr(OSUtils::jsonString(v.value()["role"],""));
-						_node->setRole(ztaddr.toInt(),((rstr == "upstream")||(rstr == "UPSTREAM")) ? ZT_PEER_ROLE_UPSTREAM : ZT_PEER_ROLE_LEAF);
-
 						const uint64_t ztaddr2 = ztaddr.toInt();
 						std::vector<InetAddress> &v4h = _v4Hints[ztaddr2];
 						std::vector<InetAddress> &v6h = _v6Hints[ztaddr2];

service/README.md

@@ -19,7 +19,6 @@ Settings available in `local.conf` (this is not valid JSON, and JSON does not al
 	},
 	"virtual": { /* Settings applied to ZeroTier virtual network devices (VL1) */
 		"##########": { /* 10-digit ZeroTier address */
-			"role": "upstream"|"leaf", /* If upstream, define this as a trusted "federated root" (default is leaf) */
 			"try": [ "IP/port"/*,...*/ ], /* Hints on where to reach this peer if no upstreams/roots are online */
 			"blacklist": [ "NETWORK/bits"/*,...*/ ] /* Blacklist a physical path for only this peer. */
 		}

world/mkworld.cpp

@@ -53,11 +53,12 @@ using namespace ZeroTier;
 class WorldMaker : public World
 {
 public:
-	static inline World make(uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector<World::Root> &roots,const C25519::Pair &signWith)
+	static inline World make(World::Type t,uint64_t id,uint64_t ts,const C25519::Public &sk,const std::vector<World::Root> &roots,const C25519::Pair &signWith)
 	{
 		WorldMaker w;
 		w._id = id;
 		w._ts = ts;
+		w._type = t;
 		w._updateSigningKey = sk;
 		w._roots = roots;
 
@@ -139,7 +140,7 @@ int main(int argc,char **argv)
 
 	fprintf(stderr,"INFO: generating and signing id==%llu ts==%llu"ZT_EOL_S,(unsigned long long)id,(unsigned long long)ts);
 
-	World nw = WorldMaker::make(id,ts,currentKP.pub,roots,previousKP);
+	World nw = WorldMaker::make(World::TYPE_PLANET,id,ts,currentKP.pub,roots,previousKP);
 
 	Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> outtmp;
 	nw.serialize(outtmp,false);