Merge branch 'edge' into android-jni-dev

.gitignore

@@ -29,8 +29,12 @@ Thumbs.db
 # *nix/Mac build droppings
 /build-*
 /ZeroTierOneInstaller-*
+/examples/docker/zerotier-one
+/examples/docker/test-*.env
+/world/mkworld
+/world/*.c25519
 
-# Miscellaneous file types that we don't want to check in
+# Miscellaneous temporaries, build files, etc.
 *.log
 *.opensdf
 *.user
@@ -45,23 +49,18 @@ Thumbs.db
 *.rpm
 *.autosave
 *.tmp
-
-# Root topology build files, temporaries, and never check in secrets
-/root-topology/bin2c
-/root-topology/mktopology
-/root-topology/*.secret
-/root-topology/test/supernodes
-/root-topology/test/test-root-topology
+node_modules
+cluster-geo/cluster-geo/config.js
+cluster-geo/cluster-geo/cache.*
+tests/http/zerotier-one
+tests/http/result_*
+tests/http/big-test-out
 
 # MacGap wrapper build files
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/project.xcworkspace/xcuserdata/*
 /ext/mac-ui-macgap1-wrapper/src/MacGap.xcodeproj/xcuserdata/*
 /ext/mac-ui-macgap1-wrapper/src/build
 
-# Web UI dev temporaries
-/ui/.module-cache
-node_modules
-
 # Java/Android/JNI build droppings
 java/obj/
 java/libs/

README.md

@@ -166,6 +166,10 @@ If you're interested, there's a [technical deep dive about NAT traversal on our
 
 If a firewall between you and the Internet blocks ZeroTier's UDP traffic, you will fall back to last-resort TCP tunneling to rootservers over port 443 (https impersonation). This will work almost anywhere but is *very slow* compared to UDP or direct peer to peer connectivity.
 
+### Contributing
+
+There are three main branches: **edge**, **test**, and **master**. Other branches may be for specific features, tests, or use cases. In general **edge** is "bleeding" and may or may not work, while **test** should be relatively stable and **master** is the latest tagged release. Pull requests should generally be done against **test** or **edge**, since pull requests against **master** may be working against a branch that is somewhat out of date.
+
 ### License
 
 The ZeroTier source code is open source and is licensed under the GNU GPL v3 (not LGPL). If you'd like to embed it in a closed-source commercial product or appliance, please e-mail [[email protected]](mailto:[email protected]) to discuss commercial licensing. Otherwise it can be used for free.

cluster-geo/cluster-geo.exe

@@ -0,0 +1,13 @@
+#!/bin/bash
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
+
+cd `dirname $0`
+if [ ! -d cluster-geo -o ! -f cluster-geo/cluster-geo.js ]; then
+	echo 'Cannot find ./cluster-geo containing NodeJS script files.'
+	exit 1
+fi
+
+cd cluster-geo
+
+exec node --harmony cluster-geo.js

cluster-geo/cluster-geo/cluster-geo.js

@@ -0,0 +1,96 @@
+"use strict";
+
+//
+// GeoIP lookup service
+//
+
+// GeoIP cache TTL in ms
+var CACHE_TTL = (60 * 60 * 24 * 60 * 1000); // 60 days
+
+var config = require(__dirname + '/config.js');
+
+if (!config.maxmind) {
+	console.error('FATAL: only MaxMind GeoIP2 is currently supported and is not configured in config.js');
+	process.exit(1);
+}
+var geo = require('geoip2ws')(config.maxmind);
+
+var cache = require('levelup')(__dirname + '/cache.leveldb');
+
+function lookup(ip,callback)
+{
+	cache.get(ip,function(err,cachedEntryJson) {
+		if ((!err)&&(cachedEntryJson)) {
+			try {
+				let cachedEntry = JSON.parse(cachedEntryJson.toString());
+				if (cachedEntry) {
+					let ts = cachedEntry.ts;
+					let r = cachedEntry.r;
+					if ((ts)&&(r)) {
+						if ((Date.now() - ts) < CACHE_TTL) {
+							r._cached = true;
+							return callback(null,r);
+						}
+					}
+				}
+			} catch (e) {}
+		}
+
+		geo(ip,function(err,result) {
+			if (err)
+				return callback(err,null);
+			if ((!result)||(!result.location))
+				return callback(new Error('null result'),null);
+
+			cache.put(ip,JSON.stringify({
+				ts: Date.now(),
+				r: result
+			}),function(err) {
+				if (err)
+					console.error('Error saving to cache: '+err);
+				return callback(null,result);
+			});
+		});
+	});
+};
+
+var linebuf = '';
+process.stdin.on('readable',function() {
+	var chunk;
+	while (null !== (chunk = process.stdin.read())) {
+		for(var i=0;i<chunk.length;++i) {
+			let c = chunk[i];
+			if ((c == 0x0d)||(c == 0x0a)) {
+				if (linebuf.length > 0) {
+					let ip = linebuf;
+					lookup(ip,function(err,result) {
+						if ((err)||(!result)||(!result.location)) {
+							return process.stdout.write(ip+',0,0,0,0,0,0\n');
+						} else {
+							let lat = parseFloat(result.location.latitude);
+							let lon = parseFloat(result.location.longitude);
+
+							// Convert to X,Y,Z coordinates from Earth's origin, Earth-as-sphere approximation.
+							let latRadians = lat * 0.01745329251994; // PI / 180
+							let lonRadians = lon * 0.01745329251994; // PI / 180
+							let cosLat = Math.cos(latRadians);
+							let x = Math.round((-6371.0) * cosLat * Math.cos(lonRadians)); // 6371 == Earth's approximate radius in kilometers
+							let y = Math.round(6371.0 * Math.sin(latRadians));
+							let z = Math.round(6371.0 * cosLat * Math.sin(lonRadians));
+
+							return process.stdout.write(ip+',1,'+lat+','+lon+','+x+','+y+','+z+'\n');
+						}
+					});
+				}
+				linebuf = '';
+			} else {
+				linebuf += String.fromCharCode(c);
+			}
+		}
+	}
+});
+
+process.stdin.on('end',function() {
+	cache.close();
+	process.exit(0);
+});

cluster-geo/cluster-geo/config.js.sample

@@ -0,0 +1,7 @@
+// MaxMind GeoIP2 config
+module.exports.maxmind = {
+	userId: 1234,
+	licenseKey: 'asdf',
+	service: 'city',
+	requestTimeout: 1000
+};

cluster-geo/cluster-geo/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "cluster-geo",
+  "version": "1.0.0",
+  "description": "Cluster GEO-IP Query Service",
+  "main": "cluster-geo.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "ZeroTier, Inc.",
+  "license": "GPL-3.0",
+  "dependencies": {
+    "geoip2ws": "^1.7.1",
+    "leveldown": "^1.4.2",
+    "levelup": "^1.2.1"
+  }
+}

controller/SqliteNetworkController.cpp

@@ -44,12 +44,15 @@
 #include "../ext/json-parser/json.h"
 
 #include "SqliteNetworkController.hpp"
+
+#include "../node/Node.hpp"
 #include "../node/Utils.hpp"
 #include "../node/CertificateOfMembership.hpp"
 #include "../node/NetworkConfig.hpp"
 #include "../node/InetAddress.hpp"
 #include "../node/MAC.hpp"
 #include "../node/Address.hpp"
+
 #include "../osdep/OSUtils.hpp"
 
 // Include ZT_NETCONF_SCHEMA_SQL constant to init database
@@ -117,8 +120,10 @@ struct NetworkRecord {
 
 } // anonymous namespace
 
-SqliteNetworkController::SqliteNetworkController(const char *dbPath) :
+SqliteNetworkController::SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath) :
+	_node(node),
 	_dbPath(dbPath),
+	_circuitTestPath(circuitTestPath),
 	_db((sqlite3 *)0)
 {
 	if (sqlite3_open_v2(dbPath,&_db,SQLITE_OPEN_READWRITE|SQLITE_OPEN_CREATE,(const char *)0) != SQLITE_OK)
@@ -253,8 +258,6 @@ SqliteNetworkController::~SqliteNetworkController()
 		sqlite3_finalize(_sCreateMember);
 		sqlite3_finalize(_sGetNodeIdentity);
 		sqlite3_finalize(_sCreateOrReplaceNode);
-		sqlite3_finalize(_sUpdateNode);
-		sqlite3_finalize(_sUpdateNode2);
 		sqlite3_finalize(_sGetEtherTypesFromRuleTable);
 		sqlite3_finalize(_sGetActiveBridges);
 		sqlite3_finalize(_sGetIpAssignmentsForNode);
@@ -500,6 +503,53 @@ unsigned int SqliteNetworkController::handleControlPlaneHttpPOST(
 					}
 
 					return _doCPGet(path,urlArgs,headers,body,responseBody,responseContentType);
+				} else if ((path.size() == 3)&&(path[2] == "test")) {
+					ZT_CircuitTest *test = (ZT_CircuitTest *)malloc(sizeof(ZT_CircuitTest));
+					memset(test,0,sizeof(ZT_CircuitTest));
+
+					Utils::getSecureRandom(&(test->testId),sizeof(test->testId));
+					test->credentialNetworkId = nwid;
+					test->ptr = (void *)this;
+
+					json_value *j = json_parse(body.c_str(),body.length());
+					if (j) {
+						if (j->type == json_object) {
+							for(unsigned int k=0;k<j->u.object.length;++k) {
+
+								if (!strcmp(j->u.object.values[k].name,"hops")) {
+									if (j->u.object.values[k].value->type == json_array) {
+										for(unsigned int kk=0;kk<j->u.object.values[k].value->u.array.length;++kk) {
+											json_value *hop = j->u.object.values[k].value->u.array.values[kk];
+											if (hop->type == json_array) {
+												for(unsigned int kkk=0;kkk<hop->u.array.length;++kkk) {
+													if (hop->u.array.values[kkk]->type == json_string) {
+														test->hops[test->hopCount].addresses[test->hops[test->hopCount].breadth++] = Utils::hexStrToU64(hop->u.array.values[kkk]->u.string.ptr) & 0xffffffffffULL;
+													}
+												}
+												++test->hopCount;
+											}
+										}
+									}
+								} else if (!strcmp(j->u.object.values[k].name,"reportAtEveryHop")) {
+									if (j->u.object.values[k].value->type == json_boolean)
+										test->reportAtEveryHop = (j->u.object.values[k].value->u.boolean == 0) ? 0 : 1;
+								}
+
+							}
+						}
+						json_value_free(j);
+					}
+
+					if (!test->hopCount) {
+						::free((void *)test);
+						return 500;
+					}
+
+					test->timestamp = OSUtils::now();
+					_circuitTests[test->testId] = test;
+					_node->circuitTestBegin(test,&(SqliteNetworkController::_circuitTestCallback));
+
+					return 200;
 				} // else 404
 
 			} else {
@@ -1742,6 +1792,8 @@ NetworkController::ResultCode SqliteNetworkController::_doNetworkConfigRequest(c
 					for(uint32_t k=ipRangeStart,l=0;(k<=ipRangeEnd)&&(l < 1000000);++k,++l) {
 						uint32_t ip = (ipRangeLen > 0) ? (ipRangeStart + (ipTrialCounter % ipRangeLen)) : ipRangeStart;
 						++ipTrialCounter;
+						if ((ip & 0x000000ff) == 0x000000ff)
+							continue; // don't allow addresses that end in .255
 
 						for(std::vector< std::pair<uint32_t,int> >::const_iterator r(routedNetworks.begin());r!=routedNetworks.end();++r) {
 							if ((ip & (0xffffffff << (32 - r->second))) == r->first) {
@@ -1812,4 +1864,75 @@ NetworkController::ResultCode SqliteNetworkController::_doNetworkConfigRequest(c
 	return NetworkController::NETCONF_QUERY_OK;
 }
 
+void SqliteNetworkController::_circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report)
+{
+	static Mutex circuitTestWriteLock;
+
+	const uint64_t now = OSUtils::now();
+
+	SqliteNetworkController *const c = reinterpret_cast<SqliteNetworkController *>(test->ptr);
+	char tmp[128];
+
+	std::string reportSavePath(c->_circuitTestPath);
+	OSUtils::mkdir(reportSavePath);
+	Utils::snprintf(tmp,sizeof(tmp),ZT_PATH_SEPARATOR_S"%.16llx",test->credentialNetworkId);
+	reportSavePath.append(tmp);
+	OSUtils::mkdir(reportSavePath);
+	Utils::snprintf(tmp,sizeof(tmp),ZT_PATH_SEPARATOR_S"%.16llx_%.16llx",test->timestamp,test->testId);
+	reportSavePath.append(tmp);
+	OSUtils::mkdir(reportSavePath);
+	Utils::snprintf(tmp,sizeof(tmp),ZT_PATH_SEPARATOR_S"%.16llx_%.10llx_%.10llx",now,report->upstream,report->current);
+	reportSavePath.append(tmp);
+
+	{
+		Mutex::Lock _l(circuitTestWriteLock);
+		FILE *f = fopen(reportSavePath.c_str(),"a");
+		if (!f)
+			return;
+		fseek(f,0,SEEK_END);
+		fprintf(f,"%s{\n"
+			"\t\"timestamp\": %llu,"ZT_EOL_S
+			"\t\"testId\": \"%.16llx\","ZT_EOL_S
+			"\t\"upstream\": \"%.10llx\","ZT_EOL_S
+			"\t\"current\": \"%.10llx\","ZT_EOL_S
+			"\t\"receivedTimestamp\": %llu,"ZT_EOL_S
+			"\t\"remoteTimestamp\": %llu,"ZT_EOL_S
+			"\t\"sourcePacketId\": \"%.16llx\","ZT_EOL_S
+			"\t\"flags\": %llu,"ZT_EOL_S
+			"\t\"sourcePacketHopCount\": %u,"ZT_EOL_S
+			"\t\"errorCode\": %u,"ZT_EOL_S
+			"\t\"vendor\": %d,"ZT_EOL_S
+			"\t\"protocolVersion\": %u,"ZT_EOL_S
+			"\t\"majorVersion\": %u,"ZT_EOL_S
+			"\t\"minorVersion\": %u,"ZT_EOL_S
+			"\t\"revision\": %u,"ZT_EOL_S
+			"\t\"platform\": %d,"ZT_EOL_S
+			"\t\"architecture\": %d,"ZT_EOL_S
+			"\t\"receivedOnLocalAddress\": \"%s\","ZT_EOL_S
+			"\t\"receivedFromRemoteAddress\": \"%s\""ZT_EOL_S
+			"}",
+			((ftell(f) > 0) ? ",\n" : ""),
+			(unsigned long long)report->timestamp,
+			(unsigned long long)test->testId,
+			(unsigned long long)report->upstream,
+			(unsigned long long)report->current,
+			(unsigned long long)now,
+			(unsigned long long)report->remoteTimestamp,
+			(unsigned long long)report->sourcePacketId,
+			(unsigned long long)report->flags,
+			report->sourcePacketHopCount,
+			report->errorCode,
+			(int)report->vendor,
+			report->protocolVersion,
+			report->majorVersion,
+			report->minorVersion,
+			report->revision,
+			(int)report->platform,
+			(int)report->architecture,
+			reinterpret_cast<const InetAddress *>(&(report->receivedOnLocalAddress))->toString().c_str(),
+			reinterpret_cast<const InetAddress *>(&(report->receivedFromRemoteAddress))->toString().c_str());
+		fclose(f);
+	}
+}
+
 } // namespace ZeroTier

controller/SqliteNetworkController.hpp

@@ -43,12 +43,17 @@
 // Number of in-memory last log entries to maintain per user
 #define ZT_SQLITENETWORKCONTROLLER_IN_MEMORY_LOG_SIZE 32
 
+// How long do circuit tests "live"? This is just to prevent buildup in memory.
+#define ZT_SQLITENETWORKCONTROLLER_CIRCUIT_TEST_TIMEOUT 300000
+
 namespace ZeroTier {
 
+class Node;
+
 class SqliteNetworkController : public NetworkController
 {
 public:
-	SqliteNetworkController(const char *dbPath);
+	SqliteNetworkController(Node *node,const char *dbPath,const char *circuitTestPath);
 	virtual ~SqliteNetworkController();
 
 	virtual NetworkController::ResultCode doNetworkConfigRequest(
@@ -104,7 +109,11 @@ private:
 		const Dictionary &metaData,
 		Dictionary &netconf);
 
+	static void _circuitTestCallback(ZT_Node *node,ZT_CircuitTest *test,const ZT_CircuitTestReport *report);
+
+	Node *_node;
 	std::string _dbPath;
+	std::string _circuitTestPath;
 	std::string _instanceId;
 
 	// A circular buffer last log
@@ -136,6 +145,9 @@ private:
 	// Last log entries by address and network ID pair
 	std::map< std::pair<Address,uint64_t>,_LLEntry > _lastLog;
 
+	// Circuit tests outstanding
+	std::map< uint64_t,ZT_CircuitTest * > _circuitTests;
+
 	sqlite3 *_db;
 
 	sqlite3_stmt *_sGetNetworkById;
@@ -143,8 +155,6 @@ private:
 	sqlite3_stmt *_sCreateMember;
 	sqlite3_stmt *_sGetNodeIdentity;
 	sqlite3_stmt *_sCreateOrReplaceNode;
-	sqlite3_stmt *_sUpdateNode;
-	sqlite3_stmt *_sUpdateNode2;
 	sqlite3_stmt *_sGetEtherTypesFromRuleTable;
 	sqlite3_stmt *_sGetActiveBridges;
 	sqlite3_stmt *_sGetIpAssignmentsForNode;

examples/api/README.md

@@ -0,0 +1,26 @@
+API Examples
+======
+
+This folder contains examples that can be posted with curl or another http query utility to a local instance.
+
+To test querying with curl:
+
+    curl -H 'X-ZT1-Auth:AUTHTOKEN' http://127.0.0.1:9993/status
+
+To create a public network on a local controller (service must be built with "make ZT\_ENABLE\_NETWORK\_CONTROLLER=1"):
+
+    curl -H 'X-ZT1-Auth:AUTHTOKEN' -X POST -d @public.json http://127.0.0.1:9993/controller/network/################
+
+Replace AUTHTOKEN with the contents of this instance's authtoken.secret file and ################ with a valid network ID. Its first 10 hex digits must be the ZeroTier address of the controller itself, while the last 6 hex digits can be anything. Also be sure to change the port if you have this instance listening somewhere other than 9993.
+
+After POSTing you can double check the network config with:
+
+    curl -H 'X-ZT1-Auth:AUTHTOKEN' http://127.0.0.1:9993/controller/network/################
+
+Once this network is created (and if your controller is online, etc.) you can then join this network from any device anywhere in the world and it will receive a valid network configuration.
+
+---
+
+**public.json**: A valid configuration for a public network that allows IPv4 and IPv6 traffic.
+
+**circuit-test-pingpong.json**: An example circuit test that can be posted to /controller/network/################/test to order a test -- you will have to edit this to insert the hops you want since the two hard coded device IDs are from our own test instances.

examples/api/circuit-test-pingpong.json

@@ -0,0 +1,13 @@
+{
+	"hops": [
+		[ "4cbc810d4c" ],
+		[ "868cd1664f" ],
+		[ "4cbc810d4c" ],
+		[ "868cd1664f" ],
+		[ "4cbc810d4c" ],
+		[ "868cd1664f" ],
+		[ "4cbc810d4c" ],
+		[ "868cd1664f" ]
+	],
+	"reportAtEveryHop": true
+}

examples/api/public.json

@@ -0,0 +1,27 @@
+{
+	"name": "public_test_network",
+	"private": false,
+	"enableBroadcast": true,
+	"allowPassiveBridging": false,
+	"v4AssignMode": "zt",
+	"v6AssignMode": "rfc4193",
+	"multicastLimit": 32,
+	"relays": [],
+	"gateways": [],
+	"ipLocalRoutes": ["10.66.0.0/16"],
+	"ipAssignmentPools": [{"ipRangeStart":"10.66.0.1","ipRangeEnd":"10.66.255.254"}],
+	"rules": [
+	{
+		"ruleNo": 10,
+		"etherType": 2048,
+		"action": "accept"
+	},{
+		"ruleNo": 20,
+		"etherType": 2054,
+		"action": "accept"
+	},{
+		"ruleNo": 30,
+		"etherType": 34525,
+		"action": "accept"
+	}]
+}

examples/docker/Dockerfile

@@ -0,0 +1,19 @@
+FROM centos:7
+
+MAINTAINER https://www.zerotier.com/
+
+RUN yum -y update && yum install -y sqlite net-tools && yum clean all
+
+EXPOSE 9993/udp
+
+RUN mkdir -p /var/lib/zerotier-one
+RUN mkdir -p /var/lib/zerotier-one/networks.d
+RUN ln -sf /var/lib/zerotier-one/zerotier-one /usr/local/bin/zerotier-cli
+RUN ln -sf /var/lib/zerotier-one/zerotier-one /usr/local/bin/zerotier-idtool
+
+ADD zerotier-one /var/lib/zerotier-one/
+
+ADD main.sh /
+RUN chmod a+x /main.sh
+
+CMD ["./main.sh"]

examples/docker/README.md

@@ -0,0 +1,8 @@
+Simple Dockerfile Example
+======
+
+This is a simple Docker example using ZeroTier One in normal tun/tap mode. It uses a Dockerfile to build an image containing ZeroTier One and a main.sh that launches it with an identity supplied via the Docker environment via the ZEROTIER\_IDENTITY\_SECRET and ZEROTIER\_NETWORK variables. The Dockerfile assumes that the zerotier-one binary is in the build folder.
+
+This is not a very secure way to load an identity secret, but it's useful for testing since it allows you to repeatedly launch Docker containers with the same identity. For production we'd recommend using something like Hashicorp Vault, or modifying main.sh to leave identities unspecified and allow the container to generate a new identity at runtime. Then you could script approval of containers using the controller API, approving them as they launch, etc. (We are working on better ways of doing mass provisioning.)
+
+To use in normal tun/tap mode with Docker, containers must be run with the options "--device=/dev/net/tun --privileged". The main.sh script supplied here will complain and exit if these options are not present (no /dev/net/tun device).

examples/docker/main.sh

@@ -0,0 +1,25 @@
+#!/bin/bash
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
+
+if [ ! -c "/dev/net/tun" ]; then
+	echo 'FATAL: must be docker run with: --device=/dev/net/tun --cap-add=NET_ADMIN'
+	exit 1
+fi
+
+if [ -z "$ZEROTIER_IDENTITY_SECRET" ]; then
+	echo 'FATAL: ZEROTIER_IDENTITY_SECRET not set -- aborting!'
+	exit 1
+fi
+
+if [ -z "$ZEROTIER_NETWORK" ]; then
+	echo 'Warning: ZEROTIER_NETWORK not set, you will need to docker exec zerotier-cli to join a network.'
+else
+	# The existence of a .conf will cause the service to "remember" this network
+	touch /var/lib/zerotier-one/networks.d/$ZEROTIER_NETWORK.conf
+fi
+
+rm -f /var/lib/zerotier-one/identity.*
+echo "$ZEROTIER_IDENTITY_SECRET" >/var/lib/zerotier-one/identity.secret
+
+/var/lib/zerotier-one/zerotier-one

examples/docker/maketestenv.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+if [ -z "$1" -o -z "$2" ]; then
+	echo 'Usage: maketestenv.sh <output file e.g. test-01.env> <network ID>'
+	exit 1
+fi
+
+newid=`../../zerotier-idtool generate`
+
+echo "ZEROTIER_IDENTITY_SECRET=$newid" >$1
+echo "ZEROTIER_NETWORK=$2" >>$1

include/ZeroTierOne.h

@@ -115,6 +115,34 @@ extern "C" {
  */
 #define ZT_FEATURE_FLAG_FIPS 0x00000002
 
+/**
+ * Maximum number of hops in a ZeroTier circuit test
+ *
+ * This is more or less the max that can be fit in a given packet (with
+ * fragmentation) and only one address per hop.
+ */
+#define ZT_CIRCUIT_TEST_MAX_HOPS 512
+
+/**
+ * Maximum number of addresses per hop in a circuit test
+ */
+#define ZT_CIRCUIT_TEST_MAX_HOP_BREADTH 256
+
+/**
+ * Maximum number of cluster members (and max member ID plus one)
+ */
+#define ZT_CLUSTER_MAX_MEMBERS 128
+
+/**
+ * Maximum number of physical ZeroTier addresses a cluster member can report
+ */
+#define ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES 16
+
+/**
+ * Maximum allowed cluster message length in bytes
+ */
+#define ZT_CLUSTER_MAX_MESSAGE_LENGTH (1500 - 48)
+
 /**
  * A null/empty sockaddr (all zero) to signify an unspecified socket address
  */
@@ -161,7 +189,17 @@ enum ZT_ResultCode
 	/**
 	 * Network ID not valid
 	 */
-	ZT_RESULT_ERROR_NETWORK_NOT_FOUND = 1000
+	ZT_RESULT_ERROR_NETWORK_NOT_FOUND = 1000,
+
+	/**
+	 * The requested operation is not supported on this version or build
+	 */
+	ZT_RESULT_ERROR_UNSUPPORTED_OPERATION = 1001,
+
+	/**
+	 * The requestion operation was given a bad parameter or was called in an invalid state
+	 */
+	ZT_RESULT_ERROR_BAD_PARAMETER = 1002
 };
 
 /**
@@ -242,31 +280,6 @@ enum ZT_Event
 	 */
 	ZT_EVENT_FATAL_ERROR_IDENTITY_COLLISION = 4,
 
-	/**
-	 * A more recent version was observed on the network
-	 *
-	 * Right now this is only triggered if a hub or rootserver reports a
-	 * more recent version, and only once. It can be used to trigger a
-	 * software update check.
-	 *
-	 * Meta-data: unsigned int[3], more recent version number
-	 */
-	ZT_EVENT_SAW_MORE_RECENT_VERSION = 5,
-
-	/**
-	 * A packet failed authentication
-	 *
-	 * Meta-data: struct sockaddr_storage containing origin address of packet
-	 */
-	ZT_EVENT_AUTHENTICATION_FAILURE = 6,
-
-	/**
-	 * A received packet was not valid
-	 *
-	 * Meta-data: struct sockaddr_storage containing origin address of packet
-	 */
-	ZT_EVENT_INVALID_PACKET = 7,
-
 	/**
 	 * Trace (debugging) message
 	 *
@@ -274,7 +287,7 @@ enum ZT_Event
 	 *
 	 * Meta-data: C string, TRACE message
 	 */
-	ZT_EVENT_TRACE = 8
+	ZT_EVENT_TRACE = 5
 };
 
 /**
@@ -287,6 +300,16 @@ typedef struct
 	 */
 	uint64_t address;
 
+	/**
+	 * Current world ID
+	 */
+	uint64_t worldId;
+
+	/**
+	 * Current world revision/timestamp
+	 */
+	uint64_t worldTimestamp;
+
 	/**
 	 * Public identity in string-serialized form (safe to send to others)
 	 *
@@ -401,6 +424,59 @@ enum ZT_VirtualNetworkConfigOperation
 	ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY = 4
 };
 
+/**
+ * What trust hierarchy role does this peer have?
+ */
+enum ZT_PeerRole {
+	ZT_PEER_ROLE_LEAF = 0,     // ordinary node
+	ZT_PEER_ROLE_RELAY = 1,    // relay node
+	ZT_PEER_ROLE_ROOT = 2      // root server
+};
+
+/**
+ * Vendor ID
+ */
+enum ZT_Vendor {
+	ZT_VENDOR_UNSPECIFIED = 0,
+	ZT_VENDOR_ZEROTIER = 1
+};
+
+/**
+ * Platform type
+ */
+enum ZT_Platform {
+	ZT_PLATFORM_UNSPECIFIED = 0,
+	ZT_PLATFORM_LINUX = 1,
+	ZT_PLATFORM_WINDOWS = 2,
+	ZT_PLATFORM_MACOS = 3,
+	ZT_PLATFORM_ANDROID = 4,
+	ZT_PLATFORM_IOS = 5,
+	ZT_PLATFORM_SOLARIS_SMARTOS = 6,
+	ZT_PLATFORM_FREEBSD = 7,
+	ZT_PLATFORM_NETBSD = 8,
+	ZT_PLATFORM_OPENBSD = 9,
+	ZT_PLATFORM_RISCOS = 10,
+	ZT_PLATFORM_VXWORKS = 11,
+	ZT_PLATFORM_FREERTOS = 12,
+	ZT_PLATFORM_SYSBIOS = 13,
+	ZT_PLATFORM_HURD = 14
+};
+
+/**
+ * Architecture type
+ */
+enum ZT_Architecture {
+	ZT_ARCHITECTURE_UNSPECIFIED = 0,
+	ZT_ARCHITECTURE_X86 = 1,
+	ZT_ARCHITECTURE_X64 = 2,
+	ZT_ARCHITECTURE_ARM32 = 3,
+	ZT_ARCHITECTURE_ARM64 = 4,
+	ZT_ARCHITECTURE_MIPS32 = 5,
+	ZT_ARCHITECTURE_MIPS64 = 6,
+	ZT_ARCHITECTURE_POWER32 = 7,
+	ZT_ARCHITECTURE_POWER64 = 8
+};
+
 /**
  * Virtual network configuration
  */
@@ -532,11 +608,6 @@ typedef struct
 	 */
 	uint64_t lastReceive;
 
-	/**
-	 * Is path fixed? (i.e. not learned, static)
-	 */
-	int fixed;
-
 	/**
 	 * Is path active?
 	 */
@@ -548,15 +619,6 @@ typedef struct
 	int preferred;
 } ZT_PeerPhysicalPath;
 
-/**
- * What trust hierarchy role does this peer have?
- */
-enum ZT_PeerRole {
-	ZT_PEER_ROLE_LEAF = 0,     // ordinary node
-	ZT_PEER_ROLE_RELAY = 1,    // relay node
-	ZT_PEER_ROLE_ROOT = 2      // root server
-};
-
 /**
  * Peer status result buffer
  */
@@ -623,13 +685,267 @@ typedef struct
 } ZT_PeerList;
 
 /**
- * Local interface trust levels
+ * ZeroTier circuit test configuration and path
+ */
+typedef struct {
+	/**
+	 * Test ID -- an arbitrary 64-bit identifier
+	 */
+	uint64_t testId;
+
+	/**
+	 * Timestamp -- sent with test and echoed back by each reporter
+	 */
+	uint64_t timestamp;
+
+	/**
+	 * Originator credential: network ID
+	 *
+	 * If this is nonzero, a network ID will be set for this test and
+	 * the originator must be its primary network controller. This is
+	 * currently the only authorization method available, so it must
+	 * be set to run a test.
+	 */
+	uint64_t credentialNetworkId;
+
+	/**
+	 * Hops in circuit test (a.k.a. FIFO for graph traversal)
+	 */
+	struct {
+		/**
+		 * Hop flags (currently unused, must be zero)
+		 */
+		unsigned int flags;
+
+		/**
+		 * Number of addresses in this hop (max: ZT_CIRCUIT_TEST_MAX_HOP_BREADTH)
+		 */
+		unsigned int breadth;
+
+		/**
+		 * 40-bit ZeroTier addresses (most significant 24 bits ignored)
+		 */
+		uint64_t addresses[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH];
+	} hops[ZT_CIRCUIT_TEST_MAX_HOPS];
+
+	/**
+	 * Number of hops (max: ZT_CIRCUIT_TEST_MAX_HOPS)
+	 */
+	unsigned int hopCount;
+
+	/**
+	 * If non-zero, circuit test will report back at every hop
+	 */
+	int reportAtEveryHop;
+
+	/**
+	 * An arbitrary user-settable pointer
+	 */
+	void *ptr;
+
+	/**
+	 * Pointer for internal use -- initialize to zero and do not modify
+	 */
+	void *_internalPtr;
+} ZT_CircuitTest;
+
+/**
+ * Circuit test result report
+ */
+typedef struct {
+	/**
+	 * Sender of report (current hop)
+	 */
+	uint64_t current;
+
+	/**
+	 * Previous hop
+	 */
+	uint64_t upstream;
+
+	/**
+	 * 64-bit test ID
+	 */
+	uint64_t testId;
+
+	/**
+	 * Timestamp from original test (echoed back at each hop)
+	 */
+	uint64_t timestamp;
+
+	/**
+	 * Timestamp on remote device
+	 */
+	uint64_t remoteTimestamp;
+
+	/**
+	 * 64-bit packet ID of packet received by the reporting device
+	 */
+	uint64_t sourcePacketId;
+
+	/**
+	 * Flags (currently unused, will be zero)
+	 */
+	uint64_t flags;
+
+	/**
+	 * ZeroTier protocol-level hop count of packet received by reporting device (>0 indicates relayed)
+	 */
+	unsigned int sourcePacketHopCount;
+
+	/**
+	 * Error code (currently unused, will be zero)
+	 */
+	unsigned int errorCode;
+
+	/**
+	 * Remote device vendor ID
+	 */
+	enum ZT_Vendor vendor;
+
+	/**
+	 * Remote device protocol compliance version
+	 */
+	unsigned int protocolVersion;
+
+	/**
+	 * Software major version
+	 */
+	unsigned int majorVersion;
+
+	/**
+	 * Software minor version
+	 */
+	unsigned int minorVersion;
+
+	/**
+	 * Software revision
+	 */
+	unsigned int revision;
+
+	/**
+	 * Platform / OS
+	 */
+	enum ZT_Platform platform;
+
+	/**
+	 * System architecture
+	 */
+	enum ZT_Architecture architecture;
+
+	/**
+	 * Local device address on which packet was received by reporting device
+	 *
+	 * This may have ss_family equal to zero (null address) if unspecified.
+	 */
+	struct sockaddr_storage receivedOnLocalAddress;
+
+	/**
+	 * Remote address from which reporter received the test packet
+	 *
+	 * This may have ss_family set to zero (null address) if unspecified.
+	 */
+	struct sockaddr_storage receivedFromRemoteAddress;
+
+	/**
+	 * Next hops to which packets are being or will be sent by the reporter
+	 *
+	 * In addition to reporting back, the reporter may send the test on if
+	 * there are more recipients in the FIFO. If it does this, it can report
+	 * back the address(es) that make up the next hop and the physical address
+	 * for each if it has one. The physical address being null/unspecified
+	 * typically indicates that no direct path exists and the next packet
+	 * will be relayed.
+	 */
+	struct {
+		/**
+		 * 40-bit ZeroTier address
+		 */
+		uint64_t address;
+
+		/**
+		 * Physical address or null address (ss_family == 0) if unspecified or unknown
+		 */
+		struct sockaddr_storage physicalAddress;
+	} nextHops[ZT_CIRCUIT_TEST_MAX_HOP_BREADTH];
+
+	/**
+	 * Number of next hops reported in nextHops[]
+	 */
+	unsigned int nextHopCount;
+} ZT_CircuitTestReport;
+
+/**
+ * A cluster member's status
  */
-typedef enum {
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL = 0,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_PRIVACY = 1,
-	ZT_LOCAL_INTERFACE_ADDRESS_TRUST_ULTIMATE = 2
-} ZT_LocalInterfaceAddressTrust;
+typedef struct {
+	/**
+	 * This cluster member's ID (from 0 to 1-ZT_CLUSTER_MAX_MEMBERS)
+	 */
+	unsigned int id;
+
+	/**
+	 * Number of milliseconds since last 'alive' heartbeat message received via cluster backplane address
+	 */
+	unsigned int msSinceLastHeartbeat;
+
+	/**
+	 * Non-zero if cluster member is alive
+	 */
+	int alive;
+
+	/**
+	 * X, Y, and Z coordinates of this member (if specified, otherwise zero)
+	 *
+	 * What these mean depends on the location scheme being used for
+	 * location-aware clustering. At present this is GeoIP and these
+	 * will be the X, Y, and Z coordinates of the location on a spherical
+	 * approximation of Earth where Earth's core is the origin (in km).
+	 * They don't have to be perfect and need only be comparable with others
+	 * to find shortest path via the standard vector distance formula.
+	 */
+	int x,y,z;
+
+	/**
+	 * Cluster member's last reported load
+	 */
+	uint64_t load;
+
+	/**
+	 * Number of peers this cluster member "has"
+	 */
+	uint64_t peers;
+
+	/**
+	 * Physical ZeroTier endpoints for this member (where peers are sent when directed here)
+	 */
+	struct sockaddr_storage zeroTierPhysicalEndpoints[ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES];
+
+	/**
+	 * Number of physical ZeroTier endpoints this member is announcing
+	 */
+	unsigned int numZeroTierPhysicalEndpoints;
+} ZT_ClusterMemberStatus;
+
+/**
+ * ZeroTier cluster status
+ */
+typedef struct {
+	/**
+	 * My cluster member ID (a record for 'self' is included in member[])
+	 */
+	unsigned int myId;
+
+	/**
+	 * Number of cluster members
+	 */
+	unsigned int clusterSize;
+
+	/**
+	 * Cluster member statuses
+	 */
+	ZT_ClusterMemberStatus members[ZT_CLUSTER_MAX_MEMBERS];
+} ZT_ClusterStatus;
 
 /**
  * An instance of a ZeroTier One node (opaque)
@@ -800,7 +1116,6 @@ typedef int (*ZT_WirePacketSendFunction)(
  * @param dataStorePutFunction Function called to put objects in persistent storage
  * @param virtualNetworkConfigFunction Function to be called when virtual LANs are created, deleted, or their config parameters change
  * @param eventCallback Function to receive status updates and non-fatal error notices
- * @param overrideRootTopology Alternative root server topology or NULL for default (mostly for test/debug use)
  * @return OK (0) or error code if a fatal error condition has occurred
  */
 enum ZT_ResultCode ZT_Node_new(
@@ -812,8 +1127,7 @@ enum ZT_ResultCode ZT_Node_new(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology);
+	ZT_EventCallback eventCallback);
 
 /**
  * Delete a node and free all resources it consumes
@@ -1014,11 +1328,6 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
 /**
  * Add a local interface address
  *
- * Local interface addresses may be added if you want remote peers
- * with whom you have a trust relatinship (e.g. common network membership)
- * to receive information about these endpoints as potential endpoints for
- * direct communication.
- *
  * Take care that these are never ZeroTier interface addresses, otherwise
  * strange things might happen or they simply won't work.
  *
@@ -1033,11 +1342,9 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr);
  * reject bad, empty, and unusable addresses.
  *
  * @param addr Local interface address
- * @param metric Local interface metric
- * @param trust How much do you trust the local network under this interface?
  * @return Boolean: non-zero if address was accepted and added
  */
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr);
 
 /**
  * Clear local interface addresses
@@ -1061,6 +1368,149 @@ void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node);
  */
 void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkConfigMasterInstance);
 
+/**
+ * Initiate a VL1 circuit test
+ *
+ * This sends an initial VERB_CIRCUIT_TEST and reports results back to the
+ * supplied callback until circuitTestEnd() is called. The supplied
+ * ZT_CircuitTest structure should be initially zeroed and then filled
+ * in with settings and hops.
+ *
+ * It is the caller's responsibility to call circuitTestEnd() and then
+ * to dispose of the test structure. Otherwise this node will listen
+ * for results forever.
+ *
+ * @param node Node instance
+ * @param test Test configuration
+ * @param reportCallback Function to call each time a report is received
+ * @return OK or error if, for example, test is too big for a packet or support isn't compiled in
+ */
+enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *, ZT_CircuitTest *,const ZT_CircuitTestReport *));
+
+/**
+ * Stop listening for results to a given circuit test
+ *
+ * This does not free the 'test' structure. The caller may do that
+ * after calling this method to unregister it.
+ *
+ * Any reports that are received for a given test ID after it is
+ * terminated are ignored.
+ *
+ * @param node Node instance
+ * @param test Test configuration to unregister
+ */
+void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test);
+
+/**
+ * Initialize cluster operation
+ *
+ * This initializes the internal structures and state for cluster operation.
+ * It takes two function pointers. The first is to a function that can be
+ * used to send data to cluster peers (mechanism is not defined by Node),
+ * and the second is to a function that can be used to get the location of
+ * a physical address in X,Y,Z coordinate space (e.g. as cartesian coordinates
+ * projected from the center of the Earth).
+ *
+ * Send function takes an arbitrary pointer followed by the cluster member ID
+ * to send data to, a pointer to the data, and the length of the data. The
+ * maximum message length is ZT_CLUSTER_MAX_MESSAGE_LENGTH (65535). Messages
+ * must be delivered whole and may be dropped or transposed, though high
+ * failure rates are undesirable and can cause problems. Validity checking or
+ * CRC is also not required since the Node validates the authenticity of
+ * cluster messages using cryptogrphic methods and will silently drop invalid
+ * messages.
+ *
+ * Address to location function is optional and if NULL geo-handoff is not
+ * enabled (in this case x, y, and z in clusterInit are also unused). It
+ * takes an arbitrary pointer followed by a physical address and three result
+ * parameters for x, y, and z. It returns zero on failure or nonzero if these
+ * three coordinates have been set. Coordinate space is arbitrary and can be
+ * e.g. coordinates on Earth relative to Earth's center. These can be obtained
+ * from latitutde and longitude with versions of the Haversine formula.
+ *
+ * See: http://stackoverflow.com/questions/1185408/converting-from-longitude-latitude-to-cartesian-coordinates
+ *
+ * Neither the send nor the address to location function should block. If the
+ * address to location function does not have a location for an address, it
+ * should return zero and then look up the address for future use since it
+ * will be called again in (typically) 1-3 minutes.
+ *
+ * Note that both functions can be called from any thread from which the
+ * various Node functions are called, and so must be thread safe if multiple
+ * threads are being used.
+ *
+ * @param node Node instance
+ * @param myId My cluster member ID (less than or equal to ZT_CLUSTER_MAX_MEMBERS)
+ * @param zeroTierPhysicalEndpoints Preferred physical address(es) for ZeroTier clients to contact this cluster member (for peer redirect)
+ * @param numZeroTierPhysicalEndpoints Number of physical endpoints in zeroTierPhysicalEndpoints[] (max allowed: 255)
+ * @param x My cluster member's X location
+ * @param y My cluster member's Y location
+ * @param z My cluster member's Z location
+ * @param sendFunction Function to be called to send data to other cluster members
+ * @param sendFunctionArg First argument to sendFunction()
+ * @param addressToLocationFunction Function to be called to get the location of a physical address or NULL to disable geo-handoff
+ * @param addressToLocationFunctionArg First argument to addressToLocationFunction()
+ * @return OK or UNSUPPORTED_OPERATION if this Node was not built with cluster support
+ */
+enum ZT_ResultCode ZT_Node_clusterInit(
+	ZT_Node *node,
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg);
+
+/**
+ * Add a member to this cluster
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param memberId Member ID (must be less than or equal to ZT_CLUSTER_MAX_MEMBERS)
+ * @return OK or error if clustering is disabled, ID invalid, etc.
+ */
+enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId);
+
+/**
+ * Remove a member from this cluster
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param memberId Member ID to remove (nothing happens if not present)
+ */
+void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId);
+
+/**
+ * Handle an incoming cluster state message
+ *
+ * The message itself contains cluster member IDs, and invalid or badly
+ * addressed messages will be silently discarded.
+ *
+ * Calling this without having called clusterInit() will do nothing.
+ *
+ * @param node Node instance
+ * @param msg Cluster message
+ * @param len Length of cluster message
+ */
+void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len);
+
+/**
+ * Get the current status of the cluster from this node's point of view
+ *
+ * Calling this without clusterInit() or without cluster support will just
+ * zero out the structure and show a cluster size of zero.
+ *
+ * @param node Node instance
+ * @param cs Cluster status structure to fill with data
+ */
+void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs);
+
 /**
  * Get ZeroTier One version
  *

make-linux.mk

@@ -29,7 +29,7 @@ endif
 UNAME_M=$(shell uname -m)
 
 INCLUDES=
-DEFS=
+DEFS=-DZT_ENABLE_CLUSTER
 LDLIBS?=
 
 include objects.mk

make-mac.mk

@@ -58,8 +58,8 @@ ifeq ($(ZT_DEBUG),1)
 	# C25519 in particular is almost UNUSABLE in heavy testing without it.
 ext/lz4/lz4.o node/Salsa20.o node/SHA512.o node/C25519.o node/Poly1305.o: CFLAGS = -Wall -O2 -g -pthread $(INCLUDES) $(DEFS)
 else
-	CFLAGS?=-O3 -fstack-protector
-	CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -fvectorize -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS)
+	CFLAGS?=-Ofast -fstack-protector
+	CFLAGS+=$(ARCH_FLAGS) -Wall -flto -fPIE -pthread -mmacosx-version-min=10.7 -DNDEBUG -Wno-unused-private-field $(INCLUDES) $(DEFS)
 	STRIP=strip
 endif
 
@@ -93,7 +93,7 @@ official: FORCE
 	make ZT_OFFICIAL_RELEASE=1 mac-dist-pkg
 
 clean:
-	rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-*
+	rm -rf *.dSYM build-* *.pkg *.dmg *.o node/*.o controller/*.o service/*.o osdep/*.o ext/http-parser/*.o ext/lz4/*.o ext/json-parser/*.o zerotier-one zerotier-idtool zerotier-selftest zerotier-cli ZeroTierOneInstaller-* mkworld
 
 # For those building from source -- installs signed binary tap driver in system ZT home
 install-mac-tap: FORCE

node/AntiRecursion.hpp

@@ -35,28 +35,28 @@
 
 namespace ZeroTier {
 
-#define ZT_ANTIRECURSION_TAIL_LEN 256
+/**
+ * Size of anti-recursion history
+ */
+#define ZT_ANTIRECURSION_HISTORY_SIZE 16
 
 /**
  * Filter to prevent recursion (ZeroTier-over-ZeroTier)
  *
  * This works by logging ZeroTier packets that we send. It's then invoked
- * again against packets read from local Ethernet taps. If the last N
+ * again against packets read from local Ethernet taps. If the last 32
  * bytes representing the ZeroTier packet match in the tap frame, then
  * the frame is a re-injection of a frame that we sent and is rejected.
  *
  * This means that ZeroTier packets simply will not traverse ZeroTier
  * networks, which would cause all sorts of weird problems.
  *
- * NOTE: this is applied to low-level packets before they are sent to
- * SocketManager and/or sockets, not to fully assembled packets before
- * (possible) fragmentation.
+ * This is highly optimized code since it's checked for every packet.
  */
 class AntiRecursion
 {
 public:
 	AntiRecursion()
-		throw()
 	{
 		memset(_history,0,sizeof(_history));
 		_ptr = 0;
@@ -68,13 +68,20 @@ public:
 	 * @param data ZT packet data
 	 * @param len Length of packet
 	 */
-	inline void logOutgoingZT(const void *data,unsigned int len)
-		throw()
+	inline void logOutgoingZT(const void *const data,const unsigned int len)
 	{
-		ArItem *i = &(_history[_ptr++ % ZT_ANTIRECURSION_HISTORY_SIZE]);
-		const unsigned int tl = (len > ZT_ANTIRECURSION_TAIL_LEN) ? ZT_ANTIRECURSION_TAIL_LEN : len;
-		memcpy(i->tail,((const unsigned char *)data) + (len - tl),tl);
-		i->len = tl;
+		if (len < 32)
+			return;
+#ifdef ZT_NO_TYPE_PUNNING
+		memcpy(_history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail,reinterpret_cast<const uint8_t *>(data) + (len - 32),32);
+#else
+		uint64_t *t = _history[++_ptr % ZT_ANTIRECURSION_HISTORY_SIZE].tail;
+		const uint64_t *p = reinterpret_cast<const uint64_t *>(reinterpret_cast<const uint8_t *>(data) + (len - 32));
+		*(t++) = *(p++);
+		*(t++) = *(p++);
+		*(t++) = *(p++);
+		*t = *p;
+#endif
 	}
 
 	/**
@@ -84,25 +91,36 @@ public:
 	 * @param len Length of frame
 	 * @return True if frame is OK to be passed, false if it's a ZT frame that we sent
 	 */
-	inline bool checkEthernetFrame(const void *data,unsigned int len)
-		throw()
+	inline bool checkEthernetFrame(const void *const data,const unsigned int len) const
 	{
-		for(unsigned int h=0;h<ZT_ANTIRECURSION_HISTORY_SIZE;++h) {
-			ArItem *i = &(_history[h]);
-			if ((i->len > 0)&&(len >= i->len)&&(!memcmp(((const unsigned char *)data) + (len - i->len),i->tail,i->len)))
+		if (len < 32)
+			return true;
+		const uint8_t *const pp = reinterpret_cast<const uint8_t *>(data) + (len - 32);
+		const _ArItem *i = _history;
+		const _ArItem *const end = i + ZT_ANTIRECURSION_HISTORY_SIZE;
+		while (i != end) {
+#ifdef ZT_NO_TYPE_PUNNING
+			if (!memcmp(pp,i->tail,32))
 				return false;
+#else
+			const uint64_t *t = i->tail;
+			const uint64_t *p = reinterpret_cast<const uint64_t *>(pp);
+			uint64_t bits = *(t++) ^ *(p++);
+			bits |= *(t++) ^ *(p++);
+			bits |= *(t++) ^ *(p++);
+			bits |= *t ^ *p;
+			if (!bits)
+				return false;
+#endif
+			++i;
 		}
 		return true;
 	}
 
 private:
-	struct ArItem
-	{
-		unsigned char tail[ZT_ANTIRECURSION_TAIL_LEN];
-		unsigned int len;
-	};
-	ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE];
-	volatile unsigned int _ptr;
+	struct _ArItem { uint64_t tail[4]; };
+	_ArItem _history[ZT_ANTIRECURSION_HISTORY_SIZE];
+	volatile unsigned long _ptr;
 };
 
 } // namespace ZeroTier

node/Buffer.hpp

@@ -391,6 +391,23 @@ public:
 		::memmove(_b,_b + at,_l -= at);
 	}
 
+	/**
+	 * Erase something from the middle of the buffer
+	 *
+	 * @param start Starting position
+	 * @param length Length of block to erase
+	 * @throw std::out_of_range Position plus length is beyond size of buffer
+	 */
+	inline void erase(const unsigned int at,const unsigned int length)
+		throw(std::out_of_range)
+	{
+		const unsigned int endr = at + length;
+		if (endr > _l)
+			throw std::out_of_range("Buffer: erase() range beyond end of buffer");
+		::memmove(_b + at,_b + endr,_l - endr);
+		_l -= length;
+	}
+
 	/**
 	 * Set buffer data length to zero
 	 */

node/CertificateOfMembership.hpp

@@ -315,78 +315,6 @@ public:
 	 */
 	inline const Address &signedBy() const throw() { return _signedBy; }
 
-	/**
-	 * Serialize to std::string or compatible class
-	 *
-	 * @param b String or other class supporting push_back() and append() like std::string
-	 */
-	template<typename T>
-	inline void serialize2(T &b) const
-	{
-		uint64_t tmp[3];
-		char tmp2[ZT_ADDRESS_LENGTH];
-		b.push_back((char)COM_UINT64_ED25519);
-		b.push_back((char)((_qualifiers.size() >> 8) & 0xff));
-		b.push_back((char)(_qualifiers.size() & 0xff));
-		for(std::vector<_Qualifier>::const_iterator q(_qualifiers.begin());q!=_qualifiers.end();++q) {
-			tmp[0] = Utils::hton(q->id);
-			tmp[1] = Utils::hton(q->value);
-			tmp[2] = Utils::hton(q->maxDelta);
-			b.append(reinterpret_cast<const char *>(reinterpret_cast<void *>(tmp)),sizeof(tmp));
-		}
-		_signedBy.copyTo(tmp2,ZT_ADDRESS_LENGTH);
-		b.append(tmp2,ZT_ADDRESS_LENGTH);
-		if (_signedBy)
-			b.append((const char *)_signature.data,_signature.size());
-	}
-
-	/**
-	 * Deserialize from std::string::iterator or compatible iterator or char* pointer
-	 *
-	 * @param p Iterator
-	 * @param end End of buffer
-	 */
-	template<typename T>
-	inline void deserialize2(T &p,const T &end)
-	{
-		uint64_t tmp[3];
-		char tmp2[ZT_ADDRESS_LENGTH];
-		unsigned int qcount;
-
-		_qualifiers.clear();
-		_signedBy.zero();
-
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		if (*(p++) != (char)COM_UINT64_ED25519) throw std::invalid_argument("unknown certificate of membership type");
-
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		qcount = (unsigned int)*(p++) << 8;
-		if (p == end) throw std::out_of_range("incomplete certificate of membership");
-		qcount |= (unsigned int)*(p++);
-
-		for(unsigned int i=0;i<qcount;++i) {
-			char *p2 = reinterpret_cast<char *>(reinterpret_cast<void *>(tmp));
-			for(unsigned int j=0;j<sizeof(tmp);++j) {
-				if (p == end) throw std::out_of_range("incomplete certificate of membership");
-				*(p2++) = *(p++);
-			}
-			_qualifiers.push_back(_Qualifier(Utils::ntoh(tmp[0]),Utils::ntoh(tmp[1]),Utils::ntoh(tmp[2])));
-		}
-
-		for(unsigned int j=0;j<ZT_ADDRESS_LENGTH;++j) {
-			if (p == end) throw std::out_of_range("incomplete certificate of membership");
-			tmp2[j] = *(p++);
-		}
-		_signedBy.setTo(tmp2,ZT_ADDRESS_LENGTH);
-
-		if (_signedBy) {
-			for(unsigned int j=0;j<_signature.size();++j) {
-				if (p == end) throw std::out_of_range("incomplete certificate of membership");
-				_signature.data[j] = (unsigned char)*(p++);
-			}
-		}
-	}
-
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b) const
 	{

node/Cluster.cpp

@@ -0,0 +1,760 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifdef ZT_ENABLE_CLUSTER
+
+#include <stdint.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <math.h>
+
+#include <algorithm>
+#include <utility>
+
+#include "../version.h"
+
+#include "Cluster.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "MulticastGroup.hpp"
+#include "CertificateOfMembership.hpp"
+#include "Salsa20.hpp"
+#include "Poly1305.hpp"
+#include "Identity.hpp"
+#include "Topology.hpp"
+#include "Packet.hpp"
+#include "Switch.hpp"
+#include "Node.hpp"
+
+namespace ZeroTier {
+
+static inline double _dist3d(int x1,int y1,int z1,int x2,int y2,int z2)
+	throw()
+{
+	double dx = ((double)x2 - (double)x1);
+	double dy = ((double)y2 - (double)y1);
+	double dz = ((double)z2 - (double)z1);
+	return sqrt((dx * dx) + (dy * dy) + (dz * dz));
+}
+
+Cluster::Cluster(
+	const RuntimeEnvironment *renv,
+	uint16_t id,
+	const std::vector<InetAddress> &zeroTierPhysicalEndpoints,
+	int32_t x,
+	int32_t y,
+	int32_t z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg) :
+	RR(renv),
+	_sendFunction(sendFunction),
+	_sendFunctionArg(sendFunctionArg),
+	_addressToLocationFunction(addressToLocationFunction),
+	_addressToLocationFunctionArg(addressToLocationFunctionArg),
+	_x(x),
+	_y(y),
+	_z(z),
+	_id(id),
+	_zeroTierPhysicalEndpoints(zeroTierPhysicalEndpoints),
+	_members(new _Member[ZT_CLUSTER_MAX_MEMBERS]),
+	_peerAffinities(65536),
+	_lastCleanedPeerAffinities(0),
+	_lastCheckedPeersForAnnounce(0),
+	_lastFlushed(0)
+{
+	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
+
+	// Generate master secret by hashing the secret from our Identity key pair
+	RR->identity.sha512PrivateKey(_masterSecret);
+
+	// Generate our inbound message key, which is the master secret XORed with our ID and hashed twice
+	memcpy(stmp,_masterSecret,sizeof(stmp));
+	stmp[0] ^= Utils::hton(id);
+	SHA512::hash(stmp,stmp,sizeof(stmp));
+	SHA512::hash(stmp,stmp,sizeof(stmp));
+	memcpy(_key,stmp,sizeof(_key));
+	Utils::burn(stmp,sizeof(stmp));
+}
+
+Cluster::~Cluster()
+{
+	Utils::burn(_masterSecret,sizeof(_masterSecret));
+	Utils::burn(_key,sizeof(_key));
+	delete [] _members;
+}
+
+void Cluster::handleIncomingStateMessage(const void *msg,unsigned int len)
+{
+	Buffer<ZT_CLUSTER_MAX_MESSAGE_LENGTH> dmsg;
+	{
+		// FORMAT: <[16] iv><[8] MAC><... data>
+		if ((len < 24)||(len > ZT_CLUSTER_MAX_MESSAGE_LENGTH))
+			return;
+
+		// 16-byte IV: first 8 bytes XORed with key, last 8 bytes used as Salsa20 64-bit IV
+		char keytmp[32];
+		memcpy(keytmp,_key,32);
+		for(int i=0;i<8;++i)
+			keytmp[i] ^= reinterpret_cast<const char *>(msg)[i];
+		Salsa20 s20(keytmp,256,reinterpret_cast<const char *>(msg) + 8);
+		Utils::burn(keytmp,sizeof(keytmp));
+
+		// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
+		char polykey[ZT_POLY1305_KEY_LEN];
+		memset(polykey,0,sizeof(polykey));
+		s20.encrypt12(polykey,polykey,sizeof(polykey));
+
+		// Compute 16-byte MAC
+		char mac[ZT_POLY1305_MAC_LEN];
+		Poly1305::compute(mac,reinterpret_cast<const char *>(msg) + 24,len - 24,polykey);
+
+		// Check first 8 bytes of MAC against 64-bit MAC in stream
+		if (!Utils::secureEq(mac,reinterpret_cast<const char *>(msg) + 16,8))
+			return;
+
+		// Decrypt!
+		dmsg.setSize(len - 24);
+		s20.decrypt12(reinterpret_cast<const char *>(msg) + 24,const_cast<void *>(dmsg.data()),dmsg.size());
+	}
+
+	if (dmsg.size() < 4)
+		return;
+	const uint16_t fromMemberId = dmsg.at<uint16_t>(0);
+	unsigned int ptr = 2;
+	if (fromMemberId == _id) // sanity check: we don't talk to ourselves
+		return;
+	const uint16_t toMemberId = dmsg.at<uint16_t>(ptr);
+	ptr += 2;
+	if (toMemberId != _id) // sanity check: message not for us?
+		return;
+
+	{	// make sure sender is actually considered a member
+		Mutex::Lock _l3(_memberIds_m);
+		if (std::find(_memberIds.begin(),_memberIds.end(),fromMemberId) == _memberIds.end())
+			return;
+	}
+
+	{
+		_Member &m = _members[fromMemberId];
+		Mutex::Lock mlck(m.lock);
+
+		try {
+			while (ptr < dmsg.size()) {
+				const unsigned int mlen = dmsg.at<uint16_t>(ptr); ptr += 2;
+				const unsigned int nextPtr = ptr + mlen;
+				if (nextPtr > dmsg.size())
+					break;
+
+				int mtype = -1;
+				try {
+					switch((StateMessageType)(mtype = (int)dmsg[ptr++])) {
+						default:
+							break;
+
+						case STATE_MESSAGE_ALIVE: {
+							ptr += 7; // skip version stuff, not used yet
+							m.x = dmsg.at<int32_t>(ptr); ptr += 4;
+							m.y = dmsg.at<int32_t>(ptr); ptr += 4;
+							m.z = dmsg.at<int32_t>(ptr); ptr += 4;
+							ptr += 8; // skip local clock, not used
+							m.load = dmsg.at<uint64_t>(ptr); ptr += 8;
+							ptr += 8; // skip flags, unused
+#ifdef ZT_TRACE
+							std::string addrs;
+#endif
+							unsigned int physicalAddressCount = dmsg[ptr++];
+							m.zeroTierPhysicalEndpoints.clear();
+							for(unsigned int i=0;i<physicalAddressCount;++i) {
+								m.zeroTierPhysicalEndpoints.push_back(InetAddress());
+								ptr += m.zeroTierPhysicalEndpoints.back().deserialize(dmsg,ptr);
+								if (!(m.zeroTierPhysicalEndpoints.back())) {
+									m.zeroTierPhysicalEndpoints.pop_back();
+								}
+#ifdef ZT_TRACE
+								else {
+									if (addrs.length() > 0)
+										addrs.push_back(',');
+									addrs.append(m.zeroTierPhysicalEndpoints.back().toString());
+								}
+#endif
+							}
+#ifdef ZT_TRACE
+							if ((RR->node->now() - m.lastReceivedAliveAnnouncement) >= ZT_CLUSTER_TIMEOUT) {
+								TRACE("[%u] I'm alive! peers close to %d,%d,%d can be redirected to: %s",(unsigned int)fromMemberId,m.x,m.y,m.z,addrs.c_str());
+							}
+#endif
+							m.lastReceivedAliveAnnouncement = RR->node->now();
+						}	break;
+
+						case STATE_MESSAGE_HAVE_PEER: {
+							const uint64_t now = RR->node->now();
+							Identity id;
+							InetAddress physicalAddress;
+							ptr += id.deserialize(dmsg,ptr);
+							ptr += physicalAddress.deserialize(dmsg,ptr);
+							if (id) {
+								// Forget any paths that we have to this peer at its address
+								if (physicalAddress) {
+									SharedPtr<Peer> myPeerRecord(RR->topology->getPeerNoCache(id.address(),now));
+									if (myPeerRecord)
+										myPeerRecord->removePathByAddress(physicalAddress);
+								}
+
+								// Always save identity to update file time
+								RR->topology->saveIdentity(id);
+
+								// Set peer affinity to its new home
+								{
+									Mutex::Lock _l2(_peerAffinities_m);
+									_PA &pa = _peerAffinities[id.address()];
+									pa.ts = now;
+									pa.mid = fromMemberId;
+		 						}
+		 						TRACE("[%u] has %s @ %s",(unsigned int)fromMemberId,id.address().toString().c_str(),physicalAddress.toString().c_str());
+		 					}
+						}	break;
+
+						case STATE_MESSAGE_MULTICAST_LIKE: {
+							const uint64_t nwid = dmsg.at<uint64_t>(ptr); ptr += 8;
+							const Address address(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
+							const MAC mac(dmsg.field(ptr,6),6); ptr += 6;
+							const uint32_t adi = dmsg.at<uint32_t>(ptr); ptr += 4;
+							RR->mc->add(RR->node->now(),nwid,MulticastGroup(mac,adi),address);
+							TRACE("[%u] %s likes %s/%.8x on %.16llx",(unsigned int)fromMemberId,address.toString().c_str(),mac.toString().c_str(),(unsigned int)adi,nwid);
+						}	break;
+
+						case STATE_MESSAGE_COM: {
+							/* not currently used so not decoded yet
+							CertificateOfMembership com;
+							ptr += com.deserialize(dmsg,ptr);
+							if (com) {
+								TRACE("[%u] COM for %s on %.16llu rev %llu",(unsigned int)fromMemberId,com.issuedTo().toString().c_str(),com.networkId(),com.revision());
+							}
+							*/
+						}	break;
+
+						case STATE_MESSAGE_PROXY_UNITE: {
+							const Address localPeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
+							const Address remotePeerAddress(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
+							const unsigned int numRemotePeerPaths = dmsg[ptr++];
+							InetAddress remotePeerPaths[256]; // size is 8-bit, so 256 is max
+							for(unsigned int i=0;i<numRemotePeerPaths;++i)
+								ptr += remotePeerPaths[i].deserialize(dmsg,ptr);
+
+							TRACE("[%u] requested that we unite local %s with remote %s",(unsigned int)fromMemberId,localPeerAddress.toString().c_str(),remotePeerAddress.toString().c_str());
+
+							const uint64_t now = RR->node->now();
+							SharedPtr<Peer> localPeer(RR->topology->getPeerNoCache(localPeerAddress,now));
+							if ((localPeer)&&(numRemotePeerPaths > 0)) {
+								InetAddress bestLocalV4,bestLocalV6;
+								localPeer->getBestActiveAddresses(now,bestLocalV4,bestLocalV6);
+
+								InetAddress bestRemoteV4,bestRemoteV6;
+								for(unsigned int i=0;i<numRemotePeerPaths;++i) {
+									if ((bestRemoteV4)&&(bestRemoteV6))
+										break;
+									switch(remotePeerPaths[i].ss_family) {
+										case AF_INET:
+											if (!bestRemoteV4)
+												bestRemoteV4 = remotePeerPaths[i];
+											break;
+										case AF_INET6:
+											if (!bestRemoteV6)
+												bestRemoteV6 = remotePeerPaths[i];
+											break;
+									}
+								}
+
+								Packet rendezvousForLocal(localPeerAddress,RR->identity.address(),Packet::VERB_RENDEZVOUS);
+								rendezvousForLocal.append((uint8_t)0);
+								remotePeerAddress.appendTo(rendezvousForLocal);
+
+								Buffer<2048> rendezvousForRemote;
+								remotePeerAddress.appendTo(rendezvousForRemote);
+								rendezvousForRemote.append((uint8_t)Packet::VERB_RENDEZVOUS);
+								const unsigned int rendezvousForOtherEndPayloadSizePtr = rendezvousForRemote.size();
+								rendezvousForRemote.addSize(2); // space for actual packet payload length
+								rendezvousForRemote.append((uint8_t)0); // flags == 0
+								localPeerAddress.appendTo(rendezvousForRemote);
+
+								bool haveMatch = false;
+								if ((bestLocalV6)&&(bestRemoteV6)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV6.port());
+									rendezvousForLocal.append((uint8_t)16);
+									rendezvousForLocal.append(bestRemoteV6.rawIpData(),16);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV6.port());
+									rendezvousForRemote.append((uint8_t)16);
+									rendezvousForRemote.append(bestLocalV6.rawIpData(),16);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 16));
+								} else if ((bestLocalV4)&&(bestRemoteV4)) {
+									haveMatch = true;
+
+									rendezvousForLocal.append((uint16_t)bestRemoteV4.port());
+									rendezvousForLocal.append((uint8_t)4);
+									rendezvousForLocal.append(bestRemoteV4.rawIpData(),4);
+
+									rendezvousForRemote.append((uint16_t)bestLocalV4.port());
+									rendezvousForRemote.append((uint8_t)4);
+									rendezvousForRemote.append(bestLocalV4.rawIpData(),4);
+									rendezvousForRemote.setAt<uint16_t>(rendezvousForOtherEndPayloadSizePtr,(uint16_t)(9 + 4));
+								}
+
+								if (haveMatch) {
+									_send(fromMemberId,STATE_MESSAGE_PROXY_SEND,rendezvousForRemote.data(),rendezvousForRemote.size());
+									_flush(fromMemberId); // we want this to go ASAP, since with port restricted cone NATs success can be timing-sensitive
+									RR->sw->send(rendezvousForLocal,true,0);
+								}
+							}
+						}	break;
+
+						case STATE_MESSAGE_PROXY_SEND: {
+							const Address rcpt(dmsg.field(ptr,ZT_ADDRESS_LENGTH),ZT_ADDRESS_LENGTH); ptr += ZT_ADDRESS_LENGTH;
+							const Packet::Verb verb = (Packet::Verb)dmsg[ptr++];
+							const unsigned int len = dmsg.at<uint16_t>(ptr); ptr += 2;
+							Packet outp(rcpt,RR->identity.address(),verb);
+							outp.append(dmsg.field(ptr,len),len); ptr += len;
+							RR->sw->send(outp,true,0);
+							TRACE("[%u] proxy send %s to %s length %u",(unsigned int)fromMemberId,Packet::verbString(verb),rcpt.toString().c_str(),len);
+						}	break;
+					}
+				} catch ( ... ) {
+					TRACE("invalid message of size %u type %d (inner decode), discarding",mlen,mtype);
+					// drop invalids
+				}
+
+				ptr = nextPtr;
+			}
+		} catch ( ... ) {
+			TRACE("invalid message (outer loop), discarding");
+			// drop invalids
+		}
+	}
+}
+
+bool Cluster::sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite)
+{
+	if (len > 16384) // sanity check
+		return false;
+
+	const uint64_t now = RR->node->now();
+	unsigned int canHasPeer = 0;
+
+	{	// Anyone got this peer?
+		Mutex::Lock _l2(_peerAffinities_m);
+		_PA *pa = _peerAffinities.get(toPeerAddress);
+		if ((pa)&&(pa->mid != _id)&&((now - pa->ts) < ZT_PEER_ACTIVITY_TIMEOUT))
+			canHasPeer = pa->mid;
+		else return false;
+	}
+
+	Buffer<1024> buf;
+	if (unite) {
+		InetAddress v4,v6;
+		if (fromPeerAddress) {
+			SharedPtr<Peer> fromPeer(RR->topology->getPeerNoCache(fromPeerAddress,now));
+			if (fromPeer)
+				fromPeer->getBestActiveAddresses(now,v4,v6);
+		}
+		uint8_t addrCount = 0;
+		if (v4)
+			++addrCount;
+		if (v6)
+			++addrCount;
+		if (addrCount) {
+			toPeerAddress.appendTo(buf);
+			fromPeerAddress.appendTo(buf);
+			buf.append(addrCount);
+			if (v4)
+				v4.serialize(buf);
+			if (v6)
+				v6.serialize(buf);
+		}
+	}
+	{
+		Mutex::Lock _l2(_members[canHasPeer].lock);
+		if (buf.size() > 0)
+			_send(canHasPeer,STATE_MESSAGE_PROXY_UNITE,buf.data(),buf.size());
+		if (_members[canHasPeer].zeroTierPhysicalEndpoints.size() > 0)
+			RR->node->putPacket(InetAddress(),_members[canHasPeer].zeroTierPhysicalEndpoints.front(),data,len);
+	}
+
+	TRACE("sendViaCluster(): relaying %u bytes from %s to %s by way of %u",len,fromPeerAddress.toString().c_str(),toPeerAddress.toString().c_str(),(unsigned int)canHasPeer);
+
+	return true;
+}
+
+void Cluster::replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress)
+{
+	const uint64_t now = RR->node->now();
+	{
+		Mutex::Lock _l2(_peerAffinities_m);
+		_PA &pa = _peerAffinities[peerId.address()];
+		if (pa.mid != _id) {
+			pa.ts = now;
+			pa.mid = _id;
+		} else if ((now - pa.ts) < ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD) {
+			return;
+		} else {
+			pa.ts = now;
+		}
+	}
+
+	// announcement
+	Buffer<4096> buf;
+	peerId.serialize(buf,false);
+	physicalAddress.serialize(buf);
+	{
+		Mutex::Lock _l(_memberIds_m);
+		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+			Mutex::Lock _l2(_members[*mid].lock);
+			_send(*mid,STATE_MESSAGE_HAVE_PEER,buf.data(),buf.size());
+		}
+	}
+}
+
+void Cluster::replicateMulticastLike(uint64_t nwid,const Address &peerAddress,const MulticastGroup &group)
+{
+	Buffer<1024> buf;
+	buf.append((uint64_t)nwid);
+	peerAddress.appendTo(buf);
+	group.mac().appendTo(buf);
+	buf.append((uint32_t)group.adi());
+	TRACE("replicating %s MULTICAST_LIKE %.16llx/%s/%u to all members",peerAddress.toString().c_str(),nwid,group.mac().toString().c_str(),(unsigned int)group.adi());
+	{
+		Mutex::Lock _l(_memberIds_m);
+		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+			Mutex::Lock _l2(_members[*mid].lock);
+			_send(*mid,STATE_MESSAGE_MULTICAST_LIKE,buf.data(),buf.size());
+		}
+	}
+}
+
+void Cluster::replicateCertificateOfNetworkMembership(const CertificateOfMembership &com)
+{
+	Buffer<4096> buf;
+	com.serialize(buf);
+	TRACE("replicating %s COM for %.16llx to all members",com.issuedTo().toString().c_str(),com.networkId());
+	{
+		Mutex::Lock _l(_memberIds_m);
+		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+			Mutex::Lock _l2(_members[*mid].lock);
+			_send(*mid,STATE_MESSAGE_COM,buf.data(),buf.size());
+		}
+	}
+}
+
+struct _ClusterAnnouncePeers
+{
+	_ClusterAnnouncePeers(const uint64_t now_,Cluster *parent_) : now(now_),parent(parent_) {}
+	const uint64_t now;
+	Cluster *const parent;
+	inline void operator()(const Topology &t,const SharedPtr<Peer> &peer) const
+	{
+		Path *p = peer->getBestPath(now);
+		if (p)
+			parent->replicateHavePeer(peer->identity(),p->address());
+	}
+};
+void Cluster::doPeriodicTasks()
+{
+	const uint64_t now = RR->node->now();
+
+	// Erase old peer affinity entries just to control table size
+	if ((now - _lastCleanedPeerAffinities) >= (ZT_PEER_ACTIVITY_TIMEOUT * 5)) {
+		_lastCleanedPeerAffinities = now;
+		Address *k = (Address *)0;
+		_PA *v = (_PA *)0;
+		Mutex::Lock _l(_peerAffinities_m);
+		Hashtable< Address,_PA >::Iterator i(_peerAffinities);
+		while (i.next(k,v)) {
+			if ((now - v->ts) >= (ZT_PEER_ACTIVITY_TIMEOUT * 5))
+				_peerAffinities.erase(*k);
+		}
+	}
+
+	// Announce peers that we have active direct paths to -- note that we forget paths
+	// that other cluster members claim they have, which prevents us from fighting
+	// with other cluster members (route flapping) over specific paths.
+	if ((now - _lastCheckedPeersForAnnounce) >= (ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD / 4)) {
+		_lastCheckedPeersForAnnounce = now;
+		_ClusterAnnouncePeers func(now,this);
+		RR->topology->eachPeer<_ClusterAnnouncePeers &>(func);
+	}
+
+	// Flush outgoing packet send queue every doPeriodicTasks()
+	if ((now - _lastFlushed) >= ZT_CLUSTER_FLUSH_PERIOD) {
+		_lastFlushed = now;
+		Mutex::Lock _l(_memberIds_m);
+		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+			Mutex::Lock _l2(_members[*mid].lock);
+
+			if ((now - _members[*mid].lastAnnouncedAliveTo) >= ((ZT_CLUSTER_TIMEOUT / 2) - 1000)) {
+				Buffer<2048> alive;
+				alive.append((uint16_t)ZEROTIER_ONE_VERSION_MAJOR);
+				alive.append((uint16_t)ZEROTIER_ONE_VERSION_MINOR);
+				alive.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
+				alive.append((uint8_t)ZT_PROTO_VERSION);
+				if (_addressToLocationFunction) {
+					alive.append((int32_t)_x);
+					alive.append((int32_t)_y);
+					alive.append((int32_t)_z);
+				} else {
+					alive.append((int32_t)0);
+					alive.append((int32_t)0);
+					alive.append((int32_t)0);
+				}
+				alive.append((uint64_t)now);
+				alive.append((uint64_t)0); // TODO: compute and send load average
+				alive.append((uint64_t)0); // unused/reserved flags
+				alive.append((uint8_t)_zeroTierPhysicalEndpoints.size());
+				for(std::vector<InetAddress>::const_iterator pe(_zeroTierPhysicalEndpoints.begin());pe!=_zeroTierPhysicalEndpoints.end();++pe)
+					pe->serialize(alive);
+				_send(*mid,STATE_MESSAGE_ALIVE,alive.data(),alive.size());
+				_members[*mid].lastAnnouncedAliveTo = now;
+			}
+
+			_flush(*mid); // does nothing if nothing to flush
+		}
+	}
+}
+
+void Cluster::addMember(uint16_t memberId)
+{
+	if ((memberId >= ZT_CLUSTER_MAX_MEMBERS)||(memberId == _id))
+		return;
+
+	Mutex::Lock _l2(_members[memberId].lock);
+
+	{
+		Mutex::Lock _l(_memberIds_m);
+		if (std::find(_memberIds.begin(),_memberIds.end(),memberId) != _memberIds.end())
+			return;
+		_memberIds.push_back(memberId);
+		std::sort(_memberIds.begin(),_memberIds.end());
+	}
+
+	_members[memberId].clear();
+
+	// Generate this member's message key from the master and its ID
+	uint16_t stmp[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
+	memcpy(stmp,_masterSecret,sizeof(stmp));
+	stmp[0] ^= Utils::hton(memberId);
+	SHA512::hash(stmp,stmp,sizeof(stmp));
+	SHA512::hash(stmp,stmp,sizeof(stmp));
+	memcpy(_members[memberId].key,stmp,sizeof(_members[memberId].key));
+	Utils::burn(stmp,sizeof(stmp));
+
+	// Prepare q
+	_members[memberId].q.clear();
+	char iv[16];
+	Utils::getSecureRandom(iv,16);
+	_members[memberId].q.append(iv,16);
+	_members[memberId].q.addSize(8); // room for MAC
+	_members[memberId].q.append((uint16_t)_id);
+	_members[memberId].q.append((uint16_t)memberId);
+}
+
+void Cluster::removeMember(uint16_t memberId)
+{
+	Mutex::Lock _l(_memberIds_m);
+	std::vector<uint16_t> newMemberIds;
+	for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+		if (*mid != memberId)
+			newMemberIds.push_back(*mid);
+	}
+	_memberIds = newMemberIds;
+}
+
+bool Cluster::findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload)
+{
+	if (_addressToLocationFunction) {
+		// Pick based on location if it can be determined
+		int px = 0,py = 0,pz = 0;
+		if (_addressToLocationFunction(_addressToLocationFunctionArg,reinterpret_cast<const struct sockaddr_storage *>(&peerPhysicalAddress),&px,&py,&pz) == 0) {
+			TRACE("no geolocation data for %s (geo-lookup is lazy/async so it may work next time)",peerPhysicalAddress.toIpString().c_str());
+			return false;
+		}
+
+		// Find member closest to this peer
+		const uint64_t now = RR->node->now();
+		std::vector<InetAddress> best;
+		const double currentDistance = _dist3d(_x,_y,_z,px,py,pz);
+		double bestDistance = (offload ? 2147483648.0 : currentDistance);
+		unsigned int bestMember = _id;
+		{
+			Mutex::Lock _l(_memberIds_m);
+			for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+				_Member &m = _members[*mid];
+				Mutex::Lock _ml(m.lock);
+
+				// Consider member if it's alive and has sent us a location and one or more physical endpoints to send peers to
+				if ( ((now - m.lastReceivedAliveAnnouncement) < ZT_CLUSTER_TIMEOUT) && ((m.x != 0)||(m.y != 0)||(m.z != 0)) && (m.zeroTierPhysicalEndpoints.size() > 0) ) {
+					const double mdist = _dist3d(m.x,m.y,m.z,px,py,pz);
+					if (mdist < bestDistance) {
+						bestDistance = mdist;
+						bestMember = *mid;
+						best = m.zeroTierPhysicalEndpoints;
+					}
+				}
+			}
+		}
+
+		// Redirect to a closer member if it has a ZeroTier endpoint address in the same ss_family
+		for(std::vector<InetAddress>::const_iterator a(best.begin());a!=best.end();++a) {
+			if (a->ss_family == peerPhysicalAddress.ss_family) {
+				TRACE("%s at [%d,%d,%d] is %f from us but %f from %u, can redirect to %s",peerAddress.toString().c_str(),px,py,pz,currentDistance,bestDistance,bestMember,a->toString().c_str());
+				redirectTo = *a;
+				return true;
+			}
+		}
+		TRACE("%s at [%d,%d,%d] is %f from us, no better endpoints found",peerAddress.toString().c_str(),px,py,pz,currentDistance);
+		return false;
+	} else {
+		// TODO: pick based on load if no location info?
+		return false;
+	}
+}
+
+void Cluster::status(ZT_ClusterStatus &status) const
+{
+	const uint64_t now = RR->node->now();
+	memset(&status,0,sizeof(ZT_ClusterStatus));
+	ZT_ClusterMemberStatus *ms[ZT_CLUSTER_MAX_MEMBERS];
+	memset(ms,0,sizeof(ms));
+
+	status.myId = _id;
+
+	ms[_id] = &(status.members[status.clusterSize++]);
+	ms[_id]->id = _id;
+	ms[_id]->alive = 1;
+	ms[_id]->x = _x;
+	ms[_id]->y = _y;
+	ms[_id]->z = _z;
+	ms[_id]->peers = RR->topology->countActive();
+	for(std::vector<InetAddress>::const_iterator ep(_zeroTierPhysicalEndpoints.begin());ep!=_zeroTierPhysicalEndpoints.end();++ep) {
+		if (ms[_id]->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
+			break;
+		memcpy(&(ms[_id]->zeroTierPhysicalEndpoints[ms[_id]->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage));
+	}
+
+	{
+		Mutex::Lock _l1(_memberIds_m);
+		for(std::vector<uint16_t>::const_iterator mid(_memberIds.begin());mid!=_memberIds.end();++mid) {
+			if (status.clusterSize >= ZT_CLUSTER_MAX_MEMBERS) // sanity check
+				break;
+			ZT_ClusterMemberStatus *s = ms[*mid] = &(status.members[status.clusterSize++]);
+			_Member &m = _members[*mid];
+			Mutex::Lock ml(m.lock);
+
+			s->id = *mid;
+			s->msSinceLastHeartbeat = (unsigned int)std::min((uint64_t)(~((unsigned int)0)),(now - m.lastReceivedAliveAnnouncement));
+			s->alive = (s->msSinceLastHeartbeat < ZT_CLUSTER_TIMEOUT) ? 1 : 0;
+			s->x = m.x;
+			s->y = m.y;
+			s->z = m.z;
+			s->load = m.load;
+			for(std::vector<InetAddress>::const_iterator ep(m.zeroTierPhysicalEndpoints.begin());ep!=m.zeroTierPhysicalEndpoints.end();++ep) {
+				if (s->numZeroTierPhysicalEndpoints >= ZT_CLUSTER_MAX_ZT_PHYSICAL_ADDRESSES) // sanity check
+					break;
+				memcpy(&(s->zeroTierPhysicalEndpoints[s->numZeroTierPhysicalEndpoints++]),&(*ep),sizeof(struct sockaddr_storage));
+			}
+		}
+	}
+
+	{
+		Mutex::Lock _l2(_peerAffinities_m);
+		Address *k = (Address *)0;
+		_PA *v = (_PA *)0;
+		Hashtable< Address,_PA >::Iterator i(const_cast<Cluster *>(this)->_peerAffinities);
+		while (i.next(k,v)) {
+			if ( (ms[v->mid]) && (v->mid != _id) && ((now - v->ts) < ZT_PEER_ACTIVITY_TIMEOUT) )
+				++ms[v->mid]->peers;
+		}
+	}
+}
+
+void Cluster::_send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len)
+{
+	if ((len + 3) > (ZT_CLUSTER_MAX_MESSAGE_LENGTH - (24 + 2 + 2))) // sanity check
+		return;
+	_Member &m = _members[memberId];
+	// assumes m.lock is locked!
+	if ((m.q.size() + len + 3) > ZT_CLUSTER_MAX_MESSAGE_LENGTH)
+		_flush(memberId);
+	m.q.append((uint16_t)(len + 1));
+	m.q.append((uint8_t)type);
+	m.q.append(msg,len);
+}
+
+void Cluster::_flush(uint16_t memberId)
+{
+	_Member &m = _members[memberId];
+	// assumes m.lock is locked!
+	if (m.q.size() > (24 + 2 + 2)) { // 16-byte IV + 8-byte MAC + 2 byte from-member-ID + 2 byte to-member-ID
+		// Create key from member's key and IV
+		char keytmp[32];
+		memcpy(keytmp,m.key,32);
+		for(int i=0;i<8;++i)
+			keytmp[i] ^= m.q[i];
+		Salsa20 s20(keytmp,256,m.q.field(8,8));
+		Utils::burn(keytmp,sizeof(keytmp));
+
+		// One-time-use Poly1305 key from first 32 bytes of Salsa20 keystream (as per DJB/NaCl "standard")
+		char polykey[ZT_POLY1305_KEY_LEN];
+		memset(polykey,0,sizeof(polykey));
+		s20.encrypt12(polykey,polykey,sizeof(polykey));
+
+		// Encrypt m.q in place
+		s20.encrypt12(reinterpret_cast<const char *>(m.q.data()) + 24,const_cast<char *>(reinterpret_cast<const char *>(m.q.data())) + 24,m.q.size() - 24);
+
+		// Add MAC for authentication (encrypt-then-MAC)
+		char mac[ZT_POLY1305_MAC_LEN];
+		Poly1305::compute(mac,reinterpret_cast<const char *>(m.q.data()) + 24,m.q.size() - 24,polykey);
+		memcpy(m.q.field(16,8),mac,8);
+
+		// Send!
+		_sendFunction(_sendFunctionArg,memberId,m.q.data(),m.q.size());
+
+		// Prepare for more
+		m.q.clear();
+		char iv[16];
+		Utils::getSecureRandom(iv,16);
+		m.q.append(iv,16);
+		m.q.addSize(8); // room for MAC
+		m.q.append((uint16_t)_id); // from member ID
+		m.q.append((uint16_t)memberId); // to member ID
+	}
+}
+
+} // namespace ZeroTier
+
+#endif // ZT_ENABLE_CLUSTER

node/Cluster.hpp

@@ -0,0 +1,370 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef ZT_CLUSTER_HPP
+#define ZT_CLUSTER_HPP
+
+#ifdef ZT_ENABLE_CLUSTER
+
+#include <vector>
+#include <algorithm>
+
+#include "Constants.hpp"
+#include "../include/ZeroTierOne.h"
+#include "Address.hpp"
+#include "InetAddress.hpp"
+#include "SHA512.hpp"
+#include "Utils.hpp"
+#include "Buffer.hpp"
+#include "Mutex.hpp"
+#include "SharedPtr.hpp"
+#include "Hashtable.hpp"
+
+/**
+ * Timeout for cluster members being considered "alive"
+ *
+ * A cluster member is considered dead and will no longer have peers
+ * redirected to it if we have not heard a heartbeat in this long.
+ */
+#define ZT_CLUSTER_TIMEOUT 10000
+
+/**
+ * How often should we announce that we have a peer?
+ */
+#define ZT_CLUSTER_HAVE_PEER_ANNOUNCE_PERIOD ZT_PEER_DIRECT_PING_DELAY
+
+/**
+ * Desired period between doPeriodicTasks() in milliseconds
+ */
+#define ZT_CLUSTER_PERIODIC_TASK_PERIOD 250
+
+/**
+ * How often to flush outgoing message queues (maximum interval)
+ */
+#define ZT_CLUSTER_FLUSH_PERIOD 500
+
+namespace ZeroTier {
+
+class RuntimeEnvironment;
+class CertificateOfMembership;
+class MulticastGroup;
+class Peer;
+class Identity;
+
+/**
+ * Multi-homing cluster state replication and packet relaying
+ *
+ * Multi-homing means more than one node sharing the same ZeroTier identity.
+ * There is nothing in the protocol to prevent this, but to make it work well
+ * requires the devices sharing an identity to cooperate and share some
+ * information.
+ *
+ * There are three use cases we want to fulfill:
+ *
+ * (1) Multi-homing of root servers with handoff for efficient routing,
+ *     HA, and load balancing across many commodity nodes.
+ * (2) Multi-homing of network controllers for the same reason.
+ * (3) Multi-homing of nodes on virtual networks, such as domain servers
+ *     and other important endpoints.
+ *
+ * These use cases are in order of escalating difficulty. The initial
+ * version of Cluster is aimed at satisfying the first, though you are
+ * free to try #2 and #3.
+ */
+class Cluster
+{
+public:
+	/**
+	 * State message types
+	 */
+	enum StateMessageType
+	{
+		STATE_MESSAGE_NOP = 0,
+
+		/**
+		 * This cluster member is alive:
+		 *   <[2] version minor>
+		 *   <[2] version major>
+		 *   <[2] version revision>
+		 *   <[1] protocol version>
+		 *   <[4] X location (signed 32-bit)>
+		 *   <[4] Y location (signed 32-bit)>
+		 *   <[4] Z location (signed 32-bit)>
+		 *   <[8] local clock at this member>
+		 *   <[8] load average>
+		 *   <[8] flags (currently unused, must be zero)>
+		 *   <[1] number of preferred ZeroTier endpoints>
+		 *   <[...] InetAddress(es) of preferred ZeroTier endpoint(s)>
+		 */
+		STATE_MESSAGE_ALIVE = 1,
+
+		/**
+		 * Cluster member has this peer:
+		 *   <[...] binary serialized peer identity>
+		 *   <[...] binary serialized peer remote physical address>
+		 *
+		 * Clusters send this message when they learn a path to a peer. The
+		 * replicated physical address is the one learned.
+		 */
+		STATE_MESSAGE_HAVE_PEER = 2,
+
+		/**
+		 * Peer subscription to multicast group:
+		 *   <[8] network ID>
+		 *   <[5] peer ZeroTier address>
+		 *   <[6] MAC address of multicast group>
+		 *   <[4] 32-bit multicast group ADI>
+		 */
+		STATE_MESSAGE_MULTICAST_LIKE = 3,
+
+		/**
+		 * Certificate of network membership for a peer:
+		 *   <[...] serialized COM>
+		 */
+		STATE_MESSAGE_COM = 4,
+
+		/**
+		 * Request that VERB_RENDEZVOUS be sent to a peer that we have:
+		 *   <[5] ZeroTier address of peer on recipient's side>
+		 *   <[5] ZeroTier address of peer on sender's side>
+		 *   <[1] 8-bit number of sender's peer's active path addresses>
+		 *   <[...] series of serialized InetAddresses of sender's peer's paths>
+		 *
+		 * This requests that we perform NAT-t introduction between a peer that
+		 * we have and one on the sender's side. The sender furnishes contact
+		 * info for its peer, and we send VERB_RENDEZVOUS to both sides: to ours
+		 * directly and with PROXY_SEND to theirs.
+		 */
+		STATE_MESSAGE_PROXY_UNITE = 5,
+
+		/**
+		 * Request that a cluster member send a packet to a locally-known peer:
+		 *   <[5] ZeroTier address of recipient>
+		 *   <[1] packet verb>
+		 *   <[2] length of packet payload>
+		 *   <[...] packet payload>
+		 *
+		 * This differs from RELAY in that it requests the receiving cluster
+		 * member to actually compose a ZeroTier Packet from itself to the
+		 * provided recipient. RELAY simply says "please forward this blob."
+		 * RELAY is used to implement peer-to-peer relaying with RENDEZVOUS,
+		 * while PROXY_SEND is used to implement proxy sending (which right
+		 * now is only used to send RENDEZVOUS).
+		 */
+		STATE_MESSAGE_PROXY_SEND = 6,
+
+		/**
+		 * Replicate a network config for a network we belong to:
+		 *   <[8] 64-bit network ID>
+		 *   <[2] 16-bit length of network config>
+		 *   <[...] serialized network config>
+		 *
+		 * This is used by clusters to avoid every member having to query
+		 * for the same netconf for networks all members belong to.
+		 *
+		 * TODO: not implemented yet!
+		 */
+		STATE_MESSAGE_NETWORK_CONFIG = 7
+	};
+
+	/**
+	 * Construct a new cluster
+	 */
+	Cluster(
+		const RuntimeEnvironment *renv,
+		uint16_t id,
+		const std::vector<InetAddress> &zeroTierPhysicalEndpoints,
+		int32_t x,
+		int32_t y,
+		int32_t z,
+		void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+		void *sendFunctionArg,
+		int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+		void *addressToLocationFunctionArg);
+
+	~Cluster();
+
+	/**
+	 * @return This cluster member's ID
+	 */
+	inline uint16_t id() const throw() { return _id; }
+
+	/**
+	 * Handle an incoming intra-cluster message
+	 *
+	 * @param data Message data
+	 * @param len Message length (max: ZT_CLUSTER_MAX_MESSAGE_LENGTH)
+	 */
+	void handleIncomingStateMessage(const void *msg,unsigned int len);
+
+	/**
+	 * Send this packet via another node in this cluster if another node has this peer
+	 *
+	 * @param fromPeerAddress Source peer address (if known, should be NULL for fragments)
+	 * @param toPeerAddress Destination peer address
+	 * @param data Packet or packet fragment data
+	 * @param len Length of packet or fragment
+	 * @param unite If true, also request proxy unite across cluster
+	 * @return True if this data was sent via another cluster member, false if none have this peer
+	 */
+	bool sendViaCluster(const Address &fromPeerAddress,const Address &toPeerAddress,const void *data,unsigned int len,bool unite);
+
+	/**
+	 * Advertise to the cluster that we have this peer
+	 *
+	 * @param peerId Identity of peer that we have
+	 * @param physicalAddress Physical address of peer (from our POV)
+	 */
+	void replicateHavePeer(const Identity &peerId,const InetAddress &physicalAddress);
+
+	/**
+	 * Advertise a multicast LIKE to the cluster
+	 *
+	 * @param nwid Network ID
+	 * @param peerAddress Peer address that sent LIKE
+	 * @param group Multicast group
+	 */
+	void replicateMulticastLike(uint64_t nwid,const Address &peerAddress,const MulticastGroup &group);
+
+	/**
+	 * Advertise a network COM to the cluster
+	 *
+	 * @param com Certificate of network membership (contains peer and network ID)
+	 */
+	void replicateCertificateOfNetworkMembership(const CertificateOfMembership &com);
+
+	/**
+	 * Call every ~ZT_CLUSTER_PERIODIC_TASK_PERIOD milliseconds.
+	 */
+	void doPeriodicTasks();
+
+	/**
+	 * Add a member ID to this cluster
+	 *
+	 * @param memberId Member ID
+	 */
+	void addMember(uint16_t memberId);
+
+	/**
+	 * Remove a member ID from this cluster
+	 *
+	 * @param memberId Member ID to remove
+	 */
+	void removeMember(uint16_t memberId);
+
+	/**
+	 * Find a better cluster endpoint for this peer (if any)
+	 *
+	 * @param redirectTo InetAddress to be set to a better endpoint (if there is one)
+	 * @param peerAddress Address of peer to (possibly) redirect
+	 * @param peerPhysicalAddress Physical address of peer's current best path (where packet was most recently received or getBestPath()->address())
+	 * @param offload Always redirect if possible -- can be used to offload peers during shutdown
+	 * @return True if redirectTo was set to a new address, false if redirectTo was not modified
+	 */
+	bool findBetterEndpoint(InetAddress &redirectTo,const Address &peerAddress,const InetAddress &peerPhysicalAddress,bool offload);
+
+	/**
+	 * Fill out ZT_ClusterStatus structure (from core API)
+	 *
+	 * @param status Reference to structure to hold result (anything there is replaced)
+	 */
+	void status(ZT_ClusterStatus &status) const;
+
+private:
+	void _send(uint16_t memberId,StateMessageType type,const void *msg,unsigned int len);
+	void _flush(uint16_t memberId);
+
+	// These are initialized in the constructor and remain immutable
+	uint16_t _masterSecret[ZT_SHA512_DIGEST_LEN / sizeof(uint16_t)];
+	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
+	const RuntimeEnvironment *RR;
+	void (*_sendFunction)(void *,unsigned int,const void *,unsigned int);
+	void *_sendFunctionArg;
+	int (*_addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *);
+	void *_addressToLocationFunctionArg;
+	const int32_t _x;
+	const int32_t _y;
+	const int32_t _z;
+	const uint16_t _id;
+	const std::vector<InetAddress> _zeroTierPhysicalEndpoints;
+	// end immutable fields
+
+	struct _Member
+	{
+		unsigned char key[ZT_PEER_SECRET_KEY_LENGTH];
+
+		uint64_t lastReceivedAliveAnnouncement;
+		uint64_t lastAnnouncedAliveTo;
+
+		uint64_t load;
+		int32_t x,y,z;
+
+		std::vector<InetAddress> zeroTierPhysicalEndpoints;
+
+		Buffer<ZT_CLUSTER_MAX_MESSAGE_LENGTH> q;
+
+		Mutex lock;
+
+		inline void clear()
+		{
+			lastReceivedAliveAnnouncement = 0;
+			lastAnnouncedAliveTo = 0;
+			load = 0;
+			x = 0;
+			y = 0;
+			z = 0;
+			zeroTierPhysicalEndpoints.clear();
+			q.clear();
+		}
+
+		_Member() { this->clear(); }
+		~_Member() { Utils::burn(key,sizeof(key)); }
+	};
+	_Member *const _members;
+
+	std::vector<uint16_t> _memberIds;
+	Mutex _memberIds_m;
+
+	struct _PA
+	{
+		_PA() : ts(0),mid(0xffff) {}
+		uint64_t ts;
+		uint16_t mid;
+	};
+	Hashtable< Address,_PA > _peerAffinities;
+	Mutex _peerAffinities_m;
+
+	uint64_t _lastCleanedPeerAffinities;
+	uint64_t _lastCheckedPeersForAnnounce;
+	uint64_t _lastFlushed;
+};
+
+} // namespace ZeroTier
+
+#endif // ZT_ENABLE_CLUSTER
+
+#endif

node/Constants.hpp

@@ -173,16 +173,11 @@
 
 /**
  * Timeout for receipt of fragmented packets in ms
- *
- * Since there's no retransmits, this is just a really bad case scenario for
- * transit time. It's short enough that a DOS attack from exhausing buffers is
- * very unlikely, as the transfer rate would have to be fast enough to fill
- * system memory in this time.
  */
-#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 1000
+#define ZT_FRAGMENTED_PACKET_RECEIVE_TIMEOUT 500
 
 /**
- * Length of secret key in bytes -- 256-bit for Salsa20
+ * Length of secret key in bytes -- 256-bit -- do not change
  */
 #define ZT_PEER_SECRET_KEY_LENGTH 32
 
@@ -194,7 +189,7 @@
 /**
  * Overriding granularity for timer tasks to prevent CPU-intensive thrashing on every packet
  */
-#define ZT_CORE_TIMER_TASK_GRANULARITY 1000
+#define ZT_CORE_TIMER_TASK_GRANULARITY 500
 
 /**
  * How long to remember peer records in RAM if they haven't been used
@@ -264,33 +259,22 @@
  * This is also how often pings will be retried to upstream peers (relays, roots)
  * constantly until something is heard.
  */
-#define ZT_PING_CHECK_INVERVAL 6250
+#define ZT_PING_CHECK_INVERVAL 9500
 
 /**
  * Delay between ordinary case pings of direct links
  */
-#define ZT_PEER_DIRECT_PING_DELAY 120000
-
-/**
- * Delay between requests for updated network autoconf information
- */
-#define ZT_NETWORK_AUTOCONF_DELAY 60000
+#define ZT_PEER_DIRECT_PING_DELAY 60000
 
 /**
  * Timeout for overall peer activity (measured from last receive)
  */
-#define ZT_PEER_ACTIVITY_TIMEOUT (ZT_PEER_DIRECT_PING_DELAY + (ZT_PING_CHECK_INVERVAL * 3))
+#define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 4) + ZT_PING_CHECK_INVERVAL)
 
 /**
- * Stop relaying via peers that have not responded to direct sends
- *
- * When we send something (including frames), we generally expect a response.
- * Switching relays if no response in a short period of time causes more
- * rapid failover if a root server goes down or becomes unreachable. In the
- * mistaken case, little harm is done as it'll pick the next-fastest
- * root server and will switch back eventually.
+ * Delay between requests for updated network autoconf information
  */
-#define ZT_PEER_RELAY_CONVERSATION_LATENCY_THRESHOLD 10000
+#define ZT_NETWORK_AUTOCONF_DELAY 60000
 
 /**
  * Minimum interval between attempts by relays to unite peers
@@ -299,7 +283,7 @@
  * a RENDEZVOUS message no more than this often. This instructs the peers
  * to attempt NAT-t and gives each the other's corresponding IP:port pair.
  */
-#define ZT_MIN_UNITE_INTERVAL 60000
+#define ZT_MIN_UNITE_INTERVAL 30000
 
 /**
  * Delay between initial direct NAT-t packet and more aggressive techniques
@@ -309,20 +293,17 @@
  */
 #define ZT_NAT_T_TACTICAL_ESCALATION_DELAY 1000
 
-/**
- * Size of anti-recursion history (see AntiRecursion.hpp)
- */
-#define ZT_ANTIRECURSION_HISTORY_SIZE 16
-
 /**
  * Minimum delay between attempts to confirm new paths to peers (to avoid HELLO flooding)
  */
-#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 5000
+#define ZT_MIN_PATH_CONFIRMATION_INTERVAL 1000
 
 /**
- * Interval between direct path pushes in milliseconds
+ * How long (max) to remember network certificates of membership?
+ *
+ * This only applies to networks we don't belong to.
  */
-#define ZT_DIRECT_PATH_PUSH_INTERVAL 300000
+#define ZT_PEER_NETWORK_COM_EXPIRATION 3600000
 
 /**
  * Sanity limit on maximum bridge routes
@@ -340,6 +321,30 @@
  */
 #define ZT_MAX_BRIDGE_SPAM 16
 
+/**
+ * Interval between direct path pushes in milliseconds
+ */
+#define ZT_DIRECT_PATH_PUSH_INTERVAL 120000
+
+/**
+ * Time horizon for push direct paths cutoff
+ */
+#define ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME 60000
+
+/**
+ * Maximum number of direct path pushes within cutoff time
+ *
+ * This limits response to PUSH_DIRECT_PATHS to CUTOFF_LIMIT responses
+ * per CUTOFF_TIME milliseconds per peer to prevent this from being
+ * useful for DOS amplification attacks.
+ */
+#define ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT 5
+
+/**
+ * Maximum number of paths per IP scope (e.g. global, link-local) and family (e.g. v4/v6)
+ */
+#define ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY 1
+
 /**
  * A test pseudo-network-ID that can be joined
  *

node/Defaults.cpp

@@ -1,82 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#include <stdio.h>
-#include <stdlib.h>
-#include <string.h>
-
-#include "../include/ZeroTierOne.h"
-
-#include "Constants.hpp"
-#include "Defaults.hpp"
-#include "Utils.hpp"
-
-// bin2c'd signed default root topology dictionary
-#include "../root-topology/ZT_DEFAULT_ROOT_TOPOLOGY.c"
-
-#ifdef __WINDOWS__
-#include <WinSock2.h>
-#include <Windows.h>
-#include <ShlObj.h>
-#endif
-
-namespace ZeroTier {
-
-const Defaults ZT_DEFAULTS;
-
-static inline std::map< Address,Identity > _mkRootTopologyAuth()
-{
-	std::map< Address,Identity > ua;
-
-	{ // 0001
-		Identity id("77792b1c02:0:b5c361e8e9c2154e82c3e902fdfc337468b092a7c4d8dc685c37eb10ee4f3c17cc0bb1d024167e8cb0824d12263428373582da3d0a9a14b36e4546c317e811e6");
-		ua[id.address()] = id;
-	}
-	{ // 0002
-		Identity id("86921e6de1:0:9ba04f9f12ed54ef567f548cb69d31e404537d7b0ee000c63f3d7c8d490a1a47a5a5b2af0cbe12d23f9194270593f298d936d7c872612ea509ef1c67ce2c7fc1");
-		ua[id.address()] = id;
-	}
-	{ // 0003
-		Identity id("90302b7025:0:358154a57af1b7afa07d0d91b69b92eaad2f11ade7f02343861f0c1b757d15626e8cb7f08fc52993d2202a39cbf5128c5647ee8c63d27d92db5a1d0fbe1eba19");
-		ua[id.address()] = id;
-	}
-	{ // 0004
-		Identity id("e5174078ee:0:c3f90daa834a74ee47105f5726ae2e29fc8ae0e939c9326788b52b16d847354de8de3b13a81896bbb509b91e1da21763073a30bbfb2b8e994550798d30a2d709");
-		ua[id.address()] = id;
-	}
-
-	return ua;
-}
-
-Defaults::Defaults() :
-	defaultRootTopology((const char *)ZT_DEFAULT_ROOT_TOPOLOGY,ZT_DEFAULT_ROOT_TOPOLOGY_LEN),
-	rootTopologyAuthorities(_mkRootTopologyAuth()),
-	v4Broadcast(((uint32_t)0xffffffff),ZT_DEFAULT_PORT)
-{
-}
-
-} // namespace ZeroTier

node/Dictionary.cpp

@@ -32,6 +32,68 @@
 
 namespace ZeroTier {
 
+Dictionary::iterator Dictionary::find(const std::string &key)
+{
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i;
+	}
+	return end();
+}
+Dictionary::const_iterator Dictionary::find(const std::string &key) const
+{
+	for(const_iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i;
+	}
+	return end();
+}
+
+bool Dictionary::getBoolean(const std::string &key,bool dfl) const
+{
+	const_iterator e(find(key));
+	if (e == end())
+		return dfl;
+	if (e->second.length() < 1)
+		return dfl;
+	switch(e->second[0]) {
+		case '1':
+		case 't':
+		case 'T':
+		case 'y':
+		case 'Y':
+			return true;
+	}
+	return false;
+}
+
+std::string &Dictionary::operator[](const std::string &key)
+{
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i->second;
+	}
+	push_back(std::pair<std::string,std::string>(key,std::string()));
+	std::sort(begin(),end());
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key)
+			return i->second;
+	}
+	return front().second; // should be unreachable!
+}
+
+std::string Dictionary::toString() const
+{
+	std::string s;
+	for(const_iterator kv(begin());kv!=end();++kv) {
+		_appendEsc(kv->first.data(),(unsigned int)kv->first.length(),s);
+		s.push_back('=');
+		_appendEsc(kv->second.data(),(unsigned int)kv->second.length(),s);
+		s.append(ZT_EOL_S);
+	}
+	return s;
+}
+
 void Dictionary::updateFromString(const char *s,unsigned int maxlen)
 {
 	bool escapeState = false;
@@ -80,6 +142,16 @@ void Dictionary::fromString(const char *s,unsigned int maxlen)
 	updateFromString(s,maxlen);
 }
 
+void Dictionary::eraseKey(const std::string &key)
+{
+	for(iterator i(begin());i!=end();++i) {
+		if (i->first == key) {
+			this->erase(i);
+			return;
+		}
+	}
+}
+
 bool Dictionary::sign(const Identity &id,uint64_t now)
 {
 	try {

node/Dictionary.hpp

@@ -31,8 +31,9 @@
 #include <stdint.h>
 
 #include <string>
-#include <map>
+#include <vector>
 #include <stdexcept>
+#include <algorithm>
 
 #include "Constants.hpp"
 #include "Utils.hpp"
@@ -56,12 +57,12 @@ class Identity;
  *
  * Keys beginning with "~!" are reserved for signature data fields.
  *
- * Note: the signature code depends on std::map<> being sorted, but no
- * other code does. So if the underlying data structure is ever swapped
- * out for an unsorted one, the signature code will have to be updated
- * to sort before composing the string to sign.
+ * It's stored as a simple vector and can be linearly scanned or
+ * binary searched. Dictionaries are only used for very small things
+ * outside the core loop, so this is not a significant performance
+ * issue and it reduces memory use and code footprint.
  */
-class Dictionary : public std::map<std::string,std::string>
+class Dictionary : public std::vector< std::pair<std::string,std::string> >
 {
 public:
 	Dictionary() {}
@@ -77,21 +78,8 @@ public:
 	 */
 	Dictionary(const std::string &s) { fromString(s.c_str(),(unsigned int)s.length()); }
 
-	/**
-	 * Get a key, throwing an exception if it is not present
-	 *
-	 * @param key Key to look up
-	 * @return Reference to value
-	 * @throws std::invalid_argument Key not found
-	 */
-	inline const std::string &get(const std::string &key) const
-		throw(std::invalid_argument)
-	{
-		const_iterator e(find(key));
-		if (e == end())
-			throw std::invalid_argument(std::string("missing required field: ")+key);
-		return e->second;
-	}
+	iterator find(const std::string &key);
+	const_iterator find(const std::string &key) const;
 
 	/**
 	 * Get a key, returning a default if not present
@@ -113,23 +101,7 @@ public:
 	 * @param dfl Default boolean result if key not found or empty (default: false)
 	 * @return Boolean value of key
 	 */
-	inline bool getBoolean(const std::string &key,bool dfl = false) const
-	{
-		const_iterator e(find(key));
-		if (e == end())
-			return dfl;
-		if (e->second.length() < 1)
-			return dfl;
-		switch(e->second[0]) {
-			case '1':
-			case 't':
-			case 'T':
-			case 'y':
-			case 'Y':
-				return true;
-		}
-		return false;
-	}
+	bool getBoolean(const std::string &key,bool dfl = false) const;
 
 	/**
 	 * @param key Key to get
@@ -170,6 +142,8 @@ public:
 		return Utils::strTo64(e->second.c_str());
 	}
 
+	std::string &operator[](const std::string &key);
+
 	/**
 	 * @param key Key to set
 	 * @param value String value
@@ -239,17 +213,7 @@ public:
 	/**
 	 * @return String-serialized dictionary
 	 */
-	inline std::string toString() const
-	{
-		std::string s;
-		for(const_iterator kv(begin());kv!=end();++kv) {
-			_appendEsc(kv->first.data(),(unsigned int)kv->first.length(),s);
-			s.push_back('=');
-			_appendEsc(kv->second.data(),(unsigned int)kv->second.length(),s);
-			s.append(ZT_EOL_S);
-		}
-		return s;
-	}
+	std::string toString() const;
 
 	/**
 	 * Clear and initialize from a string
@@ -278,14 +242,19 @@ public:
 	 */
 	uint64_t signatureTimestamp() const;
 
+	/**
+	 * @param key Key to erase
+	 */
+	void eraseKey(const std::string &key);
+
 	/**
 	 * Remove any signature from this dictionary
 	 */
 	inline void removeSignature()
 	{
-		erase(ZT_DICTIONARY_SIGNATURE);
-		erase(ZT_DICTIONARY_SIGNATURE_IDENTITY);
-		erase(ZT_DICTIONARY_SIGNATURE_TIMESTAMP);
+		eraseKey(ZT_DICTIONARY_SIGNATURE);
+		eraseKey(ZT_DICTIONARY_SIGNATURE_IDENTITY);
+		eraseKey(ZT_DICTIONARY_SIGNATURE_TIMESTAMP);
 	}
 
 	/**
@@ -305,21 +274,6 @@ public:
 	 */
 	bool verify(const Identity &id) const;
 
-  inline bool operator==(const Dictionary &d) const
-  {
-    // std::map::operator== is broken on uclibc++
-    if (size() != d.size())
-      return false;
-    const_iterator a(begin());
-    const_iterator b(d.begin());
-    while (a != end()) {
-      if (*(a++) != *(b++))
-        return false;
-    }
-    return true;
-  }
-  inline bool operator!=(const Dictionary &d) const { return (!(*this == d)); }
-
 private:
 	void _mkSigBuf(std::string &buf) const;
 	static void _appendEsc(const char *data,unsigned int len,std::string &to);

node/Hashtable.hpp

@@ -322,7 +322,6 @@ public:
 		b->next = _t[bidx];
 		_t[bidx] = b;
 		++_s;
-
 		return b->v;
 	}
 
@@ -351,7 +350,6 @@ public:
 		b->next = _t[bidx];
 		_t[bidx] = b;
 		++_s;
-
 		return b->v;
 	}
 
@@ -382,7 +380,10 @@ private:
 	}
 	static inline unsigned long _hc(const uint32_t i)
 	{
-		// In the uint32_t case we use a simple multiplier for hashing to ensure coverage
+		return ((unsigned long)i * (unsigned long)0x9e3779b1);
+	}
+	static inline unsigned long _hc(const uint16_t i)
+	{
 		return ((unsigned long)i * (unsigned long)0x9e3779b1);
 	}
 

node/Identity.cpp

@@ -41,7 +41,6 @@
 
 #define ZT_IDENTITY_GEN_HASHCASH_FIRST_BYTE_LESS_THAN 17
 #define ZT_IDENTITY_GEN_MEMORY 2097152
-#define ZT_IDENTITY_GEN_SALSA20_ROUNDS 20
 
 namespace ZeroTier {
 
@@ -55,8 +54,8 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub
 	// ordinary Salsa20 is randomly seekable. This is good for a cipher
 	// but is not what we want for sequential memory-harndess.
 	memset(genmem,0,ZT_IDENTITY_GEN_MEMORY);
-	Salsa20 s20(digest,256,(char *)digest + 32,ZT_IDENTITY_GEN_SALSA20_ROUNDS);
-	s20.encrypt((char *)genmem,(char *)genmem,64);
+	Salsa20 s20(digest,256,(char *)digest + 32);
+	s20.encrypt20((char *)genmem,(char *)genmem,64);
 	for(unsigned long i=64;i<ZT_IDENTITY_GEN_MEMORY;i+=64) {
 		unsigned long k = i - 64;
 		*((uint64_t *)((char *)genmem + i)) = *((uint64_t *)((char *)genmem + k));
@@ -67,7 +66,7 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub
 		*((uint64_t *)((char *)genmem + i + 40)) = *((uint64_t *)((char *)genmem + k + 40));
 		*((uint64_t *)((char *)genmem + i + 48)) = *((uint64_t *)((char *)genmem + k + 48));
 		*((uint64_t *)((char *)genmem + i + 56)) = *((uint64_t *)((char *)genmem + k + 56));
-		s20.encrypt((char *)genmem + i,(char *)genmem + i,64);
+		s20.encrypt20((char *)genmem + i,(char *)genmem + i,64);
 	}
 
 	// Render final digest using genmem as a lookup table
@@ -77,7 +76,7 @@ static inline void _computeMemoryHardHash(const void *publicKey,unsigned int pub
 		uint64_t tmp = ((uint64_t *)genmem)[idx2];
 		((uint64_t *)genmem)[idx2] = ((uint64_t *)digest)[idx1];
 		((uint64_t *)digest)[idx1] = tmp;
-		s20.encrypt(digest,digest,64);
+		s20.encrypt20(digest,digest,64);
 	}
 }
 

node/Identity.hpp

@@ -38,8 +38,7 @@
 #include "Address.hpp"
 #include "C25519.hpp"
 #include "Buffer.hpp"
-
-#define ZT_IDENTITY_MAX_BINARY_SERIALIZED_LENGTH (ZT_ADDRESS_LENGTH + 1 + ZT_C25519_PUBLIC_KEY_LEN + 1 + ZT_C25519_PRIVATE_KEY_LEN)
+#include "SHA512.hpp"
 
 namespace ZeroTier {
 
@@ -93,8 +92,7 @@ public:
 	}
 
 	template<unsigned int C>
-	Identity(const Buffer<C> &b,unsigned int startAt = 0)
-		throw(std::out_of_range,std::invalid_argument) :
+	Identity(const Buffer<C> &b,unsigned int startAt = 0) :
 		_privateKey((C25519::Private *)0)
 	{
 		deserialize(b,startAt);
@@ -139,6 +137,21 @@ public:
 	 */
 	inline bool hasPrivate() const throw() { return (_privateKey != (C25519::Private *)0); }
 
+	/**
+	 * Compute the SHA512 hash of our private key (if we have one)
+	 *
+	 * @param sha Buffer to receive SHA512 (MUST be ZT_SHA512_DIGEST_LEN (64) bytes in length)
+	 * @return True on success, false if no private key
+	 */
+	inline bool sha512PrivateKey(void *sha) const
+	{
+		if (_privateKey) {
+			SHA512::hash(sha,_privateKey->data,ZT_C25519_PRIVATE_KEY_LEN);
+			return true;
+		}
+		return false;
+	}
+
 	/**
 	 * Sign a message with this identity (private key required)
 	 *
@@ -220,7 +233,6 @@ public:
 	 */
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b,bool includePrivate = false) const
-		throw(std::out_of_range)
 	{
 		_address.appendTo(b);
 		b.append((unsigned char)IDENTITY_TYPE_C25519);
@@ -245,7 +257,6 @@ public:
 	 */
 	template<unsigned int C>
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
-		throw(std::out_of_range,std::invalid_argument)
 	{
 		delete _privateKey;
 		_privateKey = (C25519::Private *)0;

node/IncomingPacket.hpp

@@ -107,6 +107,27 @@ public:
 	 */
 	inline uint64_t receiveTime() const throw() { return _receiveTime; }
 
+	/**
+	 * Compute the Salsa20/12+SHA512 proof of work function
+	 *
+	 * @param difficulty Difficulty in bits (max: 64)
+	 * @param challenge Challenge string
+	 * @param challengeLength Length of challenge in bytes (max allowed: ZT_PROTO_MAX_PACKET_LENGTH)
+	 * @param result Buffer to fill with 16-byte result
+	 */
+	static void computeSalsa2012Sha512ProofOfWork(unsigned int difficulty,const void *challenge,unsigned int challengeLength,unsigned char result[16]);
+
+	/**
+	 * Verify the result of Salsa20/12+SHA512 proof of work
+	 *
+	 * @param difficulty Difficulty in bits (max: 64)
+	 * @param challenge Challenge bytes
+	 * @param challengeLength Length of challenge in bytes (max allowed: ZT_PROTO_MAX_PACKET_LENGTH)
+	 * @param proposedResult Result supplied by client
+	 * @return True if result is valid
+	 */
+	static bool testSalsa2012Sha512ProofOfWorkResult(unsigned int difficulty,const void *challenge,unsigned int challengeLength,const unsigned char proposedResult[16]);
+
 private:
 	// These are called internally to handle packet contents once it has
 	// been authenticated, decrypted, decompressed, and classified.
@@ -117,6 +138,7 @@ private:
 	bool _doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
+	bool _doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_LIKE(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
@@ -124,8 +146,11 @@ private:
 	bool _doMULTICAST_GATHER(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doMULTICAST_FRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 	bool _doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
+	bool _doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
+	bool _doCIRCUIT_TEST_REPORT(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
+	bool _doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer);
 
-	// Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to join
+	// Send an ERROR_NEED_MEMBERSHIP_CERTIFICATE to a peer indicating that an updated cert is needed to communicate
 	void _sendErrorNeedCertificate(const RuntimeEnvironment *RR,const SharedPtr<Peer> &peer,uint64_t nwid);
 
 	uint64_t _receiveTime;

node/InetAddress.cpp

@@ -77,14 +77,12 @@ InetAddress::IpScope InetAddress::ipScope() const
 					if ((ip & 0xffff0000) == 0xc0a80000) return IP_SCOPE_PRIVATE;				// 192.168.0.0/16
 					break;
 				case 0xff: return IP_SCOPE_NONE;                                      // 255.0.0.0/8 (broadcast, or unused/unusable)
-				default:
-					switch(ip >> 28) {
-						case 0xe: return IP_SCOPE_MULTICAST;                              // 224.0.0.0/4
-						case 0xf: return IP_SCOPE_PSEUDOPRIVATE;                          // 240.0.0.0/4 ("reserved," usually unusable)
-						default: return IP_SCOPE_GLOBAL;                                  // everything else
-					}
-					break;
 			}
+			switch(ip >> 28) {
+				case 0xe: return IP_SCOPE_MULTICAST;                              // 224.0.0.0/4
+				case 0xf: return IP_SCOPE_PSEUDOPRIVATE;                          // 240.0.0.0/4 ("reserved," usually unusable)
+			}
+			return IP_SCOPE_GLOBAL;
 		}	break;
 
 		case AF_INET6: {

node/InetAddress.hpp

@@ -38,16 +38,22 @@
 #include "../include/ZeroTierOne.h"
 #include "Utils.hpp"
 #include "MAC.hpp"
+#include "Buffer.hpp"
 
 namespace ZeroTier {
 
+/**
+ * Maximum integer value of enum IpScope
+ */
+#define ZT_INETADDRESS_MAX_SCOPE 7
+
 /**
  * Extends sockaddr_storage with friendly C++ methods
  *
  * This is basically a "mixin" for sockaddr_storage. It adds methods and
  * operators, but does not modify the structure. This can be cast to/from
- * sockaddr_storage and used interchangeably. Don't change this as it's
- * used in a few places.
+ * sockaddr_storage and used interchangeably. DO NOT change this by e.g.
+ * adding non-static fields, since much code depends on this identity.
  */
 struct InetAddress : public sockaddr_storage
 {
@@ -65,7 +71,8 @@ struct InetAddress : public sockaddr_storage
 	 * IP address scope
 	 *
 	 * Note that these values are in ascending order of path preference and
-	 * MUST remain that way or Path must be changed to reflect.
+	 * MUST remain that way or Path must be changed to reflect. Also be sure
+	 * to change ZT_INETADDRESS_MAX_SCOPE if the max changes.
 	 */
 	enum IpScope
 	{
@@ -99,74 +106,88 @@ struct InetAddress : public sockaddr_storage
 	inline InetAddress &operator=(const InetAddress &a)
 		throw()
 	{
-		memcpy(this,&a,sizeof(InetAddress));
+		if (&a != this)
+			memcpy(this,&a,sizeof(InetAddress));
 		return *this;
 	}
 
 	inline InetAddress &operator=(const InetAddress *a)
 		throw()
 	{
-		memcpy(this,a,sizeof(InetAddress));
+		if (a != this)
+			memcpy(this,a,sizeof(InetAddress));
 		return *this;
 	}
 
 	inline InetAddress &operator=(const struct sockaddr_storage &ss)
 		throw()
 	{
-		memcpy(this,&ss,sizeof(InetAddress));
+		if (reinterpret_cast<const InetAddress *>(&ss) != this)
+			memcpy(this,&ss,sizeof(InetAddress));
 		return *this;
 	}
 
 	inline InetAddress &operator=(const struct sockaddr_storage *ss)
 		throw()
 	{
-		memcpy(this,ss,sizeof(InetAddress));
+		if (reinterpret_cast<const InetAddress *>(ss) != this)
+			memcpy(this,ss,sizeof(InetAddress));
 		return *this;
 	}
 
 	inline InetAddress &operator=(const struct sockaddr_in &sa)
 		throw()
 	{
-		memset(this,0,sizeof(InetAddress));
-		memcpy(this,&sa,sizeof(struct sockaddr_in));
+		if (reinterpret_cast<const InetAddress *>(&sa) != this) {
+			memset(this,0,sizeof(InetAddress));
+			memcpy(this,&sa,sizeof(struct sockaddr_in));
+		}
 		return *this;
 	}
 
 	inline InetAddress &operator=(const struct sockaddr_in *sa)
 		throw()
 	{
-		memset(this,0,sizeof(InetAddress));
-		memcpy(this,sa,sizeof(struct sockaddr_in));
+		if (reinterpret_cast<const InetAddress *>(sa) != this) {
+			memset(this,0,sizeof(InetAddress));
+			memcpy(this,sa,sizeof(struct sockaddr_in));
+		}
 		return *this;
 	}
 
 	inline InetAddress &operator=(const struct sockaddr_in6 &sa)
 		throw()
 	{
-		memset(this,0,sizeof(InetAddress));
-		memcpy(this,&sa,sizeof(struct sockaddr_in6));
+		if (reinterpret_cast<const InetAddress *>(&sa) != this) {
+			memset(this,0,sizeof(InetAddress));
+			memcpy(this,&sa,sizeof(struct sockaddr_in6));
+		}
 		return *this;
 	}
 
 	inline InetAddress &operator=(const struct sockaddr_in6 *sa)
 		throw()
 	{
-		memset(this,0,sizeof(InetAddress));
-		memcpy(this,sa,sizeof(struct sockaddr_in6));
+		if (reinterpret_cast<const InetAddress *>(sa) != this) {
+			memset(this,0,sizeof(InetAddress));
+			memcpy(this,sa,sizeof(struct sockaddr_in6));
+		}
 		return *this;
 	}
 
 	inline InetAddress &operator=(const struct sockaddr &sa)
 		throw()
 	{
-		memset(this,0,sizeof(InetAddress));
-		switch(sa.sa_family) {
-			case AF_INET:
-				memcpy(this,&sa,sizeof(struct sockaddr_in));
-				break;
-			case AF_INET6:
-				memcpy(this,&sa,sizeof(struct sockaddr_in6));
-				break;
+		if (reinterpret_cast<const InetAddress *>(&sa) != this) {
+			memset(this,0,sizeof(InetAddress));
+			switch(sa.sa_family) {
+				case AF_INET:
+					memcpy(this,&sa,sizeof(struct sockaddr_in));
+					break;
+				case AF_INET6:
+					memcpy(this,&sa,sizeof(struct sockaddr_in6));
+					break;
+			}
 		}
 		return *this;
 	}
@@ -174,14 +195,16 @@ struct InetAddress : public sockaddr_storage
 	inline InetAddress &operator=(const struct sockaddr *sa)
 		throw()
 	{
-		memset(this,0,sizeof(InetAddress));
-		switch(sa->sa_family) {
-			case AF_INET:
-				memcpy(this,sa,sizeof(struct sockaddr_in));
-				break;
-			case AF_INET6:
-				memcpy(this,sa,sizeof(struct sockaddr_in6));
-				break;
+		if (reinterpret_cast<const InetAddress *>(sa) != this) {
+			memset(this,0,sizeof(InetAddress));
+			switch(sa->sa_family) {
+				case AF_INET:
+					memcpy(this,sa,sizeof(struct sockaddr_in));
+					break;
+				case AF_INET6:
+					memcpy(this,sa,sizeof(struct sockaddr_in6));
+					break;
+			}
 		}
 		return *this;
 	}
@@ -303,7 +326,7 @@ struct InetAddress : public sockaddr_storage
 	inline bool isV6() const throw() { return (ss_family == AF_INET6); }
 
 	/**
-	 * @return pointer to raw IP address bytes
+	 * @return pointer to raw address bytes or NULL if not available
 	 */
 	inline const void *rawIpData() const
 		throw()
@@ -316,27 +339,19 @@ struct InetAddress : public sockaddr_storage
 	}
 
 	/**
-	 * @return pointer to raw IP address bytes
-	 */
-	inline void *rawIpData()
-		throw()
-	{
-		switch(ss_family) {
-			case AF_INET: return (void *)&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr);
-			case AF_INET6: return (void *)(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr);
-			default: return 0;
-		}
-	}
-
-	/**
+	 * Performs an IP-only comparison or, if that is impossible, a memcmp()
+	 *
 	 * @param a InetAddress to compare again
 	 * @return True if only IP portions are equal (false for non-IP or null addresses)
 	 */
 	inline bool ipsEqual(const InetAddress &a) const
 	{
-		switch(ss_family) {
-			case AF_INET: return (reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_addr.s_addr);
-			case AF_INET6: return (memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0);
+		if (ss_family == a.ss_family) {
+			if (ss_family == AF_INET)
+				return (reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr == reinterpret_cast<const struct sockaddr_in *>(&a)->sin_addr.s_addr);
+			if (ss_family == AF_INET6)
+				return (memcmp(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,reinterpret_cast<const struct sockaddr_in6 *>(&a)->sin6_addr.s6_addr,16) == 0);
+			return (memcmp(this,&a,sizeof(InetAddress)) == 0);
 		}
 		return false;
 	}
@@ -362,6 +377,62 @@ struct InetAddress : public sockaddr_storage
 	 */
 	inline operator bool() const throw() { return (ss_family != 0); }
 
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		// This is used in the protocol and must be the same as describe in places
+		// like VERB_HELLO in Packet.hpp.
+		switch(ss_family) {
+			case AF_INET:
+				b.append((uint8_t)0x04);
+				b.append(&(reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr),4);
+				b.append((uint16_t)port()); // just in case sin_port != uint16_t
+				return;
+			case AF_INET6:
+				b.append((uint8_t)0x06);
+				b.append(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,16);
+				b.append((uint16_t)port()); // just in case sin_port != uint16_t
+				return;
+			default:
+				b.append((uint8_t)0);
+				return;
+		}
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		memset(this,0,sizeof(InetAddress));
+		unsigned int p = startAt;
+		switch(b[p++]) {
+			case 0:
+				return 1;
+			case 0x01:
+				// TODO: Ethernet address (but accept for forward compatibility)
+				return 7;
+			case 0x02:
+				// TODO: Bluetooth address (but accept for forward compatibility) 
+				return 7;
+			case 0x03:
+				// TODO: Other address types (but accept for forward compatibility)
+				// These could be extended/optional things like AF_UNIX, LTE Direct, shared memory, etc.
+				return (unsigned int)(b.template at<uint16_t>(p) + 3); // other addresses begin with 16-bit non-inclusive length
+			case 0x04:
+				ss_family = AF_INET;
+				memcpy(&(reinterpret_cast<struct sockaddr_in *>(this)->sin_addr.s_addr),b.field(p,4),4); p += 4;
+				reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2;
+				break;
+			case 0x06:
+				ss_family = AF_INET6;
+				memcpy(reinterpret_cast<struct sockaddr_in6 *>(this)->sin6_addr.s6_addr,b.field(p,16),16); p += 16;
+				reinterpret_cast<struct sockaddr_in *>(this)->sin_port = Utils::hton(b.template at<uint16_t>(p)); p += 2;
+				break;
+			default:
+				throw std::invalid_argument("invalid serialized InetAddress");
+		}
+		return (p - startAt);
+	}
+
 	bool operator==(const InetAddress &a) const throw();
 	bool operator<(const InetAddress &a) const throw();
 	inline bool operator!=(const InetAddress &a) const throw() { return !(*this == a); }

node/Multicaster.cpp

@@ -37,6 +37,7 @@
 #include "Peer.hpp"
 #include "C25519.hpp"
 #include "CertificateOfMembership.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
@@ -174,116 +175,130 @@ void Multicaster::send(
 	unsigned long idxbuf[8194];
 	unsigned long *indexes = idxbuf;
 
-	Mutex::Lock _l(_groups_m);
-	MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
-
-	if (!gs.members.empty()) {
-		// Allocate a memory buffer if group is monstrous
-		if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
-			indexes = new unsigned long[gs.members.size()];
-
-		// Generate a random permutation of member indexes
-		for(unsigned long i=0;i<gs.members.size();++i)
-			indexes[i] = i;
-		for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
-			unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
-			unsigned long tmp = indexes[j];
-			indexes[j] = indexes[i];
-			indexes[i] = tmp;
+	try {
+		Mutex::Lock _l(_groups_m);
+		MulticastGroupStatus &gs = _groups[Multicaster::Key(nwid,mg)];
+
+		if (!gs.members.empty()) {
+			// Allocate a memory buffer if group is monstrous
+			if (gs.members.size() > (sizeof(idxbuf) / sizeof(unsigned long)))
+				indexes = new unsigned long[gs.members.size()];
+
+			// Generate a random permutation of member indexes
+			for(unsigned long i=0;i<gs.members.size();++i)
+				indexes[i] = i;
+			for(unsigned long i=(unsigned long)gs.members.size()-1;i>0;--i) {
+				unsigned long j = (unsigned long)RR->node->prng() % (i + 1);
+				unsigned long tmp = indexes[j];
+				indexes[j] = indexes[i];
+				indexes[i] = tmp;
+			}
 		}
-	}
 
-	if (gs.members.size() >= limit) {
-		// Skip queue if we already have enough members to complete the send operation
-		OutboundMulticast out;
-
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			1, // we'll still gather a little from peers to keep multicast list fresh
-			src,
-			mg,
-			etherType,
-			data,
-			len);
-
-		unsigned int count = 0;
-
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendOnly(RR,*ast);
-				if (++count >= limit)
-					break;
+		if (gs.members.size() >= limit) {
+			// Skip queue if we already have enough members to complete the send operation
+			OutboundMulticast out;
+
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				1, // we'll still gather a little from peers to keep multicast list fresh
+				src,
+				mg,
+				etherType,
+				data,
+				len);
+
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendOnly(RR,*ast); // optimization: don't use dedup log if it's a one-pass send
+					if (++count >= limit)
+						break;
+				}
 			}
-		}
 
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendOnly(RR,ma);
-				++count;
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendOnly(RR,ma); // optimization: don't use dedup log if it's a one-pass send
+					++count;
+				}
 			}
-		}
-	} else {
-		unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
-
-		if ((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY) {
-			gs.lastExplicitGather = now;
-			SharedPtr<Peer> sn(RR->topology->getBestRoot());
-			if (sn) {
-				TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
-
-				Packet outp(sn->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
-				outp.append(nwid);
-				outp.append((uint8_t)0);
-				mg.mac().appendTo(outp);
-				outp.append((uint32_t)mg.adi());
-				outp.append((uint32_t)gatherLimit);
-				outp.armor(sn->key(),true);
-				sn->send(RR,outp.data(),outp.size(),now);
+		} else {
+			unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
+
+			if ((gs.members.empty())||((now - gs.lastExplicitGather) >= ZT_MULTICAST_EXPLICIT_GATHER_DELAY)) {
+				gs.lastExplicitGather = now;
+				SharedPtr<Peer> r(RR->topology->getBestRoot());
+				if (r) {
+					TRACE(">>MC upstream GATHER up to %u for group %.16llx/%s",gatherLimit,nwid,mg.toString().c_str());
+
+					const CertificateOfMembership *com = (CertificateOfMembership *)0;
+					{
+						SharedPtr<Network> nw(RR->node->network(nwid));
+						if (nw) {
+							SharedPtr<NetworkConfig> nconf(nw->config2());
+							if ((nconf)&&(nconf->com())&&(nconf->isPrivate())&&(r->needsOurNetworkMembershipCertificate(nwid,now,true)))
+								com = &(nconf->com());
+						}
+					}
+
+					Packet outp(r->address(),RR->identity.address(),Packet::VERB_MULTICAST_GATHER);
+					outp.append(nwid);
+					outp.append((uint8_t)(com ? 0x01 : 0x00));
+					mg.mac().appendTo(outp);
+					outp.append((uint32_t)mg.adi());
+					outp.append((uint32_t)gatherLimit);
+					if (com)
+						com->serialize(outp);
+					outp.armor(r->key(),true);
+					r->send(RR,outp.data(),outp.size(),now);
+				}
+				gatherLimit = 0;
 			}
-			gatherLimit = 0;
-		}
 
-		gs.txQueue.push_back(OutboundMulticast());
-		OutboundMulticast &out = gs.txQueue.back();
-
-		out.init(
-			RR,
-			now,
-			nwid,
-			com,
-			limit,
-			gatherLimit,
-			src,
-			mg,
-			etherType,
-			data,
-			len);
-
-		unsigned int count = 0;
-
-		for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
-			if (*ast != RR->identity.address()) {
-				out.sendAndLog(RR,*ast);
-				if (++count >= limit)
-					break;
+			gs.txQueue.push_back(OutboundMulticast());
+			OutboundMulticast &out = gs.txQueue.back();
+
+			out.init(
+				RR,
+				now,
+				nwid,
+				com,
+				limit,
+				gatherLimit,
+				src,
+				mg,
+				etherType,
+				data,
+				len);
+
+			unsigned int count = 0;
+
+			for(std::vector<Address>::const_iterator ast(alwaysSendTo.begin());ast!=alwaysSendTo.end();++ast) {
+				if (*ast != RR->identity.address()) {
+					out.sendAndLog(RR,*ast);
+					if (++count >= limit)
+						break;
+				}
 			}
-		}
 
-		unsigned long idx = 0;
-		while ((count < limit)&&(idx < gs.members.size())) {
-			Address ma(gs.members[indexes[idx++]].address);
-			if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
-				out.sendAndLog(RR,ma);
-				++count;
+			unsigned long idx = 0;
+			while ((count < limit)&&(idx < gs.members.size())) {
+				Address ma(gs.members[indexes[idx++]].address);
+				if (std::find(alwaysSendTo.begin(),alwaysSendTo.end(),ma) == alwaysSendTo.end()) {
+					out.sendAndLog(RR,ma);
+					++count;
+				}
 			}
 		}
-	}
+	} catch ( ... ) {} // this is a sanity check to catch any failures and make sure indexes[] still gets deleted
 
 	// Free allocated memory buffer if any
 	if (indexes != idxbuf)

node/Network.cpp

@@ -37,6 +37,7 @@
 #include "Packet.hpp"
 #include "Buffer.hpp"
 #include "NetworkController.hpp"
+#include "Node.hpp"
 
 #include "../version.h"
 
@@ -59,6 +60,9 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 	Utils::snprintf(confn,sizeof(confn),"networks.d/%.16llx.conf",_id);
 	Utils::snprintf(mcdbn,sizeof(mcdbn),"networks.d/%.16llx.mcerts",_id);
 
+	// These files are no longer used, so clean them.
+	RR->node->dataStoreDelete(mcdbn);
+
 	if (_id == ZT_TEST_NETWORK_ID) {
 		applyConfiguration(NetworkConfig::createTestNetworkConfig(RR->identity.address()));
 
@@ -79,24 +83,6 @@ Network::Network(const RuntimeEnvironment *renv,uint64_t nwid) :
 			// Save a one-byte CR to persist membership while we request a real netconf
 			RR->node->dataStorePut(confn,"\n",1,false);
 		}
-
-		try {
-			std::string mcdb(RR->node->dataStoreGet(mcdbn));
-			if (mcdb.length() > 6) {
-				const char *p = mcdb.data();
-				const char *e = p + mcdb.length();
-				if (!memcmp("ZTMCD0",p,6)) {
-					p += 6;
-					while (p != e) {
-						CertificateOfMembership com;
-						com.deserialize2(p,e);
-						if (!com)
-							break;
-						_certInfo[com.issuedTo()].com = com;
-					}
-				}
-			}
-		} catch ( ... ) {} // ignore invalid MCDB, we'll re-learn from peers
 	}
 
 	if (!_portInitialized) {
@@ -115,32 +101,10 @@ Network::~Network()
 	char n[128];
 	if (_destroyed) {
 		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DESTROY,&ctmp);
-
 		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.conf",_id);
 		RR->node->dataStoreDelete(n);
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-		RR->node->dataStoreDelete(n);
 	} else {
 		RR->node->configureVirtualNetworkPort(_id,ZT_VIRTUAL_NETWORK_CONFIG_OPERATION_DOWN,&ctmp);
-
-		clean();
-
-		Utils::snprintf(n,sizeof(n),"networks.d/%.16llx.mcerts",_id);
-
-		Mutex::Lock _l(_lock);
-		if ((!_config)||(_config->isPublic())||(_certInfo.empty())) {
-			RR->node->dataStoreDelete(n);
-		} else {
-			std::string buf("ZTMCD0");
-			Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-			Address *a = (Address *)0;
-			_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-			while (i.next(a,ci)) {
-				if (ci->com)
-					ci->com.serialize2(buf);
-			}
-			RR->node->dataStorePut(n,buf,true);
-		}
 	}
 }
 
@@ -178,6 +142,20 @@ void Network::multicastUnsubscribe(const MulticastGroup &mg)
 		_myMulticastGroups.swap(nmg);
 }
 
+bool Network::tryAnnounceMulticastGroupsTo(const SharedPtr<Peer> &peer)
+{
+	Mutex::Lock _l(_lock);
+	if (
+	    (_isAllowed(peer)) ||
+	    (peer->address() == this->controller()) ||
+	    (RR->topology->isRoot(peer->identity()))
+	   ) {
+		_announceMulticastGroupsTo(peer->address(),_allMulticastGroups());
+		return true;
+	}
+	return false;
+}
+
 bool Network::applyConfiguration(const SharedPtr<NetworkConfig> &conf)
 {
 	if (_destroyed) // sanity check
@@ -281,70 +259,6 @@ void Network::requestConfiguration()
 	RR->sw->send(outp,true,0);
 }
 
-bool Network::validateAndAddMembershipCertificate(const CertificateOfMembership &cert)
-{
-	if (!cert) // sanity check
-		return false;
-
-	Mutex::Lock _l(_lock);
-
-	{
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(cert.issuedTo());
-		if ((ci)&&(ci->com == cert))
-			return true; // we already have it
-	}
-
-	// Check signature, log and return if cert is invalid
-	if (cert.signedBy() != controller()) {
-		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,cert.signedBy().toString().c_str());
-		return false; // invalid signer
-	}
-
-	if (cert.signedBy() == RR->identity.address()) {
-
-		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
-		// So, verify that we signed th cert ourself
-		if (!cert.verify(RR->identity)) {
-			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-
-	} else {
-
-		SharedPtr<Peer> signer(RR->topology->getPeer(cert.signedBy()));
-
-		if (!signer) {
-			// This would be rather odd, since this is our controller... could happen
-			// if we get packets before we've gotten config.
-			RR->sw->requestWhois(cert.signedBy());
-			return false; // signer unknown
-		}
-
-		if (!cert.verify(signer->identity())) {
-			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,cert.signedBy().toString().c_str());
-			return false; // invalid signature
-		}
-	}
-
-	// If we made it past authentication, add or update cert in our cert info store
-	_certInfo[cert.issuedTo()].com = cert;
-
-	return true;
-}
-
-bool Network::peerNeedsOurMembershipCertificate(const Address &to,uint64_t now)
-{
-	Mutex::Lock _l(_lock);
-	if ((_config)&&(!_config->isPublic())&&(_config->com())) {
-		_RemoteMemberCertificateInfo &ci = _certInfo[to];
-		if ((now - ci.lastPushed) > (ZT_NETWORK_AUTOCONF_DELAY / 2)) {
-			ci.lastPushed = now;
-			return true;
-		}
-	}
-	return false;
-}
-
 void Network::clean()
 {
 	const uint64_t now = RR->node->now();
@@ -353,22 +267,6 @@ void Network::clean()
 	if (_destroyed)
 		return;
 
-	if ((_config)&&(_config->isPublic())) {
-		// Open (public) networks do not track certs or cert pushes at all.
-		_certInfo.clear();
-	} else if (_config) {
-		// Clean obsolete entries from private network cert info table
-		Hashtable< Address,_RemoteMemberCertificateInfo >::Iterator i(_certInfo);
-		Address *a = (Address *)0;
-		_RemoteMemberCertificateInfo *ci = (_RemoteMemberCertificateInfo *)0;
-		const uint64_t forgetIfBefore = now - (ZT_PEER_ACTIVITY_TIMEOUT * 16); // arbitrary reasonable cutoff
-		while (i.next(a,ci)) {
-			if ((ci->lastPushed < forgetIfBefore)&&(!ci->com.agreesWith(_config->com())))
-				_certInfo.erase(*a);
-		}
-	}
-
-	// Clean learned multicast groups if we haven't heard from them in a while
 	{
 		Hashtable< MulticastGroup,uint64_t >::Iterator i(_multicastGroupsBehindMe);
 		MulticastGroup *mg = (MulticastGroup *)0;
@@ -494,7 +392,7 @@ void Network::_externalConfig(ZT_VirtualNetworkConfig *ec) const
 	} else ec->assignedAddressCount = 0;
 }
 
-bool Network::_isAllowed(const Address &peer) const
+bool Network::_isAllowed(const SharedPtr<Peer> &peer) const
 {
 	// Assumes _lock is locked
 	try {
@@ -502,82 +400,99 @@ bool Network::_isAllowed(const Address &peer) const
 			return false;
 		if (_config->isPublic())
 			return true;
-		const _RemoteMemberCertificateInfo *ci = _certInfo.get(peer);
-		if (!ci)
-			return false;
-		return _config->com().agreesWith(ci->com);
+		return ((_config->com())&&(peer->networkMembershipCertificatesAgree(_id,_config->com())));
 	} catch (std::exception &exc) {
-		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer.toString().c_str(),exc.what());
+		TRACE("isAllowed() check failed for peer %s: unexpected exception: %s",peer->address().toString().c_str(),exc.what());
 	} catch ( ... ) {
-		TRACE("isAllowed() check failed for peer %s: unexpected exception: unknown exception",peer.toString().c_str());
+		TRACE("isAllowed() check failed for peer %s: unexpected exception: unknown exception",peer->address().toString().c_str());
 	}
 	return false; // default position on any failure
 }
 
-std::vector<MulticastGroup> Network::_allMulticastGroups() const
-{
-	// Assumes _lock is locked
-	std::vector<MulticastGroup> mgs;
-	mgs.reserve(_myMulticastGroups.size() + _multicastGroupsBehindMe.size() + 1);
-	mgs.insert(mgs.end(),_myMulticastGroups.begin(),_myMulticastGroups.end());
-	_multicastGroupsBehindMe.appendKeys(mgs);
-	if ((_config)&&(_config->enableBroadcast()))
-		mgs.push_back(Network::BROADCAST);
-	std::sort(mgs.begin(),mgs.end());
-	mgs.erase(std::unique(mgs.begin(),mgs.end()),mgs.end());
-	return mgs;
-}
-
-// Used in Network::_announceMulticastGroups()
-class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths
+class _GetPeersThatNeedMulticastAnnouncement
 {
 public:
-	_AnnounceMulticastGroupsToPeersWithActiveDirectPaths(const RuntimeEnvironment *renv,Network *nw) :
-		RR(renv),
+	_GetPeersThatNeedMulticastAnnouncement(const RuntimeEnvironment *renv,Network *nw) :
 		_now(renv->node->now()),
+		_controller(nw->controller()),
 		_network(nw),
-		_rootAddresses(renv->topology->rootAddresses()),
-		_allMulticastGroups(nw->_allMulticastGroups())
+		_rootAddresses(renv->topology->rootAddresses())
 	{}
-
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		if ( ( (p->hasActiveDirectPath(_now)) && ( (_network->_isAllowed(p->address())) || (p->address() == _network->controller()) ) ) || (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end()) ) {
-			Packet outp(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
-
-			for(std::vector<MulticastGroup>::iterator mg(_allMulticastGroups.begin());mg!=_allMulticastGroups.end();++mg) {
-				if ((outp.size() + 18) >= ZT_UDP_DEFAULT_PAYLOAD_MTU) {
-					outp.armor(p->key(),true);
-					p->send(RR,outp.data(),outp.size(),_now);
-					outp.reset(p->address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
-				}
-
-				// network ID, MAC, ADI
-				outp.append((uint64_t)_network->id());
-				mg->mac().appendTo(outp);
-				outp.append((uint32_t)mg->adi());
-			}
-
-			if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) {
-				outp.armor(p->key(),true);
-				p->send(RR,outp.data(),outp.size(),_now);
-			}
+		if (
+		    (_network->_isAllowed(p)) ||
+		    (p->address() == _controller) ||
+		    (std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end())
+		   ) {
+			peers.push_back(p->address());
 		}
 	}
-
+	std::vector<Address> peers;
 private:
-	const RuntimeEnvironment *RR;
 	uint64_t _now;
+	Address _controller;
 	Network *_network;
 	std::vector<Address> _rootAddresses;
-	std::vector<MulticastGroup> _allMulticastGroups;
 };
-
 void Network::_announceMulticastGroups()
 {
 	// Assumes _lock is locked
-	_AnnounceMulticastGroupsToPeersWithActiveDirectPaths afunc(RR,this);
-	RR->topology->eachPeer<_AnnounceMulticastGroupsToPeersWithActiveDirectPaths &>(afunc);
+
+	_GetPeersThatNeedMulticastAnnouncement gpfunc(RR,this);
+	RR->topology->eachPeer<_GetPeersThatNeedMulticastAnnouncement &>(gpfunc);
+
+	std::vector<MulticastGroup> allMulticastGroups(_allMulticastGroups());
+	for(std::vector<Address>::const_iterator pa(gpfunc.peers.begin());pa!=gpfunc.peers.end();++pa)
+		_announceMulticastGroupsTo(*pa,allMulticastGroups);
+}
+
+void Network::_announceMulticastGroupsTo(const Address &peerAddress,const std::vector<MulticastGroup> &allMulticastGroups) const
+{
+	// Assumes _lock is locked
+
+	// We push COMs ahead of MULTICAST_LIKE since they're used for access control -- a COM is a public
+	// credential so "over-sharing" isn't really an issue (and we only do so with roots).
+	if ((_config)&&(_config->com())&&(!_config->isPublic())) {
+		Packet outp(peerAddress,RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
+		_config->com().serialize(outp);
+		RR->sw->send(outp,true,0);
+	}
+
+	{
+		Packet outp(peerAddress,RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
+
+		for(std::vector<MulticastGroup>::const_iterator mg(allMulticastGroups.begin());mg!=allMulticastGroups.end();++mg) {
+			if ((outp.size() + 18) >= ZT_UDP_DEFAULT_PAYLOAD_MTU) {
+				RR->sw->send(outp,true,0);
+				outp.reset(peerAddress,RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
+			}
+
+			// network ID, MAC, ADI
+			outp.append((uint64_t)_id);
+			mg->mac().appendTo(outp);
+			outp.append((uint32_t)mg->adi());
+		}
+
+		if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH)
+			RR->sw->send(outp,true,0);
+	}
+}
+
+std::vector<MulticastGroup> Network::_allMulticastGroups() const
+{
+	// Assumes _lock is locked
+
+	std::vector<MulticastGroup> mgs;
+	mgs.reserve(_myMulticastGroups.size() + _multicastGroupsBehindMe.size() + 1);
+	mgs.insert(mgs.end(),_myMulticastGroups.begin(),_myMulticastGroups.end());
+	_multicastGroupsBehindMe.appendKeys(mgs);
+	if ((_config)&&(_config->enableBroadcast()))
+		mgs.push_back(Network::BROADCAST);
+	std::sort(mgs.begin(),mgs.end());
+	mgs.erase(std::unique(mgs.begin(),mgs.end()),mgs.end());
+
+	return mgs;
 }
 
 } // namespace ZeroTier

node/Network.hpp

@@ -55,7 +55,8 @@
 namespace ZeroTier {
 
 class RuntimeEnvironment;
-class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
+class Peer;
+class _GetPeersThatNeedMulticastAnnouncement;
 
 /**
  * A virtual LAN
@@ -63,7 +64,7 @@ class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
 class Network : NonCopyable
 {
 	friend class SharedPtr<Network>;
-	friend class _AnnounceMulticastGroupsToPeersWithActiveDirectPaths;
+	friend class _GetPeersThatNeedMulticastAnnouncement; // internal function object
 
 public:
 	/**
@@ -92,7 +93,13 @@ public:
 	/**
 	 * @return Address of network's controller (most significant 40 bits of ID)
 	 */
-	inline Address controller() throw() { return Address(_id >> 24); }
+	inline Address controller() const throw() { return Address(_id >> 24); }
+
+	/**
+	 * @param nwid Network ID
+	 * @return Address of network's controller
+	 */
+	static inline Address controllerFor(uint64_t nwid) throw() { return Address(nwid >> 24); }
 
 	/**
 	 * @return Multicast group memberships for this network's port (local, not learned via bridging)
@@ -133,6 +140,14 @@ public:
 	 */
 	void multicastUnsubscribe(const MulticastGroup &mg);
 
+	/**
+	 * Announce multicast groups to a peer if that peer is authorized on this network
+	 *
+	 * @param peer Peer to try to announce multicast groups to
+	 * @return True if peer was authorized and groups were announced
+	 */
+	bool tryAnnounceMulticastGroupsTo(const SharedPtr<Peer> &peer);
+
 	/**
 	 * Apply a NetworkConfig to this network
 	 *
@@ -177,33 +192,10 @@ public:
 	void requestConfiguration();
 
 	/**
-	 * Add or update a membership certificate
-	 *
-	 * @param cert Certificate of membership
-	 * @return True if certificate was accepted as valid
-	 */
-	bool validateAndAddMembershipCertificate(const CertificateOfMembership &cert);
-
-	/**
-	 * Check if we should push membership certificate to a peer, AND update last pushed
-	 *
-	 * If we haven't pushed a cert to this peer in a long enough time, this returns
-	 * true and updates the last pushed time. Otherwise it returns false.
-	 *
-	 * This doesn't actually send anything, since COMs can hitch a ride with several
-	 * different kinds of packets.
-	 *
-	 * @param to Destination peer
-	 * @param now Current time
-	 * @return True if we should include a COM with whatever we're currently sending
-	 */
-	bool peerNeedsOurMembershipCertificate(const Address &to,uint64_t now);
-
-	/**
-	 * @param peer Peer address to check
+	 * @param peer Peer to check
 	 * @return True if peer is allowed to communicate on this network
 	 */
-	inline bool isAllowed(const Address &peer) const
+	inline bool isAllowed(const SharedPtr<Peer> &peer) const
 	{
 		Mutex::Lock _l(_lock);
 		return _isAllowed(peer);
@@ -347,17 +339,12 @@ public:
 	inline bool operator>=(const Network &n) const throw() { return (_id >= n._id); }
 
 private:
-	struct _RemoteMemberCertificateInfo
-	{
-		_RemoteMemberCertificateInfo() : com(),lastPushed(0) {}
-		CertificateOfMembership com; // remote member's COM
-		uint64_t lastPushed; // when did we last push ours to them?
-	};
-
 	ZT_VirtualNetworkStatus _status() const;
 	void _externalConfig(ZT_VirtualNetworkConfig *ec) const; // assumes _lock is locked
-	bool _isAllowed(const Address &peer) const;
+	bool _isAllowed(const SharedPtr<Peer> &peer) const;
+	bool _tryAnnounceMulticastGroupsTo(const std::vector<Address> &rootAddresses,const std::vector<MulticastGroup> &allMulticastGroups,const SharedPtr<Peer> &peer,uint64_t now) const;
 	void _announceMulticastGroups();
+	void _announceMulticastGroupsTo(const Address &peerAddress,const std::vector<MulticastGroup> &allMulticastGroups) const;
 	std::vector<MulticastGroup> _allMulticastGroups() const;
 
 	const RuntimeEnvironment *RR;
@@ -370,8 +357,6 @@ private:
 	Hashtable< MulticastGroup,uint64_t > _multicastGroupsBehindMe; // multicast groups that seem to be behind us and when we last saw them (if we are a bridge)
 	Hashtable< MAC,Address > _remoteBridgeRoutes; // remote addresses where given MACs are reachable (for tracking devices behind remote bridges)
 
-	Hashtable< Address,_RemoteMemberCertificateInfo > _certInfo;
-
 	SharedPtr<NetworkConfig> _config; // Most recent network configuration, which is an immutable value-object
 	volatile uint64_t _lastConfigUpdate;
 

node/NetworkConfig.cpp

@@ -55,6 +55,9 @@ SharedPtr<NetworkConfig> NetworkConfig::createTestNetworkConfig(const Address &s
 	if ((ip & 0x000000ff) == 0x00000000) ip ^= 0x00000001; // or .0
 	nc->_staticIps.push_back(InetAddress(Utils::hton(ip),8));
 
+	// Assign an RFC4193-compliant IPv6 address -- will never collide
+	nc->_staticIps.push_back(InetAddress::makeIpv6rfc4193(ZT_TEST_NETWORK_ID,self.toInt()));
+
 	return nc;
 }
 
@@ -87,27 +90,27 @@ void NetworkConfig::_fromDictionary(const Dictionary &d)
 
 	// NOTE: d.get(name) throws if not found, d.get(name,default) returns default
 
-	_nwid = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID).c_str());
+	_nwid = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_NETWORK_ID,"0").c_str());
 	if (!_nwid)
 		throw std::invalid_argument("configuration contains zero network ID");
 
-	_timestamp = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP).c_str());
+	_timestamp = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_TIMESTAMP,"0").c_str());
 	_revision = Utils::hexStrToU64(d.get(ZT_NETWORKCONFIG_DICT_KEY_REVISION,"1").c_str()); // older controllers don't send this, so default to 1
 
 	memset(_etWhitelist,0,sizeof(_etWhitelist));
-	std::vector<std::string> ets(Utils::split(d.get(ZT_NETWORKCONFIG_DICT_KEY_ALLOWED_ETHERNET_TYPES).c_str(),",","",""));
+	std::vector<std::string> ets(Utils::split(d.get(ZT_NETWORKCONFIG_DICT_KEY_ALLOWED_ETHERNET_TYPES,"").c_str(),",","",""));
 	for(std::vector<std::string>::const_iterator et(ets.begin());et!=ets.end();++et) {
 		unsigned int tmp = Utils::hexStrToUInt(et->c_str()) & 0xffff;
 		_etWhitelist[tmp >> 3] |= (1 << (tmp & 7));
 	}
 
-	_issuedTo = Address(d.get(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO));
+	_issuedTo = Address(d.get(ZT_NETWORKCONFIG_DICT_KEY_ISSUED_TO,"0"));
 	_multicastLimit = Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_MULTICAST_LIMIT,zero).c_str());
 	if (_multicastLimit == 0) _multicastLimit = ZT_MULTICAST_DEFAULT_LIMIT;
 	_allowPassiveBridging = (Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_ALLOW_PASSIVE_BRIDGING,zero).c_str()) != 0);
 	_private = (Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_PRIVATE,one).c_str()) != 0);
 	_enableBroadcast = (Utils::hexStrToUInt(d.get(ZT_NETWORKCONFIG_DICT_KEY_ENABLE_BROADCAST,one).c_str()) != 0);
-	_name = d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME);
+	_name = d.get(ZT_NETWORKCONFIG_DICT_KEY_NAME,"");
 	if (_name.length() > ZT_MAX_NETWORK_SHORT_NAME_LENGTH)
 		throw std::invalid_argument("network short name too long (max: 255 characters)");
 

node/Node.cpp

@@ -46,7 +46,7 @@
 #include "Address.hpp"
 #include "Identity.hpp"
 #include "SelfAwareness.hpp"
-#include "Defaults.hpp"
+#include "Cluster.hpp"
 
 const struct sockaddr_storage ZT_SOCKADDR_NULL = {0};
 
@@ -64,8 +64,7 @@ Node::Node(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology) :
+	ZT_EventCallback eventCallback) :
 	_RR(this),
 	RR(&_RR),
 	_uPtr(uptr),
@@ -82,18 +81,15 @@ Node::Node(
 	_lastPingCheck(0),
 	_lastHousekeepingRun(0)
 {
-	_newestVersionSeen[0] = ZEROTIER_ONE_VERSION_MAJOR;
-	_newestVersionSeen[1] = ZEROTIER_ONE_VERSION_MINOR;
-	_newestVersionSeen[2] = ZEROTIER_ONE_VERSION_REVISION;
 	_online = false;
 
 	// Use Salsa20 alone as a high-quality non-crypto PRNG
 	{
 		char foo[32];
 		Utils::getSecureRandom(foo,32);
-		_prng.init(foo,256,foo,8);
+		_prng.init(foo,256,foo);
 		memset(_prngStream,0,sizeof(_prngStream));
-		_prng.encrypt(_prngStream,_prngStream,sizeof(_prngStream));
+		_prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream));
 	}
 
 	std::string idtmp(dataStoreGet("identity.secret"));
@@ -128,21 +124,6 @@ Node::Node(
 		throw;
 	}
 
-	Dictionary rt;
-	if (overrideRootTopology) {
-		rt.fromString(std::string(overrideRootTopology));
-	} else {
-		std::string rttmp(dataStoreGet("root-topology"));
-		if (rttmp.length() > 0) {
-			rt.fromString(rttmp);
-			if (!Topology::authenticateRootTopology(rt))
-				rt.clear();
-		}
-		if ((!rt.size())||(!rt.contains("rootservers")))
-			rt.fromString(ZT_DEFAULTS.defaultRootTopology);
-	}
-	RR->topology->setRootServers(Dictionary(rt.get("rootservers","")));
-
 	postEvent(ZT_EVENT_UP);
 }
 
@@ -155,6 +136,9 @@ Node::~Node()
 	delete RR->antiRec;
 	delete RR->mc;
 	delete RR->sw;
+#ifdef ZT_ENABLE_CLUSTER
+	delete RR->cluster;
+#endif
 }
 
 ZT_ResultCode Node::processWirePacket(
@@ -197,29 +181,91 @@ public:
 		RR(renv),
 		_now(now),
 		_relays(relays),
-		_rootAddresses(RR->topology->rootAddresses())
+		_world(RR->topology->world())
 	{
 	}
 
-	uint64_t lastReceiveFromUpstream;
+	uint64_t lastReceiveFromUpstream; // tracks last time we got a packet from an 'upstream' peer like a root or a relay
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		bool isRelay = false;
-		for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(_relays.begin());r!=_relays.end();++r) {
-			if (r->first == p->address()) {
-				isRelay = true;
+		bool upstream = false;
+		InetAddress stableEndpoint4,stableEndpoint6;
+
+		// If this is a world root, pick (if possible) both an IPv4 and an IPv6 stable endpoint to use if link isn't currently alive.
+		for(std::vector<World::Root>::const_iterator r(_world.roots().begin());r!=_world.roots().end();++r) {
+			if (r->identity.address() == p->address()) {
+				upstream = true;
+				for(unsigned long k=0,ptr=RR->node->prng();k<r->stableEndpoints.size();++k) {
+					const InetAddress &addr = r->stableEndpoints[ptr++ % r->stableEndpoints.size()];
+					if (!stableEndpoint4) {
+						if (addr.ss_family == AF_INET)
+							stableEndpoint4 = addr;
+					}
+					if (!stableEndpoint6) {
+						if (addr.ss_family == AF_INET6)
+							stableEndpoint6 = addr;
+					}
+				}
 				break;
 			}
 		}
 
-		if ((isRelay)||(std::find(_rootAddresses.begin(),_rootAddresses.end(),p->address()) != _rootAddresses.end())) {
-			p->doPingAndKeepalive(RR,_now);
-			if (p->lastReceive() > lastReceiveFromUpstream)
-				lastReceiveFromUpstream = p->lastReceive();
-		} else {
-			if (p->alive(_now))
-				p->doPingAndKeepalive(RR,_now);
+		if (!upstream) {
+			// If I am a root server, only ping other root servers -- roots don't ping "down"
+			// since that would just be a waste of bandwidth and could potentially cause route
+			// flapping in Cluster mode.
+			if (RR->topology->amRoot())
+				return;
+
+			// Check for network preferred relays, also considered 'upstream' and thus always
+			// pinged to keep links up. If they have stable addresses we will try them there.
+			for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(_relays.begin());r!=_relays.end();++r) {
+				if (r->first == p->address()) {
+					if (r->second.ss_family == AF_INET)
+						stableEndpoint4 = r->second;
+					else if (r->second.ss_family == AF_INET6)
+						stableEndpoint6 = r->second;
+					upstream = true;
+					break;
+				}
+			}
+		}
+
+		if (upstream) {
+			// "Upstream" devices are roots and relays and get special treatment -- they stay alive
+			// forever and we try to keep (if available) both IPv4 and IPv6 channels open to them.
+			bool needToContactIndirect = true;
+			if (p->doPingAndKeepalive(RR,_now,AF_INET)) {
+				needToContactIndirect = false;
+			} else {
+				if (stableEndpoint4) {
+					needToContactIndirect = false;
+					p->attemptToContactAt(RR,InetAddress(),stableEndpoint4,_now);
+				}
+			}
+			if (p->doPingAndKeepalive(RR,_now,AF_INET6)) {
+				needToContactIndirect = false;
+			} else {
+				if (stableEndpoint6) {
+					needToContactIndirect = false;
+					p->attemptToContactAt(RR,InetAddress(),stableEndpoint6,_now);
+				}
+			}
+
+			if (needToContactIndirect) {
+				// If this is an upstream and we have no stable endpoint for either IPv4 or IPv6,
+				// send a NOP indirectly if possible to see if we can get to this peer in any
+				// way whatsoever. This will e.g. find network preferred relays that lack
+				// stable endpoints by using root servers.
+				Packet outp(p->address(),RR->identity.address(),Packet::VERB_NOP);
+				RR->sw->send(outp,true,0);
+			}
+
+			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
+		} else if (p->activelyTransferringFrames(_now)) {
+			// Normal nodes get their preferred link kept alive if the node has generated frame traffic recently
+			p->doPingAndKeepalive(RR,_now,0);
 		}
 	}
 
@@ -227,7 +273,7 @@ private:
 	const RuntimeEnvironment *RR;
 	uint64_t _now;
 	const std::vector< std::pair<Address,InetAddress> > &_relays;
-	std::vector<Address> _rootAddresses;
+	World _world;
 };
 
 ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextBackgroundTaskDeadline)
@@ -259,24 +305,13 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 			for(std::vector< SharedPtr<Network> >::const_iterator n(needConfig.begin());n!=needConfig.end();++n)
 				(*n)->requestConfiguration();
 
-			// Attempt to contact network preferred relays that we don't have direct links to
-			std::sort(networkRelays.begin(),networkRelays.end());
-			networkRelays.erase(std::unique(networkRelays.begin(),networkRelays.end()),networkRelays.end());
-			for(std::vector< std::pair<Address,InetAddress> >::const_iterator nr(networkRelays.begin());nr!=networkRelays.end();++nr) {
-				if (nr->second) {
-					SharedPtr<Peer> rp(RR->topology->getPeer(nr->first));
-					if ((rp)&&(!rp->hasActiveDirectPath(now)))
-						rp->attemptToContactAt(RR,InetAddress(),nr->second,now);
-				}
-			}
-
-			// Ping living or root server/relay peers
+			// Do pings and keepalives
 			_PingPeersThatNeedPing pfunc(RR,now,networkRelays);
 			RR->topology->eachPeer<_PingPeersThatNeedPing &>(pfunc);
 
 			// Update online status, post status change as event
-			bool oldOnline = _online;
-			_online = ((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT);
+			const bool oldOnline = _online;
+			_online = (((now - pfunc.lastReceiveFromUpstream) < ZT_PEER_ACTIVITY_TIMEOUT)||(RR->topology->amRoot()));
 			if (oldOnline != _online)
 				postEvent(_online ? ZT_EVENT_ONLINE : ZT_EVENT_OFFLINE);
 		} catch ( ... ) {
@@ -298,7 +333,18 @@ ZT_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *nextB
 	}
 
 	try {
-		*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
+#ifdef ZT_ENABLE_CLUSTER
+		// If clustering is enabled we have to call cluster->doPeriodicTasks() very often, so we override normal timer deadline behavior
+		if (RR->cluster) {
+			RR->sw->doTimerTasks(now);
+			RR->cluster->doPeriodicTasks();
+			*nextBackgroundTaskDeadline = now + ZT_CLUSTER_PERIODIC_TASK_PERIOD; // this is really short so just tick at this rate
+		} else {
+#endif
+			*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
+#ifdef ZT_ENABLE_CLUSTER
+		}
+#endif
 	} catch ( ... ) {
 		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
@@ -355,6 +401,8 @@ uint64_t Node::address() const
 void Node::status(ZT_NodeStatus *status) const
 {
 	status->address = RR->identity.address().toInt();
+	status->worldId = RR->topology->worldId();
+	status->worldTimestamp = RR->topology->worldTimestamp();
 	status->publicIdentity = RR->publicIdentityStr.c_str();
 	status->secretIdentity = RR->secretIdentityStr.c_str();
 	status->online = _online ? 1 : 0;
@@ -389,14 +437,13 @@ ZT_PeerList *Node::peers() const
 		p->latency = pi->second->latency();
 		p->role = RR->topology->isRoot(pi->second->identity()) ? ZT_PEER_ROLE_ROOT : ZT_PEER_ROLE_LEAF;
 
-		std::vector<RemotePath> paths(pi->second->paths());
-		RemotePath *bestPath = pi->second->getBestPath(_now);
+		std::vector<Path> paths(pi->second->paths());
+		Path *bestPath = pi->second->getBestPath(_now);
 		p->pathCount = 0;
-		for(std::vector<RemotePath>::iterator path(paths.begin());path!=paths.end();++path) {
+		for(std::vector<Path>::iterator path(paths.begin());path!=paths.end();++path) {
 			memcpy(&(p->paths[p->pathCount].address),&(path->address()),sizeof(struct sockaddr_storage));
 			p->paths[p->pathCount].lastSend = path->lastSend();
 			p->paths[p->pathCount].lastReceive = path->lastReceived();
-			p->paths[p->pathCount].fixed = path->fixed() ? 1 : 0;
 			p->paths[p->pathCount].active = path->active(_now) ? 1 : 0;
 			p->paths[p->pathCount].preferred = ((bestPath)&&(*path == *bestPath)) ? 1 : 0;
 			++p->pathCount;
@@ -441,11 +488,11 @@ void Node::freeQueryResult(void *qr)
 		::free(qr);
 }
 
-int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+int Node::addLocalInterfaceAddress(const struct sockaddr_storage *addr)
 {
 	if (Path::isAddressValidForPath(*(reinterpret_cast<const InetAddress *>(addr)))) {
 		Mutex::Lock _l(_directPaths_m);
-		_directPaths.push_back(Path(*(reinterpret_cast<const InetAddress *>(addr)),metric,(Path::Trust)trust));
+		_directPaths.push_back(*(reinterpret_cast<const InetAddress *>(addr)));
 		std::sort(_directPaths.begin(),_directPaths.end());
 		_directPaths.erase(std::unique(_directPaths.begin(),_directPaths.end()),_directPaths.end());
 		return 1;
@@ -464,6 +511,132 @@ void Node::setNetconfMaster(void *networkControllerInstance)
 	RR->localNetworkController = reinterpret_cast<NetworkController *>(networkControllerInstance);
 }
 
+ZT_ResultCode Node::circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
+{
+	if (test->hopCount > 0) {
+		try {
+			Packet outp(Address(),RR->identity.address(),Packet::VERB_CIRCUIT_TEST);
+			RR->identity.address().appendTo(outp);
+			outp.append((uint16_t)((test->reportAtEveryHop != 0) ? 0x03 : 0x02));
+			outp.append((uint64_t)test->timestamp);
+			outp.append((uint64_t)test->testId);
+			outp.append((uint16_t)0); // originator credential length, updated later
+			if (test->credentialNetworkId) {
+				outp.append((uint8_t)0x01);
+				outp.append((uint64_t)test->credentialNetworkId);
+				outp.setAt<uint16_t>(ZT_PACKET_IDX_PAYLOAD + 23,(uint16_t)9);
+			}
+			outp.append((uint16_t)0);
+			C25519::Signature sig(RR->identity.sign(reinterpret_cast<const char *>(outp.data()) + ZT_PACKET_IDX_PAYLOAD,outp.size() - ZT_PACKET_IDX_PAYLOAD));
+			outp.append((uint16_t)sig.size());
+			outp.append(sig.data,sig.size());
+			outp.append((uint16_t)0); // originator doesn't need an extra credential, since it's the originator
+			for(unsigned int h=1;h<test->hopCount;++h) {
+				outp.append((uint8_t)0);
+				outp.append((uint8_t)(test->hops[h].breadth & 0xff));
+				for(unsigned int a=0;a<test->hops[h].breadth;++a)
+					Address(test->hops[h].addresses[a]).appendTo(outp);
+			}
+
+			for(unsigned int a=0;a<test->hops[0].breadth;++a) {
+				outp.newInitializationVector();
+				outp.setDestination(Address(test->hops[0].addresses[a]));
+				RR->sw->send(outp,true,0);
+			}
+		} catch ( ... ) {
+			return ZT_RESULT_FATAL_ERROR_INTERNAL; // probably indicates FIFO too big for packet
+		}
+	}
+
+	{
+		test->_internalPtr = reinterpret_cast<void *>(reportCallback);
+		Mutex::Lock _l(_circuitTests_m);
+		if (std::find(_circuitTests.begin(),_circuitTests.end(),test) == _circuitTests.end())
+			_circuitTests.push_back(test);
+	}
+
+	return ZT_RESULT_OK;
+}
+
+void Node::circuitTestEnd(ZT_CircuitTest *test)
+{
+	Mutex::Lock _l(_circuitTests_m);
+	for(;;) {
+		std::vector< ZT_CircuitTest * >::iterator ct(std::find(_circuitTests.begin(),_circuitTests.end(),test));
+		if (ct == _circuitTests.end())
+			break;
+		else _circuitTests.erase(ct);
+	}
+}
+
+ZT_ResultCode Node::clusterInit(
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		return ZT_RESULT_ERROR_BAD_PARAMETER;
+
+	std::vector<InetAddress> eps;
+	for(unsigned int i=0;i<numZeroTierPhysicalEndpoints;++i)
+		eps.push_back(InetAddress(zeroTierPhysicalEndpoints[i]));
+	std::sort(eps.begin(),eps.end());
+	RR->cluster = new Cluster(RR,myId,eps,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg);
+
+	return ZT_RESULT_OK;
+#else
+	return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION;
+#endif
+}
+
+ZT_ResultCode Node::clusterAddMember(unsigned int memberId)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (!RR->cluster)
+		return ZT_RESULT_ERROR_BAD_PARAMETER;
+	RR->cluster->addMember((uint16_t)memberId);
+	return ZT_RESULT_OK;
+#else
+	return ZT_RESULT_ERROR_UNSUPPORTED_OPERATION;
+#endif
+}
+
+void Node::clusterRemoveMember(unsigned int memberId)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->removeMember((uint16_t)memberId);
+#endif
+}
+
+void Node::clusterHandleIncomingMessage(const void *msg,unsigned int len)
+{
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->handleIncomingStateMessage(msg,len);
+#endif
+}
+
+void Node::clusterStatus(ZT_ClusterStatus *cs)
+{
+	if (!cs)
+		return;
+#ifdef ZT_ENABLE_CLUSTER
+	if (RR->cluster)
+		RR->cluster->status(*cs);
+	else
+#endif
+	memset(cs,0,sizeof(ZT_ClusterStatus));
+}
+
 /****************************************************************************/
 /* Node methods used only within node/                                      */
 /****************************************************************************/
@@ -482,16 +655,6 @@ std::string Node::dataStoreGet(const char *name)
 	return r;
 }
 
-void Node::postNewerVersionIfNewer(unsigned int major,unsigned int minor,unsigned int rev)
-{
-	if (Utils::compareVersion(major,minor,rev,_newestVersionSeen[0],_newestVersionSeen[1],_newestVersionSeen[2]) > 0) {
-		_newestVersionSeen[0] = major;
-		_newestVersionSeen[1] = minor;
-		_newestVersionSeen[2] = rev;
-		this->postEvent(ZT_EVENT_SAW_MORE_RECENT_VERSION,(const void *)_newestVersionSeen);
-	}
-}
-
 #ifdef ZT_TRACE
 void Node::postTrace(const char *module,unsigned int line,const char *fmt,...)
 {
@@ -529,10 +692,24 @@ uint64_t Node::prng()
 {
 	unsigned int p = (++_prngStreamPtr % (sizeof(_prngStream) / sizeof(uint64_t)));
 	if (!p)
-		_prng.encrypt(_prngStream,_prngStream,sizeof(_prngStream));
+		_prng.encrypt12(_prngStream,_prngStream,sizeof(_prngStream));
 	return _prngStream[p];
 }
 
+void Node::postCircuitTestReport(const ZT_CircuitTestReport *report)
+{
+	std::vector< ZT_CircuitTest * > toNotify;
+	{
+		Mutex::Lock _l(_circuitTests_m);
+		for(std::vector< ZT_CircuitTest * >::iterator i(_circuitTests.begin());i!=_circuitTests.end();++i) {
+			if ((*i)->testId == report->testId)
+				toNotify.push_back(*i);
+		}
+	}
+	for(std::vector< ZT_CircuitTest * >::iterator i(toNotify.begin());i!=toNotify.end();++i)
+		(reinterpret_cast<void (*)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *)>((*i)->_internalPtr))(reinterpret_cast<ZT_Node *>(this),*i,report);
+}
+
 } // namespace ZeroTier
 
 /****************************************************************************/
@@ -550,12 +727,11 @@ enum ZT_ResultCode ZT_Node_new(
 	ZT_WirePacketSendFunction wirePacketSendFunction,
 	ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-	ZT_EventCallback eventCallback,
-	const char *overrideRootTopology)
+	ZT_EventCallback eventCallback)
 {
 	*node = (ZT_Node *)0;
 	try {
-		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback,overrideRootTopology));
+		*node = reinterpret_cast<ZT_Node *>(new ZeroTier::Node(now,uptr,dataStoreGetFunction,dataStorePutFunction,wirePacketSendFunction,virtualNetworkFrameFunction,virtualNetworkConfigFunction,eventCallback));
 		return ZT_RESULT_OK;
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
@@ -587,8 +763,7 @@ enum ZT_ResultCode ZT_Node_processWirePacket(
 	} catch (std::bad_alloc &exc) {
 		return ZT_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {
-		reinterpret_cast<ZeroTier::Node *>(node)->postEvent(ZT_EVENT_INVALID_PACKET,(const void *)remoteAddress);
-		return ZT_RESULT_OK;
+		return ZT_RESULT_OK; // "OK" since invalid packets are simply dropped, but the system is still up
 	}
 }
 
@@ -714,6 +889,22 @@ void ZT_Node_freeQueryResult(ZT_Node *node,void *qr)
 	} catch ( ... ) {}
 }
 
+int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr);
+	} catch ( ... ) {
+		return 0;
+	}
+}
+
+void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
+	} catch ( ... ) {}
+}
+
 void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 {
 	try {
@@ -721,19 +912,69 @@ void ZT_Node_setNetconfMaster(ZT_Node *node,void *networkControllerInstance)
 	} catch ( ... ) {}
 }
 
-int ZT_Node_addLocalInterfaceAddress(ZT_Node *node,const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust)
+enum ZT_ResultCode ZT_Node_circuitTestBegin(ZT_Node *node,ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *))
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->addLocalInterfaceAddress(addr,metric,trust);
+		return reinterpret_cast<ZeroTier::Node *>(node)->circuitTestBegin(test,reportCallback);
 	} catch ( ... ) {
-		return 0;
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
 	}
 }
 
-void ZT_Node_clearLocalInterfaceAddresses(ZT_Node *node)
+void ZT_Node_circuitTestEnd(ZT_Node *node,ZT_CircuitTest *test)
 {
 	try {
-		reinterpret_cast<ZeroTier::Node *>(node)->clearLocalInterfaceAddresses();
+		reinterpret_cast<ZeroTier::Node *>(node)->circuitTestEnd(test);
+	} catch ( ... ) {}
+}
+
+enum ZT_ResultCode ZT_Node_clusterInit(
+	ZT_Node *node,
+	unsigned int myId,
+	const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+	unsigned int numZeroTierPhysicalEndpoints,
+	int x,
+	int y,
+	int z,
+	void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+	void *sendFunctionArg,
+	int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+	void *addressToLocationFunctionArg)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->clusterInit(myId,zeroTierPhysicalEndpoints,numZeroTierPhysicalEndpoints,x,y,z,sendFunction,sendFunctionArg,addressToLocationFunction,addressToLocationFunctionArg);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
+enum ZT_ResultCode ZT_Node_clusterAddMember(ZT_Node *node,unsigned int memberId)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->clusterAddMember(memberId);
+	} catch ( ... ) {
+		return ZT_RESULT_FATAL_ERROR_INTERNAL;
+	}
+}
+
+void ZT_Node_clusterRemoveMember(ZT_Node *node,unsigned int memberId)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterRemoveMember(memberId);
+	} catch ( ... ) {}
+}
+
+void ZT_Node_clusterHandleIncomingMessage(ZT_Node *node,const void *msg,unsigned int len)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterHandleIncomingMessage(msg,len);
+	} catch ( ... ) {}
+}
+
+void ZT_Node_clusterStatus(ZT_Node *node,ZT_ClusterStatus *cs)
+{
+	try {
+		reinterpret_cast<ZeroTier::Node *>(node)->clusterStatus(cs);
 	} catch ( ... ) {}
 }
 

node/Node.hpp

@@ -71,8 +71,7 @@ public:
 		ZT_WirePacketSendFunction wirePacketSendFunction,
 		ZT_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 		ZT_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
-		ZT_EventCallback eventCallback,
-		const char *overrideRootTopology);
+		ZT_EventCallback eventCallback);
 
 	~Node();
 
@@ -106,9 +105,26 @@ public:
 	ZT_VirtualNetworkConfig *networkConfig(uint64_t nwid) const;
 	ZT_VirtualNetworkList *networks() const;
 	void freeQueryResult(void *qr);
-	int addLocalInterfaceAddress(const struct sockaddr_storage *addr,int metric,ZT_LocalInterfaceAddressTrust trust);
+	int addLocalInterfaceAddress(const struct sockaddr_storage *addr);
 	void clearLocalInterfaceAddresses();
 	void setNetconfMaster(void *networkControllerInstance);
+	ZT_ResultCode circuitTestBegin(ZT_CircuitTest *test,void (*reportCallback)(ZT_Node *,ZT_CircuitTest *,const ZT_CircuitTestReport *));
+	void circuitTestEnd(ZT_CircuitTest *test);
+	ZT_ResultCode clusterInit(
+		unsigned int myId,
+		const struct sockaddr_storage *zeroTierPhysicalEndpoints,
+		unsigned int numZeroTierPhysicalEndpoints,
+		int x,
+		int y,
+		int z,
+		void (*sendFunction)(void *,unsigned int,const void *,unsigned int),
+		void *sendFunctionArg,
+		int (*addressToLocationFunction)(void *,const struct sockaddr_storage *,int *,int *,int *),
+		void *addressToLocationFunctionArg);
+	ZT_ResultCode clusterAddMember(unsigned int memberId);
+	void clusterRemoveMember(unsigned int memberId);
+	void clusterHandleIncomingMessage(const void *msg,unsigned int len);
+	void clusterStatus(ZT_ClusterStatus *cs);
 
 	// Internal functions ------------------------------------------------------
 
@@ -168,6 +184,16 @@ public:
 		return _network(nwid);
 	}
 
+	inline bool belongsToNetwork(uint64_t nwid) const
+	{
+		Mutex::Lock _l(_networks_m);
+		for(std::vector< std::pair< uint64_t, SharedPtr<Network> > >::const_iterator i=_networks.begin();i!=_networks.end();++i) {
+			if (i->first == nwid)
+				return true;
+		}
+		return false;
+	}
+
 	inline std::vector< SharedPtr<Network> > allNetworks() const
 	{
 		std::vector< SharedPtr<Network> > nw;
@@ -181,7 +207,7 @@ public:
 	/**
 	 * @return Potential direct paths to me a.k.a. local interface addresses
 	 */
-	inline std::vector<Path> directPaths() const
+	inline std::vector<InetAddress> directPaths() const
 	{
 		Mutex::Lock _l(_directPaths_m);
 		return _directPaths;
@@ -214,11 +240,6 @@ public:
 	 */
 	inline bool online() const throw() { return _online; }
 
-	/**
-	 * If this version is newer than the newest we've seen, post a new version seen event
-	 */
-	void postNewerVersionIfNewer(unsigned int major,unsigned int minor,unsigned int rev);
-
 #ifdef ZT_TRACE
 	void postTrace(const char *module,unsigned int line,const char *fmt,...);
 #endif
@@ -228,6 +249,13 @@ public:
 	 */
 	uint64_t prng();
 
+	/**
+	 * Post a circuit test report to any listeners for a given test ID
+	 *
+	 * @param report Report (includes test ID)
+	 */
+	void postCircuitTestReport(const ZT_CircuitTestReport *report);
+
 private:
 	inline SharedPtr<Network> _network(uint64_t nwid) const
 	{
@@ -254,7 +282,10 @@ private:
 	std::vector< std::pair< uint64_t, SharedPtr<Network> > > _networks;
 	Mutex _networks_m;
 
-	std::vector<Path> _directPaths;
+	std::vector< ZT_CircuitTest * > _circuitTests;
+	Mutex _circuitTests_m;
+
+	std::vector<InetAddress> _directPaths;
 	Mutex _directPaths_m;
 
 	Mutex _backgroundTasksLock;
@@ -266,7 +297,6 @@ private:
 	uint64_t _now;
 	uint64_t _lastPingCheck;
 	uint64_t _lastHousekeepingRun;
-	unsigned int _newestVersionSeen[3]; // major, minor, revision
 	bool _online;
 };
 

node/OutboundMulticast.cpp

@@ -103,11 +103,11 @@ void OutboundMulticast::init(
 void OutboundMulticast::sendOnly(const RuntimeEnvironment *RR,const Address &toAddr)
 {
 	if (_haveCom) {
-		SharedPtr<Network> network(RR->node->network(_nwid));
-		if ((network)&&(network->peerNeedsOurMembershipCertificate(toAddr,RR->node->now()))) {
+		SharedPtr<Peer> peer(RR->topology->getPeer(toAddr));
+		if ( (!peer) || (peer->needsOurNetworkMembershipCertificate(_nwid,RR->node->now(),true)) ) {
+			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			_packetWithCom.newInitializationVector();
 			_packetWithCom.setDestination(toAddr);
-			//TRACE(">>MC %.16llx -> %s (with COM)",(unsigned long long)this,toAddr.toString().c_str());
 			RR->sw->send(_packetWithCom,true,_nwid);
 			return;
 		}

node/Packet.cpp

@@ -31,6 +31,8 @@ namespace ZeroTier {
 
 const unsigned char Packet::ZERO_KEY[32] = { 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0 };
 
+#ifdef ZT_TRACE
+
 const char *Packet::verbString(Verb v)
 	throw()
 {
@@ -43,7 +45,7 @@ const char *Packet::verbString(Verb v)
 		case VERB_RENDEZVOUS: return "RENDEZVOUS";
 		case VERB_FRAME: return "FRAME";
 		case VERB_EXT_FRAME: return "EXT_FRAME";
-		case VERB_P5_MULTICAST_FRAME: return "P5_MULTICAST_FRAME";
+		case VERB_ECHO: return "ECHO";
 		case VERB_MULTICAST_LIKE: return "MULTICAST_LIKE";
 		case VERB_NETWORK_MEMBERSHIP_CERTIFICATE: return "NETWORK_MEMBERSHIP_CERTIFICATE";
 		case VERB_NETWORK_CONFIG_REQUEST: return "NETWORK_CONFIG_REQUEST";
@@ -52,6 +54,9 @@ const char *Packet::verbString(Verb v)
 		case VERB_MULTICAST_FRAME: return "MULTICAST_FRAME";
 		case VERB_SET_EPHEMERAL_KEY: return "SET_EPHEMERAL_KEY";
 		case VERB_PUSH_DIRECT_PATHS: return "PUSH_DIRECT_PATHS";
+		case VERB_CIRCUIT_TEST: return "CIRCUIT_TEST";
+		case VERB_CIRCUIT_TEST_REPORT: return "CIRCUIT_TEST_REPORT";
+		case VERB_REQUEST_PROOF_OF_WORK: return "REQUEST_PROOF_OF_WORK";
 	}
 	return "(unknown)";
 }
@@ -73,6 +78,8 @@ const char *Packet::errorString(ErrorCode e)
 	return "(unknown)";
 }
 
+#endif // ZT_TRACE
+
 void Packet::armor(const void *key,bool encryptPayload)
 {
 	unsigned char mangledKey[32];
@@ -85,14 +92,14 @@ void Packet::armor(const void *key,bool encryptPayload)
 	setCipher(encryptPayload ? ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012 : ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE);
 
 	_salsa20MangleKey((const unsigned char *)key,mangledKey);
-	Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8),ZT_PROTO_SALSA20_ROUNDS);
+	Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)/*,ZT_PROTO_SALSA20_ROUNDS*/);
 
 	// MAC key is always the first 32 bytes of the Salsa20 key stream
 	// This is the same construction DJB's NaCl library uses
-	s20.encrypt(ZERO_KEY,macKey,sizeof(macKey));
+	s20.encrypt12(ZERO_KEY,macKey,sizeof(macKey));
 
 	if (encryptPayload)
-		s20.encrypt(payload,payload,payloadLen);
+		s20.encrypt12(payload,payload,payloadLen);
 
 	Poly1305::compute(mac,payload,payloadLen,macKey);
 	memcpy(field(ZT_PACKET_IDX_MAC,8),mac,8);
@@ -109,15 +116,15 @@ bool Packet::dearmor(const void *key)
 
 	if ((cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_NONE)||(cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)) {
 		_salsa20MangleKey((const unsigned char *)key,mangledKey);
-		Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8),ZT_PROTO_SALSA20_ROUNDS);
+		Salsa20 s20(mangledKey,256,field(ZT_PACKET_IDX_IV,8)/*,ZT_PROTO_SALSA20_ROUNDS*/);
 
-		s20.encrypt(ZERO_KEY,macKey,sizeof(macKey));
+		s20.encrypt12(ZERO_KEY,macKey,sizeof(macKey));
 		Poly1305::compute(mac,payload,payloadLen,macKey);
 		if (!Utils::secureEq(mac,field(ZT_PACKET_IDX_MAC,8),8))
 			return false;
 
 		if (cs == ZT_PROTO_CIPHER_SUITE__C25519_POLY1305_SALSA2012)
-			s20.decrypt(payload,payload,payloadLen);
+			s20.decrypt12(payload,payload,payloadLen);
 
 		return true;
 	} else return false; // unrecognized cipher suite

node/Packet.hpp

@@ -46,22 +46,24 @@
 #include "../ext/lz4/lz4.h"
 
 /**
- * Protocol version -- incremented only for MAJOR changes
+ * Protocol version -- incremented only for major changes
  *
  * 1 - 0.2.0 ... 0.2.5
  * 2 - 0.3.0 ... 0.4.5
- *   * Added signature and originating peer to multicast frame
- *   * Double size of multicast frame bloom filter
+ *   + Added signature and originating peer to multicast frame
+ *   + Double size of multicast frame bloom filter
  * 3 - 0.5.0 ... 0.6.0
- *   * Yet another multicast redesign
- *   * New crypto completely changes key agreement cipher
- * 4 - 0.6.0 ... CURRENT
- *   * New identity format based on hashcash design
- *
- * This isn't going to change again for a long time unless your
- * author wakes up again at 4am with another great idea. :P
+ *   + Yet another multicast redesign
+ *   + New crypto completely changes key agreement cipher
+ * 4 - 0.6.0 ... 1.0.6
+ *   + New identity format based on hashcash design
+ * 5 - 1.1.0 ... CURRENT
+ *   + Supports circuit test, proof of work, and echo
+ *   + Supports in-band world (root server definition) updates
+ *   + Clustering! (Though this will work with protocol v4 clients.)
+ *   + Otherwise backward compatible with protocol v4
  */
-#define ZT_PROTO_VERSION 4
+#define ZT_PROTO_VERSION 5
 
 /**
  * Minimum supported protocol version
@@ -233,15 +235,6 @@
  */
 #define ZT_PROTO_MIN_FRAGMENT_LENGTH ZT_PACKET_FRAGMENT_IDX_PAYLOAD
 
-// Destination address types from HELLO, OK(HELLO), and other message types
-#define ZT_PROTO_DEST_ADDRESS_TYPE_NONE 0
-#define ZT_PROTO_DEST_ADDRESS_TYPE_ZEROTIER 1   // reserved but unused
-#define ZT_PROTO_DEST_ADDRESS_TYPE_ETHERNET 2   // future use
-#define ZT_PROTO_DEST_ADDRESS_TYPE_BLUETOOTH 3  // future use
-#define ZT_PROTO_DEST_ADDRESS_TYPE_IPV4 4
-#define ZT_PROTO_DEST_ADDRESS_TYPE_LTE_DIRECT 5 // future use
-#define ZT_PROTO_DEST_ADDRESS_TYPE_IPV6 6
-
 // Ephemeral key record flags
 #define ZT_PROTO_EPHEMERAL_KEY_FLAG_FIPS 0x01   // future use
 
@@ -354,11 +347,11 @@ namespace ZeroTier {
  * ZeroTier packet
  *
  * Packet format:
- *   <[8] random initialization vector (doubles as 64-bit packet ID)>
+ *   <[8] 64-bit random packet ID and crypto initialization vector>
  *   <[5] destination ZT address>
  *   <[5] source ZT address>
  *   <[1] flags/cipher (top 5 bits) and ZT hop count (last 3 bits)>
- *   <[8] 8-bit MAC (currently first 8 bytes of poly1305 tag)>
+ *   <[8] 64-bit MAC>
  *   [... -- begin encryption envelope -- ...]
  *   <[1] encrypted flags (top 3 bits) and verb (last 5 bits)>
  *   [... verb-specific payload ...]
@@ -374,6 +367,10 @@ namespace ZeroTier {
  * immutable. This is because intermediate nodes can increment the hop
  * count up to 7 (protocol max).
  *
+ * A hop count of 7 also indicates that receiving peers should not attempt
+ * to learn direct paths from this packet. (Right now direct paths are only
+ * learned from direct packets anyway.)
+ *
  * http://tonyarcieri.com/all-the-crypto-code-youve-ever-written-is-probably-broken
  *
  * For unencrypted packets, MAC is computed on plaintext. Only HELLO is ever
@@ -530,10 +527,13 @@ public:
 	 */
 	enum Verb /* Max value: 32 (5 bits) */
 	{
-		/* No operation, payload ignored, no reply */
+		/**
+		 * No operation (ignored, no reply)
+		 */
 		VERB_NOP = 0,
 
-		/* Announcement of a node's existence:
+		/**
+		 * Announcement of a node's existence:
 		 *   <[1] protocol version>
 		 *   <[1] software major version>
 		 *   <[1] software minor version>
@@ -542,6 +542,8 @@ public:
 		 *   <[...] binary serialized identity (see Identity)>
 		 *   <[1] destination address type>
 		 *   [<[...] destination address>]
+		 *   <[8] 64-bit world ID of current world>
+		 *   <[8] 64-bit timestamp of current world>
 		 *
 		 * This is the only message that ever must be sent in the clear, since it
 		 * is used to push an identity to a new peer.
@@ -553,10 +555,10 @@ public:
 		 * address that require re-establishing connectivity.
 		 *
 		 * Destination address types and formats (not all of these are used now):
-		 *   0 - None -- no destination address data present
-		 *   1 - Ethernet address -- format: <[6] Ethernet MAC>
-		 *   4 - 6-byte IPv4 UDP address/port -- format: <[4] IP>, <[2] port>
-		 *   6 - 18-byte IPv6 UDP address/port -- format: <[16] IP>, <[2] port>
+		 *   0x00 - None -- no destination address data present
+		 *   0x01 - Ethernet address -- format: <[6] Ethernet MAC>
+		 *   0x04 - 6-byte IPv4 UDP address/port -- format: <[4] IP>, <[2] port>
+		 *   0x06 - 18-byte IPv6 UDP address/port -- format: <[16] IP>, <[2] port>
 		 *
 		 * OK payload:
 		 *   <[8] timestamp (echoed from original HELLO)>
@@ -566,12 +568,15 @@ public:
 		 *   <[2] software revision (of responder)>
 		 *   <[1] destination address type (for this OK, not copied from HELLO)>
 		 *   [<[...] destination address>]
+		 *   <[2] 16-bit length of world update or 0 if none>
+		 *   [[...] world update]
 		 *
 		 * ERROR has no payload.
 		 */
 		VERB_HELLO = 1,
 
-		/* Error response:
+		/**
+		 * Error response:
 		 *   <[1] in-re verb>
 		 *   <[8] in-re packet ID>
 		 *   <[1] error code>
@@ -579,14 +584,16 @@ public:
 		 */
 		VERB_ERROR = 2,
 
-		/* Success response:
+		/**
+		 * Success response:
 		 *   <[1] in-re verb>
 		 *   <[8] in-re packet ID>
 		 *   <[...] request-specific payload>
 		 */
 		VERB_OK = 3,
 
-		/* Query an identity by address:
+		/**
+		 * Query an identity by address:
 		 *   <[5] address to look up>
 		 *
 		 * OK response payload:
@@ -597,7 +604,8 @@ public:
 		 */
 		VERB_WHOIS = 4,
 
-		/* Meet another node at a given protocol address:
+		/**
+		 * Meet another node at a given protocol address:
 		 *   <[1] flags (unused, currently 0)>
 		 *   <[5] ZeroTier address of peer that might be found at this address>
 		 *   <[2] 16-bit protocol address port>
@@ -616,11 +624,16 @@ public:
 		 * may also ignore these messages if a peer is not known or is not being
 		 * actively communicated with.
 		 *
+		 * Unfortunately the physical address format in this message pre-dates
+		 * InetAddress's serialization format. :( ZeroTier is four years old and
+		 * yes we've accumulated a tiny bit of cruft here and there.
+		 *
 		 * No OK or ERROR is generated.
 		 */
 		VERB_RENDEZVOUS = 5,
 
-		/* ZT-to-ZT unicast ethernet frame (shortened EXT_FRAME):
+		/**
+		 * ZT-to-ZT unicast ethernet frame (shortened EXT_FRAME):
 		 *   <[8] 64-bit network ID>
 		 *   <[2] 16-bit ethertype>
 		 *   <[...] ethernet payload>
@@ -635,7 +648,8 @@ public:
 		 */
 		VERB_FRAME = 6,
 
-		/* Full Ethernet frame with MAC addressing and optional fields:
+		/**
+		 * Full Ethernet frame with MAC addressing and optional fields:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
 		 *  [<[...] certificate of network membership>]
@@ -658,23 +672,44 @@ public:
 		 */
 		VERB_EXT_FRAME = 7,
 
-		/* DEPRECATED */
-		VERB_P5_MULTICAST_FRAME = 8,
+		/**
+		 * ECHO request (a.k.a. ping):
+		 *   <[...] arbitrary payload to be echoed back>
+		 *
+		 * This generates OK with a copy of the transmitted payload. No ERROR
+		 * is generated. Response to ECHO requests is optional.
+		 *
+		 * Support for fragmented echo packets is optional and their use is not
+		 * recommended.
+		 */
+		VERB_ECHO = 8,
 
-		/* Announce interest in multicast group(s):
+		/**
+		 * Announce interest in multicast group(s):
 		 *   <[8] 64-bit network ID>
 		 *   <[6] multicast Ethernet address>
 		 *   <[4] multicast additional distinguishing information (ADI)>
 		 *   [... additional tuples of network/address/adi ...]
 		 *
-		 * LIKEs are sent to peers with whom you have a direct peer to peer
-		 * connection, and always including root servers.
+		 * LIKEs may be sent to any peer, though a good implementation should
+		 * restrict them to peers on the same network they're for and to network
+		 * controllers and root servers. In the current network, root servers
+		 * will provide the service of final multicast cache.
+		 *
+		 * It is recommended that NETWORK_MEMBERSHIP_CERTIFICATE pushes be sent
+		 * along with MULTICAST_LIKE when pushing LIKEs to peers that do not
+		 * share a network membership (such as root servers), since this can be
+		 * used to authenticate GATHER requests and limit responses to peers
+		 * authorized to talk on a network. (Should be an optional field here,
+		 * but saving one or two packets every five minutes is not worth an
+		 * ugly hack or protocol rev.)
 		 *
 		 * OK/ERROR are not generated.
 		 */
 		VERB_MULTICAST_LIKE = 9,
 
-		/* Network member certificate replication/push:
+		/**
+		 * Network member certificate replication/push:
 		 *   <[...] serialized certificate of membership>
 		 *   [ ... additional certificates may follow ...]
 		 *
@@ -685,7 +720,8 @@ public:
 		 */
 		VERB_NETWORK_MEMBERSHIP_CERTIFICATE = 10,
 
-		/* Network configuration request:
+		/**
+		 * Network configuration request:
 		 *   <[8] 64-bit network ID>
 		 *   <[2] 16-bit length of request meta-data dictionary>
 		 *   <[...] string-serialized request meta-data>
@@ -720,7 +756,8 @@ public:
 		 */
 		VERB_NETWORK_CONFIG_REQUEST = 11,
 
-		/* Network configuration refresh request:
+		/**
+		 * Network configuration refresh request:
 		 *   <[...] array of 64-bit network IDs>
 		 *
 		 * This can be sent by the network controller to inform a node that it
@@ -731,7 +768,8 @@ public:
 		 */
 		VERB_NETWORK_CONFIG_REFRESH = 12,
 
-		/* Request endpoints for multicast distribution:
+		/**
+		 * Request endpoints for multicast distribution:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
 		 *   <[6] MAC address of multicast group being queried>
@@ -769,7 +807,8 @@ public:
 		 */
 		VERB_MULTICAST_GATHER = 13,
 
-		/* Multicast frame:
+		/**
+		 * Multicast frame:
 		 *   <[8] 64-bit network ID>
 		 *   <[1] flags>
 		 *  [<[...] network certificate of membership>]
@@ -810,7 +849,8 @@ public:
 		 */
 		VERB_MULTICAST_FRAME = 14,
 
-		/* Ephemeral (PFS) key push: (UNFINISHED, NOT IMPLEMENTED YET)
+		/**
+		 * Ephemeral (PFS) key push: (UNFINISHED, NOT IMPLEMENTED YET)
 		 *   <[2] flags (unused and reserved, must be 0)>
 		 *   <[2] length of padding / extra field section>
 		 *   <[...] padding / extra field section>
@@ -866,7 +906,8 @@ public:
 		 */
 		VERB_SET_EPHEMERAL_KEY = 15,
 
-		/* Push of potential endpoints for direct communication:
+		/**
+		 * Push of potential endpoints for direct communication:
 		 *   <[2] 16-bit number of paths>
 		 *   <[...] paths>
 		 *
@@ -880,13 +921,10 @@ public:
 		 *
 		 * Path record flags:
 		 *   0x01 - Forget this path if it is currently known
-		 *   0x02 - Blacklist this path, do not use
+		 *   0x02 - (Unused)
 		 *   0x04 - Disable encryption (trust: privacy)
 		 *   0x08 - Disable encryption and authentication (trust: ultimate)
 		 *
-		 * Address types and addresses are of the same format as the destination
-		 * address type and address in HELLO.
-		 *
 		 * The receiver may, upon receiving a push, attempt to establish a
 		 * direct link to one or more of the indicated addresses. It is the
 		 * responsibility of the sender to limit which peers it pushes direct
@@ -904,7 +942,166 @@ public:
 		 *
 		 * OK and ERROR are not generated.
 		 */
-		VERB_PUSH_DIRECT_PATHS = 16
+		VERB_PUSH_DIRECT_PATHS = 16,
+
+		/**
+		 * Source-routed circuit test message:
+		 *   <[5] address of originator of circuit test>
+		 *   <[2] 16-bit flags>
+		 *   <[8] 64-bit timestamp>
+		 *   <[8] 64-bit test ID (arbitrary, set by tester)>
+		 *   <[2] 16-bit originator credential length (includes type)>
+		 *   [[1] originator credential type (for authorizing test)]
+		 *   [[...] originator credential]
+		 *   <[2] 16-bit length of additional fields>
+		 *   [[...] additional fields]
+		 *   [ ... end of signed portion of request ... ]
+		 *   <[2] 16-bit length of signature of request>
+		 *   <[...] signature of request by originator>
+		 *   <[2] 16-bit previous hop credential length (including type)>
+		 *   [[1] previous hop credential type]
+		 *   [[...] previous hop credential]
+		 *   <[...] next hop(s) in path>
+		 *
+		 * Flags:
+		 *   0x01 - Report back to originator at middle hops
+		 *   0x02 - Report back to originator at last hop
+		 *
+		 * Originator credential types:
+		 *   0x01 - 64-bit network ID for which originator is controller
+		 *
+		 * Previous hop credential types:
+		 *   0x01 - Certificate of network membership
+		 *
+		 * Path record format:
+		 *   <[1] 8-bit flags (unused, must be zero)>
+		 *   <[1] 8-bit breadth (number of next hops)>
+		 *   <[...] one or more ZeroTier addresses of next hops>
+		 *
+		 * The circuit test allows a device to send a message that will traverse
+		 * the network along a specified path, with each hop optionally reporting
+		 * back to the tester via VERB_CIRCUIT_TEST_REPORT.
+		 *
+		 * Each circuit test packet includes a digital signature by the originator
+		 * of the request, as well as a credential by which that originator claims
+		 * authorization to perform the test. Currently this signature is ed25519,
+		 * but in the future flags might be used to indicate an alternative
+		 * algorithm. For example, the originator might be a network controller.
+		 * In this case the test might be authorized if the recipient is a member
+		 * of a network controlled by it, and if the previous hop(s) are also
+		 * members. Each hop may include its certificate of network membership.
+		 *
+		 * Circuit test paths consist of a series of records. When a node receives
+		 * an authorized circuit test, it:
+		 *
+		 * (1) Reports back to circuit tester as flags indicate
+		 * (2) Reads and removes the next hop from the packet's path
+		 * (3) Sends the packet along to next hop(s), if any.
+		 *
+		 * It is perfectly legal for a path to contain the same hop more than
+		 * once. In fact, this can be a very useful test to determine if a hop
+		 * can be reached bidirectionally and if so what that connectivity looks
+		 * like.
+		 *
+		 * The breadth field in source-routed path records allows a hop to forward
+		 * to more than one recipient, allowing the tester to specify different
+		 * forms of graph traversal in a test.
+		 *
+		 * There is no hard limit to the number of hops in a test, but it is
+		 * practically limited by the maximum size of a (possibly fragmented)
+		 * ZeroTier packet.
+		 *
+		 * Support for circuit tests is optional. If they are not supported, the
+		 * node should respond with an UNSUPPORTED_OPERATION error. If a circuit
+		 * test request is not authorized, it may be ignored or reported as
+		 * an INVALID_REQUEST. No OK messages are generated, but TEST_REPORT
+		 * messages may be sent (see below).
+		 *
+		 * ERROR packet format:
+		 *   <[8] 64-bit timestamp (echoed from original>
+		 *   <[8] 64-bit test ID (echoed from original)>
+		 */
+		VERB_CIRCUIT_TEST = 17,
+
+		/**
+		 * Circuit test hop report:
+		 *   <[8] 64-bit timestamp (from original test)>
+		 *   <[8] 64-bit test ID (from original test)>
+		 *   <[8] 64-bit reporter timestamp (reporter's clock, 0 if unspec)>
+		 *   <[1] 8-bit vendor ID (set to 0, currently unused)>
+		 *   <[1] 8-bit reporter protocol version>
+		 *   <[1] 8-bit reporter major version>
+		 *   <[1] 8-bit reporter minor version>
+		 *   <[2] 16-bit reporter revision>
+		 *   <[2] 16-bit reporter OS/platform>
+		 *   <[2] 16-bit reporter architecture>
+		 *   <[2] 16-bit error code (set to 0, currently unused)>
+		 *   <[8] 64-bit report flags (set to 0, currently unused)>
+		 *   <[8] 64-bit source packet ID>
+		 *   <[5] upstream ZeroTier address from which test was received>
+		 *   <[1] 8-bit source packet hop count (ZeroTier hop count)>
+		 *   <[...] local wire address on which packet was received>
+		 *   <[...] remote wire address from which packet was received>
+		 *   <[2] 16-bit length of additional fields>
+		 *   <[...] additional fields>
+		 *   <[1] 8-bit number of next hops (breadth)>
+		 *   <[...] next hop information>
+		 *
+		 * Next hop information record format:
+		 *   <[5] ZeroTier address of next hop>
+		 *   <[...] current best direct path address, if any, 0 if none>
+		 *
+		 * Circuit test reports can be sent by hops in a circuit test to report
+		 * back results. They should include information about the sender as well
+		 * as about the paths to which next hops are being sent.
+		 *
+		 * If a test report is received and no circuit test was sent, it should be
+		 * ignored. This message generates no OK or ERROR response.
+		 */
+		VERB_CIRCUIT_TEST_REPORT = 18,
+
+		/**
+		 * Request proof of work:
+		 *   <[1] 8-bit proof of work type>
+		 *   <[1] 8-bit proof of work difficulty>
+		 *   <[2] 16-bit length of proof of work challenge>
+		 *   <[...] proof of work challenge>
+		 *
+		 * This requests that a peer perform a proof of work calucation. It can be
+		 * sent by highly trusted peers (e.g. root servers, network controllers)
+		 * under suspected denial of service conditions in an attempt to filter
+		 * out "non-serious" peers and remain responsive to those proving their
+		 * intent to actually communicate.
+		 *
+		 * If the peer obliges to perform the work, it does so and responds with
+		 * an OK containing the result. Otherwise it may ignore the message or
+		 * response with an ERROR_INVALID_REQUEST or ERROR_UNSUPPORTED_OPERATION.
+		 *
+		 * Proof of work type IDs:
+		 *   0x01 - Salsa20/12+SHA512 hashcash function
+		 *
+		 * Salsa20/12+SHA512 is based on the following composite hash function:
+		 *
+		 * (1) Compute SHA512(candidate)
+		 * (2) Use the first 256 bits of the result of #1 as a key to encrypt
+		 *     131072 zero bytes with Salsa20/12 (with a zero IV).
+		 * (3) Compute SHA512(the result of step #2)
+		 * (4) Accept this candiate if the first [difficulty] bits of the result
+		 *     from step #3 are zero. Otherwise generate a new candidate and try
+		 *     again.
+		 *
+		 * This is performed repeatedly on candidates generated by appending the
+		 * supplied challenge to an arbitrary nonce until a valid candidate
+		 * is found. This chosen prepended nonce is then returned as the result
+		 * in OK.
+		 *
+		 * OK payload:
+		 *   <[2] 16-bit length of result>
+		 *   <[...] computed proof of work>
+		 *
+		 * ERROR has no payload.
+		 */
+		VERB_REQUEST_PROOF_OF_WORK = 19
 	};
 
 	/**
@@ -940,19 +1137,12 @@ public:
 		ERROR_UNWANTED_MULTICAST = 8
 	};
 
-	/**
-	 * @param v Verb
-	 * @return String representation (e.g. HELLO, OK)
-	 */
+#ifdef ZT_TRACE
 	static const char *verbString(Verb v)
 		throw();
-
-	/**
-	 * @param e Error code
-	 * @return String error name
-	 */
 	static const char *errorString(ErrorCode e)
 		throw();
+#endif
 
 	template<unsigned int C2>
 	Packet(const Buffer<C2> &b) :

node/Defaults.hpp → node/Path.cpp

@@ -25,50 +25,21 @@
  * LLC. Start here: http://www.zerotier.com/
  */
 
-#ifndef ZT_DEFAULTS_HPP
-#define ZT_DEFAULTS_HPP
-
-#include <stdexcept>
-#include <string>
-#include <vector>
-#include <map>
-
-#include "Constants.hpp"
-#include "Identity.hpp"
-#include "InetAddress.hpp"
+#include "Path.hpp"
+#include "AntiRecursion.hpp"
+#include "RuntimeEnvironment.hpp"
+#include "Node.hpp"
 
 namespace ZeroTier {
 
-/**
- * Static configuration defaults
- * 
- * These are the default values that ship baked into the ZeroTier binary. They
- * define the basic parameters required for it to connect to the rest of the
- * network and obtain software updates.
- */
-class Defaults
+bool Path::send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 {
-public:
-	Defaults();
-
-	/**
-	 * Default root topology dictionary
-	 */
-	const std::string defaultRootTopology;
-
-	/**
-	 * Identities permitted to sign root topology dictionaries
-	 */
-	const std::map< Address,Identity > rootTopologyAuthorities;
-
-	/**
-	 * Address for IPv4 LAN auto-location broadcasts: 255.255.255.255:9993
-	 */
-	const InetAddress v4Broadcast;
-};
-
-extern const Defaults ZT_DEFAULTS;
+	if (RR->node->putPacket(_localAddress,address(),data,len)) {
+		sent(now);
+		RR->antiRec->logOutgoingZT(data,len);
+		return true;
+	}
+	return false;
+}
 
 } // namespace ZeroTier
-
-#endif

node/Path.hpp

@@ -28,12 +28,19 @@
 #ifndef ZT_PATH_HPP
 #define ZT_PATH_HPP
 
+#include <stdint.h>
+#include <string.h>
+
+#include <stdexcept>
+#include <algorithm>
+
 #include "Constants.hpp"
 #include "InetAddress.hpp"
-#include "Utils.hpp"
 
 namespace ZeroTier {
 
+class RuntimeEnvironment;
+
 /**
  * Base class for paths
  *
@@ -42,44 +49,92 @@ namespace ZeroTier {
 class Path
 {
 public:
+	Path() :
+		_lastSend(0),
+		_lastReceived(0),
+		_addr(),
+		_localAddress(),
+		_ipScope(InetAddress::IP_SCOPE_NONE)
+	{
+	}
+
+	Path(const InetAddress &localAddress,const InetAddress &addr) :
+		_lastSend(0),
+		_lastReceived(0),
+		_addr(addr),
+		_localAddress(localAddress),
+		_ipScope(addr.ipScope())
+	{
+	}
+
+	inline Path &operator=(const Path &p)
+		throw()
+	{
+		if (this != &p)
+			memcpy(this,&p,sizeof(Path));
+		return *this;
+	}
+
 	/**
-	 * Path trust category
-	 *
-	 * Note that this is NOT peer trust and has nothing to do with root server
-	 * designations or other trust metrics. This indicates how much we trust
-	 * this path to be secure and/or private. A trust level of normal means
-	 * encrypt and authenticate all traffic. Privacy trust means we can send
-	 * traffic in the clear. Ultimate trust means we don't even need
-	 * authentication. Generally a private path would be a hard-wired local
-	 * LAN, while an ultimate trust path would be a physically isolated private
-	 * server backplane.
+	 * Called when a packet is sent to this remote path
 	 *
-	 * Nearly all paths will be normal trust. The other levels are for high
-	 * performance local SDN use only.
+	 * This is called automatically by Path::send().
 	 *
-	 * These values MUST match ZT_LocalInterfaceAddressTrust in ZeroTierOne.h
+	 * @param t Time of send
 	 */
-	enum Trust
+	inline void sent(uint64_t t)
+		throw()
 	{
-		TRUST_NORMAL = 0,
-		TRUST_PRIVACY = 1,
-		TRUST_ULTIMATE = 2
-	};
+		_lastSend = t;
+	}
 
-	Path() :
-		_addr(),
-		_ipScope(InetAddress::IP_SCOPE_NONE),
-		_trust(TRUST_NORMAL)
+	/**
+	 * Called when a packet is received from this remote path
+	 *
+	 * @param t Time of receive
+	 */
+	inline void received(uint64_t t)
+		throw()
 	{
+		_lastReceived = t;
 	}
 
-	Path(const InetAddress &addr,int metric,Trust trust) :
-		_addr(addr),
-		_ipScope(addr.ipScope()),
-		_trust(trust)
+	/**
+	 * @param now Current time
+	 * @return True if this path appears active
+	 */
+	inline bool active(uint64_t now) const
+		throw()
 	{
+		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
 	}
 
+	/**
+	 * Send a packet via this path
+	 *
+	 * @param RR Runtime environment
+	 * @param data Packet data
+	 * @param len Packet length
+	 * @param now Current time
+	 * @return True if transport reported success
+	 */
+	bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now);
+
+	/**
+	 * @return Address of local side of this path or NULL if unspecified
+	 */
+	inline const InetAddress &localAddress() const throw() { return _localAddress; }
+
+	/**
+	 * @return Time of last send to this path
+	 */
+	inline uint64_t lastSend() const throw() { return _lastSend; }
+
+	/**
+	 * @return Time of last receive from this path
+	 */
+	inline uint64_t lastReceived() const throw() { return _lastReceived; }
+
 	/**
 	 * @return Physical address
 	 */
@@ -104,17 +159,14 @@ public:
 		return ( ((int)_ipScope * 2) + ((_addr.ss_family == AF_INET6) ? 1 : 0) );
 	}
 
-	/**
-	 * @return Path trust level
-	 */
-	inline Trust trust() const throw() { return _trust; }
-
 	/**
 	 * @return True if path is considered reliable (no NAT keepalives etc. are needed)
 	 */
 	inline bool reliable() const throw()
 	{
-		return ((_ipScope != InetAddress::IP_SCOPE_GLOBAL)&&(_ipScope != InetAddress::IP_SCOPE_PSEUDOPRIVATE));
+		if (_addr.ss_family == AF_INET)
+			return ((_ipScope != InetAddress::IP_SCOPE_GLOBAL)&&(_ipScope != InetAddress::IP_SCOPE_PSEUDOPRIVATE));
+		return true;
 	}
 
 	/**
@@ -122,14 +174,6 @@ public:
 	 */
 	inline operator bool() const throw() { return (_addr); }
 
-	// Comparisons are by address only
-	inline bool operator==(const Path &p) const throw() { return (_addr == p._addr); }
-	inline bool operator!=(const Path &p) const throw() { return (_addr != p._addr); }
-	inline bool operator<(const Path &p) const throw() { return (_addr < p._addr); }
-	inline bool operator>(const Path &p) const throw() { return (_addr > p._addr); }
-	inline bool operator<=(const Path &p) const throw() { return (_addr <= p._addr); }
-	inline bool operator>=(const Path &p) const throw() { return (_addr >= p._addr); }
-
 	/**
 	 * Check whether this address is valid for a ZeroTier path
 	 *
@@ -163,10 +207,36 @@ public:
 		return false;
 	}
 
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		b.append((uint8_t)1); // version
+		b.append((uint64_t)_lastSend);
+		b.append((uint64_t)_lastReceived);
+		_addr.serialize(b);
+		_localAddress.serialize(b);
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+		if (b[p++] != 1)
+			throw std::invalid_argument("invalid serialized Path");
+		_lastSend = b.template at<uint64_t>(p); p += 8;
+		_lastReceived = b.template at<uint64_t>(p); p += 8;
+		p += _addr.deserialize(b,p);
+		p += _localAddress.deserialize(b,p);
+		_ipScope = _addr.ipScope();
+		return (p - startAt);
+	}
+
 private:
+	uint64_t _lastSend;
+	uint64_t _lastReceived;
 	InetAddress _addr;
+	InetAddress _localAddress;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
-	Trust _trust;
 };
 
 } // namespace ZeroTier

node/Peer.cpp

@@ -34,9 +34,13 @@
 #include "Network.hpp"
 #include "AntiRecursion.hpp"
 #include "SelfAwareness.hpp"
+#include "Cluster.hpp"
+#include "Packet.hpp"
 
 #include <algorithm>
 
+#define ZT_PEER_PATH_SORT_INTERVAL 5000
+
 namespace ZeroTier {
 
 // Used to send varying values for NAT keepalive
@@ -50,13 +54,19 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	_lastMulticastFrame(0),
 	_lastAnnouncedTo(0),
 	_lastPathConfirmationSent(0),
-	_lastDirectPathPush(0),
+	_lastDirectPathPushSent(0),
+	_lastDirectPathPushReceive(0),
+	_lastPathSort(0),
+	_vProto(0),
 	_vMajor(0),
 	_vMinor(0),
 	_vRevision(0),
 	_id(peerIdentity),
 	_numPaths(0),
-	_latency(0)
+	_latency(0),
+	_directPathPushCutoffCount(0),
+	_networkComs(4),
+	_lastPushedComs(4)
 {
 	if (!myIdentity.agree(peerIdentity,_key,ZT_PEER_SECRET_KEY_LENGTH))
 		throw std::runtime_error("new peer identity key agreement failed");
@@ -72,14 +82,78 @@ void Peer::received(
 	uint64_t inRePacketId,
 	Packet::Verb inReVerb)
 {
+#ifdef ZT_ENABLE_CLUSTER
+	InetAddress redirectTo;
+	if ((RR->cluster)&&(hops == 0)) {
+		// Note: findBetterEndpoint() is first since we still want to check
+		// for a better endpoint even if we don't actually send a redirect.
+		if ( (RR->cluster->findBetterEndpoint(redirectTo,_id.address(),remoteAddr,false)) && (verb != Packet::VERB_OK)&&(verb != Packet::VERB_ERROR)&&(verb != Packet::VERB_RENDEZVOUS)&&(verb != Packet::VERB_PUSH_DIRECT_PATHS) ) {
+			if (_vProto >= 5) {
+				// For newer peers we can send a more idiomatic verb: PUSH_DIRECT_PATHS.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
+				outp.append((uint16_t)1); // count == 1
+				outp.append((uint8_t)0); // no flags
+				outp.append((uint16_t)0); // no extensions
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append((uint8_t)6);
+					outp.append(redirectTo.rawIpData(),4);
+				} else {
+					outp.append((uint8_t)6);
+					outp.append((uint8_t)18);
+					outp.append(redirectTo.rawIpData(),16);
+				}
+				outp.append((uint16_t)redirectTo.port());
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+			} else {
+				// For older peers we use RENDEZVOUS to coax them into contacting us elsewhere.
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_RENDEZVOUS);
+				outp.append((uint8_t)0); // no flags
+				RR->identity.address().appendTo(outp);
+				outp.append((uint16_t)redirectTo.port());
+				if (redirectTo.ss_family == AF_INET) {
+					outp.append((uint8_t)4);
+					outp.append(redirectTo.rawIpData(),4);
+				} else {
+					outp.append((uint8_t)16);
+					outp.append(redirectTo.rawIpData(),16);
+				}
+				outp.armor(_key,true);
+				RR->antiRec->logOutgoingZT(outp.data(),outp.size());
+				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
+			}
+		}
+	}
+#endif
+
 	const uint64_t now = RR->node->now();
-	_lastReceive = now;
+	bool needMulticastGroupAnnounce = false;
+	bool pathIsConfirmed = false;
+
+	{	// begin _lock
+		Mutex::Lock _l(_lock);
+
+		_lastReceive = now;
+		if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
+			_lastUnicastFrame = now;
+		else if (verb == Packet::VERB_MULTICAST_FRAME)
+			_lastMulticastFrame = now;
+
+#ifdef ZT_ENABLE_CLUSTER
+		// If we think this peer belongs elsewhere, don't learn this path or
+		// do other connection init stuff.
+		if (redirectTo)
+			return;
+#endif
 
-	if (!hops) {
-		bool pathIsConfirmed = false;
+		if ((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000)) {
+			_lastAnnouncedTo = now;
+			needMulticastGroupAnnounce = true;
+		}
 
-		/* Learn new paths from direct (hops == 0) packets */
-		{
+		if (hops == 0) {
 			unsigned int np = _numPaths;
 			for(unsigned int p=0;p<np;++p) {
 				if ((_paths[p].address() == remoteAddr)&&(_paths[p].localAddress() == localAddr)) {
@@ -90,29 +164,30 @@ void Peer::received(
 			}
 
 			if (!pathIsConfirmed) {
-				if ((verb == Packet::VERB_OK)&&(inReVerb == Packet::VERB_HELLO)) {
-					// Learn paths if they've been confirmed via a HELLO
-					RemotePath *slot = (RemotePath *)0;
+				if ((verb == Packet::VERB_OK)||(RR->topology->amRoot())) {
+
+					Path *slot = (Path *)0;
 					if (np < ZT_MAX_PEER_NETWORK_PATHS) {
-						// Add new path
 						slot = &(_paths[np++]);
 					} else {
-						// Replace oldest non-fixed path
 						uint64_t slotLRmin = 0xffffffffffffffffULL;
 						for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
-							if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
+							if (_paths[p].lastReceived() <= slotLRmin) {
 								slotLRmin = _paths[p].lastReceived();
 								slot = &(_paths[p]);
 							}
 						}
 					}
 					if (slot) {
-						*slot = RemotePath(localAddr,remoteAddr,false);
+						*slot = Path(localAddr,remoteAddr);
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
+						_sortPaths(now);
 					}
+
 				} else {
+
 					/* If this path is not known, send a HELLO. We don't learn
 					 * paths without confirming that a bidirectional link is in
 					 * fact present, but any packet that decodes and authenticates
@@ -122,133 +197,104 @@ void Peer::received(
 						TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
 						attemptToContactAt(RR,localAddr,remoteAddr,now);
 					}
-				}
-			}
-		}
-
-		/* Announce multicast groups of interest to direct peers if they are
-		 * considered authorized members of a given network. Also announce to
-		 * root servers and network controllers. */
-		if ((pathIsConfirmed)&&((now - _lastAnnouncedTo) >= ((ZT_MULTICAST_LIKE_EXPIRE / 2) - 1000))) {
-			_lastAnnouncedTo = now;
-
-			const bool isRoot = RR->topology->isRoot(_id);
-
-			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
-			const std::vector< SharedPtr<Network> > networks(RR->node->allNetworks());
-			for(std::vector< SharedPtr<Network> >::const_iterator n(networks.begin());n!=networks.end();++n) {
-				if ( (isRoot) || ((*n)->isAllowed(_id.address())) || (_id.address() == (*n)->controller()) ) {
-					const std::vector<MulticastGroup> mgs((*n)->allMulticastGroups());
-					for(std::vector<MulticastGroup>::const_iterator mg(mgs.begin());mg!=mgs.end();++mg) {
-						if ((outp.size() + 18) > ZT_UDP_DEFAULT_PAYLOAD_MTU) {
-							outp.armor(_key,true);
-							RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
-							outp.reset(_id.address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
-						}
 
-						// network ID, MAC, ADI
-						outp.append((uint64_t)(*n)->id());
-						mg->mac().appendTo(outp);
-						outp.append((uint32_t)mg->adi());
-					}
 				}
 			}
-			if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) {
-				outp.armor(_key,true);
-				RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
-			}
 		}
-	}
+	}	// end _lock
 
-	if ((verb == Packet::VERB_FRAME)||(verb == Packet::VERB_EXT_FRAME))
-		_lastUnicastFrame = now;
-	else if (verb == Packet::VERB_MULTICAST_FRAME)
-		_lastMulticastFrame = now;
-}
+#ifdef ZT_ENABLE_CLUSTER
+	if ((RR->cluster)&&(pathIsConfirmed))
+		RR->cluster->replicateHavePeer(_id,remoteAddr);
+#endif
 
-RemotePath *Peer::getBestPath(uint64_t now)
-{
-	RemotePath *bestPath = (RemotePath *)0;
-	uint64_t lrMax = 0;
-	int rank = 0;
-	for(unsigned int p=0,np=_numPaths;p<np;++p) {
-		if ( (_paths[p].active(now)) && ((_paths[p].lastReceived() >= lrMax)||(_paths[p].preferenceRank() >= rank)) ) {
-			lrMax = _paths[p].lastReceived();
-			rank = _paths[p].preferenceRank();
-			bestPath = &(_paths[p]);
-		}
+	if (needMulticastGroupAnnounce) {
+		const std::vector< SharedPtr<Network> > networks(RR->node->allNetworks());
+		for(std::vector< SharedPtr<Network> >::const_iterator n(networks.begin());n!=networks.end();++n)
+			(*n)->tryAnnounceMulticastGroupsTo(SharedPtr<Peer>(this));
 	}
-	return bestPath;
 }
 
 void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now)
 {
+	// _lock not required here since _id is immutable and nothing else is accessed
+
 	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
 	outp.append((unsigned char)ZT_PROTO_VERSION);
 	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MAJOR);
 	outp.append((unsigned char)ZEROTIER_ONE_VERSION_MINOR);
 	outp.append((uint16_t)ZEROTIER_ONE_VERSION_REVISION);
 	outp.append(now);
-
 	RR->identity.serialize(outp,false);
-
-	switch(atAddress.ss_family) {
-		case AF_INET:
-			outp.append((unsigned char)ZT_PROTO_DEST_ADDRESS_TYPE_IPV4);
-			outp.append(atAddress.rawIpData(),4);
-			outp.append((uint16_t)atAddress.port());
-			break;
-		case AF_INET6:
-			outp.append((unsigned char)ZT_PROTO_DEST_ADDRESS_TYPE_IPV6);
-			outp.append(atAddress.rawIpData(),16);
-			outp.append((uint16_t)atAddress.port());
-			break;
-		default:
-			outp.append((unsigned char)ZT_PROTO_DEST_ADDRESS_TYPE_NONE);
-			break;
-	}
+	atAddress.serialize(outp);
+	outp.append((uint64_t)RR->topology->worldId());
+	outp.append((uint64_t)RR->topology->worldTimestamp());
 
 	outp.armor(_key,false); // HELLO is sent in the clear
+	RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
 }
 
-void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
+bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily)
 {
-	RemotePath *const bestPath = getBestPath(now);
-	if (bestPath) {
-		if ((now - bestPath->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
-			TRACE("PING %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
-			attemptToContactAt(RR,bestPath->localAddress(),bestPath->address(),now);
-			bestPath->sent(now);
-		} else if (((now - bestPath->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!bestPath->reliable())) {
+	Path *p = (Path *)0;
+
+	Mutex::Lock _l(_lock);
+	if (inetAddressFamily != 0) {
+		p = _getBestPath(now,inetAddressFamily);
+	} else {
+		p = _getBestPath(now);
+	}
+
+	if (p) {
+		if ((now - p->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
+			//TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
+			attemptToContactAt(RR,p->localAddress(),p->address(),now);
+			p->sent(now);
+		} else if (((now - p->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!p->reliable())) {
+			//TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
 			_natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads
-			TRACE("NAT keepalive %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
-			RR->node->putPacket(bestPath->localAddress(),bestPath->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
-			bestPath->sent(now);
+			RR->node->putPacket(p->localAddress(),p->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
+			p->sent(now);
+		} else {
+			//TRACE("no PING or NAT keepalive: addr==%s reliable==%d %llums/%llums send/receive inactivity",p->address().toString().c_str(),(int)p->reliable(),now - p->lastSend(),now - p->lastReceived());
 		}
+		return true;
 	}
+
+	return false;
 }
 
-void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force)
+void Peer::pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force)
 {
-	if (((now - _lastDirectPathPush) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
-		_lastDirectPathPush = now;
+#ifdef ZT_ENABLE_CLUSTER
+	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
+	if (RR->cluster)
+		return;
+#endif
+
+	Mutex::Lock _l(_lock);
 
-		std::vector<Path> dps(RR->node->directPaths());
+	if (((now - _lastDirectPathPushSent) >= ZT_DIRECT_PATH_PUSH_INTERVAL)||(force)) {
+		_lastDirectPathPushSent = now;
+
+		std::vector<InetAddress> dps(RR->node->directPaths());
+		if (dps.empty())
+			return;
 
 #ifdef ZT_TRACE
 		{
 			std::string ps;
-			for(std::vector<Path>::const_iterator p(dps.begin());p!=dps.end();++p) {
+			for(std::vector<InetAddress>::const_iterator p(dps.begin());p!=dps.end();++p) {
 				if (ps.length() > 0)
 					ps.push_back(',');
-				ps.append(p->address().toString());
+				ps.append(p->toString());
 			}
 			TRACE("pushing %u direct paths to %s: %s",(unsigned int)dps.size(),_id.address().toString().c_str(),ps.c_str());
 		}
 #endif
 
-		std::vector<Path>::const_iterator p(dps.begin());
+		std::vector<InetAddress>::const_iterator p(dps.begin());
 		while (p != dps.end()) {
 			Packet outp(_id.address(),RR->identity.address(),Packet::VERB_PUSH_DIRECT_PATHS);
 			outp.addSize(2); // leave room for count
@@ -256,7 +302,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 			unsigned int count = 0;
 			while ((p != dps.end())&&((outp.size() + 24) < ZT_PROTO_MAX_PACKET_LENGTH)) {
 				uint8_t addressType = 4;
-				switch(p->address().ss_family) {
+				switch(p->ss_family) {
 					case AF_INET:
 						break;
 					case AF_INET6:
@@ -268,6 +314,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 				}
 
 				uint8_t flags = 0;
+				/* TODO: path trust is not implemented yet
 				switch(p->trust()) {
 					default:
 						break;
@@ -278,13 +325,14 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 						flags |= (0x04 | 0x08); // no encryption, no authentication (redundant but go ahead and set both)
 						break;
 				}
+				*/
 
 				outp.append(flags);
 				outp.append((uint16_t)0); // no extensions
 				outp.append(addressType);
 				outp.append((uint8_t)((addressType == 4) ? 6 : 18));
-				outp.append(p->address().rawIpData(),((addressType == 4) ? 4 : 16));
-				outp.append((uint16_t)p->address().port());
+				outp.append(p->rawIpData(),((addressType == 4) ? 4 : 16));
+				outp.append((uint16_t)p->port());
 
 				++count;
 				++p;
@@ -299,76 +347,28 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_
 	}
 }
 
-void Peer::addPath(const RemotePath &newp)
-{
-	unsigned int np = _numPaths;
-
-	for(unsigned int p=0;p<np;++p) {
-		if (_paths[p].address() == newp.address()) {
-			_paths[p].setFixed(newp.fixed());
-			return;
-		}
-	}
-
-	RemotePath *slot = (RemotePath *)0;
-	if (np < ZT_MAX_PEER_NETWORK_PATHS) {
-		// Add new path
-		slot = &(_paths[np++]);
-	} else {
-		// Replace oldest non-fixed path
-		uint64_t slotLRmin = 0xffffffffffffffffULL;
-		for(unsigned int p=0;p<ZT_MAX_PEER_NETWORK_PATHS;++p) {
-			if ((!_paths[p].fixed())&&(_paths[p].lastReceived() <= slotLRmin)) {
-				slotLRmin = _paths[p].lastReceived();
-				slot = &(_paths[p]);
-			}
-		}
-	}
-	if (slot) {
-		*slot = newp;
-		_numPaths = np;
-	}
-}
-
-void Peer::clearPaths(bool fixedToo)
-{
-	if (fixedToo) {
-		_numPaths = 0;
-	} else {
-		unsigned int np = _numPaths;
-		unsigned int x = 0;
-		unsigned int y = 0;
-		while (x < np) {
-			if (_paths[x].fixed())
-				_paths[y++] = _paths[x];
-			++x;
-		}
-		_numPaths = y;
-	}
-}
-
 bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope scope,uint64_t now)
 {
+	Mutex::Lock _l(_lock);
 	unsigned int np = _numPaths;
 	unsigned int x = 0;
 	unsigned int y = 0;
 	while (x < np) {
 		if (_paths[x].address().ipScope() == scope) {
-			if (_paths[x].fixed()) {
-				attemptToContactAt(RR,_paths[x].localAddress(),_paths[x].address(),now);
-				_paths[y++] = _paths[x]; // keep fixed paths
-			}
+			attemptToContactAt(RR,_paths[x].localAddress(),_paths[x].address(),now);
 		} else {
-			_paths[y++] = _paths[x]; // keep paths not in this scope
+			_paths[y++] = _paths[x];
 		}
 		++x;
 	}
 	_numPaths = y;
+	_sortPaths(now);
 	return (y < np);
 }
 
 void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const
 {
+	Mutex::Lock _l(_lock);
 	uint64_t bestV4 = 0,bestV6 = 0;
 	for(unsigned int p=0,np=_numPaths;p<np;++p) {
 		if (_paths[p].active(now)) {
@@ -390,4 +390,169 @@ void Peer::getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6)
 	}
 }
 
+bool Peer::networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const
+{
+	Mutex::Lock _l(_lock);
+	const _NetworkCom *ourCom = _networkComs.get(nwid);
+	if (ourCom)
+		return ourCom->com.agreesWith(com);
+	return false;
+}
+
+bool Peer::validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com)
+{
+	// Sanity checks
+	if ((!com)||(com.issuedTo() != _id.address()))
+		return false;
+
+	// Return true if we already have this *exact* COM
+	{
+		Mutex::Lock _l(_lock);
+		_NetworkCom *ourCom = _networkComs.get(nwid);
+		if ((ourCom)&&(ourCom->com == com))
+			return true;
+	}
+
+	// Check signature, log and return if cert is invalid
+	if (com.signedBy() != Network::controllerFor(nwid)) {
+		TRACE("rejected network membership certificate for %.16llx signed by %s: signer not a controller of this network",(unsigned long long)_id,com.signedBy().toString().c_str());
+		return false; // invalid signer
+	}
+
+	if (com.signedBy() == RR->identity.address()) {
+
+		// We are the controller: RR->identity.address() == controller() == cert.signedBy()
+		// So, verify that we signed th cert ourself
+		if (!com.verify(RR->identity)) {
+			TRACE("rejected network membership certificate for %.16llx self signed by %s: signature check failed",(unsigned long long)_id,com.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+
+	} else {
+
+		SharedPtr<Peer> signer(RR->topology->getPeer(com.signedBy()));
+
+		if (!signer) {
+			// This would be rather odd, since this is our controller... could happen
+			// if we get packets before we've gotten config.
+			RR->sw->requestWhois(com.signedBy());
+			return false; // signer unknown
+		}
+
+		if (!com.verify(signer->identity())) {
+			TRACE("rejected network membership certificate for %.16llx signed by %s: signature check failed",(unsigned long long)_id,com.signedBy().toString().c_str());
+			return false; // invalid signature
+		}
+	}
+
+	// If we made it past all those checks, add or update cert in our cert info store
+	{
+		Mutex::Lock _l(_lock);
+		_networkComs.set(nwid,_NetworkCom(RR->node->now(),com));
+	}
+
+	return true;
+}
+
+bool Peer::needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime)
+{
+	Mutex::Lock _l(_lock);
+	uint64_t &lastPushed = _lastPushedComs[nwid];
+	const uint64_t tmp = lastPushed;
+	if (updateLastPushedTime)
+		lastPushed = now;
+	return ((now - tmp) >= (ZT_NETWORK_AUTOCONF_DELAY / 2));
+}
+
+void Peer::clean(const RuntimeEnvironment *RR,uint64_t now)
+{
+	Mutex::Lock _l(_lock);
+
+	{
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if (_paths[x].active(now))
+				_paths[y++] = _paths[x];
+			++x;
+		}
+		_numPaths = y;
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		_NetworkCom *v = (_NetworkCom *)0;
+		Hashtable< uint64_t,_NetworkCom >::Iterator i(_networkComs);
+		while (i.next(k,v)) {
+			if ( (!RR->node->belongsToNetwork(*k)) && ((now - v->ts) >= ZT_PEER_NETWORK_COM_EXPIRATION) )
+				_networkComs.erase(*k);
+		}
+	}
+
+	{
+		uint64_t *k = (uint64_t *)0;
+		uint64_t *v = (uint64_t *)0;
+		Hashtable< uint64_t,uint64_t >::Iterator i(_lastPushedComs);
+		while (i.next(k,v)) {
+			if ((now - *v) > (ZT_NETWORK_AUTOCONF_DELAY * 2))
+				_lastPushedComs.erase(*k);
+		}
+	}
+}
+
+struct _SortPathsByQuality
+{
+	uint64_t _now;
+	_SortPathsByQuality(const uint64_t now) : _now(now) {}
+	inline bool operator()(const Path &a,const Path &b) const
+	{
+		const uint64_t qa = (
+			((uint64_t)a.active(_now) << 63) |
+			(((uint64_t)(a.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)a.lastReceived() & 0x7ffffffffffffULL) );
+		const uint64_t qb = (
+			((uint64_t)b.active(_now) << 63) |
+			(((uint64_t)(b.preferenceRank() & 0xfff)) << 51) |
+			((uint64_t)b.lastReceived() & 0x7ffffffffffffULL) );
+		return (qb < qa); // invert sense to sort in descending order
+	}
+};
+void Peer::_sortPaths(const uint64_t now)
+{
+	// assumes _lock is locked
+	_lastPathSort = now;
+	std::sort(&(_paths[0]),&(_paths[_numPaths]),_SortPathsByQuality(now));
+}
+
+Path *Peer::_getBestPath(const uint64_t now)
+{
+	// assumes _lock is locked
+	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
+		_sortPaths(now);
+	if (_paths[0].active(now)) {
+		return &(_paths[0]);
+	} else {
+		_sortPaths(now);
+		if (_paths[0].active(now))
+			return &(_paths[0]);
+	}
+	return (Path *)0;
+}
+
+Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
+{
+	// assumes _lock is locked
+	if ((now - _lastPathSort) >= ZT_PEER_PATH_SORT_INTERVAL)
+		_sortPaths(now);
+	for(int k=0;k<2;++k) { // try once, and if it fails sort and try one more time
+		for(unsigned int i=0;i<_numPaths;++i) {
+			if ((_paths[i].active(now))&&((int)_paths[i].address().ss_family == inetAddressFamily))
+				return &(_paths[i]);
+		}
+		_sortPaths(now);
+	}
+	return (Path *)0;
+}
+
 } // namespace ZeroTier

node/Peer.hpp

@@ -41,7 +41,7 @@
 
 #include "RuntimeEnvironment.hpp"
 #include "CertificateOfMembership.hpp"
-#include "RemotePath.hpp"
+#include "Path.hpp"
 #include "Address.hpp"
 #include "Utils.hpp"
 #include "Identity.hpp"
@@ -49,8 +49,14 @@
 #include "Packet.hpp"
 #include "SharedPtr.hpp"
 #include "AtomicCounter.hpp"
+#include "Hashtable.hpp"
+#include "Mutex.hpp"
 #include "NonCopyable.hpp"
 
+// Very rough computed estimate: (8 + 256 + 80 + (16 * 64) + (128 * 256) + (128 * 16))
+// 1048576 provides tons of headroom -- overflow would just cause peer not to be persisted
+#define ZT_PEER_SUGGESTED_SERIALIZATION_BUFFER_SIZE 1048576
+
 namespace ZeroTier {
 
 /**
@@ -124,12 +130,16 @@ public:
 		Packet::Verb inReVerb = Packet::VERB_NOP);
 
 	/**
-	 * Get the best direct path to this peer
+	 * Get the current best direct path to this peer
 	 *
 	 * @param now Current time
-	 * @return Best path or NULL if there are no active (or fixed) direct paths
+	 * @return Best path or NULL if there are no active direct paths
 	 */
-	RemotePath *getBestPath(uint64_t now);
+	inline Path *getBestPath(uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		return _getBestPath(now);
+	}
 
 	/**
 	 * Send via best path
@@ -140,14 +150,14 @@ public:
 	 * @param now Current time
 	 * @return Path used on success or NULL on failure
 	 */
-	inline RemotePath *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+	inline Path *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 	{
-		RemotePath *bestPath = getBestPath(now);
+		Path *bestPath = getBestPath(now);
 		if (bestPath) {
 			if (bestPath->send(RR,data,len,now))
 				return bestPath;
 		}
-		return (RemotePath *)0;
+		return (Path *)0;
 	}
 
 	/**
@@ -168,8 +178,10 @@ public:
 	 *
 	 * @param RR Runtime environment
 	 * @param now Current time
+	 * @param inetAddressFamily Keep this address family alive, or 0 to simply pick current best ignoring family
+	 * @return True if at least one direct path seems alive
 	 */
-	void doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now);
+	bool doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily);
 
 	/**
 	 * Push direct paths if we haven't done so in [rate limit] milliseconds
@@ -179,43 +191,20 @@ public:
 	 * @param now Current time
 	 * @param force If true, push regardless of rate limit
 	 */
-	void pushDirectPaths(const RuntimeEnvironment *RR,RemotePath *path,uint64_t now,bool force);
+	void pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force);
 
 	/**
 	 * @return All known direct paths to this peer
 	 */
-	inline std::vector<RemotePath> paths() const
+	inline std::vector<Path> paths() const
 	{
-		std::vector<RemotePath> pp;
+		std::vector<Path> pp;
+		Mutex::Lock _l(_lock);
 		for(unsigned int p=0,np=_numPaths;p<np;++p)
 			pp.push_back(_paths[p]);
 		return pp;
 	}
 
-	/**
-	 * @return Time of last direct packet receive for any path
-	 */
-	inline uint64_t lastDirectReceive() const
-		throw()
-	{
-		uint64_t x = 0;
-		for(unsigned int p=0,np=_numPaths;p<np;++p)
-			x = std::max(x,_paths[p].lastReceived());
-		return x;
-	}
-
-	/**
-	 * @return Time of last direct packet send for any path
-	 */
-	inline uint64_t lastDirectSend() const
-		throw()
-	{
-		uint64_t x = 0;
-		for(unsigned int p=0,np=_numPaths;p<np;++p)
-			x = std::max(x,_paths[p].lastSend());
-		return x;
-	}
-
 	/**
 	 * @return Time of last receive of anything, whether direct or relayed
 	 */
@@ -242,20 +231,38 @@ public:
 	inline uint64_t lastAnnouncedTo() const throw() { return _lastAnnouncedTo; }
 
 	/**
-	 * @return True if peer has received an actual data frame within ZT_PEER_ACTIVITY_TIMEOUT milliseconds
+	 * @return True if this peer is actively sending real network frames
 	 */
-	inline uint64_t alive(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
+	inline uint64_t activelyTransferringFrames(uint64_t now) const throw() { return ((now - lastFrame()) < ZT_PEER_ACTIVITY_TIMEOUT); }
 
 	/**
-	 * @return Current latency or 0 if unknown (max: 65535)
+	 * @return Latency in milliseconds or 0 if unknown
 	 */
-	inline unsigned int latency() const
-		throw()
+	inline unsigned int latency() const { return _latency; }
+
+	/**
+	 * This computes a quality score for relays and root servers
+	 *
+	 * If we haven't heard anything from these in ZT_PEER_ACTIVITY_TIMEOUT, they
+	 * receive the worst possible quality (max unsigned int). Otherwise the
+	 * quality is a product of latency and the number of potential missed
+	 * pings. This causes roots and relays to switch over a bit faster if they
+	 * fail.
+	 *
+	 * @return Relay quality score computed from latency and other factors, lower is better
+	 */
+	inline unsigned int relayQuality(const uint64_t now) const
 	{
+		const uint64_t tsr = now - _lastReceive;
+		if (tsr >= ZT_PEER_ACTIVITY_TIMEOUT)
+			return (~(unsigned int)0);
 		unsigned int l = _latency;
-		return std::min(l,(unsigned int)65535);
+		if (!l)
+			l = 0xffff;
+		return (l * (((unsigned int)tsr / (ZT_PEER_DIRECT_PING_DELAY + 1000)) + 1));
 	}
 
+
 	/**
 	 * Update latency with a new direct measurment
 	 *
@@ -270,18 +277,14 @@ public:
 		else _latency = std::min(l,(unsigned int)65535);
 	}
 
-	/**
-	 * @return True if this peer has at least one direct IP address path
-	 */
-	inline bool hasDirectPath() const throw() { return (_numPaths != 0); }
-
 	/**
 	 * @param now Current time
-	 * @return True if this peer has at least one active or fixed direct path
+	 * @return True if this peer has at least one active direct path
 	 */
 	inline bool hasActiveDirectPath(uint64_t now) const
 		throw()
 	{
+		Mutex::Lock _l(_lock);
 		for(unsigned int p=0,np=_numPaths;p<np;++p) {
 			if (_paths[p].active(now))
 				return true;
@@ -289,26 +292,9 @@ public:
 		return false;
 	}
 
-	/**
-	 * Add a path (if we don't already have it)
-	 *
-	 * @param p New path to add
-	 */
-	void addPath(const RemotePath &newp);
-
-	/**
-	 * Clear paths
-	 *
-	 * @param fixedToo If true, clear fixed paths as well as learned ones
-	 */
-	void clearPaths(bool fixedToo);
-
 	/**
 	 * Reset paths within a given scope
 	 *
-	 * For fixed paths in this scope, a packet is sent. Non-fixed paths in this
-	 * scope are forgotten.
-	 *
 	 * @param RR Runtime environment
 	 * @param scope IP scope of paths to reset
 	 * @param now Current time
@@ -354,6 +340,7 @@ public:
 	inline bool atLeastVersion(unsigned int major,unsigned int minor,unsigned int rev)
 		throw()
 	{
+		Mutex::Lock _l(_lock);
 		if ((_vMajor > 0)||(_vMinor > 0)||(_vRevision > 0)) {
 			if (_vMajor > major)
 				return true;
@@ -381,6 +368,77 @@ public:
 	 */
 	void getBestActiveAddresses(uint64_t now,InetAddress &v4,InetAddress &v6) const;
 
+	/**
+	 * Check network COM agreement with this peer
+	 *
+	 * @param nwid Network ID
+	 * @param com Another certificate of membership
+	 * @return True if supplied COM agrees with ours, false if not or if we don't have one
+	 */
+	bool networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfMembership &com) const;
+
+	/**
+	 * Check the validity of the COM and add/update if valid and new
+	 *
+	 * @param RR Runtime Environment
+	 * @param nwid Network ID
+	 * @param com Externally supplied COM
+	 */
+	bool validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com);
+
+	/**
+	 * @param nwid Network ID
+	 * @param now Current time
+	 * @param updateLastPushedTime If true, go ahead and update the last pushed time regardless of return value
+	 * @return Whether or not this peer needs another COM push from us
+	 */
+	bool needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool updateLastPushedTime);
+
+	/**
+	 * Perform periodic cleaning operations
+	 */
+	void clean(const RuntimeEnvironment *RR,uint64_t now);
+
+	/**
+	 * Remove all paths with this remote address
+	 *
+	 * @param addr Remote address to remove
+	 */
+	inline void removePathByAddress(const InetAddress &addr)
+	{
+		Mutex::Lock _l(_lock);
+		unsigned int np = _numPaths;
+		unsigned int x = 0;
+		unsigned int y = 0;
+		while (x < np) {
+			if (_paths[x].address() != addr)
+				_paths[y++] = _paths[x];
+			++x;
+		}
+		_numPaths = y;
+	}
+
+	/**
+	 * Update direct path push stats and return true if we should respond
+	 *
+	 * This is a circuit breaker to make VERB_PUSH_DIRECT_PATHS not particularly
+	 * useful as a DDOS amplification attack vector. Otherwise a malicious peer
+	 * could send loads of these and cause others to bombard arbitrary IPs with
+	 * traffic.
+	 *
+	 * @param now Current time
+	 * @return True if we should respond
+	 */
+	inline bool shouldRespondToDirectPathPush(const uint64_t now)
+	{
+		Mutex::Lock _l(_lock);
+		if ((now - _lastDirectPathPushReceive) <= ZT_PUSH_DIRECT_PATHS_CUTOFF_TIME)
+			++_directPathPushCutoffCount;
+		else _directPathPushCutoffCount = 0;
+		_lastDirectPathPushReceive = now;
+		return (_directPathPushCutoffCount < ZT_PUSH_DIRECT_PATHS_CUTOFF_LIMIT);
+	}
+
 	/**
 	 * Find a common set of addresses by which two peers can link, if any
 	 *
@@ -401,26 +459,170 @@ public:
 		else return std::pair<InetAddress,InetAddress>();
 	}
 
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b) const
+	{
+		Mutex::Lock _l(_lock);
+
+		const unsigned int recSizePos = b.size();
+		b.addSize(4); // space for uint32_t field length
+
+		b.append((uint16_t)1); // version of serialized Peer data
+
+		_id.serialize(b,false);
+
+		b.append((uint64_t)_lastUsed);
+		b.append((uint64_t)_lastReceive);
+		b.append((uint64_t)_lastUnicastFrame);
+		b.append((uint64_t)_lastMulticastFrame);
+		b.append((uint64_t)_lastAnnouncedTo);
+		b.append((uint64_t)_lastPathConfirmationSent);
+		b.append((uint64_t)_lastDirectPathPushSent);
+		b.append((uint64_t)_lastDirectPathPushReceive);
+		b.append((uint64_t)_lastPathSort);
+		b.append((uint16_t)_vProto);
+		b.append((uint16_t)_vMajor);
+		b.append((uint16_t)_vMinor);
+		b.append((uint16_t)_vRevision);
+		b.append((uint32_t)_latency);
+		b.append((uint16_t)_directPathPushCutoffCount);
+
+		b.append((uint16_t)_numPaths);
+		for(unsigned int i=0;i<_numPaths;++i)
+			_paths[i].serialize(b);
+
+		b.append((uint32_t)_networkComs.size());
+		{
+			uint64_t *k = (uint64_t *)0;
+			_NetworkCom *v = (_NetworkCom *)0;
+			Hashtable<uint64_t,_NetworkCom>::Iterator i(const_cast<Peer *>(this)->_networkComs);
+			while (i.next(k,v)) {
+				b.append((uint64_t)*k);
+				b.append((uint64_t)v->ts);
+				v->com.serialize(b);
+			}
+		}
+
+		b.append((uint32_t)_lastPushedComs.size());
+		{
+			uint64_t *k = (uint64_t *)0;
+			uint64_t *v = (uint64_t *)0;
+			Hashtable<uint64_t,uint64_t>::Iterator i(const_cast<Peer *>(this)->_lastPushedComs);
+			while (i.next(k,v)) {
+				b.append((uint64_t)*k);
+				b.append((uint64_t)*v);
+			}
+		}
+
+		b.template setAt<uint32_t>(recSizePos,(uint32_t)(b.size() - (recSizePos + 4))); // set size
+	}
+
+	/**
+	 * Create a new Peer from a serialized instance
+	 *
+	 * @param myIdentity This node's identity
+	 * @param b Buffer containing serialized Peer data
+	 * @param p Pointer to current position in buffer, will be updated in place as buffer is read (value/result)
+	 * @return New instance of Peer or NULL if serialized data was corrupt or otherwise invalid (may also throw an exception via Buffer)
+	 */
+	template<unsigned int C>
+	static inline SharedPtr<Peer> deserializeNew(const Identity &myIdentity,const Buffer<C> &b,unsigned int &p)
+	{
+		const unsigned int recSize = b.template at<uint32_t>(p); p += 4;
+		if ((p + recSize) > b.size())
+			return SharedPtr<Peer>(); // size invalid
+		if (b.template at<uint16_t>(p) != 1)
+			return SharedPtr<Peer>(); // version mismatch
+		p += 2;
+
+		Identity npid;
+		p += npid.deserialize(b,p);
+		if (!npid)
+			return SharedPtr<Peer>();
+
+		SharedPtr<Peer> np(new Peer(myIdentity,npid));
+
+		np->_lastUsed = b.template at<uint64_t>(p); p += 8;
+		np->_lastReceive = b.template at<uint64_t>(p); p += 8;
+		np->_lastUnicastFrame = b.template at<uint64_t>(p); p += 8;
+		np->_lastMulticastFrame = b.template at<uint64_t>(p); p += 8;
+		np->_lastAnnouncedTo = b.template at<uint64_t>(p); p += 8;
+		np->_lastPathConfirmationSent = b.template at<uint64_t>(p); p += 8;
+		np->_lastDirectPathPushSent = b.template at<uint64_t>(p); p += 8;
+		np->_lastDirectPathPushReceive = b.template at<uint64_t>(p); p += 8;
+		np->_lastPathSort = b.template at<uint64_t>(p); p += 8;
+		np->_vProto = b.template at<uint16_t>(p); p += 2;
+		np->_vMajor = b.template at<uint16_t>(p); p += 2;
+		np->_vMinor = b.template at<uint16_t>(p); p += 2;
+		np->_vRevision = b.template at<uint16_t>(p); p += 2;
+		np->_latency = b.template at<uint32_t>(p); p += 4;
+		np->_directPathPushCutoffCount = b.template at<uint16_t>(p); p += 2;
+
+		const unsigned int numPaths = b.template at<uint16_t>(p); p += 2;
+		for(unsigned int i=0;i<numPaths;++i) {
+			if (i < ZT_MAX_PEER_NETWORK_PATHS) {
+				p += np->_paths[np->_numPaths++].deserialize(b,p);
+			} else {
+				// Skip any paths beyond max, but still read stream
+				Path foo;
+				p += foo.deserialize(b,p);
+			}
+		}
+
+		const unsigned int numNetworkComs = b.template at<uint32_t>(p); p += 4;
+		for(unsigned int i=0;i<numNetworkComs;++i) {
+			_NetworkCom &c = np->_networkComs[b.template at<uint64_t>(p)]; p += 8;
+			c.ts = b.template at<uint64_t>(p); p += 8;
+			p += c.com.deserialize(b,p);
+		}
+
+		const unsigned int numLastPushed = b.template at<uint32_t>(p); p += 4;
+		for(unsigned int i=0;i<numLastPushed;++i) {
+			const uint64_t nwid = b.template at<uint64_t>(p); p += 8;
+			const uint64_t ts = b.template at<uint64_t>(p); p += 8;
+			np->_lastPushedComs.set(nwid,ts);
+		}
+
+		return np;
+	}
+
 private:
-	void _announceMulticastGroups(const RuntimeEnvironment *RR,uint64_t now);
+	void _sortPaths(const uint64_t now);
+	Path *_getBestPath(const uint64_t now);
+	Path *_getBestPath(const uint64_t now,int inetAddressFamily);
+
+	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized
 
-	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH];
 	uint64_t _lastUsed;
 	uint64_t _lastReceive; // direct or indirect
 	uint64_t _lastUnicastFrame;
 	uint64_t _lastMulticastFrame;
 	uint64_t _lastAnnouncedTo;
 	uint64_t _lastPathConfirmationSent;
-	uint64_t _lastDirectPathPush;
+	uint64_t _lastDirectPathPushSent;
+	uint64_t _lastDirectPathPushReceive;
+	uint64_t _lastPathSort;
 	uint16_t _vProto;
 	uint16_t _vMajor;
 	uint16_t _vMinor;
 	uint16_t _vRevision;
 	Identity _id;
-	RemotePath _paths[ZT_MAX_PEER_NETWORK_PATHS];
+	Path _paths[ZT_MAX_PEER_NETWORK_PATHS];
 	unsigned int _numPaths;
 	unsigned int _latency;
+	unsigned int _directPathPushCutoffCount;
 
+	struct _NetworkCom
+	{
+		_NetworkCom() {}
+		_NetworkCom(uint64_t t,const CertificateOfMembership &c) : ts(t),com(c) {}
+		uint64_t ts;
+		CertificateOfMembership com;
+	};
+	Hashtable<uint64_t,_NetworkCom> _networkComs;
+	Hashtable<uint64_t,uint64_t> _lastPushedComs;
+
+	Mutex _lock;
 	AtomicCounter __refCount;
 };
 

node/Poly1305.cpp

@@ -7,14 +7,21 @@ Public domain.
 #include "Constants.hpp"
 #include "Poly1305.hpp"
 
+#include <stdio.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
 #ifdef __WINDOWS__
 #pragma warning(disable: 4146)
 #endif
 
 namespace ZeroTier {
 
-//////////////////////////////////////////////////////////////////////////////
-//////////////////////////////////////////////////////////////////////////////
+#if 0
+
+// "Naive" implementation, which is slower... might still want this on some older
+// or weird platforms if the later versions have issues.
 
 static inline void add(unsigned int h[17],const unsigned int c[17])
 {
@@ -113,13 +120,509 @@ static inline int crypto_onetimeauth(unsigned char *out,const unsigned char *in,
   return 0;
 }
 
+void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key)
+	throw()
+{
+	crypto_onetimeauth((unsigned char *)auth,(const unsigned char *)data,len,(const unsigned char *)key);
+}
+
+#endif
+
+namespace {
+
+typedef struct poly1305_context {
+  size_t aligner;
+  unsigned char opaque[136];
+} poly1305_context;
+
+#if (defined(_MSC_VER) || defined(__GNUC__)) && (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__))
+
+//////////////////////////////////////////////////////////////////////////////
+// 128-bit implementation for MSC and GCC from Poly1305-donna
+
+#if defined(_MSC_VER)
+  #include <intrin.h>
+
+  typedef struct uint128_t {
+    unsigned long long lo;
+    unsigned long long hi;
+  } uint128_t;
+
+  #define MUL(out, x, y) out.lo = _umul128((x), (y), &out.hi)
+  #define ADD(out, in) { unsigned long long t = out.lo; out.lo += in.lo; out.hi += (out.lo < t) + in.hi; }
+  #define ADDLO(out, in) { unsigned long long t = out.lo; out.lo += in; out.hi += (out.lo < t); }
+  #define SHR(in, shift) (__shiftright128(in.lo, in.hi, (shift)))
+  #define LO(in) (in.lo)
+
+//  #define POLY1305_NOINLINE __declspec(noinline)
+#elif defined(__GNUC__)
+  #if defined(__SIZEOF_INT128__)
+    typedef unsigned __int128 uint128_t;
+  #else
+    typedef unsigned uint128_t __attribute__((mode(TI)));
+  #endif
+
+  #define MUL(out, x, y) out = ((uint128_t)x * y)
+  #define ADD(out, in) out += in
+  #define ADDLO(out, in) out += in
+  #define SHR(in, shift) (unsigned long long)(in >> (shift))
+  #define LO(in) (unsigned long long)(in)
+
+//  #define POLY1305_NOINLINE __attribute__((noinline))
+#endif
+
+#define poly1305_block_size 16
+
+/* 17 + sizeof(size_t) + 8*sizeof(unsigned long long) */
+typedef struct poly1305_state_internal_t {
+  unsigned long long r[3];
+  unsigned long long h[3];
+  unsigned long long pad[2];
+  size_t leftover;
+  unsigned char buffer[poly1305_block_size];
+  unsigned char final;
+} poly1305_state_internal_t;
+
+/* interpret eight 8 bit unsigned integers as a 64 bit unsigned integer in little endian */
+static inline unsigned long long
+U8TO64(const unsigned char *p) {
+  return
+    (((unsigned long long)(p[0] & 0xff)      ) |
+     ((unsigned long long)(p[1] & 0xff) <<  8) |
+     ((unsigned long long)(p[2] & 0xff) << 16) |
+     ((unsigned long long)(p[3] & 0xff) << 24) |
+     ((unsigned long long)(p[4] & 0xff) << 32) |
+     ((unsigned long long)(p[5] & 0xff) << 40) |
+     ((unsigned long long)(p[6] & 0xff) << 48) |
+     ((unsigned long long)(p[7] & 0xff) << 56));
+}
+
+/* store a 64 bit unsigned integer as eight 8 bit unsigned integers in little endian */
+static inline void
+U64TO8(unsigned char *p, unsigned long long v) {
+  p[0] = (v      ) & 0xff;
+  p[1] = (v >>  8) & 0xff;
+  p[2] = (v >> 16) & 0xff;
+  p[3] = (v >> 24) & 0xff;
+  p[4] = (v >> 32) & 0xff;
+  p[5] = (v >> 40) & 0xff;
+  p[6] = (v >> 48) & 0xff;
+  p[7] = (v >> 56) & 0xff;
+}
+
+static inline void
+poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  unsigned long long t0,t1;
+
+  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
+  t0 = U8TO64(&key[0]);
+  t1 = U8TO64(&key[8]);
+
+  st->r[0] = ( t0                    ) & 0xffc0fffffff;
+  st->r[1] = ((t0 >> 44) | (t1 << 20)) & 0xfffffc0ffff;
+  st->r[2] = ((t1 >> 24)             ) & 0x00ffffffc0f;
+
+  /* h = 0 */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+
+  /* save pad for later */
+  st->pad[0] = U8TO64(&key[16]);
+  st->pad[1] = U8TO64(&key[24]);
+
+  st->leftover = 0;
+  st->final = 0;
+}
+
+static inline void
+poly1305_blocks(poly1305_state_internal_t *st, const unsigned char *m, size_t bytes) {
+  const unsigned long long hibit = (st->final) ? 0 : ((unsigned long long)1 << 40); /* 1 << 128 */
+  unsigned long long r0,r1,r2;
+  unsigned long long s1,s2;
+  unsigned long long h0,h1,h2;
+  unsigned long long c;
+  uint128_t d0,d1,d2,d;
+
+  r0 = st->r[0];
+  r1 = st->r[1];
+  r2 = st->r[2];
+
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+
+  s1 = r1 * (5 << 2);
+  s2 = r2 * (5 << 2);
+
+  while (bytes >= poly1305_block_size) {
+    unsigned long long t0,t1;
+
+    /* h += m[i] */
+    t0 = U8TO64(&m[0]);
+    t1 = U8TO64(&m[8]);
+
+    h0 += (( t0                    ) & 0xfffffffffff);
+    h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff);
+    h2 += (((t1 >> 24)             ) & 0x3ffffffffff) | hibit;
+
+    /* h *= r */
+    MUL(d0, h0, r0); MUL(d, h1, s2); ADD(d0, d); MUL(d, h2, s1); ADD(d0, d);
+    MUL(d1, h0, r1); MUL(d, h1, r0); ADD(d1, d); MUL(d, h2, s2); ADD(d1, d);
+    MUL(d2, h0, r2); MUL(d, h1, r1); ADD(d2, d); MUL(d, h2, r0); ADD(d2, d);
+
+    /* (partial) h %= p */
+                  c = SHR(d0, 44); h0 = LO(d0) & 0xfffffffffff;
+    ADDLO(d1, c); c = SHR(d1, 44); h1 = LO(d1) & 0xfffffffffff;
+    ADDLO(d2, c); c = SHR(d2, 42); h2 = LO(d2) & 0x3ffffffffff;
+    h0  += c * 5; c = (h0 >> 44);  h0 =    h0  & 0xfffffffffff;
+    h1  += c;
+
+    m += poly1305_block_size;
+    bytes -= poly1305_block_size;
+  }
+
+  st->h[0] = h0;
+  st->h[1] = h1;
+  st->h[2] = h2;
+}
+
+static inline void
+poly1305_finish(poly1305_context *ctx, unsigned char mac[16]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  unsigned long long h0,h1,h2,c;
+  unsigned long long g0,g1,g2;
+  unsigned long long t0,t1;
+
+  /* process the remaining block */
+  if (st->leftover) {
+    size_t i = st->leftover;
+    st->buffer[i] = 1;
+    for (i = i + 1; i < poly1305_block_size; i++)
+      st->buffer[i] = 0;
+    st->final = 1;
+    poly1305_blocks(st, st->buffer, poly1305_block_size);
+  }
+
+  /* fully carry h */
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+
+               c = (h1 >> 44); h1 &= 0xfffffffffff;
+  h2 += c;     c = (h2 >> 42); h2 &= 0x3ffffffffff;
+  h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff;
+  h1 += c;     c = (h1 >> 44); h1 &= 0xfffffffffff;
+  h2 += c;     c = (h2 >> 42); h2 &= 0x3ffffffffff;
+  h0 += c * 5; c = (h0 >> 44); h0 &= 0xfffffffffff;
+  h1 += c;
+
+  /* compute h + -p */
+  g0 = h0 + 5; c = (g0 >> 44); g0 &= 0xfffffffffff;
+  g1 = h1 + c; c = (g1 >> 44); g1 &= 0xfffffffffff;
+  g2 = h2 + c - ((unsigned long long)1 << 42);
+
+  /* select h if h < p, or h + -p if h >= p */
+  c = (g2 >> ((sizeof(unsigned long long) * 8) - 1)) - 1;
+  g0 &= c;
+  g1 &= c;
+  g2 &= c;
+  c = ~c;
+  h0 = (h0 & c) | g0;
+  h1 = (h1 & c) | g1;
+  h2 = (h2 & c) | g2;
+
+  /* h = (h + pad) */
+  t0 = st->pad[0];
+  t1 = st->pad[1];
+
+  h0 += (( t0                    ) & 0xfffffffffff)    ; c = (h0 >> 44); h0 &= 0xfffffffffff;
+  h1 += (((t0 >> 44) | (t1 << 20)) & 0xfffffffffff) + c; c = (h1 >> 44); h1 &= 0xfffffffffff;
+  h2 += (((t1 >> 24)             ) & 0x3ffffffffff) + c;                 h2 &= 0x3ffffffffff;
+
+  /* mac = h % (2^128) */
+  h0 = ((h0      ) | (h1 << 44));
+  h1 = ((h1 >> 20) | (h2 << 24));
+
+  U64TO8(&mac[0], h0);
+  U64TO8(&mac[8], h1);
+
+  /* zero out the state */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+  st->r[0] = 0;
+  st->r[1] = 0;
+  st->r[2] = 0;
+  st->pad[0] = 0;
+  st->pad[1] = 0;
+}
+
 //////////////////////////////////////////////////////////////////////////////
+
+#else
+
 //////////////////////////////////////////////////////////////////////////////
+// More portable 64-bit implementation
+
+#define poly1305_block_size 16
+
+/* 17 + sizeof(size_t) + 14*sizeof(unsigned long) */
+typedef struct poly1305_state_internal_t {
+  unsigned long r[5];
+  unsigned long h[5];
+  unsigned long pad[4];
+  size_t leftover;
+  unsigned char buffer[poly1305_block_size];
+  unsigned char final;
+} poly1305_state_internal_t;
+
+/* interpret four 8 bit unsigned integers as a 32 bit unsigned integer in little endian */
+static unsigned long
+U8TO32(const unsigned char *p) {
+  return
+    (((unsigned long)(p[0] & 0xff)      ) |
+       ((unsigned long)(p[1] & 0xff) <<  8) |
+         ((unsigned long)(p[2] & 0xff) << 16) |
+         ((unsigned long)(p[3] & 0xff) << 24));
+}
+
+/* store a 32 bit unsigned integer as four 8 bit unsigned integers in little endian */
+static void
+U32TO8(unsigned char *p, unsigned long v) {
+  p[0] = (v      ) & 0xff;
+  p[1] = (v >>  8) & 0xff;
+  p[2] = (v >> 16) & 0xff;
+  p[3] = (v >> 24) & 0xff;
+}
+
+static inline void
+poly1305_init(poly1305_context *ctx, const unsigned char key[32]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+
+  /* r &= 0xffffffc0ffffffc0ffffffc0fffffff */
+  st->r[0] = (U8TO32(&key[ 0])     ) & 0x3ffffff;
+  st->r[1] = (U8TO32(&key[ 3]) >> 2) & 0x3ffff03;
+  st->r[2] = (U8TO32(&key[ 6]) >> 4) & 0x3ffc0ff;
+  st->r[3] = (U8TO32(&key[ 9]) >> 6) & 0x3f03fff;
+  st->r[4] = (U8TO32(&key[12]) >> 8) & 0x00fffff;
+
+  /* h = 0 */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+  st->h[3] = 0;
+  st->h[4] = 0;
+
+  /* save pad for later */
+  st->pad[0] = U8TO32(&key[16]);
+  st->pad[1] = U8TO32(&key[20]);
+  st->pad[2] = U8TO32(&key[24]);
+  st->pad[3] = U8TO32(&key[28]);
+
+  st->leftover = 0;
+  st->final = 0;
+}
+
+static inline void
+poly1305_blocks(poly1305_state_internal_t *st, const unsigned char *m, size_t bytes) {
+  const unsigned long hibit = (st->final) ? 0 : (1 << 24); /* 1 << 128 */
+  unsigned long r0,r1,r2,r3,r4;
+  unsigned long s1,s2,s3,s4;
+  unsigned long h0,h1,h2,h3,h4;
+  unsigned long long d0,d1,d2,d3,d4;
+  unsigned long c;
+
+  r0 = st->r[0];
+  r1 = st->r[1];
+  r2 = st->r[2];
+  r3 = st->r[3];
+  r4 = st->r[4];
+
+  s1 = r1 * 5;
+  s2 = r2 * 5;
+  s3 = r3 * 5;
+  s4 = r4 * 5;
+
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+  h3 = st->h[3];
+  h4 = st->h[4];
+
+  while (bytes >= poly1305_block_size) {
+    /* h += m[i] */
+    h0 += (U8TO32(m+ 0)     ) & 0x3ffffff;
+    h1 += (U8TO32(m+ 3) >> 2) & 0x3ffffff;
+    h2 += (U8TO32(m+ 6) >> 4) & 0x3ffffff;
+    h3 += (U8TO32(m+ 9) >> 6) & 0x3ffffff;
+    h4 += (U8TO32(m+12) >> 8) | hibit;
+
+    /* h *= r */
+    d0 = ((unsigned long long)h0 * r0) + ((unsigned long long)h1 * s4) + ((unsigned long long)h2 * s3) + ((unsigned long long)h3 * s2) + ((unsigned long long)h4 * s1);
+    d1 = ((unsigned long long)h0 * r1) + ((unsigned long long)h1 * r0) + ((unsigned long long)h2 * s4) + ((unsigned long long)h3 * s3) + ((unsigned long long)h4 * s2);
+    d2 = ((unsigned long long)h0 * r2) + ((unsigned long long)h1 * r1) + ((unsigned long long)h2 * r0) + ((unsigned long long)h3 * s4) + ((unsigned long long)h4 * s3);
+    d3 = ((unsigned long long)h0 * r3) + ((unsigned long long)h1 * r2) + ((unsigned long long)h2 * r1) + ((unsigned long long)h3 * r0) + ((unsigned long long)h4 * s4);
+    d4 = ((unsigned long long)h0 * r4) + ((unsigned long long)h1 * r3) + ((unsigned long long)h2 * r2) + ((unsigned long long)h3 * r1) + ((unsigned long long)h4 * r0);
+
+    /* (partial) h %= p */
+                  c = (unsigned long)(d0 >> 26); h0 = (unsigned long)d0 & 0x3ffffff;
+    d1 += c;      c = (unsigned long)(d1 >> 26); h1 = (unsigned long)d1 & 0x3ffffff;
+    d2 += c;      c = (unsigned long)(d2 >> 26); h2 = (unsigned long)d2 & 0x3ffffff;
+    d3 += c;      c = (unsigned long)(d3 >> 26); h3 = (unsigned long)d3 & 0x3ffffff;
+    d4 += c;      c = (unsigned long)(d4 >> 26); h4 = (unsigned long)d4 & 0x3ffffff;
+    h0 += c * 5;  c =                (h0 >> 26); h0 =                h0 & 0x3ffffff;
+    h1 += c;
+
+    m += poly1305_block_size;
+    bytes -= poly1305_block_size;
+  }
+
+  st->h[0] = h0;
+  st->h[1] = h1;
+  st->h[2] = h2;
+  st->h[3] = h3;
+  st->h[4] = h4;
+}
+
+static inline void
+poly1305_finish(poly1305_context *ctx, unsigned char mac[16]) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  unsigned long h0,h1,h2,h3,h4,c;
+  unsigned long g0,g1,g2,g3,g4;
+  unsigned long long f;
+  unsigned long mask;
+
+  /* process the remaining block */
+  if (st->leftover) {
+    size_t i = st->leftover;
+    st->buffer[i++] = 1;
+    for (; i < poly1305_block_size; i++)
+      st->buffer[i] = 0;
+    st->final = 1;
+    poly1305_blocks(st, st->buffer, poly1305_block_size);
+  }
+
+  /* fully carry h */
+  h0 = st->h[0];
+  h1 = st->h[1];
+  h2 = st->h[2];
+  h3 = st->h[3];
+  h4 = st->h[4];
+
+               c = h1 >> 26; h1 = h1 & 0x3ffffff;
+  h2 +=     c; c = h2 >> 26; h2 = h2 & 0x3ffffff;
+  h3 +=     c; c = h3 >> 26; h3 = h3 & 0x3ffffff;
+  h4 +=     c; c = h4 >> 26; h4 = h4 & 0x3ffffff;
+  h0 += c * 5; c = h0 >> 26; h0 = h0 & 0x3ffffff;
+  h1 +=     c;
+
+  /* compute h + -p */
+  g0 = h0 + 5; c = g0 >> 26; g0 &= 0x3ffffff;
+  g1 = h1 + c; c = g1 >> 26; g1 &= 0x3ffffff;
+  g2 = h2 + c; c = g2 >> 26; g2 &= 0x3ffffff;
+  g3 = h3 + c; c = g3 >> 26; g3 &= 0x3ffffff;
+  g4 = h4 + c - (1 << 26);
+
+  /* select h if h < p, or h + -p if h >= p */
+  mask = (g4 >> ((sizeof(unsigned long) * 8) - 1)) - 1;
+  g0 &= mask;
+  g1 &= mask;
+  g2 &= mask;
+  g3 &= mask;
+  g4 &= mask;
+  mask = ~mask;
+  h0 = (h0 & mask) | g0;
+  h1 = (h1 & mask) | g1;
+  h2 = (h2 & mask) | g2;
+  h3 = (h3 & mask) | g3;
+  h4 = (h4 & mask) | g4;
+
+  /* h = h % (2^128) */
+  h0 = ((h0      ) | (h1 << 26)) & 0xffffffff;
+  h1 = ((h1 >>  6) | (h2 << 20)) & 0xffffffff;
+  h2 = ((h2 >> 12) | (h3 << 14)) & 0xffffffff;
+  h3 = ((h3 >> 18) | (h4 <<  8)) & 0xffffffff;
+
+  /* mac = (h + pad) % (2^128) */
+  f = (unsigned long long)h0 + st->pad[0]            ; h0 = (unsigned long)f;
+  f = (unsigned long long)h1 + st->pad[1] + (f >> 32); h1 = (unsigned long)f;
+  f = (unsigned long long)h2 + st->pad[2] + (f >> 32); h2 = (unsigned long)f;
+  f = (unsigned long long)h3 + st->pad[3] + (f >> 32); h3 = (unsigned long)f;
+
+  U32TO8(mac +  0, h0);
+  U32TO8(mac +  4, h1);
+  U32TO8(mac +  8, h2);
+  U32TO8(mac + 12, h3);
+
+  /* zero out the state */
+  st->h[0] = 0;
+  st->h[1] = 0;
+  st->h[2] = 0;
+  st->h[3] = 0;
+  st->h[4] = 0;
+  st->r[0] = 0;
+  st->r[1] = 0;
+  st->r[2] = 0;
+  st->r[3] = 0;
+  st->r[4] = 0;
+  st->pad[0] = 0;
+  st->pad[1] = 0;
+  st->pad[2] = 0;
+  st->pad[3] = 0;
+}
+
+//////////////////////////////////////////////////////////////////////////////
+
+#endif // MSC/GCC or not
+
+static inline void
+poly1305_update(poly1305_context *ctx, const unsigned char *m, size_t bytes) {
+  poly1305_state_internal_t *st = (poly1305_state_internal_t *)ctx;
+  size_t i;
+
+  /* handle leftover */
+  if (st->leftover) {
+    size_t want = (poly1305_block_size - st->leftover);
+    if (want > bytes)
+      want = bytes;
+    for (i = 0; i < want; i++)
+      st->buffer[st->leftover + i] = m[i];
+    bytes -= want;
+    m += want;
+    st->leftover += want;
+    if (st->leftover < poly1305_block_size)
+      return;
+    poly1305_blocks(st, st->buffer, poly1305_block_size);
+    st->leftover = 0;
+  }
+
+  /* process full blocks */
+  if (bytes >= poly1305_block_size) {
+    size_t want = (bytes & ~(poly1305_block_size - 1));
+    poly1305_blocks(st, m, want);
+    m += want;
+    bytes -= want;
+  }
+
+  /* store leftover */
+  if (bytes) {
+    for (i = 0; i < bytes; i++)
+      st->buffer[st->leftover + i] = m[i];
+    st->leftover += bytes;
+  }
+}
+
+} // anonymous namespace
 
 void Poly1305::compute(void *auth,const void *data,unsigned int len,const void *key)
-	throw()
+  throw()
 {
-	crypto_onetimeauth((unsigned char *)auth,(const unsigned char *)data,len,(const unsigned char *)key);
+  poly1305_context ctx;
+  poly1305_init(&ctx,reinterpret_cast<const unsigned char *>(key));
+  poly1305_update(&ctx,reinterpret_cast<const unsigned char *>(data),(size_t)len);
+  poly1305_finish(&ctx,reinterpret_cast<unsigned char *>(auth));
 }
 
 } // namespace ZeroTier

node/RemotePath.hpp

@@ -1,147 +0,0 @@
-/*
- * ZeroTier One - Network Virtualization Everywhere
- * Copyright (C) 2011-2015  ZeroTier, Inc.
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU General Public License for more details.
- *
- * You should have received a copy of the GNU General Public License
- * along with this program.  If not, see <http://www.gnu.org/licenses/>.
- *
- * --
- *
- * ZeroTier may be used and distributed under the terms of the GPLv3, which
- * are available at: http://www.gnu.org/licenses/gpl-3.0.html
- *
- * If you would like to embed ZeroTier into a commercial application or
- * redistribute it in a modified binary form, please contact ZeroTier Networks
- * LLC. Start here: http://www.zerotier.com/
- */
-
-#ifndef ZT_REMOTEPATH_HPP
-#define ZT_REMOTEPATH_HPP
-
-#include <stdint.h>
-#include <string.h>
-
-#include <stdexcept>
-#include <algorithm>
-
-#include "Path.hpp"
-#include "Node.hpp"
-#include "AntiRecursion.hpp"
-#include "RuntimeEnvironment.hpp"
-
-namespace ZeroTier {
-
-/**
- * Path to a remote peer
- *
- * This extends Path to include status information about path activity.
- */
-class RemotePath : public Path
-{
-public:
-	RemotePath() :
-		Path(),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(),
-		_fixed(false) {}
-
-	RemotePath(const InetAddress &localAddress,const InetAddress &addr,bool fixed) :
-		Path(addr,0,TRUST_NORMAL),
-		_lastSend(0),
-		_lastReceived(0),
-		_localAddress(localAddress),
-		_fixed(fixed) {}
-
-	inline const InetAddress &localAddress() const throw() { return _localAddress; }
-
-	inline uint64_t lastSend() const throw() { return _lastSend; }
-	inline uint64_t lastReceived() const throw() { return _lastReceived; }
-
-	/**
-	 * @return Is this a fixed path?
-	 */
-	inline bool fixed() const throw() { return _fixed; }
-
-	/**
-	 * @param f New value of fixed flag
-	 */
-	inline void setFixed(const bool f)
-		throw()
-	{
-		_fixed = f;
-	}
-
-	/**
-	 * Called when a packet is sent to this remote path
-	 *
-	 * This is called automatically by RemotePath::send().
-	 *
-	 * @param t Time of send
-	 */
-	inline void sent(uint64_t t)
-		throw()
-	{
-		_lastSend = t;
-	}
-
-	/**
-	 * Called when a packet is received from this remote path
-	 *
-	 * @param t Time of receive
-	 */
-	inline void received(uint64_t t)
-		throw()
-	{
-		_lastReceived = t;
-	}
-
-	/**
-	 * @param now Current time
-	 * @return True if this path is fixed or has received data in last ACTIVITY_TIMEOUT ms
-	 */
-	inline bool active(uint64_t now) const
-		throw()
-	{
-		return ( (_fixed) || ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT) );
-	}
-
-	/**
-	 * Send a packet via this path
-	 *
-	 * @param RR Runtime environment
-	 * @param data Packet data
-	 * @param len Packet length
-	 * @param now Current time
-	 * @return True if transport reported success
-	 */
-	inline bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
-	{
-		if (RR->node->putPacket(_localAddress,address(),data,len)) {
-			sent(now);
-			RR->antiRec->logOutgoingZT(data,len);
-			return true;
-		}
-		return false;
-	}
-
-private:
-	uint64_t _lastSend;
-	uint64_t _lastReceived;
-	InetAddress _localAddress;
-	bool _fixed;
-};
-
-} // namespace ZeroTier
-
-#endif

node/RuntimeEnvironment.hpp

@@ -43,6 +43,7 @@ class Multicaster;
 class AntiRecursion;
 class NetworkController;
 class SelfAwareness;
+class Cluster;
 
 /**
  * Holds global state for an instance of ZeroTier::Node
@@ -51,14 +52,17 @@ class RuntimeEnvironment
 {
 public:
 	RuntimeEnvironment(Node *n) :
-		node(n),
-		identity(),
-		localNetworkController((NetworkController *)0),
-		sw((Switch *)0),
-		mc((Multicaster *)0),
-		antiRec((AntiRecursion *)0),
-		topology((Topology *)0),
-		sa((SelfAwareness *)0)
+		node(n)
+		,identity()
+		,localNetworkController((NetworkController *)0)
+		,sw((Switch *)0)
+		,mc((Multicaster *)0)
+		,antiRec((AntiRecursion *)0)
+		,topology((Topology *)0)
+		,sa((SelfAwareness *)0)
+#ifdef ZT_ENABLE_CLUSTER
+		,cluster((Cluster *)0)
+#endif
 	{
 	}
 
@@ -86,6 +90,10 @@ public:
 	AntiRecursion *antiRec;
 	Topology *topology;
 	SelfAwareness *sa;
+
+#ifdef ZT_ENABLE_CLUSTER
+	Cluster *cluster;
+#endif
 };
 
 } // namespace ZeroTier

node/Salsa20.cpp

@@ -66,7 +66,7 @@ static const _s20sseconsts _S20SSECONSTANTS;
 
 namespace ZeroTier {
 
-void Salsa20::init(const void *key,unsigned int kbits,const void *iv,unsigned int rounds)
+void Salsa20::init(const void *key,unsigned int kbits,const void *iv)
 	throw()
 {
 #ifdef ZT_SALSA20_SSE
@@ -121,11 +121,9 @@ void Salsa20::init(const void *key,unsigned int kbits,const void *iv,unsigned in
 	_state.i[15] = U8TO32_LITTLE(constants + 12);
 	_state.i[0] = U8TO32_LITTLE(constants + 0);
 #endif
-
-	_roundsDiv2 = rounds / 2;
 }
 
-void Salsa20::encrypt(const void *in,void *out,unsigned int bytes)
+void Salsa20::encrypt12(const void *in,void *out,unsigned int bytes)
 	throw()
 {
 	uint8_t tmp[64];
@@ -175,68 +173,169 @@ void Salsa20::encrypt(const void *in,void *out,unsigned int bytes)
 		__m128i X1 = _mm_loadu_si128((const __m128i *)&(_state.v[1]));
 		__m128i X2 = _mm_loadu_si128((const __m128i *)&(_state.v[2]));
 		__m128i X3 = _mm_loadu_si128((const __m128i *)&(_state.v[3]));
+		__m128i T;
 		__m128i X0s = X0;
 		__m128i X1s = X1;
 		__m128i X2s = X2;
 		__m128i X3s = X3;
 
-		for (i=0;i<_roundsDiv2;++i) {
-			__m128i T = _mm_add_epi32(X0, X3);
-			X1 = _mm_xor_si128(X1, _mm_slli_epi32(T, 7));
-			X1 = _mm_xor_si128(X1, _mm_srli_epi32(T, 25));
-			T = _mm_add_epi32(X1, X0);
-			X2 = _mm_xor_si128(X2, _mm_slli_epi32(T, 9));
-			X2 = _mm_xor_si128(X2, _mm_srli_epi32(T, 23));
-			T = _mm_add_epi32(X2, X1);
-			X3 = _mm_xor_si128(X3, _mm_slli_epi32(T, 13));
-			X3 = _mm_xor_si128(X3, _mm_srli_epi32(T, 19));
-			T = _mm_add_epi32(X3, X2);
-			X0 = _mm_xor_si128(X0, _mm_slli_epi32(T, 18));
-			X0 = _mm_xor_si128(X0, _mm_srli_epi32(T, 14));
-
-			X1 = _mm_shuffle_epi32(X1, 0x93);
-			X2 = _mm_shuffle_epi32(X2, 0x4E);
-			X3 = _mm_shuffle_epi32(X3, 0x39);
-
-			T = _mm_add_epi32(X0, X1);
-			X3 = _mm_xor_si128(X3, _mm_slli_epi32(T, 7));
-			X3 = _mm_xor_si128(X3, _mm_srli_epi32(T, 25));
-			T = _mm_add_epi32(X3, X0);
-			X2 = _mm_xor_si128(X2, _mm_slli_epi32(T, 9));
-			X2 = _mm_xor_si128(X2, _mm_srli_epi32(T, 23));
-			T = _mm_add_epi32(X2, X3);
-			X1 = _mm_xor_si128(X1, _mm_slli_epi32(T, 13));
-			X1 = _mm_xor_si128(X1, _mm_srli_epi32(T, 19));
-			T = _mm_add_epi32(X1, X2);
-			X0 = _mm_xor_si128(X0, _mm_slli_epi32(T, 18));
-			X0 = _mm_xor_si128(X0, _mm_srli_epi32(T, 14));
-
-			X1 = _mm_shuffle_epi32(X1, 0x39);
-			X2 = _mm_shuffle_epi32(X2, 0x4E);
-			X3 = _mm_shuffle_epi32(X3, 0x93);
-		}
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
 
 		X0 = _mm_add_epi32(X0s,X0);
 		X1 = _mm_add_epi32(X1s,X1);
 		X2 = _mm_add_epi32(X2s,X2);
 		X3 = _mm_add_epi32(X3s,X3);
 
-		{
-			__m128i k02 = _mm_or_si128(_mm_slli_epi64(X0, 32), _mm_srli_epi64(X3, 32));
-			k02 = _mm_shuffle_epi32(k02, _MM_SHUFFLE(0, 1, 2, 3));
-			__m128i k13 = _mm_or_si128(_mm_slli_epi64(X1, 32), _mm_srli_epi64(X0, 32));
-			k13 = _mm_shuffle_epi32(k13, _MM_SHUFFLE(0, 1, 2, 3));
-			__m128i k20 = _mm_or_si128(_mm_and_si128(X2, _S20SSECONSTANTS.maskLo32), _mm_and_si128(X1, _S20SSECONSTANTS.maskHi32));
-			__m128i k31 = _mm_or_si128(_mm_and_si128(X3, _S20SSECONSTANTS.maskLo32), _mm_and_si128(X2, _S20SSECONSTANTS.maskHi32));
-
-			const float *const mv = (const float *)m;
-			float *const cv = (float *)c;
-
-			_mm_storeu_ps(cv,_mm_castsi128_ps(_mm_xor_si128(_mm_unpackhi_epi64(k02,k20),_mm_castps_si128(_mm_loadu_ps(mv)))));
-			_mm_storeu_ps(cv + 4,_mm_castsi128_ps(_mm_xor_si128(_mm_unpackhi_epi64(k13,k31),_mm_castps_si128(_mm_loadu_ps(mv + 4)))));
-			_mm_storeu_ps(cv + 8,_mm_castsi128_ps(_mm_xor_si128(_mm_unpacklo_epi64(k20,k02),_mm_castps_si128(_mm_loadu_ps(mv + 8)))));
-			_mm_storeu_ps(cv + 12,_mm_castsi128_ps(_mm_xor_si128(_mm_unpacklo_epi64(k31,k13),_mm_castps_si128(_mm_loadu_ps(mv + 12)))));
-		}
+		__m128i k02 = _mm_shuffle_epi32(_mm_or_si128(_mm_slli_epi64(X0, 32), _mm_srli_epi64(X3, 32)), _MM_SHUFFLE(0, 1, 2, 3));
+		__m128i k13 = _mm_shuffle_epi32(_mm_or_si128(_mm_slli_epi64(X1, 32), _mm_srli_epi64(X0, 32)), _MM_SHUFFLE(0, 1, 2, 3));
+		__m128i k20 = _mm_or_si128(_mm_and_si128(X2, _S20SSECONSTANTS.maskLo32), _mm_and_si128(X1, _S20SSECONSTANTS.maskHi32));
+		__m128i k31 = _mm_or_si128(_mm_and_si128(X3, _S20SSECONSTANTS.maskLo32), _mm_and_si128(X2, _S20SSECONSTANTS.maskHi32));
+		_mm_storeu_ps(reinterpret_cast<float *>(c),_mm_castsi128_ps(_mm_xor_si128(_mm_unpackhi_epi64(k02,k20),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m))))));
+		_mm_storeu_ps(reinterpret_cast<float *>(c) + 4,_mm_castsi128_ps(_mm_xor_si128(_mm_unpackhi_epi64(k13,k31),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m) + 4)))));
+		_mm_storeu_ps(reinterpret_cast<float *>(c) + 8,_mm_castsi128_ps(_mm_xor_si128(_mm_unpacklo_epi64(k20,k02),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m) + 8)))));
+		_mm_storeu_ps(reinterpret_cast<float *>(c) + 12,_mm_castsi128_ps(_mm_xor_si128(_mm_unpacklo_epi64(k31,k13),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m) + 12)))));
 
 		if (!(++_state.i[8])) {
 			++_state.i[5]; // state reordered for SSE
@@ -260,41 +359,942 @@ void Salsa20::encrypt(const void *in,void *out,unsigned int bytes)
 		x14 = j14;
 		x15 = j15;
 
-		for(i=0;i<_roundsDiv2;++i) {
-			 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
-			 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
-			x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
-			 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
-			 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
-			x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
-			 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
-			 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
-			x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
-			 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
-			 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
-			x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
-			 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
-			 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
-			x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
-			x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
-			 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
-			 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
-			 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
-			 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
-			 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
-			 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
-			 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
-			 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
-			x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
-			 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
-			 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
-			x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
-			x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
-			x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
-			x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
-			x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		x0 = PLUS(x0,j0);
+		x1 = PLUS(x1,j1);
+		x2 = PLUS(x2,j2);
+		x3 = PLUS(x3,j3);
+		x4 = PLUS(x4,j4);
+		x5 = PLUS(x5,j5);
+		x6 = PLUS(x6,j6);
+		x7 = PLUS(x7,j7);
+		x8 = PLUS(x8,j8);
+		x9 = PLUS(x9,j9);
+		x10 = PLUS(x10,j10);
+		x11 = PLUS(x11,j11);
+		x12 = PLUS(x12,j12);
+		x13 = PLUS(x13,j13);
+		x14 = PLUS(x14,j14);
+		x15 = PLUS(x15,j15);
+
+		U32TO8_LITTLE(c + 0,XOR(x0,U8TO32_LITTLE(m + 0)));
+		U32TO8_LITTLE(c + 4,XOR(x1,U8TO32_LITTLE(m + 4)));
+		U32TO8_LITTLE(c + 8,XOR(x2,U8TO32_LITTLE(m + 8)));
+		U32TO8_LITTLE(c + 12,XOR(x3,U8TO32_LITTLE(m + 12)));
+		U32TO8_LITTLE(c + 16,XOR(x4,U8TO32_LITTLE(m + 16)));
+		U32TO8_LITTLE(c + 20,XOR(x5,U8TO32_LITTLE(m + 20)));
+		U32TO8_LITTLE(c + 24,XOR(x6,U8TO32_LITTLE(m + 24)));
+		U32TO8_LITTLE(c + 28,XOR(x7,U8TO32_LITTLE(m + 28)));
+		U32TO8_LITTLE(c + 32,XOR(x8,U8TO32_LITTLE(m + 32)));
+		U32TO8_LITTLE(c + 36,XOR(x9,U8TO32_LITTLE(m + 36)));
+		U32TO8_LITTLE(c + 40,XOR(x10,U8TO32_LITTLE(m + 40)));
+		U32TO8_LITTLE(c + 44,XOR(x11,U8TO32_LITTLE(m + 44)));
+		U32TO8_LITTLE(c + 48,XOR(x12,U8TO32_LITTLE(m + 48)));
+		U32TO8_LITTLE(c + 52,XOR(x13,U8TO32_LITTLE(m + 52)));
+		U32TO8_LITTLE(c + 56,XOR(x14,U8TO32_LITTLE(m + 56)));
+		U32TO8_LITTLE(c + 60,XOR(x15,U8TO32_LITTLE(m + 60)));
+
+		if (!(++j8)) {
+			++j9;
+			/* stopping at 2^70 bytes per nonce is user's responsibility */
+		}
+#endif
+
+		if (bytes <= 64) {
+			if (bytes < 64) {
+				for (i = 0;i < bytes;++i)
+					ctarget[i] = c[i];
+			}
+
+#ifndef ZT_SALSA20_SSE
+			_state.i[8] = j8;
+			_state.i[9] = j9;
+#endif
+
+			return;
+		}
+
+		bytes -= 64;
+		c += 64;
+		m += 64;
+	}
+}
+
+void Salsa20::encrypt20(const void *in,void *out,unsigned int bytes)
+	throw()
+{
+	uint8_t tmp[64];
+	const uint8_t *m = (const uint8_t *)in;
+	uint8_t *c = (uint8_t *)out;
+	uint8_t *ctarget = c;
+	unsigned int i;
+
+#ifndef ZT_SALSA20_SSE
+	uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15;
+	uint32_t j0, j1, j2, j3, j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15;
+#endif
+
+	if (!bytes)
+		return;
+
+#ifndef ZT_SALSA20_SSE
+	j0 = _state.i[0];
+	j1 = _state.i[1];
+	j2 = _state.i[2];
+	j3 = _state.i[3];
+	j4 = _state.i[4];
+	j5 = _state.i[5];
+	j6 = _state.i[6];
+	j7 = _state.i[7];
+	j8 = _state.i[8];
+	j9 = _state.i[9];
+	j10 = _state.i[10];
+	j11 = _state.i[11];
+	j12 = _state.i[12];
+	j13 = _state.i[13];
+	j14 = _state.i[14];
+	j15 = _state.i[15];
+#endif
+
+	for (;;) {
+		if (bytes < 64) {
+			for (i = 0;i < bytes;++i)
+				tmp[i] = m[i];
+			m = tmp;
+			ctarget = c;
+			c = tmp;
 		}
 
+#ifdef ZT_SALSA20_SSE
+		__m128i X0 = _mm_loadu_si128((const __m128i *)&(_state.v[0]));
+		__m128i X1 = _mm_loadu_si128((const __m128i *)&(_state.v[1]));
+		__m128i X2 = _mm_loadu_si128((const __m128i *)&(_state.v[2]));
+		__m128i X3 = _mm_loadu_si128((const __m128i *)&(_state.v[3]));
+		__m128i T;
+		__m128i X0s = X0;
+		__m128i X1s = X1;
+		__m128i X2s = X2;
+		__m128i X3s = X3;
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		// 2X round -------------------------------------------------------------
+		T = _mm_add_epi32(X0, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X1, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X3, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x93);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x39);
+		T = _mm_add_epi32(X0, X1);
+		X3 = _mm_xor_si128(_mm_xor_si128(X3, _mm_slli_epi32(T, 7)), _mm_srli_epi32(T, 25));
+		T = _mm_add_epi32(X3, X0);
+		X2 = _mm_xor_si128(_mm_xor_si128(X2, _mm_slli_epi32(T, 9)), _mm_srli_epi32(T, 23));
+		T = _mm_add_epi32(X2, X3);
+		X1 = _mm_xor_si128(_mm_xor_si128(X1, _mm_slli_epi32(T, 13)), _mm_srli_epi32(T, 19));
+		T = _mm_add_epi32(X1, X2);
+		X0 = _mm_xor_si128(_mm_xor_si128(X0, _mm_slli_epi32(T, 18)), _mm_srli_epi32(T, 14));
+		X1 = _mm_shuffle_epi32(X1, 0x39);
+		X2 = _mm_shuffle_epi32(X2, 0x4E);
+		X3 = _mm_shuffle_epi32(X3, 0x93);
+
+		X0 = _mm_add_epi32(X0s,X0);
+		X1 = _mm_add_epi32(X1s,X1);
+		X2 = _mm_add_epi32(X2s,X2);
+		X3 = _mm_add_epi32(X3s,X3);
+
+		__m128i k02 = _mm_shuffle_epi32(_mm_or_si128(_mm_slli_epi64(X0, 32), _mm_srli_epi64(X3, 32)), _MM_SHUFFLE(0, 1, 2, 3));
+		__m128i k13 = _mm_shuffle_epi32(_mm_or_si128(_mm_slli_epi64(X1, 32), _mm_srli_epi64(X0, 32)), _MM_SHUFFLE(0, 1, 2, 3));
+		__m128i k20 = _mm_or_si128(_mm_and_si128(X2, _S20SSECONSTANTS.maskLo32), _mm_and_si128(X1, _S20SSECONSTANTS.maskHi32));
+		__m128i k31 = _mm_or_si128(_mm_and_si128(X3, _S20SSECONSTANTS.maskLo32), _mm_and_si128(X2, _S20SSECONSTANTS.maskHi32));
+		_mm_storeu_ps(reinterpret_cast<float *>(c),_mm_castsi128_ps(_mm_xor_si128(_mm_unpackhi_epi64(k02,k20),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m))))));
+		_mm_storeu_ps(reinterpret_cast<float *>(c) + 4,_mm_castsi128_ps(_mm_xor_si128(_mm_unpackhi_epi64(k13,k31),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m) + 4)))));
+		_mm_storeu_ps(reinterpret_cast<float *>(c) + 8,_mm_castsi128_ps(_mm_xor_si128(_mm_unpacklo_epi64(k20,k02),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m) + 8)))));
+		_mm_storeu_ps(reinterpret_cast<float *>(c) + 12,_mm_castsi128_ps(_mm_xor_si128(_mm_unpacklo_epi64(k31,k13),_mm_castps_si128(_mm_loadu_ps(reinterpret_cast<const float *>(m) + 12)))));
+
+		if (!(++_state.i[8])) {
+			++_state.i[5]; // state reordered for SSE
+			/* stopping at 2^70 bytes per nonce is user's responsibility */
+		}
+#else
+		x0 = j0;
+		x1 = j1;
+		x2 = j2;
+		x3 = j3;
+		x4 = j4;
+		x5 = j5;
+		x6 = j6;
+		x7 = j7;
+		x8 = j8;
+		x9 = j9;
+		x10 = j10;
+		x11 = j11;
+		x12 = j12;
+		x13 = j13;
+		x14 = j14;
+		x15 = j15;
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
+		// 2X round -------------------------------------------------------------
+		 x4 = XOR( x4,ROTATE(PLUS( x0,x12), 7));
+		 x8 = XOR( x8,ROTATE(PLUS( x4, x0), 9));
+		x12 = XOR(x12,ROTATE(PLUS( x8, x4),13));
+		 x0 = XOR( x0,ROTATE(PLUS(x12, x8),18));
+		 x9 = XOR( x9,ROTATE(PLUS( x5, x1), 7));
+		x13 = XOR(x13,ROTATE(PLUS( x9, x5), 9));
+		 x1 = XOR( x1,ROTATE(PLUS(x13, x9),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x1,x13),18));
+		x14 = XOR(x14,ROTATE(PLUS(x10, x6), 7));
+		 x2 = XOR( x2,ROTATE(PLUS(x14,x10), 9));
+		 x6 = XOR( x6,ROTATE(PLUS( x2,x14),13));
+		x10 = XOR(x10,ROTATE(PLUS( x6, x2),18));
+		 x3 = XOR( x3,ROTATE(PLUS(x15,x11), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x3,x15), 9));
+		x11 = XOR(x11,ROTATE(PLUS( x7, x3),13));
+		x15 = XOR(x15,ROTATE(PLUS(x11, x7),18));
+		 x1 = XOR( x1,ROTATE(PLUS( x0, x3), 7));
+		 x2 = XOR( x2,ROTATE(PLUS( x1, x0), 9));
+		 x3 = XOR( x3,ROTATE(PLUS( x2, x1),13));
+		 x0 = XOR( x0,ROTATE(PLUS( x3, x2),18));
+		 x6 = XOR( x6,ROTATE(PLUS( x5, x4), 7));
+		 x7 = XOR( x7,ROTATE(PLUS( x6, x5), 9));
+		 x4 = XOR( x4,ROTATE(PLUS( x7, x6),13));
+		 x5 = XOR( x5,ROTATE(PLUS( x4, x7),18));
+		x11 = XOR(x11,ROTATE(PLUS(x10, x9), 7));
+		 x8 = XOR( x8,ROTATE(PLUS(x11,x10), 9));
+		 x9 = XOR( x9,ROTATE(PLUS( x8,x11),13));
+		x10 = XOR(x10,ROTATE(PLUS( x9, x8),18));
+		x12 = XOR(x12,ROTATE(PLUS(x15,x14), 7));
+		x13 = XOR(x13,ROTATE(PLUS(x12,x15), 9));
+		x14 = XOR(x14,ROTATE(PLUS(x13,x12),13));
+		x15 = XOR(x15,ROTATE(PLUS(x14,x13),18));
+
 		x0 = PLUS(x0,j0);
 		x1 = PLUS(x1,j1);
 		x2 = PLUS(x2,j2);

node/Salsa20.hpp

@@ -12,6 +12,7 @@
 #include <stdlib.h>
 
 #include "Constants.hpp"
+#include "Utils.hpp"
 
 #if (!defined(ZT_SALSA20_SSE)) && (defined(__SSE2__) || defined(__WINDOWS__))
 #define ZT_SALSA20_SSE 1
@@ -31,16 +32,17 @@ class Salsa20
 public:
 	Salsa20() throw() {}
 
+	~Salsa20() { Utils::burn(&_state,sizeof(_state)); }
+
 	/**
 	 * @param key Key bits
 	 * @param kbits Number of key bits: 128 or 256 (recommended)
 	 * @param iv 64-bit initialization vector
-	 * @param rounds Number of rounds: 8, 12, or 20
 	 */
-	Salsa20(const void *key,unsigned int kbits,const void *iv,unsigned int rounds)
+	Salsa20(const void *key,unsigned int kbits,const void *iv)
 		throw()
 	{
-		init(key,kbits,iv,rounds);
+		init(key,kbits,iv);
 	}
 
 	/**
@@ -49,19 +51,28 @@ public:
 	 * @param key Key bits
 	 * @param kbits Number of key bits: 128 or 256 (recommended)
 	 * @param iv 64-bit initialization vector
-	 * @param rounds Number of rounds: 8, 12, or 20
 	 */
-	void init(const void *key,unsigned int kbits,const void *iv,unsigned int rounds)
+	void init(const void *key,unsigned int kbits,const void *iv)
+		throw();
+
+	/**
+	 * Encrypt data using Salsa20/12
+	 *
+	 * @param in Input data
+	 * @param out Output buffer
+	 * @param bytes Length of data
+	 */
+	void encrypt12(const void *in,void *out,unsigned int bytes)
 		throw();
 
 	/**
-	 * Encrypt data
+	 * Encrypt data using Salsa20/20
 	 *
 	 * @param in Input data
 	 * @param out Output buffer
 	 * @param bytes Length of data
 	 */
-	void encrypt(const void *in,void *out,unsigned int bytes)
+	void encrypt20(const void *in,void *out,unsigned int bytes)
 		throw();
 
 	/**
@@ -71,10 +82,23 @@ public:
 	 * @param out Output buffer
 	 * @param bytes Length of data
 	 */
-	inline void decrypt(const void *in,void *out,unsigned int bytes)
+	inline void decrypt12(const void *in,void *out,unsigned int bytes)
+		throw()
+	{
+		encrypt12(in,out,bytes);
+	}
+
+	/**
+	 * Decrypt data
+	 *
+	 * @param in Input data
+	 * @param out Output buffer
+	 * @param bytes Length of data
+	 */
+	inline void decrypt20(const void *in,void *out,unsigned int bytes)
 		throw()
 	{
-		encrypt(in,out,bytes);
+		encrypt20(in,out,bytes);
 	}
 
 private:
@@ -84,7 +108,6 @@ private:
 #endif // ZT_SALSA20_SSE
 		uint32_t i[16];
 	} _state;
-	unsigned int _roundsDiv2;
 };
 
 } // namespace ZeroTier

node/SelfAwareness.cpp

@@ -36,6 +36,7 @@
 #include "Topology.hpp"
 #include "Packet.hpp"
 #include "Peer.hpp"
+#include "Switch.hpp"
 
 // Entry timeout -- make it fairly long since this is just to prevent stale buildup
 #define ZT_SELFAWARENESS_ENTRY_TIMEOUT 3600000
@@ -65,7 +66,8 @@ private:
 };
 
 SelfAwareness::SelfAwareness(const RuntimeEnvironment *renv) :
-	RR(renv)
+	RR(renv),
+	_phy(32)
 {
 }
 
@@ -77,66 +79,62 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 {
 	const InetAddress::IpScope scope = myPhysicalAddress.ipScope();
 
+	// This would be weird, e.g. a public IP talking to 10.0.0.1, so just ignore it.
+	// If your network is this weird it's probably not reliable information.
+	if (scope != reporterPhysicalAddress.ipScope())
+		return;
+
+	// Some scopes we ignore, and global scope IPs are only used for this
+	// mechanism if they come from someone we trust (e.g. a root).
 	switch(scope) {
 		case InetAddress::IP_SCOPE_NONE:
 		case InetAddress::IP_SCOPE_LOOPBACK:
 		case InetAddress::IP_SCOPE_MULTICAST:
 			return;
 		case InetAddress::IP_SCOPE_GLOBAL:
-			if ((!trusted)||(scope != reporterPhysicalAddress.ipScope()))
+			if (!trusted)
 				return;
 			break;
 		default:
-			if (scope != reporterPhysicalAddress.ipScope())
-				return;
 			break;
 	}
 
 	Mutex::Lock _l(_phy_m);
+	PhySurfaceEntry &entry = _phy[PhySurfaceKey(reporter,reporterPhysicalAddress,scope)];
 
-	PhySurfaceEntry &entry = _phy[PhySurfaceKey(reporter,scope)];
-
-	if ((now - entry.ts) >= ZT_SELFAWARENESS_ENTRY_TIMEOUT) {
+	if ( ((now - entry.ts) < ZT_SELFAWARENESS_ENTRY_TIMEOUT) && (!entry.mySurface.ipsEqual(myPhysicalAddress)) ) {
 		entry.mySurface = myPhysicalAddress;
 		entry.ts = now;
-		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced <null>)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str());
-	} else if (entry.mySurface != myPhysicalAddress) {
-		entry.mySurface = myPhysicalAddress;
-		entry.ts = now;
-		TRACE("learned physical address %s for scope %u as seen from %s(%s) (replaced %s, resetting all in scope)",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str());
+		TRACE("physical address %s for scope %u as seen from %s(%s) differs from %s, resetting paths in scope",myPhysicalAddress.toString().c_str(),(unsigned int)scope,reporter.toString().c_str(),reporterPhysicalAddress.toString().c_str(),entry.mySurface.toString().c_str());
 
-		// Erase all entries (other than this one) for this scope to prevent thrashing
-		// Note: we should probably not use 'entry' after this
+		// Erase all entries in this scope that were not reported from this remote address to prevent 'thrashing'
+		// due to multiple reports of endpoint change.
+		// Don't use 'entry' after this since hash table gets modified.
 		{
 			Hashtable< PhySurfaceKey,PhySurfaceEntry >::Iterator i(_phy);
 			PhySurfaceKey *k = (PhySurfaceKey *)0;
 			PhySurfaceEntry *e = (PhySurfaceEntry *)0;
 			while (i.next(k,e)) {
-				if ((k->reporter != reporter)&&(k->scope == scope))
+				if ((k->reporterPhysicalAddress != reporterPhysicalAddress)&&(k->scope == scope))
 					_phy.erase(*k);
 			}
 		}
 
+		// Reset all paths within this scope
 		_ResetWithinScope rset(RR,now,(InetAddress::IpScope)scope);
 		RR->topology->eachPeer<_ResetWithinScope &>(rset);
 
-		// For all peers for whom we forgot an address, send a packet indirectly if
-		// they are still considered alive so that we will re-establish direct links.
-		SharedPtr<Peer> sn(RR->topology->getBestRoot());
-		if (sn) {
-			RemotePath *snp = sn->getBestPath(now);
-			if (snp) {
-				for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
-					if ((*p)->alive(now)) {
-						TRACE("sending indirect NOP to %s via %s(%s) to re-establish link",(*p)->address().toString().c_str(),sn->address().toString().c_str(),snp->address().toString().c_str());
-						Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
-						outp.armor((*p)->key(),true);
-						snp->send(RR,outp.data(),outp.size(),now);
-					}
-				}
+		// Send a NOP to all peers for whom we forgot a path. This will cause direct
+		// links to be re-established if possible, possibly using a root server or some
+		// other relay.
+		for(std::vector< SharedPtr<Peer> >::const_iterator p(rset.peersReset.begin());p!=rset.peersReset.end();++p) {
+			if ((*p)->activelyTransferringFrames(now)) {
+				Packet outp((*p)->address(),RR->identity.address(),Packet::VERB_NOP);
+				RR->sw->send(outp,true,0);
 			}
 		}
 	} else {
+		entry.mySurface = myPhysicalAddress;
 		entry.ts = now;
 	}
 }

node/SelfAwareness.hpp

@@ -69,14 +69,14 @@ private:
 	struct PhySurfaceKey
 	{
 		Address reporter;
+		InetAddress reporterPhysicalAddress;
 		InetAddress::IpScope scope;
 
-		inline unsigned long hashCode() const throw() { return ((unsigned long)reporter.toInt() + (unsigned long)scope); }
-
 		PhySurfaceKey() : reporter(),scope(InetAddress::IP_SCOPE_NONE) {}
-		PhySurfaceKey(const Address &r,InetAddress::IpScope s) : reporter(r),scope(s) {}
-		inline bool operator<(const PhySurfaceKey &k) const throw() { return ((reporter < k.reporter) ? true : ((reporter == k.reporter) ? ((int)scope < (int)k.scope) : false)); }
-		inline bool operator==(const PhySurfaceKey &k) const throw() { return ((reporter == k.reporter)&&(scope == k.scope)); }
+		PhySurfaceKey(const Address &r,const InetAddress &ra,InetAddress::IpScope s) : reporter(r),reporterPhysicalAddress(ra),scope(s) {}
+
+		inline unsigned long hashCode() const throw() { return ((unsigned long)reporter.toInt() + (unsigned long)scope); }
+		inline bool operator==(const PhySurfaceKey &k) const throw() { return ((reporter == k.reporter)&&(reporterPhysicalAddress == k.reporterPhysicalAddress)&&(scope == k.scope)); }
 	};
 	struct PhySurfaceEntry
 	{

node/Switch.cpp

@@ -45,6 +45,7 @@
 #include "AntiRecursion.hpp"
 #include "SelfAwareness.hpp"
 #include "Packet.hpp"
+#include "Cluster.hpp"
 
 namespace ZeroTier {
 
@@ -202,7 +203,8 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		// Destination is another ZeroTier peer on the same network
 
 		Address toZT(to.toAddress(network->id())); // since in-network MACs are derived from addresses and network IDs, we can reverse this
-		const bool includeCom = network->peerNeedsOurMembershipCertificate(toZT,RR->node->now());
+		SharedPtr<Peer> toPeer(RR->topology->getPeer(toZT));
+		const bool includeCom = ( (nconf->isPrivate()) && (nconf->com()) && ((!toPeer)||(toPeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) );
 		if ((fromBridged)||(includeCom)) {
 			Packet outp(toZT,RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
@@ -267,9 +269,10 @@ void Switch::onLocalEthernet(const SharedPtr<Network> &network,const MAC &from,c
 		}
 
 		for(unsigned int b=0;b<numBridges;++b) {
+			SharedPtr<Peer> bridgePeer(RR->topology->getPeer(bridges[b]));
 			Packet outp(bridges[b],RR->identity.address(),Packet::VERB_EXT_FRAME);
 			outp.append(network->id());
-			if (network->peerNeedsOurMembershipCertificate(bridges[b],RR->node->now())) {
+			if ( (nconf->isPrivate()) && (nconf->com()) && ((!bridgePeer)||(bridgePeer->needsOurNetworkMembershipCertificate(network->id(),RR->node->now(),true))) ) {
 				outp.append((unsigned char)0x01); // 0x01 -- COM included
 				nconf->com().serialize(outp);
 			} else {
@@ -292,17 +295,18 @@ void Switch::send(const Packet &packet,bool encrypt,uint64_t nwid)
 		return;
 	}
 
+	//TRACE(">> %s to %s (%u bytes, encrypt==%d, nwid==%.16llx)",Packet::verbString(packet.verb()),packet.destination().toString().c_str(),packet.size(),(int)encrypt,nwid);
+
 	if (!_trySend(packet,encrypt,nwid)) {
 		Mutex::Lock _l(_txQueue_m);
 		_txQueue.push_back(TXQueueEntry(packet.destination(),RR->node->now(),packet,encrypt,nwid));
 	}
 }
 
-bool Switch::unite(const Address &p1,const Address &p2,bool force)
+bool Switch::unite(const Address &p1,const Address &p2)
 {
 	if ((p1 == RR->identity.address())||(p2 == RR->identity.address()))
 		return false;
-
 	SharedPtr<Peer> p1p = RR->topology->getPeer(p1);
 	if (!p1p)
 		return false;
@@ -312,14 +316,6 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 
 	const uint64_t now = RR->node->now();
 
-	{
-		Mutex::Lock _l(_lastUniteAttempt_m);
-		uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
-		if (((now - luts) < ZT_MIN_UNITE_INTERVAL)&&(!force))
-			return false;
-		luts = now;
-	}
-
 	std::pair<InetAddress,InetAddress> cg(Peer::findCommonGround(*p1p,*p2p,now));
 	if ((!(cg.first))||(cg.first.ipScope() != cg.second.ipScope()))
 		return false;
@@ -446,8 +442,8 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		Mutex::Lock _l(_contactQueue_m);
 		for(std::list<ContactQueueEntry>::iterator qi(_contactQueue.begin());qi!=_contactQueue.end();) {
 			if (now >= qi->fireAtTime) {
-				if ((!qi->peer->alive(now))||(qi->peer->hasActiveDirectPath(now))) {
-					// Cancel attempt if we've already connected or peer is no longer "alive"
+				if (qi->peer->hasActiveDirectPath(now)) {
+					// Cancel if connection has succeeded
 					_contactQueue.erase(qi++);
 					continue;
 				} else {
@@ -543,7 +539,7 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 		_LastUniteKey *k = (_LastUniteKey *)0;
 		uint64_t *v = (uint64_t *)0;
 		while (i.next(k,v)) {
-			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 16))
+			if ((now - *v) >= (ZT_MIN_UNITE_INTERVAL * 8))
 				_lastUniteAttempt.erase(*k);
 		}
 	}
@@ -565,6 +561,11 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 			// It wouldn't hurt anything, just redundant and unnecessary.
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
 			if ((!relayTo)||(!relayTo->send(RR,fragment.data(),fragment.size(),RR->node->now()))) {
+#ifdef ZT_ENABLE_CLUSTER
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false)))
+					return; // sent by way of another member of this cluster
+#endif
+
 				// Don't know peer or no direct path -- so relay via root server
 				relayTo = RR->topology->getBestRoot();
 				if (relayTo)
@@ -624,11 +625,17 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 
 void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
-	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,RR->node->now()));
+	const uint64_t now = RR->node->now();
+	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localAddr,fromAddr,now));
 
 	Address source(packet->source());
 	Address destination(packet->destination());
 
+	// Catch this and toss it -- it would never work, but it could happen if we somehow
+	// mistakenly guessed an address we're bound to as a destination for another peer.
+	if (source == RR->identity.address())
+		return;
+
 	//TRACE("<< %.16llx %s -> %s (size: %u)",(unsigned long long)packet->packetId(),source.toString().c_str(),destination.toString().c_str(),packet->size());
 
 	if (destination != RR->identity.address()) {
@@ -637,13 +644,18 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 			packet->incrementHops();
 
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),RR->node->now())))) {
-				unite(source,destination,false);
+			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),now)))) {
+				if (_shouldTryUnite(now,source,destination))
+					unite(source,destination);
 			} else {
-				// Don't know peer or no direct path -- so relay via root server
+#ifdef ZT_ENABLE_CLUSTER
+				if ((RR->cluster)&&(RR->cluster->sendViaCluster(source,destination,packet->data(),packet->size(),_shouldTryUnite(now,source,destination))))
+					return; // sent by way of another member of this cluster
+#endif
+
 				relayTo = RR->topology->getBestRoot(&source,1,true);
 				if (relayTo)
-					relayTo->send(RR,packet->data(),packet->size(),RR->node->now());
+					relayTo->send(RR,packet->data(),packet->size(),now);
 			}
 		} else {
 			TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet->source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str());
@@ -658,7 +670,7 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 		if (!dq.creationTime) {
 			// If we have no other fragments yet, create an entry and save the head
 
-			dq.creationTime = RR->node->now();
+			dq.creationTime = now;
 			dq.frag0 = packet;
 			dq.totalFragments = 0; // 0 == unknown, waiting for Packet::Fragment
 			dq.haveFragments = 1; // head is first bit (left to right)
@@ -724,17 +736,20 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // sanity check: unconfigured network? why are we trying to talk to it?
 		}
 
-		RemotePath *viaPath = peer->getBestPath(now);
+		Path *viaPath = peer->getBestPath(now);
 		SharedPtr<Peer> relay;
 		if (!viaPath) {
 			// See if this network has a preferred relay (if packet has an associated network)
 			if (nconf) {
-				unsigned int latency = ~((unsigned int)0);
+				unsigned int bestq = ~((unsigned int)0);
 				for(std::vector< std::pair<Address,InetAddress> >::const_iterator r(nconf->relays().begin());r!=nconf->relays().end();++r) {
 					if (r->first != peer->address()) {
 						SharedPtr<Peer> rp(RR->topology->getPeer(r->first));
-						if ((rp)&&(rp->hasActiveDirectPath(now))&&(rp->latency() <= latency))
+						const unsigned int q = rp->relayQuality(now);
+						if ((rp)&&(q < bestq)) { // SUBTILE: < == don't use these if they are nil quality (unsigned int max), instead use a root
+							bestq = q;
 							rp.swap(relay);
+						}
 					}
 				}
 			}
@@ -747,7 +762,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 				return false; // no paths, no root servers?
 		}
 
-		if ((network)&&(relay)&&(network->isAllowed(peer->address()))) {
+		if ((network)&&(relay)&&(network->isAllowed(peer))) {
 			// Push hints for direct connectivity to this peer if we are relaying
 			peer->pushDirectPaths(RR,viaPath,now,false);
 		}
@@ -786,4 +801,14 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 	return false;
 }
 
+bool Switch::_shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2)
+{
+	Mutex::Lock _l(_lastUniteAttempt_m);
+	uint64_t &luts = _lastUniteAttempt[_LastUniteKey(p1,p2)];
+	if ((now - luts) < ZT_MIN_UNITE_INTERVAL)
+		return false;
+	luts = now;
+	return true;
+}
+
 } // namespace ZeroTier

node/Switch.hpp

@@ -127,15 +127,10 @@ public:
 	 * This only works if both peers are known, with known working direct
 	 * links to this peer. The best link for each peer is sent to the other.
 	 *
-	 * A rate limiter is in effect via the _lastUniteAttempt map. If force
-	 * is true, a unite attempt is made even if one has been made less than
-	 * ZT_MIN_UNITE_INTERVAL milliseconds ago.
-	 *
 	 * @param p1 One of two peers (order doesn't matter)
 	 * @param p2 Second of pair
-	 * @param force If true, send now regardless of interval
 	 */
-	bool unite(const Address &p1,const Address &p2,bool force);
+	bool unite(const Address &p1,const Address &p2);
 
 	/**
 	 * Attempt NAT traversal to peer at a given physical address
@@ -185,6 +180,7 @@ private:
 	void _handleRemotePacketHead(const InetAddress &localAddr,const InetAddress &fromAddr,const void *data,unsigned int len);
 	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(const Packet &packet,bool encrypt,uint64_t nwid);
+	bool _shouldTryUnite(const uint64_t now,const Address &p1,const Address &p2);
 
 	const RuntimeEnvironment *const RR;
 	uint64_t _lastBeaconResponse;

node/Topology.hpp

@@ -31,10 +31,10 @@
 #include <stdio.h>
 #include <string.h>
 
-#include <map>
 #include <vector>
 #include <stdexcept>
 #include <algorithm>
+#include <utility>
 
 #include "Constants.hpp"
 
@@ -43,8 +43,8 @@
 #include "Peer.hpp"
 #include "Mutex.hpp"
 #include "InetAddress.hpp"
-#include "Dictionary.hpp"
 #include "Hashtable.hpp"
+#include "World.hpp"
 
 namespace ZeroTier {
 
@@ -59,21 +59,6 @@ public:
 	Topology(const RuntimeEnvironment *renv);
 	~Topology();
 
-	/**
-	 * @param sn Root server identities and addresses
-	 */
-	void setRootServers(const std::map< Identity,std::vector<InetAddress> > &sn);
-
-	/**
-	 * Set up root servers for this network
-	 *
-	 * This performs no signature verification of any kind. The caller must
-	 * check the signature of the root topology dictionary first.
-	 *
-	 * @param sn 'rootservers' key from root-topology Dictionary (deserialized as Dictionary)
-	 */
-	void setRootServers(const Dictionary &sn);
-
 	/**
 	 * Add a peer to database
 	 *
@@ -94,23 +79,49 @@ public:
 	SharedPtr<Peer> getPeer(const Address &zta);
 
 	/**
-	 * @return Vector of peers that are root servers
+	 * Get a peer only if it is presently in memory (no disk cache)
+	 *
+	 * This also does not update the lastUsed() time for peers, which means
+	 * that it won't prevent them from falling out of RAM. This is currently
+	 * used in the Cluster code to update peer info without forcing all peers
+	 * across the entire cluster to remain in memory cache.
+	 *
+	 * @param zta ZeroTier address
+	 * @param now Current time
 	 */
-	inline std::vector< SharedPtr<Peer> > rootPeers() const
+	inline SharedPtr<Peer> getPeerNoCache(const Address &zta,const uint64_t now)
 	{
 		Mutex::Lock _l(_lock);
-		return _rootPeers;
+		const SharedPtr<Peer> *const ap = _peers.get(zta);
+		if (ap)
+			return *ap;
+		return SharedPtr<Peer>();
 	}
 
+	/**
+	 * Get the identity of a peer
+	 *
+	 * @param zta ZeroTier address of peer
+	 * @return Identity or NULL Identity if not found
+	 */
+	Identity getIdentity(const Address &zta);
+
+	/**
+	 * Cache an identity
+	 *
+	 * This is done automatically on addPeer(), and so is only useful for
+	 * cluster identity replication.
+	 *
+	 * @param id Identity to cache
+	 */
+	void saveIdentity(const Identity &id);
+
 	/**
 	 * Get the current favorite root server
 	 *
 	 * @return Root server with lowest latency or NULL if none
 	 */
-	inline SharedPtr<Peer> getBestRoot()
-	{
-		return getBestRoot((const Address *)0,0,false);
-	}
+	inline SharedPtr<Peer> getBestRoot() { return getBestRoot((const Address *)0,0,false); }
 
 	/**
 	 * Get the best root server, avoiding root servers listed in an array
@@ -128,10 +139,19 @@ public:
 
 	/**
 	 * @param id Identity to check
-	 * @return True if this is a designated root server
+	 * @return True if this is a designated root server in this world
+	 */
+	inline bool isRoot(const Identity &id) const
+	{
+		Mutex::Lock _l(_lock);
+		return (std::find(_rootAddresses.begin(),_rootAddresses.end(),id.address()) != _rootAddresses.end());
+	}
+
+	/**
+	 * @param id Identity to check
+	 * @return True if this is a root server or a network preferred relay from one of our networks
 	 */
-	bool isRoot(const Identity &id) const
-		throw();
+	bool isUpstream(const Identity &id) const;
 
 	/**
 	 * @return Vector of root server addresses
@@ -142,11 +162,49 @@ public:
 		return _rootAddresses;
 	}
 
+	/**
+	 * @return Current World (copy)
+	 */
+	inline World world() const
+	{
+		Mutex::Lock _l(_lock);
+		return _world;
+	}
+
+	/**
+	 * @return Current world ID
+	 */
+	inline uint64_t worldId() const
+	{
+		return _world.id(); // safe to read without lock, and used from within eachPeer() so don't lock
+	}
+
+	/**
+	 * @return Current world timestamp
+	 */
+	inline uint64_t worldTimestamp() const
+	{
+		return _world.timestamp(); // safe to read without lock, and used from within eachPeer() so don't lock
+	}
+
+	/**
+	 * Validate new world and update if newer and signature is okay
+	 *
+	 * @param newWorld Potential new world definition revision
+	 * @return True if an update actually occurred
+	 */
+	bool worldUpdateIfValid(const World &newWorld);
+
 	/**
 	 * Clean and flush database
 	 */
 	void clean(uint64_t now);
 
+	/**
+	 * @return Number of peers with active direct paths
+	 */
+	unsigned long countActive() const;
+
 	/**
 	 * Apply a function or function object to all peers
 	 *
@@ -164,44 +222,47 @@ public:
 	inline void eachPeer(F f)
 	{
 		Mutex::Lock _l(_lock);
-		Hashtable< Address,SharedPtr<Peer> >::Iterator i(_activePeers);
+		Hashtable< Address,SharedPtr<Peer> >::Iterator i(_peers);
 		Address *a = (Address *)0;
 		SharedPtr<Peer> *p = (SharedPtr<Peer> *)0;
-		while (i.next(a,p))
-			f(*this,*p);
+		while (i.next(a,p)) {
+#ifdef ZT_TRACE
+			if (!(*p)) {
+				fprintf(stderr,"FATAL BUG: eachPeer() caught NULL peer for %s -- peer pointers in Topology should NEVER be NULL"ZT_EOL_S,a->toString().c_str());
+				abort();
+			}
+#endif
+			f(*this,*((const SharedPtr<Peer> *)p));
+		}
 	}
 
 	/**
-	 * @return All currently active peers by address
+	 * @return All currently active peers by address (unsorted)
 	 */
 	inline std::vector< std::pair< Address,SharedPtr<Peer> > > allPeers() const
 	{
 		Mutex::Lock _l(_lock);
-		return _activePeers.entries();
+		return _peers.entries();
 	}
 
 	/**
-	 * Validate a root topology dictionary against the identities specified in Defaults
-	 *
-	 * @param rt Root topology dictionary
-	 * @return True if dictionary signature is valid
+	 * @return True if I am a root server in the current World
 	 */
-	static bool authenticateRootTopology(const Dictionary &rt);
+	inline bool amRoot() const throw() { return _amRoot; }
 
 private:
 	Identity _getIdentity(const Address &zta);
-	void _saveIdentity(const Identity &id);
+	void _setWorld(const World &newWorld);
 
 	const RuntimeEnvironment *RR;
 
-	Hashtable< Address,SharedPtr<Peer> > _activePeers;
-	std::map< Identity,std::vector<InetAddress> > _roots;
+	World _world;
+	Hashtable< Address,SharedPtr<Peer> > _peers;
 	std::vector< Address > _rootAddresses;
 	std::vector< SharedPtr<Peer> > _rootPeers;
+	bool _amRoot;
 
 	Mutex _lock;
-
-	bool _amRoot;
 };
 
 } // namespace ZeroTier

node/Utils.cpp

@@ -181,7 +181,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes)
 
 #ifdef __UNIX_LIKE__
 
-	static char randomBuf[65536];
+	static char randomBuf[131072];
 	static unsigned int randomPtr = sizeof(randomBuf);
 	static int devURandomFd = -1;
 	static Mutex globalLock;
@@ -191,7 +191,7 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes)
 	if (devURandomFd <= 0) {
 		devURandomFd = ::open("/dev/urandom",O_RDONLY);
 		if (devURandomFd <= 0) {
-			fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to open /dev/urandom\r\n");
+			fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to open /dev/urandom\n");
 			exit(1);
 			return;
 		}
@@ -199,10 +199,16 @@ void Utils::getSecureRandom(void *buf,unsigned int bytes)
 
 	for(unsigned int i=0;i<bytes;++i) {
 		if (randomPtr >= sizeof(randomBuf)) {
-			if ((int)::read(devURandomFd,randomBuf,sizeof(randomBuf)) != (int)sizeof(randomBuf)) {
-				fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to read from /dev/urandom\r\n");
-				exit(1);
-				return;
+			for(;;) {
+				if ((int)::read(devURandomFd,randomBuf,sizeof(randomBuf)) != (int)sizeof(randomBuf)) {
+					::close(devURandomFd);
+					devURandomFd = ::open("/dev/urandom",O_RDONLY);
+					if (devURandomFd <= 0) {
+						fprintf(stderr,"FATAL ERROR: Utils::getSecureRandom() unable to open /dev/urandom\n");
+						exit(1);
+						return;
+					}
+				} else break;
 			}
 			randomPtr = 0;
 		}
@@ -261,25 +267,6 @@ std::vector<std::string> Utils::split(const char *s,const char *const sep,const
 	return fields;
 }
 
-std::string Utils::trim(const std::string &s)
-{
-	unsigned long end = (unsigned long)s.length();
-	while (end) {
-		char c = s[end - 1];
-		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
-			--end;
-		else break;
-	}
-	unsigned long start = 0;
-	while (start < end) {
-		char c = s[start];
-		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
-			++start;
-		else break;
-	}
-	return s.substr(start,end - start);
-}
-
 unsigned int Utils::snprintf(char *buf,unsigned int len,const char *fmt,...)
 	throw(std::length_error)
 {

node/Utils.hpp

@@ -256,14 +256,6 @@ public:
 		return true;
 	}
 
-	/**
-	 * Trim whitespace from the start and end of a string
-	 *
-	 * @param s String to trim
-	 * @return Trimmed string
-	 */
-	static std::string trim(const std::string &s);
-
 	/**
 	 * Variant of snprintf that is portable and throws an exception
 	 *

node/World.hpp

@@ -0,0 +1,241 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef ZT_WORLD_HPP
+#define ZT_WORLD_HPP
+
+#include <vector>
+#include <string>
+
+#include "Constants.hpp"
+#include "InetAddress.hpp"
+#include "Identity.hpp"
+#include "Buffer.hpp"
+#include "C25519.hpp"
+
+/**
+ * Maximum number of roots (sanity limit, okay to increase)
+ *
+ * A given root can (through multi-homing) be distributed across any number of
+ * physical endpoints, but having more than one is good to permit total failure
+ * of one root or its withdrawal due to compromise without taking the whole net
+ * down.
+ */
+#define ZT_WORLD_MAX_ROOTS 4
+
+/**
+ * Maximum number of stable endpoints per root (sanity limit, okay to increase)
+ */
+#define ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT 32
+
+/**
+ * The (more than) maximum length of a serialized World
+ */
+#define ZT_WORLD_MAX_SERIALIZED_LENGTH (((1024 + (32 * ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT)) * ZT_WORLD_MAX_ROOTS) + ZT_C25519_PUBLIC_KEY_LEN + ZT_C25519_SIGNATURE_LEN + 128)
+
+/**
+ * World ID indicating null / empty World object
+ */
+#define ZT_WORLD_ID_NULL 0
+
+/**
+ * World ID for a test network with ephemeral or temporary roots
+ */
+#define ZT_WORLD_ID_TESTNET 1
+
+/**
+ * World ID for Earth
+ *
+ * This is the ID for the ZeroTier World used on planet Earth. It is unrelated
+ * to the public network 8056c2e21c000001 of the same name. It was chosen
+ * from Earth's approximate distance from the sun in kilometers.
+ */
+#define ZT_WORLD_ID_EARTH 149604618
+
+/**
+ * World ID for Mars -- for future use by SpaceX or others
+ */
+#define ZT_WORLD_ID_MARS 227883110
+
+namespace ZeroTier {
+
+/**
+ * A world definition (formerly known as a root topology)
+ *
+ * Think of a World as a single data center. Within this data center a set
+ * of distributed fault tolerant root servers provide stable anchor points
+ * for a peer to peer network that provides VLAN service. Updates to a world
+ * definition can be published by signing them with the previous revision's
+ * signing key, and should be very infrequent.
+ *
+ * The maximum data center size is approximately 2.5 cubic light seconds,
+ * since many protocols have issues with >5s RTT latencies.
+ *
+ * ZeroTier operates a World for Earth capable of encompassing the planet, its
+ * orbits, the Moon (about 1.3 light seconds), and nearby Lagrange points. A
+ * world ID for Mars and nearby space is defined but not yet used, and a test
+ * world ID is provided for testing purposes.
+ *
+ * If you absolutely must run your own "unofficial" ZeroTier network, please
+ * define your world IDs above 0xffffffff (4294967295). Code to make a World
+ * is in mkworld.cpp in the parent directory and must be edited to change
+ * settings.
+ */
+class World
+{
+public:
+	struct Root
+	{
+		Identity identity;
+		std::vector<InetAddress> stableEndpoints;
+
+		inline bool operator==(const Root &r) const throw() { return ((identity == r.identity)&&(stableEndpoints == r.stableEndpoints)); }
+		inline bool operator!=(const Root &r) const throw() { return (!(*this == r)); }
+		inline bool operator<(const Root &r) const throw() { return (identity < r.identity); } // for sorting
+	};
+
+	/**
+	 * Construct an empty / null World
+	 */
+	World() :
+		_id(ZT_WORLD_ID_NULL),
+		_ts(0) {}
+
+	/**
+	 * @return Root servers for this world and their stable endpoints
+	 */
+	inline const std::vector<World::Root> &roots() const throw() { return _roots; }
+
+	/**
+	 * @return World unique identifier
+	 */
+	inline uint64_t id() const throw() { return _id; }
+
+	/**
+	 * @return World definition timestamp
+	 */
+	inline uint64_t timestamp() const throw() { return _ts; }
+
+	/**
+	 * Check whether a world update should replace this one
+	 *
+	 * A new world update is valid if it is for the same world ID, is newer,
+	 * and is signed by the current world's signing key. If this world object
+	 * is null, it can always be updated.
+	 *
+	 * @param update Candidate update
+	 * @param fullSignatureCheck Perform full cryptographic signature check (true == yes, false == skip)
+	 * @return True if update is newer than current and is properly signed
+	 */
+	inline bool shouldBeReplacedBy(const World &update,bool fullSignatureCheck)
+	{
+		if (_id == ZT_WORLD_ID_NULL)
+			return true;
+		if ((_id == update._id)&&(_ts < update._ts)) {
+			if (fullSignatureCheck) {
+				Buffer<ZT_WORLD_MAX_SERIALIZED_LENGTH> tmp;
+				update.serialize(tmp,true);
+				return C25519::verify(_updateSigningKey,tmp.data(),tmp.size(),update._signature);
+			} else return true;
+		}
+		return false;
+	}
+
+	/**
+	 * @return True if this World is non-empty
+	 */
+	inline operator bool() const throw() { return (_id != ZT_WORLD_ID_NULL); }
+
+	template<unsigned int C>
+	inline void serialize(Buffer<C> &b,bool forSign = false) const
+	{
+		if (forSign)
+			b.append((uint64_t)0x7f7f7f7f7f7f7f7fULL);
+		b.append((uint8_t)0x01); // version -- only one valid value for now
+		b.append((uint64_t)_id);
+		b.append((uint64_t)_ts);
+		b.append(_updateSigningKey.data,ZT_C25519_PUBLIC_KEY_LEN);
+		if (!forSign)
+			b.append(_signature.data,ZT_C25519_SIGNATURE_LEN);
+		b.append((uint8_t)_roots.size());
+		for(std::vector<Root>::const_iterator r(_roots.begin());r!=_roots.end();++r) {
+			r->identity.serialize(b);
+			b.append((uint8_t)r->stableEndpoints.size());
+			for(std::vector<InetAddress>::const_iterator ep(r->stableEndpoints.begin());ep!=r->stableEndpoints.end();++ep)
+				ep->serialize(b);
+		}
+		if (forSign)
+			b.append((uint64_t)0xf7f7f7f7f7f7f7f7ULL);
+	}
+
+	template<unsigned int C>
+	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
+	{
+		unsigned int p = startAt;
+
+		_roots.clear();
+
+		if (b[p++] != 0x01)
+			throw std::invalid_argument("invalid World serialized version");
+
+		_id = b.template at<uint64_t>(p); p += 8;
+		_ts = b.template at<uint64_t>(p); p += 8;
+		memcpy(_updateSigningKey.data,b.field(p,ZT_C25519_PUBLIC_KEY_LEN),ZT_C25519_PUBLIC_KEY_LEN); p += ZT_C25519_PUBLIC_KEY_LEN;
+		memcpy(_signature.data,b.field(p,ZT_C25519_SIGNATURE_LEN),ZT_C25519_SIGNATURE_LEN); p += ZT_C25519_SIGNATURE_LEN;
+		unsigned int numRoots = b[p++];
+		if (numRoots > ZT_WORLD_MAX_ROOTS)
+			throw std::invalid_argument("too many roots in World");
+		for(unsigned int k=0;k<numRoots;++k) {
+			_roots.push_back(Root());
+			Root &r = _roots.back();
+			p += r.identity.deserialize(b,p);
+			unsigned int numStableEndpoints = b[p++];
+			if (numStableEndpoints > ZT_WORLD_MAX_STABLE_ENDPOINTS_PER_ROOT)
+				throw std::invalid_argument("too many stable endpoints in World/Root");
+			for(unsigned int kk=0;kk<numStableEndpoints;++kk) {
+				r.stableEndpoints.push_back(InetAddress());
+				p += r.stableEndpoints.back().deserialize(b,p);
+			}
+		}
+
+		return (p - startAt);
+	}
+
+	inline bool operator==(const World &w) const throw() { return ((_id == w._id)&&(_ts == w._ts)&&(_updateSigningKey == w._updateSigningKey)&&(_signature == w._signature)&&(_roots == w._roots)); }
+	inline bool operator!=(const World &w) const throw() { return (!(*this == w)); }
+
+protected:
+	uint64_t _id;
+	uint64_t _ts;
+	C25519::Public _updateSigningKey;
+	C25519::Signature _signature;
+	std::vector<Root> _roots;
+};
+
+} // namespace ZeroTier
+
+#endif

objects.mk

@@ -4,7 +4,7 @@ OBJS=\
 	ext/http-parser/http_parser.o \
 	node/C25519.o \
 	node/CertificateOfMembership.o \
-	node/Defaults.o \
+	node/Cluster.o \
 	node/Dictionary.o \
 	node/Identity.o \
 	node/IncomingPacket.o \
@@ -15,6 +15,7 @@ OBJS=\
 	node/Node.o \
 	node/OutboundMulticast.o \
 	node/Packet.o \
+	node/Path.o \
 	node/Peer.o \
 	node/Poly1305.o \
 	node/Salsa20.o \
@@ -26,5 +27,6 @@ OBJS=\
 	osdep/BackgroundResolver.o \
 	osdep/Http.o \
 	osdep/OSUtils.o \
+	service/ClusterGeoIpService.o \
 	service/ControlPlane.o \
 	service/OneService.o

one.cpp

@@ -911,7 +911,6 @@ static void printHelp(const char *cn,FILE *out)
 	fprintf(out,"  -v                - Show version"ZT_EOL_S);
 	fprintf(out,"  -U                - Run as unprivileged user (skip privilege check)"ZT_EOL_S);
 	fprintf(out,"  -p<port>          - Port for UDP and TCP/HTTP (default: 9993, 0 for random)"ZT_EOL_S);
-	//fprintf(out,"  -T<path>          - Override root topology, do not authenticate or update"ZT_EOL_S);
 
 #ifdef __UNIX_LIKE__
 	fprintf(out,"  -d                - Fork and run as daemon (Unix-ish OSes)"ZT_EOL_S);
@@ -974,7 +973,6 @@ int main(int argc,char **argv)
 	if ((strstr(argv[0],"zerotier-cli"))||(strstr(argv[0],"ZEROTIER-CLI")))
 		return cli(argc,argv);
 
-	std::string overrideRootTopology;
 	std::string homeDir;
 	unsigned int port = ZT_DEFAULT_PORT;
 	bool skipRootCheck = false;
@@ -1001,18 +999,6 @@ int main(int argc,char **argv)
 					skipRootCheck = true;
 					break;
 
-				case 'T': // Override root topology
-					if (argv[i][2]) {
-						if (!OSUtils::readFile(argv[i] + 2,overrideRootTopology)) {
-							fprintf(stderr,"%s: cannot read root topology from %s"ZT_EOL_S,argv[0],argv[i] + 2);
-							return 1;
-						}
-					} else {
-						printHelp(argv[0],stdout);
-						return 1;
-					}
-					break;
-
 				case 'v': // Display version
 					printf("%d.%d.%d"ZT_EOL_S,ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION);
 					return 0;
@@ -1169,7 +1155,7 @@ int main(int argc,char **argv)
 
 	try {
 		for(;;) {
-			zt1Service = OneService::newInstance(homeDir.c_str(),port,(overrideRootTopology.length() > 0) ? overrideRootTopology.c_str() : (const char *)0);
+			zt1Service = OneService::newInstance(homeDir.c_str(),port);
 			switch(zt1Service->run()) {
 				case OneService::ONE_STILL_RUNNING: // shouldn't happen, run() won't return until done
 				case OneService::ONE_NORMAL_TERMINATION:

osdep/Http.cpp

@@ -100,6 +100,14 @@ struct HttpPhyHandler
 			phy->setNotifyWritable(sock,false);
 	}
 
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
+#ifdef __UNIX_LIKE__
+	inline void phyOnUnixAccept(PhySocket *sockL,PhySocket *sockN,void **uptrL,void **uptrN) {}
+	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
+	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
+	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
+#endif // __UNIX_LIKE__
+
 	http_parser parser;
 	std::string currentHeaderField;
 	std::string currentHeaderValue;

osdep/Phy.hpp

@@ -64,6 +64,12 @@
 #include <netinet/in.h>
 #include <netinet/tcp.h>
 
+#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
+#ifndef IPV6_DONTFRAG
+#define IPV6_DONTFRAG 62
+#endif
+#endif
+
 #define ZT_PHY_SOCKFD_TYPE int
 #define ZT_PHY_SOCKFD_NULL (-1)
 #define ZT_PHY_SOCKFD_VALID(s) ((s) > -1)
@@ -72,11 +78,6 @@
 #define ZT_PHY_MAX_INTERCEPTS ZT_PHY_MAX_SOCKETS
 #define ZT_PHY_SOCKADDR_STORAGE_TYPE struct sockaddr_storage
 
-#if defined(__linux__) || defined(linux) || defined(__LINUX__) || defined(__linux)
-#define ZT_PHY_HAVE_EVENTFD 1
-#include <sys/eventfd.h>
-#endif
-
 #endif // Windows or not
 
 namespace ZeroTier {
@@ -103,6 +104,7 @@ typedef void PhySocket;
  * phyOnTcpClose(PhySocket *sock,void **uptr)
  * phyOnTcpData(PhySocket *sock,void **uptr,void *data,unsigned long len)
  * phyOnTcpWritable(PhySocket *sock,void **uptr)
+ * phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable)
  *
  * On Linux/OSX/Unix only (not required/used on Windows or elsewhere):
  *
@@ -110,9 +112,6 @@ typedef void PhySocket;
  * phyOnUnixClose(PhySocket *sock,void **uptr)
  * phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len)
  * phyOnUnixWritable(PhySocket *sock,void **uptr)
- * phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr)
- * phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len)
- * phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr)
  *
  * These templates typically refer to function objects. Templates are used to
  * avoid the call overhead of indirection, which is surprisingly high for high
@@ -148,11 +147,10 @@ private:
 		ZT_PHY_SOCKET_TCP_OUT_CONNECTED = 0x02,
 		ZT_PHY_SOCKET_TCP_IN = 0x03,
 		ZT_PHY_SOCKET_TCP_LISTEN = 0x04,
-		ZT_PHY_SOCKET_RAW = 0x05,
-		ZT_PHY_SOCKET_UDP = 0x06,
+		ZT_PHY_SOCKET_UDP = 0x05,
+		ZT_PHY_SOCKET_FD = 0x06,
 		ZT_PHY_SOCKET_UNIX_IN = 0x07,
-		ZT_PHY_SOCKET_UNIX_LISTEN = 0x08,
-		ZT_PHY_SOCKET_PAIR_ENDPOINT = 0x09
+		ZT_PHY_SOCKET_UNIX_LISTEN = 0x08
 	};
 
 	struct PhySocketImpl
@@ -271,57 +269,47 @@ public:
 	 */
 	inline unsigned long maxCount() const throw() { return ZT_PHY_MAX_SOCKETS; }
 
-#ifdef __UNIX_LIKE__
 	/**
-	 * Create a two-way socket pair
+	 * Wrap a raw file descriptor in a PhySocket structure
 	 *
-	 * This uses socketpair() to create a local domain pair. The returned
-	 * PhySocket holds the local side of the socket pair, while the
-	 * supplied fd variable is set to the descriptor for the remote side.
+	 * This can be used to select/poll on a raw file descriptor as part of this
+	 * class's I/O loop. By default the fd is set for read notification but
+	 * this can be controlled with setNotifyReadable(). When any detected
+	 * condition is present, the phyOnFileDescriptorActivity() callback is
+	 * called with one or both of its arguments 'true'.
 	 *
-	 * The local side is set to O_NONBLOCK to work with our poll loop, but
-	 * the remote descriptor is left untouched. It's up to the caller to
-	 * set any required fcntl(), ioctl(), or setsockopt() settings there.
-	 * It's also up to the caller to close the remote descriptor when
-	 * done, if necessary.
+	 * The Phy<>::close() method *must* be called when you're done with this
+	 * file descriptor to remove it from the select/poll set, but unlike other
+	 * types of sockets Phy<> does not actually close the underlying fd or
+	 * otherwise manage its life cycle. There is also no close notification
+	 * callback for this fd, since Phy<> doesn't actually perform reading or
+	 * writing or detect error conditions. This is only useful for adding a
+	 * file descriptor to Phy<> to select/poll on it.
 	 *
-	 * @param remoteSocketDescriptor Result parameter set to remote end of socket pair's socket FD
-	 * @param uptr Pointer to associate with local side of socket pair
-	 * @return PhySocket for local side of socket pair
+	 * @param fd Raw file descriptor
+	 * @param uptr User pointer to supply to callbacks
+	 * @return PhySocket wrapping fd or NULL on failure (out of memory or too many sockets)
 	 */
-	inline PhySocket *createSocketPair(ZT_PHY_SOCKFD_TYPE &remoteSocketDescriptor,void *uptr = (void *)0)
+	inline PhySocket *wrapSocket(ZT_PHY_SOCKFD_TYPE fd,void *uptr = (void *)0)
 	{
 		if (_socks.size() >= ZT_PHY_MAX_SOCKETS)
 			return (PhySocket *)0;
-
-		int fd[2]; fd[0] = -1; fd[1] = -1;
-		if ((::socketpair(PF_LOCAL,SOCK_STREAM,0,fd) != 0)||(fd[0] <= 0)||(fd[1] <= 0))
-			return (PhySocket *)0;
-		fcntl(fd[0],F_SETFL,O_NONBLOCK);
-
 		try {
 			_socks.push_back(PhySocketImpl());
 		} catch ( ... ) {
-			ZT_PHY_CLOSE_SOCKET(fd[0]);
-			ZT_PHY_CLOSE_SOCKET(fd[1]);
 			return (PhySocket *)0;
 		}
 		PhySocketImpl &sws = _socks.back();
-
-		if ((long)fd[0] > _nfds)
-			_nfds = (long)fd[0];
-		FD_SET(fd[0],&_readfds);
-		sws.type = ZT_PHY_SOCKET_PAIR_ENDPOINT;
-		sws.sock = fd[0];
+		if ((long)fd > _nfds)
+			_nfds = (long)fd;
+		FD_SET(fd,&_readfds);
+		sws.type = ZT_PHY_SOCKET_FD;
+		sws.sock = fd;
 		sws.uptr = uptr;
 		memset(&(sws.saddr),0,sizeof(struct sockaddr_storage));
 		// no sockaddr for this socket type, leave saddr null
-
-		remoteSocketDescriptor = fd[1];
-
 		return (PhySocket *)&sws;
 	}
-#endif // __UNIX_LIKE__
 
 	/**
 	 * Bind a UDP socket
@@ -374,6 +362,9 @@ public:
 				f = 1; setsockopt(s,IPPROTO_IPV6,IPV6_V6ONLY,(void *)&f,sizeof(f));
 #ifdef IPV6_MTU_DISCOVER
 				f = 0; setsockopt(s,IPPROTO_IPV6,IPV6_MTU_DISCOVER,&f,sizeof(f));
+#endif
+#ifdef IPV6_DONTFRAG
+				f = 0; setsockopt(s,IPPROTO_IPV6,IPV6_DONTFRAG,&f,sizeof(f));
 #endif
 			}
 			f = 0; setsockopt(s,SOL_SOCKET,SO_REUSEADDR,(void *)&f,sizeof(f));
@@ -664,6 +655,36 @@ public:
 		return (PhySocket *)&sws;
 	}
 
+	/**
+	 * Try to set buffer sizes as close to the given value as possible
+	 *
+	 * This will try the specified value and then lower values in 16K increments
+	 * until one works.
+	 *
+	 * @param sock Socket
+	 * @param bufferSize Desired buffer sizes
+	 */
+	inline void setBufferSizes(const PhySocket *sock,int bufferSize)
+	{
+		PhySocketImpl &sws = *(reinterpret_cast<PhySocketImpl *>(sock));
+		if (bufferSize > 0) {
+			int bs = bufferSize;
+			while (bs >= 65536) {
+				int tmpbs = bs;
+				if (::setsockopt(sws.sock,SOL_SOCKET,SO_RCVBUF,(const char *)&tmpbs,sizeof(tmpbs)) == 0)
+					break;
+				bs -= 16384;
+			}
+			bs = bufferSize;
+			while (bs >= 65536) {
+				int tmpbs = bs;
+				if (::setsockopt(sws.sock,SOL_SOCKET,SO_SNDBUF,(const char *)&tmpbs,sizeof(tmpbs)) == 0)
+					break;
+				bs -= 16384;
+			}
+		}
+	}
+
 	/**
 	 * Attempt to send data to a stream socket (non-blocking)
 	 *
@@ -778,6 +799,26 @@ public:
 		}
 	}
 
+	/**
+	 * Set whether we want to be notified that a socket is readable
+	 *
+	 * This is primarily for raw sockets added with wrapSocket(). It could be
+	 * used with others, but doing so would essentially lock them and prevent
+	 * data from being read from them until this is set to 'true' again.
+	 *
+	 * @param sock Socket to modify
+	 * @param notifyReadable True if socket should be monitored for readability
+	 */
+	inline const void setNotifyReadable(PhySocket *sock,bool notifyReadable)
+	{
+		PhySocketImpl &sws = *(reinterpret_cast<PhySocketImpl *>(sock));
+		if (notifyReadable) {
+			FD_SET(sws.sock,&_readfds);
+		} else {
+			FD_CLR(sws.sock,&_readfds);
+		}
+	}
+
 	/**
 	 * Wait for activity and handle one or more events
 	 *
@@ -927,7 +968,7 @@ public:
 					}
 					if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) {
 						try {
-							_handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr));
+							//_handler->phyOnUnixWritable((PhySocket *)&(*s),&(s->uptr));
 						} catch ( ... ) {}
 					}
 #endif // __UNIX_LIKE__
@@ -962,25 +1003,15 @@ public:
 #endif // __UNIX_LIKE__
 					break;
 
-				case ZT_PHY_SOCKET_PAIR_ENDPOINT: {
-#ifdef __UNIX_LIKE__
-					ZT_PHY_SOCKFD_TYPE sock = s->sock; // if closed, s->sock becomes invalid as s is no longer dereferencable
-					if (FD_ISSET(sock,&rfds)) {
-						long n = (long)::read(sock,buf,sizeof(buf));
-						if (n <= 0) {
-							this->close((PhySocket *)&(*s),true);
-						} else {
-							try {
-								_handler->phyOnSocketPairEndpointData((PhySocket *)&(*s),&(s->uptr),(void *)buf,(unsigned long)n);
-							} catch ( ... ) {}
-						}
-					}
-					if ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds))) {
+				case ZT_PHY_SOCKET_FD: {
+					ZT_PHY_SOCKFD_TYPE sock = s->sock;
+					const bool readable = ((FD_ISSET(sock,&rfds))&&(FD_ISSET(sock,&_readfds)));
+					const bool writable = ((FD_ISSET(sock,&wfds))&&(FD_ISSET(sock,&_writefds)));
+					if ((readable)||(writable)) {
 						try {
-							_handler->phyOnSocketPairEndpointWritable((PhySocket *)&(*s),&(s->uptr));
+							_handler->phyOnFileDescriptorActivity((PhySocket *)&(*s),&(s->uptr),readable,writable);
 						} catch ( ... ) {}
 					}
-#endif // __UNIX_LIKE__
 				}	break;
 
 				default:
@@ -1012,7 +1043,8 @@ public:
 		FD_CLR(sws.sock,&_exceptfds);
 #endif
 
-		ZT_PHY_CLOSE_SOCKET(sws.sock);
+		if (sws.type != ZT_PHY_SOCKET_FD)
+			ZT_PHY_CLOSE_SOCKET(sws.sock);
 
 #ifdef __UNIX_LIKE__
 		if (sws.type == ZT_PHY_SOCKET_UNIX_LISTEN)
@@ -1039,12 +1071,6 @@ public:
 					} catch ( ... ) {}
 #endif // __UNIX_LIKE__
 					break;
-				case ZT_PHY_SOCKET_PAIR_ENDPOINT:
-#ifdef __UNIX_LIKE__
-					try {
-						_handler->phyOnSocketPairEndpointClose(sock,&(sws.uptr));
-					} catch ( ... ) {}
-#endif // __UNIX_LIKE__
 				default:
 					break;
 			}

root-topology/Makefile

@@ -1,17 +0,0 @@
-all:	FORCE
-	g++ -o mktopology mktopology.cpp ../osdep/OSUtils.cpp ../node/Utils.cpp ../node/InetAddress.cpp ../node/Identity.cpp ../node/C25519.cpp ../node/Salsa20.cpp ../node/Dictionary.cpp ../node/SHA512.cpp
-	gcc -o bin2c bin2c.c
-
-official: FORCE
-	rm -f ZT_DEFAULT_ROOT_TOPOLOGY.dict
-	./mktopology >ZT_DEFAULT_ROOT_TOPOLOGY.dict
-	./bin2c ZT_DEFAULT_ROOT_TOPOLOGY < ZT_DEFAULT_ROOT_TOPOLOGY.dict > ZT_DEFAULT_ROOT_TOPOLOGY.c
-	ls -l ZT_DEFAULT_ROOT_TOPOLOGY.c
-
-clean:
-	rm -f *.o mktopology bin2c
-
-realclean: clean
-	rm -f ZT_DEFAULT_ROOT_TOPOLOGY.c ZT_DEFAULT_ROOT_TOPOLOGY.dict
-
-FORCE:

root-topology/README.md

@@ -1,18 +0,0 @@
-This folder contains the source files to compile the signed network root topology dictionary. Users outside ZeroTier won't find this useful except for testing, since the root topology must be signed by the root topology authority (public identity in root-topology-authority.public) to be considered valid.
-
-Keys in the root topology dictionary are:
-
- * **rootservers**: contains another Dictionary mapping rootserver address to rootserver definition
-   * **##########**: rootserver address, contains rootserver definition
-     * **id**: rootserver identity (public) in string-serialized format
-     * **udp**: comma-delimited list of ip/port UDP addresses of node
-     * **tcp**: *DEPRECATED* comma-delimited list of ip/port TCP addresses of node
-     * **desc**: human-readable description (optional)
-     * **dns**: DNS name (optional, not currently used for anything)
- * **noupdate**: if the value of this is '1', do not auto-update from ZeroTier's servers
-
-ZT_DEFAULT_ROOT_TOPOLOGY.c contains the current default value, and this URL is periodically checked for updates:
-
-http://download.zerotier.com/net/topology/ROOT
-
-Obviously nothing prevents OSS users from replacing this topology with their own, changing the hard coded topology signing identity and update URL in Defaults, and signing their own dictionary. But doing so would yield a network that would have a tough(ish) time talking to the main one. Since the main network is a free service, why bother? (Except for building testnets, which ZeroTier already does for internal testing.)

root-topology/ZT_DEFAULT_ROOT_TOPOLOGY.c

@@ -1,90 +0,0 @@
-static unsigned char ZT_DEFAULT_ROOT_TOPOLOGY[] = {
-	0x72, 0x6f, 0x6f, 0x74, 0x73, 0x65, 0x72, 0x76, 0x65, 0x72, 0x73, 0x3d, 0x37, 0x65, 0x31, 0x39,
-	0x38, 0x37, 0x36, 0x61, 0x62, 0x61, 0x5c, 0x3d, 0x69, 0x64, 0x5c, 0x5c, 0x5c, 0x3d, 0x37, 0x65,
-	0x31, 0x39, 0x38, 0x37, 0x36, 0x61, 0x62, 0x61, 0x3a, 0x30, 0x3a, 0x32, 0x61, 0x36, 0x65, 0x32,
-	0x62, 0x32, 0x33, 0x31, 0x38, 0x39, 0x33, 0x30, 0x66, 0x36, 0x30, 0x65, 0x62, 0x30, 0x39, 0x37,
-	0x66, 0x37, 0x30, 0x64, 0x30, 0x66, 0x34, 0x62, 0x30, 0x32, 0x38, 0x62, 0x32, 0x63, 0x64, 0x36,
-	0x64, 0x33, 0x64, 0x30, 0x63, 0x36, 0x33, 0x63, 0x30, 0x31, 0x34, 0x62, 0x39, 0x30, 0x33, 0x39,
-	0x66, 0x66, 0x33, 0x35, 0x33, 0x39, 0x30, 0x65, 0x34, 0x31, 0x31, 0x38, 0x31, 0x66, 0x32, 0x31,
-	0x36, 0x66, 0x62, 0x32, 0x65, 0x36, 0x66, 0x61, 0x38, 0x64, 0x39, 0x35, 0x63, 0x31, 0x65, 0x65,
-	0x39, 0x36, 0x36, 0x37, 0x31, 0x35, 0x36, 0x34, 0x31, 0x31, 0x39, 0x30, 0x35, 0x63, 0x33, 0x64,
-	0x63, 0x63, 0x66, 0x65, 0x61, 0x37, 0x38, 0x64, 0x38, 0x63, 0x36, 0x64, 0x66, 0x61, 0x66, 0x62,
-	0x61, 0x36, 0x38, 0x38, 0x31, 0x37, 0x30, 0x62, 0x33, 0x66, 0x61, 0x5c, 0x5c, 0x6e, 0x75, 0x64,
-	0x70, 0x5c, 0x5c, 0x5c, 0x3d, 0x31, 0x39, 0x38, 0x2e, 0x31, 0x39, 0x39, 0x2e, 0x39, 0x37, 0x2e,
-	0x32, 0x32, 0x30, 0x2f, 0x39, 0x39, 0x39, 0x33, 0x5c, 0x5c, 0x6e, 0x74, 0x63, 0x70, 0x5c, 0x5c,
-	0x5c, 0x3d, 0x31, 0x39, 0x38, 0x2e, 0x31, 0x39, 0x39, 0x2e, 0x39, 0x37, 0x2e, 0x32, 0x32, 0x30,
-	0x2f, 0x34, 0x34, 0x33, 0x5c, 0x5c, 0x6e, 0x64, 0x65, 0x73, 0x63, 0x5c, 0x5c, 0x5c, 0x3d, 0x53,
-	0x61, 0x6e, 0x20, 0x46, 0x72, 0x61, 0x6e, 0x63, 0x69, 0x73, 0x63, 0x6f, 0x2c, 0x20, 0x43, 0x61,
-	0x6c, 0x69, 0x66, 0x6f, 0x72, 0x6e, 0x69, 0x61, 0x2c, 0x20, 0x55, 0x53, 0x41, 0x5c, 0x5c, 0x6e,
-	0x5c, 0x6e, 0x38, 0x38, 0x34, 0x31, 0x34, 0x30, 0x38, 0x61, 0x32, 0x65, 0x5c, 0x3d, 0x69, 0x64,
-	0x5c, 0x5c, 0x5c, 0x3d, 0x38, 0x38, 0x34, 0x31, 0x34, 0x30, 0x38, 0x61, 0x32, 0x65, 0x3a, 0x30,
-	0x3a, 0x62, 0x62, 0x31, 0x64, 0x33, 0x31, 0x66, 0x32, 0x63, 0x33, 0x32, 0x33, 0x65, 0x32, 0x36,
-	0x34, 0x65, 0x39, 0x65, 0x36, 0x34, 0x31, 0x37, 0x32, 0x63, 0x31, 0x61, 0x37, 0x34, 0x66, 0x37,
-	0x37, 0x38, 0x39, 0x39, 0x35, 0x35, 0x35, 0x65, 0x64, 0x31, 0x30, 0x37, 0x35, 0x31, 0x63, 0x64,
-	0x35, 0x36, 0x65, 0x38, 0x36, 0x34, 0x30, 0x35, 0x63, 0x64, 0x65, 0x31, 0x31, 0x38, 0x64, 0x30,
-	0x32, 0x64, 0x66, 0x66, 0x65, 0x35, 0x35, 0x35, 0x64, 0x34, 0x36, 0x32, 0x63, 0x63, 0x66, 0x36,
-	0x61, 0x38, 0x35, 0x62, 0x35, 0x36, 0x33, 0x31, 0x63, 0x31, 0x32, 0x33, 0x35, 0x30, 0x63, 0x38,
-	0x64, 0x35, 0x64, 0x63, 0x34, 0x30, 0x39, 0x62, 0x61, 0x31, 0x30, 0x62, 0x39, 0x30, 0x32, 0x35,
-	0x64, 0x30, 0x66, 0x34, 0x34, 0x35, 0x63, 0x66, 0x34, 0x34, 0x39, 0x64, 0x39, 0x32, 0x62, 0x31,
-	0x63, 0x5c, 0x5c, 0x6e, 0x75, 0x64, 0x70, 0x5c, 0x5c, 0x5c, 0x3d, 0x31, 0x30, 0x37, 0x2e, 0x31,
-	0x39, 0x31, 0x2e, 0x34, 0x36, 0x2e, 0x32, 0x31, 0x30, 0x2f, 0x39, 0x39, 0x39, 0x33, 0x5c, 0x5c,
-	0x6e, 0x74, 0x63, 0x70, 0x5c, 0x5c, 0x5c, 0x3d, 0x31, 0x30, 0x37, 0x2e, 0x31, 0x39, 0x31, 0x2e,
-	0x34, 0x36, 0x2e, 0x32, 0x31, 0x30, 0x2f, 0x34, 0x34, 0x33, 0x5c, 0x5c, 0x6e, 0x64, 0x65, 0x73,
-	0x63, 0x5c, 0x5c, 0x5c, 0x3d, 0x50, 0x61, 0x72, 0x69, 0x73, 0x2c, 0x20, 0x46, 0x72, 0x61, 0x6e,
-	0x63, 0x65, 0x5c, 0x5c, 0x6e, 0x5c, 0x6e, 0x38, 0x61, 0x63, 0x66, 0x30, 0x35, 0x39, 0x66, 0x65,
-	0x33, 0x5c, 0x3d, 0x69, 0x64, 0x5c, 0x5c, 0x5c, 0x3d, 0x38, 0x61, 0x63, 0x66, 0x30, 0x35, 0x39,
-	0x66, 0x65, 0x33, 0x3a, 0x30, 0x3a, 0x34, 0x38, 0x32, 0x66, 0x36, 0x65, 0x65, 0x35, 0x64, 0x66,
-	0x65, 0x39, 0x30, 0x32, 0x33, 0x31, 0x39, 0x62, 0x34, 0x31, 0x39, 0x64, 0x65, 0x35, 0x62, 0x64,
-	0x63, 0x37, 0x36, 0x35, 0x32, 0x30, 0x39, 0x63, 0x30, 0x65, 0x63, 0x64, 0x61, 0x33, 0x38, 0x63,
-	0x34, 0x64, 0x36, 0x65, 0x34, 0x66, 0x63, 0x66, 0x30, 0x64, 0x33, 0x33, 0x36, 0x35, 0x38, 0x33,
-	0x39, 0x38, 0x62, 0x34, 0x35, 0x32, 0x37, 0x64, 0x63, 0x64, 0x32, 0x32, 0x66, 0x39, 0x33, 0x31,
-	0x31, 0x32, 0x66, 0x62, 0x39, 0x62, 0x65, 0x66, 0x64, 0x30, 0x32, 0x66, 0x64, 0x37, 0x38, 0x62,
-	0x66, 0x37, 0x32, 0x36, 0x31, 0x62, 0x33, 0x33, 0x33, 0x66, 0x63, 0x31, 0x30, 0x35, 0x64, 0x31,
-	0x39, 0x32, 0x61, 0x36, 0x32, 0x33, 0x63, 0x61, 0x39, 0x65, 0x35, 0x30, 0x66, 0x63, 0x36, 0x30,
-	0x62, 0x33, 0x37, 0x34, 0x61, 0x35, 0x5c, 0x5c, 0x6e, 0x75, 0x64, 0x70, 0x5c, 0x5c, 0x5c, 0x3d,
-	0x31, 0x36, 0x32, 0x2e, 0x32, 0x34, 0x33, 0x2e, 0x37, 0x37, 0x2e, 0x31, 0x31, 0x31, 0x2f, 0x39,
-	0x39, 0x39, 0x33, 0x5c, 0x5c, 0x6e, 0x74, 0x63, 0x70, 0x5c, 0x5c, 0x5c, 0x3d, 0x31, 0x36, 0x32,
-	0x2e, 0x32, 0x34, 0x33, 0x2e, 0x37, 0x37, 0x2e, 0x31, 0x31, 0x31, 0x2f, 0x34, 0x34, 0x33, 0x5c,
-	0x5c, 0x6e, 0x64, 0x65, 0x73, 0x63, 0x5c, 0x5c, 0x5c, 0x3d, 0x4e, 0x65, 0x77, 0x20, 0x59, 0x6f,
-	0x72, 0x6b, 0x2c, 0x20, 0x4e, 0x65, 0x77, 0x20, 0x59, 0x6f, 0x72, 0x6b, 0x2c, 0x20, 0x55, 0x53,
-	0x41, 0x5c, 0x5c, 0x6e, 0x5c, 0x6e, 0x39, 0x64, 0x32, 0x31, 0x39, 0x30, 0x33, 0x39, 0x66, 0x33,
-	0x5c, 0x3d, 0x69, 0x64, 0x5c, 0x5c, 0x5c, 0x3d, 0x39, 0x64, 0x32, 0x31, 0x39, 0x30, 0x33, 0x39,
-	0x66, 0x33, 0x3a, 0x30, 0x3a, 0x30, 0x31, 0x66, 0x30, 0x39, 0x32, 0x32, 0x61, 0x39, 0x38, 0x65,
-	0x33, 0x62, 0x33, 0x34, 0x65, 0x62, 0x63, 0x62, 0x66, 0x66, 0x33, 0x33, 0x33, 0x32, 0x36, 0x39,
-	0x64, 0x63, 0x32, 0x36, 0x35, 0x64, 0x37, 0x61, 0x30, 0x32, 0x30, 0x61, 0x61, 0x62, 0x36, 0x39,
-	0x64, 0x37, 0x32, 0x62, 0x65, 0x34, 0x64, 0x34, 0x61, 0x63, 0x63, 0x39, 0x63, 0x38, 0x63, 0x39,
-	0x32, 0x39, 0x34, 0x37, 0x38, 0x35, 0x37, 0x37, 0x31, 0x32, 0x35, 0x36, 0x63, 0x64, 0x31, 0x64,
-	0x39, 0x34, 0x32, 0x61, 0x39, 0x30, 0x64, 0x31, 0x62, 0x64, 0x31, 0x64, 0x32, 0x64, 0x63, 0x61,
-	0x33, 0x65, 0x61, 0x38, 0x34, 0x65, 0x66, 0x37, 0x64, 0x38, 0x35, 0x61, 0x66, 0x65, 0x36, 0x36,
-	0x31, 0x31, 0x66, 0x62, 0x34, 0x33, 0x66, 0x66, 0x30, 0x62, 0x37, 0x34, 0x31, 0x32, 0x36, 0x64,
-	0x39, 0x30, 0x61, 0x36, 0x65, 0x5c, 0x5c, 0x6e, 0x75, 0x64, 0x70, 0x5c, 0x5c, 0x5c, 0x3d, 0x31,
-	0x32, 0x38, 0x2e, 0x31, 0x39, 0x39, 0x2e, 0x31, 0x39, 0x37, 0x2e, 0x32, 0x31, 0x37, 0x2f, 0x39,
-	0x39, 0x39, 0x33, 0x5c, 0x5c, 0x6e, 0x74, 0x63, 0x70, 0x5c, 0x5c, 0x5c, 0x3d, 0x31, 0x32, 0x38,
-	0x2e, 0x31, 0x39, 0x39, 0x2e, 0x31, 0x39, 0x37, 0x2e, 0x32, 0x31, 0x37, 0x2f, 0x34, 0x34, 0x33,
-	0x5c, 0x5c, 0x6e, 0x64, 0x65, 0x73, 0x63, 0x5c, 0x5c, 0x5c, 0x3d, 0x53, 0x69, 0x6e, 0x67, 0x61,
-	0x70, 0x6f, 0x72, 0x65, 0x5c, 0x5c, 0x6e, 0x5c, 0x6e, 0x0a, 0x7e, 0x21, 0x65, 0x64, 0x32, 0x35,
-	0x35, 0x31, 0x39, 0x3d, 0x38, 0x33, 0x32, 0x62, 0x33, 0x35, 0x64, 0x61, 0x64, 0x64, 0x37, 0x66,
-	0x35, 0x36, 0x66, 0x66, 0x33, 0x38, 0x31, 0x66, 0x61, 0x37, 0x32, 0x31, 0x64, 0x65, 0x37, 0x64,
-	0x35, 0x62, 0x65, 0x34, 0x63, 0x65, 0x62, 0x66, 0x63, 0x63, 0x63, 0x32, 0x30, 0x30, 0x32, 0x30,
-	0x38, 0x33, 0x38, 0x30, 0x64, 0x33, 0x30, 0x38, 0x34, 0x66, 0x36, 0x34, 0x38, 0x65, 0x32, 0x63,
-	0x31, 0x61, 0x35, 0x63, 0x66, 0x34, 0x33, 0x65, 0x35, 0x39, 0x66, 0x39, 0x32, 0x61, 0x36, 0x36,
-	0x35, 0x64, 0x66, 0x34, 0x64, 0x62, 0x63, 0x62, 0x38, 0x33, 0x37, 0x38, 0x38, 0x66, 0x36, 0x62,
-	0x64, 0x36, 0x37, 0x37, 0x66, 0x30, 0x32, 0x62, 0x32, 0x31, 0x30, 0x65, 0x35, 0x30, 0x63, 0x61,
-	0x66, 0x65, 0x66, 0x64, 0x32, 0x65, 0x66, 0x31, 0x38, 0x39, 0x62, 0x62, 0x66, 0x34, 0x38, 0x31,
-	0x62, 0x64, 0x30, 0x32, 0x63, 0x64, 0x63, 0x39, 0x38, 0x34, 0x35, 0x33, 0x38, 0x37, 0x64, 0x38,
-	0x34, 0x39, 0x62, 0x63, 0x35, 0x36, 0x66, 0x39, 0x63, 0x37, 0x32, 0x35, 0x31, 0x65, 0x35, 0x64,
-	0x30, 0x65, 0x61, 0x34, 0x34, 0x34, 0x66, 0x66, 0x63, 0x66, 0x38, 0x66, 0x37, 0x32, 0x32, 0x63,
-	0x32, 0x66, 0x65, 0x62, 0x38, 0x39, 0x36, 0x30, 0x33, 0x61, 0x30, 0x65, 0x35, 0x62, 0x61, 0x32,
-	0x39, 0x35, 0x66, 0x63, 0x0a, 0x7e, 0x21, 0x73, 0x69, 0x67, 0x69, 0x64, 0x3d, 0x37, 0x37, 0x37,
-	0x39, 0x32, 0x62, 0x31, 0x63, 0x30, 0x32, 0x3a, 0x30, 0x3a, 0x62, 0x35, 0x63, 0x33, 0x36, 0x31,
-	0x65, 0x38, 0x65, 0x39, 0x63, 0x32, 0x31, 0x35, 0x34, 0x65, 0x38, 0x32, 0x63, 0x33, 0x65, 0x39,
-	0x30, 0x32, 0x66, 0x64, 0x66, 0x63, 0x33, 0x33, 0x37, 0x34, 0x36, 0x38, 0x62, 0x30, 0x39, 0x32,
-	0x61, 0x37, 0x63, 0x34, 0x64, 0x38, 0x64, 0x63, 0x36, 0x38, 0x35, 0x63, 0x33, 0x37, 0x65, 0x62,
-	0x31, 0x30, 0x65, 0x65, 0x34, 0x66, 0x33, 0x63, 0x31, 0x37, 0x63, 0x63, 0x30, 0x62, 0x62, 0x31,
-	0x64, 0x30, 0x32, 0x34, 0x31, 0x36, 0x37, 0x65, 0x38, 0x63, 0x62, 0x30, 0x38, 0x32, 0x34, 0x64,
-	0x31, 0x32, 0x32, 0x36, 0x33, 0x34, 0x32, 0x38, 0x33, 0x37, 0x33, 0x35, 0x38, 0x32, 0x64, 0x61,
-	0x33, 0x64, 0x30, 0x61, 0x39, 0x61, 0x31, 0x34, 0x62, 0x33, 0x36, 0x65, 0x34, 0x35, 0x34, 0x36,
-	0x63, 0x33, 0x31, 0x37, 0x65, 0x38, 0x31, 0x31, 0x65, 0x36, 0x0a, 0x7e, 0x21, 0x73, 0x69, 0x67,
-	0x74, 0x73, 0x3d, 0x31, 0x34, 0x65, 0x30, 0x63, 0x62, 0x62, 0x39, 0x38, 0x64, 0x36, 0x0a 
-};
-#define ZT_DEFAULT_ROOT_TOPOLOGY_LEN 1391

root-topology/ZT_DEFAULT_ROOT_TOPOLOGY.dict

@@ -1,4 +0,0 @@
-rootservers=7e19876aba\=id\\\=7e19876aba:0:2a6e2b2318930f60eb097f70d0f4b028b2cd6d3d0c63c014b9039ff35390e41181f216fb2e6fa8d95c1ee9667156411905c3dccfea78d8c6dfafba688170b3fa\\nudp\\\=198.199.97.220/9993\\ntcp\\\=198.199.97.220/443\\ndesc\\\=San Francisco, California, USA\\n\n8841408a2e\=id\\\=8841408a2e:0:bb1d31f2c323e264e9e64172c1a74f77899555ed10751cd56e86405cde118d02dffe555d462ccf6a85b5631c12350c8d5dc409ba10b9025d0f445cf449d92b1c\\nudp\\\=107.191.46.210/9993\\ntcp\\\=107.191.46.210/443\\ndesc\\\=Paris, France\\n\n8acf059fe3\=id\\\=8acf059fe3:0:482f6ee5dfe902319b419de5bdc765209c0ecda38c4d6e4fcf0d33658398b4527dcd22f93112fb9befd02fd78bf7261b333fc105d192a623ca9e50fc60b374a5\\nudp\\\=162.243.77.111/9993\\ntcp\\\=162.243.77.111/443\\ndesc\\\=New York, New York, USA\\n\n9d219039f3\=id\\\=9d219039f3:0:01f0922a98e3b34ebcbff333269dc265d7a020aab69d72be4d4acc9c8c9294785771256cd1d942a90d1bd1d2dca3ea84ef7d85afe6611fb43ff0b74126d90a6e\\nudp\\\=128.199.197.217/9993\\ntcp\\\=128.199.197.217/443\\ndesc\\\=Singapore\\n\n
-~!ed25519=832b35dadd7f56ff381fa721de7d5be4cebfccc200208380d3084f648e2c1a5cf43e59f92a665df4dbcb83788f6bd677f02b210e50cafefd2ef189bbf481bd02cdc9845387d849bc56f9c7251e5d0ea444ffcf8f722c2feb89603a0e5ba295fc
-~!sigid=77792b1c02:0:b5c361e8e9c2154e82c3e902fdfc337468b092a7c4d8dc685c37eb10ee4f3c17cc0bb1d024167e8cb0824d12263428373582da3d0a9a14b36e4546c317e811e6
-~!sigts=14e0cbb98d6

root-topology/bin2c.c

@@ -1,57 +0,0 @@
-/**
-   Converts input from stdin into an array of binary data for use in C.
-
-   License: Public Domain
-
-   Usage: app VariableName < input > output.c
-*/
-
-#include <stdint.h> /* uintXX_t */
-#include <inttypes.h> /* PRIuXX macros */
-#include <stdio.h>
-
-static char const * appName = 0;
-
-static void usage()
-{
-    printf("Usage: %s OBJECT_NAME < input > output.c\n\n", appName );
-}
-
-int main( int argc, char const ** argv )
-{
-    appName = argv[0];
-    if( (argc != 2) || (argv[1][0] == '-') )
-    {
-	usage();
-	return 1;
-    }
-    char const * varname = argv[1];
-    enum { bufSize = 1024 * 8 };
-    unsigned char buf[bufSize];
-    size_t rd = 0;
-    size_t i = 0;
-    size_t flip = 0;
-
-    printf( "static unsigned char %s[] = {\n\t", varname);
-    uint32_t size = 0;
-    while( 0 != (rd = fread( buf, 1, bufSize, stdin ) ) )
-    {
-	size += rd;
-	for(i = 0; i < rd; ++i )
-	{
-	    printf( "0x%02x", buf[i] );
-	    if( !( (rd < bufSize) && (i == rd-1)) ) putchar(',');
-	    if( 16 == ++flip )
-	    {
-		flip = 0;
-		printf("\n\t");
-	    }
-	    else putchar(' ');
-	}
-    }
-    printf("\n};\n");
-    printf("#define %s_LEN %llu\n",varname,(unsigned long long)size);
-    //printf( "enum { %s_length = %"PRIu32"%s }; ", varname, size,"UL");
-    //printf("enum { %s_length = sizeof(%s) };\n", varname, varname );
-    return 0;
-}

root-topology/mktopology.cpp

@@ -1,68 +0,0 @@
-#include <stdio.h>
-#include <string.h>
-#include <stdlib.h>
-
-#include <string>
-#include <iostream>
-#include <map>
-
-#include "../osdep/OSUtils.hpp"
-#include "../node/Identity.hpp"
-#include "../node/Dictionary.hpp"
-
-using namespace ZeroTier;
-
-int main(int argc,char **argv)
-{
-	std::string buf;
-
-	// Read root-topology-authority.secret signing authority, must be symlinked and online
-	Identity topologyAuthority;
-	if (OSUtils::readFile("root-topology-authority.secret",buf))
-		topologyAuthority.fromString(buf);
-	else std::cerr << "Warning: root-topology-authority.secret not found, creating unsigned topology." << std::endl;
-
-	Dictionary topology;
-
-	// Read template.dict to populate default fields in root topology
-	// if this file exists. Otherwise we just start empty.
-	buf.clear();
-	if (OSUtils::readFile("template.dict",buf))
-		topology.fromString(buf);
-
-	// Read all entries in rootservers/ that correspond to rootserver entry dictionaries
-	// and add them to topology under rootservers/ subkey.
-	Dictionary rootservers;
-	std::vector<std::string> rootserverDictionaries(OSUtils::listDirectory("rootservers"));
-	for(std::vector<std::string>::const_iterator sn(rootserverDictionaries.begin());sn!=rootserverDictionaries.end();++sn) {
-		if (sn->length() == 10) {
-			buf.clear();
-			if (!OSUtils::readFile((std::string("rootservers/")+(*sn)).c_str(),buf)) {
-				std::cerr << "Cannot read rootservers/" << *sn << std::endl;
-				return 1;
-			}
-			rootservers[*sn] = buf;
-		}
-	}
-	topology["rootservers"] = rootservers.toString();
-
-	if ((topologyAuthority)&&(topologyAuthority.hasPrivate())) {
-		// Sign topology with root-topology-authority.secret
-		if (!topology.sign(topologyAuthority,OSUtils::now())) {
-			std::cerr << "Unable to sign!" << std::endl;
-			return 1;
-		}
-
-		// Test signature to make sure signing worked
-		Dictionary test(topology.toString());
-		if (!test.verify(topologyAuthority)) {
-			std::cerr << "Test verification of signed dictionary failed!" << std::endl;
-			return 1;
-		}
-	}
-
-	// Output to stdout
-	std::cout << topology.toString();
-
-	return 0;
-}

root-topology/rootservers/7e19876aba

@@ -1,4 +0,0 @@
-id=7e19876aba:0:2a6e2b2318930f60eb097f70d0f4b028b2cd6d3d0c63c014b9039ff35390e41181f216fb2e6fa8d95c1ee9667156411905c3dccfea78d8c6dfafba688170b3fa
-udp=198.199.97.220/9993
-tcp=198.199.97.220/443
-desc=San Francisco, California, USA

root-topology/rootservers/8841408a2e

@@ -1,4 +0,0 @@
-id=8841408a2e:0:bb1d31f2c323e264e9e64172c1a74f77899555ed10751cd56e86405cde118d02dffe555d462ccf6a85b5631c12350c8d5dc409ba10b9025d0f445cf449d92b1c
-udp=107.191.46.210/9993
-tcp=107.191.46.210/443
-desc=Paris, France

root-topology/rootservers/8acf059fe3

@@ -1,4 +0,0 @@
-id=8acf059fe3:0:482f6ee5dfe902319b419de5bdc765209c0ecda38c4d6e4fcf0d33658398b4527dcd22f93112fb9befd02fd78bf7261b333fc105d192a623ca9e50fc60b374a5
-udp=162.243.77.111/9993
-tcp=162.243.77.111/443
-desc=New York, New York, USA

root-topology/rootservers/9d219039f3

@@ -1,4 +0,0 @@
-id=9d219039f3:0:01f0922a98e3b34ebcbff333269dc265d7a020aab69d72be4d4acc9c8c9294785771256cd1d942a90d1bd1d2dca3ea84ef7d85afe6611fb43ff0b74126d90a6e
-udp=128.199.197.217/9993
-tcp=128.199.197.217/443
-desc=Singapore

root-topology/test/README.md

@@ -1,6 +0,0 @@
-Test Root Topology Script
-======
-
-This builds a test-root-topology from any number of running test-rootserver-# Docker containers. This can then be used with the (undocumented) -T (override root topology) option to run test networks under Docker.
-
-Once you have a local Docker test network running you can use iptables rules to simulate a variety of network pathologies, or you can just use it to test any new changes to the protocol or node behavior at some limited scale.

root-topology/test/create-test-root-topology.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-
-if [ ! -e ../mktopology ]; then
-	echo 'Build ../mktopology first!'
-	exit 1
-fi
-
-echo 'Populating rootservers/* with all Docker test-rootserver-* container IPs and identities...'
-
-rm -rf rootservers
-mkdir rootservers
-
-for cid in `docker ps -f 'name=test-rootserver-*' -q`; do
-	id=`docker exec $cid cat /var/lib/zerotier-one/identity.secret | cut -d : -f 1-3`
-	ztaddr=`echo $id | cut -d : -f 1`
-	ip=`docker exec $cid ifconfig | sed -En 's/127.0.0.1//;s/.*inet (addr:)?(([0-9]*\.){3}[0-9]*).*/\2/p'`
-	echo $cid $ztaddr $id $ip
-	echo "id=$id" >rootservers/$ztaddr
-	echo "udp=$ip/9993" >>rootservers/$ztaddr
-done
-
-echo 'Creating test-root-topology...'
-
-rm -f test-root-topology
-../mktopology >test-root-topology
-
-echo 'Done!'
-echo
-cat test-root-topology
-
-exit 0

selftest.cpp

@@ -50,8 +50,8 @@
 #include "node/C25519.hpp"
 #include "node/Poly1305.hpp"
 #include "node/CertificateOfMembership.hpp"
-#include "node/Defaults.hpp"
 #include "node/Node.hpp"
+#include "node/IncomingPacket.hpp"
 
 #include "osdep/OSUtils.hpp"
 #include "osdep/Phy.hpp"
@@ -161,27 +161,27 @@ static int testCrypto()
 		memset(buf2,0,sizeof(buf2));
 		memset(buf3,0,sizeof(buf3));
 		Salsa20 s20;
-		s20.init("12345678123456781234567812345678",256,"12345678",20);
-		s20.encrypt(buf1,buf2,sizeof(buf1));
-		s20.init("12345678123456781234567812345678",256,"12345678",20);
-		s20.decrypt(buf2,buf3,sizeof(buf2));
+		s20.init("12345678123456781234567812345678",256,"12345678");
+		s20.encrypt20(buf1,buf2,sizeof(buf1));
+		s20.init("12345678123456781234567812345678",256,"12345678");
+		s20.decrypt20(buf2,buf3,sizeof(buf2));
 		if (memcmp(buf1,buf3,sizeof(buf1))) {
 			std::cout << "FAIL (encrypt/decrypt test)" << std::endl;
 			return -1;
 		}
 	}
-	Salsa20 s20(s20TV0Key,256,s20TV0Iv,20);
+	Salsa20 s20(s20TV0Key,256,s20TV0Iv);
 	memset(buf1,0,sizeof(buf1));
 	memset(buf2,0,sizeof(buf2));
-	s20.encrypt(buf1,buf2,64);
+	s20.encrypt20(buf1,buf2,64);
 	if (memcmp(buf2,s20TV0Ks,64)) {
 		std::cout << "FAIL (test vector 0)" << std::endl;
 		return -1;
 	}
-	s20.init(s2012TV0Key,256,s2012TV0Iv,12);
+	s20.init(s2012TV0Key,256,s2012TV0Iv);
 	memset(buf1,0,sizeof(buf1));
 	memset(buf2,0,sizeof(buf2));
-	s20.encrypt(buf1,buf2,64);
+	s20.encrypt12(buf1,buf2,64);
 	if (memcmp(buf2,s2012TV0Ks,64)) {
 		std::cout << "FAIL (test vector 1)" << std::endl;
 		return -1;
@@ -194,34 +194,16 @@ static int testCrypto()
 	std::cout << "[crypto] Salsa20 SSE: DISABLED" << std::endl;
 #endif
 
-	std::cout << "[crypto] Benchmarking Salsa20/8... "; std::cout.flush();
-	{
-		unsigned char *bb = (unsigned char *)::malloc(1234567);
-		for(unsigned int i=0;i<1234567;++i)
-			bb[i] = (unsigned char)i;
-		Salsa20 s20(s20TV0Key,256,s20TV0Iv,8);
-		double bytes = 0.0;
-		uint64_t start = OSUtils::now();
-		for(unsigned int i=0;i<200;++i) {
-			s20.encrypt(bb,bb,1234567);
-			bytes += 1234567.0;
-		}
-		uint64_t end = OSUtils::now();
-		SHA512::hash(buf1,bb,1234567);
-		std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second (" << Utils::hex(buf1,16) << ')' << std::endl;
-		::free((void *)bb);
-	}
-
 	std::cout << "[crypto] Benchmarking Salsa20/12... "; std::cout.flush();
 	{
 		unsigned char *bb = (unsigned char *)::malloc(1234567);
 		for(unsigned int i=0;i<1234567;++i)
 			bb[i] = (unsigned char)i;
-		Salsa20 s20(s20TV0Key,256,s20TV0Iv,12);
+		Salsa20 s20(s20TV0Key,256,s20TV0Iv);
 		double bytes = 0.0;
 		uint64_t start = OSUtils::now();
 		for(unsigned int i=0;i<200;++i) {
-			s20.encrypt(bb,bb,1234567);
+			s20.encrypt12(bb,bb,1234567);
 			bytes += 1234567.0;
 		}
 		uint64_t end = OSUtils::now();
@@ -235,11 +217,11 @@ static int testCrypto()
 		unsigned char *bb = (unsigned char *)::malloc(1234567);
 		for(unsigned int i=0;i<1234567;++i)
 			bb[i] = (unsigned char)i;
-		Salsa20 s20(s20TV0Key,256,s20TV0Iv,20);
+		Salsa20 s20(s20TV0Key,256,s20TV0Iv);
 		double bytes = 0.0;
 		uint64_t start = OSUtils::now();
 		for(unsigned int i=0;i<200;++i) {
-			s20.encrypt(bb,bb,1234567);
+			s20.encrypt20(bb,bb,1234567);
 			bytes += 1234567.0;
 		}
 		uint64_t end = OSUtils::now();
@@ -269,6 +251,35 @@ static int testCrypto()
 	}
 	std::cout << "PASS" << std::endl;
 
+	std::cout << "[crypto] Benchmarking Poly1305... "; std::cout.flush();
+	{
+		unsigned char *bb = (unsigned char *)::malloc(1234567);
+		for(unsigned int i=0;i<1234567;++i)
+			bb[i] = (unsigned char)i;
+		double bytes = 0.0;
+		uint64_t start = OSUtils::now();
+		for(unsigned int i=0;i<200;++i) {
+			Poly1305::compute(buf1,bb,1234567,poly1305TV0Key);
+			bytes += 1234567.0;
+		}
+		uint64_t end = OSUtils::now();
+		std::cout << ((bytes / 1048576.0) / ((double)(end - start) / 1000.0)) << " MiB/second" << std::endl;
+		::free((void *)bb);
+	}
+
+	/*
+	for(unsigned int d=8;d<=10;++d) {
+		for(int k=0;k<8;++k) {
+			std::cout << "[crypto] computeSalsa2012Sha512ProofOfWork(" << d << ",\"foobarbaz\",9) == "; std::cout.flush();
+			unsigned char result[16];
+			uint64_t start = OSUtils::now();
+			IncomingPacket::computeSalsa2012Sha512ProofOfWork(d,"foobarbaz",9,result);
+			uint64_t end = OSUtils::now();
+			std::cout << Utils::hex(result,16) << " -- valid: " << IncomingPacket::testSalsa2012Sha512ProofOfWorkResult(d,"foobarbaz",9,result) << ", " << (end - start) << "ms" << std::endl;
+		}
+	}
+	*/
+
 	std::cout << "[crypto] Testing C25519 and Ed25519 against test vectors... "; std::cout.flush();
 	for(int k=0;k<ZT_NUM_C25519_TEST_VECTORS;++k) {
 		C25519::Pair p1,p2;
@@ -582,23 +593,44 @@ static int testOther()
 	std::cout << "[other] Testing Hashtable... "; std::cout.flush();
 	{
 		Hashtable<uint64_t,std::string> ht;
-		Hashtable<uint64_t,std::string> ht2;
 		std::map<uint64_t,std::string> ref; // assume std::map works correctly :)
 		for(int x=0;x<2;++x) {
-			for(int i=0;i<25000;++i) {
+			for(int i=0;i<77777;++i) {
 				uint64_t k = rand();
 				while ((k == 0)||(ref.count(k) > 0))
 					++k;
 				std::string v("!");
 				for(int j=0;j<(int)(k % 64);++j)
 					v.push_back("0123456789"[rand() % 10]);
-				ht.set(k,v);
 				ref[k] = v;
+				ht.set(0xffffffffffffffffULL,v);
+				std::string &vref = ht[k];
+				vref = v;
+				ht.erase(0xffffffffffffffffULL);
 			}
 			if (ht.size() != ref.size()) {
 				std::cout << "FAILED! (size mismatch, original)" << std::endl;
 				return -1;
 			}
+			{
+				Hashtable<uint64_t,std::string>::Iterator i(ht);
+				uint64_t *k = (uint64_t *)0;
+				std::string *v = (std::string *)0;
+				while(i.next(k,v)) {
+					if (ref.find(*k)->second != *v) {
+						std::cout << "FAILED! (data mismatch!)" << std::endl;
+						return -1;
+					}
+				}
+			}
+			for(std::map<uint64_t,std::string>::const_iterator i(ref.begin());i!=ref.end();++i) {
+				if (ht[i->first] != i->second) {
+					std::cout << "FAILED! (data mismatch!)" << std::endl;
+					return -1;
+				}
+			}
+
+			Hashtable<uint64_t,std::string> ht2;
 			ht2 = ht;
 			Hashtable<uint64_t,std::string> ht3(ht2);
 			if (ht2.size() != ref.size()) {
@@ -609,6 +641,7 @@ static int testOther()
 				std::cout << "FAILED! (size mismatch, copied)" << std::endl;
 				return -1;
 			}
+
 			for(std::map<uint64_t,std::string>::iterator i(ref.begin());i!=ref.end();++i) {
 				std::string *v = ht.get(i->first);
 				if (!v) {
@@ -826,9 +859,7 @@ struct TestPhyHandlers
 	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
 	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
-	inline void phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr) {}
-  inline void phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
-  inline void phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr) {}
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
 #endif // __UNIX_LIKE__
 };
 static int testPhy()

service/ClusterDefinition.hpp

@@ -0,0 +1,125 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef ZT_CLUSTERDEFINITION_HPP
+#define ZT_CLUSTERDEFINITION_HPP
+
+#ifdef ZT_ENABLE_CLUSTER
+
+#include <vector>
+#include <algorithm>
+
+#include "../node/Constants.hpp"
+#include "../node/Utils.hpp"
+#include "../osdep/OSUtils.hpp"
+
+namespace ZeroTier {
+
+/**
+ * Parser for cluster definition file
+ */
+class ClusterDefinition
+{
+public:
+	struct MemberDefinition
+	{
+		MemberDefinition() : id(0),x(0),y(0),z(0) { name[0] = (char)0; }
+
+		unsigned int id;
+		int x,y,z;
+		char name[256];
+		InetAddress clusterEndpoint;
+		std::vector<InetAddress> zeroTierEndpoints;
+	};
+
+	ClusterDefinition(uint64_t myAddress,const char *pathToClusterFile)
+	{
+		std::string cf;
+		if (!OSUtils::readFile(pathToClusterFile,cf))
+			return;
+
+		char myAddressStr[64];
+		Utils::snprintf(myAddressStr,sizeof(myAddressStr),"%.10llx",myAddress);
+
+		std::vector<std::string> lines(Utils::split(cf.c_str(),"\r\n","",""));
+		for(std::vector<std::string>::iterator l(lines.begin());l!=lines.end();++l) {
+			std::vector<std::string> fields(Utils::split(l->c_str()," \t","",""));
+			if ((fields.size() < 5)||(fields[0][0] == '#')||(fields[0] != myAddressStr))
+				continue;
+
+			int id = Utils::strToUInt(fields[1].c_str());
+			if ((id < 0)||(id > ZT_CLUSTER_MAX_MEMBERS))
+				continue;
+			MemberDefinition &md = _md[id];
+
+			md.id = (unsigned int)id;
+			if (fields.size() >= 6) {
+				std::vector<std::string> xyz(Utils::split(fields[5].c_str(),",","",""));
+				md.x = (xyz.size() > 0) ? Utils::strToInt(xyz[0].c_str()) : 0;
+				md.y = (xyz.size() > 1) ? Utils::strToInt(xyz[1].c_str()) : 0;
+				md.z = (xyz.size() > 2) ? Utils::strToInt(xyz[2].c_str()) : 0;
+			}
+			Utils::scopy(md.name,sizeof(md.name),fields[2].c_str());
+			md.clusterEndpoint.fromString(fields[3]);
+			if (!md.clusterEndpoint)
+				continue;
+			std::vector<std::string> zips(Utils::split(fields[4].c_str(),",","",""));
+			for(std::vector<std::string>::iterator zip(zips.begin());zip!=zips.end();++zip) {
+				InetAddress i;
+				i.fromString(*zip);
+				if (i)
+					md.zeroTierEndpoints.push_back(i);
+			}
+
+			_ids.push_back((unsigned int)id);
+		}
+
+		std::sort(_ids.begin(),_ids.end());
+	}
+
+	inline const MemberDefinition &operator[](unsigned int id) const throw() { return _md[id]; }
+	inline unsigned int size() const throw() { return (unsigned int)_ids.size(); }
+	inline const std::vector<unsigned int> &ids() const throw() { return _ids; }
+
+	inline std::vector<MemberDefinition> members() const
+	{
+		std::vector<MemberDefinition> m;
+		for(std::vector<unsigned int>::const_iterator i(_ids.begin());i!=_ids.end();++i)
+			m.push_back(_md[*i]);
+		return m;
+	}
+
+private:
+	MemberDefinition _md[ZT_CLUSTER_MAX_MEMBERS];
+	std::vector<unsigned int> _ids;
+};
+
+} // namespace ZeroTier
+
+#endif // ZT_ENABLE_CLUSTER
+
+#endif

service/ClusterGeoIpService.cpp

@@ -0,0 +1,196 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifdef ZT_ENABLE_CLUSTER
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdint.h>
+#include <unistd.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <signal.h>
+#include <errno.h>
+
+#include <iostream>
+
+#include "ClusterGeoIpService.hpp"
+#include "../node/Utils.hpp"
+#include "../osdep/OSUtils.hpp"
+
+#define ZT_CLUSTERGEOIPSERVICE_INTERNAL_CACHE_TTL (60 * 60 * 1000)
+
+namespace ZeroTier {
+
+ClusterGeoIpService::ClusterGeoIpService(const char *pathToExe) :
+	_pathToExe(pathToExe),
+	_sOutputFd(-1),
+	_sInputFd(-1),
+	_sPid(0),
+	_run(true)
+{
+	_thread = Thread::start(this);
+}
+
+ClusterGeoIpService::~ClusterGeoIpService()
+{
+	_run = false;
+	long p = _sPid;
+	if (p > 0) {
+		::kill(p,SIGTERM);
+		Thread::sleep(500);
+		::kill(p,SIGKILL);
+	}
+	Thread::join(_thread);
+}
+
+bool ClusterGeoIpService::locate(const InetAddress &ip,int &x,int &y,int &z)
+{
+	InetAddress ipNoPort(ip);
+	ipNoPort.setPort(0); // we index cache by IP only
+	const uint64_t now = OSUtils::now();
+
+	bool r = false;
+	{
+		Mutex::Lock _l(_cache_m);
+		std::map< InetAddress,_CE >::iterator c(_cache.find(ipNoPort));
+		if (c != _cache.end()) {
+			x = c->second.x;
+			y = c->second.y;
+			z = c->second.z;
+			if ((now - c->second.ts) < ZT_CLUSTERGEOIPSERVICE_INTERNAL_CACHE_TTL)
+				return true;
+			else r = true; // return true but refresh as well
+		}
+	}
+
+	{
+		Mutex::Lock _l(_sOutputLock);
+		if (_sOutputFd >= 0) {
+			std::string ips(ipNoPort.toIpString());
+			ips.push_back('\n');
+			//fprintf(stderr,"ClusterGeoIpService: << %s",ips.c_str());
+			::write(_sOutputFd,ips.data(),ips.length());
+		}
+	}
+
+	return r;
+}
+
+void ClusterGeoIpService::threadMain()
+	throw()
+{
+	char linebuf[65536];
+	char buf[65536];
+	long n,lineptr;
+
+	while (_run) {
+		{
+			Mutex::Lock _l(_sOutputLock);
+
+			_sOutputFd = -1;
+			_sInputFd = -1;
+			_sPid = 0;
+
+			int stdinfds[2] = { 0,0 };  // sub-process's stdin, our output
+			int stdoutfds[2] = { 0,0 }; // sub-process's stdout, our input
+			::pipe(stdinfds);
+			::pipe(stdoutfds);
+
+			long p = (long)::vfork();
+			if (p < 0) {
+				Thread::sleep(500);
+				continue;
+			} else if (p == 0) {
+				::close(stdinfds[1]);
+				::close(stdoutfds[0]);
+				::dup2(stdinfds[0],STDIN_FILENO);
+				::dup2(stdoutfds[1],STDOUT_FILENO);
+				::execl(_pathToExe.c_str(),_pathToExe.c_str(),(const char *)0);
+				::exit(1);
+			} else {
+				::close(stdinfds[0]);
+				::close(stdoutfds[1]);
+				_sOutputFd = stdinfds[1];
+				_sInputFd = stdoutfds[0];
+				_sPid = p;
+			}
+		}
+
+		lineptr = 0;
+		while (_run) {
+			n = ::read(_sInputFd,buf,sizeof(buf));
+			if (n <= 0) {
+				if (errno == EINTR)
+					continue;
+				else break;
+			}
+			for(long i=0;i<n;++i) {
+				if (lineptr > (long)sizeof(linebuf))
+					lineptr = 0;
+				if ((buf[i] == '\n')||(buf[i] == '\r')) {
+					linebuf[lineptr] = (char)0;
+					if (lineptr > 0) {
+						//fprintf(stderr,"ClusterGeoIpService: >> %s\n",linebuf);
+						try {
+							std::vector<std::string> result(Utils::split(linebuf,",","",""));
+							if ((result.size() >= 7)&&(result[1] == "1")) {
+								InetAddress rip(result[0],0);
+								if ((rip.ss_family == AF_INET)||(rip.ss_family == AF_INET6)) {
+									_CE ce;
+									ce.ts = OSUtils::now();
+									ce.x = (int)::strtol(result[4].c_str(),(char **)0,10);
+									ce.y = (int)::strtol(result[5].c_str(),(char **)0,10);
+									ce.z = (int)::strtol(result[6].c_str(),(char **)0,10);
+									//fprintf(stderr,"ClusterGeoIpService: %s is at %d,%d,%d\n",rip.toIpString().c_str(),ce.x,ce.y,ce.z);
+									{
+										Mutex::Lock _l2(_cache_m);
+										_cache[rip] = ce;
+									}
+								}
+							}
+						} catch ( ... ) {}
+					}
+					lineptr = 0;
+				} else linebuf[lineptr++] = buf[i];
+			}
+		}
+
+		::close(_sOutputFd);
+		::close(_sInputFd);
+		::kill(_sPid,SIGTERM);
+		Thread::sleep(250);
+		::kill(_sPid,SIGKILL);
+		::waitpid(_sPid,(int *)0,0);
+	}
+}
+
+} // namespace ZeroTier
+
+#endif // ZT_ENABLE_CLUSTER

service/ClusterGeoIpService.hpp

@@ -0,0 +1,94 @@
+/*
+ * ZeroTier One - Network Virtualization Everywhere
+ * Copyright (C) 2011-2015  ZeroTier, Inc.
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ *
+ * --
+ *
+ * ZeroTier may be used and distributed under the terms of the GPLv3, which
+ * are available at: http://www.gnu.org/licenses/gpl-3.0.html
+ *
+ * If you would like to embed ZeroTier into a commercial application or
+ * redistribute it in a modified binary form, please contact ZeroTier Networks
+ * LLC. Start here: http://www.zerotier.com/
+ */
+
+#ifndef ZT_CLUSTERGEOIPSERVICE_HPP
+#define ZT_CLUSTERGEOIPSERVICE_HPP
+
+#ifdef ZT_ENABLE_CLUSTER
+
+#include <vector>
+#include <map>
+#include <string>
+
+#include "../node/Constants.hpp"
+#include "../node/InetAddress.hpp"
+#include "../node/Mutex.hpp"
+#include "../osdep/Thread.hpp"
+
+namespace ZeroTier {
+
+/**
+ * Runs the Cluster GeoIP service in the background and resolves geoIP queries
+ */
+class ClusterGeoIpService
+{
+public:
+	/**
+	 * @param pathToExe Path to cluster geo-resolution service executable
+	 */
+	ClusterGeoIpService(const char *pathToExe);
+
+	~ClusterGeoIpService();
+
+	/**
+	 * Attempt to locate an IP
+	 *
+	 * This returns true if x, y, and z are set. Otherwise it returns false
+	 * and a geo-locate job is ordered in the background. This usually takes
+	 * 500-1500ms to complete, after which time results will be available.
+	 * If false is returned the supplied coordinate variables are unchanged.
+	 *
+	 * @param ip IPv4 or IPv6 address
+	 * @param x Reference to variable to receive X
+	 * @param y Reference to variable to receive Y
+	 * @param z Reference to variable to receive Z
+	 * @return True if coordinates were set
+	 */
+	bool locate(const InetAddress &ip,int &x,int &y,int &z);
+
+	void threadMain()
+		throw();
+
+private:
+	const std::string _pathToExe;
+	int _sOutputFd;
+	int _sInputFd;
+	volatile long _sPid;
+	volatile bool _run;
+	Thread _thread;
+	Mutex _sOutputLock;
+
+	struct _CE { uint64_t ts; int x,y,z; };
+	std::map< InetAddress,_CE > _cache;
+	Mutex _cache_m;
+};
+
+} // namespace ZeroTier
+
+#endif // ZT_ENABLE_CLUSTER
+
+#endif

service/ControlPlane.cpp

@@ -182,14 +182,12 @@ static std::string _jsonEnumerate(unsigned int depth,const ZT_PeerPhysicalPath *
 			"%s\t\"address\": \"%s\",\n"
 			"%s\t\"lastSend\": %llu,\n"
 			"%s\t\"lastReceive\": %llu,\n"
-			"%s\t\"fixed\": %s,\n"
 			"%s\t\"active\": %s,\n"
 			"%s\t\"preferred\": %s\n"
 			"%s}",
 			prefix,_jsonEscape(reinterpret_cast<const InetAddress *>(&(pp[i].address))->toString()).c_str(),
 			prefix,pp[i].lastSend,
 			prefix,pp[i].lastReceive,
-			prefix,(pp[i].fixed == 0) ? "false" : "true",
 			prefix,(pp[i].active == 0) ? "false" : "true",
 			prefix,(pp[i].preferred == 0) ? "false" : "true",
 			prefix);
@@ -267,7 +265,7 @@ unsigned int ControlPlane::handleRequest(
 	std::string &responseBody,
 	std::string &responseContentType)
 {
-	char json[1024];
+	char json[8194];
 	unsigned int scode = 404;
 	std::vector<std::string> ps(Utils::split(path.c_str(),"/","",""));
 	std::map<std::string,std::string> urlArgs;
@@ -356,29 +354,65 @@ unsigned int ControlPlane::handleRequest(
 
 			if (ps[0] == "status") {
 				responseContentType = "application/json";
+
 				ZT_NodeStatus status;
 				_node->status(&status);
+
+				std::string clusterJson;
+#ifdef ZT_ENABLE_CLUSTER
+				{
+					ZT_ClusterStatus cs;
+					_node->clusterStatus(&cs);
+
+					if (cs.clusterSize >= 1) {
+						char t[1024];
+						Utils::snprintf(t,sizeof(t),"{\n\t\t\"myId\": %u,\n\t\t\"clusterSize\": %u,\n\t\t\"members\": [",cs.myId,cs.clusterSize);
+						clusterJson.append(t);
+						for(unsigned int i=0;i<cs.clusterSize;++i) {
+							Utils::snprintf(t,sizeof(t),"%s\t\t\t{\n\t\t\t\t\"id\": %u,\n\t\t\t\t\"msSinceLastHeartbeat\": %u,\n\t\t\t\t\"alive\": %s,\n\t\t\t\t\"x\": %d,\n\t\t\t\t\"y\": %d,\n\t\t\t\t\"z\": %d,\n\t\t\t\t\"load\": %llu,\n\t\t\t\t\"peers\": %llu\n\t\t\t}",
+								((i == 0) ? "\n" : ",\n"),
+								cs.members[i].id,
+								cs.members[i].msSinceLastHeartbeat,
+								(cs.members[i].alive != 0) ? "true" : "false",
+								cs.members[i].x,
+								cs.members[i].y,
+								cs.members[i].z,
+								cs.members[i].load,
+								cs.members[i].peers);
+							clusterJson.append(t);
+						}
+						clusterJson.append(" ]\n\t\t}");
+					}
+				}
+#endif
+
 				Utils::snprintf(json,sizeof(json),
 					"{\n"
 					"\t\"address\": \"%.10llx\",\n"
 					"\t\"publicIdentity\": \"%s\",\n"
+					"\t\"worldId\": %llu,\n"
+					"\t\"worldTimestamp\": %llu,\n"
 					"\t\"online\": %s,\n"
 					"\t\"tcpFallbackActive\": %s,\n"
 					"\t\"versionMajor\": %d,\n"
 					"\t\"versionMinor\": %d,\n"
 					"\t\"versionRev\": %d,\n"
 					"\t\"version\": \"%d.%d.%d\",\n"
-					"\t\"clock\": %llu\n"
+					"\t\"clock\": %llu,\n"
+					"\t\"cluster\": %s\n"
 					"}\n",
 					status.address,
 					status.publicIdentity,
+					status.worldId,
+					status.worldTimestamp,
 					(status.online) ? "true" : "false",
 					(_svc->tcpFallbackActive()) ? "true" : "false",
 					ZEROTIER_ONE_VERSION_MAJOR,
 					ZEROTIER_ONE_VERSION_MINOR,
 					ZEROTIER_ONE_VERSION_REVISION,
 					ZEROTIER_ONE_VERSION_MAJOR,ZEROTIER_ONE_VERSION_MINOR,ZEROTIER_ONE_VERSION_REVISION,
-					(unsigned long long)OSUtils::now());
+					(unsigned long long)OSUtils::now(),
+					((clusterJson.length() > 0) ? clusterJson.c_str() : "null"));
 				responseBody = json;
 				scode = 200;
 			} else if (ps[0] == "config") {

service/OneService.cpp

@@ -58,6 +58,8 @@
 
 #include "OneService.hpp"
 #include "ControlPlane.hpp"
+#include "ClusterGeoIpService.hpp"
+#include "ClusterDefinition.hpp"
 
 /**
  * Uncomment to enable UDP breakage switch
@@ -338,6 +340,25 @@ public:
 static BackgroundSoftwareUpdateChecker backgroundSoftwareUpdateChecker;
 #endif // ZT_AUTO_UPDATE
 
+static std::string _trimString(const std::string &s)
+{
+	unsigned long end = (unsigned long)s.length();
+	while (end) {
+		char c = s[end - 1];
+		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
+			--end;
+		else break;
+	}
+	unsigned long start = 0;
+	while (start < end) {
+		char c = s[start];
+		if ((c == ' ')||(c == '\r')||(c == '\n')||(!c)||(c == '\t'))
+			++start;
+		else break;
+	}
+	return s.substr(start,end - start);
+}
+
 class OneServiceImpl;
 
 static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t nwid,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwconf);
@@ -347,6 +368,11 @@ static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,c
 static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len);
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
+#ifdef ZT_ENABLE_CLUSTER
+static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len);
+static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z);
+#endif
+
 static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
 static int ShttpOnMessageBegin(http_parser *parser);
@@ -399,28 +425,33 @@ struct TcpConnection
 class OneServiceImpl : public OneService
 {
 public:
-	OneServiceImpl(const char *hp,unsigned int port,const char *overrideRootTopology) :
-		_homePath((hp) ? hp : "."),
-		_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY),
+	OneServiceImpl(const char *hp,unsigned int port) :
+		_homePath((hp) ? hp : ".")
+		,_tcpFallbackResolver(ZT_TCP_FALLBACK_RELAY)
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-		_controller((_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str()),
+		,_controller((SqliteNetworkController *)0)
 #endif
-		_phy(this,false,true),
-		_overrideRootTopology((overrideRootTopology) ? overrideRootTopology : ""),
-		_node((Node *)0),
-		_controlPlane((ControlPlane *)0),
-		_lastDirectReceiveFromGlobal(0),
-		_lastSendToGlobal(0),
-		_lastRestart(0),
-		_nextBackgroundTaskDeadline(0),
-		_tcpFallbackTunnel((TcpConnection *)0),
-		_termReason(ONE_STILL_RUNNING),
-		_port(0),
+		,_phy(this,false,true)
+		,_node((Node *)0)
+		,_controlPlane((ControlPlane *)0)
+		,_lastDirectReceiveFromGlobal(0)
+		,_lastSendToGlobal(0)
+		,_lastRestart(0)
+		,_nextBackgroundTaskDeadline(0)
+		,_tcpFallbackTunnel((TcpConnection *)0)
+		,_termReason(ONE_STILL_RUNNING)
+		,_port(0)
 #ifdef ZT_USE_MINIUPNPC
-		_v4UpnpUdpSocket((PhySocket *)0),
-		_upnpClient((UPNPClient *)0),
+		,_v4UpnpUdpSocket((PhySocket *)0)
+		,_upnpClient((UPNPClient *)0)
+#endif
+#ifdef ZT_ENABLE_CLUSTER
+		,_clusterMessageSocket((PhySocket *)0)
+		,_clusterGeoIpService((ClusterGeoIpService *)0)
+		,_clusterDefinition((ClusterDefinition *)0)
+		,_clusterMemberId(0)
 #endif
-		_run(true)
+		,_run(true)
 	{
 		const int portTrials = (port == 0) ? 256 : 1; // if port is 0, pick random
 		for(int k=0;k<portTrials;++k) {
@@ -470,15 +501,12 @@ public:
 		OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "zerotier-one.port").c_str(),std::string(portstr));
 
 #ifdef ZT_USE_MINIUPNPC
-		// Bind a random secondary port for use with uPnP, since some NAT routers
+		// Bind a secondary port for use with uPnP, since some NAT routers
 		// (cough Ubiquity Edge cough) barf up a lung if you do both conventional
 		// NAT-t and uPnP from behind the same port. I think this is a bug, but
 		// everyone else's router bugs are our problem. :P
-		for(int k=0;k<256;++k) {
-			unsigned int randp = 0;
-			Utils::getSecureRandom(&randp,sizeof(randp));
-			unsigned int upnport = 40000 + (randp % 25500);
-
+		for(int k=0;k<512;++k) {
+			const unsigned int upnport = 40000 + (((port + 1) * (k + 1)) % 25500);
 			_v4UpnpLocalAddress = InetAddress(0,upnport);
 			_v4UpnpUdpSocket = _phy.udpBind((const struct sockaddr *)&_v4UpnpLocalAddress,reinterpret_cast<void *>(&_v4UpnpLocalAddress),131072);
 			if (_v4UpnpUdpSocket) {
@@ -495,9 +523,19 @@ public:
 		_phy.close(_v6UdpSocket);
 		_phy.close(_v4TcpListenSocket);
 		_phy.close(_v6TcpListenSocket);
+#ifdef ZT_ENABLE_CLUSTER
+		_phy.close(_clusterMessageSocket);
+#endif
 #ifdef ZT_USE_MINIUPNPC
 		_phy.close(_v4UpnpUdpSocket);
 		delete _upnpClient;
+#endif
+#ifdef ZT_ENABLE_NETWORK_CONTROLLER
+		delete _controller;
+#endif
+#ifdef ZT_ENABLE_CLUSTER
+		delete _clusterGeoIpService;
+		delete _clusterDefinition;
 #endif
 	}
 
@@ -521,7 +559,7 @@ public:
 					} else OSUtils::lockDownFile(authTokenPath.c_str(),false);
 				}
 			}
-			authToken = Utils::trim(authToken);
+			authToken = _trimString(authToken);
 
 			_node = new Node(
 				OSUtils::now(),
@@ -531,18 +569,82 @@ public:
 				SnodeWirePacketSendFunction,
 				SnodeVirtualNetworkFrameFunction,
 				SnodeVirtualNetworkConfigFunction,
-				SnodeEventCallback,
-				((_overrideRootTopology.length() > 0) ? _overrideRootTopology.c_str() : (const char *)0));
+				SnodeEventCallback);
 
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-			_node->setNetconfMaster((void *)&_controller);
+			_controller = new SqliteNetworkController(_node,(_homePath + ZT_PATH_SEPARATOR_S + ZT_CONTROLLER_DB_PATH).c_str(),(_homePath + ZT_PATH_SEPARATOR_S + "circuitTestResults.d").c_str());
+			_node->setNetconfMaster((void *)_controller);
+#endif
+
+#ifdef ZT_ENABLE_CLUSTER
+		if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str())) {
+			_clusterDefinition = new ClusterDefinition(_node->address(),(_homePath + ZT_PATH_SEPARATOR_S + "cluster").c_str());
+			if (_clusterDefinition->size() > 0) {
+				std::vector<ClusterDefinition::MemberDefinition> members(_clusterDefinition->members());
+				for(std::vector<ClusterDefinition::MemberDefinition>::iterator m(members.begin());m!=members.end();++m) {
+					PhySocket *cs = _phy.udpBind(reinterpret_cast<const struct sockaddr *>(&(m->clusterEndpoint)));
+					if (cs) {
+						if (_clusterMessageSocket) {
+							_phy.close(_clusterMessageSocket,false);
+							_phy.close(cs,false);
+
+							Mutex::Lock _l(_termReason_m);
+							_termReason = ONE_UNRECOVERABLE_ERROR;
+							_fatalErrorMessage = "Cluster: can't determine my cluster member ID: able to bind more than one cluster message socket IP/port!";
+							return _termReason;
+						}
+						_clusterMessageSocket = cs;
+						_clusterMemberId = m->id;
+					}
+				}
+
+				if (!_clusterMessageSocket) {
+					Mutex::Lock _l(_termReason_m);
+					_termReason = ONE_UNRECOVERABLE_ERROR;
+					_fatalErrorMessage = "Cluster: can't determine my cluster member ID: unable to bind to any cluster message socket IP/port.";
+					return _termReason;
+				}
+
+				if (OSUtils::fileExists((_homePath + ZT_PATH_SEPARATOR_S + "cluster-geo.exe").c_str()))
+					_clusterGeoIpService = new ClusterGeoIpService((_homePath + ZT_PATH_SEPARATOR_S + "cluster-geo.exe").c_str());
+
+				const ClusterDefinition::MemberDefinition &me = (*_clusterDefinition)[_clusterMemberId];
+				InetAddress endpoints[255];
+				unsigned int numEndpoints = 0;
+				for(std::vector<InetAddress>::const_iterator i(me.zeroTierEndpoints.begin());i!=me.zeroTierEndpoints.end();++i)
+					endpoints[numEndpoints++] = *i;
+
+				if (_node->clusterInit(
+					_clusterMemberId,
+					reinterpret_cast<const struct sockaddr_storage *>(endpoints),
+					numEndpoints,
+					me.x,
+					me.y,
+					me.z,
+					&SclusterSendFunction,
+					this,
+					(_clusterGeoIpService) ? &SclusterGeoIpFunction : 0,
+					this) == ZT_RESULT_OK) {
+
+					std::vector<ClusterDefinition::MemberDefinition> members(_clusterDefinition->members());
+					for(std::vector<ClusterDefinition::MemberDefinition>::iterator m(members.begin());m!=members.end();++m) {
+						if (m->id != _clusterMemberId)
+							_node->clusterAddMember(m->id);
+					}
+
+				}
+			} else {
+				delete _clusterDefinition;
+				_clusterDefinition = (ClusterDefinition *)0;
+			}
+		}
 #endif
 
 			_controlPlane = new ControlPlane(this,_node,(_homePath + ZT_PATH_SEPARATOR_S + "ui").c_str());
 			_controlPlane->addAuthToken(authToken.c_str());
 
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-			_controlPlane->setController(&_controller);
+			_controlPlane->setController(_controller);
 #endif
 
 			{	// Remember networks from previous session
@@ -629,7 +731,7 @@ public:
 #ifdef ZT_USE_MINIUPNPC
 					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
-						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)));
 #endif
 
 					struct ifaddrs *ifatbl = (struct ifaddrs *)0;
@@ -647,7 +749,7 @@ public:
 								if (!isZT) {
 									InetAddress ip(ifa->ifa_addr);
 									ip.setPort(_port);
-									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip),0,ZT_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
+									_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&ip));
 								}
 							}
 							ifa = ifa->ifa_next;
@@ -763,10 +865,19 @@ public:
 
 	inline void phyOnDatagram(PhySocket *sock,void **uptr,const struct sockaddr *from,void *data,unsigned long len)
 	{
+#ifdef ZT_ENABLE_CLUSTER
+		if (sock == _clusterMessageSocket) {
+			_lastDirectReceiveFromGlobal = OSUtils::now();
+			_node->clusterHandleIncomingMessage(data,len);
+			return;
+		}
+#endif
+
 #ifdef ZT_BREAK_UDP
 		if (OSUtils::fileExists("/tmp/ZT_BREAK_UDP"))
 			return;
 #endif
+
 		if ((len >= 16)&&(reinterpret_cast<const InetAddress *>(from)->ipScope() == InetAddress::IP_SCOPE_GLOBAL))
 			_lastDirectReceiveFromGlobal = OSUtils::now();
 		ZT_ResultCode rc = _node->processWirePacket(
@@ -920,7 +1031,7 @@ public:
 							if (from) {
 								ZT_ResultCode rc = _node->processWirePacket(
 									OSUtils::now(),
-									0,
+									&ZT_SOCKADDR_NULL,
 									reinterpret_cast<struct sockaddr_storage *>(&from),
 									data,
 									plen,
@@ -974,9 +1085,7 @@ public:
 	inline void phyOnUnixClose(PhySocket *sock,void **uptr) {}
 	inline void phyOnUnixData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
 	inline void phyOnUnixWritable(PhySocket *sock,void **uptr) {}
-	inline void phyOnSocketPairEndpointClose(PhySocket *sock,void **uptr) {}
-  inline void phyOnSocketPairEndpointData(PhySocket *sock,void **uptr,void *data,unsigned long len) {}
-  inline void phyOnSocketPairEndpointWritable(PhySocket *sock,void **uptr) {}
+	inline void phyOnFileDescriptorActivity(PhySocket *sock,void **uptr,bool readable,bool writable) {}
 
 	inline int nodeVirtualNetworkConfigFunction(uint64_t nwid,enum ZT_VirtualNetworkConfigOperation op,const ZT_VirtualNetworkConfig *nwc)
 	{
@@ -1285,7 +1394,6 @@ public:
 			_phy.close(tc->sock); // will call close handler, which deletes from _tcpConnections
 	}
 
-private:
 	std::string _dataStorePrepPath(const char *name) const
 	{
 		std::string p(_homePath);
@@ -1306,10 +1414,9 @@ private:
 	const std::string _homePath;
 	BackgroundResolver _tcpFallbackResolver;
 #ifdef ZT_ENABLE_NETWORK_CONTROLLER
-	SqliteNetworkController _controller;
+	SqliteNetworkController *_controller;
 #endif
 	Phy<OneServiceImpl *> _phy;
-	std::string _overrideRootTopology;
 	Node *_node;
 	InetAddress _v4LocalAddress,_v6LocalAddress;
 	PhySocket *_v4UdpSocket;
@@ -1341,6 +1448,13 @@ private:
 	UPNPClient *_upnpClient;
 #endif
 
+#ifdef ZT_ENABLE_CLUSTER
+	PhySocket *_clusterMessageSocket;
+	ClusterGeoIpService *_clusterGeoIpService;
+	ClusterDefinition *_clusterDefinition;
+	unsigned int _clusterMemberId;
+#endif
+
 	bool _run;
 	Mutex _run_m;
 };
@@ -1358,6 +1472,21 @@ static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct soc
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,sourceMac,destMac,etherType,vlanId,data,len); }
 
+#ifdef ZT_ENABLE_CLUSTER
+static void SclusterSendFunction(void *uptr,unsigned int toMemberId,const void *data,unsigned int len)
+{
+	OneServiceImpl *const impl = reinterpret_cast<OneServiceImpl *>(uptr);
+	const ClusterDefinition::MemberDefinition &md = (*(impl->_clusterDefinition))[toMemberId];
+	if (md.clusterEndpoint)
+		impl->_phy.udpSend(impl->_clusterMessageSocket,reinterpret_cast<const struct sockaddr *>(&(md.clusterEndpoint)),data,len);
+}
+static int SclusterGeoIpFunction(void *uptr,const struct sockaddr_storage *addr,int *x,int *y,int *z)
+{
+	OneServiceImpl *const impl = reinterpret_cast<OneServiceImpl *>(uptr);
+	return (int)(impl->_clusterGeoIpService->locate(*(reinterpret_cast<const InetAddress *>(addr)),*x,*y,*z));
+}
+#endif
+
 static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->tapFrameHandler(nwid,from,to,etherType,vlanId,data,len); }
 
@@ -1506,7 +1635,7 @@ std::string OneService::autoUpdateUrl()
 	return std::string();
 }
 
-OneService *OneService::newInstance(const char *hp,unsigned int port,const char *overrideRootTopology) { return new OneServiceImpl(hp,port,overrideRootTopology); }
+OneService *OneService::newInstance(const char *hp,unsigned int port) { return new OneServiceImpl(hp,port); }
 OneService::~OneService() {}
 
 } // namespace ZeroTier

service/OneService.hpp

@@ -43,6 +43,9 @@ namespace ZeroTier {
  * periodically checked and updates are automatically downloaded, verified
  * against a built-in list of update signing keys, and installed. This is
  * only supported for certain platforms.
+ *
+ * If built with ZT_ENABLE_CLUSTER, a 'cluster' file is checked and if
+ * present is read to determine the identity of other cluster members.
  */
 class OneService
 {
@@ -95,12 +98,10 @@ public:
 	 *
 	 * @param hp Home path
 	 * @param port TCP and UDP port for packets and HTTP control (if 0, pick random port)
-	 * @param overrideRootTopology String-serialized root topology (for testing, default: NULL)
 	 */
 	static OneService *newInstance(
 		const char *hp,
-		unsigned int port,
-		const char *overrideRootTopology = (const char *)0);
+		unsigned int port);
 
 	virtual ~OneService();
 

tests/http/Dockerfile

@@ -0,0 +1,24 @@
+FROM centos:latest
+
+MAINTAINER https://www.zerotier.com/
+
+EXPOSE 9993/udp
+
+ADD nodesource-el.repo /etc/yum.repos.d/nodesource-el.repo
+RUN yum -y update && yum install -y nodejs && yum clean all
+
+RUN mkdir -p /var/lib/zerotier-one
+RUN mkdir -p /var/lib/zerotier-one/networks.d
+RUN touch /var/lib/zerotier-one/networks.d/ffffffffffffffff.conf
+
+ADD package.json /
+RUN npm install
+
+ADD zerotier-one /
+RUN chmod a+x /zerotier-one
+
+ADD agent.js /
+ADD docker-main.sh /
+RUN chmod a+x /docker-main.sh
+
+CMD ["./docker-main.sh"]

tests/http/README.md

@@ -0,0 +1,12 @@
+HTTP one-to-all test
+======
+
+*This is really internal use code. You're free to test it out but expect to do some editing/tweaking to make it work. We used this to run some massive scale tests of our new geo-cluster-based root server infrastructure prior to taking it live.*
+
+Before using this code you will want to edit agent.js to change SERVER_HOST to the IP address of where you will run server.js. This should typically be an open Internet IP, since this makes reporting not dependent upon the thing being tested. Also note that this thing does no security of any kind. It's designed for one-off tests run over a short period of time, not to be anything that runs permanently. You will also want to edit the Dockerfile if you want to build containers and change the network ID to the network you want to run tests over.
+
+This code can be deployed across a large number of VMs or containers to test and benchmark HTTP traffic within a virtual network at scale. The agent acts as a server and can query other agents, while the server collects agent data and tells agents about each other. It's designed to use RFC4193-based ZeroTier IPv6 addresses within the cluster, which allows the easy provisioning of a large cluster without IP conflicts.
+
+The Dockerfile builds an image that launches the agent. The image must be "docker run" with "--device=/dev/net/tun --privileged" to permit it to open a tun/tap device within the container. (Unfortunately CAP_NET_ADMIN may not work due to a bug in Docker and/or Linux.) You can run a bunch with a command like:
+
+    for ((n=0;n<10;n++)); do docker run --device=/dev/net/tun --privileged -d zerotier/http-test; done

tests/http/agent.js

@@ -0,0 +1,277 @@
+// ZeroTier distributed HTTP test agent
+
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+// Maximum interval between test attempts
+var TEST_INTERVAL_MAX = 60000;
+
+// Test timeout in ms
+var TEST_TIMEOUT = 30000;
+
+// Where should I contact to register and query a list of other test agents?
+var SERVER_HOST = '104.238.141.145';
+var SERVER_PORT = 18080;
+
+// Which port should agents use for their HTTP?
+var AGENT_PORT = 18888;
+
+// Payload size in bytes
+var PAYLOAD_SIZE = 10000;
+
+// ---------------------------------------------------------------------------
+
+var ipaddr = require('ipaddr.js');
+var os = require('os');
+var http = require('http');
+var async = require('async');
+
+var express = require('express');
+var app = express();
+
+// Find our ZeroTier-assigned RFC4193 IPv6 address
+var thisAgentId = null;
+var interfaces = os.networkInterfaces();
+if (!interfaces) {
+	console.error('FATAL: os.networkInterfaces() failed.');
+	process.exit(1);
+}
+for(var ifname in interfaces) {
+	var ifaddrs = interfaces[ifname];
+	if (Array.isArray(ifaddrs)) {
+		for(var i=0;i<ifaddrs.length;++i) {
+			if (ifaddrs[i].family == 'IPv6') {
+				try {
+					var ipbytes = ipaddr.parse(ifaddrs[i].address).toByteArray();
+					if ((ipbytes.length === 16)&&(ipbytes[0] == 0xfd)&&(ipbytes[9] == 0x99)&&(ipbytes[10] == 0x93)) {
+						thisAgentId = '';
+						for(var j=0;j<16;++j) {
+							var tmp = ipbytes[j].toString(16);
+							if (tmp.length === 1)
+								thisAgentId += '0';
+							thisAgentId += tmp;
+						}
+					}
+				} catch (e) {
+					console.error(e);
+				}
+			}
+		}
+	}
+}
+if (thisAgentId === null) {
+	console.error('FATAL: no ZeroTier-assigned RFC4193 IPv6 addresses found on any local interface!');
+	process.exit(1);
+}
+
+//console.log(thisAgentId);
+
+// Create a random (and therefore not very compressable) payload
+var payload = new Buffer(PAYLOAD_SIZE);
+for(var xx=0;xx<PAYLOAD_SIZE;++xx) {
+	payload.writeUInt8(Math.round(Math.random() * 255.0),xx);
+}
+
+function agentIdToIp(agentId)
+{
+	var ip = '';
+	ip += agentId.substr(0,4);
+	ip += ':';
+	ip += agentId.substr(4,4);
+	ip += ':';
+	ip += agentId.substr(8,4);
+	ip += ':';
+	ip += agentId.substr(12,4);
+	ip += ':';
+	ip += agentId.substr(16,4);
+	ip += ':';
+	ip += agentId.substr(20,4);
+	ip += ':';
+	ip += agentId.substr(24,4);
+	ip += ':';
+	ip += agentId.substr(28,4);
+	return ip;
+};
+
+var lastTestResult = null;
+var allOtherAgents = [];
+
+function doTest()
+{
+	var submit = http.request({
+		host: SERVER_HOST,
+		port: SERVER_PORT,
+		path: '/'+thisAgentId,
+		method: 'POST'
+	},function(res) {
+		var body = '';
+		res.on('data',function(chunk) { body += chunk.toString(); });
+		res.on('end',function() {
+
+			if (body) {
+				try {
+					var peers = JSON.parse(body);
+					if (Array.isArray(peers))
+						allOtherAgents = peers;
+				} catch (e) {}
+			}
+
+			if (allOtherAgents.length > 1) {
+
+				var target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+				while (target === thisAgentId)
+					target = allOtherAgents[Math.floor(Math.random() * allOtherAgents.length)];
+
+				var testRequest = null;
+				var timeoutId = null;
+				timeoutId = setTimeout(function() {
+					if (testRequest !== null)
+						testRequest.abort();
+					timeoutId = null;
+				},TEST_TIMEOUT);
+				var startTime = Date.now();
+
+				testRequest = http.get({
+					host: agentIdToIp(target),
+					port: AGENT_PORT,
+					path: '/'
+				},function(res) {
+					var bytes = 0;
+					res.on('data',function(chunk) { bytes += chunk.length; });
+					res.on('end',function() {
+						lastTestResult = {
+							source: thisAgentId,
+							target: target,
+							time: (Date.now() - startTime),
+							bytes: bytes,
+							timedOut: (timeoutId === null),
+							error: null
+						};
+						if (timeoutId !== null)
+							clearTimeout(timeoutId);
+						return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+					});
+				}).on('error',function(e) {
+					lastTestResult = {
+						source: thisAgentId,
+						target: target,
+						time: (Date.now() - startTime),
+						bytes: 0,
+						timedOut: (timeoutId === null),
+						error: e.toString()
+					};
+					if (timeoutId !== null)
+						clearTimeout(timeoutId);
+					return setTimeout(doTest,Math.round(Math.random() * TEST_INTERVAL_MAX) + 1);
+				});
+
+			} else {
+				return setTimeout(doTest,1000);
+			}
+
+		});
+	}).on('error',function(e) {
+		console.log('POST failed: '+e.toString());
+		return setTimeout(doTest,1000);
+	});
+	if (lastTestResult !== null) {
+		submit.write(JSON.stringify(lastTestResult));
+		lastTestResult = null;
+	}
+	submit.end();
+};
+
+/*
+function performTestOnAllPeers(peers,callback)
+{
+	var allResults = {};
+	var allRequests = [];
+	var timedOut = false;
+	var endOfTestTimer = setTimeout(function() {
+		timedOut = true;
+		for(var x=0;x<allRequests.length;++x)
+			allRequests[x].abort();
+	},TEST_DURATION);
+
+	async.each(peers,function(peer,next) {
+		if (timedOut)
+			return next(null);
+		if (peer.length !== 32)
+			return next(null);
+
+		var connectionStartTime = Date.now();
+		allResults[peer] = {
+			start: connectionStartTime,
+			end: 0,
+			error: null,
+			timedOut: false,
+			bytes: 0
+		};
+
+		allRequests.push(http.get({
+			host: agentIdToIp(peer),
+			port: AGENT_PORT,
+			path: '/'
+		},function(res) {
+			var bytes = 0;
+			res.on('data',function(chunk) {
+				bytes += chunk.length;
+			});
+			res.on('end',function() {
+				allResults[peer] = {
+					start: connectionStartTime,
+					end: Date.now(),
+					error: null,
+					timedOut: timedOut,
+					bytes: bytes
+				};
+				return next(null);
+			});
+		}).on('error',function(e) {
+			allResults[peer] = {
+				start: connectionStartTime,
+				end: Date.now(),
+				error: e.toString(),
+				timedOut: timedOut,
+				bytes: 0
+			};
+			return next(null);
+		}));
+	},function(err) {
+		if (!timedOut)
+			clearTimeout(endOfTestTimer);
+		return callback(allResults);
+	});
+};
+
+function doTestsAndReport()
+{
+	registerAndGetPeers(function(err,peers) {
+		if (err) {
+			console.error('WARNING: skipping test: unable to contact or query server: '+err.toString());
+		} else {
+			performTestOnAllPeers(peers,function(results) {
+				var submit = http.request({
+					host: SERVER_HOST,
+					port: SERVER_PORT,
+					path: '/'+thisAgentId,
+					method: 'POST'
+				},function(res) {
+				}).on('error',function(e) {
+					console.error('WARNING: unable to submit results to server: '+err.toString());
+				});
+				submit.write(JSON.stringify(results));
+				submit.end();
+			});
+		}
+	});
+};
+*/
+
+// Agents just serve up a test payload
+app.get('/',function(req,res) { return res.status(200).send(payload); });
+
+var expressServer = app.listen(AGENT_PORT,function () {
+	// Start timeout-based loop
+	doTest();
+});

tests/http/big-test-hosts

@@ -0,0 +1,2 @@
[email protected]
[email protected]

tests/http/big-test-kill.sh

@@ -0,0 +1,18 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+pssh -h big-test-hosts -i -t 0 -p 256 "docker ps -aq | xargs -r docker rm -f"
+
+exit 0

tests/http/big-test-ready.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=100
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -i -t 0 -p 256 "docker pull $CONTAINER_IMAGE"
+
+exit 0

tests/http/big-test-start.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+
+# Edit as needed -- note that >1000 per host is likely problematic due to Linux kernel limits
+NUM_CONTAINERS=50
+CONTAINER_IMAGE=zerotier/http-test
+
+#
+# This script is designed to be run on Docker hosts to run NUM_CONTAINERS
+#
+# It can then be run on each Docker host via pssh or similar to run very
+# large scale tests.
+#
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/usr/sbin:/sbin
+
+# Kill and clean up old test containers if any -- note that this kills all containers on the system!
+#docker ps -q | xargs -n 1 docker kill
+#docker ps -aq | xargs -n 1 docker rm
+
+# Pull latest if needed -- change this to your image name and/or where to pull it from
+#docker pull $CONTAINER_IMAGE
+
+# Run NUM_CONTAINERS
+#for ((n=0;n<$NUM_CONTAINERS;n++)); do
+#	docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE
+#done
+
+pssh -h big-test-hosts -o big-test-out -t 0 -p 256 "for ((n=0;n<$NUM_CONTAINERS;n++)); do docker run --device=/dev/net/tun --privileged -d $CONTAINER_IMAGE; sleep 0.25; done"
+
+exit 0

tests/http/docker-main.sh

@@ -0,0 +1,14 @@
+#!/bin/bash
+
+export PATH=/bin:/usr/bin:/usr/local/bin:/sbin:/usr/sbin
+
+/zerotier-one -d >>zerotier-one.out 2>&1
+
+while [ ! -d "/proc/sys/net/ipv6/conf/zt0" ]; do
+	sleep 0.25
+done
+
+sleep 2
+
+exec node --harmony /agent.js >>agent.out 2>&1
+#exec node --harmony /agent.js

tests/http/nodesource-el.repo

@@ -0,0 +1,6 @@
+[nodesource]
+name=Node.js Packages for Enterprise Linux 7 - $basearch
+baseurl=https://rpm.nodesource.com/pub_4.x/el/7/$basearch
+failovermethod=priority
+enabled=1
+gpgcheck=0

tests/http/package.json

@@ -0,0 +1,16 @@
+{
+  "name": "zerotier-test-http",
+  "version": "1.0.0",
+  "description": "ZeroTier in-network HTTP test",
+  "main": "agent.js",
+  "scripts": {
+    "test": "echo \"Error: no test specified\" && exit 1"
+  },
+  "author": "ZeroTier, Inc.",
+  "license": "GPL-3.0",
+  "dependencies": {
+    "async": "^1.5.0",
+    "express": "^4.13.3",
+    "ipaddr.js": "^1.0.3"
+  }
+}

tests/http/server.js

@@ -0,0 +1,44 @@
+// ZeroTier distributed HTTP test coordinator and result-reporting server
+
+// ---------------------------------------------------------------------------
+// Customizable parameters:
+
+var SERVER_PORT = 18080;
+
+// ---------------------------------------------------------------------------
+
+var fs = require('fs');
+
+var express = require('express');
+var app = express();
+
+app.use(function(req,res,next) {
+	req.rawBody = '';
+	req.on('data', function(chunk) { req.rawBody += chunk.toString(); });
+	req.on('end', function() { return next(); });
+});
+
+var knownAgents = {};
+
+app.post('/:agentId',function(req,res) {
+	var agentId = req.params.agentId;
+	if ((!agentId)||(agentId.length !== 32))
+		return res.status(404).send('');
+
+	if (req.rawBody) {
+		var receiveTime = Date.now();
+		var resultData = null;
+		try {
+			resultData = JSON.parse(req.rawBody);
+			console.log(resultData.source+','+resultData.target+','+resultData.time+','+resultData.bytes+','+resultData.timedOut+',"'+((resultData.error) ? resultData.error : '')+'"');
+		} catch (e) {}
+	}
+
+	knownAgents[agentId] = Date.now();
+	return res.status(200).send(JSON.stringify(Object.keys(knownAgents)));
+});
+
+var expressServer = app.listen(SERVER_PORT,function () {
+	console.log('LISTENING ON '+SERVER_PORT);
+	console.log('');
+});

version.h

@@ -36,11 +36,11 @@
 /**
  * Minor version
  */
-#define ZEROTIER_ONE_VERSION_MINOR 0
+#define ZEROTIER_ONE_VERSION_MINOR 1
 
 /**
  * Revision
  */
-#define ZEROTIER_ONE_VERSION_REVISION 6
+#define ZEROTIER_ONE_VERSION_REVISION 0
 
 #endif

world/README.md

@@ -0,0 +1,7 @@
+World Definitions and Generator Code
+======
+
+This little bit of code is used to generate world updates. Ordinary users probably will never need this unless they want to test or experiment.
+
+See mkworld.cpp for documentation. To build from this directory use 'source ./build.sh'.
+