Merge branch 'dev' of https://github.com/zerotier/ZeroTierOne into dev

ext/installfiles/windows/ZeroTier One.aip

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
-<DOCUMENT Type="Advanced Installer" CreateVersion="10.9" version="17.5" Modules="enterprise" RootPath="." Language="en" Id="{DC564647-6BF0-4550-87F4-89C938D0159C}">
+<DOCUMENT Type="Advanced Installer" CreateVersion="10.9" version="17.6" Modules="enterprise" RootPath="." Language="en" Id="{DC564647-6BF0-4550-87F4-89C938D0159C}">
   <COMPONENT cid="caphyon.advinst.msicomp.MsiPropsComponent">
     <ROW Property="AI_BITMAP_DISPLAY_MODE" Value="0"/>
     <ROW Property="AI_EMBD_MSI_EXTR_PATH" Value="[TempFolder]" ValueLocId="-"/>
@@ -25,10 +25,10 @@
     <ROW Property="CTRLS" Value="2"/>
     <ROW Property="MSIFASTINSTALL" MultiBuildValue="DefaultBuild:2"/>
     <ROW Property="Manufacturer" Value="ZeroTier, Inc."/>
-    <ROW Property="ProductCode" Value="1033:{14C0E8A0-190B-4059-85D1-E00BA6EE37D9} " Type="16"/>
+    <ROW Property="ProductCode" Value="1033:{A54427E1-76AC-4415-8A2C-C8928ACAE994} " Type="16"/>
     <ROW Property="ProductLanguage" Value="1033"/>
     <ROW Property="ProductName" Value="ZeroTier One"/>
-    <ROW Property="ProductVersion" Value="1.5.0" Type="32"/>
+    <ROW Property="ProductVersion" Value="1.5.1" Type="32"/>
     <ROW Property="REBOOT" MultiBuildValue="DefaultBuild:ReallySuppress"/>
     <ROW Property="RUNAPPLICATION" Value="1" Type="4"/>
     <ROW Property="SecureCustomProperties" Value="OLDPRODUCTS;AI_NEWERPRODUCTFOUND;AI_SETUPEXEPATH;SETUPEXEDIR"/>
@@ -58,7 +58,7 @@
     <ROW Directory="x86_Dir" Directory_Parent="tapwindows_Dir" DefaultDir="x86"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.MsiCompsComponent">
-    <ROW Component="AI_CustomARPName" ComponentId="{A1B82D56-799A-488A-A764-594F66740565}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
+    <ROW Component="AI_CustomARPName" ComponentId="{551D28C1-E4A1-4CC5-9FB1-A840D70DC5AB}" Directory_="APPDIR" Attributes="4" KeyPath="DisplayName" Options="1"/>
     <ROW Component="AI_DisableModify" ComponentId="{020DCABD-5D56-49B9-AF48-F07F0B55E590}" Directory_="APPDIR" Attributes="4" KeyPath="NoModify" Options="1"/>
     <ROW Component="AI_ExePath" ComponentId="{8E02B36C-7A19-429B-A93E-77A9261AC918}" Directory_="APPDIR" Attributes="4" KeyPath="AI_ExePath"/>
     <ROW Component="APPDIR" ComponentId="{4DD7907D-D7FE-4CD6-B1A0-B5C1625F5133}" Directory_="APPDIR" Attributes="0"/>
@@ -131,7 +131,7 @@
     <ROW Path="&lt;AI_DICTS&gt;ui_en.ail"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.DigCertStoreComponent">
-    <ROW TimeStampUrl="http://timestamp.verisign.com/scripts/timstamp.dll" SignerDescription="ZeroTier One" DescriptionUrl="https://www.zerotier.com/" SignOptions="0" SignTool="0" UseSha256="1" Subject="CN=&quot;Red Hat, Inc.&quot;, O=&quot;Red Hat, Inc.&quot;, L=Raleigh, S=North Carolina, C=US"/>
+    <ROW TimeStampUrl="http://timestamp.verisign.com/scripts/timstamp.dll" SignerDescription="ZeroTier One" DescriptionUrl="https://www.zerotier.com/" SignOptions="7" SignTool="5" UseSha256="1" KVTenantId="5300bf3b-0eff-4a5f-a63f-821e22ed1730" KVAppId="5f94d77e-b795-41fd-afe7-ec913b03c1d3" KVName="ZeroTier-CS" KVCertName="ZT-EV-CS" KVCertVersion="442c2d6f77874ff99eed4b36f5cb401c"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.FirewallExceptionComponent">
     <ROW FirewallException="ZeroTierOneControl" DisplayName="ZeroTier One TCP/9993" GroupName="ZeroTierOne" Enabled="1" Scope="*" Condition="1" Profiles="7" Port="9993" Protocol="TCP"/>
@@ -465,28 +465,28 @@
     <ROW XmlAttribute="xsischemaLocation" XmlElement="swidsoftware_identification_tag" Name="xsi:schemaLocation" Flags="14" Order="3" Value="http://standards.iso.org/iso/19770/-2/2008/schema.xsd software_identification_tag.xsd"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.XmlElementComponent">
-    <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="0"/>
-    <ROW XmlElement="swidentitlement_required_indicator" ParentElement="swidsoftware_identification_tag" Name="swid:entitlement_required_indicator" Condition="1" Order="0" Flags="14" Text="false"/>
-    <ROW XmlElement="swidmajor" ParentElement="swidnumeric" Name="swid:major" Condition="1" Order="0" Flags="14" Text="1"/>
-    <ROW XmlElement="swidminor" ParentElement="swidnumeric" Name="swid:minor" Condition="1" Order="1" Flags="14" Text="5"/>
-    <ROW XmlElement="swidname" ParentElement="swidproduct_version" Name="swid:name" Condition="1" Order="0" Flags="14" Text="[ProductVersion]"/>
-    <ROW XmlElement="swidname_1" ParentElement="swidsoftware_creator" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc."/>
-    <ROW XmlElement="swidname_2" ParentElement="swidsoftware_licensor" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc."/>
-    <ROW XmlElement="swidname_3" ParentElement="swidtag_creator" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc."/>
-    <ROW XmlElement="swidnumeric" ParentElement="swidproduct_version" Name="swid:numeric" Condition="1" Order="1" Flags="14"/>
-    <ROW XmlElement="swidproduct_title" ParentElement="swidsoftware_identification_tag" Name="swid:product_title" Condition="1" Order="1" Flags="14" Text="[ProductName]"/>
-    <ROW XmlElement="swidproduct_version" ParentElement="swidsoftware_identification_tag" Name="swid:product_version" Condition="1" Order="2" Flags="14"/>
-    <ROW XmlElement="swidregid" ParentElement="swidsoftware_creator" Name="swid:regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier"/>
-    <ROW XmlElement="swidregid_1" ParentElement="swidsoftware_licensor" Name="swid:regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier"/>
-    <ROW XmlElement="swidregid_2" ParentElement="swidtag_creator" Name="swid:regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier"/>
-    <ROW XmlElement="swidreview" ParentElement="swidnumeric" Name="swid:review" Condition="1" Order="3" Flags="14" Text="0"/>
-    <ROW XmlElement="swidsoftware_creator" ParentElement="swidsoftware_identification_tag" Name="swid:software_creator" Condition="1" Order="3" Flags="14"/>
-    <ROW XmlElement="swidsoftware_id" ParentElement="swidsoftware_identification_tag" Name="swid:software_id" Condition="1" Order="5" Flags="14"/>
-    <ROW XmlElement="swidsoftware_identification_tag" Name="swid:software_identification_tag" Condition="1" Order="0" Flags="14"/>
-    <ROW XmlElement="swidsoftware_licensor" ParentElement="swidsoftware_identification_tag" Name="swid:software_licensor" Condition="1" Order="4" Flags="14"/>
-    <ROW XmlElement="swidtag_creator" ParentElement="swidsoftware_identification_tag" Name="swid:tag_creator" Condition="1" Order="6" Flags="14"/>
-    <ROW XmlElement="swidtag_creator_regid" ParentElement="swidsoftware_id" Name="swid:tag_creator_regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier"/>
-    <ROW XmlElement="swidunique_id" ParentElement="swidsoftware_id" Name="swid:unique_id" Condition="1" Order="0" Flags="14" Text="ZeroTierOne"/>
+    <ROW XmlElement="swidbuild" ParentElement="swidnumeric" Name="swid:build" Condition="1" Order="2" Flags="14" Text="1" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidentitlement_required_indicator" ParentElement="swidsoftware_identification_tag" Name="swid:entitlement_required_indicator" Condition="1" Order="0" Flags="14" Text="false" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidmajor" ParentElement="swidnumeric" Name="swid:major" Condition="1" Order="0" Flags="14" Text="1" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidminor" ParentElement="swidnumeric" Name="swid:minor" Condition="1" Order="1" Flags="14" Text="5" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidname" ParentElement="swidproduct_version" Name="swid:name" Condition="1" Order="0" Flags="14" Text="[ProductVersion]" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidname_1" ParentElement="swidsoftware_creator" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc." UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidname_2" ParentElement="swidsoftware_licensor" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc." UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidname_3" ParentElement="swidtag_creator" Name="swid:name" Condition="1" Order="0" Flags="14" Text="ZeroTier, Inc." UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidnumeric" ParentElement="swidproduct_version" Name="swid:numeric" Condition="1" Order="1" Flags="14" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidproduct_title" ParentElement="swidsoftware_identification_tag" Name="swid:product_title" Condition="1" Order="1" Flags="14" Text="[ProductName]" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidproduct_version" ParentElement="swidsoftware_identification_tag" Name="swid:product_version" Condition="1" Order="2" Flags="14" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidregid" ParentElement="swidsoftware_creator" Name="swid:regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidregid_1" ParentElement="swidsoftware_licensor" Name="swid:regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidregid_2" ParentElement="swidtag_creator" Name="swid:regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidreview" ParentElement="swidnumeric" Name="swid:review" Condition="1" Order="3" Flags="14" Text="0" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidsoftware_creator" ParentElement="swidsoftware_identification_tag" Name="swid:software_creator" Condition="1" Order="3" Flags="14" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidsoftware_id" ParentElement="swidsoftware_identification_tag" Name="swid:software_id" Condition="1" Order="5" Flags="14" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidsoftware_identification_tag" Name="swid:software_identification_tag" Condition="1" Order="0" Flags="14" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidsoftware_licensor" ParentElement="swidsoftware_identification_tag" Name="swid:software_licensor" Condition="1" Order="4" Flags="14" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidtag_creator" ParentElement="swidsoftware_identification_tag" Name="swid:tag_creator" Condition="1" Order="6" Flags="14" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidtag_creator_regid" ParentElement="swidsoftware_id" Name="swid:tag_creator_regid" Condition="1" Order="1" Flags="14" Text="regid.2010-01.com.zerotier" UpdateIndexInParent="0"/>
+    <ROW XmlElement="swidunique_id" ParentElement="swidsoftware_id" Name="swid:unique_id" Condition="1" Order="0" Flags="14" Text="ZeroTierOne" UpdateIndexInParent="0"/>
   </COMPONENT>
   <COMPONENT cid="caphyon.advinst.msicomp.XmlFileComponent">
     <ROW XmlFile="regid.199509.com.example_ProductName.swidtag" FileName="REGID2~1.SWI|regid.2010-01.com.zerotier_ZeroTierOne.swidtag" DirProperty="APPDIR" Component="ProductInformation" RootElement="swidsoftware_identification_tag" Flags="25" Version="1.0" Encoding="UTF-8" IndentUnits="2"/>

make-bsd.mk

@@ -80,16 +80,16 @@ ifeq ($(CC_MACH),armv6kz)
 endif
 ifeq ($(CC_MACH),armv7)
 	ZT_ARCHITECTURE=3
-	override DEFS+=-DZT_NO_TYPE_PUNNING
+	override DEFS+=-DZT_NO_TYPE_PUNNING -DZT_AES_NO_ACCEL
 	ZT_USE_ARM32_NEON_ASM_SALSA2012=1
 endif
 ifeq ($(CC_MACH),arm64)
 	ZT_ARCHITECTURE=4
-	override DEFS+=-DZT_NO_TYPE_PUNNING
+	override DEFS+=-DZT_NO_TYPE_PUNNING -march=armv8-a+crypto
 endif
 ifeq ($(CC_MACH),aarch64)
 	ZT_ARCHITECTURE=4
-	override DEFS+=-DZT_NO_TYPE_PUNNING
+	override DEFS+=-DZT_NO_TYPE_PUNNING -march=armv8-a+crypto
 endif
 ifeq ($(CC_MACH),mipsel)
 	ZT_ARCHITECTURE=5
@@ -124,6 +124,7 @@ ifeq ($(ZT_USE_ARM32_NEON_ASM_SALSA2012),1)
 	override DEFS+=-DZT_USE_ARM32_NEON_ASM_SALSA2012
 	override CORE_OBJS+=ext/arm32-neon-salsa2012-asm/salsa2012.o
 	override ASFLAGS+=-meabi=5
+	override LDFLAGS+=-Wl,-z,notext
 endif
 
 override DEFS+=-DZT_BUILD_PLATFORM=$(ZT_BUILD_PLATFORM) -DZT_BUILD_ARCHITECTURE=$(ZT_ARCHITECTURE) -DZT_SOFTWARE_UPDATE_DEFAULT="\"disable\""

node/AES_aesni.cpp

@@ -26,7 +26,9 @@ namespace {
 
 const __m128i s_sseSwapBytes = _mm_set_epi8(0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15);
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul")))
+#endif
 __m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept
 {
 	y = _mm_shuffle_epi8(y, s_sseSwapBytes);
@@ -57,7 +59,9 @@ __m128i p_gmacPCLMUL128(const __m128i h, __m128i y) noexcept
 
 #define ZT_AES_VAES512 1
 
+#ifdef __GNUC__
 __attribute__((__target__("sse4,aes,avx,avx2,vaes,avx512f,avx512bw")))
+#endif
 void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept
 {
 	const __m512i kk0 = _mm512_broadcast_i32x4(k[0]);
@@ -107,7 +111,9 @@ void p_aesCtrInnerVAES512(unsigned int &len, const uint64_t c0, uint64_t &c1, co
 
 #define ZT_AES_VAES256 1
 
+#ifdef __GNUC__
 __attribute__((__target__("sse4,aes,avx,avx2,vaes")))
+#endif
 void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, const uint8_t *&in, uint8_t *&out, const __m128i *const k) noexcept
 {
 	const __m256i kk0 = _mm256_broadcastsi128_si256(k[0]);
@@ -175,7 +181,9 @@ void p_aesCtrInnerVAES256(unsigned int &len, const uint64_t c0, uint64_t &c1, co
 
 #endif // does compiler support AVX2 and AVX512 AES intrinsics?
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
+#endif
 __m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept
 {
 	__m128i x, y;
@@ -190,7 +198,9 @@ __m128i p_init256_1_aesni(__m128i a, __m128i b) noexcept
 	return x;
 }
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
+#endif
 __m128i p_init256_2_aesni(__m128i a, __m128i b) noexcept
 {
 	__m128i x, y, z;
@@ -208,7 +218,9 @@ __m128i p_init256_2_aesni(__m128i a, __m128i b) noexcept
 
 } // anonymous namespace
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul")))
+#endif
 void AES::GMAC::p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept
 {
 	__m128i y = _mm_loadu_si128(reinterpret_cast<const __m128i *>(_y));
@@ -274,7 +286,9 @@ void AES::GMAC::p_aesNIUpdate(const uint8_t *in, unsigned int len) noexcept
 	_rp = len; // len is always less than 16 here
 }
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,pclmul,aes")))
+#endif
 void AES::GMAC::p_aesNIFinish(uint8_t tag[16]) noexcept
 {
 	__m128i y = _mm_loadu_si128(reinterpret_cast<const __m128i *>(_y));
@@ -345,7 +359,9 @@ void AES::GMAC::p_aesNIFinish(uint8_t tag[16]) noexcept
 	_mm_storeu_si128(reinterpret_cast<__m128i *>(tag), _mm_xor_si128(_mm_shuffle_epi8(t4, s_sseSwapBytes), encIV));
 }
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes")))
+#endif
 void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) noexcept
 {
 	const __m128i dd = _mm_set_epi64x(0, (long long)_ctr[0]);
@@ -542,7 +558,9 @@ void AES::CTR::p_aesNICrypt(const uint8_t *in, uint8_t *out, unsigned int len) n
 	_ctr[1] = Utils::hton(c1);
 }
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
+#endif
 void AES::p_init_aesni(const uint8_t *key) noexcept
 {
 	__m128i t1, t2, k1, k2, k3, k4, k5, k6, k7, k8, k9, k10, k11, k12, k13;
@@ -604,7 +622,9 @@ void AES::p_init_aesni(const uint8_t *key) noexcept
 	p_k.ni.h2[3] = _mm_xor_si128(_mm_shuffle_epi32(hhhh, 78), hhhh);
 }
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
+#endif
 void AES::p_encrypt_aesni(const void *const in, void *const out) const noexcept
 {
 	__m128i tmp = _mm_loadu_si128((const __m128i *)in);
@@ -625,7 +645,9 @@ void AES::p_encrypt_aesni(const void *const in, void *const out) const noexcept
 	_mm_storeu_si128((__m128i *)out, _mm_aesenclast_si128(tmp, p_k.ni.k[14]));
 }
 
+#ifdef __GNUC__
 __attribute__((__target__("ssse3,sse4,sse4.1,sse4.2,aes,pclmul")))
+#endif
 void AES::p_decrypt_aesni(const void *in, void *out) const noexcept
 {
 	__m128i tmp = _mm_loadu_si128((const __m128i *)in);

node/Constants.hpp

@@ -107,8 +107,6 @@
 #include <xmmintrin.h>
 #include <emmintrin.h>
 #include <immintrin.h>
-#include <tmmintrin.h>
-#include <mmintrin.h>
 #endif
 
 #if (defined(__ARM_NEON) || defined(__ARM_NEON__) || defined(ZT_ARCH_ARM_HAS_NEON))
@@ -188,6 +186,9 @@
  */
 #define ZT_ADDRESS_LENGTH_HEX 10
 
+/**
+ * Size of symmetric key (only the first 32 bits are used for some ciphers)
+ */
 #define ZT_SYMMETRIC_KEY_SIZE 48
 
 /**
@@ -223,7 +224,7 @@
 /**
  * How often Topology::clean() and Network::clean() and similar are called, in ms
  */
-#define ZT_HOUSEKEEPING_PERIOD 60000
+#define ZT_HOUSEKEEPING_PERIOD 30000
 
 /**
  * Delay between WHOIS retries in ms
@@ -255,7 +256,7 @@
 /**
  * Period for multicast LIKE announcements
  */
-#define ZT_MULTICAST_ANNOUNCE_PERIOD 120000
+#define ZT_MULTICAST_ANNOUNCE_PERIOD 60000
 
 /**
  * Delay between explicit MULTICAST_GATHER requests for a given multicast channel

node/InetAddress.hpp

@@ -453,6 +453,52 @@ struct InetAddress : public sockaddr_storage
 	 */
 	bool isNetwork() const;
 
+	/**
+	 * Find the total number of prefix bits that match between this IP and another
+	 * 
+	 * @param b Second IP to compare with
+	 * @return Number of matching prefix bits or 0 if none match or IPs are of different families (e.g. v4 and v6)
+	 */
+	inline unsigned int matchingPrefixBits(const InetAddress &b) const
+	{
+		unsigned int c = 0;
+		if (ss_family == b.ss_family) {
+			switch(ss_family) {
+				case AF_INET: {
+					uint32_t ip0 = Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(this)->sin_addr.s_addr);
+					uint32_t ip1 = Utils::ntoh((uint32_t)reinterpret_cast<const struct sockaddr_in *>(&b)->sin_addr.s_addr);
+					while ((ip0 >> 31) == (ip1 >> 31)) {
+						ip0 <<= 1;
+						ip1 <<= 1;
+						if (++c == 32)
+							break;
+					}
+				}	break;
+				case AF_INET6: {
+					const uint8_t *ip0 = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(this)->sin6_addr.s6_addr);
+					const uint8_t *ip1 = reinterpret_cast<const uint8_t *>(reinterpret_cast<const struct sockaddr_in6 *>(&b)->sin6_addr.s6_addr);
+					for(unsigned int i=0;i<16;++i) {
+						if (ip0[i] == ip1[i]) {
+							c += 8;
+						} else {
+							uint8_t ip0b = ip0[i];
+							uint8_t ip1b = ip1[i];
+							uint8_t bit = 0x80;
+							while (bit != 0) {
+								if ((ip0b & bit) != (ip1b & bit))
+									break;
+								++c;
+								bit >>= 1;
+							}
+							break;
+						}
+					}
+				}	break;
+			}
+		}
+		return c;
+	}
+
 	/**
 	 * @return 14-bit (0-16383) hash of this IP's first 24 or 48 bits (for V4 or V6) for rate limiting code, or 0 if non-IP
 	 */

node/Multicaster.cpp

@@ -262,9 +262,8 @@ void Multicaster::send(
 				}
 			}
 		} else {
-			if (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) {
-				RR->t->outgoingNetworkFrameDropped(tPtr,network,src,mg.mac(),etherType,0,len,"multicast TX queue is full");
-				return;
+			while (gs.txQueue.size() >= ZT_TX_QUEUE_SIZE) {
+				gs.txQueue.pop_front();
 			}
 
 			const unsigned int gatherLimit = (limit - (unsigned int)gs.members.size()) + 1;
@@ -371,39 +370,37 @@ void Multicaster::send(
 
 void Multicaster::clean(int64_t now)
 {
-	{
-		Mutex::Lock _l(_groups_m);
-		Multicaster::Key *k = (Multicaster::Key *)0;
-		MulticastGroupStatus *s = (MulticastGroupStatus *)0;
-		Hashtable<Multicaster::Key,MulticastGroupStatus>::Iterator mm(_groups);
-		while (mm.next(k,s)) {
-			for(std::list<OutboundMulticast>::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) {
-				if ((tx->expired(now))||(tx->atLimit()))
-					s->txQueue.erase(tx++);
-				else ++tx;
-			}
+	Mutex::Lock _l(_groups_m);
+	Multicaster::Key *k = (Multicaster::Key *)0;
+	MulticastGroupStatus *s = (MulticastGroupStatus *)0;
+	Hashtable<Multicaster::Key,MulticastGroupStatus>::Iterator mm(_groups);
+	while (mm.next(k,s)) {
+		for(std::list<OutboundMulticast>::iterator tx(s->txQueue.begin());tx!=s->txQueue.end();) {
+			if ((tx->expired(now))||(tx->atLimit()))
+				s->txQueue.erase(tx++);
+			else ++tx;
+		}
 
-			unsigned long count = 0;
-			{
-				std::vector<MulticastGroupMember>::iterator reader(s->members.begin());
-				std::vector<MulticastGroupMember>::iterator writer(reader);
-				while (reader != s->members.end()) {
-					if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
-						*writer = *reader;
-						++writer;
-						++count;
-					}
-					++reader;
+		unsigned long count = 0;
+		{
+			std::vector<MulticastGroupMember>::iterator reader(s->members.begin());
+			std::vector<MulticastGroupMember>::iterator writer(reader);
+			while (reader != s->members.end()) {
+				if ((now - reader->timestamp) < ZT_MULTICAST_LIKE_EXPIRE) {
+					*writer = *reader;
+					++writer;
+					++count;
 				}
+				++reader;
 			}
+		}
 
-			if (count) {
-				s->members.resize(count);
-			} else if (s->txQueue.empty()) {
-				_groups.erase(*k);
-			} else {
-				s->members.clear();
-			}
+		if (count) {
+			s->members.resize(count);
+		} else if (s->txQueue.empty()) {
+			_groups.erase(*k);
+		} else {
+			s->members.clear();
 		}
 	}
 }

node/Multicaster.hpp

@@ -138,7 +138,7 @@ public:
 		unsigned int len);
 
 	/**
-	 * Clean up and resort database
+	 * Clean database
 	 *
 	 * @param RR Runtime environment
 	 * @param now Current time
@@ -172,14 +172,14 @@ private:
 		inline bool operator!=(const Address &a) const { return (address != a); }
 
 		Address address;
-		uint64_t timestamp; // time of last notification
+		int64_t timestamp; // time of last notification
 	};
 
 	struct MulticastGroupStatus
 	{
 		MulticastGroupStatus() : lastExplicitGather(0) {}
 
-		uint64_t lastExplicitGather;
+		int64_t lastExplicitGather;
 		std::list<OutboundMulticast> txQueue; // pending outbound multicasts
 		std::vector<MulticastGroupMember> members; // members of this group
 	};

node/Peer.cpp

@@ -409,7 +409,7 @@ void Peer::sendHELLO(void *tPtr,const int64_t localSocket,const InetAddress &atA
 	outp.cryptField(_key,startCryptedPortionAt,outp.size() - startCryptedPortionAt);
 
 	if (atAddress) {
-		outp.armor(_key,false,aesKeysIfSupported()); // false == don't encrypt full payload, but add MAC
+		outp.armor(_key,false,nullptr); // false == don't encrypt full payload, but add MAC
 		RR->node->expectReplyTo(outp.packetId());
 		RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
 	} else {

node/Utils.cpp

@@ -50,6 +50,43 @@
 #include <asm/hwcap.h>
 #endif
 
+#ifdef ZT_ARCH_ARM_HAS_NEON
+
+#ifdef __LINUX__
+#include <sys/auxv.h>
+#include <asm/hwcap.h>
+#endif
+
+#if defined(__FreeBSD__)
+#include <elf.h>
+#include <sys/auxv.h>
+static inline long getauxval(int caps)
+{
+	long hwcaps = 0;
+	elf_aux_info(caps, &hwcaps, sizeof(hwcaps));
+	return hwcaps;
+}
+#endif
+
+// If these are not even defined, obviously they are not supported.
+#ifndef HWCAP_AES
+#define HWCAP_AES 0
+#endif
+#ifndef HWCAP_CRC32
+#define HWCAP_CRC32 0
+#endif
+#ifndef HWCAP_PMULL
+#define HWCAP_PMULL 0
+#endif
+#ifndef HWCAP_SHA1
+#define HWCAP_SHA1 0
+#endif
+#ifndef HWCAP_SHA2
+#define HWCAP_SHA2 0
+#endif
+
+#endif // ZT_ARCH_ARM_HAS_NEON
+
 namespace ZeroTier {
 
 const uint64_t Utils::ZERO256[4] = {0ULL,0ULL,0ULL,0ULL};

node/Utils.hpp

@@ -94,6 +94,22 @@ public:
 	static const CPUIDRegisters CPUID;
 #endif
 
+	/**
+	 * Compute the log2 (most significant bit set) of a 32-bit integer
+	 * 
+	 * @param v Integer to compute
+	 * @return log2 or 0 if v is 0
+	 */
+	static inline unsigned int log2(uint32_t v)
+	{
+		uint32_t r = (v > 0xffff) << 4; v >>= r;
+		uint32_t shift = (v > 0xff) << 3; v >>= shift; r |= shift;
+		shift = (v > 0xf) << 2; v >>= shift; r |= shift;
+		shift = (v > 0x3) << 1; v >>= shift; r |= shift;
+		r |= (v >> 1);
+		return (unsigned int)r;
+	}
+
 	/**
 	 * Perform a time-invariant binary comparison
 	 *

one.cpp

@@ -1480,8 +1480,13 @@ static int idtool(int argc,char **argv)
 static void _sighandlerHup(int sig)
 {
 }
+static void _sighandlerReallyQuit(int sig)
+{
+	exit(0);
+}
 static void _sighandlerQuit(int sig)
 {
+	alarm(5); // force exit after 5s
 	OneService *s = zt1Service;
 	if (s)
 		s->terminate();
@@ -1873,7 +1878,7 @@ int main(int argc,char **argv)
 	signal(SIGIO,SIG_IGN);
 	signal(SIGUSR1,SIG_IGN);
 	signal(SIGUSR2,SIG_IGN);
-	signal(SIGALRM,SIG_IGN);
+	signal(SIGALRM,&_sighandlerReallyQuit);
 	signal(SIGINT,&_sighandlerQuit);
 	signal(SIGTERM,&_sighandlerQuit);
 	signal(SIGQUIT,&_sighandlerQuit);

osdep/LinuxEthernetTap.cpp

@@ -80,6 +80,7 @@ LinuxEthernetTap::LinuxEthernetTap(
 	_handler(handler),
 	_arg(arg),
 	_nwid(nwid),
+	_mac(mac),
 	_homePath(homePath),
 	_mtu(mtu),
 	_fd(0),
@@ -166,73 +167,14 @@ LinuxEthernetTap::LinuxEthernetTap(
 		throw std::runtime_error("unable to configure TUN/TAP device for TAP operation");
 	}
 
-	_dev = ifr.ifr_name;
-
 	::ioctl(_fd,TUNSETPERSIST,0); // valgrind may generate a false alarm here
-
-	// Open an arbitrary socket to talk to netlink
-	int sock = socket(AF_INET,SOCK_DGRAM,0);
-	if (sock <= 0) {
-		::close(_fd);
-		throw std::runtime_error("unable to open netlink socket");
-	}
-
-	// Set MAC address
-	ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
-	mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6);
-	if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to configure TAP hardware (MAC) address");
-		return;
-	}
-
-	// Set MTU
-	ifr.ifr_ifru.ifru_mtu = (int)mtu;
-	if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to configure TAP MTU");
-	}
-
-	if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) {
-		::close(_fd);
-		throw std::runtime_error("unable to set flags on file descriptor for TAP device");
-	}
-
-	/* Bring interface up */
-	if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to get TAP interface flags");
-	}
-	ifr.ifr_flags |= IFF_UP;
-	if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) {
-		::close(_fd);
-		::close(sock);
-		throw std::runtime_error("unable to set TAP interface flags");
-	}
-
-	::close(sock);
+	_dev = ifr.ifr_name;
 
 	// Set close-on-exec so that devices cannot persist if we fork/exec for update
 	::fcntl(_fd,F_SETFD,fcntl(_fd,F_GETFD) | FD_CLOEXEC);
 
 	(void)::pipe(_shutdownSignalPipe);
 
-	/*
-	globalDeviceMap[nwids] = _dev;
-	devmapf = fopen((_homePath + ZT_PATH_SEPARATOR_S + "devicemap").c_str(),"w");
-	if (devmapf) {
-		gdmEntry = globalDeviceMap.begin();
-		while (gdmEntry != globalDeviceMap.end()) {
-			fprintf(devmapf,"%s=%s\n",gdmEntry->first.c_str(),gdmEntry->second.c_str());
-			++gdmEntry;
-		}
-		fclose(devmapf);
-	}
-	*/
-
 	_thread = Thread::start(this);
 }
 
@@ -464,7 +406,55 @@ void LinuxEthernetTap::threadMain()
 	int n,nfds,r;
 	char getBuf[ZT_MAX_MTU + 64];
 
-	Thread::sleep(500);
+	Thread::sleep(100);
+
+	{
+		struct ifreq ifr;
+		memset(&ifr,0,sizeof(ifr));
+
+		strcpy(ifr.ifr_name,_dev.c_str());
+
+		const int sock = socket(AF_INET,SOCK_DGRAM,0);
+		if (sock <= 0)
+			return;
+
+		if (ioctl(sock,SIOCGIFFLAGS,(void *)&ifr) < 0) {
+			::close(sock);
+			printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
+			return;
+		}
+		ifr.ifr_flags |= IFF_UP;
+		if (ioctl(sock,SIOCSIFFLAGS,(void *)&ifr) < 0) {
+			::close(sock);
+			printf("WARNING: ioctl() failed setting up Linux tap device (bring interface up)\n");
+			return;
+		}
+
+		Thread::sleep(500);
+
+		ifr.ifr_ifru.ifru_hwaddr.sa_family = ARPHRD_ETHER;
+		_mac.copyTo(ifr.ifr_ifru.ifru_hwaddr.sa_data,6);
+		if (ioctl(sock,SIOCSIFHWADDR,(void *)&ifr) < 0) {
+			::close(sock);
+			printf("WARNING: ioctl() failed setting up Linux tap device (set MAC)\n");
+			return;
+		}
+
+		ifr.ifr_ifru.ifru_mtu = (int)_mtu;
+		if (ioctl(sock,SIOCSIFMTU,(void *)&ifr) < 0) {
+			::close(sock);
+			printf("WARNING: ioctl() failed setting up Linux tap device (set MTU)\n");
+			return;
+		}
+
+		if (fcntl(_fd,F_SETFL,fcntl(_fd,F_GETFL) & ~O_NONBLOCK) == -1) {
+			::close(sock);
+			printf("WARNING: ioctl() failed setting up Linux tap device (set non-blocking)\n");
+			return;
+		}
+
+		::close(sock);
+	}
 
 	FD_ZERO(&readfds);
 	FD_ZERO(&nullfds);

osdep/LinuxEthernetTap.hpp

@@ -63,6 +63,7 @@ private:
 	void (*_handler)(void *,void *,uint64_t,const MAC &,const MAC &,unsigned int,unsigned int,const void *,unsigned int);
 	void *_arg;
 	uint64_t _nwid;
+	MAC _mac;
 	Thread _thread;
 	std::string _homePath;
 	std::string _dev;

osdep/LinuxNetLink.cpp

@@ -13,6 +13,8 @@
 
 #include "../node/Constants.hpp"
 
+//#define ZT_NETLINK_TRACE
+
 #ifdef __LINUX__
 
 #include "LinuxNetLink.hpp"
@@ -43,10 +45,6 @@ struct nl_adr_req {
 LinuxNetLink::LinuxNetLink()
 	: _t()
 	, _running(false)
-	, _routes_ipv4()
-	, _rv4_m()
-	, _routes_ipv6()
-	, _rv6_m()
 	, _seq(0)
 	, _interfaces()
 	, _if_m()
@@ -85,7 +83,7 @@ void LinuxNetLink::_setSocketTimeout(int fd, int seconds)
 	tv.tv_sec = seconds;
 	tv.tv_usec = 0;
 	if(setsockopt(fd, SOL_SOCKET, SO_RCVTIMEO, (const char*)&tv, sizeof(tv)) != 0) {
-#ifdef ZT_TRACE
+#ifdef ZT_NETLINK_TRACE
 		fprintf(stderr, "setsockopt failed: %s\n", strerror(errno));
 #endif
 	}
@@ -119,8 +117,8 @@ int LinuxNetLink::_doRecv(int fd)
 			if(nlp->nlmsg_type == NLMSG_ERROR && (nlp->nlmsg_flags & NLM_F_ACK) != NLM_F_ACK) {
 				struct nlmsgerr *err = (struct nlmsgerr*)NLMSG_DATA(nlp);
 				if (err->error != 0) {
-#ifdef ZT_TRACE
-					//fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error)));
+#ifdef ZT_NETLINK_TRACE
+					fprintf(stderr, "rtnetlink error: %s\n", strerror(-(err->error)));
 #endif
 				}
 				p = buf;
@@ -145,9 +143,9 @@ int LinuxNetLink::_doRecv(int fd)
 			}
 
 			if (nlp->nlmsg_type == NLMSG_OVERRUN) {
-//#ifdef ZT_TRACE
+#ifdef ZT_NETLINK_TRACE
 				fprintf(stderr, "NLMSG_OVERRUN: Data lost\n");
-//#endif
+#endif
 				p = buf;
 				nll = 0;
 				break;
@@ -173,11 +171,10 @@ int LinuxNetLink::_doRecv(int fd)
 void LinuxNetLink::threadMain() throw()
 {
 	int rtn = 0;
-
 	while(_running) {
 		rtn = _doRecv(_fd);
 		if (rtn <= 0) {
-			Thread::sleep(100);
+			Thread::sleep(250);
 			continue;
 		}
 	}
@@ -215,6 +212,7 @@ void LinuxNetLink::_processMessage(struct nlmsghdr *nlp, int nll)
 
 void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp)
 {
+#ifdef ZT_NETLINK_TRACE
 	struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp);
 	struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap);
 	int ifal = IFA_PAYLOAD(nlp);
@@ -242,13 +240,13 @@ void LinuxNetLink::_ipAddressAdded(struct nlmsghdr *nlp)
 		}
 	}
 
-#ifdef ZT_TRACE
-	//fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
+	fprintf(stderr,"Added IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
 #endif
 }
 
 void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp)
 {
+#ifdef ZT_NETLINK_TRACE
 	struct ifaddrmsg *ifap = (struct ifaddrmsg *)NLMSG_DATA(nlp);
 	struct rtattr *rtap = (struct rtattr *)IFA_RTA(ifap);
 	int ifal = IFA_PAYLOAD(nlp);
@@ -276,8 +274,7 @@ void LinuxNetLink::_ipAddressDeleted(struct nlmsghdr *nlp)
 		}
 	}
 
-#ifdef ZT_TRACE
-	//fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
+	fprintf(stderr, "Removed IP Address %s local: %s label: %s broadcast: %s\n", addr, local, label, bcast);
 #endif
 }
 
@@ -293,28 +290,79 @@ void LinuxNetLink::_routeAdded(struct nlmsghdr *nlp)
 	struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp);
 	int rtl = RTM_PAYLOAD(nlp);
 
+	Route r;
+	bool wecare = false;
+
 	for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl))
 	{
 		switch(rtap->rta_type)
 		{
 		case RTA_DST:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_SRC:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24: 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_GATEWAY:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_OIF:
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+				case AF_INET6:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+			}
 			sprintf(ifs, "%d", *((int*)RTA_DATA(rtap)));
 			break;
 		}
 	}
-	sprintf(ms, "%d", rtp->rtm_dst_len);
 
-#ifdef ZT_TRACE
-	//fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
+	if (wecare) {
+		Mutex::Lock rl(_routes_m);
+		_routes[r.target].insert(r);
+	}
+
+#ifdef ZT_NETLINK_TRACE
+	sprintf(ms, "%d", rtp->rtm_dst_len);
+	fprintf(stderr, "Route Added: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
 #endif
 }
 
@@ -330,28 +378,79 @@ void LinuxNetLink::_routeDeleted(struct nlmsghdr *nlp)
 	struct rtattr *rtap = (struct rtattr *)RTM_RTA(rtp);
 	int rtl = RTM_PAYLOAD(nlp);
 
+	Route r;
+	bool wecare = false;
+
 	for(;RTA_OK(rtap, rtl); rtap=RTA_NEXT(rtap, rtl))
 	{
 		switch(rtap->rta_type)
 		{
 		case RTA_DST:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), dsts, 24);
+					r.target.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_SRC:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), srcs, 24);
+					r.src.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_GATEWAY:
-			inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, rtp->rtm_family == AF_INET ? 24 : 40);
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 4, 0);
+					wecare = true;
+					break;
+				case AF_INET6:
+					inet_ntop(rtp->rtm_family, RTA_DATA(rtap), gws, 24);
+					r.via.set(RTA_DATA(rtap), 16, 0);
+					wecare = true;
+					break;
+			}
 			break;
 		case RTA_OIF:
+			switch(rtp->rtm_family) {
+				case AF_INET:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+				case AF_INET6:
+					r.ifidx = *((int*)RTA_DATA(rtap));
+					wecare = true;
+					break;
+			}
 			sprintf(ifs, "%d", *((int*)RTA_DATA(rtap)));
 			break;
 		}
 	}
-	sprintf(ms, "%d", rtp->rtm_dst_len);
 
-#ifdef ZT_TRACE
-	//fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
+	if (wecare) {
+		Mutex::Lock rl(_routes_m);
+		_routes[r.target].erase(r);
+	}
+
+#ifdef ZT_NETLINK_TRACE
+	sprintf(ms, "%d", rtp->rtm_dst_len);
+	fprintf(stderr, "Route Deleted: dst %s/%s gw %s src %s if %s\n", dsts, ms, gws, srcs, ifs);
 #endif
 }
 
@@ -605,11 +704,11 @@ void LinuxNetLink::addRoute(const InetAddress &target, const InetAddress &via, c
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char  tmp[64];
-	//char tmp2[64];
-	//char tmp3[64];
-	//fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
+#ifdef ZT_NETLINK_TRACE
+	char  tmp[64];
+	char tmp2[64];
+	char tmp3[64];
+	fprintf(stderr, "Adding Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
 #endif
 
 	int rtl = sizeof(struct rtmsg);
@@ -720,11 +819,11 @@ void LinuxNetLink::delRoute(const InetAddress &target, const InetAddress &via, c
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char  tmp[64];
-	//char tmp2[64];
-	//char tmp3[64];
-	//fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
+#ifdef ZT_NETLINK_TRACE
+	char  tmp[64];
+	char tmp2[64];
+	char tmp3[64];
+	fprintf(stderr, "Removing Route. target: %s via: %s src: %s iface: %s\n", target.toString(tmp), via.toString(tmp2), src.toString(tmp3), ifaceName);
 #endif
 
 	int rtl = sizeof(struct rtmsg);
@@ -839,9 +938,9 @@ void LinuxNetLink::addAddress(const InetAddress &addr, const char *iface)
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char tmp[128];
-	//fprintf(stderr, "Adding IP address %s to interface %s", addr.toString(tmp), iface);
+#ifdef ZT_NETLINK_TRACE
+	char tmp[128];
+	fprintf(stderr, "Adding IP address %s to interface %s\n", addr.toString(tmp), iface);
 #endif
 
 	int interface_index = _indexForInterface(iface);
@@ -955,9 +1054,9 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface)
 		return;
 	}
 
-#ifdef ZT_TRACE
-	//char tmp[128];
-	//fprintf(stderr, "Removing IP address %s from interface %s", addr.toString(tmp), iface);
+#ifdef ZT_NETLINK_TRACE
+	char tmp[128];
+	fprintf(stderr, "Removing IP address %s from interface %s\n", addr.toString(tmp), iface);
 #endif
 
 	int interface_index = _indexForInterface(iface);
@@ -1043,14 +1142,23 @@ void LinuxNetLink::removeAddress(const InetAddress &addr, const char *iface)
 	close(fd);
 }
 
-RouteList LinuxNetLink::getIPV4Routes() const
-{
-	return _routes_ipv4;
-}
-
-RouteList LinuxNetLink::getIPV6Routes() const
+bool LinuxNetLink::routeIsSet(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname)
 {
-	return _routes_ipv6;
+	Mutex::Lock rl(_routes_m);
+	const std::set<LinuxNetLink::Route> &rs = _routes[target];
+	for(std::set<LinuxNetLink::Route>::const_iterator ri(rs.begin());ri!=rs.end();++ri) {
+		if ((ri->via == via)&&(ri->src == src)) {
+			if (ifname) {
+				Mutex::Lock ifl(_if_m);
+				const iface_entry *ife = _interfaces.get(ri->ifidx);
+				if ((ife)&&(!strncmp(ife->ifacename,ifname,IFNAMSIZ)))
+					return true;
+			} else {
+				return true;
+			}
+		}
+	}
+	return false;
 }
 
 int LinuxNetLink::_indexForInterface(const char *iface)

osdep/LinuxNetLink.hpp

@@ -19,6 +19,8 @@
 #ifdef __LINUX__
 
 #include <vector>
+#include <map>
+#include <set>
 
 #include <sys/socket.h>
 #include <asm/types.h>
@@ -35,84 +37,112 @@
 
 namespace ZeroTier {
 
-struct route_entry {
-	InetAddress target;
-	InetAddress via;
-	int if_index;
-	char iface[IFNAMSIZ];
-};
-typedef std::vector<route_entry> RouteList;
-
 /**
  * Interface with Linux's RTNETLINK
  */
 class LinuxNetLink
 {
 private:
-    LinuxNetLink();
-    ~LinuxNetLink();
+	LinuxNetLink();
+	~LinuxNetLink();
 
 public:
-    static LinuxNetLink& getInstance()
-    {
-        static LinuxNetLink instance;
-        return instance;
-    }
-
-    LinuxNetLink(LinuxNetLink const&) = delete;
-    void operator=(LinuxNetLink const&) = delete;
-
-    void addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
-    void delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
-    RouteList getIPV4Routes() const;
-    RouteList getIPV6Routes() const;
-
-    void addAddress(const InetAddress &addr, const char *iface);
-    void removeAddress(const InetAddress &addr, const char *iface);
+	struct Route {
+		InetAddress target;
+		InetAddress via;
+		InetAddress src;
+		int ifidx;
+
+		inline bool operator==(const Route &r) const
+		{ return ((target == r.target)&&(via == r.via)&&(src == r.src)&&(ifidx == r.ifidx)); }
+		inline bool operator!=(const Route &r) const
+		{ return (!(*this == r)); }
+		inline bool operator<(const Route &r) const
+		{
+			if (target < r.target) {
+				return true;
+			} else if (target == r.target) {
+				if (via < r.via) {
+					return true;
+				} else if (via == r.via) {
+					if (src < r.src) {
+						return true;
+					} else if (src == r.src) {
+						return (ifidx < r.ifidx);
+					}
+				}
+			}
+			return false;
+		}
+		inline bool operator>(const Route &r) const
+		{ return (r < *this); }
+		inline bool operator<=(const Route &r) const
+		{ return !(r < *this); }
+		inline bool operator>=(const Route &r) const
+		{ return !(*this < r); }
+	};
+
+	static LinuxNetLink& getInstance()
+	{
+		static LinuxNetLink instance;
+		return instance;
+	}
+
+	LinuxNetLink(LinuxNetLink const&) = delete;
+	void operator=(LinuxNetLink const&) = delete;
+
+	void addRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
+	void delRoute(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifaceName);
+
+	void addAddress(const InetAddress &addr, const char *iface);
+	void removeAddress(const InetAddress &addr, const char *iface);
+
+	bool routeIsSet(const InetAddress &target, const InetAddress &via, const InetAddress &src, const char *ifname);
+
+	void threadMain() throw();
 
-    void threadMain() throw();
 private:
-    int _doRecv(int fd);
-
-    void _processMessage(struct nlmsghdr *nlp, int nll);
-    void _routeAdded(struct nlmsghdr *nlp);
-    void _routeDeleted(struct nlmsghdr *nlp);
-    void _linkAdded(struct nlmsghdr *nlp);
-    void _linkDeleted(struct nlmsghdr *nlp);
-    void _ipAddressAdded(struct nlmsghdr *nlp);
-    void _ipAddressDeleted(struct nlmsghdr *nlp);
-
-    void _requestInterfaceList();
-    void _requestIPv4Routes();
-    void _requestIPv6Routes();
-
-    int _indexForInterface(const char *iface);
-
-    void _setSocketTimeout(int fd, int seconds = 1);
-
-    Thread _t;
-    bool _running;
-
-    RouteList _routes_ipv4;
-    Mutex _rv4_m;
-    RouteList _routes_ipv6;
-    Mutex _rv6_m;
-
-    uint32_t _seq;
-
-    struct iface_entry {
-        int index;
-        char ifacename[IFNAMSIZ];
-        char mac[18];
-        char mac_bin[6];
-        unsigned int mtu;
-    };
-    Hashtable<int, iface_entry> _interfaces;
-    Mutex _if_m;
-
-    // socket communication vars;
-    int _fd;
-    struct sockaddr_nl _la;
+	int _doRecv(int fd);
+
+	void _processMessage(struct nlmsghdr *nlp, int nll);
+	void _routeAdded(struct nlmsghdr *nlp);
+	void _routeDeleted(struct nlmsghdr *nlp);
+	void _linkAdded(struct nlmsghdr *nlp);
+	void _linkDeleted(struct nlmsghdr *nlp);
+	void _ipAddressAdded(struct nlmsghdr *nlp);
+	void _ipAddressDeleted(struct nlmsghdr *nlp);
+
+	void _requestInterfaceList();
+	void _requestIPv4Routes();
+	void _requestIPv6Routes();
+
+	int _indexForInterface(const char *iface);
+
+	void _setSocketTimeout(int fd, int seconds = 1);
+
+	Thread _t;
+	bool _running;
+
+	uint32_t _seq;
+
+	std::map< InetAddress,std::set<LinuxNetLink::Route> > _routes;
+	Mutex _routes_m;
+
+	struct iface_entry {
+		iface_entry()
+		{ memset(this,0,sizeof(iface_entry)); }
+		int index;
+		char ifacename[IFNAMSIZ];
+		char mac[18];
+		char mac_bin[6];
+		unsigned int mtu;
+	};
+	Hashtable<int, iface_entry> _interfaces;
+	Mutex _if_m;
+
+	// socket communication vars;
+	int _fd;
+	struct sockaddr_nl _la;
 };
 
 }

osdep/MacEthernetTap.cpp

@@ -79,6 +79,7 @@ MacEthernetTap::MacEthernetTap(
 	_homePath(homePath),
 	_mtu(mtu),
 	_metric(metric),
+	_devNo(0),
 	_agentStdin(-1),
 	_agentStdout(-1),
 	_agentStderr(-1),
@@ -97,7 +98,7 @@ MacEthernetTap::MacEthernetTap(
 	agentPath.push_back(ZT_PATH_SEPARATOR);
 	agentPath.append("MacEthernetTapAgent");
 	if (!OSUtils::fileExists(agentPath.c_str()))
-		throw std::runtime_error("MacEthernetTapAgent not installed in ZeroTier home");
+		throw std::runtime_error("MacEthernetTapAgent not present in ZeroTier home");
 
 	Mutex::Lock _gl(globalTapCreateLock); // only make one at a time
 
@@ -112,7 +113,7 @@ MacEthernetTap::MacEthernetTap(
 			while (p) {
 				int nameLen = (int)strlen(p->ifa_name);
 				// Delete feth# from feth0 to feth9999, but don't touch >10000.
-				if ((!strncmp(p->ifa_name,"feth",4))&&(nameLen >= 5)&&(nameLen < 9)&&(deleted.count(std::string(p->ifa_name)) == 0)) {
+				if ((!strncmp(p->ifa_name,"feth",4))&&(nameLen >= 5)&&(nameLen <= 8)&&(deleted.count(std::string(p->ifa_name)) == 0)) {
 					deleted.insert(std::string(p->ifa_name));
 					const char *args[4];
 					args[0] = "/sbin/ifconfig";
@@ -156,10 +157,11 @@ MacEthernetTap::MacEthernetTap(
 			if (devNo < 100)
 				devNo = 100;
 		} else {
+			_dev = devstr;
+			_devNo = devNo;
 			break;
 		}
 	}
-	_dev = devstr;
 
 	if (::pipe(_shutdownSignalPipe))
 		throw std::runtime_error("pipe creation failed");
@@ -204,22 +206,50 @@ MacEthernetTap::MacEthernetTap(
 
 MacEthernetTap::~MacEthernetTap()
 {
+	char tmp[64];
+	const char *args[4];
+	pid_t pid0,pid1;
+
 	MacDNSHelper::removeDNS(_nwid);
-	
+
 	Mutex::Lock _gl(globalTapCreateLock);
 	::write(_shutdownSignalPipe[1],"\0",1); // causes thread to exit
-	Thread::join(_thread);
-	::close(_shutdownSignalPipe[0]);
-	::close(_shutdownSignalPipe[1]);
+
 	int ec = 0;
-	::kill(_agentPid,SIGTERM);
+	::kill(_agentPid,SIGKILL);
 	::waitpid(_agentPid,&ec,0);
-	::close(_agentStdin);
-	::close(_agentStdout);
-	::close(_agentStderr);
-	::close(_agentStdin2);
-	::close(_agentStdout2);
-	::close(_agentStderr2);
+
+	args[0] = "/sbin/ifconfig";
+	args[1] = _dev.c_str();
+	args[2] = "destroy";
+	args[3] = (char *)0;
+	pid0 = vfork();
+	if (pid0 == 0) {
+		execv(args[0],const_cast<char **>(args));
+		_exit(-1);
+	}
+
+	snprintf(tmp,sizeof(tmp),"feth%u",_devNo + 5000);
+	//args[0] = "/sbin/ifconfig";
+	args[1] = tmp;
+	//args[2] = "destroy";
+	//args[3] = (char *)0;
+	pid1 = vfork();
+	if (pid1 == 0) {
+		execv(args[0],const_cast<char **>(args));
+		_exit(-1);
+	}
+
+	if (pid0 > 0) {
+		int rv = 0;
+		waitpid(pid0,&rv,0);
+	}
+	if (pid1 > 0) {
+		int rv = 0;
+		waitpid(pid1,&rv,0);
+	}
+
+	Thread::join(_thread);
 }
 
 void MacEthernetTap::setEnabled(bool en) { _enabled = en; }
@@ -456,6 +486,15 @@ void MacEthernetTap::threadMain()
 			*/
 		}
 	}
+
+	::close(_agentStdin);
+	::close(_agentStdout);
+	::close(_agentStderr);
+	::close(_agentStdin2);
+	::close(_agentStdout2);
+	::close(_agentStderr2);
+	::close(_shutdownSignalPipe[0]);
+	::close(_shutdownSignalPipe[1]);
 }
 
 void MacEthernetTap::setDns(const char *domain, const std::vector<InetAddress> &servers)

osdep/MacEthernetTap.hpp

@@ -72,6 +72,7 @@ private:
 	Mutex _putLock;
 	unsigned int _mtu;
 	unsigned int _metric;
+	unsigned int _devNo;
 	int _shutdownSignalPipe[2];
 	int _agentStdin,_agentStdout,_agentStderr,_agentStdin2,_agentStdout2,_agentStderr2;
 	long _agentPid;

osdep/MacEthernetTapAgent.c

@@ -29,13 +29,13 @@
  * is limited to 2048. AF_NDRV packet injection is required to inject
  * ZeroTier's large MTU frames.
  *
- * Benchmarks show that this performs similarly to the old tap.kext driver,
- * and a kext is no longer required. Splitting it off into an agent will
- * also make it easier to have zerotier-one itself drop permissions.
- *
  * All this stuff is basically undocumented. A lot of tracing through
  * the Darwin/XNU kernel source was required to figure out how to make
  * this actually work.
+ * 
+ * We hope to develop a DriverKit-based driver in the near-mid future to
+ * replace this weird hack, but it works for now through Big Sur in our
+ * testing.
  *
  * See also:
  *

osdep/ManagedRoute.cpp

@@ -49,6 +49,9 @@
 #include <utility>
 
 #include "ManagedRoute.hpp"
+#ifdef __LINUX__
+#include "LinuxNetLink.hpp"
+#endif
 
 #define ZT_BSD_ROUTE_CMD "/sbin/route"
 #define ZT_LINUX_IP_COMMAND "/sbin/ip"
@@ -269,6 +272,8 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress
 #ifdef __LINUX__ // ----------------------------------------------------------
 #define ZT_ROUTING_SUPPORT_FOUND 1
 
+// This has been replaced by LinuxNetLink
+/*
 static void _routeCmd(const char *op,const InetAddress &target,const InetAddress &via,const char *localInterface)
 {
 	long p = (long)fork();
@@ -289,6 +294,7 @@ static void _routeCmd(const char *op,const InetAddress &target,const InetAddress
 		::_exit(-1);
 	}
 }
+*/
 
 #endif // __LINUX__ ----------------------------------------------------------
 
@@ -388,6 +394,33 @@ static bool _winHasRoute(const NET_LUID &interfaceLuid, const NET_IFINDEX &inter
 
 } // anonymous namespace
 
+ManagedRoute::ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device)
+{
+	_target = target;
+	_via = via;
+	_src = src;
+
+	if (_via.ss_family == AF_INET) {
+		_via.setPort(32);
+	} else if (_via.ss_family == AF_INET6) {
+		_via.setPort(128);
+	}
+
+	if (_src.ss_family == AF_INET) {
+		_src.setPort(32);
+	} else if (_src.ss_family == AF_INET6) {
+		_src.setPort(128);
+	}
+
+	Utils::scopy(_device,sizeof(_device),device);
+	_systemDevice[0] = (char)0;
+}
+
+ManagedRoute::~ManagedRoute()
+{
+	this->remove();
+}
+
 /* Linux NOTE: for default route override, some Linux distributions will
  * require a change to the rp_filter parameter. A value of '1' will prevent
  * default route override from working properly.
@@ -485,13 +518,14 @@ bool ManagedRoute::sync()
 
 #ifdef __LINUX__ // ----------------------------------------------------------
 
-	if (!_applied.count(leftt)) {
+	const char *const devptr = (_via) ? (const char *)0 : _device;
+	if ((leftt)&&(!LinuxNetLink::getInstance().routeIsSet(leftt,_via,_src,devptr))) {
 		_applied[leftt] = false; // boolean unused
-		_routeCmd("replace",leftt,_via,(_via) ? (const char *)0 : _device);
+		LinuxNetLink::getInstance().addRoute(leftt, _via, _src, devptr);
 	}
-	if ((rightt)&&(!_applied.count(rightt))) {
+	if ((rightt)&&(!LinuxNetLink::getInstance().routeIsSet(rightt,_via,_src,devptr))) {
 		_applied[rightt] = false; // boolean unused
-		_routeCmd("replace",rightt,_via,(_via) ? (const char *)0 : _device);
+		LinuxNetLink::getInstance().addRoute(rightt, _via, _src, devptr);
 	}
 
 #endif // __LINUX__ ----------------------------------------------------------
@@ -539,7 +573,8 @@ void ManagedRoute::remove()
 #endif // __BSD__ ------------------------------------------------------------
 
 #ifdef __LINUX__ // ----------------------------------------------------------
-		_routeCmd("del",r->first,_via,(_via) ? (const char *)0 : _device);
+		//_routeCmd("del",r->first,_via,(_via) ? (const char *)0 : _device);
+		LinuxNetLink::getInstance().delRoute(r->first,_via,_src,(_via) ? (const char *)0 : _device);
 #endif // __LINUX__ ----------------------------------------------------------
 
 #ifdef __WINDOWS__ // --------------------------------------------------------

osdep/ManagedRoute.hpp

@@ -36,28 +36,8 @@ class ManagedRoute
 	friend class SharedPtr<ManagedRoute>;
 
 public:
-	ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device)
-	{
-		_target = target;
-		_via = via;
-		_src = src;
-		if (via.ss_family == AF_INET)
-			_via.setPort(32);
-		else if (via.ss_family == AF_INET6)
-			_via.setPort(128);
-		if (src.ss_family == AF_INET) {
-			_src.setPort(32);
-		} else if (src.ss_family == AF_INET6) {
-			_src.setPort(128);
-		}
-		Utils::scopy(_device,sizeof(_device),device);
-		_systemDevice[0] = (char)0;
-	}
-
-	~ManagedRoute()
-	{
-		this->remove();
-	}
+	ManagedRoute(const InetAddress &target,const InetAddress &via,const InetAddress &src,const char *device);
+	~ManagedRoute();
 
 	/**
 	 * Set or update currently set route

service/OneService.cpp

@@ -43,7 +43,6 @@
 #include "../node/Peer.hpp"
 
 #include "../osdep/Phy.hpp"
-#include "../osdep/Thread.hpp"
 #include "../osdep/OSUtils.hpp"
 #include "../osdep/Http.hpp"
 #include "../osdep/PortMapper.hpp"
@@ -529,7 +528,7 @@ public:
 		std::shared_ptr<EthernetTap> tap;
 		ZT_VirtualNetworkConfig config; // memcpy() of raw config from core
 		std::vector<InetAddress> managedIps;
-		std::list< SharedPtr<ManagedRoute> > managedRoutes;
+		std::map< InetAddress, SharedPtr<ManagedRoute> > managedRoutes;
 		NetworkSettings settings;
 	};
 	std::map<uint64_t,NetworkState> _nets;
@@ -918,8 +917,8 @@ public:
 					OSUtils::cleanDirectory((_homePath + ZT_PATH_SEPARATOR_S "peers.d").c_str(),now - 2592000000LL); // delete older than 30 days
 				}
 
-				const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 100;
-				clockShouldBe = now + (uint64_t)delay;
+				const unsigned long delay = (dl > now) ? (unsigned long)(dl - now) : 500;
+				clockShouldBe = now + (int64_t)delay;
 				_phy.poll(delay);
 			}
 		} catch (std::exception &e) {
@@ -1885,9 +1884,9 @@ public:
 	}
 
 	// Match only an IP from a vector of IPs -- used in syncManagedStuff()
-	bool matchIpOnly(const std::vector<InetAddress> &ips,const InetAddress &ip) const
+	inline bool matchIpOnly(const std::set<InetAddress> &ips,const InetAddress &ip) const
 	{
-		for(std::vector<InetAddress>::const_iterator i(ips.begin());i!=ips.end();++i) {
+		for(std::set<InetAddress>::const_iterator i(ips.begin());i!=ips.end();++i) {
 			if (i->ipsEqual(ip))
 				return true;
 		}
@@ -1932,88 +1931,83 @@ public:
 		}
 
 		if (syncRoutes) {
-			char tapdev[64];
+			// Get tap device name (use LUID in hex on Windows) and IP addresses.
 #if defined(__WINDOWS__) && !defined(ZT_SDK)
-			OSUtils::ztsnprintf(tapdev,sizeof(tapdev),"%.16llx",(unsigned long long)((WindowsEthernetTap *)(n.tap.get()))->luid().Value);
+			char tapdevbuf[64];
+			OSUtils::ztsnprintf(tapdevbuf,sizeof(tapdevbuf),"%.16llx",(unsigned long long)((WindowsEthernetTap *)(n.tap.get()))->luid().Value);
+			std::string tapdev(tapdevbuf);
 #else
-			Utils::scopy(tapdev,sizeof(tapdev),n.tap->deviceName().c_str());
+			std::string tapdev(n.tap->deviceName());
 #endif
 
-			std::vector<InetAddress> myIps(n.tap->ips());
+			std::vector<InetAddress> tapIps(n.tap->ips());
+			std::set<InetAddress> myIps(tapIps.begin(), tapIps.end());
+			for(unsigned int i=0;i<n.config.assignedAddressCount;++i)
+				myIps.insert(InetAddress(n.config.assignedAddresses[i]));
 
-			// Nuke applied routes that are no longer in n.config.routes[] and/or are not allowed
-			for(std::list< SharedPtr<ManagedRoute> >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();) {
-				bool haveRoute = false;
-				if ( (checkIfManagedIsAllowed(n,(*mr)->target())) && (((*mr)->via().ss_family != (*mr)->target().ss_family)||(!matchIpOnly(myIps,(*mr)->via()))) ) {
-					for(unsigned int i=0;i<n.config.routeCount;++i) {
-						const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target));
-						const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via));
-						if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (strcmp(tapdev,(*mr)->device())==0) ) ) {
-							haveRoute = true;
-							break;
-						}
-					}
-				}
-				if (haveRoute) {
-					++mr;
-				} else {
-					n.managedRoutes.erase(mr++);
-				}
-			}
-
-			// Apply routes in n.config.routes[] that we haven't applied yet, and sync those we have in case shadow routes need to change
+			std::set<InetAddress> haveRouteTargets;
 			for(unsigned int i=0;i<n.config.routeCount;++i) {
 				const InetAddress *const target = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].target));
 				const InetAddress *const via = reinterpret_cast<const InetAddress *>(&(n.config.routes[i].via));
 
-				const InetAddress *src = NULL;
-				for (unsigned int j=0; j<n.config.assignedAddressCount; ++j) {
-					const InetAddress *const tmp = reinterpret_cast<const InetAddress *>(&(n.config.assignedAddresses[j]));
-					if (target->isV4() && tmp->isV4()) {
-						src = reinterpret_cast<InetAddress *>(&(n.config.assignedAddresses[j]));
-						break;
-					} else if (target->isV6() && tmp->isV6()) {
-						src = reinterpret_cast<InetAddress *>(&(n.config.assignedAddresses[j]));
-						break;
+				// Make sure we are allowed to set this managed route, and that 'via' is not our IP. The latter
+				// avoids setting routes via the router on the router.
+				if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) )
+					continue;
+
+				// Find an IP on the interface that can be a source IP, abort if no IPs assigned.
+				const InetAddress *src = nullptr;
+				unsigned int mostMatchingPrefixBits = 0;
+				for(std::set<InetAddress>::const_iterator i(myIps.begin());i!=myIps.end();++i) {
+					const unsigned int matchingPrefixBits = i->matchingPrefixBits(*target);
+					if (matchingPrefixBits >= mostMatchingPrefixBits) {
+						mostMatchingPrefixBits = matchingPrefixBits;
+						src = &(*i);
 					}
 				}
 				if (!src)
-					src = &NULL_INET_ADDR;
-
-				if ( (!checkIfManagedIsAllowed(n,*target)) || ((via->ss_family == target->ss_family)&&(matchIpOnly(myIps,*via))) )
 					continue;
 
-				bool haveRoute = false;
-
-				// Ignore routes implied by local managed IPs since adding the IP adds the route
+				// Ignore routes implied by local managed IPs since adding the IP adds the route.
+				// Apple on the other hand seems to need this at least on some versions.
 #ifndef __APPLE__
+				bool haveRoute = false;
 				for(std::vector<InetAddress>::iterator ip(n.managedIps.begin());ip!=n.managedIps.end();++ip) {
 					if ((target->netmaskBits() == ip->netmaskBits())&&(target->containsAddress(*ip))) {
 						haveRoute = true;
 						break;
 					}
 				}
-#endif
-				if (haveRoute)
-					continue;
-#ifndef ZT_SDK
-				// If we've already applied this route, just sync it and continue
-				for(std::list< SharedPtr<ManagedRoute> >::iterator mr(n.managedRoutes.begin());mr!=n.managedRoutes.end();++mr) {
-					if ( ((*mr)->target() == *target) && ( ((via->ss_family == target->ss_family)&&((*mr)->via().ipsEqual(*via))) || (tapdev == (*mr)->device()) ) ) {
-						haveRoute = true;
-						(*mr)->sync();
-						break;
-					}
-				}
 				if (haveRoute)
 					continue;
+#endif
+
+				haveRouteTargets.insert(*target);
 
-				// Add and apply new routes
-				n.managedRoutes.push_back(SharedPtr<ManagedRoute>(new ManagedRoute(*target,*via,*src,tapdev)));
-				if (!n.managedRoutes.back()->sync())
-					n.managedRoutes.pop_back();
+#ifndef ZT_SDK
+				SharedPtr<ManagedRoute> &mr = n.managedRoutes[*target];
+				if (!mr)
+					mr.set(new ManagedRoute(*target, *via, *src, tapdev.c_str()));
 #endif
 			}
+
+			for(std::map< InetAddress, SharedPtr<ManagedRoute> >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();) {
+				if (haveRouteTargets.find(r->first) == haveRouteTargets.end())
+					n.managedRoutes.erase(r++);
+				else ++r;
+			}
+
+			// Sync device-local managed routes first, then indirect results. That way
+			// we don't get destination unreachable for routes that are via things
+			// that do not yet have routes in the system.
+			for(std::map< InetAddress, SharedPtr<ManagedRoute> >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) {
+				if (!r->second->via())
+					r->second->sync();
+			}
+			for(std::map< InetAddress, SharedPtr<ManagedRoute> >::iterator r(n.managedRoutes.begin());r!=n.managedRoutes.end();++r) {
+				if (r->second->via())
+					r->second->sync();
+			}
 		}
 
 		if (syncDns) {

version.h

@@ -27,7 +27,7 @@
 /**
  * Revision
  */
-#define ZEROTIER_ONE_VERSION_REVISION 0
+#define ZEROTIER_ONE_VERSION_REVISION 1
 
 /**
  * Build version

windows/ZeroTierOne/ZeroTierOne.vcxproj

@@ -51,6 +51,8 @@
     <ClCompile Include="..\..\ext\miniupnpc\upnperrors.c" />
     <ClCompile Include="..\..\ext\miniupnpc\upnpreplyparse.c" />
     <ClCompile Include="..\..\node\AES.cpp" />
+    <ClCompile Include="..\..\node\AES_aesni.cpp" />
+    <ClCompile Include="..\..\node\AES_armcrypto.cpp" />
     <ClCompile Include="..\..\node\Bond.cpp" />
     <ClCompile Include="..\..\node\BondController.cpp" />
     <ClCompile Include="..\..\node\C25519.cpp">
@@ -443,7 +445,7 @@
       </AdditionalIncludeDirectories>
       <PreprocessorDefinitions>ZT_EXPORT;FD_SETSIZE=1024;STATICLIB;ZT_SOFTWARE_UPDATE_DEFAULT="apply";ZT_SALSA20_SSE;ZT_USE_MINIUPNPC;MINIUPNP_STATICLIB;WIN32;NOMINMAX;ZT_BUILD_PLATFORM=2;ZT_BUILD_ARCHITECTURE=2;%(PreprocessorDefinitions)</PreprocessorDefinitions>
       <RuntimeLibrary>MultiThreaded</RuntimeLibrary>
-      <EnableEnhancedInstructionSet>NotSet</EnableEnhancedInstructionSet>
+      <EnableEnhancedInstructionSet>StreamingSIMDExtensions2</EnableEnhancedInstructionSet>
       <StringPooling>true</StringPooling>
       <InlineFunctionExpansion>AnySuitable</InlineFunctionExpansion>
       <FavorSizeOrSpeed>Speed</FavorSizeOrSpeed>

windows/ZeroTierOne/ZeroTierOne.vcxproj.filters

@@ -285,6 +285,12 @@
     <ClCompile Include="..\..\node\AES.cpp">
       <Filter>Source Files\node</Filter>
     </ClCompile>
+    <ClCompile Include="..\..\node\AES_aesni.cpp">
+      <Filter>Source Files\node</Filter>
+    </ClCompile>
+    <ClCompile Include="..\..\node\AES_armcrypto.cpp">
+      <Filter>Source Files\node</Filter>
+    </ClCompile>
   </ItemGroup>
   <ItemGroup>
     <ClInclude Include="resource.h">