Merge branch 'adamierymenko-dev' into netcon

include/ZeroTierOne.h

@@ -656,7 +656,12 @@ typedef void ZT1_Node;
  * on failure, and this results in the network being placed into the
  * PORT_ERROR state.
  */
-typedef int (*ZT1_VirtualNetworkConfigFunction)(ZT1_Node *,void *,uint64_t,enum ZT1_VirtualNetworkConfigOperation,const ZT1_VirtualNetworkConfig *);
+typedef int (*ZT1_VirtualNetworkConfigFunction)(
+	ZT1_Node *,
+	void *,
+	uint64_t,
+	enum ZT1_VirtualNetworkConfigOperation,
+	const ZT1_VirtualNetworkConfig *);
 
 /**
  * Function to send a frame out to a virtual network port
@@ -665,7 +670,16 @@ typedef int (*ZT1_VirtualNetworkConfigFunction)(ZT1_Node *,void *,uint64_t,enum
  * (5) destination MAC, (6) ethertype, (7) VLAN ID, (8) frame data,
  * (9) frame length.
  */
-typedef void (*ZT1_VirtualNetworkFrameFunction)(ZT1_Node *,void *,uint64_t,uint64_t,uint64_t,unsigned int,unsigned int,const void *,unsigned int);
+typedef void (*ZT1_VirtualNetworkFrameFunction)(
+	ZT1_Node *,
+	void *,
+	uint64_t,
+	uint64_t,
+	uint64_t,
+	unsigned int,
+	unsigned int,
+	const void *,
+	unsigned int);
 
 /**
  * Callback for events
@@ -676,7 +690,11 @@ typedef void (*ZT1_VirtualNetworkFrameFunction)(ZT1_Node *,void *,uint64_t,uint6
  * whether it is present at all) is event type dependent. See the comments
  * in the definition of ZT1_Event.
  */
-typedef void (*ZT1_EventCallback)(ZT1_Node *,void *,enum ZT1_Event,const void *);
+typedef void (*ZT1_EventCallback)(
+	ZT1_Node *,
+	void *,
+	enum ZT1_Event,
+	const void *);
 
 /**
  * Function to get an object from the data store
@@ -698,7 +716,14 @@ typedef void (*ZT1_EventCallback)(ZT1_Node *,void *,enum ZT1_Event,const void *)
  * read. The caller may call the function multiple times to read the whole
  * object.
  */
-typedef long (*ZT1_DataStoreGetFunction)(ZT1_Node *,void *,const char *,void *,unsigned long,unsigned long,unsigned long *);
+typedef long (*ZT1_DataStoreGetFunction)(
+	ZT1_Node *,
+	void *,
+	const char *,
+	void *,
+	unsigned long,
+	unsigned long,
+	unsigned long *);
 
 /**
  * Function to store an object in the data store
@@ -716,19 +741,40 @@ typedef long (*ZT1_DataStoreGetFunction)(ZT1_Node *,void *,const char *,void *,u
  * If the data pointer is null, this must be interpreted as a delete
  * operation.
  */
-typedef int (*ZT1_DataStorePutFunction)(ZT1_Node *,void *,const char *,const void *,unsigned long,int);
+typedef int (*ZT1_DataStorePutFunction)(
+	ZT1_Node *,
+	void *,
+	const char *,
+	const void *,
+	unsigned long,
+	int);
 
 /**
  * Function to send a ZeroTier packet out over the wire
  *
- * Parameters: (1) node, (2) user ptr, (3) address, (4) packet data,
- * (5) packet data length.
+ * Parameters:
+ *  (1) Node
+ *  (2) User pointer
+ *  (3) Local interface ID, -1==unspcified/random
+ *  (4) Remote address
+ *  (5) Packet data
+ *  (6) Packet length
+ *
+ * If you have only one local interface it is fine to ignore the local
+ * interface ID field. This is used to support different local interface
+ * endpoints and differentiation between them.
  *
  * The function must return zero on success and may return any error code
  * on failure. Note that success does not (of course) guarantee packet
  * delivery. It only means that the packet appears to have been sent.
  */
-typedef int (*ZT1_WirePacketSendFunction)(ZT1_Node *,void *,const struct sockaddr_storage *,const void *,unsigned int);
+typedef int (*ZT1_WirePacketSendFunction)(
+	ZT1_Node *,                      /* Node */
+	void *,                          /* User ptr */
+	int,                             /* Local interface ID, -1 for unspecified/random */
+	const struct sockaddr_storage *, /* Remote address */
+	const void *,                    /* Packet data */
+	unsigned int);                   /* Packet length */
 
 /****************************************************************************/
 /* C Node API                                                               */
@@ -747,7 +793,7 @@ typedef int (*ZT1_WirePacketSendFunction)(ZT1_Node *,void *,const struct sockadd
  * @param dataStorePutFunction Function called to put objects in persistent storage
  * @param virtualNetworkConfigFunction Function to be called when virtual LANs are created, deleted, or their config parameters change
  * @param eventCallback Function to receive status updates and non-fatal error notices
- * @param overrideRootTopology If not NULL, must contain string-serialize root topology (for testing, default: NULL)
+ * @param overrideRootTopology Alternative root server topology or NULL for default (mostly for test/debug use)
  * @return OK (0) or error code if a fatal error condition has occurred
  */
 enum ZT1_ResultCode ZT1_Node_new(
@@ -760,11 +806,7 @@ enum ZT1_ResultCode ZT1_Node_new(
 	ZT1_VirtualNetworkFrameFunction virtualNetworkFrameFunction,
 	ZT1_VirtualNetworkConfigFunction virtualNetworkConfigFunction,
 	ZT1_EventCallback eventCallback,
-	const char *overrideRootTopology
-#ifdef __cplusplus
-    = (const char *)0               
-#endif
-    );
+	const char *overrideRootTopology);
 
 /**
  * Delete a node and free all resources it consumes
@@ -781,6 +823,7 @@ void ZT1_Node_delete(ZT1_Node *node);
  *
  * @param node Node instance
  * @param now Current clock in milliseconds
+ * @param localInterfaceId Local interface ID on which packet was received (use 0 if only one interface or unsure)
  * @param remoteAddress Origin of packet
  * @param packetData Packet data
  * @param packetLength Packet length
@@ -790,6 +833,7 @@ void ZT1_Node_delete(ZT1_Node *node);
 enum ZT1_ResultCode ZT1_Node_processWirePacket(
 	ZT1_Node *node,
 	uint64_t now,
+	const int localInterfaceId,
 	const struct sockaddr_storage *remoteAddress,
 	const void *packetData,
 	unsigned int packetLength,
@@ -882,14 +926,10 @@ enum ZT1_ResultCode ZT1_Node_leave(ZT1_Node *node,uint64_t nwid);
  * @param node Node instance
  * @param nwid 64-bit network ID
  * @param multicastGroup Ethernet multicast or broadcast MAC (least significant 48 bits)
- * @param multicastAdi Multicast ADI (least significant 32 bits only, default: 0)
+ * @param multicastAdi Multicast ADI (least significant 32 bits only, use 0 if not needed)
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-enum ZT1_ResultCode ZT1_Node_multicastSubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi
-#ifdef __cplusplus
-    = 0
-#endif
-    );
+enum ZT1_ResultCode ZT1_Node_multicastSubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 
 /**
  * Unsubscribe from an Ethernet multicast group (or all groups)
@@ -902,14 +942,10 @@ enum ZT1_ResultCode ZT1_Node_multicastSubscribe(ZT1_Node *node,uint64_t nwid,uin
  * @param node Node instance
  * @param nwid 64-bit network ID
  * @param multicastGroup Ethernet multicast or broadcast MAC (least significant 48 bits)
- * @param multicastAdi Multicast ADI (least significant 32 bits only, default: 0)
+ * @param multicastAdi Multicast ADI (least significant 32 bits only, use 0 if not needed)
  * @return OK (0) or error code if a fatal error condition has occurred
  */
-enum ZT1_ResultCode ZT1_Node_multicastUnsubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi
-#ifdef __cplusplus
-    = 0
-#endif
-    );
+enum ZT1_ResultCode ZT1_Node_multicastUnsubscribe(ZT1_Node *node,uint64_t nwid,uint64_t multicastGroup,unsigned long multicastAdi);
 
 /**
  * Get this node's 40-bit ZeroTier address

node/Hashtable.hpp

@@ -383,7 +383,7 @@ private:
 	static inline unsigned long _hc(const uint32_t i)
 	{
 		// In the uint32_t case we use a simple multiplier for hashing to ensure coverage
-		return ((unsigned long)i * (unsigned long)2654435761);
+		return ((unsigned long)i * (unsigned long)0x9e3779b1);
 	}
 
 	inline void _grow()

node/IncomingPacket.cpp

@@ -69,7 +69,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR)
 			switch(verb()) {
 				//case Packet::VERB_NOP:
 				default: // ignore unknown verbs, but if they pass auth check they are "received"
-					peer->received(RR,_remoteAddress,hops(),packetId(),verb(),0,Packet::VERB_NOP);
+					peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),verb(),0,Packet::VERB_NOP);
 					return true;
 				case Packet::VERB_HELLO:                          return _doHELLO(RR);
 				case Packet::VERB_ERROR:                          return _doERROR(RR,peer);
@@ -144,7 +144,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 						Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE);
 						nconf->com().serialize(outp);
 						outp.armor(peer->key(),true);
-						RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+						RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 					}
 				}
 			}	break;
@@ -165,7 +165,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 			default: break;
 		}
 
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb);
 	} catch (std::exception &ex) {
 		TRACE("dropped ERROR from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -231,7 +231,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 						outp.append(packetId());
 						outp.append((unsigned char)Packet::ERROR_IDENTITY_COLLISION);
 						outp.armor(key,true);
-						RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+						RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 					} else {
 						RR->node->postEvent(ZT1_EVENT_AUTHENTICATION_FAILURE,(const void *)&_remoteAddress);
 						TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
@@ -278,7 +278,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 
 		// VALID -- continues here
 
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_HELLO,0,Packet::VERB_NOP);
 		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision);
 
 		bool trusted = false;
@@ -316,7 +316,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR)
 		}
 
 		outp.armor(peer->key(),true);
-		RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+		RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 	} catch (std::exception &ex) {
 		TRACE("dropped HELLO from %s(%s): %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -436,7 +436,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 			default: break;
 		}
 
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb);
 	} catch (std::exception &ex) {
 		TRACE("dropped OK from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -456,7 +456,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append(packetId());
 				queried->identity().serialize(outp,false);
 				outp.armor(peer->key(),true);
-				RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+				RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 			} else {
 				Packet outp(peer->address(),RR->identity.address(),Packet::VERB_ERROR);
 				outp.append((unsigned char)Packet::VERB_WHOIS);
@@ -464,12 +464,12 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 				outp.append(payload(),ZT_ADDRESS_LENGTH);
 				outp.armor(peer->key(),true);
-				RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+				RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 			}
 		} else {
 			TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str());
 		}
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -487,8 +487,8 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<
 			if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
 				InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
 				TRACE("RENDEZVOUS from %s says %s might be at %s, starting NAT-t",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
-				peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP);
-				RR->sw->rendezvous(withPeer,atAddr);
+				peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP);
+				RR->sw->rendezvous(withPeer,_localInterfaceId,atAddr);
 			} else {
 				TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
 			}
@@ -525,7 +525,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				RR->node->putFrame(network->id(),MAC(peer->address(),network->id()),network->mac(),etherType,0,field(ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,payloadLen),payloadLen);
 			}
 
-			peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP);
+			peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP);
 		} else {
 			TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
@@ -602,7 +602,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				RR->node->putFrame(network->id(),from,to,etherType,0,field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,payloadLen),payloadLen);
 			}
 
-			peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP);
+			peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP);
 		} else {
 			TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
@@ -623,7 +623,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 		for(unsigned int ptr=ZT_PACKET_IDX_PAYLOAD;ptr<size();ptr+=18)
 			RR->mc->add(now,at<uint64_t>(ptr),MulticastGroup(MAC(field(ptr + 8,6),6),at<uint32_t>(ptr + 14)),peer->address());
 
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP);
 	} catch (std::exception &ex) {
 		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -647,7 +647,7 @@ bool IncomingPacket::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment
 			}
 		}
 
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
 	} catch (std::exception &ex) {
 		TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),ex.what());
 	} catch ( ... ) {
@@ -666,7 +666,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 
 		const unsigned int h = hops();
 		const uint64_t pid = packetId();
-		peer->received(RR,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP);
 
 		if (RR->localNetworkController) {
 			Dictionary netconf;
@@ -688,7 +688,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 						if (outp.size() > ZT_PROTO_MAX_PACKET_LENGTH) {
 							TRACE("NETWORK_CONFIG_REQUEST failed: internal error: netconf size %u is too large",(unsigned int)netconfStr.length());
 						} else {
-							RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+							RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 						}
 					}
 				}	break;
@@ -700,7 +700,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_OBJ_NOT_FOUND);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
-					RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+					RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 				}	break;
 
 				case NetworkController::NETCONF_QUERY_ACCESS_DENIED: {
@@ -710,7 +710,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 					outp.append((unsigned char)Packet::ERROR_NETWORK_ACCESS_DENIED_);
 					outp.append(nwid);
 					outp.armor(peer->key(),true);
-					RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+					RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 				} break;
 
 				case NetworkController::NETCONF_QUERY_INTERNAL_SERVER_ERROR:
@@ -732,7 +732,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 			outp.append((unsigned char)Packet::ERROR_UNSUPPORTED_OPERATION);
 			outp.append(nwid);
 			outp.armor(peer->key(),true);
-			RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+			RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 		}
 	} catch (std::exception &exc) {
 		TRACE("dropped NETWORK_CONFIG_REQUEST from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
@@ -753,7 +753,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 				nw->requestConfiguration();
 			ptr += 8;
 		}
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP);
 	} catch (std::exception &exc) {
 		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -780,11 +780,11 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 			outp.append((uint32_t)mg.adi());
 			if (RR->mc->gather(peer->address(),nwid,mg,outp,gatherLimit)) {
 				outp.armor(peer->key(),true);
-				RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+				RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 			}
 		}
 
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP);
 	} catch (std::exception &exc) {
 		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -871,12 +871,12 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 				outp.append((unsigned char)0x02); // flag 0x02 = contains gather results
 				if (RR->mc->gather(peer->address(),nwid,to,outp,gatherLimit)) {
 					outp.armor(peer->key(),true);
-					RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+					RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 				}
 			}
 		} // else ignore -- not a member of this network
 
-		peer->received(RR,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP);
+		peer->received(RR,_localInterfaceId,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP);
 	} catch (std::exception &exc) {
 		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception: %s",source().toString().c_str(),_remoteAddress.toString().c_str(),exc.what());
 	} catch ( ... ) {
@@ -905,14 +905,14 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 					InetAddress a(field(ptr,4),4,at<uint16_t>(ptr + 4));
 					if ( ((flags & (0x01 | 0x02)) == 0) && (Path::isAddressValidForPath(a)) ) {
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						peer->attemptToContactAt(RR,a,RR->node->now());
+						peer->attemptToContactAt(RR,_localInterfaceId,a,RR->node->now());
 					}
 				}	break;
 				case 6: {
 					InetAddress a(field(ptr,16),16,at<uint16_t>(ptr + 16));
 					if ( ((flags & (0x01 | 0x02)) == 0) && (Path::isAddressValidForPath(a)) ) {
 						TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-						peer->attemptToContactAt(RR,a,RR->node->now());
+						peer->attemptToContactAt(RR,_localInterfaceId,a,RR->node->now());
 					}
 				}	break;
 			}
@@ -934,7 +934,7 @@ void IncomingPacket::_sendErrorNeedCertificate(const RuntimeEnvironment *RR,cons
 	outp.append((unsigned char)Packet::ERROR_NEED_MEMBERSHIP_CERTIFICATE);
 	outp.append(nwid);
 	outp.armor(peer->key(),true);
-	RR->node->putPacket(_remoteAddress,outp.data(),outp.size());
+	RR->node->putPacket(_localInterfaceId,_remoteAddress,outp.data(),outp.size());
 }
 
 } // namespace ZeroTier

node/IncomingPacket.hpp

@@ -72,14 +72,16 @@ public:
 	 *
 	 * @param data Packet data
 	 * @param len Packet length
+	 * @param localInterfaceId Local interface ID
 	 * @param remoteAddress Address from which packet came
 	 * @param now Current time
 	 * @throws std::out_of_range Range error processing packet
 	 */
-	IncomingPacket(const void *data,unsigned int len,const InetAddress &remoteAddress,uint64_t now) :
+	IncomingPacket(const void *data,unsigned int len,int localInterfaceId,const InetAddress &remoteAddress,uint64_t now) :
  		Packet(data,len),
  		_receiveTime(now),
  		_remoteAddress(remoteAddress),
+ 		_localInterfaceId(localInterfaceId),
  		__refCount()
 	{
 	}
@@ -128,6 +130,7 @@ private:
 
 	uint64_t _receiveTime;
 	InetAddress _remoteAddress;
+	int _localInterfaceId;
 	AtomicCounter __refCount;
 };
 

node/Node.cpp

@@ -157,13 +157,14 @@ Node::~Node()
 
 ZT1_ResultCode Node::processWirePacket(
 	uint64_t now,
+	int localInterfaceId,
 	const struct sockaddr_storage *remoteAddress,
 	const void *packetData,
 	unsigned int packetLength,
 	volatile uint64_t *nextBackgroundTaskDeadline)
 {
 	_now = now;
-	RR->sw->onRemotePacket(*(reinterpret_cast<const InetAddress *>(remoteAddress)),packetData,packetLength);
+	RR->sw->onRemotePacket(localInterfaceId,*(reinterpret_cast<const InetAddress *>(remoteAddress)),packetData,packetLength);
 	return ZT1_RESULT_OK;
 }
 
@@ -232,7 +233,9 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 	_now = now;
 	Mutex::Lock bl(_backgroundTasksLock);
 
-	if ((now - _lastPingCheck) >= ZT_PING_CHECK_INVERVAL) {
+	unsigned long timeUntilNextPingCheck = ZT_PING_CHECK_INVERVAL;
+	const uint64_t timeSinceLastPingCheck = now - _lastPingCheck;
+	if (timeSinceLastPingCheck >= ZT_PING_CHECK_INVERVAL) {
 		try {
 			_lastPingCheck = now;
 
@@ -261,7 +264,7 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 				if (nr->second) {
 					SharedPtr<Peer> rp(RR->topology->getPeer(nr->first));
 					if ((rp)&&(!rp->hasActiveDirectPath(now)))
-						rp->attemptToContactAt(RR,nr->second,now);
+						rp->attemptToContactAt(RR,-1,nr->second,now);
 				}
 			}
 
@@ -277,6 +280,8 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 		} catch ( ... ) {
 			return ZT1_RESULT_FATAL_ERROR_INTERNAL;
 		}
+	} else {
+		timeUntilNextPingCheck -= (unsigned long)timeSinceLastPingCheck;
 	}
 
 	if ((now - _lastHousekeepingRun) >= ZT_HOUSEKEEPING_PERIOD) {
@@ -291,7 +296,7 @@ ZT1_ResultCode Node::processBackgroundTasks(uint64_t now,volatile uint64_t *next
 	}
 
 	try {
-		*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min((unsigned long)ZT_PING_CHECK_INVERVAL,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
+		*nextBackgroundTaskDeadline = now + (uint64_t)std::max(std::min(timeUntilNextPingCheck,RR->sw->doTimerTasks(now)),(unsigned long)ZT_CORE_TIMER_TASK_GRANULARITY);
 	} catch ( ... ) {
 		return ZT1_RESULT_FATAL_ERROR_INTERNAL;
 	}
@@ -569,13 +574,14 @@ void ZT1_Node_delete(ZT1_Node *node)
 enum ZT1_ResultCode ZT1_Node_processWirePacket(
 	ZT1_Node *node,
 	uint64_t now,
+	int localInterfaceId,
 	const struct sockaddr_storage *remoteAddress,
 	const void *packetData,
 	unsigned int packetLength,
 	volatile uint64_t *nextBackgroundTaskDeadline)
 {
 	try {
-		return reinterpret_cast<ZeroTier::Node *>(node)->processWirePacket(now,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline);
+		return reinterpret_cast<ZeroTier::Node *>(node)->processWirePacket(now,localInterfaceId,remoteAddress,packetData,packetLength,nextBackgroundTaskDeadline);
 	} catch (std::bad_alloc &exc) {
 		return ZT1_RESULT_FATAL_ERROR_OUT_OF_MEMORY;
 	} catch ( ... ) {

node/Node.hpp

@@ -80,6 +80,7 @@ public:
 
 	ZT1_ResultCode processWirePacket(
 		uint64_t now,
+		int localInterfaceId,
 		const struct sockaddr_storage *remoteAddress,
 		const void *packetData,
 		unsigned int packetLength,
@@ -119,16 +120,18 @@ public:
 	/**
 	 * Enqueue a ZeroTier message to be sent
 	 *
+	 * @param localInterfaceId Local interface ID, -1 for unspecified/random
 	 * @param addr Destination address
 	 * @param data Packet data
 	 * @param len Packet length
 	 * @return True if packet appears to have been sent
 	 */
-	inline bool putPacket(const InetAddress &addr,const void *data,unsigned int len)
+	inline bool putPacket(int localInterfaceId,const InetAddress &addr,const void *data,unsigned int len)
 	{
 		return (_wirePacketSendFunction(
 			reinterpret_cast<ZT1_Node *>(this),
 			_uPtr,
+			localInterfaceId,
 			reinterpret_cast<const struct sockaddr_storage *>(&addr),
 			data,
 			len) == 0);

node/Path.hpp

@@ -93,7 +93,16 @@ public:
 	/**
 	 * @return Preference rank, higher == better
 	 */
-	inline int preferenceRank() const throw() { return (int)_ipScope; } // IP scopes are in ascending rank order in InetAddress.hpp
+	inline int preferenceRank() const throw()
+	{
+		// First, since the scope enum values in InetAddress.hpp are in order of
+		// use preference rank, we take that. Then we multiple by two, yielding
+		// a sequence like 0, 2, 4, 6, etc. Then if it's IPv6 we add one. This
+		// makes IPv6 addresses of a given scope outrank IPv4 addresses of the
+		// same scope -- e.g. 1 outranks 0. This makes us prefer IPv6, but not
+		// if the address scope/class is of a fundamentally lower rank.
+		return ( ((int)_ipScope * 2) + ((_addr.ss_family == AF_INET6) ? 1 : 0) );
+	}
 
 	/**
 	 * @return Path trust level

node/Peer.cpp

@@ -39,6 +39,9 @@
 
 namespace ZeroTier {
 
+// Used to send varying values for NAT keepalive
+static uint32_t _natKeepaliveBuf = 0;
+
 Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 	throw(std::runtime_error) :
 	_lastUsed(0),
@@ -61,6 +64,7 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 
 void Peer::received(
 	const RuntimeEnvironment *RR,
+	int localInterfaceId,
 	const InetAddress &remoteAddr,
 	unsigned int hops,
 	uint64_t packetId,
@@ -78,7 +82,7 @@ void Peer::received(
 		{
 			unsigned int np = _numPaths;
 			for(unsigned int p=0;p<np;++p) {
-				if (_paths[p].address() == remoteAddr) {
+				if ((_paths[p].address() == remoteAddr)&&(_paths[p].localInterfaceId() == localInterfaceId)) {
 					_paths[p].received(now);
 					pathIsConfirmed = true;
 					break;
@@ -103,7 +107,7 @@ void Peer::received(
 						}
 					}
 					if (slot) {
-						*slot = RemotePath(remoteAddr,false);
+						*slot = RemotePath(localInterfaceId,remoteAddr,false);
 						slot->received(now);
 						_numPaths = np;
 						pathIsConfirmed = true;
@@ -116,7 +120,7 @@ void Peer::received(
 					if ((now - _lastPathConfirmationSent) >= ZT_MIN_PATH_CONFIRMATION_INTERVAL) {
 						_lastPathConfirmationSent = now;
 						TRACE("got %s via unknown path %s(%s), confirming...",Packet::verbString(verb),_id.address().toString().c_str(),remoteAddr.toString().c_str());
-						attemptToContactAt(RR,remoteAddr,now);
+						attemptToContactAt(RR,localInterfaceId,remoteAddr,now);
 					}
 				}
 			}
@@ -138,7 +142,7 @@ void Peer::received(
 					for(std::vector<MulticastGroup>::const_iterator mg(mgs.begin());mg!=mgs.end();++mg) {
 						if ((outp.size() + 18) > ZT_UDP_DEFAULT_PAYLOAD_MTU) {
 							outp.armor(_key,true);
-							RR->node->putPacket(remoteAddr,outp.data(),outp.size());
+							RR->node->putPacket(localInterfaceId,remoteAddr,outp.data(),outp.size());
 							outp.reset(_id.address(),RR->identity.address(),Packet::VERB_MULTICAST_LIKE);
 						}
 
@@ -151,7 +155,7 @@ void Peer::received(
 			}
 			if (outp.size() > ZT_PROTO_MIN_PACKET_LENGTH) {
 				outp.armor(_key,true);
-				RR->node->putPacket(remoteAddr,outp.data(),outp.size());
+				RR->node->putPacket(localInterfaceId,remoteAddr,outp.data(),outp.size());
 			}
 		}
 	}
@@ -177,7 +181,7 @@ RemotePath *Peer::getBestPath(uint64_t now)
 	return bestPath;
 }
 
-void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &atAddress,uint64_t now)
+void Peer::attemptToContactAt(const RuntimeEnvironment *RR,int localInterfaceId,const InetAddress &atAddress,uint64_t now)
 {
 	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
 	outp.append((unsigned char)ZT_PROTO_VERSION);
@@ -205,7 +209,7 @@ void Peer::attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &at
 	}
 
 	outp.armor(_key,false); // HELLO is sent in the clear
-	RR->node->putPacket(atAddress,outp.data(),outp.size());
+	RR->node->putPacket(localInterfaceId,atAddress,outp.data(),outp.size());
 }
 
 void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
@@ -214,11 +218,12 @@ void Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now)
 	if (bestPath) {
 		if ((now - bestPath->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
 			TRACE("PING %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
-			attemptToContactAt(RR,bestPath->address(),now);
+			attemptToContactAt(RR,bestPath->localInterfaceId(),bestPath->address(),now);
 			bestPath->sent(now);
 		} else if (((now - bestPath->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!bestPath->reliable())) {
+			_natKeepaliveBuf += (uint32_t)((now * 0x9e3779b1) >> 1); // tumble this around to send constantly varying (meaningless) payloads
 			TRACE("NAT keepalive %s(%s)",_id.address().toString().c_str(),bestPath->address().toString().c_str());
-			RR->node->putPacket(bestPath->address(),"",0);
+			RR->node->putPacket(bestPath->localInterfaceId(),bestPath->address(),&_natKeepaliveBuf,sizeof(_natKeepaliveBuf));
 			bestPath->sent(now);
 		}
 	}
@@ -350,7 +355,7 @@ bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope sc
 	while (x < np) {
 		if (_paths[x].address().ipScope() == scope) {
 			if (_paths[x].fixed()) {
-				attemptToContactAt(RR,_paths[x].address(),now);
+				attemptToContactAt(RR,_paths[x].localInterfaceId(),_paths[x].address(),now);
 				_paths[y++] = _paths[x]; // keep fixed paths
 			}
 		} else {

node/Peer.hpp

@@ -105,6 +105,7 @@ public:
 	 * and appears to be valid.
 	 *
 	 * @param RR Runtime environment
+	 * @param localInterfaceId Local interface ID or -1 if unspecified
 	 * @param remoteAddr Internet address of sender
 	 * @param hops ZeroTier (not IP) hops
 	 * @param packetId Packet ID
@@ -114,6 +115,7 @@ public:
 	 */
 	void received(
 		const RuntimeEnvironment *RR,
+		int localInterfaceId,
 		const InetAddress &remoteAddr,
 		unsigned int hops,
 		uint64_t packetId,
@@ -155,10 +157,11 @@ public:
 	 * for NAT traversal and path verification.
 	 *
 	 * @param RR Runtime environment
+	 * @param localInterfaceId Local interface ID or -1 for unspecified
 	 * @param atAddress Destination address
 	 * @param now Current time
 	 */
-	void attemptToContactAt(const RuntimeEnvironment *RR,const InetAddress &atAddress,uint64_t now);
+	void attemptToContactAt(const RuntimeEnvironment *RR,int localInterfaceId,const InetAddress &atAddress,uint64_t now);
 
 	/**
 	 * Send pings or keepalives depending on configured timeouts

node/RemotePath.hpp

@@ -53,14 +53,18 @@ public:
 		Path(),
 		_lastSend(0),
 		_lastReceived(0),
+		_localInterfaceId(-1),
 		_fixed(false) {}
 
-	RemotePath(const InetAddress &addr,bool fixed) :
+	RemotePath(int localInterfaceId,const InetAddress &addr,bool fixed) :
 		Path(addr,0,TRUST_NORMAL),
 		_lastSend(0),
 		_lastReceived(0),
+		_localInterfaceId(localInterfaceId),
 		_fixed(fixed) {}
 
+	inline int localInterfaceId() const throw() { return _localInterfaceId; }
+
 	inline uint64_t lastSend() const throw() { return _lastSend; }
 	inline uint64_t lastReceived() const throw() { return _lastReceived; }
 
@@ -123,7 +127,7 @@ public:
 	 */
 	inline bool send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
 	{
-		if (RR->node->putPacket(address(),data,len)) {
+		if (RR->node->putPacket(_localInterfaceId,address(),data,len)) {
 			sent(now);
 			RR->antiRec->logOutgoingZT(data,len);
 			return true;
@@ -134,6 +138,7 @@ public:
 private:
 	uint64_t _lastSend;
 	uint64_t _lastReceived;
+	int _localInterfaceId;
 	bool _fixed;
 };
 

node/SHA512.cpp

@@ -48,10 +48,8 @@ Public domain.
 
 #define uint64 uint64_t
 
-#define load_bigendian(x) Utils::ntoh(*((const uint64_t *)(x)))
-#define store_bigendian(x,u) (*((uint64_t *)(x)) = Utils::hton((u)))
+#ifdef ZT_NO_TYPE_PUNNING
 
-#if 0
 static uint64 load_bigendian(const unsigned char *x)
 {
   return
@@ -77,7 +75,13 @@ static void store_bigendian(unsigned char *x,uint64 u)
   x[1] = u; u >>= 8;
   x[0] = u;
 }
-#endif
+
+#else // !ZT_NO_TYPE_PUNNING
+
+#define load_bigendian(x) Utils::ntoh(*((const uint64_t *)(x)))
+#define store_bigendian(x,u) (*((uint64_t *)(x)) = Utils::hton((u)))
+
+#endif // ZT_NO_TYPE_PUNNING
 
 #define SHR(x,c) ((x) >> (c))
 #define ROTR(x,c) (((x) >> (c)) | ((x) << (64 - (c))))

node/Switch.cpp

@@ -78,8 +78,11 @@ Switch::~Switch()
 {
 }
 
-void Switch::onRemotePacket(const InetAddress &fromAddr,const void *data,unsigned int len)
+void Switch::onRemotePacket(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
+	if (localInterfaceId < 0)
+		localInterfaceId = 0;
+
 	try {
 		if (len == 13) {
 			/* LEGACY: before VERB_PUSH_DIRECT_PATHS, peers used broadcast
@@ -96,14 +99,14 @@ void Switch::onRemotePacket(const InetAddress &fromAddr,const void *data,unsigne
 					_lastBeaconResponse = now;
 					Packet outp(peer->address(),RR->identity.address(),Packet::VERB_NOP);
 					outp.armor(peer->key(),false);
-					RR->node->putPacket(fromAddr,outp.data(),outp.size());
+					RR->node->putPacket(localInterfaceId,fromAddr,outp.data(),outp.size());
 				}
 			}
 		} else if (len > ZT_PROTO_MIN_FRAGMENT_LENGTH) {
 			if (((const unsigned char *)data)[ZT_PACKET_FRAGMENT_IDX_FRAGMENT_INDICATOR] == ZT_PACKET_FRAGMENT_INDICATOR) {
-				_handleRemotePacketFragment(fromAddr,data,len);
+				_handleRemotePacketFragment(localInterfaceId,fromAddr,data,len);
 			} else if (len >= ZT_PROTO_MIN_PACKET_LENGTH) {
-				_handleRemotePacketHead(fromAddr,data,len);
+				_handleRemotePacketHead(localInterfaceId,fromAddr,data,len);
 			}
 		}
 	} catch (std::exception &ex) {
@@ -376,14 +379,14 @@ bool Switch::unite(const Address &p1,const Address &p2,bool force)
 	return true;
 }
 
-void Switch::rendezvous(const SharedPtr<Peer> &peer,const InetAddress &atAddr)
+void Switch::rendezvous(const SharedPtr<Peer> &peer,int localInterfaceId,const InetAddress &atAddr)
 {
 	TRACE("sending NAT-t message to %s(%s)",peer->address().toString().c_str(),atAddr.toString().c_str());
 	const uint64_t now = RR->node->now();
-	peer->attemptToContactAt(RR,atAddr,now);
+	peer->attemptToContactAt(RR,localInterfaceId,atAddr,now);
 	{
 		Mutex::Lock _l(_contactQueue_m);
-		_contactQueue.push_back(ContactQueueEntry(peer,now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY,atAddr));
+		_contactQueue.push_back(ContactQueueEntry(peer,now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY,localInterfaceId,atAddr));
 	}
 }
 
@@ -453,14 +456,14 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 				} else {
 					if (qi->strategyIteration == 0) {
 						// First strategy: send packet directly to destination
-						qi->peer->attemptToContactAt(RR,qi->inaddr,now);
+						qi->peer->attemptToContactAt(RR,qi->localInterfaceId,qi->inaddr,now);
 					} else if (qi->strategyIteration <= 4) {
 						// Strategies 1-4: try escalating ports for symmetric NATs that remap sequentially
 						InetAddress tmpaddr(qi->inaddr);
 						int p = (int)qi->inaddr.port() + qi->strategyIteration;
 						if (p < 0xffff) {
 							tmpaddr.setPort((unsigned int)p);
-							qi->peer->attemptToContactAt(RR,tmpaddr,now);
+							qi->peer->attemptToContactAt(RR,qi->localInterfaceId,tmpaddr,now);
 						} else qi->strategyIteration = 5;
 					} else {
 						// All strategies tried, expire entry
@@ -551,7 +554,7 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 	return nextDelay;
 }
 
-void Switch::_handleRemotePacketFragment(const InetAddress &fromAddr,const void *data,unsigned int len)
+void Switch::_handleRemotePacketFragment(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
 	Packet::Fragment fragment(data,len);
 	Address destination(fragment.destination());
@@ -622,9 +625,9 @@ void Switch::_handleRemotePacketFragment(const InetAddress &fromAddr,const void
 	}
 }
 
-void Switch::_handleRemotePacketHead(const InetAddress &fromAddr,const void *data,unsigned int len)
+void Switch::_handleRemotePacketHead(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len)
 {
-	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,fromAddr,RR->node->now()));
+	SharedPtr<IncomingPacket> packet(new IncomingPacket(data,len,localInterfaceId,fromAddr,RR->node->now()));
 
 	Address source(packet->source());
 	Address destination(packet->destination());

node/Switch.hpp

@@ -69,11 +69,12 @@ public:
 	/**
 	 * Called when a packet is received from the real network
 	 *
+	 * @param localInterfaceId Local interface ID or -1 for unspecified
 	 * @param fromAddr Internet IP address of origin
 	 * @param data Packet data
 	 * @param len Packet length
 	 */
-	void onRemotePacket(const InetAddress &fromAddr,const void *data,unsigned int len);
+	void onRemotePacket(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len);
 
 	/**
 	 * Called when a packet comes from a local Ethernet tap
@@ -130,9 +131,10 @@ public:
 	 * Attempt NAT traversal to peer at a given physical address
 	 *
 	 * @param peer Peer to contact
+	 * @param localInterfaceId Local interface ID or -1 if unspecified
 	 * @param atAddr Address of peer
 	 */
-	void rendezvous(const SharedPtr<Peer> &peer,const InetAddress &atAddr);
+	void rendezvous(const SharedPtr<Peer> &peer,int localInterfaceId,const InetAddress &atAddr);
 
 	/**
 	 * Request WHOIS on a given address
@@ -169,8 +171,8 @@ public:
 	unsigned long doTimerTasks(uint64_t now);
 
 private:
-	void _handleRemotePacketFragment(const InetAddress &fromAddr,const void *data,unsigned int len);
-	void _handleRemotePacketHead(const InetAddress &fromAddr,const void *data,unsigned int len);
+	void _handleRemotePacketFragment(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len);
+	void _handleRemotePacketHead(int localInterfaceId,const InetAddress &fromAddr,const void *data,unsigned int len);
 	Address _sendWhoisRequest(const Address &addr,const Address *peersAlreadyConsulted,unsigned int numPeersAlreadyConsulted);
 	bool _trySend(const Packet &packet,bool encrypt,uint64_t nwid);
 
@@ -250,15 +252,17 @@ private:
 	struct ContactQueueEntry
 	{
 		ContactQueueEntry() {}
-		ContactQueueEntry(const SharedPtr<Peer> &p,uint64_t ft,const InetAddress &a) :
+		ContactQueueEntry(const SharedPtr<Peer> &p,uint64_t ft,int liid,const InetAddress &a) :
 			peer(p),
 			fireAtTime(ft),
 			inaddr(a),
+			localInterfaceId(liid),
 			strategyIteration(0) {}
 
 		SharedPtr<Peer> peer;
 		uint64_t fireAtTime;
 		InetAddress inaddr;
+		int localInterfaceId;
 		unsigned int strategyIteration;
 	};
 	std::list<ContactQueueEntry> _contactQueue;

node/Topology.cpp

@@ -62,7 +62,7 @@ void Topology::setRootServers(const std::map< Identity,std::vector<InetAddress>
 			if (!p)
 				p = SharedPtr<Peer>(new Peer(RR->identity,i->first));
 			for(std::vector<InetAddress>::const_iterator j(i->second.begin());j!=i->second.end();++j)
-				p->addPath(RemotePath(*j,true));
+				p->addPath(RemotePath(0,*j,true));
 			p->use(now);
 			_rootPeers.push_back(p);
 		}

one.cpp

@@ -910,7 +910,7 @@ static void printHelp(const char *cn,FILE *out)
 	fprintf(out,"  -h                - Display this help"ZT_EOL_S);
 	fprintf(out,"  -v                - Show version"ZT_EOL_S);
 	fprintf(out,"  -U                - Run as unprivileged user (skip privilege check)"ZT_EOL_S);
-	fprintf(out,"  -p<port>          - Port for UDP and TCP/HTTP (default: 9993)"ZT_EOL_S);
+	fprintf(out,"  -p<port>          - Port for UDP and TCP/HTTP (default: 9993, 0 for random)"ZT_EOL_S);
 	//fprintf(out,"  -T<path>          - Override root topology, do not authenticate or update"ZT_EOL_S);
 
 #ifdef __UNIX_LIKE__
@@ -985,7 +985,7 @@ int main(int argc,char **argv)
 
 				case 'p': // port -- for both UDP and TCP, packets and control plane
 					port = Utils::strToUInt(argv[i] + 2);
-					if ((port > 0xffff)||(port == 0)) {
+					if (port > 0xffff) {
 						printHelp(argv[0],stdout);
 						return 1;
 					}

service/OneService.cpp

@@ -359,7 +359,7 @@ static int SnodeVirtualNetworkConfigFunction(ZT1_Node *node,void *uptr,uint64_t
 static void SnodeEventCallback(ZT1_Node *node,void *uptr,enum ZT1_Event event,const void *metaData);
 static long SnodeDataStoreGetFunction(ZT1_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize);
 static int SnodeDataStorePutFunction(ZT1_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure);
-static int SnodeWirePacketSendFunction(ZT1_Node *node,void *uptr,const struct sockaddr_storage *addr,const void *data,unsigned int len);
+static int SnodeWirePacketSendFunction(ZT1_Node *node,void *uptr,int localInterfaceId,const struct sockaddr_storage *addr,const void *data,unsigned int len);
 static void SnodeVirtualNetworkFrameFunction(ZT1_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
 static void StapFrameHandler(void *uptr,uint64_t nwid,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
@@ -411,6 +411,10 @@ struct TcpConnection
 	Mutex writeBuf_m;
 };
 
+// Interface IDs -- the uptr for UDP sockets is set to point to one of these
+static const int ZT1_INTERFACE_ID_DEFAULT = 0; // default, usually port 9993
+static const int ZT1_INTERFACE_ID_UPNP = 1; // a randomly chosen UDP socket used with uPnP mappings, if enabled
+
 class OneServiceImpl : public OneService
 {
 public:
@@ -430,38 +434,82 @@ public:
 		_nextBackgroundTaskDeadline(0),
 		_tcpFallbackTunnel((TcpConnection *)0),
 		_termReason(ONE_STILL_RUNNING),
-		_port(port),
+		_port(0),
 #ifdef ZT_USE_MINIUPNPC
-		_upnpClient((int)port),
+		_v4UpnpUdpSocket((PhySocket *)0),
+		_upnpClient((UPNPClient *)0),
 #endif
 		_run(true)
 	{
 		struct sockaddr_in in4;
 		struct sockaddr_in6 in6;
 
-		::memset((void *)&in4,0,sizeof(in4));
-		in4.sin_family = AF_INET;
-		in4.sin_port = Utils::hton((uint16_t)port);
-		_v4UdpSocket = _phy.udpBind((const struct sockaddr *)&in4,this,131072);
-		if (!_v4UdpSocket)
-			throw std::runtime_error("cannot bind to port (UDP/IPv4)");
-		in4.sin_addr.s_addr = Utils::hton((uint32_t)0x7f000001); // right now we just listen for TCP @localhost
-		_v4TcpListenSocket = _phy.tcpListen((const struct sockaddr *)&in4,this);
-		if (!_v4TcpListenSocket) {
-			_phy.close(_v4UdpSocket);
-			throw std::runtime_error("cannot bind to port (TCP/IPv4)");
+		const int portTrials = (port == 0) ? 256 : 1; // if port is 0, pick random
+		for(int k=0;k<portTrials;++k) {
+			if (port == 0) {
+				unsigned int randp = 0;
+				Utils::getSecureRandom(&randp,sizeof(randp));
+				port = 40000 + (randp % 25500);
+			}
+
+			memset((void *)&in4,0,sizeof(in4));
+			in4.sin_family = AF_INET;
+			in4.sin_port = Utils::hton((uint16_t)port);
+
+			_v4UdpSocket = _phy.udpBind((const struct sockaddr *)&in4,reinterpret_cast<void *>(const_cast<int *>(&ZT1_INTERFACE_ID_DEFAULT)),131072);
+
+			if (_v4UdpSocket) {
+				in4.sin_addr.s_addr = Utils::hton((uint32_t)0x7f000001); // right now we just listen for TCP @localhost
+				_v4TcpListenSocket = _phy.tcpListen((const struct sockaddr *)&in4,this);
+
+				if (_v4TcpListenSocket) {
+					memset((void *)&in6,0,sizeof(in6));
+					in6.sin6_family = AF_INET6;
+					in6.sin6_port = in4.sin_port;
+
+					_v6UdpSocket = _phy.udpBind((const struct sockaddr *)&in6,reinterpret_cast<void *>(const_cast<int *>(&ZT1_INTERFACE_ID_DEFAULT)),131072);
+
+					in6.sin6_addr.s6_addr[15] = 1; // listen for TCP only at localhost
+					_v6TcpListenSocket = _phy.tcpListen((const struct sockaddr *)&in6,this);
+
+					_port = port;
+					break; // success!
+				} else {
+					_phy.close(_v4UdpSocket,false);
+				}
+			}
+
+			port = 0;
 		}
 
-		::memset((void *)&in6,0,sizeof(in6));
-		in6.sin6_family = AF_INET6;
-		in6.sin6_port = in4.sin_port;
-		_v6UdpSocket = _phy.udpBind((const struct sockaddr *)&in6,this,131072);
-		in6.sin6_addr.s6_addr[15] = 1; // listen for TCP only at localhost
-		_v6TcpListenSocket = _phy.tcpListen((const struct sockaddr *)&in6,this);
+		if (_port == 0)
+			throw std::runtime_error("cannot bind to port");
 
 		char portstr[64];
-		Utils::snprintf(portstr,sizeof(portstr),"%u",port);
+		Utils::snprintf(portstr,sizeof(portstr),"%u",_port);
 		OSUtils::writeFile((_homePath + ZT_PATH_SEPARATOR_S + "zerotier-one.port").c_str(),std::string(portstr));
+
+#ifdef ZT_USE_MINIUPNPC
+		// Bind a random secondary port for use with uPnP, since some NAT routers
+		// (cough Ubiquity Edge cough) barf up a lung if you do both conventional
+		// NAT-t and uPnP from behind the same port. I think this is a bug, but
+		// everyone else's router bugs are our problem. :P
+		for(int k=0;k<256;++k) {
+			unsigned int randp = 0;
+			Utils::getSecureRandom(&randp,sizeof(randp));
+			unsigned int upnport = 40000 + (randp % 25500);
+
+			memset((void *)&in4,0,sizeof(in4));
+			in4.sin_family = AF_INET;
+			in4.sin_port = Utils::hton((uint16_t)upnport);
+
+			_v4UpnpUdpSocket = _phy.udpBind((const struct sockaddr *)&in4,reinterpret_cast<void *>(const_cast<int *>(&ZT1_INTERFACE_ID_UPNP)),131072);
+			if (_v4UpnpUdpSocket) {
+				_upnpClient = new UPNPClient(upnport);
+				break;
+			}
+		}
+#endif
 	}
 
 	virtual ~OneServiceImpl()
@@ -470,6 +518,10 @@ public:
 		_phy.close(_v6UdpSocket);
 		_phy.close(_v4TcpListenSocket);
 		_phy.close(_v6TcpListenSocket);
+#ifdef ZT_USE_MINIUPNPC
+		_phy.close(_v4UpnpUdpSocket);
+		delete _upnpClient;
+#endif
 	}
 
 	virtual ReasonForTermination run()
@@ -558,7 +610,7 @@ public:
 
 #ifdef ZT_AUTO_UPDATE
 				if ((now - lastSoftwareUpdateCheck) >= ZT_AUTO_UPDATE_CHECK_PERIOD) {
-					lastSoftwareUpdateCheck = OSUtils::now();
+					lastSoftwareUpdateCheck = now;
 					Thread::start(&backgroundSoftwareUpdateChecker);
 				}
 #endif // ZT_AUTO_UPDATE
@@ -598,7 +650,7 @@ public:
 					_node->clearLocalInterfaceAddresses();
 
 #ifdef ZT_USE_MINIUPNPC
-					std::vector<InetAddress> upnpAddresses(_upnpClient.get());
+					std::vector<InetAddress> upnpAddresses(_upnpClient->get());
 					for(std::vector<InetAddress>::const_iterator ext(upnpAddresses.begin());ext!=upnpAddresses.end();++ext)
 						_node->addLocalInterfaceAddress(reinterpret_cast<const struct sockaddr_storage *>(&(*ext)),0,ZT1_LOCAL_INTERFACE_ADDRESS_TRUST_NORMAL);
 #endif
@@ -742,6 +794,7 @@ public:
 			_lastDirectReceiveFromGlobal = OSUtils::now();
 		ZT1_ResultCode rc = _node->processWirePacket(
 			OSUtils::now(),
+			*(reinterpret_cast<const int *>(*uptr)), // for UDP sockets, we set uptr to point to their interface ID
 			(const struct sockaddr_storage *)from, // Phy<> uses sockaddr_storage, so it'll always be that big
 			data,
 			len,
@@ -890,6 +943,7 @@ public:
 							if (from) {
 								ZT1_ResultCode rc = _node->processWirePacket(
 									OSUtils::now(),
+									0,
 									reinterpret_cast<struct sockaddr_storage *>(&from),
 									data,
 									plen,
@@ -1098,8 +1152,22 @@ public:
 		}
 	}
 
-	inline int nodeWirePacketSendFunction(const struct sockaddr_storage *addr,const void *data,unsigned int len)
+	inline int nodeWirePacketSendFunction(int localInterfaceId,const struct sockaddr_storage *addr,const void *data,unsigned int len)
 	{
+#ifdef ZT_USE_MINIUPNPC
+		if (localInterfaceId == ZT1_INTERFACE_ID_UPNP) {
+#ifdef ZT_BREAK_UDP
+			if (!OSUtils::fileExists("/tmp/ZT_BREAK_UDP")) {
+#endif
+			if (addr->ss_family == AF_INET)
+				return ((_phy.udpSend(_v4UpnpUdpSocket,(const struct sockaddr *)addr,data,len) != 0) ? 0 : -1);
+			else return -1;
+#ifdef ZT_BREAK_UDP
+			}
+#endif
+		}
+#endif // ZT_USE_MINIUPNPC
+
 		int result = -1;
 		switch(addr->ss_family) {
 			case AF_INET:
@@ -1155,6 +1223,7 @@ public:
 #endif // ZT1_TCP_FALLBACK_RELAY
 
 				break;
+
 			case AF_INET6:
 #ifdef ZT_BREAK_UDP
 				if (!OSUtils::fileExists("/tmp/ZT_BREAK_UDP")) {
@@ -1165,6 +1234,7 @@ public:
 				}
 #endif
 				break;
+
 			default:
 				return -1;
 		}
@@ -1286,7 +1356,8 @@ private:
 	unsigned int _port;
 
 #ifdef ZT_USE_MINIUPNPC
-	UPNPClient _upnpClient;
+	PhySocket *_v4UpnpUdpSocket;
+	UPNPClient *_upnpClient;
 #endif
 
 	bool _run;
@@ -1301,8 +1372,8 @@ static long SnodeDataStoreGetFunction(ZT1_Node *node,void *uptr,const char *name
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeDataStoreGetFunction(name,buf,bufSize,readIndex,totalSize); }
 static int SnodeDataStorePutFunction(ZT1_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure)
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeDataStorePutFunction(name,data,len,secure); }
-static int SnodeWirePacketSendFunction(ZT1_Node *node,void *uptr,const struct sockaddr_storage *addr,const void *data,unsigned int len)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(addr,data,len); }
+static int SnodeWirePacketSendFunction(ZT1_Node *node,void *uptr,int localInterfaceId,const struct sockaddr_storage *addr,const void *data,unsigned int len)
+{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localInterfaceId,addr,data,len); }
 static void SnodeVirtualNetworkFrameFunction(ZT1_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,sourceMac,destMac,etherType,vlanId,data,len); }
 

service/OneService.hpp

@@ -89,8 +89,12 @@ public:
 	 * Once created, you must call the run() method to actually start
 	 * processing.
 	 *
+	 * The port is saved to a file in the home path called zerotier-one.port,
+	 * which is used by the CLI and can be used to see which port was chosen if
+	 * 0 (random port) is picked.
+	 *
 	 * @param hp Home path
-	 * @param port TCP and UDP port for packets and HTTP control
+	 * @param port TCP and UDP port for packets and HTTP control (if 0, pick random port)
 	 * @param overrideRootTopology String-serialized root topology (for testing, default: NULL)
 	 */
 	static OneService *newInstance(

version.h

@@ -41,6 +41,6 @@
 /**
  * Revision
  */
-#define ZEROTIER_ONE_VERSION_REVISION 5
+#define ZEROTIER_ONE_VERSION_REVISION 6
 
 #endif