Try bringing back TTL escalation -- may help with Docker (IP-MASQ) type NAT

include/ZeroTierOne.h

@@ -1080,6 +1080,7 @@ typedef int (*ZT_DataStorePutFunction)(
  *  (4) Remote address
  *  (5) Packet data
  *  (6) Packet length
+ *  (7) Desired IP TTL or 0 to use default
  *
  * If there is only one local interface it is safe to ignore the local
  * interface address. Otherwise if running with multiple interfaces, the
@@ -1087,17 +1088,22 @@ typedef int (*ZT_DataStorePutFunction)(
  * the ss_family field is zero (NULL address), a random or preferred
  * default interface should be used.
  *
+ * If TTL is nonzero, packets should have their IP TTL value set to this
+ * value if possible. If this is not possible it is acceptable to ignore
+ * this value and send anyway with normal or default TTL.
+ *
  * The function must return zero on success and may return any error code
  * on failure. Note that success does not (of course) guarantee packet
  * delivery. It only means that the packet appears to have been sent.
  */
 typedef int (*ZT_WirePacketSendFunction)(
-	ZT_Node *,                      /* Node */
+	ZT_Node *,                       /* Node */
 	void *,                          /* User ptr */
 	const struct sockaddr_storage *, /* Local address */
 	const struct sockaddr_storage *, /* Remote address */
 	const void *,                    /* Packet data */
-	unsigned int);                   /* Packet length */
+	unsigned int,                    /* Packet length */
+	unsigned int);                   /* TTL or 0 to use default */
 
 /****************************************************************************/
 /* C Node API                                                               */

node/Node.hpp

@@ -149,9 +149,10 @@ public:
 	 * @param addr Destination address
 	 * @param data Packet data
 	 * @param len Packet length
+	 * @param ttl Desired TTL (default: 0 for unchanged/default TTL)
 	 * @return True if packet appears to have been sent
 	 */
-	inline bool putPacket(const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len)
+	inline bool putPacket(const InetAddress &localAddress,const InetAddress &addr,const void *data,unsigned int len,unsigned int ttl = 0)
 	{
 		return (_wirePacketSendFunction(
 			reinterpret_cast<ZT_Node *>(this),
@@ -159,7 +160,8 @@ public:
 			reinterpret_cast<const struct sockaddr_storage *>(&localAddress),
 			reinterpret_cast<const struct sockaddr_storage *>(&addr),
 			data,
-			len) == 0);
+			len,
+			ttl) == 0);
 	}
 
 	/**

node/Peer.cpp

@@ -211,7 +211,7 @@ void Peer::received(
 	}
 }
 
-void Peer::sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now)
+void Peer::sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl)
 {
 	// _lock not required here since _id is immutable and nothing else is accessed
 
@@ -228,7 +228,7 @@ void Peer::sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,c
 
 	outp.armor(_key,false); // HELLO is sent in the clear
 	RR->antiRec->logOutgoingZT(outp.data(),outp.size());
-	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size());
+	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size(),ttl);
 }
 
 bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily)

node/Peer.hpp

@@ -170,8 +170,9 @@ public:
 	 * @param localAddr Local address
 	 * @param atAddress Destination address
 	 * @param now Current time
+	 * @param ttl Desired IP TTL (default: 0 to leave alone)
 	 */
-	void sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now);
+	void sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl = 0);
 
 	/**
 	 * Send pings or keepalives depending on configured timeouts

node/Switch.cpp

@@ -435,7 +435,7 @@ void Switch::rendezvous(const SharedPtr<Peer> &peer,const InetAddress &localAddr
 {
 	TRACE("sending NAT-t message to %s(%s)",peer->address().toString().c_str(),atAddr.toString().c_str());
 	const uint64_t now = RR->node->now();
-	peer->sendHELLO(RR,localAddr,atAddr,now);
+	peer->sendHELLO(RR,localAddr,atAddr,now,2); // first attempt: send low-TTL packet to 'open' local NAT
 	{
 		Mutex::Lock _l(_contactQueue_m);
 		_contactQueue.push_back(ContactQueueEntry(peer,now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY,localAddr,atAddr));
@@ -509,8 +509,8 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 					if (qi->strategyIteration == 0) {
 						// First strategy: send packet directly to destination
 						qi->peer->sendHELLO(RR,qi->localAddr,qi->inaddr,now);
-					} else if (qi->strategyIteration <= 4) {
-						// Strategies 1-4: try escalating ports for symmetric NATs that remap sequentially
+					} else if (qi->strategyIteration <= 3) {
+						// Strategies 1-3: try escalating ports for symmetric NATs that remap sequentially
 						InetAddress tmpaddr(qi->inaddr);
 						int p = (int)qi->inaddr.port() + qi->strategyIteration;
 						if (p < 0xffff) {

osdep/Phy.hpp

@@ -414,6 +414,24 @@ public:
 		return (PhySocket *)&sws;
 	}
 
+	/**
+	 * Set the IP TTL for the next outgoing packet (for IPv4 UDP sockets only)
+	 *
+	 * @param ttl New TTL (0 or >255 will set it to 255)
+	 * @return True on success
+	 */
+	inline bool setIp4UdpTtl(PhySocket *sock,unsigned int ttl)
+	{
+		PhySocketImpl &sws = *(reinterpret_cast<PhySocketImpl *>(sock));
+#if defined(_WIN32) || defined(_WIN64)
+		DWORD tmp = ((ttl == 0)||(ttl > 255)) ? 255 : (DWORD)ttl;
+		return (::setsockopt(sws.sock,IPPROTO_IP,IP_TTL,(const char *)&tmp,sizeof(tmp)) == 0);
+#else
+		int tmp = ((ttl == 0)||(ttl > 255)) ? 255 : (int)ttl;
+		return (::setsockopt(sws.sock,IPPROTO_IP,IP_TTL,(void *)&tmp,sizeof(tmp)) == 0);
+#endif
+	}
+
 	/**
 	 * Send a UDP packet
 	 *

service/OneService.cpp

@@ -365,7 +365,7 @@ static int SnodeVirtualNetworkConfigFunction(ZT_Node *node,void *uptr,uint64_t n
 static void SnodeEventCallback(ZT_Node *node,void *uptr,enum ZT_Event event,const void *metaData);
 static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,void *buf,unsigned long bufSize,unsigned long readIndex,unsigned long *totalSize);
 static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure);
-static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len);
+static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl);
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
 
 #ifdef ZT_ENABLE_CLUSTER
@@ -1253,16 +1253,23 @@ public:
 		}
 	}
 
-	inline int nodeWirePacketSendFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len)
+	inline int nodeWirePacketSendFunction(const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl)
 	{
 #ifdef ZT_USE_MINIUPNPC
 		if ((localAddr->ss_family == AF_INET)&&(reinterpret_cast<const struct sockaddr_in *>(localAddr)->sin_port == reinterpret_cast<const struct sockaddr_in *>(&_v4UpnpLocalAddress)->sin_port)) {
 #ifdef ZT_BREAK_UDP
 			if (!OSUtils::fileExists("/tmp/ZT_BREAK_UDP")) {
 #endif
-			if (addr->ss_family == AF_INET)
-				return ((_phy.udpSend(_v4UpnpUdpSocket,(const struct sockaddr *)addr,data,len) != 0) ? 0 : -1);
-			else return -1;
+				if (addr->ss_family == AF_INET) {
+					if (ttl)
+						_phy.setIp4UdpTtl(_v4UpnpUdpSocket,ttl);
+					const int result = ((_phy.udpSend(_v4UpnpUdpSocket,(const struct sockaddr *)addr,data,len) != 0) ? 0 : -1);
+					if (ttl)
+						_phy.setIp4UdlTtl(_v4UpnpUdpSocket,255);
+					return result;
+				} else {
+					return -1;
+				}
 #ifdef ZT_BREAK_UDP
 			}
 #endif
@@ -1275,8 +1282,13 @@ public:
 #ifdef ZT_BREAK_UDP
 				if (!OSUtils::fileExists("/tmp/ZT_BREAK_UDP")) {
 #endif
-				if (_v4UdpSocket)
-					result = ((_phy.udpSend(_v4UdpSocket,(const struct sockaddr *)addr,data,len) != 0) ? 0 : -1);
+					if (_v4UdpSocket) {
+						if (ttl)
+							_phy.setIp4UdpTtl(_v4UdpSocket,ttl);
+						result = ((_phy.udpSend(_v4UdpSocket,(const struct sockaddr *)addr,data,len) != 0) ? 0 : -1);
+						if (ttl)
+							_phy.setIp4UdpTtl(_v4UdpSocket,255);
+					}
 #ifdef ZT_BREAK_UDP
 				}
 #endif
@@ -1480,8 +1492,8 @@ static long SnodeDataStoreGetFunction(ZT_Node *node,void *uptr,const char *name,
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeDataStoreGetFunction(name,buf,bufSize,readIndex,totalSize); }
 static int SnodeDataStorePutFunction(ZT_Node *node,void *uptr,const char *name,const void *data,unsigned long len,int secure)
 { return reinterpret_cast<OneServiceImpl *>(uptr)->nodeDataStorePutFunction(name,data,len,secure); }
-static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len)
-{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len); }
+static int SnodeWirePacketSendFunction(ZT_Node *node,void *uptr,const struct sockaddr_storage *localAddr,const struct sockaddr_storage *addr,const void *data,unsigned int len,unsigned int ttl)
+{ return reinterpret_cast<OneServiceImpl *>(uptr)->nodeWirePacketSendFunction(localAddr,addr,data,len,ttl); }
 static void SnodeVirtualNetworkFrameFunction(ZT_Node *node,void *uptr,uint64_t nwid,uint64_t sourceMac,uint64_t destMac,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
 { reinterpret_cast<OneServiceImpl *>(uptr)->nodeVirtualNetworkFrameFunction(nwid,sourceMac,destMac,etherType,vlanId,data,len); }