Implement first pass on rapid dead path detection, and increment version to 1.1.3 (dev)

node/Constants.hpp

@@ -264,13 +264,23 @@
 /**
  * Delay between ordinary case pings of direct links
  */
-#define ZT_PEER_DIRECT_PING_DELAY 60000
+#define ZT_PEER_DIRECT_PING_DELAY 90000
 
 /**
  * Timeout for overall peer activity (measured from last receive)
  */
 #define ZT_PEER_ACTIVITY_TIMEOUT ((ZT_PEER_DIRECT_PING_DELAY * 4) + ZT_PING_CHECK_INVERVAL)
 
+/**
+ * No answer timeout to trigger dead path detection
+ */
+#define ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT 3000
+
+/**
+ * Probation threshold after which a path becomes dead
+ */
+#define ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION 3
+
 /**
  * Delay between requests for updated network autoconf information
  */

node/IncomingPacket.cpp

@@ -86,7 +86,7 @@ bool IncomingPacket::tryDecode(const RuntimeEnvironment *RR,bool deferred)
 			switch(v) {
 				//case Packet::VERB_NOP:
 				default: // ignore unknown verbs, but if they pass auth check they are "received"
-					peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),v,0,Packet::VERB_NOP);
+					peer->received(_localAddress,_remoteAddress,hops(),packetId(),v,0,Packet::VERB_NOP);
 					return true;
 
 				case Packet::VERB_HELLO:                          return _doHELLO(RR,peer);
@@ -185,7 +185,7 @@ bool IncomingPacket::_doERROR(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 			default: break;
 		}
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_ERROR,inRePacketId,inReVerb);
 	} catch ( ... ) {
 		TRACE("dropped ERROR from %s(%s): unexpected exception",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -279,7 +279,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 				}
 
 				// Check packet integrity and authentication
-				SharedPtr<Peer> newPeer(new Peer(RR->identity,id));
+				SharedPtr<Peer> newPeer(new Peer(RR,RR->identity,id));
 				if (!dearmor(newPeer->key())) {
 					TRACE("rejected HELLO from %s(%s): packet failed authentication",id.address().toString().c_str(),_remoteAddress.toString().c_str());
 					return true;
@@ -349,7 +349,7 @@ bool IncomingPacket::_doHELLO(const RuntimeEnvironment *RR,SharedPtr<Peer> &peer
 		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
 
 		peer->setRemoteVersion(protoVersion,vMajor,vMinor,vRevision); // important for this to go first so received() knows the version
-		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_HELLO,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped HELLO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -410,7 +410,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					// roots. In the future it should be done if we query less trusted
 					// sources.
 					//if (id.locallyValidate())
-						RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr<Peer>(new Peer(RR->identity,id))));
+						RR->sw->doAnythingWaitingForPeer(RR->topology->addPeer(SharedPtr<Peer>(new Peer(RR,RR->identity,id))));
 				}
 			} break;
 
@@ -450,7 +450,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 					// OK(MULTICAST_FRAME) includes certificate of membership update
 					CertificateOfMembership com;
 					offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME__OK__IDX_COM_AND_GATHER_RESULTS);
-					peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
+					peer->validateAndSetNetworkMembershipCertificate(nwid,com);
 				}
 
 				if ((flags & 0x02) != 0) {
@@ -465,7 +465,7 @@ bool IncomingPacket::_doOK(const RuntimeEnvironment *RR,const SharedPtr<Peer> &p
 			default: break;
 		}
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_OK,inRePacketId,inReVerb);
 	} catch ( ... ) {
 		TRACE("dropped OK from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -494,7 +494,7 @@ bool IncomingPacket::_doWHOIS(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		} else {
 			TRACE("dropped WHOIS from %s(%s): missing or invalid address",source().toString().c_str(),_remoteAddress.toString().c_str());
 		}
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_WHOIS,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped WHOIS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -513,7 +513,7 @@ bool IncomingPacket::_doRENDEZVOUS(const RuntimeEnvironment *RR,const SharedPtr<
 				if ((port > 0)&&((addrlen == 4)||(addrlen == 16))) {
 					InetAddress atAddr(field(ZT_PROTO_VERB_RENDEZVOUS_IDX_ADDRESS,addrlen),addrlen,port);
 					TRACE("RENDEZVOUS from %s says %s might be at %s, starting NAT-t",peer->address().toString().c_str(),with.toString().c_str(),atAddr.toString().c_str());
-					peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP);
+					peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_RENDEZVOUS,0,Packet::VERB_NOP);
 					RR->sw->rendezvous(withPeer,_localAddress,atAddr);
 				} else {
 					TRACE("dropped corrupt RENDEZVOUS from %s(%s) (bad address or port)",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
@@ -553,7 +553,7 @@ bool IncomingPacket::_doFRAME(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 				RR->node->putFrame(network->id(),MAC(peer->address(),network->id()),network->mac(),etherType,0,field(ZT_PROTO_VERB_FRAME_IDX_PAYLOAD,payloadLen),payloadLen);
 			}
 
-			peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP);
+			peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_FRAME,0,Packet::VERB_NOP);
 		} else {
 			TRACE("dropped FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
@@ -575,7 +575,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				if ((flags & 0x01) != 0) {
 					CertificateOfMembership com;
 					comLen = com.deserialize(*this,ZT_PROTO_VERB_EXT_FRAME_IDX_COM);
-					peer->validateAndSetNetworkMembershipCertificate(RR,network->id(),com);
+					peer->validateAndSetNetworkMembershipCertificate(network->id(),com);
 				}
 
 				if (!network->isAllowed(peer)) {
@@ -624,7 +624,7 @@ bool IncomingPacket::_doEXT_FRAME(const RuntimeEnvironment *RR,const SharedPtr<P
 				RR->node->putFrame(network->id(),from,to,etherType,0,field(comLen + ZT_PROTO_VERB_EXT_FRAME_IDX_PAYLOAD,payloadLen),payloadLen);
 			}
 
-			peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP);
+			peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_EXT_FRAME,0,Packet::VERB_NOP);
 		} else {
 			TRACE("dropped EXT_FRAME from %s(%s): we are not connected to network %.16llx",source().toString().c_str(),_remoteAddress.toString().c_str(),at<uint64_t>(ZT_PROTO_VERB_FRAME_IDX_NETWORK_ID));
 		}
@@ -646,7 +646,7 @@ bool IncomingPacket::_doECHO(const RuntimeEnvironment *RR,const SharedPtr<Peer>
 		outp.armor(peer->key(),true);
 		RR->antiRec->logOutgoingZT(outp.data(),outp.size());
 		RR->node->putPacket(_localAddress,_remoteAddress,outp.data(),outp.size());
-		peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_ECHO,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped ECHO from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -665,7 +665,7 @@ bool IncomingPacket::_doMULTICAST_LIKE(const RuntimeEnvironment *RR,const Shared
 			RR->mc->add(now,nwid,group,peer->address());
 		}
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_LIKE,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped MULTICAST_LIKE from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -680,10 +680,10 @@ bool IncomingPacket::_doNETWORK_MEMBERSHIP_CERTIFICATE(const RuntimeEnvironment
 		unsigned int ptr = ZT_PACKET_IDX_PAYLOAD;
 		while (ptr < size()) {
 			ptr += com.deserialize(*this,ptr);
-			peer->validateAndSetNetworkMembershipCertificate(RR,com.networkId(),com);
+			peer->validateAndSetNetworkMembershipCertificate(com.networkId(),com);
 		}
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_MEMBERSHIP_CERTIFICATE,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped NETWORK_MEMBERSHIP_CERTIFICATE from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -700,7 +700,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REQUEST(const RuntimeEnvironment *RR,cons
 
 		const unsigned int h = hops();
 		const uint64_t pid = packetId();
-		peer->received(RR,_localAddress,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,h,pid,Packet::VERB_NETWORK_CONFIG_REQUEST,0,Packet::VERB_NOP);
 
 		if (RR->localNetworkController) {
 			Dictionary netconf;
@@ -789,7 +789,7 @@ bool IncomingPacket::_doNETWORK_CONFIG_REFRESH(const RuntimeEnvironment *RR,cons
 				nw->requestConfiguration();
 			ptr += 8;
 		}
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_NETWORK_CONFIG_REFRESH,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped NETWORK_CONFIG_REFRESH from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -825,7 +825,7 @@ bool IncomingPacket::_doMULTICAST_GATHER(const RuntimeEnvironment *RR,const Shar
 #endif
 		}
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_GATHER,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped MULTICAST_GATHER from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -846,7 +846,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			if ((flags & 0x01) != 0) {
 				CertificateOfMembership com;
 				offset += com.deserialize(*this,ZT_PROTO_VERB_MULTICAST_FRAME_IDX_COM);
-				peer->validateAndSetNetworkMembershipCertificate(RR,nwid,com);
+				peer->validateAndSetNetworkMembershipCertificate(nwid,com);
 			}
 
 			// Check membership after we've read any included COM, since
@@ -915,7 +915,7 @@ bool IncomingPacket::_doMULTICAST_FRAME(const RuntimeEnvironment *RR,const Share
 			}
 		} // else ignore -- not a member of this network
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_MULTICAST_FRAME,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped MULTICAST_FRAME from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -955,7 +955,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) && (!peer->hasActivePathTo(now,a)) ) {
 						if (++countPerScope[(int)a.ipScope()][0] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-							peer->sendHELLO(RR,_localAddress,a,now);
+							peer->sendHELLO(_localAddress,a,now);
 						} else {
 							TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str());
 						}
@@ -966,7 +966,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 					if ( ((flags & 0x01) == 0) && (Path::isAddressValidForPath(a)) && (!peer->hasActivePathTo(now,a)) ) {
 						if (++countPerScope[(int)a.ipScope()][1] <= ZT_PUSH_DIRECT_PATHS_MAX_PER_SCOPE_AND_FAMILY) {
 							TRACE("attempting to contact %s at pushed direct path %s",peer->address().toString().c_str(),a.toString().c_str());
-							peer->sendHELLO(RR,_localAddress,a,now);
+							peer->sendHELLO(_localAddress,a,now);
 						} else {
 							TRACE("ignoring contact for %s at %s -- too many per scope",peer->address().toString().c_str(),a.toString().c_str());
 						}
@@ -976,7 +976,7 @@ bool IncomingPacket::_doPUSH_DIRECT_PATHS(const RuntimeEnvironment *RR,const Sha
 			ptr += addrLen;
 		}
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_PUSH_DIRECT_PATHS,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped PUSH_DIRECT_PATHS from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -1143,7 +1143,7 @@ bool IncomingPacket::_doCIRCUIT_TEST(const RuntimeEnvironment *RR,const SharedPt
 			}
 		}
 
-		peer->received(RR,_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP);
+		peer->received(_localAddress,_remoteAddress,hops(),packetId(),Packet::VERB_CIRCUIT_TEST,0,Packet::VERB_NOP);
 	} catch ( ... ) {
 		TRACE("dropped CIRCUIT_TEST from %s(%s): unexpected exception",source().toString().c_str(),_remoteAddress.toString().c_str());
 	}
@@ -1238,7 +1238,7 @@ bool IncomingPacket::_doREQUEST_PROOF_OF_WORK(const RuntimeEnvironment *RR,const
 					break;
 			}
 
-			peer->received(RR,_localAddress,_remoteAddress,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP);
+			peer->received(_localAddress,_remoteAddress,hops(),pid,Packet::VERB_REQUEST_PROOF_OF_WORK,0,Packet::VERB_NOP);
 		} else {
 			TRACE("dropped REQUEST_PROOF_OF_WORK from %s(%s): not trusted enough",peer->address().toString().c_str(),_remoteAddress.toString().c_str());
 		}

node/Node.cpp

@@ -244,20 +244,20 @@ public:
 			// "Upstream" devices are roots and relays and get special treatment -- they stay alive
 			// forever and we try to keep (if available) both IPv4 and IPv6 channels open to them.
 			bool needToContactIndirect = true;
-			if (p->doPingAndKeepalive(RR,_now,AF_INET)) {
+			if (p->doPingAndKeepalive(_now,AF_INET)) {
 				needToContactIndirect = false;
 			} else {
 				if (stableEndpoint4) {
 					needToContactIndirect = false;
-					p->sendHELLO(RR,InetAddress(),stableEndpoint4,_now);
+					p->sendHELLO(InetAddress(),stableEndpoint4,_now);
 				}
 			}
-			if (p->doPingAndKeepalive(RR,_now,AF_INET6)) {
+			if (p->doPingAndKeepalive(_now,AF_INET6)) {
 				needToContactIndirect = false;
 			} else {
 				if (stableEndpoint6) {
 					needToContactIndirect = false;
-					p->sendHELLO(RR,InetAddress(),stableEndpoint6,_now);
+					p->sendHELLO(InetAddress(),stableEndpoint6,_now);
 				}
 			}
 
@@ -273,7 +273,7 @@ public:
 			lastReceiveFromUpstream = std::max(p->lastReceive(),lastReceiveFromUpstream);
 		} else if (p->activelyTransferringFrames(_now)) {
 			// Normal nodes get their preferred link kept alive if the node has generated frame traffic recently
-			p->doPingAndKeepalive(RR,_now,0);
+			p->doPingAndKeepalive(_now,0);
 		}
 	}
 

node/Packet.hpp

@@ -655,32 +655,11 @@ public:
 
 		/**
 		 * ECHO request (a.k.a. ping):
-		 *   <[1] 8-bit purpose of echo request>
-		 *   <[...] additional arbitrary payload>
+		 *   <[...] arbitrary payload>
 		 *
 		 * This generates OK with a copy of the transmitted payload. No ERROR
 		 * is generated. Response to ECHO requests is optional and ECHO may be
 		 * ignored if a node detects a possible flood.
-		 *
-		 * An empty payload is permitted. This is used in some versions for
-		 * path checking and validation. If a payload is present it must
-		 * follow the above format, though the recipient does not have to check
-		 * this. It can simply echo it back.
-		 *
-		 * Echo purpose codes:
-		 *   0x00 - User ECHO request
-		 *   0x01 - Dead path detection
-		 *
-		 * Support for fragmented echo packets is optional and their use is not
-		 * recommended.
-		 *
-		 * Dead path detection is performed by sending ECHOs with the same random
-		 * payload to the best (or every) direct path and then once indirectly
-		 * (such as via a root server). When an OK is received echoing back this
-		 * test payload, all paths that have not yet received this OK are cancelled
-		 * or re-tested. This can be done after a short period of inactivity to
-		 * detect and automatically cancel dead paths without requiring any
-		 * special logic (other than support for ECHO) at the remote end.
 		 */
 		VERB_ECHO = 8,
 

node/Path.hpp

@@ -98,14 +98,21 @@ public:
 	 *
 	 * @param t Time of send
 	 */
-	inline void sent(uint64_t t) { _lastSend = t; }
+	inline void sent(uint64_t t)
+	{
+		_lastSend = t;
+	}
 
 	/**
 	 * Called when a packet is received from this remote path
 	 *
 	 * @param t Time of receive
 	 */
-	inline void received(uint64_t t) { _lastReceived = t; }
+	inline void received(uint64_t t)
+	{
+		_lastReceived = t;
+		_probation = 0;
+	}
 
 	/**
 	 * @param now Current time
@@ -114,7 +121,7 @@ public:
 	inline bool active(uint64_t now) const
 		throw()
 	{
-		return ((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT);
+		return (((now - _lastReceived) < ZT_PEER_ACTIVITY_TIMEOUT)&&(_probation < ZT_PEER_DEAD_PATH_DETECTION_MAX_PROBATION));
 	}
 
 	/**
@@ -240,28 +247,40 @@ public:
 	inline bool isClusterSuboptimal() const { return ((_flags & ZT_PATH_FLAG_CLUSTER_SUBOPTIMAL) != 0); }
 #endif
 
+	/**
+	 * @return Current path probation count (for dead path detect)
+	 */
+	inline unsigned int probation() const { return _probation; }
+
+	/**
+	 * Increase this path's probation violation count (for dead path detect)
+	 */
+	inline void increaseProbation() { ++_probation; }
+
 	template<unsigned int C>
 	inline void serialize(Buffer<C> &b) const
 	{
-		b.append((uint8_t)0); // version
+		b.append((uint8_t)1); // version
 		b.append((uint64_t)_lastSend);
 		b.append((uint64_t)_lastReceived);
 		_addr.serialize(b);
 		_localAddress.serialize(b);
 		b.append((uint16_t)_flags);
+		b.append((uint16_t)_probation);
 	}
 
 	template<unsigned int C>
 	inline unsigned int deserialize(const Buffer<C> &b,unsigned int startAt = 0)
 	{
 		unsigned int p = startAt;
-		if (b[p++] != 0)
+		if (b[p++] != 1)
 			throw std::invalid_argument("invalid serialized Path");
 		_lastSend = b.template at<uint64_t>(p); p += 8;
 		_lastReceived = b.template at<uint64_t>(p); p += 8;
 		p += _addr.deserialize(b,p);
 		p += _localAddress.deserialize(b,p);
 		_flags = b.template at<uint16_t>(p); p += 2;
+		_probation = b.template at<uint16_t>(p); p += 2;
 		_ipScope = _addr.ipScope();
 		return (p - startAt);
 	}
@@ -275,6 +294,7 @@ private:
 	InetAddress _addr;
 	InetAddress _localAddress;
 	unsigned int _flags;
+	unsigned int _probation;
 	InetAddress::IpScope _ipScope; // memoize this since it's a computed value checked often
 };
 

node/Peer.cpp

@@ -46,8 +46,8 @@ namespace ZeroTier {
 // Used to send varying values for NAT keepalive
 static uint32_t _natKeepaliveBuf = 0;
 
-Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
-	throw(std::runtime_error) :
+Peer::Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity) :
+	RR(renv),
 	_lastUsed(0),
 	_lastReceive(0),
 	_lastUnicastFrame(0),
@@ -72,7 +72,6 @@ Peer::Peer(const Identity &myIdentity,const Identity &peerIdentity)
 }
 
 void Peer::received(
-	const RuntimeEnvironment *RR,
 	const InetAddress &localAddr,
 	const InetAddress &remoteAddr,
 	unsigned int hops,
@@ -199,7 +198,7 @@ void Peer::received(
 						outp.armor(_key,true);
 						RR->node->putPacket(localAddr,remoteAddr,outp.data(),outp.size());
 					} else {
-						sendHELLO(RR,localAddr,remoteAddr,now);
+						sendHELLO(localAddr,remoteAddr,now);
 					}
 
 				}
@@ -214,7 +213,7 @@ void Peer::received(
 	}
 }
 
-void Peer::sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl)
+void Peer::sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl)
 {
 	// _lock not required here since _id is immutable and nothing else is accessed
 
@@ -234,7 +233,7 @@ void Peer::sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,c
 	RR->node->putPacket(localAddr,atAddress,outp.data(),outp.size(),ttl);
 }
 
-bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily)
+bool Peer::doPingAndKeepalive(uint64_t now,int inetAddressFamily)
 {
 	Path *p = (Path *)0;
 
@@ -248,7 +247,7 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 	if (p) {
 		if ((now - p->lastReceived()) >= ZT_PEER_DIRECT_PING_DELAY) {
 			//TRACE("PING %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
-			sendHELLO(RR,p->localAddress(),p->address(),now);
+			sendHELLO(p->localAddress(),p->address(),now);
 			p->sent(now);
 		} else if (((now - p->lastSend()) >= ZT_NAT_KEEPALIVE_DELAY)&&(!p->reliable())) {
 			//TRACE("NAT keepalive %s(%s) after %llums/%llums send/receive inactivity",_id.address().toString().c_str(),p->address().toString().c_str(),now - p->lastSend(),now - p->lastReceived());
@@ -264,7 +263,7 @@ bool Peer::doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inet
 	return false;
 }
 
-void Peer::pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force)
+void Peer::pushDirectPaths(Path *path,uint64_t now,bool force)
 {
 #ifdef ZT_ENABLE_CLUSTER
 	// Cluster mode disables normal PUSH_DIRECT_PATHS in favor of cluster-based peer redirection
@@ -332,7 +331,7 @@ void Peer::pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,
 	}
 }
 
-bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope scope,uint64_t now)
+bool Peer::resetWithinScope(InetAddress::IpScope scope,uint64_t now)
 {
 	Mutex::Lock _l(_lock);
 	unsigned int np = _numPaths;
@@ -340,7 +339,7 @@ bool Peer::resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope sc
 	unsigned int y = 0;
 	while (x < np) {
 		if (_paths[x].address().ipScope() == scope) {
-			sendHELLO(RR,_paths[x].localAddress(),_paths[x].address(),now);
+			sendHELLO(_paths[x].localAddress(),_paths[x].address(),now);
 		} else {
 			_paths[y++] = _paths[x];
 		}
@@ -383,7 +382,7 @@ bool Peer::networkMembershipCertificatesAgree(uint64_t nwid,const CertificateOfM
 	return false;
 }
 
-bool Peer::validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com)
+bool Peer::validateAndSetNetworkMembershipCertificate(uint64_t nwid,const CertificateOfMembership &com)
 {
 	// Sanity checks
 	if ((!com)||(com.issuedTo() != _id.address()))
@@ -448,7 +447,7 @@ bool Peer::needsOurNetworkMembershipCertificate(uint64_t nwid,uint64_t now,bool
 	return ((now - tmp) >= (ZT_NETWORK_AUTOCONF_DELAY / 2));
 }
 
-void Peer::clean(const RuntimeEnvironment *RR,uint64_t now)
+void Peer::clean(uint64_t now)
 {
 	Mutex::Lock _l(_lock);
 
@@ -485,6 +484,34 @@ void Peer::clean(const RuntimeEnvironment *RR,uint64_t now)
 	}
 }
 
+bool Peer::_checkPath(Path &p,const uint64_t now)
+{
+	// assumes _lock is locked
+
+	if (!p.active(now))
+		return false;
+
+	if (p.lastSend() > p.lastReceived()) {
+		if ((p.lastSend() - p.lastReceived()) >= ZT_PEER_DEAD_PATH_DETECTION_NO_ANSWER_TIMEOUT) {
+			TRACE("%s(%s) has not answered, checking if dead (probation: %u)",_id.address().toString().c_str(),p.address().toString().c_str(),p.probation());
+
+			if ( (_vProto >= 5) && ( !((_vMajor == 1)&&(_vMinor == 1)&&(_vRevision == 0)) ) ) {
+				// 1.1.1 and newer nodes support ECHO, which is smaller -- but 1.1.0 has a bug so use HELLO there too
+				Packet outp(_id.address(),RR->identity.address(),Packet::VERB_ECHO);
+				outp.armor(_key,true);
+				p.send(RR,outp.data(),outp.size(),now);
+			} else {
+				sendHELLO(p.localAddress(),p.address(),now);
+				p.sent(now);
+			}
+
+			p.increaseProbation();
+		}
+	}
+
+	return true;
+}
+
 Path *Peer::_getBestPath(const uint64_t now)
 {
 	// assumes _lock is locked
@@ -492,7 +519,7 @@ Path *Peer::_getBestPath(const uint64_t now)
 	uint64_t bestPathScore = 0;
 	for(unsigned int i=0;i<_numPaths;++i) {
 		const uint64_t score = _paths[i].score();
-		if ((score >= bestPathScore)&&(_paths[i].active(now))) {
+		if ((score >= bestPathScore)&&(_checkPath(_paths[i],now))) {
 			bestPathScore = score;
 			bestPath = &(_paths[i]);
 		}
@@ -507,7 +534,7 @@ Path *Peer::_getBestPath(const uint64_t now,int inetAddressFamily)
 	uint64_t bestPathScore = 0;
 	for(unsigned int i=0;i<_numPaths;++i) {
 		const uint64_t score = _paths[i].score();
-		if (((int)_paths[i].address().ss_family == inetAddressFamily)&&(score >= bestPathScore)&&(_paths[i].active(now))) {
+		if (((int)_paths[i].address().ss_family == inetAddressFamily)&&(score >= bestPathScore)&&(_checkPath(_paths[i],now))) {
 			bestPathScore = score;
 			bestPath = &(_paths[i]);
 		}

node/Peer.hpp

@@ -75,12 +75,12 @@ public:
 	/**
 	 * Construct a new peer
 	 *
+	 * @param renv Runtime environment
 	 * @param myIdentity Identity of THIS node (for key agreement)
 	 * @param peerIdentity Identity of peer
 	 * @throws std::runtime_error Key agreement with peer's identity failed
 	 */
-	Peer(const Identity &myIdentity,const Identity &peerIdentity)
-		throw(std::runtime_error);
+	Peer(const RuntimeEnvironment *renv,const Identity &myIdentity,const Identity &peerIdentity);
 
 	/**
 	 * @return Time peer record was last used in any way
@@ -120,7 +120,6 @@ public:
 	 * @param inReVerb Verb in reply to (for OK/ERROR, default: VERB_NOP)
 	 */
 	void received(
-		const RuntimeEnvironment *RR,
 		const InetAddress &localAddr,
 		const InetAddress &remoteAddr,
 		unsigned int hops,
@@ -144,13 +143,12 @@ public:
 	/**
 	 * Send via best path
 	 *
-	 * @param RR Runtime environment
 	 * @param data Packet data
 	 * @param len Packet length
 	 * @param now Current time
 	 * @return Path used on success or NULL on failure
 	 */
-	inline Path *send(const RuntimeEnvironment *RR,const void *data,unsigned int len,uint64_t now)
+	inline Path *send(const void *data,unsigned int len,uint64_t now)
 	{
 		Path *const bestPath = getBestPath(now);
 		if (bestPath) {
@@ -166,33 +164,30 @@ public:
 	 * This does not update any statistics. It's used to send initial HELLOs
 	 * for NAT traversal and path verification.
 	 *
-	 * @param RR Runtime environment
 	 * @param localAddr Local address
 	 * @param atAddress Destination address
 	 * @param now Current time
 	 * @param ttl Desired IP TTL (default: 0 to leave alone)
 	 */
-	void sendHELLO(const RuntimeEnvironment *RR,const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl = 0);
+	void sendHELLO(const InetAddress &localAddr,const InetAddress &atAddress,uint64_t now,unsigned int ttl = 0);
 
 	/**
 	 * Send pings or keepalives depending on configured timeouts
 	 *
-	 * @param RR Runtime environment
 	 * @param now Current time
 	 * @param inetAddressFamily Keep this address family alive, or 0 to simply pick current best ignoring family
 	 * @return True if at least one direct path seems alive
 	 */
-	bool doPingAndKeepalive(const RuntimeEnvironment *RR,uint64_t now,int inetAddressFamily);
+	bool doPingAndKeepalive(uint64_t now,int inetAddressFamily);
 
 	/**
 	 * Push direct paths back to self if we haven't done so in the configured timeout
 	 *
-	 * @param RR Runtime environment
 	 * @param path Remote path to use to send the push
 	 * @param now Current time
 	 * @param force If true, push regardless of rate limit
 	 */
-	void pushDirectPaths(const RuntimeEnvironment *RR,Path *path,uint64_t now,bool force);
+	void pushDirectPaths(Path *path,uint64_t now,bool force);
 
 	/**
 	 * @return All known direct paths to this peer
@@ -324,12 +319,11 @@ public:
 	/**
 	 * Reset paths within a given scope
 	 *
-	 * @param RR Runtime environment
 	 * @param scope IP scope of paths to reset
 	 * @param now Current time
 	 * @return True if at least one path was forgotten
 	 */
-	bool resetWithinScope(const RuntimeEnvironment *RR,InetAddress::IpScope scope,uint64_t now);
+	bool resetWithinScope(InetAddress::IpScope scope,uint64_t now);
 
 	/**
 	 * @return 256-bit secret symmetric encryption key
@@ -383,11 +377,10 @@ public:
 	/**
 	 * Check the validity of the COM and add/update if valid and new
 	 *
-	 * @param RR Runtime Environment
 	 * @param nwid Network ID
 	 * @param com Externally supplied COM
 	 */
-	bool validateAndSetNetworkMembershipCertificate(const RuntimeEnvironment *RR,uint64_t nwid,const CertificateOfMembership &com);
+	bool validateAndSetNetworkMembershipCertificate(uint64_t nwid,const CertificateOfMembership &com);
 
 	/**
 	 * @param nwid Network ID
@@ -399,8 +392,10 @@ public:
 
 	/**
 	 * Perform periodic cleaning operations
+	 *
+	 * @param now Current time
 	 */
-	void clean(const RuntimeEnvironment *RR,uint64_t now);
+	void clean(uint64_t now);
 
 	/**
 	 * Update direct path push stats and return true if we should respond
@@ -503,13 +498,14 @@ public:
 	/**
 	 * Create a new Peer from a serialized instance
 	 *
+	 * @param renv Runtime environment
 	 * @param myIdentity This node's identity
 	 * @param b Buffer containing serialized Peer data
 	 * @param p Pointer to current position in buffer, will be updated in place as buffer is read (value/result)
 	 * @return New instance of Peer or NULL if serialized data was corrupt or otherwise invalid (may also throw an exception via Buffer)
 	 */
 	template<unsigned int C>
-	static inline SharedPtr<Peer> deserializeNew(const Identity &myIdentity,const Buffer<C> &b,unsigned int &p)
+	static inline SharedPtr<Peer> deserializeNew(const RuntimeEnvironment *renv,const Identity &myIdentity,const Buffer<C> &b,unsigned int &p)
 	{
 		const unsigned int recSize = b.template at<uint32_t>(p); p += 4;
 		if ((p + recSize) > b.size())
@@ -523,7 +519,7 @@ public:
 		if (!npid)
 			return SharedPtr<Peer>();
 
-		SharedPtr<Peer> np(new Peer(myIdentity,npid));
+		SharedPtr<Peer> np(new Peer(renv,myIdentity,npid));
 
 		np->_lastUsed = b.template at<uint64_t>(p); p += 8;
 		np->_lastReceive = b.template at<uint64_t>(p); p += 8;
@@ -569,11 +565,13 @@ public:
 	}
 
 private:
+	bool _checkPath(Path &p,const uint64_t now);
 	Path *_getBestPath(const uint64_t now);
 	Path *_getBestPath(const uint64_t now,int inetAddressFamily);
 
 	unsigned char _key[ZT_PEER_SECRET_KEY_LENGTH]; // computed with key agreement, not serialized
 
+	const RuntimeEnvironment *RR;
 	uint64_t _lastUsed;
 	uint64_t _lastReceive; // direct or indirect
 	uint64_t _lastUnicastFrame;

node/SelfAwareness.cpp

@@ -46,21 +46,19 @@ namespace ZeroTier {
 class _ResetWithinScope
 {
 public:
-	_ResetWithinScope(const RuntimeEnvironment *renv,uint64_t now,InetAddress::IpScope scope) :
-		RR(renv),
+	_ResetWithinScope(uint64_t now,InetAddress::IpScope scope) :
 		_now(now),
 		_scope(scope) {}
 
 	inline void operator()(Topology &t,const SharedPtr<Peer> &p)
 	{
-		if (p->resetWithinScope(RR,_scope,_now))
+		if (p->resetWithinScope(_scope,_now))
 			peersReset.push_back(p);
 	}
 
 	std::vector< SharedPtr<Peer> > peersReset;
 
 private:
-	const RuntimeEnvironment *RR;
 	uint64_t _now;
 	InetAddress::IpScope _scope;
 };
@@ -121,7 +119,7 @@ void SelfAwareness::iam(const Address &reporter,const InetAddress &reporterPhysi
 		}
 
 		// Reset all paths within this scope
-		_ResetWithinScope rset(RR,now,(InetAddress::IpScope)scope);
+		_ResetWithinScope rset(now,(InetAddress::IpScope)scope);
 		RR->topology->eachPeer<_ResetWithinScope &>(rset);
 
 		// Send a NOP to all peers for whom we forgot a path. This will cause direct

node/Switch.cpp

@@ -408,7 +408,7 @@ bool Switch::unite(const Address &p1,const Address &p2)
 				outp.append(cg.first.rawIpData(),4);
 			}
 			outp.armor(p1p->key(),true);
-			p1p->send(RR,outp.data(),outp.size(),now);
+			p1p->send(outp.data(),outp.size(),now);
 		} else {
 			// Tell p2 where to find p1.
 			Packet outp(p2,RR->identity.address(),Packet::VERB_RENDEZVOUS);
@@ -423,7 +423,7 @@ bool Switch::unite(const Address &p1,const Address &p2)
 				outp.append(cg.second.rawIpData(),4);
 			}
 			outp.armor(p2p->key(),true);
-			p2p->send(RR,outp.data(),outp.size(),now);
+			p2p->send(outp.data(),outp.size(),now);
 		}
 		++alt; // counts up and also flips LSB
 	}
@@ -435,7 +435,7 @@ void Switch::rendezvous(const SharedPtr<Peer> &peer,const InetAddress &localAddr
 {
 	TRACE("sending NAT-t message to %s(%s)",peer->address().toString().c_str(),atAddr.toString().c_str());
 	const uint64_t now = RR->node->now();
-	peer->sendHELLO(RR,localAddr,atAddr,now,2); // first attempt: send low-TTL packet to 'open' local NAT
+	peer->sendHELLO(localAddr,atAddr,now,2); // first attempt: send low-TTL packet to 'open' local NAT
 	{
 		Mutex::Lock _l(_contactQueue_m);
 		_contactQueue.push_back(ContactQueueEntry(peer,now + ZT_NAT_T_TACTICAL_ESCALATION_DELAY,localAddr,atAddr));
@@ -508,14 +508,14 @@ unsigned long Switch::doTimerTasks(uint64_t now)
 				} else {
 					if (qi->strategyIteration == 0) {
 						// First strategy: send packet directly to destination
-						qi->peer->sendHELLO(RR,qi->localAddr,qi->inaddr,now);
+						qi->peer->sendHELLO(qi->localAddr,qi->inaddr,now);
 					} else if (qi->strategyIteration <= 3) {
 						// Strategies 1-3: try escalating ports for symmetric NATs that remap sequentially
 						InetAddress tmpaddr(qi->inaddr);
 						int p = (int)qi->inaddr.port() + qi->strategyIteration;
 						if (p < 0xffff) {
 							tmpaddr.setPort((unsigned int)p);
-							qi->peer->sendHELLO(RR,qi->localAddr,tmpaddr,now);
+							qi->peer->sendHELLO(qi->localAddr,tmpaddr,now);
 						} else qi->strategyIteration = 5;
 					} else {
 						// All strategies tried, expire entry
@@ -619,7 +619,7 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 			// Note: we don't bother initiating NAT-t for fragments, since heads will set that off.
 			// It wouldn't hurt anything, just redundant and unnecessary.
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-			if ((!relayTo)||(!relayTo->send(RR,fragment.data(),fragment.size(),RR->node->now()))) {
+			if ((!relayTo)||(!relayTo->send(fragment.data(),fragment.size(),RR->node->now()))) {
 #ifdef ZT_ENABLE_CLUSTER
 				if (RR->cluster) {
 					RR->cluster->sendViaCluster(Address(),destination,fragment.data(),fragment.size(),false);
@@ -630,7 +630,7 @@ void Switch::_handleRemotePacketFragment(const InetAddress &localAddr,const Inet
 				// Don't know peer or no direct path -- so relay via root server
 				relayTo = RR->topology->getBestRoot();
 				if (relayTo)
-					relayTo->send(RR,fragment.data(),fragment.size(),RR->node->now());
+					relayTo->send(fragment.data(),fragment.size(),RR->node->now());
 			}
 		} else {
 			TRACE("dropped relay [fragment](%s) -> %s, max hops exceeded",fromAddr.toString().c_str(),destination.toString().c_str());
@@ -705,7 +705,7 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 			packet->incrementHops();
 
 			SharedPtr<Peer> relayTo = RR->topology->getPeer(destination);
-			if ((relayTo)&&((relayTo->send(RR,packet->data(),packet->size(),now)))) {
+			if ((relayTo)&&((relayTo->send(packet->data(),packet->size(),now)))) {
 				Mutex::Lock _l(_lastUniteAttempt_m);
 				uint64_t &luts = _lastUniteAttempt[_LastUniteKey(source,destination)];
 				if ((now - luts) >= ZT_MIN_UNITE_INTERVAL) {
@@ -730,7 +730,7 @@ void Switch::_handleRemotePacketHead(const InetAddress &localAddr,const InetAddr
 
 				relayTo = RR->topology->getBestRoot(&source,1,true);
 				if (relayTo)
-					relayTo->send(RR,packet->data(),packet->size(),now);
+					relayTo->send(packet->data(),packet->size(),now);
 			}
 		} else {
 			TRACE("dropped relay %s(%s) -> %s, max hops exceeded",packet->source().toString().c_str(),fromAddr.toString().c_str(),destination.toString().c_str());
@@ -787,7 +787,7 @@ Address Switch::_sendWhoisRequest(const Address &addr,const Address *peersAlread
 		Packet outp(root->address(),RR->identity.address(),Packet::VERB_WHOIS);
 		addr.appendTo(outp);
 		outp.armor(root->key(),true);
-		if (root->send(RR,outp.data(),outp.size(),RR->node->now()))
+		if (root->send(outp.data(),outp.size(),RR->node->now()))
 			return root->address();
 	}
 	return Address();
@@ -841,7 +841,7 @@ bool Switch::_trySend(const Packet &packet,bool encrypt,uint64_t nwid)
 
 		if ((network)&&(relay)&&(network->isAllowed(peer))) {
 			// Push hints for direct connectivity to this peer if we are relaying
-			peer->pushDirectPaths(RR,viaPath,now,false);
+			peer->pushDirectPaths(viaPath,now,false);
 		}
 
 		Packet tmp(packet);

node/Topology.cpp

@@ -67,7 +67,7 @@ Topology::Topology(const RuntimeEnvironment *renv) :
 				);
 			unsigned int pos = 0;
 			deserializeBuf->copyFrom(all + ptr,reclen + 4);
-			SharedPtr<Peer> p(Peer::deserializeNew(RR->identity,*deserializeBuf,pos));
+			SharedPtr<Peer> p(Peer::deserializeNew(RR,RR->identity,*deserializeBuf,pos));
 			ptr += pos;
 			if (!p)
 				break; // stop if invalid records
@@ -180,7 +180,7 @@ SharedPtr<Peer> Topology::getPeer(const Address &zta)
 	try {
 		Identity id(_getIdentity(zta));
 		if (id) {
-			SharedPtr<Peer> np(new Peer(RR->identity,id));
+			SharedPtr<Peer> np(new Peer(RR,RR->identity,id));
 			{
 				Mutex::Lock _l(_lock);
 				SharedPtr<Peer> &ap = _peers[zta];
@@ -327,7 +327,7 @@ void Topology::clean(uint64_t now)
 		if (((now - (*p)->lastUsed()) >= ZT_PEER_IN_MEMORY_EXPIRATION)&&(std::find(_rootAddresses.begin(),_rootAddresses.end(),*a) == _rootAddresses.end())) {
 			_peers.erase(*a);
 		} else {
-			(*p)->clean(RR,now);
+			(*p)->clean(now);
 		}
 	}
 }
@@ -361,7 +361,7 @@ void Topology::_setWorld(const World &newWorld)
 			if (rp) {
 				_rootPeers.push_back(*rp);
 			} else {
-				SharedPtr<Peer> newrp(new Peer(RR->identity,r->identity));
+				SharedPtr<Peer> newrp(new Peer(RR,RR->identity,r->identity));
 				_peers.set(r->identity.address(),newrp);
 				_rootPeers.push_back(newrp);
 			}

version.h

@@ -41,6 +41,6 @@
 /**
  * Revision
  */
-#define ZEROTIER_ONE_VERSION_REVISION 2
+#define ZEROTIER_ONE_VERSION_REVISION 3
 
 #endif