More VL1 work after re-re-re-refactor...

node/AES.hpp

@@ -218,8 +218,22 @@ public:
 		 */
 		ZT_INLINE void init(const uint8_t iv[16],void *const output) noexcept
 		{
-			_ctr[0] = Utils::loadAsIsEndian<uint64_t>(iv);
-			_ctr[1] = Utils::loadAsIsEndian<uint64_t>(iv + 8);
+			Utils::copy<16>(_ctr,iv);
+			_out = reinterpret_cast<uint8_t *>(output);
+			_len = 0;
+		}
+
+		/**
+		 * Initialize this CTR instance to encrypt a new stream
+		 *
+		 * @param iv Unique initialization vector
+		 * @param ic Initial counter (must be in big-endian byte order!)
+		 * @param output Buffer to which to store output (MUST be large enough for total bytes processed!)
+		 */
+		ZT_INLINE void init(const uint8_t iv[12],const uint32_t ic,void *const output) noexcept
+		{
+			Utils::copy<12>(_ctr,iv);
+			reinterpret_cast<uint32_t *>(_ctr)[3] = ic;
 			_out = reinterpret_cast<uint8_t *>(output);
 			_len = 0;
 		}

node/Buf.hpp

@@ -747,6 +747,32 @@ public:
 			Utils::copy(unsafeData + s,bytes,len);
 	}
 
+	/**
+	 * Write zeroes
+	 *
+	 * @param ii Index value-result parameter (incremented by len)
+	 * @param len Number of zero bytes to write
+	 */
+	ZT_INLINE void wZ(int &ii,const unsigned int len) noexcept
+	{
+		const int s = ii;
+		if (likely((ii += (int)len) <= ZT_BUF_MEM_SIZE))
+			Utils::zero(unsafeData + s,len);
+	}
+
+	/**
+	 * Write secure random bytes
+	 *
+	 * @param ii Index value-result parameter (incremented by len)
+	 * @param len Number of random bytes to write
+	 */
+	ZT_INLINE void wR(int &ii,const unsigned int len) noexcept
+	{
+		const int s = ii;
+		if (likely((ii += (int)len) <= ZT_BUF_MEM_SIZE))
+			Utils::getSecureRandom(unsafeData + s,len);
+	}
+
 	/**
 	 * Store a byte without advancing the index
 	 */

node/Constants.hpp

@@ -134,11 +134,6 @@
  */
 #define ZT_PATH_ALIVE_TIMEOUT ((ZT_PATH_KEEPALIVE_PERIOD * 2) + 5000)
 
-/**
- * Number of ports to try for each BFG1024 scan attempt (if enabled).
- */
-#define ZT_NAT_T_BFG1024_PORTS_PER_ATTEMPT 256
-
 /**
  * Maximum number of queued endpoints to try per "pulse."
  */

node/Peer.cpp

@@ -16,7 +16,6 @@
 #include "Trace.hpp"
 #include "Peer.hpp"
 #include "Topology.hpp"
-#include "Node.hpp"
 #include "SelfAwareness.hpp"
 #include "InetAddress.hpp"
 #include "Protocol.hpp"
@@ -38,7 +37,6 @@ Peer::Peer(const RuntimeEnvironment *renv) :
 	m_alivePathCount(0),
 	m_tryQueue(),
 	m_tryQueuePtr(m_tryQueue.end()),
-	m_probe(0),
 	m_vProto(0),
 	m_vMajor(0),
 	m_vMinor(0),
@@ -144,50 +142,64 @@ void Peer::received(
 	}
 }
 
-void Peer::send(void *const tPtr,const int64_t now,const void *const data,const unsigned int len) noexcept
-{
-	SharedPtr<Path> via(this->path(now));
-	if (via) {
-		via->send(RR,tPtr,data,len,now);
-	} else {
-		const SharedPtr<Peer> root(RR->topology->root());
-		if ((root)&&(root.ptr() != this)) {
-			via = root->path(now);
-			if (via) {
-				via->send(RR,tPtr,data,len,now);
-				root->relayed(now,len);
-			} else {
-				return;
-			}
-		} else {
-			return;
-		}
-	}
-	sent(now,len);
-}
-
 unsigned int Peer::hello(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now)
 {
-#if 0
-	Packet outp(_id.address(),RR->identity.address(),Packet::VERB_HELLO);
-
-	outp.append((unsigned char)ZT_PROTO_VERSION);
-	outp.append((unsigned char)ZEROTIER_VERSION_MAJOR);
-	outp.append((unsigned char)ZEROTIER_VERSION_MINOR);
-	outp.append((uint16_t)ZEROTIER_VERSION_REVISION);
-	outp.append(now);
-	RR->identity.serialize(outp,false);
-	atAddress.serialize(outp);
-
-	RR->node->expectReplyTo(outp.packetId());
-
-	if (atAddress) {
-		outp.armor(_key,false); // false == don't encrypt full payload, but add MAC
-		RR->node->putPacket(tPtr,localSocket,atAddress,outp.data(),outp.size());
-	} else {
-		RR->sw->send(tPtr,outp,false); // false == don't encrypt full payload, but add MAC
-	}
-#endif
+	Buf outp;
+
+	const int64_t now = RR->node->now();
+	const uint64_t packetId = m_identityKey->nextMessage(RR->identity.address(),m_id.address());
+	int ii = Protocol::newPacket(outp,packetId,m_id.address(),RR->identity.address(),Protocol::VERB_HELLO);
+
+	outp.wI8(ii,ZT_PROTO_VERSION);
+	outp.wI8(ii,ZEROTIER_VERSION_MAJOR);
+	outp.wI8(ii,ZEROTIER_VERSION_MINOR);
+	outp.wI16(ii,ZEROTIER_VERSION_REVISION);
+	outp.wI64(ii,(uint64_t)now);
+	outp.wO(ii,RR->identity);
+	outp.wO(ii,atAddress);
+
+	const int ivStart = ii;
+	outp.wR(ii,12);
+
+	// LEGACY: the six reserved bytes after the IV exist for legacy compatibility with v1.x nodes.
+	// Once those are dead they'll become just reserved bytes for future use as flags etc.
+	outp.wI32(ii,0); // reserved bytes
+	void *const legacyMoonCountStart = outp.unsafeData + ii;
+	outp.wI16(ii,0);
+	const uint64_t legacySalsaIv = packetId & ZT_CONST_TO_BE_UINT64(0xfffffffffffffff8ULL);
+	Salsa20(m_identityKey->secret,&legacySalsaIv).crypt12(legacyMoonCountStart,legacyMoonCountStart,2);
+
+	const int cryptSectionStart = ii;
+	FCV<uint8_t,4096> md;
+	Dictionary::append(md,ZT_PROTO_HELLO_NODE_META_INSTANCE_ID,RR->instanceId);
+	outp.wI16(ii,(uint16_t)md.size());
+	outp.wB(ii,md.data(),(unsigned int)md.size());
+
+	if (unlikely((ii + ZT_HMACSHA384_LEN) > ZT_BUF_SIZE)) // sanity check: should be impossible
+		return 0;
+
+	AES::CTR ctr(m_helloCipher);
+	void *const cryptSection = outp.unsafeData + ii;
+	ctr.init(outp.unsafeData + ivStart,0,cryptSection);
+	ctr.crypt(cryptSection,ii - cryptSectionStart);
+	ctr.finish();
+
+	HMACSHA384(m_helloMacKey,outp.unsafeData,ii,outp.unsafeData + ii);
+	ii += ZT_HMACSHA384_LEN;
+
+	// LEGACY: we also need Poly1305 for v1.x peers.
+	uint8_t polyKey[ZT_POLY1305_KEY_SIZE],perPacketKey[ZT_SALSA20_KEY_SIZE];
+	Protocol::salsa2012DeriveKey(m_identityKey->secret,perPacketKey,outp,ii);
+	Salsa20(perPacketKey,&packetId).crypt12(Utils::ZERO256,polyKey,sizeof(polyKey));
+	Poly1305 p1305(polyKey);
+	p1305.update(outp.unsafeData + ZT_PROTO_PACKET_ENCRYPTED_SECTION_START,ii - ZT_PROTO_PACKET_ENCRYPTED_SECTION_START);
+	uint64_t polyMac[2];
+	p1305.finish(polyMac);
+	Utils::storeAsIsEndian<uint64_t>(outp.unsafeData + ZT_PROTO_PACKET_MAC_INDEX,polyMac[0]);
+
+	if (likely(RR->node->putPacket(tPtr,localSocket,atAddress,outp.unsafeData,ii)))
+		return ii;
+	return 0;
 }
 
 void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
@@ -197,7 +209,7 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 	// Determine if we need to send a full HELLO because we are refreshing ephemeral
 	// keys or it's simply been too long.
 	bool needHello = false;
-	if ( ((now - m_ephemeralPairTimestamp) >= (ZT_SYMMETRIC_KEY_TTL / 2)) || ((m_ephemeralKeys[0])&&(m_ephemeralKeys[0]->odometer() >= (ZT_SYMMETRIC_KEY_TTL_MESSAGES / 2))) ) {
+	if ( (m_vProto >= 11) && ( ((now - m_ephemeralPairTimestamp) >= (ZT_SYMMETRIC_KEY_TTL / 2)) || ((m_ephemeralKeys[0])&&(m_ephemeralKeys[0]->odometer() >= (ZT_SYMMETRIC_KEY_TTL_MESSAGES / 2))) ) ) {
 		m_ephemeralPair.generate();
 		needHello = true;
 	} else if ((now - m_lastSentHello) >= ZT_PEER_HELLO_INTERVAL) {
@@ -212,7 +224,7 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 		if (RR->node->externalPathLookup(tPtr, m_id, -1, addr)) {
 			if ((addr)&&(RR->node->shouldUsePathForZeroTierTraffic(tPtr, m_id, -1, addr))) {
 				RR->t->tryingNewPath(tPtr, 0x84a10000, m_id, addr, InetAddress::NIL, 0, 0, Identity::NIL);
-				sent(now,m_sendProbe(tPtr,-1,addr,now));
+				sent(now,m_sendProbe(tPtr,-1,addr,nullptr,0,now));
 			}
 		}
 
@@ -224,7 +236,7 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 				} else {
 					if ((i->second.isInetAddr())&&(!i->second.ip().ipsEqual(addr))) {
 						RR->t->tryingNewPath(tPtr, 0x0a009444, m_id, i->second.ip(), InetAddress::NIL, 0, 0, Identity::NIL);
-						sent(now,m_sendProbe(tPtr,-1,i->second.ip(),now));
+						sent(now,m_sendProbe(tPtr,-1,i->second.ip(),nullptr,0,now));
 						break;
 					}
 				}
@@ -232,66 +244,61 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 		}
 	}
 
+	// Sort paths and forget expired ones.
 	m_prioritizePaths(now);
 
-	// Attempt queued paths to try.
-	for(int k=0;(k<ZT_NAT_T_MAX_QUEUED_ATTEMPTS_PER_PULSE)&&(!m_tryQueue.empty());++k) {
-		// This is a global circular pointer that iterates through the list of
-		// endpoints to attempt.
-		if (m_tryQueuePtr == m_tryQueue.end())
-			m_tryQueuePtr = m_tryQueue.begin();
-
-		// Delete timed out entries.
-		if ((now - m_tryQueuePtr->ts) > ZT_PATH_ALIVE_TIMEOUT) {
-			m_tryQueue.erase(m_tryQueuePtr++);
-			continue;
-		}
-
-		if (m_tryQueuePtr->target.isInetAddr()) {
-			// Delete entries that duplicate existing alive paths.
-			bool duplicate = false;
-			for(unsigned int i=0;i<m_alivePathCount;++i) {
-				if (m_paths[i]->address() == m_tryQueuePtr->target.ip()) {
-					duplicate = true;
+	// Attempt queued endpoints if they don't overlap with paths.
+	if (!m_tryQueue.empty()) {
+		for(int k=0;k<ZT_NAT_T_MAX_QUEUED_ATTEMPTS_PER_PULSE;++k) {
+			// This is a global circular pointer that iterates through the list of
+			// endpoints to attempt.
+			if (m_tryQueuePtr == m_tryQueue.end()) {
+				if (m_tryQueue.empty())
 					break;
-				}
-			}
-			if (duplicate) {
-				m_tryQueue.erase(m_tryQueuePtr++);
-				continue;
+				m_tryQueuePtr = m_tryQueue.begin();
 			}
 
-			if (m_tryQueuePtr->breakSymmetricBFG1024 && RR->node->natMustDie()) {
-				// Attempt aggressive NAT traversal if both requested and enabled.
-				uint16_t ports[1023];
-				for (unsigned int i=0;i<1023;++i)
-					ports[i] = (uint64_t)(i + 1);
-				for (unsigned int i=0;i<512;++i) {
-					const uint64_t rn = Utils::random();
-					const unsigned int a = (unsigned int)rn % 1023;
-					const unsigned int b = (unsigned int)(rn >> 32U) % 1023;
-					if (a != b) {
-						uint16_t tmp = ports[a];
-						ports[a] = ports[b];
-						ports[b] = tmp;
+			if (likely((now - m_tryQueuePtr->ts) < ZT_PATH_ALIVE_TIMEOUT)) {
+				if (m_tryQueuePtr->target.isInetAddr()) {
+					for(unsigned int i=0;i<m_alivePathCount;++i) {
+						if (m_paths[i]->address().ipsEqual(m_tryQueuePtr->target.ip()))
+							goto skip_tryQueue_item;
+					}
+
+					if ((m_alivePathCount == 0) && (m_tryQueuePtr->breakSymmetricBFG1024) && (RR->node->natMustDie())) {
+						// Attempt aggressive NAT traversal if both requested and enabled. This sends a probe
+						// to all ports under 1024, which assumes that the peer has bound to such a port and
+						// has attempted to initiate a connection through it. This can traverse a decent number
+						// of symmetric NATs at the cost of 32KiB per attempt and the potential to trigger IDS
+						// systems by looking like a port scan (because it is).
+						uint16_t ports[1023];
+						for (unsigned int i=0;i<1023;++i)
+							ports[i] = (uint64_t)(i + 1);
+						for (unsigned int i=0;i<512;++i) {
+							const uint64_t rn = Utils::random();
+							const unsigned int a = (unsigned int)rn % 1023;
+							const unsigned int b = (unsigned int)(rn >> 32U) % 1023;
+							if (a != b) {
+								const uint16_t tmp = ports[a];
+								ports[a] = ports[b];
+								ports[b] = tmp;
+							}
+						}
+						sent(now,m_sendProbe(tPtr, -1, m_tryQueuePtr->target.ip(), ports, 1023, now));
+					} else {
+						sent(now,m_sendProbe(tPtr, -1, m_tryQueuePtr->target.ip(), nullptr, 0, now));
 					}
 				}
-				InetAddress addr(m_tryQueuePtr->target.ip());
-				for (unsigned int i=0;i<ZT_NAT_T_BFG1024_PORTS_PER_ATTEMPT;++i) {
-					addr.setPort(ports[i]);
-					sent(now,m_sendProbe(tPtr,-1,addr,now));
-				}
-			} else {
-				// Otherwise send a normal probe.
-				sent(now,m_sendProbe(tPtr, -1, m_tryQueuePtr->target.ip(), now));
 			}
-		}
 
-		++m_tryQueuePtr;
+skip_tryQueue_item:
+			m_tryQueue.erase(m_tryQueuePtr++);
+		}
 	}
 
 	// Do keepalive on all currently active paths, sending HELLO to the first
 	// if needHello is true and sending small keepalives to others.
+	uint64_t randomJunk = Utils::random();
 	for(unsigned int i=0;i<m_alivePathCount;++i) {
 		if (needHello) {
 			needHello = false;
@@ -300,13 +307,12 @@ void Peer::pulse(void *const tPtr,const int64_t now,const bool isRoot)
 			sent(now,bytes);
 			m_lastSentHello = now;
 		} else if ((now - m_paths[i]->lastOut()) >= ZT_PATH_KEEPALIVE_PERIOD) {
-			m_paths[i]->send(RR, tPtr, &now, 1, now);
+			m_paths[i]->send(RR, tPtr, reinterpret_cast<uint8_t *>(&randomJunk) + (i & 7U), 1, now);
 			sent(now,1);
 		}
 	}
 
-	// If we need a HELLO and were not able to send one via any other path,
-	// send one indirectly.
+	// Send a HELLO indirectly if we were not able to send one via any direct path.
 	if (needHello) {
 		const SharedPtr<Peer> root(RR->topology->root());
 		if (root) {
@@ -335,21 +341,25 @@ void Peer::contact(void *tPtr,const int64_t now,const Endpoint &ep,const bool br
 		++foo;
 	}
 
-	// Check to see if this endpoint overlaps an existing queue item. If so, just update it.
-	for(List<p_TryQueueItem>::iterator i(m_tryQueue.begin());i!=m_tryQueue.end();++i) {
-		if (i->target == ep) {
-			i->ts = now;
-			i->breakSymmetricBFG1024 = breakSymmetricBFG1024;
-			return;
+	const bool wasEmpty = m_tryQueue.empty();
+	if (!wasEmpty) {
+		for(List<p_TryQueueItem>::iterator i(m_tryQueue.begin());i!=m_tryQueue.end();++i) {
+			if (i->target == ep) {
+				i->ts = now;
+				i->breakSymmetricBFG1024 = breakSymmetricBFG1024;
+				return;
+			}
 		}
 	}
 
-	// Add endpoint to endpoint attempt queue.
 #ifdef __CPP11__
 	m_tryQueue.emplace_back(now, ep, breakSymmetricBFG1024);
 #else
 	_tryQueue.push_back(_TryQueueItem(now,ep,breakSymmetricBFG1024));
 #endif
+
+	if (wasEmpty)
+		m_tryQueuePtr = m_tryQueue.begin();
 }
 
 void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddressFamily,int64_t now)
@@ -358,7 +368,7 @@ void Peer::resetWithinScope(void *tPtr,InetAddress::IpScope scope,int inetAddres
 	unsigned int pc = 0;
 	for(unsigned int i=0;i<m_alivePathCount;++i) {
 		if ((m_paths[i]) && ((m_paths[i]->address().family() == inetAddressFamily) && (m_paths[i]->address().ipScope() == scope))) {
-			const unsigned int bytes = m_sendProbe(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), now);
+			const unsigned int bytes = m_sendProbe(tPtr, m_paths[i]->localSocket(), m_paths[i]->address(), nullptr, 0, now);
 			m_paths[i]->sent(now, bytes);
 			sent(now,bytes);
 		} else if (pc != i) {
@@ -517,8 +527,6 @@ int Peer::unmarshal(const uint8_t *restrict data,const int len) noexcept
 		m_bootstrap[tmp.type()] = tmp;
 	}
 
-	m_probe = 0; // ephemeral token, reset on unmarshal
-
 	if ((p + 10) > len)
 		return -1;
 	m_vProto = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
@@ -562,30 +570,34 @@ void Peer::m_prioritizePaths(int64_t now)
 	}
 }
 
-unsigned int Peer::m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now)
+unsigned int Peer::m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,const uint16_t *ports,const unsigned int numPorts,int64_t now)
 {
 	// Assumes m_lock is locked
-	if ((m_vProto < 11)||(m_probe == 0)) {
-		const SharedPtr<SymmetricKey> k(m_key());
-		const uint64_t packetId = k->nextMessage(RR->identity.address(),m_id.address());
-
-		uint8_t p[ZT_PROTO_MIN_PACKET_LENGTH + 1];
-		Utils::storeAsIsEndian<uint64_t>(p + ZT_PROTO_PACKET_ID_INDEX,packetId);
-		m_id.address().copyTo(p + ZT_PROTO_PACKET_DESTINATION_INDEX);
-		RR->identity.address().copyTo(p + ZT_PROTO_PACKET_SOURCE_INDEX);
-		p[ZT_PROTO_PACKET_FLAGS_INDEX] = 0;
-		p[ZT_PROTO_PACKET_VERB_INDEX] = Protocol::VERB_ECHO;
-		p[ZT_PROTO_PACKET_VERB_INDEX + 1] = (uint8_t)now; // arbitrary byte
-
-		Protocol::armor(p,ZT_PROTO_MIN_PACKET_LENGTH,k,cipher());
-
-		RR->expect->sending(packetId,now);
-		RR->node->putPacket(tPtr,-1,atAddress,p,ZT_PROTO_MIN_PACKET_LENGTH);
-
-		return ZT_PROTO_MIN_PACKET_LENGTH;
+	const SharedPtr<SymmetricKey> k(m_key());
+	const uint64_t packetId = k->nextMessage(RR->identity.address(),m_id.address());
+
+	uint8_t p[ZT_PROTO_MIN_PACKET_LENGTH + 1];
+	Utils::storeAsIsEndian<uint64_t>(p + ZT_PROTO_PACKET_ID_INDEX,packetId);
+	m_id.address().copyTo(p + ZT_PROTO_PACKET_DESTINATION_INDEX);
+	RR->identity.address().copyTo(p + ZT_PROTO_PACKET_SOURCE_INDEX);
+	p[ZT_PROTO_PACKET_FLAGS_INDEX] = 0;
+	p[ZT_PROTO_PACKET_VERB_INDEX] = Protocol::VERB_ECHO;
+	p[ZT_PROTO_PACKET_VERB_INDEX + 1] = 0; // arbitrary payload
+
+	Protocol::armor(p,ZT_PROTO_MIN_PACKET_LENGTH + 1,k,cipher());
+
+	RR->expect->sending(packetId,now);
+
+	if (numPorts > 0) {
+		InetAddress tmp(atAddress);
+		for(unsigned int i=0;i<numPorts;++i) {
+			tmp.setPort(ports[i]);
+			RR->node->putPacket(tPtr,-1,tmp,p,ZT_PROTO_MIN_PACKET_LENGTH + 1);
+		}
+		return ZT_PROTO_MIN_PACKET_LENGTH * numPorts;
 	} else {
-		RR->node->putPacket(tPtr,-1,atAddress,&m_probe,4);
-		return 4;
+		RR->node->putPacket(tPtr,-1,atAddress,p,ZT_PROTO_MIN_PACKET_LENGTH + 1);
+		return ZT_PROTO_MIN_PACKET_LENGTH;
 	}
 }
 

node/Peer.hpp

@@ -89,32 +89,6 @@ public:
 		return m_locator;
 	}
 
-	/**
-	 * Set this peer's probe token
-	 *
-	 * This doesn't update the mapping in Topology. The caller must do
-	 * this, which is the HELLO handler in VL1.
-	 *
-	 * @param t New probe token
-	 * @return Old probe token
-	 */
-	ZT_INLINE uint32_t setProbeToken(const uint32_t t) noexcept
-	{
-		RWMutex::Lock l(m_lock);
-		const uint32_t pt = m_probe;
-		m_probe = t;
-		return pt;
-	}
-
-	/**
-	 * @return This peer's probe token or 0 if unknown
-	 */
-	ZT_INLINE uint32_t probeToken() const noexcept
-	{
-		RWMutex::RLock l(m_lock);
-		return m_probe;
-	}
-
 	/**
 	 * Log receipt of an authenticated packet
 	 *
@@ -167,13 +141,13 @@ public:
 	 */
 	ZT_INLINE SharedPtr<Path> path(const int64_t now) noexcept
 	{
-		if ((now - m_lastPrioritizedPaths) > ZT_PEER_PRIORITIZE_PATHS_INTERVAL) {
-			RWMutex::Lock l(m_lock);
-			m_prioritizePaths(now);
+		if (likely((now - m_lastPrioritizedPaths) < ZT_PEER_PRIORITIZE_PATHS_INTERVAL)) {
+			RWMutex::RLock l(m_lock);
 			if (m_alivePathCount > 0)
 				return m_paths[0];
 		} else {
-			RWMutex::RLock l(m_lock);
+			RWMutex::Lock l(m_lock);
+			m_prioritizePaths(now);
 			if (m_alivePathCount > 0)
 				return m_paths[0];
 		}
@@ -206,7 +180,27 @@ public:
 	 * @param data Data to send
 	 * @param len Length in bytes
 	 */
-	void send(void *tPtr,int64_t now,const void *data,unsigned int len) noexcept;
+	ZT_INLINE void send(void *tPtr,int64_t now,const void *data,unsigned int len) noexcept
+	{
+		SharedPtr<Path> via(this->path(now));
+		if (via) {
+			via->send(RR,tPtr,data,len,now);
+		} else {
+			const SharedPtr<Peer> root(RR->topology->root());
+			if ((root)&&(root.ptr() != this)) {
+				via = root->path(now);
+				if (via) {
+					via->send(RR,tPtr,data,len,now);
+					root->relayed(now,len);
+				} else {
+					return;
+				}
+			} else {
+				return;
+			}
+		}
+		sent(now,len);
+	}
 
 	/**
 	 * Send a HELLO to this peer at a specified physical address.
@@ -463,7 +457,7 @@ public:
 
 private:
 	void m_prioritizePaths(int64_t now);
-	unsigned int m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,int64_t now);
+	unsigned int m_sendProbe(void *tPtr,int64_t localSocket,const InetAddress &atAddress,const uint16_t *ports,unsigned int numPorts,int64_t now);
 	void m_deriveSecondaryIdentityKeys() noexcept;
 
 	ZT_INLINE SharedPtr<SymmetricKey> m_key() noexcept
@@ -547,9 +541,6 @@ private:
 	List<p_TryQueueItem> m_tryQueue;
 	List<p_TryQueueItem>::iterator m_tryQueuePtr; // loops over _tryQueue like a circular buffer
 
-	// 32-bit probe token or 0 if unknown.
-	uint32_t m_probe;
-
 	uint16_t m_vProto;
 	uint16_t m_vMajor;
 	uint16_t m_vMinor;

node/Protocol.hpp

@@ -251,10 +251,7 @@
 
 #define ZT_PROTO_HELLO_NODE_META_INSTANCE_ID      "i"
 #define ZT_PROTO_HELLO_NODE_META_LOCATOR          "l"
-#define ZT_PROTO_HELLO_NODE_META_PROBE_TOKEN      "p"
 #define ZT_PROTO_HELLO_NODE_META_SOFTWARE_VENDOR  "s"
-#define ZT_PROTO_HELLO_NODE_META_SOFTWARE_VERSION "v"
-#define ZT_PROTO_HELLO_NODE_META_PHYSICAL_DEST    "d"
 #define ZT_PROTO_HELLO_NODE_META_COMPLIANCE       "c"
 #define ZT_PROTO_HELLO_NODE_META_EPHEMERAL_PUBLIC "e"
 #define ZT_PROTO_HELLO_NODE_META_EPHEMERAL_ACK    "E"
@@ -282,18 +279,19 @@ enum Verb
 	/**
 	 * Announcement of a node's existence and vitals:
 	 *   <[1] protocol version>
-	 *   <[1] software major version (LEGACY)>
-	 *   <[1] software minor version (LEGACY)>
-	 *   <[2] software revision (LEGACY)>
+	 *   <[1] software major version (optional, 0 if unspecified)>
+	 *   <[1] software minor version (optional, 0 if unspecified)>
+	 *   <[2] software revision (optional, 0 if unspecified)>
 	 *   <[8] timestamp>
 	 *   <[...] binary serialized full sender identity>
-	 *   <[...] physical destination address of packet (LEGACY)>
+	 *   <[...] physical destination of packet>
 	 *   <[12] 96-bit CTR IV>
+	 *   <[6] reserved bytes, currently used for legacy compatibility>
 	 *   [... start of encrypted section ...]
 	 *   <[2] 16-bit length of encrypted dictionary>
 	 *   <[...] encrypted dictionary>
 	 *   [... end of encrypted section ...]
-	 *   <[48] HMAC-SHA384 of plaintext packet>
+	 *   <[48] HMAC-SHA384 of packet>
 	 *
 	 * HELLO is sent to initiate a new pairing between two nodes and
 	 * periodically to refresh information.
@@ -342,7 +340,6 @@ enum Verb
 	 * 
 	 *   INSTANCE_ID - a 64-bit unique value generated on each node start
 	 *   LOCATOR - signed record enumerating this node's trusted contact points
-	 *   PROBE_TOKEN - 32-bit probe token
 	 *   EPHEMERAL_PUBLIC - Ephemeral public key(s)
 	 * 
 	 * OK will contain EPHEMERAL_PUBLIC (of the sender) and:
@@ -354,8 +351,6 @@ enum Verb
 	 *   HOSTNAME - arbitrary short host name for this node
 	 *   CONTACT - arbitrary short contact information string for this node
 	 *   SOFTWARE_VENDOR - short name or description of vendor, such as a URL
-	 *   SOFTWARE_VERSION - major, minor, revision, and build (packed 64-bit int)
-	 *   PHYSICAL_DEST - serialized Endpoint to which this message was sent
 	 *   COMPLIANCE - bit mask containing bits for e.g. a FIPS-compliant node
 	 *
 	 * The timestamp field in OK is echoed but the others represent the sender
@@ -363,22 +358,21 @@ enum Verb
 	 * only contains the EPHEMERAL fields, allowing the receiver of the OK to
 	 * confirm that both sides know the correct keys and thus begin using the
 	 * ephemeral shared secret to send packets.
+	 * 
+	 * OK is sent encrypted with the usual AEAD, but still includes a full HMAC
+	 * as well (inside the cryptographic envelope).
 	 *
 	 * OK payload:
 	 *   <[8] timestamp echoed from original HELLO>
 	 *   <[1] protocol version of responding node>
+	 *   <[1] software major version (optional)>
+	 *   <[1] software minor version (optional)>
+	 *   <[2] software revision (optional)>
+	 *   <[...] physical destination address of packet>
+	 *   <[2] 16-bit reserved field (zero for legacy compatibility)>
 	 *   <[2] 16-bit length of dictionary>
 	 *   <[...] dictionary>
 	 *   <[48] HMAC-SHA384 of plaintext packet>
-	 *
-	 * Legacy OK payload (sent to pre-2.x nodes):
-	 *   <[8] timestamp echoed from original HELLO>
-	 *   <[1] protocol version of responding node>
-	 *   <[1] software major version>
-	 *   <[1] software minor version>
-	 *   <[2] software revision>
-	 *   <[...] physical destination address of packet>
-	 *   <[2] 16-bit zero length of additional fields>
 	 */
 	VERB_HELLO = 0x01,
 

node/RuntimeEnvironment.hpp

@@ -40,7 +40,8 @@ class Expect;
 class RuntimeEnvironment
 {
 public:
-	ZT_INLINE RuntimeEnvironment(Node *n) noexcept :
+	ZT_INLINE RuntimeEnvironment(Node *const n) noexcept :
+		instanceId(Utils::getSecureRandomU64()),
 		node(n),
 		localNetworkController(nullptr),
 		rtmem(nullptr),
@@ -51,8 +52,8 @@ public:
 		topology(nullptr),
 		sa(nullptr)
 	{
-		publicIdentityStr[0] = nullptr;
-		secretIdentityStr[0] = nullptr;
+		publicIdentityStr[0] = 0;
+		secretIdentityStr[0] = 0;
 	}
 
 	ZT_INLINE ~RuntimeEnvironment() noexcept
@@ -60,6 +61,9 @@ public:
 		Utils::burn(secretIdentityStr,sizeof(secretIdentityStr));
 	}
 
+	// Unique ID generated on startup
+	const uint64_t instanceId;
+
 	// Node instance that owns this RuntimeEnvironment
 	Node *const node;
 

node/Speck128.hpp

@@ -17,6 +17,8 @@
 #include "Constants.hpp"
 #include "Utils.hpp"
 
+#define ZT_SPECK128_KEY_SIZE 16
+
 namespace ZeroTier {
 
 /**

node/Topology.cpp

@@ -61,38 +61,6 @@ SharedPtr<Peer> Topology::add(void *tPtr,const SharedPtr<Peer> &peer)
 	return peer;
 }
 
-PeerList Topology::peersByProbeToken(const uint32_t probeToken) const
-{
-	Mutex::Lock l(m_peersByProbeToken_l);
-	std::pair< MultiMap< uint32_t,SharedPtr<Peer> >::const_iterator,MultiMap< uint32_t,SharedPtr<Peer> >::const_iterator > r(m_peersByProbeToken.equal_range(probeToken));
-	PeerList pl;
-	if (r.first == r.second)
-		return pl;
-	const unsigned int cnt = (unsigned int)std::distance(r.first,r.second);
-	pl.resize(cnt);
-	MultiMap< uint32_t,SharedPtr<Peer> >::const_iterator pi(r.first);
-	for(unsigned int i=0;i<cnt;++i) {
-		pl[i] = pi->second;
-		++pi;
-	}
-	return pl;
-}
-
-void Topology::updateProbeToken(const SharedPtr<Peer> &peer,const uint32_t oldToken,const uint32_t newToken)
-{
-	Mutex::Lock l(m_peersByProbeToken_l);
-	if (oldToken != 0) {
-		std::pair< MultiMap< uint32_t,SharedPtr<Peer> >::iterator,MultiMap< uint32_t,SharedPtr<Peer> >::iterator > r(m_peersByProbeToken.equal_range(oldToken));
-		for(MultiMap< uint32_t,SharedPtr<Peer> >::iterator i(r.first);i!=r.second;) {
-			if (i->second == peer)
-				m_peersByProbeToken.erase(i++);
-			else ++i;
-		}
-	}
-	if (newToken != 0)
-		m_peersByProbeToken.insert(std::pair< uint32_t,SharedPtr<Peer> >(newToken,peer));
-}
-
 void Topology::setPhysicalPathConfiguration(const struct sockaddr_storage *pathNetwork,const ZT_PhysicalPathConfiguration *pathConfig)
 {
 	if (!pathNetwork) {
@@ -191,7 +159,6 @@ void Topology::doPeriodicTasks(void *tPtr,const int64_t now)
 		RWMutex::Lock l1(m_peers_l);
 		for(Map< Address,SharedPtr<Peer> >::iterator i(m_peers.begin());i != m_peers.end();) {
 			if ( ((now - i->second->lastReceive()) > ZT_PEER_ALIVE_TIMEOUT) && (m_roots.count(i->second->identity()) == 0) ) {
-				updateProbeToken(i->second,i->second->probeToken(),0);
 				i->second->save(tPtr);
 				m_peers.erase(i++);
 			} else ++i;

node/Topology.hpp

@@ -82,23 +82,6 @@ public:
 		}
 	}
 
-	/**
-	 * Get peer(s) by 32-bit probe token
-	 *
-	 * @param probeToken Probe token
-	 * @return List of peers
-	 */
-	PeerList peersByProbeToken(uint32_t probeToken) const;
-
-	/**
-	 * Set or update the probe token associated with a peer
-	 *
-	 * @param peer Peer to update
-	 * @param oldToken Old probe token or 0 if none
-	 * @param newToken New probe token or 0 to erase old mapping but not set a new token
-	 */
-	void updateProbeToken(const SharedPtr<Peer> &peer,uint32_t oldToken,uint32_t newToken);
-
 	/**
 	 * Get a Path object for a given local and remote physical address, creating if needed
 	 *
@@ -321,7 +304,6 @@ private:
 	const RuntimeEnvironment *const RR;
 
 	RWMutex m_paths_l;
-	Mutex m_peersByProbeToken_l;
 	RWMutex m_peers_l;
 
 	std::pair< InetAddress,ZT_PhysicalPathConfiguration > m_physicalPathConfig[ZT_MAX_CONFIGURABLE_PATHS];
@@ -329,8 +311,6 @@ private:
 
 	Map< uint64_t,SharedPtr<Path> > m_paths;
 
-	MultiMap< uint32_t,SharedPtr<Peer> > m_peersByProbeToken;
-
 	Map< Address,SharedPtr<Peer> > m_peers;
 	Set< Identity > m_roots;
 	Vector< SharedPtr<Peer> > m_rootPeers;

node/VL1.cpp

@@ -39,16 +39,26 @@ struct p_SalsaPolyCopyFunction
 {
 	Salsa20 s20;
 	Poly1305 poly1305;
+	unsigned int hdrRemaining;
 	ZT_INLINE p_SalsaPolyCopyFunction(const void *salsaKey,const void *salsaIv) :
 		s20(salsaKey,salsaIv),
-		poly1305()
+		poly1305(),
+		hdrRemaining(ZT_PROTO_PACKET_ENCRYPTED_SECTION_START)
 	{
 		uint8_t macKey[ZT_POLY1305_KEY_SIZE];
 		s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
 		poly1305.init(macKey);
 	}
-	ZT_INLINE void operator()(void *dest,const void *src,const unsigned int len) noexcept
+	ZT_INLINE void operator()(void *dest,const void *src,unsigned int len) noexcept
 	{
+		if (hdrRemaining != 0) {
+			unsigned int hdrBytes = (len > hdrRemaining) ? hdrRemaining : len;
+			Utils::copy(dest,src,hdrBytes);
+			hdrRemaining -= hdrBytes;
+			dest = reinterpret_cast<uint8_t *>(dest) + hdrBytes;
+			src = reinterpret_cast<const uint8_t *>(src) + hdrBytes;
+			len -= hdrBytes;
+		}
 		poly1305.update(src,len);
 		s20.crypt12(src,dest,len);
 	}
@@ -57,15 +67,25 @@ struct p_SalsaPolyCopyFunction
 struct p_PolyCopyFunction
 {
 	Poly1305 poly1305;
+	unsigned int hdrRemaining;
 	ZT_INLINE p_PolyCopyFunction(const void *salsaKey,const void *salsaIv) :
-		poly1305()
+		poly1305(),
+		hdrRemaining(ZT_PROTO_PACKET_ENCRYPTED_SECTION_START)
 	{
 		uint8_t macKey[ZT_POLY1305_KEY_SIZE];
 		Salsa20(salsaKey,salsaIv).crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_SIZE);
 		poly1305.init(macKey);
 	}
-	ZT_INLINE void operator()(void *dest,const void *src,const unsigned int len) noexcept
+	ZT_INLINE void operator()(void *dest,const void *src,unsigned int len) noexcept
 	{
+		if (hdrRemaining != 0) {
+			unsigned int hdrBytes = (len > hdrRemaining) ? hdrRemaining : len;
+			Utils::copy(dest,src,hdrBytes);
+			hdrRemaining -= hdrBytes;
+			dest = reinterpret_cast<uint8_t *>(dest) + hdrBytes;
+			src = reinterpret_cast<const uint8_t *>(src) + hdrBytes;
+			len -= hdrBytes;
+		}
 		poly1305.update(src,len);
 		Utils::copy(dest,src,len);
 	}
@@ -103,21 +123,8 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 	 */
 
 	try {
-		// If this is too short to be a packet or fragment, check if it's a probe and if not simply drop it.
-		if (unlikely(len < ZT_PROTO_MIN_FRAGMENT_LENGTH)) {
-			if (len == ZT_PROTO_PROBE_LENGTH) {
-				const uint32_t probeToken = data->lI32(0);
-				PeerList peers(RR->topology->peersByProbeToken(probeToken));
-				ZT_SPEW("probe %.8lx matches %u peers",(unsigned long)probeToken,peers.size());
-				for(unsigned int pi=0;pi<peers.size();++pi) {
-					if (peers[pi]->rateGateProbeRequest(now)) {
-						ZT_SPEW("HELLO -> %s(%s)",peers[pi]->address().toString().c_str(),fromAddr.toString().c_str());
-						peers[pi]->hello(tPtr,localSocket,fromAddr,now);
-					}
-				}
-			}
+		if (unlikely(len < ZT_PROTO_MIN_FRAGMENT_LENGTH))
 			return;
-		}
 
 		static_assert((ZT_PROTO_PACKET_ID_INDEX + sizeof(uint64_t)) < ZT_PROTO_MIN_FRAGMENT_LENGTH,"overflow");
 		const uint64_t packetId = Utils::loadAsIsEndian<uint64_t>(data->unsafeData + ZT_PROTO_PACKET_ID_INDEX);
@@ -445,9 +452,13 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 	const uint8_t hops = pkt.unsafeData[ZT_PROTO_PACKET_FLAGS_INDEX] & ZT_PROTO_FLAG_FIELD_HOPS_MASK;
 
 	const uint8_t protoVersion = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START>();
-	unsigned int versionMajor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 1>(); // LEGACY
-	unsigned int versionMinor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 2>(); // LEGACY
-	unsigned int versionRev = pkt.lI16<ZT_PROTO_PACKET_PAYLOAD_START + 3>(); // LEGACY
+	if (unlikely(protoVersion < ZT_PROTO_VERSION_MIN)) {
+		RR->t->incomingPacketDropped(tPtr,0x907a9891,packetId,0,Identity::NIL,path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_PEER_TOO_OLD);
+		return SharedPtr<Peer>();
+	}
+	const unsigned int versionMajor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 1>();
+	const unsigned int versionMinor = pkt.lI8<ZT_PROTO_PACKET_PAYLOAD_START + 2>();
+	const unsigned int versionRev = pkt.lI16<ZT_PROTO_PACKET_PAYLOAD_START + 3>();
 	const uint64_t timestamp = pkt.lI64<ZT_PROTO_PACKET_PAYLOAD_START + 5>();
 
 	int ii = ZT_PROTO_PACKET_PAYLOAD_START + 13;
@@ -533,7 +544,6 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 	// This far means we passed MAC (Poly1305 or HMAC-SHA384 for newer peers)
 	// ------------------------------------------------------------------------------------------------------------------
 
-	// LEGACY: this is superseded by the sent-to field in the meta-data dictionary if present.
 	InetAddress sentTo;
 	if (unlikely(pkt.rO(ii,sentTo) < 0)) {
 		RR->t->incomingPacketDropped(tPtr,0x707a9811,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
@@ -544,20 +554,16 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 
 	if (protoVersion >= 11) {
 		// V2.x and newer supports an encrypted section and has a new OK format.
+		ii += 4; // skip reserved field
 		if (likely((ii + 12) < packetSize)) {
-			uint64_t ctrNonce[2];
-			ctrNonce[0] = Utils::loadAsIsEndian<uint64_t>(pkt.unsafeData + ii);
-#if __BYTE_ORDER == __BIG_ENDIAN
-			ctrNonce[1] = ((uint64_t)Utils::loadAsIsEndian<uint32_t>(pkt.unsafeData + ii + 8)) << 32U;
-#else
-			ctrNonce[1] = Utils::loadAsIsEndian<uint32_t>(pkt.unsafeData + ii + 8);
-#endif
-			ii += 12;
 			AES::CTR ctr(peer->identityHelloDictionaryEncryptionCipher());
-			ctr.init(reinterpret_cast<uint8_t *>(ctrNonce),pkt.unsafeData + ii);
+			const uint8_t *const ctrNonce = pkt.unsafeData + ii;
+			ii += 12;
+			ctr.init(ctrNonce,0,pkt.unsafeData + ii);
 			ctr.crypt(pkt.unsafeData + ii,packetSize - ii);
 			ctr.finish();
 
+			ii += 2; // skip reserved field
 			const unsigned int dictSize = pkt.rI16(ii);
 			if (unlikely((ii + dictSize) > packetSize)) {
 				RR->t->incomingPacketDropped(tPtr,0x707a9815,packetId,0,identityFromPeerPtr(peer),path->address(),hops,Protocol::VERB_HELLO,ZT_TRACE_PACKET_DROP_REASON_INVALID_OBJECT);
@@ -570,48 +576,38 @@ SharedPtr<Peer> VL1::m_HELLO(void *tPtr, const SharedPtr<Path> &path, Buf &pkt,
 			}
 
 			if (!md.empty()) {
-				InetAddress sentTo2;
-				if (md.getO(ZT_PROTO_HELLO_NODE_META_PHYSICAL_DEST,sentTo2))
-					sentTo = sentTo2;
-				const uint64_t packedVer = md.getUI(ZT_PROTO_HELLO_NODE_META_SOFTWARE_VERSION);
-				if (packedVer != 0) {
-					versionMajor = (unsigned int)(packedVer >> 48U) & 0xffffU;
-					versionMinor = (unsigned int)(packedVer >> 32U) & 0xffffU;
-					versionRev = (unsigned int)(packedVer >> 16U) & 0xffffU;
-				}
-				const uint32_t probeToken = (uint32_t)md.getUI(ZT_PROTO_HELLO_NODE_META_PROBE_TOKEN);
-				if (probeToken != 0)
-					peer->setProbeToken(probeToken);
+				// TODO
 			}
 		}
+	}
 
-		Protocol::newPacket(pkt,key->nextMessage(RR->identity.address(),peer->address()),peer->address(),RR->identity.address(),Protocol::VERB_OK);
-		ii = ZT_PROTO_PACKET_PAYLOAD_START;
-		pkt.wI8(ii,Protocol::VERB_HELLO);
-		pkt.wI64(ii,packetId);
-		pkt.wI64(ii,timestamp);
-		pkt.wI8(ii,(uint8_t)protoVersion);
+	Protocol::newPacket(pkt,key->nextMessage(RR->identity.address(),peer->address()),peer->address(),RR->identity.address(),Protocol::VERB_OK);
+	ii = ZT_PROTO_PACKET_PAYLOAD_START;
+	pkt.wI8(ii,Protocol::VERB_HELLO);
+	pkt.wI64(ii,packetId);
+	pkt.wI64(ii,timestamp);
+	pkt.wI8(ii,ZT_PROTO_VERSION);
+	pkt.wI8(ii,ZEROTIER_VERSION_MAJOR);
+	pkt.wI8(ii,ZEROTIER_VERSION_MINOR);
+	pkt.wI16(ii,ZEROTIER_VERSION_REVISION);
+	pkt.wO(ii,path->address());
+	pkt.wI16(ii,0); // reserved, specifies no "moons" for older versions
 
+	if (protoVersion >= 11) {
 		FCV<uint8_t,1024> okmd;
 		pkt.wI16(ii,(uint16_t)okmd.size());
 		pkt.wB(ii,okmd.data(),okmd.size());
-	} else {
-		// V1.x has nothing more for this version to parse, and has an older OK format.
-		Protocol::newPacket(pkt,key->nextMessage(RR->identity.address(),peer->address()),peer->address(),RR->identity.address(),Protocol::VERB_OK);
-		ii = ZT_PROTO_PACKET_PAYLOAD_START;
-		pkt.wI8(ii,Protocol::VERB_HELLO);
-		pkt.wI64(ii,packetId);
-		pkt.wI64(ii,timestamp);
-		pkt.wI8(ii,(uint8_t)protoVersion);
-		pkt.wI8(ii,(uint8_t)versionMajor);
-		pkt.wI8(ii,(uint8_t)versionMinor);
-		pkt.wI16(ii,(uint16_t)versionRev);
-		pkt.wO(ii,path->address());
-		pkt.wI16(ii,0);
+
+		if (unlikely((ii + ZT_HMACSHA384_LEN) > ZT_BUF_MEM_SIZE)) // sanity check, should be impossible
+			return SharedPtr<Peer>();
+
+		HMACSHA384(peer->identityHelloHmacKey(),pkt.unsafeData,ii,pkt.unsafeData + ii);
+		ii += ZT_HMACSHA384_LEN;
 	}
 
 	peer->setRemoteVersion(protoVersion,versionMajor,versionMinor,versionRev);
 	peer->send(tPtr,RR->node->now(),pkt.unsafeData,ii,path);
+	return peer;
 }
 
 bool VL1::m_ERROR(void *tPtr,const uint64_t packetId,const unsigned int auth, const SharedPtr<Path> &path, const SharedPtr<Peer> &peer, Buf &pkt, int packetSize, Protocol::Verb &inReVerb)