More minor refactoring

node/AES.cpp

@@ -447,10 +447,6 @@ void AES::CTR::crypt(const void *const input,unsigned int len) noexcept
 
 #ifdef ZT_AES_AESNI
 	if (likely(Utils::CPUID.aes)) {
-		_mm_prefetch(in,_MM_HINT_T0);
-		_mm_prefetch(in + 64,_MM_HINT_T0);
-		_mm_prefetch(in + 128,_MM_HINT_T0);
-
 		uint64_t c0 = _ctr[0];
 		uint64_t c1 = Utils::ntoh(_ctr[1]);
 
@@ -496,10 +492,6 @@ void AES::CTR::crypt(const void *const input,unsigned int len) noexcept
 		// This is the largest chunk size that will fit in SSE registers with four
 		// registers left over for round key data and temporaries.
 		while (len >= 192) {
-			_mm_prefetch(in + 192,_MM_HINT_T0);
-			_mm_prefetch(in + 256,_MM_HINT_T0);
-			_mm_prefetch(in + 320,_MM_HINT_T0);
-
 			__m128i d0,d1,d2,d3,d4,d5,d6,d7,d8,d9,d10,d11;
 			if (likely(c1 < 0xfffffffffffffff4ULL)) {
 				d0 = _mm_set_epi64x((long long)Utils::hton(c1),(long long)c0);

node/AES.hpp

@@ -21,6 +21,7 @@
 #include <cstdint>
 #include <cstring>
 
+#ifndef ZT_AES_NO_ACCEL
 #if (defined(__amd64) || defined(__amd64__) || defined(__x86_64) || defined(__x86_64__) || defined(__AMD64) || defined(__AMD64__) || defined(_M_X64))
 #include <wmmintrin.h>
 #include <emmintrin.h>
@@ -28,6 +29,7 @@
 #include <immintrin.h>
 #define ZT_AES_AESNI 1
 #endif
+#endif
 
 namespace ZeroTier {
 
@@ -197,7 +199,8 @@ public:
 		 */
 		ZT_ALWAYS_INLINE void init(const uint8_t iv[16],void *const output) noexcept
 		{
-			memcpy(_ctr,iv,16);
+			_ctr[0] = Utils::loadAsIsEndian<uint64_t>(iv);
+			_ctr[1] = Utils::loadAsIsEndian<uint64_t>(iv + 8);
 			_out = reinterpret_cast<uint8_t *>(output);
 			_len = 0;
 		}

node/Buf.cpp

@@ -17,7 +17,7 @@ namespace ZeroTier {
 
 static std::atomic<uintptr_t> s_pool(0);
 
-void *Buf::operator new(std::size_t sz) noexcept
+void *Buf::operator new(std::size_t sz)
 {
 	uintptr_t bb;
 	for (;;) {
@@ -34,14 +34,14 @@ void *Buf::operator new(std::size_t sz) noexcept
 		s_pool.store(0);
 		b = (Buf *)malloc(sz);
 		if (!b)
-			return nullptr;
+			throw std::bad_alloc();
 	}
 
 	b->__refCount.store(0);
 	return (void *)b;
 }
 
-void Buf::operator delete(void *ptr) noexcept
+void Buf::operator delete(void *ptr)
 {
 	if (ptr) {
 		uintptr_t bb;

node/Buf.hpp

@@ -83,8 +83,8 @@ class Buf
 
 public:
 	// New and delete operators that allocate Buf instances from a shared lock-free memory pool.
-	static void *operator new(std::size_t sz) noexcept;
-	static void operator delete(void *ptr) noexcept;
+	static void *operator new(std::size_t sz);
+	static void operator delete(void *ptr);
 
 	/**
 	 * Free all instances of Buf in shared pool.

node/Capability.cpp

@@ -18,7 +18,7 @@
 
 namespace ZeroTier {
 
-bool Capability::sign(const Identity &from,const Address &to)
+bool Capability::sign(const Identity &from,const Address &to) noexcept
 {
 	uint8_t buf[ZT_CAPABILITY_MARSHAL_SIZE_MAX + 16];
 	try {
@@ -34,7 +34,7 @@ bool Capability::sign(const Identity &from,const Address &to)
 	return false;
 }
 
-int Capability::marshal(uint8_t data[ZT_CAPABILITY_MARSHAL_SIZE_MAX],const bool forSign) const
+int Capability::marshal(uint8_t data[ZT_CAPABILITY_MARSHAL_SIZE_MAX],const bool forSign) const noexcept
 {
 	int p = 0;
 	if (forSign) {
@@ -72,7 +72,7 @@ int Capability::marshal(uint8_t data[ZT_CAPABILITY_MARSHAL_SIZE_MAX],const bool
 	return p;
 }
 
-int Capability::unmarshal(const uint8_t *data,int len)
+int Capability::unmarshal(const uint8_t *data,int len) noexcept
 {
 	if (len < 22)
 		return -1;
@@ -122,7 +122,7 @@ int Capability::unmarshal(const uint8_t *data,int len)
 	return p;
 }
 
-int Capability::marshalVirtualNetworkRules(uint8_t *data,const ZT_VirtualNetworkRule *const rules,const unsigned int ruleCount)
+int Capability::marshalVirtualNetworkRules(uint8_t *data,const ZT_VirtualNetworkRule *const rules,const unsigned int ruleCount) noexcept
 {
 	int p = 0;
 	for(unsigned int i=0;i<ruleCount;++i) {
@@ -239,7 +239,7 @@ int Capability::marshalVirtualNetworkRules(uint8_t *data,const ZT_VirtualNetwork
 	return p;
 }
 
-int Capability::unmarshalVirtualNetworkRules(const uint8_t *const data,const int len,ZT_VirtualNetworkRule *const rules,unsigned int &ruleCount,const unsigned int maxRuleCount)
+int Capability::unmarshalVirtualNetworkRules(const uint8_t *const data,const int len,ZT_VirtualNetworkRule *const rules,unsigned int &ruleCount,const unsigned int maxRuleCount) noexcept
 {
 	int p = 0;
 	unsigned int rc = 0;

node/Capability.hpp

@@ -62,9 +62,9 @@ class Capability : public Credential
 	friend class Credential;
 
 public:
-	static ZT_ALWAYS_INLINE ZT_CredentialType credentialType() { return ZT_CREDENTIAL_TYPE_CAPABILITY; }
+	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_CAPABILITY; }
 
-	ZT_ALWAYS_INLINE Capability() { memoryZero(this); }
+	ZT_ALWAYS_INLINE Capability() noexcept { memoryZero(this); }
 
 	/**
 	 * @param id Capability ID
@@ -74,7 +74,7 @@ public:
 	 * @param rules Network flow rules for this capability
 	 * @param ruleCount Number of flow rules
 	 */
-	ZT_ALWAYS_INLINE Capability(const uint32_t id,const uint64_t nwid,const int64_t ts,const unsigned int mccl,const ZT_VirtualNetworkRule *const rules,const unsigned int ruleCount) :
+	ZT_ALWAYS_INLINE Capability(const uint32_t id,const uint64_t nwid,const int64_t ts,const unsigned int mccl,const ZT_VirtualNetworkRule *const rules,const unsigned int ruleCount) noexcept :
 		_nwid(nwid),
 		_ts(ts),
 		_id(id),
@@ -88,32 +88,32 @@ public:
 	/**
 	 * @return Rules -- see ruleCount() for size of array
 	 */
-	ZT_ALWAYS_INLINE const ZT_VirtualNetworkRule *rules() const { return _rules; }
+	ZT_ALWAYS_INLINE const ZT_VirtualNetworkRule *rules() const noexcept { return _rules; }
 
 	/**
 	 * @return Number of rules in rules()
 	 */
-	ZT_ALWAYS_INLINE unsigned int ruleCount() const { return _ruleCount; }
+	ZT_ALWAYS_INLINE unsigned int ruleCount() const noexcept { return _ruleCount; }
 
 	/**
 	 * @return ID and evaluation order of this capability in network
 	 */
-	ZT_ALWAYS_INLINE uint32_t id() const { return _id; }
+	ZT_ALWAYS_INLINE uint32_t id() const noexcept { return _id; }
 
 	/**
 	 * @return Network ID for which this capability was issued
 	 */
-	ZT_ALWAYS_INLINE uint64_t networkId() const { return _nwid; }
+	ZT_ALWAYS_INLINE uint64_t networkId() const noexcept { return _nwid; }
 
 	/**
 	 * @return Timestamp
 	 */
-	ZT_ALWAYS_INLINE int64_t timestamp() const { return _ts; }
+	ZT_ALWAYS_INLINE int64_t timestamp() const noexcept { return _ts; }
 
 	/**
 	 * @return Last 'to' address in chain of custody
 	 */
-	ZT_ALWAYS_INLINE Address issuedTo() const
+	ZT_ALWAYS_INLINE Address issuedTo() const noexcept
 	{
 		Address i2;
 		for(int i=0;i<ZT_MAX_CAPABILITY_CUSTODY_CHAIN_LENGTH;++i) {
@@ -137,27 +137,46 @@ public:
 	 * @param to Recipient of this signature
 	 * @return True if signature successful and chain of custody appended
 	 */
-	bool sign(const Identity &from,const Address &to);
+	bool sign(const Identity &from,const Address &to) noexcept;
 
 	/**
 	 * Verify this capability's chain of custody and signatures
 	 *
 	 * @param RR Runtime environment to provide for peer lookup, etc.
 	 */
-	ZT_ALWAYS_INLINE Credential::VerifyResult verify(const RuntimeEnvironment *RR,void *tPtr) const { return _verify(RR,tPtr,*this); }
+	ZT_ALWAYS_INLINE Credential::VerifyResult verify(const RuntimeEnvironment *RR,void *tPtr) const noexcept { return _verify(RR,tPtr,*this); }
 
-	static ZT_ALWAYS_INLINE int marshalSizeMax() { return ZT_CAPABILITY_MARSHAL_SIZE_MAX; }
-	int marshal(uint8_t data[ZT_CAPABILITY_MARSHAL_SIZE_MAX],bool forSign = false) const;
-	int unmarshal(const uint8_t *data,int len);
+	static constexpr int marshalSizeMax() noexcept { return ZT_CAPABILITY_MARSHAL_SIZE_MAX; }
+	int marshal(uint8_t data[ZT_CAPABILITY_MARSHAL_SIZE_MAX],bool forSign = false) const noexcept;
+	int unmarshal(const uint8_t *data,int len) noexcept;
 
-	static int marshalVirtualNetworkRules(uint8_t *data,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount);
-	static int unmarshalVirtualNetworkRules(const uint8_t *data,int len,ZT_VirtualNetworkRule *rules,unsigned int &ruleCount,unsigned int maxRuleCount);
+	/**
+	 * Marshal a set of virtual network rules
+	 *
+	 * @param data Buffer to store rules (must be at least ruleCount * ZT_VIRTUALNETWORKRULE_MARSHAL_SIZE_MAX)
+	 * @param rules Network rules
+	 * @param ruleCount Number of rules
+	 * @return Number of bytes written or -1 on error
+	 */
+	static int marshalVirtualNetworkRules(uint8_t *data,const ZT_VirtualNetworkRule *rules,unsigned int ruleCount) noexcept;
+
+	/**
+	 * Unmarshal a set of virtual network rules
+	 *
+	 * @param data Rule set to unmarshal
+	 * @param len Length of data
+	 * @param rules Buffer to store rules
+	 * @param ruleCount Result parameter to set to the number of rules decoded
+	 * @param maxRuleCount Capacity of rules buffer
+	 * @return Number of bytes unmarshaled or -1 on error
+	 */
+	static int unmarshalVirtualNetworkRules(const uint8_t *data,int len,ZT_VirtualNetworkRule *rules,unsigned int &ruleCount,unsigned int maxRuleCount) noexcept;
 
 	// Provides natural sort order by ID
-	ZT_ALWAYS_INLINE bool operator<(const Capability &c) const { return (_id < c._id); }
+	ZT_ALWAYS_INLINE bool operator<(const Capability &c) const noexcept { return (_id < c._id); }
 
-	ZT_ALWAYS_INLINE bool operator==(const Capability &c) const { return (memcmp(this,&c,sizeof(Capability)) == 0); }
-	ZT_ALWAYS_INLINE bool operator!=(const Capability &c) const { return (memcmp(this,&c,sizeof(Capability)) != 0); }
+	ZT_ALWAYS_INLINE bool operator==(const Capability &c) const noexcept { return (memcmp(this,&c,sizeof(Capability)) == 0); }
+	ZT_ALWAYS_INLINE bool operator!=(const Capability &c) const noexcept { return (memcmp(this,&c,sizeof(Capability)) != 0); }
 
 private:
 	uint64_t _nwid;

node/VL1.cpp

@@ -254,11 +254,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 					s20.crypt12(Utils::ZERO256,macKey,ZT_POLY1305_KEY_LEN);
 
 					// Get a buffer to store the decrypted and fully contiguous packet.
-					pkt.b = Buf::get();
-					if (!pkt.b) {
-						RR->t->unexpectedError(tPtr,0x1de16991,"Buf::get() failed (out of memory?)");
-						return;
-					}
+					pkt.b.set(new Buf());
 
 					// Salsa20 is a stream cipher but it's only seekable to multiples of 64 bytes.
 					// This moves data in slices around so that all slices have sizes that are
@@ -351,12 +347,7 @@ void VL1::onRemotePacket(void *const tPtr,const int64_t localSocket,const InetAd
 				return;
 			}
 
-			SharedPtr<Buf> nb(Buf::get());
-			if (!nb) {
-				RR->t->unexpectedError(tPtr,0xffe169fa,"Buf::get() failed (out of memory?)");
-				return;
-			}
-
+			SharedPtr<Buf> nb(new Buf());
 			const int uncompressedLen = LZ4_decompress_safe(
 				reinterpret_cast<const char *>(pkt.b->b + ZT_PROTO_PACKET_PAYLOAD_START),
 				reinterpret_cast<char *>(nb->b),

node/VL2.cpp

@@ -31,43 +31,43 @@ VL2::~VL2()
 {
 }
 
-bool VL2::onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len)
+void VL2::onLocalEthernet(void *const tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,const unsigned int etherType,unsigned int vlanId,SharedPtr<Buf> &data,unsigned int len)
 {
 }
 
-bool VL2::_FRAME(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_FRAME(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_EXT_FRAME(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_EXT_FRAME(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_MULTICAST_LIKE(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_MULTICAST_LIKE(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_NETWORK_CREDENTIALS(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_NETWORK_CREDENTIALS(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_NETWORK_CONFIG_REQUEST(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_NETWORK_CONFIG_REQUEST(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_NETWORK_CONFIG(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_NETWORK_CONFIG(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_MULTICAST_GATHER(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_MULTICAST_GATHER(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_MULTICAST_FRAME_deprecated(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_MULTICAST_FRAME_deprecated(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 
-bool VL2::_MULTICAST(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
+bool VL2::_MULTICAST(void *const tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize)
 {
 }
 

node/VL2.hpp

@@ -51,7 +51,7 @@ public:
 	 * @param data Ethernet payload
 	 * @param len Frame length
 	 */
-	void onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,const void *data,unsigned int len);
+	void onLocalEthernet(void *tPtr,const SharedPtr<Network> &network,const MAC &from,const MAC &to,unsigned int etherType,unsigned int vlanId,SharedPtr<Buf> &data,unsigned int len);
 
 protected:
 	bool _FRAME(void *tPtr,const SharedPtr<Path> &path,SharedPtr<Peer> &peer,Buf &pkt,int packetSize);