Implement AES-GMAC-SIV and benchmark, rework COM and add a lot of comments and docs, and comments and docs elsewhere too.

node/AES.hpp

@@ -118,11 +118,15 @@ public:
 		_decryptSW(reinterpret_cast<const uint8_t *>(in),reinterpret_cast<uint8_t *>(out));
 	}
 
+	class GMACSIVEncryptor;
+
 	/**
 	 * Streaming GMAC calculator
 	 */
 	class GMAC
 	{
+		friend class GMACSIVEncryptor;
+
 	public:
 		/**
 		 * Create a new instance of GMAC (must be initialized with init() before use)
@@ -190,6 +194,8 @@ public:
 	 */
 	class CTR
 	{
+		friend class GMACSIVEncryptor;
+
 	public:
 		ZT_INLINE CTR(const AES &aes) noexcept : _aes(aes) {}
 
@@ -229,6 +235,76 @@ public:
 		unsigned int _len;
 	};
 
+	/**
+	 * Encrypt with AES-GMAC-SIV
+	 */
+	class GMACSIVEncryptor
+	{
+	public:
+		/**
+		 * Create a new AES-GMAC-SIV encryptor keyed with the provided AES instances
+		 *
+		 * @param k0 First of two AES instances keyed with K0
+		 * @param k1 Second of two AES instances keyed with K1
+		 */
+		ZT_INLINE GMACSIVEncryptor(const AES &k0,const AES &k1) noexcept :
+			_gmac(k0),
+			_ctr(k1) {}
+
+		/*
+		 * Initialize AES-GMAC-SIV
+		 *
+		 * @param iv IV in network byte order (byte order in which it will appear on the wire)
+		 * @param output Pointer to buffer to receive ciphertext, must be large enough for all to-be-processed data!
+		 */
+		ZT_INLINE void init(const uint64_t iv,void *const output) noexcept
+		{
+			_output = output;
+			_iv[0] = iv;
+			_iv[1] = 0;
+			_gmac.init(reinterpret_cast<const uint8_t *>(_iv));
+		}
+
+		ZT_INLINE void update1(const void *const input,const unsigned int len) noexcept
+		{
+			_gmac.update(input,len);
+		}
+
+		/**
+		 * Finish first pass, compute CTR IV, initialize second pass.
+		 */
+		ZT_INLINE void finish1() noexcept
+		{
+			uint64_t gmacTag[2];
+			_gmac.finish(reinterpret_cast<uint8_t *>(gmacTag));
+			_iv[1] = gmacTag[0];
+			_ctr._aes.encrypt(_iv,_iv);
+			_ctr.init(reinterpret_cast<const uint8_t *>(_iv),_output);
+		}
+
+		ZT_INLINE void update2(const void *const input,const unsigned int len) noexcept
+		{
+			_ctr.crypt(input,len);
+		}
+
+		/**
+		 * Finish second pass and return a pointer to the opaque 128-bit IV+MAC block
+		 *
+		 * @return Pointer to 128-bit opaque IV+MAC
+		 */
+		ZT_INLINE const uint8_t *finish2()
+		{
+			_ctr.finish();
+			return reinterpret_cast<const uint8_t *>(_iv);
+		}
+
+	private:
+		void *_output;
+		uint64_t _iv[2];
+		AES::GMAC _gmac;
+		AES::CTR _ctr;
+	};
+
 private:
 	static const uint32_t Te0[256];
 	static const uint32_t Te1[256];

node/CertificateOfMembership.cpp

@@ -15,167 +15,267 @@
 
 namespace ZeroTier {
 
-CertificateOfMembership::CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Address &issuedTo)
-{
-	_qualifiers[COM_RESERVED_ID_TIMESTAMP].id = COM_RESERVED_ID_TIMESTAMP;
-	_qualifiers[COM_RESERVED_ID_TIMESTAMP].value = timestamp;
-	_qualifiers[COM_RESERVED_ID_TIMESTAMP].maxDelta = timestampMaxDelta;
-	_qualifiers[COM_RESERVED_ID_NETWORK_ID].id = COM_RESERVED_ID_NETWORK_ID;
-	_qualifiers[COM_RESERVED_ID_NETWORK_ID].value = nwid;
-	_qualifiers[COM_RESERVED_ID_NETWORK_ID].maxDelta = 0;
-	_qualifiers[COM_RESERVED_ID_ISSUED_TO].id = COM_RESERVED_ID_ISSUED_TO;
-	_qualifiers[COM_RESERVED_ID_ISSUED_TO].value = issuedTo.toInt();
-	_qualifiers[COM_RESERVED_ID_ISSUED_TO].maxDelta = 0xffffffffffffffffULL;
-	_qualifierCount = 3;
-	_signatureLength = 0;
-}
+CertificateOfMembership::CertificateOfMembership(const int64_t timestamp,const int64_t timestampMaxDelta,const uint64_t nwid,const Identity &issuedTo) noexcept :
+	_timestamp(timestamp),
+	_timestampMaxDelta(timestampMaxDelta),
+	_networkId(nwid),
+	_issuedTo(issuedTo.fingerprint()),
+	_signatureLength(0) {}
 
-void CertificateOfMembership::setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta)
+bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other) const noexcept
 {
-	_signedBy.zero();
-	for(unsigned int i=0;i<_qualifierCount;++i) {
-		if (_qualifiers[i].id == id) {
-			_qualifiers[i].value = value;
-			_qualifiers[i].maxDelta = maxDelta;
-			return;
-		}
-	}
-	if (_qualifierCount < ZT_NETWORK_COM_MAX_QUALIFIERS) {
-		_qualifiers[_qualifierCount].id = id;
-		_qualifiers[_qualifierCount].value = value;
-		_qualifiers[_qualifierCount].maxDelta = maxDelta;
-		++_qualifierCount;
-		std::sort(&(_qualifiers[0]),&(_qualifiers[_qualifierCount]));
-	}
-}
+	// NOTE: we always do explicit absolute value with an if() since llabs() can have overflow
+	// conditions that could introduce a vulnerability.
 
-bool CertificateOfMembership::agreesWith(const CertificateOfMembership &other) const
-{
-	unsigned int myidx = 0;
-	unsigned int otheridx = 0;
-
-	if ((_qualifierCount == 0)||(other._qualifierCount == 0))
-		return false;
-
-	while (myidx < _qualifierCount) {
-		// Fail if we're at the end of other, since this means the field is
-		// missing.
-		if (otheridx >= other._qualifierCount)
+	if (other._timestamp > _timestamp) {
+		if ((other._timestamp - _timestamp) > _timestampMaxDelta)
+			return false;
+	} else {
+		if ((_timestamp - other._timestamp) > _timestampMaxDelta)
 			return false;
+	}
 
-		// Seek to corresponding tuple in other, ignoring tuples that
-		// we may not have. If we run off the end of other, the tuple is
-		// missing. This works because tuples are sorted by ID.
-		while (other._qualifiers[otheridx].id != _qualifiers[myidx].id) {
-			++otheridx;
-			if (otheridx >= other._qualifierCount)
+	// us <> them
+	for(FCV<_Qualifier,ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS>::const_iterator i(_additionalQualifiers.begin());i != _additionalQualifiers.end();++i) {
+		if (i->delta != 0xffffffffffffffffULL) {
+			const uint64_t *v2 = nullptr;
+			for(FCV<_Qualifier,ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS>::const_iterator j(other._additionalQualifiers.begin());j != other._additionalQualifiers.end();++i) {
+				if (j->id == i->id) {
+					v2 = &(j->value);
+					break;
+				}
+			}
+			if (!v2)
 				return false;
+			if (*v2 > i->value) {
+				if ((*v2 - i->value) > i->delta)
+					return false;
+			} else {
+				if ((i->value - *v2) > i->delta)
+					return false;
+			}
 		}
+	}
 
-		// Compare to determine if the absolute value of the difference
-		// between these two parameters is within our maxDelta.
-		const uint64_t a = _qualifiers[myidx].value;
-		const uint64_t b = other._qualifiers[myidx].value;
-		if (((a >= b) ? (a - b) : (b - a)) > _qualifiers[myidx].maxDelta)
-			return false;
-
-		++myidx;
+	// them <> us
+	for(FCV<_Qualifier,ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS>::const_iterator i(other._additionalQualifiers.begin());i != other._additionalQualifiers.end();++i) {
+		if (i->delta != 0xffffffffffffffffULL) {
+			const uint64_t *v2 = nullptr;
+			for(FCV<_Qualifier,ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS>::const_iterator j(_additionalQualifiers.begin());j != _additionalQualifiers.end();++i) {
+				if (j->id == i->id) {
+					v2 = &(j->value);
+					break;
+				}
+			}
+			if (!v2)
+				return false;
+			if (*v2 > i->value) {
+				if ((*v2 - i->value) > i->delta)
+					return false;
+			} else {
+				if ((i->value - *v2) > i->delta)
+					return false;
+			}
+		}
 	}
 
-	return true;
+	// SECURITY: check for issued-to inequality is a sanity check. This should be impossible since elsewhere
+	// in the code COMs are checked to ensure that they do in fact belong to their issued-to identities.
+	return (other._networkId != _networkId) && (other._issuedTo.address() != _issuedTo.address());
 }
 
-bool CertificateOfMembership::sign(const Identity &with)
+bool CertificateOfMembership::sign(const Identity &with) noexcept
 {
-	uint64_t buf[ZT_NETWORK_COM_MAX_QUALIFIERS * 3];
-	unsigned int ptr = 0;
-	for(unsigned int i=0;i<_qualifierCount;++i) {
-		buf[ptr++] = Utils::hton(_qualifiers[i].id);
-		buf[ptr++] = Utils::hton(_qualifiers[i].value);
-		buf[ptr++] = Utils::hton(_qualifiers[i].maxDelta);
-	}
-
-	try {
-		_signatureLength = with.sign(buf,ptr * sizeof(uint64_t),_signature,sizeof(_signature));
-		_signedBy = with.address();
-		return true;
-	} catch ( ... ) {
-		_signedBy.zero();
-		return false;
-	}
+	_signedBy = with.address();
+	uint64_t buf[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX / 8];
+	const unsigned int bufSize = _fillSigningBuf(buf);
+	_signatureLength = with.sign(buf,bufSize,_signature,sizeof(_signature));
+	return _signatureLength > 0;
 }
 
-int CertificateOfMembership::marshal(uint8_t data[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX]) const noexcept
+int CertificateOfMembership::marshal(uint8_t data[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX],const bool v2) const noexcept
 {
-	data[0] = 1; // 96-byte signature length; a v2 is supported in unmarshal code where signature is length prefixed
-	Utils::storeBigEndian<uint16_t>(data + 1,(uint16_t)_qualifierCount);
+	data[0] = v2 ? 2 : 1;
+
+	// All formats start with the standard three qualifiers: timestamp with delta, network ID as a strict
+	// equality compare, and the address of the issued-to node as an informational tuple.
 	int p = 3;
-	for(unsigned int i=0;i<_qualifierCount;++i) {
-		Utils::storeBigEndian<uint64_t>(data + p,_qualifiers[i].id); p += 8;
-		Utils::storeBigEndian<uint64_t>(data + p,_qualifiers[i].value); p += 8;
-		Utils::storeBigEndian<uint64_t>(data + p,_qualifiers[i].maxDelta); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,0); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,(uint64_t)_timestamp); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,(uint64_t)_timestampMaxDelta); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,1); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,_networkId); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,0); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,2); p += 8;
+	Utils::storeBigEndian<uint64_t>(data + p,_issuedTo.address().toInt()); p += 8;
+	Utils::storeAsIsEndian<uint64_t>(data + p,0xffffffffffffffffULL); p += 8;
+
+	if (v2) {
+		// V2 marshal format will have three tuples followed by the fingerprint hash.
+		Utils::storeBigEndian<uint16_t>(data + 1,3);
+		memcpy(data + p,_issuedTo.hash(),48);
+		p += 48;
+	} else {
+		// V1 marshal format must shove everything into tuples, resulting in nine.
+		Utils::storeBigEndian<uint16_t>(data + 1,9);
+		for(int k=0;k<6;++k) {
+			Utils::storeBigEndian<uint64_t>(data + p,(uint64_t)k + 3); p += 8;
+			Utils::storeAsIsEndian<uint64_t>(data + p,Utils::loadAsIsEndian<uint64_t>(_issuedTo.hash() + (k * 8))); p += 8;
+			Utils::storeAsIsEndian<uint64_t>(data + p,0xffffffffffffffffULL); p += 8;
+		}
 	}
-	_signedBy.copyTo(data + p); p += ZT_ADDRESS_LENGTH;
-	if ((_signedBy)&&(_signatureLength == 96)) {
-		memcpy(data + p,_signature,96); p += 96;
+
+	_signedBy.copyTo(data + p); p += 5;
+
+	if (v2) {
+		// V2 marshal format prefixes signatures with a 16-bit length to support future signature types.
+		Utils::storeBigEndian<uint16_t>(data + p,(uint16_t)_signatureLength); p += 2;
+		memcpy(data + p,_signature,_signatureLength);
+		p += (int)_signatureLength;
+	} else {
+		// V1 only supports 96-byte signature fields.
+		memcpy(data + p,_signature,96);
+		p += 96;
 	}
+
 	return p;
 }
 
 int CertificateOfMembership::unmarshal(const uint8_t *data,int len) noexcept
 {
-	if ((len < 3)||(data[0] == 0))
+	if (len < (1 + 2 + 72))
 		return -1;
-	unsigned int numq = Utils::loadBigEndian<uint16_t>(data + 1);
-	if (numq > ZT_NETWORK_COM_MAX_QUALIFIERS)
+
+	TriviallyCopyable::memoryZero(this);
+
+	const unsigned int numq = Utils::loadBigEndian<uint16_t>(data + 1);
+	if ((numq < 3)||(numq > (ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS + 3)))
 		return -1;
-	_qualifierCount = numq;
 	int p = 3;
-	for(unsigned int i=0;i<numq;++i) {
+	for(unsigned int q=0;q<numq;++q) {
 		if ((p + 24) > len)
 			return -1;
-		_qualifiers[i].id = Utils::loadBigEndian<uint64_t>(data + p); p += 8;
-		_qualifiers[i].value = Utils::loadBigEndian<uint64_t>(data + p); p += 8;
-		_qualifiers[i].maxDelta = Utils::loadBigEndian<uint64_t>(data + p); p += 8;
-	}
-	if ((p + ZT_ADDRESS_LENGTH) > len)
-		return -1;
-	_signedBy.setTo(data + p); p += ZT_ADDRESS_LENGTH;
-	if (_signedBy) {
-		_signatureLength = 96;
-		if (data[0] > 1) {
-			// If the version byte is >1, signatures come prefixed by a length. This is the
-			// way it should have been in the first place. Version byte 1 indicates 96 byte
-			// signatures and is backward compatible with <2.x nodes.
-			if ((p + 2) >= len)
-				return -1;
-			_signatureLength = Utils::loadBigEndian<uint16_t>(data + p); p += 2;
-			if (_signatureLength == 0)
-				return -1;
+		const uint64_t id = Utils::loadBigEndian<uint64_t>(data + p); p += 8;
+		const uint64_t value = Utils::loadBigEndian<uint64_t>(data + p); p += 8;
+		const uint64_t delta = Utils::loadBigEndian<uint64_t>(data + p); p += 8;
+		switch(id) {
+			case 0:
+				_timestamp = (int64_t)value;
+				_timestampMaxDelta = (int64_t)delta;
+				break;
+			case 1:
+				_networkId = value;
+				break;
+			case 2:
+				_issuedTo.apiFingerprint()->address = value;
+				break;
+
+				// V1 nodes will pack the hash into qualifier tuples.
+			case 3:
+				Utils::storeBigEndian<uint64_t>(_issuedTo.apiFingerprint()->hash,value);
+				break;
+			case 4:
+				Utils::storeBigEndian<uint64_t>(_issuedTo.apiFingerprint()->hash + 8,value);
+				break;
+			case 5:
+				Utils::storeBigEndian<uint64_t>(_issuedTo.apiFingerprint()->hash + 16,value);
+				break;
+			case 6:
+				Utils::storeBigEndian<uint64_t>(_issuedTo.apiFingerprint()->hash + 24,value);
+				break;
+			case 7:
+				Utils::storeBigEndian<uint64_t>(_issuedTo.apiFingerprint()->hash + 32,value);
+				break;
+			case 8:
+				Utils::storeBigEndian<uint64_t>(_issuedTo.apiFingerprint()->hash + 40,value);
+				break;
+
+			default:
+				if (_additionalQualifiers.size() == _additionalQualifiers.capacity())
+					return -1;
+				_additionalQualifiers.push_back(_Qualifier(id,value,delta));
+				break;
 		}
-		if ((int)(p + _signatureLength) > len)
+	}
+
+	std::sort(_additionalQualifiers.begin(),_additionalQualifiers.end());
+
+	if (data[0] == 1) {
+		if ((p + 96) > len)
 			return -1;
+		_signatureLength = 96;
 		memcpy(_signature,data + p,96);
-		p += 96;
+		return p + 96;
+	} else if (data[0] == 2) {
+		if ((p + 48) > len)
+			return -1;
+		memcpy(_issuedTo.apiFingerprint()->hash,data + p,48);
+		p += 48;
+		if ((p + 2) > len)
+			return -1;
+		_signatureLength = Utils::loadBigEndian<uint16_t>(data + p);
+		if ((_signatureLength > sizeof(_signature))||((p + _signatureLength) > len))
+			return -1;
+		memcpy(_signature,data + p,_signatureLength);
+		return p + (int)_signatureLength;
 	}
-	return p;
+
+	return -1;
 }
 
-bool CertificateOfMembership::operator==(const CertificateOfMembership &c) const
+unsigned int CertificateOfMembership::_fillSigningBuf(uint64_t buf[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX / 8]) const noexcept
 {
-	if (_signedBy != c._signedBy)
-		return false;
-	if (_qualifierCount != c._qualifierCount)
-		return false;
-	if (_signatureLength != c._signatureLength)
-		return false;
-	for(unsigned int i=0;i<_qualifierCount;++i) {
-		const _Qualifier &a = _qualifiers[i];
-		const _Qualifier &b = c._qualifiers[i];
-		if ((a.id != b.id)||(a.value != b.value)||(a.maxDelta != b.maxDelta))
-			return false;
+	const uint64_t informational = 0xffffffffffffffffULL;
+
+	/*
+	 * Signing always embeds all data to be signed in qualifier tuple format for
+	 * backward compatibility with V1 nodes, since otherwise we'd need a signature
+	 * for v1 nodes to verify and another for v2 nodes to verify.
+	 */
+
+	// The standard three tuples that must begin every COM.
+	buf[0] = 0;
+	buf[1] = Utils::hton((uint64_t)_timestamp);
+	buf[2] = Utils::hton((uint64_t)_timestampMaxDelta);
+	buf[3] = ZT_CONST_TO_BE_UINT64(1);
+	buf[4] = Utils::hton(_networkId);
+	buf[5] = 0;
+	buf[6] = ZT_CONST_TO_BE_UINT64(2);
+	buf[7] = Utils::hton(_issuedTo.address().toInt());
+	buf[8] = informational;
+
+	unsigned int p = 9;
+
+	// The full identity fingerprint of the peer to whom the COM was issued,
+	// embeded as a series of informational tuples.
+	if (_issuedTo.haveHash()) {
+		buf[p++] = ZT_CONST_TO_BE_UINT64(3);
+		buf[p++] = Utils::loadAsIsEndian<uint64_t>(_issuedTo.hash());
+		buf[p++] = informational;
+		buf[p++] = ZT_CONST_TO_BE_UINT64(4);
+		buf[p++] = Utils::loadAsIsEndian<uint64_t>(_issuedTo.hash() + 8);
+		buf[p++] = informational;
+		buf[p++] = ZT_CONST_TO_BE_UINT64(5);
+		buf[p++] = Utils::loadAsIsEndian<uint64_t>(_issuedTo.hash() + 16);
+		buf[p++] = informational;
+		buf[p++] = ZT_CONST_TO_BE_UINT64(6);
+		buf[p++] = Utils::loadAsIsEndian<uint64_t>(_issuedTo.hash() + 24);
+		buf[p++] = informational;
+		buf[p++] = ZT_CONST_TO_BE_UINT64(7);
+		buf[p++] = Utils::loadAsIsEndian<uint64_t>(_issuedTo.hash() + 32);
+		buf[p++] = informational;
+		buf[p++] = ZT_CONST_TO_BE_UINT64(8);
+		buf[p++] = Utils::loadAsIsEndian<uint64_t>(_issuedTo.hash() + 40);
+		buf[p++] = informational;
 	}
-	return (memcmp(_signature,c._signature,_signatureLength) == 0);
+
+	for(FCV<_Qualifier,ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS>::const_iterator i(_additionalQualifiers.begin());i != _additionalQualifiers.end();++i) {
+		buf[p++] = Utils::hton(i->id);
+		buf[p++] = Utils::hton(i->value);
+		buf[p++] = Utils::hton(i->delta);
+	}
+
+	return p * 8;
 }
 
 } // namespace ZeroTier

node/CertificateOfMembership.hpp

@@ -14,6 +14,8 @@
 #ifndef ZT_CERTIFICATEOFMEMBERSHIP_HPP
 #define ZT_CERTIFICATEOFMEMBERSHIP_HPP
 
+// TODO: redo
+
 #include <cstdint>
 #include <cstring>
 
@@ -27,13 +29,13 @@
 #include "C25519.hpp"
 #include "Identity.hpp"
 #include "Utils.hpp"
+#include "FCV.hpp"
 
-/**
- * Maximum number of qualifiers allowed in a COM (absolute max: 65535)
- */
-#define ZT_NETWORK_COM_MAX_QUALIFIERS 8
+// Maximum number of additional tuples beyond the standard always-present three.
+#define ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS 8
 
-#define ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX (1 + 2 + (24 * ZT_NETWORK_COM_MAX_QUALIFIERS) + 5 + ZT_SIGNATURE_BUFFER_SIZE)
+// version + qualifier count + three required qualifiers + additional qualifiers +
+#define ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX (1 + 2 + (3 * 3 * 8) + (ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS * 3 * 8) + 144 + 5 + 2 + 96)
 
 namespace ZeroTier {
 
@@ -42,28 +44,62 @@ class RuntimeEnvironment;
 /**
  * Certificate of network membership
  *
- * The COM contains a sorted set of three-element tuples called qualifiers.
- * These contain an id, a value, and a maximum delta.
+ * This is the fundamental permission object issued by network controllers to members of networks
+ * to admit them into networks.
+ *
+ * A certificate of membership (COM) consists of a series of tuples called qualifiers as well
+ * as the full identity fingerprint of the node being admitted, the address of the controller
+ * (for sanity checking), and a signature.
+ *
+ * A qualifier is a tuple of three 64-bit unsigned integers: an id, a value, and a delta.
+ *
+ * Certiciates are checked between peers by determining if they agree. If the absolute value
+ * of the difference between any two qualifier values exceeds its delta, the certificates do
+ * not agree. A delta if 1 for example means that the values of two peers may differ by no more
+ * than one. A delta of 0 indicates values that must be the same. A delta of uint64_max is for
+ * informational tuples that are not included in certificate checking, as this means they may
+ * differ by any amount.
+ *
+ * All COMs contain three initial tuples: timestamp, network ID, and the address of the
+ * issued-to node. The latter is informational. The network ID must equal exactly, though in
+ * theory a controller could allow a delta there to e.g. allow cross-communication between all
+ * of its networks. (This has never been done in practice.) The most important field is the
+ * timestamp, whose delta defines a moving window within which certificates must be timestamped
+ * by the network controller to agree. A certificate that is too old will fall out of this
+ * window vs its peers and will no longer be considered valid.
+ *
+ * (Revocations are a method to rapidly revoke access that works alongside this slower but
+ * more definitive method.)
  *
- * The ID is arbitrary and should be assigned using a scheme that makes
- * every ID globally unique. IDs beneath 65536 are reserved for global
- * assignment by ZeroTier Networks.
+ * Certificate of membership wire format:
  *
- * The value's meaning is ID-specific and isn't important here. What's
- * important is the value and the third member of the tuple: the maximum
- * delta. The maximum delta is the maximum difference permitted between
- * values for a given ID between certificates for the two certificates to
- * themselves agree.
+ * This wire format comes in two versions: version 1 for ZeroTier 1.x, which will
+ * eventually go away once 1.x is out of support, and version 2 for ZeroTier 2.x and later.
  *
- * Network membership is checked by checking whether a peer's certificate
- * agrees with your own. The timestamp provides the fundamental criterion--
- * each member of a private network must constantly obtain new certificates
- * often enough to stay within the max delta for this qualifier. But other
- * criteria could be added in the future for very special behaviors, things
- * like latitude and longitude for instance.
+ * Version 2:
  *
- * This is a memcpy()'able structure and is safe (in a crash sense) to modify
- * without locks.
+ * <[1] wire format type byte: 1 or 2>
+ * <[2] 16-bit number of qualifier tuples>
+ * <[...] qualifier tuples>
+ * <[48] fingerprint hash of identity of peer to whom COM was issued>
+ * <[5] address of network controller>
+ * <[2] 16-bit size of signature>
+ * <[...] signature>
+ *
+ * Version 1 is identical except the fingerprint hash is omitted and is instead loaded
+ * into a series of six informational tuples. The signature size is also omitted and a
+ * 96-byte signature field is assumed.
+ *
+ * Qualifier tuples must appear in numeric order of ID, and the first three tuples
+ * must have IDs 0, 1, and 2 being the timestamp, network ID, and issued-to address
+ * respectively. In version 1 COMs the IDs 3-8 are used to pack in the full identity
+ * fingerprint, so these are reserved as well. Optional additional tuples (not currently
+ * used) must use ID 65536 or higher.
+ *
+ * Signatures are computed over tuples only for backward compatibility with v1, and we
+ * don't plan to change this. Tuples are emitted into a buffer in ascending numeric
+ * order with the fingerprint hash being packed into tuple IDs 3-8 and this buffer is
+ * then signed.
  */
 class CertificateOfMembership : public Credential
 {
@@ -72,33 +108,6 @@ class CertificateOfMembership : public Credential
 public:
 	static constexpr ZT_CredentialType credentialType() noexcept { return ZT_CREDENTIAL_TYPE_COM; }
 
-	/**
-	 * Reserved qualifier IDs
-	 *
-	 * IDs below 1024 are reserved for use as standard IDs. Others are available
-	 * for user-defined use.
-	 *
-	 * Addition of new required fields requires that code in hasRequiredFields
-	 * be updated as well.
-	 */
-	enum ReservedId
-	{
-		/**
-		 * Timestamp of certificate
-		 */
-		COM_RESERVED_ID_TIMESTAMP = 0,
-
-		/**
-		 * Network ID for which certificate was issued
-		 */
-		COM_RESERVED_ID_NETWORK_ID = 1,
-
-		/**
-		 * ZeroTier address to whom certificate was issued
-		 */
-		COM_RESERVED_ID_ISSUED_TO = 2
-	};
-
 	/**
 	 * Create an empty certificate of membership
 	 */
@@ -112,12 +121,12 @@ public:
 	 * @param nwid Network ID
 	 * @param issuedTo Certificate recipient
 	 */
-	CertificateOfMembership(uint64_t timestamp,uint64_t timestampMaxDelta,uint64_t nwid,const Address &issuedTo);
+	CertificateOfMembership(int64_t timestamp,int64_t timestampMaxDelta,uint64_t nwid,const Identity &issuedTo) noexcept;
 
 	/**
 	 * @return True if there's something here
 	 */
-	ZT_INLINE operator bool() const noexcept { return (_qualifierCount != 0); }
+	ZT_INLINE operator bool() const noexcept { return (_networkId != 0); }
 
 	/**
 	 * @return Credential ID, always 0 for COMs
@@ -127,57 +136,17 @@ public:
 	/**
 	 * @return Timestamp for this cert and maximum delta for timestamp
 	 */
-	ZT_INLINE int64_t timestamp() const noexcept
-	{
-		if (_qualifiers[COM_RESERVED_ID_TIMESTAMP].id == COM_RESERVED_ID_TIMESTAMP)
-			return (int64_t)_qualifiers[0].value;
-		for(unsigned int i=0;i<_qualifierCount;++i) {
-			if (_qualifiers[i].id == COM_RESERVED_ID_TIMESTAMP)
-				return (int64_t)_qualifiers[i].value;
-		}
-		return 0;
-	}
+	ZT_INLINE int64_t timestamp() const noexcept { return _timestamp; }
 
 	/**
-	 * @return Address to which this cert was issued
+	 * @return Fingerprint of identity to which this cert was issued
 	 */
-	ZT_INLINE Address issuedTo() const noexcept
-	{
-		if (_qualifiers[COM_RESERVED_ID_ISSUED_TO].id == COM_RESERVED_ID_ISSUED_TO)
-			return Address(_qualifiers[2].value);
-		for(unsigned int i=0;i<_qualifierCount;++i) {
-			if (_qualifiers[i].id == COM_RESERVED_ID_ISSUED_TO)
-				return Address(_qualifiers[i].value);
-		}
-		return Address();
-	}
+	ZT_INLINE const Fingerprint &issuedTo() const noexcept { return _issuedTo; }
 
 	/**
 	 * @return Network ID for which this cert was issued
 	 */
-	ZT_INLINE uint64_t networkId() const noexcept
-	{
-		if (_qualifiers[COM_RESERVED_ID_NETWORK_ID].id == COM_RESERVED_ID_NETWORK_ID)
-			return _qualifiers[COM_RESERVED_ID_NETWORK_ID].value;
-		for(unsigned int i=0;i<_qualifierCount;++i) {
-			if (_qualifiers[i].id == COM_RESERVED_ID_NETWORK_ID)
-				return _qualifiers[i].value;
-		}
-		return 0ULL;
-	}
-
-	/**
-	 * Add or update a qualifier in this certificate
-	 *
-	 * Any signature is invalidated and signedBy is set to null.
-	 *
-	 * @param id Qualifier ID
-	 * @param value Qualifier value
-	 * @param maxDelta Qualifier maximum allowed difference (absolute value of difference)
-	 */
-	void setQualifier(uint64_t id,uint64_t value,uint64_t maxDelta);
-
-	ZT_INLINE void setQualifier(ReservedId id,uint64_t value,uint64_t maxDelta) { setQualifier((uint64_t)id,value,maxDelta); }
+	ZT_INLINE uint64_t networkId() const noexcept { return _networkId; }
 
 	/**
 	 * Compare two certificates for parameter agreement
@@ -192,7 +161,7 @@ public:
 	 * @param other Cert to compare with
 	 * @return True if certs agree and 'other' may be communicated with
 	 */
-	bool agreesWith(const CertificateOfMembership &other) const;
+	bool agreesWith(const CertificateOfMembership &other) const noexcept;
 
 	/**
 	 * Sign this certificate
@@ -200,7 +169,7 @@ public:
 	 * @param with Identity to sign with, must include private key
 	 * @return True if signature was successful
 	 */
-	bool sign(const Identity &with);
+	bool sign(const Identity &with) noexcept;
 
 	/**
 	 * Verify this COM and its signature
@@ -210,31 +179,29 @@ public:
 	 */
 	ZT_INLINE Credential::VerifyResult verify(const RuntimeEnvironment *RR,void *tPtr) const { return _verify(RR,tPtr,*this); }
 
-	/**
-	 * @return Address that signed this certificate or null address if none
-	 */
-	ZT_INLINE const Address &signedBy() const noexcept { return _signedBy; }
-
 	static constexpr int marshalSizeMax() noexcept { return ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX; }
-	int marshal(uint8_t data[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX]) const noexcept;
+	int marshal(uint8_t data[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX],bool v2) const noexcept;
 	int unmarshal(const uint8_t *data,int len) noexcept;
 
-	bool operator==(const CertificateOfMembership &c) const;
-	ZT_INLINE bool operator!=(const CertificateOfMembership &c) const { return (!(*this == c)); }
-
 private:
+	unsigned int _fillSigningBuf(uint64_t buf[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX / 8]) const noexcept;
+
 	struct _Qualifier
 	{
-		ZT_INLINE _Qualifier() noexcept : id(0),value(0),maxDelta(0) {}
+		ZT_INLINE _Qualifier() noexcept : id(0),value(0),delta(0) {}
+		ZT_INLINE _Qualifier(const uint64_t id_,const uint64_t value_,const uint64_t delta_) noexcept : id(id_),value(value_),delta(delta_) {}
 		uint64_t id;
 		uint64_t value;
-		uint64_t maxDelta;
+		uint64_t delta;
 		ZT_INLINE bool operator<(const _Qualifier &q) const noexcept { return (id < q.id); } // sort order
 	};
 
+	FCV<_Qualifier,ZT_CERTIFICATEOFMEMBERSHIP_MAX_ADDITIONAL_QUALIFIERS> _additionalQualifiers;
+	int64_t _timestamp;
+	int64_t _timestampMaxDelta;
+	uint64_t _networkId;
+	Fingerprint _issuedTo;
 	Address _signedBy;
-	_Qualifier _qualifiers[ZT_NETWORK_COM_MAX_QUALIFIERS];
-	unsigned int _qualifierCount;
 	unsigned int _signatureLength;
 	uint8_t _signature[ZT_SIGNATURE_BUFFER_SIZE];
 };

node/Credential.cpp

@@ -72,22 +72,19 @@ Credential::VerifyResult Credential::_verify(const RuntimeEnvironment *const RR,
 
 Credential::VerifyResult Credential::_verify(const RuntimeEnvironment *const RR,void *tPtr,const CertificateOfMembership &credential) const
 {
-	if ((!credential._signedBy)||(credential._signedBy != Network::controllerFor(credential.networkId()))||(credential._qualifierCount > ZT_NETWORK_COM_MAX_QUALIFIERS))
+	// Sanity check network ID.
+	if ((!credential._signedBy)||(credential._signedBy != Network::controllerFor(credential._networkId)))
 		return Credential::VERIFY_BAD_SIGNATURE;
 
+	// If we don't know the peer, get its identity. This shouldn't happen here but should be handled.
 	const SharedPtr<Peer> peer(RR->topology->peer(tPtr,credential._signedBy));
 	if (!peer)
 		return Credential::VERIFY_NEED_IDENTITY;
 
-	uint64_t buf[ZT_NETWORK_COM_MAX_QUALIFIERS * 3];
-	unsigned int ptr = 0;
-	for(unsigned int i=0;i<credential._qualifierCount;++i) {
-		buf[ptr++] = Utils::hton(credential._qualifiers[i].id);
-		buf[ptr++] = Utils::hton(credential._qualifiers[i].value);
-		buf[ptr++] = Utils::hton(credential._qualifiers[i].maxDelta);
-	}
-
-	return (peer->identity().verify(buf,ptr * sizeof(uint64_t),credential._signature,credential._signatureLength) ? Credential::VERIFY_OK : Credential::VERIFY_BAD_SIGNATURE);
+	// Now verify the controller's signature.
+	uint64_t buf[ZT_CERTIFICATEOFMEMBERSHIP_MARSHAL_SIZE_MAX / 8];
+	const unsigned int bufSize = credential._fillSigningBuf(buf);
+	return peer->identity().verify(buf,bufSize,credential._signature,credential._signatureLength) ? Credential::VERIFY_OK : Credential::VERIFY_BAD_SIGNATURE;
 }
 
 Credential::VerifyResult Credential::_verify(const RuntimeEnvironment *RR,void *tPtr,const Capability &credential) const

node/Credential.hpp

@@ -36,6 +36,11 @@ class RuntimeEnvironment;
 
 /**
  * Base class for credentials
+ *
+ * Note that all credentials are and must be trivially copyable.
+ *
+ * All credential verification methods are implemented in Credential.cpp as they share a lot
+ * of common code and logic and grouping them makes auditing easier.
  */
 class Credential : public TriviallyCopyable
 {

node/Defragmenter.hpp

@@ -145,7 +145,7 @@ public:
 		const unsigned int fragmentNo,
 		const unsigned int totalFragmentsExpected,
 		const int64_t now,
-		const SharedPtr< Path > &via,
+		const SharedPtr<Path> &via,
 		const unsigned int maxIncomingFragmentsPerPath)
 	{
 		// Sanity checks for malformed fragments or invalid input parameters.

node/FCV.hpp

@@ -33,6 +33,9 @@ namespace ZeroTier {
  * This doesn't implement everything in std::vector, just what we need. It
  * also adds a few special things for use in ZT core code.
  *
+ * Note that an FCV will be TriviallyCopyable IF and only if its contained
+ * type is TriviallyCopyable. There's a const static checker for this.
+ *
  * @tparam T Type to contain
  * @tparam C Maximum capacity of vector
  */
@@ -43,6 +46,11 @@ public:
 	typedef T * iterator;
 	typedef const T * const_iterator;
 
+	/**
+	 * @return True if this FCV is trivially copyable, which means its type is also.
+	 */
+	static constexpr bool isTriviallyCopyable() noexcept { return isTriviallyCopyable(reinterpret_cast<const T *>(0)); }
+
 	ZT_INLINE FCV() noexcept : _s(0) {}
 	ZT_INLINE FCV(const FCV &v) : _s(0) { *this = v; }
 

node/Fingerprint.hpp

@@ -45,18 +45,13 @@ public:
 
 	ZT_INLINE Address address() const noexcept { return Address(_fp.address); }
 	ZT_INLINE const uint8_t *hash() const noexcept { return _fp.hash; }
+	ZT_INLINE ZT_Fingerprint *apiFingerprint() noexcept { return &_fp; }
+	ZT_INLINE const ZT_Fingerprint *apiFingerprint() const noexcept { return &_fp; }
 
 	/**
-	 * Copy into ZT_Fingerprint struct as used in API and trace messages
-	 *
-	 * @param fp ZT_Fingerprint
-	 */
-	ZT_INLINE void getAPIFingerprint(ZT_Fingerprint *fp) const noexcept { memcpy(fp,&_fp,sizeof(ZT_Fingerprint)); }
-
-	/**
-	 * @return Pointer to ZT_Fingerprint for API use
+	 * @return True if hash is not all zero (missing/unspecified)
 	 */
-	ZT_INLINE const ZT_Fingerprint *apiFingerprint() const noexcept { return &_fp; }
+	ZT_INLINE bool haveHash() const noexcept { return (!Utils::allZero(_fp.hash,sizeof(_fp.hash))); }
 
 	/**
 	 * Get a base32-encoded representation of this fingerprint

node/Protocol.hpp

@@ -23,6 +23,8 @@
 #include "Address.hpp"
 #include "Identity.hpp"
 
+// TODO: mlock
+
 /*
  * Core ZeroTier protocol packet formats ------------------------------------------------------------------------------
  *

node/Tests.cpp

@@ -1039,6 +1039,24 @@ extern "C" const char *ZTT_benchmarkCrypto()
 			ZT_T_PRINTF("%.4f MiB/sec" ZT_EOL_S,((16384.0 * 350000.0) / 1048576.0) / ((double)(end - start) / 1000.0));
 		}
 
+		{
+			ZT_T_PRINTF("[crypto] Benchmarking AES-GMAC-SIV... ");
+			AES k0(AES_CTR_TEST_VECTOR_0_KEY);
+			AES k1(AES_GMAC_VECTOR_0_KEY);
+			AES::GMACSIVEncryptor enc(k0,k1);
+			int64_t start = now();
+			for(long i=0;i<350000;++i) {
+				enc.init((uint64_t)i,tmp);
+				enc.update1(tmp,sizeof(tmp));
+				enc.finish1();
+				enc.update2(tmp,sizeof(tmp));
+				enc.finish2();
+			}
+			int64_t end = now();
+			foo = tmp[0]; // prevent optimization
+			ZT_T_PRINTF("%.4f MiB/sec" ZT_EOL_S,((16384.0 * 350000.0) / 1048576.0) / ((double)(end - start) / 1000.0));
+		}
+
 		{
 			ZT_T_PRINTF("[crypto] Benchmarking Poly1305... ");
 			int64_t start = now();

node/TriviallyCopyable.hpp

@@ -166,6 +166,12 @@ ZT_PACKED_STRUCT(struct TriviallyCopyable
 	}
 });
 
+static constexpr bool isTriviallyCopyable(const TriviallyCopyable *const anything) noexcept { return true; }
+static constexpr bool isTriviallyCopyable(const void *const anything) noexcept { return false; }
+
+template<typename T>
+static constexpr bool isTriviallyCopyable(const T &anything) noexcept { return isTriviallyCopyable(&anything); }
+
 } // namespace ZeroTier
 
 #endif

node/Utils.hpp

@@ -16,16 +16,27 @@
 
 #include "Constants.hpp"
 
+namespace ZeroTier {
+
+namespace Utils {
+
+// Macros to convert endian-ness at compile time for constants.
 #if __BYTE_ORDER == __LITTLE_ENDIAN
 #define ZT_CONST_TO_BE_UINT16(x) ((uint16_t)((uint16_t)((uint16_t)(x) << 8U) | (uint16_t)((uint16_t)(x) >> 8U)))
+#define ZT_CONST_TO_BE_UINT64(x) ( \
+	(((uint64_t)(x) & 0x00000000000000ffULL) << 56U) | \
+	(((uint64_t)(x) & 0x000000000000ff00ULL) << 40U) | \
+	(((uint64_t)(x) & 0x0000000000ff0000ULL) << 24U) | \
+	(((uint64_t)(x) & 0x00000000ff000000ULL) <<  8U) | \
+	(((uint64_t)(x) & 0x000000ff00000000ULL) >>  8U) | \
+	(((uint64_t)(x) & 0x0000ff0000000000ULL) >> 24U) | \
+	(((uint64_t)(x) & 0x00ff000000000000ULL) >> 40U) | \
+	(((uint64_t)(x) & 0xff00000000000000ULL) >> 56U))
 #else
 #define ZT_CONST_TO_BE_UINT16(x) ((uint16_t)(x))
+#define ZT_CONST_TO_BE_UINT64(x) ((uint64_t)(x))
 #endif
 
-namespace ZeroTier {
-
-namespace Utils {
-
 #ifdef ZT_ARCH_X64
 struct CPUIDRegisters
 {