Certificates, and rename credentials to credential since they are not truly certificates (according to the common definition).

core/Address.hpp

@@ -44,19 +44,14 @@ public:
 	{}
 
 	ZT_INLINE Address &operator=(const uint64_t a) noexcept
-	{
-		_a = a;
-		return *this;
-	}
+	{ _a = a; return *this; }
 
 	/**
 	 * @param bits Raw address -- 5 bytes, big-endian byte order
 	 * @param len Length of array
 	 */
 	ZT_INLINE void setTo(const uint8_t b[5]) noexcept
-	{
-		_a = ((uint64_t)b[0] << 32U) | ((uint64_t)b[1] << 24U) | ((uint64_t)b[2] << 16U) | ((uint64_t)b[3] << 8U) | (uint64_t)b[4];
-	}
+	{ _a = ((uint64_t)b[0] << 32U) | ((uint64_t)b[1] << 24U) | ((uint64_t)b[2] << 16U) | ((uint64_t)b[3] << 8U) | (uint64_t)b[4]; }
 
 	/**
 	 * @param bits Buffer to hold 5-byte address in big-endian byte order

core/Blob.hpp

@@ -42,6 +42,9 @@ struct Blob
 	ZT_INLINE bool operator>=(const Blob &b) const noexcept { return (memcmp(data,b.data,S) >= 0); }
 };
 
+typedef Blob<48> SHA384Hash;
+typedef Blob<16> GUID;
+
 } // namespace ZeroTier
 
 #endif

core/CMakeLists.txt

@@ -94,13 +94,18 @@ set(core_src
 )
 
 add_library(${PROJECT_NAME} STATIC ${core_src} ${core_headers})
-target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_11)
 target_include_directories(${PROJECT_NAME} PRIVATE ${CMAKE_BINARY_DIR})
 
 if(WIN32)
 	set(libs ${libs} wsock32 ws2_32 rpcrt4 iphlpapi)
+	target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_17)
 else(WIN32)
 	set(libs ${libs} pthread)
+	if (APPLE)
+		target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_17)
+	else(APPLE)
+		target_compile_features(${PROJECT_NAME} PUBLIC cxx_std_11)
+	endif(APPLE)
 endif(WIN32)
 
 add_executable(zt_core_tests Tests.h Tests.cpp)

core/Certificate.cpp

@@ -13,6 +13,7 @@
 
 #include "Certificate.hpp"
 #include "SHA512.hpp"
+#include "ECC384.hpp"
 
 namespace ZeroTier {
 
@@ -47,6 +48,7 @@ Certificate &Certificate::operator=(const Certificate &cert)
 	this->subject.identityCount = 0;
 	this->subject.networkCount = 0;
 	this->subject.certificateCount = 0;
+	this->subject.updateUrlCount = 0;
 
 	for (unsigned int i = 0; i < cert.subject.identityCount; ++i) {
 		if (cert.subject.identities[i].identity) {
@@ -66,16 +68,16 @@ Certificate &Certificate::operator=(const Certificate &cert)
 			addSubjectCertificate(cert.subject.certificates[i]);
 	}
 
-	if (cert.issuer) {
-		m_identities.push_back(*reinterpret_cast<const Identity *>(cert.issuer));
-		this->issuer = reinterpret_cast<ZT_Identity *>(&(m_identities.back()));
+	if (cert.subject.updateUrls) {
+		for (unsigned int i = 0; i < cert.subject.updateUrlCount; ++i) {
+			if (cert.subject.updateUrls[i])
+				addUpdateUrl(cert.subject.updateUrls[i]);
+		}
 	}
 
-	if (cert.updateUrls) {
-		for (unsigned int i = 0; i < cert.updateUrlCount; ++i) {
-			if (cert.updateUrls[i])
-				addUpdateUrl(cert.updateUrls[i]);
-		}
+	if (cert.issuer) {
+		m_identities.push_back(*reinterpret_cast<const Identity *>(cert.issuer));
+		this->issuer = &(m_identities.back());
 	}
 
 	return *this;
@@ -91,7 +93,7 @@ ZT_Certificate_Identity *Certificate::addSubjectNode(const Identity &id)
 	m_identities.push_back(id);
 
 	// Set ZT_Certificate_Identity struct fields to point to local copy of identity.
-	m_subjectIdentities.back().identity = reinterpret_cast<ZT_Identity *>(&(m_identities.back()));
+	m_subjectIdentities.back().identity = &(m_identities.back());
 	m_subjectIdentities.back().locator = nullptr;
 
 	return &(m_subjectIdentities.back());
@@ -106,7 +108,7 @@ ZT_Certificate_Identity *Certificate::addSubjectNode(const Identity &id, const L
 	m_locators.push_back(loc);
 
 	// Set pointer to stored local copy of locator.
-	n->locator = reinterpret_cast<ZT_Locator *>(&(m_locators.back()));
+	n->locator = &(m_locators.back());
 
 	return n;
 }
@@ -127,7 +129,7 @@ ZT_Certificate_Network *Certificate::addSubjectNetwork(const uint64_t id, const
 void Certificate::addSubjectCertificate(const uint8_t serialNo[ZT_SHA384_DIGEST_SIZE])
 {
 	// Store local copy of serial in m_serials container.
-	m_serials.push_back(Blob< ZT_SHA384_DIGEST_SIZE >(serialNo));
+	m_serials.push_back(SHA384Hash(serialNo));
 
 	// Enlarge array of uint8_t pointers, set new pointer to local copy of serial, and set
 	// certificates to point to potentially reallocated array.
@@ -144,8 +146,8 @@ void Certificate::addUpdateUrl(const char *url)
 	// Add pointer to local copy to pointer array and update C structure to point to
 	// potentially reallocated array.
 	m_updateUrls.push_back(m_strings.back().c_str());
-	this->updateUrls = m_updateUrls.data();
-	this->updateUrlCount = (unsigned int)m_updateUrls.size();
+	this->subject.updateUrls = m_updateUrls.data();
+	this->subject.updateUrlCount = (unsigned int)m_updateUrls.size();
 }
 
 Vector< uint8_t > Certificate::encode(const bool omitSignature) const
@@ -159,42 +161,12 @@ Vector< uint8_t > Certificate::encode(const bool omitSignature) const
 	// and faster to marshal/unmarshal.
 
 	d.add("f", this->flags);
-	d.add("v0", this->validity[0]);
-	d.add("v1", this->validity[1]);
+	d.add("t", (uint64_t)this->timestamp);
+	d.add("v0", (uint64_t)this->validity[0]);
+	d.add("v1", (uint64_t)this->validity[1]);
 	d.add("mP", (uint64_t)this->maxPathLength);
 
-	d.add("s.i$", (uint64_t)this->subject.identityCount);
-	for (unsigned int i = 0; i < this->subject.identityCount; ++i) {
-		if (this->subject.identities[i].identity)
-			d.addO(Dictionary::arraySubscript(tmp, "s.i$.i", i), *reinterpret_cast<const Identity *>(this->subject.identities[i].identity));
-		if (this->subject.identities[i].locator)
-			d.addO(Dictionary::arraySubscript(tmp, "s.i$.l", i), *reinterpret_cast<const Locator *>(this->subject.identities[i].locator));
-	}
-
-	d.add("s.n$", (uint64_t)this->subject.networkCount);
-	for (unsigned int i = 0; i < this->subject.networkCount; ++i) {
-		d.add(Dictionary::arraySubscript(tmp, "s.n$.i", i), this->subject.networks[i].id);
-		Fingerprint fp(this->subject.networks[i].controller);
-		d.addO(Dictionary::arraySubscript(tmp, "s.n$.c", i), fp);
-	}
-
-	d.add("s.c$", (uint64_t)this->subject.certificateCount);
-	for (unsigned int i = 0; i < this->subject.certificateCount; ++i) {
-		if (this->subject.certificates[i])
-			d[Dictionary::arraySubscript(tmp, "s.c$", i)].assign(this->subject.certificates[i], this->subject.certificates[i] + ZT_SHA384_DIGEST_SIZE);
-	}
-
-	d.add("s.n.c", this->subject.name.country);
-	d.add("s.n.o", this->subject.name.organization);
-	d.add("s.n.u", this->subject.name.unit);
-	d.add("s.n.l", this->subject.name.locality);
-	d.add("s.n.p", this->subject.name.province);
-	d.add("s.n.sA", this->subject.name.streetAddress);
-	d.add("s.n.pC", this->subject.name.postalCode);
-	d.add("s.n.cN", this->subject.name.commonName);
-	d.add("s.n.sN", this->subject.name.serialNo);
-	d.add("s.n.e", this->subject.name.email);
-	d.add("s.n.ur", this->subject.name.url);
+	m_encodeSubject(d, false);
 
 	if (this->issuer)
 		d.addO("i", *reinterpret_cast<const Identity *>(this->issuer));
@@ -211,12 +183,6 @@ Vector< uint8_t > Certificate::encode(const bool omitSignature) const
 	d.add("iN.e", this->issuerName.email);
 	d.add("iN.ur", this->issuerName.url);
 
-	d.add("u$", (uint64_t)this->updateUrlCount);
-	if (this->updateUrls) {
-		for (unsigned int i = 0; i < this->updateUrlCount; ++i)
-			d.add(Dictionary::arraySubscript(tmp, "u$", i), this->updateUrls[i]);
-	}
-
 	if ((!omitSignature) && (this->signatureSize > 0) && (this->signatureSize <= sizeof(this->signature)))
 		d["si"].assign(this->signature, this->signature + this->signatureSize);
 
@@ -235,10 +201,13 @@ bool Certificate::decode(const Vector< uint8_t > &data)
 		return false;
 
 	this->flags = d.getUI("f");
+	this->timestamp = (int64_t)d.getUI("t");
 	this->validity[0] = (int64_t)d.getUI("v0");
 	this->validity[1] = (int64_t)d.getUI("v1");
 	this->maxPathLength = (unsigned int)d.getUI("mP");
 
+	this->subject.timestamp = (int64_t)d.getUI("s.t");
+
 	unsigned int cnt = (unsigned int)d.getUI("s.i$");
 	for (unsigned int i = 0; i < cnt; ++i) {
 		const Vector< uint8_t > &identityData = d[Dictionary::arraySubscript(tmp, "s.i$.i", i)];
@@ -271,13 +240,15 @@ bool Certificate::decode(const Vector< uint8_t > &data)
 	}
 
 	cnt = (unsigned int)d.getUI("s.c$");
-	for (unsigned int i=0;i<cnt;++i) {
+	for (unsigned int i = 0; i < cnt; ++i) {
 		const Vector< uint8_t > &serial = d[Dictionary::arraySubscript(tmp, "s.c$", i)];
 		if (serial.size() != ZT_SHA384_DIGEST_SIZE)
 			return false;
 		this->addSubjectCertificate(serial.data());
 	}
 
+	d.getS("s.n.sN", this->subject.name.serialNo, sizeof(this->subject.name.serialNo));
+	d.getS("s.n.cN", this->subject.name.commonName, sizeof(this->subject.name.commonName));
 	d.getS("s.n.c", this->subject.name.country, sizeof(this->subject.name.country));
 	d.getS("s.n.o", this->subject.name.organization, sizeof(this->subject.name.organization));
 	d.getS("s.n.u", this->subject.name.unit, sizeof(this->subject.name.unit));
@@ -285,8 +256,6 @@ bool Certificate::decode(const Vector< uint8_t > &data)
 	d.getS("s.n.p", this->subject.name.province, sizeof(this->subject.name.province));
 	d.getS("s.n.sA", this->subject.name.streetAddress, sizeof(this->subject.name.streetAddress));
 	d.getS("s.n.pC", this->subject.name.postalCode, sizeof(this->subject.name.postalCode));
-	d.getS("s.n.cN", this->subject.name.commonName, sizeof(this->subject.name.commonName));
-	d.getS("s.n.sN", this->subject.name.serialNo, sizeof(this->subject.name.serialNo));
 	d.getS("s.n.e", this->subject.name.email, sizeof(this->subject.name.email));
 	d.getS("s.n.ur", this->subject.name.url, sizeof(this->subject.name.url));
 
@@ -299,6 +268,8 @@ bool Certificate::decode(const Vector< uint8_t > &data)
 		}
 	}
 
+	d.getS("iN.sN", this->issuerName.serialNo, sizeof(this->issuerName.serialNo));
+	d.getS("iN.cN", this->issuerName.commonName, sizeof(this->issuerName.commonName));
 	d.getS("iN.c", this->issuerName.country, sizeof(this->issuerName.country));
 	d.getS("iN.o", this->issuerName.organization, sizeof(this->issuerName.organization));
 	d.getS("iN.u", this->issuerName.unit, sizeof(this->issuerName.unit));
@@ -306,8 +277,6 @@ bool Certificate::decode(const Vector< uint8_t > &data)
 	d.getS("iN.p", this->issuerName.province, sizeof(this->issuerName.province));
 	d.getS("iN.sA", this->issuerName.streetAddress, sizeof(this->issuerName.streetAddress));
 	d.getS("iN.pC", this->issuerName.postalCode, sizeof(this->issuerName.postalCode));
-	d.getS("iN.cN", this->issuerName.commonName, sizeof(this->issuerName.commonName));
-	d.getS("iN.sN", this->issuerName.serialNo, sizeof(this->issuerName.serialNo));
 	d.getS("iN.e", this->issuerName.email, sizeof(this->issuerName.email));
 	d.getS("iN.ur", this->issuerName.url, sizeof(this->issuerName.url));
 
@@ -338,15 +307,117 @@ bool Certificate::sign(const Identity &issuer)
 	return (this->signatureSize = issuer.sign(enc.data(), (unsigned int)enc.size(), this->signature, sizeof(this->signature))) > 0;
 }
 
-bool Certificate::verify() const
+ZT_CertificateError Certificate::verify() const
 {
 	try {
 		if (this->issuer) {
-			Vector< uint8_t > enc(encode(true));
-			return reinterpret_cast<const Identity *>(this->issuer)->verify(enc.data(), (unsigned int)enc.size(), this->signature, this->signatureSize);
+			const Vector< uint8_t > enc(encode(true));
+			if (!reinterpret_cast<const Identity *>(this->issuer)->verify(enc.data(), (unsigned int)enc.size(), this->signature, this->signatureSize))
+				return ZT_CERTIFICATE_ERROR_INVALID_PRIMARY_SIGNATURE;
+		} else {
+			return ZT_CERTIFICATE_ERROR_INVALID_PRIMARY_SIGNATURE;
 		}
-	} catch ( ... ) {}
-	return false;
+
+		if (this->subject.uniqueIdProofSignatureSize > 0) {
+			static_assert(ZT_ECC384_SIGNATURE_SIZE <= ZT_CERTIFICATE_MAX_SIGNATURE_SIZE, "overflow");
+			static_assert((ZT_ECC384_PUBLIC_KEY_SIZE + 1) <= ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE, "overflow");
+			if (
+				(this->subject.uniqueIdProofSignatureSize != ZT_ECC384_SIGNATURE_SIZE) ||
+				(this->subject.uniqueIdSize != (ZT_ECC384_PUBLIC_KEY_SIZE + 1)) ||
+				(this->subject.uniqueId[0] != ZT_CERTIFICATE_UNIQUE_ID_PUBLIC_KEY_TYPE_NIST_P_384))
+				return ZT_CERTIFICATE_ERROR_INVALID_UNIQUE_ID_PROOF;
+			Dictionary tmp;
+			m_encodeSubject(tmp, true);
+			Vector< uint8_t > enc;
+			tmp.encode(enc);
+			uint8_t h[ZT_SHA384_DIGEST_SIZE];
+			SHA384(h, enc.data(), (unsigned int)enc.size());
+			if (!ECC384ECDSAVerify(this->subject.uniqueId + 1, h, this->subject.uniqueIdProofSignature))
+				return ZT_CERTIFICATE_ERROR_INVALID_UNIQUE_ID_PROOF;
+		} else if (this->subject.uniqueIdSize > ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE) {
+			return ZT_CERTIFICATE_ERROR_INVALID_FORMAT;
+		}
+
+		for (unsigned int i = 0; i < this->subject.identityCount; ++i) {
+			if (!this->subject.identities[i].identity)
+				return ZT_CERTIFICATE_ERROR_MISSING_REQUIRED_FIELDS;
+			if (!reinterpret_cast<const Identity *>(this->subject.identities[i].identity)->locallyValidate())
+				return ZT_CERTIFICATE_ERROR_INVALID_IDENTITY;
+			if (this->subject.identities[i].locator) {
+				if (!reinterpret_cast<const Locator *>(this->subject.identities[i].locator)->verify(*reinterpret_cast<const Identity *>(this->subject.identities[i].identity)))
+					return ZT_CERTIFICATE_ERROR_INVALID_COMPONENT_SIGNATURE;
+			}
+		}
+
+		for (unsigned int i = 0; i < this->subject.networkCount; ++i) {
+			if (!this->subject.networks[i].id)
+				return ZT_CERTIFICATE_ERROR_MISSING_REQUIRED_FIELDS;
+		}
+
+		if (this->subject.updateUrlCount) {
+			if (!this->subject.updateUrls)
+				return ZT_CERTIFICATE_ERROR_INVALID_FORMAT;
+			for (unsigned int i = 0; i < this->subject.updateUrlCount; ++i) {
+				if (!this->subject.updateUrls[i])
+					return ZT_CERTIFICATE_ERROR_MISSING_REQUIRED_FIELDS;
+			}
+		} else if (this->subject.updateUrls) {
+			return ZT_CERTIFICATE_ERROR_INVALID_FORMAT;
+		}
+	} catch (...) {}
+
+	return ZT_CERTIFICATE_ERROR_NONE;
+}
+
+void Certificate::m_encodeSubject(Dictionary &d, bool omitUniqueIdProofSignature) const
+{
+	char tmp[256];
+
+	d.add("s.t", (uint64_t)this->subject.timestamp);
+
+	d.add("s.i$", (uint64_t)this->subject.identityCount);
+	for (unsigned int i = 0; i < this->subject.identityCount; ++i) {
+		if (this->subject.identities[i].identity)
+			d.addO(Dictionary::arraySubscript(tmp, "s.i$.i", i), *reinterpret_cast<const Identity *>(this->subject.identities[i].identity));
+		if (this->subject.identities[i].locator)
+			d.addO(Dictionary::arraySubscript(tmp, "s.i$.l", i), *reinterpret_cast<const Locator *>(this->subject.identities[i].locator));
+	}
+
+	d.add("s.n$", (uint64_t)this->subject.networkCount);
+	for (unsigned int i = 0; i < this->subject.networkCount; ++i) {
+		d.add(Dictionary::arraySubscript(tmp, "s.n$.i", i), this->subject.networks[i].id);
+		Fingerprint fp(this->subject.networks[i].controller);
+		d.addO(Dictionary::arraySubscript(tmp, "s.n$.c", i), fp);
+	}
+
+	d.add("s.c$", (uint64_t)this->subject.certificateCount);
+	for (unsigned int i = 0; i < this->subject.certificateCount; ++i) {
+		if (this->subject.certificates[i])
+			d[Dictionary::arraySubscript(tmp, "s.c$", i)].assign(this->subject.certificates[i], this->subject.certificates[i] + ZT_SHA384_DIGEST_SIZE);
+	}
+
+	d.add("s.u$", (uint64_t)this->subject.updateUrlCount);
+	if (this->subject.updateUrls) {
+		for (unsigned int i = 0; i < this->subject.updateUrlCount; ++i)
+			d.add(Dictionary::arraySubscript(tmp, "s.u$", i), this->subject.updateUrls[i]);
+	}
+
+	d.add("s.n.c", this->subject.name.country);
+	d.add("s.n.o", this->subject.name.organization);
+	d.add("s.n.u", this->subject.name.unit);
+	d.add("s.n.l", this->subject.name.locality);
+	d.add("s.n.p", this->subject.name.province);
+	d.add("s.n.sA", this->subject.name.streetAddress);
+	d.add("s.n.pC", this->subject.name.postalCode);
+	d.add("s.n.cN", this->subject.name.commonName);
+	d.add("s.n.sN", this->subject.name.serialNo);
+	d.add("s.n.e", this->subject.name.email);
+	d.add("s.n.ur", this->subject.name.url);
+
+	if ((this->subject.uniqueIdSize > 0) && (this->subject.uniqueIdSize <= ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE))
+		d["s.uI"].assign(this->subject.uniqueId, this->subject.uniqueId + this->subject.uniqueIdSize);
+	if ((!omitUniqueIdProofSignature) && (this->subject.uniqueIdProofSignatureSize > 0) && (this->subject.uniqueIdProofSignatureSize <= ZT_CERTIFICATE_MAX_SIGNATURE_SIZE))
+		d["s.uS"].assign(this->subject.uniqueIdProofSignature, this->subject.uniqueIdProofSignature + this->subject.uniqueIdProofSignatureSize);
 }
 
 } // namespace ZeroTier

core/Certificate.hpp

@@ -63,7 +63,6 @@ public:
 	void clear();
 
 	Certificate &operator=(const ZT_Certificate &apiCert);
-
 	Certificate &operator=(const Certificate &cert);
 
 	/**
@@ -134,41 +133,41 @@ public:
 	bool sign(const Identity &issuer);
 
 	/**
-	 * Verify certificate signature against the issuer contained therein
+	 * Verify self-contained signatures and validity of certificate structure
+	 *
+	 * This doesn't check the entire certificate chain, just the validity of
+	 * the certificate's internal signature and fields.
 	 *
-	 * @return True if certificate is signed and signature is valid
+	 * @return OK (0) or error code indicating why certificate failed verification.
 	 */
-	bool verify() const;
+	ZT_CertificateError verify() const;
 
 	ZT_INLINE unsigned long hashCode() const noexcept
 	{ return (unsigned long)Utils::loadAsIsEndian< uint32_t >(this->serialNo); }
 
 	ZT_INLINE bool operator==(const ZT_Certificate &c) const noexcept
 	{ return memcmp(this->serialNo, c.serialNo, ZT_SHA384_DIGEST_SIZE) == 0; }
-
 	ZT_INLINE bool operator!=(const ZT_Certificate &c) const noexcept
 	{ return memcmp(this->serialNo, c.serialNo, ZT_SHA384_DIGEST_SIZE) != 0; }
-
 	ZT_INLINE bool operator<(const ZT_Certificate &c) const noexcept
 	{ return memcmp(this->serialNo, c.serialNo, ZT_SHA384_DIGEST_SIZE) < 0; }
-
 	ZT_INLINE bool operator<=(const ZT_Certificate &c) const noexcept
 	{ return memcmp(this->serialNo, c.serialNo, ZT_SHA384_DIGEST_SIZE) <= 0; }
-
 	ZT_INLINE bool operator>(const ZT_Certificate &c) const noexcept
 	{ return memcmp(this->serialNo, c.serialNo, ZT_SHA384_DIGEST_SIZE) > 0; }
-
 	ZT_INLINE bool operator>=(const ZT_Certificate &c) const noexcept
 	{ return memcmp(this->serialNo, c.serialNo, ZT_SHA384_DIGEST_SIZE) >= 0; }
 
 private:
+	void m_encodeSubject(Dictionary &d, bool omitUniqueIdProofSignature) const;
+
 	// These hold any identity or locator objects that are owned by and should
 	// be deleted with this certificate. Lists are used so the pointers never
 	// change.
 	List< Identity > m_identities;
 	List< Locator > m_locators;
 	List< String > m_strings;
-	List< Blob< ZT_SHA384_DIGEST_SIZE > > m_serials;
+	List< SHA384Hash > m_serials;
 
 	// These are stored in a vector because the memory needs to be contiguous.
 	Vector< ZT_Certificate_Identity > m_subjectIdentities;

core/FCV.hpp

@@ -32,7 +32,7 @@ namespace ZeroTier {
  * @tparam T Type to contain
  * @tparam C Maximum capacity of vector
  */
-template<typename T, unsigned int C>
+template< typename T, unsigned int C >
 class FCV
 {
 public:
@@ -46,13 +46,14 @@ public:
 	{ *this = v; }
 
 	ZT_INLINE FCV(const T *const contents, const unsigned int len) :
-		_s(0)
+		_s(len)
 	{
-		for (unsigned int i = 0;i < len;++i)
-			push_back(contents[i]);
+		const unsigned int l = std::min(len, C);
+		for (unsigned int i = 0; i < l; ++i)
+			new(reinterpret_cast<T *>(_m) + i) T(contents[i]);
 	}
 
-	template<typename I>
+	template< typename I >
 	ZT_INLINE FCV(I i, I end) :
 		_s(0)
 	{
@@ -71,7 +72,7 @@ public:
 			this->clear();
 			const unsigned int s = v._s;
 			_s = s;
-			for (unsigned int i = 0;i < s;++i)
+			for (unsigned int i = 0; i < s; ++i)
 				new(reinterpret_cast<T *>(_m) + i) T(*(reinterpret_cast<const T *>(v._m) + i));
 		}
 		return *this;
@@ -84,7 +85,7 @@ public:
 	{
 		const unsigned int s = _s;
 		_s = 0;
-		for (unsigned int i = 0;i < s;++i)
+		for (unsigned int i = 0; i < s; ++i)
 			(reinterpret_cast<T *>(_m) + i)->~T();
 	}
 
@@ -248,13 +249,13 @@ public:
 	 * @param start Starting iterator
 	 * @param end Ending iterator (must be greater than start)
 	 */
-	template<typename X>
+	template< typename X >
 	ZT_INLINE void assign(X start, const X &end)
 	{
-		const int l = std::min((int) std::distance(start, end), (int) C);
+		const int l = std::min((int)std::distance(start, end), (int)C);
 		if (l > 0) {
-			this->resize((unsigned int) l);
-			for (int i = 0;i < l;++i)
+			this->resize((unsigned int)l);
+			for (int i = 0; i < l; ++i)
 				reinterpret_cast<T *>(_m)[i] = *(start++);
 		} else {
 			this->clear();
@@ -264,7 +265,7 @@ public:
 	ZT_INLINE bool operator==(const FCV &v) const noexcept
 	{
 		if (_s == v._s) {
-			for (unsigned int i = 0;i < _s;++i) {
+			for (unsigned int i = 0; i < _s; ++i) {
 				if (!(*(reinterpret_cast<const T *>(_m) + i) == *(reinterpret_cast<const T *>(v._m) + i)))
 					return false;
 			}

core/Mutex.hpp

@@ -16,23 +16,39 @@
 
 #include "Constants.hpp"
 
-#include <cstdint>
-#include <cstdlib>
-
+// If C++17 is available use std::mutex and std::shared_mutex as
+// these will probably use whatever is fastest on a given platform.
+// Older compilers require pthreads to be available. The compiler
+// now used on Windows is new enough to use C++17 stuff, so no more
+// need for Windows-specific implementations here.
+#if __cplusplus >= 201703L
+#include <mutex>
+#include <shared_mutex>
+#else
+#define ZT_USE_PTHREADS
 #ifndef __WINDOWS__
 #include <pthread.h>
 #endif
+#endif
 
 namespace ZeroTier {
 
+/**
+ * A simple mutual exclusion lock.
+ */
 class Mutex
 {
 public:
+#ifdef ZT_USE_PTHREADS
 	ZT_INLINE Mutex() noexcept { pthread_mutex_init(&_mh,nullptr); }
 	ZT_INLINE ~Mutex() noexcept { pthread_mutex_destroy(&_mh); }
-
 	ZT_INLINE void lock() const noexcept { pthread_mutex_lock(&((const_cast <Mutex *> (this))->_mh)); }
 	ZT_INLINE void unlock() const noexcept { pthread_mutex_unlock(&((const_cast <Mutex *> (this))->_mh)); }
+#else
+	ZT_INLINE Mutex() noexcept : _m() {}
+	ZT_INLINE void lock() const noexcept { const_cast<Mutex *>(this)->_m.lock(); }
+	ZT_INLINE void unlock() const noexcept { const_cast<Mutex *>(this)->_m.unlock(); }
+#endif
 
 	class Lock
 	{
@@ -48,19 +64,33 @@ private:
 	ZT_INLINE Mutex(const Mutex &) noexcept {}
 	ZT_INLINE const Mutex &operator=(const Mutex &) noexcept { return *this; }
 
+#ifdef ZT_USE_PTHREADS
 	pthread_mutex_t _mh;
+#else
+	std::mutex _m;
+#endif
 };
 
+/**
+ * A lock allowing multiple threads to read but making all wait on any writing thread.
+ */
 class RWMutex
 {
 public:
+#ifdef ZT_USE_PTHREADS
 	ZT_INLINE RWMutex() noexcept { pthread_rwlock_init(&_mh,nullptr); }
 	ZT_INLINE ~RWMutex() noexcept { pthread_rwlock_destroy(&_mh); }
-
 	ZT_INLINE void lock() const noexcept { pthread_rwlock_wrlock(&((const_cast <RWMutex *> (this))->_mh)); }
 	ZT_INLINE void rlock() const noexcept { pthread_rwlock_rdlock(&((const_cast <RWMutex *> (this))->_mh)); }
 	ZT_INLINE void unlock() const noexcept { pthread_rwlock_unlock(&((const_cast <RWMutex *> (this))->_mh)); }
 	ZT_INLINE void runlock() const noexcept { pthread_rwlock_unlock(&((const_cast <RWMutex *> (this))->_mh)); }
+#else
+	ZT_INLINE RWMutex() noexcept : _m() {}
+	ZT_INLINE void lock() const noexcept { const_cast<RWMutex *>(this)->_m.lock(); }
+	ZT_INLINE void rlock() const noexcept { const_cast<RWMutex *>(this)->_m.lock_shared(); }
+	ZT_INLINE void unlock() const noexcept { const_cast<RWMutex *>(this)->_m.unlock(); }
+	ZT_INLINE void runlock() const noexcept { const_cast<RWMutex *>(this)->_m.unlock_shared(); }
+#endif
 
 	/**
 	 * RAAI locker that acquires only the read lock (shared read)
@@ -89,10 +119,13 @@ public:
 	};
 
 	/**
-	 * RAAI locker that acquires the read lock first and can switch modes
+	 * RAAI locker that acquires the read lock first and can switch to writing.
 	 *
 	 * Use writing() to acquire the write lock if not already acquired. Use reading() to
-	 * let go of the write lock and go back to only holding the read lock.
+	 * let go of the write lock and go back to only holding the read lock. Note that on
+	 * most platforms there's a brief moment where the lock is unlocked during the
+	 * transition, meaning protected variable states can change. Code must not assume
+	 * that the lock is held constantly if writing() is used to change mode.
 	 */
 	class RMaybeWLock
 	{
@@ -101,6 +134,7 @@ public:
 		explicit ZT_INLINE RMaybeWLock(const RWMutex &m) noexcept : _m(const_cast<RWMutex *>(&m)),_w(false) { _m->rlock(); }
 		ZT_INLINE void writing() noexcept { if (!_w) { _w = true; _m->runlock(); _m->lock(); } }
 		ZT_INLINE void reading() noexcept { if (_w) { _w = false; _m->unlock(); _m->rlock(); } }
+		ZT_INLINE bool isWriting() const noexcept { return _w; }
 		ZT_INLINE ~RMaybeWLock() { if (_w) _m->unlock(); else _m->runlock(); }
 	private:
 		RWMutex *const _m;
@@ -111,44 +145,13 @@ private:
 	ZT_INLINE RWMutex(const RWMutex &) noexcept {}
 	ZT_INLINE const RWMutex &operator=(const RWMutex &) noexcept { return *this; }
 
+#ifdef ZT_USE_PTHREADS
 	pthread_rwlock_t _mh;
+#else
+	std::shared_mutex _m;
+#endif
 };
 
 } // namespace ZeroTier
 
-#if 0
-#include <Windows.h>
-
-namespace ZeroTier {
-
-class Mutex
-{
-public:
-	ZT_INLINE Mutex() { InitializeCriticalSection(&_cs); }
-	ZT_INLINE ~Mutex() { DeleteCriticalSection(&_cs); }
-	ZT_INLINE void lock() { EnterCriticalSection(&_cs); }
-	ZT_INLINE void unlock() { LeaveCriticalSection(&_cs); }
-	ZT_INLINE void lock() const { (const_cast <Mutex *> (this))->lock(); }
-	ZT_INLINE void unlock() const { (const_cast <Mutex *> (this))->unlock(); }
-
-	class Lock
-	{
-	public:
-		ZT_INLINE Lock(Mutex &m) : _m(&m) { m.lock(); }
-		ZT_INLINE Lock(const Mutex &m) : _m(const_cast<Mutex *>(&m)) { _m->lock(); }
-		ZT_INLINE ~Lock() { _m->unlock(); }
-	private:
-		Mutex *const _m;
-	};
-
-private:
-	ZT_INLINE Mutex(const Mutex &) {}
-	ZT_INLINE const Mutex &operator=(const Mutex &) { return *this; }
-
-	CRITICAL_SECTION _cs;
-};
-
-} // namespace ZeroTier
-#endif
-
 #endif

core/Topology.cpp

@@ -23,7 +23,7 @@ Topology::Topology(const RuntimeEnvironment *renv, void *tPtr) :
 	idtmp[1] = 0;
 	Vector< uint8_t > data(RR->node->stateObjectGet(tPtr, ZT_STATE_OBJECT_ROOTS, idtmp));
 	// TODO
-	m_updateRootPeers(tPtr);
+	m_updateRootPeers_l_roots_certs(tPtr);
 }
 
 SharedPtr< Peer > Topology::add(void *tPtr, const SharedPtr< Peer > &peer)
@@ -42,12 +42,17 @@ SharedPtr< Peer > Topology::add(void *tPtr, const SharedPtr< Peer > &peer)
 SharedPtr< Peer > Topology::addRoot(void *const tPtr, const Identity &id)
 {
 	if ((id != RR->identity) && id.locallyValidate()) {
-		RWMutex::Lock l1(m_peers_l);
-		// TODO
-		//m_roots.insert(id);
+		RWMutex::Lock l1(m_roots_l);
 
-		m_updateRootPeers(tPtr);
-		m_writeRootList(tPtr);
+		// A null pointer in the set of certificates specifying a root indicates that
+		// the root has been directly added.
+		m_roots[id.fingerprint()].insert(SharedPtr< const Certificate >());
+
+		{
+			Mutex::Lock certsLock(m_certs_l);
+			m_updateRootPeers_l_roots_certs(tPtr);
+		}
+		m_writeRootList_l_roots(tPtr);
 
 		for (Vector< SharedPtr< Peer > >::const_iterator p(m_rootPeers.begin()); p != m_rootPeers.end(); ++p) {
 			if ((*p)->identity() == id)
@@ -57,13 +62,9 @@ SharedPtr< Peer > Topology::addRoot(void *const tPtr, const Identity &id)
 	return SharedPtr< Peer >();
 }
 
-ZT_CertificateError addRootSet(void *tPtr, const Certificate &cert)
-{
-}
-
 bool Topology::removeRoot(void *const tPtr, Address address)
 {
-	RWMutex::Lock l1(m_peers_l);
+	RWMutex::Lock l1(m_roots_l);
 	// TODO
 	return true;
 }
@@ -72,44 +73,82 @@ struct p_RootRankingComparisonOperator
 {
 	ZT_INLINE bool operator()(const SharedPtr< Peer > &a, const SharedPtr< Peer > &b) const noexcept
 	{
-		// Sort in inverse order of latency with lowest latency first (and -1 last).
-		const int bb = b->latency();
-		if (bb < 0)
+		// Sort roots first in order of which root has spoken most recently, but
+		// only at a resolution of ZT_PATH_KEEPALIVE_PERIOD/2 units of time. This
+		// means that living roots that seem responsive are ranked the same. Then
+		// they're sorted in descending order of latency so that the apparently
+		// fastest root is ranked first.
+		const int64_t alr = a->lastReceive() / (ZT_PATH_KEEPALIVE_PERIOD / 2);
+		const int64_t blr = b->lastReceive() / (ZT_PATH_KEEPALIVE_PERIOD / 2);
+		if (alr < blr) {
 			return true;
-		return bb < a->latency();
+		} else if (blr == alr) {
+			const int bb = b->latency();
+			if (bb < 0)
+				return true;
+			return bb < a->latency();
+		}
 	}
 };
 
 void Topology::rankRoots()
 {
-	RWMutex::Lock l1(m_peers_l);
+	RWMutex::Lock l1(m_roots_l);
 	std::sort(m_rootPeers.begin(), m_rootPeers.end(), p_RootRankingComparisonOperator());
 }
 
 void Topology::doPeriodicTasks(void *tPtr, const int64_t now)
 {
-	// Delete peers that haven't said anything in ZT_PEER_ALIVE_TIMEOUT.
+	// Peer and path delete operations are batched to avoid holding write locks on
+	// these structures for any length of time. A list is compiled in read mode,
+	// then the write lock is acquired for each delete. This adds overhead if there
+	// are a lot of deletions, but that's not common.
+
+	// Delete peers that are stale or offline.
 	{
-		RWMutex::Lock l1(m_peers_l);
-		for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end();) {
-			// TODO: also delete if the peer has not exchanged meaningful communication in a while, such as
-			// a network frame or non-trivial control packet.
-			if (((now - i->second->lastReceive()) > ZT_PEER_ALIVE_TIMEOUT) && (m_roots.count(i->second->identity()) == 0)) {
-				i->second->save(tPtr);
-				m_peers.erase(i++);
-			} else ++i;
+		Vector< Address > toDelete;
+		{
+			RWMutex::RLock l1(m_peers_l);
+			for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end(); ++i) {
+				// TODO: also delete if the peer has not exchanged meaningful communication in a while, such as
+				// a network frame or non-trivial control packet.
+				if (((now - i->second->lastReceive()) > ZT_PEER_ALIVE_TIMEOUT) && (m_roots.find(i->second->identity().fingerprint()) == m_roots.end()))
+					toDelete.push_back(i->first);
+			}
+		}
+		for (Vector< Address >::iterator i(toDelete.begin()); i != toDelete.end(); ++i) {
+			RWMutex::Lock l1(m_peers_l);
+			const Map< Address, SharedPtr< Peer > >::iterator p(m_peers.find(*i));
+			if (likely(p != m_peers.end())) {
+				p->second->save(tPtr);
+				m_peers.erase(p);
+			}
 		}
 	}
 
 	// Delete paths that are no longer held by anyone else ("weak reference" type behavior).
 	{
-		RWMutex::Lock l1(m_paths_l);
-		for (Map< uint64_t, SharedPtr< Path > >::iterator i(m_paths.begin()); i != m_paths.end();) {
-			if (i->second.weakGC())
-				m_paths.erase(i++);
-			else ++i;
+		Vector< uint64_t > toDelete;
+		{
+			RWMutex::RLock l1(m_paths_l);
+			for (Map< uint64_t, SharedPtr< Path > >::iterator i(m_paths.begin()); i != m_paths.end(); ++i) {
+				if (i->second.weakGC())
+					toDelete.push_back(i->first);
+			}
+		}
+		for (Vector< uint64_t >::iterator i(toDelete.begin()); i != toDelete.end(); ++i) {
+			RWMutex::Lock l1(m_paths_l);
+			const Map< uint64_t, SharedPtr< Path > >::iterator p(m_paths.find(*i));
+			if (likely(p != m_paths.end()))
+				m_paths.erase(p);
 		}
 	}
+
+	// Clean any expired certificates
+	{
+		Mutex::Lock l1(m_certs_l);
+		m_cleanCertificates_l_certs(now);
+	}
 }
 
 void Topology::saveAll(void *tPtr)
@@ -119,6 +158,146 @@ void Topology::saveAll(void *tPtr)
 		i->second->save(tPtr);
 }
 
+ZT_CertificateError Topology::addCertificate(void *tPtr, const Certificate &cert, const int64_t now, const unsigned int localTrust)
+{
+	Mutex::Lock certsLock(m_certs_l);
+
+	// Check to see if we already have this specific certificate.
+	const SHA384Hash serial(cert.serialNo);
+	if (m_certs.find(serial) != m_certs.end())
+		return ZT_CERTIFICATE_ERROR_NONE;
+
+	// Verify certificate all the way to a trusted root.
+	const ZT_CertificateError err = m_verifyCertificate_l_certs(cert, now, localTrust, false);
+	if (err != ZT_CERTIFICATE_ERROR_NONE)
+		return err;
+
+	// Create entry containing copy of certificate and trust flags.
+	const std::pair< SharedPtr< const Certificate >, unsigned int > certEntry(SharedPtr< const Certificate >(new Certificate(cert)), localTrust);
+
+	// If the subject contains a unique ID, check if we already have a cert for the
+	// same uniquely identified subject. If so, check its subject timestamp and keep
+	// the one we have if newer. Otherwise replace it. Note that the verification
+	// function will have checked the unique ID proof signature already if a unique
+	// ID was present.
+	FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE > uniqueId(cert.subject.uniqueId, cert.subject.uniqueIdSize);
+	if (!uniqueId.empty()) {
+		std::pair< SharedPtr< const Certificate >, unsigned int > &bySubjectUniqueId = m_certsBySubjectUniqueId[uniqueId];
+		if (bySubjectUniqueId.first) {
+			if (bySubjectUniqueId.first->subject.timestamp >= cert.subject.timestamp)
+				return ZT_CERTIFICATE_ERROR_HAVE_NEWER_CERT;
+			m_eraseCertificate_l_certs(bySubjectUniqueId.first);
+			m_certsBySubjectUniqueId[uniqueId] = certEntry;
+		} else {
+			bySubjectUniqueId = certEntry;
+		}
+	}
+
+	// Save certificate by serial number.
+	m_certs[serial] = certEntry;
+
+	// Add certificate to sets of certificates whose subject references a given identity.
+	for (unsigned int i = 0; i < cert.subject.identityCount; ++i) {
+		const Identity *const ii = reinterpret_cast<const Identity *>(cert.subject.identities[i].identity);
+		m_certsBySubjectIdentity[ii->fingerprint()].insert(certEntry);
+	}
+
+	// Clean any certificates whose chains are now broken, which can happen if there was
+	// an update that replaced an old cert with a given unique ID. Otherwise this generally
+	// does nothing here.
+	m_cleanCertificates_l_certs(now);
+
+	// Refresh the root peers lists, since certs may enumerate roots.
+	{
+		RWMutex::Lock rootsLock(m_roots_l);
+		m_updateRootPeers_l_roots_certs(tPtr);
+	}
+
+	return ZT_CERTIFICATE_ERROR_NONE;
+}
+
+void Topology::m_eraseCertificate_l_certs(const SharedPtr< const Certificate > &cert)
+{
+	// assumes m_certs is locked for writing
+
+	m_certsBySubjectUniqueId.erase(FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE >(cert->subject.uniqueId, cert->subject.uniqueIdSize));
+	m_certs.erase(SHA384Hash(cert->serialNo));
+	for (unsigned int i = 0; i < cert->subject.identityCount; ++i) {
+		const Identity *const ii = reinterpret_cast<const Identity *>(cert->subject.identities[i].identity);
+		Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > >::iterator bySubjectIdentity(m_certsBySubjectIdentity.find(ii->fingerprint()));
+		if (bySubjectIdentity != m_certsBySubjectIdentity.end()) {
+			bySubjectIdentity->second.erase(cert);
+			if (bySubjectIdentity->second.empty())
+				m_certsBySubjectIdentity.erase(bySubjectIdentity);
+		}
+	}
+}
+
+void Topology::m_cleanCertificates_l_certs(int64_t now)
+{
+	// assumes m_certs is locked for writing
+
+	Vector< SharedPtr< const Certificate > > toDelete;
+	for (;;) {
+		for (Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > >::iterator c(m_certs.begin()); c != m_certs.end(); ++c) {
+			const ZT_CertificateError err = m_verifyCertificate_l_certs(*(c->second.first), now, c->second.second, true);
+			if (err != ZT_CERTIFICATE_ERROR_NONE)
+				toDelete.push_back(c->second.first);
+		}
+
+		if (toDelete.empty())
+			break;
+
+		for (Vector< SharedPtr< const Certificate > >::iterator c(toDelete.begin()); c != toDelete.end(); ++c)
+			m_eraseCertificate_l_certs(*c);
+		toDelete.clear();
+	}
+}
+
+bool Topology::m_verifyCertificateChain_l_certs(const Certificate *current, const int64_t now) const
+{
+	// assumes m_certs is at least locked for reading
+
+	Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > >::const_iterator c = m_certsBySubjectIdentity.find(reinterpret_cast<const Identity *>(current->issuer)->fingerprint());
+	if (c != m_certsBySubjectIdentity.end()) {
+		for (Map< SharedPtr< const Certificate >, unsigned int >::const_iterator cc(c->second.begin()); cc != c->second.end(); ++cc) {
+			if (
+				(cc->first->maxPathLength > current->maxPathLength) &&
+				(cc->first->validity[0] <= now) &&                  // not before now
+				(cc->first->validity[1] >= now) &&                  // not after now
+				(cc->first->validity[0] <= current->timestamp) &&   // not before child cert's timestamp
+				(cc->first->validity[1] >= current->timestamp)      // not after child cert's timestamp
+				) {
+				if ((cc->second & ZT_CERTIFICATE_LOCAL_TRUST_FLAG_ROOT_CA) != 0)
+					return true;
+				if (m_verifyCertificateChain_l_certs(cc->first.ptr(), now))
+					return true;
+			}
+		}
+	}
+
+	return false;
+}
+
+ZT_CertificateError Topology::m_verifyCertificate_l_certs(const Certificate &cert, const int64_t now, unsigned int localTrust, bool skipSignatureCheck) const
+{
+	// assumes m_certs is at least locked for reading
+
+	if ((cert.validity[0] > now) || (cert.validity[1] < now))
+		return ZT_CERTIFICATE_ERROR_OUT_OF_VALID_TIME_WINDOW;
+
+	if (!skipSignatureCheck) {
+		const ZT_CertificateError ce = cert.verify();
+		if (ce != ZT_CERTIFICATE_ERROR_NONE)
+			return ce;
+	}
+
+	if ((localTrust & ZT_CERTIFICATE_LOCAL_TRUST_FLAG_ROOT_CA) == 0) {
+		if (!m_verifyCertificateChain_l_certs(&cert, now))
+			return ZT_CERTIFICATE_ERROR_INVALID_CHAIN;
+	}
+}
+
 void Topology::m_loadCached(void *tPtr, const Address &zta, SharedPtr< Peer > &peer)
 {
 	try {
@@ -148,7 +327,7 @@ void Topology::m_loadCached(void *tPtr, const Address &zta, SharedPtr< Peer > &p
 	}
 }
 
-void Topology::m_writeRootList(void *tPtr)
+void Topology::m_writeRootList_l_roots(void *tPtr)
 {
 	// assumes m_peers_l is locked for read or write
 	// TODO
@@ -170,9 +349,9 @@ void Topology::m_writeRootList(void *tPtr)
 #endif
 }
 
-void Topology::m_updateRootPeers(void *tPtr)
+void Topology::m_updateRootPeers_l_roots_certs(void *tPtr)
 {
-	// assumes m_peers_l is locked for write
+	// assumes m_peers_l and m_certs_l are locked for write
 	// TODO
 #if 0
 	Vector< SharedPtr< Peer > > rp;

core/Topology.hpp

@@ -25,6 +25,7 @@
 #include "ScopedPtr.hpp"
 #include "Fingerprint.hpp"
 #include "Blob.hpp"
+#include "FCV.hpp"
 #include "Certificate.hpp"
 #include "Containers.hpp"
 
@@ -46,6 +47,7 @@ public:
 	 * This will not replace existing peers. In that case the existing peer
 	 * record is returned.
 	 *
+	 * @param tPtr Thread pointer to be handed through to any callbacks called as a result of this call
 	 * @param peer Peer to add
 	 * @return New or existing peer (should replace 'peer')
 	 */
@@ -115,7 +117,7 @@ public:
 	 */
 	ZT_INLINE SharedPtr< Peer > root() const
 	{
-		RWMutex::RLock l(m_peers_l);
+		RWMutex::RLock l(m_roots_l);
 		if (unlikely(m_rootPeers.empty()))
 			return SharedPtr< Peer >();
 		return m_rootPeers.front();
@@ -127,8 +129,8 @@ public:
 	 */
 	ZT_INLINE bool isRoot(const Identity &id) const
 	{
-		RWMutex::RLock l(m_peers_l);
-		return (m_roots.find(id) != m_roots.end());
+		RWMutex::RLock l(m_roots_l);
+		return (m_roots.find(id.fingerprint()) != m_roots.end());
 	}
 
 	/**
@@ -182,20 +184,6 @@ public:
 	 */
 	SharedPtr< Peer > addRoot(void *tPtr, const Identity &id);
 
-	/**
-	 * Add or update a root set
-	 *
-	 * This does not check the certificate's validity. That must be done
-	 * first. It may however return a certificate error if something is
-	 * missing or wrong that prevents the certificate from being used
-	 * as a root set.
-	 *
-	 * @param tPtr Thread pointer
-	 * @param cert Certificate whose subject enumerates root identities
-	 * @return Zero on success or an error code
-	 */
-	ZT_CertificateError addRootSet(void *tPtr, const Certificate &cert);
-
 	/**
 	 * Remove a root server's identity from the root server set
 	 *
@@ -222,12 +210,22 @@ public:
 	 */
 	void saveAll(void *tPtr);
 
+	/**
+	 * Add a certificate to the local certificate store
+	 *
+	 * @param cert Certificate to add (a copy will be made if added)
+	 * @return Error or 0 on success
+	 */
+	ZT_CertificateError addCertificate(void *tPtr, const Certificate &cert, const int64_t now, unsigned int localTrust);
+
 private:
+	void m_eraseCertificate_l_certs(const SharedPtr< const Certificate > &cert);
+	void m_cleanCertificates_l_certs(int64_t now);
+	bool m_verifyCertificateChain_l_certs(const Certificate *current, const int64_t now) const;
+	ZT_CertificateError m_verifyCertificate_l_certs(const Certificate &cert, const int64_t now, unsigned int localTrust, bool skipSignatureCheck) const;
 	void m_loadCached(void *tPtr, const Address &zta, SharedPtr< Peer > &peer);
-
-	void m_writeRootList(void *tPtr);
-
-	void m_updateRootPeers(void *tPtr);
+	void m_writeRootList_l_roots(void *tPtr);
+	void m_updateRootPeers_l_roots_certs(void *tPtr);
 
 	// This gets an integer key from an InetAddress for looking up paths.
 	static ZT_INLINE uint64_t s_getPathKey(const int64_t l, const InetAddress &r) noexcept
@@ -249,13 +247,22 @@ private:
 	}
 
 	const RuntimeEnvironment *const RR;
-	RWMutex m_paths_l; // locks m_paths
-	RWMutex m_peers_l; // locks m_peers, m_roots, and m_rootPeers
+
+	RWMutex m_paths_l; // m_paths
+	RWMutex m_peers_l; // m_peers
+	RWMutex m_roots_l; // m_roots, m_rootPeers
+	Mutex m_certs_l;   // m_certs, m_certsBySubjectIdentity
+
 	Map< uint64_t, SharedPtr< Path > > m_paths;
+
 	Map< Address, SharedPtr< Peer > > m_peers;
-	Map< Identity, Set< Blob<ZT_SHA384_DIGEST_SIZE> > > m_roots;
-	Map< String, Certificate > m_rootSets;
+
+	Map< Fingerprint, Set< SharedPtr< const Certificate > > > m_roots;
 	Vector< SharedPtr< Peer > > m_rootPeers;
+
+	Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > > m_certs;
+	Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > > m_certsBySubjectIdentity;
+	Map< FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE >, std::pair< SharedPtr< const Certificate >, unsigned int > > m_certsBySubjectUniqueId;
 };
 
 } // namespace ZeroTier

core/zerotier.h

@@ -296,7 +296,7 @@ typedef struct
 /**
  * Maximum length of string fields in certificates
  */
-#define ZT_CERTIFICATE_MAX_STRING_LENGTH 63
+#define ZT_CERTIFICATE_MAX_STRING_LENGTH 127
 
 /**
  * Maximum length of a signature
@@ -304,9 +304,24 @@ typedef struct
 #define ZT_CERTIFICATE_MAX_SIGNATURE_SIZE 96
 
 /**
- * Flag indicating that the nodes in the subject are a set of roots
+ * Maximum size of a unique ID field in a certificate subject
  */
-#define ZT_CERTIFICATE_FLAG_CERTIFICATE_USE_ROOT_SET 0x0000000000000001ULL
+#define ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE 64
+
+/**
+ * Certificate is a root CA
+ */
+#define ZT_CERTIFICATE_LOCAL_TRUST_FLAG_ROOT_CA 0x0001U
+
+/**
+ * Certificate's subject describes a set of roots
+ */
+#define ZT_CERTIFICATE_LOCAL_TRUST_FLAG_ZEROTIER_ROOT_SET 0x0002U
+
+/**
+ * Public key type for NIST P-384 public keys used as subject unique IDs.
+ */
+#define ZT_CERTIFICATE_UNIQUE_ID_PUBLIC_KEY_TYPE_NIST_P_384 1
 
 /**
  * Errors returned by functions that verify or handle certificates.
@@ -318,35 +333,55 @@ enum ZT_CertificateError
 	 */
 	ZT_CERTIFICATE_ERROR_NONE = 0,
 
+	/**
+	 * A newer certificate with the same issuer and subject serial plus CN exists.
+	 */
+	ZT_CERTIFICATE_ERROR_HAVE_NEWER_CERT = 1,
+
 	/**
 	 * Certificate format is invalid or required fields are missing
 	 */
-	ZT_CERTIFICATE_ERROR_INVALID_FORMAT = 1,
+	ZT_CERTIFICATE_ERROR_INVALID_FORMAT = -1,
 
 	/**
 	 * One or more identities in the certificate are invalid or fail consistency check
 	 */
-	ZT_CERTIFICATE_ERROR_INVALID_IDENTITY = 2,
+	ZT_CERTIFICATE_ERROR_INVALID_IDENTITY = -2,
 
 	/**
 	 * Certificate primary signature is invalid
 	 */
-	ZT_CERTIFICATE_ERROR_INVALID_PRIMARY_SIGNATURE = 3,
+	ZT_CERTIFICATE_ERROR_INVALID_PRIMARY_SIGNATURE = -3,
 
 	/**
 	 * Full chain validation of certificate failed
 	 */
-	ZT_CERTIFICATE_ERROR_INVALID_CHAIN = 4,
+	ZT_CERTIFICATE_ERROR_INVALID_CHAIN = -4,
 
 	/**
 	 * One or more signed components (e.g. a Locator) has an invalid signature.
 	 */
-	ZT_CERTIFICATE_ERROR_INVALID_COMPONENT_SIGNATURE = 5,
+	ZT_CERTIFICATE_ERROR_INVALID_COMPONENT_SIGNATURE = -5,
+
+	/**
+	 * Unique ID proof signature in subject was not valid.
+	 */
+	ZT_CERTIFICATE_ERROR_INVALID_UNIQUE_ID_PROOF = -6,
 
 	/**
 	 * Certificate is not appropriate for this use
 	 */
-	ZT_CERTIFICATE_ERROR_INAPPROPRIATE_FOR_USE = 6
+	ZT_CERTIFICATE_ERROR_INAPPROPRIATE_FOR_USE = -7,
+
+	/**
+	 * Certificate is missing a required field
+	 */
+	ZT_CERTIFICATE_ERROR_MISSING_REQUIRED_FIELDS = -8,
+
+	/**
+	 * Certificate is expired or not yet in effect
+	 */
+	ZT_CERTIFICATE_ERROR_OUT_OF_VALID_TIME_WINDOW = -9
 };
 
 /**
@@ -354,6 +389,8 @@ enum ZT_CertificateError
  */
 typedef struct
 {
+	char serialNo[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
+	char commonName[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 	char country[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 	char organization[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 	char unit[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
@@ -361,8 +398,6 @@ typedef struct
 	char province[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 	char streetAddress[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 	char postalCode[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
-	char commonName[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
-	char serialNo[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 	char email[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 	char url[ZT_CERTIFICATE_MAX_STRING_LENGTH + 1];
 } ZT_Certificate_Name;
@@ -404,6 +439,11 @@ typedef struct
  */
 typedef struct
 {
+	/**
+	 * Timestamp of subject, can also be a revision ID for this subject's name.
+	 */
+	int64_t timestamp;
+
 	/**
 	 * Identities and optional locators of nodes
 	 */
@@ -419,6 +459,11 @@ typedef struct
 	 */
 	const uint8_t *const *certificates;
 
+	/**
+	 * URLs that can be consulted for updates to this certificate.
+	 */
+	const char *const *updateUrls;
+
 	/**
 	 * Number of identities
 	 */
@@ -434,10 +479,35 @@ typedef struct
 	 */
 	unsigned int certificateCount;
 
+	/**
+	 * Number of update URLs
+	 */
+	unsigned int updateUrlCount;
+
 	/**
 	 * Information about owner of items.
 	 */
 	ZT_Certificate_Name name;
+
+	/**
+	 * Unique ID, which can be a public key prefixed by a key type.
+	 */
+	uint8_t uniqueId[ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE];
+
+	/**
+	 * If unique ID is a public key, this can be a signature of the subject.
+	 */
+	uint8_t uniqueIdProofSignature[ZT_CERTIFICATE_MAX_SIGNATURE_SIZE];
+
+	/**
+	 * Size of unique ID in bytes or 0 if none.
+	 */
+	unsigned int uniqueIdSize;
+
+	/**
+	 * Proof signature size or 0 if none.
+	 */
+	unsigned int uniqueIdProofSignatureSize;
 } ZT_Certificate_Subject;
 
 /**
@@ -462,11 +532,13 @@ typedef struct
 	 */
 	uint64_t flags;
 
+	/**
+	 * Certificate timestamp in milliseconds since epoch.
+	 */
+	int64_t timestamp;
+
 	/**
 	 * Valid time range: not before, not after.
-	 *
-	 * In ZeroTier the not before field is also the certificate issued time
-	 * and timestamp.
 	 */
 	int64_t validity[2];
 
@@ -485,16 +557,6 @@ typedef struct
 	 */
 	ZT_Certificate_Name issuerName;
 
-	/**
-	 * URLs that can be consulted for updates to this certificate.
-	 */
-	const char *const *updateUrls;
-
-	/**
-	 * Number of update URLs
-	 */
-	unsigned int updateUrlCount;
-
 	/**
 	 * Maximum path length from this certificate toward further certificates.
 	 *
@@ -505,14 +567,14 @@ typedef struct
 	unsigned int maxPathLength;
 
 	/**
-	 * Size of signature in bytes.
+	 * Signature by issuer (algorithm determined by identity type).
 	 */
-	unsigned int signatureSize;
+	uint8_t signature[ZT_CERTIFICATE_MAX_SIGNATURE_SIZE];
 
 	/**
-	 * Signature by issuer (algorithm determined by identity type).
+	 * Size of signature in bytes.
 	 */
-	uint8_t signature[ZT_CERTIFICATE_MAX_SIGNATURE_SIZE];
+	unsigned int signatureSize;
 } ZT_Certificate;
 
 /**
@@ -916,7 +978,7 @@ enum ZT_VirtualNetworkStatus
 enum ZT_VirtualNetworkType
 {
 	/**
-	 * Private networks are authorized via certificates of membership
+	 * Private networks are authorized via membership credentials
 	 */
 	ZT_NETWORK_TYPE_PRIVATE = 0,
 

pkg/zerotier/endpoint.go

@@ -1,11 +1,24 @@
+/*
+ * Copyright (c)2013-2020 ZeroTier, Inc.
+ *
+ * Use of this software is governed by the Business Source License included
+ * in the LICENSE.TXT file in the project's root directory.
+ *
+ * Change Date: 2024-01-01
+ *
+ * On the date above, in accordance with the Business Source License, use
+ * of this software will be governed by version 2.0 of the Apache License.
+ */
+/****/
+
 package zerotier
 
 // #include "../../serviceiocore/GoGlue.h"
-// static inline const ZT_Fingerprint *_getFP(const ZT_Endpoint *ep) { return &(ep->value.fp); }
-// static inline uint64_t _getAddress(const ZT_Endpoint *ep) { return ep->value.fp.address; }
-// static inline uint64_t _getMAC(const ZT_Endpoint *ep) { return ep->value.mac; }
-// static inline const struct sockaddr_storage *_getSS(const ZT_Endpoint *ep) { return &(ep->value.ss); }
-// static inline void _setSS(ZT_Endpoint *ep,const void *ss) { memcpy(&(ep->value.ss),ss,sizeof(struct sockaddr_storage)); }
+// const ZT_Fingerprint *_getFP(const ZT_Endpoint *ep) { return &(ep->value.fp); }
+// uint64_t _getAddress(const ZT_Endpoint *ep) { return ep->value.fp.address; }
+// uint64_t _getMAC(const ZT_Endpoint *ep) { return ep->value.mac; }
+// const struct sockaddr_storage *_getSS(const ZT_Endpoint *ep) { return &(ep->value.ss); }
+// void _setSS(ZT_Endpoint *ep,const void *ss) { memcpy(&(ep->value.ss),ss,sizeof(struct sockaddr_storage)); }
 import "C"
 
 import (