Certificate work, and add an Endpoint attributes field to Locator for future use.

core/Locator.cpp

@@ -28,21 +28,35 @@ Locator::Locator(const char *const str) noexcept
 	}
 }
 
-bool Locator::add(const Endpoint &ep)
+bool Locator::add(const Endpoint &ep, const EndpointAttributes &a)
 {
+	for (Vector< std::pair< Endpoint, EndpointAttributes > >::iterator i(m_endpoints.begin());i!=m_endpoints.end();++i) {
+		if (i->first == ep) {
+			i->second = a;
+			return true;
+		}
+	}
 	if (m_endpoints.size() < ZT_LOCATOR_MAX_ENDPOINTS) {
-		if (std::find(m_endpoints.begin(), m_endpoints.end(), ep) == m_endpoints.end())
-			m_endpoints.push_back(ep);
+		m_endpoints.push_back(std::pair<Endpoint, EndpointAttributes>(ep, a));
 		return true;
 	}
 	return false;
 }
 
+struct p_SortByEndpoint
+{
+	// There can't be more than one of the same endpoint, so only need to sort
+	// by endpoint.
+	ZT_INLINE bool operator()(const std::pair< Endpoint, Locator::EndpointAttributes > &a,const std::pair< Endpoint, Locator::EndpointAttributes > &b) const noexcept
+	{ return a.first < b.first; }
+};
+
 bool Locator::sign(const int64_t ts, const Identity &id) noexcept
 {
 	m_ts = ts;
 	m_signer = id.fingerprint();
-	std::sort(m_endpoints.begin(), m_endpoints.end());
+
+	std::sort(m_endpoints.begin(), m_endpoints.end(), p_SortByEndpoint());
 
 	uint8_t signdata[ZT_LOCATOR_MARSHAL_SIZE_MAX];
 	const unsigned int signlen = marshal(signdata, true);
@@ -102,11 +116,15 @@ int Locator::marshal(uint8_t data[ZT_LOCATOR_MARSHAL_SIZE_MAX], const bool exclu
 
 	Utils::storeBigEndian<uint16_t>(data + p, (uint16_t) m_endpoints.size());
 	p += 2;
-	for (Vector<Endpoint>::const_iterator e(m_endpoints.begin());e != m_endpoints.end();++e) {
-		l = e->marshal(data + p);
+	for (Vector< std::pair< Endpoint, EndpointAttributes> >::const_iterator e(m_endpoints.begin());e != m_endpoints.end();++e) {
+		l = e->first.marshal(data + p);
 		if (l <= 0)
 			return -1;
 		p += l;
+
+		l = (int)e->second.data[0] + 1;
+		Utils::copy(data + p, e->second.data, (unsigned int)l);
+		p += l;
 	}
 
 	Utils::storeMachineEndian< uint16_t >(data + p, 0); // length of meta-data, currently always 0
@@ -143,10 +161,14 @@ int Locator::unmarshal(const uint8_t *data, const int len) noexcept
 	m_endpoints.resize(endpointCount);
 	m_endpoints.shrink_to_fit();
 	for (unsigned int i = 0;i < endpointCount;++i) {
-		l = m_endpoints[i].unmarshal(data + p, len - p);
+		l = m_endpoints[i].first.unmarshal(data + p, len - p);
 		if (l <= 0)
 			return -1;
 		p += l;
+
+		l = (int)data[p] + 1;
+		Utils::copy(m_endpoints[i].second.data, data + p, (unsigned int)l);
+		p += l;
 	}
 
 	if (unlikely((p + 2) > len))
@@ -175,15 +197,17 @@ extern "C" {
 ZT_Locator *ZT_Locator_create(
 	int64_t ts,
 	const ZT_Endpoint *endpoints,
+	const ZT_EndpointAttributes *endpointAttributes, // TODO: not used yet
 	unsigned int endpointCount,
 	const ZT_Identity *signer)
 {
 	try {
 		if ((ts <= 0) || (!endpoints) || (endpointCount == 0) || (!signer))
 			return nullptr;
+		ZeroTier::Locator::EndpointAttributes emptyAttributes;
 		ZeroTier::Locator *loc = new ZeroTier::Locator();
 		for (unsigned int i = 0;i < endpointCount;++i)
-			loc->add(reinterpret_cast<const ZeroTier::Endpoint *>(endpoints)[i]);
+			loc->add(reinterpret_cast<const ZeroTier::Endpoint *>(endpoints)[i], emptyAttributes);
 		if (!loc->sign(ts, *reinterpret_cast<const ZeroTier::Identity *>(signer))) {
 			delete loc;
 			return nullptr;

core/Locator.hpp

@@ -21,10 +21,26 @@
 #include "SharedPtr.hpp"
 #include "FCV.hpp"
 #include "Containers.hpp"
+#include "Dictionary.hpp"
 
-#define ZT_LOCATOR_MAX_ENDPOINTS 8
-#define ZT_LOCATOR_MARSHAL_SIZE_MAX (8 + ZT_FINGERPRINT_MARSHAL_SIZE + 2 + (ZT_LOCATOR_MAX_ENDPOINTS * ZT_ENDPOINT_MARSHAL_SIZE_MAX) + 2 + 2 + ZT_SIGNATURE_BUFFER_SIZE)
-#define ZT_LOCATOR_STRING_SIZE_MAX 4096
+/**
+ * Maximum size of endpoint attributes dictionary plus one byte for size.
+ *
+ * This cannot be (easily) changed.
+ */
+#define ZT_LOCATOR_MAX_ENDPOINT_ATTRIBUTES_SIZE 256
+
+/**
+ * Maximum number of endpoints, which can be increased.
+ */
+#define ZT_LOCATOR_MAX_ENDPOINTS 16
+
+#define ZT_LOCATOR_MARSHAL_SIZE_MAX (8 + ZT_FINGERPRINT_MARSHAL_SIZE + 2 + (ZT_LOCATOR_MAX_ENDPOINTS * (ZT_ENDPOINT_MARSHAL_SIZE_MAX + ZT_LOCATOR_MAX_ENDPOINT_ATTRIBUTES_SIZE)) + 2 + 2 + ZT_SIGNATURE_BUFFER_SIZE)
+
+/**
+ * Maximum size of a string format Locator (this is way larger than needed)
+ */
+#define ZT_LOCATOR_STRING_SIZE_MAX 16384
 
 namespace ZeroTier {
 
@@ -36,17 +52,56 @@ namespace ZeroTier {
  */
 class Locator
 {
-	friend class SharedPtr<Locator>;
-	friend class SharedPtr<const Locator>;
+	friend class SharedPtr< Locator >;
+
+	friend class SharedPtr< const Locator >;
 
 public:
-	ZT_INLINE Locator() noexcept :
+	/**
+	 * Attributes of an endpoint in this locator
+	 *
+	 * This is specified for future use, but there are currently no attributes
+	 * defined. A Dictionary is used for serialization for extensibility.
+	 */
+	struct EndpointAttributes
+	{
+		/**
+		 * Raw attributes data in the form of a dictionary prefixed by its size.
+		 *
+		 * The maximum size of attributes is 256, which is more than enough for
+		 * tiny things like bandwidth and priority.
+		 */
+		uint8_t data[ZT_LOCATOR_MAX_ENDPOINT_ATTRIBUTES_SIZE];
+
+		ZT_INLINE EndpointAttributes() noexcept
+		{ Utils::zero< ZT_LOCATOR_MAX_ENDPOINT_ATTRIBUTES_SIZE >(data); }
+
+		ZT_INLINE bool operator==(const EndpointAttributes &a) const noexcept
+		{ return ((data[0] == a.data[0]) && (memcmp(data, a.data, data[0]) == 0)); }
+
+		ZT_INLINE bool operator<(const EndpointAttributes &a) const noexcept
+		{ return ((data[0] < a.data[0]) || ((data[0] == a.data[0]) && (memcmp(data, a.data, data[0]) < 0))); }
+
+		ZT_INLINE bool operator!=(const EndpointAttributes &a) const noexcept
+		{ return !(*this == a); }
+
+		ZT_INLINE bool operator>(const EndpointAttributes &a) const noexcept
+		{ return (a < *this); }
+
+		ZT_INLINE bool operator<=(const EndpointAttributes &a) const noexcept
+		{ return !(a < *this); }
+
+		ZT_INLINE bool operator>=(const EndpointAttributes &a) const noexcept
+		{ return !(*this < a); }
+	};
+
+	ZT_INLINE Locator() noexcept:
 		m_ts(0)
 	{}
 
 	explicit Locator(const char *const str) noexcept;
 
-	ZT_INLINE Locator(const Locator &loc) noexcept :
+	ZT_INLINE Locator(const Locator &loc) noexcept:
 		m_ts(loc.m_ts),
 		m_signer(loc.m_signer),
 		m_endpoints(loc.m_endpoints),
@@ -69,13 +124,13 @@ public:
 	/**
 	 * @return Endpoints specified in locator
 	 */
-	ZT_INLINE const Vector<Endpoint> &endpoints() const noexcept
+	ZT_INLINE const Vector< std::pair< Endpoint, EndpointAttributes > > &endpoints() const noexcept
 	{ return m_endpoints; }
 
 	/**
 	 * @return Signature data
 	 */
-	ZT_INLINE const FCV<uint8_t, ZT_SIGNATURE_BUFFER_SIZE> &signature() const noexcept
+	ZT_INLINE const FCV< uint8_t, ZT_SIGNATURE_BUFFER_SIZE > &signature() const noexcept
 	{ return m_signature; }
 
 	/**
@@ -85,9 +140,10 @@ public:
 	 * care not to add duplicates.
 	 *
 	 * @param ep Endpoint to add
+	 * @param a Endpoint attributes
 	 * @return True if endpoint was added (or already present), false if locator is full
 	 */
-	bool add(const Endpoint &ep);
+	bool add(const Endpoint &ep, const EndpointAttributes &a);
 
 	/**
 	 * Sign this locator
@@ -117,7 +173,10 @@ public:
 	char *toString(char s[ZT_LOCATOR_STRING_SIZE_MAX]) const noexcept;
 
 	ZT_INLINE String toString() const
-	{ char tmp[ZT_LOCATOR_STRING_SIZE_MAX]; return String(toString(tmp)); }
+	{
+		char tmp[ZT_LOCATOR_STRING_SIZE_MAX];
+		return String(toString(tmp));
+	}
 
 	/**
 	 * Decode a string format locator
@@ -130,8 +189,11 @@ public:
 	explicit ZT_INLINE operator bool() const noexcept
 	{ return m_ts > 0; }
 
-	static constexpr int marshalSizeMax() noexcept { return ZT_LOCATOR_MARSHAL_SIZE_MAX; }
+	static constexpr int marshalSizeMax() noexcept
+	{ return ZT_LOCATOR_MARSHAL_SIZE_MAX; }
+
 	int marshal(uint8_t data[ZT_LOCATOR_MARSHAL_SIZE_MAX], bool excludeSignature = false) const noexcept;
+
 	int unmarshal(const uint8_t *data, int len) noexcept;
 
 	ZT_INLINE bool operator==(const Locator &l) const noexcept
@@ -142,15 +204,16 @@ public:
 			(m_endpoints == l.m_endpoints) &&
 			(m_signature == l.m_signature));
 	}
+
 	ZT_INLINE bool operator!=(const Locator &l) const noexcept
 	{ return !(*this == l); }
 
 private:
 	int64_t m_ts;
 	Fingerprint m_signer;
-	Vector<Endpoint> m_endpoints;
-	FCV<uint8_t, ZT_SIGNATURE_BUFFER_SIZE> m_signature;
-	std::atomic<int> __refCount;
+	Vector< std::pair< Endpoint, EndpointAttributes > > m_endpoints;
+	FCV< uint8_t, ZT_SIGNATURE_BUFFER_SIZE > m_signature;
+	std::atomic< int > __refCount;
 };
 
 } // namespace ZeroTier

core/Node.cpp

@@ -564,6 +564,26 @@ int Node::tryPeer(
 	return 0;
 }
 
+ZT_CertificateError Node::addCertificate(
+	void *tptr,
+	int64_t now,
+	unsigned int localTrust,
+	const ZT_Certificate *cert,
+	const void *certData,
+	unsigned int certSize)
+{
+	Certificate c;
+	if (cert) {
+		c = *cert;
+	} else {
+		if ((!certData) || (!certSize))
+			return ZT_CERTIFICATE_ERROR_INVALID_FORMAT;
+		if (!c.decode(certData, certSize))
+			return ZT_CERTIFICATE_ERROR_INVALID_FORMAT;
+	}
+	return RR->topology->addCertificate(tptr, c, now, localTrust, true, true, true);
+}
+
 int Node::sendUserMessage(
 	void *tptr,
 	uint64_t dest,
@@ -1030,6 +1050,22 @@ int ZT_Node_tryPeer(
 	}
 }
 
+enum ZT_CertificateError ZT_Node_addCertificate(
+	ZT_Node *node,
+	void *tptr,
+	int64_t now,
+	unsigned int localTrust,
+	const ZT_Certificate *cert,
+	const void *certData,
+	unsigned int certSize)
+{
+	try {
+		return reinterpret_cast<ZeroTier::Node *>(node)->addCertificate(tptr, now, localTrust, cert, certData, certSize);
+	} catch (...) {
+		return ZT_CERTIFICATE_ERROR_INVALID_FORMAT;
+	}
+}
+
 void ZT_Node_setNetworkUserPtr(ZT_Node *node, uint64_t nwid, void *ptr)
 {
 	try {

core/Node.hpp

@@ -139,6 +139,14 @@ public:
 		const ZT_Endpoint *endpoint,
 		int retries);
 
+	ZT_CertificateError addCertificate(
+		void *tptr,
+		int64_t now,
+		unsigned int localTrust,
+		const ZT_Certificate *cert,
+		const void *certData,
+		unsigned int certSize);
+
 	int sendUserMessage(
 		void *tptr,
 		uint64_t dest,

core/Peer.cpp

@@ -254,14 +254,14 @@ void Peer::pulse(void *const tPtr, const int64_t now, const bool isRoot)
 			// callback (if one was supplied).
 
 			if (m_locator) {
-				for (Vector< Endpoint >::const_iterator ep(m_locator->endpoints().begin()); ep != m_locator->endpoints().end(); ++ep) {
-					if (ep->type == ZT_ENDPOINT_TYPE_IP_UDP) {
-						if (RR->node->shouldUsePathForZeroTierTraffic(tPtr, m_id, -1, ep->ip())) {
-							int64_t &lt = m_lastTried[*ep];
+				for (Vector< std::pair<Endpoint, Locator::EndpointAttributes > >::const_iterator ep(m_locator->endpoints().begin()); ep != m_locator->endpoints().end(); ++ep) {
+					if (ep->first.type == ZT_ENDPOINT_TYPE_IP_UDP) {
+						if (RR->node->shouldUsePathForZeroTierTraffic(tPtr, m_id, -1, ep->first.ip())) {
+							int64_t &lt = m_lastTried[ep->first];
 							if ((now - lt) > ZT_PATH_MIN_TRY_INTERVAL) {
 								lt = now;
-								RR->t->tryingNewPath(tPtr, 0x84b22322, m_id, ep->ip(), InetAddress::NIL, 0, 0, Identity::NIL);
-								sent(now, m_sendProbe(tPtr, -1, ep->ip(), nullptr, 0, now));
+								RR->t->tryingNewPath(tPtr, 0x84b22322, m_id, ep->first.ip(), InetAddress::NIL, 0, 0, Identity::NIL);
+								sent(now, m_sendProbe(tPtr, -1, ep->first.ip(), nullptr, 0, now));
 							}
 						}
 					}

core/Tests.cpp

@@ -898,8 +898,8 @@ extern "C" const char *ZTT_general()
 			Endpoint ep0(InetAddress::LO4);
 			Endpoint ep1(InetAddress::LO6);
 			Locator loc;
-			loc.add(ep0);
-			loc.add(ep1);
+			loc.add(ep0, Locator::EndpointAttributes());
+			loc.add(ep1, Locator::EndpointAttributes());
 			loc.sign(now(), v1id);
 			String locStr(loc.toString());
 			//ZT_T_PRINTF("%s %s %s ",locStr.c_str(),loc.endpoints()[0].toString().c_str(),loc.endpoints()[1].toString().c_str());

core/Topology.cpp

@@ -77,13 +77,16 @@ void Topology::allPeers(Vector< SharedPtr< Peer > > &allPeers, Vector< SharedPtr
 
 void Topology::doPeriodicTasks(void *tPtr, const int64_t now)
 {
-	// Clean any expired certificates
+	// Clean any expired certificates, updating roots if they have changed.
 	{
 		Mutex::Lock l1(m_certs_l);
 		if (m_cleanCertificates(tPtr, now)) {
-			RWMutex::Lock l3(m_peers_l);
-			RWMutex::Lock l2(m_roots_l);
-			m_updateRootPeers(tPtr, now);
+			m_writeTrustStore(tPtr);
+			{
+				RWMutex::Lock l3(m_peers_l);
+				RWMutex::Lock l2(m_roots_l);
+				m_updateRootPeers(tPtr, now);
+			}
 		}
 	}
 
@@ -109,19 +112,24 @@ void Topology::doPeriodicTasks(void *tPtr, const int64_t now)
 			for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end(); ++i) {
 				// TODO: also delete if the peer has not exchanged meaningful communication in a while, such as
 				// a network frame or non-trivial control packet.
-				if (((now - i->second->lastReceive()) > ZT_PEER_ALIVE_TIMEOUT) && (std::find(rootLookup.begin(), rootLookup.end(), (uintptr_t)i->second.ptr()) == rootLookup.end()))
+				if (((now - i->second->lastReceive()) > ZT_PEER_ALIVE_TIMEOUT) && (std::find(rootLookup.begin(), rootLookup.end(), (uintptr_t)(i->second.ptr())) == rootLookup.end()))
 					toDelete.push_back(i->first);
 			}
 		}
 		if (!toDelete.empty()) {
 			ZT_SPEW("garbage collecting %u offline or stale peer objects", (unsigned int)toDelete.size());
 			for (Vector< Address >::iterator i(toDelete.begin()); i != toDelete.end(); ++i) {
-				RWMutex::Lock l1(m_peers_l);
-				const Map< Address, SharedPtr< Peer > >::iterator p(m_peers.find(*i));
-				if (likely(p != m_peers.end())) {
-					p->second->save(tPtr);
-					m_peers.erase(p);
+				SharedPtr< Peer > toSave;
+				{
+					RWMutex::Lock l1(m_peers_l);
+					const Map< Address, SharedPtr< Peer > >::iterator p(m_peers.find(*i));
+					if (p != m_peers.end()) {
+						p->second.swap(toSave);
+						m_peers.erase(p);
+					}
 				}
+				if (toSave)
+					toSave->save(tPtr);
 			}
 		}
 	}
@@ -156,97 +164,75 @@ void Topology::saveAll(void *tPtr)
 {
 	{
 		RWMutex::RLock l(m_peers_l);
-		for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end(); ++i) {
+		for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end(); ++i)
 			i->second->save(tPtr);
-		}
 	}
 	{
-		char tmp[32];
-		Dictionary d;
-		{
-			Mutex::Lock l(m_certs_l);
-			unsigned long idx = 0;
-			d.add("c$", (uint64_t)m_certs.size());
-			for (Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > >::const_iterator c(m_certs.begin()); c != m_certs.end(); ++c) {
-				d[Dictionary::arraySubscript(tmp, sizeof(tmp), "c$.s", idx)].assign(c->first.data, c->first.data + ZT_SHA384_DIGEST_SIZE);
-				d.add(Dictionary::arraySubscript(tmp, sizeof(tmp), "c$.lt", idx), (uint64_t)c->second.second);
-				++idx;
-			}
-		}
-		Vector< uint8_t > trustStore;
-		d.encode(trustStore);
-		RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_TRUST_STORE, Utils::ZERO256, trustStore.data(), (unsigned int)trustStore.size());
+		Mutex::Lock l(m_certs_l);
+		m_writeTrustStore(tPtr);
 	}
 }
 
 ZT_CertificateError Topology::addCertificate(void *tPtr, const Certificate &cert, const int64_t now, const unsigned int localTrust, const bool writeToLocalStore, const bool refreshRootSets, const bool verify)
 {
 	{
-		Mutex::Lock l1(m_certs_l);
-
-		// Check to see if we already have this specific certificate.
 		const SHA384Hash serial(cert.serialNo);
-		if (m_certs.find(serial) != m_certs.end())
-			return ZT_CERTIFICATE_ERROR_NONE;
+		p_CertEntry certEntry;
+		Mutex::Lock l1(m_certs_l);
 
-		// Verify certificate all the way to a trusted root. This also verifies inner
-		// signatures such as those of locators or the subject unique ID.
-		if (verify) {
-			const ZT_CertificateError err = m_verifyCertificate(cert, now, localTrust, false);
-			if (err != ZT_CERTIFICATE_ERROR_NONE)
-				return err;
+		{
+			Map< SHA384Hash, p_CertEntry >::iterator c(m_certs.find(serial));
+			if (c != m_certs.end()) {
+				if (c->second.localTrust == localTrust)
+					return ZT_CERTIFICATE_ERROR_NONE;
+				certEntry.certificate = c->second.certificate;
+			}
+		}
+		if (!certEntry.certificate) {
+			certEntry.certificate.set(new Certificate(cert));
+			if (verify) {
+				m_cleanCertificates(tPtr, now);
+				const ZT_CertificateError err = m_verifyCertificate(cert, now, localTrust, false);
+				if (err != ZT_CERTIFICATE_ERROR_NONE)
+					return err;
+			}
 		}
 
-		// Create entry containing copy of certificate and trust flags.
-		const std::pair< SharedPtr< const Certificate >, unsigned int > certEntry(SharedPtr< const Certificate >(new Certificate(cert)), localTrust);
+		certEntry.localTrust = localTrust;
 
-		// If the subject contains a unique ID, check if we already have a cert for the
-		// same uniquely identified subject. If so, check its subject timestamp and keep
-		// the one we have if newer. Otherwise replace it. Note that the verification
-		// function will have checked the unique ID proof signature already if a unique
-		// ID was present.
 		if ((cert.subject.uniqueId) && (cert.subject.uniqueIdSize > 0)) {
 			SHA384Hash uniqueIdHash;
 			SHA384(uniqueIdHash.data, cert.subject.uniqueId, cert.subject.uniqueIdSize);
-			std::pair< SharedPtr< const Certificate >, unsigned int > &bySubjectUniqueId = m_certsBySubjectUniqueId[uniqueIdHash];
-			if (bySubjectUniqueId.first) {
-				if (bySubjectUniqueId.first->subject.timestamp >= cert.subject.timestamp)
+			p_CertEntry &bySubjectUniqueId = m_certsBySubjectUniqueID[uniqueIdHash];
+			if (bySubjectUniqueId.certificate) {
+				if (bySubjectUniqueId.certificate->subject.timestamp >= cert.subject.timestamp)
 					return ZT_CERTIFICATE_ERROR_HAVE_NEWER_CERT;
-				m_eraseCertificate(tPtr, bySubjectUniqueId.first, &uniqueIdHash);
-				m_certsBySubjectUniqueId[uniqueIdHash] = certEntry;
+				m_eraseCertificate(tPtr, bySubjectUniqueId.certificate, &uniqueIdHash);
+				m_certsBySubjectUniqueID[uniqueIdHash] = certEntry;
 			} else {
 				bySubjectUniqueId = certEntry;
 			}
 		}
 
-		// Save certificate by serial number.
-		m_certs[serial] = certEntry;
-
-		// Add certificate to sets of certificates whose subject references a given identity.
 		for (unsigned int i = 0; i < cert.subject.identityCount; ++i) {
 			const Identity *const ii = reinterpret_cast<const Identity *>(cert.subject.identities[i].identity);
 			if (ii)
-				m_certsBySubjectIdentity[ii->fingerprint()].insert(certEntry);
+				m_certsBySubjectIdentity[ii->fingerprint()][certEntry.certificate] = localTrust;
 		}
 
-		// Clean any certificates whose chains are now broken, which can happen if there was
-		// an update that replaced an old cert with a given unique ID. Otherwise this generally
-		// does nothing here. Skip if verify is false since this means we're mindlessly loading
-		// certificates, which right now only happens on startup when they're loaded from the
-		// local certificate cache.
-		if (verify)
-			m_cleanCertificates(tPtr, now);
+		m_certs[serial] = certEntry;
 
-		// Refresh the root peers lists, since certs may enumerate roots.
 		if (refreshRootSets) {
 			RWMutex::Lock l3(m_peers_l);
 			RWMutex::Lock l2(m_roots_l);
 			m_updateRootPeers(tPtr, now);
 		}
+
+		if (writeToLocalStore)
+			m_writeTrustStore(tPtr);
 	}
 
 	if (writeToLocalStore) {
-		// Write certificate data prefixed by local trust flags as a 32-bit integer.
 		Vector< uint8_t > certData(cert.encode());
 		uint64_t id[6];
 		Utils::copy< 48 >(id, cert.serialNo);
@@ -293,10 +279,8 @@ void Topology::m_eraseCertificate(void *tPtr, const SharedPtr< const Certificate
 	const SHA384Hash serialNo(cert->serialNo);
 	m_certs.erase(serialNo);
 
-	RR->node->stateObjectDelete(tPtr, ZT_STATE_OBJECT_CERT, serialNo.data);
-
 	if (uniqueIdHash)
-		m_certsBySubjectUniqueId.erase(*uniqueIdHash);
+		m_certsBySubjectUniqueID.erase(*uniqueIdHash);
 
 	for (unsigned int i = 0; i < cert->subject.identityCount; ++i) {
 		const Identity *const ii = reinterpret_cast<const Identity *>(cert->subject.identities[i].identity);
@@ -308,6 +292,8 @@ void Topology::m_eraseCertificate(void *tPtr, const SharedPtr< const Certificate
 				m_certsBySubjectIdentity.erase(bySubjectIdentity);
 		}
 	}
+
+	RR->node->stateObjectDelete(tPtr, ZT_STATE_OBJECT_CERT, serialNo.data);
 }
 
 bool Topology::m_cleanCertificates(void *tPtr, int64_t now)
@@ -317,13 +303,13 @@ bool Topology::m_cleanCertificates(void *tPtr, int64_t now)
 	bool deleted = false;
 	Vector< SharedPtr< const Certificate >> toDelete;
 	for (;;) {
-		for (Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > >::iterator c(m_certs.begin()); c != m_certs.end(); ++c) {
+		for (Map< SHA384Hash, p_CertEntry >::iterator c(m_certs.begin()); c != m_certs.end(); ++c) {
 			// Verify, but the last boolean option tells it to skip signature checks as this would
 			// already have been done. This will therefore just check the path and validity times
 			// of the certificate.
-			const ZT_CertificateError err = m_verifyCertificate(*(c->second.first), now, c->second.second, true);
+			const ZT_CertificateError err = m_verifyCertificate(*(c->second.certificate), now, c->second.localTrust, true);
 			if (err != ZT_CERTIFICATE_ERROR_NONE)
-				toDelete.push_back(c->second.first);
+				toDelete.push_back(c->second.certificate);
 		}
 
 		if (toDelete.empty())
@@ -481,4 +467,24 @@ void Topology::m_updateRootPeers(void *tPtr, const int64_t now)
 	m_rankRoots(now);
 }
 
+void Topology::m_writeTrustStore(void *tPtr)
+{
+	// assumes m_certs is locked
+
+	char tmp[32];
+	Dictionary d;
+
+	unsigned long idx = 0;
+	d.add("c$", (uint64_t)m_certs.size());
+	for (Map< SHA384Hash, p_CertEntry >::const_iterator c(m_certs.begin()); c != m_certs.end(); ++c) {
+		d[Dictionary::arraySubscript(tmp, sizeof(tmp), "c$.s", idx)].assign(c->first.data, c->first.data + ZT_SHA384_DIGEST_SIZE);
+		d.add(Dictionary::arraySubscript(tmp, sizeof(tmp), "c$.lt", idx), (uint64_t)c->second.localTrust);
+		++idx;
+	}
+
+	Vector< uint8_t > trustStore;
+	d.encode(trustStore);
+	RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_TRUST_STORE, Utils::ZERO256, trustStore.data(), (unsigned int)trustStore.size());
+}
+
 } // namespace ZeroTier

core/Topology.hpp

@@ -152,6 +152,7 @@ private:
 	SharedPtr< Peer > m_peerFromCached(void *tPtr, const Address &zta);
 	SharedPtr< Path > m_newPath(const int64_t l, const InetAddress &r, const UniqueID &k);
 	void m_updateRootPeers(void *tPtr, int64_t now);
+	void m_writeTrustStore(void *tPtr);
 
 	const RuntimeEnvironment *const RR;
 
@@ -160,9 +161,19 @@ private:
 	Map< Address, SharedPtr< Peer > > m_peers;
 	Map< UniqueID, SharedPtr< Path > > m_paths;
 
-	Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > > m_certs;
+	struct p_CertEntry
+	{
+		ZT_INLINE p_CertEntry() :
+			certificate(),
+			localTrust(0)
+		{}
+		SharedPtr< const Certificate > certificate;
+		unsigned int localTrust;
+	};
+
+	Map< SHA384Hash, p_CertEntry > m_certs;
+	Map< SHA384Hash, p_CertEntry > m_certsBySubjectUniqueID;
 	Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > > m_certsBySubjectIdentity;
-	Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > > m_certsBySubjectUniqueId;
 
 	RWMutex m_paths_l; // m_paths
 	RWMutex m_peers_l; // m_peers

core/zerotier.h

@@ -1492,6 +1492,14 @@ typedef struct
 	} value;
 } ZT_Endpoint;
 
+/**
+ * Endpoint attributes
+ *
+ * Right now this is typedef'd to void because there are none. It will become
+ * a struct once there's something to specify.
+ */
+typedef void ZT_EndpointAttributes;
+
 /**
  * Network path to a peer
  */
@@ -2361,6 +2369,29 @@ ZT_SDK_API int ZT_Node_tryPeer(
 	const ZT_Endpoint *endpoint,
 	int retries);
 
+/**
+ * Add a certificate to this node's certificate store
+ *
+ * This supports adding of certificates as expanded ZT_Certificate structures
+ * or as raw data. If 'cert' is NULL then certData/certSize must be set.
+ *
+ * @param node Node instance
+ * @param tptr Thread pointer to pass to functions/callbacks resulting from this call
+ * @param localTrust Local trust flags (ORed together)
+ * @param cert Certificate object, or set to NULL if certData and certSize are to be used
+ * @param certData Certificate binary data if 'cert' is NULL, NULL otherwise
+ * @param certSize Size of certificate binary data, 0 if none
+ * @return
+ */
+ZT_SDK_API enum ZT_CertificateError ZT_Node_addCertificate(
+	ZT_Node *node,
+	void *tptr,
+	int64_t now,
+	unsigned int localTrust,
+	const ZT_Certificate *cert,
+	const void *certData,
+	unsigned int certSize);
+
 /**
  * Send a VERB_USER_MESSAGE to another ZeroTier node
  *
@@ -2561,8 +2592,12 @@ ZT_SDK_API int ZT_Endpoint_fromString(
 /**
  * Create and sign a new locator
  *
+ * Note that attributes must be either NULL to use defaults for all or there
+ * must be an attributes object for each endpoint.
+ *
  * @param ts Locator timestamp
  * @param endpoints List of endpoints to store in locator
+ * @param endpointAttributes Array of ZT_EndpointAttributes objects or NULL to use defaults
  * @param endpointCount Number of endpoints (maximum: 8)
  * @param signer Identity to sign locator (must include private key)
  * @return Locator or NULL on error (too many endpoints or identity does not have private key)
@@ -2570,6 +2605,7 @@ ZT_SDK_API int ZT_Endpoint_fromString(
 ZT_SDK_API ZT_Locator *ZT_Locator_create(
 	int64_t ts,
 	const ZT_Endpoint *endpoints,
+	const ZT_EndpointAttributes *endpointAttributes,
 	unsigned int endpointCount,
 	const ZT_Identity *signer);
 
@@ -2807,7 +2843,7 @@ ZT_SDK_API enum ZT_CertificateError ZT_Certificate_verify(const ZT_Certificate *
 ZT_SDK_API const ZT_Certificate *ZT_Certificate_clone(const ZT_Certificate *cert);
 
 /**
- * Free a certificate created with ZT_Certificate_decode()
+ * Free a certificate created with ZT_Certificate_decode() or ZT_Certificate_clone()
  *
  * @param cert Certificate to free
  */

pkg/zerotier/locator.go

@@ -48,7 +48,7 @@ func NewLocator(ts int64, endpoints []Endpoint, signer *Identity) (*Locator, err
 	for _, e := range endpoints {
 		eps = append(eps, e.cep)
 	}
-	loc := C.ZT_Locator_create(C.int64_t(ts), &eps[0], C.uint(len(eps)), signer.cIdentity())
+	loc := C.ZT_Locator_create(C.int64_t(ts), &eps[0], nil, C.uint(len(eps)), signer.cIdentity())
 	if uintptr(loc) == 0 {
 		return nil, ErrInvalidParameter
 	}