More cert work, support certs in data store, and manually merge CMakefile.

CMakeLists.txt

@@ -1,10 +1,7 @@
 cmake_minimum_required (VERSION 3.8)
 project(zerotier DESCRIPTION "ZeroTier Network Hypervisor" LANGUAGES CXX C)
 
-set(ZEROTIER_VERSION_MAJOR 1 CACHE INTERNAL "")
-set(ZEROTIER_VERSION_MINOR 9 CACHE INTERNAL "")
-set(ZEROTIER_VERSION_REVISION 0 CACHE INTERNAL "")
-set(ZEROTIER_VERSION_BUILD 0 CACHE INTERNAL "")
+cmake_policy(SET CMP0048 NEW)
 
 if(${CMAKE_VERSION} VERSION_LESS 3.15)
 	cmake_policy(VERSION ${CMAKE_MAJOR_VERSION}.${CMAKE_MINOR_VERSION})
@@ -12,121 +9,75 @@ else()
 	cmake_policy(VERSION 3.15)
 endif()
 
-find_program(
-	GO go
-	HINTS "/usr/local/go/bin" "C:/go/bin"
-)
-
-if(NOT GO)
-	message(FATAL_ERROR "Golang not found")
-else(NOT GO)
-	message(STATUS "Found Golang at ${GO}")
-endif(NOT GO)
-
-set(CMAKE_CXX_STANDARD 11)
-set(default_build_type "Release")
-
-if(WIN32)
-	set(CMAKE_SYSTEM_VERSION "7" CACHE STRING INTERNAL FORCE)
-endif(WIN32)
-
-if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
-	message(STATUS "Setting build type to '${default_build_type}' as none was specified.")
-	set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING "Choose the type of build." FORCE)
-	set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
-endif()
-
-option(BUILD_CENTRAL_CONTROLLER "Build ZeroTier Central Controller" OFF)
-if(BUILD_CENTRAL_CONTROLLER)
-	find_package(PkgConfig REQUIRED)
-	if(APPLE)
-		set(CMAKE_PREFIX_PATH 
-			${CMAKE_PREFIX_PATH}
-			/usr/local/opt/libpq
-			/usr/local/lib
-		)
-	endif(APPLE)
-	find_package(PostgreSQL REQUIRED)
-	
-	pkg_check_modules(hiredis REQUIRED IMPORTED_TARGET hiredis)
-	
-	add_subdirectory(controller/thirdparty/redis-plus-plus-1.1.1)
-	set(redispp_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/controller/thirdparty/redis-plus-plus-1.1.1/src/sw)
-	set(redispp_STATIC_LIB redispp_static)
-endif(BUILD_CENTRAL_CONTROLLER)
-
-if(CMAKE_BUILD_TYPE STREQUAL "Debug")
-	add_definitions(-DZT_DEBUG)
-endif(CMAKE_BUILD_TYPE STREQUAL "Debug")
-
-if(WIN32)
-
-	message("++ Setting Windows Compiler Flags ${CMAKE_BUILD_TYPE}")
-
-	add_definitions(-DNOMINMAX)
-	add_compile_options(
-		-Wall
-		-Wno-deprecated
-		-Wno-unused-function
-		-Wno-format
-		$<$<CONFIG:DEBUG>:-g>
-		$<$<CONFIG:DEBUG>:-O0>
-		$<$<CONFIG:RELEASE>:-O3>
-		$<$<CONFIG:RELEASE>:-ffast-math>
-		$<$<CONFIG:RELWITHDEBINFO>:-O3>
-		$<$<CONFIG:RELWITHDEBINFO>:-g>
-	)
+set(ZEROTIER_VERSION_MAJOR 1 CACHE INTERNAL "")
+set(ZEROTIER_VERSION_MINOR 9 CACHE INTERNAL "")
+set(ZEROTIER_VERSION_REVISION 0 CACHE INTERNAL "")
+set(ZEROTIER_VERSION_BUILD 0 CACHE INTERNAL "")
 
-	set(GOFLAGS
-		-a
-		-trimpath
-		-ldflags '-w'
-	)
+project(zerotier
+	VERSION ${ZEROTIER_VERSION_MAJOR}.${ZEROTIER_VERSION_MINOR}.${ZEROTIER_VERSION_REVISION}.${ZEROTIER_VERSION_BUILD}
+	DESCRIPTION "ZeroTier Network Hypervisor"
+	LANGUAGES CXX C)
 
-else(WIN32)
+if(NOT PACKAGE_STATIC)
 
-	set(GOFLAGS
-		-trimpath
-		-buildmode=pie
+	find_program(
+		GO go
+		HINTS "/usr/local/go/bin" "/usr/bin" "/usr/local/bin" "C:/go/bin"
 	)
+	if(NOT GO)
+		message(FATAL_ERROR "Golang not found")
+	else(NOT GO)
+		message(STATUS "Found Golang at ${GO}")
+	endif(NOT GO)
+
+	set(default_build_type "Release")
+
+	if(WIN32)
+		set(CMAKE_CXX_STANDARD 17)
+		set(CMAKE_SYSTEM_VERSION "7" CACHE STRING INTERNAL FORCE)
+	else(WIN32)
+		if(APPLE)
+			set(CMAKE_CXX_STANDARD 17)
+		else(APPLE)
+			set(CMAKE_CXX_STANDARD 11)
+		endif(APPLE)
+	endif(WIN32)
+
+	if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+		message(STATUS "Setting build type to '${default_build_type}' as none was specified.")
+		set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE STRING "Choose the type of build." FORCE)
+		set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS "Debug" "Release" "MinSizeRel" "RelWithDebInfo")
+	endif()
 
-	set(MACOS_VERSION_MIN "10.12")
-
-	if(APPLE)
-
-		message("++ Setting MacOS Compiler Flags ${CMAKE_BUILD_TYPE}")
+	option(BUILD_CENTRAL_CONTROLLER "Build ZeroTier Central Controller" OFF)
+	if(BUILD_CENTRAL_CONTROLLER)
+		find_package(PkgConfig REQUIRED)
+		if(APPLE)
+			set(CMAKE_PREFIX_PATH
+				${CMAKE_PREFIX_PATH}
+				/usr/local/opt/libpq
+				/usr/local/lib
+			)
+		endif(APPLE)
+		find_package(PostgreSQL REQUIRED)
 
-		add_compile_options(
-			-Wall
-			-Wno-deprecated
-			-Wno-unused-function
-			-mmacosx-version-min=${MACOS_VERSION_MIN}
-			$<$<CONFIG:DEBUG>:-g>
-			$<$<CONFIG:DEBUG>:-O0>
-			$<$<CONFIG:RELEASE>:-Ofast>
-			$<$<CONFIG:RELEASE>:-ffast-math>
-			$<$<CONFIG:RELEASE>:-fPIE>
-			$<$<CONFIG:RELEASE>:-flto>
-			$<$<CONFIG:RELWITHDEBINFO>:-O1>
-			$<$<CONFIG:RELWITHDEBINFO>:-fPIE>
-			$<$<CONFIG:RELWITHDEBINFO>:-g>
-		)
+		pkg_check_modules(hiredis REQUIRED IMPORTED_TARGET hiredis)
 
-		add_link_options(
-			-mmacosx-version-min=${MACOS_VERSION_MIN}
-			$<$<CONFIG:RELEASE>:-flto>
-		)
+		add_subdirectory(controller/thirdparty/redis-plus-plus-1.1.1)
+		set(redispp_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/controller/thirdparty/redis-plus-plus-1.1.1/src/sw)
+		set(redispp_STATIC_LIB redispp_static)
+	endif(BUILD_CENTRAL_CONTROLLER)
 
-		set(GOFLAGS
-			${GOFLAGS}
-			-a
-			-ldflags '-w -extldflags \"-Wl,-undefined -Wl,dynamic_lookup\"'
-		)
+	if(CMAKE_BUILD_TYPE STREQUAL "Debug")
+		add_definitions(-DZT_DEBUG)
+	endif(CMAKE_BUILD_TYPE STREQUAL "Debug")
 
-	else(APPLE)
+	if(WIN32)
 
-		message("++ Setting Linux/BSD/Posix Compiler Flags (${CMAKE_BUILD_TYPE})")
+		message("++ Setting Windows Compiler Flags ${CMAKE_BUILD_TYPE}")
 
+		add_definitions(-DNOMINMAX)
 		add_compile_options(
 			-Wall
 			-Wno-deprecated
@@ -136,92 +87,227 @@ else(WIN32)
 			$<$<CONFIG:DEBUG>:-O0>
 			$<$<CONFIG:RELEASE>:-O3>
 			$<$<CONFIG:RELEASE>:-ffast-math>
-			$<$<CONFIG:RELEASE>:-fPIE>
 			$<$<CONFIG:RELWITHDEBINFO>:-O3>
-			$<$<CONFIG:RELWITHDEBINFO>:-fPIE>
 			$<$<CONFIG:RELWITHDEBINFO>:-g>
 		)
 
-		option(BUILD_32BIT "Force building as 32-bit binary" OFF)
-		option(BUILD_STATIC "Build statically linked executable" OFF)
+		set(GOFLAGS
+			-a
+			-trimpath
+			-ldflags '-w'
+		)
 
 		if(BUILD_32BIT)
 			set(CMAKE_SYSTEM_PROCESSOR "x86" CACHE STRING "system processor")
 			set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags")
-			set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags")
+			set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags")
 			set(GOARCH "GOARCH=386" CACHE STRING "go architecture")
 			add_compile_options(
 				-m32
 			)
+			add_link_options(
+				-m32
+			)
 		endif(BUILD_32BIT)
 
-		if(BUILD_STATIC)
-			add_link_options(
-				-static
+	else(WIN32)
+
+		set(GOFLAGS
+			-trimpath
+			-buildmode=pie
+		)
+
+		if(APPLE)
+
+			message("++ Setting MacOS Compiler Flags ${CMAKE_BUILD_TYPE}")
+
+			set(MACOS_VERSION_MIN "10.12")
+
+			add_compile_options(
+				-Wall
+				-Wno-deprecated
+				-Wno-unused-function
+				-mmacosx-version-min=${MACOS_VERSION_MIN}
+				$<$<CONFIG:DEBUG>:-g>
+				$<$<CONFIG:DEBUG>:-O0>
+				$<$<CONFIG:RELEASE>:-Ofast>
+				$<$<CONFIG:RELEASE>:-ffast-math>
+				$<$<CONFIG:RELEASE>:-fPIE>
+				$<$<CONFIG:RELEASE>:-flto>
+				$<$<CONFIG:RELWITHDEBINFO>:-O1>
+				$<$<CONFIG:RELWITHDEBINFO>:-fPIE>
+				$<$<CONFIG:RELWITHDEBINFO>:-g>
 			)
-			set(CMAKE_EXE_LINKER_FLAGS "-static ${CMAKE_EXE_LINKER_FLAGS}")
-			set(GOFLAGS 
-				${GOFLAGS}
-				-a 
-				-tags netgo
-				-ldflags '-w -extldflags \"-static -Wl,-unresolved-symbols=ignore-all\"'
+
+			add_link_options(
+				-mmacosx-version-min=${MACOS_VERSION_MIN}
+				$<$<CONFIG:RELEASE>:-flto>
 			)
-		else(BUILD_STATIC)
+
 			set(GOFLAGS
 				${GOFLAGS}
 				-a
-				-tags netgo
-				-ldflags '-w -extldflags \"-Wl,-unresolved-symbols=ignore-all\"'
+				-ldflags '-w -extldflags \"-Wl,-undefined -Wl,dynamic_lookup\"'
 			)
-		endif(BUILD_STATIC)
-
-	endif(APPLE)
-endif(WIN32)
-
-if (
-	CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64" OR
-	CMAKE_SYSTEM_PROCESSOR MATCHES "amd64" OR
-	CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64"
-)
-	message("++ Adding flags for processor ${CMAKE_SYSTEM_PROCESSOR}")
-	add_compile_options(-maes -mrdrnd -mpclmul -msse -msse2 -mssse3)
-endif()
 
-set(GO_BUILD_TAGS)
-
-if(BUILD_CENTRAL_CONTROLLER)
-	add_definitions(-DZT_CONTROLLER_USE_LIBPQ=1)
-	set(GO_BUILD_TAGS -tags central)
-endif(BUILD_CENTRAL_CONTROLLER)
-
-add_subdirectory(core)
-add_subdirectory(controller)
-add_subdirectory(osdep)
-add_subdirectory(serviceiocore)
-
-file(GLOB go_src 
-	${CMAKE_SOURCE_DIR}/cmd/*.go
-	${CMAKE_SOURCE_DIR}/cmd/cmd/*.go
-	${CMAKE_SOURCE_DIR}/pkg/zerotier/*.go
-)
-
-if(WIN32)
-	set(GO_EXE_NAME "zerotier.exe")
-	set(GO_EXTRA_LIBRARIES "-lstdc++ -lwsock32 -lws2_32 -liphlpapi -lole32 -loleaut32 -lrpcrt4 -luuid")
-else(WIN32)
-	set(GO_EXE_NAME "zerotier")
-	if(CMAKE_SYSTEM_NAME MATCHES "Linux")
-		set(GO_EXTRA_LIBRARIES "-lstdc++")
+		else(APPLE)
+
+			message("++ Setting Linux/BSD/Posix Compiler Flags (${CMAKE_BUILD_TYPE})")
+
+			add_compile_options(
+				-Wall
+				-Wno-deprecated
+				-Wno-unused-function
+				-Wno-format
+				$<$<CONFIG:DEBUG>:-g>
+				$<$<CONFIG:DEBUG>:-O0>
+				$<$<CONFIG:RELEASE>:-O3>
+				$<$<CONFIG:RELEASE>:-ffast-math>
+				$<$<CONFIG:RELEASE>:-fPIE>
+				$<$<CONFIG:RELWITHDEBINFO>:-O3>
+				$<$<CONFIG:RELWITHDEBINFO>:-fPIE>
+				$<$<CONFIG:RELWITHDEBINFO>:-g>
+			)
+
+			option(BUILD_32BIT "Force building as 32-bit binary" OFF)
+			option(BUILD_STATIC "Build statically linked executable" OFF)
+			option(BUILD_ARM_V5 "Build ARMv5" OFF)
+			option(BUILD_ARM_v6 "Build ARMv6" OFF)
+
+			if(BUILD_ARM_V5 AND BUILD_ARM_V6)
+				message(FATAL_ERROR "BUILD_ARM_V5 and BUILD_ARM_V6 are mutually exclusive!")
+			endif(BUILD_ARM_V5 AND BUILD_ARM_V6)
+
+			if(BUILD_32BIT)
+				set(CMAKE_SYSTEM_PROCESSOR "x86" CACHE STRING "system processor")
+				set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags")
+				set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags")
+				set(GOARCH "GOARCH=386" CACHE STRING "go architecture")
+				add_compile_options(
+					-m32
+				)
+			endif(BUILD_32BIT)
+
+			if(BUILD_STATIC)
+				add_link_options(
+					-static
+				)
+				set(CMAKE_EXE_LINKER_FLAGS "-static ${CMAKE_EXE_LINKER_FLAGS}")
+				set(GOFLAGS
+					${GOFLAGS}
+					-a
+					-tags osusergo,netgo
+					-ldflags '-w -extldflags \"-static -Wl,-unresolved-symbols=ignore-all\"'
+				)
+			else(BUILD_STATIC)
+				set(GOFLAGS
+					${GOFLAGS}
+					-a
+					-ldflags '-w -extldflags \"-Wl,-unresolved-symbols=ignore-all\"'
+				)
+			endif(BUILD_STATIC)
+
+			if(BUILD_ARM_V5)
+				set(GOARM "GOARM=5")
+			endif(BUILD_ARM_V5)
+
+			if(BUILD_ARM_V6)
+				set(GOARM "GOARM=6")
+			endif(BUILD_ARM_V6)
+
+		endif(APPLE)
+	endif(WIN32)
+
+	if (
+		CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64" OR
+		CMAKE_SYSTEM_PROCESSOR MATCHES "amd64" OR
+		CMAKE_SYSTEM_PROCESSOR MATCHES "AMD64"
+	)
+		message("++ Adding flags for processor ${CMAKE_SYSTEM_PROCESSOR}")
+		add_compile_options(-maes -mrdrnd -mpclmul -msse -msse2 -mssse3)
+	endif()
+
+	set(GO_BUILD_TAGS)
+
+	if(BUILD_CENTRAL_CONTROLLER)
+		add_definitions(-DZT_CONTROLLER_USE_LIBPQ=1)
+		set(GO_BUILD_TAGS -tags central)
+	endif(BUILD_CENTRAL_CONTROLLER)
+
+	add_subdirectory(core)
+	add_subdirectory(controller)
+	add_subdirectory(osdep)
+	add_subdirectory(serviceiocore)
+
+	file(GLOB go_src
+		${CMAKE_SOURCE_DIR}/cmd/*.go
+		${CMAKE_SOURCE_DIR}/cmd/cmd/*.go
+		${CMAKE_SOURCE_DIR}/pkg/zerotier/*.go
+	)
+
+	if(WIN32)
+		set(GO_EXE_NAME "zerotier.exe")
+		set(GO_EXTRA_LIBRARIES "-lstdc++ -lwsock32 -lws2_32 -liphlpapi -lole32 -loleaut32 -lrpcrt4 -luuid")
+	else(WIN32)
+		set(GO_EXE_NAME "zerotier")
+		if(CMAKE_SYSTEM_NAME MATCHES "Linux")
+			set(GO_EXTRA_LIBRARIES "-lstdc++")
+		else()
+			set(GO_EXTRA_LIBRARIES "-lc++")
+		endif()
+	endif(WIN32)
+
+	add_custom_target(
+		zerotier ALL
+		BYPRODUCTS ${CMAKE_BINARY_DIR}/zerotier
+		SOURCES ${go_src}
+		COMMAND ${CMAKE_COMMAND} -E env ${GOARCH} ${GOARM} CGO_ENABLED=1 CGO_CFLAGS=\"-O3\" CGO_LDFLAGS=\"$<TARGET_FILE:zt_core> $<TARGET_FILE:zt_controller> $<TARGET_FILE:zt_service_io_core> $<TARGET_FILE:zt_osdep> ${GO_EXTRA_LIBRARIES}\" ${GO} build ${GOFLAGS} -o ${CMAKE_BINARY_DIR}/${GO_EXE_NAME} ${CMAKE_SOURCE_DIR}/cmd/zerotier/zerotier.go
+		COMMENT "Compiling Go Code..."
+	)
+	add_dependencies(zerotier zt_osdep zt_core zt_controller zt_service_io_core)
+
+else(NOT PACKAGE_STATIC)
+
+	if(BUILD_32BIT)
+		set(CMAKE_SYSTEM_PROCESSOR "x86" CACHE STRING "system processor")
+		set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32" CACHE STRING "c++ flags")
+		set(CMAKE_C_FLAGS   "${CMAKE_C_FLAGS} -m32" CACHE STRING "c flags")
+		add_compile_options(
+			-m32
+		)
+	endif(BUILD_32BIT)
+
+	set(STATIC_BINARY ${CMAKE_BINARY_DIR}/zerotier)
+	set(IMPORTED_LOCATION ${CMAKE_BINARY_DIR})
+	add_executable(zerotier IMPORTED GLOBAL)
+	install(PROGRAMS ${STATIC_BINARY} DESTINATION bin)
+
+endif(NOT PACKAGE_STATIC)
+
+# Linux packaging
+
+if("${CMAKE_SYSTEM_NAME}" EQUAL "Linux")
+	if(IS_DIRECTORY /lib/systemd/system)
+		install(
+			FILES ${CMAKE_CURRENT_SOURCE_DIR}/packaging/debian/zerotier.service
+			DESTINATION /lib/systemd/system
+		)
+	elseif(IS_DIRECTORY /usr/lib/systemd/system)
+		install(
+			FILES ${CMAKE_CURRENT_SOURCE_DIR}/packaging/debian/zerotier.service
+			DESTINATION /usr/lib/systemd/system
+		)
 	else()
-		set(GO_EXTRA_LIBRARIES "-lc++")
+		install(
+			FILES ${CMAKE_CURRENT_SOURCE_DIR}/packaging/debian/zerotier.init
+			DESTINATION /etc/init.d
+		)
 	endif()
-endif(WIN32)
-
-add_custom_target(
-	zerotier ALL
-	BYPRODUCTS ${CMAKE_BINARY_DIR}/zerotier
-	SOURCES ${go_src}
-	COMMAND ${CMAKE_COMMAND} -E env ${GOARCH} CGO_ENABLED=1 CGO_CFLAGS=\"-O3\" CGO_LDFLAGS=\"$<TARGET_FILE:zt_core> $<TARGET_FILE:zt_controller> $<TARGET_FILE:zt_service_io_core> $<TARGET_FILE:zt_osdep> ${GO_EXTRA_LIBRARIES}\" ${GO} build ${GOFLAGS} -o ${CMAKE_BINARY_DIR}/${GO_EXE_NAME} ${CMAKE_SOURCE_DIR}/cmd/zerotier/zerotier.go
-	COMMENT "Compiling Go Code..."
-)
-add_dependencies(zerotier zt_osdep zt_core zt_controller zt_service_io_core)
+endif()
+
+if("${ZT_PACKAGE_FORMAT}" MATCHES "DEB")
+	include(packaging/debian.cmake)
+elseif("${ZT_PACKAGE_FORMAT}" MATCHES "RPM")
+	include(packaging/rpm.cmake)
+else()
+endif()

core/Node.cpp

@@ -34,13 +34,13 @@ namespace {
 // Structure containing all the core objects for a ZeroTier node to reduce memory allocations.
 struct _NodeObjects
 {
-	ZT_INLINE _NodeObjects(RuntimeEnvironment *const RR, void *const tPtr) :
+	ZT_INLINE _NodeObjects(RuntimeEnvironment *const RR, void *const tPtr, const int64_t now) :
 		t(RR),
 		expect(),
 		vl2(RR),
 		vl1(RR),
 		sa(RR),
-		topology(RR, tPtr)
+		topology(RR, tPtr, now)
 	{
 		RR->t = &t;
 		RR->expect = &expect;
@@ -143,7 +143,7 @@ Node::Node(
 	// This constructs all the components of the ZeroTier core within a single contiguous memory container,
 	// which reduces memory fragmentation and may improve cache locality.
 	ZT_SPEW("initializing subsystem objects...");
-	m_objects = new _NodeObjects(RR, tPtr);
+	m_objects = new _NodeObjects(RR, tPtr, now);
 	ZT_SPEW("node initialized!");
 
 	postEvent(tPtr, ZT_EVENT_UP);
@@ -612,7 +612,7 @@ void Node::setController(void *networkControllerInstance)
 
 // Methods used only within the core ----------------------------------------------------------------------------------
 
-Vector< uint8_t > Node::stateObjectGet(void *const tPtr, ZT_StateObjectType type, const uint64_t id[2])
+Vector< uint8_t > Node::stateObjectGet(void *const tPtr, ZT_StateObjectType type, const uint64_t *id)
 {
 	Vector< uint8_t > r;
 	if (m_cb.stateGetFunction) {

core/Node.hpp

@@ -277,10 +277,10 @@ public:
 	 *
 	 * @param tPtr Thread pointer
 	 * @param type Object type to get
-	 * @param id Object ID
+	 * @param id Object ID or NULL if this type does not use one
 	 * @return Vector containing data or empty vector if not found or empty
 	 */
-	Vector<uint8_t> stateObjectGet(void *tPtr, ZT_StateObjectType type, const uint64_t id[2]);
+	Vector<uint8_t> stateObjectGet(void *tPtr, ZT_StateObjectType type, const uint64_t *id);
 
 	/**
 	 * Store a state object

core/Topology.cpp

@@ -15,14 +15,41 @@
 
 namespace ZeroTier {
 
-Topology::Topology(const RuntimeEnvironment *renv, void *tPtr) :
+static const SharedPtr< const Certificate > s_nullCert;
+
+Topology::Topology(const RuntimeEnvironment *renv, void *tPtr, const int64_t now) :
 	RR(renv)
 {
-	uint64_t idtmp[2];
-	idtmp[0] = 0;
-	idtmp[1] = 0;
-	Vector< uint8_t > data(RR->node->stateObjectGet(tPtr, ZT_STATE_OBJECT_ROOTS, idtmp));
-	// TODO
+	char tmp[256];
+	Vector< uint8_t > trustData(RR->node->stateObjectGet(tPtr, ZT_STATE_OBJECT_TRUST_STORE, Utils::ZERO256));
+
+	Dictionary d;
+	if (trustData.empty() || (!d.decode(trustData.data(), (unsigned int)trustData.size()))) {
+		// TODO: import default certificates including default root set
+	} else {
+		const unsigned long certCount = (unsigned long)d.getUI("c$");
+		for (unsigned long idx = 0; idx < certCount; ++idx) {
+			uint64_t id[6];
+			const Vector< uint8_t > &serialNo = d[Dictionary::arraySubscript(tmp, "c$.s", idx)];
+			if (serialNo.size() == ZT_SHA384_DIGEST_SIZE) {
+				Utils::copy< 48 >(id, serialNo.data());
+				Certificate cert;
+				if (cert.decode(RR->node->stateObjectGet(tPtr, ZT_STATE_OBJECT_CERT, id)))
+					addCertificate(tPtr, cert, now, (unsigned int)d.getUI(Dictionary::arraySubscript(tmp, "c$.lt", idx)), false, false, false);
+			}
+		}
+
+		const unsigned long localRootCount = (unsigned long)d.getUI("lr$");
+		for (unsigned long idx = 0; idx < localRootCount; ++idx) {
+			Identity lr;
+			if (d.getO(Dictionary::arraySubscript(tmp, "lr$.i", idx), lr)) {
+				if (lr)
+					m_roots[lr].insert(s_nullCert);
+			}
+		}
+	}
+
+	m_cleanCertificates_l_certs(now);
 	m_updateRootPeers_l_roots_certs(tPtr);
 }
 
@@ -46,13 +73,13 @@ SharedPtr< Peer > Topology::addRoot(void *const tPtr, const Identity &id)
 
 		// A null pointer in the set of certificates specifying a root indicates that
 		// the root has been directly added.
-		m_roots[id.fingerprint()].insert(SharedPtr< const Certificate >());
+		m_roots[id].insert(s_nullCert);
 
 		{
 			Mutex::Lock certsLock(m_certs_l);
 			m_updateRootPeers_l_roots_certs(tPtr);
+			m_writeTrustStore_l_roots_certs(tPtr);
 		}
-		m_writeRootList_l_roots(tPtr);
 
 		for (Vector< SharedPtr< Peer > >::const_iterator p(m_rootPeers.begin()); p != m_rootPeers.end(); ++p) {
 			if ((*p)->identity() == id)
@@ -65,8 +92,24 @@ SharedPtr< Peer > Topology::addRoot(void *const tPtr, const Identity &id)
 bool Topology::removeRoot(void *const tPtr, Address address)
 {
 	RWMutex::Lock l1(m_roots_l);
-	// TODO
-	return true;
+	bool removed = false;
+	for (Map< Identity, Set< SharedPtr< const Certificate > > >::iterator r(m_roots.begin()); r != m_roots.end();) {
+		if (r->first.address() == address) {
+			r->second.erase(s_nullCert);
+			if (r->second.empty()) {
+				m_roots.erase(r++);
+				{
+					Mutex::Lock certsLock(m_certs_l);
+					m_updateRootPeers_l_roots_certs(tPtr);
+					m_writeTrustStore_l_roots_certs(tPtr);
+				}
+				removed = true;
+			} else {
+				++r;
+			}
+		} else ++r;
+	}
+	return removed;
 }
 
 struct p_RootRankingComparisonOperator
@@ -104,15 +147,26 @@ void Topology::doPeriodicTasks(void *tPtr, const int64_t now)
 	// then the write lock is acquired for each delete. This adds overhead if there
 	// are a lot of deletions, but that's not common.
 
+	// Clean any expired certificates
+	{
+		Mutex::Lock l1(m_certs_l);
+		if (m_cleanCertificates_l_certs(now)) {
+			RWMutex::Lock l2(m_roots_l);
+			m_updateRootPeers_l_roots_certs(tPtr);
+		}
+	}
+
 	// Delete peers that are stale or offline.
 	{
 		Vector< Address > toDelete;
 		{
 			RWMutex::RLock l1(m_peers_l);
-			for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end(); ++i) {
+			RWMutex::RLock l2(m_roots_l);
+			for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end();
+			     ++i) {
 				// TODO: also delete if the peer has not exchanged meaningful communication in a while, such as
 				// a network frame or non-trivial control packet.
-				if (((now - i->second->lastReceive()) > ZT_PEER_ALIVE_TIMEOUT) && (m_roots.find(i->second->identity().fingerprint()) == m_roots.end()))
+				if (((now - i->second->lastReceive()) > ZT_PEER_ALIVE_TIMEOUT) && (m_roots.find(i->second->identity()) == m_roots.end()))
 					toDelete.push_back(i->first);
 			}
 		}
@@ -131,7 +185,8 @@ void Topology::doPeriodicTasks(void *tPtr, const int64_t now)
 		Vector< uint64_t > toDelete;
 		{
 			RWMutex::RLock l1(m_paths_l);
-			for (Map< uint64_t, SharedPtr< Path > >::iterator i(m_paths.begin()); i != m_paths.end(); ++i) {
+			for (Map< uint64_t, SharedPtr< Path > >::iterator i(m_paths.begin()); i != m_paths.end();
+			     ++i) {
 				if (i->second.weakGC())
 					toDelete.push_back(i->first);
 			}
@@ -143,74 +198,85 @@ void Topology::doPeriodicTasks(void *tPtr, const int64_t now)
 				m_paths.erase(p);
 		}
 	}
-
-	// Clean any expired certificates
-	{
-		Mutex::Lock l1(m_certs_l);
-		m_cleanCertificates_l_certs(now);
-	}
 }
 
 void Topology::saveAll(void *tPtr)
 {
 	RWMutex::RLock l(m_peers_l);
-	for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end(); ++i)
+	for (Map< Address, SharedPtr< Peer > >::iterator i(m_peers.begin()); i != m_peers.end();
+	     ++i)
 		i->second->save(tPtr);
 }
 
-ZT_CertificateError Topology::addCertificate(void *tPtr, const Certificate &cert, const int64_t now, const unsigned int localTrust)
+ZT_CertificateError Topology::addCertificate(void *tPtr, const Certificate &cert, const int64_t now, const unsigned int localTrust, const bool writeToLocalStore, const bool refreshRootSets, const bool verify)
 {
-	Mutex::Lock certsLock(m_certs_l);
-
-	// Check to see if we already have this specific certificate.
-	const SHA384Hash serial(cert.serialNo);
-	if (m_certs.find(serial) != m_certs.end())
-		return ZT_CERTIFICATE_ERROR_NONE;
-
-	// Verify certificate all the way to a trusted root.
-	const ZT_CertificateError err = m_verifyCertificate_l_certs(cert, now, localTrust, false);
-	if (err != ZT_CERTIFICATE_ERROR_NONE)
-		return err;
-
-	// Create entry containing copy of certificate and trust flags.
-	const std::pair< SharedPtr< const Certificate >, unsigned int > certEntry(SharedPtr< const Certificate >(new Certificate(cert)), localTrust);
-
-	// If the subject contains a unique ID, check if we already have a cert for the
-	// same uniquely identified subject. If so, check its subject timestamp and keep
-	// the one we have if newer. Otherwise replace it. Note that the verification
-	// function will have checked the unique ID proof signature already if a unique
-	// ID was present.
-	FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE > uniqueId(cert.subject.uniqueId, cert.subject.uniqueIdSize);
-	if (!uniqueId.empty()) {
-		std::pair< SharedPtr< const Certificate >, unsigned int > &bySubjectUniqueId = m_certsBySubjectUniqueId[uniqueId];
-		if (bySubjectUniqueId.first) {
-			if (bySubjectUniqueId.first->subject.timestamp >= cert.subject.timestamp)
-				return ZT_CERTIFICATE_ERROR_HAVE_NEWER_CERT;
-			m_eraseCertificate_l_certs(bySubjectUniqueId.first);
-			m_certsBySubjectUniqueId[uniqueId] = certEntry;
-		} else {
-			bySubjectUniqueId = certEntry;
+	{
+		Mutex::Lock certsLock(m_certs_l);
+
+		// Check to see if we already have this specific certificate.
+		const SHA384Hash serial(cert.serialNo);
+		if (m_certs.find(serial) != m_certs.end())
+			return ZT_CERTIFICATE_ERROR_NONE;
+
+		// Verify certificate all the way to a trusted root. This also verifies inner
+		// signatures such as those of locators or the subject unique ID.
+		if (verify) {
+			const ZT_CertificateError err = m_verifyCertificate_l_certs(cert, now, localTrust, false);
+			if (err != ZT_CERTIFICATE_ERROR_NONE)
+				return err;
+		}
+
+		// Create entry containing copy of certificate and trust flags.
+		const std::pair< SharedPtr< const Certificate >, unsigned int > certEntry(SharedPtr< const Certificate >(new Certificate(cert)), localTrust);
+
+		// If the subject contains a unique ID, check if we already have a cert for the
+		// same uniquely identified subject. If so, check its subject timestamp and keep
+		// the one we have if newer. Otherwise replace it. Note that the verification
+		// function will have checked the unique ID proof signature already if a unique
+		// ID was present.
+		FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE > uniqueId(cert.subject.uniqueId, cert.subject.uniqueIdSize);
+		if (!uniqueId.empty()) {
+			std::pair< SharedPtr< const Certificate >, unsigned int > &bySubjectUniqueId = m_certsBySubjectUniqueId[uniqueId];
+			if (bySubjectUniqueId.first) {
+				if (bySubjectUniqueId.first->subject.timestamp >= cert.subject.timestamp)
+					return ZT_CERTIFICATE_ERROR_HAVE_NEWER_CERT;
+				m_eraseCertificate_l_certs(bySubjectUniqueId.first);
+				m_certsBySubjectUniqueId[uniqueId] = certEntry; // reference bySubjectUniqueId no longer valid
+			} else {
+				bySubjectUniqueId = certEntry;
+			}
 		}
-	}
 
-	// Save certificate by serial number.
-	m_certs[serial] = certEntry;
+		// Save certificate by serial number.
+		m_certs[serial] = certEntry;
 
-	// Add certificate to sets of certificates whose subject references a given identity.
-	for (unsigned int i = 0; i < cert.subject.identityCount; ++i) {
-		const Identity *const ii = reinterpret_cast<const Identity *>(cert.subject.identities[i].identity);
-		m_certsBySubjectIdentity[ii->fingerprint()].insert(certEntry);
-	}
+		// Add certificate to sets of certificates whose subject references a given identity.
+		for (unsigned int i = 0; i < cert.subject.identityCount; ++i) {
+			const Identity *const ii = reinterpret_cast<const Identity *>(cert.subject.identities[i].identity);
+			m_certsBySubjectIdentity[ii->fingerprint()].insert(certEntry);
+		}
 
-	// Clean any certificates whose chains are now broken, which can happen if there was
-	// an update that replaced an old cert with a given unique ID. Otherwise this generally
-	// does nothing here.
-	m_cleanCertificates_l_certs(now);
+		// Clean any certificates whose chains are now broken, which can happen if there was
+		// an update that replaced an old cert with a given unique ID. Otherwise this generally
+		// does nothing here. Skip if verify is false since this means we're mindlessly loading
+		// certificates, which right now only happens on startup when they're loaded from the
+		// local certificate cache.
+		if (verify)
+			m_cleanCertificates_l_certs(now);
+
+		// Refresh the root peers lists, since certs may enumerate roots.
+		if (refreshRootSets) {
+			RWMutex::Lock rootsLock(m_roots_l);
+			m_updateRootPeers_l_roots_certs(tPtr);
+		}
+	}
 
-	// Refresh the root peers lists, since certs may enumerate roots.
-	{
-		RWMutex::Lock rootsLock(m_roots_l);
-		m_updateRootPeers_l_roots_certs(tPtr);
+	if (writeToLocalStore) {
+		// Write certificate data prefixed by local trust flags as a 32-bit integer.
+		Vector< uint8_t > certData(cert.encode());
+		uint64_t id[6];
+		Utils::copy< 48 >(id, cert.serialNo);
+		RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_CERT, id, certData.data(), (unsigned int)certData.size());
 	}
 
 	return ZT_CERTIFICATE_ERROR_NONE;
@@ -220,11 +286,15 @@ void Topology::m_eraseCertificate_l_certs(const SharedPtr< const Certificate > &
 {
 	// assumes m_certs is locked for writing
 
-	m_certsBySubjectUniqueId.erase(FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE >(cert->subject.uniqueId, cert->subject.uniqueIdSize));
 	m_certs.erase(SHA384Hash(cert->serialNo));
+
+	if (cert->subject.uniqueIdSize > 0)
+		m_certsBySubjectUniqueId.erase(FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE >(cert->subject.uniqueId, cert->subject.uniqueIdSize));
+
 	for (unsigned int i = 0; i < cert->subject.identityCount; ++i) {
 		const Identity *const ii = reinterpret_cast<const Identity *>(cert->subject.identities[i].identity);
-		Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > >::iterator bySubjectIdentity(m_certsBySubjectIdentity.find(ii->fingerprint()));
+		Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > >::iterator
+			bySubjectIdentity(m_certsBySubjectIdentity.find(ii->fingerprint()));
 		if (bySubjectIdentity != m_certsBySubjectIdentity.end()) {
 			bySubjectIdentity->second.erase(cert);
 			if (bySubjectIdentity->second.empty())
@@ -233,13 +303,17 @@ void Topology::m_eraseCertificate_l_certs(const SharedPtr< const Certificate > &
 	}
 }
 
-void Topology::m_cleanCertificates_l_certs(int64_t now)
+bool Topology::m_cleanCertificates_l_certs(int64_t now)
 {
 	// assumes m_certs is locked for writing
 
-	Vector< SharedPtr< const Certificate > > toDelete;
+	bool deleted = false;
+	Vector< SharedPtr< const Certificate >> toDelete;
 	for (;;) {
 		for (Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > >::iterator c(m_certs.begin()); c != m_certs.end(); ++c) {
+			// Verify, but the last boolean option tells it to skip signature checks as this would
+			// already have been done. This will therefore just check the path and validity times
+			// of the certificate.
 			const ZT_CertificateError err = m_verifyCertificate_l_certs(*(c->second.first), now, c->second.second, true);
 			if (err != ZT_CERTIFICATE_ERROR_NONE)
 				toDelete.push_back(c->second.first);
@@ -248,17 +322,21 @@ void Topology::m_cleanCertificates_l_certs(int64_t now)
 		if (toDelete.empty())
 			break;
 
+		deleted = true;
 		for (Vector< SharedPtr< const Certificate > >::iterator c(toDelete.begin()); c != toDelete.end(); ++c)
 			m_eraseCertificate_l_certs(*c);
 		toDelete.clear();
 	}
+
+	return deleted;
 }
 
 bool Topology::m_verifyCertificateChain_l_certs(const Certificate *current, const int64_t now) const
 {
 	// assumes m_certs is at least locked for reading
 
-	Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > >::const_iterator c = m_certsBySubjectIdentity.find(reinterpret_cast<const Identity *>(current->issuer)->fingerprint());
+	Map< Fingerprint, Map< SharedPtr< const Certificate >, unsigned int > >::const_iterator
+		c = m_certsBySubjectIdentity.find(reinterpret_cast<const Identity *>(current->issuer)->fingerprint());
 	if (c != m_certsBySubjectIdentity.end()) {
 		for (Map< SharedPtr< const Certificate >, unsigned int >::const_iterator cc(c->second.begin()); cc != c->second.end(); ++cc) {
 			if (
@@ -283,23 +361,32 @@ ZT_CertificateError Topology::m_verifyCertificate_l_certs(const Certificate &cer
 {
 	// assumes m_certs is at least locked for reading
 
+	// Check certificate time window against current time.
 	if ((cert.validity[0] > now) || (cert.validity[1] < now))
 		return ZT_CERTIFICATE_ERROR_OUT_OF_VALID_TIME_WINDOW;
 
+	// Verify primary and internal signatures and other objects unless the caller
+	// elected to skip, which is done to re-check certs already in the DB.
 	if (!skipSignatureCheck) {
-		const ZT_CertificateError ce = cert.verify();
-		if (ce != ZT_CERTIFICATE_ERROR_NONE)
-			return ce;
+		const ZT_CertificateError err = cert.verify();
+		if (err != ZT_CERTIFICATE_ERROR_NONE)
+			return err;
 	}
 
+	// If this is a root CA, we can skip this as we're already there. Otherwise we
+	// recurse up the tree until we hit a root CA.
 	if ((localTrust & ZT_CERTIFICATE_LOCAL_TRUST_FLAG_ROOT_CA) == 0) {
 		if (!m_verifyCertificateChain_l_certs(&cert, now))
 			return ZT_CERTIFICATE_ERROR_INVALID_CHAIN;
 	}
+
+	return ZT_CERTIFICATE_ERROR_NONE;
 }
 
 void Topology::m_loadCached(void *tPtr, const Address &zta, SharedPtr< Peer > &peer)
 {
+	// does not require any locks to be held
+
 	try {
 		uint64_t id[2];
 		id[0] = zta.toInt();
@@ -327,54 +414,70 @@ void Topology::m_loadCached(void *tPtr, const Address &zta, SharedPtr< Peer > &p
 	}
 }
 
-void Topology::m_writeRootList_l_roots(void *tPtr)
+void Topology::m_updateRootPeers_l_roots_certs(void *tPtr)
 {
-	// assumes m_peers_l is locked for read or write
-	// TODO
-#if 0
-	uint8_t *const roots = (uint8_t *)malloc((ZT_IDENTITY_MARSHAL_SIZE_MAX + ZT_LOCATOR_MARSHAL_SIZE_MAX + 2) * m_roots.size());
-	if (roots) { // sanity check
-		int p = 0;
-		for (Set< Identity >::const_iterator r(m_roots.begin()); r != m_roots.end(); ++r) {
-			const int pp = r->marshal(roots + p, false);
-			if (pp > 0)
-				p += pp;
+	// assumes m_roots_l and m_certs_l are locked for write
+
+	// Clear m_roots but preserve locally added roots (indicated by a null cert ptr entry).
+	for (Map< Identity, Set< SharedPtr< const Certificate > > >::iterator r(m_roots.begin()); r != m_roots.end();) {
+		if (r->second.find(s_nullCert) == r->second.end()) {
+			m_roots.erase(r++);
+		} else {
+			r->second.clear();
+			r->second.insert(s_nullCert);
+			++r;
 		}
-		uint64_t id[2];
-		id[0] = 0;
-		id[1] = 0;
-		RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_ROOTS, id, roots, (unsigned int)p);
-		free(roots);
 	}
-#endif
+
+	// Populate m_roots from certificate subject identities from certificates flagged
+	// as local root set certificates.
+	for (Map< FCV< uint8_t, ZT_CERTIFICATE_MAX_UNIQUE_ID_SIZE >, std::pair< SharedPtr< const Certificate >, unsigned int > >::const_iterator c(m_certsBySubjectUniqueId.begin()); c != m_certsBySubjectUniqueId.end();
+	     ++c) {
+		if ((c->second.second & ZT_CERTIFICATE_LOCAL_TRUST_FLAG_ZEROTIER_ROOT_SET) != 0) {
+			for (unsigned int i = 0; i < c->second.first->subject.identityCount; ++i)
+				m_roots[*reinterpret_cast<const Identity *>(c->second.first->subject.identities[i].identity)].insert(c->second.first);
+		}
+	}
+
+	// Create a new rootPeers vector and swap.
+	Vector< SharedPtr< Peer >> newRootPeers;
+	newRootPeers.reserve(m_roots.size());
+	for (Map< Identity, Set< SharedPtr< const Certificate > > >::iterator r(m_roots.begin()); r != m_roots.end();) {
+		const SharedPtr< Peer > p(this->peer(tPtr, r->first.address(), true));
+		if ((p) && (p->identity() == r->first))
+			newRootPeers.push_back(p);
+	}
+	std::sort(newRootPeers.begin(), newRootPeers.end(), p_RootRankingComparisonOperator());
+	m_rootPeers.swap(newRootPeers);
 }
 
-void Topology::m_updateRootPeers_l_roots_certs(void *tPtr)
+void Topology::m_writeTrustStore_l_roots_certs(void *tPtr) const
 {
-	// assumes m_peers_l and m_certs_l are locked for write
-	// TODO
-#if 0
-	Vector< SharedPtr< Peer > > rp;
-	for (Map< Identity, Set< SubscriptionKeyHash > >::iterator r(m_roots.begin()); r != m_roots.end(); ++r) {
-		Map< Address, SharedPtr< Peer > >::iterator pp(m_peers.find(r->first.address()));
-		SharedPtr< Peer > p;
-		if (pp != m_peers.end())
-			p = pp->second;
-
-		if (!p)
-			m_loadCached(tPtr, r->first.address(), p);
-
-		if ((!p) || (p->identity() != r->first)) {
-			p.set(new Peer(RR));
-			p->init(r->first);
-			m_peers[r->first.address()] = p;
-		}
+	// assumes m_roots_l and m_certs_l are locked for write
+
+	char tmp[256];
+	Dictionary d;
+
+	d.add("v", (uint64_t)0); // version
 
-		rp.push_back(p);
+	unsigned long idx = 0;
+	d.add("c$", (uint64_t)m_certs.size());
+	for (Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > >::const_iterator c(m_certs.begin()); c != m_certs.end(); ++c) {
+		d[Dictionary::arraySubscript(tmp, "c$.s", idx)].assign(c->first.data, c->first.data + ZT_SHA384_DIGEST_SIZE);
+		d.add(Dictionary::arraySubscript(tmp, "c$.lt", idx), (uint64_t)c->second.second);
+		++idx;
 	}
-	std::sort(rp.begin(), rp.end(), p_RootSortComparisonOperator());
-	m_rootPeers.swap(rp);
-#endif
+
+	unsigned long localRootCount = 0;
+	for (Map< Identity, Set< SharedPtr< const Certificate > > >::const_iterator r(m_roots.begin()); r != m_roots.end();) {
+		if (r->second.find(s_nullCert) != r->second.end())
+			d.addO(Dictionary::arraySubscript(tmp, "lr$.i", localRootCount++), r->first);
+	}
+	d.add("lr$", (uint64_t)localRootCount);
+
+	Vector< uint8_t > trustStore;
+	d.encode(trustStore);
+	RR->node->stateObjectPut(tPtr, ZT_STATE_OBJECT_TRUST_STORE, Utils::ZERO256, trustStore.data(), (unsigned int)trustStore.size());
 }
 
 } // namespace ZeroTier

core/Topology.hpp

@@ -39,7 +39,7 @@ class RuntimeEnvironment;
 class Topology
 {
 public:
-	Topology(const RuntimeEnvironment *renv, void *tPtr);
+	Topology(const RuntimeEnvironment *renv, void *tPtr, int64_t now);
 
 	/**
 	 * Add peer to database
@@ -130,7 +130,7 @@ public:
 	ZT_INLINE bool isRoot(const Identity &id) const
 	{
 		RWMutex::RLock l(m_roots_l);
-		return (m_roots.find(id.fingerprint()) != m_roots.end());
+		return (m_roots.find(id) != m_roots.end());
 	}
 
 	/**
@@ -202,30 +202,41 @@ public:
 
 	/**
 	 * Do periodic tasks such as database cleanup
+	 *
+	 * @param tPtr Thread pointer
+	 * @param now Current time
 	 */
 	void doPeriodicTasks(void *tPtr, int64_t now);
 
 	/**
 	 * Save all currently known peers to data store
+	 *
+	 * @param tPtr Thread pointer
 	 */
 	void saveAll(void *tPtr);
 
 	/**
 	 * Add a certificate to the local certificate store
 	 *
+	 * @param tPtr Thread pointer
 	 * @param cert Certificate to add (a copy will be made if added)
+	 * @param now Current time
+	 * @param localTrust Local trust bit flags
+	 * @param writeToLocalStore If true, write to local object store (via API callbacks)
+	 * @param refreshRootSets If true, refresh root sets in case a root set changed (default: true)
+	 * @param verify If true, verify certificate and certificate chain (default: true)
 	 * @return Error or 0 on success
 	 */
-	ZT_CertificateError addCertificate(void *tPtr, const Certificate &cert, const int64_t now, unsigned int localTrust);
+	ZT_CertificateError addCertificate(void *tPtr, const Certificate &cert, const int64_t now, unsigned int localTrust, bool writeToLocalStore, bool refreshRootSets = true, bool verify = true);
 
 private:
 	void m_eraseCertificate_l_certs(const SharedPtr< const Certificate > &cert);
-	void m_cleanCertificates_l_certs(int64_t now);
+	bool m_cleanCertificates_l_certs(int64_t now);
 	bool m_verifyCertificateChain_l_certs(const Certificate *current, const int64_t now) const;
 	ZT_CertificateError m_verifyCertificate_l_certs(const Certificate &cert, const int64_t now, unsigned int localTrust, bool skipSignatureCheck) const;
 	void m_loadCached(void *tPtr, const Address &zta, SharedPtr< Peer > &peer);
-	void m_writeRootList_l_roots(void *tPtr);
 	void m_updateRootPeers_l_roots_certs(void *tPtr);
+	void m_writeTrustStore_l_roots_certs(void *tPtr) const;
 
 	// This gets an integer key from an InetAddress for looking up paths.
 	static ZT_INLINE uint64_t s_getPathKey(const int64_t l, const InetAddress &r) noexcept
@@ -257,7 +268,7 @@ private:
 
 	Map< Address, SharedPtr< Peer > > m_peers;
 
-	Map< Fingerprint, Set< SharedPtr< const Certificate > > > m_roots;
+	Map< Identity, Set< SharedPtr< const Certificate > > > m_roots;
 	Vector< SharedPtr< Peer > > m_rootPeers;
 
 	Map< SHA384Hash, std::pair< SharedPtr< const Certificate >, unsigned int > > m_certs;

core/zerotier.h

@@ -1584,7 +1584,7 @@ enum ZT_StateObjectType
 	/**
 	 * Public address and public key
 	 *
-	 * Object ID: (unused)
+	 * Object ID: (none)
 	 * Canonical path: <HOME>/identity.public
    * Persistence: required
 	 */
@@ -1593,7 +1593,7 @@ enum ZT_StateObjectType
 	/**
 	 * Full identity with secret key
 	 *
-	 * Object ID: (unused)
+	 * Object ID: (none)
 	 * Canonical path: <HOME>/identity.secret
    * Persistence: required, should be stored with restricted permissions e.g. mode 0600 on *nix
 	 */
@@ -1602,7 +1602,7 @@ enum ZT_StateObjectType
 	/**
 	 * This node's locator
 	 *
-	 * Object ID: (unused)
+	 * Object ID: (none)
 	 * Canonical path: <HOME>/locator
 	 * Persistence: optional
 	 */
@@ -1611,7 +1611,7 @@ enum ZT_StateObjectType
 	/**
 	 * Peer and related state
 	 *
-	 * Object ID: peer address
+	 * Object ID: [1]address (40 bits, in least significant 64 bits)
 	 * Canonical path: <HOME>/peers.d/<ID> (10-digit address)
 	 * Persistence: optional, can be cleared at any time
 	 */
@@ -1620,22 +1620,45 @@ enum ZT_StateObjectType
 	/**
 	 * Network configuration
 	 *
-	 * Object ID: network ID
+	 * Object ID: [1]id (64-bit network ID)
 	 * Canonical path: <HOME>/networks.d/<NETWORKID>.conf (16-digit hex ID)
 	 * Persistence: required if network memberships should persist
 	 */
 	ZT_STATE_OBJECT_NETWORK_CONFIG = 6,
 
 	/**
-	 * Root list
+	 * List of certificates and their local trust, and locally added roots
 	 *
-	 * Object ID: (unused)
-	 * Canonical path: <HOME>/roots
+	 * Object ID: (none)
+	 * Canonical path: <HOME>/trust
 	 * Persistence: required if root settings should persist
 	 */
-	ZT_STATE_OBJECT_ROOTS = 7
+	ZT_STATE_OBJECT_TRUST_STORE = 7,
+
+	/**
+	 * Certificate
+	 *
+	 * Object ID: [6]serial (384-bit serial packed into 6 uint64_t's)
+	 * Canonical path: <HOME>/certs.d/<serial> (96-digit hex serial)
+	 */
+	ZT_STATE_OBJECT_CERT = 8
 };
 
+/**
+ * Size of the object ID for peers (in 64-bit uint64_t's)
+ */
+#define ZT_STATE_OBJECT_PEER_ID_SIZE 1
+
+/**
+ * Size of the object ID for network configurations (in 64-bit uint64_t's)
+ */
+#define ZT_STATE_OBJECT_NETWORK_CONFIG_ID_SIZE 1
+
+/**
+ * Size of the object ID for certificates (in 64-bit uint64_t's)
+ */
+#define ZT_STATE_OBJECT_CERT_ID_SIZE 6
+
 /**
  * An instance of a ZeroTier One node (opaque)
  */
@@ -1711,6 +1734,10 @@ typedef void (*ZT_EventCallback)(
  * See ZT_StateObjectType docs for information about each state object type
  * and when and if it needs to be persisted.
  *
+ * The state object ID's size depends on the object type, and is always
+ * in the form of one or more 64-bit unsigned integers. Some object types
+ * do not use this field, and for these it may be NULL.
+ *
  * An object of length -1 is sent to indicate that an object should be
  * deleted.
  */
@@ -1719,7 +1746,7 @@ typedef void (*ZT_StatePutFunction)(
 	void *,                                /* User ptr */
 	void *,                                /* Thread ptr */
 	enum ZT_StateObjectType,               /* State object type */
-	const uint64_t [2],                    /* State object ID (if applicable) */
+	const uint64_t *,                      /* State object ID (if applicable) */
 	const void *,                          /* State object data */
 	int);                                  /* Length of data or -1 to delete */
 
@@ -1737,7 +1764,7 @@ typedef int (*ZT_StateGetFunction)(
 	void *,                                /* User ptr */
 	void *,                                /* Thread ptr */
 	enum ZT_StateObjectType,               /* State object type */
-	const uint64_t [2],                    /* State object ID (if applicable) */
+	const uint64_t *,                      /* State object ID (if applicable) */
 	void **,                               /* Result parameter: data */
 	void (**)(void *));                    /* Result parameter: data free function */
 

serviceiocore/GoGlue.cpp

@@ -107,8 +107,8 @@ const char *const ZT_PLATFORM_DEFAULT_HOMEPATH = defaultHomePath.c_str();
 // These are implemented in Go code.
 extern "C" int goPathCheckFunc(void *, const ZT_Identity *, int, const void *, int);
 extern "C" int goPathLookupFunc(void *, uint64_t, int, const ZT_Identity *, int *, uint8_t [16], int *);
-extern "C" void goStateObjectPutFunc(void *, int, const uint64_t [2], const void *, int);
-extern "C" int goStateObjectGetFunc(void *, int, const uint64_t [2], void **);
+extern "C" void goStateObjectPutFunc(void *, int, const uint64_t *, const void *, int);
+extern "C" int goStateObjectGetFunc(void *, int, const uint64_t *, void **);
 extern "C" void goVirtualNetworkConfigFunc(void *, ZT_GoTap *, uint64_t, int, const ZT_VirtualNetworkConfig *);
 extern "C" void goZtEvent(void *, int, const void *);
 extern "C" void goHandleTapAddedMulticastGroup(void *, ZT_GoTap *, uint64_t, uint64_t, uint32_t);