began instrumentation of http control plane and controller

controller/EmbeddedNetworkController.cpp

@@ -47,6 +47,7 @@
 #include "../node/MAC.hpp"
 #include "../node/NetworkConfig.hpp"
 #include "../node/Node.hpp"
+#include "opentelemetry/trace/provider.h"
 
 using json = nlohmann::json;
 
@@ -65,6 +66,11 @@ namespace {
 
 static json _renderRule(ZT_VirtualNetworkRule& rule)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::renderRule");
+	auto scope = tracer->WithActiveSpan(span);
+
 	char tmp[128];
 	json r = json::object();
 	const ZT_VirtualNetworkRuleType rt = (ZT_VirtualNetworkRuleType)(rule.t & 0x3f);
@@ -272,6 +278,11 @@ static json _renderRule(ZT_VirtualNetworkRule& rule)
 
 static bool _parseRule(json& r, ZT_VirtualNetworkRule& rule)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::parseRule");
+	auto scope = tracer->WithActiveSpan(span);
+
 	if (! r.is_object())
 		return false;
 
@@ -588,6 +599,11 @@ void EmbeddedNetworkController::setSSORedirectURL(const std::string& url)
 
 void EmbeddedNetworkController::init(const Identity& signingId, Sender* sender)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::init");
+	auto scope = tracer->WithActiveSpan(span);
+
 	char tmp[64];
 	_signingId = signingId;
 	_sender = sender;
@@ -645,6 +661,11 @@ void EmbeddedNetworkController::init(const Identity& signingId, Sender* sender)
 
 void EmbeddedNetworkController::request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>& metaData)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::request");
+	auto scope = tracer->WithActiveSpan(span);
+
 	if (((! _signingId) || (! _signingId.hasPrivate())) || (_signingId.address().toInt() != (nwid >> 24)) || (! _sender))
 		return;
 	_startThreads();
@@ -672,6 +693,11 @@ void EmbeddedNetworkController::request(uint64_t nwid, const InetAddress& fromAd
 
 std::string EmbeddedNetworkController::networkUpdateFromPostData(uint64_t networkID, const std::string& body)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::networkUpdateFromPostData");
+	auto scope = tracer->WithActiveSpan(span);
+
 	json b = OSUtils::jsonParse(body);
 
 	char nwids[24];
@@ -959,6 +985,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	std::string memberPath = "/controller/network/([0-9a-fA-F]{16})/member/([0-9a-fA-F]{10})";
 
 	auto controllerGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::controllerGet");
+		auto scope = tracer->WithActiveSpan(span);
+
 		char tmp[4096];
 		const bool dbOk = _db.isReady();
 		OSUtils::ztsnprintf(
@@ -979,6 +1010,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Get(controllerPath, controllerGet);
 
 	auto networkListGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::networkListGet");
+		auto scope = tracer->WithActiveSpan(span);
+
 		std::set<uint64_t> networkIds;
 		_db.networks(networkIds);
 		char tmp[64];
@@ -995,6 +1031,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Get(networkListPath, networkListGet);
 
 	auto networkListGet2 = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::networkListGet2");
+		auto scope = tracer->WithActiveSpan(span);
+
 		std::set<uint64_t> networkIds;
 		_db.networks(networkIds);
 
@@ -1043,6 +1084,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Get(networkListPath2, networkListGet2);
 
 	auto networkGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::networkGet");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1];
 		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
 		json network;
@@ -1057,6 +1103,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Get(networkPath, networkGet);
 
 	auto createNewNetwork = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::createNewNetwork");
+		auto scope = tracer->WithActiveSpan(span);
+
 		// fprintf(stderr, "creating new network (new style)\n");
 		uint64_t nwid = 0;
 		uint64_t nwidPrefix = (Utils::hexStrToU64(_signingIdAddressString.c_str()) << 24) & 0xffffffffff000000ULL;
@@ -1084,6 +1135,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Post(networkListPath, createNewNetwork);
 
 	auto createNewNetworkOldAndBusted = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::createNewNetworkOldAndBusted");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto inID = req.matches[1].str();
 
 		if (inID != _signingIdAddressString) {
@@ -1116,6 +1172,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Post(oldAndBustedNetworkCreatePath, createNewNetworkOldAndBusted);
 
 	auto networkPost = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::networkPost");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1].str();
 		uint64_t nwid = Utils::hexStrToU64(networkID.c_str());
 
@@ -1128,6 +1189,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Post(networkPath, networkPost);
 
 	auto networkDelete = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::networkDelete");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1].str();
 		uint64_t nwid = Utils::hexStrToU64(networkID.c_str());
 
@@ -1144,6 +1210,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Delete(networkPath, networkDelete);
 
 	auto memberListGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::memberListGet");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1];
 		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
 		json network;
@@ -1170,6 +1241,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Get(memberListPath, memberListGet);
 
 	auto memberListGet2 = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::memberListGet2");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1];
 		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
 		json network;
@@ -1208,6 +1284,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Get(memberListPath2, memberListGet2);
 
 	auto memberGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::memberGet");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1];
 		auto memberID = req.matches[2];
 		uint64_t nwid = Utils::hexStrToU64(networkID.str().c_str());
@@ -1225,6 +1306,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Get(memberPath, memberGet);
 
 	auto memberPost = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::memberPost");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1].str();
 		auto memberID = req.matches[2].str();
 		uint64_t nwid = Utils::hexStrToU64(networkID.c_str());
@@ -1347,6 +1433,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 	sv6.Post(memberPath, memberPost);
 
 	auto memberDelete = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_controller");
+		auto span = tracer->StartSpan("embedded_controller::memberDelete");
+		auto scope = tracer->WithActiveSpan(span);
+
 		auto networkID = req.matches[1].str();
 		auto memberID = req.matches[2].str();
 
@@ -1374,6 +1465,11 @@ void EmbeddedNetworkController::configureHTTPControlPlane(httplib::Server& s, ht
 
 void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace& rt)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::handleRemoteTrace");
+	auto scope = tracer->WithActiveSpan(span);
+
 	static volatile unsigned long idCounter = 0;
 	char id[128], tmp[128];
 	std::string k, v;
@@ -1436,6 +1532,11 @@ void EmbeddedNetworkController::handleRemoteTrace(const ZT_RemoteTrace& rt)
 
 void EmbeddedNetworkController::onNetworkUpdate(const void* db, uint64_t networkId, const nlohmann::json& network)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::onNetworkUpdate");
+	auto scope = tracer->WithActiveSpan(span);
+
 	// Send an update to all members of the network that are online
 	const int64_t now = OSUtils::now();
 	std::lock_guard<std::mutex> l(_memberStatus_l);
@@ -1447,6 +1548,11 @@ void EmbeddedNetworkController::onNetworkUpdate(const void* db, uint64_t network
 
 void EmbeddedNetworkController::onNetworkMemberUpdate(const void* db, uint64_t networkId, uint64_t memberId, const nlohmann::json& member)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::onNetworkMemberUpdate");
+	auto scope = tracer->WithActiveSpan(span);
+
 	// Push update to member if online
 	try {
 		std::lock_guard<std::mutex> l(_memberStatus_l);
@@ -1460,6 +1566,11 @@ void EmbeddedNetworkController::onNetworkMemberUpdate(const void* db, uint64_t n
 
 void EmbeddedNetworkController::onNetworkMemberDeauthorize(const void* db, uint64_t networkId, uint64_t memberId)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::onNetworkMemberDeauthorize");
+	auto scope = tracer->WithActiveSpan(span);
+
 	const int64_t now = OSUtils::now();
 	Revocation rev((uint32_t)_node->prng(), networkId, 0, now, ZT_REVOCATION_FLAG_FAST_PROPAGATE, Address(memberId), Revocation::CREDENTIAL_TYPE_COM);
 	rev.sign(_signingId);
@@ -1474,6 +1585,11 @@ void EmbeddedNetworkController::onNetworkMemberDeauthorize(const void* db, uint6
 
 void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromAddr, uint64_t requestPacketId, const Identity& identity, const Dictionary<ZT_NETWORKCONFIG_METADATA_DICT_CAPACITY>& metaData)
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_controller");
+	auto span = tracer->StartSpan("embedded_controller::_request");
+	auto scope = tracer->WithActiveSpan(span);
+
 	Metrics::network_config_request++;
 	auto tid = std::this_thread::get_id();
 	std::stringstream ss;
@@ -2193,6 +2309,11 @@ void EmbeddedNetworkController::_request(uint64_t nwid, const InetAddress& fromA
 
 void EmbeddedNetworkController::_startThreads()
 {
+	auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+	auto tracer = provider->GetTracer("embedded_network_controller");
+	auto span = tracer->StartSpan("embedded_network_controller::_startThreads");
+	auto scope = tracer->WithActiveSpan(span);
+
 	std::lock_guard<std::mutex> l(_threads_l);
 	if (! _threads.empty()) {
 		return;
@@ -2232,6 +2353,11 @@ void EmbeddedNetworkController::_startThreads()
 void EmbeddedNetworkController::_ssoExpiryThread()
 {
 	while (_ssoExpiryRunning) {
+		auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+		auto tracer = provider->GetTracer("embedded_network_controller");
+		auto span = tracer->StartSpan("embedded_network_controller::_ssoExpiryThread");
+		auto scope = tracer->WithActiveSpan(span);
+
 		std::vector<_MemberStatusKey> expired;
 		nlohmann::json network, member;
 		int64_t now = OSUtils::now();

service/OneService.cpp

@@ -64,6 +64,8 @@
 #include "opentelemetry/sdk/resource/resource.h"
 #endif
 
+#include "opentelemetry/trace/provider.h"
+
 #if ZT_SSO_ENABLED
 #include <zeroidc.h>
 #endif
@@ -1836,7 +1838,14 @@ class OneServiceImpl : public OneService {
 		}
 
 		auto authCheck = [=](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::authCheck");
+			auto scope = tracer->WithActiveSpan(span);
+
 			if (req.path == "/metrics") {
+				auto mspan = tracer->StartSpan("http_control_plane::metricsAuth");
+				auto mscope = tracer->WithActiveSpan(mspan);
 				if (req.has_header("x-zt1-auth")) {
 					std::string token = req.get_header_value("x-zt1-auth");
 					if (token == _metricsToken || token == _authToken) {
@@ -1856,6 +1865,7 @@ class OneServiceImpl : public OneService {
 					}
 				}
 
+				span->SetAttribute("auth", "failed");
 				setContent(req, res, "{}");
 				res.status = 401;
 				return httplib::Server::HandlerResponse::Handled;
@@ -1912,9 +1922,13 @@ class OneServiceImpl : public OneService {
 					}
 				}
 
+				span->SetAttribute("ipAllowed", ipAllowed);
+
 				if (ipAllowed && isAuth) {
 					return httplib::Server::HandlerResponse::Unhandled;
 				}
+
+				span->SetAttribute("auth", "failed");
 				setContent(req, res, "{}");
 				res.status = 401;
 				return httplib::Server::HandlerResponse::Handled;
@@ -1922,6 +1936,11 @@ class OneServiceImpl : public OneService {
 		};
 
 		auto bondShow = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::bondShow");
+			auto scope = tracer->WithActiveSpan(span);
+
 			if (! _node->bondController()->inUse()) {
 				setContent(req, res, "");
 				res.status = 400;
@@ -1960,6 +1979,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(bondShowPath, bondShow);
 
 		auto bondRotate = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::bondRotate");
+			auto scope = tracer->WithActiveSpan(span);
+
 			if (! _node->bondController()->inUse()) {
 				setContent(req, res, "");
 				res.status = 400;
@@ -1988,6 +2012,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Put(bondRotatePath, bondRotate);
 
 		auto setMtu = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::setMtu");
+			auto scope = tracer->WithActiveSpan(span);
+
 			if (! _node->bondController()->inUse()) {
 				setContent(req, res, "Bonding layer isn't active yet");
 				res.status = 400;
@@ -2012,6 +2041,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Put(setBondMtuPath, setMtu);
 
 		auto getConfig = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::getConfig");
+			auto scope = tracer->WithActiveSpan(span);
+
 			std::string config;
 			{
 				Mutex::Lock lc(_localConfig_m);
@@ -2026,6 +2060,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(configPath, getConfig);
 
 		auto configPost = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::configPost");
+			auto scope = tracer->WithActiveSpan(span);
+
 			json j(OSUtils::jsonParse(req.body));
 			if (j.is_object()) {
 				Mutex::Lock lcl(_localConfig_m);
@@ -2046,6 +2085,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Put(configPostPath, configPost);
 
 		auto healthGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::healthGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			json out = json::object();
 
 			char tmp[256];
@@ -2068,6 +2112,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(healthPath, healthGet);
 
 		auto moonListGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::moonListGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			std::vector<World> moons(_node->moons());
 
 			auto out = json::array();
@@ -2082,6 +2131,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(moonListPath, moonListGet);
 
 		auto moonGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::moonGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			std::vector<World> moons(_node->moons());
 			auto input = req.matches[1];
 			auto out = json::object();
@@ -2098,6 +2152,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(moonPath, moonGet);
 
 		auto moonPost = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::moonPost");
+			auto scope = tracer->WithActiveSpan(span);
+
 			auto input = req.matches[1];
 			uint64_t seed = 0;
 			try {
@@ -2141,6 +2200,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Put(moonPath, moonPost);
 
 		auto moonDelete = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::moonDelete");
+			auto scope = tracer->WithActiveSpan(span);
+
 			auto input = req.matches[1];
 			uint64_t id = Utils::hexStrToU64(input.str().c_str());
 			auto out = json::object();
@@ -2151,6 +2215,11 @@ class OneServiceImpl : public OneService {
 		_controlPlane.Delete(moonPath, moonDelete);
 
 		auto networkListGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::networkListGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			Mutex::Lock _l(_nets_m);
 			auto out = json::array();
 
@@ -2166,6 +2235,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(networkListPath, networkListGet);
 
 		auto networkGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::networkGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			Mutex::Lock _l(_nets_m);
 
 			auto input = req.matches[1];
@@ -2184,6 +2258,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(networkPath, networkGet);
 
 		auto networkPost = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::networkPost");
+			auto scope = tracer->WithActiveSpan(span);
+
 			auto input = req.matches[1];
 			uint64_t wantnw = Utils::hexStrToU64(input.str().c_str());
 			_node->join(wantnw, (void*)0, (void*)0);
@@ -2242,6 +2321,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Put(networkPath, networkPost);
 
 		auto networkDelete = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::networkDelete");
+			auto scope = tracer->WithActiveSpan(span);
+
 			auto input = req.matches[1];
 			auto out = json::object();
 			ZT_VirtualNetworkList* nws = _node->networks();
@@ -2259,6 +2343,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Delete(networkPath, networkDelete);
 
 		auto peerListGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::peerListGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			ZT_PeerList* pl = _node->peers();
 			auto out = nlohmann::json::array();
 
@@ -2279,6 +2368,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(peerListPath, peerListGet);
 
 		auto peerGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::peerGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			ZT_PeerList* pl = _node->peers();
 
 			auto input = req.matches[1];
@@ -2301,6 +2395,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(peerPath, peerGet);
 
 		auto statusGet = [&, setContent](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::statusGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			ZT_NodeStatus status;
 			_node->status(&status);
 
@@ -2442,6 +2541,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(ssoPath, ssoGet);
 #endif
 		auto metricsGet = [this](const httplib::Request& req, httplib::Response& res) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::metricsGet");
+			auto scope = tracer->WithActiveSpan(span);
+
 			std::string statspath = _homePath + ZT_PATH_SEPARATOR + "metrics.prom";
 			std::string metrics;
 			if (OSUtils::readFile(statspath.c_str(), metrics)) {
@@ -2456,6 +2560,11 @@ class OneServiceImpl : public OneService {
 		_controlPlaneV6.Get(metricsPath, metricsGet);
 
 		auto exceptionHandler = [&, setContent](const httplib::Request& req, httplib::Response& res, std::exception_ptr ep) {
+			auto provider = opentelemetry::trace::Provider::GetTracerProvider();
+			auto tracer = provider->GetTracer("http_control_plane");
+			auto span = tracer->StartSpan("http_control_plane::exceptionHandler");
+			auto scope = tracer->WithActiveSpan(span);
+
 			char buf[1024];
 			auto fmt = "{\"error\": %d, \"description\": \"%s\"}";
 			try {