ZeroTierOne/node/Metrics.cpp

/*
 * Copyright (c)2013-2023 ZeroTier, Inc.
 *
 * Use of this software is governed by the Business Source License included
 * in the LICENSE.TXT file in the project's root directory.
 *
 * Change Date: 2026-01-01
 *
 * On the date above, in accordance with the Business Source License, use
 * of this software will be governed by version 2.0 of the Apache License.
 */

// clang-format off
#include <prometheus/simpleapi.h>
#include <prometheus/histogram.h>
#include "Metrics.hpp"
// clang-format on

namespace prometheus {
namespace simpleapi {
std::shared_ptr<Registry> registry_ptr = std::make_shared<Registry>();
Registry& registry = *registry_ptr;
SaveToFile saver;
}	// namespace simpleapi
}	// namespace prometheus

namespace ZeroTier {
namespace Metrics {
// Packet Type Counts
prometheus::simpleapi::counter_family_t packets { "zt_packet", "ZeroTier packet type counts" };

// Incoming packets
prometheus::simpleapi::counter_metric_t pkt_nop_in { packets.Add({ { "packet_type", "nop" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_in { packets.Add({ { "packet_type", "error" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_ack_in { packets.Add({ { "packet_type", "ack" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_qos_in { packets.Add({ { "packet_type", "qos" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_hello_in { packets.Add({ { "packet_type", "hello" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_ok_in { packets.Add({ { "packet_type", "ok" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_whois_in { packets.Add({ { "packet_type", "whois" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_rendezvous_in { packets.Add({ { "packet_type", "rendezvous" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_frame_in { packets.Add({ { "packet_type", "frame" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_ext_frame_in { packets.Add({ { "packet_type", "ext_frame" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_echo_in { packets.Add({ { "packet_type", "echo" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_multicast_like_in { packets.Add({ { "packet_type", "multicast_like" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_network_credentials_in { packets.Add({ { "packet_type", "network_credentials" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_network_config_request_in { packets.Add({ { "packet_type", "network_config_request" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_network_config_in { packets.Add({ { "packet_type", "network_config" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_multicast_gather_in { packets.Add({ { "packet_type", "multicast_gather" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_multicast_frame_in { packets.Add({ { "packet_type", "multicast_frame" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_in { packets.Add({ { "packet_type", "push_direct_paths" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_user_message_in { packets.Add({ { "packet_type", "user_message" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_remote_trace_in { packets.Add({ { "packet_type", "remote_trace" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_in { packets.Add({ { "packet_type", "path_negotiation_request" }, { "direction", "rx" } }) };

// Outgoing packets
prometheus::simpleapi::counter_metric_t pkt_nop_out { packets.Add({ { "packet_type", "nop" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_out { packets.Add({ { "packet_type", "error" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_ack_out { packets.Add({ { "packet_type", "ack" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_qos_out { packets.Add({ { "packet_type", "qos" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_hello_out { packets.Add({ { "packet_type", "hello" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_ok_out { packets.Add({ { "packet_type", "ok" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_whois_out { packets.Add({ { "packet_type", "whois" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_rendezvous_out { packets.Add({ { "packet_type", "rendezvous" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_frame_out { packets.Add({ { "packet_type", "frame" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_ext_frame_out { packets.Add({ { "packet_type", "ext_frame" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_echo_out { packets.Add({ { "packet_type", "echo" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_multicast_like_out { packets.Add({ { "packet_type", "multicast_like" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_network_credentials_out { packets.Add({ { "packet_type", "network_credentials" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_network_config_request_out { packets.Add({ { "packet_type", "network_config_request" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_network_config_out { packets.Add({ { "packet_type", "network_config" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_multicast_gather_out { packets.Add({ { "packet_type", "multicast_gather" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_multicast_frame_out { packets.Add({ { "packet_type", "multicast_frame" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_push_direct_paths_out { packets.Add({ { "packet_type", "push_direct_paths" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_user_message_out { packets.Add({ { "packet_type", "user_message" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_remote_trace_out { packets.Add({ { "packet_type", "remote_trace" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_path_negotiation_request_out { packets.Add({ { "packet_type", "path_negotiation_request" }, { "direction", "tx" } }) };

// Packet Error Counts
prometheus::simpleapi::counter_family_t packet_errors { "zt_packet_error", "ZeroTier packet errors" };

// Incoming Error Counts
prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_in { packet_errors.Add({ { "error_type", "obj_not_found" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_in { packet_errors.Add({ { "error_type", "unsupported_operation" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_in { packet_errors.Add({ { "error_type", "identity_collision" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_in { packet_errors.Add({ { "error_type", "need_membership_certificate" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_in { packet_errors.Add({ { "error_type", "network_access_denied" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_in { packet_errors.Add({ { "error_type", "unwanted_multicast" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_in { packet_errors.Add({ { "error_type", "authentication_required" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_in { packet_errors.Add({ { "error_type", "internal_server_error" }, { "direction", "rx" } }) };

// Outgoing Error Counts
prometheus::simpleapi::counter_metric_t pkt_error_obj_not_found_out { packet_errors.Add({ { "error_type", "obj_not_found" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_unsupported_op_out { packet_errors.Add({ { "error_type", "unsupported_operation" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_identity_collision_out { packet_errors.Add({ { "error_type", "identity_collision" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_need_membership_cert_out { packet_errors.Add({ { "error_type", "need_membership_certificate" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_network_access_denied_out { packet_errors.Add({ { "error_type", "network_access_denied" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_unwanted_multicast_out { packet_errors.Add({ { "error_type", "unwanted_multicast" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_authentication_required_out { packet_errors.Add({ { "error_type", "authentication_required" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t pkt_error_internal_server_error_out { packet_errors.Add({ { "error_type", "internal_server_error" }, { "direction", "tx" } }) };

// Data Sent/Received Metrics
prometheus::simpleapi::counter_family_t data { "zt_data", "number of bytes ZeroTier has transmitted or received" };
prometheus::simpleapi::counter_metric_t udp_recv { data.Add({ { "protocol", "udp" }, { "direction", "rx" } }) };
prometheus::simpleapi::counter_metric_t udp_send { data.Add({ { "protocol", "udp" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t tcp_send { data.Add({ { "protocol", "tcp" }, { "direction", "tx" } }) };
prometheus::simpleapi::counter_metric_t tcp_recv { data.Add({ { "protocol", "tcp" }, { "direction", "rx" } }) };

// Network Metrics
prometheus::simpleapi::gauge_metric_t network_num_joined { "zt_num_networks", "number of networks this instance is joined to" };
prometheus::simpleapi::gauge_family_t network_num_multicast_groups { "zt_network_multicast_groups_subscribed", "number of multicast groups networks are subscribed to" };
prometheus::simpleapi::counter_family_t network_packets { "zt_network_packets", "number of incoming/outgoing packets per network" };

#ifndef ZT_NO_PEER_METRICS
// PeerMetrics
prometheus::CustomFamily<prometheus::Histogram<uint64_t> >& peer_latency = prometheus::Builder<prometheus::Histogram<uint64_t> >().Name("zt_peer_latency").Help("peer latency (ms)").Register(prometheus::simpleapi::registry);

prometheus::simpleapi::gauge_family_t peer_path_count { "zt_peer_path_count", "number of paths to peer" };
prometheus::simpleapi::counter_family_t peer_packets { "zt_peer_packets", "number of packets to/from a peer" };
prometheus::simpleapi::counter_family_t peer_packet_errors { "zt_peer_packet_errors", "number of incoming packet errors from a peer" };
#endif

// General Controller Metrics
prometheus::simpleapi::gauge_metric_t network_count { "controller_network_count", "number of networks the controller is serving" };
prometheus::simpleapi::gauge_metric_t member_count { "controller_member_count", "number of network members the controller is serving" };
prometheus::simpleapi::counter_metric_t network_changes { "controller_network_change_count", "number of times a network configuration is changed" };
prometheus::simpleapi::counter_metric_t member_changes { "controller_member_change_count", "number of times a network member configuration is changed" };
prometheus::simpleapi::counter_metric_t member_auths { "controller_member_auth_count", "number of network member auths" };
prometheus::simpleapi::counter_metric_t member_deauths { "controller_member_deauth_count", "number of network member deauths" };

prometheus::simpleapi::gauge_metric_t network_config_request_queue_size { "controller_network_config_request_queue", "number of entries in the request queue for network configurations" };

prometheus::simpleapi::counter_metric_t sso_expiration_checks { "controller_sso_expiration_checks", "number of sso expiration checks done" };

prometheus::simpleapi::counter_metric_t sso_member_deauth { "controller_sso_timeouts", "number of sso timeouts" };

prometheus::simpleapi::counter_metric_t network_config_request { "controller_network_config_request", "count of config requests handled" };
prometheus::simpleapi::gauge_metric_t network_config_request_threads { "controller_network_config_request_threads", "number of active network config handling threads" };
prometheus::simpleapi::counter_metric_t db_get_network { "controller_db_get_network", "counter" };
prometheus::simpleapi::counter_metric_t db_get_network_and_member { "controller_db_get_network_and_member", "counter" };
prometheus::simpleapi::counter_metric_t db_get_network_and_member_and_summary { "controller_db_get_networK_and_member_summary", "counter" };
prometheus::simpleapi::counter_metric_t db_get_member_list { "controller_db_get_member_list", "counter" };
prometheus::simpleapi::counter_metric_t db_get_network_list { "controller_db_get_network_list", "counter" };
prometheus::simpleapi::counter_metric_t db_member_change { "controller_db_member_change", "counter" };
prometheus::simpleapi::counter_metric_t db_network_change { "controller_db_network_change", "counter" };

#ifdef ZT_CONTROLLER_USE_LIBPQ
// Central Controller Metrics
prometheus::simpleapi::counter_metric_t pgsql_mem_notification { "controller_pgsql_member_notifications_received", "number of member change notifications received via pgsql NOTIFY" };
prometheus::simpleapi::counter_metric_t pgsql_net_notification { "controller_pgsql_network_notifications_received", "number of network change notifications received via pgsql NOTIFY" };
prometheus::simpleapi::counter_metric_t pgsql_node_checkin { "controller_pgsql_node_checkin_count", "number of node check-ins (pgsql)" };
prometheus::simpleapi::counter_metric_t pgsql_commit_ticks { "controller_pgsql_commit_ticks", "number of commit ticks run (pgsql)" };
prometheus::simpleapi::counter_metric_t db_get_sso_info { "controller_db_get_sso_info", "counter" };

prometheus::simpleapi::counter_metric_t redis_mem_notification { "controller_redis_member_notifications_received", "number of member change notifications received via redis" };
prometheus::simpleapi::counter_metric_t redis_net_notification { "controller_redis_network_notifications_received", "number of network change notifications received via redis" };
prometheus::simpleapi::counter_metric_t redis_node_checkin { "controller_redis_node_checkin_count", "number of node check-ins (redis)" };

// Central DB Pool Metrics
prometheus::simpleapi::counter_metric_t conn_counter { "controller_pgsql_connections_created", "number of pgsql connections created" };
prometheus::simpleapi::counter_metric_t max_pool_size { "controller_pgsql_max_conn_pool_size", "max connection pool size for postgres" };
prometheus::simpleapi::counter_metric_t min_pool_size { "controller_pgsql_min_conn_pool_size", "minimum connection pool size for postgres" };
prometheus::simpleapi::gauge_metric_t pool_avail { "controller_pgsql_available_conns", "number of available postgres connections" };
prometheus::simpleapi::gauge_metric_t pool_in_use { "controller_pgsql_in_use_conns", "number of postgres database connections in use" };
prometheus::simpleapi::counter_metric_t pool_errors { "controller_pgsql_connection_errors", "number of connection errors the connection pool has seen" };
#endif

        // Fragmentation Metrics
        prometheus::simpleapi::counter_family_t packet_fragmentation
        { "zt_packet_fragmentation", "ZeroTier packet fragmentation events" };

        // VL2 Fragmentation Metrics
        prometheus::simpleapi::counter_metric_t vl2_oversized_frame_tx
        { packet_fragmentation.Add({{"layer", "VL2"}, {"direction", "tx"}, {"reason", "oversized_frame"}}) };
        prometheus::simpleapi::counter_metric_t vl2_would_fragment_or_drop_rx
        { packet_fragmentation.Add({{"layer", "VL2"}, {"direction", "rx"}, {"reason", "would_fragment_or_drop"}}) };

        // VL1 Fragmentation Metrics
        prometheus::simpleapi::counter_metric_t vl1_fragmented_tx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "tx"}, {"reason", "mtu_exceeded"}}) };
        prometheus::simpleapi::counter_metric_t vl1_fragment_rx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "fragment"}}) };
        prometheus::simpleapi::counter_metric_t vl1_reassembly_failed_rx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "reassembly_failed"}}) };
        prometheus::simpleapi::counter_metric_t vl1_fragment_without_head_rx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "fragment_without_head"}}) };
        prometheus::simpleapi::counter_metric_t vl1_fragment_before_head_rx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "fragment_before_head"}}) };
        prometheus::simpleapi::counter_metric_t vl1_duplicate_fragment_rx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "duplicate_fragment"}}) };
        prometheus::simpleapi::counter_metric_t vl1_duplicate_head_rx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "duplicate_head"}}) };

        // VL1 Fragmentation Histogram and Counters
        prometheus::CustomFamily<prometheus::Histogram<uint64_t>> &vl1_fragments_per_packet_histogram =
            prometheus::Builder<prometheus::Histogram<uint64_t>>()
                .Name("zt_vl1_fragments_per_packet")
                .Help("Histogram of fragments per packet at VL1")
                .Register(prometheus::simpleapi::registry);
        prometheus::simpleapi::counter_metric_t vl1_incomplete_reassembly_rx
        { packet_fragmentation.Add({{"layer", "VL1"}, {"direction", "rx"}, {"reason", "incomplete_reassembly"}}) };
        prometheus::simpleapi::counter_metric_t vl1_vl2_double_fragmentation_tx
        { packet_fragmentation.Add({{"layer", "VL1_VL2"}, {"direction", "tx"}, {"reason", "double_fragmentation"}}) };

        prometheus::Histogram<uint64_t> &vl1_fragments_per_packet_hist =
            vl1_fragments_per_packet_histogram.Add(
                {},
                std::vector<uint64_t>(std::begin(ZeroTier::Metrics::VL1_FRAGMENTS_PER_PACKET_BUCKETS), std::end(ZeroTier::Metrics::VL1_FRAGMENTS_PER_PACKET_BUCKETS))
            );

        // VL2 Frame Size Histogram
        // Buckets: 512 (IoT/legacy), 576 (min IPv4), 1200 (QUIC/mobile), 1280 (min IPv6),
        // 1332, 1380, 1400 (VPN/overlay), 1420 (cloud), 1460 (TCP MSS), 1472 (ICMP/MTU),
        // 1480 (ICMP/MTU), 1492 (PPPoE), 1500 (Ethernet), 2800 (VL2 default), 9000 (jumbo)
        prometheus::CustomFamily<prometheus::Histogram<uint64_t>> &vl2_frame_size_histogram =
            prometheus::Builder<prometheus::Histogram<uint64_t>>()
                .Name("zt_vl2_frame_size")
                .Help("Histogram of frame sizes delivered to TAP (VL2)")
                .Register(prometheus::simpleapi::registry);
        prometheus::simpleapi::counter_metric_t vl2_incomplete_reassembly_rx
        { packet_fragmentation.Add({{"layer", "VL2"}, {"direction", "rx"}, {"reason", "incomplete_reassembly"}}) };
        prometheus::simpleapi::counter_metric_t vl2_vl1_double_fragmentation_tx
        { packet_fragmentation.Add({{"layer", "VL2_VL1"}, {"direction", "tx"}, {"reason", "double_fragmentation"}}) };

        prometheus::Histogram<uint64_t> &vl2_frame_size_hist =
            vl2_frame_size_histogram.Add(
                {},
                std::vector<uint64_t>(std::begin(ZeroTier::Metrics::VL2_FRAME_SIZE_BUCKETS), std::end(ZeroTier::Metrics::VL2_FRAME_SIZE_BUCKETS))
            );
    }
}