Merge branch 'develop' of https://github.com/gravitl/netmaker into NET-1911

Dockerfile

@@ -6,7 +6,7 @@ COPY . .
 
 RUN GOOS=linux CGO_ENABLED=1 go build -ldflags="-s -w " -tags ${tags} .
 # RUN go build -tags=ee . -o netmaker main.go
-FROM alpine:3.21.0
+FROM alpine:3.21.2
 
 # add a c lib
 # set the working directory

Dockerfile-quick

@@ -1,5 +1,5 @@
 #first stage - builder
-FROM alpine:3.21.0
+FROM alpine:3.21.2
 ARG version 
 WORKDIR /app
 COPY ./netmaker /root/netmaker

compose/docker-compose.yml

@@ -12,7 +12,7 @@ services:
       - sqldata:/root/data
     environment:
       # config-dependant vars
-      - STUN_SERVERS=stun1.l.google.com:19302,stun2.l.google.com:19302,stun3.l.google.com:19302,stun4.l.google.com:19302
+      - STUN_SERVERS=stun.${NM_DOMAIN}:3478,stun1.l.google.com:19302,stun2.l.google.com:19302,stun3.l.google.com:19302,stun4.l.google.com:19302
       # The domain/host IP indicating the mq broker address
       - BROKER_ENDPOINT=wss://broker.${NM_DOMAIN} # For EMQX broker use `BROKER_ENDPOINT=wss://broker.${NM_DOMAIN}/mqtt`
       # For EMQX broker (uncomment the two lines below)
@@ -39,6 +39,14 @@ services:
     links:
       - "netmaker:api"
     restart: always
+  stun:
+    container_name: stun
+    image: coturn/coturn
+    restart: always
+    ports:
+      - "3478:3478/udp"   # STUN UDP
+    environment:
+     - LISTENING_PORT=3478
 
   caddy:
     image: caddy:2.8.4
@@ -52,8 +60,8 @@ services:
       - caddy_data:/data
       - caddy_conf:/config
     ports:
-      - "80:80"
-      - "443:443"
+      - "80:80/tcp"
+      - "443:443/tcp"
 
   coredns:
     #network_mode: host

config/config.go

@@ -92,14 +92,15 @@ type ServerConfig struct {
 	JwtValidityDuration        time.Duration `yaml:"jwt_validity_duration" swaggertype:"primitive,integer" format:"int64"`
 	RacAutoDisable             bool          `yaml:"rac_auto_disable"`
 	CacheEnabled               string        `yaml:"caching_enabled"`
-	EndpointDetection          bool          `json:"endpoint_detection"`
+	EndpointDetection          bool          `yaml:"endpoint_detection"`
 	AllowedEmailDomains        string        `yaml:"allowed_email_domains"`
-	EmailSenderAddr            string        `json:"email_sender_addr"`
-	EmailSenderUser            string        `json:"email_sender_user"`
-	EmailSenderPassword        string        `json:"email_sender_password"`
-	SmtpHost                   string        `json:"smtp_host"`
-	SmtpPort                   int           `json:"smtp_port"`
+	EmailSenderAddr            string        `yaml:"email_sender_addr"`
+	EmailSenderUser            string        `yaml:"email_sender_user"`
+	EmailSenderPassword        string        `yaml:"email_sender_password"`
+	SmtpHost                   string        `yaml:"smtp_host"`
+	SmtpPort                   int           `yaml:"smtp_port"`
 	MetricInterval             string        `yaml:"metric_interval"`
+	MetricsPort                int           `yaml:"metrics_port"`
 	ManageDNS                  bool          `yaml:"manage_dns"`
 	Stun                       bool          `yaml:"stun"`
 	StunServers                string        `yaml:"stun_servers"`

controllers/hosts.go

@@ -5,7 +5,6 @@ import (
 	"errors"
 	"fmt"
 	"net/http"
-	"os"
 
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
@@ -239,11 +238,7 @@ func pull(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	if sendPeerUpdate {
-		reset := true
-		if os.Getenv("RESET_PEER_UPDATE") != "" {
-			reset = os.Getenv("RESET_PEER_UPDATE") == "true"
-		}
-		if err := mq.PublishPeerUpdate(reset); err != nil {
+		if err := mq.PublishPeerUpdate(false); err != nil {
 			logger.Log(0, "fail to publish peer update: ", err.Error())
 		}
 	}
@@ -373,11 +368,11 @@ func hostUpdateFallback(w http.ResponseWriter, r *http.Request) {
 	var hostUpdate models.HostUpdate
 	err = json.NewDecoder(r.Body).Decode(&hostUpdate)
 	if err != nil {
-		logger.Log(0, r.Header.Get("user"), "failed to update a host:", err.Error())
+		slog.Error("failed to update a host:", "user", r.Header.Get("user"), "error", err.Error())
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
-	slog.Info("recieved host update", "name", hostUpdate.Host.Name, "id", hostUpdate.Host.ID)
+	slog.Info("recieved host update", "name", hostUpdate.Host.Name, "id", hostUpdate.Host.ID, "action", hostUpdate.Action)
 	switch hostUpdate.Action {
 	case models.CheckIn:
 		sendPeerUpdate = mq.HandleHostCheckin(&hostUpdate.Host, currentHost)
@@ -594,7 +589,7 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) {
 					w,
 					r,
 					logic.FormatError(
-						fmt.Errorf("failed to force delete daemon node: "+err.Error()),
+						fmt.Errorf("failed to force delete daemon node: %s", err.Error()),
 						"internal",
 					),
 				)
@@ -634,7 +629,7 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) {
 					w,
 					r,
 					logic.FormatError(
-						fmt.Errorf("failed to force delete daemon node: "+err.Error()),
+						fmt.Errorf("failed to force delete daemon node: %s", err.Error()),
 						"internal",
 					),
 				)

controllers/network.go

@@ -434,6 +434,7 @@ func getNetworkACL(w http.ResponseWriter, r *http.Request) {
 // @Tags        Networks
 // @Security    oauth
 // @Param       networkname path string true "Network name"
+// @Param       force query bool false "Force Delete"
 // @Produce     json
 // @Success     200 {object} models.SuccessResponse
 // @Failure     400 {object} models.ErrorResponse
@@ -441,10 +442,18 @@ func getNetworkACL(w http.ResponseWriter, r *http.Request) {
 func deleteNetwork(w http.ResponseWriter, r *http.Request) {
 	// Set header
 	w.Header().Set("Content-Type", "application/json")
-
+	force := r.URL.Query().Get("force") == "true"
 	var params = mux.Vars(r)
 	network := params["networkname"]
-	err := logic.DeleteNetwork(network)
+	doneCh := make(chan struct{}, 1)
+	networkNodes, err := logic.GetNetworkNodes(network)
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"),
+			fmt.Sprintf("failed to get network nodes [%s]: %v", network, err))
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	err = logic.DeleteNetwork(network, force, doneCh)
 	if err != nil {
 		errtype := "badrequest"
 		if strings.Contains(err.Error(), "Node check failed") {
@@ -459,7 +468,22 @@ func deleteNetwork(w http.ResponseWriter, r *http.Request) {
 	go logic.DeleteDefaultNetworkPolicies(models.NetworkID(network))
 	//delete network from allocated ip map
 	go logic.RemoveNetworkFromAllocatedIpMap(network)
-
+	go func() {
+		<-doneCh
+		mq.PublishPeerUpdate(true)
+		// send node update to clean up locally
+		for _, node := range networkNodes {
+			node := node
+			node.PendingDelete = true
+			node.Action = models.NODE_DELETE
+			if err := mq.NodeUpdate(&node); err != nil {
+				slog.Error("error publishing node update to node", "node", node.ID, "error", err)
+			}
+		}
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
+		}
+	}()
 	logger.Log(1, r.Header.Get("user"), "deleted network", network)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode("success")
@@ -608,15 +632,15 @@ func updateNetwork(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	netOld1, err := logic.GetNetwork(payload.NetID)
+	netOld, err := logic.GetNetwork(payload.NetID)
 	if err != nil {
 		slog.Info("error fetching network", "user", r.Header.Get("user"), "err", err)
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 		return
 	}
-	// partial update
-	netOld2 := netOld1
-	_, _, _, err = logic.UpdateNetwork(&netOld1, &netOld2)
+	netNew := netOld
+	netNew.DefaultACL = payload.DefaultACL
+	_, _, _, err = logic.UpdateNetwork(&netOld, &netNew)
 	if err != nil {
 		slog.Info("failed to update network", "user", r.Header.Get("user"), "err", err)
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))

controllers/network_test.go

@@ -75,11 +75,19 @@ func TestDeleteNetwork(t *testing.T) {
 	t.Run("NetworkwithNodes", func(t *testing.T) {
 	})
 	t.Run("DeleteExistingNetwork", func(t *testing.T) {
-		err := logic.DeleteNetwork("skynet")
+		doneCh := make(chan struct{}, 1)
+		err := logic.DeleteNetwork("skynet", false, doneCh)
 		assert.Nil(t, err)
 	})
 	t.Run("NonExistentNetwork", func(t *testing.T) {
-		err := logic.DeleteNetwork("skynet")
+		doneCh := make(chan struct{}, 1)
+		err := logic.DeleteNetwork("skynet", false, doneCh)
+		assert.Nil(t, err)
+	})
+	createNetv1("test")
+	t.Run("ForceDeleteNetwork", func(t *testing.T) {
+		doneCh := make(chan struct{}, 1)
+		err := logic.DeleteNetwork("test", true, doneCh)
 		assert.Nil(t, err)
 	})
 }
@@ -214,6 +222,15 @@ func createNet() {
 		logic.CreateNetwork(network)
 	}
 }
+func createNetv1(netId string) {
+	var network models.Network
+	network.NetID = netId
+	network.AddressRange = "100.0.0.1/24"
+	_, err := logic.GetNetwork(netId)
+	if err != nil {
+		logic.CreateNetwork(network)
+	}
+}
 
 func createNetDualStack() {
 	var network models.Network

docker/Caddyfile

@@ -32,3 +32,7 @@ broker.{$NM_DOMAIN} {
 		}
 	reverse_proxy @ws mq:8883   # For EMQX websockets use `reverse_proxy @ws mq:8083`
 }
+
+https://stun.{$NM_DOMAIN} {
+	reverse_proxy stun:3478
+}

go.mod

@@ -5,7 +5,7 @@ go 1.23
 require (
 	github.com/blang/semver v3.5.1+incompatible
 	github.com/eclipse/paho.mqtt.golang v1.4.3
-	github.com/go-playground/validator/v10 v10.23.0
+	github.com/go-playground/validator/v10 v10.24.0
 	github.com/golang-jwt/jwt/v4 v4.5.1
 	github.com/google/uuid v1.6.0
 	github.com/gorilla/handlers v1.5.2
@@ -18,10 +18,10 @@ require (
 	github.com/stretchr/testify v1.10.0
 	github.com/txn2/txeh v1.5.5
 	go.uber.org/automaxprocs v1.6.0
-	golang.org/x/crypto v0.30.0
-	golang.org/x/net v0.27.0 // indirect
+	golang.org/x/crypto v0.32.0
+	golang.org/x/net v0.34.0 // indirect
 	golang.org/x/oauth2 v0.24.0
-	golang.org/x/sys v0.28.0 // indirect
+	golang.org/x/sys v0.29.0 // indirect
 	golang.org/x/text v0.21.0 // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb
 	gopkg.in/yaml.v3 v3.0.1
@@ -50,7 +50,7 @@ require (
 
 require (
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
-	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.8 // indirect
 	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/kr/text v0.2.0 // indirect

go.sum

@@ -17,8 +17,8 @@ github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQ
 github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
-github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
-github.com/gabriel-vasile/mimetype v1.4.3/go.mod h1:d8uq/6HKRL6CGdk+aubisF/M5GcPfT7nKyLpA0lbSSk=
+github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM=
+github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8=
 github.com/go-jose/go-jose/v3 v3.0.3 h1:fFKWeig/irsp7XD2zBxvnmA/XaRWp5V3CBsZXJF7G7k=
 github.com/go-jose/go-jose/v3 v3.0.3/go.mod h1:5b+7YgP7ZICgJDBdfjZaIt+H/9L9T/YQrVfLAMboGkQ=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
@@ -27,8 +27,8 @@ github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/o
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o=
-github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM=
+github.com/go-playground/validator/v10 v10.24.0 h1:KHQckvo8G6hlWnrPX4NJJ+aBfWNAE/HH+qdL2cBpCmg=
+github.com/go-playground/validator/v10 v10.24.0/go.mod h1:GGzBIJMuE98Ic/kJsBXbz1x/7cByt++cQ+YOuDM5wus=
 github.com/golang-jwt/jwt/v4 v4.5.1 h1:JdqV9zKUdtaa9gdPlywC3aeoEsR681PlKC+4F5gQgeo=
 github.com/golang-jwt/jwt/v4 v4.5.1/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
@@ -101,8 +101,8 @@ go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwE
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
-golang.org/x/crypto v0.30.0 h1:RwoQn3GkWiMkzlX562cLB7OxWvjH1L8xutO2WoJcRoY=
-golang.org/x/crypto v0.30.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc=
+golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -112,8 +112,8 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
 golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
+golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0=
+golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k=
 golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE=
 golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -129,8 +129,8 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU=
+golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

logic/networks.go

@@ -102,7 +102,7 @@ func RemoveIpFromAllocatedIpMap(networkName string, ip string) {
 // AddNetworkToAllocatedIpMap - add network to allocated ip map when network is added
 func AddNetworkToAllocatedIpMap(networkName string) {
 	networkCacheMutex.Lock()
-	allocatedIpMap[networkName] = map[string]net.IP{}
+	allocatedIpMap[networkName] = make(map[string]net.IP)
 	networkCacheMutex.Unlock()
 }
 
@@ -171,23 +171,8 @@ func GetNetworks() ([]models.Network, error) {
 }
 
 // DeleteNetwork - deletes a network
-func DeleteNetwork(network string) error {
-	// remove ACL for network
-	err := nodeacls.DeleteACLContainer(nodeacls.NetworkID(network))
-	if err != nil {
-		logger.Log(1, "failed to remove the node acls during network delete for network,", network)
-	}
-	// Delete default network enrollment key
-	keys, _ := GetAllEnrollmentKeys()
-	for _, key := range keys {
-		if key.Tags[0] == network {
-			if key.Default {
-				DeleteEnrollmentKey(key.Value, true)
-				break
-			}
+func DeleteNetwork(network string, force bool, done chan struct{}) error {
 
-		}
-	}
 	nodeCount, err := GetNetworkNonServerNodeCount(network)
 	if nodeCount == 0 || database.IsEmptyRecord(err) {
 		// delete server nodes first then db records
@@ -200,7 +185,50 @@ func DeleteNetwork(network string) error {
 		}
 		return nil
 	}
-	return errors.New("node check failed. All nodes must be deleted before deleting network")
+
+	// Remove All Nodes
+	go func() {
+		nodes, err := GetNetworkNodes(network)
+		if err == nil {
+			for _, node := range nodes {
+				node := node
+				host, err := GetHost(node.HostID.String())
+				if err != nil {
+					continue
+				}
+				DissasociateNodeFromHost(&node, host)
+			}
+		}
+		// remove ACL for network
+		err = nodeacls.DeleteACLContainer(nodeacls.NetworkID(network))
+		if err != nil {
+			logger.Log(1, "failed to remove the node acls during network delete for network,", network)
+		}
+		// delete server nodes first then db records
+		err = database.DeleteRecord(database.NETWORKS_TABLE_NAME, network)
+		if err != nil {
+			return
+		}
+		if servercfg.CacheEnabled() {
+			deleteNetworkFromCache(network)
+		}
+		done <- struct{}{}
+		close(done)
+	}()
+
+	// Delete default network enrollment key
+	keys, _ := GetAllEnrollmentKeys()
+	for _, key := range keys {
+		if key.Tags[0] == network {
+			if key.Default {
+				DeleteEnrollmentKey(key.Value, true)
+				break
+			}
+
+		}
+	}
+
+	return nil
 }
 
 // CreateNetwork - creates a network in database

logic/nodes.go

@@ -239,7 +239,7 @@ func UpdateNode(currentNode *models.Node, newNode *models.Node) error {
 		}
 	}
 
-	return fmt.Errorf("failed to update node " + currentNode.ID.String() + ", cannot change ID.")
+	return fmt.Errorf("failed to update node %s, cannot change ID", currentNode.ID.String())
 }
 
 // DeleteNode - marks node for deletion (and adds to zombie list) if called by UI or deletes node if called by node

logic/peers.go

@@ -79,11 +79,11 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 			IngressInfo: make(map[string]models.IngressInfo),
 			AclRules:    make(map[string]models.AclRule),
 		},
-		PeerIDs:           make(models.PeerMap, 0),
-		Peers:             []wgtypes.PeerConfig{},
-		NodePeers:         []wgtypes.PeerConfig{},
-		HostNetworkInfo:   models.HostInfoMap{},
-		EndpointDetection: servercfg.IsEndpointDetectionEnabled(),
+		PeerIDs:         make(models.PeerMap, 0),
+		Peers:           []wgtypes.PeerConfig{},
+		NodePeers:       []wgtypes.PeerConfig{},
+		HostNetworkInfo: models.HostInfoMap{},
+		ServerConfig:    servercfg.ServerInfo,
 	}
 	defer func() {
 		if !hostPeerUpdate.FwUpdate.AllowAll {
@@ -187,7 +187,14 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 
 		} else {
 			hostPeerUpdate.FwUpdate.AllowAll = false
-			hostPeerUpdate.FwUpdate.AclRules = GetAclRulesForNode(&node)
+			rules := GetAclRulesForNode(&node)
+			if len(hostPeerUpdate.FwUpdate.AclRules) == 0 {
+				hostPeerUpdate.FwUpdate.AclRules = rules
+			} else {
+				for aclID, rule := range rules {
+					hostPeerUpdate.FwUpdate.AclRules[aclID] = rule
+				}
+			}
 		}
 
 		currentPeers := GetNetworkNodesMemory(allNodes, node.Network)
@@ -457,10 +464,6 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 			}
 		}
 	}
-
-	hostPeerUpdate.ManageDNS = servercfg.GetManageDNS()
-	hostPeerUpdate.Stun = servercfg.IsStunEnabled()
-	hostPeerUpdate.StunServers = servercfg.GetStunServers()
 	return hostPeerUpdate, nil
 }
 

logic/util.go

@@ -7,6 +7,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"log/slog"
 	"net"
 	"os"
 	"strings"
@@ -91,6 +92,30 @@ func StringSliceContains(slice []string, item string) bool {
 	}
 	return false
 }
+func SetVerbosity(logLevel int) {
+	var level slog.Level
+	switch logLevel {
+
+	case 0:
+		level = slog.LevelInfo
+	case 1:
+		level = slog.LevelError
+	case 2:
+		level = slog.LevelWarn
+	case 3:
+		level = slog.LevelDebug
+
+	default:
+		level = slog.LevelInfo
+	}
+	// Create the logger with the chosen level
+	handler := slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{
+		Level: level,
+	})
+	logger := slog.New(handler)
+	slog.SetDefault(logger)
+
+}
 
 // NormalizeCIDR - returns the first address of CIDR
 func NormalizeCIDR(address string) (string, error) {

models/metrics.go

@@ -48,6 +48,7 @@ type HostNetworkInfo struct {
 	ListenPort   int     `json:"listen_port" yaml:"listen_port"`
 	IsStaticPort bool    `json:"is_static_port"`
 	IsStatic     bool    `json:"is_static"`
+	Version      string  `json:"version"`
 }
 
 // PeerMap - peer map for ids and addresses in metrics

models/mqtt.go

@@ -8,25 +8,29 @@ import (
 
 // HostPeerUpdate - struct for host peer updates
 type HostPeerUpdate struct {
-	Host              Host                 `json:"host" bson:"host" yaml:"host"`
-	ChangeDefaultGw   bool                 `json:"change_default_gw"`
-	DefaultGwIp       net.IP               `json:"default_gw_ip"`
-	IsInternetGw      bool                 `json:"is_inet_gw"`
-	NodeAddrs         []net.IPNet          `json:"nodes_addrs" yaml:"nodes_addrs"`
-	Server            string               `json:"server" bson:"server" yaml:"server"`
-	ServerVersion     string               `json:"serverversion" bson:"serverversion" yaml:"serverversion"`
-	ServerAddrs       []ServerAddr         `json:"serveraddrs" bson:"serveraddrs" yaml:"serveraddrs"`
+	Host            Host                  `json:"host"`
+	ChangeDefaultGw bool                  `json:"change_default_gw"`
+	DefaultGwIp     net.IP                `json:"default_gw_ip"`
+	IsInternetGw    bool                  `json:"is_inet_gw"`
+	NodeAddrs       []net.IPNet           `json:"nodes_addrs"`
+	Server          string                `json:"server"`
+	ServerVersion   string                `json:"serverversion"`
+	ServerAddrs     []ServerAddr          `json:"serveraddrs"`
+	NodePeers       []wgtypes.PeerConfig  `json:"node_peers"`
+	Peers           []wgtypes.PeerConfig  `json:"host_peers"`
+	PeerIDs         PeerMap               `json:"peerids"`
+	HostNetworkInfo HostInfoMap           `json:"host_network_info,omitempty"`
+	EgressRoutes    []EgressNetworkRoutes `json:"egress_network_routes"`
+	FwUpdate        FwUpdate              `json:"fw_update"`
+	ReplacePeers    bool                  `json:"replace_peers"`
+	ServerConfig
+	OldPeerUpdateFields
+}
+
+type OldPeerUpdateFields struct {
 	NodePeers         []wgtypes.PeerConfig `json:"peers" bson:"peers" yaml:"peers"`
-	Peers             []wgtypes.PeerConfig
-	PeerIDs           PeerMap               `json:"peerids" bson:"peerids" yaml:"peerids"`
-	HostNetworkInfo   HostInfoMap           `json:"host_network_info,omitempty" bson:"host_network_info,omitempty" yaml:"host_network_info,omitempty"`
-	EgressRoutes      []EgressNetworkRoutes `json:"egress_network_routes"`
-	FwUpdate          FwUpdate              `json:"fw_update"`
-	ReplacePeers      bool                  `json:"replace_peers"`
-	EndpointDetection bool                  `json:"endpoint_detection"`
-	ManageDNS         bool                  `yaml:"manage_dns"`
-	Stun              bool                  `yaml:"stun"`
-	StunServers       string                `yaml:"stun_servers"`
+	OldPeers          []wgtypes.PeerConfig `json:"Peers"`
+	EndpointDetection bool                 `json:"endpoint_detection"`
 }
 
 type FwRule struct {

models/network.go

@@ -42,9 +42,10 @@ func (network *Network) SetNetworkLastModified() {
 }
 
 // Network.SetDefaults - sets default values for a network struct
-func (network *Network) SetDefaults() {
+func (network *Network) SetDefaults() (upsert bool) {
 	if network.DefaultUDPHolePunch == "" {
 		network.DefaultUDPHolePunch = "no"
+		upsert = true
 	}
 	if network.DefaultInterface == "" {
 		if len(network.NetID) < 33 {
@@ -52,35 +53,45 @@ func (network *Network) SetDefaults() {
 		} else {
 			network.DefaultInterface = network.NetID
 		}
+		upsert = true
 	}
 	if network.DefaultListenPort == 0 {
 		network.DefaultListenPort = 51821
+		upsert = true
 	}
 	if network.NodeLimit == 0 {
 		network.NodeLimit = 999999999
+		upsert = true
 	}
 	if network.DefaultKeepalive == 0 {
 		network.DefaultKeepalive = 20
+		upsert = true
 	}
 	if network.AllowManualSignUp == "" {
 		network.AllowManualSignUp = "no"
+		upsert = true
 	}
 
 	if network.IsIPv4 == "" {
 		network.IsIPv4 = "yes"
+		upsert = true
 	}
 
 	if network.IsIPv6 == "" {
 		network.IsIPv6 = "no"
+		upsert = true
 	}
 
 	if network.DefaultMTU == 0 {
 		network.DefaultMTU = 1280
+		upsert = true
 	}
 
 	if network.DefaultACL == "" {
 		network.DefaultACL = "yes"
+		upsert = true
 	}
+	return
 }
 
 func (network *Network) GetNetworkNetworkCIDR4() *net.IPNet {

models/structs.go

@@ -252,24 +252,26 @@ type NodeJoinResponse struct {
 
 // ServerConfig - struct for dealing with the server information for a netclient
 type ServerConfig struct {
-	CoreDNSAddr    string `yaml:"corednsaddr"`
-	API            string `yaml:"api"`
-	APIPort        string `yaml:"apiport"`
-	DNSMode        string `yaml:"dnsmode"`
-	Version        string `yaml:"version"`
-	MQPort         string `yaml:"mqport"`
-	MQUserName     string `yaml:"mq_username"`
-	MQPassword     string `yaml:"mq_password"`
-	BrokerType     string `yaml:"broker_type"`
-	Server         string `yaml:"server"`
-	Broker         string `yaml:"broker"`
-	IsPro          bool   `yaml:"isee" json:"Is_EE"`
-	TrafficKey     []byte `yaml:"traffickey"`
-	MetricInterval string `yaml:"metric_interval"`
-	ManageDNS      bool   `yaml:"manage_dns"`
-	Stun           bool   `yaml:"stun"`
-	StunServers    string `yaml:"stun_servers"`
-	DefaultDomain  string `yaml:"default_domain"`
+	CoreDNSAddr       string `yaml:"corednsaddr"`
+	API               string `yaml:"api"`
+	APIPort           string `yaml:"apiport"`
+	DNSMode           string `yaml:"dnsmode"`
+	Version           string `yaml:"version"`
+	MQPort            string `yaml:"mqport"`
+	MQUserName        string `yaml:"mq_username"`
+	MQPassword        string `yaml:"mq_password"`
+	BrokerType        string `yaml:"broker_type"`
+	Server            string `yaml:"server"`
+	Broker            string `yaml:"broker"`
+	IsPro             bool   `yaml:"isee" json:"Is_EE"`
+	TrafficKey        []byte `yaml:"traffickey"`
+	MetricInterval    string `yaml:"metric_interval"`
+	MetricsPort       int    `yaml:"metrics_port"`
+	ManageDNS         bool   `yaml:"manage_dns"`
+	Stun              bool   `yaml:"stun"`
+	StunServers       string `yaml:"stun_servers"`
+	EndpointDetection bool   `yaml:"endpoint_detection"`
+	DefaultDomain     string `yaml:"default_domain"`
 }
 
 // User.NameInCharset - returns if name is in charset below or not

mq/handlers.go

@@ -280,6 +280,7 @@ func HandleHostCheckin(h, currentHost *models.Host) bool {
 		(h.ListenPort != 0 && h.ListenPort != currentHost.ListenPort) ||
 		(h.WgPublicListenPort != 0 && h.WgPublicListenPort != currentHost.WgPublicListenPort) || (!h.EndpointIPv6.Equal(currentHost.EndpointIPv6))
 	if ifaceDelta { // only save if something changes
+
 		currentHost.EndpointIP = h.EndpointIP
 		currentHost.EndpointIPv6 = h.EndpointIPv6
 		currentHost.Interfaces = h.Interfaces

mq/publishers.go

@@ -17,7 +17,6 @@ import (
 
 // PublishPeerUpdate --- determines and publishes a peer update to all the hosts
 func PublishPeerUpdate(replacePeers bool) error {
-
 	if !servercfg.IsMessageQueueBackend() {
 		return nil
 	}
@@ -114,6 +113,11 @@ func PublishSingleHostPeerUpdate(host *models.Host, allNodes []models.Node, dele
 	if err != nil {
 		return err
 	}
+	peerUpdate.OldPeerUpdateFields = models.OldPeerUpdateFields{
+		NodePeers:         peerUpdate.NodePeers,
+		OldPeers:          peerUpdate.Peers,
+		EndpointDetection: peerUpdate.ServerConfig.EndpointDetection,
+	}
 	peerUpdate.ReplacePeers = replacePeers
 	data, err := json.Marshal(&peerUpdate)
 	if err != nil {

scripts/netmaker.default.env

@@ -96,3 +96,7 @@ MANAGE_DNS=false
 OLD_ACL_SUPPORT=true
 # if STUN is set to true, hole punch is called
 STUN=true
+# Metrics Collection Port
+METRICS_PORT=51821
+# Metrics Collection interval in minutes
+PUBLISH_METRIC_INTERVAL=15

scripts/nm-quick.sh

@@ -6,7 +6,7 @@ SCRIPT_DIR=$(dirname "$(realpath "$0")")
 CONFIG_PATH="$SCRIPT_DIR/$CONFIG_FILE"
 NM_QUICK_VERSION="0.1.1"
 LATEST=$(curl -s https://api.github.com/repos/gravitl/netmaker/releases/latest | grep "tag_name" | cut -d : -f 2,3 | tr -d [:space:],\")
-
+BRANCH=master
 if [ $(id -u) -ne 0 ]; then
 	echo "This script must be run as root"
 	exit 1
@@ -617,7 +617,7 @@ install_netmaker() {
 
 	echo "Pulling config files..."
 
-	local BASE_URL="https://raw.githubusercontent.com/gravitl/netmaker/master"
+	local BASE_URL="https://raw.githubusercontent.com/gravitl/netmaker/$BRANCH"
 	local COMPOSE_URL="$BASE_URL/compose/docker-compose.yml"
 	local CADDY_URL="$BASE_URL/docker/Caddyfile"
 	if [ "$INSTALL_TYPE" = "pro" ]; then

servercfg/serverconf.go

@@ -14,6 +14,8 @@ import (
 	"github.com/gravitl/netmaker/models"
 )
 
+var ServerInfo = GetServerInfo()
+
 // EmqxBrokerType denotes the broker type for EMQX MQTT
 const EmqxBrokerType = "emqx"
 
@@ -141,10 +143,12 @@ func GetServerInfo() models.ServerConfig {
 	cfg.Version = GetVersion()
 	cfg.IsPro = IsPro
 	cfg.MetricInterval = GetMetricInterval()
+	cfg.MetricsPort = GetMetricsPort()
 	cfg.ManageDNS = GetManageDNS()
 	cfg.Stun = IsStunEnabled()
 	cfg.StunServers = GetStunServers()
 	cfg.DefaultDomain = GetDefaultDomain()
+	cfg.EndpointDetection = IsEndpointDetectionEnabled()
 	return cfg
 }
 
@@ -654,6 +658,19 @@ func GetMqUserName() string {
 	return password
 }
 
+// GetMetricsPort - get metrics port
+func GetMetricsPort() int {
+	p := 51821
+	if os.Getenv("METRICS_PORT") != "" {
+		pStr := os.Getenv("METRICS_PORT")
+		pInt, err := strconv.Atoi(pStr)
+		if err == nil && pInt != 0 {
+			p = pInt
+		}
+	}
+	return p
+}
+
 // GetMetricInterval - get the publish metric interval
 func GetMetricIntervalInMinutes() time.Duration {
 	//default 15 minutes

serverctl/serverctl.go

@@ -59,32 +59,8 @@ func setNetworkDefaults() error {
 		return err
 	}
 	for _, network := range networks {
-		update := false
-		newNet := network
-		if strings.Contains(network.NetID, ".") {
-			newNet.NetID = strings.ReplaceAll(network.NetID, ".", "")
-			newNet.DefaultInterface = strings.ReplaceAll(network.DefaultInterface, ".", "")
-			update = true
-		}
-		if strings.ContainsAny(network.NetID, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") {
-			newNet.NetID = strings.ToLower(network.NetID)
-			newNet.DefaultInterface = strings.ToLower(network.DefaultInterface)
-			update = true
-		}
-		if update {
-			newNet.SetDefaults()
-			if err := logic.SaveNetwork(&newNet); err != nil {
-				logger.Log(0, "error saving networks during initial update:", err.Error())
-			}
-			if err := logic.DeleteNetwork(network.NetID); err != nil {
-				logger.Log(0, "error deleting old network:", err.Error())
-			}
-		} else {
-			network.SetDefaults()
-			_, _, _, err = logic.UpdateNetwork(&network, &network)
-			if err != nil {
-				logger.Log(0, "could not set defaults on network", network.NetID)
-			}
+		if network.SetDefaults() {
+			logic.SaveNetwork(&network)
 		}
 	}
 	return nil

utils/utils.go

@@ -1,6 +1,10 @@
 package utils
 
-import "time"
+import (
+	"log/slog"
+	"runtime"
+	"time"
+)
 
 // RetryStrategy specifies a strategy to retry an operation after waiting a while,
 // with hooks for successful and unsuccessful (>=max) tries.
@@ -39,3 +43,19 @@ func (rs RetryStrategy) DoStrategy() {
 		return
 	}
 }
+
+func TraceCaller() {
+	// Skip 1 frame to get the caller of this function
+	pc, file, line, ok := runtime.Caller(2)
+	if !ok {
+		slog.Debug("Unable to get caller information")
+		return
+	}
+
+	// Get function name from the program counter (pc)
+	funcName := runtime.FuncForPC(pc).Name()
+
+	// Print trace details
+	slog.Debug("Called from function: %s\n", "func-name", funcName)
+	slog.Debug("File: %s, Line: %d\n", "file", file, "line-no", line)
+}