Merge branch 'develop' of https://github.com/gravitl/netmaker into NM-120

.github/workflows/docs.yml

@@ -19,7 +19,7 @@ jobs:
           ref: ${{ github.event.inputs.branch || 'master' }}
 
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: go.mod
 

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
       - name: Build
@@ -27,7 +27,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
       - name: Setup go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
       - name: Build
@@ -44,7 +44,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
       - name: run tests
@@ -64,7 +64,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v5
       - name: Setup Go
-        uses: actions/setup-go@v5
+        uses: actions/setup-go@v6
         with:
           go-version-file: 'go.mod'
       - name: run static checks

Dockerfile

@@ -1,5 +1,5 @@
 #first stage - builder
-FROM gravitl/go-builder:1.23.0 AS builder
+FROM gravitl/go-builder:1.24.0 AS builder
 ARG tags 
 WORKDIR /app
 COPY . .

auth/host_session.go

@@ -226,7 +226,7 @@ func SessionHandler(conn *websocket.Conn) {
 		if err = conn.WriteMessage(messageType, reponseData); err != nil {
 			logger.Log(0, "error during message writing:", err.Error())
 		}
-		go CheckNetRegAndHostUpdate(models.EnrollmentKey{Networks: netsToAdd}, &result.Host, "")
+		go CheckNetRegAndHostUpdate(models.EnrollmentKey{Value: "user auth", Tags: []string{registerMessage.User}, Networks: netsToAdd}, &result.Host, "")
 	case <-timeout: // the read from req.answerCh has timed out
 		logger.Log(0, "timeout signal recv,exiting oauth socket conn")
 		break

controllers/egress.go

@@ -45,22 +45,23 @@ func createEgress(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	var egressRange string
-	var cidrErr error
 	if !req.IsInetGw {
 		if req.Range != "" {
-			egressRange, cidrErr = logic.NormalizeCIDR(req.Range)
+			var err error
+			egressRange, err = logic.NormalizeCIDR(req.Range)
+			if err != nil {
+				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+				return
+			}
 		}
-		isDomain := logic.IsFQDN(req.Range)
-		if cidrErr != nil && !isDomain {
-			if cidrErr != nil {
-				logic.ReturnErrorResponse(w, r, logic.FormatError(cidrErr, "badrequest"))
-			} else {
+
+		if req.Domain != "" {
+			isDomain := logic.IsFQDN(req.Domain)
+			if !isDomain {
 				logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("bad domain name"), "badrequest"))
+				return
 			}
-			return
-		}
-		if isDomain {
-			req.Domain = req.Range
+
 			egressRange = ""
 		}
 	} else {
@@ -204,20 +205,23 @@ func updateEgress(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	var egressRange string
-	var cidrErr error
 	if !req.IsInetGw {
-		egressRange, cidrErr = logic.NormalizeCIDR(req.Range)
-		isDomain := logic.IsFQDN(req.Range)
-		if cidrErr != nil && !isDomain {
-			if cidrErr != nil {
-				logic.ReturnErrorResponse(w, r, logic.FormatError(cidrErr, "badrequest"))
-			} else {
-				logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("bad domain name"), "badrequest"))
+		if req.Range != "" {
+			var err error
+			egressRange, err = logic.NormalizeCIDR(req.Range)
+			if err != nil {
+				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+				return
 			}
-			return
 		}
-		if isDomain {
-			req.Domain = req.Range
+
+		if req.Domain != "" {
+			isDomain := logic.IsFQDN(req.Domain)
+			if !isDomain {
+				logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("bad domain name"), "badrequest"))
+				return
+			}
+
 			egressRange = ""
 		}
 	} else {
@@ -233,12 +237,20 @@ func updateEgress(w http.ResponseWriter, r *http.Request) {
 	}
 	var updateNat bool
 	var updateStatus bool
+	var resetDomain bool
+	var resetRange bool
 	if req.Nat != e.Nat {
 		updateNat = true
 	}
 	if req.Status != e.Status {
 		updateStatus = true
 	}
+	if req.Domain == "" {
+		resetDomain = true
+	}
+	if req.Range == "" || egressRange == "" {
+		resetRange = true
+	}
 	event := &models.Event{
 		Action: models.Update,
 		Source: models.Subject{
@@ -294,6 +306,12 @@ func updateEgress(w http.ResponseWriter, r *http.Request) {
 		e.Status = req.Status
 		e.UpdateEgressStatus(db.WithContext(context.TODO()))
 	}
+	if resetDomain {
+		_ = e.ResetDomain(db.WithContext(context.TODO()))
+	}
+	if resetRange {
+		_ = e.ResetRange(db.WithContext(context.TODO()))
+	}
 	event.Diff.New = e
 	logic.LogEvent(event)
 	if req.Domain != "" {
@@ -321,8 +339,6 @@ func updateEgress(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-	} else {
-		go mq.PublishPeerUpdate(false)
 	}
 	go mq.PublishPeerUpdate(false)
 	logic.ReturnSuccessResponseWithJson(w, r, e, "updated egress resource")

controllers/ext_client.go

@@ -466,23 +466,6 @@ func getExtClientHAConf(w http.ResponseWriter, r *http.Request) {
 	extclient.IngressGatewayID = targetGwID
 	extclient.Network = networkid
 	extclient.Tags = make(map[models.TagID]struct{})
-	// extclient.Tags[models.TagID(fmt.Sprintf("%s.%s", extclient.Network,
-	// 	models.RemoteAccessTagName))] = struct{}{}
-	// set extclient dns to ingressdns if extclient dns is not explicitly set
-	if (extclient.DNS == "") && (gwnode.IngressDNS != "") {
-		network, _ := logic.GetNetwork(gwnode.Network)
-		dns := gwnode.IngressDNS
-		if len(network.NameServers) > 0 {
-			if dns == "" {
-				dns = strings.Join(network.NameServers, ",")
-			} else {
-				dns += "," + strings.Join(network.NameServers, ",")
-			}
-
-		}
-		extclient.DNS = dns
-
-	}
 
 	listenPort := logic.GetPeerListenPort(host)
 	extclient.IngressGatewayEndpoint = fmt.Sprintf("%s:%d", host.EndpointIP.String(), listenPort)
@@ -506,6 +489,11 @@ func getExtClientHAConf(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+	logic.SetDNSOnWgConfig(&gwnode, &client)
+	defaultDNS := ""
+	if client.DNS != "" {
+		defaultDNS = "DNS = " + client.DNS
+	}
 	addrString := client.Address
 	if addrString != "" {
 		addrString += "/32"
@@ -551,13 +539,6 @@ func getExtClientHAConf(w http.ResponseWriter, r *http.Request) {
 	} else {
 		gwendpoint = fmt.Sprintf("%s:%d", host.EndpointIP.String(), host.ListenPort)
 	}
-	defaultDNS := ""
-	if client.DNS != "" {
-		defaultDNS = "DNS = " + client.DNS
-	} else if gwnode.IngressDNS != "" {
-		defaultDNS = "DNS = " + gwnode.IngressDNS
-	}
-
 	defaultMTU := 1420
 	if host.MTU != 0 {
 		defaultMTU = host.MTU
@@ -630,6 +611,7 @@ Endpoint = %s
 
 	name := client.ClientID + ".conf"
 	w.Header().Set("Content-Type", "application/config")
+	w.Header().Set("Client-ID", client.ClientID)
 	w.Header().Set("Content-Disposition", "attachment; filename=\""+name+"\"")
 	w.WriteHeader(http.StatusOK)
 	_, err = fmt.Fprint(w, config)

controllers/hosts.go

@@ -207,10 +207,10 @@ func pull(w http.ResponseWriter, r *http.Request) {
 	for _, nodeID := range host.Nodes {
 		node, err := logic.GetNodeByID(nodeID)
 		if err != nil {
-			slog.Error("failed to get node:", "id", node.ID, "error", err)
+			//slog.Error("failed to get node:", "id", node.ID, "error", err)
 			continue
 		}
-		if node.FailedOverBy != uuid.Nil && r.URL.Query().Get("reset_failovered") == "true" {
+		if r.URL.Query().Get("reset_failovered") == "true" {
 			logic.ResetFailedOverPeer(&node)
 			sendPeerUpdate = true
 		}
@@ -232,19 +232,11 @@ func pull(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
-	serverConf := logic.GetServerInfo()
-	key, keyErr := logic.RetrievePublicTrafficKey()
-	if keyErr != nil {
-		logger.Log(0, "error retrieving key:", keyErr.Error())
-		logic.ReturnErrorResponse(w, r, logic.FormatError(keyErr, "internal"))
-		return
-	}
 	_ = logic.CheckHostPorts(host)
-	serverConf.TrafficKey = key
 	response := models.HostPull{
 		Host:              *host,
 		Nodes:             logic.GetHostNodes(host),
-		ServerConfig:      serverConf,
+		ServerConfig:      hPU.ServerConfig,
 		Peers:             hPU.Peers,
 		PeerIDs:           hPU.PeerIDs,
 		HostNetworkInfo:   hPU.HostNetworkInfo,
@@ -257,6 +249,7 @@ func pull(w http.ResponseWriter, r *http.Request) {
 		EgressWithDomains: hPU.EgressWithDomains,
 		EndpointDetection: logic.IsEndpointDetectionEnabled(),
 		DnsNameservers:    hPU.DnsNameservers,
+		ReplacePeers:      hPU.ReplacePeers,
 	}
 
 	logger.Log(1, hostID, host.Name, "completed a pull")
@@ -363,8 +356,7 @@ func hostUpdateFallback(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 		return
 	}
-	var sendPeerUpdate bool
-	var replacePeers bool
+	var sendPeerUpdate, sendDeletedNodeUpdate, replacePeers bool
 	var hostUpdate models.HostUpdate
 	err = json.NewDecoder(r.Body).Decode(&hostUpdate)
 	if err != nil {
@@ -376,6 +368,10 @@ func hostUpdateFallback(w http.ResponseWriter, r *http.Request) {
 	switch hostUpdate.Action {
 	case models.CheckIn:
 		sendPeerUpdate = mq.HandleHostCheckin(&hostUpdate.Host, currentHost)
+		changed := logic.CheckHostPorts(currentHost)
+		if changed {
+			mq.HostUpdate(&models.HostUpdate{Action: models.UpdateHost, Host: *currentHost})
+		}
 	case models.UpdateHost:
 		if hostUpdate.Host.PublicKey != currentHost.PublicKey {
 			//remove old peer entry
@@ -388,7 +384,8 @@ func hostUpdateFallback(w http.ResponseWriter, r *http.Request) {
 			logic.ReturnErrorResponse(w, r, logic.FormatError(err, logic.Internal))
 			return
 		}
-
+	case models.UpdateNode:
+		sendDeletedNodeUpdate, sendPeerUpdate = logic.UpdateHostNode(&hostUpdate.Host, &hostUpdate.Node)
 	case models.UpdateMetrics:
 		mq.UpdateMetricsFallBack(hostUpdate.Node.ID.String(), hostUpdate.NewMetrics)
 	case models.EgressUpdate:
@@ -403,14 +400,24 @@ func hostUpdateFallback(w http.ResponseWriter, r *http.Request) {
 			e.Update(db.WithContext(r.Context()))
 		}
 		sendPeerUpdate = true
+	case models.SignalHost:
+		mq.SignalPeer(hostUpdate.Signal)
+	case models.DeleteHost:
+		mq.DeleteAndCleanupHost(currentHost)
+		sendPeerUpdate = true
 	}
-
-	if sendPeerUpdate {
-		err := mq.PublishPeerUpdate(replacePeers)
-		if err != nil {
-			slog.Error("failed to publish peer update", "error", err)
+	go func() {
+		if sendDeletedNodeUpdate {
+			mq.PublishDeletedNodePeerUpdate(&hostUpdate.Node)
 		}
-	}
+		if sendPeerUpdate {
+			err := mq.PublishPeerUpdate(replacePeers)
+			if err != nil {
+				slog.Error("failed to publish peer update", "error", err)
+			}
+		}
+	}()
+
 	logic.ReturnSuccessResponse(w, r, "updated host data")
 }
 

controllers/middleware.go

@@ -71,7 +71,7 @@ func userMiddleWare(handler http.Handler) http.Handler {
 		if strings.Contains(route, "tags") {
 			r.Header.Set("TARGET_RSRC", models.TagRsrc.String())
 		}
-		if strings.Contains(route, "extclients") {
+		if strings.Contains(route, "extclients") || strings.Contains(route, "client_conf") {
 			r.Header.Set("TARGET_RSRC", models.ExtClientsRsrc.String())
 		}
 		if strings.Contains(route, "enrollment-keys") {

controllers/network.go

@@ -575,21 +575,40 @@ func createNetwork(w http.ResponseWriter, r *http.Request) {
 
 	// validate address ranges: must be private
 	if network.AddressRange != "" {
-		_, _, err := net.ParseCIDR(network.AddressRange)
+		_, cidr, err := net.ParseCIDR(network.AddressRange)
 		if err != nil {
 			logger.Log(0, r.Header.Get("user"), "failed to create network: ",
 				err.Error())
 			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 			return
+		} else {
+			ones, bits := cidr.Mask.Size()
+			if bits-ones <= 1 {
+				err = fmt.Errorf("cannot create network with /31 or /32 cidr")
+				logger.Log(0, r.Header.Get("user"), "failed to create network: ",
+					err.Error())
+				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+				return
+			}
 		}
 	}
+
 	if network.AddressRange6 != "" {
-		_, _, err := net.ParseCIDR(network.AddressRange6)
+		_, cidr, err := net.ParseCIDR(network.AddressRange6)
 		if err != nil {
 			logger.Log(0, r.Header.Get("user"), "failed to create network: ",
 				err.Error())
 			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 			return
+		} else {
+			ones, bits := cidr.Mask.Size()
+			if bits-ones <= 1 {
+				err = fmt.Errorf("cannot create network with /127 or /128 cidr")
+				logger.Log(0, r.Header.Get("user"), "failed to create network: ",
+					err.Error())
+				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+				return
+			}
 		}
 	}
 

controllers/node.go

@@ -729,6 +729,9 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 		if err := mq.NodeUpdate(newNode); err != nil {
 			slog.Error("error publishing node update to node", "node", newNode.ID, "error", err)
 		}
+		if !newNode.Connected {
+			mq.HostUpdate(&models.HostUpdate{Host: *host, Action: models.RequestPull})
+		}
 		mq.PublishPeerUpdate(false)
 		if servercfg.IsDNSMode() {
 			logic.SetDNS()

docker/Dockerfile-go-builder

@@ -1,4 +1,4 @@
-FROM golang:1.23.0-alpine3.20
+FROM golang:1.24.0-alpine3.20
 ARG version 
 RUN apk add --no-cache build-base
 WORKDIR /app

go.mod

@@ -1,8 +1,6 @@
 module github.com/gravitl/netmaker
 
-go 1.23.0
-
-toolchain go1.23.7
+go 1.24.0
 
 require (
 	github.com/blang/semver v3.5.1+incompatible
@@ -17,14 +15,14 @@ require (
 	github.com/rqlite/gorqlite v0.0.0-20240122221808-a8a425b1a6aa
 	github.com/seancfoley/ipaddress-go v1.7.1
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
-	github.com/stretchr/testify v1.11.0
+	github.com/stretchr/testify v1.11.1
 	github.com/txn2/txeh v1.5.5
 	go.uber.org/automaxprocs v1.6.0
-	golang.org/x/crypto v0.41.0
+	golang.org/x/crypto v0.42.0
 	golang.org/x/net v0.43.0 // indirect
 	golang.org/x/oauth2 v0.30.0
-	golang.org/x/sys v0.35.0 // indirect
-	golang.org/x/text v0.28.0 // indirect
+	golang.org/x/sys v0.36.0 // indirect
+	golang.org/x/text v0.29.0 // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb
 	gopkg.in/yaml.v3 v3.0.1
 )
@@ -32,7 +30,7 @@ require (
 require (
 	filippo.io/edwards25519 v1.1.0
 	github.com/c-robinson/iplib v1.0.8
-	github.com/posthog/posthog-go v1.6.4
+	github.com/posthog/posthog-go v1.6.8
 )
 
 require (
@@ -48,13 +46,13 @@ require (
 	github.com/matryer/is v1.4.1
 	github.com/okta/okta-sdk-golang/v5 v5.0.6
 	github.com/pquerna/otp v1.5.0
-	github.com/spf13/cobra v1.9.1
+	github.com/spf13/cobra v1.10.1
 	google.golang.org/api v0.248.0
 	gopkg.in/mail.v2 v2.3.1
 	gorm.io/datatypes v1.2.6
 	gorm.io/driver/postgres v1.6.0
 	gorm.io/driver/sqlite v1.6.0
-	gorm.io/gorm v1.30.1
+	gorm.io/gorm v1.31.0
 )
 
 require (
@@ -94,7 +92,7 @@ require (
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/rogpeppe/go-internal v1.14.1 // indirect
 	github.com/seancfoley/bintree v1.3.1 // indirect
-	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/spf13/pflag v1.0.9 // indirect
 	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.61.0 // indirect
 	go.opentelemetry.io/otel v1.36.0 // indirect
@@ -116,5 +114,5 @@ require (
 	github.com/leodido/go-urn v1.4.0 // indirect
 	github.com/mattn/go-runewidth v0.0.16 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sync v0.17.0 // indirect
 )

go.sum

@@ -140,8 +140,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
 github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posthog/posthog-go v1.6.4 h1:vPo6Z8T1R+aUBugXg1+psD8qZYSOtFktzhj6H8rzOBI=
-github.com/posthog/posthog-go v1.6.4/go.mod h1:LcC1Nu4AgvV22EndTtrMXTy+7RGVC0MhChSw7Qk5XkY=
+github.com/posthog/posthog-go v1.6.8 h1:l5H05oKqiZbLYAjxrus3Rvj606bdEu+7EDAhKnSgU6I=
+github.com/posthog/posthog-go v1.6.8/go.mod h1:LcC1Nu4AgvV22EndTtrMXTy+7RGVC0MhChSw7Qk5XkY=
 github.com/pquerna/otp v1.5.0 h1:NMMR+WrmaqXU4EzdGJEE1aUUI0AMRzsp96fFFWNPwxs=
 github.com/pquerna/otp v1.5.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg=
 github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g=
@@ -160,10 +160,10 @@ github.com/seancfoley/ipaddress-go v1.7.1 h1:fDWryS+L8iaaH5RxIKbY0xB5Z+Zxk8xoXLN
 github.com/seancfoley/ipaddress-go v1.7.1/go.mod h1:TQRZgv+9jdvzHmKoPGBMxyiaVmoI0rYpfEk8Q/sL/Iw=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
-github.com/spf13/cobra v1.9.1 h1:CXSaggrXdbHK9CF+8ywj8Amf7PBRmPCOJugH954Nnlo=
-github.com/spf13/cobra v1.9.1/go.mod h1:nDyEzZ8ogv936Cinf6g1RU9MRY64Ir93oCnqb9wxYW0=
-github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o=
-github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
+github.com/spf13/cobra v1.10.1 h1:lJeBwCfmrnXthfAupyUTzJ/J4Nc1RsHC/mSRU2dll/s=
+github.com/spf13/cobra v1.10.1/go.mod h1:7SmJGaTHFVBY0jW4NXGluQoLvhqFQM+6XSKD+P4XaB0=
+github.com/spf13/pflag v1.0.9 h1:9exaQaMOCwffKiiiYk6/BndUBv+iRViNW+4lEMi0PvY=
+github.com/spf13/pflag v1.0.9/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
 github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
@@ -175,8 +175,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
 github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/stretchr/testify v1.11.0 h1:ib4sjIrwZKxE5u/Japgo/7SJV3PvgjGiRNAvTVGqQl8=
-github.com/stretchr/testify v1.11.0/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/txn2/txeh v1.5.5 h1:UN4e/lCK5HGw/gGAi2GCVrNKg0GTCUWs7gs5riaZlz4=
 github.com/txn2/txeh v1.5.5/go.mod h1:qYzGG9kCzeVEI12geK4IlanHWY8X4uy/I3NcW7mk8g4=
 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
@@ -202,8 +202,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs=
-golang.org/x/crypto v0.41.0 h1:WKYxWedPGCTVVl5+WHSSrOBT0O8lx32+zxmHxijgXp4=
-golang.org/x/crypto v0.41.0/go.mod h1:pO5AFd7FA68rFak7rOAGVuygIISepHftHnr8dr6+sUc=
+golang.org/x/crypto v0.42.0 h1:chiH31gIWm57EkTXpwnqf8qeuMUi0yekh6mT2AvFlqI=
+golang.org/x/crypto v0.42.0/go.mod h1:4+rDnOTJhQCx2q7/j6rAN5XDw8kPjeaXEUR2eL94ix8=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -221,8 +221,8 @@ golang.org/x/oauth2 v0.30.0/go.mod h1:B++QgG3ZKulg6sRPGD/mqlHQs5rB3Ml9erfeDY7xKl
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.16.0 h1:ycBJEhp9p4vXvUZNszeOq0kGTPghopOL8q0fq3vstxw=
-golang.org/x/sync v0.16.0/go.mod h1:1dzgHSNfp02xaA81J2MS99Qcpr2w7fw1gpm99rleRqA=
+golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug=
+golang.org/x/sync v0.17.0/go.mod h1:9KTHXmSnoGruLpwFjVSX0lNNA75CykiMECbovNTZqGI=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -232,8 +232,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.35.0 h1:vz1N37gP5bs89s7He8XuIYXpyY0+QlsKmzipCbUtyxI=
-golang.org/x/sys v0.35.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
+golang.org/x/sys v0.36.0 h1:KVRy2GtZBrk1cBYA7MKu5bEZFxQk4NIDV6RLVcC8o0k=
+golang.org/x/sys v0.36.0/go.mod h1:OgkHotnGiDImocRcuBABYBEXf8A9a87e/uXjp9XT3ks=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -246,8 +246,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8=
 golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
-golang.org/x/text v0.28.0 h1:rhazDwis8INMIwQ4tpjLDzUhx6RlXqZNPEM0huQojng=
-golang.org/x/text v0.28.0/go.mod h1:U8nCwOR8jO/marOQ0QbDiOngZVEBB7MAiitBuMjXiNU=
+golang.org/x/text v0.29.0 h1:1neNs90w9YzJ9BocxfsQNHKuAT4pkghyXc4nhZ6sJvk=
+golang.org/x/text v0.29.0/go.mod h1:7MhJOA9CD2qZyOKYazxdYMF85OwPdEr9jTtBpO7ydH4=
 golang.org/x/time v0.12.0 h1:ScB/8o8olJvc+CQPWrK3fPZNfh7qgwCrY0zJmoEQLSE=
 golang.org/x/time v0.12.0/go.mod h1:CDIdPxbZBQxdj6cxyCIdrNogrJKMJ7pr37NYpMcMDSg=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -290,5 +290,5 @@ gorm.io/driver/sqlite v1.6.0/go.mod h1:AO9V1qIQddBESngQUKWL9yoH93HIeA1X6V633rBwy
 gorm.io/driver/sqlserver v1.6.0 h1:VZOBQVsVhkHU/NzNhRJKoANt5pZGQAS1Bwc6m6dgfnc=
 gorm.io/driver/sqlserver v1.6.0/go.mod h1:WQzt4IJo/WHKnckU9jXBLMJIVNMVeTu25dnOzehntWw=
 gorm.io/gorm v1.25.7/go.mod h1:hbnx/Oo0ChWMn1BIhpy1oYozzpM15i4YPuHDmfYtwg8=
-gorm.io/gorm v1.30.1 h1:lSHg33jJTBxs2mgJRfRZeLDG+WZaHYCk3Wtfl6Ngzo4=
-gorm.io/gorm v1.30.1/go.mod h1:8Z33v652h4//uMA76KjeDH8mJXPm1QNCYrMeatR0DOE=
+gorm.io/gorm v1.31.0 h1:0VlycGreVhK7RF/Bwt51Fk8v0xLiiiFdbGDPIZQ7mJY=
+gorm.io/gorm v1.31.0/go.mod h1:XyQVbO2k6YkOis7C2437jSit3SsDK72s7n7rsSHd+Gs=

logic/acls.go

@@ -50,16 +50,31 @@ func GetFwRulesOnIngressGateway(node models.Node) (rules []models.FwRule) {
 	if defaultDevicePolicy.Enabled {
 		return
 	}
+	defer func() {
+		if len(rules) == 0 && IsNodeAllowedToCommunicateWithAllRsrcs(node) {
+			if node.NetworkRange.IP != nil {
+				rules = append(rules, models.FwRule{
+					SrcIP: node.NetworkRange,
+					Allow: true,
+				})
+			}
+			if node.NetworkRange6.IP != nil {
+				rules = append(rules, models.FwRule{
+					SrcIP: node.NetworkRange6,
+					Allow: true,
+				})
+			}
+			return
+		}
+	}()
+
 	for _, nodeI := range nodes {
 		if !nodeI.IsStatic || nodeI.IsUserNode {
 			continue
 		}
-		if !node.StaticNode.Enabled {
+		if !nodeI.StaticNode.Enabled {
 			continue
 		}
-		// if nodeI.StaticNode.IngressGatewayID != node.ID.String() {
-		// 	continue
-		// }
 		if IsNodeAllowedToCommunicateWithAllRsrcs(nodeI) {
 			if nodeI.Address.IP != nil {
 				rules = append(rules, models.FwRule{
@@ -525,7 +540,18 @@ func GetAclRulesForNode(targetnodeI *models.Node) (rules map[string]models.AclRu
 						continue
 					}
 					if _, ok := eI.Nodes[targetnode.ID.String()]; ok {
-						if eI.Range != "" {
+						if servercfg.IsPro && eI.Domain != "" && len(eI.DomainAns) > 0 {
+							for _, domainAnsI := range eI.DomainAns {
+								ip, cidr, err := net.ParseCIDR(domainAnsI)
+								if err == nil {
+									if ip.To4() != nil {
+										egressRanges4 = append(egressRanges4, *cidr)
+									} else {
+										egressRanges6 = append(egressRanges6, *cidr)
+									}
+								}
+							}
+						} else if eI.Range != "" {
 							_, cidr, err := net.ParseCIDR(eI.Range)
 							if err == nil {
 								if cidr.IP.To4() != nil {
@@ -535,6 +561,7 @@ func GetAclRulesForNode(targetnodeI *models.Node) (rules map[string]models.AclRu
 								}
 							}
 						}
+						dstTags[targetnode.ID.String()] = struct{}{}
 					}
 				}
 				break
@@ -544,7 +571,18 @@ func GetAclRulesForNode(targetnodeI *models.Node) (rules map[string]models.AclRu
 				err := e.Get(db.WithContext(context.TODO()))
 				if err == nil && e.Status && len(e.Nodes) > 0 {
 					if _, ok := e.Nodes[targetnode.ID.String()]; ok {
-						if e.Range != "" {
+						if servercfg.IsPro && e.Domain != "" && len(e.DomainAns) > 0 {
+							for _, domainAnsI := range e.DomainAns {
+								ip, cidr, err := net.ParseCIDR(domainAnsI)
+								if err == nil {
+									if ip.To4() != nil {
+										egressRanges4 = append(egressRanges4, *cidr)
+									} else {
+										egressRanges6 = append(egressRanges6, *cidr)
+									}
+								}
+							}
+						} else if e.Range != "" {
 							_, cidr, err := net.ParseCIDR(e.Range)
 							if err == nil {
 								if cidr.IP.To4() != nil {
@@ -554,6 +592,7 @@ func GetAclRulesForNode(targetnodeI *models.Node) (rules map[string]models.AclRu
 								}
 							}
 						}
+						dstTags[targetnode.ID.String()] = struct{}{}
 					}
 
 				}
@@ -800,10 +839,10 @@ func GetEgressRulesForNode(targetnode models.Node) (rules map[string]models.AclR
 						if node.ID == targetnode.ID {
 							continue
 						}
-						if node.Address.IP != nil {
+						if !node.IsStatic && node.Address.IP != nil {
 							aclRule.IPList = append(aclRule.IPList, node.AddressIPNet4())
 						}
-						if node.Address6.IP != nil {
+						if !node.IsStatic && node.Address6.IP != nil {
 							aclRule.IP6List = append(aclRule.IP6List, node.AddressIPNet6())
 						}
 						if node.IsStatic && node.StaticNode.Address != "" {
@@ -1426,6 +1465,18 @@ func GetDefaultPolicy(netID models.NetworkID, ruleType models.AclPolicyType) (mo
 	return acl, nil
 }
 
+// ListUserPolicies - lists all user policies in a network
+func ListUserPolicies(netID models.NetworkID) []models.Acl {
+	allAcls := ListAcls()
+	userAcls := []models.Acl{}
+	for _, acl := range allAcls {
+		if acl.NetworkID == netID && acl.RuleType == models.UserPolicy {
+			userAcls = append(userAcls, acl)
+		}
+	}
+	return userAcls
+}
+
 // ListAcls - lists all acl policies
 func ListAclsByNetwork(netID models.NetworkID) ([]models.Acl, error) {
 

logic/dns.go

@@ -226,9 +226,7 @@ func GetGwDNS(node *models.Node) string {
 }
 
 func SetDNSOnWgConfig(gwNode *models.Node, extclient *models.ExtClient) {
-	if extclient.DNS == "" {
-		extclient.DNS = GetGwDNS(gwNode)
-	}
+	extclient.DNS = GetGwDNS(gwNode)
 }
 
 // GetCustomDNS - gets the custom DNS of a network
@@ -434,6 +432,25 @@ func validateNameserverReq(ns schema.Nameserver) error {
 	if len(ns.Servers) == 0 {
 		return errors.New("atleast one nameserver should be specified")
 	}
+	network, err := GetNetwork(ns.NetworkID)
+	if err != nil {
+		return errors.New("invalid network id")
+	}
+	_, cidr, err4 := net.ParseCIDR(network.AddressRange)
+	_, cidr6, err6 := net.ParseCIDR(network.AddressRange6)
+	for _, nsIPStr := range ns.Servers {
+		nsIP := net.ParseIP(nsIPStr)
+		if nsIP == nil {
+			return errors.New("invalid nameserver " + nsIPStr)
+		}
+		if err4 == nil && nsIP.To4() != nil {
+			if cidr.Contains(nsIP) {
+				return errors.New("cannot use netmaker IP as nameserver")
+			}
+		} else if err6 == nil && cidr6.Contains(nsIP) {
+			return errors.New("cannot use netmaker IP as nameserver")
+		}
+	}
 	if !ns.MatchAll && len(ns.MatchDomains) == 0 {
 		return errors.New("atleast one match domain is required")
 	}

logic/egress.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"errors"
 	"maps"
+	"strings"
 
 	"github.com/gravitl/netmaker/db"
 	"github.com/gravitl/netmaker/models"
@@ -37,7 +38,6 @@ func ValidateEgressReq(e *schema.Egress) error {
 }
 
 func DoesUserHaveAccessToEgress(user *models.User, e *schema.Egress, acls []models.Acl) bool {
-
 	if !e.Status {
 		return false
 	}
@@ -183,7 +183,7 @@ func AddEgressInfoToPeerByAccess(node, targetNode *models.Node, eli []schema.Egr
 func GetEgressDomainsByAccess(user *models.User, network models.NetworkID) (domains []string) {
 	acls, _ := ListAclsByNetwork(network)
 	eli, _ := (&schema.Egress{Network: network.String()}).ListByNetwork(db.WithContext(context.TODO()))
-	defaultDevicePolicy, _ := GetDefaultPolicy(network, models.DevicePolicy)
+	defaultDevicePolicy, _ := GetDefaultPolicy(network, models.UserPolicy)
 	isDefaultPolicyActive := defaultDevicePolicy.Enabled
 	for _, e := range eli {
 		if !e.Status || e.Network != network.String() {
@@ -195,7 +195,8 @@ func GetEgressDomainsByAccess(user *models.User, network models.NetworkID) (doma
 			}
 		}
 		if e.Domain != "" && len(e.DomainAns) > 0 {
-			domains = append(domains, e.Domain)
+			domains = append(domains, BaseDomain(e.Domain))
+
 		}
 	}
 	return
@@ -302,26 +303,36 @@ func GetEgressRanges(netID models.NetworkID) (map[string][]string, map[string]st
 }
 
 func ListAllByRoutingNodeWithDomain(egs []schema.Egress, nodeID string) (egWithDomain []models.EgressDomain) {
+	node, err := GetNodeByID(nodeID)
+	if err != nil {
+		return
+	}
+	host, err := GetHost(node.HostID.String())
+	if err != nil {
+		return
+	}
 	for _, egI := range egs {
 		if !egI.Status || egI.Domain == "" {
 			continue
 		}
 		if _, ok := egI.Nodes[nodeID]; ok {
-			node, err := GetNodeByID(nodeID)
-			if err != nil {
-				continue
-			}
-			host, err := GetHost(node.HostID.String())
-			if err != nil {
-				continue
-			}
+
 			egWithDomain = append(egWithDomain, models.EgressDomain{
 				ID:     egI.ID,
 				Domain: egI.Domain,
 				Node:   node,
 				Host:   *host,
 			})
+
 		}
 	}
 	return
 }
+
+func BaseDomain(host string) string {
+	parts := strings.Split(host, ".")
+	if len(parts) < 2 {
+		return host // not a FQDN
+	}
+	return strings.Join(parts[len(parts)-2:], ".")
+}

logic/extpeers.go

@@ -70,23 +70,43 @@ func storeExtClientInCache(key string, extclient models.ExtClient) {
 func GetEgressRangesOnNetwork(client *models.ExtClient) ([]string, error) {
 
 	var result []string
-	networkNodes, err := GetNetworkNodes(client.Network)
-	if err != nil {
-		return []string{}, err
-	}
 	eli, _ := (&schema.Egress{Network: client.Network}).ListByNetwork(db.WithContext(context.TODO()))
-	acls, _ := ListAclsByNetwork(models.NetworkID(client.Network))
-	// clientNode := client.ConvertToStaticNode()
-	for _, currentNode := range networkNodes {
-		if currentNode.Network != client.Network {
+	staticNode := client.ConvertToStaticNode()
+	userPolicies := ListUserPolicies(models.NetworkID(client.Network))
+	defaultUserPolicy, _ := GetDefaultPolicy(models.NetworkID(client.Network), models.UserPolicy)
+
+	for _, eI := range eli {
+		if !eI.Status {
+			continue
+		}
+		if eI.Domain == "" && eI.Range == "" {
+			continue
+		}
+		if eI.Domain != "" && len(eI.DomainAns) == 0 {
 			continue
 		}
-		GetNodeEgressInfo(&currentNode, eli, acls)
-		if currentNode.EgressDetails.IsEgressGateway { // add the egress gateway range(s) to the result
-			if len(currentNode.EgressDetails.EgressGatewayRanges) > 0 {
-				result = append(result, currentNode.EgressDetails.EgressGatewayRanges...)
+		rangesToBeAdded := []string{}
+		if eI.Domain != "" {
+			rangesToBeAdded = append(rangesToBeAdded, eI.DomainAns...)
+		} else {
+			rangesToBeAdded = append(rangesToBeAdded, eI.Range)
+		}
+		if defaultUserPolicy.Enabled {
+			result = append(result, rangesToBeAdded...)
+		} else {
+			if staticNode.IsUserNode && staticNode.StaticNode.OwnerID != "" {
+				user, err := GetUser(staticNode.StaticNode.OwnerID)
+				if err != nil {
+					return []string{}, errors.New("user not found")
+				}
+				if DoesUserHaveAccessToEgress(user, &eI, userPolicies) {
+					result = append(result, rangesToBeAdded...)
+				}
+			} else {
+				result = append(result, rangesToBeAdded...)
 			}
 		}
+
 	}
 	extclients, _ := GetNetworkExtClients(client.Network)
 	for _, extclient := range extclients {

logic/hosts.go

@@ -377,6 +377,29 @@ func UpsertHost(h *models.Host) error {
 	return nil
 }
 
+// UpdateHostNode -  handles updates from client nodes
+func UpdateHostNode(h *models.Host, newNode *models.Node) (publishDeletedNodeUpdate, publishPeerUpdate bool) {
+	currentNode, err := GetNodeByID(newNode.ID.String())
+	if err != nil {
+		return
+	}
+	ifaceDelta := IfaceDelta(&currentNode, newNode)
+	newNode.SetLastCheckIn()
+	if err := UpdateNode(&currentNode, newNode); err != nil {
+		slog.Error("error saving node", "name", h.Name, "network", newNode.Network, "error", err)
+		return
+	}
+	if ifaceDelta { // reduce number of unneeded updates, by only sending on iface changes
+		if !newNode.Connected {
+			publishDeletedNodeUpdate = true
+		}
+		publishPeerUpdate = true
+		// reset failover data for this node
+		ResetFailedOverPeer(newNode)
+	}
+	return
+}
+
 // RemoveHost - removes a given host from server
 func RemoveHost(h *models.Host, forceDelete bool) error {
 	if !forceDelete && len(h.Nodes) > 0 {

logic/networks.go

@@ -388,7 +388,7 @@ func UniqueAddressCache(networkName string, reverse bool) (net.IP, error) {
 	}
 
 	if network.IsIPv4 == "no" {
-		return add, fmt.Errorf("IPv4 not active on network " + networkName)
+		return add, fmt.Errorf("IPv4 not active on network %s", networkName)
 	}
 	//ensure AddressRange is valid
 	if _, _, err := net.ParseCIDR(network.AddressRange); err != nil {
@@ -431,7 +431,7 @@ func UniqueAddressDB(networkName string, reverse bool) (net.IP, error) {
 	}
 
 	if network.IsIPv4 == "no" {
-		return add, fmt.Errorf("IPv4 not active on network " + networkName)
+		return add, fmt.Errorf("IPv4 not active on network %s", networkName)
 	}
 	//ensure AddressRange is valid
 	if _, _, err := net.ParseCIDR(network.AddressRange); err != nil {
@@ -529,7 +529,7 @@ func UniqueAddress6DB(networkName string, reverse bool) (net.IP, error) {
 		return add, err
 	}
 	if network.IsIPv6 == "no" {
-		return add, fmt.Errorf("IPv6 not active on network " + networkName)
+		return add, fmt.Errorf("IPv6 not active on network %s", networkName)
 	}
 
 	//ensure AddressRange is valid
@@ -573,7 +573,7 @@ func UniqueAddress6Cache(networkName string, reverse bool) (net.IP, error) {
 		return add, err
 	}
 	if network.IsIPv6 == "no" {
-		return add, fmt.Errorf("IPv6 not active on network " + networkName)
+		return add, fmt.Errorf("IPv6 not active on network %s", networkName)
 	}
 
 	//ensure AddressRange is valid

logic/nodes.go

@@ -685,7 +685,7 @@ func createNode(node *models.Node) error {
 			node.Address.Mask = net.CIDRMask(cidr.Mask.Size())
 		}
 	} else if !IsIPUnique(node.Network, node.Address.String(), database.NODES_TABLE_NAME, false) {
-		return fmt.Errorf("invalid address: ipv4 " + node.Address.String() + " is not unique")
+		return fmt.Errorf("invalid address: ipv4 %s is not unique", node.Address.String())
 	}
 	if node.Address6.IP == nil {
 		if parentNetwork.IsIPv6 == "yes" {
@@ -699,7 +699,7 @@ func createNode(node *models.Node) error {
 			node.Address6.Mask = net.CIDRMask(cidr.Mask.Size())
 		}
 	} else if !IsIPUnique(node.Network, node.Address6.String(), database.NODES_TABLE_NAME, true) {
-		return fmt.Errorf("invalid address: ipv6 " + node.Address6.String() + " is not unique")
+		return fmt.Errorf("invalid address: ipv6 %s is not unique", node.Address6.String())
 	}
 	node.ID = uuid.New()
 	//Create a JWT for the node

logic/peers.go

@@ -149,10 +149,11 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 	}
 	defer func() {
 		if !hostPeerUpdate.FwUpdate.AllowAll {
-
-			hostPeerUpdate.FwUpdate.EgressInfo["allowed-network-rules"] = models.EgressInfo{
-				EgressID:      "allowed-network-rules",
-				EgressFwRules: make(map[string]models.AclRule),
+			if len(hostPeerUpdate.FwUpdate.AllowedNetworks) > 0 {
+				hostPeerUpdate.FwUpdate.EgressInfo["allowed-network-rules"] = models.EgressInfo{
+					EgressID:      "allowed-network-rules",
+					EgressFwRules: make(map[string]models.AclRule),
+				}
 			}
 			for _, aclRule := range hostPeerUpdate.FwUpdate.AllowedNetworks {
 				hostPeerUpdate.FwUpdate.AclRules[aclRule.ID] = aclRule
@@ -182,8 +183,8 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 		acls, _ := ListAclsByNetwork(models.NetworkID(node.Network))
 		eli, _ := (&schema.Egress{Network: node.Network}).ListByNetwork(db.WithContext(context.TODO()))
 		GetNodeEgressInfo(&node, eli, acls)
-		if node.EgressDetails.IsEgressGateway {
-			egsWithDomain := ListAllByRoutingNodeWithDomain(eli, node.ID.String())
+		egsWithDomain := ListAllByRoutingNodeWithDomain(eli, node.ID.String())
+		if len(egsWithDomain) > 0 {
 			hostPeerUpdate.EgressWithDomains = append(hostPeerUpdate.EgressWithDomains, egsWithDomain...)
 		}
 		hostPeerUpdate = SetDefaultGw(node, hostPeerUpdate)

logic/settings.go

@@ -245,6 +245,8 @@ func GetServerInfo() models.ServerConfig {
 	cfg.StunServers = serverSettings.StunServers
 	cfg.DefaultDomain = serverSettings.DefaultDomain
 	cfg.EndpointDetection = serverSettings.EndpointDetection
+	key, _ := RetrievePublicTrafficKey()
+	cfg.TrafficKey = key
 	return cfg
 }
 

logic/wireguard.go

@@ -9,20 +9,9 @@ func IfaceDelta(currentNode *models.Node, newNode *models.Node) bool {
 	// single comparison statements
 	if newNode.Address.String() != currentNode.Address.String() ||
 		newNode.Address6.String() != currentNode.Address6.String() ||
-		newNode.IsRelay != currentNode.IsRelay ||
 		newNode.Connected != currentNode.Connected {
 		return true
 	}
-	if newNode.IsRelay {
-		if len(currentNode.RelayedNodes) != len(newNode.RelayedNodes) {
-			return true
-		}
-		for _, node := range newNode.RelayedNodes {
-			if !StringSliceContains(currentNode.RelayedNodes, node) {
-				return true
-			}
-		}
-	}
 	return false
 }
 

migrate/migrate.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"log"
+	"net"
 	"os"
 	"time"
 
@@ -63,6 +64,10 @@ func migrateNameservers() {
 	}
 
 	for _, netI := range nets {
+		_, cidr, err := net.ParseCIDR(netI.AddressRange)
+		if err != nil {
+			continue
+		}
 		if len(netI.NameServers) > 0 {
 			ns := schema.Nameserver{
 				ID:           uuid.NewString(),
@@ -78,8 +83,14 @@ func migrateNameservers() {
 				Status:    true,
 				CreatedBy: user.UserName,
 			}
-			for _, ip := range netI.NameServers {
-				ns.Servers = append(ns.Servers, ip)
+
+			for _, nsIP := range netI.NameServers {
+				if net.ParseIP(nsIP) == nil {
+					continue
+				}
+				if !cidr.Contains(net.ParseIP(nsIP)) {
+					ns.Servers = append(ns.Servers, nsIP)
+				}
 			}
 			ns.Create(db.WithContext(context.TODO()))
 			netI.NameServers = []string{}

models/extclient.go

@@ -66,7 +66,7 @@ func (ext *ExtClient) ConvertToStaticNode() Node {
 		Tags:       ext.Tags,
 		IsStatic:   true,
 		StaticNode: *ext,
-		IsUserNode: ext.RemoteAccessClientID != "",
+		IsUserNode: ext.RemoteAccessClientID != "" || ext.DeviceID != "",
 		Mutex:      ext.Mutex,
 	}
 }

models/host.go

@@ -106,6 +106,8 @@ const (
 	SignalHost HostMqAction = "SIGNAL_HOST"
 	// UpdateHost - constant for host update action
 	UpdateHost HostMqAction = "UPDATE_HOST"
+	// UpdateNode - constant for Node update action
+	UpdateNode HostMqAction = "UPDATE_NODE"
 	// DeleteHost - constant for host delete action
 	DeleteHost HostMqAction = "DELETE_HOST"
 	// JoinHostToNetwork - constant for host network join action

models/node.go

@@ -481,6 +481,7 @@ func (newNode *Node) Fill(
 	if newNode.IsFailOver != currentNode.IsFailOver {
 		newNode.IsFailOver = currentNode.IsFailOver
 	}
+	newNode.FailOverPeers = currentNode.FailOverPeers
 	if newNode.Tags == nil {
 		if currentNode.Tags == nil {
 			currentNode.Tags = make(map[TagID]struct{})

models/structs.go

@@ -17,6 +17,7 @@ const (
 )
 
 type FeatureFlags struct {
+	EnableEgressHA          bool `json:"enable_egress_ha"`
 	EnableNetworkActivity   bool `json:"enable_network_activity"`
 	EnableOAuth             bool `json:"enable_oauth"`
 	EnableIDPIntegration    bool `json:"enable_idp_integration"`
@@ -266,6 +267,7 @@ type HostPull struct {
 	NameServers       []string              `json:"name_servers"`
 	EgressWithDomains []EgressDomain        `json:"egress_with_domains"`
 	DnsNameservers    []Nameserver          `json:"dns_nameservers"`
+	ReplacePeers      bool                  `json:"replace_peers"`
 }
 
 type DefaultGwInfo struct {

mq/handlers.go

@@ -136,42 +136,10 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) {
 			return
 		}
 	case models.DeleteHost:
-		if servercfg.GetBrokerType() == servercfg.EmqxBrokerType {
-			// delete EMQX credentials for host
-			if err := emqx.DeleteEmqxUser(currentHost.ID.String()); err != nil {
-				slog.Error("failed to remove host credentials from EMQX", "id", currentHost.ID, "error", err)
-			}
-		}
-
-		// notify of deleted peer change
-		go func(host models.Host) {
-			for _, nodeID := range host.Nodes {
-				node, err := logic.GetNodeByID(nodeID)
-				if err == nil {
-					var gwClients []models.ExtClient
-					if node.IsIngressGateway {
-						gwClients = logic.GetGwExtclients(node.ID.String(), node.Network)
-					}
-					go PublishMqUpdatesForDeletedNode(node, false, gwClients)
-				}
-
-			}
-		}(*currentHost)
-
-		if err := logic.DisassociateAllNodesFromHost(currentHost.ID.String()); err != nil {
-			slog.Error("failed to delete all nodes of host", "id", currentHost.ID, "error", err)
-			return
-		}
-		if err := logic.RemoveHostByID(currentHost.ID.String()); err != nil {
-			slog.Error("failed to delete host", "id", currentHost.ID, "error", err)
-			return
-		}
-		if servercfg.IsDNSMode() {
-			logic.SetDNS()
-		}
+		DeleteAndCleanupHost(currentHost)
 		sendPeerUpdate = true
 	case models.SignalHost:
-		signalPeer(hostUpdate.Signal)
+		SignalPeer(hostUpdate.Signal)
 
 	}
 
@@ -183,7 +151,43 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) {
 	}
 }
 
-func signalPeer(signal models.Signal) {
+func DeleteAndCleanupHost(h *models.Host) {
+	if servercfg.GetBrokerType() == servercfg.EmqxBrokerType {
+		// delete EMQX credentials for host
+		if err := emqx.DeleteEmqxUser(h.ID.String()); err != nil {
+			slog.Error("failed to remove host credentials from EMQX", "id", h.ID, "error", err)
+		}
+	}
+
+	// notify of deleted peer change
+	go func(host models.Host) {
+		for _, nodeID := range host.Nodes {
+			node, err := logic.GetNodeByID(nodeID)
+			if err == nil {
+				var gwClients []models.ExtClient
+				if node.IsIngressGateway {
+					gwClients = logic.GetGwExtclients(node.ID.String(), node.Network)
+				}
+				go PublishMqUpdatesForDeletedNode(node, false, gwClients)
+			}
+
+		}
+	}(*h)
+
+	if err := logic.DisassociateAllNodesFromHost(h.ID.String()); err != nil {
+		slog.Error("failed to delete all nodes of host", "id", h.ID, "error", err)
+		return
+	}
+	if err := logic.RemoveHostByID(h.ID.String()); err != nil {
+		slog.Error("failed to delete host", "id", h.ID, "error", err)
+		return
+	}
+	if servercfg.IsDNSMode() {
+		logic.SetDNS()
+	}
+}
+
+func SignalPeer(signal models.Signal) {
 
 	if signal.ToHostPubKey == "" {
 		msg := "insufficient data to signal peer"

pro/auth/sync.go

@@ -11,11 +11,13 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/pro/idp"
 	"github.com/gravitl/netmaker/pro/idp/azure"
 	"github.com/gravitl/netmaker/pro/idp/google"
 	"github.com/gravitl/netmaker/pro/idp/okta"
 	proLogic "github.com/gravitl/netmaker/pro/logic"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 var (
@@ -149,7 +151,8 @@ func syncUsers(idpUsers []idp.User) error {
 	for _, user := range idpUsers {
 		if user.AccountArchived {
 			// delete the user if it has been archived.
-			_ = logic.DeleteUser(user.Username)
+			user := dbUsersMap[user.Username]
+			_ = deleteAndCleanUpUser(&user)
 			continue
 		}
 
@@ -209,14 +212,14 @@ func syncUsers(idpUsers []idp.User) error {
 	}
 
 	for _, user := range dbUsersMap {
-		if user.ExternalIdentityProviderID == "" {
-			continue
-		}
-		if _, ok := idpUsersMap[user.UserName]; !ok {
-			// delete the user if it has been deleted on idp.
-			err = logic.DeleteUser(user.UserName)
-			if err != nil {
-				return err
+		if user.ExternalIdentityProviderID != "" {
+			if _, ok := idpUsersMap[user.UserName]; !ok {
+				// delete the user if it has been deleted on idp
+				// or is filtered out.
+				err = deleteAndCleanUpUser(&user)
+				if err != nil {
+					return err
+				}
 			}
 		}
 	}
@@ -277,7 +280,11 @@ func syncGroups(idpGroups []idp.Group) error {
 			dbGroup.ExternalIdentityProviderID = group.ID
 			dbGroup.Name = group.Name
 			dbGroup.Default = false
-			dbGroup.NetworkRoles = make(map[models.NetworkID]map[models.UserRoleID]struct{})
+			dbGroup.NetworkRoles = map[models.NetworkID]map[models.UserRoleID]struct{}{
+				models.AllNetworks: {
+					proLogic.GetDefaultGlobalUserRoleID(): {},
+				},
+			}
 			err := proLogic.CreateUserGroup(&dbGroup)
 			if err != nil {
 				return err
@@ -324,8 +331,9 @@ func syncGroups(idpGroups []idp.Group) error {
 	for _, group := range dbGroups {
 		if group.ExternalIdentityProviderID != "" {
 			if _, ok := idpGroupsMap[group.ExternalIdentityProviderID]; !ok {
-				// delete the group if it has been deleted on idp.
-				err = proLogic.DeleteUserGroup(group.ID)
+				// delete the group if it has been deleted on idp
+				// or is filtered out.
+				err = proLogic.DeleteAndCleanUpGroup(&group)
 				if err != nil {
 					return err
 				}
@@ -355,6 +363,7 @@ func GetIDPSyncStatus() models.IDPSyncStatus {
 		}
 	}
 }
+
 func filterUsersByGroupMembership(idpUsers []idp.User, idpGroups []idp.Group) []idp.User {
 	usersMap := make(map[string]int)
 	for i, user := range idpUsers {
@@ -395,14 +404,14 @@ func filterGroupsByMembers(idpGroups []idp.Group, idpUsers []idp.User) []idp.Gro
 			if _, ok := usersMap[member]; ok {
 				members = append(members, member)
 			}
+		}
 
-			if len(members) > 0 {
-				// the group at index `i` has members from the `idpUsers` list,
-				// so we keep it.
-				filteredGroupsMap[i] = true
-				// filter out members that were not provided in the `idpUsers` list.
-				idpGroups[i].Members = members
-			}
+		if len(members) > 0 {
+			// the group at index `i` has members from the `idpUsers` list,
+			// so we keep it.
+			filteredGroupsMap[i] = true
+			// filter out members that were not provided in the `idpUsers` list.
+			idpGroups[i].Members = members
 		}
 	}
 
@@ -415,3 +424,37 @@ func filterGroupsByMembers(idpGroups []idp.Group, idpUsers []idp.User) []idp.Gro
 
 	return filteredGroups
 }
+
+// TODO: deduplicate
+// The cyclic import between the package logic and mq requires this
+// function to be duplicated in multiple places.
+func deleteAndCleanUpUser(user *models.User) error {
+	err := logic.DeleteUser(user.UserName)
+	if err != nil {
+		return err
+	}
+
+	// check and delete extclient with this ownerID
+	go func() {
+		extclients, err := logic.GetAllExtClients()
+		if err != nil {
+			return
+		}
+		for _, extclient := range extclients {
+			if extclient.OwnerID == user.UserName {
+				err = logic.DeleteExtClientAndCleanup(extclient)
+				if err == nil {
+					_ = mq.PublishDeletedClientPeerUpdate(&extclient)
+				}
+			}
+		}
+
+		go logic.DeleteUserInvite(user.UserName)
+		go mq.PublishPeerUpdate(false)
+		if servercfg.IsDNSMode() {
+			go logic.SetDNS()
+		}
+	}()
+
+	return nil
+}

pro/controllers/users.go

@@ -808,11 +808,13 @@ func deleteUserGroup(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("cannot delete default user group"), "badrequest"))
 		return
 	}
-	err = proLogic.DeleteUserGroup(models.UserGroupID(gid))
+	err = proLogic.DeleteAndCleanUpGroup(&userG)
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+
+	// TODO: log event in proLogic.DeleteAndCleanUpGroup so that all deletions are logged.
 	logic.LogEvent(&models.Event{
 		Action: models.Delete,
 		Source: models.Subject{
@@ -828,42 +830,7 @@ func deleteUserGroup(w http.ResponseWriter, r *http.Request) {
 		},
 		Origin: models.Dashboard,
 	})
-	replacePeers := false
-	go func() {
-		for networkID := range userG.NetworkRoles {
-			acls, err := logic.ListAclsByNetwork(networkID)
-			if err != nil {
-				continue
-			}
 
-			for _, acl := range acls {
-				var hasGroupSrc bool
-				newAclSrc := make([]models.AclPolicyTag, 0)
-				for _, src := range acl.Src {
-					if src.ID == models.UserGroupAclID && src.Value == userG.ID.String() {
-						hasGroupSrc = true
-					} else {
-						newAclSrc = append(newAclSrc, src)
-					}
-				}
-
-				if hasGroupSrc {
-					if len(newAclSrc) == 0 {
-						// no other src exists, delete acl.
-						_ = logic.DeleteAcl(acl)
-					} else {
-						// other sources exist, update acl.
-						acl.Src = newAclSrc
-						_ = logic.UpsertAcl(acl)
-					}
-					replacePeers = true
-				}
-			}
-		}
-	}()
-
-	go proLogic.UpdatesUserGwAccessOnGrpUpdates(userG.ID, userG.NetworkRoles, make(map[models.NetworkID]map[models.UserRoleID]struct{}))
-	go mq.PublishPeerUpdate(replacePeers)
 	logic.ReturnSuccessResponseWithJson(w, r, nil, "deleted user group")
 }
 
@@ -1499,6 +1466,10 @@ func getUserRemoteAccessGwsV1(w http.ResponseWriter, r *http.Request) {
 			continue
 		}
 
+		if extClient.RemoteAccessClientID == "" {
+			continue
+		}
+
 		_, ok := userExtClients[extClient.IngressGatewayID]
 		if !ok {
 			userExtClients[extClient.IngressGatewayID] = []models.ExtClient{}
@@ -1527,13 +1498,21 @@ func getUserRemoteAccessGwsV1(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		if !found {
-			// TODO: prevent ip clashes.
-			if len(extClients) > 0 {
-				gwClient = extClients[0]
+		if !found && req.RemoteAccessClientID != "" {
+			for _, extClient := range extClients {
+				if extClient.RemoteAccessClientID == req.RemoteAccessClientID {
+					gwClient = extClient
+					found = true
+					break
+				}
 			}
 		}
 
+		if !found && len(extClients) > 0 {
+			// TODO: prevent ip clashes.
+			gwClient = extClients[0]
+		}
+
 		host, err := logic.GetHost(node.HostID.String())
 		if err != nil {
 			continue

pro/idp/azure/azure.go

@@ -226,15 +226,19 @@ func (a *Client) getAccessToken() (string, error) {
 }
 
 func buildPrefixFilter(field string, prefixes []string) string {
+	return url.PathEscape("$filter=" + buildCondition(field, prefixes))
+}
+
+func buildCondition(field string, prefixes []string) string {
 	if len(prefixes) == 0 {
 		return ""
 	}
 
 	if len(prefixes) == 1 {
-		return fmt.Sprintf("$filter=startswith(%s,'%s')", field, prefixes[0])
+		return fmt.Sprintf("startswith(%s,'%s')", field, prefixes[0])
 	}
 
-	return buildPrefixFilter(field, prefixes[:1]) + "%20or%20" + buildPrefixFilter(field, prefixes[1:])
+	return buildCondition(field, prefixes[:1]) + " or " + buildCondition(field, prefixes[1:])
 }
 
 type getUsersResponse struct {

pro/idp/okta/okta.go

@@ -17,6 +17,7 @@ func NewOktaClient(oktaOrgURL, oktaAPIToken string) (*Client, error) {
 	config, err := okta.NewConfiguration(
 		okta.WithOrgUrl(oktaOrgURL),
 		okta.WithToken(oktaAPIToken),
+		okta.WithRateLimitPrevent(true),
 	)
 	if err != nil {
 		return nil, err
@@ -45,18 +46,17 @@ func (o *Client) Verify() error {
 
 func (o *Client) GetUsers(filters []string) ([]idp.User, error) {
 	var retval []idp.User
-	var allUsersFetched bool
-
-	for !allUsersFetched {
-		users, resp, err := o.client.UserAPI.ListUsers(context.TODO()).
-			Search(buildPrefixFilter("profile.login", filters)).
-			Execute()
-		if err != nil {
-			return nil, err
-		}
 
-		allUsersFetched = !resp.HasNextPage()
+	users, resp, err := o.client.UserAPI.ListUsers(context.TODO()).
+		Search(buildPrefixFilter("profile.login", filters)).
+		Execute()
+	if err != nil {
+		return nil, err
+	}
 
+	usersProcessingPending := len(users) > 0 || resp.HasNextPage()
+
+	for usersProcessingPending {
 		for _, user := range users {
 			id := *user.Id
 			username := *user.Profile.Login
@@ -79,6 +79,19 @@ func (o *Client) GetUsers(filters []string) ([]idp.User, error) {
 				AccountArchived: false,
 			})
 		}
+
+		if resp.HasNextPage() {
+			users = make([]okta.User, 0)
+
+			resp, err = resp.Next(&users)
+			if err != nil {
+				return nil, err
+			}
+
+			usersProcessingPending = len(users) > 0 || resp.HasNextPage()
+		} else {
+			usersProcessingPending = false
+		}
 	}
 
 	return retval, nil
@@ -86,35 +99,46 @@ func (o *Client) GetUsers(filters []string) ([]idp.User, error) {
 
 func (o *Client) GetGroups(filters []string) ([]idp.Group, error) {
 	var retval []idp.Group
-	var allGroupsFetched bool
-
-	for !allGroupsFetched {
-		groups, resp, err := o.client.GroupAPI.ListGroups(context.TODO()).
-			Search(buildPrefixFilter("profile.name", filters)).
-			Execute()
-		if err != nil {
-			return nil, err
-		}
 
-		allGroupsFetched = !resp.HasNextPage()
+	groups, resp, err := o.client.GroupAPI.ListGroups(context.TODO()).
+		Search(buildPrefixFilter("profile.name", filters)).
+		Execute()
+	if err != nil {
+		return nil, err
+	}
+
+	groupsProcessingPending := len(groups) > 0 || resp.HasNextPage()
 
+	for groupsProcessingPending {
 		for _, group := range groups {
-			var allMembersFetched bool
 			id := *group.Id
 			name := *group.Profile.Name
 
 			var members []string
-			for !allMembersFetched {
-				groupUsers, resp, err := o.client.GroupAPI.ListGroupUsers(context.TODO(), id).Execute()
-				if err != nil {
-					return nil, err
-				}
+			groupUsers, groupUsersResp, err := o.client.GroupAPI.ListGroupUsers(context.TODO(), id).Execute()
+			if err != nil {
+				return nil, err
+			}
 
-				allMembersFetched = !resp.HasNextPage()
+			groupUsersProcessingPending := len(groupUsers) > 0 || groupUsersResp.HasNextPage()
 
+			for groupUsersProcessingPending {
 				for _, groupUser := range groupUsers {
 					members = append(members, *groupUser.Id)
 				}
+
+				if groupUsersResp.HasNextPage() {
+					groupUsers = make([]okta.GroupMember, 0)
+
+					groupUsersResp, err = groupUsersResp.Next(&groupUsers)
+					if err != nil {
+						return nil, err
+					}
+
+					groupUsersProcessingPending = len(groupUsers) > 0 || groupUsersResp.HasNextPage()
+				} else {
+					groupUsersProcessingPending = false
+				}
 			}
 
 			retval = append(retval, idp.Group{
@@ -123,6 +147,19 @@ func (o *Client) GetGroups(filters []string) ([]idp.Group, error) {
 				Members: members,
 			})
 		}
+
+		if resp.HasNextPage() {
+			groups = make([]okta.Group, 0)
+
+			resp, err = resp.Next(&groups)
+			if err != nil {
+				return nil, err
+			}
+
+			groupsProcessingPending = len(groups) > 0 || resp.HasNextPage()
+		} else {
+			groupsProcessingPending = false
+		}
 	}
 
 	return retval, nil

pro/logic/dns.go

@@ -3,6 +3,7 @@ package logic
 import (
 	"context"
 	"errors"
+	"net"
 
 	"github.com/gravitl/netmaker/db"
 	"github.com/gravitl/netmaker/logic"
@@ -20,6 +21,25 @@ func ValidateNameserverReq(ns schema.Nameserver) error {
 	if len(ns.Servers) == 0 {
 		return errors.New("atleast one nameserver should be specified")
 	}
+	network, err := logic.GetNetwork(ns.NetworkID)
+	if err != nil {
+		return errors.New("invalid network id")
+	}
+	_, cidr, err4 := net.ParseCIDR(network.AddressRange)
+	_, cidr6, err6 := net.ParseCIDR(network.AddressRange6)
+	for _, nsIPStr := range ns.Servers {
+		nsIP := net.ParseIP(nsIPStr)
+		if nsIP == nil {
+			return errors.New("invalid nameserver " + nsIPStr)
+		}
+		if err4 == nil && nsIP.To4() != nil {
+			if cidr.Contains(nsIP) {
+				return errors.New("cannot use netmaker IP as nameserver")
+			}
+		} else if err6 == nil && cidr6.Contains(nsIP) {
+			return errors.New("cannot use netmaker IP as nameserver")
+		}
+	}
 	if !ns.MatchAll && len(ns.MatchDomains) == 0 {
 		return errors.New("atleast one match domain is required")
 	}

pro/logic/user_mgmt.go

@@ -620,6 +620,22 @@ func GetUserGroup(gid models.UserGroupID) (models.UserGroup, error) {
 	return ug, nil
 }
 
+func GetDefaultGlobalAdminGroupID() models.UserGroupID {
+	return globalNetworksAdminGroupID
+}
+
+func GetDefaultGlobalUserGroupID() models.UserGroupID {
+	return globalNetworksUserGroupID
+}
+
+func GetDefaultGlobalAdminRoleID() models.UserRoleID {
+	return globalNetworksAdminRoleID
+}
+
+func GetDefaultGlobalUserRoleID() models.UserRoleID {
+	return globalNetworksUserRoleID
+}
+
 func GetDefaultNetworkAdminGroupID(networkID models.NetworkID) models.UserGroupID {
 	return models.UserGroupID(fmt.Sprintf("%s-%s-grp", networkID, models.NetworkAdmin))
 }
@@ -672,6 +688,52 @@ func UpdateUserGroup(g models.UserGroup) error {
 	return database.Insert(g.ID.String(), string(d), database.USER_GROUPS_TABLE_NAME)
 }
 
+func DeleteAndCleanUpGroup(group *models.UserGroup) error {
+	err := DeleteUserGroup(group.ID)
+	if err != nil {
+		return err
+	}
+
+	go func() {
+		var replacePeers bool
+		for networkID := range group.NetworkRoles {
+			acls, err := logic.ListAclsByNetwork(networkID)
+			if err != nil {
+				continue
+			}
+
+			for _, acl := range acls {
+				var hasGroupSrc bool
+				newAclSrc := make([]models.AclPolicyTag, 0)
+				for _, src := range acl.Src {
+					if src.ID == models.UserGroupAclID && src.Value == group.ID.String() {
+						hasGroupSrc = true
+					} else {
+						newAclSrc = append(newAclSrc, src)
+					}
+				}
+
+				if hasGroupSrc {
+					if len(newAclSrc) == 0 {
+						// no other src exists, delete acl.
+						_ = logic.DeleteAcl(acl)
+					} else {
+						// other sources exist, update acl.
+						acl.Src = newAclSrc
+						_ = logic.UpsertAcl(acl)
+					}
+					replacePeers = true
+				}
+			}
+		}
+
+		go UpdatesUserGwAccessOnGrpUpdates(group.ID, group.NetworkRoles, make(map[models.NetworkID]map[models.UserRoleID]struct{}))
+		go mq.PublishPeerUpdate(replacePeers)
+	}()
+
+	return nil
+}
+
 // DeleteUserGroup - deletes user group
 func DeleteUserGroup(gid models.UserGroupID) error {
 	g, err := GetUserGroup(gid)

schema/egress.go

@@ -52,6 +52,18 @@ func (e *Egress) UpdateEgressStatus(ctx context.Context) error {
 	}).Error
 }
 
+func (e *Egress) ResetDomain(ctx context.Context) error {
+	return db.FromContext(ctx).Table(e.Table()).Where("id = ?", e.ID).Updates(map[string]any{
+		"domain": "",
+	}).Error
+}
+
+func (e *Egress) ResetRange(ctx context.Context) error {
+	return db.FromContext(ctx).Table(e.Table()).Where("id = ?", e.ID).Updates(map[string]any{
+		"range": "",
+	}).Error
+}
+
 func (e *Egress) DoesEgressRouteExists(ctx context.Context) error {
 	return db.FromContext(ctx).Table(e.Table()).Where("range = ?", e.Range).First(&e).Error
 }

scripts/netmaker-ci-runner.sh

@@ -0,0 +1,189 @@
+#!/usr/bin/env bash
+# Netmaker CI helper: bring WireGuard up/down and manage ephemeral client lifecycle.
+# Subcommands:
+#   up   - fetch config, capture Client-ID, bring interface up, save state
+#   down - bring interface down, delete local conf, delete client via API
+#
+# Env vars (can be overridden by flags):
+#   NETMAKER_BASE_URL   (required)  e.g. https://nm.example.com   or pass --base-url
+#   NETMAKER_NETWORK    (required)  e.g. corpnet                  or pass --network
+#   NETMAKER_API_JWT    (required)  Bearer token                  or pass --jwt
+#   WG_IFACE            (default netmaker)                           or pass --iface
+#   WG_CONF_DIR         (default /etc/wireguard)                  or pass --confdir
+#   NETMAKER_STATE_FILE (default RUNNER_TEMP or /tmp)
+# You may also pass --client-id on `down` to avoid relying on the state file.
+
+set -euo pipefail
+
+# ---------- defaults ----------
+WG_IFACE="${WG_IFACE:-netmaker}"
+WG_CONF_DIR="${WG_CONF_DIR:-/etc/wireguard}"
+SUBCMD=""
+CLIENT_ID_OVERRIDE=""
+
+usage() {
+  cat <<USAGE
+Usage:
+  $0 up   [--iface IFACE] [--confdir DIR] [--base-url URL] [--network NET] [--jwt TOKEN]
+  $0 down [--iface IFACE] [--confdir DIR] [--base-url URL] [--network NET] [--jwt TOKEN] [--client-id ID]
+
+Flags override env vars. Env vars documented at top of the script.
+Examples:
+  NETMAKER_BASE_URL=https://nm.example.com NETMAKER_NETWORK=corpnet NETMAKER_API_JWT=... $0 up
+  $0 down --base-url https://nm.example.com --network corpnet --jwt ... --client-id icy-water
+USAGE
+}
+
+# ---------- arg parse ----------
+if [[ $# -lt 1 ]]; then usage; exit 2; fi
+SUBCMD="$1"; shift || true
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --iface)      WG_IFACE="$2"; shift 2;;
+    --confdir)    WG_CONF_DIR="$2"; shift 2;;
+    --base-url)   NETMAKER_BASE_URL="$2"; shift 2;;
+    --network)    NETMAKER_NETWORK="$2"; shift 2;;
+    --jwt)        NETMAKER_API_JWT="$2"; shift 2;;
+    --client-id)  CLIENT_ID_OVERRIDE="$2"; shift 2;;
+    -h|--help)    usage; exit 0;;
+    *) echo "Unknown arg: $1" >&2; usage; exit 2;;
+  esac
+done
+
+STATE_FILE="${NETMAKER_STATE_FILE:-${RUNNER_TEMP:-/tmp}/netmaker_ci_${WG_IFACE}.env}"
+
+require_env() {
+  : "${NETMAKER_BASE_URL:?ERROR: NETMAKER_BASE_URL not set}"
+  : "${NETMAKER_NETWORK:?ERROR: NETMAKER_NETWORK not set}"
+  : "${NETMAKER_API_JWT:?ERROR: NETMAKER_API_JWT not set}"
+}
+
+install_deps() {
+  echo "[*] Checking dependencies ..."
+  local need=(curl jq wg-quick ip)
+  local miss=()
+  for b in "${need[@]}"; do command -v "$b" >/dev/null 2>&1 || miss+=("$b"); done
+  if [[ ${#miss[@]} -eq 0 ]]; then
+    echo "[*] All dependencies present."
+    return
+  fi
+  echo "[*] Installing missing deps: ${miss[*]}"
+  if command -v apt-get >/dev/null 2>&1; then
+    sudo apt-get update -y
+    sudo apt-get install -y wireguard-tools jq curl iproute2 resolvconf
+  elif command -v yum >/dev/null 2>&1; then
+    sudo yum install -y wireguard-tools jq curl iproute iproute-tc
+  elif command -v dnf >/dev/null 2>&1; then
+    sudo dnf install -y wireguard-tools jq curl iproute
+  else
+    echo "ERROR: no supported package manager found; install: curl jq wireguard-tools iproute" >&2
+    exit 1
+  fi
+}
+
+do_up() {
+  require_env
+  install_deps
+
+  local ep="${NETMAKER_BASE_URL}/api/v1/client_conf/${NETMAKER_NETWORK}"
+  local tmp_conf="/tmp/${WG_IFACE}.conf"
+  local tmp_hdr="/tmp/${WG_IFACE}.headers"
+
+  echo "[*] Requesting client config: ${ep}"
+  # Optional headers
+  declare -a hdrs
+  hdrs=(-H "Authorization: Bearer ${NETMAKER_API_JWT}")
+  [[ -n "${NM_CLIENT_LABEL:-}"   ]] && hdrs+=(-H "X-NM-Client-Label: ${NM_CLIENT_LABEL}")
+  [[ -n "${NM_REQUESTED_NAME:-}" ]] && hdrs+=(-H "X-NM-Requested-Name: ${NM_REQUESTED_NAME}")
+
+  local code
+  code="$(curl -sS -L --dump-header "${tmp_hdr}" -w '%{http_code}' -o "${tmp_conf}" "${hdrs[@]}" "${ep}")"
+  if [[ "${code}" != "200" ]]; then
+    echo "ERROR: client_conf HTTP ${code}" >&2
+    curl -sS -L "${hdrs[@]}" "${ep}" | head -c 400 >&2 || true
+    exit 1
+  fi
+  grep -q "^\[Interface\]" "${tmp_conf}" || { echo "ERROR: not a WireGuard conf"; head -n 20 "${tmp_conf}"; exit 1; }
+
+  # --- Extract Client-ID (one-liner, trim spaces/quotes) ---
+  local client_id
+  client_id="$(grep -i '^Client-ID:' "${tmp_hdr}" | head -n1 | cut -d: -f2- | tr -d '\r' | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//' -e 's/^"//; s/"$//' -e "s/^'//; s/'$//")"
+  if [[ -z "${client_id}" ]]; then
+    echo "ERROR: Client-ID header missing in response; cannot manage lifecycle." >&2
+    exit 1
+  fi
+  echo "[*] Client-ID: ${client_id}"
+
+  # Optional marker
+  if ! grep -q "^#interface-name=" "${tmp_conf}"; then
+    echo "#interface-name=${WG_IFACE}" | cat - "${tmp_conf}" > "${tmp_conf}.tmp" && mv "${tmp_conf}.tmp" "${tmp_conf}"
+  fi
+
+  # Install & bring up
+  sudo mkdir -p "${WG_CONF_DIR}"
+  sudo mv "${tmp_conf}" "${WG_CONF_DIR}/${WG_IFACE}.conf"
+  sudo chmod 600 "${WG_CONF_DIR}/${WG_IFACE}.conf"
+  echo "[*] Bringing up ${WG_IFACE} ..."
+  sudo wg-quick up "${WG_IFACE}"
+
+  echo "==== ${WG_IFACE} is up ===="
+  ip addr show "${WG_IFACE}" || true
+  wg show "${WG_IFACE}" || true
+
+  # Persist state
+  cat > "${STATE_FILE}" <<EOF
+NETMAKER_BASE_URL='${NETMAKER_BASE_URL}'
+NETMAKER_NETWORK='${NETMAKER_NETWORK}'
+NETMAKER_API_JWT='${NETMAKER_API_JWT}'
+WG_IFACE='${WG_IFACE}'
+WG_CONF_DIR='${WG_CONF_DIR}'
+CLIENT_ID='${client_id}'
+EOF
+  chmod 600 "${STATE_FILE}"
+  echo "[*] Saved state: ${STATE_FILE}"
+}
+
+do_down() {
+  # Load state if present; flags/env can still override
+  if [[ -f "${STATE_FILE}" ]]; then
+    # shellcheck disable=SC1090
+    source "${STATE_FILE}"
+  fi
+
+  require_env
+
+  local client_id="${CLIENT_ID_OVERRIDE:-${CLIENT_ID:-}}"
+  echo "[*] Bringing down ${WG_IFACE} ..."
+  sudo wg-quick down "${WG_IFACE}" || echo "WARN: wg-quick down failed (already down?)."
+
+  # Remove local conf
+  if [[ -f "${WG_CONF_DIR}/${WG_IFACE}.conf" ]]; then
+    sudo shred -u "${WG_CONF_DIR}/${WG_IFACE}.conf" 2>/dev/null || sudo rm -f "${WG_CONF_DIR}/${WG_IFACE}.conf"
+  fi
+
+  # Delete ephemeral client on server (if we know its ID)
+  if [[ -n "${client_id}" ]]; then
+    local del_ep="${NETMAKER_BASE_URL}/api/extclients/${NETMAKER_NETWORK}/${client_id}"
+    echo "[*] Deleting client: DELETE ${del_ep}"
+    local http
+    http="$(curl -sS -o /dev/null -w '%{http_code}' -X DELETE -H "Authorization: Bearer ${NETMAKER_API_JWT}" "${del_ep}")"
+    if [[ "${http}" =~ ^20[0-9]$ ]]; then
+      echo "[*] Client deleted (HTTP ${http})."
+    else
+      echo "WARN: deletion returned HTTP ${http}; verify server state."
+    fi
+  else
+    echo "WARN: client id not known (missing --client-id and state file); skipping server delete."
+  fi
+
+  rm -f "${STATE_FILE}" || true
+  echo "[*] Teardown finished."
+}
+
+case "${SUBCMD}" in
+  up)   do_up ;;
+  down) do_down ;;
+  *)    usage; exit 2 ;;
+esac
+