Merge branch 'develop' into NET-655

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -31,6 +31,7 @@ body:
       label: Version
       description: What version are you running?
       options:
+        - v0.21.3
         - v0.21.2
         - v0.21.1
         - v0.21.0

Dockerfile

@@ -6,7 +6,7 @@ COPY . .
 
 RUN GOOS=linux CGO_ENABLED=1 go build -ldflags="-s -w " -tags ${tags} .
 # RUN go build -tags=ee . -o netmaker main.go
-FROM alpine:3.18.4
+FROM alpine:3.18.5
 
 # add a c lib
 # set the working directory

Dockerfile-quick

@@ -1,5 +1,5 @@
 #first stage - builder
-FROM alpine:3.18.4
+FROM alpine:3.18.5
 ARG version 
 WORKDIR /app
 COPY ./netmaker /root/netmaker

README.md

@@ -16,7 +16,7 @@
 
 <p align="center">
   <a href="https://github.com/gravitl/netmaker/releases">
-    <img src="https://img.shields.io/badge/Version-0.21.2-informational?style=flat-square" />
+    <img src="https://img.shields.io/badge/Version-0.21.3-informational?style=flat-square" />
   </a>
   <a href="https://hub.docker.com/r/gravitl/netmaker/tags">
     <img src="https://img.shields.io/docker/pulls/gravitl/netmaker?label=downloads" />

compose/docker-compose.netclient.yml

@@ -3,7 +3,7 @@ version: "3.4"
 services:
   netclient:
     container_name: netclient
-    image: 'gravitl/netclient:v0.21.2'
+    image: 'gravitl/netclient:v0.21.3'
     hostname: netmaker-1
     network_mode: host
     restart: on-failure

config/config.go

@@ -91,6 +91,7 @@ type ServerConfig struct {
 	Environment                string        `yaml:"environment"`
 	JwtValidityDuration        time.Duration `yaml:"jwt_validity_duration"`
 	RacAutoDisable             bool          `yaml:"rac_auto_disable"`
+	CacheEnabled               bool          `yaml:"caching_enabled"`
 }
 
 // SQLConfig - Generic SQL Config

controllers/docs.go

@@ -10,7 +10,7 @@
 //
 //	Schemes: https
 //	BasePath: /
-//	Version: 0.21.2
+//	Version: 0.21.3
 //	Host: api.demo.netmaker.io
 //
 //	Consumes:

controllers/ext_client.go

@@ -481,6 +481,9 @@ func updateExtClient(w http.ResponseWriter, r *http.Request) {
 		sendPeerUpdate = true
 		logic.SetClientACLs(&oldExtClient, update.DeniedACLs)
 	}
+	if !logic.IsSlicesEqual(update.ExtraAllowedIPs, oldExtClient.ExtraAllowedIPs) {
+		sendPeerUpdate = true
+	}
 
 	if update.Enabled != oldExtClient.Enabled {
 		sendPeerUpdate = true

controllers/hosts.go

@@ -6,7 +6,9 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/google/uuid"
 	"github.com/gorilla/mux"
+	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
@@ -98,6 +100,16 @@ func pull(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+	for _, nodeID := range host.Nodes {
+		node, err := logic.GetNodeByID(nodeID)
+		if err != nil {
+			slog.Error("failed to get node:", "id", node.ID, "error", err)
+			continue
+		}
+		if node.FailedOverBy != uuid.Nil {
+			go logic.ResetFailedOverPeer(&node)
+		}
+	}
 	allNodes, err := logic.GetAllNodes()
 	if err != nil {
 		logger.Log(0, "failed to get nodes: ", hostID)
@@ -331,6 +343,24 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) {
 	// confirm host exists
 	currHost, err := logic.GetHost(hostid)
 	if err != nil {
+		if database.IsEmptyRecord(err) {
+			// check if there is any daemon nodes that needs to be deleted
+			node, err := logic.GetNodeByHostRef(hostid, network)
+			if err != nil {
+				slog.Error("couldn't get node for host", "hostid", hostid, "network", network, "error", err)
+				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+				return
+			}
+			if err = logic.DeleteNodeByID(&node); err != nil {
+				slog.Error("failed to force delete daemon node",
+					"nodeid", node.ID.String(), "hostid", hostid, "network", network, "error", err)
+				logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to force delete daemon node: "+err.Error()), "internal"))
+				return
+			}
+			logic.ReturnSuccessResponse(w, r, "force deleted daemon node successfully")
+			return
+		}
+
 		logger.Log(0, r.Header.Get("user"), "failed to find host:", err.Error())
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
@@ -461,6 +491,27 @@ func authenticateHost(response http.ResponseWriter, request *http.Request) {
 		logic.ReturnErrorResponse(response, request, errorResponse)
 		return
 	}
+
+	// Create EMQX creds and ACLs if not found
+	if servercfg.GetBrokerType() == servercfg.EmqxBrokerType {
+		if err := mq.CreateEmqxUser(host.ID.String(), host.HostPass, false); err != nil {
+			slog.Error("failed to create host credentials for EMQX: ", err.Error())
+		} else {
+			if err := mq.CreateHostACL(host.ID.String(), servercfg.GetServerInfo().Server); err != nil {
+				slog.Error("failed to add host ACL rules to EMQX: ", err.Error())
+			}
+			for _, nodeID := range host.Nodes {
+				if node, err := logic.GetNodeByID(nodeID); err == nil {
+					if err = mq.AppendNodeUpdateACL(host.ID.String(), node.Network, node.ID.String(), servercfg.GetServer()); err != nil {
+						slog.Error("failed to add ACLs for EMQX node", "error", err)
+					}
+				} else {
+					slog.Error("failed to get node", "nodeid", nodeID, "error", err)
+				}
+			}
+		}
+	}
+
 	response.WriteHeader(http.StatusOK)
 	response.Header().Set("Content-Type", "application/json")
 	response.Write(successJSONResponse)
@@ -495,39 +546,33 @@ func signalPeer(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 		return
 	}
-	if signal.ToHostPubKey == "" || signal.TurnRelayEndpoint == "" {
+	if signal.ToHostPubKey == "" || (!servercfg.IsPro && signal.TurnRelayEndpoint == "") {
 		msg := "insufficient data to signal peer"
 		logger.Log(0, r.Header.Get("user"), msg)
 		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New(msg), "badrequest"))
 		return
 	}
-	hosts, err := logic.GetAllHosts()
+	signal.IsPro = servercfg.IsPro
+	var peerHost *models.Host
+	if signal.ToHostID == "" {
+		peerHost, err = logic.GetHostByPubKey(signal.ToHostPubKey)
+	} else {
+		peerHost, err = logic.GetHost(signal.ToHostID)
+	}
 	if err != nil {
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to signal, peer not found"), "badrequest"))
 		return
 	}
-	// push the signal to host through mq
-	found := false
-	for _, hostI := range hosts {
-		if hostI.PublicKey.String() == signal.ToHostPubKey {
-			// found host publish message and break
-			found = true
-			err = mq.HostUpdate(&models.HostUpdate{
-				Action: models.SignalHost,
-				Host:   hostI,
-				Signal: signal,
-			})
-			if err != nil {
-				logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to publish signal to peer: "+err.Error()), "badrequest"))
-				return
-			}
-			break
-		}
-	}
-	if !found {
-		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to signal, peer not found"), "badrequest"))
+	err = mq.HostUpdate(&models.HostUpdate{
+		Action: models.SignalHost,
+		Host:   *peerHost,
+		Signal: signal,
+	})
+	if err != nil {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to publish signal to peer: "+err.Error()), "badrequest"))
 		return
 	}
+
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(signal)
 }

controllers/migrate.go

@@ -218,7 +218,5 @@ func convertLegacyNode(legacy models.LegacyNode, hostID uuid.UUID) models.Node {
 	node.IngressGatewayRange6 = legacy.IngressGatewayRange6
 	node.DefaultACL = legacy.DefaultACL
 	node.OwnerID = legacy.OwnerID
-	node.FailoverNode, _ = uuid.Parse(legacy.FailoverNode)
-	node.Failover = models.ParseBool(legacy.Failover)
 	return node
 }

controllers/node.go

@@ -341,7 +341,6 @@ func getAllNodes(w http.ResponseWriter, r *http.Request) {
 func getNode(w http.ResponseWriter, r *http.Request) {
 	// set header.
 	w.Header().Set("Content-Type", "application/json")
-	nodeRequest := r.Header.Get("requestfrom") == "node"
 
 	var params = mux.Vars(r)
 	nodeid := params["nodeid"]
@@ -386,12 +385,6 @@ func getNode(w http.ResponseWriter, r *http.Request) {
 		PeerIDs:      hostPeerUpdate.PeerIDs,
 	}
 
-	if servercfg.IsPro && nodeRequest {
-		if err = logic.EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
-			logger.Log(1, "failed to reset failover list during node config pull", node.ID.String(), node.Network)
-		}
-	}
-
 	logger.Log(2, r.Header.Get("user"), "fetched node", params["nodeid"])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(response)
@@ -524,12 +517,6 @@ func createIngressGateway(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if servercfg.IsPro && request.Failover {
-		if err = logic.EnterpriseResetFailoverFunc(node.Network); err != nil {
-			logger.Log(1, "failed to reset failover list during failover create", node.ID.String(), node.Network)
-		}
-	}
-
 	apiNode := node.ConvertToAPINode()
 	logger.Log(1, r.Header.Get("user"), "created ingress gateway on node", nodeid, "on network", netid)
 	w.WriteHeader(http.StatusOK)
@@ -562,7 +549,7 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request"))
 		return
 	}
-	node, wasFailover, removedClients, err := logic.DeleteIngressGateway(nodeid)
+	node, removedClients, err := logic.DeleteIngressGateway(nodeid)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
 			fmt.Sprintf("failed to delete ingress gateway on node [%s] on network [%s]: %v",
@@ -572,11 +559,6 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if servercfg.IsPro {
-		if wasFailover {
-			if err = logic.EnterpriseResetFailoverFunc(node.Network); err != nil {
-				logger.Log(1, "failed to reset failover list during failover create", node.ID.String(), node.Network)
-			}
-		}
 		go func() {
 			users, err := logic.GetUsersDB()
 			if err == nil {
@@ -662,11 +644,6 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 	}
 	ifaceDelta := logic.IfaceDelta(&currentNode, newNode)
 	aclUpdate := currentNode.DefaultACL != newNode.DefaultACL
-	if ifaceDelta && servercfg.IsPro {
-		if err = logic.EnterpriseResetAllPeersFailovers(currentNode.ID, currentNode.Network); err != nil {
-			logger.Log(0, "failed to reset failover lists during node update for node", currentNode.ID.String(), currentNode.Network)
-		}
-	}
 
 	err = logic.UpdateNode(&currentNode, newNode)
 	if err != nil {

controllers/node_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gravitl/netmaker/logic/acls"
 	"github.com/gravitl/netmaker/logic/acls/nodeacls"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 	"github.com/stretchr/testify/assert"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -217,7 +218,9 @@ func TestNodeACLs(t *testing.T) {
 }
 
 func deleteAllNodes() {
-	logic.ClearNodeCache()
+	if servercfg.CacheEnabled() {
+		logic.ClearNodeCache()
+	}
 	database.DeleteAllRecords(database.NODES_TABLE_NAME)
 }
 

controllers/server.go

@@ -4,8 +4,10 @@ import (
 	"encoding/json"
 	"net/http"
 	"strings"
+	"syscall"
 
 	"github.com/gorilla/mux"
+	"golang.org/x/exp/slog"
 
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic"
@@ -18,16 +20,26 @@ func serverHandlers(r *mux.Router) {
 	// r.HandleFunc("/api/server/addnetwork/{network}", securityCheckServer(true, http.HandlerFunc(addNetwork))).Methods(http.MethodPost)
 	r.HandleFunc(
 		"/api/server/health",
-		http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		func(resp http.ResponseWriter, req *http.Request) {
 			resp.WriteHeader(http.StatusOK)
 			resp.Write([]byte("Server is up and running!!"))
-		}),
-	)
+		},
+	).Methods(http.MethodGet)
+	r.HandleFunc(
+		"/api/server/shutdown",
+		func(w http.ResponseWriter, _ *http.Request) {
+			msg := "received api call to shutdown server, sending interruption..."
+			slog.Warn(msg)
+			_, _ = w.Write([]byte(msg))
+			w.WriteHeader(http.StatusOK)
+			_ = syscall.Kill(syscall.Getpid(), syscall.SIGINT)
+		},
+	).Methods(http.MethodPost)
 	r.HandleFunc("/api/server/getconfig", allowUsers(http.HandlerFunc(getConfig))).
 		Methods(http.MethodGet)
 	r.HandleFunc("/api/server/getserverinfo", Authorize(true, false, "node", http.HandlerFunc(getServerInfo))).
 		Methods(http.MethodGet)
-	r.HandleFunc("/api/server/status", http.HandlerFunc(getStatus)).Methods(http.MethodGet)
+	r.HandleFunc("/api/server/status", getStatus).Methods(http.MethodGet)
 	r.HandleFunc("/api/server/usage", Authorize(true, false, "user", http.HandlerFunc(getUsage))).
 		Methods(http.MethodGet)
 }

controllers/user.go

@@ -350,6 +350,10 @@ func createUser(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "forbidden"))
 		return
 	}
+	if !servercfg.IsPro && !user.IsAdmin {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "forbidden"))
+		return
+	}
 
 	err = logic.CreateUser(&user)
 	if err != nil {
@@ -502,14 +506,14 @@ func deleteUser(w http.ResponseWriter, r *http.Request) {
 	if user.IsSuperAdmin {
 		slog.Error(
 			"failed to delete user: ", "user", username, "error", "superadmin cannot be deleted")
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("superadmin cannot be deleted"), "internal"))
 		return
 	}
 	if !caller.IsSuperAdmin {
 		if caller.IsAdmin && user.IsAdmin {
 			slog.Error(
-				"failed to delete user: ", "user", username, "error", "admin cannot delete another admin user")
-			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+				"failed to delete user: ", "user", username, "error", "admin cannot delete another admin user, including oneself")
+			logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("admin cannot delete another admin user, including oneself"), "internal"))
 			return
 		}
 	}

go.mod

@@ -4,22 +4,22 @@ go 1.19
 
 require (
 	github.com/eclipse/paho.mqtt.golang v1.4.3
-	github.com/go-playground/validator/v10 v10.15.5
+	github.com/go-playground/validator/v10 v10.16.0
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/uuid v1.4.0
-	github.com/gorilla/handlers v1.5.1
-	github.com/gorilla/mux v1.8.0
+	github.com/gorilla/handlers v1.5.2
+	github.com/gorilla/mux v1.8.1
 	github.com/lib/pq v1.10.9
-	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/mattn/go-sqlite3 v1.14.18
 	github.com/rqlite/gorqlite v0.0.0-20210514125552-08ff1e76b22f
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stretchr/testify v1.8.4
 	github.com/txn2/txeh v1.5.5
-	golang.org/x/crypto v0.14.0
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/oauth2 v0.13.0
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/crypto v0.16.0
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/oauth2 v0.15.0
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1
@@ -32,18 +32,18 @@ require (
 )
 
 require (
-	github.com/coreos/go-oidc/v3 v3.7.0
-	github.com/gorilla/websocket v1.5.0
+	github.com/coreos/go-oidc/v3 v3.8.0
+	github.com/gorilla/websocket v1.5.1
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
 )
 
 require (
 	github.com/devilcove/httpclient v0.6.0
-	github.com/go-jose/go-jose/v3 v3.0.0
+	github.com/go-jose/go-jose/v3 v3.0.1
 	github.com/guumaster/tablewriter v0.0.10
 	github.com/matryer/is v1.4.1
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 )
 
 require (

go.sum

@@ -7,10 +7,10 @@ filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5E
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/c-robinson/iplib v1.0.7 h1:Dh9AINAlkc+NsNzZuFiVs+pi3AjN+0B7mu01KHdJKHU=
 github.com/c-robinson/iplib v1.0.7/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo=
-github.com/coreos/go-oidc/v3 v3.7.0 h1:FTdj0uexT4diYIPlF4yoFVI5MRO1r5+SEcIpEw9vC0o=
-github.com/coreos/go-oidc/v3 v3.7.0/go.mod h1:yQzSCqBnK3e6Fs5l+f5i0F8Kwf0zpH9bPEsbY00KanM=
+github.com/coreos/go-oidc/v3 v3.8.0 h1:s3e30r6VEl3/M7DTSCEuImmrfu1/1WBgA0cXkdzkrAY=
+github.com/coreos/go-oidc/v3 v3.8.0/go.mod h1:yQzSCqBnK3e6Fs5l+f5i0F8Kwf0zpH9bPEsbY00KanM=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -18,20 +18,19 @@ github.com/devilcove/httpclient v0.6.0 h1:M5YAfHeNbu+0QxCiOCo/fKN+Hf0BtF/6aovu3N
 github.com/devilcove/httpclient v0.6.0/go.mod h1:ctrAO2gRgTT+GxtRdWBp2SMQ+vacuxXlbhmlM4oWhs8=
 github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQNCyp70xik=
 github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE=
-github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
-github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
-github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
+github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
-github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
+github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -43,12 +42,12 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
 github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
 github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
-github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
+github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
+github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
 github.com/guumaster/tablewriter v0.0.10 h1:A0HD94yMdt4usgxBjoEceNeE0XMJ027euoHAzsPqBQs=
 github.com/guumaster/tablewriter v0.0.10/go.mod h1:p4FRFhyfo0UD9ZLmMRbbJooTUsxo6b80qZTERVDWrH8=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
@@ -65,8 +64,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.18 h1:JL0eqdCOq6DJVNPSvArO/bIV9/P7fbGrV00LZHc+5aI=
+github.com/mattn/go-sqlite3 v1.14.18/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -83,8 +82,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -105,8 +104,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY=
+golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -114,10 +113,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
+golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
@@ -128,16 +127,16 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

k8s/client/netclient-daemonset.yaml

@@ -16,7 +16,7 @@ spec:
       hostNetwork: true
       containers:
       - name: netclient
-        image: gravitl/netclient:v0.21.2
+        image: gravitl/netclient:v0.21.3
         env:
         - name: TOKEN
           value: "TOKEN_VALUE"

k8s/client/netclient.yaml

@@ -28,7 +28,7 @@ spec:
       #           - "<node label value>"
       containers:
       - name: netclient
-        image: gravitl/netclient:v0.21.2
+        image: gravitl/netclient:v0.21.3
         env:
         - name: TOKEN
           value: "TOKEN_VALUE"

k8s/server/netmaker-ui.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: netmaker-ui
-        image: gravitl/netmaker-ui:v0.21.2
+        image: gravitl/netmaker-ui:v0.21.3
         ports:
         - containerPort: 443
         env:

logic/acls/common.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/gravitl/netmaker/database"
+	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
 )
 
@@ -128,8 +129,10 @@ func (aclContainer ACLContainer) Get(containerID ContainerID) (ACLContainer, err
 func fetchACLContainer(containerID ContainerID) (ACLContainer, error) {
 	aclMutex.RLock()
 	defer aclMutex.RUnlock()
-	if aclContainer, ok := fetchAclContainerFromCache(containerID); ok {
-		return aclContainer, nil
+	if servercfg.CacheEnabled() {
+		if aclContainer, ok := fetchAclContainerFromCache(containerID); ok {
+			return aclContainer, nil
+		}
 	}
 	aclJson, err := fetchACLContainerJson(ContainerID(containerID))
 	if err != nil {
@@ -139,7 +142,9 @@ func fetchACLContainer(containerID ContainerID) (ACLContainer, error) {
 	if err := json.Unmarshal([]byte(aclJson), &currentNetworkACL); err != nil {
 		return nil, err
 	}
-	storeAclContainerInCache(containerID, currentNetworkACL)
+	if servercfg.CacheEnabled() {
+		storeAclContainerInCache(containerID, currentNetworkACL)
+	}
 	return currentNetworkACL, nil
 }
 
@@ -176,7 +181,9 @@ func upsertACLContainer(containerID ContainerID, aclContainer ACLContainer) (ACL
 	if err != nil {
 		return aclContainer, err
 	}
-	storeAclContainerInCache(containerID, aclContainer)
+	if servercfg.CacheEnabled() {
+		storeAclContainerInCache(containerID, aclContainer)
+	}
 	return aclContainer, nil
 }
 

logic/acls/nodeacls/modify.go

@@ -3,6 +3,7 @@ package nodeacls
 import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic/acls"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 // CreateNodeACL - inserts or updates a node ACL on given network and adds to state
@@ -87,6 +88,8 @@ func DeleteACLContainer(network NetworkID) error {
 	if err != nil {
 		return err
 	}
-	acls.DeleteAclFromCache(acls.ContainerID(network))
+	if servercfg.CacheEnabled() {
+		acls.DeleteAclFromCache(acls.ContainerID(network))
+	}
 	return nil
 }

logic/auth.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/go-playground/validator/v10"
 	"golang.org/x/crypto/bcrypt"
+	"golang.org/x/exp/slog"
 
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
@@ -165,11 +166,19 @@ func VerifyAuthRequest(authRequest models.UserAuthParams) (string, error) {
 	}
 
 	// Create a new JWT for the node
-	tokenString, _ := CreateUserJWT(authRequest.UserName, result.IsSuperAdmin, result.IsAdmin)
+	tokenString, err := CreateUserJWT(authRequest.UserName, result.IsSuperAdmin, result.IsAdmin)
+	if err != nil {
+		slog.Error("error creating jwt", "error", err)
+		return "", err
+	}
 
 	// update last login time
 	result.LastLoginTime = time.Now()
-	UpsertUser(result)
+	err = UpsertUser(result)
+	if err != nil {
+		slog.Error("error upserting user", "error", err)
+		return "", err
+	}
 
 	return tokenString, nil
 }
@@ -178,9 +187,11 @@ func VerifyAuthRequest(authRequest models.UserAuthParams) (string, error) {
 func UpsertUser(user models.User) error {
 	data, err := json.Marshal(&user)
 	if err != nil {
+		slog.Error("error marshalling user", "user", user.UserName, "error", err.Error())
 		return err
 	}
 	if err = database.Insert(user.UserName, string(data), database.USERS_TABLE_NAME); err != nil {
+		slog.Error("error inserting user", "user", user.UserName, "error", err.Error())
 		return err
 	}
 	return nil

logic/errors.go

@@ -44,6 +44,17 @@ func ReturnSuccessResponse(response http.ResponseWriter, request *http.Request,
 	json.NewEncoder(response).Encode(httpResponse)
 }
 
+// ReturnSuccessResponseWithJson - processes message and adds header
+func ReturnSuccessResponseWithJson(response http.ResponseWriter, request *http.Request, res interface{}, message string) {
+	var httpResponse models.SuccessResponse
+	httpResponse.Code = http.StatusOK
+	httpResponse.Response = res
+	httpResponse.Message = message
+	response.Header().Set("Content-Type", "application/json")
+	response.WriteHeader(http.StatusOK)
+	json.NewEncoder(response).Encode(httpResponse)
+}
+
 // ReturnErrorResponse - processes error and adds header
 func ReturnErrorResponse(response http.ResponseWriter, request *http.Request, errorMessage models.ErrorResponse) {
 	httpResponse := &models.ErrorResponse{Code: errorMessage.Code, Message: errorMessage.Message}

logic/extpeers.go

@@ -3,12 +3,15 @@ package logic
 import (
 	"encoding/json"
 	"fmt"
+	"net"
 	"reflect"
 	"sync"
 	"time"
 
 	"github.com/gravitl/netmaker/database"
+	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -78,21 +81,25 @@ func DeleteExtClient(network string, clientid string) error {
 	if err != nil {
 		return err
 	}
-	deleteExtClientFromCache(key)
+	if servercfg.CacheEnabled() {
+		deleteExtClientFromCache(key)
+	}
 	return nil
 }
 
 // GetNetworkExtClients - gets the ext clients of given network
 func GetNetworkExtClients(network string) ([]models.ExtClient, error) {
 	var extclients []models.ExtClient
-	allextclients := getAllExtClientsFromCache()
-	if len(allextclients) != 0 {
-		for _, extclient := range allextclients {
-			if extclient.Network == network {
-				extclients = append(extclients, extclient)
+	if servercfg.CacheEnabled() {
+		allextclients := getAllExtClientsFromCache()
+		if len(allextclients) != 0 {
+			for _, extclient := range allextclients {
+				if extclient.Network == network {
+					extclients = append(extclients, extclient)
+				}
 			}
+			return extclients, nil
 		}
-		return extclients, nil
 	}
 	records, err := database.FetchRecords(database.EXT_CLIENT_TABLE_NAME)
 	if err != nil {
@@ -109,7 +116,9 @@ func GetNetworkExtClients(network string) ([]models.ExtClient, error) {
 		}
 		key, err := GetRecordKey(extclient.ClientID, extclient.Network)
 		if err == nil {
-			storeExtClientInCache(key, extclient)
+			if servercfg.CacheEnabled() {
+				storeExtClientInCache(key, extclient)
+			}
 		}
 		if extclient.Network == network {
 			extclients = append(extclients, extclient)
@@ -125,15 +134,19 @@ func GetExtClient(clientid string, network string) (models.ExtClient, error) {
 	if err != nil {
 		return extclient, err
 	}
-	if extclient, ok := getExtClientFromCache(key); ok {
-		return extclient, nil
+	if servercfg.CacheEnabled() {
+		if extclient, ok := getExtClientFromCache(key); ok {
+			return extclient, nil
+		}
 	}
 	data, err := database.FetchRecord(database.EXT_CLIENT_TABLE_NAME, key)
 	if err != nil {
 		return extclient, err
 	}
 	err = json.Unmarshal([]byte(data), &extclient)
-	storeExtClientInCache(key, extclient)
+	if servercfg.CacheEnabled() {
+		storeExtClientInCache(key, extclient)
+	}
 	return extclient, err
 }
 
@@ -184,6 +197,9 @@ func CreateExtClient(extclient *models.ExtClient) error {
 	} else if len(extclient.PrivateKey) == 0 && len(extclient.PublicKey) > 0 {
 		extclient.PrivateKey = "[ENTER PRIVATE KEY]"
 	}
+	if extclient.ExtraAllowedIPs == nil {
+		extclient.ExtraAllowedIPs = []string{}
+	}
 
 	parentNetwork, err := GetNetwork(extclient.Network)
 	if err != nil {
@@ -230,7 +246,9 @@ func SaveExtClient(extclient *models.ExtClient) error {
 	if err = database.Insert(key, string(data), database.EXT_CLIENT_TABLE_NAME); err != nil {
 		return err
 	}
-	storeExtClientInCache(key, *extclient)
+	if servercfg.CacheEnabled() {
+		storeExtClientInCache(key, *extclient)
+	}
 	return SetNetworkNodesLastModified(extclient.Network)
 }
 
@@ -247,9 +265,7 @@ func UpdateExtClient(old *models.ExtClient, update *models.CustomExtClient) mode
 	if update.Enabled != old.Enabled {
 		new.Enabled = update.Enabled
 	}
-	if update.ExtraAllowedIPs != nil && StringDifference(old.ExtraAllowedIPs, update.ExtraAllowedIPs) != nil {
-		new.ExtraAllowedIPs = update.ExtraAllowedIPs
-	}
+	new.ExtraAllowedIPs = update.ExtraAllowedIPs
 	if update.DeniedACLs != nil && !reflect.DeepEqual(old.DeniedACLs, update.DeniedACLs) {
 		new.DeniedACLs = update.DeniedACLs
 	}
@@ -318,3 +334,107 @@ func ToggleExtClientConnectivity(client *models.ExtClient, enable bool) (models.
 
 	return newClient, nil
 }
+
+func getExtPeers(node, peer *models.Node) ([]wgtypes.PeerConfig, []models.IDandAddr, []models.EgressNetworkRoutes, error) {
+	var peers []wgtypes.PeerConfig
+	var idsAndAddr []models.IDandAddr
+	var egressRoutes []models.EgressNetworkRoutes
+	extPeers, err := GetNetworkExtClients(node.Network)
+	if err != nil {
+		return peers, idsAndAddr, egressRoutes, err
+	}
+	host, err := GetHost(node.HostID.String())
+	if err != nil {
+		return peers, idsAndAddr, egressRoutes, err
+	}
+	for _, extPeer := range extPeers {
+		extPeer := extPeer
+		if !IsClientNodeAllowed(&extPeer, peer.ID.String()) {
+			continue
+		}
+		pubkey, err := wgtypes.ParseKey(extPeer.PublicKey)
+		if err != nil {
+			logger.Log(1, "error parsing ext pub key:", err.Error())
+			continue
+		}
+
+		if host.PublicKey.String() == extPeer.PublicKey ||
+			extPeer.IngressGatewayID != node.ID.String() || !extPeer.Enabled {
+			continue
+		}
+
+		var allowedips []net.IPNet
+		var peer wgtypes.PeerConfig
+		if extPeer.Address != "" {
+			var peeraddr = net.IPNet{
+				IP:   net.ParseIP(extPeer.Address),
+				Mask: net.CIDRMask(32, 32),
+			}
+			if peeraddr.IP != nil && peeraddr.Mask != nil {
+				allowedips = append(allowedips, peeraddr)
+			}
+		}
+
+		if extPeer.Address6 != "" {
+			var addr6 = net.IPNet{
+				IP:   net.ParseIP(extPeer.Address6),
+				Mask: net.CIDRMask(128, 128),
+			}
+			if addr6.IP != nil && addr6.Mask != nil {
+				allowedips = append(allowedips, addr6)
+			}
+		}
+		for _, extraAllowedIP := range extPeer.ExtraAllowedIPs {
+			_, cidr, err := net.ParseCIDR(extraAllowedIP)
+			if err == nil {
+				allowedips = append(allowedips, *cidr)
+			}
+		}
+		egressRoutes = append(egressRoutes, getExtPeerEgressRoute(extPeer)...)
+		primaryAddr := extPeer.Address
+		if primaryAddr == "" {
+			primaryAddr = extPeer.Address6
+		}
+		peer = wgtypes.PeerConfig{
+			PublicKey:         pubkey,
+			ReplaceAllowedIPs: true,
+			AllowedIPs:        allowedips,
+		}
+		peers = append(peers, peer)
+		idsAndAddr = append(idsAndAddr, models.IDandAddr{
+			ID:          peer.PublicKey.String(),
+			Name:        extPeer.ClientID,
+			Address:     primaryAddr,
+			IsExtClient: true,
+		})
+	}
+	return peers, idsAndAddr, egressRoutes, nil
+
+}
+
+func getExtPeerEgressRoute(extPeer models.ExtClient) (egressRoutes []models.EgressNetworkRoutes) {
+	if extPeer.Address != "" {
+		egressRoutes = append(egressRoutes, models.EgressNetworkRoutes{
+			NodeAddr:     extPeer.AddressIPNet4(),
+			EgressRanges: extPeer.ExtraAllowedIPs,
+		})
+	}
+	if extPeer.Address6 != "" {
+		egressRoutes = append(egressRoutes, models.EgressNetworkRoutes{
+			NodeAddr:     extPeer.AddressIPNet6(),
+			EgressRanges: extPeer.ExtraAllowedIPs,
+		})
+	}
+	return
+}
+
+func getExtpeersExtraRoutes(network string) (egressRoutes []models.EgressNetworkRoutes) {
+	extPeers, err := GetNetworkExtClients(network)
+	if err != nil {
+		return
+	}
+	for _, extPeer := range extPeers {
+		egressRoutes = append(egressRoutes, getExtPeerEgressRoute(extPeer)...)
+	}
+	return
+}

logic/gateway.go

@@ -8,7 +8,6 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
-	"github.com/gravitl/netmaker/servercfg"
 )
 
 // GetInternetGateways - gets all the nodes that are internet gateways
@@ -168,9 +167,6 @@ func CreateIngressGateway(netid string, nodeid string, ingress models.IngressReq
 	node.IngressGatewayRange6 = network.AddressRange6
 	node.IngressDNS = ingress.ExtclientDNS
 	node.SetLastModified()
-	if ingress.Failover && servercfg.IsPro {
-		node.Failover = true
-	}
 	err = UpsertNode(&node)
 	if err != nil {
 		return models.Node{}, err
@@ -199,35 +195,33 @@ func GetIngressGwUsers(node models.Node) (models.IngressGwUsers, error) {
 }
 
 // DeleteIngressGateway - deletes an ingress gateway
-func DeleteIngressGateway(nodeid string) (models.Node, bool, []models.ExtClient, error) {
+func DeleteIngressGateway(nodeid string) (models.Node, []models.ExtClient, error) {
 	removedClients := []models.ExtClient{}
 	node, err := GetNodeByID(nodeid)
 	if err != nil {
-		return models.Node{}, false, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 	clients, err := GetExtClientsByID(nodeid, node.Network)
 	if err != nil && !database.IsEmptyRecord(err) {
-		return models.Node{}, false, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 
 	removedClients = clients
 
 	// delete ext clients belonging to ingress gateway
 	if err = DeleteGatewayExtClients(node.ID.String(), node.Network); err != nil {
-		return models.Node{}, false, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 	logger.Log(3, "deleting ingress gateway")
-	wasFailover := node.Failover
 	node.LastModified = time.Now()
 	node.IsIngressGateway = false
 	node.IngressGatewayRange = ""
-	node.Failover = false
 	err = UpsertNode(&node)
 	if err != nil {
-		return models.Node{}, wasFailover, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 	err = SetNetworkNodesLastModified(node.Network)
-	return node, wasFailover, removedClients, err
+	return node, removedClients, err
 }
 
 // DeleteGatewayExtClients - deletes ext clients based on gateway (mac) of ingress node and network

logic/hosts.go

@@ -81,16 +81,21 @@ const (
 
 // GetAllHosts - returns all hosts in flat list or error
 func GetAllHosts() ([]models.Host, error) {
-	currHosts := getHostsFromCache()
-	if len(currHosts) != 0 {
-		return currHosts, nil
+	var currHosts []models.Host
+	if servercfg.CacheEnabled() {
+		currHosts := getHostsFromCache()
+		if len(currHosts) != 0 {
+			return currHosts, nil
+		}
 	}
 	records, err := database.FetchRecords(database.HOSTS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return nil, err
 	}
 	currHostsMap := make(map[string]models.Host)
-	defer loadHostsIntoCache(currHostsMap)
+	if servercfg.CacheEnabled() {
+		defer loadHostsIntoCache(currHostsMap)
+	}
 	for k := range records {
 		var h models.Host
 		err = json.Unmarshal([]byte(records[k]), &h)
@@ -116,16 +121,20 @@ func GetAllHostsAPI(hosts []models.Host) []models.ApiHost {
 
 // GetHostsMap - gets all the current hosts on machine in a map
 func GetHostsMap() (map[string]models.Host, error) {
-	hostsMap := getHostsMapFromCache()
-	if len(hostsMap) != 0 {
-		return hostsMap, nil
+	if servercfg.CacheEnabled() {
+		hostsMap := getHostsMapFromCache()
+		if len(hostsMap) != 0 {
+			return hostsMap, nil
+		}
 	}
 	records, err := database.FetchRecords(database.HOSTS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return nil, err
 	}
 	currHostMap := make(map[string]models.Host)
-	defer loadHostsIntoCache(currHostMap)
+	if servercfg.CacheEnabled() {
+		defer loadHostsIntoCache(currHostMap)
+	}
 	for k := range records {
 		var h models.Host
 		err = json.Unmarshal([]byte(records[k]), &h)
@@ -140,8 +149,10 @@ func GetHostsMap() (map[string]models.Host, error) {
 
 // GetHost - gets a host from db given id
 func GetHost(hostid string) (*models.Host, error) {
-	if host, ok := getHostFromCache(hostid); ok {
-		return &host, nil
+	if servercfg.CacheEnabled() {
+		if host, ok := getHostFromCache(hostid); ok {
+			return &host, nil
+		}
 	}
 	record, err := database.FetchRecord(database.HOSTS_TABLE_NAME, hostid)
 	if err != nil {
@@ -152,10 +163,27 @@ func GetHost(hostid string) (*models.Host, error) {
 	if err = json.Unmarshal([]byte(record), &h); err != nil {
 		return nil, err
 	}
-	storeHostInCache(h)
+	if servercfg.CacheEnabled() {
+		storeHostInCache(h)
+	}
+
 	return &h, nil
 }
 
+// GetHostByPubKey - gets a host from db given pubkey
+func GetHostByPubKey(hostPubKey string) (*models.Host, error) {
+	hosts, err := GetAllHosts()
+	if err != nil {
+		return nil, err
+	}
+	for _, host := range hosts {
+		if host.PublicKey.String() == hostPubKey {
+			return &host, nil
+		}
+	}
+	return nil, errors.New("host not found")
+}
+
 // CreateHost - creates a host if not exist
 func CreateHost(h *models.Host) error {
 	hosts, hErr := GetAllHosts()
@@ -265,7 +293,10 @@ func UpsertHost(h *models.Host) error {
 	if err != nil {
 		return err
 	}
-	storeHostInCache(*h)
+	if servercfg.CacheEnabled() {
+		storeHostInCache(*h)
+	}
+
 	return nil
 }
 
@@ -289,8 +320,10 @@ func RemoveHost(h *models.Host, forceDelete bool) error {
 	if err != nil {
 		return err
 	}
+	if servercfg.CacheEnabled() {
+		deleteHostFromCache(h.ID.String())
+	}
 
-	deleteHostFromCache(h.ID.String())
 	return nil
 }
 
@@ -304,7 +337,9 @@ func RemoveHostByID(hostID string) error {
 	if err != nil {
 		return err
 	}
-	deleteHostFromCache(hostID)
+	if servercfg.CacheEnabled() {
+		deleteHostFromCache(hostID)
+	}
 	return nil
 }
 

logic/jwts.go

@@ -108,7 +108,7 @@ func VerifyUserToken(tokenString string) (username string, issuperadmin, isadmin
 		}
 
 		if user.UserName != "" {
-			return claims.UserName, claims.IsSuperAdmin, claims.IsAdmin, nil
+			return user.UserName, user.IsSuperAdmin, user.IsAdmin, nil
 		}
 		err = errors.New("user does not exist")
 	}

logic/nodes.go

@@ -119,7 +119,9 @@ func UpdateNodeCheckin(node *models.Node) error {
 	if err != nil {
 		return err
 	}
-	storeNodeInCache(*node)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(*node)
+	}
 	return nil
 }
 
@@ -134,7 +136,9 @@ func UpsertNode(newNode *models.Node) error {
 	if err != nil {
 		return err
 	}
-	storeNodeInCache(*newNode)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(*newNode)
+	}
 	return nil
 }
 
@@ -171,7 +175,9 @@ func UpdateNode(currentNode *models.Node, newNode *models.Node) error {
 			if err != nil {
 				return err
 			}
-			storeNodeInCache(*newNode)
+			if servercfg.CacheEnabled() {
+				storeNodeInCache(*newNode)
+			}
 			return nil
 		}
 	}
@@ -205,6 +211,9 @@ func DeleteNode(node *models.Node, purge bool) error {
 			UpsertNode(&relayNode)
 		}
 	}
+	if node.FailedOverBy != uuid.Nil {
+		ResetFailedOverPeer(node)
+	}
 	if node.IsRelay {
 		// unset all the relayed nodes
 		SetRelayedNodes(false, node.ID.String(), node.RelayedNodes)
@@ -233,11 +242,6 @@ func DeleteNode(node *models.Node, purge bool) error {
 	if err := DissasociateNodeFromHost(node, host); err != nil {
 		return err
 	}
-	if servercfg.IsPro {
-		if err := EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
-			logger.Log(0, "failed to reset failover lists during node delete for node", host.Name, node.Network)
-		}
-	}
 
 	return nil
 }
@@ -266,7 +270,9 @@ func DeleteNodeByID(node *models.Node) error {
 			return err
 		}
 	}
-	deleteNodeFromCache(node.ID.String())
+	if servercfg.CacheEnabled() {
+		deleteNodeFromCache(node.ID.String())
+	}
 	if servercfg.IsDNSMode() {
 		SetDNS()
 	}
@@ -309,29 +315,19 @@ func ValidateNode(node *models.Node, isUpdate bool) error {
 	return err
 }
 
-// IsFailoverPresent - checks if a node is marked as a failover in given network
-func IsFailoverPresent(network string) bool {
-	netNodes, err := GetNetworkNodes(network)
-	if err != nil {
-		return false
-	}
-	for i := range netNodes {
-		if netNodes[i].Failover {
-			return true
-		}
-	}
-	return false
-}
-
 // GetAllNodes - returns all nodes in the DB
 func GetAllNodes() ([]models.Node, error) {
 	var nodes []models.Node
-	nodes = getNodesFromCache()
-	if len(nodes) != 0 {
-		return nodes, nil
+	if servercfg.CacheEnabled() {
+		nodes = getNodesFromCache()
+		if len(nodes) != 0 {
+			return nodes, nil
+		}
 	}
 	nodesMap := make(map[string]models.Node)
-	defer loadNodesIntoCache(nodesMap)
+	if servercfg.CacheEnabled() {
+		defer loadNodesIntoCache(nodesMap)
+	}
 	collection, err := database.FetchRecords(database.NODES_TABLE_NAME)
 	if err != nil {
 		if database.IsEmptyRecord(err) {
@@ -385,6 +381,9 @@ func SetNodeDefaults(node *models.Node) {
 	if node.DefaultACL == "" {
 		node.DefaultACL = parentNetwork.DefaultACL
 	}
+	if node.FailOverPeers == nil {
+		node.FailOverPeers = make(map[string]struct{})
+	}
 
 	node.SetLastModified()
 	node.SetLastCheckIn()
@@ -402,8 +401,10 @@ func GetRecordKey(id string, network string) (string, error) {
 }
 
 func GetNodeByID(uuid string) (models.Node, error) {
-	if node, ok := getNodeFromCache(uuid); ok {
-		return node, nil
+	if servercfg.CacheEnabled() {
+		if node, ok := getNodeFromCache(uuid); ok {
+			return node, nil
+		}
 	}
 	var record, err = database.FetchRecord(database.NODES_TABLE_NAME, uuid)
 	if err != nil {
@@ -413,7 +414,9 @@ func GetNodeByID(uuid string) (models.Node, error) {
 	if err = json.Unmarshal([]byte(record), &node); err != nil {
 		return models.Node{}, err
 	}
-	storeNodeInCache(node)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(node)
+	}
 	return node, nil
 }
 
@@ -569,7 +572,9 @@ func createNode(node *models.Node) error {
 	if err != nil {
 		return err
 	}
-	storeNodeInCache(*node)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(*node)
+	}
 	_, err = nodeacls.CreateNodeACL(nodeacls.NetworkID(node.Network), nodeacls.NodeID(node.ID.String()), defaultACLVal)
 	if err != nil {
 		logger.Log(1, "failed to create node ACL for node,", node.ID.String(), "err:", err.Error())

logic/peers.go

@@ -15,6 +15,17 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
+var (
+	// ResetFailOver - function to reset failOvered peers on this node
+	ResetFailOver = func(failOverNode *models.Node) error {
+		return nil
+	}
+	// ResetFailedOverPeer - removes failed over node from network peers
+	ResetFailedOverPeer = func(failedOverNode *models.Node) error {
+		return nil
+	}
+)
+
 // GetPeerUpdateForHost - gets the consolidated peer update for the host from all networks
 func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.Node,
 	deletedNode *models.Node, deletedClients []models.ExtClient) (models.HostPeerUpdate, error) {
@@ -129,7 +140,12 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 					EgressRanges: peer.EgressGatewayRanges,
 				})
 			}
-			if (node.IsRelayed && node.RelayedBy != peer.ID.String()) || (peer.IsRelayed && peer.RelayedBy != node.ID.String()) {
+			if peer.IsIngressGateway {
+				hostPeerUpdate.EgressRoutes = append(hostPeerUpdate.EgressRoutes, getExtpeersExtraRoutes(peer.Network)...)
+			}
+			_, isFailOverPeer := node.FailOverPeers[peer.ID.String()]
+			if (node.IsRelayed && node.RelayedBy != peer.ID.String()) ||
+				(peer.IsRelayed && peer.RelayedBy != node.ID.String()) || isFailOverPeer {
 				// if node is relayed and peer is not the relay, set remove to true
 				if _, ok := peerIndexMap[peerHost.PublicKey.String()]; ok {
 					continue
@@ -194,9 +210,10 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 				nodePeer = hostPeerUpdate.Peers[peerIndexMap[peerHost.PublicKey.String()]]
 			}
 
-			if node.Network == network { // add to peers map for metrics
+			if node.Network == network && !peerConfig.Remove { // add to peers map for metrics
 				hostPeerUpdate.PeerIDs[peerHost.PublicKey.String()] = models.IDandAddr{
 					ID:         peer.ID.String(),
+					HostID:     peerHost.ID.String(),
 					Address:    peer.PrimaryAddress(),
 					Name:       peerHost.Name,
 					Network:    peer.Network,
@@ -207,9 +224,11 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 		}
 		var extPeers []wgtypes.PeerConfig
 		var extPeerIDAndAddrs []models.IDandAddr
+		var egressRoutes []models.EgressNetworkRoutes
 		if node.IsIngressGateway {
-			extPeers, extPeerIDAndAddrs, err = getExtPeers(&node, &node)
+			extPeers, extPeerIDAndAddrs, egressRoutes, err = getExtPeers(&node, &node)
 			if err == nil {
+				hostPeerUpdate.EgressRoutes = append(hostPeerUpdate.EgressRoutes, egressRoutes...)
 				hostPeerUpdate.Peers = append(hostPeerUpdate.Peers, extPeers...)
 				for _, extPeerIdAndAddr := range extPeerIDAndAddrs {
 					extPeerIdAndAddr := extPeerIdAndAddr
@@ -290,76 +309,6 @@ func GetPeerListenPort(host *models.Host) int {
 	return peerPort
 }
 
-func getExtPeers(node, peer *models.Node) ([]wgtypes.PeerConfig, []models.IDandAddr, error) {
-	var peers []wgtypes.PeerConfig
-	var idsAndAddr []models.IDandAddr
-	extPeers, err := GetNetworkExtClients(node.Network)
-	if err != nil {
-		return peers, idsAndAddr, err
-	}
-	host, err := GetHost(node.HostID.String())
-	if err != nil {
-		return peers, idsAndAddr, err
-	}
-	for _, extPeer := range extPeers {
-		extPeer := extPeer
-		if !IsClientNodeAllowed(&extPeer, peer.ID.String()) {
-			continue
-		}
-		pubkey, err := wgtypes.ParseKey(extPeer.PublicKey)
-		if err != nil {
-			logger.Log(1, "error parsing ext pub key:", err.Error())
-			continue
-		}
-
-		if host.PublicKey.String() == extPeer.PublicKey ||
-			extPeer.IngressGatewayID != node.ID.String() || !extPeer.Enabled {
-			continue
-		}
-
-		var allowedips []net.IPNet
-		var peer wgtypes.PeerConfig
-		if extPeer.Address != "" {
-			var peeraddr = net.IPNet{
-				IP:   net.ParseIP(extPeer.Address),
-				Mask: net.CIDRMask(32, 32),
-			}
-			if peeraddr.IP != nil && peeraddr.Mask != nil {
-				allowedips = append(allowedips, peeraddr)
-			}
-		}
-
-		if extPeer.Address6 != "" {
-			var addr6 = net.IPNet{
-				IP:   net.ParseIP(extPeer.Address6),
-				Mask: net.CIDRMask(128, 128),
-			}
-			if addr6.IP != nil && addr6.Mask != nil {
-				allowedips = append(allowedips, addr6)
-			}
-		}
-
-		primaryAddr := extPeer.Address
-		if primaryAddr == "" {
-			primaryAddr = extPeer.Address6
-		}
-		peer = wgtypes.PeerConfig{
-			PublicKey:         pubkey,
-			ReplaceAllowedIPs: true,
-			AllowedIPs:        allowedips,
-		}
-		peers = append(peers, peer)
-		idsAndAddr = append(idsAndAddr, models.IDandAddr{
-			ID:          peer.PublicKey.String(),
-			Name:        extPeer.ClientID,
-			Address:     primaryAddr,
-			IsExtClient: true,
-		})
-	}
-	return peers, idsAndAddr, nil
-
-}
-
 // GetAllowedIPs - calculates the wireguard allowedip field for a peer of a node based on the peer and node settings
 func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet {
 	var allowedips []net.IPNet
@@ -367,7 +316,7 @@ func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet
 
 	// handle ingress gateway peers
 	if peer.IsIngressGateway {
-		extPeers, _, err := getExtPeers(peer, node)
+		extPeers, _, _, err := getExtPeers(peer, node)
 		if err != nil {
 			logger.Log(2, "could not retrieve ext peers for ", peer.ID.String(), err.Error())
 		}
@@ -381,6 +330,31 @@ func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet
 	return allowedips
 }
 
+func GetFailOverPeerIps(peer, node *models.Node) []net.IPNet {
+	allowedips := []net.IPNet{}
+	for failOverpeerID := range node.FailOverPeers {
+		failOverpeer, err := GetNodeByID(failOverpeerID)
+		if err == nil && failOverpeer.FailedOverBy == peer.ID {
+			if failOverpeer.Address.IP != nil {
+				allowed := net.IPNet{
+					IP:   failOverpeer.Address.IP,
+					Mask: net.CIDRMask(32, 32),
+				}
+				allowedips = append(allowedips, allowed)
+			}
+			if failOverpeer.Address6.IP != nil {
+				allowed := net.IPNet{
+					IP:   failOverpeer.Address6.IP,
+					Mask: net.CIDRMask(128, 128),
+				}
+				allowedips = append(allowedips, allowed)
+			}
+
+		}
+	}
+	return allowedips
+}
+
 func GetEgressIPs(peer *models.Node) []net.IPNet {
 
 	peerHost, err := GetHost(peer.HostID.String())
@@ -444,6 +418,9 @@ func getNodeAllowedIPs(peer, node *models.Node) []net.IPNet {
 	if peer.IsRelay {
 		allowedips = append(allowedips, RelayedAllowedIPs(peer, node)...)
 	}
+	if peer.IsFailOver {
+		allowedips = append(allowedips, GetFailOverPeerIps(peer, node)...)
+	}
 	return allowedips
 }
 

logic/server.go

@@ -1,22 +1,8 @@
 package logic
 
-import (
-	"github.com/google/uuid"
-	"github.com/gravitl/netmaker/models"
-)
-
 // EnterpriseCheckFuncs - can be set to run functions for EE
 var EnterpriseCheckFuncs []func()
 
-// EnterpriseFailoverFunc - interface to control failover funcs
-var EnterpriseFailoverFunc func(node *models.Node) error
-
-// EnterpriseResetFailoverFunc - interface to control reset failover funcs
-var EnterpriseResetFailoverFunc func(network string) error
-
-// EnterpriseResetAllPeersFailovers - resets all nodes that are considering a node to be failover worthy (inclusive)
-var EnterpriseResetAllPeersFailovers func(nodeid uuid.UUID, network string) error
-
 // == Join, Checkin, and Leave for Server ==
 
 // KUBERNETES_LISTEN_PORT - starting port for Kubernetes in order to use NodePort range

logic/util.go

@@ -134,4 +134,18 @@ func RemoveStringSlice(slice []string, i int) []string {
 	return append(slice[:i], slice[i+1:]...)
 }
 
+// IsSlicesEqual tells whether a and b contain the same elements.
+// A nil argument is equivalent to an empty slice.
+func IsSlicesEqual(a, b []string) bool {
+	if len(a) != len(b) {
+		return false
+	}
+	for i, v := range a {
+		if v != b[i] {
+			return false
+		}
+	}
+	return true
+}
+
 // == private ==

main.go

@@ -28,8 +28,7 @@ import (
 	"golang.org/x/exp/slog"
 )
 
-
-var version = "v0.21.2"
+var version = "v0.21.3"
 
 // Start DB Connection and start API Request Handler
 func main() {

migrate/migrate.go

@@ -2,6 +2,7 @@ package migrate
 
 import (
 	"encoding/json"
+	"log"
 
 	"golang.org/x/exp/slog"
 
@@ -9,6 +10,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 // Run - runs all migrations
@@ -28,6 +30,26 @@ func assignSuperAdmin() {
 		return
 	}
 	createdSuperAdmin := false
+	owner := servercfg.GetOwnerEmail()
+	if owner != "" {
+		user, err := logic.GetUser(owner)
+		if err != nil {
+			log.Fatal("error getting user", "user", owner, "error", err.Error())
+		}
+		user.IsSuperAdmin = true
+		user.IsAdmin = false
+		err = logic.UpsertUser(*user)
+		if err != nil {
+			log.Fatal(
+				"error updating user to superadmin",
+				"user",
+				user.UserName,
+				"error",
+				err.Error(),
+			)
+		}
+		return
+	}
 	for _, u := range users {
 		if u.IsAdmin {
 			user, err := logic.GetUser(u.UserName)

models/api_host.go

@@ -26,10 +26,6 @@ type ApiHost struct {
 	MacAddress          string   `json:"macaddress"`
 	Nodes               []string `json:"nodes"`
 	IsDefault           bool     `json:"isdefault"             yaml:"isdefault"`
-	IsRelayed           bool     `json:"isrelayed"             yaml:"isrelayed"             bson:"isrelayed"`
-	RelayedBy           string   `json:"relayed_by"            yaml:"relayed_by"            bson:"relayed_by"`
-	IsRelay             bool     `json:"isrelay"               yaml:"isrelay"               bson:"isrelay"`
-	RelayedHosts        []string `json:"relay_hosts"           yaml:"relay_hosts"           bson:"relay_hosts"`
 	NatType             string   `json:"nat_type"              yaml:"nat_type"`
 	PersistentKeepalive int      `json:"persistentkeepalive"   yaml:"persistentkeepalive"`
 	AutoUpdate          bool     `json:"autoupdate"              yaml:"autoupdate"`

models/api_node.go

@@ -30,7 +30,6 @@ type ApiNode struct {
 	IsIngressGateway        bool     `json:"isingressgateway"`
 	EgressGatewayRanges     []string `json:"egressgatewayranges"`
 	EgressGatewayNatEnabled bool     `json:"egressgatewaynatenabled"`
-	FailoverNode            string   `json:"failovernode"`
 	DNSOn                   bool     `json:"dnson"`
 	IngressDns              string   `json:"ingressdns"`
 	Server                  string   `json:"server"`
@@ -38,8 +37,10 @@ type ApiNode struct {
 	Connected               bool     `json:"connected"`
 	PendingDelete           bool     `json:"pendingdelete"`
 	// == PRO ==
-	DefaultACL string `json:"defaultacl,omitempty" validate:"checkyesornoorunset"`
-	Failover   bool   `json:"failover"`
+	DefaultACL    string              `json:"defaultacl,omitempty" validate:"checkyesornoorunset"`
+	IsFailOver    bool                `json:"is_fail_over"`
+	FailOverPeers map[string]struct{} `json:"fail_over_peers" yaml:"fail_over_peers"`
+	FailedOverBy  uuid.UUID           `json:"failed_over_by" yaml:"failed_over_by"`
 }
 
 // ApiNode.ConvertToServerNode - converts an api node to a server node
@@ -56,7 +57,8 @@ func (a *ApiNode) ConvertToServerNode(currentNode *Node) *Node {
 	convertedNode.RelayedBy = a.RelayedBy
 	convertedNode.RelayedNodes = a.RelayedNodes
 	convertedNode.PendingDelete = a.PendingDelete
-	convertedNode.Failover = a.Failover
+	convertedNode.FailedOverBy = currentNode.FailedOverBy
+	convertedNode.FailOverPeers = currentNode.FailOverPeers
 	convertedNode.IsEgressGateway = a.IsEgressGateway
 	convertedNode.IsIngressGateway = a.IsIngressGateway
 	// prevents user from changing ranges, must delete and recreate
@@ -100,7 +102,6 @@ func (a *ApiNode) ConvertToServerNode(currentNode *Node) *Node {
 		convertedNode.Address6 = *addr6
 		convertedNode.Address6.IP = ip6
 	}
-	convertedNode.FailoverNode, _ = uuid.Parse(a.FailoverNode)
 	convertedNode.LastModified = time.Unix(a.LastModified, 0)
 	convertedNode.LastCheckIn = time.Unix(a.LastCheckIn, 0)
 	convertedNode.LastPeerUpdate = time.Unix(a.LastPeerUpdate, 0)
@@ -146,10 +147,6 @@ func (nm *Node) ConvertToAPINode() *ApiNode {
 	apiNode.IsIngressGateway = nm.IsIngressGateway
 	apiNode.EgressGatewayRanges = nm.EgressGatewayRanges
 	apiNode.EgressGatewayNatEnabled = nm.EgressGatewayNatEnabled
-	apiNode.FailoverNode = nm.FailoverNode.String()
-	if isUUIDSet(apiNode.FailoverNode) {
-		apiNode.FailoverNode = ""
-	}
 	apiNode.DNSOn = nm.DNSOn
 	apiNode.IngressDns = nm.IngressDNS
 	apiNode.Server = nm.Server
@@ -160,14 +157,12 @@ func (nm *Node) ConvertToAPINode() *ApiNode {
 	apiNode.Connected = nm.Connected
 	apiNode.PendingDelete = nm.PendingDelete
 	apiNode.DefaultACL = nm.DefaultACL
-	apiNode.Failover = nm.Failover
+	apiNode.IsFailOver = nm.IsFailOver
+	apiNode.FailOverPeers = nm.FailOverPeers
+	apiNode.FailedOverBy = nm.FailedOverBy
 	return &apiNode
 }
 
 func isEmptyAddr(addr string) bool {
 	return addr == "<nil>" || addr == ":0"
 }
-
-func isUUIDSet(uuid string) bool {
-	return uuid != "00000000-0000-0000-0000-000000000000"
-}

models/host.go

@@ -126,6 +126,8 @@ const (
 	Disconnect SignalAction = "DISCONNECT"
 	// ConnNegotiation - action to negotiate connection between peers
 	ConnNegotiation SignalAction = "CONNECTION_NEGOTIATION"
+	// RelayME - action to relay the peer
+	RelayME SignalAction = "RELAY_ME"
 )
 
 // HostUpdate - struct for host update
@@ -148,8 +150,13 @@ type Signal struct {
 	FromHostPubKey    string       `json:"from_host_pubkey"`
 	TurnRelayEndpoint string       `json:"turn_relay_addr"`
 	ToHostPubKey      string       `json:"to_host_pubkey"`
+	FromHostID        string       `json:"from_host_id"`
+	ToHostID          string       `json:"to_host_id"`
+	FromNodeID        string       `json:"from_node_id"`
+	ToNodeID          string       `json:"to_node_id"`
 	Reply             bool         `json:"reply"`
 	Action            SignalAction `json:"action"`
+	IsPro             bool         `json:"is_pro"`
 	TimeStamp         int64        `json:"timestamp"`
 }
 

models/metrics.go

@@ -29,6 +29,7 @@ type Metric struct {
 // IDandAddr - struct to hold ID and primary Address
 type IDandAddr struct {
 	ID          string `json:"id" bson:"id" yaml:"id"`
+	HostID      string `json:"host_id"`
 	Address     string `json:"address" bson:"address" yaml:"address"`
 	Name        string `json:"name" bson:"name" yaml:"name"`
 	IsServer    string `json:"isserver" bson:"isserver" yaml:"isserver" validate:"checkyesorno"`

models/mqtt.go

@@ -70,3 +70,8 @@ type FwUpdate struct {
 	IsEgressGw bool                  `json:"is_egress_gw"`
 	EgressInfo map[string]EgressInfo `json:"egress_info"`
 }
+
+// FailOverMeReq - struct for failover req
+type FailOverMeReq struct {
+	NodeID string `json:"node_id"`
+}

models/node.go

@@ -90,10 +90,11 @@ type Node struct {
 	IngressGatewayRange     string               `json:"ingressgatewayrange" bson:"ingressgatewayrange" yaml:"ingressgatewayrange"`
 	IngressGatewayRange6    string               `json:"ingressgatewayrange6" bson:"ingressgatewayrange6" yaml:"ingressgatewayrange6"`
 	// == PRO ==
-	DefaultACL   string    `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"`
-	OwnerID      string    `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"`
-	FailoverNode uuid.UUID `json:"failovernode" bson:"failovernode" yaml:"failovernode"`
-	Failover     bool      `json:"failover" bson:"failover" yaml:"failover"`
+	DefaultACL    string              `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"`
+	OwnerID       string              `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"`
+	IsFailOver    bool                `json:"is_fail_over" yaml:"is_fail_over"`
+	FailOverPeers map[string]struct{} `json:"fail_over_peers" yaml:"fail_over_peers"`
+	FailedOverBy  uuid.UUID           `json:"failed_over_by" yaml:"failed_over_by"`
 }
 
 // LegacyNode - legacy struct for node model
@@ -192,6 +193,22 @@ func (node *Node) PrimaryAddress() string {
 	return node.Address6.IP.String()
 }
 
+// ExtClient.PrimaryAddress - returns ipv4 IPNet format
+func (extPeer *ExtClient) AddressIPNet4() net.IPNet {
+	return net.IPNet{
+		IP:   net.ParseIP(extPeer.Address),
+		Mask: net.CIDRMask(32, 32),
+	}
+}
+
+// ExtClient.AddressIPNet6 - return ipv6 IPNet format
+func (extPeer *ExtClient) AddressIPNet6() net.IPNet {
+	return net.IPNet{
+		IP:   net.ParseIP(extPeer.Address),
+		Mask: net.CIDRMask(128, 128),
+	}
+}
+
 // Node.PrimaryNetworkRange - returns node's parent network, returns ipv4 address if present, else return ipv6
 func (node *Node) PrimaryNetworkRange() net.IPNet {
 	if node.NetworkRange.IP != nil {
@@ -416,8 +433,8 @@ func (newNode *Node) Fill(currentNode *Node, isPro bool) { // TODO add new field
 	if newNode.DefaultACL == "" {
 		newNode.DefaultACL = currentNode.DefaultACL
 	}
-	if newNode.Failover != currentNode.Failover {
-		newNode.Failover = currentNode.Failover
+	if newNode.IsFailOver != currentNode.IsFailOver {
+		newNode.IsFailOver = currentNode.IsFailOver
 	}
 }
 

mq/handlers.go

@@ -49,18 +49,27 @@ func UpdateNode(client mqtt.Client, msg mqtt.Message) {
 	}
 
 	ifaceDelta := logic.IfaceDelta(&currentNode, &newNode)
-	if servercfg.IsPro && ifaceDelta {
-		if err = logic.EnterpriseResetAllPeersFailovers(currentNode.ID, currentNode.Network); err != nil {
-			slog.Warn("failed to reset failover list during node update", "nodeid", currentNode.ID, "network", currentNode.Network)
-		}
-	}
 	newNode.SetLastCheckIn()
 	if err := logic.UpdateNode(&currentNode, &newNode); err != nil {
 		slog.Error("error saving node", "id", id, "error", err)
 		return
 	}
 	if ifaceDelta { // reduce number of unneeded updates, by only sending on iface changes
-		if err = PublishPeerUpdate(); err != nil {
+		if !newNode.Connected {
+			err = PublishDeletedNodePeerUpdate(&newNode)
+			host, err := logic.GetHost(newNode.HostID.String())
+			if err != nil {
+				slog.Error("failed to get host for the node", "nodeid", newNode.ID.String(), "error", err)
+				return
+			}
+			allNodes, err := logic.GetAllNodes()
+			if err == nil {
+				PublishSingleHostPeerUpdate(host, allNodes, nil, nil)
+			}
+		} else {
+			err = PublishPeerUpdate()
+		}
+		if err != nil {
 			slog.Warn("error updating peers when node informed the server of an interface change", "nodeid", currentNode.ID, "error", err)
 		}
 	}

mq/publishers.go

@@ -96,10 +96,6 @@ func PublishSingleHostPeerUpdate(host *models.Host, allNodes []models.Node, dele
 	if err != nil {
 		return err
 	}
-	if len(peerUpdate.Peers) == 0 { // no peers to send
-		return nil
-	}
-
 	data, err := json.Marshal(&peerUpdate)
 	if err != nil {
 		return err

mq/util.go

@@ -73,6 +73,7 @@ func encryptMsg(host *models.Host, msg []byte) ([]byte, error) {
 }
 
 func publish(host *models.Host, dest string, msg []byte) error {
+
 	encrypted, encryptErr := encryptMsg(host, msg)
 	if encryptErr != nil {
 		return encryptErr
@@ -80,6 +81,7 @@ func publish(host *models.Host, dest string, msg []byte) error {
 	if mqclient == nil {
 		return errors.New("cannot publish ... mqclient not connected")
 	}
+
 	if token := mqclient.Publish(dest, 0, true, encrypted); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
 		var err error
 		if token.Error() == nil {

pro/controllers/failover.go

@@ -0,0 +1,201 @@
+package controllers
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/mux"
+	controller "github.com/gravitl/netmaker/controllers"
+	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/logic"
+	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
+	proLogic "github.com/gravitl/netmaker/pro/logic"
+	"golang.org/x/exp/slog"
+)
+
+// FailOverHandlers - handlers for FailOver
+func FailOverHandlers(r *mux.Router) {
+	r.HandleFunc("/api/v1/node/{nodeid}/failover", logic.SecurityCheck(true, http.HandlerFunc(createfailOver))).Methods(http.MethodPost)
+	r.HandleFunc("/api/v1/node/{nodeid}/failover", logic.SecurityCheck(true, http.HandlerFunc(deletefailOver))).Methods(http.MethodDelete)
+	r.HandleFunc("/api/v1/node/{network}/failover/reset", logic.SecurityCheck(true, http.HandlerFunc(resetFailOver))).Methods(http.MethodPost)
+	r.HandleFunc("/api/v1/node/{nodeid}/failover_me", controller.Authorize(true, false, "host", http.HandlerFunc(failOverME))).Methods(http.MethodPost)
+}
+
+// swagger:route POST /api/v1/node/failover node createfailOver
+//
+// Create a relay.
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: nodeResponse
+func createfailOver(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	nodeid := params["nodeid"]
+	// confirm host exists
+	node, err := logic.GetNodeByID(nodeid)
+	if err != nil {
+		slog.Error("failed to get node:", "error", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+	if _, exists := proLogic.FailOverExists(node.Network); exists {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failover exists already in the network"), "badrequest"))
+		return
+	}
+	host, err := logic.GetHost(node.HostID.String())
+	if err != nil {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("error getting host"+err.Error()), "badrequest"))
+		return
+	}
+	if host.OS != models.OS_Types.Linux {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("only linux nodes can act as failovers"), "badrequest"))
+		return
+	}
+	if node.IsRelayed {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("cannot set relayed node as failover"), "badrequest"))
+		return
+	}
+	node.IsFailOver = true
+	err = logic.UpsertNode(&node)
+	if err != nil {
+		slog.Error("failed to upsert node", "node", node.ID.String(), "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	go mq.PublishPeerUpdate()
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponseWithJson(w, r, node, "created failover successfully")
+}
+
+func resetFailOver(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	net := params["network"]
+	nodes, err := logic.GetNetworkNodes(net)
+	if err != nil {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	for _, node := range nodes {
+		if node.FailedOverBy != uuid.Nil {
+			node.FailedOverBy = uuid.Nil
+			node.FailOverPeers = make(map[string]struct{})
+			logic.UpsertNode(&node)
+		}
+	}
+	go mq.PublishPeerUpdate()
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponse(w, r, "failover has been reset successfully")
+}
+
+// swagger:route DELETE /api/v1/node/failover node deletefailOver
+//
+// Create a relay.
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: nodeResponse
+func deletefailOver(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	nodeid := params["nodeid"]
+	// confirm host exists
+	node, err := logic.GetNodeByID(nodeid)
+	if err != nil {
+		slog.Error("failed to get node:", "error", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+	node.IsFailOver = false
+	// Reset FailOvered Peers
+	err = logic.UpsertNode(&node)
+	if err != nil {
+		slog.Error("failed to upsert node", "node", node.ID.String(), "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	go func() {
+		proLogic.ResetFailOver(&node)
+		mq.PublishPeerUpdate()
+	}()
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponseWithJson(w, r, node, "deleted failover successfully")
+}
+
+// swagger:route POST /api/node/{nodeid}/failOverME node failOver_me
+//
+// Create a relay.
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: nodeResponse
+func failOverME(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	nodeid := params["nodeid"]
+	// confirm host exists
+	node, err := logic.GetNodeByID(nodeid)
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"), "failed to get node:", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+
+	failOverNode, exists := proLogic.FailOverExists(node.Network)
+	if !exists {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failover node doesn't exist in the network"), "badrequest"))
+		return
+	}
+	var failOverReq models.FailOverMeReq
+	err = json.NewDecoder(r.Body).Decode(&failOverReq)
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"), "error decoding request body: ", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+	var sendPeerUpdate bool
+	peerNode, err := logic.GetNodeByID(failOverReq.NodeID)
+	if err != nil {
+		slog.Error("peer not found: ", "nodeid", failOverReq.NodeID, "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("peer not found"), "badrequest"))
+		return
+	}
+	if node.IsRelayed || node.IsFailOver {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("node is relayed or acting as failover"), "badrequest"))
+		return
+	}
+	if peerNode.IsRelayed || peerNode.IsFailOver {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("peer node is relayed or acting as failover"), "badrequest"))
+		return
+	}
+
+	err = proLogic.SetFailOverCtx(failOverNode, node, peerNode)
+	if err != nil {
+		slog.Error("failed to create failover", "id", node.ID.String(),
+			"network", node.Network, "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to create failover: %v", err), "internal"))
+		return
+	}
+	slog.Info("[auto-relay] created relay on node", "node", node.ID.String(), "network", node.Network)
+	sendPeerUpdate = true
+
+	if sendPeerUpdate {
+		go mq.PublishPeerUpdate()
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponse(w, r, "relayed successfully")
+}

pro/controllers/relay.go

@@ -3,9 +3,11 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
-	proLogic "github.com/gravitl/netmaker/pro/logic"
 	"net/http"
 
+	"github.com/google/uuid"
+	proLogic "github.com/gravitl/netmaker/pro/logic"
+
 	"github.com/gorilla/mux"
 	controller "github.com/gravitl/netmaker/controllers"
 	"github.com/gravitl/netmaker/logger"
@@ -19,6 +21,7 @@ func RelayHandlers(r *mux.Router) {
 
 	r.HandleFunc("/api/nodes/{network}/{nodeid}/createrelay", controller.Authorize(false, true, "user", http.HandlerFunc(createRelay))).Methods(http.MethodPost)
 	r.HandleFunc("/api/nodes/{network}/{nodeid}/deleterelay", controller.Authorize(false, true, "user", http.HandlerFunc(deleteRelay))).Methods(http.MethodDelete)
+	r.HandleFunc("/api/v1/host/{hostid}/failoverme", controller.Authorize(true, false, "host", http.HandlerFunc(failOverME))).Methods(http.MethodPost)
 }
 
 // swagger:route POST /api/nodes/{network}/{nodeid}/createrelay nodes createRelay
@@ -51,6 +54,15 @@ func createRelay(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+	for _, relayedNodeID := range relayNode.RelayedNodes {
+		relayedNode, err := logic.GetNodeByID(relayedNodeID)
+		if err == nil {
+			if relayedNode.FailedOverBy != uuid.Nil {
+				go logic.ResetFailedOverPeer(&relayedNode)
+			}
+
+		}
+	}
 	go mq.PublishPeerUpdate()
 	logger.Log(1, r.Header.Get("user"), "created relay on node", relayRequest.NodeID, "on network", relayRequest.NetID)
 	apiNode := relayNode.ConvertToAPINode()

pro/initialize.go

@@ -27,9 +27,11 @@ func InitPro() {
 		proControllers.MetricHandlers,
 		proControllers.RelayHandlers,
 		proControllers.UserHandlers,
+		proControllers.FailOverHandlers,
 	)
 	logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() {
 		// == License Handling ==
+		ClearLicenseCache()
 		if err := ValidateLicense(); err != nil {
 			slog.Error(err.Error())
 			return
@@ -41,11 +43,9 @@ func InitPro() {
 		if servercfg.GetServerConfig().RacAutoDisable {
 			AddRacHooks()
 		}
-		resetFailover()
 	})
-	logic.EnterpriseFailoverFunc = proLogic.SetFailover
-	logic.EnterpriseResetFailoverFunc = proLogic.ResetFailover
-	logic.EnterpriseResetAllPeersFailovers = proLogic.WipeAffectedFailoversOnly
+	logic.ResetFailOver = proLogic.ResetFailOver
+	logic.ResetFailedOverPeer = proLogic.ResetFailedOverPeer
 	logic.DenyClientNodeAccess = proLogic.DenyClientNode
 	logic.IsClientNodeAllowed = proLogic.IsClientNodeAllowed
 	logic.AllowClientNodeAccess = proLogic.RemoveDeniedNodeFromClient
@@ -64,18 +64,6 @@ func InitPro() {
 	mq.UpdateMetrics = proLogic.MQUpdateMetrics
 }
 
-func resetFailover() {
-	nets, err := logic.GetNetworks()
-	if err == nil {
-		for _, net := range nets {
-			err = proLogic.ResetFailover(net.NetID)
-			if err != nil {
-				slog.Error("failed to reset failover", "network", net.NetID, "error", err.Error())
-			}
-		}
-	}
-}
-
 func retrieveProLogo() string {
 	return `              
  __   __     ______     ______   __    __     ______     __  __     ______     ______    

pro/license.go

@@ -38,10 +38,10 @@ func AddLicenseHooks() {
 		Hook:     ValidateLicense,
 		Interval: time.Hour,
 	}
-	logic.HookManagerCh <- models.HookDetails{
-		Hook:     ClearLicenseCache,
-		Interval: time.Hour,
-	}
+	// logic.HookManagerCh <- models.HookDetails{
+	// 	Hook:     ClearLicenseCache,
+	// 	Interval: time.Hour,
+	// }
 }
 
 // ValidateLicense - the initial and periodic license check for netmaker server
@@ -97,11 +97,14 @@ func ValidateLicense() (err error) {
 		return err
 	}
 
-	validationResponse, err := validateLicenseKey(encryptedData, tempPubKey)
+	validationResponse, timedOut, err := validateLicenseKey(encryptedData, tempPubKey)
 	if err != nil {
 		err = fmt.Errorf("failed to validate license key: %w", err)
 		return err
 	}
+	if timedOut {
+		return
+	}
 	if len(validationResponse) == 0 {
 		err = errors.New("empty validation response")
 		return err
@@ -185,12 +188,11 @@ func getLicensePublicKey(licensePubKeyEncoded string) (*[32]byte, error) {
 	return ncutils.ConvertBytesToKey(decodedPubKey)
 }
 
-func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, error) {
+func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, bool, error) {
 	publicKeyBytes, err := ncutils.ConvertKeyToBytes(publicKey)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
-
 	msg := ValidateLicenseRequest{
 		LicenseKey:     servercfg.GetLicenseKey(),
 		NmServerPubKey: base64encode(publicKeyBytes),
@@ -199,7 +201,7 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro
 
 	requestBody, err := json.Marshal(msg)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 
 	req, err := http.NewRequest(
@@ -208,40 +210,49 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro
 		bytes.NewReader(requestBody),
 	)
 	if err != nil {
-		return nil, err
+		return nil, false, err
 	}
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Accept", "application/json")
 	client := &http.Client{}
-	var body []byte
 	validateResponse, err := client.Do(req)
 	if err != nil { // check cache
-		body, err = getCachedResponse()
-		if err != nil {
-			return nil, err
-		}
 		slog.Warn("proceeding with cached response, Netmaker API may be down")
-	} else {
-		defer validateResponse.Body.Close()
-		if validateResponse.StatusCode != http.StatusOK {
-			err := fmt.Errorf("could not validate license, got status code %d", validateResponse.StatusCode)
-			// if it's a temp error, just log it, don't consider license invalid
-			if validateResponse.StatusCode == http.StatusServiceUnavailable ||
-				validateResponse.StatusCode == http.StatusGatewayTimeout {
-				slog.Warn(err.Error())
-				return nil, nil
-			}
-			return nil, err
-		} // if you received a 200 cache the response locally
-
-		body, err = io.ReadAll(validateResponse.Body)
+		cachedResp, err := getCachedResponse()
+		return cachedResp, false, err
+	}
+	defer validateResponse.Body.Close()
+	code := validateResponse.StatusCode
+
+	// if we received a 200, cache the response locally
+	if code == http.StatusOK {
+		body, err := io.ReadAll(validateResponse.Body)
 		if err != nil {
-			return nil, err
+			slog.Warn("failed to parse response", "error", err)
+			return nil, false, err
+		}
+		if err := cacheResponse(body); err != nil {
+			slog.Warn("failed to cache response", "error", err)
 		}
-		cacheResponse(body)
+		return body, false, nil
+	}
+
+	// at this point the backend returned some undesired state
+
+	// inform failure via logs
+	body, _ := io.ReadAll(validateResponse.Body)
+	err = fmt.Errorf("could not validate license with validation backend (status={%d}, body={%s})",
+		validateResponse.StatusCode, string(body))
+	slog.Warn(err.Error())
+
+	// try to use cache if we had a temporary error
+	if code == http.StatusServiceUnavailable || code == http.StatusGatewayTimeout {
+		slog.Warn("Netmaker API may be down, will retry later...", "code", code)
+		return nil, true, nil
 	}
 
-	return body, err
+	// at this point the error is irreversible, return it
+	return nil, false, err
 }
 
 func getAccountsHost() string {

pro/logic/failover.go

@@ -1,122 +1,97 @@
 package logic
 
 import (
+	"errors"
+
 	"github.com/google/uuid"
-	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
 )
 
-// SetFailover - finds a suitable failover candidate and sets it
-func SetFailover(node *models.Node) error {
-	failoverNode := determineFailoverCandidate(node)
-	if failoverNode != nil {
-		return setFailoverNode(failoverNode, node)
+func SetFailOverCtx(failOverNode, victimNode, peerNode models.Node) error {
+	if peerNode.FailOverPeers == nil {
+		peerNode.FailOverPeers = make(map[string]struct{})
+	}
+	if victimNode.FailOverPeers == nil {
+		victimNode.FailOverPeers = make(map[string]struct{})
+	}
+	peerNode.FailOverPeers[victimNode.ID.String()] = struct{}{}
+	victimNode.FailOverPeers[peerNode.ID.String()] = struct{}{}
+	victimNode.FailedOverBy = failOverNode.ID
+	peerNode.FailedOverBy = failOverNode.ID
+	if err := logic.UpsertNode(&failOverNode); err != nil {
+		return err
+	}
+	if err := logic.UpsertNode(&victimNode); err != nil {
+		return err
+	}
+	if err := logic.UpsertNode(&peerNode); err != nil {
+		return err
 	}
 	return nil
 }
 
-// ResetFailover - sets the failover node and wipes disconnected status
-func ResetFailover(network string) error {
-	nodes, err := logic.GetNetworkNodes(network)
-	if err != nil {
-		return err
-	}
+// GetFailOverNode - gets the host acting as failOver
+func GetFailOverNode(network string, allNodes []models.Node) (models.Node, error) {
+	nodes := logic.GetNetworkNodesMemory(allNodes, network)
 	for _, node := range nodes {
-		node := node
-		err = SetFailover(&node)
-		if err != nil {
-			logger.Log(2, "error setting failover for node", node.ID.String(), ":", err.Error())
-		}
-		err = WipeFailover(node.ID.String())
-		if err != nil {
-			logger.Log(2, "error wiping failover for node", node.ID.String(), ":", err.Error())
+		if node.IsFailOver {
+			return node, nil
 		}
 	}
-	return nil
+	return models.Node{}, errors.New("auto relay not found")
 }
 
-// determineFailoverCandidate - returns a list of nodes that
-// are suitable for relaying a given node
-func determineFailoverCandidate(nodeToBeRelayed *models.Node) *models.Node {
-
-	currentNetworkNodes, err := logic.GetNetworkNodes(nodeToBeRelayed.Network)
+// FailOverExists - checks if failOver exists already in the network
+func FailOverExists(network string) (failOverNode models.Node, exists bool) {
+	nodes, err := logic.GetNetworkNodes(network)
 	if err != nil {
-		return nil
-	}
-
-	currentMetrics, err := GetMetrics(nodeToBeRelayed.ID.String())
-	if err != nil || currentMetrics == nil || currentMetrics.Connectivity == nil {
-		return nil
+		return
 	}
-
-	minLatency := int64(9223372036854775807) // max signed int64 value
-	var fastestCandidate *models.Node
-	for i := range currentNetworkNodes {
-		if currentNetworkNodes[i].ID == nodeToBeRelayed.ID {
-			continue
-		}
-
-		if currentMetrics.Connectivity[currentNetworkNodes[i].ID.String()].Connected && (currentNetworkNodes[i].Failover) {
-			if currentMetrics.Connectivity[currentNetworkNodes[i].ID.String()].Latency < int64(minLatency) {
-				fastestCandidate = &currentNetworkNodes[i]
-				minLatency = currentMetrics.Connectivity[currentNetworkNodes[i].ID.String()].Latency
-			}
+	for _, node := range nodes {
+		if node.IsFailOver {
+			exists = true
+			failOverNode = node
+			return
 		}
 	}
-
-	return fastestCandidate
+	return
 }
 
-// setFailoverNode - changes node's failover node
-func setFailoverNode(failoverNode, node *models.Node) error {
-
-	node.FailoverNode = failoverNode.ID
-	nodeToUpdate, err := logic.GetNodeByID(node.ID.String())
+// ResetFailedOverPeer - removes failed over node from network peers
+func ResetFailedOverPeer(failedOveredNode *models.Node) error {
+	nodes, err := logic.GetNetworkNodes(failedOveredNode.Network)
 	if err != nil {
 		return err
 	}
-	if nodeToUpdate.FailoverNode == failoverNode.ID {
-		return nil
-	}
-	return logic.UpdateNode(&nodeToUpdate, node)
-}
-
-// WipeFailover - removes the failover peers of given node (ID)
-func WipeFailover(nodeid string) error {
-	metrics, err := GetMetrics(nodeid)
+	failedOveredNode.FailedOverBy = uuid.Nil
+	failedOveredNode.FailOverPeers = make(map[string]struct{})
+	err = logic.UpsertNode(failedOveredNode)
 	if err != nil {
 		return err
 	}
-	if metrics != nil {
-		metrics.FailoverPeers = make(map[string]string)
-		return logic.UpdateMetrics(nodeid, metrics)
+	for _, node := range nodes {
+		if node.FailOverPeers == nil || node.ID == failedOveredNode.ID {
+			continue
+		}
+		delete(node.FailOverPeers, failedOveredNode.ID.String())
+		logic.UpsertNode(&node)
 	}
 	return nil
 }
 
-// WipeAffectedFailoversOnly - wipes failovers for nodes that have given node (ID)
-// in their respective failover lists
-func WipeAffectedFailoversOnly(nodeid uuid.UUID, network string) error {
-	currentNetworkNodes, err := logic.GetNetworkNodes(network)
+// ResetFailOver - reset failovered peers
+func ResetFailOver(failOverNode *models.Node) error {
+	// Unset FailedOverPeers
+	nodes, err := logic.GetNetworkNodes(failOverNode.Network)
 	if err != nil {
-		return nil
+		return err
 	}
-	WipeFailover(nodeid.String())
-
-	for i := range currentNetworkNodes {
-		currNodeID := currentNetworkNodes[i].ID
-		if currNodeID == nodeid {
-			continue
-		}
-		currMetrics, err := GetMetrics(currNodeID.String())
-		if err != nil || currMetrics == nil {
-			continue
-		}
-		if currMetrics.FailoverPeers != nil {
-			if len(currMetrics.FailoverPeers[nodeid.String()]) > 0 {
-				WipeFailover(currNodeID.String())
-			}
+	for _, node := range nodes {
+		if node.FailedOverBy == failOverNode.ID {
+			node.FailedOverBy = uuid.Nil
+			node.FailOverPeers = make(map[string]struct{})
+			logic.UpsertNode(&node)
 		}
 	}
 	return nil

pro/logic/metrics.go

@@ -2,6 +2,9 @@ package logic
 
 import (
 	"encoding/json"
+	"math"
+	"time"
+
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic"
@@ -10,8 +13,6 @@ import (
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
-	"math"
-	"time"
 )
 
 // GetMetrics - gets the metrics
@@ -80,13 +81,6 @@ func MQUpdateMetrics(client mqtt.Client, msg mqtt.Message) {
 		}
 	}
 
-	if newMetrics.Connectivity != nil {
-		err := logic.EnterpriseFailoverFunc(&currentNode)
-		if err != nil {
-			slog.Error("failed to failover for node", "id", currentNode.ID, "network", currentNode.Network, "error", err)
-		}
-	}
-
 	if shouldUpdate {
 		slog.Info("updating peers after node detected connectivity issues", "id", currentNode.ID, "network", currentNode.Network)
 		host, err := logic.GetHost(currentNode.HostID.String())
@@ -170,21 +164,6 @@ func updateNodeMetrics(currentNode *models.Node, newMetrics *models.Metrics) boo
 
 	}
 
-	// add nodes that need failover
-	nodes, err := logic.GetNetworkNodes(currentNode.Network)
-	if err != nil {
-		slog.Error("failed to retrieve nodes while updating metrics", "error", err)
-		return false
-	}
-	for _, node := range nodes {
-		if !newMetrics.Connectivity[node.ID.String()].Connected &&
-			len(newMetrics.Connectivity[node.ID.String()].NodeName) > 0 &&
-			node.Connected &&
-			len(node.FailoverNode) > 0 &&
-			!node.Failover {
-			newMetrics.FailoverPeers[node.ID.String()] = node.FailoverNode.String()
-		}
-	}
 	shouldUpdate := len(oldMetrics.FailoverPeers) == 0 && len(newMetrics.FailoverPeers) > 0
 	for k, v := range oldMetrics.FailoverPeers {
 		if len(newMetrics.FailoverPeers[k]) > 0 && len(v) == 0 {

pro/logic/relays.go

@@ -69,7 +69,7 @@ func SetRelayedNodes(setRelayed bool, relay string, relayed []string) []models.N
 			continue
 		}
 		node.IsRelayed = setRelayed
-		if node.IsRelayed {
+		if setRelayed {
 			node.RelayedBy = relay
 		} else {
 			node.RelayedBy = ""
@@ -155,6 +155,7 @@ func UpdateRelayed(currentNode, newNode *models.Node) {
 	if len(updatenodes) > 0 {
 		for _, relayedNode := range updatenodes {
 			node := relayedNode
+			ResetFailedOverPeer(&node)
 			go func() {
 				if err := mq.NodeUpdate(&node); err != nil {
 					slog.Error("error publishing node update to node", "node", node.ID, "error", err)

pro/types.go

@@ -1,3 +1,4 @@
+// go:build ee
 //go:build ee
 // +build ee
 

release.md

@@ -1,20 +1,19 @@
 
-# Netmaker v0.21.2
+# Netmaker v0.21.3
 
 ## Whats New
-- Remote access client session management, refer users section in docs for more details
-- Can now create generic DNS entries
-- Upgrade client version to match server version from UI
-- Moved PersistentKeepAlive setting from node to host level
+- Auto Relay via Enrollment key
+- Local Routing Improvements
 ## What's Fixed
-- Extclients DNS now properly set from ingress dns value provided
-- Allow role update of OAuth user
-- Fixed zombie node issue
+- Inconsistency in DNS Entries for Network has been fixed
+- Unique network CIDR validation
+- Fixed extclient caching decrepancies
+- Deleted node peer update fixes when disconnected from network
+- Force Deletion of Daemon Nodes that are stuck in removing state
 ## known issues
 - Windows installer does not install WireGuard
 - netclient-gui will continously display error dialog if netmaker server is offline
 - Mac IPv6 addresses/route issues
-- Docker client can not re-join after complete deletion
 - netclient-gui network tab blank after disconnect
 
 

scripts/netmaker.default.env

@@ -81,3 +81,5 @@ OIDC_ISSUER=
 JWT_VALIDITY_DURATION=43200
 # Auto disable a user's connecteds clients bassed on JWT token expiration
 RAC_AUTO_DISABLE="true"
+# if turned on data will be cached on to improve performance significantly (IMPORTANT: If HA set to `false` )
+CACHING_ENABLED="true

scripts/nm-upgrade-0-17-1-to-0-19-0.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-LATEST="v0.21.2"
+LATEST="v0.21.3"
 INSTALL_PATH="/root"
 
 trap restore_old_netmaker_instructions

servercfg/serverconf.go

@@ -207,6 +207,17 @@ func GetDB() string {
 	return database
 }
 
+// CacheEnabled - checks if cache is enabled
+func CacheEnabled() bool {
+	caching := false
+	if os.Getenv("CACHING_ENABLED") != "" {
+		caching = os.Getenv("CACHING_ENABLED") == "true"
+	} else if config.Config.Server.Database != "" {
+		caching = config.Config.Server.CacheEnabled
+	}
+	return caching
+}
+
 // GetAPIHost - gets the api host
 func GetAPIHost() string {
 	serverhost := "127.0.0.1"
@@ -256,6 +267,11 @@ func GetPublicBrokerEndpoint() string {
 	}
 }
 
+// GetOwnerEmail - gets the owner email (saas)
+func GetOwnerEmail() string {
+	return os.Getenv("SAAS_OWNER_EMAIL")
+}
+
 // GetMessageQueueEndpoint - gets the message queue endpoint
 func GetMessageQueueEndpoint() (string, bool) {
 	host, _ := GetPublicIP()

swagger.yml

@@ -1149,7 +1149,7 @@ info:
 
         API calls must be authenticated via a header of the format -H “Authorization: Bearer <YOUR_SECRET_KEY>” There are two methods to obtain YOUR_SECRET_KEY: 1. Using the masterkey. By default, this value is “secret key,” but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [Netmaker](https://docs.netmaker.org/index.html) documentation for more details. 2. Using a JWT received for a node. This can be retrieved by calling the /api/nodes/<network>/authenticate endpoint, as documented below.
     title: Netmaker
-    version: 0.21.2
+    version: 0.21.3
 paths:
     /api/dns:
         get: