Merge pull request #2769 from gravitl/release-v0.22.0

v0.22.0

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -31,6 +31,7 @@ body:
       label: Version
       description: What version are you running?
       options:
+        - v0.22.0
         - v0.21.2
         - v0.21.1
         - v0.21.0
@@ -91,7 +92,6 @@ body:
       multiple: true
       options:
         - Linux
-        - FreeBSD
         - Windows
         - Mac
         - Unlisted

.github/workflows/deletedroplets.yml

@@ -12,7 +12,7 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'success' }}
     steps:
       - name: get logs
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@v3
         with:
           run_id: ${{ github.event.workflow_run.id}}
           if_no_artifact_found: warn
@@ -60,7 +60,7 @@ jobs:
     if: ${{ github.event.workflow_run.conclusion == 'failure' }}
     steps:
       - name: get logs
-        uses: dawidd6/action-download-artifact@v2
+        uses: dawidd6/action-download-artifact@v3
         with:
           run_id: ${{ github.event.workflow_run.id}}
           if_no_artifact_found: warn

.github/workflows/test.yml

@@ -13,7 +13,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - name: Build
@@ -27,7 +27,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - name: Build
@@ -44,7 +44,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - name: run tests
@@ -64,7 +64,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v4
       - name: Setup Go
-        uses: actions/setup-go@v4
+        uses: actions/setup-go@v5
         with:
           go-version: 1.19
       - name: run static checks

.goreleaser.prerelease.yaml

@@ -23,7 +23,6 @@ builds:
       - linux_arm64
       - darwin_amd64
       - darwin_arm64
-      - freebsd_amd64
       - windows_amd64
     binary: 'nmctl'
 archives:

.goreleaser.yaml

@@ -23,7 +23,6 @@ builds:
       - linux_arm64
       - darwin_amd64
       - darwin_arm64
-      - freebsd_amd64
       - windows_amd64
     binary: 'nmctl'
 archives:

Dockerfile

@@ -6,7 +6,7 @@ COPY . .
 
 RUN GOOS=linux CGO_ENABLED=1 go build -ldflags="-s -w " -tags ${tags} .
 # RUN go build -tags=ee . -o netmaker main.go
-FROM alpine:3.18.4
+FROM alpine:3.19.0
 
 # add a c lib
 # set the working directory

Dockerfile-quick

@@ -1,5 +1,5 @@
 #first stage - builder
-FROM alpine:3.18.4
+FROM alpine:3.19.0
 ARG version 
 WORKDIR /app
 COPY ./netmaker /root/netmaker

README.md

@@ -16,7 +16,7 @@
 
 <p align="center">
   <a href="https://github.com/gravitl/netmaker/releases">
-    <img src="https://img.shields.io/badge/Version-0.21.2-informational?style=flat-square" />
+    <img src="https://img.shields.io/badge/Version-0.22.0-informational?style=flat-square" />
   </a>
   <a href="https://hub.docker.com/r/gravitl/netmaker/tags">
     <img src="https://img.shields.io/docker/pulls/gravitl/netmaker?label=downloads" />
@@ -43,7 +43,7 @@
 | Create                                    | Manage                                  | Automate                                |
 |-------------------------------------------|-----------------------------------------|-----------------------------------------|
 | :heavy_check_mark: WireGuard Networks     | :heavy_check_mark: Admin UI             | :heavy_check_mark: Linux                |
-| :heavy_check_mark: Remote Access Gateways | :heavy_check_mark: OAuth                | :heavy_check_mark: FreeBSD              |
+| :heavy_check_mark: Remote Access Gateways | :heavy_check_mark: OAuth                | :heavy_check_mark: Docker              |
 | :heavy_check_mark: Mesh VPNs              | :heavy_check_mark: Private DNS          | :heavy_check_mark: Mac                  |
 | :heavy_check_mark: Site-to-Site           | :heavy_check_mark: Access Control Lists | :heavy_check_mark: Windows              |
 

auth/azure-ad.go

@@ -66,6 +66,15 @@ func handleAzureCallback(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	user, err := logic.GetUser(content.Email)
+	if err != nil {
+		handleOauthUserNotFound(w)
+		return
+	}
+	if !(user.IsSuperAdmin || user.IsAdmin) {
+		handleOauthUserNotAllowed(w)
+		return
+	}
 	var newPass, fetchErr = fetchPassValue("")
 	if fetchErr != nil {
 		return

auth/error.go

@@ -10,6 +10,31 @@ const oauthNotConfigured = `<!DOCTYPE html><html>
 </body>
 </html>`
 
+const userNotAllowed = `<!DOCTYPE html><html>
+<body>
+<h3>Only Admins are allowed to access Dashboard.</h3>
+<p>Non-Admins can access the netmaker networks using <a href="https://docs.netmaker.io/pro/rac.html" target="_blank" rel="noopener">RemoteAccessClient.</a></p>
+</body>
+</html>
+`
+const userNotFound = `<!DOCTYPE html><html>
+<body>
+<h3>User Not Found.</h3>
+</body>
+</html>`
+
+func handleOauthUserNotFound(response http.ResponseWriter) {
+	response.Header().Set("Content-Type", "text/html; charset=utf-8")
+	response.WriteHeader(http.StatusNotFound)
+	response.Write([]byte(userNotFound))
+}
+
+func handleOauthUserNotAllowed(response http.ResponseWriter) {
+	response.Header().Set("Content-Type", "text/html; charset=utf-8")
+	response.WriteHeader(http.StatusForbidden)
+	response.Write([]byte(userNotAllowed))
+}
+
 // handleOauthNotConfigured - returns an appropriate html page when oauth is not configured on netmaker server but an oauth login was attempted
 func handleOauthNotConfigured(response http.ResponseWriter) {
 	response.Header().Set("Content-Type", "text/html; charset=utf-8")

auth/github.go

@@ -66,6 +66,15 @@ func handleGithubCallback(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	user, err := logic.GetUser(content.Email)
+	if err != nil {
+		handleOauthUserNotFound(w)
+		return
+	}
+	if !(user.IsSuperAdmin || user.IsAdmin) {
+		handleOauthUserNotAllowed(w)
+		return
+	}
 	var newPass, fetchErr = fetchPassValue("")
 	if fetchErr != nil {
 		return

auth/google.go

@@ -68,6 +68,15 @@ func handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	user, err := logic.GetUser(content.Email)
+	if err != nil {
+		handleOauthUserNotFound(w)
+		return
+	}
+	if !(user.IsSuperAdmin || user.IsAdmin) {
+		handleOauthUserNotAllowed(w)
+		return
+	}
 	var newPass, fetchErr = fetchPassValue("")
 	if fetchErr != nil {
 		return

auth/host_session.go

@@ -253,7 +253,7 @@ func CheckNetRegAndHostUpdate(networks []string, h *models.Host, relayNodeId uui
 			Action: models.RequestAck,
 			Host:   *h,
 		})
-		if err := mq.PublishPeerUpdate(); err != nil {
+		if err := mq.PublishPeerUpdate(false); err != nil {
 			logger.Log(0, "failed to publish peer update during registration -", err.Error())
 		}
 	}

auth/oidc.go

@@ -79,6 +79,15 @@ func handleOIDCCallback(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	user, err := logic.GetUser(content.Email)
+	if err != nil {
+		handleOauthUserNotFound(w)
+		return
+	}
+	if !(user.IsSuperAdmin || user.IsAdmin) {
+		handleOauthUserNotAllowed(w)
+		return
+	}
 	var newPass, fetchErr = fetchPassValue("")
 	if fetchErr != nil {
 		return

cli/cmd/acl/allow.go

@@ -2,6 +2,7 @@ package acl
 
 import (
 	"fmt"
+	"log"
 
 	"github.com/gravitl/netmaker/cli/functions"
 	"github.com/gravitl/netmaker/logic/acls"
@@ -14,17 +15,34 @@ var aclAllowCmd = &cobra.Command{
 	Short: "Allow access from one node to another",
 	Long:  `Allow access from one node to another`,
 	Run: func(cmd *cobra.Command, args []string) {
+		network := args[0]
 		fromNodeID := args[1]
 		toNodeID := args[2]
-		payload := acls.ACLContainer(map[acls.AclID]acls.ACL{
-			acls.AclID(fromNodeID): map[acls.AclID]byte{
-				acls.AclID(toNodeID): acls.Allowed,
-			},
-			acls.AclID(toNodeID): map[acls.AclID]byte{
-				acls.AclID(fromNodeID): acls.Allowed,
-			},
-		})
-		functions.UpdateACL(args[0], &payload)
+
+		if fromNodeID == toNodeID {
+			log.Fatal("Cannot allow access from a node to itself")
+		}
+
+		// get current acls
+		res := functions.GetACL(network)
+		if res == nil {
+			log.Fatalf("Could not load network ACLs")
+		}
+
+		payload := *res
+
+		if _, ok := payload[acls.AclID(fromNodeID)]; !ok {
+			log.Fatalf("Node %s does not exist", fromNodeID)
+		}
+		if _, ok := payload[acls.AclID(toNodeID)]; !ok {
+			log.Fatalf("Node %s does not exist", toNodeID)
+		}
+
+		// update acls
+		payload[acls.AclID(fromNodeID)][acls.AclID(toNodeID)] = acls.Allowed
+		payload[acls.AclID(toNodeID)][acls.AclID(fromNodeID)] = acls.Allowed
+
+		functions.UpdateACL(network, &payload)
 		fmt.Println("Success")
 	},
 }

cli/cmd/acl/deny.go

@@ -2,6 +2,7 @@ package acl
 
 import (
 	"fmt"
+	"log"
 
 	"github.com/gravitl/netmaker/cli/functions"
 	"github.com/gravitl/netmaker/logic/acls"
@@ -14,17 +15,34 @@ var aclDenyCmd = &cobra.Command{
 	Short: "Deny access from one node to another",
 	Long:  `Deny access from one node to another`,
 	Run: func(cmd *cobra.Command, args []string) {
+		network := args[0]
 		fromNodeID := args[1]
 		toNodeID := args[2]
-		payload := acls.ACLContainer(map[acls.AclID]acls.ACL{
-			acls.AclID(fromNodeID): map[acls.AclID]byte{
-				acls.AclID(toNodeID): acls.NotAllowed,
-			},
-			acls.AclID(toNodeID): map[acls.AclID]byte{
-				acls.AclID(fromNodeID): acls.NotAllowed,
-			},
-		})
-		functions.UpdateACL(args[0], &payload)
+
+		if fromNodeID == toNodeID {
+			log.Fatal("Cannot deny access to self")
+		}
+
+		// get current acls
+		res := functions.GetACL(network)
+		if res == nil {
+			log.Fatalf("Could not load network ACLs")
+		}
+
+		payload := *res
+
+		if _, ok := payload[acls.AclID(fromNodeID)]; !ok {
+			log.Fatalf("Node [%s] does not exist", fromNodeID)
+		}
+		if _, ok := payload[acls.AclID(toNodeID)]; !ok {
+			log.Fatalf("Node [%s] does not exist", toNodeID)
+		}
+
+		// update acls
+		payload[acls.AclID(fromNodeID)][acls.AclID(toNodeID)] = acls.NotAllowed
+		payload[acls.AclID(toNodeID)][acls.AclID(fromNodeID)] = acls.NotAllowed
+
+		functions.UpdateACL(network, &payload)
 		fmt.Println("Success")
 	},
 }

cli/cmd/context/set.go

@@ -1,9 +1,11 @@
 package context
 
 import (
+	"fmt"
 	"log"
 
 	"github.com/gravitl/netmaker/cli/config"
+	"github.com/gravitl/netmaker/cli/functions"
 	"github.com/spf13/cobra"
 )
 
@@ -13,6 +15,8 @@ var (
 	password  string
 	masterKey string
 	sso       bool
+	tenantId  string
+	saas      bool
 )
 
 var contextSetCmd = &cobra.Command{
@@ -27,10 +31,28 @@ var contextSetCmd = &cobra.Command{
 			Password:  password,
 			MasterKey: masterKey,
 			SSO:       sso,
+			TenantId:  tenantId,
+			Saas:      saas,
 		}
-		if ctx.Username == "" && ctx.MasterKey == "" && !ctx.SSO {
-			cmd.Usage()
-			log.Fatal("Either username/password or master key is required")
+		if !ctx.Saas {
+			if ctx.Username == "" && ctx.MasterKey == "" && !ctx.SSO {
+				log.Fatal("Either username/password or master key is required")
+				cmd.Usage()
+			}
+			if ctx.Endpoint == "" {
+				log.Fatal("Endpoint is required when for self-hosted tenants")
+				cmd.Usage()
+			}
+		} else {
+			if ctx.TenantId == "" {
+				log.Fatal("Tenant ID is required for SaaS tenants")
+				cmd.Usage()
+			}
+			ctx.Endpoint = fmt.Sprintf(functions.TenantUrlTemplate, tenantId)
+			if ctx.Username == "" && ctx.Password == "" && !ctx.SSO {
+				log.Fatal("Username/password is required for non-SSO SaaS contexts")
+				cmd.Usage()
+			}
 		}
 		config.SetContext(args[0], ctx)
 	},
@@ -38,11 +60,12 @@ var contextSetCmd = &cobra.Command{
 
 func init() {
 	contextSetCmd.Flags().StringVar(&endpoint, "endpoint", "", "Endpoint of the API Server")
-	contextSetCmd.MarkFlagRequired("endpoint")
 	contextSetCmd.Flags().StringVar(&username, "username", "", "Username")
 	contextSetCmd.Flags().StringVar(&password, "password", "", "Password")
 	contextSetCmd.MarkFlagsRequiredTogether("username", "password")
-	contextSetCmd.Flags().BoolVar(&sso, "sso", false, "Login via Single Sign On (SSO) ?")
+	contextSetCmd.Flags().BoolVar(&sso, "sso", false, "Login via Single Sign On (SSO)?")
 	contextSetCmd.Flags().StringVar(&masterKey, "master_key", "", "Master Key")
+	contextSetCmd.Flags().StringVar(&tenantId, "tenant_id", "", "Tenant ID")
+	contextSetCmd.Flags().BoolVar(&saas, "saas", false, "Is this context for a SaaS tenant?")
 	rootCmd.AddCommand(contextSetCmd)
 }

cli/config/config.go

@@ -18,6 +18,8 @@ type Context struct {
 	Current   bool   `yaml:"current,omitempty"`
 	AuthToken string `yaml:"auth_token,omitempty"`
 	SSO       bool   `yaml:"sso,omitempty"`
+	TenantId  string `yaml:"tenant_id,omitempty"`
+	Saas      bool   `yaml:"saas,omitempty"`
 }
 
 var (

cli/functions/http_client.go

@@ -11,11 +11,19 @@ import (
 	"os"
 	"os/signal"
 	"strings"
+	"time"
 
 	"github.com/gorilla/websocket"
 	"github.com/gravitl/netmaker/cli/config"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
+	"golang.org/x/exp/slog"
+)
+
+const (
+	ambBaseUrl        = "https://api.accounts.netmaker.io"
+	TenantUrlTemplate = "https://api-%s.app.prod.netmaker.io"
+	ambOauthWssUrl    = "wss://api.accounts.netmaker.io/api/v1/auth/sso"
 )
 
 func ssoLogin(endpoint string) string {
@@ -81,34 +89,57 @@ func getAuthToken(ctx config.Context, force bool) string {
 	if !force && ctx.AuthToken != "" {
 		return ctx.AuthToken
 	}
-	if ctx.SSO {
-		authToken := ssoLogin(ctx.Endpoint)
+	if !ctx.Saas {
+		if ctx.SSO {
+			authToken := ssoLogin(ctx.Endpoint)
+			config.SetAuthToken(authToken)
+			return authToken
+		}
+		authParams := &models.UserAuthParams{UserName: ctx.Username, Password: ctx.Password}
+		payload, err := json.Marshal(authParams)
+		if err != nil {
+			log.Fatal(err)
+		}
+		res, err := http.Post(ctx.Endpoint+"/api/users/adm/authenticate", "application/json", bytes.NewReader(payload))
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer res.Body.Close()
+		resBodyBytes, err := io.ReadAll(res.Body)
+		if err != nil {
+			log.Fatalf("Client could not read response body: %s", err)
+		}
+		if res.StatusCode != http.StatusOK {
+			log.Fatalf("Error Status: %d Response: %s", res.StatusCode, string(resBodyBytes))
+		}
+		body := new(models.SuccessResponse)
+		if err := json.Unmarshal(resBodyBytes, body); err != nil {
+			log.Fatalf("Error unmarshalling JSON: %s", err)
+		}
+		authToken := body.Response.(map[string]any)["AuthToken"].(string)
 		config.SetAuthToken(authToken)
 		return authToken
 	}
-	authParams := &models.UserAuthParams{UserName: ctx.Username, Password: ctx.Password}
-	payload, err := json.Marshal(authParams)
-	if err != nil {
-		log.Fatal(err)
+
+	if !ctx.SSO {
+		sToken, _, err := basicAuthSaasSignin(ctx.Username, ctx.Password)
+		if err != nil {
+			log.Fatal(err)
+		}
+		authToken, _, err := tenantLogin(ctx, sToken)
+		if err != nil {
+			log.Fatal(err)
+		}
+		config.SetAuthToken(authToken)
+		return authToken
 	}
-	res, err := http.Post(ctx.Endpoint+"/api/users/adm/authenticate", "application/json", bytes.NewReader(payload))
+
+	accessToken, err := loginSaaSOauth(&models.SsoLoginReqDto{OauthProvider: "oidc"}, ctx.TenantId)
 	if err != nil {
 		log.Fatal(err)
 	}
-	resBodyBytes, err := io.ReadAll(res.Body)
-	if err != nil {
-		log.Fatalf("Client could not read response body: %s", err)
-	}
-	if res.StatusCode != http.StatusOK {
-		log.Fatalf("Error Status: %d Response: %s", res.StatusCode, string(resBodyBytes))
-	}
-	body := new(models.SuccessResponse)
-	if err := json.Unmarshal(resBodyBytes, body); err != nil {
-		log.Fatalf("Error unmarshalling JSON: %s", err)
-	}
-	authToken := body.Response.(map[string]any)["AuthToken"].(string)
-	config.SetAuthToken(authToken)
-	return authToken
+	config.SetAuthToken(accessToken)
+	return accessToken
 }
 
 func request[T any](method, route string, payload any) *T {
@@ -188,3 +219,213 @@ func get(route string) string {
 	}
 	return string(bodyBytes)
 }
+
+func basicAuthSaasSignin(email, password string) (string, http.Header, error) {
+	payload := models.SignInReqDto{
+		FormFields: []models.FormField{
+			{
+				Id:    "email",
+				Value: email,
+			},
+			{
+				Id:    "password",
+				Value: password,
+			},
+		},
+	}
+
+	var res models.SignInResDto
+
+	// Create a new HTTP client with a timeout
+	client := &http.Client{
+		Timeout: 30 * time.Second,
+	}
+
+	// Create the request body
+	payloadBuf := new(bytes.Buffer)
+	json.NewEncoder(payloadBuf).Encode(payload)
+
+	// Create the request
+	req, err := http.NewRequest("POST", ambBaseUrl+"/auth/signin", payloadBuf)
+	if err != nil {
+		return "", http.Header{}, err
+	}
+	req.Header.Set("Content-Type", "application/json; charset=utf-8")
+	req.Header.Set("rid", "thirdpartyemailpassword")
+
+	// Send the request
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", http.Header{}, err
+	}
+	defer resp.Body.Close()
+
+	// Check the response status code
+	if resp.StatusCode != http.StatusOK {
+		return "", http.Header{}, fmt.Errorf("error authenticating: %s", resp.Status)
+	}
+
+	// Copy the response headers
+	resHeaders := resp.Header
+
+	// Decode the response body
+	err = json.NewDecoder(resp.Body).Decode(&res)
+	if err != nil {
+		return "", http.Header{}, err
+	}
+
+	sToken := resHeaders.Get(models.ResHeaderKeyStAccessToken)
+	encodedAccessToken := url.QueryEscape(sToken)
+
+	return encodedAccessToken, resHeaders, nil
+}
+
+func tenantLogin(ctx config.Context, sToken string) (string, string, error) {
+	url := fmt.Sprintf("%s/api/v1/tenant/login?tenant_id=%s", ambBaseUrl, ctx.TenantId)
+
+	client := &http.Client{}
+	req, err := http.NewRequest(http.MethodPost, url, nil)
+
+	if err != nil {
+		return "", "", err
+	}
+	req.Header.Add("Cookie", fmt.Sprintf("sAccessToken=%s", sToken))
+
+	res, err := client.Do(req)
+	if err != nil {
+		return "", "", err
+	}
+	defer res.Body.Close()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		return "", "", err
+	}
+
+	data := models.TenantLoginResDto{}
+	json.Unmarshal(body, &data)
+
+	return data.Response.AuthToken, fmt.Sprintf(TenantUrlTemplate, ctx.TenantId), nil
+}
+
+func loginSaaSOauth(payload *models.SsoLoginReqDto, tenantId string) (string, error) {
+	socketUrl := ambOauthWssUrl
+	// Dial the netmaker server controller
+	conn, _, err := websocket.DefaultDialer.Dial(socketUrl, nil)
+	if err != nil {
+		slog.Error("error connecting to endpoint ", "url", socketUrl, "err", err)
+		return "", err
+	}
+
+	defer conn.Close()
+	return handleServerSSORegisterConn(payload, conn, tenantId)
+}
+
+func handleServerSSORegisterConn(payload *models.SsoLoginReqDto, conn *websocket.Conn, tenantId string) (string, error) {
+	reqData, err := json.Marshal(payload)
+	if err != nil {
+		return "", err
+	}
+	if err := conn.WriteMessage(websocket.TextMessage, reqData); err != nil {
+		return "", err
+	}
+	dataCh := make(chan string)
+	defer close(dataCh)
+	interrupt := make(chan os.Signal, 1)
+	signal.Notify(interrupt, os.Interrupt)
+
+	go func() {
+		for {
+			msgType, msg, err := conn.ReadMessage()
+			if err != nil {
+				if msgType < 0 {
+					slog.Info("received close message from server")
+					return
+				}
+				if !strings.Contains(err.Error(), "normal") { // Error reading a message from the server
+					slog.Error("error msg", "err", err)
+				}
+				return
+			}
+			if msgType == websocket.CloseMessage {
+				slog.Info("received close message from server")
+				return
+			}
+			if strings.Contains(string(msg), "auth/sso") {
+				fmt.Printf("Please visit:\n %s \nto authenticate\n", string(msg))
+			} else {
+				var res models.SsoLoginData
+				if err := json.Unmarshal(msg, &res); err != nil {
+					return
+				}
+				accessToken, _, err := tenantLoginV2(res.AmbAccessToken, tenantId, res.Username)
+				if err != nil {
+					slog.Error("error logging in tenant", "err", err)
+					dataCh <- ""
+					return
+				}
+				dataCh <- accessToken
+				return
+			}
+		}
+	}()
+
+	for {
+		select {
+		case accessToken := <-dataCh:
+			if accessToken == "" {
+				slog.Info("error getting access token")
+				return "", fmt.Errorf("error getting access token")
+			}
+			return accessToken, nil
+		case <-time.After(30 * time.Second):
+			slog.Error("authentiation timed out")
+			os.Exit(1)
+		case <-interrupt:
+			slog.Info("interrupt received, closing connection")
+			// Cleanly close the connection by sending a close message and then
+			// waiting (with timeout) for the server to close the connection.
+			err := conn.WriteMessage(websocket.CloseMessage, websocket.FormatCloseMessage(websocket.CloseNormalClosure, ""))
+			if err != nil {
+				log.Fatal(err)
+			}
+			os.Exit(1)
+		}
+	}
+}
+
+func tenantLoginV2(ambJwt, tenantId, email string) (string, string, error) {
+	url := fmt.Sprintf("%s/api/v1/tenant/login/custom", ambBaseUrl)
+	payload := models.LoginReqDto{
+		Email:    email,
+		TenantID: tenantId,
+	}
+	payloadBuf := new(bytes.Buffer)
+	json.NewEncoder(payloadBuf).Encode(payload)
+
+	client := &http.Client{}
+	req, err := http.NewRequest("POST", url, payloadBuf)
+	if err != nil {
+		slog.Error("error creating request", "err", err)
+		return "", "", err
+	}
+	req.Header.Add("Authorization", fmt.Sprintf("Bearer %s", ambJwt))
+
+	res, err := client.Do(req)
+	if err != nil {
+		slog.Error("error sending request", "err", err)
+		return "", "", err
+	}
+	defer res.Body.Close()
+
+	body, err := io.ReadAll(res.Body)
+	if err != nil {
+		slog.Error("error reading response body", "err", err)
+		return "", "", err
+	}
+
+	data := models.TenantLoginResDto{}
+	json.Unmarshal(body, &data)
+
+	return data.Response.AuthToken, fmt.Sprintf(TenantUrlTemplate, tenantId), nil
+}

compose/docker-compose.netclient.yml

@@ -3,7 +3,7 @@ version: "3.4"
 services:
   netclient:
     container_name: netclient
-    image: 'gravitl/netclient:v0.21.2'
+    image: 'gravitl/netclient:v0.22.0'
     hostname: netmaker-1
     network_mode: host
     restart: on-failure

compose/docker-compose.yml

@@ -25,10 +25,6 @@ services:
       - COREDNS_ADDR=${SERVER_HOST}
       # Overrides SERVER_HOST if set. Useful for making HTTP available via different interfaces/networks.
       - SERVER_HTTP_HOST=api.${NM_DOMAIN}
-      # domain for your turn server
-      - TURN_SERVER_HOST=turn.${NM_DOMAIN}
-      # domain of the turn api server
-      - TURN_SERVER_API_HOST=https://turnapi.${NM_DOMAIN}
 
   netmaker-ui:
     container_name: netmaker-ui
@@ -60,6 +56,7 @@ services:
       - "443:443"
 
   coredns:
+    #network_mode: host
     container_name: coredns
     image: coredns/coredns:1.10.1
     command: -conf /root/dnsconfig/Corefile
@@ -82,22 +79,6 @@ services:
       - ./wait.sh:/mosquitto/config/wait.sh
       - mosquitto_logs:/mosquitto/log
       - mosquitto_data:/mosquitto/data
-
-  turn:
-    container_name: turn
-    image: gravitl/turnserver:v1.0.0
-    env_file: ./netmaker.env
-    environment:
-      # config-dependant vars
-      - USERNAME=${TURN_USERNAME}
-      - PASSWORD=${TURN_PASSWORD}
-      # domain for your turn server
-      - TURN_SERVER_HOST=turn.${NM_DOMAIN}
-    network_mode: "host"
-    volumes:
-      - turn_server:/etc/config
-    restart: always
-
 volumes:
   caddy_data: { } # runtime data for caddy
   caddy_conf: { } # configuration file for Caddy
@@ -105,4 +86,4 @@ volumes:
   dnsconfig: { } # storage for coredns
   mosquitto_logs: { } # storage for mqtt logs
   mosquitto_data: { } # storage for mqtt data
-  turn_server: { }
+

config/config.go

@@ -91,6 +91,7 @@ type ServerConfig struct {
 	Environment                string        `yaml:"environment"`
 	JwtValidityDuration        time.Duration `yaml:"jwt_validity_duration"`
 	RacAutoDisable             bool          `yaml:"rac_auto_disable"`
+	CacheEnabled               string        `yaml:"caching_enabled"`
 }
 
 // SQLConfig - Generic SQL Config

controllers/controller.go

@@ -41,7 +41,7 @@ func HandleRESTRequests(wg *sync.WaitGroup, ctx context.Context) {
 
 	// Currently allowed dev origin is all. Should change in prod
 	// should consider analyzing the allowed methods further
-	headersOk := handlers.AllowedHeaders([]string{"Access-Control-Allow-Origin", "X-Requested-With", "Content-Type", "authorization"})
+	headersOk := handlers.AllowedHeaders([]string{"Access-Control-Allow-Origin", "X-Requested-With", "Content-Type", "authorization", "From-Ui"})
 	originsOk := handlers.AllowedOrigins(strings.Split(servercfg.GetAllowedOrigin(), ","))
 	methodsOk := handlers.AllowedMethods([]string{http.MethodGet, http.MethodPut, http.MethodPost, http.MethodDelete})
 

controllers/dns.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"encoding/json"
+	"errors"
 	"fmt"
 	"net/http"
 
@@ -10,7 +11,6 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
-	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/servercfg"
 )
 
@@ -170,24 +170,17 @@ func createDNS(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
-	err = logic.SetDNS()
-	if err != nil {
-		logger.Log(0, r.Header.Get("user"),
-			fmt.Sprintf("Failed to set DNS entries on file: %v", err))
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
-		return
+	if servercfg.IsDNSMode() {
+		err = logic.SetDNS()
+		if err != nil {
+			logger.Log(0, r.Header.Get("user"),
+				fmt.Sprintf("Failed to set DNS entries on file: %v", err))
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+			return
+		}
 	}
+
 	logger.Log(1, "new DNS record added:", entry.Name)
-	if servercfg.IsMessageQueueBackend() {
-		go func() {
-			if err = mq.PublishPeerUpdate(); err != nil {
-				logger.Log(0, "failed to publish peer update after ACL update on", entry.Network)
-			}
-			if err := mq.PublishCustomDNS(&entry); err != nil {
-				logger.Log(0, "error publishing custom dns", err.Error())
-			}
-		}()
-	}
 	logger.Log(2, r.Header.Get("user"),
 		fmt.Sprintf("DNS entry is set: %+v", entry))
 	w.WriteHeader(http.StatusOK)
@@ -221,23 +214,17 @@ func deleteDNS(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	logger.Log(1, "deleted dns entry: ", entrytext)
-	err = logic.SetDNS()
-	if err != nil {
-		logger.Log(0, r.Header.Get("user"),
-			fmt.Sprintf("Failed to set DNS entries on file: %v", err))
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
-		return
+	if servercfg.IsDNSMode() {
+		err = logic.SetDNS()
+		if err != nil {
+			logger.Log(0, r.Header.Get("user"),
+				fmt.Sprintf("Failed to set DNS entries on file: %v", err))
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+			return
+		}
 	}
+
 	json.NewEncoder(w).Encode(entrytext + " deleted.")
-	go func() {
-		dns := models.DNSUpdate{
-			Action: models.DNSDeleteByName,
-			Name:   entrytext,
-		}
-		if err := mq.PublishDNSUpdate(params["network"], dns); err != nil {
-			logger.Log(0, "failed to publish dns update", err.Error())
-		}
-	}()
 
 }
 
@@ -271,7 +258,10 @@ func GetDNSEntry(domain string, network string) (models.DNSEntry, error) {
 func pushDNS(w http.ResponseWriter, r *http.Request) {
 	// Set header
 	w.Header().Set("Content-Type", "application/json")
-
+	if !servercfg.IsDNSMode() {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("DNS Mode is set to off"), "badrequest"))
+		return
+	}
 	err := logic.SetDNS()
 
 	if err != nil {

controllers/docs.go

@@ -10,7 +10,7 @@
 //
 //	Schemes: https
 //	BasePath: /
-//	Version: 0.21.2
+//	Version: 0.22.0
 //	Host: api.demo.netmaker.io
 //
 //	Consumes:

controllers/enrollmentkeys.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"time"
 
+	"github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 
@@ -115,6 +116,35 @@ func createEnrollmentKey(w http.ResponseWriter, r *http.Request) {
 	if enrollmentKeyBody.Expiration > 0 {
 		newTime = time.Unix(enrollmentKeyBody.Expiration, 0)
 	}
+	v := validator.New()
+	err = v.Struct(enrollmentKeyBody)
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"), "error validating request body: ",
+			err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("validation error: name length must be between 3 and 32: %w", err), "badrequest"))
+		return
+	}
+
+	if existingKeys, err := logic.GetAllEnrollmentKeys(); err != nil {
+		logger.Log(0, r.Header.Get("user"), "error validating request body: ",
+			err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	} else {
+		// check if any tags are duplicate
+		existingTags := make(map[string]struct{})
+		for _, existingKey := range existingKeys {
+			for _, t := range existingKey.Tags {
+				existingTags[t] = struct{}{}
+			}
+		}
+		for _, t := range enrollmentKeyBody.Tags {
+			if _, ok := existingTags[t]; ok {
+				logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("key names must be unique"), "badrequest"))
+				return
+			}
+		}
+	}
 
 	relayId := uuid.Nil
 	if enrollmentKeyBody.Relay != "" {
@@ -231,13 +261,6 @@ func handleHostRegister(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
-	// re-register host with turn just in case.
-	if servercfg.IsUsingTurn() {
-		err = logic.RegisterHostWithTurn(newHost.ID.String(), newHost.HostPass)
-		if err != nil {
-			logger.Log(0, "failed to register host with turn server: ", err.Error())
-		}
-	}
 	// check if host already exists
 	hostExists := false
 	if hostExists = logic.HostExists(&newHost); hostExists && len(enrollmentKey.Networks) == 0 {

controllers/ext_client.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
+	"github.com/gravitl/netmaker/servercfg"
 
 	"github.com/gravitl/netmaker/models"
 
@@ -216,18 +217,28 @@ func getExtClientConf(w http.ResponseWriter, r *http.Request) {
 	} else {
 		gwendpoint = fmt.Sprintf("%s:%d", host.EndpointIP.String(), host.ListenPort)
 	}
-	newAllowedIPs := network.AddressRange
-	if newAllowedIPs != "" && network.AddressRange6 != "" {
-		newAllowedIPs += ","
-	}
-	if network.AddressRange6 != "" {
-		newAllowedIPs += network.AddressRange6
-	}
-	if egressGatewayRanges, err := logic.GetEgressRangesOnNetwork(&client); err == nil {
-		for _, egressGatewayRange := range egressGatewayRanges {
-			newAllowedIPs += "," + egressGatewayRange
+	var newAllowedIPs string
+	if logic.IsInternetGw(gwnode) {
+		egressrange := "0.0.0.0/0"
+		if gwnode.Address6.IP != nil && client.Address6 != "" {
+			egressrange += "," + "::/0"
+		}
+		newAllowedIPs = egressrange
+	} else {
+		newAllowedIPs = network.AddressRange
+		if newAllowedIPs != "" && network.AddressRange6 != "" {
+			newAllowedIPs += ","
+		}
+		if network.AddressRange6 != "" {
+			newAllowedIPs += network.AddressRange6
+		}
+		if egressGatewayRanges, err := logic.GetEgressRangesOnNetwork(&client); err == nil {
+			for _, egressGatewayRange := range egressGatewayRanges {
+				newAllowedIPs += "," + egressGatewayRange
+			}
 		}
 	}
+
 	defaultDNS := ""
 	if client.DNS != "" {
 		defaultDNS = "DNS = " + client.DNS
@@ -345,30 +356,28 @@ func createExtClient(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 		userName = caller.UserName
-		if !caller.IsAdmin && !caller.IsSuperAdmin {
-			if _, ok := caller.RemoteGwIDs[nodeid]; !ok {
-				err = errors.New("permission denied")
-				slog.Error("failed to create extclient", "error", err)
-				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "forbidden"))
-				return
-			}
-			// check if user has a config already for remote access client
-			extclients, err := logic.GetNetworkExtClients(node.Network)
-			if err != nil {
-				slog.Error("failed to get extclients", "error", err)
-				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		if _, ok := caller.RemoteGwIDs[nodeid]; (!caller.IsAdmin && !caller.IsSuperAdmin) && !ok {
+			err = errors.New("permission denied")
+			slog.Error("failed to create extclient", "error", err)
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "forbidden"))
+			return
+		}
+		// check if user has a config already for remote access client
+		extclients, err := logic.GetNetworkExtClients(node.Network)
+		if err != nil {
+			slog.Error("failed to get extclients", "error", err)
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+			return
+		}
+		for _, extclient := range extclients {
+			if extclient.RemoteAccessClientID != "" &&
+				extclient.RemoteAccessClientID == customExtClient.RemoteAccessClientID && nodeid == extclient.IngressGatewayID {
+				// extclient on the gw already exists for the remote access client
+				err = errors.New("remote client config already exists on the gateway. it may have been created by another user with this same remote client machine")
+				slog.Error("failed to create extclient", "user", userName, "error", err)
+				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 				return
 			}
-			for _, extclient := range extclients {
-				if extclient.RemoteAccessClientID != "" &&
-					extclient.RemoteAccessClientID == customExtClient.RemoteAccessClientID && nodeid == extclient.IngressGatewayID {
-					// extclient on the gw already exists for the remote access client
-					err = errors.New("remote client config already exists on the gateway")
-					slog.Error("failed to create extclient", "user", userName, "error", err)
-					logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
-					return
-				}
-			}
 		}
 	}
 
@@ -413,11 +422,11 @@ func createExtClient(w http.ResponseWriter, r *http.Request) {
 	slog.Info("created extclient", "user", r.Header.Get("user"), "network", node.Network, "clientid", extclient.ClientID)
 	w.WriteHeader(http.StatusOK)
 	go func() {
-		if err := mq.PublishPeerUpdate(); err != nil {
+		if err := mq.PublishPeerUpdate(false); err != nil {
 			logger.Log(1, "error setting ext peers on "+nodeid+": "+err.Error())
 		}
-		if err := mq.PublishExtClientDNS(&extclient); err != nil {
-			logger.Log(1, "error publishing extclient dns", err.Error())
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
 		}
 	}()
 }
@@ -501,22 +510,37 @@ func updateExtClient(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	logger.Log(0, r.Header.Get("user"), "updated ext client", update.ClientID)
-	if sendPeerUpdate { // need to send a peer update to the ingress node as enablement of one of it's clients has changed
-		if ingressNode, err := logic.GetNodeByID(newclient.IngressGatewayID); err == nil {
-			if err = mq.PublishPeerUpdate(); err != nil {
-				logger.Log(1, "error setting ext peers on", ingressNode.ID.String(), ":", err.Error())
-			}
-		}
-	}
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(newclient)
-	if changedID {
-		go func() {
-			if err := mq.PublishExtClientDNSUpdate(oldExtClient, newclient, oldExtClient.Network); err != nil {
-				logger.Log(1, "error pubishing dns update for extcient update", err.Error())
+
+	go func() {
+		if changedID && servercfg.IsDNSMode() {
+			logic.SetDNS()
+		}
+		if sendPeerUpdate { // need to send a peer update to the ingress node as enablement of one of it's clients has changed
+			ingressNode, err := logic.GetNodeByID(newclient.IngressGatewayID)
+			if err == nil {
+				if err = mq.PublishPeerUpdate(false); err != nil {
+					logger.Log(1, "error setting ext peers on", ingressNode.ID.String(), ":", err.Error())
+				}
 			}
-		}()
-	}
+			if !update.Enabled {
+				ingressHost, err := logic.GetHost(ingressNode.HostID.String())
+				if err != nil {
+					slog.Error("Failed to get ingress host", "node", ingressNode.ID.String(), "error", err)
+					return
+				}
+				nodes, err := logic.GetAllNodes()
+				if err != nil {
+					slog.Error("Failed to get nodes", "error", err)
+					return
+				}
+				go mq.PublishSingleHostPeerUpdate(ingressHost, nodes, nil, []models.ExtClient{oldExtClient}, false)
+			}
+		}
+
+	}()
+
 }
 
 // swagger:route DELETE /api/extclients/{network}/{clientid} ext_client deleteExtClient
@@ -573,8 +597,8 @@ func deleteExtClient(w http.ResponseWriter, r *http.Request) {
 		if err := mq.PublishDeletedClientPeerUpdate(&extclient); err != nil {
 			logger.Log(1, "error setting ext peers on "+ingressnode.ID.String()+": "+err.Error())
 		}
-		if err = mq.PublishDeleteExtClientDNS(&extclient); err != nil {
-			logger.Log(1, "error publishing dns update for extclient deletion", err.Error())
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
 		}
 	}()
 

controllers/hosts.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"net/http"
 
+	"github.com/google/uuid"
 	"github.com/gorilla/mux"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
@@ -30,6 +31,7 @@ func hostHandlers(r *mux.Router) {
 	r.HandleFunc("/api/hosts/adm/authenticate", authenticateHost).Methods(http.MethodPost)
 	r.HandleFunc("/api/v1/host", Authorize(true, false, "host", http.HandlerFunc(pull))).Methods(http.MethodGet)
 	r.HandleFunc("/api/v1/host/{hostid}/signalpeer", Authorize(true, false, "host", http.HandlerFunc(signalPeer))).Methods(http.MethodPost)
+	r.HandleFunc("/api/v1/fallback/host/{hostid}", Authorize(true, false, "host", http.HandlerFunc(hostUpdateFallback))).Methods(http.MethodPut)
 	r.HandleFunc("/api/v1/auth-register/host", socketHandler)
 }
 
@@ -99,6 +101,16 @@ func pull(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+	for _, nodeID := range host.Nodes {
+		node, err := logic.GetNodeByID(nodeID)
+		if err != nil {
+			slog.Error("failed to get node:", "id", node.ID, "error", err)
+			continue
+		}
+		if node.FailedOverBy != uuid.Nil {
+			go logic.ResetFailedOverPeer(&node)
+		}
+	}
 	allNodes, err := logic.GetAllNodes()
 	if err != nil {
 		logger.Log(0, "failed to get nodes: ", hostID)
@@ -130,6 +142,8 @@ func pull(w http.ResponseWriter, r *http.Request) {
 		Peers:           hPU.Peers,
 		PeerIDs:         hPU.PeerIDs,
 		HostNetworkInfo: hPU.HostNetworkInfo,
+		EgressRoutes:    hPU.EgressRoutes,
+		FwUpdate:        hPU.FwUpdate,
 	}
 
 	logger.Log(1, hostID, "completed a pull")
@@ -181,20 +195,12 @@ func updateHost(w http.ResponseWriter, r *http.Request) {
 		logger.Log(0, r.Header.Get("user"), "failed to send host update: ", currHost.ID.String(), err.Error())
 	}
 	go func() {
-		if err := mq.PublishPeerUpdate(); err != nil {
+		if err := mq.PublishPeerUpdate(false); err != nil {
 			logger.Log(0, "fail to publish peer update: ", err.Error())
 		}
 		if newHost.Name != currHost.Name {
-			networks := logic.GetHostNetworks(currHost.ID.String())
-			if err := mq.PublishHostDNSUpdate(currHost, newHost, networks); err != nil {
-				var dnsError *models.DNSError
-				if errors.Is(err, dnsError) {
-					for _, message := range err.(models.DNSError).ErrorStrings {
-						logger.Log(0, message)
-					}
-				} else {
-					logger.Log(0, err.Error())
-				}
+			if servercfg.IsDNSMode() {
+				logic.SetDNS()
 			}
 		}
 	}()
@@ -205,6 +211,55 @@ func updateHost(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(apiHostData)
 }
 
+// swagger:route PUT /api/v1/fallback/host/{hostid} hosts hostUpdateFallback
+//
+// Updates a Netclient host on Netmaker server.
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: apiHostResponse
+func hostUpdateFallback(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	hostid := params["hostid"]
+	currentHost, err := logic.GetHost(hostid)
+	if err != nil {
+		slog.Error("error getting host", "id", hostid, "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+
+	var hostUpdate models.HostUpdate
+	err = json.NewDecoder(r.Body).Decode(&hostUpdate)
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"), "failed to update a host:", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	slog.Info("recieved host update", "name", hostUpdate.Host.Name, "id", hostUpdate.Host.ID)
+	switch hostUpdate.Action {
+	case models.CheckIn:
+		_ = mq.HandleHostCheckin(&hostUpdate.Host, currentHost)
+
+	case models.UpdateHost:
+
+		_ = logic.UpdateHostFromClient(&hostUpdate.Host, currentHost)
+		err := logic.UpsertHost(currentHost)
+		if err != nil {
+			slog.Error("failed to update host", "id", currentHost.ID, "error", err)
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+			return
+		}
+	case models.UpdateMetrics:
+		mq.UpdateMetricsFallBack(hostUpdate.Node.ID.String(), hostUpdate.NewMetrics)
+	}
+	logic.ReturnSuccessResponse(w, r, "updated host data")
+
+}
+
 // swagger:route DELETE /api/hosts/{hostid} hosts deleteHost
 //
 // Deletes a Netclient host from Netmaker server.
@@ -241,6 +296,12 @@ func deleteHost(w http.ResponseWriter, r *http.Request) {
 		go mq.PublishMqUpdatesForDeletedNode(node, false, gwClients)
 
 	}
+	if servercfg.GetBrokerType() == servercfg.EmqxBrokerType {
+		// delete EMQX credentials for host
+		if err := mq.DeleteEmqxUser(currHost.ID.String()); err != nil {
+			slog.Error("failed to remove host credentials from EMQX", "id", currHost.ID, "error", err)
+		}
+	}
 	if err = logic.RemoveHost(currHost, forceDelete); err != nil {
 		logger.Log(0, r.Header.Get("user"), "failed to delete a host:", err.Error())
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
@@ -299,8 +360,10 @@ func addHostToNetwork(w http.ResponseWriter, r *http.Request) {
 			Host:   *currHost,
 			Node:   *newNode,
 		})
-		mq.PublishPeerUpdate()
-		mq.HandleNewNodeDNS(currHost, newNode)
+		mq.PublishPeerUpdate(false)
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
+		}
 	}()
 	logger.Log(2, r.Header.Get("user"), fmt.Sprintf("added host %s to network %s", currHost.Name, network))
 	w.WriteHeader(http.StatusOK)
@@ -385,7 +448,12 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to delete node"), "internal"))
 		return
 	}
-	go mq.PublishMqUpdatesForDeletedNode(*node, true, gwClients)
+	go func() {
+		mq.PublishMqUpdatesForDeletedNode(*node, true, gwClients)
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
+		}
+	}()
 	logger.Log(2, r.Header.Get("user"), fmt.Sprintf("removed host %s from network %s", currHost.Name, network))
 	w.WriteHeader(http.StatusOK)
 }
@@ -478,6 +546,27 @@ func authenticateHost(response http.ResponseWriter, request *http.Request) {
 		logic.ReturnErrorResponse(response, request, errorResponse)
 		return
 	}
+
+	// Create EMQX creds and ACLs if not found
+	if servercfg.GetBrokerType() == servercfg.EmqxBrokerType {
+		if err := mq.CreateEmqxUser(host.ID.String(), authRequest.Password, false); err != nil {
+			slog.Error("failed to create host credentials for EMQX: ", err.Error())
+		} else {
+			if err := mq.CreateHostACL(host.ID.String(), servercfg.GetServerInfo().Server); err != nil {
+				slog.Error("failed to add host ACL rules to EMQX: ", err.Error())
+			}
+			for _, nodeID := range host.Nodes {
+				if node, err := logic.GetNodeByID(nodeID); err == nil {
+					if err = mq.AppendNodeUpdateACL(host.ID.String(), node.Network, node.ID.String(), servercfg.GetServer()); err != nil {
+						slog.Error("failed to add ACLs for EMQX node", "error", err)
+					}
+				} else {
+					slog.Error("failed to get node", "nodeid", nodeID, "error", err)
+				}
+			}
+		}
+	}
+
 	response.WriteHeader(http.StatusOK)
 	response.Header().Set("Content-Type", "application/json")
 	response.Write(successJSONResponse)
@@ -512,39 +601,28 @@ func signalPeer(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 		return
 	}
-	if signal.ToHostPubKey == "" || signal.TurnRelayEndpoint == "" {
+	if signal.ToHostPubKey == "" {
 		msg := "insufficient data to signal peer"
 		logger.Log(0, r.Header.Get("user"), msg)
 		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New(msg), "badrequest"))
 		return
 	}
-	hosts, err := logic.GetAllHosts()
+	signal.IsPro = servercfg.IsPro
+	peerHost, err := logic.GetHost(signal.ToHostID)
 	if err != nil {
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to signal, peer not found"), "badrequest"))
 		return
 	}
-	// push the signal to host through mq
-	found := false
-	for _, hostI := range hosts {
-		if hostI.PublicKey.String() == signal.ToHostPubKey {
-			// found host publish message and break
-			found = true
-			err = mq.HostUpdate(&models.HostUpdate{
-				Action: models.SignalHost,
-				Host:   hostI,
-				Signal: signal,
-			})
-			if err != nil {
-				logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to publish signal to peer: "+err.Error()), "badrequest"))
-				return
-			}
-			break
-		}
-	}
-	if !found {
-		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to signal, peer not found"), "badrequest"))
+	err = mq.HostUpdate(&models.HostUpdate{
+		Action: models.SignalHost,
+		Host:   *peerHost,
+		Signal: signal,
+	})
+	if err != nil {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failed to publish signal to peer: "+err.Error()), "badrequest"))
 		return
 	}
+
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(signal)
 }

controllers/migrate.go

@@ -95,7 +95,7 @@ func migrate(w http.ResponseWriter, r *http.Request) {
 	if err := logic.UpsertHost(&host); err != nil {
 		slog.Error("save host", "error", err)
 	}
-	go mq.PublishPeerUpdate()
+	go mq.PublishPeerUpdate(false)
 	response := models.HostPull{
 		Host:         host,
 		Nodes:        nodes,
@@ -218,7 +218,5 @@ func convertLegacyNode(legacy models.LegacyNode, hostID uuid.UUID) models.Node {
 	node.IngressGatewayRange6 = legacy.IngressGatewayRange6
 	node.DefaultACL = legacy.DefaultACL
 	node.OwnerID = legacy.OwnerID
-	node.FailoverNode, _ = uuid.Parse(legacy.FailoverNode)
-	node.Failover = models.ParseBool(legacy.Failover)
 	return node
 }

controllers/network.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 	"strings"
 
@@ -16,7 +17,6 @@ import (
 	"github.com/gravitl/netmaker/logic/acls"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/mq"
-	"github.com/gravitl/netmaker/servercfg"
 )
 
 func networkHandlers(r *mux.Router) {
@@ -127,11 +127,12 @@ func updateNetworkACL(w http.ResponseWriter, r *http.Request) {
 	logger.Log(1, r.Header.Get("user"), "updated ACLs for network", netname)
 
 	// send peer updates
-	if servercfg.IsMessageQueueBackend() {
-		if err = mq.PublishPeerUpdate(); err != nil {
+	go func() {
+		if err = mq.PublishPeerUpdate(false); err != nil {
 			logger.Log(0, "failed to publish peer update after ACL update on", netname)
 		}
-	}
+	}()
+
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(newNetACL)
 }
@@ -246,6 +247,40 @@ func createNetwork(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// validate address ranges: must be private
+	if network.AddressRange != "" {
+		_, ipNet, err := net.ParseCIDR(network.AddressRange)
+		if err != nil {
+			logger.Log(0, r.Header.Get("user"), "failed to create network: ",
+				err.Error())
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+			return
+		}
+		if !ipNet.IP.IsPrivate() {
+			err := errors.New("address range must be private")
+			logger.Log(0, r.Header.Get("user"), "failed to create network: ",
+				err.Error())
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+			return
+		}
+	}
+	if network.AddressRange6 != "" {
+		_, ipNet, err := net.ParseCIDR(network.AddressRange6)
+		if err != nil {
+			logger.Log(0, r.Header.Get("user"), "failed to create network: ",
+				err.Error())
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+			return
+		}
+		if !ipNet.IP.IsPrivate() {
+			err := errors.New("address range must be private")
+			logger.Log(0, r.Header.Get("user"), "failed to create network: ",
+				err.Error())
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+			return
+		}
+	}
+
 	network, err = logic.CreateNetwork(network)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"), "failed to create network: ",

controllers/node.go

@@ -341,7 +341,6 @@ func getAllNodes(w http.ResponseWriter, r *http.Request) {
 func getNode(w http.ResponseWriter, r *http.Request) {
 	// set header.
 	w.Header().Set("Content-Type", "application/json")
-	nodeRequest := r.Header.Get("requestfrom") == "node"
 
 	var params = mux.Vars(r)
 	nodeid := params["nodeid"]
@@ -386,12 +385,6 @@ func getNode(w http.ResponseWriter, r *http.Request) {
 		PeerIDs:      hostPeerUpdate.PeerIDs,
 	}
 
-	if servercfg.IsPro && nodeRequest {
-		if err = logic.EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
-			logger.Log(1, "failed to reset failover list during node config pull", node.ID.String(), node.Network)
-		}
-	}
-
 	logger.Log(2, r.Header.Get("user"), "fetched node", params["nodeid"])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(response)
@@ -443,7 +436,7 @@ func createEgressGateway(w http.ResponseWriter, r *http.Request) {
 		if err := mq.NodeUpdate(&node); err != nil {
 			slog.Error("error publishing node update to node", "node", node.ID, "error", err)
 		}
-		mq.PublishPeerUpdate()
+		mq.PublishPeerUpdate(false)
 	}()
 }
 
@@ -486,7 +479,7 @@ func deleteEgressGateway(w http.ResponseWriter, r *http.Request) {
 		if err := mq.NodeUpdate(&node); err != nil {
 			slog.Error("error publishing node update to node", "node", node.ID, "error", err)
 		}
-		mq.PublishPeerUpdate()
+		mq.PublishPeerUpdate(false)
 	}()
 }
 
@@ -524,12 +517,6 @@ func createIngressGateway(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if servercfg.IsPro && request.Failover {
-		if err = logic.EnterpriseResetFailoverFunc(node.Network); err != nil {
-			logger.Log(1, "failed to reset failover list during failover create", node.ID.String(), node.Network)
-		}
-	}
-
 	apiNode := node.ConvertToAPINode()
 	logger.Log(1, r.Header.Get("user"), "created ingress gateway on node", nodeid, "on network", netid)
 	w.WriteHeader(http.StatusOK)
@@ -562,7 +549,7 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "bad request"))
 		return
 	}
-	node, wasFailover, removedClients, err := logic.DeleteIngressGateway(nodeid)
+	node, removedClients, err := logic.DeleteIngressGateway(nodeid)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
 			fmt.Sprintf("failed to delete ingress gateway on node [%s] on network [%s]: %v",
@@ -572,11 +559,6 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if servercfg.IsPro {
-		if wasFailover {
-			if err = logic.EnterpriseResetFailoverFunc(node.Network); err != nil {
-				logger.Log(1, "failed to reset failover list during failover create", node.ID.String(), node.Network)
-			}
-		}
 		go func() {
 			users, err := logic.GetUsersDB()
 			if err == nil {
@@ -608,13 +590,15 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 				return
 			}
 			go func() {
-				if err := mq.PublishSingleHostPeerUpdate(host, allNodes, nil, removedClients[:]); err != nil {
+				if err := mq.PublishSingleHostPeerUpdate(host, allNodes, nil, removedClients[:], false); err != nil {
 					slog.Error("publishSingleHostUpdate", "host", host.Name, "error", err)
 				}
 				if err := mq.NodeUpdate(&node); err != nil {
 					slog.Error("error publishing node update to node", "node", node.ID, "error", err)
 				}
-				mq.PublishDeleteAllExtclientsDNS(node.Network, removedClients)
+				if servercfg.IsDNSMode() {
+					logic.SetDNS()
+				}
 			}()
 		}
 	}
@@ -653,7 +637,7 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 	}
 	newNode := newData.ConvertToServerNode(&currentNode)
 	relayUpdate := logic.RelayUpdates(&currentNode, newNode)
-	host, err := logic.GetHost(newNode.HostID.String())
+	_, err = logic.GetHost(newNode.HostID.String())
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
 			fmt.Sprintf("failed to get host for node  [ %s ] info: %v", nodeid, err))
@@ -662,11 +646,6 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 	}
 	ifaceDelta := logic.IfaceDelta(&currentNode, newNode)
 	aclUpdate := currentNode.DefaultACL != newNode.DefaultACL
-	if ifaceDelta && servercfg.IsPro {
-		if err = logic.EnterpriseResetAllPeersFailovers(currentNode.ID, currentNode.Network); err != nil {
-			logger.Log(0, "failed to reset failover lists during node update for node", currentNode.ID.String(), currentNode.Network)
-		}
-	}
 
 	err = logic.UpdateNode(&currentNode, newNode)
 	if err != nil {
@@ -678,9 +657,6 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 	if relayUpdate {
 		logic.UpdateRelayed(&currentNode, newNode)
 	}
-	if servercfg.IsDNSMode() {
-		logic.SetDNS()
-	}
 
 	apiNode := newNode.ConvertToAPINode()
 	logger.Log(1, r.Header.Get("user"), "updated node", currentNode.ID.String(), "on network", currentNode.Network)
@@ -691,12 +667,12 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 			slog.Error("error publishing node update to node", "node", newNode.ID, "error", err)
 		}
 		if aclUpdate || relayupdate || ifaceDelta {
-			if err := mq.PublishPeerUpdate(); err != nil {
+			if err := mq.PublishPeerUpdate(false); err != nil {
 				logger.Log(0, "error during node ACL update for node", newNode.ID.String())
 			}
 		}
-		if err := mq.PublishReplaceDNS(&currentNode, newNode, host); err != nil {
-			logger.Log(1, "failed to publish dns update", err.Error())
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
 		}
 	}(aclUpdate, relayUpdate, newNode)
 }

controllers/node_test.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gravitl/netmaker/logic/acls"
 	"github.com/gravitl/netmaker/logic/acls/nodeacls"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 	"github.com/stretchr/testify/assert"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -217,7 +218,9 @@ func TestNodeACLs(t *testing.T) {
 }
 
 func deleteAllNodes() {
-	logic.ClearNodeCache()
+	if servercfg.CacheEnabled() {
+		logic.ClearNodeCache()
+	}
 	database.DeleteAllRecords(database.NODES_TABLE_NAME)
 }
 

controllers/server.go

@@ -4,8 +4,10 @@ import (
 	"encoding/json"
 	"net/http"
 	"strings"
+	"syscall"
 
 	"github.com/gorilla/mux"
+	"golang.org/x/exp/slog"
 
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic"
@@ -18,16 +20,26 @@ func serverHandlers(r *mux.Router) {
 	// r.HandleFunc("/api/server/addnetwork/{network}", securityCheckServer(true, http.HandlerFunc(addNetwork))).Methods(http.MethodPost)
 	r.HandleFunc(
 		"/api/server/health",
-		http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		func(resp http.ResponseWriter, req *http.Request) {
 			resp.WriteHeader(http.StatusOK)
 			resp.Write([]byte("Server is up and running!!"))
-		}),
-	)
+		},
+	).Methods(http.MethodGet)
+	r.HandleFunc(
+		"/api/server/shutdown",
+		func(w http.ResponseWriter, _ *http.Request) {
+			msg := "received api call to shutdown server, sending interruption..."
+			slog.Warn(msg)
+			_, _ = w.Write([]byte(msg))
+			w.WriteHeader(http.StatusOK)
+			_ = syscall.Kill(syscall.Getpid(), syscall.SIGINT)
+		},
+	).Methods(http.MethodPost)
 	r.HandleFunc("/api/server/getconfig", allowUsers(http.HandlerFunc(getConfig))).
 		Methods(http.MethodGet)
 	r.HandleFunc("/api/server/getserverinfo", Authorize(true, false, "node", http.HandlerFunc(getServerInfo))).
 		Methods(http.MethodGet)
-	r.HandleFunc("/api/server/status", http.HandlerFunc(getStatus)).Methods(http.MethodGet)
+	r.HandleFunc("/api/server/status", getStatus).Methods(http.MethodGet)
 	r.HandleFunc("/api/server/usage", Authorize(true, false, "user", http.HandlerFunc(getUsage))).
 		Methods(http.MethodGet)
 }

controllers/user.go

@@ -71,6 +71,20 @@ func authenticateUser(response http.ResponseWriter, request *http.Request) {
 		logic.ReturnErrorResponse(response, request, errorResponse)
 		return
 	}
+	if val := request.Header.Get("From-Ui"); val == "true" {
+		// request came from UI, if normal user block Login
+		user, err := logic.GetUser(authRequest.UserName)
+		if err != nil {
+			logger.Log(0, authRequest.UserName, "user validation failed: ",
+				err.Error())
+			logic.ReturnErrorResponse(response, request, logic.FormatError(err, "unauthorized"))
+			return
+		}
+		if !(user.IsAdmin || user.IsSuperAdmin) {
+			logic.ReturnErrorResponse(response, request, logic.FormatError(errors.New("only admins can access dashboard"), "unauthorized"))
+			return
+		}
+	}
 	username := authRequest.UserName
 	jwt, err := logic.VerifyAuthRequest(authRequest)
 	if err != nil {
@@ -119,12 +133,12 @@ func authenticateUser(response http.ResponseWriter, request *http.Request) {
 				if client.OwnerID == username && !client.Enabled {
 					slog.Info(fmt.Sprintf("enabling ext client %s for user %s due to RAC autodisabling feature", client.ClientID, client.OwnerID))
 					if newClient, err := logic.ToggleExtClientConnectivity(&client, true); err != nil {
-						slog.Error("error disabling ext client in RAC autodisable hook", "error", err)
+						slog.Error("error enabling ext client in RAC autodisable hook", "error", err)
 						continue // dont return but try for other clients
 					} else {
 						// publish peer update to ingress gateway
 						if ingressNode, err := logic.GetNodeByID(newClient.IngressGatewayID); err == nil {
-							if err = mq.PublishPeerUpdate(); err != nil {
+							if err = mq.PublishPeerUpdate(false); err != nil {
 								slog.Error("error updating ext clients on", "ingress", ingressNode.ID.String(), "err", err.Error())
 							}
 						}
@@ -329,6 +343,7 @@ func createUser(w http.ResponseWriter, r *http.Request) {
 	caller, err := logic.GetUser(r.Header.Get("user"))
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
 	}
 	var user models.User
 	err = json.NewDecoder(r.Body).Decode(&user)
@@ -350,6 +365,10 @@ func createUser(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "forbidden"))
 		return
 	}
+	if !servercfg.IsPro && !user.IsAdmin {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("non-admins users can only be created on Pro version"), "forbidden"))
+		return
+	}
 
 	err = logic.CreateUser(&user)
 	if err != nil {
@@ -502,14 +521,14 @@ func deleteUser(w http.ResponseWriter, r *http.Request) {
 	if user.IsSuperAdmin {
 		slog.Error(
 			"failed to delete user: ", "user", username, "error", "superadmin cannot be deleted")
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("superadmin cannot be deleted"), "internal"))
 		return
 	}
 	if !caller.IsSuperAdmin {
 		if caller.IsAdmin && user.IsAdmin {
 			slog.Error(
-				"failed to delete user: ", "user", username, "error", "admin cannot delete another admin user")
-			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+				"failed to delete user: ", "user", username, "error", "admin cannot delete another admin user, including oneself")
+			logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("admin cannot delete another admin user, including oneself"), "internal"))
 			return
 		}
 	}
@@ -541,6 +560,9 @@ func deleteUser(w http.ResponseWriter, r *http.Request) {
 				}
 			}
 		}
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
+		}
 	}()
 	logger.Log(1, username, "was deleted")
 	json.NewEncoder(w).Encode(params["username"] + " deleted.")

docker/Caddyfile

@@ -24,16 +24,6 @@ https://api.{$NM_DOMAIN} {
 	reverse_proxy http://netmaker:8081
 }
 
-# TURN
-https://turn.{$NM_DOMAIN} {
-	reverse_proxy host.docker.internal:3479
-}
-
-# TURN API
-https://turnapi.{$NM_DOMAIN} {
-	reverse_proxy http://host.docker.internal:8089
-}
-
 # MQ
 wss://broker.{$NM_DOMAIN} {
 	reverse_proxy ws://mq:8883 # For EMQX websockets use `reverse_proxy ws://mq:8083`

docker/Caddyfile-pro

@@ -39,16 +39,6 @@ https://api.{$NM_DOMAIN} {
 	reverse_proxy http://netmaker:8081
 }
 
-# TURN
-https://turn.{$NM_DOMAIN} {
-	reverse_proxy host.docker.internal:3479
-}
-
-# TURN API
-https://turnapi.{$NM_DOMAIN} {
-	reverse_proxy http://host.docker.internal:8089
-}
-
 # MQ
 wss://broker.{$NM_DOMAIN} {
 	reverse_proxy ws://mq:8883

go.mod

@@ -4,46 +4,45 @@ go 1.19
 
 require (
 	github.com/eclipse/paho.mqtt.golang v1.4.3
-	github.com/go-playground/validator/v10 v10.15.5
+	github.com/go-playground/validator/v10 v10.16.0
 	github.com/golang-jwt/jwt/v4 v4.5.0
-	github.com/google/uuid v1.4.0
-	github.com/gorilla/handlers v1.5.1
-	github.com/gorilla/mux v1.8.0
+	github.com/google/uuid v1.5.0
+	github.com/gorilla/handlers v1.5.2
+	github.com/gorilla/mux v1.8.1
 	github.com/lib/pq v1.10.9
-	github.com/mattn/go-sqlite3 v1.14.17
+	github.com/mattn/go-sqlite3 v1.14.19
 	github.com/rqlite/gorqlite v0.0.0-20210514125552-08ff1e76b22f
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
 	github.com/stretchr/testify v1.8.4
 	github.com/txn2/txeh v1.5.5
-	golang.org/x/crypto v0.14.0
-	golang.org/x/net v0.17.0 // indirect
-	golang.org/x/oauth2 v0.13.0
-	golang.org/x/sys v0.13.0 // indirect
-	golang.org/x/text v0.13.0 // indirect
+	golang.org/x/crypto v0.17.0
+	golang.org/x/net v0.19.0 // indirect
+	golang.org/x/oauth2 v0.15.0
+	golang.org/x/sys v0.15.0 // indirect
+	golang.org/x/text v0.14.0 // indirect
 	golang.zx2c4.com/wireguard/wgctrl v0.0.0-20221104135756-97bc4ad4a1cb
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
-	filippo.io/edwards25519 v1.0.0
+	filippo.io/edwards25519 v1.1.0
 	github.com/c-robinson/iplib v1.0.7
 	github.com/posthog/posthog-go v0.0.0-20211028072449-93c17c49e2b0
 )
 
 require (
-	github.com/coreos/go-oidc/v3 v3.7.0
-	github.com/gorilla/websocket v1.5.0
+	github.com/coreos/go-oidc/v3 v3.9.0
+	github.com/gorilla/websocket v1.5.1
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
 )
 
 require (
-	github.com/devilcove/httpclient v0.6.0
-	github.com/go-jose/go-jose/v3 v3.0.0
+	github.com/go-jose/go-jose/v3 v3.0.1
 	github.com/guumaster/tablewriter v0.0.10
 	github.com/matryer/is v1.4.1
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/spf13/cobra v1.7.0
+	github.com/spf13/cobra v1.8.0
 )
 
 require (

go.sum

@@ -2,36 +2,33 @@ cloud.google.com/go/compute v1.20.1 h1:6aKEtlUiwEpJzM001l0yFkpXmUVXaN8W+fbkb2AZN
 cloud.google.com/go/compute v1.20.1/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM=
 cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
 cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
-filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
-filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
+filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA=
+filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/c-robinson/iplib v1.0.7 h1:Dh9AINAlkc+NsNzZuFiVs+pi3AjN+0B7mu01KHdJKHU=
 github.com/c-robinson/iplib v1.0.7/go.mod h1:i3LuuFL1hRT5gFpBRnEydzw8R6yhGkF4szNDIbF8pgo=
-github.com/coreos/go-oidc/v3 v3.7.0 h1:FTdj0uexT4diYIPlF4yoFVI5MRO1r5+SEcIpEw9vC0o=
-github.com/coreos/go-oidc/v3 v3.7.0/go.mod h1:yQzSCqBnK3e6Fs5l+f5i0F8Kwf0zpH9bPEsbY00KanM=
+github.com/coreos/go-oidc/v3 v3.9.0 h1:0J/ogVOd4y8P0f0xUh8l9t07xRP/d8tccvjHl2dcsSo=
+github.com/coreos/go-oidc/v3 v3.9.0/go.mod h1:rTKz2PYwftcrtoCzV5g5kvfJoWcm0Mk8AF8y1iAQro4=
 github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU=
-github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
+github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/devilcove/httpclient v0.6.0 h1:M5YAfHeNbu+0QxCiOCo/fKN+Hf0BtF/6aovu3NNgcKk=
-github.com/devilcove/httpclient v0.6.0/go.mod h1:ctrAO2gRgTT+GxtRdWBp2SMQ+vacuxXlbhmlM4oWhs8=
 github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQNCyp70xik=
 github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE=
-github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/gabriel-vasile/mimetype v1.4.2 h1:w5qFW6JKBz9Y393Y4q372O9A7cUSequkh1Q7OhCmWKU=
 github.com/gabriel-vasile/mimetype v1.4.2/go.mod h1:zApsH/mKG4w07erKIaJPFiX0Tsq9BFQgN3qGY5GnNgA=
-github.com/go-jose/go-jose/v3 v3.0.0 h1:s6rrhirfEP/CGIoc6p+PZAeogN2SxKav6Wp7+dyMWVo=
-github.com/go-jose/go-jose/v3 v3.0.0/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
+github.com/go-jose/go-jose/v3 v3.0.1 h1:pWmKFVtt+Jl0vBZTIpz/eAKwsm6LkIxDVVbFHKkchhA=
+github.com/go-jose/go-jose/v3 v3.0.1/go.mod h1:RNkWWRld676jZEYoV3+XK8L2ZnNSvIsxFMht0mSX+u8=
 github.com/go-playground/assert/v2 v2.2.0 h1:JvknZsQTYeFEAhQwI4qEt9cyV5ONwRHC+lYKSsYSR8s=
 github.com/go-playground/locales v0.14.1 h1:EWaQ/wswjilfKLTECiXz7Rh+3BjFhfDFKv/oXslEjJA=
 github.com/go-playground/locales v0.14.1/go.mod h1:hxrqLVvrK65+Rwrd5Fc6F2O76J/NuW9t0sjnWqG1slY=
 github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJnYK9S473LQFuzCbDbfSFY=
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
-github.com/go-playground/validator/v10 v10.15.5 h1:LEBecTWb/1j5TNY1YYG2RcOUN3R7NLylN+x8TTueE24=
-github.com/go-playground/validator/v10 v10.15.5/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-playground/validator/v10 v10.16.0 h1:x+plE831WK4vaKHO/jpgUGsvLKIqRRkz6M78GuJAfGE=
+github.com/go-playground/validator/v10 v10.16.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
 github.com/golang-jwt/jwt/v4 v4.5.0 h1:7cYmW1XlMY7h7ii7UhUyChSgS5wUJEnm9uZVTGqOWzg=
 github.com/golang-jwt/jwt/v4 v4.5.0/go.mod h1:m21LjoU+eqJr34lmDMbreY2eSTRJ1cv77w39/MY0Ch0=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
@@ -41,14 +38,14 @@ github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38=
-github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4=
-github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4=
-github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
-github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
-github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
-github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
+github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/gorilla/handlers v1.5.2 h1:cLTUSsNkgcwhgRqvCNmdbRWG0A3N4F+M2nWKdScwyEE=
+github.com/gorilla/handlers v1.5.2/go.mod h1:dX+xVpaxdSw+q0Qek8SSsl3dfMk3jNddUkMzo0GtH0w=
+github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
+github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
+github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY=
+github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY=
 github.com/guumaster/tablewriter v0.0.10 h1:A0HD94yMdt4usgxBjoEceNeE0XMJ027euoHAzsPqBQs=
 github.com/guumaster/tablewriter v0.0.10/go.mod h1:p4FRFhyfo0UD9ZLmMRbbJooTUsxo6b80qZTERVDWrH8=
 github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek=
@@ -65,8 +62,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m
 github.com/mattn/go-runewidth v0.0.10/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk=
 github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU=
 github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w=
-github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM=
-github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
+github.com/mattn/go-sqlite3 v1.14.19 h1:fhGleo2h1p8tVChob4I9HpmVFIAkKGpiukdrgQbWfGI=
+github.com/mattn/go-sqlite3 v1.14.19/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
@@ -83,8 +80,8 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e h1:MRM5ITcdelLK2j1vwZ3Je0FKVCfqOLp5zO6trqMLYs0=
 github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e/go.mod h1:XV66xRDqSt+GTGFMVlhk3ULuV0y9ZmzeVGR4mloJI3M=
-github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I=
-github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0=
+github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0=
+github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho=
 github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA=
 github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -105,8 +102,8 @@ github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5t
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
 golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc=
-golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4=
+golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k=
+golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -114,10 +111,10 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM=
-golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE=
-golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY=
-golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0=
+golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c=
+golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U=
+golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ=
+golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
@@ -128,16 +125,16 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
-golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE=
-golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc=
+golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k=
-golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE=
+golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
+golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc=

k8s/client/netclient-daemonset.yaml

@@ -16,7 +16,7 @@ spec:
       hostNetwork: true
       containers:
       - name: netclient
-        image: gravitl/netclient:v0.21.2
+        image: gravitl/netclient:v0.22.0
         env:
         - name: TOKEN
           value: "TOKEN_VALUE"

k8s/client/netclient.yaml

@@ -28,7 +28,7 @@ spec:
       #           - "<node label value>"
       containers:
       - name: netclient
-        image: gravitl/netclient:v0.21.2
+        image: gravitl/netclient:v0.22.0
         env:
         - name: TOKEN
           value: "TOKEN_VALUE"

k8s/server/netmaker-ui.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: netmaker-ui
-        image: gravitl/netmaker-ui:v0.21.2
+        image: gravitl/netmaker-ui:v0.22.0
         ports:
         - containerPort: 443
         env:

logic/acls/common.go

@@ -5,6 +5,7 @@ import (
 	"sync"
 
 	"github.com/gravitl/netmaker/database"
+	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
 )
 
@@ -128,8 +129,10 @@ func (aclContainer ACLContainer) Get(containerID ContainerID) (ACLContainer, err
 func fetchACLContainer(containerID ContainerID) (ACLContainer, error) {
 	aclMutex.RLock()
 	defer aclMutex.RUnlock()
-	if aclContainer, ok := fetchAclContainerFromCache(containerID); ok {
-		return aclContainer, nil
+	if servercfg.CacheEnabled() {
+		if aclContainer, ok := fetchAclContainerFromCache(containerID); ok {
+			return aclContainer, nil
+		}
 	}
 	aclJson, err := fetchACLContainerJson(ContainerID(containerID))
 	if err != nil {
@@ -139,7 +142,9 @@ func fetchACLContainer(containerID ContainerID) (ACLContainer, error) {
 	if err := json.Unmarshal([]byte(aclJson), &currentNetworkACL); err != nil {
 		return nil, err
 	}
-	storeAclContainerInCache(containerID, currentNetworkACL)
+	if servercfg.CacheEnabled() {
+		storeAclContainerInCache(containerID, currentNetworkACL)
+	}
 	return currentNetworkACL, nil
 }
 
@@ -176,7 +181,9 @@ func upsertACLContainer(containerID ContainerID, aclContainer ACLContainer) (ACL
 	if err != nil {
 		return aclContainer, err
 	}
-	storeAclContainerInCache(containerID, aclContainer)
+	if servercfg.CacheEnabled() {
+		storeAclContainerInCache(containerID, aclContainer)
+	}
 	return aclContainer, nil
 }
 

logic/acls/nodeacls/modify.go

@@ -3,6 +3,7 @@ package nodeacls
 import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic/acls"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 // CreateNodeACL - inserts or updates a node ACL on given network and adds to state
@@ -87,6 +88,8 @@ func DeleteACLContainer(network NetworkID) error {
 	if err != nil {
 		return err
 	}
-	acls.DeleteAclFromCache(acls.ContainerID(network))
+	if servercfg.CacheEnabled() {
+		acls.DeleteAclFromCache(acls.ContainerID(network))
+	}
 	return nil
 }

logic/auth.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/go-playground/validator/v10"
 	"golang.org/x/crypto/bcrypt"
+	"golang.org/x/exp/slog"
 
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
@@ -165,11 +166,19 @@ func VerifyAuthRequest(authRequest models.UserAuthParams) (string, error) {
 	}
 
 	// Create a new JWT for the node
-	tokenString, _ := CreateUserJWT(authRequest.UserName, result.IsSuperAdmin, result.IsAdmin)
+	tokenString, err := CreateUserJWT(authRequest.UserName, result.IsSuperAdmin, result.IsAdmin)
+	if err != nil {
+		slog.Error("error creating jwt", "error", err)
+		return "", err
+	}
 
 	// update last login time
 	result.LastLoginTime = time.Now()
-	UpsertUser(result)
+	err = UpsertUser(result)
+	if err != nil {
+		slog.Error("error upserting user", "error", err)
+		return "", err
+	}
 
 	return tokenString, nil
 }
@@ -178,9 +187,11 @@ func VerifyAuthRequest(authRequest models.UserAuthParams) (string, error) {
 func UpsertUser(user models.User) error {
 	data, err := json.Marshal(&user)
 	if err != nil {
+		slog.Error("error marshalling user", "user", user.UserName, "error", err.Error())
 		return err
 	}
 	if err = database.Insert(user.UserName, string(data), database.USERS_TABLE_NAME); err != nil {
+		slog.Error("error inserting user", "user", user.UserName, "error", err.Error())
 		return err
 	}
 	return nil

logic/dns.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 	"github.com/txn2/txeh"
 )
 
@@ -36,6 +37,10 @@ func SetDNS() error {
 			hostfile.AddHost(entry.Address, entry.Name)
 		}
 	}
+	dns := GetExtclientDNS()
+	for _, entry := range dns {
+		hostfile.AddHost(entry.Address, entry.Name)
+	}
 	if corefilestring == "" {
 		corefilestring = "example.com"
 	}
@@ -69,6 +74,28 @@ func GetDNS(network string) ([]models.DNSEntry, error) {
 	return dns, nil
 }
 
+// GetExtclientDNS - gets all extclients dns entries
+func GetExtclientDNS() []models.DNSEntry {
+	extclients, err := GetAllExtClients()
+	if err != nil {
+		return []models.DNSEntry{}
+	}
+	var dns []models.DNSEntry
+	for _, extclient := range extclients {
+		var entry = models.DNSEntry{}
+		entry.Name = fmt.Sprintf("%s.%s", extclient.ClientID, extclient.Network)
+		entry.Network = extclient.Network
+		if extclient.Address != "" {
+			entry.Address = extclient.Address
+		}
+		if extclient.Address6 != "" {
+			entry.Address6 = extclient.Address6
+		}
+		dns = append(dns, entry)
+	}
+	return dns
+}
+
 // GetNodeDNS - gets the DNS of a network node
 func GetNodeDNS(network string) ([]models.DNSEntry, error) {
 
@@ -142,6 +169,7 @@ func SetCorefile(domains string) error {
 	}
 
 	corefile := domains + ` {
+	bind %s
     reload 15s
     hosts /root/dnsconfig/netmaker.hosts {
 	fallthrough	
@@ -150,8 +178,7 @@ func SetCorefile(domains string) error {
     log
 }
 `
-	corebytes := []byte(corefile)
-
+	corebytes := []byte(fmt.Sprintf(corefile, servercfg.GetCoreDNSAddr()))
 	err = os.WriteFile(dir+"/config/dnsconfig/Corefile", corebytes, 0644)
 	if err != nil {
 		return err

logic/enrollmentkey.go

@@ -22,7 +22,7 @@ var EnrollmentErrors = struct {
 	FailedToTokenize   error
 	FailedToDeTokenize error
 }{
-	InvalidCreate:      fmt.Errorf("invalid enrollment key created"),
+	InvalidCreate:      fmt.Errorf("failed to create enrollment key. paramters invalid"),
 	NoKeyFound:         fmt.Errorf("no enrollmentkey found"),
 	InvalidKey:         fmt.Errorf("invalid key provided"),
 	NoUsesRemaining:    fmt.Errorf("no uses remaining"),
@@ -61,8 +61,8 @@ func CreateEnrollmentKey(uses int, expiration time.Time, networks, tags []string
 	if len(tags) > 0 {
 		k.Tags = tags
 	}
-	if ok := k.Validate(); !ok {
-		return nil, EnrollmentErrors.InvalidCreate
+	if err := k.Validate(); err != nil {
+		return nil, err
 	}
 	if relay != uuid.Nil {
 		relayNode, err := GetNodeByID(relay.String())

logic/enrollmentkey_test.go

@@ -17,7 +17,7 @@ func TestCreateEnrollmentKey(t *testing.T) {
 		newKey, err := CreateEnrollmentKey(0, time.Time{}, nil, nil, false, uuid.Nil)
 		assert.Nil(t, newKey)
 		assert.NotNil(t, err)
-		assert.Equal(t, err, EnrollmentErrors.InvalidCreate)
+		assert.ErrorIs(t, err, models.ErrInvalidEnrollmentKey)
 	})
 	t.Run("Can_Create_Key_Uses", func(t *testing.T) {
 		newKey, err := CreateEnrollmentKey(1, time.Time{}, nil, nil, false, uuid.Nil)

logic/errors.go

@@ -44,6 +44,17 @@ func ReturnSuccessResponse(response http.ResponseWriter, request *http.Request,
 	json.NewEncoder(response).Encode(httpResponse)
 }
 
+// ReturnSuccessResponseWithJson - processes message and adds header
+func ReturnSuccessResponseWithJson(response http.ResponseWriter, request *http.Request, res interface{}, message string) {
+	var httpResponse models.SuccessResponse
+	httpResponse.Code = http.StatusOK
+	httpResponse.Response = res
+	httpResponse.Message = message
+	response.Header().Set("Content-Type", "application/json")
+	response.WriteHeader(http.StatusOK)
+	json.NewEncoder(response).Encode(httpResponse)
+}
+
 // ReturnErrorResponse - processes error and adds header
 func ReturnErrorResponse(response http.ResponseWriter, request *http.Request, errorMessage models.ErrorResponse) {
 	httpResponse := &models.ErrorResponse{Code: errorMessage.Code, Message: errorMessage.Message}

logic/extpeers.go

@@ -11,6 +11,7 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -80,21 +81,25 @@ func DeleteExtClient(network string, clientid string) error {
 	if err != nil {
 		return err
 	}
-	deleteExtClientFromCache(key)
+	if servercfg.CacheEnabled() {
+		deleteExtClientFromCache(key)
+	}
 	return nil
 }
 
 // GetNetworkExtClients - gets the ext clients of given network
 func GetNetworkExtClients(network string) ([]models.ExtClient, error) {
 	var extclients []models.ExtClient
-	allextclients := getAllExtClientsFromCache()
-	if len(allextclients) != 0 {
-		for _, extclient := range allextclients {
-			if extclient.Network == network {
-				extclients = append(extclients, extclient)
+	if servercfg.CacheEnabled() {
+		allextclients := getAllExtClientsFromCache()
+		if len(allextclients) != 0 {
+			for _, extclient := range allextclients {
+				if extclient.Network == network {
+					extclients = append(extclients, extclient)
+				}
 			}
+			return extclients, nil
 		}
-		return extclients, nil
 	}
 	records, err := database.FetchRecords(database.EXT_CLIENT_TABLE_NAME)
 	if err != nil {
@@ -111,7 +116,9 @@ func GetNetworkExtClients(network string) ([]models.ExtClient, error) {
 		}
 		key, err := GetRecordKey(extclient.ClientID, extclient.Network)
 		if err == nil {
-			storeExtClientInCache(key, extclient)
+			if servercfg.CacheEnabled() {
+				storeExtClientInCache(key, extclient)
+			}
 		}
 		if extclient.Network == network {
 			extclients = append(extclients, extclient)
@@ -127,15 +134,19 @@ func GetExtClient(clientid string, network string) (models.ExtClient, error) {
 	if err != nil {
 		return extclient, err
 	}
-	if extclient, ok := getExtClientFromCache(key); ok {
-		return extclient, nil
+	if servercfg.CacheEnabled() {
+		if extclient, ok := getExtClientFromCache(key); ok {
+			return extclient, nil
+		}
 	}
 	data, err := database.FetchRecord(database.EXT_CLIENT_TABLE_NAME, key)
 	if err != nil {
 		return extclient, err
 	}
 	err = json.Unmarshal([]byte(data), &extclient)
-	storeExtClientInCache(key, extclient)
+	if servercfg.CacheEnabled() {
+		storeExtClientInCache(key, extclient)
+	}
 	return extclient, err
 }
 
@@ -235,7 +246,9 @@ func SaveExtClient(extclient *models.ExtClient) error {
 	if err = database.Insert(key, string(data), database.EXT_CLIENT_TABLE_NAME); err != nil {
 		return err
 	}
-	storeExtClientInCache(key, *extclient)
+	if servercfg.CacheEnabled() {
+		storeExtClientInCache(key, *extclient)
+	}
 	return SetNetworkNodesLastModified(extclient.Network)
 }
 
@@ -425,3 +438,35 @@ func getExtpeersExtraRoutes(network string) (egressRoutes []models.EgressNetwork
 	}
 	return
 }
+
+func GetExtclientAllowedIPs(client models.ExtClient) (allowedIPs []string) {
+	gwnode, err := GetNodeByID(client.IngressGatewayID)
+	if err != nil {
+		logger.Log(0,
+			fmt.Sprintf("failed to get ingress gateway node [%s] info: %v", client.IngressGatewayID, err))
+		return
+	}
+
+	network, err := GetParentNetwork(client.Network)
+	if err != nil {
+		logger.Log(1, "Could not retrieve Ingress Gateway Network", client.Network)
+		return
+	}
+	if IsInternetGw(gwnode) {
+		egressrange := "0.0.0.0/0"
+		if gwnode.Address6.IP != nil && client.Address6 != "" {
+			egressrange += "," + "::/0"
+		}
+		allowedIPs = []string{egressrange}
+	} else {
+		allowedIPs = []string{network.AddressRange}
+
+		if network.AddressRange6 != "" {
+			allowedIPs = append(allowedIPs, network.AddressRange6)
+		}
+		if egressGatewayRanges, err := GetEgressRangesOnNetwork(&client); err == nil {
+			allowedIPs = append(allowedIPs, egressGatewayRanges...)
+		}
+	}
+	return
+}

logic/gateway.go

@@ -8,7 +8,16 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
-	"github.com/gravitl/netmaker/servercfg"
+)
+
+var (
+	// SetInternetGw - sets the node as internet gw based on flag bool
+	SetInternetGw = func(node *models.Node, flag bool) {
+	}
+	// IsInternetGw - checks if node is acting as internet gw
+	IsInternetGw = func(node models.Node) bool {
+		return false
+	}
 )
 
 // GetInternetGateways - gets all the nodes that are internet gateways
@@ -79,12 +88,8 @@ func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, erro
 	}
 	for i := len(gateway.Ranges) - 1; i >= 0; i-- {
 		// check if internet gateway IPv4
-		if gateway.Ranges[i] == "0.0.0.0/0" && FreeTier {
-			return models.Node{}, fmt.Errorf("currently IPv4 internet gateways are not supported on the free tier: %s", gateway.Ranges[i])
-		}
-		// check if internet gateway IPv6
-		if gateway.Ranges[i] == "::/0" {
-			return models.Node{}, fmt.Errorf("currently IPv6 internet gateways are not supported: %s", gateway.Ranges[i])
+		if gateway.Ranges[i] == "0.0.0.0/0" || gateway.Ranges[i] == "::/0" {
+			return models.Node{}, fmt.Errorf("create internet gateways on the remote client gateway")
 		}
 		normalized, err := NormalizeCIDR(gateway.Ranges[i])
 		if err != nil {
@@ -164,13 +169,11 @@ func CreateIngressGateway(netid string, nodeid string, ingress models.IngressReq
 		return models.Node{}, err
 	}
 	node.IsIngressGateway = true
+	SetInternetGw(&node, ingress.IsInternetGateway)
 	node.IngressGatewayRange = network.AddressRange
 	node.IngressGatewayRange6 = network.AddressRange6
 	node.IngressDNS = ingress.ExtclientDNS
 	node.SetLastModified()
-	if ingress.Failover && servercfg.IsPro {
-		node.Failover = true
-	}
 	err = UpsertNode(&node)
 	if err != nil {
 		return models.Node{}, err
@@ -199,35 +202,34 @@ func GetIngressGwUsers(node models.Node) (models.IngressGwUsers, error) {
 }
 
 // DeleteIngressGateway - deletes an ingress gateway
-func DeleteIngressGateway(nodeid string) (models.Node, bool, []models.ExtClient, error) {
+func DeleteIngressGateway(nodeid string) (models.Node, []models.ExtClient, error) {
 	removedClients := []models.ExtClient{}
 	node, err := GetNodeByID(nodeid)
 	if err != nil {
-		return models.Node{}, false, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 	clients, err := GetExtClientsByID(nodeid, node.Network)
 	if err != nil && !database.IsEmptyRecord(err) {
-		return models.Node{}, false, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 
 	removedClients = clients
 
 	// delete ext clients belonging to ingress gateway
 	if err = DeleteGatewayExtClients(node.ID.String(), node.Network); err != nil {
-		return models.Node{}, false, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 	logger.Log(3, "deleting ingress gateway")
-	wasFailover := node.Failover
 	node.LastModified = time.Now()
 	node.IsIngressGateway = false
+	node.IsInternetGateway = false
 	node.IngressGatewayRange = ""
-	node.Failover = false
 	err = UpsertNode(&node)
 	if err != nil {
-		return models.Node{}, wasFailover, removedClients, err
+		return models.Node{}, removedClients, err
 	}
 	err = SetNetworkNodesLastModified(node.Network)
-	return node, wasFailover, removedClients, err
+	return node, removedClients, err
 }
 
 // DeleteGatewayExtClients - deletes ext clients based on gateway (mac) of ingress node and network

logic/hosts.go

@@ -2,16 +2,12 @@ package logic
 
 import (
 	"crypto/md5"
-	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
-	"net/http"
 	"sort"
-	"strconv"
 	"sync"
 
-	"github.com/devilcove/httpclient"
 	"github.com/google/uuid"
 	"golang.org/x/crypto/bcrypt"
 
@@ -81,16 +77,21 @@ const (
 
 // GetAllHosts - returns all hosts in flat list or error
 func GetAllHosts() ([]models.Host, error) {
-	currHosts := getHostsFromCache()
-	if len(currHosts) != 0 {
-		return currHosts, nil
+	var currHosts []models.Host
+	if servercfg.CacheEnabled() {
+		currHosts := getHostsFromCache()
+		if len(currHosts) != 0 {
+			return currHosts, nil
+		}
 	}
 	records, err := database.FetchRecords(database.HOSTS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return nil, err
 	}
 	currHostsMap := make(map[string]models.Host)
-	defer loadHostsIntoCache(currHostsMap)
+	if servercfg.CacheEnabled() {
+		defer loadHostsIntoCache(currHostsMap)
+	}
 	for k := range records {
 		var h models.Host
 		err = json.Unmarshal([]byte(records[k]), &h)
@@ -116,16 +117,20 @@ func GetAllHostsAPI(hosts []models.Host) []models.ApiHost {
 
 // GetHostsMap - gets all the current hosts on machine in a map
 func GetHostsMap() (map[string]models.Host, error) {
-	hostsMap := getHostsMapFromCache()
-	if len(hostsMap) != 0 {
-		return hostsMap, nil
+	if servercfg.CacheEnabled() {
+		hostsMap := getHostsMapFromCache()
+		if len(hostsMap) != 0 {
+			return hostsMap, nil
+		}
 	}
 	records, err := database.FetchRecords(database.HOSTS_TABLE_NAME)
 	if err != nil && !database.IsEmptyRecord(err) {
 		return nil, err
 	}
 	currHostMap := make(map[string]models.Host)
-	defer loadHostsIntoCache(currHostMap)
+	if servercfg.CacheEnabled() {
+		defer loadHostsIntoCache(currHostMap)
+	}
 	for k := range records {
 		var h models.Host
 		err = json.Unmarshal([]byte(records[k]), &h)
@@ -140,8 +145,10 @@ func GetHostsMap() (map[string]models.Host, error) {
 
 // GetHost - gets a host from db given id
 func GetHost(hostid string) (*models.Host, error) {
-	if host, ok := getHostFromCache(hostid); ok {
-		return &host, nil
+	if servercfg.CacheEnabled() {
+		if host, ok := getHostFromCache(hostid); ok {
+			return &host, nil
+		}
 	}
 	record, err := database.FetchRecord(database.HOSTS_TABLE_NAME, hostid)
 	if err != nil {
@@ -152,10 +159,27 @@ func GetHost(hostid string) (*models.Host, error) {
 	if err = json.Unmarshal([]byte(record), &h); err != nil {
 		return nil, err
 	}
-	storeHostInCache(h)
+	if servercfg.CacheEnabled() {
+		storeHostInCache(h)
+	}
+
 	return &h, nil
 }
 
+// GetHostByPubKey - gets a host from db given pubkey
+func GetHostByPubKey(hostPubKey string) (*models.Host, error) {
+	hosts, err := GetAllHosts()
+	if err != nil {
+		return nil, err
+	}
+	for _, host := range hosts {
+		if host.PublicKey.String() == hostPubKey {
+			return &host, nil
+		}
+	}
+	return nil, errors.New("host not found")
+}
+
 // CreateHost - creates a host if not exist
 func CreateHost(h *models.Host) error {
 	hosts, hErr := GetAllHosts()
@@ -169,12 +193,6 @@ func CreateHost(h *models.Host) error {
 	if (err != nil && !database.IsEmptyRecord(err)) || (err == nil) {
 		return ErrHostExists
 	}
-	if servercfg.IsUsingTurn() {
-		err = RegisterHostWithTurn(h.ID.String(), h.HostPass)
-		if err != nil {
-			logger.Log(0, "failed to register host with turn server: ", err.Error())
-		}
-	}
 
 	// encrypt that password so we never see it
 	hash, err := bcrypt.GenerateFromPassword([]byte(h.HostPass), 5)
@@ -265,7 +283,10 @@ func UpsertHost(h *models.Host) error {
 	if err != nil {
 		return err
 	}
-	storeHostInCache(*h)
+	if servercfg.CacheEnabled() {
+		storeHostInCache(*h)
+	}
+
 	return nil
 }
 
@@ -275,10 +296,6 @@ func RemoveHost(h *models.Host, forceDelete bool) error {
 		return fmt.Errorf("host still has associated nodes")
 	}
 
-	if servercfg.IsUsingTurn() {
-		DeRegisterHostWithTurn(h.ID.String())
-	}
-
 	if len(h.Nodes) > 0 {
 		if err := DisassociateAllNodesFromHost(h.ID.String()); err != nil {
 			return err
@@ -289,22 +306,28 @@ func RemoveHost(h *models.Host, forceDelete bool) error {
 	if err != nil {
 		return err
 	}
+	if servercfg.CacheEnabled() {
+		deleteHostFromCache(h.ID.String())
+	}
+	go func() {
+		if servercfg.IsDNSMode() {
+			SetDNS()
+		}
+	}()
 
-	deleteHostFromCache(h.ID.String())
 	return nil
 }
 
 // RemoveHostByID - removes a given host by id from server
 func RemoveHostByID(hostID string) error {
-	if servercfg.IsUsingTurn() {
-		DeRegisterHostWithTurn(hostID)
-	}
 
 	err := database.DeleteRecord(database.HOSTS_TABLE_NAME, hostID)
 	if err != nil {
 		return err
 	}
-	deleteHostFromCache(hostID)
+	if servercfg.CacheEnabled() {
+		deleteHostFromCache(hostID)
+	}
 	return nil
 }
 
@@ -533,52 +556,6 @@ func ConvHostPassToHash(hostPass string) string {
 	return fmt.Sprintf("%x", md5.Sum([]byte(hostPass)))
 }
 
-// RegisterHostWithTurn - registers the host with the given turn server
-func RegisterHostWithTurn(hostID, hostPass string) error {
-	auth := servercfg.GetTurnUserName() + ":" + servercfg.GetTurnPassword()
-	api := httpclient.JSONEndpoint[models.SuccessResponse, models.ErrorResponse]{
-		URL:           servercfg.GetTurnApiHost(),
-		Route:         "/api/v1/host/register",
-		Method:        http.MethodPost,
-		Authorization: fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(auth))),
-		Data: models.HostTurnRegister{
-			HostID:       hostID,
-			HostPassHash: ConvHostPassToHash(hostPass),
-		},
-		Response:      models.SuccessResponse{},
-		ErrorResponse: models.ErrorResponse{},
-	}
-	_, errData, err := api.GetJSON(models.SuccessResponse{}, models.ErrorResponse{})
-	if err != nil {
-		if errors.Is(err, httpclient.ErrStatus) {
-			logger.Log(1, "error server status", strconv.Itoa(errData.Code), errData.Message)
-		}
-		return err
-	}
-	return nil
-}
-
-// DeRegisterHostWithTurn - to be called when host need to be deregistered from a turn server
-func DeRegisterHostWithTurn(hostID string) error {
-	auth := servercfg.GetTurnUserName() + ":" + servercfg.GetTurnPassword()
-	api := httpclient.JSONEndpoint[models.SuccessResponse, models.ErrorResponse]{
-		URL:           servercfg.GetTurnApiHost(),
-		Route:         fmt.Sprintf("/api/v1/host/deregister?host_id=%s", hostID),
-		Method:        http.MethodPost,
-		Authorization: fmt.Sprintf("Basic %s", base64.StdEncoding.EncodeToString([]byte(auth))),
-		Response:      models.SuccessResponse{},
-		ErrorResponse: models.ErrorResponse{},
-	}
-	_, errData, err := api.GetJSON(models.SuccessResponse{}, models.ErrorResponse{})
-	if err != nil {
-		if errors.Is(err, httpclient.ErrStatus) {
-			logger.Log(1, "error server status", strconv.Itoa(errData.Code), errData.Message)
-		}
-		return err
-	}
-	return nil
-}
-
 // SortApiHosts - Sorts slice of ApiHosts by their ID alphabetically with numbers first
 func SortApiHosts(unsortedHosts []models.ApiHost) {
 	sort.Slice(unsortedHosts, func(i, j int) bool {

logic/nodes.go

@@ -119,7 +119,9 @@ func UpdateNodeCheckin(node *models.Node) error {
 	if err != nil {
 		return err
 	}
-	storeNodeInCache(*node)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(*node)
+	}
 	return nil
 }
 
@@ -134,7 +136,9 @@ func UpsertNode(newNode *models.Node) error {
 	if err != nil {
 		return err
 	}
-	storeNodeInCache(*newNode)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(*newNode)
+	}
 	return nil
 }
 
@@ -171,7 +175,9 @@ func UpdateNode(currentNode *models.Node, newNode *models.Node) error {
 			if err != nil {
 				return err
 			}
-			storeNodeInCache(*newNode)
+			if servercfg.CacheEnabled() {
+				storeNodeInCache(*newNode)
+			}
 			return nil
 		}
 	}
@@ -205,6 +211,9 @@ func DeleteNode(node *models.Node, purge bool) error {
 			UpsertNode(&relayNode)
 		}
 	}
+	if node.FailedOverBy != uuid.Nil {
+		ResetFailedOverPeer(node)
+	}
 	if node.IsRelay {
 		// unset all the relayed nodes
 		SetRelayedNodes(false, node.ID.String(), node.RelayedNodes)
@@ -233,11 +242,6 @@ func DeleteNode(node *models.Node, purge bool) error {
 	if err := DissasociateNodeFromHost(node, host); err != nil {
 		return err
 	}
-	if servercfg.IsPro {
-		if err := EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
-			logger.Log(0, "failed to reset failover lists during node delete for node", host.Name, node.Network)
-		}
-	}
 
 	return nil
 }
@@ -266,7 +270,9 @@ func DeleteNodeByID(node *models.Node) error {
 			return err
 		}
 	}
-	deleteNodeFromCache(node.ID.String())
+	if servercfg.CacheEnabled() {
+		deleteNodeFromCache(node.ID.String())
+	}
 	if servercfg.IsDNSMode() {
 		SetDNS()
 	}
@@ -309,29 +315,19 @@ func ValidateNode(node *models.Node, isUpdate bool) error {
 	return err
 }
 
-// IsFailoverPresent - checks if a node is marked as a failover in given network
-func IsFailoverPresent(network string) bool {
-	netNodes, err := GetNetworkNodes(network)
-	if err != nil {
-		return false
-	}
-	for i := range netNodes {
-		if netNodes[i].Failover {
-			return true
-		}
-	}
-	return false
-}
-
 // GetAllNodes - returns all nodes in the DB
 func GetAllNodes() ([]models.Node, error) {
 	var nodes []models.Node
-	nodes = getNodesFromCache()
-	if len(nodes) != 0 {
-		return nodes, nil
+	if servercfg.CacheEnabled() {
+		nodes = getNodesFromCache()
+		if len(nodes) != 0 {
+			return nodes, nil
+		}
 	}
 	nodesMap := make(map[string]models.Node)
-	defer loadNodesIntoCache(nodesMap)
+	if servercfg.CacheEnabled() {
+		defer loadNodesIntoCache(nodesMap)
+	}
 	collection, err := database.FetchRecords(database.NODES_TABLE_NAME)
 	if err != nil {
 		if database.IsEmptyRecord(err) {
@@ -385,6 +381,9 @@ func SetNodeDefaults(node *models.Node) {
 	if node.DefaultACL == "" {
 		node.DefaultACL = parentNetwork.DefaultACL
 	}
+	if node.FailOverPeers == nil {
+		node.FailOverPeers = make(map[string]struct{})
+	}
 
 	node.SetLastModified()
 	node.SetLastCheckIn()
@@ -402,8 +401,10 @@ func GetRecordKey(id string, network string) (string, error) {
 }
 
 func GetNodeByID(uuid string) (models.Node, error) {
-	if node, ok := getNodeFromCache(uuid); ok {
-		return node, nil
+	if servercfg.CacheEnabled() {
+		if node, ok := getNodeFromCache(uuid); ok {
+			return node, nil
+		}
 	}
 	var record, err = database.FetchRecord(database.NODES_TABLE_NAME, uuid)
 	if err != nil {
@@ -413,7 +414,9 @@ func GetNodeByID(uuid string) (models.Node, error) {
 	if err = json.Unmarshal([]byte(record), &node); err != nil {
 		return models.Node{}, err
 	}
-	storeNodeInCache(node)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(node)
+	}
 	return node, nil
 }
 
@@ -472,13 +475,8 @@ func DeleteExpiredNodes(ctx context.Context, peerUpdate chan *models.Node) {
 				return
 			}
 			for _, node := range allnodes {
+				node := node
 				if time.Now().After(node.ExpirationDateTime) {
-					if err := DeleteNode(&node, false); err != nil {
-						slog.Error("error deleting expired node", "nodeid", node.ID.String(), "error", err.Error())
-						continue
-					}
-					node.Action = models.NODE_DELETE
-					node.PendingDelete = true
 					peerUpdate <- &node
 					slog.Info("deleting expired node", "nodeid", node.ID.String())
 				}
@@ -569,7 +567,9 @@ func createNode(node *models.Node) error {
 	if err != nil {
 		return err
 	}
-	storeNodeInCache(*node)
+	if servercfg.CacheEnabled() {
+		storeNodeInCache(*node)
+	}
 	_, err = nodeacls.CreateNodeACL(nodeacls.NetworkID(node.Network), nodeacls.NodeID(node.ID.String()), defaultACLVal)
 	if err != nil {
 		logger.Log(1, "failed to create node ACL for node,", node.ID.String(), "err:", err.Error())

logic/peers.go

@@ -15,6 +15,17 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
+var (
+	// ResetFailOver - function to reset failOvered peers on this node
+	ResetFailOver = func(failOverNode *models.Node) error {
+		return nil
+	}
+	// ResetFailedOverPeer - removes failed over node from network peers
+	ResetFailedOverPeer = func(failedOverNode *models.Node) error {
+		return nil
+	}
+)
+
 // GetPeerUpdateForHost - gets the consolidated peer update for the host from all networks
 func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.Node,
 	deletedNode *models.Node, deletedClients []models.ExtClient) (models.HostPeerUpdate, error) {
@@ -132,7 +143,9 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 			if peer.IsIngressGateway {
 				hostPeerUpdate.EgressRoutes = append(hostPeerUpdate.EgressRoutes, getExtpeersExtraRoutes(peer.Network)...)
 			}
-			if (node.IsRelayed && node.RelayedBy != peer.ID.String()) || (peer.IsRelayed && peer.RelayedBy != node.ID.String()) {
+			_, isFailOverPeer := node.FailOverPeers[peer.ID.String()]
+			if (node.IsRelayed && node.RelayedBy != peer.ID.String()) ||
+				(peer.IsRelayed && peer.RelayedBy != node.ID.String()) || isFailOverPeer {
 				// if node is relayed and peer is not the relay, set remove to true
 				if _, ok := peerIndexMap[peerHost.PublicKey.String()]; ok {
 					continue
@@ -197,9 +210,10 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 				nodePeer = hostPeerUpdate.Peers[peerIndexMap[peerHost.PublicKey.String()]]
 			}
 
-			if node.Network == network { // add to peers map for metrics
+			if node.Network == network && !peerConfig.Remove && len(peerConfig.AllowedIPs) > 0 { // add to peers map for metrics
 				hostPeerUpdate.PeerIDs[peerHost.PublicKey.String()] = models.IDandAddr{
 					ID:         peer.ID.String(),
+					HostID:     peerHost.ID.String(),
 					Address:    peer.PrimaryAddress(),
 					Name:       peerHost.Name,
 					Network:    peer.Network,
@@ -227,8 +241,18 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 				logger.Log(1, "error retrieving external clients:", err.Error())
 			}
 		}
+		addedInetGwRanges := false
 		if node.IsEgressGateway && node.EgressGatewayRequest.NatEnabled == "yes" && len(node.EgressGatewayRequest.Ranges) > 0 {
 			hostPeerUpdate.FwUpdate.IsEgressGw = true
+			if IsInternetGw(node) {
+				hostPeerUpdate.FwUpdate.IsEgressGw = true
+				egressrange := []string{"0.0.0.0/0"}
+				if node.Address6.IP != nil {
+					egressrange = append(egressrange, "::/0")
+				}
+				node.EgressGatewayRequest.Ranges = append(node.EgressGatewayRequest.Ranges, egressrange...)
+				addedInetGwRanges = true
+			}
 			hostPeerUpdate.FwUpdate.EgressInfo[node.ID.String()] = models.EgressInfo{
 				EgressID: node.ID.String(),
 				Network:  node.PrimaryNetworkRange(),
@@ -238,6 +262,28 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 				},
 				EgressGWCfg: node.EgressGatewayRequest,
 			}
+
+		}
+		if IsInternetGw(node) && !addedInetGwRanges {
+			hostPeerUpdate.FwUpdate.IsEgressGw = true
+			egressrange := []string{"0.0.0.0/0"}
+			if node.Address6.IP != nil {
+				egressrange = append(egressrange, "::/0")
+			}
+			hostPeerUpdate.FwUpdate.EgressInfo[node.ID.String()] = models.EgressInfo{
+				EgressID: node.ID.String(),
+				Network:  node.PrimaryAddressIPNet(),
+				EgressGwAddr: net.IPNet{
+					IP:   net.ParseIP(node.PrimaryAddress()),
+					Mask: getCIDRMaskFromAddr(node.PrimaryAddress()),
+				},
+				EgressGWCfg: models.EgressGatewayRequest{
+					NodeID:     node.ID.String(),
+					NetID:      node.Network,
+					NatEnabled: "yes",
+					Ranges:     egressrange,
+				},
+			}
 		}
 	}
 	// == post peer calculations ==
@@ -316,6 +362,31 @@ func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet
 	return allowedips
 }
 
+func GetFailOverPeerIps(peer, node *models.Node) []net.IPNet {
+	allowedips := []net.IPNet{}
+	for failOverpeerID := range node.FailOverPeers {
+		failOverpeer, err := GetNodeByID(failOverpeerID)
+		if err == nil && failOverpeer.FailedOverBy == peer.ID {
+			if failOverpeer.Address.IP != nil {
+				allowed := net.IPNet{
+					IP:   failOverpeer.Address.IP,
+					Mask: net.CIDRMask(32, 32),
+				}
+				allowedips = append(allowedips, allowed)
+			}
+			if failOverpeer.Address6.IP != nil {
+				allowed := net.IPNet{
+					IP:   failOverpeer.Address6.IP,
+					Mask: net.CIDRMask(128, 128),
+				}
+				allowedips = append(allowedips, allowed)
+			}
+
+		}
+	}
+	return allowedips
+}
+
 func GetEgressIPs(peer *models.Node) []net.IPNet {
 
 	peerHost, err := GetHost(peer.HostID.String())
@@ -379,6 +450,9 @@ func getNodeAllowedIPs(peer, node *models.Node) []net.IPNet {
 	if peer.IsRelay {
 		allowedips = append(allowedips, RelayedAllowedIPs(peer, node)...)
 	}
+	if peer.IsFailOver {
+		allowedips = append(allowedips, GetFailOverPeerIps(peer, node)...)
+	}
 	return allowedips
 }
 

logic/server.go

@@ -1,22 +1,8 @@
 package logic
 
-import (
-	"github.com/google/uuid"
-	"github.com/gravitl/netmaker/models"
-)
-
 // EnterpriseCheckFuncs - can be set to run functions for EE
 var EnterpriseCheckFuncs []func()
 
-// EnterpriseFailoverFunc - interface to control failover funcs
-var EnterpriseFailoverFunc func(node *models.Node) error
-
-// EnterpriseResetFailoverFunc - interface to control reset failover funcs
-var EnterpriseResetFailoverFunc func(network string) error
-
-// EnterpriseResetAllPeersFailovers - resets all nodes that are considering a node to be failover worthy (inclusive)
-var EnterpriseResetAllPeersFailovers func(nodeid uuid.UUID, network string) error
-
 // == Join, Checkin, and Leave for Server ==
 
 // KUBERNETES_LISTEN_PORT - starting port for Kubernetes in order to use NodePort range

main.go

@@ -28,8 +28,7 @@ import (
 	"golang.org/x/exp/slog"
 )
 
-
-var version = "v0.21.2"
+var version = "v0.22.0"
 
 // Start DB Connection and start API Request Handler
 func main() {
@@ -169,9 +168,19 @@ func runMessageQueue(wg *sync.WaitGroup, ctx context.Context) {
 		go logic.ManageZombies(ctx, peerUpdate)
 		go logic.DeleteExpiredNodes(ctx, peerUpdate)
 		for nodeUpdate := range peerUpdate {
-			if err := mq.NodeUpdate(nodeUpdate); err != nil {
-				logger.Log(0, "failed to send peer update for deleted node: ", nodeUpdate.ID.String(), err.Error())
+			if nodeUpdate == nil {
+				continue
+			}
+			node := nodeUpdate
+			node.Action = models.NODE_DELETE
+			node.PendingDelete = true
+			if err := mq.NodeUpdate(node); err != nil {
+				logger.Log(0, "failed to send peer update for deleted node: ", node.ID.String(), err.Error())
+			}
+			if err := logic.DeleteNode(node, true); err != nil {
+				slog.Error("error deleting expired node", "nodeid", node.ID.String(), "error", err.Error())
 			}
+			go mq.PublishDeletedNodePeerUpdate(node)
 		}
 	}()
 	<-ctx.Done()

migrate/migrate.go

@@ -18,6 +18,7 @@ func Run() {
 	updateEnrollmentKeys()
 	assignSuperAdmin()
 	updateHosts()
+	updateNodes()
 }
 
 func assignSuperAdmin() {
@@ -137,3 +138,32 @@ func updateHosts() {
 		}
 	}
 }
+
+func updateNodes() {
+	nodes, err := logic.GetAllNodes()
+	if err != nil {
+		slog.Error("migration failed for nodes", "error", err)
+		return
+	}
+	for _, node := range nodes {
+		if node.IsEgressGateway {
+			egressRanges, update := removeInterGw(node.EgressGatewayRanges)
+			if update {
+				node.EgressGatewayRequest.Ranges = egressRanges
+				node.EgressGatewayRanges = egressRanges
+				logic.UpsertNode(&node)
+			}
+		}
+	}
+}
+
+func removeInterGw(egressRanges []string) ([]string, bool) {
+	update := false
+	for i := len(egressRanges) - 1; i >= 0; i-- {
+		if egressRanges[i] == "0.0.0.0/0" || egressRanges[i] == "::/0" {
+			update = true
+			egressRanges = append(egressRanges[:i], egressRanges[i+1:]...)
+		}
+	}
+	return egressRanges, update
+}

models/api_host.go

@@ -26,10 +26,6 @@ type ApiHost struct {
 	MacAddress          string   `json:"macaddress"`
 	Nodes               []string `json:"nodes"`
 	IsDefault           bool     `json:"isdefault"             yaml:"isdefault"`
-	IsRelayed           bool     `json:"isrelayed"             yaml:"isrelayed"             bson:"isrelayed"`
-	RelayedBy           string   `json:"relayed_by"            yaml:"relayed_by"            bson:"relayed_by"`
-	IsRelay             bool     `json:"isrelay"               yaml:"isrelay"               bson:"isrelay"`
-	RelayedHosts        []string `json:"relay_hosts"           yaml:"relay_hosts"           bson:"relay_hosts"`
 	NatType             string   `json:"nat_type"              yaml:"nat_type"`
 	PersistentKeepalive int      `json:"persistentkeepalive"   yaml:"persistentkeepalive"`
 	AutoUpdate          bool     `json:"autoupdate"              yaml:"autoupdate"`

models/api_node.go

@@ -28,9 +28,9 @@ type ApiNode struct {
 	RelayedNodes            []string `json:"relaynodes" yaml:"relayedNodes"`
 	IsEgressGateway         bool     `json:"isegressgateway"`
 	IsIngressGateway        bool     `json:"isingressgateway"`
+	IsInternetGateway       bool     `json:"isinternetgateway" yaml:"isinternetgateway"`
 	EgressGatewayRanges     []string `json:"egressgatewayranges"`
 	EgressGatewayNatEnabled bool     `json:"egressgatewaynatenabled"`
-	FailoverNode            string   `json:"failovernode"`
 	DNSOn                   bool     `json:"dnson"`
 	IngressDns              string   `json:"ingressdns"`
 	Server                  string   `json:"server"`
@@ -38,8 +38,10 @@ type ApiNode struct {
 	Connected               bool     `json:"connected"`
 	PendingDelete           bool     `json:"pendingdelete"`
 	// == PRO ==
-	DefaultACL string `json:"defaultacl,omitempty" validate:"checkyesornoorunset"`
-	Failover   bool   `json:"failover"`
+	DefaultACL    string              `json:"defaultacl,omitempty" validate:"checkyesornoorunset"`
+	IsFailOver    bool                `json:"is_fail_over"`
+	FailOverPeers map[string]struct{} `json:"fail_over_peers" yaml:"fail_over_peers"`
+	FailedOverBy  uuid.UUID           `json:"failed_over_by" yaml:"failed_over_by"`
 }
 
 // ApiNode.ConvertToServerNode - converts an api node to a server node
@@ -56,7 +58,8 @@ func (a *ApiNode) ConvertToServerNode(currentNode *Node) *Node {
 	convertedNode.RelayedBy = a.RelayedBy
 	convertedNode.RelayedNodes = a.RelayedNodes
 	convertedNode.PendingDelete = a.PendingDelete
-	convertedNode.Failover = a.Failover
+	convertedNode.FailedOverBy = currentNode.FailedOverBy
+	convertedNode.FailOverPeers = currentNode.FailOverPeers
 	convertedNode.IsEgressGateway = a.IsEgressGateway
 	convertedNode.IsIngressGateway = a.IsIngressGateway
 	// prevents user from changing ranges, must delete and recreate
@@ -65,6 +68,7 @@ func (a *ApiNode) ConvertToServerNode(currentNode *Node) *Node {
 	convertedNode.IngressGatewayRange6 = currentNode.IngressGatewayRange6
 	convertedNode.DNSOn = a.DNSOn
 	convertedNode.IngressDNS = a.IngressDns
+	convertedNode.IsInternetGateway = a.IsInternetGateway
 	convertedNode.EgressGatewayRequest = currentNode.EgressGatewayRequest
 	convertedNode.EgressGatewayNatEnabled = currentNode.EgressGatewayNatEnabled
 	convertedNode.RelayedNodes = a.RelayedNodes
@@ -86,10 +90,6 @@ func (a *ApiNode) ConvertToServerNode(currentNode *Node) *Node {
 	} else if !isEmptyAddr(currentNode.LocalAddress.String()) {
 		convertedNode.LocalAddress = currentNode.LocalAddress
 	}
-	udpAddr, err := net.ResolveUDPAddr("udp", a.InternetGateway)
-	if err == nil {
-		convertedNode.InternetGateway = udpAddr
-	}
 	ip, addr, err := net.ParseCIDR(a.Address)
 	if err == nil {
 		convertedNode.Address = *addr
@@ -100,7 +100,6 @@ func (a *ApiNode) ConvertToServerNode(currentNode *Node) *Node {
 		convertedNode.Address6 = *addr6
 		convertedNode.Address6.IP = ip6
 	}
-	convertedNode.FailoverNode, _ = uuid.Parse(a.FailoverNode)
 	convertedNode.LastModified = time.Unix(a.LastModified, 0)
 	convertedNode.LastCheckIn = time.Unix(a.LastCheckIn, 0)
 	convertedNode.LastPeerUpdate = time.Unix(a.LastPeerUpdate, 0)
@@ -146,28 +145,22 @@ func (nm *Node) ConvertToAPINode() *ApiNode {
 	apiNode.IsIngressGateway = nm.IsIngressGateway
 	apiNode.EgressGatewayRanges = nm.EgressGatewayRanges
 	apiNode.EgressGatewayNatEnabled = nm.EgressGatewayNatEnabled
-	apiNode.FailoverNode = nm.FailoverNode.String()
-	if isUUIDSet(apiNode.FailoverNode) {
-		apiNode.FailoverNode = ""
-	}
 	apiNode.DNSOn = nm.DNSOn
 	apiNode.IngressDns = nm.IngressDNS
 	apiNode.Server = nm.Server
-	apiNode.InternetGateway = nm.InternetGateway.String()
 	if isEmptyAddr(apiNode.InternetGateway) {
 		apiNode.InternetGateway = ""
 	}
 	apiNode.Connected = nm.Connected
 	apiNode.PendingDelete = nm.PendingDelete
 	apiNode.DefaultACL = nm.DefaultACL
-	apiNode.Failover = nm.Failover
+	apiNode.IsInternetGateway = nm.IsInternetGateway
+	apiNode.IsFailOver = nm.IsFailOver
+	apiNode.FailOverPeers = nm.FailOverPeers
+	apiNode.FailedOverBy = nm.FailedOverBy
 	return &apiNode
 }
 
 func isEmptyAddr(addr string) bool {
 	return addr == "<nil>" || addr == ":0"
 }
-
-func isUUIDSet(uuid string) bool {
-	return uuid != "00000000-0000-0000-0000-000000000000"
-}

models/enrollment_key.go

@@ -1,6 +1,8 @@
 package models
 
 import (
+	"errors"
+	"fmt"
 	"time"
 
 	"github.com/google/uuid"
@@ -13,6 +15,14 @@ const (
 	Unlimited
 )
 
+var (
+	ErrNilEnrollmentKey          = errors.New("enrollment key is nil")
+	ErrNilNetworksEnrollmentKey  = errors.New("enrollment key networks is nil")
+	ErrNilTagsEnrollmentKey      = errors.New("enrollment key tags is nil")
+	ErrInvalidEnrollmentKey      = errors.New("enrollment key is not valid")
+	ErrInvalidEnrollmentKeyValue = errors.New("enrollment key value is not valid")
+)
+
 // KeyType - the type of enrollment key
 type KeyType int
 
@@ -50,7 +60,7 @@ type APIEnrollmentKey struct {
 	UsesRemaining int      `json:"uses_remaining"`
 	Networks      []string `json:"networks"`
 	Unlimited     bool     `json:"unlimited"`
-	Tags          []string `json:"tags"`
+	Tags          []string `json:"tags" validate:"required,dive,min=3,max=32"`
 	Type          KeyType  `json:"type"`
 	Relay         string   `json:"relay"`
 }
@@ -81,9 +91,18 @@ func (k *EnrollmentKey) IsValid() bool {
 
 // EnrollmentKey.Validate - validate's an EnrollmentKey
 // should be used during creation
-func (k *EnrollmentKey) Validate() bool {
-	return k.Networks != nil &&
-		k.Tags != nil &&
-		len(k.Value) == EnrollmentKeyLength &&
-		k.IsValid()
+func (k *EnrollmentKey) Validate() error {
+	if k == nil {
+		return ErrNilEnrollmentKey
+	}
+	if k.Tags == nil {
+		return ErrNilTagsEnrollmentKey
+	}
+	if len(k.Value) != EnrollmentKeyLength {
+		return fmt.Errorf("%w: length not %d characters", ErrInvalidEnrollmentKeyValue, EnrollmentKeyLength)
+	}
+	if !k.IsValid() {
+		return fmt.Errorf("%w: uses remaining: %d, expiration: %s, unlimited: %t", ErrInvalidEnrollmentKey, k.UsesRemaining, k.Expiration, k.Unlimited)
+	}
+	return nil
 }

models/extclient.go

@@ -10,13 +10,14 @@ type ExtClient struct {
 	Address                string              `json:"address" bson:"address"`
 	Address6               string              `json:"address6" bson:"address6"`
 	ExtraAllowedIPs        []string            `json:"extraallowedips" bson:"extraallowedips"`
+	AllowedIPs             []string            `json:"allowed_ips"`
 	IngressGatewayID       string              `json:"ingressgatewayid" bson:"ingressgatewayid"`
 	IngressGatewayEndpoint string              `json:"ingressgatewayendpoint" bson:"ingressgatewayendpoint"`
 	LastModified           int64               `json:"lastmodified" bson:"lastmodified"`
 	Enabled                bool                `json:"enabled" bson:"enabled"`
 	OwnerID                string              `json:"ownerid" bson:"ownerid"`
 	DeniedACLs             map[string]struct{} `json:"deniednodeacls" bson:"acls,omitempty"`
-	RemoteAccessClientID   string              `json:"remote_access_client_id"`
+	RemoteAccessClientID   string              `json:"remote_access_client_id"` // unique ID (MAC address) of RAC machine
 }
 
 // CustomExtClient - struct for CustomExtClient params
@@ -27,5 +28,5 @@ type CustomExtClient struct {
 	ExtraAllowedIPs      []string            `json:"extraallowedips,omitempty"`
 	Enabled              bool                `json:"enabled,omitempty"`
 	DeniedACLs           map[string]struct{} `json:"deniednodeacls" bson:"acls,omitempty"`
-	RemoteAccessClientID string              `json:"remote_access_client_id"`
+	RemoteAccessClientID string              `json:"remote_access_client_id"` // unique ID (MAC address) of RAC machine
 }

models/host.go

@@ -110,30 +110,31 @@ const (
 	RequestAck HostMqAction = "REQ_ACK"
 	// CheckIn - update last check in times and public address and interfaces
 	CheckIn HostMqAction = "CHECK_IN"
-	// RegisterWithTurn - registers host with turn server if configured
-	RegisterWithTurn HostMqAction = "REGISTER_WITH_TURN"
 	// UpdateKeys - update wireguard private/public keys
 	UpdateKeys HostMqAction = "UPDATE_KEYS"
 	// RequestPull - request a pull from a host
 	RequestPull HostMqAction = "REQ_PULL"
+	// UpdateMetrics - updates metrics data
+	UpdateMetrics HostMqAction = "UPDATE_METRICS"
 )
 
 // SignalAction - turn peer signal action
 type SignalAction string
 
 const (
-	// Disconnect - action to stop using turn connection
-	Disconnect SignalAction = "DISCONNECT"
 	// ConnNegotiation - action to negotiate connection between peers
 	ConnNegotiation SignalAction = "CONNECTION_NEGOTIATION"
+	// RelayME - action to relay the peer
+	RelayME SignalAction = "RELAY_ME"
 )
 
 // HostUpdate - struct for host update
 type HostUpdate struct {
-	Action HostMqAction
-	Host   Host
-	Node   Node
-	Signal Signal
+	Action     HostMqAction
+	Host       Host
+	Node       Node
+	Signal     Signal
+	NewMetrics Metrics
 }
 
 // HostTurnRegister - struct for host turn registration
@@ -144,13 +145,17 @@ type HostTurnRegister struct {
 
 // Signal - struct for signalling peer
 type Signal struct {
-	Server            string       `json:"server"`
-	FromHostPubKey    string       `json:"from_host_pubkey"`
-	TurnRelayEndpoint string       `json:"turn_relay_addr"`
-	ToHostPubKey      string       `json:"to_host_pubkey"`
-	Reply             bool         `json:"reply"`
-	Action            SignalAction `json:"action"`
-	TimeStamp         int64        `json:"timestamp"`
+	Server         string       `json:"server"`
+	FromHostPubKey string       `json:"from_host_pubkey"`
+	ToHostPubKey   string       `json:"to_host_pubkey"`
+	FromHostID     string       `json:"from_host_id"`
+	ToHostID       string       `json:"to_host_id"`
+	FromNodeID     string       `json:"from_node_id"`
+	ToNodeID       string       `json:"to_node_id"`
+	Reply          bool         `json:"reply"`
+	Action         SignalAction `json:"action"`
+	IsPro          bool         `json:"is_pro"`
+	TimeStamp      int64        `json:"timestamp"`
 }
 
 // RegisterMsg - login message struct for hosts to join via SSO login

models/metrics.go

@@ -6,11 +6,10 @@ import (
 
 // Metrics - metrics struct
 type Metrics struct {
-	Network       string            `json:"network" bson:"network" yaml:"network"`
-	NodeID        string            `json:"node_id" bson:"node_id" yaml:"node_id"`
-	NodeName      string            `json:"node_name" bson:"node_name" yaml:"node_name"`
-	Connectivity  map[string]Metric `json:"connectivity" bson:"connectivity" yaml:"connectivity"`
-	FailoverPeers map[string]string `json:"needsfailover" bson:"needsfailover" yaml:"needsfailover"`
+	Network      string            `json:"network" bson:"network" yaml:"network"`
+	NodeID       string            `json:"node_id" bson:"node_id" yaml:"node_id"`
+	NodeName     string            `json:"node_name" bson:"node_name" yaml:"node_name"`
+	Connectivity map[string]Metric `json:"connectivity" bson:"connectivity" yaml:"connectivity"`
 }
 
 // Metric - holds a metric for data between nodes
@@ -29,6 +28,7 @@ type Metric struct {
 // IDandAddr - struct to hold ID and primary Address
 type IDandAddr struct {
 	ID          string `json:"id" bson:"id" yaml:"id"`
+	HostID      string `json:"host_id"`
 	Address     string `json:"address" bson:"address" yaml:"address"`
 	Name        string `json:"name" bson:"name" yaml:"name"`
 	IsServer    string `json:"isserver" bson:"isserver" yaml:"isserver" validate:"checkyesorno"`

models/mqtt.go

@@ -19,6 +19,7 @@ type HostPeerUpdate struct {
 	HostNetworkInfo HostInfoMap           `json:"host_network_info,omitempty" bson:"host_network_info,omitempty" yaml:"host_network_info,omitempty"`
 	EgressRoutes    []EgressNetworkRoutes `json:"egress_network_routes"`
 	FwUpdate        FwUpdate              `json:"fw_update"`
+	ReplacePeers    bool                  `json:"replace_peers"`
 }
 
 // IngressInfo - struct for ingress info
@@ -70,3 +71,8 @@ type FwUpdate struct {
 	IsEgressGw bool                  `json:"is_egress_gw"`
 	EgressInfo map[string]EgressInfo `json:"egress_info"`
 }
+
+// FailOverMeReq - struct for failover req
+type FailOverMeReq struct {
+	NodeID string `json:"node_id"`
+}

models/node.go

@@ -14,8 +14,6 @@ import (
 const (
 	// NODE_SERVER_NAME - the default server name
 	NODE_SERVER_NAME = "netmaker"
-	// TEN_YEARS_IN_SECONDS - ten years in seconds
-	TEN_YEARS_IN_SECONDS = 315670000000000000
 	// MAX_NAME_LENGTH - max name length of node
 	MAX_NAME_LENGTH = 62
 	// == ACTIONS == (can only be set by server)
@@ -54,27 +52,27 @@ type Iface struct {
 
 // CommonNode - represents a commonn node data elements shared by netmaker and netclient
 type CommonNode struct {
-	ID                  uuid.UUID    `json:"id" yaml:"id"`
-	HostID              uuid.UUID    `json:"hostid" yaml:"hostid"`
-	Network             string       `json:"network" yaml:"network"`
-	NetworkRange        net.IPNet    `json:"networkrange" yaml:"networkrange"`
-	NetworkRange6       net.IPNet    `json:"networkrange6" yaml:"networkrange6"`
-	InternetGateway     *net.UDPAddr `json:"internetgateway" yaml:"internetgateway"`
-	Server              string       `json:"server" yaml:"server"`
-	Connected           bool         `json:"connected" yaml:"connected"`
-	Address             net.IPNet    `json:"address" yaml:"address"`
-	Address6            net.IPNet    `json:"address6" yaml:"address6"`
-	Action              string       `json:"action" yaml:"action"`
-	LocalAddress        net.IPNet    `json:"localaddress" yaml:"localaddress"`
-	IsEgressGateway     bool         `json:"isegressgateway" yaml:"isegressgateway"`
-	EgressGatewayRanges []string     `json:"egressgatewayranges" bson:"egressgatewayranges" yaml:"egressgatewayranges"`
-	IsIngressGateway    bool         `json:"isingressgateway" yaml:"isingressgateway"`
-	IsRelayed           bool         `json:"isrelayed" bson:"isrelayed" yaml:"isrelayed"`
-	RelayedBy           string       `json:"relayedby" bson:"relayedby" yaml:"relayedby"`
-	IsRelay             bool         `json:"isrelay" bson:"isrelay" yaml:"isrelay"`
-	RelayedNodes        []string     `json:"relaynodes" yaml:"relayedNodes"`
-	IngressDNS          string       `json:"ingressdns" yaml:"ingressdns"`
-	DNSOn               bool         `json:"dnson" yaml:"dnson"`
+	ID                  uuid.UUID `json:"id" yaml:"id"`
+	HostID              uuid.UUID `json:"hostid" yaml:"hostid"`
+	Network             string    `json:"network" yaml:"network"`
+	NetworkRange        net.IPNet `json:"networkrange" yaml:"networkrange"`
+	NetworkRange6       net.IPNet `json:"networkrange6" yaml:"networkrange6"`
+	Server              string    `json:"server" yaml:"server"`
+	Connected           bool      `json:"connected" yaml:"connected"`
+	Address             net.IPNet `json:"address" yaml:"address"`
+	Address6            net.IPNet `json:"address6" yaml:"address6"`
+	Action              string    `json:"action" yaml:"action"`
+	LocalAddress        net.IPNet `json:"localaddress" yaml:"localaddress"`
+	IsEgressGateway     bool      `json:"isegressgateway" yaml:"isegressgateway"`
+	EgressGatewayRanges []string  `json:"egressgatewayranges" bson:"egressgatewayranges" yaml:"egressgatewayranges"`
+	IsIngressGateway    bool      `json:"isingressgateway" yaml:"isingressgateway"`
+	IsInternetGateway   bool      `json:"isinternetgateway" yaml:"isinternetgateway"`
+	IsRelayed           bool      `json:"isrelayed" bson:"isrelayed" yaml:"isrelayed"`
+	RelayedBy           string    `json:"relayedby" bson:"relayedby" yaml:"relayedby"`
+	IsRelay             bool      `json:"isrelay" bson:"isrelay" yaml:"isrelay"`
+	RelayedNodes        []string  `json:"relaynodes" yaml:"relayedNodes"`
+	IngressDNS          string    `json:"ingressdns" yaml:"ingressdns"`
+	DNSOn               bool      `json:"dnson" yaml:"dnson"`
 }
 
 // Node - a model of a network node
@@ -90,10 +88,11 @@ type Node struct {
 	IngressGatewayRange     string               `json:"ingressgatewayrange" bson:"ingressgatewayrange" yaml:"ingressgatewayrange"`
 	IngressGatewayRange6    string               `json:"ingressgatewayrange6" bson:"ingressgatewayrange6" yaml:"ingressgatewayrange6"`
 	// == PRO ==
-	DefaultACL   string    `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"`
-	OwnerID      string    `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"`
-	FailoverNode uuid.UUID `json:"failovernode" bson:"failovernode" yaml:"failovernode"`
-	Failover     bool      `json:"failover" bson:"failover" yaml:"failover"`
+	DefaultACL    string              `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"`
+	OwnerID       string              `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"`
+	IsFailOver    bool                `json:"is_fail_over" yaml:"is_fail_over"`
+	FailOverPeers map[string]struct{} `json:"fail_over_peers" yaml:"fail_over_peers"`
+	FailedOverBy  uuid.UUID           `json:"failed_over_by" yaml:"failed_over_by"`
 }
 
 // LegacyNode - legacy struct for node model
@@ -353,7 +352,7 @@ func (node *Node) SetLastPeerUpdate() {
 // Node.SetExpirationDateTime - sets node expiry time
 func (node *Node) SetExpirationDateTime() {
 	if node.ExpirationDateTime.IsZero() {
-		node.ExpirationDateTime = time.Now().Add(TEN_YEARS_IN_SECONDS)
+		node.ExpirationDateTime = time.Now().AddDate(100, 1, 0)
 	}
 }
 
@@ -432,8 +431,8 @@ func (newNode *Node) Fill(currentNode *Node, isPro bool) { // TODO add new field
 	if newNode.DefaultACL == "" {
 		newNode.DefaultACL = currentNode.DefaultACL
 	}
-	if newNode.Failover != currentNode.Failover {
-		newNode.Failover = currentNode.Failover
+	if newNode.IsFailOver != currentNode.IsFailOver {
+		newNode.IsFailOver = currentNode.IsFailOver
 	}
 }
 

models/structs.go

@@ -64,11 +64,13 @@ type IngressGwUsers struct {
 
 // UserRemoteGws - struct to hold user's remote gws
 type UserRemoteGws struct {
-	GwID      string    `json:"remote_access_gw_id"`
-	GWName    string    `json:"gw_name"`
-	Network   string    `json:"network"`
-	Connected bool      `json:"connected"`
-	GwClient  ExtClient `json:"gw_client"`
+	GwID              string    `json:"remote_access_gw_id"`
+	GWName            string    `json:"gw_name"`
+	Network           string    `json:"network"`
+	Connected         bool      `json:"connected"`
+	IsInternetGateway bool      `json:"is_internet_gateway"`
+	GwClient          ExtClient `json:"gw_client"`
+	GwPeerPublicKey   string    `json:"gw_peer_public_key"`
 }
 
 // UserRemoteGwsReq - struct to hold user remote acccess gws req
@@ -189,8 +191,8 @@ type HostRelayRequest struct {
 
 // IngressRequest - ingress request struct
 type IngressRequest struct {
-	ExtclientDNS string `json:"extclientdns"`
-	Failover     bool   `json:"failover"`
+	ExtclientDNS      string `json:"extclientdns"`
+	IsInternetGateway bool   `json:"is_internet_gw"`
 }
 
 // ServerUpdateData - contains data to configure server
@@ -223,12 +225,14 @@ type TrafficKeys struct {
 
 // HostPull - response of a host's pull
 type HostPull struct {
-	Host            Host                 `json:"host" yaml:"host"`
-	Nodes           []Node               `json:"nodes" yaml:"nodes"`
-	Peers           []wgtypes.PeerConfig `json:"peers" yaml:"peers"`
-	ServerConfig    ServerConfig         `json:"server_config" yaml:"server_config"`
-	PeerIDs         PeerMap              `json:"peer_ids,omitempty" yaml:"peer_ids,omitempty"`
-	HostNetworkInfo HostInfoMap          `json:"host_network_info,omitempty"  yaml:"host_network_info,omitempty"`
+	Host            Host                  `json:"host" yaml:"host"`
+	Nodes           []Node                `json:"nodes" yaml:"nodes"`
+	Peers           []wgtypes.PeerConfig  `json:"peers" yaml:"peers"`
+	ServerConfig    ServerConfig          `json:"server_config" yaml:"server_config"`
+	PeerIDs         PeerMap               `json:"peer_ids,omitempty" yaml:"peer_ids,omitempty"`
+	HostNetworkInfo HostInfoMap           `json:"host_network_info,omitempty"  yaml:"host_network_info,omitempty"`
+	EgressRoutes    []EgressNetworkRoutes `json:"egress_network_routes"`
+	FwUpdate        FwUpdate              `json:"fw_update"`
 }
 
 // NodeGet - struct for a single node get response
@@ -259,14 +263,11 @@ type ServerConfig struct {
 	MQPort      string `yaml:"mqport"`
 	MQUserName  string `yaml:"mq_username"`
 	MQPassword  string `yaml:"mq_password"`
+	BrokerType  string `yaml:"broker_type"`
 	Server      string `yaml:"server"`
 	Broker      string `yaml:"broker"`
 	IsPro       bool   `yaml:"isee" json:"Is_EE"`
-	StunPort    int    `yaml:"stun_port"`
 	TrafficKey  []byte `yaml:"traffickey"`
-	TurnDomain  string `yaml:"turn_domain"`
-	TurnPort    int    `yaml:"turn_port"`
-	UseTurn     bool   `yaml:"use_turn"`
 }
 
 // User.NameInCharset - returns if name is in charset below or not
@@ -306,3 +307,54 @@ type LicenseLimits struct {
 	Clients  int `json:"clients"`
 	Networks int `json:"networks"`
 }
+
+type SignInReqDto struct {
+	FormFields FormFields `json:"formFields"`
+}
+
+type FormField struct {
+	Id    string `json:"id"`
+	Value any    `json:"value"`
+}
+
+type FormFields []FormField
+
+type SignInResDto struct {
+	Status string `json:"status"`
+	User   User   `json:"user"`
+}
+
+type TenantLoginResDto struct {
+	Code     int    `json:"code"`
+	Message  string `json:"message"`
+	Response struct {
+		UserName  string `json:"UserName"`
+		AuthToken string `json:"AuthToken"`
+	} `json:"response"`
+}
+
+type SsoLoginReqDto struct {
+	OauthProvider string `json:"oauthprovider"`
+}
+
+type SsoLoginResDto struct {
+	User      string `json:"UserName"`
+	AuthToken string `json:"AuthToken"`
+}
+
+type SsoLoginData struct {
+	Expiration     time.Time `json:"expiration"`
+	OauthProvider  string    `json:"oauthprovider,omitempty"`
+	OauthCode      string    `json:"oauthcode,omitempty"`
+	Username       string    `json:"username,omitempty"`
+	AmbAccessToken string    `json:"ambaccesstoken,omitempty"`
+}
+
+type LoginReqDto struct {
+	Email    string `json:"email"`
+	TenantID string `json:"tenant_id"`
+}
+
+const (
+	ResHeaderKeyStAccessToken = "St-Access-Token"
+)

mq/emqx.go

@@ -286,16 +286,6 @@ func CreateHostACL(hostID, serverName string) error {
 				Permission: "allow",
 				Action:     "all",
 			},
-			{
-				Topic:      fmt.Sprintf("dns/all/%s/%s", hostID, serverName),
-				Permission: "allow",
-				Action:     "all",
-			},
-			{
-				Topic:      fmt.Sprintf("dns/update/%s/%s", hostID, serverName),
-				Permission: "allow",
-				Action:     "all",
-			},
 			{
 				Topic:      fmt.Sprintf("host/serverupdate/%s/%s", serverName, hostID),
 				Permission: "allow",

mq/handlers.go

@@ -2,24 +2,25 @@ package mq
 
 import (
 	"encoding/json"
-	"fmt"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/google/uuid"
 	"github.com/gravitl/netmaker/database"
+	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/logic/hostactions"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
-	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
 // UpdateMetrics  message Handler -- handles updates from client nodes for metrics
 var UpdateMetrics = func(client mqtt.Client, msg mqtt.Message) {
 }
 
+var UpdateMetricsFallBack = func(nodeid string, newMetrics models.Metrics) {}
+
 // DefaultHandler default message queue handler  -- NOT USED
 func DefaultHandler(client mqtt.Client, msg mqtt.Message) {
 	slog.Info("mqtt default handler", "topic", msg.Topic(), "message", msg.Payload())
@@ -49,11 +50,6 @@ func UpdateNode(client mqtt.Client, msg mqtt.Message) {
 	}
 
 	ifaceDelta := logic.IfaceDelta(&currentNode, &newNode)
-	if servercfg.IsPro && ifaceDelta {
-		if err = logic.EnterpriseResetAllPeersFailovers(currentNode.ID, currentNode.Network); err != nil {
-			slog.Warn("failed to reset failover list during node update", "nodeid", currentNode.ID, "network", currentNode.Network)
-		}
-	}
 	newNode.SetLastCheckIn()
 	if err := logic.UpdateNode(&currentNode, &newNode); err != nil {
 		slog.Error("error saving node", "id", id, "error", err)
@@ -69,10 +65,10 @@ func UpdateNode(client mqtt.Client, msg mqtt.Message) {
 			}
 			allNodes, err := logic.GetAllNodes()
 			if err == nil {
-				PublishSingleHostPeerUpdate(host, allNodes, nil, nil)
+				PublishSingleHostPeerUpdate(host, allNodes, nil, nil, false)
 			}
 		} else {
-			err = PublishPeerUpdate()
+			err = PublishPeerUpdate(false)
 		}
 		if err != nil {
 			slog.Warn("error updating peers when node informed the server of an interface change", "nodeid", currentNode.ID, "error", err)
@@ -106,9 +102,10 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) {
 	}
 	slog.Info("recieved host update", "name", hostUpdate.Host.Name, "id", hostUpdate.Host.ID)
 	var sendPeerUpdate bool
+	var replacePeers bool
 	switch hostUpdate.Action {
 	case models.CheckIn:
-		sendPeerUpdate = handleHostCheckin(&hostUpdate.Host, currentHost)
+		sendPeerUpdate = HandleHostCheckin(&hostUpdate.Host, currentHost)
 	case models.Acknowledgement:
 		hu := hostactions.GetAction(currentHost.ID.String())
 		if hu != nil {
@@ -126,38 +123,16 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) {
 				if err != nil {
 					return
 				}
-				if err = PublishSingleHostPeerUpdate(currentHost, nodes, nil, nil); err != nil {
+				if err = PublishSingleHostPeerUpdate(currentHost, nodes, nil, nil, false); err != nil {
 					slog.Error("failed peers publish after join acknowledged", "name", hostUpdate.Host.Name, "id", currentHost.ID, "error", err)
 					return
 				}
-				if err = HandleNewNodeDNS(&hu.Host, &hu.Node); err != nil {
-					slog.Error("failed to send dns update after node added to host", "name", hostUpdate.Host.Name, "id", currentHost.ID, "error", err)
-					return
-				}
 			}
 		}
 	case models.UpdateHost:
 		if hostUpdate.Host.PublicKey != currentHost.PublicKey {
 			//remove old peer entry
-			peerUpdate := models.HostPeerUpdate{
-				ServerVersion: servercfg.GetVersion(),
-				Peers: []wgtypes.PeerConfig{
-					{
-						PublicKey: currentHost.PublicKey,
-						Remove:    true,
-					},
-				},
-			}
-			data, err := json.Marshal(&peerUpdate)
-			if err != nil {
-				slog.Error("failed to marshal peer update", "error", err)
-			}
-			hosts := logic.GetRelatedHosts(hostUpdate.Host.ID.String())
-			server := servercfg.GetServer()
-			for _, host := range hosts {
-				publish(&host, fmt.Sprintf("peers/host/%s/%s", host.ID.String(), server), data)
-			}
-
+			replacePeers = true
 		}
 		sendPeerUpdate = logic.UpdateHostFromClient(&hostUpdate.Host, currentHost)
 		err := logic.UpsertHost(currentHost)
@@ -170,7 +145,6 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) {
 			// delete EMQX credentials for host
 			if err := DeleteEmqxUser(currentHost.ID.String()); err != nil {
 				slog.Error("failed to remove host credentials from EMQX", "id", currentHost.ID, "error", err)
-				return
 			}
 		}
 
@@ -197,26 +171,46 @@ func UpdateHost(client mqtt.Client, msg mqtt.Message) {
 			slog.Error("failed to delete host", "id", currentHost.ID, "error", err)
 			return
 		}
-		sendPeerUpdate = true
-	case models.RegisterWithTurn:
-		if servercfg.IsUsingTurn() {
-			err = logic.RegisterHostWithTurn(hostUpdate.Host.ID.String(), hostUpdate.Host.HostPass)
-			if err != nil {
-				slog.Error("failed to register host with turn server", "id", currentHost.ID, "error", err)
-				return
-			}
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
 		}
+		sendPeerUpdate = true
+	case models.SignalHost:
+		signalPeer(hostUpdate.Signal)
 
 	}
 
 	if sendPeerUpdate {
-		err := PublishPeerUpdate()
+		err := PublishPeerUpdate(replacePeers)
 		if err != nil {
 			slog.Error("failed to publish peer update", "error", err)
 		}
 	}
 }
 
+func signalPeer(signal models.Signal) {
+
+	if signal.ToHostPubKey == "" {
+		msg := "insufficient data to signal peer"
+		logger.Log(0, msg)
+		return
+	}
+	signal.IsPro = servercfg.IsPro
+	peerHost, err := logic.GetHost(signal.ToHostID)
+	if err != nil {
+		slog.Error("failed to signal, peer not found", "error", err)
+		return
+	}
+	err = HostUpdate(&models.HostUpdate{
+		Action: models.SignalHost,
+		Host:   *peerHost,
+		Signal: signal,
+	})
+	if err != nil {
+		slog.Error("failed to publish signal to peer", "error", err)
+	}
+}
+
 // ClientPeerUpdate  message handler -- handles updating peers after signal from client nodes
 func ClientPeerUpdate(client mqtt.Client, msg mqtt.Message) {
 	id, err := GetID(msg.Topic())
@@ -238,7 +232,7 @@ func ClientPeerUpdate(client mqtt.Client, msg mqtt.Message) {
 	case ncutils.ACK:
 		// do we still need this
 	case ncutils.DONE:
-		if err = PublishPeerUpdate(); err != nil {
+		if err = PublishPeerUpdate(false); err != nil {
 			slog.Error("error publishing peer update for node", "id", currentNode.ID, "error", err)
 			return
 		}
@@ -247,29 +241,7 @@ func ClientPeerUpdate(client mqtt.Client, msg mqtt.Message) {
 	slog.Info("sent peer updates after signal received from", "id", id)
 }
 
-func HandleNewNodeDNS(host *models.Host, node *models.Node) error {
-	dns := models.DNSUpdate{
-		Action: models.DNSInsert,
-		Name:   host.Name + "." + node.Network,
-	}
-	if node.Address.IP != nil {
-		dns.Address = node.Address.IP.String()
-		if err := PublishDNSUpdate(node.Network, dns); err != nil {
-			return err
-		}
-	} else if node.Address6.IP != nil {
-		dns.Address = node.Address6.IP.String()
-		if err := PublishDNSUpdate(node.Network, dns); err != nil {
-			return err
-		}
-	}
-	if err := PublishAllDNS(node); err != nil {
-		return err
-	}
-	return nil
-}
-
-func handleHostCheckin(h, currentHost *models.Host) bool {
+func HandleHostCheckin(h, currentHost *models.Host) bool {
 	if h == nil {
 		return false
 	}

mq/publishers.go

@@ -14,7 +14,7 @@ import (
 )
 
 // PublishPeerUpdate --- determines and publishes a peer update to all the hosts
-func PublishPeerUpdate() error {
+func PublishPeerUpdate(replacePeers bool) error {
 	if !servercfg.IsMessageQueueBackend() {
 		return nil
 	}
@@ -30,9 +30,12 @@ func PublishPeerUpdate() error {
 	}
 	for _, host := range hosts {
 		host := host
-		if err = PublishSingleHostPeerUpdate(&host, allNodes, nil, nil); err != nil {
-			logger.Log(1, "failed to publish peer update to host", host.ID.String(), ": ", err.Error())
-		}
+		go func(host models.Host) {
+			if err = PublishSingleHostPeerUpdate(&host, allNodes, nil, nil, replacePeers); err != nil {
+				logger.Log(1, "failed to publish peer update to host", host.ID.String(), ": ", err.Error())
+			}
+		}(host)
+
 	}
 	return err
 }
@@ -55,7 +58,7 @@ func PublishDeletedNodePeerUpdate(delNode *models.Node) error {
 	}
 	for _, host := range hosts {
 		host := host
-		if err = PublishSingleHostPeerUpdate(&host, allNodes, delNode, nil); err != nil {
+		if err = PublishSingleHostPeerUpdate(&host, allNodes, delNode, nil, false); err != nil {
 			logger.Log(1, "failed to publish peer update to host", host.ID.String(), ": ", err.Error())
 		}
 	}
@@ -81,7 +84,7 @@ func PublishDeletedClientPeerUpdate(delClient *models.ExtClient) error {
 	for _, host := range hosts {
 		host := host
 		if host.OS != models.OS_Types.IoT {
-			if err = PublishSingleHostPeerUpdate(&host, nodes, nil, []models.ExtClient{*delClient}); err != nil {
+			if err = PublishSingleHostPeerUpdate(&host, nodes, nil, []models.ExtClient{*delClient}, false); err != nil {
 				logger.Log(1, "failed to publish peer update to host", host.ID.String(), ": ", err.Error())
 			}
 		}
@@ -90,12 +93,13 @@ func PublishDeletedClientPeerUpdate(delClient *models.ExtClient) error {
 }
 
 // PublishSingleHostPeerUpdate --- determines and publishes a peer update to one host
-func PublishSingleHostPeerUpdate(host *models.Host, allNodes []models.Node, deletedNode *models.Node, deletedClients []models.ExtClient) error {
+func PublishSingleHostPeerUpdate(host *models.Host, allNodes []models.Node, deletedNode *models.Node, deletedClients []models.ExtClient, replacePeers bool) error {
 
 	peerUpdate, err := logic.GetPeerUpdateForHost("", host, allNodes, deletedNode, deletedClients)
 	if err != nil {
 		return err
 	}
+	peerUpdate.ReplacePeers = replacePeers
 	data, err := json.Marshal(&peerUpdate)
 	if err != nil {
 		return err
@@ -166,73 +170,6 @@ func ServerStartNotify() error {
 	return nil
 }
 
-// PublishDNSUpdatev1 - published dns updates to all nodes passed
-func PublishDNSUpdatev1(network string, dns models.DNSUpdate, nodes []models.Node) error {
-	for _, node := range nodes {
-		host, err := logic.GetHost(node.HostID.String())
-		if err != nil {
-			logger.Log(0, "error retrieving host for dns update", node.HostID.String(), err.Error())
-			continue
-		}
-		data, err := json.Marshal(dns)
-		if err != nil {
-			logger.Log(0, "failed to encode dns data for node", node.ID.String(), err.Error())
-		}
-		if err := publish(host, "dns/update/"+host.ID.String()+"/"+servercfg.GetServer(), data); err != nil {
-			logger.Log(0, "error publishing dns update to host", host.ID.String(), err.Error())
-			continue
-		}
-		logger.Log(3, "published dns update to host", host.ID.String())
-	}
-	return nil
-}
-
-// PublishDNSUpdate publishes a dns update to all nodes on a network
-func PublishDNSUpdate(network string, dns models.DNSUpdate) error {
-	nodes, err := logic.GetNetworkNodes(network)
-	if err != nil {
-		return err
-	}
-	for _, node := range nodes {
-		host, err := logic.GetHost(node.HostID.String())
-		if err != nil {
-			logger.Log(0, "error retrieving host for dns update", node.HostID.String(), err.Error())
-			continue
-		}
-		data, err := json.Marshal(dns)
-		if err != nil {
-			logger.Log(0, "failed to encode dns data for node", node.ID.String(), err.Error())
-		}
-		if err := publish(host, "dns/update/"+host.ID.String()+"/"+servercfg.GetServer(), data); err != nil {
-			logger.Log(0, "error publishing dns update to host", host.ID.String(), err.Error())
-			continue
-		}
-		logger.Log(3, "published dns update to host", host.ID.String())
-	}
-	return nil
-}
-
-// PublishAllDNS publishes an array of dns updates (ip / host.network) for each peer to a node joining a network
-func PublishAllDNS(newnode *models.Node) error {
-	alldns := []models.DNSUpdate{}
-	newnodeHost, err := logic.GetHost(newnode.HostID.String())
-	if err != nil {
-		return fmt.Errorf("error retrieving host for dns update %w", err)
-	}
-	alldns = append(alldns, getNodeDNS(newnode.Network)...)
-	alldns = append(alldns, getExtClientDNS(newnode.Network)...)
-	alldns = append(alldns, getCustomDNS(newnode.Network)...)
-	data, err := json.Marshal(alldns)
-	if err != nil {
-		return fmt.Errorf("error encoding dns data %w", err)
-	}
-	if err := publish(newnodeHost, "dns/all/"+newnodeHost.ID.String()+"/"+servercfg.GetServer(), data); err != nil {
-		return fmt.Errorf("error publishing full dns update to %s, %w", newnodeHost.ID.String(), err)
-	}
-	logger.Log(3, "published full dns update to %s", newnodeHost.ID.String())
-	return nil
-}
-
 // PublishMqUpdatesForDeletedNode - published all the required updates for deleted node
 func PublishMqUpdatesForDeletedNode(node models.Node, sendNodeUpdate bool, gwClients []models.ExtClient) {
 	// notify of peer change
@@ -246,162 +183,10 @@ func PublishMqUpdatesForDeletedNode(node models.Node, sendNodeUpdate bool, gwCli
 	if err := PublishDeletedNodePeerUpdate(&node); err != nil {
 		logger.Log(1, "error publishing peer update ", err.Error())
 	}
-	host, err := logic.GetHost(node.HostID.String())
-	if err != nil {
-		logger.Log(1, "failed to retrieve host for node", node.ID.String(), err.Error())
-	}
-	if err := PublishDNSDelete(&node, host); err != nil {
-		logger.Log(1, "error publishing dns update", err.Error())
-	}
-	if err := PublishDeleteAllExtclientsDNS(node.Network, gwClients); err != nil {
-		logger.Log(1, "error publishing ext dns update", err.Error())
-	}
-
-}
-
-// PublishDNSDelete publish a dns update deleting a node to all hosts on a network
-func PublishDNSDelete(node *models.Node, host *models.Host) error {
-	dns := models.DNSUpdate{
-		Action: models.DNSDeleteByIP,
-		Name:   host.Name + "." + node.Network,
-	}
-	if node.Address.IP != nil {
-		dns.Address = node.Address.IP.String()
-		if err := PublishDNSUpdate(node.Network, dns); err != nil {
-			return fmt.Errorf("dns update node deletion %w", err)
-		}
-	}
-	if node.Address6.IP != nil {
-		dns.Address = node.Address6.IP.String()
-		if err := PublishDNSUpdate(node.Network, dns); err != nil {
-			return fmt.Errorf("dns update node deletion %w", err)
-		}
-	}
-	return nil
-}
-
-// PublishReplaceDNS publish a dns update to replace a dns entry on all hosts in network
-func PublishReplaceDNS(oldNode, newNode *models.Node, host *models.Host) error {
-	dns := models.DNSUpdate{
-		Action: models.DNSReplaceIP,
-		Name:   host.Name + "." + oldNode.Network,
-	}
-	if !oldNode.Address.IP.Equal(newNode.Address.IP) {
-		dns.Address = oldNode.Address.IP.String()
-		dns.NewAddress = newNode.Address.IP.String()
-		if err := PublishDNSUpdate(oldNode.Network, dns); err != nil {
-			return err
-		}
-	}
-	if !oldNode.Address6.IP.Equal(newNode.Address6.IP) {
-		dns.Address = oldNode.Address6.IP.String()
-		dns.NewAddress = newNode.Address6.IP.String()
-		if err := PublishDNSUpdate(oldNode.Network, dns); err != nil {
-			return err
-		}
-	}
-	return nil
-}
-
-// PublishExtClientDNS publish dns update for new extclient
-func PublishExtClientDNS(client *models.ExtClient) error {
-	errMsgs := models.DNSError{}
-	dns := models.DNSUpdate{
-		Action:  models.DNSInsert,
-		Name:    client.ClientID + "." + client.Network,
-		Address: client.Address,
-	}
-	if client.Address != "" {
-		dns.Address = client.Address
-		if err := PublishDNSUpdate(client.Network, dns); err != nil {
-			errMsgs.ErrorStrings = append(errMsgs.ErrorStrings, err.Error())
-		}
-
-	}
-	if client.Address6 != "" {
-		dns.Address = client.Address6
-		if err := PublishDNSUpdate(client.Network, dns); err != nil {
-			errMsgs.ErrorStrings = append(errMsgs.ErrorStrings, err.Error())
-		}
-	}
-	if len(errMsgs.ErrorStrings) > 0 {
-		return errMsgs
-	}
-	return nil
-}
-
-// PublishExtClientDNSUpdate update for extclient name change
-func PublishExtClientDNSUpdate(old, new models.ExtClient, network string) error {
-	dns := models.DNSUpdate{
-		Action:  models.DNSReplaceName,
-		Name:    old.ClientID + "." + network,
-		NewName: new.ClientID + "." + network,
-	}
-	if err := PublishDNSUpdate(network, dns); err != nil {
-		return err
-	}
-	return nil
-}
-
-// PublishDeleteAllExtclientsDNS - publish to delete all passed ext clients dns entries
-func PublishDeleteAllExtclientsDNS(network string, clients []models.ExtClient) error {
-	nodes, err := logic.GetNetworkNodes(network)
-	if err != nil {
-		return err
-	}
-	for _, client := range clients {
-		dns := models.DNSUpdate{
-			Action: models.DNSDeleteByName,
-			Name:   client.ClientID + "." + client.Network,
-		}
-		go PublishDNSUpdatev1(client.Network, dns, nodes)
-	}
-	return nil
-}
-
-// PublishDeleteExtClientDNS publish dns update to delete extclient entry
-func PublishDeleteExtClientDNS(client *models.ExtClient) error {
-	dns := models.DNSUpdate{
-		Action: models.DNSDeleteByName,
-		Name:   client.ClientID + "." + client.Network,
-	}
-	if err := PublishDNSUpdate(client.Network, dns); err != nil {
-		return err
-	}
-	return nil
-}
-
-// PublishCustomDNS publish dns update for new custom dns entry
-func PublishCustomDNS(entry *models.DNSEntry) error {
-	dns := models.DNSUpdate{
-		Action: models.DNSInsert,
-		Name:   entry.Name,
-		//entry.Address6 is never used
-		Address: entry.Address,
-	}
-	if err := PublishDNSUpdate(entry.Network, dns); err != nil {
-		return err
+	if servercfg.IsDNSMode() {
+		logic.SetDNS()
 	}
-	return nil
-}
 
-// PublishHostDNSUpdate publishes dns update on host name change
-func PublishHostDNSUpdate(old, new *models.Host, networks []string) error {
-	errMsgs := models.DNSError{}
-	for _, network := range networks {
-		dns := models.DNSUpdate{
-			Action:  models.DNSReplaceName,
-			Name:    old.Name + "." + network,
-			NewName: new.Name + "." + network,
-		}
-		if err := PublishDNSUpdate(network, dns); err != nil {
-			errMsgs.ErrorStrings = append(errMsgs.ErrorStrings, err.Error())
-		}
-	}
-	if len(errMsgs.ErrorStrings) > 0 {
-		return errMsgs
-	}
-	return nil
 }
 
 func PushMetricsToExporter(metrics models.Metrics) error {
@@ -422,71 +207,6 @@ func PushMetricsToExporter(metrics models.Metrics) error {
 	return nil
 }
 
-func getNodeDNS(network string) []models.DNSUpdate {
-	alldns := []models.DNSUpdate{}
-	dns := models.DNSUpdate{}
-	nodes, err := logic.GetNetworkNodes(network)
-	if err != nil {
-		logger.Log(0, "error retreiving network nodes for network", network, err.Error())
-	}
-	for _, node := range nodes {
-		host, err := logic.GetHost(node.HostID.String())
-		if err != nil {
-			logger.Log(0, "error retrieving host for dns update", node.HostID.String(), err.Error())
-			continue
-		}
-		dns.Action = models.DNSInsert
-		dns.Name = host.Name + "." + node.Network
-		if node.Address.IP != nil {
-			dns.Address = node.Address.IP.String()
-			alldns = append(alldns, dns)
-		}
-		if node.Address6.IP != nil {
-			dns.Address = node.Address6.IP.String()
-			alldns = append(alldns, dns)
-		}
-	}
-	return alldns
-}
-
-func getExtClientDNS(network string) []models.DNSUpdate {
-	alldns := []models.DNSUpdate{}
-	dns := models.DNSUpdate{}
-	clients, err := logic.GetNetworkExtClients(network)
-	if err != nil {
-		logger.Log(0, "error retrieving extclients", err.Error())
-	}
-	for _, client := range clients {
-		dns.Action = models.DNSInsert
-		dns.Name = client.ClientID + "." + client.Network
-		if client.Address != "" {
-			dns.Address = client.Address
-			alldns = append(alldns, dns)
-		}
-		if client.Address6 != "" {
-			dns.Address = client.Address
-			alldns = append(alldns, dns)
-		}
-	}
-	return alldns
-}
-
-func getCustomDNS(network string) []models.DNSUpdate {
-	alldns := []models.DNSUpdate{}
-	dns := models.DNSUpdate{}
-	customdns, err := logic.GetCustomDNS(network)
-	if err != nil {
-		logger.Log(0, "error retrieving custom dns entries", err.Error())
-	}
-	for _, custom := range customdns {
-		dns.Action = models.DNSInsert
-		dns.Address = custom.Address
-		dns.Name = custom.Name + "." + custom.Network
-		alldns = append(alldns, dns)
-	}
-	return alldns
-}
-
 // sendPeers - retrieve networks, send peer ports to all peers
 func sendPeers() {
 
@@ -515,7 +235,7 @@ func sendPeers() {
 		for _, host := range hosts {
 			host := host
 			logger.Log(2, "sending scheduled peer update (5 min)")
-			if err = PublishSingleHostPeerUpdate(&host, nodes, nil, nil); err != nil {
+			if err = PublishSingleHostPeerUpdate(&host, nodes, nil, nil, false); err != nil {
 				logger.Log(1, "error publishing peer updates for host: ", host.ID.String(), " Err: ", err.Error())
 			}
 		}

mq/util.go

@@ -73,13 +73,15 @@ func encryptMsg(host *models.Host, msg []byte) ([]byte, error) {
 }
 
 func publish(host *models.Host, dest string, msg []byte) error {
+
 	encrypted, encryptErr := encryptMsg(host, msg)
 	if encryptErr != nil {
 		return encryptErr
 	}
-	if mqclient == nil {
+	if mqclient == nil || !mqclient.IsConnectionOpen() {
 		return errors.New("cannot publish ... mqclient not connected")
 	}
+
 	if token := mqclient.Publish(dest, 0, true, encrypted); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
 		var err error
 		if token.Error() == nil {

pro/controllers/failover.go

@@ -0,0 +1,201 @@
+package controllers
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"net/http"
+
+	"github.com/google/uuid"
+	"github.com/gorilla/mux"
+	controller "github.com/gravitl/netmaker/controllers"
+	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/logic"
+	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
+	proLogic "github.com/gravitl/netmaker/pro/logic"
+	"golang.org/x/exp/slog"
+)
+
+// FailOverHandlers - handlers for FailOver
+func FailOverHandlers(r *mux.Router) {
+	r.HandleFunc("/api/v1/node/{nodeid}/failover", logic.SecurityCheck(true, http.HandlerFunc(createfailOver))).Methods(http.MethodPost)
+	r.HandleFunc("/api/v1/node/{nodeid}/failover", logic.SecurityCheck(true, http.HandlerFunc(deletefailOver))).Methods(http.MethodDelete)
+	r.HandleFunc("/api/v1/node/{network}/failover/reset", logic.SecurityCheck(true, http.HandlerFunc(resetFailOver))).Methods(http.MethodPost)
+	r.HandleFunc("/api/v1/node/{nodeid}/failover_me", controller.Authorize(true, false, "host", http.HandlerFunc(failOverME))).Methods(http.MethodPost)
+}
+
+// swagger:route POST /api/v1/node/failover node createfailOver
+//
+// Create a relay.
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: nodeResponse
+func createfailOver(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	nodeid := params["nodeid"]
+	// confirm host exists
+	node, err := logic.GetNodeByID(nodeid)
+	if err != nil {
+		slog.Error("failed to get node:", "error", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+	if _, exists := proLogic.FailOverExists(node.Network); exists {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failover exists already in the network"), "badrequest"))
+		return
+	}
+	host, err := logic.GetHost(node.HostID.String())
+	if err != nil {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("error getting host"+err.Error()), "badrequest"))
+		return
+	}
+	if host.OS != models.OS_Types.Linux {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("only linux nodes can act as failovers"), "badrequest"))
+		return
+	}
+	if node.IsRelayed {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("cannot set relayed node as failover"), "badrequest"))
+		return
+	}
+	node.IsFailOver = true
+	err = logic.UpsertNode(&node)
+	if err != nil {
+		slog.Error("failed to upsert node", "node", node.ID.String(), "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	go mq.PublishPeerUpdate(false)
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponseWithJson(w, r, node, "created failover successfully")
+}
+
+func resetFailOver(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	net := params["network"]
+	nodes, err := logic.GetNetworkNodes(net)
+	if err != nil {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	for _, node := range nodes {
+		if node.FailedOverBy != uuid.Nil {
+			node.FailedOverBy = uuid.Nil
+			node.FailOverPeers = make(map[string]struct{})
+			logic.UpsertNode(&node)
+		}
+	}
+	go mq.PublishPeerUpdate(false)
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponse(w, r, "failover has been reset successfully")
+}
+
+// swagger:route DELETE /api/v1/node/failover node deletefailOver
+//
+// Create a relay.
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: nodeResponse
+func deletefailOver(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	nodeid := params["nodeid"]
+	// confirm host exists
+	node, err := logic.GetNodeByID(nodeid)
+	if err != nil {
+		slog.Error("failed to get node:", "error", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+	node.IsFailOver = false
+	// Reset FailOvered Peers
+	err = logic.UpsertNode(&node)
+	if err != nil {
+		slog.Error("failed to upsert node", "node", node.ID.String(), "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	go func() {
+		proLogic.ResetFailOver(&node)
+		mq.PublishPeerUpdate(false)
+	}()
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponseWithJson(w, r, node, "deleted failover successfully")
+}
+
+// swagger:route POST /api/node/{nodeid}/failOverME node failOver_me
+//
+// Create a relay.
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: nodeResponse
+func failOverME(w http.ResponseWriter, r *http.Request) {
+	var params = mux.Vars(r)
+	nodeid := params["nodeid"]
+	// confirm host exists
+	node, err := logic.GetNodeByID(nodeid)
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"), "failed to get node:", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+
+	failOverNode, exists := proLogic.FailOverExists(node.Network)
+	if !exists {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("failover node doesn't exist in the network"), "badrequest"))
+		return
+	}
+	var failOverReq models.FailOverMeReq
+	err = json.NewDecoder(r.Body).Decode(&failOverReq)
+	if err != nil {
+		logger.Log(0, r.Header.Get("user"), "error decoding request body: ", err.Error())
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+		return
+	}
+	var sendPeerUpdate bool
+	peerNode, err := logic.GetNodeByID(failOverReq.NodeID)
+	if err != nil {
+		slog.Error("peer not found: ", "nodeid", failOverReq.NodeID, "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("peer not found"), "badrequest"))
+		return
+	}
+	if node.IsRelayed || node.IsFailOver {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("node is relayed or acting as failover"), "badrequest"))
+		return
+	}
+	if peerNode.IsRelayed || peerNode.IsFailOver {
+		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("peer node is relayed or acting as failover"), "badrequest"))
+		return
+	}
+
+	err = proLogic.SetFailOverCtx(failOverNode, node, peerNode)
+	if err != nil {
+		slog.Error("failed to create failover", "id", node.ID.String(),
+			"network", node.Network, "error", err)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to create failover: %v", err), "internal"))
+		return
+	}
+	slog.Info("[auto-relay] created relay on node", "node", node.ID.String(), "network", node.Network)
+	sendPeerUpdate = true
+
+	if sendPeerUpdate {
+		go mq.PublishPeerUpdate(false)
+	}
+
+	w.Header().Set("Content-Type", "application/json")
+	logic.ReturnSuccessResponse(w, r, "relayed successfully")
+}

pro/controllers/relay.go

@@ -3,9 +3,11 @@ package controllers
 import (
 	"encoding/json"
 	"fmt"
-	proLogic "github.com/gravitl/netmaker/pro/logic"
 	"net/http"
 
+	"github.com/google/uuid"
+	proLogic "github.com/gravitl/netmaker/pro/logic"
+
 	"github.com/gorilla/mux"
 	controller "github.com/gravitl/netmaker/controllers"
 	"github.com/gravitl/netmaker/logger"
@@ -19,6 +21,7 @@ func RelayHandlers(r *mux.Router) {
 
 	r.HandleFunc("/api/nodes/{network}/{nodeid}/createrelay", controller.Authorize(false, true, "user", http.HandlerFunc(createRelay))).Methods(http.MethodPost)
 	r.HandleFunc("/api/nodes/{network}/{nodeid}/deleterelay", controller.Authorize(false, true, "user", http.HandlerFunc(deleteRelay))).Methods(http.MethodDelete)
+	r.HandleFunc("/api/v1/host/{hostid}/failoverme", controller.Authorize(true, false, "host", http.HandlerFunc(failOverME))).Methods(http.MethodPost)
 }
 
 // swagger:route POST /api/nodes/{network}/{nodeid}/createrelay nodes createRelay
@@ -51,7 +54,16 @@ func createRelay(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
-	go mq.PublishPeerUpdate()
+	for _, relayedNodeID := range relayNode.RelayedNodes {
+		relayedNode, err := logic.GetNodeByID(relayedNodeID)
+		if err == nil {
+			if relayedNode.FailedOverBy != uuid.Nil {
+				go logic.ResetFailedOverPeer(&relayedNode)
+			}
+
+		}
+	}
+	go mq.PublishPeerUpdate(false)
 	logger.Log(1, r.Header.Get("user"), "created relay on node", relayRequest.NodeID, "on network", relayRequest.NetID)
 	apiNode := relayNode.ConvertToAPINode()
 	w.WriteHeader(http.StatusOK)
@@ -96,13 +108,13 @@ func deleteRelay(w http.ResponseWriter, r *http.Request) {
 						return
 					}
 					node.IsRelay = true // for iot update to recognise that it has to delete relay peer
-					if err = mq.PublishSingleHostPeerUpdate(h, nodes, &node, nil); err != nil {
+					if err = mq.PublishSingleHostPeerUpdate(h, nodes, &node, nil, false); err != nil {
 						logger.Log(1, "failed to publish peer update to host", h.ID.String(), ": ", err.Error())
 					}
 				}
 			}
 		}
-		mq.PublishPeerUpdate()
+		mq.PublishPeerUpdate(false)
 	}()
 	logger.Log(1, r.Header.Get("user"), "deleted relay on node", node.ID.String(), "on network", node.Network)
 	apiNode := node.ConvertToAPINode()

pro/controllers/users.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
 )
 
@@ -116,6 +117,9 @@ func removeUserFromRemoteAccessGW(w http.ResponseWriter, r *http.Request) {
 				logic.DeleteExtClient(extclient.Network, extclient.ClientID)
 			}
 		}
+		if servercfg.IsDNSMode() {
+			logic.SetDNS()
+		}
 	}(*user, remoteGwID)
 
 	err = logic.UpsertUser(*user)
@@ -148,17 +152,24 @@ func getUserRemoteAccessGws(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("required params username"), "badrequest"))
 		return
 	}
+	remoteAccessClientID := r.URL.Query().Get("remote_access_clientid")
 	var req models.UserRemoteGwsReq
-	err := json.NewDecoder(r.Body).Decode(&req)
-	if err != nil {
-		slog.Error("error decoding request body: ", "error", err)
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
-		return
+	if remoteAccessClientID == "" {
+		err := json.NewDecoder(r.Body).Decode(&req)
+		if err != nil {
+			slog.Error("error decoding request body: ", "error", err)
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+			return
+		}
 	}
-	if req.RemoteAccessClientID == "" {
+	reqFromMobile := r.URL.Query().Get("from_mobile") == "true"
+	if req.RemoteAccessClientID == "" && remoteAccessClientID == "" {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("remote access client id cannot be empty"), "badrequest"))
 		return
 	}
+	if req.RemoteAccessClientID == "" {
+		req.RemoteAccessClientID = remoteAccessClientID
+	}
 	userGws := make(map[string][]models.UserRemoteGws)
 	user, err := logic.GetUser(username)
 	if err != nil {
@@ -166,16 +177,13 @@ func getUserRemoteAccessGws(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("failed to fetch user %s, error: %v", username, err), "badrequest"))
 		return
 	}
-	if user.IsAdmin || user.IsSuperAdmin {
-		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("admins can visit dashboard to create remote clients"), "badrequest"))
-		return
-	}
 	allextClients, err := logic.GetAllExtClients()
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
 
+	processedAdminNodeIds := make(map[string]struct{})
 	for _, extClient := range allextClients {
 		if extClient.RemoteAccessClientID == req.RemoteAccessClientID && extClient.OwnerID == username {
 			node, err := logic.GetNodeByID(extClient.IngressGatewayID)
@@ -193,50 +201,104 @@ func getUserRemoteAccessGws(w http.ResponseWriter, r *http.Request) {
 				continue
 			}
 
-			if _, ok := user.RemoteGwIDs[node.ID.String()]; ok {
+			if _, ok := user.RemoteGwIDs[node.ID.String()]; (!user.IsAdmin && !user.IsSuperAdmin) && ok {
 				gws := userGws[node.Network]
-
+				extClient.AllowedIPs = logic.GetExtclientAllowedIPs(extClient)
 				gws = append(gws, models.UserRemoteGws{
-					GwID:      node.ID.String(),
-					GWName:    host.Name,
-					Network:   node.Network,
-					GwClient:  extClient,
-					Connected: true,
+					GwID:              node.ID.String(),
+					GWName:            host.Name,
+					Network:           node.Network,
+					GwClient:          extClient,
+					Connected:         true,
+					IsInternetGateway: node.IsInternetGateway,
+					GwPeerPublicKey:   host.PublicKey.String(),
 				})
 				userGws[node.Network] = gws
 				delete(user.RemoteGwIDs, node.ID.String())
-
+			} else {
+				gws := userGws[node.Network]
+				extClient.AllowedIPs = logic.GetExtclientAllowedIPs(extClient)
+				gws = append(gws, models.UserRemoteGws{
+					GwID:              node.ID.String(),
+					GWName:            host.Name,
+					Network:           node.Network,
+					GwClient:          extClient,
+					Connected:         true,
+					IsInternetGateway: node.IsInternetGateway,
+					GwPeerPublicKey:   host.PublicKey.String(),
+				})
+				userGws[node.Network] = gws
+				processedAdminNodeIds[node.ID.String()] = struct{}{}
 			}
 		}
-
 	}
 
 	// add remaining gw nodes to resp
-	for gwID := range user.RemoteGwIDs {
-		node, err := logic.GetNodeByID(gwID)
-		if err != nil {
-			continue
-		}
-		if !node.IsIngressGateway {
-			continue
-		}
-		if node.PendingDelete {
-			continue
+	if !user.IsAdmin && !user.IsSuperAdmin {
+		for gwID := range user.RemoteGwIDs {
+			node, err := logic.GetNodeByID(gwID)
+			if err != nil {
+				continue
+			}
+			if !node.IsIngressGateway {
+				continue
+			}
+			if node.PendingDelete {
+				continue
+			}
+			host, err := logic.GetHost(node.HostID.String())
+			if err != nil {
+				continue
+			}
+			gws := userGws[node.Network]
+
+			gws = append(gws, models.UserRemoteGws{
+				GwID:              node.ID.String(),
+				GWName:            host.Name,
+				Network:           node.Network,
+				IsInternetGateway: node.IsInternetGateway,
+				GwPeerPublicKey:   host.PublicKey.String(),
+			})
+			userGws[node.Network] = gws
 		}
-		host, err := logic.GetHost(node.HostID.String())
+	} else {
+		allNodes, err := logic.GetAllNodes()
 		if err != nil {
-			continue
+			slog.Error("failed to fetch all nodes", "error", err)
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+			return
 		}
-		gws := userGws[node.Network]
+		for _, node := range allNodes {
+			_, ok := processedAdminNodeIds[node.ID.String()]
+			if node.IsIngressGateway && !node.PendingDelete && !ok {
+				host, err := logic.GetHost(node.HostID.String())
+				if err != nil {
+					slog.Error("failed to fetch host", "error", err)
+					continue
+				}
+				gws := userGws[node.Network]
 
-		gws = append(gws, models.UserRemoteGws{
-			GwID:    node.ID.String(),
-			GWName:  host.Name,
-			Network: node.Network,
-		})
-		userGws[node.Network] = gws
+				gws = append(gws, models.UserRemoteGws{
+					GwID:              node.ID.String(),
+					GWName:            host.Name,
+					Network:           node.Network,
+					IsInternetGateway: node.IsInternetGateway,
+					GwPeerPublicKey:   host.PublicKey.String(),
+				})
+				userGws[node.Network] = gws
+			}
+		}
 	}
-
+	if reqFromMobile {
+		// send resp in array format
+		userGwsArr := []models.UserRemoteGws{}
+		for _, userGwI := range userGws {
+			userGwsArr = append(userGwsArr, userGwI...)
+		}
+		logic.ReturnSuccessResponseWithJson(w, r, userGwsArr, "fetched gateways for user"+username)
+		return
+	}
+	slog.Debug("returned user gws", "user", username, "gws", userGws)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(userGws)
 }

pro/initialize.go

@@ -27,6 +27,7 @@ func InitPro() {
 		proControllers.MetricHandlers,
 		proControllers.RelayHandlers,
 		proControllers.UserHandlers,
+		proControllers.FailOverHandlers,
 	)
 	logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() {
 		// == License Handling ==
@@ -42,11 +43,9 @@ func InitPro() {
 		if servercfg.GetServerConfig().RacAutoDisable {
 			AddRacHooks()
 		}
-		resetFailover()
 	})
-	logic.EnterpriseFailoverFunc = proLogic.SetFailover
-	logic.EnterpriseResetFailoverFunc = proLogic.ResetFailover
-	logic.EnterpriseResetAllPeersFailovers = proLogic.WipeAffectedFailoversOnly
+	logic.ResetFailOver = proLogic.ResetFailOver
+	logic.ResetFailedOverPeer = proLogic.ResetFailedOverPeer
 	logic.DenyClientNodeAccess = proLogic.DenyClientNode
 	logic.IsClientNodeAllowed = proLogic.IsClientNodeAllowed
 	logic.AllowClientNodeAccess = proLogic.RemoveDeniedNodeFromClient
@@ -62,19 +61,10 @@ func InitPro() {
 	logic.UpdateRelayed = proLogic.UpdateRelayed
 	logic.SetRelayedNodes = proLogic.SetRelayedNodes
 	logic.RelayUpdates = proLogic.RelayUpdates
+	logic.IsInternetGw = proLogic.IsInternetGw
+	logic.SetInternetGw = proLogic.SetInternetGw
 	mq.UpdateMetrics = proLogic.MQUpdateMetrics
-}
-
-func resetFailover() {
-	nets, err := logic.GetNetworks()
-	if err == nil {
-		for _, net := range nets {
-			err = proLogic.ResetFailover(net.NetID)
-			if err != nil {
-				slog.Error("failed to reset failover", "network", net.NetID, "error", err.Error())
-			}
-		}
-	}
+	mq.UpdateMetricsFallBack = proLogic.MQUpdateMetricsFallBack
 }
 
 func retrieveProLogo() string {

pro/logic/failover.go

@@ -1,122 +1,97 @@
 package logic
 
 import (
+	"errors"
+
 	"github.com/google/uuid"
-	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
 )
 
-// SetFailover - finds a suitable failover candidate and sets it
-func SetFailover(node *models.Node) error {
-	failoverNode := determineFailoverCandidate(node)
-	if failoverNode != nil {
-		return setFailoverNode(failoverNode, node)
+func SetFailOverCtx(failOverNode, victimNode, peerNode models.Node) error {
+	if peerNode.FailOverPeers == nil {
+		peerNode.FailOverPeers = make(map[string]struct{})
+	}
+	if victimNode.FailOverPeers == nil {
+		victimNode.FailOverPeers = make(map[string]struct{})
+	}
+	peerNode.FailOverPeers[victimNode.ID.String()] = struct{}{}
+	victimNode.FailOverPeers[peerNode.ID.String()] = struct{}{}
+	victimNode.FailedOverBy = failOverNode.ID
+	peerNode.FailedOverBy = failOverNode.ID
+	if err := logic.UpsertNode(&failOverNode); err != nil {
+		return err
+	}
+	if err := logic.UpsertNode(&victimNode); err != nil {
+		return err
+	}
+	if err := logic.UpsertNode(&peerNode); err != nil {
+		return err
 	}
 	return nil
 }
 
-// ResetFailover - sets the failover node and wipes disconnected status
-func ResetFailover(network string) error {
-	nodes, err := logic.GetNetworkNodes(network)
-	if err != nil {
-		return err
-	}
+// GetFailOverNode - gets the host acting as failOver
+func GetFailOverNode(network string, allNodes []models.Node) (models.Node, error) {
+	nodes := logic.GetNetworkNodesMemory(allNodes, network)
 	for _, node := range nodes {
-		node := node
-		err = SetFailover(&node)
-		if err != nil {
-			logger.Log(2, "error setting failover for node", node.ID.String(), ":", err.Error())
-		}
-		err = WipeFailover(node.ID.String())
-		if err != nil {
-			logger.Log(2, "error wiping failover for node", node.ID.String(), ":", err.Error())
+		if node.IsFailOver {
+			return node, nil
 		}
 	}
-	return nil
+	return models.Node{}, errors.New("auto relay not found")
 }
 
-// determineFailoverCandidate - returns a list of nodes that
-// are suitable for relaying a given node
-func determineFailoverCandidate(nodeToBeRelayed *models.Node) *models.Node {
-
-	currentNetworkNodes, err := logic.GetNetworkNodes(nodeToBeRelayed.Network)
+// FailOverExists - checks if failOver exists already in the network
+func FailOverExists(network string) (failOverNode models.Node, exists bool) {
+	nodes, err := logic.GetNetworkNodes(network)
 	if err != nil {
-		return nil
-	}
-
-	currentMetrics, err := GetMetrics(nodeToBeRelayed.ID.String())
-	if err != nil || currentMetrics == nil || currentMetrics.Connectivity == nil {
-		return nil
+		return
 	}
-
-	minLatency := int64(9223372036854775807) // max signed int64 value
-	var fastestCandidate *models.Node
-	for i := range currentNetworkNodes {
-		if currentNetworkNodes[i].ID == nodeToBeRelayed.ID {
-			continue
-		}
-
-		if currentMetrics.Connectivity[currentNetworkNodes[i].ID.String()].Connected && (currentNetworkNodes[i].Failover) {
-			if currentMetrics.Connectivity[currentNetworkNodes[i].ID.String()].Latency < int64(minLatency) {
-				fastestCandidate = &currentNetworkNodes[i]
-				minLatency = currentMetrics.Connectivity[currentNetworkNodes[i].ID.String()].Latency
-			}
+	for _, node := range nodes {
+		if node.IsFailOver {
+			exists = true
+			failOverNode = node
+			return
 		}
 	}
-
-	return fastestCandidate
+	return
 }
 
-// setFailoverNode - changes node's failover node
-func setFailoverNode(failoverNode, node *models.Node) error {
-
-	node.FailoverNode = failoverNode.ID
-	nodeToUpdate, err := logic.GetNodeByID(node.ID.String())
+// ResetFailedOverPeer - removes failed over node from network peers
+func ResetFailedOverPeer(failedOveredNode *models.Node) error {
+	nodes, err := logic.GetNetworkNodes(failedOveredNode.Network)
 	if err != nil {
 		return err
 	}
-	if nodeToUpdate.FailoverNode == failoverNode.ID {
-		return nil
-	}
-	return logic.UpdateNode(&nodeToUpdate, node)
-}
-
-// WipeFailover - removes the failover peers of given node (ID)
-func WipeFailover(nodeid string) error {
-	metrics, err := GetMetrics(nodeid)
+	failedOveredNode.FailedOverBy = uuid.Nil
+	failedOveredNode.FailOverPeers = make(map[string]struct{})
+	err = logic.UpsertNode(failedOveredNode)
 	if err != nil {
 		return err
 	}
-	if metrics != nil {
-		metrics.FailoverPeers = make(map[string]string)
-		return logic.UpdateMetrics(nodeid, metrics)
+	for _, node := range nodes {
+		if node.FailOverPeers == nil || node.ID == failedOveredNode.ID {
+			continue
+		}
+		delete(node.FailOverPeers, failedOveredNode.ID.String())
+		logic.UpsertNode(&node)
 	}
 	return nil
 }
 
-// WipeAffectedFailoversOnly - wipes failovers for nodes that have given node (ID)
-// in their respective failover lists
-func WipeAffectedFailoversOnly(nodeid uuid.UUID, network string) error {
-	currentNetworkNodes, err := logic.GetNetworkNodes(network)
+// ResetFailOver - reset failovered peers
+func ResetFailOver(failOverNode *models.Node) error {
+	// Unset FailedOverPeers
+	nodes, err := logic.GetNetworkNodes(failOverNode.Network)
 	if err != nil {
-		return nil
+		return err
 	}
-	WipeFailover(nodeid.String())
-
-	for i := range currentNetworkNodes {
-		currNodeID := currentNetworkNodes[i].ID
-		if currNodeID == nodeid {
-			continue
-		}
-		currMetrics, err := GetMetrics(currNodeID.String())
-		if err != nil || currMetrics == nil {
-			continue
-		}
-		if currMetrics.FailoverPeers != nil {
-			if len(currMetrics.FailoverPeers[nodeid.String()]) > 0 {
-				WipeFailover(currNodeID.String())
-			}
+	for _, node := range nodes {
+		if node.FailedOverBy == failOverNode.ID {
+			node.FailedOverBy = uuid.Nil
+			node.FailOverPeers = make(map[string]struct{})
+			logic.UpsertNode(&node)
 		}
 	}
 	return nil

pro/logic/metrics.go

@@ -2,6 +2,9 @@ package logic
 
 import (
 	"encoding/json"
+	"math"
+	"time"
+
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic"
@@ -10,8 +13,6 @@ import (
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slog"
-	"math"
-	"time"
 )
 
 // GetMetrics - gets the metrics
@@ -45,6 +46,28 @@ func DeleteMetrics(nodeid string) error {
 	return database.DeleteRecord(database.METRICS_TABLE_NAME, nodeid)
 }
 
+// MQUpdateMetricsFallBack - called when mq fallback thread is triggered on client
+func MQUpdateMetricsFallBack(nodeid string, newMetrics models.Metrics) {
+
+	currentNode, err := logic.GetNodeByID(nodeid)
+	if err != nil {
+		slog.Error("error getting node", "id", nodeid, "error", err)
+		return
+	}
+
+	updateNodeMetrics(&currentNode, &newMetrics)
+	if err = logic.UpdateMetrics(nodeid, &newMetrics); err != nil {
+		slog.Error("failed to update node metrics", "id", nodeid, "error", err)
+		return
+	}
+	if servercfg.IsMetricsExporter() {
+		if err := mq.PushMetricsToExporter(newMetrics); err != nil {
+			slog.Error("failed to push node metrics to exporter", "id", currentNode.ID, "error", err)
+		}
+	}
+	slog.Debug("updated node metrics", "id", nodeid)
+}
+
 func MQUpdateMetrics(client mqtt.Client, msg mqtt.Message) {
 	id, err := mq.GetID(msg.Topic())
 	if err != nil {
@@ -67,9 +90,7 @@ func MQUpdateMetrics(client mqtt.Client, msg mqtt.Message) {
 		slog.Error("error unmarshaling payload", "error", err)
 		return
 	}
-
-	shouldUpdate := updateNodeMetrics(&currentNode, &newMetrics)
-
+	updateNodeMetrics(&currentNode, &newMetrics)
 	if err = logic.UpdateMetrics(id, &newMetrics); err != nil {
 		slog.Error("failed to update node metrics", "id", id, "error", err)
 		return
@@ -79,41 +100,15 @@ func MQUpdateMetrics(client mqtt.Client, msg mqtt.Message) {
 			slog.Error("failed to push node metrics to exporter", "id", currentNode.ID, "error", err)
 		}
 	}
-
-	if newMetrics.Connectivity != nil {
-		err := logic.EnterpriseFailoverFunc(&currentNode)
-		if err != nil {
-			slog.Error("failed to failover for node", "id", currentNode.ID, "network", currentNode.Network, "error", err)
-		}
-	}
-
-	if shouldUpdate {
-		slog.Info("updating peers after node detected connectivity issues", "id", currentNode.ID, "network", currentNode.Network)
-		host, err := logic.GetHost(currentNode.HostID.String())
-		if err == nil {
-			nodes, err := logic.GetAllNodes()
-			if err != nil {
-				return
-			}
-			if err = mq.PublishSingleHostPeerUpdate(host, nodes, nil, nil); err != nil {
-				slog.Warn("failed to publish update after failover peer change for node", "id", currentNode.ID, "network", currentNode.Network, "error", err)
-			}
-		}
-	}
 	slog.Debug("updated node metrics", "id", id)
 }
 
-func updateNodeMetrics(currentNode *models.Node, newMetrics *models.Metrics) bool {
-	if newMetrics.FailoverPeers == nil {
-		newMetrics.FailoverPeers = make(map[string]string)
-	}
+func updateNodeMetrics(currentNode *models.Node, newMetrics *models.Metrics) {
+
 	oldMetrics, err := logic.GetMetrics(currentNode.ID.String())
 	if err != nil {
 		slog.Error("error finding old metrics for node", "id", currentNode.ID, "error", err)
-		return false
-	}
-	if oldMetrics.FailoverPeers == nil {
-		oldMetrics.FailoverPeers = make(map[string]string)
+		return
 	}
 
 	var attachedClients []models.ExtClient
@@ -170,34 +165,7 @@ func updateNodeMetrics(currentNode *models.Node, newMetrics *models.Metrics) boo
 
 	}
 
-	// add nodes that need failover
-	nodes, err := logic.GetNetworkNodes(currentNode.Network)
-	if err != nil {
-		slog.Error("failed to retrieve nodes while updating metrics", "error", err)
-		return false
-	}
-	for _, node := range nodes {
-		if !newMetrics.Connectivity[node.ID.String()].Connected &&
-			len(newMetrics.Connectivity[node.ID.String()].NodeName) > 0 &&
-			node.Connected &&
-			len(node.FailoverNode) > 0 &&
-			!node.Failover {
-			newMetrics.FailoverPeers[node.ID.String()] = node.FailoverNode.String()
-		}
-	}
-	shouldUpdate := len(oldMetrics.FailoverPeers) == 0 && len(newMetrics.FailoverPeers) > 0
-	for k, v := range oldMetrics.FailoverPeers {
-		if len(newMetrics.FailoverPeers[k]) > 0 && len(v) == 0 {
-			shouldUpdate = true
-		}
-
-		if len(v) > 0 && len(newMetrics.FailoverPeers[k]) == 0 {
-			newMetrics.FailoverPeers[k] = v
-		}
-	}
-
 	for k := range oldMetrics.Connectivity { // cleanup any left over data, self healing
 		delete(newMetrics.Connectivity, k)
 	}
-	return shouldUpdate
 }

pro/logic/nodes.go

@@ -5,6 +5,16 @@ import (
 	"github.com/gravitl/netmaker/models"
 )
 
+// IsInternetGw - checks if node is acting as internet gw
+func IsInternetGw(node models.Node) bool {
+	return node.IsInternetGateway
+}
+
+// SetInternetGw - sets the node as internet gw based on flag bool
+func SetInternetGw(node *models.Node, flag bool) {
+	node.IsInternetGateway = flag
+}
+
 // GetNetworkIngresses - gets the gateways of a network
 func GetNetworkIngresses(network string) ([]models.Node, error) {
 	var ingresses []models.Node

pro/logic/relays.go

@@ -69,7 +69,7 @@ func SetRelayedNodes(setRelayed bool, relay string, relayed []string) []models.N
 			continue
 		}
 		node.IsRelayed = setRelayed
-		if node.IsRelayed {
+		if setRelayed {
 			node.RelayedBy = relay
 		} else {
 			node.RelayedBy = ""
@@ -155,6 +155,7 @@ func UpdateRelayed(currentNode, newNode *models.Node) {
 	if len(updatenodes) > 0 {
 		for _, relayedNode := range updatenodes {
 			node := relayedNode
+			ResetFailedOverPeer(&node)
 			go func() {
 				if err := mq.NodeUpdate(&node); err != nil {
 					slog.Error("error publishing node update to node", "node", node.ID, "error", err)

pro/remote_access_client.go

@@ -64,9 +64,18 @@ func disableExtClient(client *models.ExtClient) error {
 	} else {
 		// publish peer update to ingress gateway
 		if ingressNode, err := logic.GetNodeByID(newClient.IngressGatewayID); err == nil {
-			if err = mq.PublishPeerUpdate(); err != nil {
+			if err = mq.PublishPeerUpdate(false); err != nil {
 				slog.Error("error updating ext clients on", "ingress", ingressNode.ID.String(), "err", err.Error())
 			}
+			ingressHost, err := logic.GetHost(ingressNode.HostID.String())
+			if err != nil {
+				return err
+			}
+			nodes, err := logic.GetAllNodes()
+			if err != nil {
+				return err
+			}
+			go mq.PublishSingleHostPeerUpdate(ingressHost, nodes, nil, []models.ExtClient{*client}, false)
 		} else {
 			return err
 		}

pro/types.go

@@ -1,3 +1,4 @@
+// go:build ee
 //go:build ee
 // +build ee
 

release.md

@@ -1,19 +1,19 @@
 
-# Netmaker v0.21.2
+# Netmaker v0.22.0
 
 ## Whats New
-- Auto Relay via Enrollment key
-- Local Routing Improvements
-## What's Fixed
-- Inconsistency in DNS Entries for Network has been fixed
-- Unique network CIDR validation
-- Fixed extclient caching decrepancies
-- Deleted node peer update fixes when disconnected from network
-- Force Deletion of Daemon Nodes that are stuck in removing state
-## known issues
-- Windows installer does not install WireGuard
-- netclient-gui will continously display error dialog if netmaker server is offline
-- Mac IPv6 addresses/route issues
-- netclient-gui network tab blank after disconnect
+- Revamped Internet Gateways
+- MQ fallback
+- Deprecating TURN in favour of failover hosts on Pro
+- Switch to CoreDNS for DNS resolution
+- DNS is no longer managed with OS hosts file (/etc/hosts file)
+- Add support for RAC on mobile
 
+## What's Fixed
+- Expired nodes not getting deleted
+- NMCTL acl subcommand leading to dirty state
+- Enforce private network ranges
+- Minor bugs and enhacements with user management
+- Scalability issues
 
+## Known issues

scripts/netmaker.default.env

@@ -6,10 +6,6 @@ NM_DOMAIN=
 SERVER_HOST=
 # The admin master key for accessing the API. Change this in any production installation.
 MASTER_KEY=
-# The username to set for turn api access
-TURN_USERNAME=
-# The password to set for turn api access
-TURN_PASSWORD=
 # The username to set for MQ access
 MQ_USERNAME=
 # The password to set for MQ access
@@ -42,18 +38,10 @@ DATABASE=sqlite
 # If using "host networking", it will find and detect the IP of the mq container.
 # For EMQX websockets use `SERVER_BROKER_ENDPOINT=ws://mq:8083/mqtt`
 SERVER_BROKER_ENDPOINT=ws://mq:1883 
-# The reachable port of STUN on the server
-STUN_PORT=3478
 # Logging verbosity level - 1, 2, or 3
 VERBOSITY=1
-# Port to access turn server
-TURN_PORT=3479
-# Config for using turn, accepts either true/false
-USE_TURN=true
 DEBUG_MODE=off
-TURN_API_PORT=8089
 # Enables the REST backend (API running on API_PORT at SERVER_HTTP_HOST).
-# Change to "off" to turn off.
 REST_BACKEND=on
 # If turned "on", Server will not set Host based on remote IP check.
 # This is already overridden if SERVER_HOST is set. Turned "off" by default.
@@ -80,4 +68,6 @@ OIDC_ISSUER=
 # Duration of JWT token validity in seconds
 JWT_VALIDITY_DURATION=43200
 # Auto disable a user's connecteds clients bassed on JWT token expiration
-RAC_AUTO_DISABLE="true"
+RAC_AUTO_DISABLE=true
+# if turned on data will be cached on to improve performance significantly (IMPORTANT: If HA set to `false` )
+CACHING_ENABLED=true

scripts/nm-quick.sh

@@ -305,11 +305,10 @@ save_config() { (
 		save_config_item SERVER_IMAGE_TAG "$IMAGE_TAG"
 	fi
 	# copy entries from the previous config
-	local toCopy=("SERVER_HOST" "MASTER_KEY" "TURN_USERNAME" "TURN_PASSWORD" "MQ_USERNAME" "MQ_PASSWORD"
+	local toCopy=("SERVER_HOST" "MASTER_KEY" "MQ_USERNAME" "MQ_PASSWORD"
 		"INSTALL_TYPE" "NODE_ID" "DNS_MODE" "NETCLIENT_AUTO_UPDATE" "API_PORT"
-		"CORS_ALLOWED_ORIGIN" "DISPLAY_KEYS" "DATABASE" "SERVER_BROKER_ENDPOINT" "STUN_PORT" "VERBOSITY"
-		"TURN_PORT" "USE_TURN" "DEBUG_MODE" "TURN_API_PORT" "REST_BACKEND"
-		"DISABLE_REMOTE_IP_CHECK" "TELEMETRY" "AUTH_PROVIDER" "CLIENT_ID" "CLIENT_SECRET"
+		"CORS_ALLOWED_ORIGIN" "DISPLAY_KEYS" "DATABASE" "SERVER_BROKER_ENDPOINT" "VERBOSITY"
+		"DEBUG_MODE"  "REST_BACKEND" "DISABLE_REMOTE_IP_CHECK" "TELEMETRY" "AUTH_PROVIDER" "CLIENT_ID" "CLIENT_SECRET"
 		"FRONTEND_URL" "AZURE_TENANT" "OIDC_ISSUER" "EXPORTER_API_PORT" "JWT_VALIDITY_DURATION" "RAC_AUTO_DISABLE")
 	for name in "${toCopy[@]}"; do
 		save_config_item $name "${!name}"
@@ -550,8 +549,6 @@ set_install_vars() {
 	echo "          dashboard.$NETMAKER_BASE_DOMAIN"
 	echo "                api.$NETMAKER_BASE_DOMAIN"
 	echo "             broker.$NETMAKER_BASE_DOMAIN"
-	echo "               turn.$NETMAKER_BASE_DOMAIN"
-	echo "            turnapi.$NETMAKER_BASE_DOMAIN"
 
 	if [ "$INSTALL_TYPE" = "pro" ]; then
 		echo "         prometheus.$NETMAKER_BASE_DOMAIN"
@@ -657,55 +654,6 @@ set_install_vars() {
 		done
 	fi
 
-	unset GET_TURN_USERNAME
-	unset GET_TURN_PASSWORD
-	unset CONFIRM_TURN_PASSWORD
-	echo "Enter Credentials For TURN..."
-	if [ -z $AUTO_BUILD ]; then
-		read -p "TURN Username (click 'enter' to use 'netmaker'): " GET_TURN_USERNAME
-	fi
-	if [ -z "$GET_TURN_USERNAME" ]; then
-		echo "using default username for TURN"
-		TURN_USERNAME="netmaker"
-	else
-		TURN_USERNAME="$GET_TURN_USERNAME"
-	fi
-
-	if test -z "$TURN_PASSWORD"; then
-		TURN_PASSWORD=$(
-			tr -dc A-Za-z0-9 </dev/urandom | head -c 30
-			echo ''
-		)
-	fi
-
-	if [ -z $AUTO_BUILD ]; then
-		select domain_option in "Auto Generated / Config Password" "Input Your Own Password"; do
-			case $REPLY in
-			1)
-				echo "using random password for turn"
-				break
-				;;
-			2)
-				while true; do
-					echo "Enter your Password For TURN: "
-					read -s GET_TURN_PASSWORD
-					echo "Enter your password again to confirm: "
-					read -s CONFIRM_TURN_PASSWORD
-					if [ ${GET_TURN_PASSWORD} != ${CONFIRM_TURN_PASSWORD} ]; then
-						echo "wrong password entered, try again..."
-						continue
-					fi
-					TURN_PASSWORD="$GET_TURN_PASSWORD"
-					echo "TURN Password Saved Successfully!!"
-					break
-				done
-				break
-				;;
-			*) echo "invalid option $REPLY" ;;
-			esac
-		done
-	fi
-
 	wait_seconds 2
 
 	echo "-----------------------------------------------------------------"
@@ -827,7 +775,7 @@ setup_mesh() {
 
 	echo "Obtaining a netmaker enrollment key..."
 
-	local tokenJson=$(nmctl enrollment_key create --unlimited --networks netmaker)
+	local tokenJson=$(nmctl enrollment_key create --tags netmaker --unlimited --networks netmaker)
 	TOKEN=$(jq -r '.token' <<<${tokenJson})
 	if test -z "$TOKEN"; then
 		echo "Error creating an enrollment key"

scripts/nm-upgrade-0-17-1-to-0-19-0.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-LATEST="v0.21.2"
+LATEST="v0.21.0"
 INSTALL_PATH="/root"
 
 trap restore_old_netmaker_instructions

scripts/nm-upgrade.sh

@@ -356,8 +356,6 @@ set_install_vars() {
 	echo "          dashboard.$NETMAKER_BASE_DOMAIN"
 	echo "                api.$NETMAKER_BASE_DOMAIN"
 	echo "             broker.$NETMAKER_BASE_DOMAIN"
-	echo "               turn.$NETMAKER_BASE_DOMAIN"
-	echo "            turnapi.$NETMAKER_BASE_DOMAIN"
 
 	if [ "$INSTALL_TYPE" = "pro" ]; then
 		echo "         prometheus.$NETMAKER_BASE_DOMAIN"

servercfg/serverconf.go

@@ -45,7 +45,6 @@ func GetServerConfig() config.ServerConfig {
 	cfg.AllowedOrigin = GetAllowedOrigin()
 	cfg.RestBackend = "off"
 	cfg.NodeID = GetNodeID()
-	cfg.StunPort = GetStunPort()
 	cfg.BrokerType = GetBrokerType()
 	cfg.EmqxRestEndpoint = GetEmqxRestEndpoint()
 	if AutoUpdateEnabled() {
@@ -120,50 +119,15 @@ func GetServerInfo() models.ServerConfig {
 	cfg.APIPort = GetAPIPort()
 	cfg.DNSMode = "off"
 	cfg.Broker = GetPublicBrokerEndpoint()
+	cfg.BrokerType = GetBrokerType()
 	if IsDNSMode() {
 		cfg.DNSMode = "on"
 	}
 	cfg.Version = GetVersion()
 	cfg.IsPro = IsPro
-	cfg.StunPort = GetStunPort()
-	cfg.TurnDomain = GetTurnHost()
-	cfg.TurnPort = GetTurnPort()
-	cfg.UseTurn = IsUsingTurn()
 	return cfg
 }
 
-// GetTurnHost - fetches the turn host domain
-func GetTurnHost() string {
-	turnServer := ""
-	if os.Getenv("TURN_SERVER_HOST") != "" {
-		turnServer = os.Getenv("TURN_SERVER_HOST")
-	} else if config.Config.Server.TurnServer != "" {
-		turnServer = config.Config.Server.TurnServer
-	}
-	return turnServer
-}
-
-// IsUsingTurn - check if server has turn configured
-func IsUsingTurn() (b bool) {
-	if os.Getenv("USE_TURN") != "" {
-		b = os.Getenv("USE_TURN") == "true"
-	} else {
-		b = config.Config.Server.UseTurn
-	}
-	return
-}
-
-// GetTurnApiHost - fetches the turn api host domain
-func GetTurnApiHost() string {
-	turnApiServer := ""
-	if os.Getenv("TURN_SERVER_API_HOST") != "" {
-		turnApiServer = os.Getenv("TURN_SERVER_API_HOST")
-	} else if config.Config.Server.TurnApiServer != "" {
-		turnApiServer = config.Config.Server.TurnApiServer
-	}
-	return turnApiServer
-}
-
 // GetFrontendURL - gets the frontend url
 func GetFrontendURL() string {
 	var frontend = ""
@@ -207,6 +171,17 @@ func GetDB() string {
 	return database
 }
 
+// CacheEnabled - checks if cache is enabled
+func CacheEnabled() bool {
+	caching := true
+	if os.Getenv("CACHING_ENABLED") == "false" {
+		caching = false
+	} else if config.Config.Server.CacheEnabled == "false" {
+		caching = false
+	}
+	return caching
+}
+
 // GetAPIHost - gets the api host
 func GetAPIHost() string {
 	serverhost := "127.0.0.1"
@@ -635,58 +610,6 @@ func GetNetmakerTenantID() string {
 	return netmakerTenantID
 }
 
-// GetStunPort - Get the port to run the stun server on
-func GetStunPort() int {
-	port := 3478 //default
-	if os.Getenv("STUN_PORT") != "" {
-		portInt, err := strconv.Atoi(os.Getenv("STUN_PORT"))
-		if err == nil {
-			port = portInt
-		}
-	} else if config.Config.Server.StunPort != 0 {
-		port = config.Config.Server.StunPort
-	}
-	return port
-}
-
-// GetTurnPort - Get the port to run the turn server on
-func GetTurnPort() int {
-	port := 3479 //default
-	if os.Getenv("TURN_PORT") != "" {
-		portInt, err := strconv.Atoi(os.Getenv("TURN_PORT"))
-		if err == nil {
-			port = portInt
-		}
-	} else if config.Config.Server.TurnPort != 0 {
-		port = config.Config.Server.TurnPort
-	}
-	return port
-}
-
-// GetTurnUserName - fetches the turn server username
-func GetTurnUserName() string {
-	userName := ""
-	if os.Getenv("TURN_USERNAME") != "" {
-		userName = os.Getenv("TURN_USERNAME")
-	} else {
-		userName = config.Config.Server.TurnUserName
-	}
-	return userName
-
-}
-
-// GetTurnPassword - fetches the turn server password
-func GetTurnPassword() string {
-	pass := ""
-	if os.Getenv("TURN_PASSWORD") != "" {
-		pass = os.Getenv("TURN_PASSWORD")
-	} else {
-		pass = config.Config.Server.TurnPassword
-	}
-	return pass
-
-}
-
 // GetNetworkLimit - fetches free tier limits on users
 func GetUserLimit() int {
 	var userslimit int

swagger.yml

@@ -1149,7 +1149,7 @@ info:
 
         API calls must be authenticated via a header of the format -H “Authorization: Bearer <YOUR_SECRET_KEY>” There are two methods to obtain YOUR_SECRET_KEY: 1. Using the masterkey. By default, this value is “secret key,” but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [Netmaker](https://docs.netmaker.org/index.html) documentation for more details. 2. Using a JWT received for a node. This can be retrieved by calling the /api/nodes/<network>/authenticate endpoint, as documented below.
     title: Netmaker
-    version: 0.21.2
+    version: 0.22.0
 paths:
     /api/dns:
         get: