Merge pull request #1640 from gravitl/release_v0.16.1

Release v0.16.1

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -31,6 +31,7 @@ body:
       label: Version
       description: What version are you running?
       options:
+        - v0.16.1
         - v0.16.0      
         - v0.15.2
         - v0.15.1

.github/workflows/buildandrelease.yml

@@ -402,7 +402,7 @@ jobs:
           cd netclient
           env GOOS=darwin GOARCH=amd64 go build -tags=gui -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-darwin/netclient .
           env CGO_ENABLED=1 GOOS=darwin GOARCH=arm64 go build -tags=gui -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-darwin-arm64/netclient main.go
-          env GOOS=darwin GOARCH=amd64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-darwin/netclient-darwin-headless .
+          env GOOS=darwin GOARCH=amd64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-darwin-headless/netclient .
       - name: Upload darwin-amd64 to Release
         uses: svenstaro/upload-release-action@v2
         with:

.github/workflows/docker-builder.yml

@@ -23,6 +23,6 @@ jobs:
       with:
         context: .
         push: true
-        platforms: linux/amd64, linux/arm64
+        platforms: linux/amd64, linux/arm64, linux/armv7l
         file: ./docker/Dockerfile-go-builder
         tags: gravitl/go-builder:latest

.github/workflows/publish-docker.yml

@@ -55,7 +55,7 @@ jobs:
             sleep 10
             kill %1
       -
-        name: Build arm and export to Docker
+        name: Build arm64 and export to Docker
         uses: docker/build-push-action@v2
         with:
           context: .
@@ -64,7 +64,22 @@ jobs:
           tags: ${{ env.TAG }}
           build-args: version=${{ env.TAG }}
       -
-        name: Test arm
+        name: Test arm64
+        run: |
+            docker run --rm ${{ env.TAG }}&
+            sleep 10
+            kill %1
+      -
+        name: Build armv7l and export to Docker
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          load: true
+          platforms: linux/armv7l
+          tags: ${{ env.TAG }}
+          build-args: version=${{ env.TAG }}
+      -
+        name: Test armv7l
         run: |
             docker run --rm ${{ env.TAG }}&
             sleep 10
@@ -74,7 +89,7 @@ jobs:
         uses: docker/build-push-action@v2
         with:
           context: .
-          platforms: linux/amd64, linux/arm64
+          platforms: linux/amd64, linux/arm64, linux/armv7l
           push: true
           tags: ${{ github.repository }}:${{ env.TAG }}, ${{ github.repository }}:latest
           build-args: version=${{ env.TAG }}

.github/workflows/publish-netclient-docker.yml

@@ -56,7 +56,7 @@ jobs:
             sleep 10
             kill %1
       -
-        name: Build arm and export to Docker
+        name: Build arm64 and export to Docker
         uses: docker/build-push-action@v2
         with:
           context: .
@@ -66,7 +66,7 @@ jobs:
           tags: ${{ env.TAG }}
           build-args: version=${{ env.TAG }}  
       -
-        name: Test arm
+        name: Test arm64
         run: |
             docker run --rm ${{ env.TAG }}&
             sleep 10

.github/workflows/test.yml

@@ -17,6 +17,7 @@ jobs:
       - name: Build
         run: |
          env CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build main.go
+         env CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -tags=ee main.go
          cd netclient
          env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build main.go
          env CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build main.go

README.md

@@ -17,7 +17,7 @@
 
 <p align="center">
   <a href="https://github.com/gravitl/netmaker/releases">
-    <img src="https://img.shields.io/badge/Version-0.16.0-informational?style=flat-square" />
+    <img src="https://img.shields.io/badge/Version-0.16.1-informational?style=flat-square" />
   </a>
   <a href="https://hub.docker.com/r/gravitl/netmaker/tags">
     <img src="https://img.shields.io/docker/pulls/gravitl/netmaker?label=downloads" />
@@ -99,6 +99,8 @@ After installing Netmaker, check out the [Walkthrough](https://itnext.io/getting
 
 - [Netmaker K3S](https://github.com/geragcp/netmaker-k3s)
 
+- [Run Netmaker + Netclient with Podman](https://github.com/agorgl/nm-setup)
+
 ## Disclaimer
  [WireGuard](https://wireguard.com/) is a registered trademark of Jason A. Donenfeld.
 

compose/docker-compose.ee.yml

@@ -3,7 +3,7 @@ version: "3.4"
 services:
   netmaker:
     container_name: netmaker
-    image: gravitl/netmaker:v0.16.0-ee
+    image: gravitl/netmaker:v0.16.1-ee
     cap_add: 
       - NET_ADMIN
       - NET_RAW
@@ -17,7 +17,7 @@ services:
     volumes:
       - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
-      - shared_certs:/etc/netmaker
+      - mosquitto_data:/etc/netmaker
     environment:
       SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
       SERVER_HOST: "SERVER_PUBLIC_IP"
@@ -42,6 +42,7 @@ services:
       METRICS_EXPORTER: "on"
       LICENSE_KEY: "YOUR_LICENSE_KEY"
       NETMAKER_ACCOUNT_ID: "YOUR_ACCOUNT_ID"
+      MQ_ADMIN_PASSWORD: "REPLACE_MQ_ADMIN_PASSWORD"
     ports:
       - "51821-51830:51821-51830/udp"
     expose:
@@ -54,7 +55,7 @@ services:
       - traefik.http.services.netmaker-api.loadbalancer.server.port=8081
   netmaker-ui:
     container_name: netmaker-ui
-    image: gravitl/netmaker-ui:v0.16.0
+    image: gravitl/netmaker-ui:v0.16.1
     depends_on:
       - netmaker
     links:
@@ -112,26 +113,28 @@ services:
     depends_on:
       - netmaker
     restart: unless-stopped
+    command: ["/mosquitto/config/wait.sh"]
+    environment:
+      NETMAKER_SERVER_HOST: "https://api.NETMAKER_BASE_DOMAIN"
     volumes:
       - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
-      - /root/mosquitto.passwords:/etc/mosquitto.passwords
+      - /root/wait.sh:/mosquitto/config/wait.sh
       - mosquitto_data:/mosquitto/data
       - mosquitto_logs:/mosquitto/log
-      - shared_certs:/mosquitto/certs
     expose:
       - "8883"
     labels:
       - traefik.enable=true
-      - traefik.tcp.routers.mqtts.rule=HostSNI(`broker.NETMAKER_BASE_DOMAIN`)
-      - traefik.tcp.routers.mqtts.tls.passthrough=true
-      - traefik.tcp.services.mqtts-svc.loadbalancer.server.port=8883
-      - traefik.tcp.routers.mqtts.service=mqtts-svc
-      - traefik.tcp.routers.mqtts.entrypoints=websecure
+      - traefik.tcp.routers.mqtt.rule=HostSNI(`broker.NETMAKER_BASE_DOMAIN`)
+      - traefik.tcp.routers.mqtt.tls.certresolver=http
+      - traefik.tcp.services.mqtt.loadbalancer.server.port=8883
+      - traefik.tcp.routers.mqtt.entrypoints=websecure
   prometheus:
     container_name: prometheus
     image: gravitl/netmaker-prometheus:latest
     environment:
       NETMAKER_METRICS_TARGET: "netmaker-exporter.NETMAKER_BASE_DOMAIN"
+      LICENSE_KEY: "YOUR_LICENSE_KEY"
     labels:
       - traefik.enable=true
       - traefik.http.routers.prometheus.entrypoints=websecure
@@ -157,6 +160,9 @@ services:
     environment:
       PROMETHEUS_HOST: "prometheus.NETMAKER_BASE_DOMAIN"
       NETMAKER_METRICS_TARGET: "netmaker-exporter.NETMAKER_BASE_DOMAIN"
+      LICENSE_KEY: "YOUR_LICENSE_KEY"
+    volumes:
+      - grafana_data:/var/lib/grafana
     ports:
       - 3000:3000
     restart: always
@@ -180,18 +186,19 @@ services:
     environment:
       MQ_HOST: "mq"
       MQ_PORT: "443"
-      MQ_SERVER_PORT: "1884"
+      MQ_SERVER_PORT: "1883"
       PROMETHEUS: "on"
       VERBOSITY: "1"
       API_PORT: "8085"
+      LICENSE_KEY: "YOUR_LICENSE_KEY"
       PROMETHEUS_HOST: https://prometheus.NETMAKER_BASE_DOMAIN
     expose:
       - "8085"
 volumes:
   traefik_certs: {}
-  shared_certs: {}
   sqldata: {}
   dnsconfig: {}
   mosquitto_data: {}
   mosquitto_logs: {}
   prometheus_data: {}
+  grafana_data: {}

compose/docker-compose.reference.yml

@@ -3,7 +3,7 @@ version: "3.4"
 services:
   netmaker: # The Primary Server for running Netmaker
     container_name: netmaker
-    image: gravitl/netmaker:v0.16.0
+    image: gravitl/netmaker:v0.16.1
     cap_add: 
       - NET_ADMIN
       - NET_RAW
@@ -62,7 +62,7 @@ services:
       - traefik.http.services.netmaker-api.loadbalancer.server.port=8081
   netmaker-ui:  # The Netmaker UI Component
     container_name: netmaker-ui
-    image: gravitl/netmaker-ui:v0.16.0
+    image: gravitl/netmaker-ui:v0.16.1
     depends_on:
       - netmaker
     links:

compose/docker-compose.yml

@@ -3,7 +3,7 @@ version: "3.4"
 services:
   netmaker:
     container_name: netmaker
-    image: gravitl/netmaker:v0.16.0
+    image: gravitl/netmaker:v0.16.1
     cap_add: 
       - NET_ADMIN
       - NET_RAW
@@ -17,7 +17,7 @@ services:
     volumes:
       - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
-      - shared_certs:/etc/netmaker
+      - mosquitto_data:/etc/netmaker
     environment:
       SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
       SERVER_HOST: "SERVER_PUBLIC_IP"
@@ -39,6 +39,7 @@ services:
       VERBOSITY: "1"
       MANAGE_IPTABLES: "on"
       PORT_FORWARD_SERVICES: "dns"
+      MQ_ADMIN_PASSWORD: "REPLACE_MQ_ADMIN_PASSWORD"
     ports:
       - "51821-51830:51821-51830/udp"
     expose:
@@ -51,7 +52,7 @@ services:
       - traefik.http.services.netmaker-api.loadbalancer.server.port=8081
   netmaker-ui:
     container_name: netmaker-ui
-    image: gravitl/netmaker-ui:v0.16.0
+    image: gravitl/netmaker-ui:v0.16.1
     depends_on:
       - netmaker
     links:
@@ -109,24 +110,25 @@ services:
     depends_on:
       - netmaker
     restart: unless-stopped
+    command: ["/mosquitto/config/wait.sh"]
+    environment:
+      NETMAKER_SERVER_HOST: "https://api.NETMAKER_BASE_DOMAIN"
     volumes:
       - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - /root/wait.sh:/mosquitto/config/wait.sh
       - mosquitto_data:/mosquitto/data
       - mosquitto_logs:/mosquitto/log
-      - shared_certs:/mosquitto/certs
     expose:
       - "8883"
     labels:
       - traefik.enable=true
-      - traefik.tcp.routers.mqtts.rule=HostSNI(`broker.NETMAKER_BASE_DOMAIN`)
-      - traefik.tcp.routers.mqtts.tls.passthrough=true
-      - traefik.tcp.services.mqtts-svc.loadbalancer.server.port=8883
-      - traefik.tcp.routers.mqtts.service=mqtts-svc
-      - traefik.tcp.routers.mqtts.entrypoints=websecure
+      - traefik.tcp.routers.mqtt.rule=HostSNI(`broker.NETMAKER_BASE_DOMAIN`)
+      - traefik.tcp.routers.mqtt.tls.certresolver=http
+      - traefik.tcp.services.mqtt.loadbalancer.server.port=8883
+      - traefik.tcp.routers.mqtt.entrypoints=websecure
 volumes:
   traefik_certs: {}
-  shared_certs: {}
   sqldata: {}
   dnsconfig: {}
   mosquitto_data: {}
-  mosquitto_logs: {}
+  mosquitto_logs: {}

config/config.go

@@ -70,6 +70,7 @@ type ServerConfig struct {
 	MQServerPort          string `yaml:"mqserverport"`
 	Server                string `yaml:"server"`
 	PublicIPService       string `yaml:"publicipservice"`
+	MQAdminPassword       string `yaml:"mqadminpassword"`
 	MetricsExporter       string `yaml:"metrics_exporter"`
 	BasicAuth             string `yaml:"basic_auth"`
 	LicenseValue          string `yaml:"license_value"`

controllers/controller.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"os/signal"
 	"sync"
+	"syscall"
 	"time"
 
 	"github.com/gorilla/handlers"
@@ -59,7 +60,7 @@ func HandleRESTRequests(wg *sync.WaitGroup) {
 
 	// Relay os.Interrupt to our channel (os.Interrupt = CTRL+C)
 	// Ignore other incoming signals
-	ctx, stop := signal.NotifyContext(context.TODO(), os.Interrupt)
+	ctx, stop := signal.NotifyContext(context.TODO(), syscall.SIGTERM, os.Interrupt)
 	defer stop()
 
 	// Block main routine until a signal is received

controllers/docs.go

@@ -1,27 +1,26 @@
-//Package classification Netmaker
+// Package classification Netmaker
 //
-// API Usage
+// # API Usage
 //
 // Most actions that can be performed via API can be performed via UI. We recommend managing your networks using the official netmaker-ui project. However, Netmaker can also be run without the UI, and all functions can be achieved via API calls. If your use case requires using Netmaker without the UI or you need to do some troubleshooting/advanced configuration, using the API directly may help.
 //
-//
-// Authentication
+// # Authentication
 //
 // API calls must be authenticated via a header of the format -H “Authorization: Bearer <YOUR_SECRET_KEY>” There are two methods to obtain YOUR_SECRET_KEY: 1. Using the masterkey. By default, this value is “secret key,” but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [Netmaker](https://docs.netmaker.org/index.html) documentation for more details. 2. Using a JWT received for a node. This can be retrieved by calling the /api/nodes/<network>/authenticate endpoint, as documented below.
 //
-//     Schemes: https
-//     BasePath: /
-//     Version: 0.16.0
-//     Host: netmaker.io
+//	Schemes: https
+//	BasePath: /
+//	Version: 0.16.1
+//	Host: netmaker.io
 //
-//     Consumes:
-//     - application/json
+//	Consumes:
+//	- application/json
 //
-//     Produces:
-//     - application/json
+//	Produces:
+//	- application/json
 //
-//     Security:
-//     - oauth
+//	Security:
+//	- oauth
 //
 // swagger:meta
 package controller
@@ -310,7 +309,7 @@ type registerRequestBodyParam struct {
 	RegisterRequest config.RegisterRequest `json:"register_request"`
 }
 
-// swagger:response registerResponse 
+// swagger:response registerResponse
 type registerResponse struct {
 	// Register Response
 	// in: body

controllers/limits.go

@@ -6,6 +6,7 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 // limit consts
@@ -22,7 +23,7 @@ func checkFreeTierLimits(limit_choice int, next http.Handler) http.HandlerFunc {
 			Code: http.StatusUnauthorized, Message: "free tier limits exceeded on networks",
 		}
 
-		if logic.Free_Tier && logic.Is_EE { // check that free tier limits not exceeded
+		if logic.Free_Tier && servercfg.Is_EE { // check that free tier limits not exceeded
 			if limit_choice == networks_l {
 				currentNetworks, err := logic.GetNetworks()
 				if (err != nil && !database.IsEmptyRecord(err)) || len(currentNetworks) >= logic.Networks_Limit {

controllers/network.go

@@ -442,6 +442,20 @@ func deleteNetwork(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, errtype))
 		return
 	}
+	// Deletes the network role from MQ
+	event := mq.MqDynsecPayload{
+		Commands: []mq.MqDynSecCmd{
+			{
+				Command:  mq.DeleteRoleCmd,
+				RoleName: network,
+			},
+		},
+	}
+
+	if err := mq.PublishEventToDynSecTopic(event); err != nil {
+		logger.Log(0, fmt.Sprintf("failed to send DynSec command [%v]: %v",
+			event.Commands, err.Error()))
+	}
 	logger.Log(1, r.Header.Get("user"), "deleted network", network)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode("success")
@@ -488,6 +502,22 @@ func createNetwork(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 		return
 	}
+	// Create Role with acls for the network
+	event := mq.MqDynsecPayload{
+		Commands: []mq.MqDynSecCmd{
+			{
+				Command:  mq.CreateRoleCmd,
+				RoleName: network.NetID,
+				Textname: "Network wide role with Acls for nodes",
+				Acls:     mq.FetchNetworkAcls(network.NetID),
+			},
+		},
+	}
+
+	if err := mq.PublishEventToDynSecTopic(event); err != nil {
+		logger.Log(0, fmt.Sprintf("failed to send DynSec command [%v]: %v",
+			event.Commands, err.Error()))
+	}
 
 	if servercfg.IsClientMode() != "off" {
 		_, err := logic.ServerJoin(&network)

controllers/node.go

@@ -67,75 +67,112 @@ func authenticate(response http.ResponseWriter, request *http.Request) {
 			decoderErr.Error())
 		logic.ReturnErrorResponse(response, request, errorResponse)
 		return
-	} else {
-		errorResponse.Code = http.StatusBadRequest
-		if authRequest.ID == "" {
-			errorResponse.Message = "W1R3: ID can't be empty"
-			logger.Log(0, request.Header.Get("user"), errorResponse.Message)
-			logic.ReturnErrorResponse(response, request, errorResponse)
-			return
-		} else if authRequest.Password == "" {
-			errorResponse.Message = "W1R3: Password can't be empty"
-			logger.Log(0, request.Header.Get("user"), errorResponse.Message)
+	}
+	errorResponse.Code = http.StatusBadRequest
+	if authRequest.ID == "" {
+		errorResponse.Message = "W1R3: ID can't be empty"
+		logger.Log(0, request.Header.Get("user"), errorResponse.Message)
+		logic.ReturnErrorResponse(response, request, errorResponse)
+		return
+	} else if authRequest.Password == "" {
+		errorResponse.Message = "W1R3: Password can't be empty"
+		logger.Log(0, request.Header.Get("user"), errorResponse.Message)
+		logic.ReturnErrorResponse(response, request, errorResponse)
+		return
+	}
+	var err error
+	result, err = logic.GetNodeByID(authRequest.ID)
+	if err != nil {
+		result, err = logic.GetDeletedNodeByID(authRequest.ID)
+		if err != nil {
+			errorResponse.Code = http.StatusBadRequest
+			errorResponse.Message = err.Error()
+			logger.Log(0, request.Header.Get("user"),
+				fmt.Sprintf("failed to get node info [%s]: %v", authRequest.ID, err))
 			logic.ReturnErrorResponse(response, request, errorResponse)
 			return
-		} else {
-			var err error
-			result, err = logic.GetNodeByID(authRequest.ID)
+		}
+	}
 
-			if err != nil {
-				errorResponse.Code = http.StatusBadRequest
-				errorResponse.Message = err.Error()
-				logger.Log(0, request.Header.Get("user"),
-					fmt.Sprintf("failed to get node info [%s]: %v", authRequest.ID, err))
-				logic.ReturnErrorResponse(response, request, errorResponse)
-				return
-			}
+	err = bcrypt.CompareHashAndPassword([]byte(result.Password), []byte(authRequest.Password))
+	if err != nil {
+		errorResponse.Code = http.StatusBadRequest
+		errorResponse.Message = err.Error()
+		logger.Log(0, request.Header.Get("user"),
+			"error validating user password: ", err.Error())
+		logic.ReturnErrorResponse(response, request, errorResponse)
+		return
+	}
+	// creates network role,node client (added here to resolve any missing configuration in MQ)
+	event := mq.MqDynsecPayload{
+		Commands: []mq.MqDynSecCmd{
+
+			{
+				Command:  mq.CreateRoleCmd,
+				RoleName: result.Network,
+				Textname: "Network wide role with Acls for nodes",
+				Acls:     mq.FetchNetworkAcls(result.Network),
+			},
+			{
+				Command:  mq.CreateClientCmd,
+				Username: result.ID,
+				Password: authRequest.Password,
+				Textname: result.Name,
+				Roles: []mq.MqDynSecRole{
+					{
+						Rolename: mq.NodeRole,
+						Priority: -1,
+					},
+					{
+						Rolename: result.Network,
+						Priority: -1,
+					},
+				},
+				Groups: make([]mq.MqDynSecGroup, 0),
+			},
+		},
+	}
 
-			err = bcrypt.CompareHashAndPassword([]byte(result.Password), []byte(authRequest.Password))
-			if err != nil {
-				errorResponse.Code = http.StatusBadRequest
-				errorResponse.Message = err.Error()
-				logger.Log(0, request.Header.Get("user"),
-					"error validating user password: ", err.Error())
-				logic.ReturnErrorResponse(response, request, errorResponse)
-				return
-			} else {
-				tokenString, err := logic.CreateJWT(authRequest.ID, authRequest.MacAddress, result.Network)
-
-				if tokenString == "" {
-					errorResponse.Code = http.StatusBadRequest
-					errorResponse.Message = "Could not create Token"
-					logger.Log(0, request.Header.Get("user"),
-						fmt.Sprintf("%s: %v", errorResponse.Message, err))
-					logic.ReturnErrorResponse(response, request, errorResponse)
-					return
-				}
+	if err := mq.PublishEventToDynSecTopic(event); err != nil {
+		logger.Log(0, fmt.Sprintf("failed to send DynSec command [%v]: %v",
+			event.Commands, err.Error()))
+		errorResponse.Code = http.StatusInternalServerError
+		errorResponse.Message = fmt.Sprintf("could not create mq client for node [%s]: %v", result.ID, err)
+		return
+	}
 
-				var successResponse = models.SuccessResponse{
-					Code:    http.StatusOK,
-					Message: "W1R3: Device " + authRequest.ID + " Authorized",
-					Response: models.SuccessfulLoginResponse{
-						AuthToken: tokenString,
-						ID:        authRequest.ID,
-					},
-				}
-				successJSONResponse, jsonError := json.Marshal(successResponse)
-
-				if jsonError != nil {
-					errorResponse.Code = http.StatusBadRequest
-					errorResponse.Message = err.Error()
-					logger.Log(0, request.Header.Get("user"),
-						"error marshalling resp: ", err.Error())
-					logic.ReturnErrorResponse(response, request, errorResponse)
-					return
-				}
-				response.WriteHeader(http.StatusOK)
-				response.Header().Set("Content-Type", "application/json")
-				response.Write(successJSONResponse)
-			}
-		}
+	tokenString, err := logic.CreateJWT(authRequest.ID, authRequest.MacAddress, result.Network)
+	if tokenString == "" {
+		errorResponse.Code = http.StatusBadRequest
+		errorResponse.Message = "Could not create Token"
+		logger.Log(0, request.Header.Get("user"),
+			fmt.Sprintf("%s: %v", errorResponse.Message, err))
+		logic.ReturnErrorResponse(response, request, errorResponse)
+		return
+	}
+
+	var successResponse = models.SuccessResponse{
+		Code:    http.StatusOK,
+		Message: "W1R3: Device " + authRequest.ID + " Authorized",
+		Response: models.SuccessfulLoginResponse{
+			AuthToken: tokenString,
+			ID:        authRequest.ID,
+		},
 	}
+	successJSONResponse, jsonError := json.Marshal(successResponse)
+
+	if jsonError != nil {
+		errorResponse.Code = http.StatusBadRequest
+		errorResponse.Message = err.Error()
+		logger.Log(0, request.Header.Get("user"),
+			"error marshalling resp: ", err.Error())
+		logic.ReturnErrorResponse(response, request, errorResponse)
+		return
+	}
+	response.WriteHeader(http.StatusOK)
+	response.Header().Set("Content-Type", "application/json")
+	response.Write(successJSONResponse)
+
 }
 
 // auth middleware for api calls from nodes where node is has not yet joined the server (register, join)
@@ -226,6 +263,9 @@ func authorize(nodesAllowed, networkCheck bool, authNetwork string, next http.Ha
 			if nodesAllowed {
 				// TODO --- should ensure that node is only operating on itself
 				if _, _, _, err := logic.VerifyToken(authToken); err == nil {
+
+					// this indicates request is from a node
+					// used for failover - if a getNode comes from node, this will trigger a metrics wipe
 					next.ServeHTTP(w, r)
 					return
 				}
@@ -411,6 +451,8 @@ func getNode(w http.ResponseWriter, r *http.Request) {
 	// set header.
 	w.Header().Set("Content-Type", "application/json")
 
+	nodeRequest := r.Header.Get("requestfrom") == "node"
+
 	var params = mux.Vars(r)
 	nodeid := params["nodeid"]
 	node, err := logic.GetNodeByID(nodeid)
@@ -440,6 +482,12 @@ func getNode(w http.ResponseWriter, r *http.Request) {
 		PeerIDs:      peerUpdate.PeerIDs,
 	}
 
+	if servercfg.Is_EE && nodeRequest {
+		if err = logic.EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
+			logger.Log(1, "failed to reset failover list during node config pull", node.Name, node.Network)
+		}
+	}
+
 	logger.Log(2, r.Header.Get("user"), "fetched node", params["nodeid"])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(response)
@@ -585,7 +633,8 @@ func createNode(w http.ResponseWriter, r *http.Request) {
 		Mine:   node.TrafficKeys.Mine,
 		Server: key,
 	}
-
+	// consume password before hashing for mq client creation
+	nodePassword := node.Password
 	err = logic.CreateNode(&node)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
@@ -621,6 +670,44 @@ func createNode(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// Create client for this node in Mq
+	event := mq.MqDynsecPayload{
+		Commands: []mq.MqDynSecCmd{
+			{ // delete if any client exists already
+				Command:  mq.DeleteClientCmd,
+				Username: node.ID,
+			},
+			{
+				Command:  mq.CreateRoleCmd,
+				RoleName: node.Network,
+				Textname: "Network wide role with Acls for nodes",
+				Acls:     mq.FetchNetworkAcls(node.Network),
+			},
+			{
+				Command:  mq.CreateClientCmd,
+				Username: node.ID,
+				Password: nodePassword,
+				Textname: node.Name,
+				Roles: []mq.MqDynSecRole{
+					{
+						Rolename: mq.NodeRole,
+						Priority: -1,
+					},
+					{
+						Rolename: node.Network,
+						Priority: -1,
+					},
+				},
+				Groups: make([]mq.MqDynSecGroup, 0),
+			},
+		},
+	}
+
+	if err := mq.PublishEventToDynSecTopic(event); err != nil {
+		logger.Log(0, fmt.Sprintf("failed to send DynSec command [%v]: %v",
+			event.Commands, err.Error()))
+	}
+
 	response := models.NodeGet{
 		Node:         node,
 		Peers:        peerUpdate.Peers,
@@ -756,7 +843,13 @@ func createIngressGateway(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	nodeid := params["nodeid"]
 	netid := params["network"]
-	node, err := logic.CreateIngressGateway(netid, nodeid)
+	type failoverData struct {
+		Failover bool `json:"failover"`
+	}
+	var failoverReqBody failoverData
+	json.NewDecoder(r.Body).Decode(&failoverReqBody)
+
+	node, err := logic.CreateIngressGateway(netid, nodeid, failoverReqBody.Failover)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
 			fmt.Sprintf("failed to create ingress gateway on node [%s] on network [%s]: %v",
@@ -765,6 +858,12 @@ func createIngressGateway(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if servercfg.Is_EE && failoverReqBody.Failover {
+		if err = logic.EnterpriseResetFailoverFunc(node.Network); err != nil {
+			logger.Log(1, "failed to reset failover list during failover create", node.Name, node.Network)
+		}
+	}
+
 	logger.Log(1, r.Header.Get("user"), "created ingress gateway on node", nodeid, "on network", netid)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
@@ -788,7 +887,7 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 	var params = mux.Vars(r)
 	nodeid := params["nodeid"]
 	netid := params["network"]
-	node, err := logic.DeleteIngressGateway(netid, nodeid)
+	node, wasFailover, err := logic.DeleteIngressGateway(netid, nodeid)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
 			fmt.Sprintf("failed to delete ingress gateway on node [%s] on network [%s]: %v",
@@ -797,6 +896,12 @@ func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	if servercfg.Is_EE && wasFailover {
+		if err = logic.EnterpriseResetFailoverFunc(node.Network); err != nil {
+			logger.Log(1, "failed to reset failover list during failover create", node.Name, node.Network)
+		}
+	}
+
 	logger.Log(1, r.Header.Get("user"), "deleted ingress gateway", nodeid)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
@@ -880,6 +985,12 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
+	if ifaceDelta && servercfg.Is_EE {
+		if err = logic.EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
+			logger.Log(0, "failed to reset failover lists during node update for node", node.Name, node.Network)
+		}
+	}
+
 	err = logic.UpdateNode(&node, &newNode)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
@@ -927,12 +1038,23 @@ func deleteNode(w http.ResponseWriter, r *http.Request) {
 	// get params
 	var params = mux.Vars(r)
 	var nodeid = params["nodeid"]
+	fromNode := r.Header.Get("requestfrom") == "node"
 	var node, err = logic.GetNodeByID(nodeid)
 	if err != nil {
-		logger.Log(0, r.Header.Get("user"),
-			fmt.Sprintf("error fetching node [ %s ] info: %v", nodeid, err))
-		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
-		return
+		if fromNode {
+			node, err = logic.GetDeletedNodeByID(nodeid)
+			if err != nil {
+				logger.Log(0, r.Header.Get("user"),
+					fmt.Sprintf("error fetching node from deleted nodes [ %s ] info: %v", nodeid, err))
+				logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+				return
+			}
+		} else {
+			logger.Log(0, r.Header.Get("user"),
+				fmt.Sprintf("error fetching node [ %s ] info: %v", nodeid, err))
+			logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
+			return
+		}
 	}
 	if isServer(&node) {
 		err := fmt.Errorf("cannot delete server node")
@@ -951,14 +1073,35 @@ func deleteNode(w http.ResponseWriter, r *http.Request) {
 	//send update to node to be deleted before deleting on server otherwise message cannot be sent
 	node.Action = models.NODE_DELETE
 
-	err = logic.DeleteNodeByID(&node, false)
+	err = logic.DeleteNodeByID(&node, fromNode)
 	if err != nil {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+	if fromNode {
+		// deletes node related role and client
+		event := mq.MqDynsecPayload{
+			Commands: []mq.MqDynSecCmd{
+				{
+					Command:  mq.DeleteClientCmd,
+					Username: nodeid,
+				},
+			},
+		}
 
-	logic.ReturnSuccessResponse(w, r, nodeid+" deleted.")
+		if err := mq.PublishEventToDynSecTopic(event); err != nil {
+			logger.Log(0, fmt.Sprintf("failed to send DynSec command [%v]: %v",
+				event.Commands, err.Error()))
+		}
+	}
 
+	if servercfg.Is_EE {
+		if err = logic.EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
+			logger.Log(0, "failed to reset failover lists during node delete for node", node.Name, node.Network)
+		}
+	}
+
+	logic.ReturnSuccessResponse(w, r, nodeid+" deleted.")
 	logger.Log(1, r.Header.Get("user"), "Deleted node", nodeid, "from network", params["network"])
 	runUpdates(&node, false)
 	runForceServerUpdate(&node, false)

controllers/server.go

@@ -1,28 +1,23 @@
 package controller
 
 import (
-	"crypto/ed25519"
-	"crypto/x509"
-	"crypto/x509/pkix"
 	"encoding/json"
-	"fmt"
 	"net/http"
 	"strings"
 
 	"github.com/gorilla/mux"
-	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
-	"github.com/gravitl/netmaker/netclient/config"
 	"github.com/gravitl/netmaker/servercfg"
-	"github.com/gravitl/netmaker/serverctl"
-	"github.com/gravitl/netmaker/tls"
 )
 
 func serverHandlers(r *mux.Router) {
 	// r.HandleFunc("/api/server/addnetwork/{network}", securityCheckServer(true, http.HandlerFunc(addNetwork))).Methods("POST")
+	r.HandleFunc("/api/server/health", http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) {
+		resp.WriteHeader(http.StatusOK)
+		resp.Write([]byte("Server is up and running!!"))
+	}))
 	r.HandleFunc("/api/server/getconfig", allowUsers(http.HandlerFunc(getConfig))).Methods("GET")
-	r.HandleFunc("/api/server/register", authorize(true, false, "node", http.HandlerFunc(register))).Methods("POST")
 	r.HandleFunc("/api/server/getserverinfo", authorize(true, false, "node", http.HandlerFunc(getServerInfo))).Methods("GET")
 }
 
@@ -54,13 +49,13 @@ func allowUsers(next http.Handler) http.HandlerFunc {
 //
 // Get the server configuration.
 //
-//		Schemes: https
+//			Schemes: https
 //
-// 		Security:
-//   		oauth
+//			Security:
+//	  		oauth
 //
-//		Responses:
-//			200: serverConfigResponse
+//			Responses:
+//				200: serverConfigResponse
 func getServerInfo(w http.ResponseWriter, r *http.Request) {
 	// Set header
 	w.Header().Set("Content-Type", "application/json")
@@ -75,13 +70,13 @@ func getServerInfo(w http.ResponseWriter, r *http.Request) {
 //
 // Get the server configuration.
 //
-//		Schemes: https
+//			Schemes: https
 //
-// 		Security:
-//   		oauth
+//			Security:
+//	  		oauth
 //
-//		Responses:
-//			200: serverConfigResponse
+//			Responses:
+//				200: serverConfigResponse
 func getConfig(w http.ResponseWriter, r *http.Request) {
 	// Set header
 	w.Header().Set("Content-Type", "application/json")
@@ -90,83 +85,9 @@ func getConfig(w http.ResponseWriter, r *http.Request) {
 
 	scfg := servercfg.GetServerConfig()
 	scfg.IsEE = "no"
-	if logic.Is_EE {
+	if servercfg.Is_EE {
 		scfg.IsEE = "yes"
 	}
 	json.NewEncoder(w).Encode(scfg)
 	//w.WriteHeader(http.StatusOK)
 }
-
-// swagger:route POST /api/server/register server register
-//
-// Registers a client with the server and return the Certificate Authority and certificate.
-//
-//		Schemes: https
-//
-// 		Security:
-//   		oauth
-//
-//		Responses:
-//			200: registerResponse
-func register(w http.ResponseWriter, r *http.Request) {
-	logger.Log(2, "processing registration request")
-	w.Header().Set("Content-Type", "application/json")
-	//decode body
-	var request config.RegisterRequest
-	if err := json.NewDecoder(r.Body).Decode(&request); err != nil {
-		logger.Log(0, "error decoding request", err.Error())
-		errorResponse := models.ErrorResponse{
-			Code: http.StatusBadRequest, Message: err.Error(),
-		}
-		logic.ReturnErrorResponse(w, r, errorResponse)
-		return
-	}
-	cert, ca, err := genCerts(&request.Key, &request.CommonName)
-	if err != nil {
-		logger.Log(0, "failed to generater certs ", err.Error())
-		errorResponse := models.ErrorResponse{
-			Code: http.StatusNotFound, Message: err.Error(),
-		}
-		logic.ReturnErrorResponse(w, r, errorResponse)
-		return
-	}
-	//x509.Certificate.PublicKey is an interface therefore json encoding/decoding result in a string value rather than a []byte
-	//include the actual public key so the certificate can be properly reassembled on the other end.
-	response := config.RegisterResponse{
-		CA:         *ca,
-		CAPubKey:   (ca.PublicKey).(ed25519.PublicKey),
-		Cert:       *cert,
-		CertPubKey: (cert.PublicKey).(ed25519.PublicKey),
-		Broker:     servercfg.GetServer(),
-		Port:       servercfg.GetMQPort(),
-	}
-	logger.Log(2, r.Header.Get("user"),
-		fmt.Sprintf("registered client [%+v] with server", request))
-	w.WriteHeader(http.StatusOK)
-	json.NewEncoder(w).Encode(response)
-}
-
-// genCerts generates a client certificate and returns the certificate and root CA
-func genCerts(clientKey *ed25519.PrivateKey, name *pkix.Name) (*x509.Certificate, *x509.Certificate, error) {
-	ca, err := serverctl.ReadCertFromDB(tls.ROOT_PEM_NAME)
-	if err != nil {
-		logger.Log(2, "root ca not found ", err.Error())
-		return nil, nil, fmt.Errorf("root ca not found %w", err)
-	}
-	key, err := serverctl.ReadKeyFromDB(tls.ROOT_KEY_NAME)
-	if err != nil {
-		logger.Log(2, "root key not found ", err.Error())
-		return nil, nil, fmt.Errorf("root key not found %w", err)
-	}
-	csr, err := tls.NewCSR(*clientKey, *name)
-	if err != nil {
-		logger.Log(2, "failed to generate client certificate requests", err.Error())
-		return nil, nil, fmt.Errorf("client certification request generation failed %w", err)
-	}
-	cert, err := tls.NewEndEntityCert(*key, csr, ca, tls.CERTIFICATE_VALIDITY)
-	if err != nil {
-		logger.Log(2, "unable to generate client certificate", err.Error())
-		return nil, nil, fmt.Errorf("client certification generation failed %w", err)
-	}
-	return cert, ca, nil
-}

docker/mosquitto-ee.conf

@@ -1,16 +0,0 @@
-per_listener_settings true
-
-listener 8883
-allow_anonymous false
-require_certificate true
-use_identity_as_username true
-cafile /mosquitto/certs/root.pem
-certfile /mosquitto/certs/server.pem
-keyfile /mosquitto/certs/server.key
-
-listener 1883
-allow_anonymous true
-
-listener 1884
-allow_anonymous false
-password_file /etc/mosquitto.passwords

docker/mosquitto.conf

@@ -1,12 +1,9 @@
-per_listener_settings true
-
+per_listener_settings false
 listener 8883
 allow_anonymous false
-require_certificate true
-use_identity_as_username true
-cafile /mosquitto/certs/root.pem
-certfile /mosquitto/certs/server.pem
-keyfile /mosquitto/certs/server.key
 
 listener 1883
-allow_anonymous true
+allow_anonymous false
+
+plugin /usr/lib/mosquitto_dynamic_security.so
+plugin_opt_config_file /mosquitto/data/dynamic-security.json

docker/mosquitto.passwords

@@ -1 +0,0 @@
-netmaker-exporter:$7$101$9kcXwXP+nUMh06gm$MND2YjtRSvcZTXjMn7xYKoqUFQxG6NOgqWmXIcxxxZksM9cA8732URQWOsPHqpGEvVF9mSVagM1MBEMIKwZm2A==

docker/wait.sh

@@ -0,0 +1,23 @@
+#!/bin/ash
+
+wait_for_netmaker() {
+  echo "SERVER: ${NETMAKER_SERVER_HOST}"
+  until curl --output /dev/null --silent --fail --head \
+    --location "${NETMAKER_SERVER_HOST}/api/server/health"; do
+    echo "Waiting for netmaker server to startup"
+    sleep 1
+  done
+}
+
+main(){
+ # wait for netmaker to startup
+ apk add curl
+ wait_for_netmaker
+ echo "Starting MQ..."
+ # Run the main container command.
+ /docker-entrypoint.sh
+ /usr/sbin/mosquitto -c /mosquitto/config/mosquitto.conf
+
+}
+
+main "${@}"

ee/initialize.go

@@ -6,9 +6,11 @@ package ee
 import (
 	controller "github.com/gravitl/netmaker/controllers"
 	"github.com/gravitl/netmaker/ee/ee_controllers"
+	eelogic "github.com/gravitl/netmaker/ee/logic"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 // InitEE - Initialize EE Logic
@@ -21,12 +23,18 @@ func InitEE() {
 		ValidateLicense()
 		if Limits.FreeTier {
 			logger.Log(0, "proceeding with Free Tier license")
+			logic.SetFreeTierForTelemetry(true)
 		} else {
 			logger.Log(0, "proceeding with Paid Tier license")
+			logic.SetFreeTierForTelemetry(false)
 		}
 		// == End License Handling ==
 		AddLicenseHooks()
+		resetFailover()
 	})
+	logic.EnterpriseFailoverFunc = eelogic.SetFailover
+	logic.EnterpriseResetFailoverFunc = eelogic.ResetFailover
+	logic.EnterpriseResetAllPeersFailovers = eelogic.WipeAffectedFailoversOnly
 }
 
 func setControllerLimits() {
@@ -34,7 +42,19 @@ func setControllerLimits() {
 	logic.Users_Limit = Limits.Users
 	logic.Clients_Limit = Limits.Clients
 	logic.Free_Tier = Limits.FreeTier
-	logic.Is_EE = true
+	servercfg.Is_EE = true
+}
+
+func resetFailover() {
+	nets, err := logic.GetNetworks()
+	if err == nil {
+		for _, net := range nets {
+			err = eelogic.ResetFailover(net.NetID)
+			if err != nil {
+				logger.Log(0, "failed to reset failover on network", net.NetID, ":", err.Error())
+			}
+		}
+	}
 }
 
 func retrieveEELogo() string {

ee/logic/failover.go

@@ -0,0 +1,121 @@
+package logic
+
+import (
+	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/logic"
+	"github.com/gravitl/netmaker/models"
+)
+
+// SetFailover - finds a suitable failover candidate and sets it
+func SetFailover(node *models.Node) error {
+	failoverNode := determineFailoverCandidate(node)
+	if failoverNode != nil {
+		return setFailoverNode(failoverNode, node)
+	}
+	return nil
+}
+
+// ResetFailover - sets the failover node and wipes disconnected status
+func ResetFailover(network string) error {
+	nodes, err := logic.GetNetworkNodes(network)
+	if err != nil {
+		return err
+	}
+	for _, node := range nodes {
+		err = SetFailover(&node)
+		if err != nil {
+			logger.Log(2, "error setting failover for node", node.Name, ":", err.Error())
+		}
+		err = WipeFailover(node.ID)
+		if err != nil {
+			logger.Log(2, "error wiping failover for node", node.Name, ":", err.Error())
+		}
+	}
+	return nil
+}
+
+// determineFailoverCandidate - returns a list of nodes that
+// are suitable for relaying a given node
+func determineFailoverCandidate(nodeToBeRelayed *models.Node) *models.Node {
+
+	currentNetworkNodes, err := logic.GetNetworkNodes(nodeToBeRelayed.Network)
+	if err != nil {
+		return nil
+	}
+
+	currentMetrics, err := logic.GetMetrics(nodeToBeRelayed.ID)
+	if err != nil || currentMetrics == nil || currentMetrics.Connectivity == nil {
+		return nil
+	}
+
+	minLatency := int64(9223372036854775807) // max signed int64 value
+	var fastestCandidate *models.Node
+	for i := range currentNetworkNodes {
+		if currentNetworkNodes[i].ID == nodeToBeRelayed.ID {
+			continue
+		}
+
+		if currentMetrics.Connectivity[currentNetworkNodes[i].ID].Connected && (currentNetworkNodes[i].Failover == "yes") {
+			if currentMetrics.Connectivity[currentNetworkNodes[i].ID].Latency < int64(minLatency) {
+				fastestCandidate = &currentNetworkNodes[i]
+				minLatency = currentMetrics.Connectivity[currentNetworkNodes[i].ID].Latency
+			}
+		}
+	}
+
+	return fastestCandidate
+}
+
+// setFailoverNode - changes node's failover node
+func setFailoverNode(failoverNode, node *models.Node) error {
+
+	node.FailoverNode = failoverNode.ID
+	nodeToUpdate, err := logic.GetNodeByID(node.ID)
+	if err != nil {
+		return err
+	}
+	if nodeToUpdate.FailoverNode == failoverNode.ID {
+		return nil
+	}
+	return logic.UpdateNode(&nodeToUpdate, node)
+}
+
+// WipeFailover - removes the failover peers of given node (ID)
+func WipeFailover(nodeid string) error {
+	metrics, err := logic.GetMetrics(nodeid)
+	if err != nil {
+		return err
+	}
+	if metrics != nil {
+		metrics.FailoverPeers = make(map[string]string)
+		return logic.UpdateMetrics(nodeid, metrics)
+	}
+	return nil
+}
+
+// WipeAffectedFailoversOnly - wipes failovers for nodes that have given node (ID)
+// in their respective failover lists
+func WipeAffectedFailoversOnly(nodeid, network string) error {
+	currentNetworkNodes, err := logic.GetNetworkNodes(network)
+	if err != nil {
+		return nil
+	}
+	WipeFailover(nodeid)
+
+	for i := range currentNetworkNodes {
+		currNodeID := currentNetworkNodes[i].ID
+		if currNodeID == nodeid {
+			continue
+		}
+		currMetrics, err := logic.GetMetrics(currNodeID)
+		if err != nil || currMetrics == nil {
+			continue
+		}
+		if currMetrics.FailoverPeers != nil {
+			if len(currMetrics.FailoverPeers[nodeid]) > 0 {
+				WipeFailover(currNodeID)
+			}
+		}
+	}
+	return nil
+}

ee/util.go

@@ -8,14 +8,10 @@ import (
 
 var isEnterprise bool
 
-// IsEnterprise - checks if enterprise binary or not
-func IsEnterprise() bool {
-	return isEnterprise
-}
-
 // setIsEnterprise - sets server to use enterprise features
 func setIsEnterprise() {
 	isEnterprise = true
+	logic.SetEEForTelemetry(isEnterprise)
 }
 
 // base64encode - base64 encode helper function

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/eclipse/paho.mqtt.golang v1.4.1
-	github.com/go-playground/validator/v10 v10.11.0
+	github.com/go-playground/validator/v10 v10.11.1
 	github.com/golang-jwt/jwt/v4 v4.4.2
 	github.com/google/uuid v1.3.0
 	github.com/gorilla/handlers v1.5.1
@@ -41,7 +41,7 @@ require (
 
 require (
 	github.com/coreos/go-oidc/v3 v3.4.0
-	github.com/gorilla/websocket v1.4.2
+	github.com/gorilla/websocket v1.5.0
 	golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
 	golang.org/x/term v0.0.0-20220722155259-a9ba230a4035
 )

go.sum

@@ -171,8 +171,8 @@ github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb
 github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs=
 github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho=
 github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA=
-github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2BOGlCyvTqsp/xIw=
-github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
+github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ=
+github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
 github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
@@ -274,8 +274,9 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0Ufc=
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
+github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
+github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/goxjs/gl v0.0.0-20210104184919-e3fafc6f8f2a/go.mod h1:dy/f2gjY09hwVfIyATps4G2ai7/hLwLkc5TrPqONuXY=
 github.com/goxjs/glfw v0.0.0-20191126052801-d2efb5f20838/go.mod h1:oS8P8gVOT4ywTcjV6wZlOU4GuVFQ8F5328KY3MJ79CY=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
@@ -667,7 +668,6 @@ golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

k8s/client/netclient-daemonset.yaml

@@ -16,7 +16,7 @@ spec:
       hostNetwork: true
       containers:
       - name: netclient
-        image: gravitl/netclient-go:v0.16.0
+        image: gravitl/netclient:v0.16.1
         env:
         - name: TOKEN
           value: "TOKEN_VALUE"

k8s/client/netclient.yaml

@@ -28,7 +28,7 @@ spec:
       #           - "<node label value>"
       containers:
       - name: netclient
-        image: gravitl/netclient:v0.16.0
+        image: gravitl/netclient:v0.16.1
         env:
         - name: TOKEN
           value: "TOKEN_VALUE"

k8s/server/netmaker-server.yaml

@@ -83,7 +83,7 @@ spec:
           value: "Kubernetes"
         - name: VERBOSITY
           value: "3"
-        image: gravitl/netmaker:v0.16.0
+        image: gravitl/netmaker:v0.16.1
         imagePullPolicy: Always
         name: netmaker
         ports:

k8s/server/netmaker-ui.yaml

@@ -15,7 +15,7 @@ spec:
     spec:
       containers:
       - name: netmaker-ui
-        image: gravitl/netmaker-ui:v0.16.0
+        image: gravitl/netmaker-ui:v0.16.1
         ports:
         - containerPort: 443
         env:

logic/accesskeys.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"sync"
 
-	"github.com/go-playground/validator/v10"
+	validator "github.com/go-playground/validator/v10"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
@@ -221,7 +221,14 @@ func genKeyName() string {
 	return strings.Join([]string{"key", entropy.Text(16)[:16]}, "-")
 }
 
+// GenKey - generates random key of length 16
 func GenKey() string {
 	entropy, _ := rand.Int(rand.Reader, maxentropy)
 	return entropy.Text(16)[:16]
 }
+
+// GenPassWord - generates random password of length 64
+func GenPassWord() string {
+	entropy, _ := rand.Int(rand.Reader, maxentropy)
+	return entropy.Text(62)[:64]
+}

logic/auth.go

@@ -6,12 +6,13 @@ import (
 	"fmt"
 	"time"
 
-	"github.com/go-playground/validator/v10"
+	validator "github.com/go-playground/validator/v10"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic/pro"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/models/promodels"
+	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/crypto/bcrypt"
 )
 
@@ -140,7 +141,7 @@ func CreateUser(user models.User) (models.User, error) {
 
 		// legacy
 		if StringSliceContains(user.Networks, currentNets[i].NetID) {
-			if !Is_EE {
+			if !servercfg.Is_EE {
 				newUser.AccessLevel = pro.NET_ADMIN
 			}
 		}

logic/dns.go

@@ -4,7 +4,7 @@ import (
 	"encoding/json"
 	"os"
 
-	"github.com/go-playground/validator/v10"
+	validator "github.com/go-playground/validator/v10"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"

logic/gateway.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 // CreateEgressGateway - creates an egress gateway
@@ -46,6 +47,10 @@ func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, erro
 	postUpCmd := ""
 	postDownCmd := ""
 	ipv4, ipv6 := getNetworkProtocols(gateway.Ranges)
+	//no support for ipv6 and ip6tables in netmaker container
+	if node.IsServer == "yes" {
+		ipv6 = false
+	}
 	logger.Log(3, "creating egress gateway firewall in use is '", node.FirewallInUse, "'")
 	if node.OS == "linux" {
 		switch node.FirewallInUse {
@@ -87,12 +92,12 @@ func CreateEgressGateway(gateway models.EgressGatewayRequest) (models.Node, erro
 	}
 	if node.PostUp != "" {
 		if !strings.Contains(node.PostUp, postUpCmd) {
-			postUpCmd = node.PostUp + " ; " + postUpCmd
+			postUpCmd = node.PostUp + postUpCmd
 		}
 	}
 	if node.PostDown != "" {
 		if !strings.Contains(node.PostDown, postDownCmd) {
-			postDownCmd = node.PostDown + " ; " + postDownCmd
+			postDownCmd = node.PostDown + postDownCmd
 		}
 	}
 
@@ -172,7 +177,7 @@ func DeleteEgressGateway(network, nodeid string) (models.Node, error) {
 }
 
 // CreateIngressGateway - creates an ingress gateway
-func CreateIngressGateway(netid string, nodeid string) (models.Node, error) {
+func CreateIngressGateway(netid string, nodeid string, failover bool) (models.Node, error) {
 
 	var postUpCmd, postDownCmd string
 	node, err := GetNodeByID(nodeid)
@@ -198,6 +203,10 @@ func CreateIngressGateway(netid string, nodeid string) (models.Node, error) {
 	node.IngressGatewayRange = network.AddressRange
 	node.IngressGatewayRange6 = network.AddressRange6
 	ipv4, ipv6 := getNetworkProtocols(cidrs)
+	//no support for ipv6 and ip6tables in netmaker container
+	if node.IsServer == "yes" {
+		ipv6 = false
+	}
 	logger.Log(3, "creating ingress gateway firewall in use is '", node.FirewallInUse, "'")
 	switch node.FirewallInUse {
 	case models.FIREWALL_NFTABLES:
@@ -224,7 +233,9 @@ func CreateIngressGateway(netid string, nodeid string) (models.Node, error) {
 	node.PostUp = postUpCmd
 	node.PostDown = postDownCmd
 	node.UDPHolePunch = "no"
-
+	if failover && servercfg.Is_EE {
+		node.Failover = "yes"
+	}
 	data, err := json.Marshal(&node)
 	if err != nil {
 		return models.Node{}, err
@@ -238,26 +249,27 @@ func CreateIngressGateway(netid string, nodeid string) (models.Node, error) {
 }
 
 // DeleteIngressGateway - deletes an ingress gateway
-func DeleteIngressGateway(networkName string, nodeid string) (models.Node, error) {
+func DeleteIngressGateway(networkName string, nodeid string) (models.Node, bool, error) {
 
 	node, err := GetNodeByID(nodeid)
 	if err != nil {
-		return models.Node{}, err
+		return models.Node{}, false, err
 	}
 	network, err := GetParentNetwork(networkName)
 	if err != nil {
-		return models.Node{}, err
+		return models.Node{}, false, err
 	}
 	// delete ext clients belonging to ingress gateway
 	if err = DeleteGatewayExtClients(node.ID, networkName); err != nil {
-		return models.Node{}, err
+		return models.Node{}, false, err
 	}
 	logger.Log(3, "deleting ingress gateway")
-
+	wasFailover := node.Failover == "yes"
 	node.UDPHolePunch = network.DefaultUDPHolePunch
 	node.LastModified = time.Now().Unix()
 	node.IsIngressGateway = "no"
 	node.IngressGatewayRange = ""
+	node.Failover = "no"
 
 	// default to removing postup and postdown
 	node.PostUp = ""
@@ -274,14 +286,14 @@ func DeleteIngressGateway(networkName string, nodeid string) (models.Node, error
 
 	data, err := json.Marshal(&node)
 	if err != nil {
-		return models.Node{}, err
+		return models.Node{}, false, err
 	}
 	err = database.Insert(node.ID, string(data), database.NODES_TABLE_NAME)
 	if err != nil {
-		return models.Node{}, err
+		return models.Node{}, wasFailover, err
 	}
 	err = SetNetworkNodesLastModified(networkName)
-	return node, err
+	return node, wasFailover, err
 }
 
 // DeleteGatewayExtClients - deletes ext clients based on gateway (mac) of ingress node and network

logic/metrics/metrics.go

@@ -1,12 +1,14 @@
 package metrics
 
 import (
+	"runtime"
 	"time"
 
 	"github.com/go-ping/ping"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/netclient/wireguard"
 	"golang.zx2c4.com/wireguard/wgctrl"
 )
 
@@ -20,6 +22,14 @@ func Collect(iface string, peerMap models.PeerMap) (*models.Metrics, error) {
 		return &metrics, err
 	}
 	defer wgclient.Close()
+
+	if runtime.GOOS == "darwin" {
+		iface, err = wireguard.GetRealIface(iface)
+		if err != nil {
+			fillUnconnectedData(&metrics, peerMap)
+			return &metrics, err
+		}
+	}
 	device, err := wgclient.Device(iface)
 	if err != nil {
 		fillUnconnectedData(&metrics, peerMap)
@@ -58,11 +68,24 @@ func Collect(iface string, peerMap models.PeerMap) (*models.Metrics, error) {
 				newMetric.Latency = 999
 			} else {
 				pingStats := pinger.Statistics()
-				newMetric.Uptime = 1
+				if pingStats.PacketsRecv > 0 {
+					newMetric.Uptime = 1
+					newMetric.Connected = true
+					newMetric.Latency = pingStats.AvgRtt.Milliseconds()
+				}
+			}
+		}
+
+		// check device peer to see if WG is working if ping failed
+		if !newMetric.Connected {
+			if currPeer.ReceiveBytes > 0 &&
+				currPeer.TransmitBytes > 0 &&
+				time.Now().Before(currPeer.LastHandshakeTime.Add(time.Minute<<1)) {
 				newMetric.Connected = true
-				newMetric.Latency = pingStats.AvgRtt.Milliseconds()
+				newMetric.Uptime = 1
 			}
 		}
+
 		newMetric.TotalTime = 1
 		metrics.Connectivity[id] = newMetric
 	}

logic/networks.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 
 	"github.com/c-robinson/iplib"
-	"github.com/go-playground/validator/v10"
+	validator "github.com/go-playground/validator/v10"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic/acls/nodeacls"

logic/nodes.go

@@ -7,7 +7,7 @@ import (
 	"sort"
 	"time"
 
-	"github.com/go-playground/validator/v10"
+	validator "github.com/go-playground/validator/v10"
 	"github.com/google/uuid"
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
@@ -186,7 +186,9 @@ func DeleteNodeByID(node *models.Node, exterminate bool) error {
 		}
 	}
 	if err = database.DeleteRecord(database.NODES_TABLE_NAME, key); err != nil {
-		return err
+		if !database.IsEmptyRecord(err) {
+			return err
+		}
 	}
 
 	if servercfg.IsDNSMode() {
@@ -212,6 +214,7 @@ func DeleteNodeByID(node *models.Node, exterminate bool) error {
 	if node.IsServer == "yes" {
 		return removeLocalServer(node)
 	}
+
 	return nil
 }
 
@@ -250,6 +253,20 @@ func ValidateNode(node *models.Node, isUpdate bool) error {
 	return err
 }
 
+// IsFailoverPresent - checks if a node is marked as a failover in given network
+func IsFailoverPresent(network string) bool {
+	netNodes, err := GetNetworkNodes(network)
+	if err != nil {
+		return false
+	}
+	for i := range netNodes {
+		if netNodes[i].Failover == "yes" {
+			return true
+		}
+	}
+	return false
+}
+
 // CreateNode - creates a node in database
 func CreateNode(node *models.Node) error {
 
@@ -480,6 +497,7 @@ func SetNodeDefaults(node *models.Node) {
 	node.SetDefaultIsHub()
 	node.SetDefaultConnected()
 	node.SetDefaultACL()
+	node.SetDefaultFailover()
 }
 
 // GetRecordKey - get record key

logic/peers.go

@@ -33,6 +33,16 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 	}
 	var peerMap = make(models.PeerMap)
 
+	var metrics *models.Metrics
+	if servercfg.Is_EE {
+		metrics, _ = GetMetrics(node.ID)
+	}
+	if metrics == nil {
+		metrics = &models.Metrics{}
+	}
+	if metrics.FailoverPeers == nil {
+		metrics.FailoverPeers = make(map[string]string)
+	}
 	// udppeers = the peers parsed from the local interface
 	// gives us correct port to reach
 	udppeers, errN := database.GetPeers(node.Network)
@@ -61,6 +71,10 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 		if node.NetworkSettings.IsPointToSite == "yes" && node.IsHub == "no" && peer.IsHub == "no" {
 			continue
 		}
+		if node.Connected != "yes" {
+			//skip unconnected nodes
+			continue
+		}
 
 		// if the node is not a server, set the endpoint
 		var setEndpoint = !(node.IsServer == "yes")
@@ -81,7 +95,10 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 		if isP2S && peer.IsHub != "yes" {
 			continue
 		}
-
+		if len(metrics.FailoverPeers[peer.ID]) > 0 && IsFailoverPresent(node.Network) {
+			logger.Log(2, "peer", peer.Name, peer.PrimaryAddress(), "was found to be in failover peers list for node", node.Name, node.PrimaryAddress())
+			continue
+		}
 		pubkey, err := wgtypes.ParseKey(peer.PublicKey)
 		if err != nil {
 			return models.PeerUpdate{}, err
@@ -134,8 +151,8 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 				return models.PeerUpdate{}, err
 			}
 		}
-		// set_allowedips
-		allowedips := GetAllowedIPs(node, &peer)
+
+		allowedips := GetAllowedIPs(node, &peer, metrics)
 		var keepalive time.Duration
 		if node.PersistentKeepalive != 0 {
 			// set_keepalive
@@ -243,64 +260,9 @@ func getExtPeers(node *models.Node) ([]wgtypes.PeerConfig, []models.IDandAddr, e
 }
 
 // GetAllowedIPs - calculates the wireguard allowedip field for a peer of a node based on the peer and node settings
-func GetAllowedIPs(node, peer *models.Node) []net.IPNet {
+func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet {
 	var allowedips []net.IPNet
-
-	if peer.Address != "" {
-		var peeraddr = net.IPNet{
-			IP:   net.ParseIP(peer.Address),
-			Mask: net.CIDRMask(32, 32),
-		}
-		allowedips = append(allowedips, peeraddr)
-	}
-
-	if peer.Address6 != "" {
-		var addr6 = net.IPNet{
-			IP:   net.ParseIP(peer.Address6),
-			Mask: net.CIDRMask(128, 128),
-		}
-		allowedips = append(allowedips, addr6)
-	}
-	// handle manually set peers
-	for _, allowedIp := range peer.AllowedIPs {
-
-		// parsing as a CIDR first. If valid CIDR, append
-		if _, ipnet, err := net.ParseCIDR(allowedIp); err == nil {
-			nodeEndpointArr := strings.Split(node.Endpoint, ":")
-			if !ipnet.Contains(net.IP(nodeEndpointArr[0])) && ipnet.IP.String() != peer.Address { // don't need to add an allowed ip that already exists..
-				allowedips = append(allowedips, *ipnet)
-			}
-
-		} else { // parsing as an IP second. If valid IP, check if ipv4 or ipv6, then append
-			if iplib.Version(net.ParseIP(allowedIp)) == 4 && allowedIp != peer.Address {
-				ipnet := net.IPNet{
-					IP:   net.ParseIP(allowedIp),
-					Mask: net.CIDRMask(32, 32),
-				}
-				allowedips = append(allowedips, ipnet)
-			} else if iplib.Version(net.ParseIP(allowedIp)) == 6 && allowedIp != peer.Address6 {
-				ipnet := net.IPNet{
-					IP:   net.ParseIP(allowedIp),
-					Mask: net.CIDRMask(128, 128),
-				}
-				allowedips = append(allowedips, ipnet)
-			}
-		}
-	}
-	// handle egress gateway peers
-	if peer.IsEgressGateway == "yes" {
-		//hasGateway = true
-		egressIPs := getEgressIPs(node, peer)
-		// remove internet gateway if server
-		if node.IsServer == "yes" {
-			for i := len(egressIPs) - 1; i >= 0; i-- {
-				if egressIPs[i].String() == "0.0.0.0/0" || egressIPs[i].String() == "::/0" {
-					egressIPs = append(egressIPs[:i], egressIPs[i+1:]...)
-				}
-			}
-		}
-		allowedips = append(allowedips, egressIPs...)
-	}
+	allowedips = getNodeAllowedIPs(peer, node)
 
 	// handle ingress gateway peers
 	if peer.IsIngressGateway == "yes" {
@@ -311,6 +273,27 @@ func GetAllowedIPs(node, peer *models.Node) []net.IPNet {
 		for _, extPeer := range extPeers {
 			allowedips = append(allowedips, extPeer.AllowedIPs...)
 		}
+		// if node is a failover node, add allowed ips from nodes it is handling
+		if peer.Failover == "yes" && metrics.FailoverPeers != nil {
+			// traverse through nodes that need handling
+			logger.Log(3, "peer", peer.Name, "was found to be failover for", node.Name, "checking failover peers...")
+			for k := range metrics.FailoverPeers {
+				// if FailoverNode is me for this node, add allowedips
+				if metrics.FailoverPeers[k] == peer.ID {
+					// get original node so we can traverse the allowed ips
+					nodeToFailover, err := GetNodeByID(k)
+					if err == nil {
+						failoverNodeMetrics, err := GetMetrics(nodeToFailover.ID)
+						if err == nil && failoverNodeMetrics != nil {
+							if len(failoverNodeMetrics.NodeName) > 0 {
+								allowedips = append(allowedips, getNodeAllowedIPs(&nodeToFailover, peer)...)
+								logger.Log(0, "failing over node", nodeToFailover.Name, nodeToFailover.PrimaryAddress(), "to failover node", peer.Name)
+							}
+						}
+					}
+				}
+			}
+		}
 	}
 	// handle relay gateway peers
 	if peer.IsRelay == "yes" {
@@ -462,6 +445,17 @@ func GetPeerUpdateForRelayedNode(node *models.Node, udppeers map[string]string)
 			}
 		}
 	}
+	//add egress range if relay is egress
+	if relay.IsEgressGateway == "yes" {
+		var ip *net.IPNet
+		for _, cidr := range relay.EgressGatewayRanges {
+			_, ip, err = net.ParseCIDR(cidr)
+			if err != nil {
+				continue
+			}
+		}
+		allowedips = append(allowedips, *ip)
+	}
 
 	pubkey, err := wgtypes.ParseKey(relay.PublicKey)
 	if err != nil {
@@ -555,3 +549,64 @@ func getEgressIPs(node, peer *models.Node) []net.IPNet {
 	}
 	return allowedips
 }
+
+func getNodeAllowedIPs(peer, node *models.Node) []net.IPNet {
+	var allowedips = []net.IPNet{}
+
+	if peer.Address != "" {
+		var peeraddr = net.IPNet{
+			IP:   net.ParseIP(peer.Address),
+			Mask: net.CIDRMask(32, 32),
+		}
+		allowedips = append(allowedips, peeraddr)
+	}
+
+	if peer.Address6 != "" {
+		var addr6 = net.IPNet{
+			IP:   net.ParseIP(peer.Address6),
+			Mask: net.CIDRMask(128, 128),
+		}
+		allowedips = append(allowedips, addr6)
+	}
+	// handle manually set peers
+	for _, allowedIp := range peer.AllowedIPs {
+
+		// parsing as a CIDR first. If valid CIDR, append
+		if _, ipnet, err := net.ParseCIDR(allowedIp); err == nil {
+			nodeEndpointArr := strings.Split(node.Endpoint, ":")
+			if !ipnet.Contains(net.IP(nodeEndpointArr[0])) && ipnet.IP.String() != peer.Address { // don't need to add an allowed ip that already exists..
+				allowedips = append(allowedips, *ipnet)
+			}
+
+		} else { // parsing as an IP second. If valid IP, check if ipv4 or ipv6, then append
+			if iplib.Version(net.ParseIP(allowedIp)) == 4 && allowedIp != peer.Address {
+				ipnet := net.IPNet{
+					IP:   net.ParseIP(allowedIp),
+					Mask: net.CIDRMask(32, 32),
+				}
+				allowedips = append(allowedips, ipnet)
+			} else if iplib.Version(net.ParseIP(allowedIp)) == 6 && allowedIp != peer.Address6 {
+				ipnet := net.IPNet{
+					IP:   net.ParseIP(allowedIp),
+					Mask: net.CIDRMask(128, 128),
+				}
+				allowedips = append(allowedips, ipnet)
+			}
+		}
+	}
+	// handle egress gateway peers
+	if peer.IsEgressGateway == "yes" {
+		//hasGateway = true
+		egressIPs := getEgressIPs(node, peer)
+		// remove internet gateway if server
+		if node.IsServer == "yes" {
+			for i := len(egressIPs) - 1; i >= 0; i-- {
+				if egressIPs[i].String() == "0.0.0.0/0" || egressIPs[i].String() == "::/0" {
+					egressIPs = append(egressIPs[:i], egressIPs[i+1:]...)
+				}
+			}
+		}
+		allowedips = append(allowedips, egressIPs...)
+	}
+	return allowedips
+}

logic/server.go

@@ -18,7 +18,17 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
-var EnterpriseCheckFuncs []interface{}
+// EnterpriseCheckFuncs - can be set to run functions for EE
+var EnterpriseCheckFuncs []func()
+
+// EnterpriseFailoverFunc - interface to control failover funcs
+var EnterpriseFailoverFunc func(node *models.Node) error
+
+// EnterpriseResetFailoverFunc - interface to control reset failover funcs
+var EnterpriseResetFailoverFunc func(network string) error
+
+// EnterpriseResetAllPeersFailovers - resets all nodes that are considering a node to be failover worthy (inclusive)
+var EnterpriseResetAllPeersFailovers func(nodeid, network string) error
 
 // == Join, Checkin, and Leave for Server ==
 
@@ -169,7 +179,7 @@ func ServerJoin(networkSettings *models.Network) (models.Node, error) {
 // EnterpriseCheck - Runs enterprise functions if presented
 func EnterpriseCheck() {
 	for _, check := range EnterpriseCheckFuncs {
-		check.(func())()
+		check()
 	}
 }
 

logic/serverconf.go

@@ -17,8 +17,6 @@ var (
 	Clients_Limit = 1000000000
 	// Free_Tier - specifies if free tier
 	Free_Tier = false
-	// Is_EE - specifies if enterprise
-	Is_EE = false
 )
 
 // constant for database key for storing server ids

logic/telemetry.go

@@ -10,12 +10,28 @@ import (
 	"github.com/posthog/posthog-go"
 )
 
+// flags to keep for telemetry
+var isFreeTier bool
+var isEE bool
+
 // posthog_pub_key - Key for sending data to PostHog
 const posthog_pub_key = "phc_1vEXhPOA1P7HP5jP2dVU9xDTUqXHAelmtravyZ1vvES"
 
 // posthog_endpoint - Endpoint of PostHog server
 const posthog_endpoint = "https://app.posthog.com"
 
+// setEEForTelemetry - store EE flag without having an import cycle when used for telemetry
+// (as the ee package needs the logic package as currently written).
+func SetEEForTelemetry(eeFlag bool) {
+	isEE = eeFlag
+}
+
+// setFreeTierForTelemetry - store free tier flag without having an import cycle when used for telemetry
+// (as the ee package needs the logic package as currently written).
+func SetFreeTierForTelemetry(freeTierFlag bool) {
+	isFreeTier = freeTierFlag
+}
+
 // sendTelemetry - gathers telemetry data and sends to posthog
 func sendTelemetry() error {
 	if servercfg.Telemetry() == "off" {
@@ -54,7 +70,9 @@ func sendTelemetry() error {
 			Set("freebsd", d.Count.FreeBSD).
 			Set("docker", d.Count.Docker).
 			Set("k8s", d.Count.K8S).
-			Set("version", d.Version),
+			Set("version", d.Version).
+			Set("is_ee", isEE).
+			Set("is_free_tier", isFreeTier),
 	})
 }
 
@@ -144,6 +162,8 @@ type telemetryData struct {
 	Networks   int
 	Servers    int
 	Version    string
+	IsEE       bool
+	IsFreeTier bool
 }
 
 // clientCount - What types of netclients we're tallying

logic/util.go

@@ -218,3 +218,11 @@ func StringDifference(a, b []string) []string {
 	}
 	return diff
 }
+
+// CheckIfFileExists - checks if file exists or not in the given path
+func CheckIfFileExists(filePath string) bool {
+	if _, err := os.Stat(filePath); os.IsNotExist(err) {
+		return false
+	}
+	return true
+}

main.go

@@ -3,9 +3,6 @@ package main
 
 import (
 	"context"
-	"crypto/ed25519"
-	"crypto/rand"
-	"errors"
 	"flag"
 	"fmt"
 	"os"
@@ -14,7 +11,6 @@ import (
 	"strconv"
 	"sync"
 	"syscall"
-	"time"
 
 	"github.com/gravitl/netmaker/auth"
 	"github.com/gravitl/netmaker/config"
@@ -29,7 +25,6 @@ import (
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
 	"github.com/gravitl/netmaker/serverctl"
-	"github.com/gravitl/netmaker/tls"
 )
 
 var version = "dev"
@@ -134,10 +129,6 @@ func initialize() { // Client Mode Prereq Check
 		}
 	}
 
-	if err = genCerts(); err != nil {
-		logger.Log(0, "something went wrong when generating broker certs", err.Error())
-	}
-
 	if servercfg.IsMessageQueueBackend() {
 		if err = mq.ServerStartNotify(); err != nil {
 			logger.Log(0, "error occurred when notifying nodes of startup", err.Error())
@@ -154,6 +145,12 @@ func startControllers() {
 			logger.Log(0, "error occurred initializing DNS: ", err.Error())
 		}
 	}
+	if servercfg.IsMessageQueueBackend() {
+		if err := mq.Configure(); err != nil {
+			logger.FatalLog("failed to configure MQ: ", err.Error())
+		}
+	}
+
 	//Run Rest Server
 	if servercfg.IsRestBackend() {
 		if !servercfg.DisableRemoteIPCheck() && servercfg.GetAPIHost() == "127.0.0.1" {
@@ -165,7 +162,6 @@ func startControllers() {
 		waitnetwork.Add(1)
 		go controller.HandleRESTRequests(&waitnetwork)
 	}
-
 	//Run MessageQueue
 	if servercfg.IsMessageQueueBackend() {
 		waitnetwork.Add(1)
@@ -184,6 +180,7 @@ func runMessageQueue(wg *sync.WaitGroup) {
 	defer wg.Done()
 	brokerHost, secure := servercfg.GetMessageQueueEndpoint()
 	logger.Log(0, "connecting to mq broker at", brokerHost, "with TLS?", fmt.Sprintf("%v", secure))
+	mq.SetUpAdminClient()
 	mq.SetupMQTT()
 	ctx, cancel := context.WithCancel(context.Background())
 	go mq.Keepalive(ctx)
@@ -206,142 +203,3 @@ func setGarbageCollection() {
 		debug.SetGCPercent(ncutils.DEFAULT_GC_PERCENT)
 	}
 }
-
-func genCerts() error {
-	logger.Log(0, "checking keys and certificates")
-	var private *ed25519.PrivateKey
-	var err error
-
-	// == ROOT key handling ==
-
-	private, err = serverctl.ReadKeyFromDB(tls.ROOT_KEY_NAME)
-	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) {
-		logger.Log(0, "generating new root key")
-		_, newKey, err := ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			return err
-		}
-		private = &newKey
-	} else if err != nil {
-		return err
-	}
-	logger.Log(2, "saving root.key")
-	if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.ROOT_KEY_NAME, *private); err != nil {
-		return err
-	}
-
-	// == ROOT cert handling ==
-
-	ca, err := serverctl.ReadCertFromDB(tls.ROOT_PEM_NAME)
-	//if cert doesn't exist or will expire within 10 days --- but can't do this as clients won't be able to connect
-	//if errors.Is(err, os.ErrNotExist) || cert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
-	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) || ca.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
-		logger.Log(0, "generating new root CA")
-		caName := tls.NewName("CA Root", "US", "Gravitl")
-		csr, err := tls.NewCSR(*private, caName)
-		if err != nil {
-			return err
-		}
-		rootCA, err := tls.SelfSignedCA(*private, csr, tls.CERTIFICATE_VALIDITY)
-		if err != nil {
-			return err
-		}
-		ca = rootCA
-	} else if err != nil {
-		return err
-	}
-	logger.Log(2, "saving root.pem")
-	if err := serverctl.SaveCert(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.ROOT_PEM_NAME, ca); err != nil {
-		return err
-	}
-
-	// == SERVER cert handling ==
-
-	cert, err := serverctl.ReadCertFromDB(tls.SERVER_PEM_NAME)
-	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) || cert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
-		//gen new key
-		logger.Log(0, "generating new server key/certificate")
-		_, key, err := ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			return err
-		}
-		serverName := tls.NewCName(servercfg.GetServer())
-		csr, err := tls.NewCSR(key, serverName)
-		if err != nil {
-			return err
-		}
-		newCert, err := tls.NewEndEntityCert(*private, csr, ca, tls.CERTIFICATE_VALIDITY)
-		if err != nil {
-			return err
-		}
-		if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_KEY_NAME, key); err != nil {
-			return err
-		}
-		cert = newCert
-	} else if err != nil {
-		return err
-	} else if err == nil {
-		if serverKey, err := serverctl.ReadKeyFromDB(tls.SERVER_KEY_NAME); err == nil {
-			logger.Log(2, "saving server.key")
-			if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_KEY_NAME, *serverKey); err != nil {
-				return err
-			}
-		} else {
-			return err
-		}
-	}
-	logger.Log(2, "saving server.pem")
-	if err := serverctl.SaveCert(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_PEM_NAME, cert); err != nil {
-		return err
-	}
-
-	// == SERVER-CLIENT connection cert handling ==
-
-	serverClientCert, err := serverctl.ReadCertFromDB(tls.SERVER_CLIENT_PEM)
-	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) || serverClientCert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
-		//gen new key
-		logger.Log(0, "generating new server client key/certificate")
-		_, key, err := ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			return err
-		}
-		serverName := tls.NewCName(tls.SERVER_CLIENT_ENTRY)
-		csr, err := tls.NewCSR(key, serverName)
-		if err != nil {
-			return err
-		}
-		newServerClientCert, err := tls.NewEndEntityCert(*private, csr, ca, tls.CERTIFICATE_VALIDITY)
-		if err != nil {
-			return err
-		}
-
-		if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_CLIENT_KEY, key); err != nil {
-			return err
-		}
-		serverClientCert = newServerClientCert
-	} else if err != nil {
-		return err
-	} else if err == nil {
-		logger.Log(2, "saving serverclient.key")
-		if serverClientKey, err := serverctl.ReadKeyFromDB(tls.SERVER_CLIENT_KEY); err == nil {
-			if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_CLIENT_KEY, *serverClientKey); err != nil {
-				return err
-			}
-		} else {
-			return err
-		}
-	}
-
-	logger.Log(2, "saving serverclient.pem")
-	if err := serverctl.SaveCert(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_CLIENT_PEM, serverClientCert); err != nil {
-		return err
-	}
-
-	logger.Log(1, "ensure the root.pem, root.key, server.pem, and server.key files are updated on your broker")
-
-	return serverctl.SetClientTLSConf(
-		functions.GetNetmakerPath()+ncutils.GetSeparator()+tls.SERVER_CLIENT_PEM,
-		functions.GetNetmakerPath()+ncutils.GetSeparator()+tls.SERVER_CLIENT_KEY,
-		ca,
-	)
-}

models/metrics.go

@@ -4,11 +4,12 @@ import "time"
 
 // Metrics - metrics struct
 type Metrics struct {
-	Network      string            `json:"network" bson:"network" yaml:"network"`
-	NodeID       string            `json:"node_id" bson:"node_id" yaml:"node_id"`
-	NodeName     string            `json:"node_name" bson:"node_name" yaml:"node_name"`
-	IsServer     string            `json:"isserver" bson:"isserver" yaml:"isserver" validate:"checkyesorno"`
-	Connectivity map[string]Metric `json:"connectivity" bson:"connectivity" yaml:"connectivity"`
+	Network       string            `json:"network" bson:"network" yaml:"network"`
+	NodeID        string            `json:"node_id" bson:"node_id" yaml:"node_id"`
+	NodeName      string            `json:"node_name" bson:"node_name" yaml:"node_name"`
+	IsServer      string            `json:"isserver" bson:"isserver" yaml:"isserver" validate:"checkyesorno"`
+	Connectivity  map[string]Metric `json:"connectivity" bson:"connectivity" yaml:"connectivity"`
+	FailoverPeers map[string]string `json:"needsfailover" bson:"needsfailover" yaml:"needsfailover"`
 }
 
 // Metric - holds a metric for data between nodes

models/node.go

@@ -82,6 +82,7 @@ type Node struct {
 	EgressGatewayNatEnabled string               `json:"egressgatewaynatenabled" bson:"egressgatewaynatenabled" yaml:"egressgatewaynatenabled"`
 	EgressGatewayRequest    EgressGatewayRequest `json:"egressgatewayrequest" bson:"egressgatewayrequest" yaml:"egressgatewayrequest"`
 	RelayAddrs              []string             `json:"relayaddrs" bson:"relayaddrs" yaml:"relayaddrs"`
+	FailoverNode            string               `json:"failovernode" bson:"failovernode" yaml:"failovernode"`
 	IngressGatewayRange     string               `json:"ingressgatewayrange" bson:"ingressgatewayrange" yaml:"ingressgatewayrange"`
 	IngressGatewayRange6    string               `json:"ingressgatewayrange6" bson:"ingressgatewayrange6" yaml:"ingressgatewayrange6"`
 	// IsStatic - refers to if the Endpoint is set manually or dynamically
@@ -104,6 +105,7 @@ type Node struct {
 	// == PRO ==
 	DefaultACL string `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"`
 	OwnerID    string `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"`
+	Failover   string `json:"failover" bson:"failover" yaml:"failover" validate:"checkyesorno"`
 }
 
 // NodesArray - used for node sorting
@@ -297,6 +299,13 @@ func (node *Node) SetDefaultName() {
 	}
 }
 
+// Node.SetDefaultFailover - sets default value of failover status to no if not set
+func (node *Node) SetDefaultFailover() {
+	if node.Failover == "" {
+		node.Failover = "no"
+	}
+}
+
 // Node.Fill - fills other node data into calling node data if not set on calling node
 func (newNode *Node) Fill(currentNode *Node) { // TODO add new field for nftables present
 	newNode.ID = currentNode.ID
@@ -452,6 +461,10 @@ func (newNode *Node) Fill(currentNode *Node) { // TODO add new field for nftable
 		newNode.DefaultACL = currentNode.DefaultACL
 	}
 
+	if newNode.Failover == "" {
+		newNode.Failover = currentNode.Failover
+	}
+
 	newNode.TrafficKeys = currentNode.TrafficKeys
 }
 

models/structs.go

@@ -218,6 +218,7 @@ type ServerConfig struct {
 	Version     string `yaml:"version"`
 	MQPort      string `yaml:"mqport"`
 	Server      string `yaml:"server"`
+	Is_EE       bool   `yaml:"isee"`
 }
 
 // User.NameInCharset - returns if name is in charset below or not

mq/dynsec.go

@@ -0,0 +1,196 @@
+package mq
+
+import (
+	"crypto/sha512"
+	"encoding/base64"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"os"
+	"time"
+
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/gravitl/netmaker/functions"
+	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/logic"
+	"github.com/gravitl/netmaker/netclient/ncutils"
+	"github.com/gravitl/netmaker/servercfg"
+	"golang.org/x/crypto/pbkdf2"
+)
+
+// mq client for admin
+var mqAdminClient mqtt.Client
+
+const (
+	// constant for client command
+	CreateClientCmd = "createClient"
+	// constant for disable command
+	DisableClientCmd = "disableClient"
+	// constant for delete client command
+	DeleteClientCmd = "deleteClient"
+	// constant for modify client command
+	ModifyClientCmd = "modifyClient"
+
+	// constant for create role command
+	CreateRoleCmd = "createRole"
+	// constant for delete role command
+	DeleteRoleCmd = "deleteRole"
+
+	// constant for admin user name
+	mqAdminUserName = "Netmaker-Admin"
+	// constant for server user name
+	mqNetmakerServerUserName = "Netmaker-Server"
+	// constant for exporter user name
+	mqExporterUserName = "Netmaker-Exporter"
+
+	// DynamicSecSubTopic - constant for dynamic security subscription topic
+	dynamicSecSubTopic = "$CONTROL/dynamic-security/#"
+	// DynamicSecPubTopic - constant for dynamic security subscription topic
+	dynamicSecPubTopic = "$CONTROL/dynamic-security/v1"
+)
+
+// struct for dynamic security file
+type dynJSON struct {
+	Clients    []client         `json:"clients"`
+	Roles      []role           `json:"roles"`
+	DefaultAcl defaultAccessAcl `json:"defaultACLAccess"`
+}
+
+// struct for client role
+type clientRole struct {
+	Rolename string `json:"rolename"`
+}
+
+// struct for MQ client
+type client struct {
+	Username   string       `json:"username"`
+	TextName   string       `json:"textName"`
+	Password   string       `json:"password"`
+	Salt       string       `json:"salt"`
+	Iterations int          `json:"iterations"`
+	Roles      []clientRole `json:"roles"`
+}
+
+// struct for MQ role
+type role struct {
+	Rolename string `json:"rolename"`
+	Acls     []Acl  `json:"acls"`
+}
+
+// struct for default acls
+type defaultAccessAcl struct {
+	PublishClientSend    bool `json:"publishClientSend"`
+	PublishClientReceive bool `json:"publishClientReceive"`
+	Subscribe            bool `json:"subscribe"`
+	Unsubscribe          bool `json:"unsubscribe"`
+}
+
+// MqDynSecGroup - struct for MQ client group
+type MqDynSecGroup struct {
+	Groupname string `json:"groupname"`
+	Priority  int    `json:"priority"`
+}
+
+// MqDynSecRole - struct for MQ client role
+type MqDynSecRole struct {
+	Rolename string `json:"rolename"`
+	Priority int    `json:"priority"`
+}
+
+// Acl - struct for MQ acls
+type Acl struct {
+	AclType  string `json:"acltype"`
+	Topic    string `json:"topic"`
+	Priority int    `json:"priority,omitempty"`
+	Allow    bool   `json:"allow"`
+}
+
+// MqDynSecCmd - struct for MQ dynamic security command
+type MqDynSecCmd struct {
+	Command         string          `json:"command"`
+	Username        string          `json:"username"`
+	Password        string          `json:"password"`
+	RoleName        string          `json:"rolename,omitempty"`
+	Acls            []Acl           `json:"acls,omitempty"`
+	Clientid        string          `json:"clientid"`
+	Textname        string          `json:"textname"`
+	Textdescription string          `json:"textdescription"`
+	Groups          []MqDynSecGroup `json:"groups"`
+	Roles           []MqDynSecRole  `json:"roles"`
+}
+
+// MqDynsecPayload - struct for dynamic security command payload
+type MqDynsecPayload struct {
+	Commands []MqDynSecCmd `json:"commands"`
+}
+
+// encodePasswordToPBKDF2 - encodes the given password with PBKDF2 hashing for MQ
+func encodePasswordToPBKDF2(password string, salt string, iterations int, keyLength int) string {
+	binaryEncoded := pbkdf2.Key([]byte(password), []byte(salt), iterations, keyLength, sha512.New)
+	return base64.StdEncoding.EncodeToString(binaryEncoded)
+}
+
+// Configure - configures the dynamic initial configuration for MQ
+func Configure() error {
+	path := functions.GetNetmakerPath() + ncutils.GetSeparator() + dynamicSecurityFile
+	if logic.CheckIfFileExists(path) {
+		logger.Log(0, "MQ Is Already Configured, Skipping...")
+		return nil
+	}
+	if servercfg.Is_EE {
+		dynConfig.Clients = append(dynConfig.Clients, exporterMQClient)
+		dynConfig.Roles = append(dynConfig.Roles, exporterMQRole)
+	}
+	password := servercfg.GetMqAdminPassword()
+	if password == "" {
+		return errors.New("MQ admin password not provided")
+	}
+	for i, cI := range dynConfig.Clients {
+		if cI.Username == mqAdminUserName || cI.Username == mqNetmakerServerUserName {
+			salt := logic.RandomString(12)
+			hashed := encodePasswordToPBKDF2(password, salt, 101, 64)
+			cI.Password = hashed
+			cI.Iterations = 101
+			cI.Salt = base64.StdEncoding.EncodeToString([]byte(salt))
+			dynConfig.Clients[i] = cI
+		} else if servercfg.Is_EE && cI.Username == mqExporterUserName {
+			exporterPassword := servercfg.GetLicenseKey()
+			salt := logic.RandomString(12)
+			hashed := encodePasswordToPBKDF2(exporterPassword, salt, 101, 64)
+			cI.Password = hashed
+			cI.Iterations = 101
+			cI.Salt = base64.StdEncoding.EncodeToString([]byte(salt))
+			dynConfig.Clients[i] = cI
+		}
+	}
+	data, err := json.MarshalIndent(dynConfig, "", " ")
+	if err != nil {
+		return err
+	}
+	return os.WriteFile(path, data, 0755)
+}
+
+// PublishEventToDynSecTopic - publishes the message to dynamic security topic
+func PublishEventToDynSecTopic(payload MqDynsecPayload) error {
+
+	d, err := json.Marshal(payload)
+	if err != nil {
+		return err
+	}
+	var connecterr error
+	if token := mqAdminClient.Publish(dynamicSecPubTopic, 2, false, d); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
+		if token.Error() == nil {
+			connecterr = errors.New("connect timeout")
+		} else {
+			connecterr = token.Error()
+		}
+	}
+	return connecterr
+}
+
+// watchDynSecTopic - message handler for dynamic security responses
+func watchDynSecTopic(client mqtt.Client, msg mqtt.Message) {
+
+	logger.Log(1, fmt.Sprintf("----->WatchDynSecTopic Message: %+v", string(msg.Payload())))
+
+}

mq/dynsec_helper.go

@@ -0,0 +1,384 @@
+package mq
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"time"
+
+	mqtt "github.com/eclipse/paho.mqtt.golang"
+	"github.com/gravitl/netmaker/servercfg"
+)
+
+const (
+	// constant for admin role
+	adminRole = "admin"
+	// constant for server role
+	serverRole = "server"
+	// constant for exporter role
+	exporterRole = "exporter"
+	// constant for node role
+	NodeRole = "node"
+
+	// const for dynamic security file
+	dynamicSecurityFile = "dynamic-security.json"
+)
+
+var (
+	// default configuration of dynamic security
+	dynConfig = dynJSON{
+		Clients: []client{
+			{
+				Username:   mqAdminUserName,
+				TextName:   "netmaker admin user",
+				Password:   "",
+				Salt:       "",
+				Iterations: 0,
+				Roles: []clientRole{
+					{
+						Rolename: adminRole,
+					},
+				},
+			},
+			{
+				Username:   mqNetmakerServerUserName,
+				TextName:   "netmaker server user",
+				Password:   "",
+				Salt:       "",
+				Iterations: 0,
+				Roles: []clientRole{
+					{
+						Rolename: serverRole,
+					},
+				},
+			},
+		},
+		Roles: []role{
+			{
+				Rolename: adminRole,
+				Acls:     fetchAdminAcls(),
+			},
+			{
+				Rolename: serverRole,
+				Acls:     fetchServerAcls(),
+			},
+			{
+				Rolename: NodeRole,
+				Acls:     fetchNodeAcls(),
+			},
+		},
+		DefaultAcl: defaultAccessAcl{
+			PublishClientSend:    false,
+			PublishClientReceive: true,
+			Subscribe:            false,
+			Unsubscribe:          true,
+		},
+	}
+
+	exporterMQClient = client{
+		Username:   mqExporterUserName,
+		TextName:   "netmaker metrics exporter",
+		Password:   "",
+		Salt:       "",
+		Iterations: 101,
+		Roles: []clientRole{
+			{
+				Rolename: exporterRole,
+			},
+		},
+	}
+	exporterMQRole = role{
+		Rolename: exporterRole,
+		Acls:     fetchExporterAcls(),
+	}
+)
+
+// DynListCLientsCmdResp - struct for list clients response from MQ
+type DynListCLientsCmdResp struct {
+	Responses []struct {
+		Command string          `json:"command"`
+		Error   string          `json:"error"`
+		Data    ListClientsData `json:"data"`
+	} `json:"responses"`
+}
+
+// ListClientsData - struct for list clients data
+type ListClientsData struct {
+	Clients    []string `json:"clients"`
+	TotalCount int      `json:"totalCount"`
+}
+
+// GetAdminClient - fetches admin client of the MQ
+func GetAdminClient() (mqtt.Client, error) {
+	opts := mqtt.NewClientOptions()
+	setMqOptions(mqAdminUserName, servercfg.GetMqAdminPassword(), opts)
+	mqclient := mqtt.NewClient(opts)
+	var connecterr error
+	if token := mqclient.Connect(); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
+		if token.Error() == nil {
+			connecterr = errors.New("connect timeout")
+		} else {
+			connecterr = token.Error()
+		}
+	}
+	return mqclient, connecterr
+}
+
+// ListClients -  to list all clients in the MQ
+func ListClients(client mqtt.Client) (ListClientsData, error) {
+	respChan := make(chan mqtt.Message, 10)
+	defer close(respChan)
+	command := "listClients"
+	resp := ListClientsData{}
+	msg := MqDynsecPayload{
+		Commands: []MqDynSecCmd{
+			{
+				Command: command,
+			},
+		},
+	}
+	client.Subscribe("$CONTROL/dynamic-security/v1/response", 2, mqtt.MessageHandler(func(c mqtt.Client, m mqtt.Message) {
+		respChan <- m
+	}))
+	defer client.Unsubscribe()
+	d, _ := json.Marshal(msg)
+	token := client.Publish("$CONTROL/dynamic-security/v1", 2, true, d)
+	if !token.WaitTimeout(30) || token.Error() != nil {
+		var err error
+		if token.Error() == nil {
+			err = errors.New("connection timeout")
+		} else {
+			err = token.Error()
+		}
+		return resp, err
+	}
+
+	for m := range respChan {
+		msg := DynListCLientsCmdResp{}
+		json.Unmarshal(m.Payload(), &msg)
+		for _, mI := range msg.Responses {
+			if mI.Command == command {
+				return mI.Data, nil
+			}
+		}
+	}
+	return resp, errors.New("resp not found")
+}
+
+// FetchNetworkAcls - fetches network acls
+func FetchNetworkAcls(network string) []Acl {
+	return []Acl{
+		{
+			AclType:  "publishClientReceive",
+			Topic:    fmt.Sprintf("update/%s/#", network),
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    fmt.Sprintf("peers/%s/#", network),
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "subscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "unsubscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+	}
+}
+
+// serverAcls - fetches server role related acls
+func fetchServerAcls() []Acl {
+	return []Acl{
+		{
+			AclType:  "publishClientSend",
+			Topic:    "peers/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientSend",
+			Topic:    "update/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientSend",
+			Topic:    "metrics_exporter",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "ping/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "update/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "signal/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "metrics/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "subscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "unsubscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+	}
+}
+
+// fetchNodeAcls - fetches node related acls
+func fetchNodeAcls() []Acl {
+	// keeping node acls generic as of now.
+	return []Acl{
+
+		{
+			AclType:  "publishClientSend",
+			Topic:    "signal/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientSend",
+			Topic:    "update/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientSend",
+			Topic:    "ping/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientSend",
+			Topic:    "metrics/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "subscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "unsubscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+	}
+}
+
+// fetchExporterAcls - fetch exporter role related acls
+func fetchExporterAcls() []Acl {
+	return []Acl{
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "metrics_exporter",
+			Allow:    true,
+			Priority: -1,
+		},
+		{
+			AclType:  "subscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "unsubscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+	}
+}
+
+// fetchAdminAcls - fetches admin role related acls
+func fetchAdminAcls() []Acl {
+	return []Acl{
+		{
+			AclType:  "publishClientSend",
+			Topic:    "$CONTROL/dynamic-security/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "$CONTROL/dynamic-security/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "subscribePattern",
+			Topic:    "$CONTROL/dynamic-security/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "$SYS/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "subscribePattern",
+			Topic:    "$SYS/#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "subscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "unsubscribePattern",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+		{
+			AclType:  "publishClientSend",
+			Topic:    "#",
+			Priority: -1,
+			Allow:    true,
+		},
+	}
+}

mq/handlers.go

@@ -86,6 +86,12 @@ func UpdateNode(client mqtt.Client, msg mqtt.Message) {
 			logger.Log(1, "error unmarshaling payload ", err.Error())
 			return
 		}
+		ifaceDelta := logic.IfaceDelta(&currentNode, &newNode)
+		if servercfg.Is_EE && ifaceDelta {
+			if err = logic.EnterpriseResetAllPeersFailovers(currentNode.ID, currentNode.Network); err != nil {
+				logger.Log(1, "failed to reset failover list during node update", currentNode.Name, currentNode.Network)
+			}
+		}
 		newNode.SetLastCheckIn()
 		if err := logic.UpdateNode(&currentNode, &newNode); err != nil {
 			logger.Log(1, "error saving node", err.Error())
@@ -98,7 +104,7 @@ func UpdateNode(client mqtt.Client, msg mqtt.Message) {
 
 // UpdateMetrics  message Handler -- handles updates from client nodes for metrics
 func UpdateMetrics(client mqtt.Client, msg mqtt.Message) {
-	if logic.Is_EE {
+	if servercfg.Is_EE {
 		go func() {
 			id, err := getID(msg.Topic())
 			if err != nil {
@@ -122,7 +128,7 @@ func UpdateMetrics(client mqtt.Client, msg mqtt.Message) {
 				return
 			}
 
-			updateNodeMetrics(&currentNode, &newMetrics)
+			shouldUpdate := updateNodeMetrics(&currentNode, &newMetrics)
 
 			if err = logic.UpdateMetrics(id, &newMetrics); err != nil {
 				logger.Log(1, "faield to update node metrics", id, currentNode.Name, err.Error())
@@ -135,6 +141,20 @@ func UpdateMetrics(client mqtt.Client, msg mqtt.Message) {
 				}
 			}
 
+			if newMetrics.Connectivity != nil {
+				err := logic.EnterpriseFailoverFunc(&currentNode)
+				if err != nil {
+					logger.Log(0, "failed to failover for node", currentNode.Name, "on network", currentNode.Network, "-", err.Error())
+				}
+			}
+
+			if shouldUpdate {
+				logger.Log(2, "updating peers after node", currentNode.Name, currentNode.Network, "detected connectivity issues")
+				if err = PublishSinglePeerUpdate(&currentNode); err != nil {
+					logger.Log(0, "failed to publish update after failover peer change for node", currentNode.Name, currentNode.Network)
+				}
+			}
+
 			logger.Log(1, "updated node metrics", id, currentNode.Name)
 		}()
 	}
@@ -194,11 +214,17 @@ func updateNodePeers(currentNode *models.Node) {
 	}
 }
 
-func updateNodeMetrics(currentNode *models.Node, newMetrics *models.Metrics) {
+func updateNodeMetrics(currentNode *models.Node, newMetrics *models.Metrics) bool {
+	if newMetrics.FailoverPeers == nil {
+		newMetrics.FailoverPeers = make(map[string]string)
+	}
 	oldMetrics, err := logic.GetMetrics(currentNode.ID)
 	if err != nil {
 		logger.Log(1, "error finding old metrics for node", currentNode.ID, currentNode.Name)
-		return
+		return false
+	}
+	if oldMetrics.FailoverPeers == nil {
+		oldMetrics.FailoverPeers = make(map[string]string)
 	}
 
 	var attachedClients []models.ExtClient
@@ -225,14 +251,45 @@ func updateNodeMetrics(currentNode *models.Node, newMetrics *models.Metrics) {
 		oldMetric := oldMetrics.Connectivity[k]
 		currMetric.TotalTime += oldMetric.TotalTime
 		currMetric.Uptime += oldMetric.Uptime // get the total uptime for this connection
-		currMetric.PercentUp = 100.0 * (float64(currMetric.Uptime) / float64(currMetric.TotalTime))
-		totalUpMinutes := currMetric.Uptime * 5
+		if currMetric.Uptime == 0 || currMetric.TotalTime == 0 {
+			currMetric.PercentUp = 0
+		} else {
+			currMetric.PercentUp = 100.0 * (float64(currMetric.Uptime) / float64(currMetric.TotalTime))
+		}
+		totalUpMinutes := currMetric.Uptime * ncutils.CheckInInterval
 		currMetric.ActualUptime = time.Duration(totalUpMinutes) * time.Minute
 		delete(oldMetrics.Connectivity, k) // remove from old data
 		newMetrics.Connectivity[k] = currMetric
 	}
 
+	// add nodes that need failover
+	nodes, err := logic.GetNetworkNodes(currentNode.Network)
+	if err != nil {
+		logger.Log(0, "failed to retrieve nodes while updating metrics")
+		return false
+	}
+	for _, node := range nodes {
+		if !newMetrics.Connectivity[node.ID].Connected &&
+			len(newMetrics.Connectivity[node.ID].NodeName) > 0 &&
+			node.Connected == "yes" &&
+			len(node.FailoverNode) > 0 &&
+			node.Failover != "yes" {
+			newMetrics.FailoverPeers[node.ID] = node.FailoverNode
+		}
+	}
+	shouldUpdate := len(oldMetrics.FailoverPeers) == 0 && len(newMetrics.FailoverPeers) > 0
+	for k, v := range oldMetrics.FailoverPeers {
+		if len(newMetrics.FailoverPeers[k]) > 0 && len(v) == 0 {
+			shouldUpdate = true
+		}
+
+		if len(v) > 0 && len(newMetrics.FailoverPeers[k]) == 0 {
+			newMetrics.FailoverPeers[k] = v
+		}
+	}
+
 	for k := range oldMetrics.Connectivity { // cleanup any left over data, self healing
 		delete(newMetrics.Connectivity, k)
 	}
+	return shouldUpdate
 }

mq/mq.go

@@ -2,13 +2,13 @@ package mq
 
 import (
 	"context"
+	"fmt"
 	"time"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
-	"github.com/gravitl/netmaker/serverctl"
 )
 
 // KEEPALIVE_TIMEOUT - time in seconds for timeout
@@ -23,21 +23,57 @@ var peer_force_send = 0
 
 var mqclient mqtt.Client
 
-// SetupMQTT creates a connection to broker and return client
-func SetupMQTT() {
+// SetUpAdminClient - sets up admin client for the MQ
+func SetUpAdminClient() {
 	opts := mqtt.NewClientOptions()
-	broker, secure := servercfg.GetMessageQueueEndpoint()
+	setMqOptions(mqAdminUserName, servercfg.GetMqAdminPassword(), opts)
+	mqAdminClient = mqtt.NewClient(opts)
+	opts.SetOnConnectHandler(func(client mqtt.Client) {
+		if token := client.Subscribe(dynamicSecSubTopic, 2, mqtt.MessageHandler(watchDynSecTopic)); token.WaitTimeout(MQ_TIMEOUT*time.Second) && token.Error() != nil {
+			client.Disconnect(240)
+			logger.Log(0, fmt.Sprintf("Dynamic security client subscription failed: %v ", token.Error()))
+		}
+
+		opts.SetOrderMatters(true)
+		opts.SetResumeSubs(true)
+	})
+	tperiod := time.Now().Add(10 * time.Second)
+	for {
+		if token := mqAdminClient.Connect(); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
+			logger.Log(2, "Admin: unable to connect to broker, retrying ...")
+			if time.Now().After(tperiod) {
+				if token.Error() == nil {
+					logger.FatalLog("Admin: could not connect to broker, token timeout, exiting ...")
+				} else {
+					logger.FatalLog("Admin: could not connect to broker, exiting ...", token.Error().Error())
+				}
+			}
+		} else {
+			break
+		}
+		time.Sleep(2 * time.Second)
+	}
+
+}
+
+func setMqOptions(user, password string, opts *mqtt.ClientOptions) {
+	broker, _ := servercfg.GetMessageQueueEndpoint()
 	opts.AddBroker(broker)
 	id := ncutils.MakeRandomString(23)
 	opts.ClientID = id
-	if secure {
-		opts.SetTLSConfig(&serverctl.TlsConfig)
-	}
+	opts.SetUsername(user)
+	opts.SetPassword(password)
 	opts.SetAutoReconnect(true)
 	opts.SetConnectRetry(true)
 	opts.SetConnectRetryInterval(time.Second << 2)
 	opts.SetKeepAlive(time.Minute)
 	opts.SetWriteTimeout(time.Minute)
+}
+
+// SetupMQTT creates a connection to broker and return client
+func SetupMQTT() {
+	opts := mqtt.NewClientOptions()
+	setMqOptions(mqNetmakerServerUserName, servercfg.GetMqAdminPassword(), opts)
 	opts.SetOnConnectHandler(func(client mqtt.Client) {
 		if token := client.Subscribe("ping/#", 2, mqtt.MessageHandler(Ping)); token.WaitTimeout(MQ_TIMEOUT*time.Second) && token.Error() != nil {
 			client.Disconnect(240)

mq/publishers.go

@@ -33,23 +33,25 @@ func PublishPeerUpdate(newNode *models.Node, publishToSelf bool) error {
 			//skip self
 			continue
 		}
-		peerUpdate, err := logic.GetPeerUpdate(&node)
+		err = PublishSinglePeerUpdate(&node)
 		if err != nil {
-			logger.Log(1, "error getting peer update for node", node.ID, err.Error())
-			continue
-		}
-		data, err := json.Marshal(&peerUpdate)
-		if err != nil {
-			logger.Log(2, "error marshaling peer update for node", node.ID, err.Error())
-			continue
-		}
-		if err = publish(&node, fmt.Sprintf("peers/%s/%s", node.Network, node.ID), data); err != nil {
-			logger.Log(1, "failed to publish peer update for node", node.ID)
-		} else {
-			logger.Log(1, "sent peer update for node", node.Name, "on network:", node.Network)
+			logger.Log(1, "failed to publish peer update to node", node.Name, "on network", node.Network, ":", err.Error())
 		}
 	}
-	return nil
+	return err
+}
+
+// PublishSinglePeerUpdate --- determines and publishes a peer update to one node
+func PublishSinglePeerUpdate(node *models.Node) error {
+	peerUpdate, err := logic.GetPeerUpdate(node)
+	if err != nil {
+		return err
+	}
+	data, err := json.Marshal(&peerUpdate)
+	if err != nil {
+		return err
+	}
+	return publish(node, fmt.Sprintf("peers/%s/%s", node.Network, node.ID), data)
 }
 
 // PublishPeerUpdate --- publishes a peer update to all the peers of a node
@@ -184,7 +186,7 @@ func ServerStartNotify() error {
 
 // function to collect and store metrics for server nodes
 func collectServerMetrics(networks []models.Network) {
-	if !logic.Is_EE {
+	if !servercfg.Is_EE {
 		return
 	}
 	if len(networks) > 0 {
@@ -217,9 +219,7 @@ func collectServerMetrics(networks []models.Network) {
 									logger.Log(2, "failed to push server metrics to exporter: ", err.Error())
 								}
 							}
-
 						}
-
 					}
 				}
 			}
@@ -233,7 +233,7 @@ func pushMetricsToExporter(metrics models.Metrics) error {
 	if err != nil {
 		return errors.New("failed to marshal metrics: " + err.Error())
 	}
-	if token := mqclient.Publish("metrics_exporter", 0, true, data); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
+	if token := mqclient.Publish("metrics_exporter", 2, true, data); !token.WaitTimeout(MQ_TIMEOUT*time.Second) || token.Error() != nil {
 		var err error
 		if token.Error() == nil {
 			err = errors.New("connection timeout")

netclient/command/commands.go

@@ -1,8 +1,6 @@
 package command
 
 import (
-	"crypto/ed25519"
-	"crypto/rand"
 	"errors"
 	"fmt"
 	"strings"
@@ -12,7 +10,6 @@ import (
 	"github.com/gravitl/netmaker/netclient/daemon"
 	"github.com/gravitl/netmaker/netclient/functions"
 	"github.com/gravitl/netmaker/netclient/ncutils"
-	"github.com/gravitl/netmaker/tls"
 )
 
 // Join - join command to run from cli
@@ -115,29 +112,8 @@ func Pull(cfg *config.ClientConfig) error {
 
 		currentServers[currCfg.Server.Server] = *currCfg
 	}
-	//generate new client key if one doesn' exist
-	var private *ed25519.PrivateKey
-	private, err = tls.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
-	if err != nil {
-		_, newKey, err := ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			return err
-		}
-		if err := tls.SaveKeyToFile(ncutils.GetNetclientPath(), ncutils.GetSeparator()+"client.key", newKey); err != nil {
-			return err
-		}
-		private = &newKey
-	}
-	// re-register with server -- get new certs for broker
-	for _, clientCfg := range currentServers {
-		if err = functions.RegisterWithServer(private, &clientCfg); err != nil {
-			logger.Log(0, "registration error", err.Error())
-		} else {
-			daemon.Restart()
-		}
-	}
+	daemon.Restart()
 	logger.Log(1, "reset network", cfg.Network, "and peer configs")
-
 	return err
 }
 

netclient/functions/common.go

@@ -192,37 +192,10 @@ func LeaveNetwork(network string) error {
 	if err := removeHostDNS(cfg.Node.Interface, ncutils.IsWindows()); err != nil {
 		logger.Log(0, "failed to delete dns entries for", cfg.Node.Interface, err.Error())
 	}
-	logger.Log(2, "deleting broker keys as required")
-	if !brokerInUse(cfg.Server.Server) {
-		if err := deleteBrokerFiles(cfg.Server.Server); err != nil {
-			logger.Log(0, "failed to deleter certs for", cfg.Server.Server, err.Error())
-		}
-	}
 	logger.Log(2, "restarting daemon")
 	return daemon.Restart()
 }
 
-func brokerInUse(broker string) bool {
-	networks, _ := ncutils.GetSystemNetworks()
-	for _, net := range networks {
-		cfg := config.ClientConfig{}
-		cfg.Network = net
-		cfg.ReadConfig()
-		if cfg.Server.Server == broker {
-			return true
-		}
-	}
-	return false
-}
-
-func deleteBrokerFiles(broker string) error {
-	dir := ncutils.GetNetclientServerPath(broker)
-	if err := os.RemoveAll(dir); err != nil {
-		return err
-	}
-	return nil
-}
-
 func deleteNodeFromServer(cfg *config.ClientConfig) error {
 	node := cfg.Node
 	if node.IsServer == "yes" {
@@ -340,6 +313,7 @@ func API(data any, method, url, authorization string) (*http.Response, error) {
 	if authorization != "" {
 		request.Header.Set("authorization", "Bearer "+authorization)
 	}
+	request.Header.Set("requestfrom", "node")
 	return HTTPClient.Do(request)
 }
 

netclient/functions/daemon.go

@@ -2,13 +2,8 @@ package functions
 
 import (
 	"context"
-	"crypto/ed25519"
-	"crypto/rand"
-	"crypto/tls"
-	"crypto/x509"
 	"errors"
 	"fmt"
-	"log"
 	"os"
 	"os/signal"
 	"strings"
@@ -21,12 +16,10 @@ import (
 	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/netclient/auth"
 	"github.com/gravitl/netmaker/netclient/config"
-	"github.com/gravitl/netmaker/netclient/daemon"
 	"github.com/gravitl/netmaker/netclient/global_settings"
 	"github.com/gravitl/netmaker/netclient/local"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/netclient/wireguard"
-	ssl "github.com/gravitl/netmaker/tls"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
@@ -73,12 +66,18 @@ func Daemon() error {
 			cancel()
 			logger.Log(0, "shutting down netclient daemon")
 			wg.Wait()
+			if mqclient != nil {
+				mqclient.Disconnect(250)
+			}
 			logger.Log(0, "shutdown complete")
 			return nil
 		case <-reset:
 			logger.Log(0, "received reset")
 			cancel()
 			wg.Wait()
+			if mqclient != nil {
+				mqclient.Disconnect(250)
+			}
 			logger.Log(0, "restarting daemon")
 			cancel = startGoRoutines(&wg)
 		}
@@ -94,11 +93,13 @@ func startGoRoutines(wg *sync.WaitGroup) context.CancelFunc {
 		cfg := config.ClientConfig{}
 		cfg.Network = network
 		cfg.ReadConfig()
-		if err := wireguard.ApplyConf(&cfg.Node, cfg.Node.Interface, ncutils.GetNetclientPathSpecific()+cfg.Node.Interface+".conf"); err != nil {
-			logger.Log(0, "failed to start ", cfg.Node.Interface, "wg interface", err.Error())
-		}
-		if cfg.PublicIPService != "" {
-			global_settings.PublicIPServices[network] = cfg.PublicIPService
+		if cfg.Node.Connected == "yes" {
+			if err := wireguard.ApplyConf(&cfg.Node, cfg.Node.Interface, ncutils.GetNetclientPathSpecific()+cfg.Node.Interface+".conf"); err != nil {
+				logger.Log(0, "failed to start ", cfg.Node.Interface, "wg interface", err.Error())
+			}
+			if cfg.PublicIPService != "" {
+				global_settings.PublicIPServices[network] = cfg.PublicIPService
+			}
 		}
 
 		server := cfg.Server.Server
@@ -201,47 +202,19 @@ func messageQueue(ctx context.Context, wg *sync.WaitGroup, cfg *config.ClientCon
 	logger.Log(0, "shutting down message queue for server", cfg.Server.Server)
 }
 
-// NewTLSConf sets up tls configuration to connect to broker securely
-func NewTLSConfig(server string) (*tls.Config, error) {
-	file := ncutils.GetNetclientServerPath(server) + ncutils.GetSeparator() + "root.pem"
-	certpool := x509.NewCertPool()
-	ca, err := os.ReadFile(file)
-	if err != nil {
-		logger.Log(0, "could not read CA file", err.Error())
-	}
-	ok := certpool.AppendCertsFromPEM(ca)
-	if !ok {
-		logger.Log(0, "failed to append cert")
-	}
-	clientKeyPair, err := tls.LoadX509KeyPair(ncutils.GetNetclientServerPath(server)+ncutils.GetSeparator()+"client.pem", ncutils.GetNetclientPath()+ncutils.GetSeparator()+"client.key")
-	if err != nil {
-		logger.Log(0, "could not read client cert/key", err.Error())
-		return nil, err
-	}
-	certs := []tls.Certificate{clientKeyPair}
-	return &tls.Config{
-		RootCAs:            certpool,
-		ClientAuth:         tls.NoClientCert,
-		ClientCAs:          nil,
-		Certificates:       certs,
-		InsecureSkipVerify: false,
-	}, nil
-
-}
-
 // func setMQTTSingenton creates a connection to broker for single use (ie to publish a message)
 // only to be called from cli (eg. connect/disconnect, join, leave) and not from daemon ---
 func setupMQTTSingleton(cfg *config.ClientConfig) error {
 	opts := mqtt.NewClientOptions()
 	server := cfg.Server.Server
 	port := cfg.Server.MQPort
-	opts.AddBroker("ssl://" + server + ":" + port)
-	tlsConfig, err := NewTLSConfig(server)
+	pass, err := os.ReadFile(ncutils.GetNetclientPathSpecific() + "secret-" + cfg.Network)
 	if err != nil {
-		logger.Log(0, "failed to get TLS config for", server, err.Error())
-		return err
+		return fmt.Errorf("could not read secrets file %w", err)
 	}
-	opts.SetTLSConfig(tlsConfig)
+	opts.AddBroker("mqtts://" + server + ":" + port)
+	opts.SetUsername(cfg.Node.ID)
+	opts.SetPassword(string(pass))
 	mqclient = mqtt.NewClient(opts)
 	var connecterr error
 	opts.SetClientID(ncutils.MakeRandomString(23))
@@ -262,13 +235,13 @@ func setupMQTT(cfg *config.ClientConfig) error {
 	opts := mqtt.NewClientOptions()
 	server := cfg.Server.Server
 	port := cfg.Server.MQPort
-	opts.AddBroker("ssl://" + server + ":" + port)
-	tlsConfig, err := NewTLSConfig(server)
+	pass, err := os.ReadFile(ncutils.GetNetclientPathSpecific() + "secret-" + cfg.Network)
 	if err != nil {
-		logger.Log(0, "failed to get TLS config for", server, err.Error())
-		return err
+		return fmt.Errorf("could not read secrets file %w", err)
 	}
-	opts.SetTLSConfig(tlsConfig)
+	opts.AddBroker(fmt.Sprintf("mqtts://%s:%s", server, port))
+	opts.SetUsername(cfg.Node.ID)
+	opts.SetPassword(string(pass))
 	opts.SetClientID(ncutils.MakeRandomString(23))
 	opts.SetDefaultPublishHandler(All)
 	opts.SetAutoReconnect(true)
@@ -311,29 +284,13 @@ func setupMQTT(cfg *config.ClientConfig) error {
 		}
 	}
 	if connecterr != nil {
-		reRegisterWithServer(cfg)
-		//try after re-registering
-		if token := mqclient.Connect(); !token.WaitTimeout(30*time.Second) || token.Error() != nil {
-			return errors.New("unable to connect to broker")
-		}
+		logger.Log(0, "failed to establish connection to broker: ", connecterr.Error())
+		return connecterr
 	}
 
 	return nil
 }
 
-func reRegisterWithServer(cfg *config.ClientConfig) {
-	logger.Log(0, "connection issue detected.. attempt connection with new certs and broker information")
-	key, err := ssl.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
-	if err != nil {
-		_, *key, err = ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			log.Fatal("could not generate new key")
-		}
-	}
-	RegisterWithServer(key, cfg)
-	daemon.Restart()
-}
-
 // publishes a message to server to update peers on this peer's behalf
 func publishSignal(nodeCfg *config.ClientConfig, signal byte) error {
 	if err := publish(nodeCfg, fmt.Sprintf("signal/%s", nodeCfg.Node.ID), []byte{signal}, 1); err != nil {

netclient/functions/join.go

@@ -199,7 +199,7 @@ func JoinNetwork(cfg *config.ClientConfig, privateKey string) error {
 		return err
 	}
 	if cfg.Node.Password == "" {
-		cfg.Node.Password = logic.GenKey()
+		cfg.Node.Password = logic.GenPassWord()
 	}
 	//check if ListenPort was set on command line
 	if cfg.Node.ListenPort != 0 {
@@ -362,10 +362,6 @@ func JoinNetwork(cfg *config.ClientConfig, privateKey string) error {
 
 	local.SetNetmakerDomainRoute(cfg.Server.API)
 	cfg.Node = node
-	if err := Register(cfg); err != nil {
-		return err
-	}
-
 	logger.Log(0, "starting wireguard")
 	err = wireguard.InitWireguard(&node, privateKey, nodeGET.Peers[:])
 	if err != nil {

netclient/functions/mqpublish.go

@@ -8,7 +8,6 @@ import (
 	"io"
 	"net"
 	"net/http"
-	"os"
 	"strconv"
 	"sync"
 	"time"
@@ -20,7 +19,6 @@ import (
 	"github.com/gravitl/netmaker/netclient/auth"
 	"github.com/gravitl/netmaker/netclient/config"
 	"github.com/gravitl/netmaker/netclient/ncutils"
-	"github.com/gravitl/netmaker/tls"
 )
 
 var metricsCache = new(sync.Map)
@@ -31,27 +29,25 @@ var metricsCache = new(sync.Map)
 func Checkin(ctx context.Context, wg *sync.WaitGroup) {
 	logger.Log(2, "starting checkin goroutine")
 	defer wg.Done()
-	currentRun := 0
-	checkin(currentRun)
-	ticker := time.NewTicker(time.Second * 60)
+	ticker := time.NewTicker(time.Minute * ncutils.CheckInInterval)
 	defer ticker.Stop()
 	for {
 		select {
 		case <-ctx.Done():
 			logger.Log(0, "checkin routine closed")
 			return
-			//delay should be configuraable -> use cfg.Node.NetworkSettings.DefaultCheckInInterval ??
 		case <-ticker.C:
-			currentRun++
-			checkin(currentRun)
-			if currentRun >= 5 {
-				currentRun = 0
+			if mqclient != nil && mqclient.IsConnected() {
+				checkin()
+			} else {
+				logger.Log(0, "MQ client is not connected, skipping checkin...")
 			}
+
 		}
 	}
 }
 
-func checkin(currentRun int) {
+func checkin() {
 	networks, _ := ncutils.GetSystemNetworks()
 	logger.Log(3, "checkin with server(s) for all networks")
 	for _, network := range networks {
@@ -69,41 +65,43 @@ func checkin(currentRun int) {
 			// defaults to iptables for now, may need another default for non-Linux OSes
 			nodeCfg.Node.FirewallInUse = models.FIREWALL_IPTABLES
 		}
-		if nodeCfg.Node.IsStatic != "yes" {
-			extIP, err := ncutils.GetPublicIP(nodeCfg.Server.API)
-			if err != nil {
-				logger.Log(1, "error encountered checking public ip addresses: ", err.Error())
-			}
-			if nodeCfg.Node.Endpoint != extIP && extIP != "" {
-				logger.Log(1, "network:", nodeCfg.Node.Network, "endpoint has changed from ", nodeCfg.Node.Endpoint, " to ", extIP)
-				nodeCfg.Node.Endpoint = extIP
-				if err := PublishNodeUpdate(&nodeCfg); err != nil {
-					logger.Log(0, "network:", nodeCfg.Node.Network, "could not publish endpoint change")
+		if nodeCfg.Node.Connected == "yes" {
+			if nodeCfg.Node.IsStatic != "yes" {
+				extIP, err := ncutils.GetPublicIP(nodeCfg.Server.API)
+				if err != nil {
+					logger.Log(1, "error encountered checking public ip addresses: ", err.Error())
 				}
-			}
-			intIP, err := getPrivateAddr()
-			if err != nil {
-				logger.Log(1, "network:", nodeCfg.Node.Network, "error encountered checking private ip addresses: ", err.Error())
-			}
-			if nodeCfg.Node.LocalAddress != intIP && intIP != "" {
-				logger.Log(1, "network:", nodeCfg.Node.Network, "local Address has changed from ", nodeCfg.Node.LocalAddress, " to ", intIP)
-				nodeCfg.Node.LocalAddress = intIP
-				if err := PublishNodeUpdate(&nodeCfg); err != nil {
-					logger.Log(0, "Network: ", nodeCfg.Node.Network, " could not publish local address change")
+				if nodeCfg.Node.Endpoint != extIP && extIP != "" {
+					logger.Log(1, "network:", nodeCfg.Node.Network, "endpoint has changed from ", nodeCfg.Node.Endpoint, " to ", extIP)
+					nodeCfg.Node.Endpoint = extIP
+					if err := PublishNodeUpdate(&nodeCfg); err != nil {
+						logger.Log(0, "network:", nodeCfg.Node.Network, "could not publish endpoint change")
+					}
 				}
-			}
-			_ = UpdateLocalListenPort(&nodeCfg)
+				intIP, err := getPrivateAddr()
+				if err != nil {
+					logger.Log(1, "network:", nodeCfg.Node.Network, "error encountered checking private ip addresses: ", err.Error())
+				}
+				if nodeCfg.Node.LocalAddress != intIP && intIP != "" {
+					logger.Log(1, "network:", nodeCfg.Node.Network, "local Address has changed from ", nodeCfg.Node.LocalAddress, " to ", intIP)
+					nodeCfg.Node.LocalAddress = intIP
+					if err := PublishNodeUpdate(&nodeCfg); err != nil {
+						logger.Log(0, "Network: ", nodeCfg.Node.Network, " could not publish local address change")
+					}
+				}
+				_ = UpdateLocalListenPort(&nodeCfg)
 
-		} else if nodeCfg.Node.IsLocal == "yes" && nodeCfg.Node.LocalRange != "" {
-			localIP, err := ncutils.GetLocalIP(nodeCfg.Node.LocalRange)
-			if err != nil {
-				logger.Log(1, "network:", nodeCfg.Node.Network, "error encountered checking local ip addresses: ", err.Error())
-			}
-			if nodeCfg.Node.Endpoint != localIP && localIP != "" {
-				logger.Log(1, "network:", nodeCfg.Node.Network, "endpoint has changed from "+nodeCfg.Node.Endpoint+" to ", localIP)
-				nodeCfg.Node.Endpoint = localIP
-				if err := PublishNodeUpdate(&nodeCfg); err != nil {
-					logger.Log(0, "network:", nodeCfg.Node.Network, "could not publish localip change")
+			} else if nodeCfg.Node.IsLocal == "yes" && nodeCfg.Node.LocalRange != "" {
+				localIP, err := ncutils.GetLocalIP(nodeCfg.Node.LocalRange)
+				if err != nil {
+					logger.Log(1, "network:", nodeCfg.Node.Network, "error encountered checking local ip addresses: ", err.Error())
+				}
+				if nodeCfg.Node.Endpoint != localIP && localIP != "" {
+					logger.Log(1, "network:", nodeCfg.Node.Network, "endpoint has changed from "+nodeCfg.Node.Endpoint+" to ", localIP)
+					nodeCfg.Node.Endpoint = localIP
+					if err := PublishNodeUpdate(&nodeCfg); err != nil {
+						logger.Log(0, "network:", nodeCfg.Node.Network, "could not publish localip change")
+					}
 				}
 			}
 		}
@@ -113,8 +111,7 @@ func checkin(currentRun int) {
 			config.Write(&nodeCfg, nodeCfg.Network)
 		}
 		Hello(&nodeCfg)
-		checkCertExpiry(&nodeCfg)
-		if currentRun >= 5 {
+		if nodeCfg.Server.Is_EE && nodeCfg.Node.Connected == "yes" {
 			logger.Log(0, "collecting metrics for node", nodeCfg.Node.Name)
 			publishMetrics(&nodeCfg)
 		}
@@ -266,22 +263,6 @@ func publish(nodeCfg *config.ClientConfig, dest string, msg []byte, qos byte) er
 	return nil
 }
 
-func checkCertExpiry(cfg *config.ClientConfig) error {
-	cert, err := tls.ReadCertFromFile(ncutils.GetNetclientServerPath(cfg.Server.Server) + ncutils.GetSeparator() + "client.pem")
-	//if cert doesn't exist or will expire within 10 days
-	if errors.Is(err, os.ErrNotExist) || cert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
-		key, err := tls.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
-		if err != nil {
-			return err
-		}
-		return RegisterWithServer(key, cfg)
-	}
-	if err != nil {
-		return err
-	}
-	return nil
-}
-
 func checkBroker(broker string, port string) error {
 	if broker == "" {
 		return errors.New("error: broker address is blank")

netclient/functions/register.go

@@ -1,100 +0,0 @@
-package functions
-
-import (
-	"crypto/ed25519"
-	"crypto/rand"
-	"encoding/json"
-	"errors"
-	"net/http"
-	"os"
-
-	"github.com/gravitl/netmaker/logger"
-	"github.com/gravitl/netmaker/netclient/config"
-	"github.com/gravitl/netmaker/netclient/ncutils"
-	"github.com/gravitl/netmaker/tls"
-)
-
-// Register - the function responsible for registering with the server and acquiring certs
-func Register(cfg *config.ClientConfig) error {
-
-	//generate new key if one doesn' exist
-	var private *ed25519.PrivateKey
-	var err error
-	private, err = tls.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
-	if err != nil {
-		_, newKey, err := ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			return err
-		}
-		if err := tls.SaveKeyToFile(ncutils.GetNetclientPath(), ncutils.GetSeparator()+"client.key", newKey); err != nil {
-			return err
-		}
-		private = &newKey
-	}
-	//check if cert exists
-	_, err = tls.ReadCertFromFile(ncutils.GetNetclientServerPath(cfg.Server.Server) + ncutils.GetSeparator() + "client.pem")
-	if errors.Is(err, os.ErrNotExist) {
-		if err := RegisterWithServer(private, cfg); err != nil {
-			return err
-		}
-	} else if err != nil {
-		return err
-	}
-	return nil
-}
-
-// RegisterWithServer calls the register endpoint with privatekey and commonname - api returns ca and client certificate
-func RegisterWithServer(private *ed25519.PrivateKey, cfg *config.ClientConfig) error {
-	data := config.RegisterRequest{
-		Key:        *private,
-		CommonName: tls.NewCName(cfg.Node.Name),
-	}
-	url := "https://" + cfg.Server.API + "/api/server/register"
-	logger.Log(1, "register at "+url)
-
-	token, err := Authenticate(cfg)
-	if err != nil {
-		return err
-	}
-	response, err := API(data, http.MethodPost, url, token)
-	if err != nil {
-		return err
-	}
-	if response.StatusCode != http.StatusOK {
-		return errors.New(response.Status)
-	}
-	var resp config.RegisterResponse
-	if err := json.NewDecoder(response.Body).Decode(&resp); err != nil {
-		return errors.New("unmarshal cert error " + err.Error())
-	}
-
-	// set broker information on register
-	var modServer bool
-	if resp.Broker != "" && resp.Broker != cfg.Server.Server {
-		cfg.Server.Server = resp.Broker
-		modServer = true
-	}
-	if resp.Port != "" && resp.Port != cfg.Server.MQPort {
-		cfg.Server.MQPort = resp.Port
-		modServer = true
-	}
-	if modServer {
-		if err = config.ModServerConfig(&cfg.Server, cfg.Node.Network); err != nil {
-			logger.Log(0, "network:", cfg.Node.Network, "error overwriting config with broker information: "+err.Error())
-		}
-	}
-
-	//x509.Certificate.PublicKey is an interface so json encoding/decoding results in a string rather that []byte
-	//the pubkeys are included in the response so the values in the certificate can be updated appropriately
-	resp.CA.PublicKey = resp.CAPubKey
-	resp.Cert.PublicKey = resp.CertPubKey
-	if err := tls.SaveCertToFile(ncutils.GetNetclientServerPath(cfg.Server.Server)+ncutils.GetSeparator(), tls.ROOT_PEM_NAME, &resp.CA); err != nil {
-		return err
-	}
-	if err := tls.SaveCertToFile(ncutils.GetNetclientServerPath(cfg.Server.Server)+ncutils.GetSeparator(), "client.pem", &resp.Cert); err != nil {
-		return err
-	}
-	logger.Log(0, "network:", cfg.Network, "certificates/key saved ")
-	//join the network defined in the token
-	return nil
-}

netclient/functions/upgrades/upgrades.go

@@ -6,6 +6,7 @@ func InitializeUpgrades() {
 		upgrade0145,
 		upgrade0146,
 		upgrade0160,
+		upgrade0161,
 	})
 }
 

netclient/functions/upgrades/v0-16-1.go

@@ -0,0 +1,24 @@
+package upgrades
+
+import (
+	"github.com/gravitl/netmaker/netclient/config"
+)
+
+var upgrade0161 = UpgradeInfo{
+	RequiredVersions: []string{
+		"v0.14.6",
+		"v0.15.0",
+		"v0.15.1",
+		"v0.15.2",
+		"v0.16.1",
+	},
+	NewVersion: "v0.16.1",
+	OP:         update0161,
+}
+
+func update0161(cfg *config.ClientConfig) {
+	// set connect default if not present 15.X -> 16.0
+	if cfg.Node.Connected == "" {
+		cfg.Node.SetDefaultConnected()
+	}
+}

netclient/ncutils/util.go

@@ -7,6 +7,9 @@ import (
 	"github.com/gravitl/netmaker/logger"
 )
 
+// CheckInInterval - the interval for check-in time in units/minute
+const CheckInInterval = 1
+
 // BackOff - back off any function while there is an error
 func BackOff(isExponential bool, maxTime int, f interface{}) (interface{}, error) {
 	// maxTime seconds

netclient/netclient.exe.manifest.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
     <assemblyIdentity
-            version="0.16.0.0"
+            version="0.16.1.0"
             processorArchitecture="*"
             name="netclient.exe"
             type="win32"

netclient/versioninfo.json

@@ -3,13 +3,13 @@
         "FileVersion": {
             "Major": 0,
             "Minor": 16,
-            "Patch": 0,
+            "Patch": 1,
             "Build": 0
         },
         "ProductVersion": {
             "Major": 0,
             "Minor": 16,
-            "Patch": 0,
+            "Patch": 1,
             "Build": 0
         },
         "FileFlagsMask": "3f",
@@ -29,7 +29,7 @@
         "OriginalFilename": "",
         "PrivateBuild": "",
         "ProductName": "Netclient",
-        "ProductVersion": "v0.16.0.0",
+        "ProductVersion": "v0.16.1.0",
         "SpecialBuild": ""
     },
     "VarFileInfo": {

netclient/wireguard/mac.go

@@ -26,7 +26,7 @@ func WgQuickDownMac(node *models.Node, iface string) error {
 
 // RemoveConfMac - bring down mac interface and remove routes
 func RemoveConfMac(iface string) error {
-	realIface, err := getRealIface(iface)
+	realIface, err := GetRealIface(iface)
 	if realIface != "" {
 		err = deleteInterface(iface, realIface)
 	}
@@ -37,7 +37,7 @@ func RemoveConfMac(iface string) error {
 func WgQuickUpMac(node *models.Node, iface string, confPath string) error {
 	var err error
 	var realIface string
-	realIface, err = getRealIface(iface)
+	realIface, err = GetRealIface(iface)
 	if realIface != "" && err == nil {
 		deleteInterface(iface, realIface)
 		deleteRoutes(realIface)
@@ -101,8 +101,8 @@ func addInterface(iface string) (string, error) {
 	return realIface, err
 }
 
-// getRealIface - retrieves tun iface based on reference iface name from config file
-func getRealIface(iface string) (string, error) {
+// GetRealIface - retrieves tun iface based on reference iface name from config file
+func GetRealIface(iface string) (string, error) {
 	ncutils.RunCmd("wg show interfaces", false)
 	ifacePath := "/var/run/wireguard/" + iface + ".name"
 	if !(ncutils.FileExists(ifacePath)) {
@@ -120,7 +120,7 @@ func getRealIface(iface string) (string, error) {
 
 // deleteRoutes - deletes network routes associated with interface
 func deleteRoutes(iface string) error {
-	realIface, err := getRealIface(iface)
+	realIface, err := GetRealIface(iface)
 	if err != nil {
 		return err
 	}

scripts/netclient.sh

@@ -23,7 +23,7 @@ if [ "$TOKEN" != "" ]; then
     TOKEN_CMD="-t $TOKEN"
 fi
 
-/root/netclient join $TOKEN_CMD -dnson no -udpholepunch no
+/root/netclient join $TOKEN_CMD -udpholepunch no
 if [ $? -ne 0 ]; then { echo "Failed to join, quitting." ; exit 1; } fi
 
 echo "[netclient] Starting netclient daemon"

scripts/nm-quick.sh

@@ -80,7 +80,7 @@ COREDNS_IP=$(ip route get 1 | sed -n 's/^.*src \([0-9.]*\) .*$/\1/p')
 SERVER_PUBLIC_IP=$(curl -s ifconfig.me)
 MASTER_KEY=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 30 ; echo '')
 EMAIL="$(echo $RANDOM | md5sum  | head -c 32)@email.com"
-
+MQ_ADMIN_PASSWORD=$(tr -dc A-Za-z0-9 </dev/urandom | head -c 64 ; echo '')
 if [ -n "$domain" ]; then
   NETMAKER_BASE_DOMAIN=$domain
 fi
@@ -128,7 +128,8 @@ sleep 5
 echo "setting mosquitto.conf..."
 
 wget -q -O /root/mosquitto.conf https://raw.githubusercontent.com/gravitl/netmaker/master/docker/mosquitto.conf
-
+wget -q -O /root/wait.sh https://raw.githubusercontent.com/gravitl/netmaker/master/docker/wait.sh
+chmod +x /root/wait.sh
 echo "setting docker-compose..."
 
 mkdir -p /etc/netmaker
@@ -139,7 +140,7 @@ sed -i "s/SERVER_PUBLIC_IP/$SERVER_PUBLIC_IP/g" /root/docker-compose.yml
 sed -i "s/COREDNS_IP/$COREDNS_IP/g" /root/docker-compose.yml
 sed -i "s/REPLACE_MASTER_KEY/$MASTER_KEY/g" /root/docker-compose.yml
 sed -i "s/YOUR_EMAIL/$EMAIL/g" /root/docker-compose.yml
-
+sed -i "s/REPLACE_MQ_ADMIN_PASSWORD/$MQ_ADMIN_PASSWORD/g" /root/docker-compose.yml
 echo "starting containers..."
 
 docker-compose -f /root/docker-compose.yml up -d

servercfg/serverconf.go

@@ -15,6 +15,7 @@ import (
 
 var (
 	Version = "dev"
+	Is_EE   = false
 )
 
 // SetHost - sets the host ip
@@ -84,6 +85,10 @@ func GetServerConfig() config.ServerConfig {
 	cfg.PortForwardServices = services
 	cfg.Server = GetServer()
 	cfg.Verbosity = GetVerbosity()
+	cfg.IsEE = "no"
+	if Is_EE {
+		cfg.IsEE = "yes"
+	}
 
 	return cfg
 }
@@ -101,6 +106,7 @@ func GetServerInfo() models.ServerConfig {
 	}
 	cfg.Version = GetVersion()
 	cfg.Server = GetServer()
+	cfg.Is_EE = Is_EE
 
 	return cfg
 }
@@ -616,6 +622,17 @@ func GetMQServerPort() string {
 	return port
 }
 
+// GetMqAdminPassword - fetches the MQ Admin password
+func GetMqAdminPassword() string {
+	password := ""
+	if os.Getenv("MQ_ADMIN_PASSWORD") != "" {
+		password = os.Getenv("MQ_ADMIN_PASSWORD")
+	} else if config.Config.Server.MQAdminPassword != "" {
+		password = config.Config.Server.MQAdminPassword
+	}
+	return password
+}
+
 // IsBasicAuthEnabled - checks if basic auth has been configured to be turned off
 func IsBasicAuthEnabled() bool {
 	var enabled = true //default

serverctl/serverctl.go

@@ -132,9 +132,32 @@ func setNetworkDefaults() error {
 			logger.Log(0, "could not initialize NetworkUsers on network", net.NetID)
 		}
 		pro.AddProNetDefaults(&net)
-		_, _, _, _, _, _, err = logic.UpdateNetwork(&net, &net)
-		if err != nil {
-			logger.Log(0, "could not set defaults on network", net.NetID)
+		update := false
+		newNet := net
+		if strings.Contains(net.NetID, ".") {
+			newNet.NetID = strings.ReplaceAll(net.NetID, ".", "")
+			newNet.DefaultInterface = strings.ReplaceAll(net.DefaultInterface, ".", "")
+			update = true
+		}
+		if strings.ContainsAny(net.NetID, "ABCDEFGHIJKLMNOPQRSTUVWXYZ") {
+			newNet.NetID = strings.ToLower(net.NetID)
+			newNet.DefaultInterface = strings.ToLower(net.DefaultInterface)
+			update = true
+		}
+		if update {
+			newNet.SetDefaults()
+			if err := logic.SaveNetwork(&newNet); err != nil {
+				logger.Log(0, "error saving networks during initial update:", err.Error())
+			}
+			if err := logic.DeleteNetwork(net.NetID); err != nil {
+				logger.Log(0, "error deleting old network:", err.Error())
+			}
+		} else {
+			net.SetDefaults()
+			_, _, _, _, _, _, err = logic.UpdateNetwork(&net, &net)
+			if err != nil {
+				logger.Log(0, "could not set defaults on network", net.NetID)
+			}
 		}
 	}
 	return nil

swagger.yaml

@@ -746,7 +746,7 @@ info:
 
         API calls must be authenticated via a header of the format -H “Authorization: Bearer <YOUR_SECRET_KEY>” There are two methods to obtain YOUR_SECRET_KEY: 1. Using the masterkey. By default, this value is “secret key,” but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [Netmaker](https://docs.netmaker.org/index.html) documentation for more details. 2. Using a JWT received for a node. This can be retrieved by calling the /api/nodes/<network>/authenticate endpoint, as documented below.
     title: Netmaker
-    version: 0.16.0
+    version: 0.16.1
 paths:
     /api/dns:
         get:

validation/validation.go

@@ -3,7 +3,7 @@ package validation
 import (
 	"regexp"
 
-	"github.com/go-playground/validator/v10"
+	validator "github.com/go-playground/validator/v10"
 )
 
 // CheckYesOrNo - checks if a field on a struct is yes or no