Merge pull request #1363 from gravitl/release_v0.14.5

Release v0.14.5

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -31,6 +31,7 @@ body:
       label: Version
       description: What version are you running?
       options:
+        - v0.14.5
         - v0.14.4
         - v0.14.3      
         - v0.14.2

.github/workflows/pr-approval.yml

@@ -0,0 +1,12 @@
+name: Manual PR Testing
+on: [pull_request]
+
+jobs:  
+  test:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: trstringer/manual-approval@v1
+        with:
+          secret: ${{ secrets.DEVOPS_API_TOKEN }}
+          approvers: theguy951357,afeiszli,mattkasun,0xdcarns
+          minimum-approvals: 1

.github/workflows/test.yml

@@ -67,7 +67,7 @@ jobs:
   tests:
     env:
       DATABASE: sqlite
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
     steps:
       - name: Checkout
         uses: actions/checkout@v2
@@ -77,8 +77,10 @@ jobs:
           go-version: 1.18
       - name: run tests
         run: |
-            sudo apt-get install -y gcc libgl1-mesa-dev xorg-dev
+            sudo apt-get install -y gcc libgl1-mesa-dev xorg-dev 
             go test -p 1 ./... -v
+            go install honnef.co/go/tools/cmd/staticcheck@latest
+            ~/go/bin/staticcheck ./...
         env:
           DATABASE: sqlite
           CLIENT_MODE: "off"

README.md

@@ -17,7 +17,7 @@
 
 <p align="center">
   <a href="https://github.com/gravitl/netmaker/releases">
-    <img src="https://img.shields.io/badge/Version-0.14.4-informational?style=flat-square" />
+    <img src="https://img.shields.io/badge/Version-0.14.5-informational?style=flat-square" />
   </a>
   <a href="https://hub.docker.com/r/gravitl/netmaker/tags">
     <img src="https://img.shields.io/docker/pulls/gravitl/netmaker?label=downloads" />
@@ -41,51 +41,35 @@
 
 # WireGuard<sup>®</sup> automation from homelab to enterprise
 
-| Create & Automate                         | Manage                                  |
-|-------------------------------------------|-----------------------------------------|
-| :heavy_check_mark: WireGuard Networks     | :heavy_check_mark: Admin UI             |
-| :heavy_check_mark: Remote Access Gateways | :heavy_check_mark: OAuth                |
-| :heavy_check_mark: Mesh VPNs              | :heavy_check_mark: Private DNS          |
-| :heavy_check_mark: Site-to-Site           | :heavy_check_mark: Access Control Lists |
+| Create                                    | Manage                                  | Automate                                |
+|-------------------------------------------|-----------------------------------------|-----------------------------------------|
+| :heavy_check_mark: WireGuard Networks     | :heavy_check_mark: Admin UI             | :heavy_check_mark: Linux                |
+| :heavy_check_mark: Remote Access Gateways | :heavy_check_mark: OAuth                | :heavy_check_mark: FreeBSD              |
+| :heavy_check_mark: Mesh VPNs              | :heavy_check_mark: Private DNS          | :heavy_check_mark: Mac                  |
+| :heavy_check_mark: Site-to-Site           | :heavy_check_mark: Access Control Lists | :heavy_check_mark: Windows              |
 
 # Get Started in 5 Minutes  
 
-**For DigitalOcean, use the 1-Click App:** <a href="https://marketplace.digitalocean.com/apps/netmaker?refcode=496ffcf1e252"><img src="https://www.deploytodo.com/do-btn-blue.svg" width="15%" /></a>  
-**For production-grade installations, visit the [Install Docs](https://netmaker.readthedocs.io/en/master/install.html).**  
-**For an HA install using helm on k8s, visit the [Helm Repo](https://github.com/gravitl/netmaker-helm/).**
-1. Get a cloud VM with Ubuntu 20.04 and a public IP.
-2. Open ports 443, 53, and 51821-51830/udp on the VM firewall and in cloud security settings.
-3. Run the script **(see below for optional configurations)**:
+(For production-grade installations, visit the [Install Docs](https://netmaker.readthedocs.io/en/master/install.html).)  
 
-`wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash`
+1. Get a cloud VM with Ubuntu 20.04 and a public IP.
+2. Open ports 443 and 51821-51830/udp on the VM firewall and in cloud security settings.
+3. Run the script: `wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash`  
+3.a. (with custom domain + email): `wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash -s -- -d mynetmaker.domain.com -e [email protected]`    
 
 <p float="left" align="middle">
 <img src="./img/readme.gif" />
 </p>
 
-Upon completion, the logs will display the instructions to connect various devices. These can also be retrieved from the UI under "Access Keys."
-
-After installing Netmaker, check out the [Walkthrough](https://itnext.io/getting-started-with-netmaker-a-wireguard-virtual-networking-platform-3d563fbd87f0) and [Getting Started](https://netmaker.readthedocs.io/en/master/getting-started.html) guides to learn more about configuring networks. Or, check out some of our other [Tutorials](https://gravitl.com/resources) for different use cases, including Kubernetes.
+After installing Netmaker, check out the [Walkthrough](https://itnext.io/getting-started-with-netmaker-a-wireguard-virtual-networking-platform-3d563fbd87f0) and [Getting Started](https://netmaker.readthedocs.io/en/master/getting-started.html) guides to learn more about configuring networks. Or, check out some of our other [Tutorials](https://www.netmaker.io/blog) for different use cases, including Kubernetes.
 
-### Optional configurations
+# Get Support
 
-**Deploy a "Hub-And-Spoke VPN" on the server**  
-*This will configure a standard VPN (non-meshed) for private internet access, with 10 clients (-c).*  
-`wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash -s -- -v true -c 10`  
+- [Discord](https://discord.gg/zRb9Vfhk8A)
 
-**Specify Domain and Email**  
-*Make sure your wildcard domain is pointing towards the server ip.*  
-`wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash -s -- -d mynetmaker.domain.com -e [email protected]`  
+- [Reddit](https://reddit.com/r/netmaker)
 
-**Script Options**  
-```
-./nm-quick
--d domain.example.com # specify a wildcard domain for netmaker to use (DNS must point to this server)
--e [email protected] # specify your email (for SSL certificates)
--m true # create a default 'mesh network' (on by default)
--v false # create a default 'VPN network' (off by default)
--c 7 # number of client configs to create (for VPN network, 5 by default)
-```
+- [Learning Resources](https://netmaker.io/blog)
 
 # Why Netmaker + WireGuard?
 
@@ -97,17 +81,6 @@ After installing Netmaker, check out the [Walkthrough](https://itnext.io/getting
 
 - Netmaker with WireGuard can be highly customized for peer-to-peer, site-to-site, Kubernetes, and more.
 
-# Get Support
-
-- [Community (Discord)](https://discord.gg/zRb9Vfhk8A)
-
-- [Community (Reddit)](https://reddit.com/r/netmaker)
-
-
-- [Business (Subscription)](https://gravitl.com/plans/business)
-
-- [Learning Resources](https://gravitl.com/resources)
-
 # Community Projects
 
 - [Netmaker + Traefik Proxy](https://github.com/bsherman/netmaker-traefik)

auth/auth.go

@@ -24,6 +24,7 @@ const (
 	google_provider_name   = "google"
 	azure_ad_provider_name = "azure-ad"
 	github_provider_name   = "github"
+	oidc_provider_name     = "oidc"
 	verify_user            = "verifyuser"
 	auth_key               = "netmaker_auth"
 )
@@ -41,6 +42,8 @@ func getCurrentAuthFunctions() map[string]interface{} {
 		return azure_ad_functions
 	case github_provider_name:
 		return github_functions
+	case oidc_provider_name:
+		return oidc_functions
 	default:
 		return nil
 	}
@@ -71,6 +74,11 @@ func InitializeAuthProvider() string {
 		logger.Log(1, "external OAuth detected, proceeding with https redirect: ("+serverConn+")")
 	}
 
+	if authInfo[0] == "oidc" {
+		functions[init_provider].(func(string, string, string, string))(serverConn+"/api/oauth/callback", authInfo[1], authInfo[2], authInfo[3])
+		return authInfo[0]
+	}
+
 	functions[init_provider].(func(string, string, string))(serverConn+"/api/oauth/callback", authInfo[1], authInfo[2])
 	return authInfo[0]
 }

auth/oidc.go

@@ -0,0 +1,142 @@
+package auth
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"time"
+
+	"github.com/coreos/go-oidc/v3/oidc"
+	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/logic"
+	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
+	"golang.org/x/oauth2"
+)
+
+const OIDC_TIMEOUT = 10 * time.Second
+
+var oidc_functions = map[string]interface{}{
+	init_provider:   initOIDC,
+	get_user_info:   getOIDCUserInfo,
+	handle_callback: handleOIDCCallback,
+	handle_login:    handleOIDCLogin,
+	verify_user:     verifyOIDCUser,
+}
+
+var oidc_verifier *oidc.IDTokenVerifier
+
+type OIDCUser struct {
+	Name  string `json:"name" bson:"name"`
+	Email string `json:"email" bson:"email"`
+}
+
+// == handle OIDC authentication here ==
+
+func initOIDC(redirectURL string, clientID string, clientSecret string, issuer string) {
+	ctx, cancel := context.WithTimeout(context.Background(), OIDC_TIMEOUT)
+	defer cancel()
+
+	provider, err := oidc.NewProvider(ctx, issuer)
+	if err != nil {
+		logger.Log(1, "error when initializing OIDC provider with issuer \""+issuer+"\"", err.Error())
+		return
+	}
+
+	oidc_verifier = provider.Verifier(&oidc.Config{ClientID: clientID})
+	auth_provider = &oauth2.Config{
+		ClientID:     clientID,
+		ClientSecret: clientSecret,
+		RedirectURL:  redirectURL,
+		Endpoint:     provider.Endpoint(),
+		Scopes:       []string{oidc.ScopeOpenID, "profile", "email"},
+	}
+}
+
+func handleOIDCLogin(w http.ResponseWriter, r *http.Request) {
+	oauth_state_string = logic.RandomString(16)
+	if auth_provider == nil && servercfg.GetFrontendURL() != "" {
+		http.Redirect(w, r, servercfg.GetFrontendURL()+"/login?oauth=callback-error", http.StatusTemporaryRedirect)
+		return
+	} else if auth_provider == nil {
+		fmt.Fprintf(w, "%s", []byte("no frontend URL was provided and an OAuth login was attempted\nplease reconfigure server to use OAuth or use basic credentials"))
+		return
+	}
+	var url = auth_provider.AuthCodeURL(oauth_state_string)
+	http.Redirect(w, r, url, http.StatusTemporaryRedirect)
+}
+
+func handleOIDCCallback(w http.ResponseWriter, r *http.Request) {
+
+	var content, err = getOIDCUserInfo(r.FormValue("state"), r.FormValue("code"))
+	if err != nil {
+		logger.Log(1, "error when getting user info from callback:", err.Error())
+		http.Redirect(w, r, servercfg.GetFrontendURL()+"/login?oauth=callback-error", http.StatusTemporaryRedirect)
+		return
+	}
+	_, err = logic.GetUser(content.Email)
+	if err != nil { // user must not exists, so try to make one
+		if err = addUser(content.Email); err != nil {
+			return
+		}
+	}
+	var newPass, fetchErr = fetchPassValue("")
+	if fetchErr != nil {
+		return
+	}
+	// send a netmaker jwt token
+	var authRequest = models.UserAuthParams{
+		UserName: content.Email,
+		Password: newPass,
+	}
+
+	var jwt, jwtErr = logic.VerifyAuthRequest(authRequest)
+	if jwtErr != nil {
+		logger.Log(1, "could not parse jwt for user", authRequest.UserName)
+		return
+	}
+
+	logger.Log(1, "completed OIDC OAuth signin in for", content.Email)
+	http.Redirect(w, r, servercfg.GetFrontendURL()+"/login?login="+jwt+"&user="+content.Email, http.StatusPermanentRedirect)
+}
+
+func getOIDCUserInfo(state string, code string) (u *OIDCUser, e error) {
+	if state != oauth_state_string {
+		return nil, fmt.Errorf("invalid OAuth state")
+	}
+
+	defer func() {
+		if p := recover(); p != nil {
+			e = fmt.Errorf("getOIDCUserInfo panic: %v", p)
+		}
+	}()
+
+	ctx, cancel := context.WithTimeout(context.Background(), OIDC_TIMEOUT)
+	defer cancel()
+
+	oauth2Token, err := auth_provider.Exchange(ctx, code)
+	if err != nil {
+		return nil, fmt.Errorf("failed to exchange oauth2 token using code \"%s\"", code)
+	}
+
+	rawIDToken, ok := oauth2Token.Extra("id_token").(string)
+	if !ok {
+		return nil, fmt.Errorf("failed to get raw id_token from oauth2 token")
+	}
+
+	idToken, err := oidc_verifier.Verify(ctx, rawIDToken)
+	if err != nil {
+		return nil, fmt.Errorf("failed to verify raw id_token: \"%s\"", err.Error())
+	}
+
+	u = &OIDCUser{}
+	if err := idToken.Claims(u); err != nil {
+		e = fmt.Errorf("error when claiming OIDCUser: \"%s\"", err.Error())
+	}
+
+	return
+}
+
+func verifyOIDCUser(token *oauth2.Token) bool {
+	return token.Valid()
+}

compose/docker-compose.contained.yml

@@ -1,92 +0,0 @@
-version: "3.4"
-
-services:
-  netmaker:
-    container_name: netmaker
-    image: gravitl/netmaker:v0.14.4
-    volumes:
-      - dnsconfig:/root/config/dnsconfig
-      - sqldata:/root/data
-      - /root/certs:/etc/netmaker/
-    cap_add: 
-      - NET_ADMIN
-      - NET_RAW
-      - SYS_MODULE
-    sysctls:
-      - net.ipv4.ip_forward=1
-      - net.ipv4.conf.all.src_valid_mark=1
-      - net.ipv6.conf.all.disable_ipv6=0
-      - net.ipv6.conf.all.forwarding=1
-    restart: always
-    environment:
-      SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
-      SERVER_HOST: "SERVER_PUBLIC_IP"
-      SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
-      COREDNS_ADDR: "SERVER_PUBLIC_IP"
-      DNS_MODE: "on"
-      SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
-      API_PORT: "8081"
-      CLIENT_MODE: "on"
-      MASTER_KEY: "REPLACE_MASTER_KEY"
-      CORS_ALLOWED_ORIGIN: "*"
-      DISPLAY_KEYS: "on"
-      DATABASE: "sqlite"
-      NODE_ID: "netmaker-server-1"
-      MQ_HOST: "mq"
-      HOST_NETWORK: "off"
-      VERBOSITY: "1"
-      MANAGE_IPTABLES: "on"
-      PORT_FORWARD_SERVICES: "dns"
-    ports:
-      - "51821-51830:51821-51830/udp"
-  netmaker-ui:
-    container_name: netmaker-ui
-    depends_on:
-      - netmaker
-    image: gravitl/netmaker-ui:v0.14.4
-    links:
-      - "netmaker:api"
-    environment:
-      BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN"
-    restart: always
-  coredns:
-    depends_on:
-      - netmaker 
-    image: coredns/coredns
-    command: -conf /root/dnsconfig/Corefile
-    container_name: coredns
-    restart: always
-    volumes:
-      - dnsconfig:/root/dnsconfig
-  caddy:
-    image: caddy:latest
-    container_name: caddy
-    restart: unless-stopped
-    ports:
-      - "80:80"
-      - "443:443"
-    volumes:
-      - /root/Caddyfile:/etc/caddy/Caddyfile
-      # - $PWD/site:/srv # you could also serve a static site in site folder
-      - caddy_data:/data
-      - caddy_conf:/config
-  mq:
-    image: eclipse-mosquitto:2.0.11-openssl
-    depends_on:
-      - netmaker
-    container_name: mq
-    restart: unless-stopped
-    ports:
-      - "8883:8883"
-    volumes:
-      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
-      - /root/certs/:/mosquitto/certs/
-      - mosquitto_data:/mosquitto/data
-      - mosquitto_logs:/mosquitto/log
-volumes:
-  caddy_data: {}
-  caddy_conf: {}
-  sqldata: {}
-  dnsconfig: {}
-  mosquitto_data: {}
-  mosquitto_logs: {}

compose/docker-compose.coredns.yml

@@ -1,12 +0,0 @@
-version: "3.4"
-
-services:
-  coredns:
-    image: coredns/coredns
-    command: -conf /root/dnsconfig/Corefile
-    container_name: coredns
-    restart: always
-    ports:
-      - "53:53/udp"
-    volumes:
-      - /root/netmaker/config/dnsconfig:/root/dnsconfig

compose/docker-compose.hostnetwork.yml

@@ -1,89 +0,0 @@
-version: "3.4"
-
-services:
-  netmaker:
-    container_name: netmaker
-    image: gravitl/netmaker:v0.14.4
-    volumes:
-      - dnsconfig:/root/config/dnsconfig
-      - /usr/bin/wg:/usr/bin/wg
-      - sqldata:/root/data
-      - /run/xtables.lock:/run/xtables.lock
-      - /root/certs:/etc/netmaker/
-    cap_add:
-      - NET_ADMIN
-      - NET_RAW
-      - SYS_MODULE
-    network_mode: host
-    restart: always
-    environment:
-      SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
-      SERVER_HOST: "SERVER_PUBLIC_IP"
-      SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
-      COREDNS_ADDR: "SERVER_PUBLIC_IP"
-      DNS_MODE: "on"
-      SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
-      API_PORT: "8081"
-      CLIENT_MODE: "on"
-      MASTER_KEY: "REPLACE_MASTER_KEY"
-      CORS_ALLOWED_ORIGIN: "*"
-      DISPLAY_KEYS: "on"
-      DATABASE: "sqlite"
-      HOST_NETWORK: "on"
-      NODE_ID: "netmaker-server-1"
-      VERBOSITY: "1"
-      MANAGE_IPTABLES: "on"
-      PORT_FORWARD_SERVICES: "dns"
-  netmaker-ui:
-    container_name: netmaker-ui
-    depends_on:
-      - netmaker
-    image: gravitl/netmaker-ui:v0.14.4
-    links:
-      - "netmaker:api"
-    ports:
-      - "8082:80"
-    environment:
-      BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN"
-    restart: always
-  coredns:
-    depends_on:
-      - netmaker 
-    image: coredns/coredns
-    command: -conf /root/dnsconfig/Corefile
-    container_name: coredns
-    restart: always
-    ports:
-      - "53053:53/udp"
-      - "53053:53/tcp"
-    volumes:
-      - dnsconfig:/root/dnsconfig
-  caddy:
-    image: caddy:latest
-    container_name: caddy
-    restart: unless-stopped
-    network_mode: host # Wants ports 80 and 443!
-    volumes:
-      - /root/Caddyfile:/etc/caddy/Caddyfile
-      # - $PWD/site:/srv # you could also serve a static site in site folder
-      - caddy_data:/data
-      - caddy_conf:/config
-  mq:
-    image: eclipse-mosquitto:2.0.11-openssl
-    container_name: mq
-    restart: unless-stopped
-    ports:
-      - "127.0.0.1:1883:1883"
-      - "8883:8883"    
-    volumes:
-      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
-      - /root/certs/:/mosquitto/certs/
-      - mosquitto_data:/mosquitto/data
-      - mosquitto_logs:/mosquitto/log
-volumes:
-  caddy_data: {}
-  caddy_conf: {}
-  sqldata: {}
-  dnsconfig: {}
-  mosquitto_data: {}
-  mosquitto_logs: {}

compose/docker-compose.nocaddy.yml

@@ -1,83 +0,0 @@
-version: "3.4"
-
-services:
-  netmaker:
-    container_name: netmaker
-    image: gravitl/netmaker:v0.14.4
-    volumes:
-      - dnsconfig:/root/config/dnsconfig
-      - sqldata:/root/data
-      - /root/certs:/etc/netmaker/
-    cap_add: 
-      - NET_ADMIN
-      - NET_RAW
-      - SYS_MODULE
-    sysctls:
-      - net.ipv4.ip_forward=1
-      - net.ipv4.conf.all.src_valid_mark=1
-      - net.ipv6.conf.all.disable_ipv6=0
-      - net.ipv6.conf.all.forwarding=1
-    restart: always
-    environment:
-      SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
-      SERVER_HOST: "SERVER_PUBLIC_IP"
-      SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
-      COREDNS_ADDR: "SERVER_PUBLIC_IP"
-      DNS_MODE: "on"
-      SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
-      API_PORT: "8081"
-      CLIENT_MODE: "on"
-      MASTER_KEY: "REPLACE_MASTER_KEY"
-      CORS_ALLOWED_ORIGIN: "*"
-      DISPLAY_KEYS: "on"
-      DATABASE: "sqlite"
-      NODE_ID: "netmaker-server-1"
-      MQ_HOST: "mq"
-      HOST_NETWORK: "off"
-      VERBOSITY: "1"
-      MANAGE_IPTABLES: "on"
-      PORT_FORWARD_SERVICES: "dns"
-    ports:
-      - "51821-51830:51821-51830/udp"
-      - "8081:8081"
-  netmaker-ui:
-    container_name: netmaker-ui
-    depends_on:
-      - netmaker
-    image: gravitl/netmaker-ui:v0.14.4
-    links:
-      - "netmaker:api"
-    ports:
-      - "8082:80"
-    environment:
-      BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN"
-    restart: always
-  coredns:
-    depends_on:
-      - netmaker 
-    image: coredns/coredns
-    command: -conf /root/dnsconfig/Corefile
-    container_name: coredns
-    restart: always
-    ports:
-      - "COREDNS_IP:53:53/udp"
-      - "COREDNS_IP:53:53/tcp"
-    volumes:
-      - dnsconfig:/root/dnsconfig
-  mq:
-    image: eclipse-mosquitto:2.0.11-openssl
-    container_name: mq
-    restart: unless-stopped
-    ports:
-      - "127.0.0.1:1883:1883"
-      - "8883:8883"
-    volumes:
-      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
-      - /root/certs/:/mosquitto/certs/
-      - mosquitto_data:/mosquitto/data
-      - mosquitto_logs:/mosquitto/log
-volumes:
-  sqldata: {}
-  dnsconfig: {}
-  mosquitto_data: {}
-  mosquitto_logs: {}

compose/docker-compose.nodns.yml

@@ -1,83 +0,0 @@
-version: "3.4"
-
-services:
-  netmaker:
-    container_name: netmaker
-    image: gravitl/netmaker:v0.14.4
-    volumes:
-      - dnsconfig:/root/config/dnsconfig
-      - sqldata:/root/data
-      - /root/certs:/etc/netmaker/
-    cap_add: 
-      - NET_ADMIN
-      - NET_RAW
-      - SYS_MODULE
-    sysctls:
-      - net.ipv4.ip_forward=1
-      - net.ipv4.conf.all.src_valid_mark=1
-      - net.ipv6.conf.all.disable_ipv6=0
-      - net.ipv6.conf.all.forwarding=1
-    restart: always
-    environment:
-      SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
-      SERVER_HOST: "SERVER_PUBLIC_IP"
-      SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
-      COREDNS_ADDR: "SERVER_PUBLIC_IP"
-      DNS_MODE: "off"
-      SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
-      API_PORT: "8081"
-      CLIENT_MODE: "on"
-      MASTER_KEY: "REPLACE_MASTER_KEY"
-      CORS_ALLOWED_ORIGIN: "*"
-      DISPLAY_KEYS: "on"
-      DATABASE: "sqlite"
-      NODE_ID: "netmaker-server-1"
-      MQ_HOST: "mq"
-      HOST_NETWORK: "off"
-      VERBOSITY: "1"
-      MANAGE_IPTABLES: "on"
-      PORT_FORWARD_SERVICES: "dns"
-    ports:
-      - "51821-51830:51821-51830/udp"
-      - "8081:8081"
-      - "50051:50051"
-  netmaker-ui:
-    container_name: netmaker-ui
-    depends_on:
-      - netmaker
-    image: gravitl/netmaker-ui:v0.14.4
-    links:
-      - "netmaker:api"
-    ports:
-      - "8082:80"
-    environment:
-      BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN"
-    restart: always
-  caddy:
-    image: caddy:latest
-    container_name: caddy
-    restart: unless-stopped
-    network_mode: host # Wants ports 80 and 443!
-    volumes:
-      - /root/Caddyfile:/etc/caddy/Caddyfile
-      # - $PWD/site:/srv # you could also serve a static site in site folder
-      - caddy_data:/data
-      - caddy_conf:/config
-  mq:
-    image: eclipse-mosquitto:2.0.11-openssl
-    container_name: mq
-    restart: unless-stopped
-    ports:
-      - "127.0.0.1:1883:1883"
-      - "8883:8883" 
-    volumes:
-      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
-      - mosquitto_data:/mosquitto/data
-      - mosquitto_logs:/mosquitto/log
-      - /root/certs/:/mosquitto/certs/
-volumes:
-  caddy_data: {}
-  caddy_conf: {}
-  sqldata: {}
-  mosquitto_data: {}
-  mosquitto_logs: {}

compose/docker-compose.reference.yml

@@ -1,13 +1,10 @@
+version: "3.4"
+
 services:
   netmaker: # The Primary Server for running Netmaker
-    privileged: true # Necessary to run sudo/root level commands on host system. Likely using this if running with host networking on.
     container_name: netmaker
-    image: gravitl/netmaker:v0.14.4
-    volumes: # Volume mounts necessary for CLIENT_MODE to control wireguard networking on host (except dnsconfig, which is where dns config files are stored for use by CoreDNS)
-      - dnsconfig:/root/config/dnsconfig # Netmaker writes Corefile to this location, which gets mounted by CoreDNS for DNS configuration.
-      - sqldata:/root/data
-      - /root/certs:/etc/netmaker/ # cert management location
-    cap_add: # Necessary capabilities to set iptables when running in container
+    image: gravitl/netmaker:v0.14.5
+    cap_add: 
       - NET_ADMIN
       - NET_RAW
       - SYS_MODULE
@@ -17,78 +14,130 @@ services:
       - net.ipv6.conf.all.disable_ipv6=0
       - net.ipv6.conf.all.forwarding=1
     restart: always
-    network_mode: host # Must configure with very particular settngs for host networking to work. Do not just set on!
-    environment:
-      SERVER_NAME: "" # The domain/host IP indicating the mq broker address
-      SERVER_HOST: "" # All the Docker Compose files pre-populate this with HOST_IP, which you replace as part of the install instructions. This will set the HTTP host.
-      SERVER_HTTP_HOST: "127.0.0.1" # Overrides SERVER_HOST if set. Useful for making HTTP available via different interfaces/networks.
-      API_PORT: 8081 # The HTTP API port for Netmaker. Used for API calls / communication from front end. If changed, need to change port of BACKEND_URL for netmaker-ui.
-      CLIENT_MODE: "on" # on if netmaker should run its own client, off if not.
-      MASTER_KEY: "secretkey" # The admin master key for accessing the API. Change this in any production installation.
-      CORS_ALLOWED_ORIGIN: "*" # The "allowed origin" for API requests. Change to restrict where API requests can come from.
+    volumes: # Volume mounts necessary for sql, coredns, and mqtt
+      - dnsconfig:/root/config/dnsconfig
+      - sqldata:/root/data
+      - shared_certs:/etc/netmaker
+    environment: # Necessary capabilities to set iptables when running in container
+      SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN" # The domain/host IP indicating the mq broker address
+      SERVER_HOST: "SERVER_PUBLIC_IP" # Set to public IP of machine.
+      SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN" # Overrides SERVER_HOST if set. Useful for making HTTP available via different interfaces/networks.
+      SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
+      COREDNS_ADDR: "SERVER_PUBLIC_IP" # Address of the CoreDNS server. Defaults to SERVER_HOST
+      DNS_MODE: "on" # Enables DNS Mode, meaning all nodes will set hosts file for private dns settings.
+      API_PORT: "8081" # The HTTP API port for Netmaker. Used for API calls / communication from front end. If changed, need to change port of BACKEND_URL for netmaker-ui.
+      CLIENT_MODE: "on" # Depricated. CLIENT_MODE should always be ON
       REST_BACKEND: "on" # Enables the REST backend (API running on API_PORT at SERVER_HTTP_HOST). Change to "off" to turn off.
-      DNS_MODE: "on" # Enables DNS Mode, meaning config files will be generated for CoreDNS. Note, turning "off" does not remove CoreDNS. You still need to remove CoreDNS from compose file.
       DISABLE_REMOTE_IP_CHECK: "off" # If turned "on", Server will not set Host based on remote IP check. This is already overridden if SERVER_HOST is set. Turned "off" by default.
-      COREDNS_ADDR: "" # Address of the CoreDNS server. Defaults to SERVER_HOST
-      DISPLAY_KEYS: "on" # Show keys permanently in UI (until deleted) as opposed to 1-time display.
-      SERVER_API_CONN_STRING: "" # Changes the api connection string. IP:PORT format. By default is empty and uses SERVER_HOST:API_PORT
-      RCE: "off" # Enables setting PostUp and PostDown (arbitrary commands) on nodes from the server. Off by default.
-      NODE_ID: "" # Sets the name/id of the nodes that the server creates. Necessary for HA configurations to identify between servers (for instance, netmaker-1, netmaker-2, etc). For non-HA deployments, is not necessary.
       TELEMETRY: "on" # Whether or not to send telemetry data to help improve Netmaker. Switch to "off" to opt out of sending telemetry.
-      MQ_HOST: "mq" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container.
+      RCE: "off" # Enables setting PostUp and PostDown (arbitrary commands) on nodes from the server. Off by default.
+      MASTER_KEY: "REPLACE_MASTER_KEY" # The admin master key for accessing the API. Change this in any production installation.
+      CORS_ALLOWED_ORIGIN: "*" # The "allowed origin" for API requests. Change to restrict where API requests can come from.
+      DISPLAY_KEYS: "on" # Show keys permanently in UI (until deleted) as opposed to 1-time display.
+      DATABASE: "sqlite" # Database to use - sqlite, postgres, or rqlite
+      NODE_ID: "netmaker-server-1" # used for HA - identifies this server vs other servers
+      MQ_HOST: "mq"  # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container.
+      MQ_SERVER_PORT: "1883" # the reachable port of MQ by the server - change if internal MQ port changes (or use external port if MQ is not on the same machine)
+      MQ_PORT: "443" # the reachable port of MQ - change if external MQ port changes (port on proxy, not necessarily the one exposed in docker-compose)
       HOST_NETWORK: "off" # whether or not host networking is turned on. Only turn on if configured for host networking (see docker-compose.hostnetwork.yml). Will set host-level settings like iptables.
+      VERBOSITY: "1" # logging verbosity level - 1, 2, or 3
       MANAGE_IPTABLES: "on" # deprecated
       PORT_FORWARD_SERVICES: "dns" # decide which services to port forward ("dns","ssh", or "mq")
+      # this section is for OAuth
+      AUTH_PROVIDER: "" # "<azure-ad|github|google|oidc>"
+      CLIENT_ID: "" # "<client id of your oauth provider>"
+      CLIENT_SECRET: "" # "<client secret of your oauth provider>"
+      FRONTEND_URL: "" # "https://dashboard.<netmaker base domain>"
+      AZURE_TENANT: "" # "<only for azure, you may optionally specify the tenant for the OAuth>"
+      OIDC_ISSUER: "" # https://oidc.yourprovider.com - URL of oidc provider
     ports:
-      - "51821-51830:51821-51830/udp"
-      - "8081:8081"
-  netmaker-ui: # The Netmaker UI Component
+      - "51821-51830:51821-51830/udp" # wireguard ports
+    expose:
+      - "8081" # api port
+    labels: # only for use with traefik proxy (default)
+      - traefik.enable=true
+      - traefik.http.routers.netmaker-api.entrypoints=websecure
+      - traefik.http.routers.netmaker-api.rule=Host(`api.NETMAKER_BASE_DOMAIN`)
+      - traefik.http.routers.netmaker-api.service=netmaker-api
+      - traefik.http.services.netmaker-api.loadbalancer.server.port=8081
+  netmaker-ui:  # The Netmaker UI Component
     container_name: netmaker-ui
+    image: gravitl/netmaker-ui:v0.14.5
     depends_on:
       - netmaker
-    image: gravitl/netmaker-ui:v0.14.4
     links:
       - "netmaker:api"
-    ports:
-      - "8082:80"
+    restart: always
     environment:
-      BACKEND_URL: "http://HOST_IP:8081" # URL where UI will send API requests. Change based on SERVER_HOST, SERVER_HTTP_HOST, and API_PORT
-  restart: always
-  coredns: # The DNS Server. Remove this section if DNS_MODE="off"
-    depends_on:
-      - netmaker 
+      BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN" # URL where UI will send API requests. Change based on SERVER_HOST, SERVER_HTTP_HOST, and API_PORT
+    expose:
+      - "80"
+    labels:
+      - traefik.enable=true
+      - traefik.http.middlewares.nmui-security.headers.accessControlAllowOriginList=*.NETMAKER_BASE_DOMAIN
+      - traefik.http.middlewares.nmui-security.headers.stsSeconds=31536000
+      - traefik.http.middlewares.nmui-security.headers.browserXssFilter=true
+      - traefik.http.middlewares.nmui-security.headers.customFrameOptionsValue=SAMEORIGIN
+      - traefik.http.middlewares.nmui-security.headers.customResponseHeaders.X-Robots-Tag=none
+      - traefik.http.middlewares.nmui-security.headers.customResponseHeaders.Server= # Remove the server name
+      - traefik.http.routers.netmaker-ui.entrypoints=websecure
+      - traefik.http.routers.netmaker-ui.middlewares=nmui-security@docker
+      - traefik.http.routers.netmaker-ui.rule=Host(`dashboard.NETMAKER_BASE_DOMAIN`)
+      - traefik.http.routers.netmaker-ui.service=netmaker-ui
+      - traefik.http.services.netmaker-ui.loadbalancer.server.port=80
+  coredns: # The DNS Server. CoreDNS can be removed unless doing special advanced use cases
+    container_name: coredns
     image: coredns/coredns
     command: -conf /root/dnsconfig/Corefile
-    container_name: coredns
+    depends_on:
+      - netmaker
     restart: always
     volumes:
       - dnsconfig:/root/dnsconfig
-  caddy:
-    image: caddy:latest
-    container_name: caddy
-    restart: unless-stopped
-    network_mode: host # Wants ports 80 and 443!
+  traefik: # the default proxy - can be replaced with caddy or nginx, but requires careful configuration
+    image: traefik:v2.6
+    container_name: traefik
+    command:
+      - "--certificatesresolvers.http.acme.email=YOUR_EMAIL"
+      - "--certificatesresolvers.http.acme.storage=/letsencrypt/acme.json"
+      - "--certificatesresolvers.http.acme.tlschallenge=true"
+      - "--entrypoints.websecure.address=:443"
+      - "--entrypoints.websecure.http.tls=true"
+      - "--entrypoints.websecure.http.tls.certResolver=http"
+      - "--log.level=INFO"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedByDefault=false"
+      - "--serverstransport.insecureskipverify=true"
+    restart: always
     volumes:
-      - /root/Caddyfile:/etc/caddy/Caddyfile
-      # - $PWD/site:/srv # you could also serve a static site in site folder
-      - caddy_data:/data
-      - caddy_conf:/config  
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - traefik_certs:/letsencrypt
+    ports:
+      - "443:443"
   mq: # the MQTT broker for netmaker
-    image: eclipse-mosquitto:2.0.11-openssl
     container_name: mq
+    image: eclipse-mosquitto:2.0.11-openssl
+    depends_on:
+      - netmaker
     restart: unless-stopped
-    ports:
-      - "127.0.0.1:1883:1883"
-      - "8883:8883" 
     volumes:
       - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf # need to pull conf file from github before running (under docker/mosquitto.conf)
       - mosquitto_data:/mosquitto/data
       - mosquitto_logs:/mosquitto/log
-      - /root/certs/:/mosquitto/certs/
+      - shared_certs:/mosquitto/certs
+    expose:
+      - "8883"
+    labels:
+      - traefik.enable=true
+      - traefik.tcp.routers.mqtts.rule=HostSNI(`broker.NETMAKER_BASE_DOMAIN`)
+      - traefik.tcp.routers.mqtts.tls.passthrough=true
+      - traefik.tcp.services.mqtts-svc.loadbalancer.server.port=8883
+      - traefik.tcp.routers.mqtts.service=mqtts-svc
+      - traefik.tcp.routers.mqtts.entrypoints=websecure
 volumes:
-  caddy_data: {} # storage for caddy data
-  caddy_conf: {} # storage for caddy configuration file
+  traefik_certs: {} # ssl certificates - auto generated
+  shared_certs: {} # netmaker certs generated for MQ comms - used by nodes/servers
   sqldata: {} # storage for embedded sqlite
   dnsconfig: {} # storage for coredns
   mosquitto_data: {} # storage for mqtt data
-  mosquitto_logs: {} # storage for mqtt logs
+  mosquitto_logs: {} # storage for mqtt logs

compose/docker-compose.traefik.yml

@@ -1,137 +0,0 @@
-version: "3.4"
-
-services:
-  netmaker:
-    container_name: netmaker
-    image: gravitl/netmaker:v0.14.4
-    cap_add: 
-      - NET_ADMIN
-      - NET_RAW
-      - SYS_MODULE
-    sysctls:
-      - net.ipv4.ip_forward=1
-      - net.ipv4.conf.all.src_valid_mark=1
-      - net.ipv6.conf.all.disable_ipv6=0
-      - net.ipv6.conf.all.forwarding=1
-    restart: always
-    volumes:
-      - dnsconfig:/root/config/dnsconfig
-      - sqldata:/root/data
-      - shared_certs:/etc/netmaker
-    environment:
-      SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
-      SERVER_HOST: "SERVER_PUBLIC_IP"
-      SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
-      COREDNS_ADDR: "SERVER_PUBLIC_IP"
-      DNS_MODE: "on"
-      SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
-      API_PORT: "8081"
-      CLIENT_MODE: "on"
-      MASTER_KEY: "REPLACE_MASTER_KEY"
-      CORS_ALLOWED_ORIGIN: "*"
-      DISPLAY_KEYS: "on"
-      DATABASE: "sqlite"
-      NODE_ID: "netmaker-server-1"
-      MQ_HOST: "mq"
-      # uncomment once netmaker supports changing MQ port
-      MQ_PORT: "443"
-      HOST_NETWORK: "off"
-      VERBOSITY: "1"
-      MANAGE_IPTABLES: "on"
-      PORT_FORWARD_SERVICES: "dns"
-    ports:
-      - "51821-51830:51821-51830/udp"
-    expose:
-      - "8081"
-    labels:
-      - traefik.enable=true
-      - traefik.http.routers.netmaker-api.entrypoints=websecure
-      - traefik.http.routers.netmaker-api.rule=Host(`api.NETMAKER_BASE_DOMAIN`)
-      - traefik.http.routers.netmaker-api.service=netmaker-api
-      - traefik.http.services.netmaker-api.loadbalancer.server.port=8081
-  netmaker-ui:
-    container_name: netmaker-ui
-    image: gravitl/netmaker-ui:v0.14.4
-    depends_on:
-      - netmaker
-    links:
-      - "netmaker:api"
-    restart: always
-    environment:
-      BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN"
-    expose:
-      - "80"
-    labels:
-      - traefik.enable=true
-      - traefik.http.middlewares.nmui-security.headers.accessControlAllowOriginList=*.NETMAKER_BASE_DOMAIN
-      - traefik.http.middlewares.nmui-security.headers.stsSeconds=31536000
-      - traefik.http.middlewares.nmui-security.headers.browserXssFilter=true
-      - traefik.http.middlewares.nmui-security.headers.customFrameOptionsValue=SAMEORIGIN
-      - traefik.http.middlewares.nmui-security.headers.customResponseHeaders.X-Robots-Tag=none
-      - traefik.http.middlewares.nmui-security.headers.customResponseHeaders.Server= # Remove the server name
-      - traefik.http.routers.netmaker-ui.entrypoints=websecure
-      - traefik.http.routers.netmaker-ui.middlewares=nmui-security@docker
-      - traefik.http.routers.netmaker-ui.rule=Host(`dashboard.NETMAKER_BASE_DOMAIN`)
-      - traefik.http.routers.netmaker-ui.service=netmaker-ui
-      - traefik.http.services.netmaker-ui.loadbalancer.server.port=80
-  coredns:
-    container_name: coredns
-    image: coredns/coredns
-    command: -conf /root/dnsconfig/Corefile
-    depends_on:
-      - netmaker
-    restart: always
-    volumes:
-      - dnsconfig:/root/dnsconfig
-  traefik:
-    image: traefik:v2.6
-    container_name: traefik
-    command:
-      - "--certificatesresolvers.http.acme.email=YOUR_EMAIL"
-      - "--certificatesresolvers.http.acme.storage=/letsencrypt/acme.json"
-      - "--certificatesresolvers.http.acme.tlschallenge=true"
-      - "--entrypoints.websecure.address=:443"
-      - "--entrypoints.websecure.http.tls=true"
-      - "--entrypoints.websecure.http.tls.certResolver=http"
-      - "--log.level=INFO"
-      - "--providers.docker=true"
-      - "--providers.docker.exposedByDefault=false"
-      - "--serverstransport.insecureskipverify=true"
-    restart: always
-    volumes:
-      - /var/run/docker.sock:/var/run/docker.sock:ro
-      - traefik_certs:/letsencrypt
-    ports:
-      - "443:443"
-  mq:
-    container_name: mq
-    image: eclipse-mosquitto:2.0.11-openssl
-    depends_on:
-      - netmaker
-    restart: unless-stopped
-    volumes:
-      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
-      - mosquitto_data:/mosquitto/data
-      - mosquitto_logs:/mosquitto/log
-      - shared_certs:/mosquitto/certs
-    ports:
-      - "127.0.0.1:1883:1883"
-    # comment once netmaker supports changing MQ port
-    #  - "8883:8883"
-    # uncomment once netmaker supports changing MQ port
-    expose:
-      - "8883"
-    labels:
-      - traefik.enable=true
-      - traefik.tcp.routers.mqtts.rule=HostSNI(`broker.NETMAKER_BASE_DOMAIN`)
-      - traefik.tcp.routers.mqtts.tls.passthrough=true
-      - traefik.tcp.services.mqtts-svc.loadbalancer.server.port=8883
-      - traefik.tcp.routers.mqtts.service=mqtts-svc
-      - traefik.tcp.routers.mqtts.entrypoints=websecure
-volumes:
-  traefik_certs: {}
-  shared_certs: {}
-  sqldata: {}
-  dnsconfig: {}
-  mosquitto_data: {}
-  mosquitto_logs: {}

compose/docker-compose.yml

@@ -3,11 +3,7 @@ version: "3.4"
 services:
   netmaker:
     container_name: netmaker
-    image: gravitl/netmaker:v0.14.4
-    volumes:
-      - dnsconfig:/root/config/dnsconfig
-      - sqldata:/root/data
-      - /root/certs:/etc/netmaker/
+    image: gravitl/netmaker:v0.14.5
     cap_add: 
       - NET_ADMIN
       - NET_RAW
@@ -18,6 +14,10 @@ services:
       - net.ipv6.conf.all.disable_ipv6=0
       - net.ipv6.conf.all.forwarding=1
     restart: always
+    volumes:
+      - dnsconfig:/root/config/dnsconfig
+      - sqldata:/root/data
+      - shared_certs:/etc/netmaker
     environment:
       SERVER_NAME: "broker.NETMAKER_BASE_DOMAIN"
       SERVER_HOST: "SERVER_PUBLIC_IP"
@@ -33,59 +33,99 @@ services:
       DATABASE: "sqlite"
       NODE_ID: "netmaker-server-1"
       MQ_HOST: "mq"
+      MQ_PORT: "443"
+      MQ_SERVER_PORT: "1883"
       HOST_NETWORK: "off"
       VERBOSITY: "1"
       MANAGE_IPTABLES: "on"
       PORT_FORWARD_SERVICES: "dns"
     ports:
       - "51821-51830:51821-51830/udp"
+    expose:
+      - "8081"
+    labels:
+      - traefik.enable=true
+      - traefik.http.routers.netmaker-api.entrypoints=websecure
+      - traefik.http.routers.netmaker-api.rule=Host(`api.NETMAKER_BASE_DOMAIN`)
+      - traefik.http.routers.netmaker-api.service=netmaker-api
+      - traefik.http.services.netmaker-api.loadbalancer.server.port=8081
   netmaker-ui:
     container_name: netmaker-ui
+    image: gravitl/netmaker-ui:v0.14.5
     depends_on:
       - netmaker
-    image: gravitl/netmaker-ui:v0.14.4
     links:
       - "netmaker:api"
+    restart: always
     environment:
       BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN"
-    restart: always
+    expose:
+      - "80"
+    labels:
+      - traefik.enable=true
+      - traefik.http.middlewares.nmui-security.headers.accessControlAllowOriginList=*.NETMAKER_BASE_DOMAIN
+      - traefik.http.middlewares.nmui-security.headers.stsSeconds=31536000
+      - traefik.http.middlewares.nmui-security.headers.browserXssFilter=true
+      - traefik.http.middlewares.nmui-security.headers.customFrameOptionsValue=SAMEORIGIN
+      - traefik.http.middlewares.nmui-security.headers.customResponseHeaders.X-Robots-Tag=none
+      - traefik.http.middlewares.nmui-security.headers.customResponseHeaders.Server= # Remove the server name
+      - traefik.http.routers.netmaker-ui.entrypoints=websecure
+      - traefik.http.routers.netmaker-ui.middlewares=nmui-security@docker
+      - traefik.http.routers.netmaker-ui.rule=Host(`dashboard.NETMAKER_BASE_DOMAIN`)
+      - traefik.http.routers.netmaker-ui.service=netmaker-ui
+      - traefik.http.services.netmaker-ui.loadbalancer.server.port=80
   coredns:
-    depends_on:
-      - netmaker 
+    container_name: coredns
     image: coredns/coredns
     command: -conf /root/dnsconfig/Corefile
-    container_name: coredns
+    depends_on:
+      - netmaker
     restart: always
     volumes:
       - dnsconfig:/root/dnsconfig
-  caddy:
-    image: caddy:latest
-    container_name: caddy
-    restart: unless-stopped
+  traefik:
+    image: traefik:v2.6
+    container_name: traefik
+    command:
+      - "--certificatesresolvers.http.acme.email=YOUR_EMAIL"
+      - "--certificatesresolvers.http.acme.storage=/letsencrypt/acme.json"
+      - "--certificatesresolvers.http.acme.tlschallenge=true"
+      - "--entrypoints.websecure.address=:443"
+      - "--entrypoints.websecure.http.tls=true"
+      - "--entrypoints.websecure.http.tls.certResolver=http"
+      - "--log.level=INFO"
+      - "--providers.docker=true"
+      - "--providers.docker.exposedByDefault=false"
+      - "--serverstransport.insecureskipverify=true"
+    restart: always
+    volumes:
+      - /var/run/docker.sock:/var/run/docker.sock:ro
+      - traefik_certs:/letsencrypt
     ports:
-      - "80:80"
       - "443:443"
-    volumes:
-      - /root/Caddyfile:/etc/caddy/Caddyfile
-      # - $PWD/site:/srv # you could also serve a static site in site folder
-      - caddy_data:/data
-      - caddy_conf:/config
   mq:
+    container_name: mq
     image: eclipse-mosquitto:2.0.11-openssl
     depends_on:
       - netmaker
-    container_name: mq
     restart: unless-stopped
-    ports:
-      - "8883:8883"
     volumes:
       - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
-      - /root/certs/:/mosquitto/certs/
       - mosquitto_data:/mosquitto/data
       - mosquitto_logs:/mosquitto/log
+      - shared_certs:/mosquitto/certs
+    expose:
+      - "8883"
+    labels:
+      - traefik.enable=true
+      - traefik.tcp.routers.mqtts.rule=HostSNI(`broker.NETMAKER_BASE_DOMAIN`)
+      - traefik.tcp.routers.mqtts.tls.passthrough=true
+      - traefik.tcp.services.mqtts-svc.loadbalancer.server.port=8883
+      - traefik.tcp.routers.mqtts.service=mqtts-svc
+      - traefik.tcp.routers.mqtts.entrypoints=websecure
 volumes:
-  caddy_data: {}
-  caddy_conf: {}
+  traefik_certs: {}
+  shared_certs: {}
   sqldata: {}
   dnsconfig: {}
   mosquitto_data: {}

config/config.go

@@ -55,6 +55,7 @@ type ServerConfig struct {
 	Verbosity             int32  `yaml:"verbosity"`
 	ServerCheckinInterval int64  `yaml:"servercheckininterval"`
 	AuthProvider          string `yaml:"authprovider"`
+	OIDCIssuer            string `yaml:"oidcissuer"`
 	ClientID              string `yaml:"clientid"`
 	ClientSecret          string `yaml:"clientsecret"`
 	FrontendURL           string `yaml:"frontendurl"`

controllers/dns.go

@@ -127,7 +127,7 @@ func createDNS(w http.ResponseWriter, r *http.Request) {
 			if err = logic.ServerUpdate(&serverNode, false); err != nil {
 				logger.Log(1, "failed to update server node after DNS update on", entry.Network)
 			}
-			if err = mq.PublishPeerUpdate(&serverNode); err != nil {
+			if err = mq.PublishPeerUpdate(&serverNode, false); err != nil {
 				logger.Log(0, "failed to publish peer update after ACL update on", entry.Network)
 			}
 		}

controllers/network.go

@@ -255,7 +255,7 @@ func updateNetworkACL(w http.ResponseWriter, r *http.Request) {
 			if err = logic.ServerUpdate(&serverNode, false); err != nil {
 				logger.Log(1, "failed to update server node after ACL update on", netname)
 			}
-			if err = mq.PublishPeerUpdate(&serverNode); err != nil {
+			if err = mq.PublishPeerUpdate(&serverNode, false); err != nil {
 				logger.Log(0, "failed to publish peer update after ACL update on", netname)
 			}
 		}

controllers/network_test.go

@@ -182,32 +182,32 @@ func TestSecurityCheck(t *testing.T) {
 	database.InitializeDatabase()
 	os.Setenv("MASTER_KEY", "secretkey")
 	t.Run("NoNetwork", func(t *testing.T) {
-		err, networks, username := SecurityCheck(false, "", "Bearer secretkey")
+		networks, username, err := SecurityCheck(false, "", "Bearer secretkey")
 		assert.Nil(t, err)
 		t.Log(networks, username)
 	})
 	t.Run("WithNetwork", func(t *testing.T) {
-		err, networks, username := SecurityCheck(false, "skynet", "Bearer secretkey")
+		networks, username, err := SecurityCheck(false, "skynet", "Bearer secretkey")
 		assert.Nil(t, err)
 		t.Log(networks, username)
 	})
 	t.Run("BadNet", func(t *testing.T) {
 		t.Skip()
-		err, networks, username := SecurityCheck(false, "badnet", "Bearer secretkey")
+		networks, username, err := SecurityCheck(false, "badnet", "Bearer secretkey")
 		assert.NotNil(t, err)
 		t.Log(err)
 		t.Log(networks, username)
 	})
 	t.Run("BadToken", func(t *testing.T) {
-		err, networks, username := SecurityCheck(false, "skynet", "Bearer badkey")
+		networks, username, err := SecurityCheck(false, "skynet", "Bearer badkey")
 		assert.NotNil(t, err)
 		t.Log(err)
 		t.Log(networks, username)
 	})
 }
 
-func TestValidateNetworkUpdate(t *testing.T) {
-	t.Skip()
+func TestValidateNetwork(t *testing.T) {
+	//t.Skip()
 	//This functions is not called by anyone
 	//it panics as validation function 'display_name_valid' is not defined
 	database.InitializeDatabase()
@@ -220,23 +220,25 @@ func TestValidateNetworkUpdate(t *testing.T) {
 		{
 			testname: "InvalidAddress",
 			network: models.Network{
+				NetID:        "skynet",
 				AddressRange: "10.0.0.256",
 			},
 			errMessage: "Field validation for 'AddressRange' failed on the 'cidr' tag",
 		},
-		{
-			testname: "InvalidAddress6",
-			network: models.Network{
-				AddressRange6: "2607::ag",
-			},
-			errMessage: "Field validation for 'AddressRange6' failed on the 'cidr' tag",
-		},
+		//{
+		//	testname: "InvalidAddress6",
+		//	network: models.Network{
+		//		NetID:         "skynet1",
+		//		AddressRange6: "2607::ffff/130",
+		//	},
+		//	errMessage: "Field validation for 'AddressRange6' failed on the 'cidr' tag",
+		//},
 		{
 			testname: "InvalidNetID",
 			network: models.Network{
-				NetID: "contains spaces",
+				NetID: "with spaces",
 			},
-			errMessage: "Field validation for 'NetID' failed on the 'alphanum' tag",
+			errMessage: "Field validation for 'NetID' failed on the 'netid_valid' tag",
 		},
 		{
 			testname: "NetIDTooLong",
@@ -248,6 +250,7 @@ func TestValidateNetworkUpdate(t *testing.T) {
 		{
 			testname: "ListenPortTooLow",
 			network: models.Network{
+				NetID:             "skynet",
 				DefaultListenPort: 1023,
 			},
 			errMessage: "Field validation for 'DefaultListenPort' failed on the 'min' tag",
@@ -255,6 +258,7 @@ func TestValidateNetworkUpdate(t *testing.T) {
 		{
 			testname: "ListenPortTooHigh",
 			network: models.Network{
+				NetID:             "skynet",
 				DefaultListenPort: 65536,
 			},
 			errMessage: "Field validation for 'DefaultListenPort' failed on the 'max' tag",
@@ -262,6 +266,7 @@ func TestValidateNetworkUpdate(t *testing.T) {
 		{
 			testname: "KeepAliveTooBig",
 			network: models.Network{
+				NetID:            "skynet",
 				DefaultKeepalive: 1010,
 			},
 			errMessage: "Field validation for 'DefaultKeepalive' failed on the 'max' tag",
@@ -269,6 +274,7 @@ func TestValidateNetworkUpdate(t *testing.T) {
 		{
 			testname: "InvalidLocalRange",
 			network: models.Network{
+				NetID:      "skynet",
 				LocalRange: "192.168.0.1",
 			},
 			errMessage: "Field validation for 'LocalRange' failed on the 'cidr' tag",
@@ -276,8 +282,10 @@ func TestValidateNetworkUpdate(t *testing.T) {
 	}
 	for _, tc := range cases {
 		t.Run(tc.testname, func(t *testing.T) {
+			t.Log(tc.testname)
 			network := models.Network(tc.network)
-			err := logic.ValidateNetworkUpdate(network)
+			network.SetDefaults()
+			err := logic.ValidateNetwork(&network, false)
 			assert.NotNil(t, err)
 			assert.Contains(t, err.Error(), tc.errMessage)
 		})

controllers/node.go

@@ -501,7 +501,7 @@ func createNode(w http.ResponseWriter, r *http.Request) {
 	logger.Log(1, r.Header.Get("user"), "created new node", node.Name, "on network", node.Network)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(response)
-	runForceServerUpdate(&node)
+	runForceServerUpdate(&node, true)
 }
 
 // Takes node out of pending state
@@ -722,7 +722,7 @@ func deleteNode(w http.ResponseWriter, r *http.Request) {
 
 	logger.Log(1, r.Header.Get("user"), "Deleted node", nodeid, "from network", params["network"])
 	runUpdates(&node, false)
-	runForceServerUpdate(&node)
+	runForceServerUpdate(&node, false)
 }
 
 func runUpdates(node *models.Node, ifaceDelta bool) {
@@ -751,7 +751,7 @@ func runServerUpdate(node *models.Node, ifaceDelta bool) error {
 	}
 
 	if ifaceDelta && logic.IsLeader(&currentServerNode) {
-		if err := mq.PublishPeerUpdate(&currentServerNode); err != nil {
+		if err := mq.PublishPeerUpdate(&currentServerNode, false); err != nil {
 			logger.Log(1, "failed to publish peer update "+err.Error())
 		}
 	}
@@ -763,9 +763,9 @@ func runServerUpdate(node *models.Node, ifaceDelta bool) error {
 	return nil
 }
 
-func runForceServerUpdate(node *models.Node) {
+func runForceServerUpdate(node *models.Node, publishPeerUpdateToNode bool) {
 	go func() {
-		if err := mq.PublishPeerUpdate(node); err != nil {
+		if err := mq.PublishPeerUpdate(node, publishPeerUpdateToNode); err != nil {
 			logger.Log(1, "failed a peer update after creation of node", node.Name)
 		}
 

controllers/security.go

@@ -31,7 +31,7 @@ func securityCheck(reqAdmin bool, next http.Handler) http.HandlerFunc {
 			return
 		}
 
-		err, networks, username := SecurityCheck(reqAdmin, params["networkname"], bearerToken)
+		networks, username, err := SecurityCheck(reqAdmin, params["networkname"], bearerToken)
 		if err != nil {
 			if strings.Contains(err.Error(), "does not exist") {
 				errorResponse.Code = http.StatusNotFound
@@ -53,7 +53,7 @@ func securityCheck(reqAdmin bool, next http.Handler) http.HandlerFunc {
 }
 
 // SecurityCheck - checks token stuff
-func SecurityCheck(reqAdmin bool, netname string, token string) (error, []string, string) {
+func SecurityCheck(reqAdmin bool, netname string, token string) ([]string, string, error) {
 
 	var hasBearer = true
 	var tokenSplit = strings.Split(token, " ")
@@ -72,10 +72,10 @@ func SecurityCheck(reqAdmin bool, netname string, token string) (error, []string
 		userName, networks, isadmin, err := logic.VerifyUserToken(authToken)
 		username = userName
 		if err != nil {
-			return errors.New("error verifying user token"), nil, username
+			return nil, username, errors.New("error verifying user token")
 		}
 		if !isadmin && reqAdmin {
-			return errors.New("you are unauthorized to access this endpoint"), nil, username
+			return nil, username, errors.New("you are unauthorized to access this endpoint")
 		}
 		userNetworks = networks
 		if isadmin {
@@ -83,10 +83,10 @@ func SecurityCheck(reqAdmin bool, netname string, token string) (error, []string
 		} else {
 			networkexists, err := functions.NetworkExists(netname)
 			if err != nil && !database.IsEmptyRecord(err) {
-				return err, nil, ""
+				return nil, "", err
 			}
 			if netname != "" && !networkexists {
-				return errors.New("this network does not exist"), nil, ""
+				return nil, "", errors.New("this network does not exist")
 			}
 		}
 	} else if isMasterAuthenticated {
@@ -95,7 +95,7 @@ func SecurityCheck(reqAdmin bool, netname string, token string) (error, []string
 	if len(userNetworks) == 0 {
 		userNetworks = append(userNetworks, NO_NETWORKS_PRESENT)
 	}
-	return nil, userNetworks, username
+	return userNetworks, username, nil
 }
 
 // Consider a more secure way of setting master key

controllers/server.go

@@ -15,6 +15,7 @@ import (
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/netclient/config"
 	"github.com/gravitl/netmaker/servercfg"
+	"github.com/gravitl/netmaker/serverctl"
 	"github.com/gravitl/netmaker/tls"
 )
 
@@ -142,12 +143,12 @@ func register(w http.ResponseWriter, r *http.Request) {
 
 // genCerts generates a client certificate and returns the certificate and root CA
 func genCerts(clientKey *ed25519.PrivateKey, name *pkix.Name) (*x509.Certificate, *x509.Certificate, error) {
-	ca, err := tls.ReadCert("/etc/netmaker/root.pem")
+	ca, err := serverctl.ReadCertFromDB(tls.ROOT_PEM_NAME)
 	if err != nil {
 		logger.Log(2, "root ca not found ", err.Error())
 		return nil, nil, fmt.Errorf("root ca not found %w", err)
 	}
-	key, err := tls.ReadKey("/etc/netmaker/root.key")
+	key, err := serverctl.ReadKeyFromDB(tls.ROOT_KEY_NAME)
 	if err != nil {
 		logger.Log(2, "root key not found ", err.Error())
 		return nil, nil, fmt.Errorf("root key not found %w", err)

database/database.go

@@ -26,6 +26,9 @@ const DELETED_NODES_TABLE_NAME = "deletednodes"
 // USERS_TABLE_NAME - users table
 const USERS_TABLE_NAME = "users"
 
+// CERTS_TABLE_NAME - certificates table
+const CERTS_TABLE_NAME = "certs"
+
 // DNS_TABLE_NAME - dns table
 const DNS_TABLE_NAME = "dns"
 
@@ -122,6 +125,7 @@ func InitializeDatabase() error {
 func createTables() {
 	createTable(NETWORKS_TABLE_NAME)
 	createTable(NODES_TABLE_NAME)
+	createTable(CERTS_TABLE_NAME)
 	createTable(DELETED_NODES_TABLE_NAME)
 	createTable(USERS_TABLE_NAME)
 	createTable(DNS_TABLE_NAME)

database/rqlite.go

@@ -83,17 +83,6 @@ func rqliteDeleteAllRecords(tableName string) error {
 	return nil
 }
 
-func rqliteFetchRecord(tableName string, key string) (string, error) {
-	results, err := FetchRecords(tableName)
-	if err != nil {
-		return "", err
-	}
-	if results[key] == "" {
-		return "", errors.New(NO_RECORD)
-	}
-	return results[key], nil
-}
-
 func rqliteFetchRecords(tableName string) (map[string]string, error) {
 	row, err := RQliteDatabase.QueryOne("SELECT * FROM " + tableName + " ORDER BY key")
 	if err != nil {

docker/mosquitto.conf

@@ -8,5 +8,5 @@ cafile /mosquitto/certs/root.pem
 certfile /mosquitto/certs/server.pem
 keyfile /mosquitto/certs/server.key
 
-listener 1883 
-allow_anonymous true
+listener 1883
+allow_anonymous true

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/mattn/go-sqlite3 v1.14.10
 	github.com/rqlite/gorqlite v0.0.0-20210514125552-08ff1e76b22f
 	github.com/skip2/go-qrcode v0.0.0-20200617195104-da1b6568686e
-	github.com/stretchr/testify v1.7.5
+	github.com/stretchr/testify v1.8.0
 	github.com/txn2/txeh v1.3.0
 	github.com/urfave/cli/v2 v2.10.3
 	golang.org/x/crypto v0.0.0-20220315160706-3147a52a75dd
@@ -31,7 +31,7 @@ require (
 
 require (
 	filippo.io/edwards25519 v1.0.0
-	fyne.io/fyne/v2 v2.2.1
+	fyne.io/fyne/v2 v2.2.2
 	github.com/c-robinson/iplib v1.0.3
 	github.com/cloverstd/tcping v0.1.1
 	github.com/guumaster/hostctl v1.1.2
@@ -41,8 +41,9 @@ require (
 
 require (
 	cloud.google.com/go v0.81.0 // indirect
-	fyne.io/systray v1.10.0 // indirect
+	fyne.io/systray v1.10.1-0.20220621085403-9a2652634e93 // indirect
 	github.com/Microsoft/go-winio v0.4.14 // indirect
+	github.com/coreos/go-oidc/v3 v3.2.0
 	github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/docker/distribution v2.7.1+incompatible // indirect
@@ -89,6 +90,7 @@ require (
 	golang.org/x/mobile v0.0.0-20211207041440-4e6c2922fdee // indirect
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
 	google.golang.org/appengine v1.6.7 // indirect
+	gopkg.in/square/go-jose.v2 v2.5.1 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	honnef.co/go/js/dom v0.0.0-20210725211120-f030747120f2 // indirect
 )

go.sum

@@ -40,10 +40,10 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 filippo.io/edwards25519 v1.0.0 h1:0wAIcmJUqRdI8IJ/3eGi5/HwXZWPujYXXlkrQogz0Ek=
 filippo.io/edwards25519 v1.0.0/go.mod h1:N1IkdkCkiLB6tki+MYJoSx2JTY9NUlxZE7eHn5EwJns=
-fyne.io/fyne/v2 v2.2.1 h1:yq8TgB2hrSjnRXcth07G9/+EcjvH7+o2YIK8rmMcIaI=
-fyne.io/fyne/v2 v2.2.1/go.mod h1:drswO92dAguX9sHBuQWRaSAu7lp8BJBxPcXAarMW3rc=
-fyne.io/systray v1.10.0 h1:Yr1D9Lxeiw3+vSuZWPlaHC8BMjIHZXJKkek706AfYQk=
-fyne.io/systray v1.10.0/go.mod h1:oM2AQqGJ1AMo4nNqZFYU8xYygSBZkW2hmdJ7n4yjedE=
+fyne.io/fyne/v2 v2.2.2 h1:/ox4o1JeDYb7LmcQS1LiJBmAnnsgKUEn25wZlAR8Ph0=
+fyne.io/fyne/v2 v2.2.2/go.mod h1:MBoGuHzLLSXdQOWFAwWhIhYTEMp33zqtGCReSWhaQTA=
+fyne.io/systray v1.10.1-0.20220621085403-9a2652634e93 h1:V2IC9t0Zj9Ur6qDbfhUuzVmIvXKFyxZXRJyigUvovs4=
+fyne.io/systray v1.10.1-0.20220621085403-9a2652634e93/go.mod h1:oM2AQqGJ1AMo4nNqZFYU8xYygSBZkW2hmdJ7n4yjedE=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/toml v1.1.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
@@ -78,6 +78,8 @@ github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnht
 github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk=
 github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE=
 github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk=
+github.com/coreos/go-oidc/v3 v3.2.0 h1:2eR2MGR7thBXSQ2YbODlF0fcmgtliLCfr9iX6RW11fc=
+github.com/coreos/go-oidc/v3 v3.2.0/go.mod h1:rEJ/idjfUyfkBit1eI1fvyr+64/g9dcKpAm8MJMesvo=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -431,8 +433,8 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
 github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals=
-github.com/stretchr/testify v1.7.5 h1:s5PTfem8p8EbKQOctVV53k6jCJt3UX4IEJzwh+C324Q=
-github.com/stretchr/testify v1.7.5/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
+github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk=
+github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tevino/abool v1.2.0 h1:heAkClL8H6w+mK5md9dzsuohKeXHUpY7Vw0ZCKW+huA=
 github.com/tevino/abool v1.2.0/go.mod h1:qc66Pna1RiIsPa7O4Egxxs9OqkuxDX55zznh9K07Tzg=
@@ -556,6 +558,7 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
+golang.org/x/net v0.0.0-20200505041828-1ed23360d12c/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
@@ -864,6 +867,8 @@ gopkg.in/ini.v1 v1.62.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ini.v1 v1.66.6 h1:LATuAqN/shcYAOkv3wl2L4rkaKqkcgTBQjOyYDvcPKI=
 gopkg.in/ini.v1 v1.66.6/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w=
+gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

k8s/client/netclient-daemonset.yaml

@@ -0,0 +1,33 @@
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  name: netclient
+  labels:
+    app: netclient
+spec:
+  selector:
+    matchLabels:
+      app: netclient
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: netclient
+    spec:
+      hostNetwork: true
+      containers:
+      - name: netclient
+        image: gravitl/netclient:v0.14.5
+        env:
+        - name: TOKEN
+          value: "TOKEN_VALUE"
+        volumeMounts:
+        - mountPath: /etc/netclient
+          name: etc-netclient
+        securityContext:
+          privileged: true
+  volumes:
+  - hostPath:
+      path: /etc/netclient
+      type: DirectoryOrCreate
+    name: etc-netclient

k8s/client/netclient.yaml

@@ -0,0 +1,44 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netclient
+  labels:
+    app: netclient
+spec:
+  selector:
+    matchLabels:
+      app: netclient
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: netclient
+    spec:
+      hostNetwork: true
+      # affinity:
+      #   nodeAffinity:
+      #     preferredDuringSchedulingIgnoredDuringExecution:
+      #     - weight: 1
+      #       preference:
+      #         matchExpressions:
+      #         - key: <node label>
+      #           operator: In
+      #           values:
+      #           - "<node label value>"
+      containers:
+      - name: netclient
+        image: gravitl/netclient:v0.14.5
+        env:
+        - name: TOKEN
+          value: "TOKEN_VALUE"
+        volumeMounts:
+        - mountPath: /etc/netclient
+          name: etc-netclient
+        securityContext:
+          privileged: true
+      volumes:
+      - hostPath:
+          path: /etc/netclient
+          type: DirectoryOrCreate
+        name: etc-netclient

k8s/server/README.md

@@ -0,0 +1,117 @@
+# Netmaker K8S YAML Templates - Run Netmaker on Kubernetes
+
+This will walk you through setting up a highly-available netmaker deployment on Kubernetes. Note: do not attempt to control networking on a Kubernetes cluster where Netmaker is deployed. This can result in circular logic and unreachable networks! Typically, a cluster should be designated as a "control" cluster.
+
+You may want a more simple Kubernetes setup. We recommend [this community project](https://github.com/geragcp/netmaker-k3s). It is specific to K3S, but should be editable to work on most Kubernetes distributions.
+
+### 0. Prerequisites
+
+Your cluster must meet a few conditions to host a netmaker server. Primarily:  
+a) **Nodes:** You must have at least 3 worker nodes available for Netmaker to deploy. Netmaker nodes have anti-affinity and will not deploy on the same kubernetes node.  
+b) **Storage:** RWX and RWO storage classes must be available  
+c) **Ingress:** Ingress must be configured with certs. Nginx + LetsEncrypt configs are provided by default. In addition, Traefik allows special ingress for TCP that may be preferable for HA MQTT. Additionally, be sure to have a wildcard DNS entry for use with Ingress/Netmaker  
+e) **Helm:** For our Postgresql installation we rely on a helm chart, so you must have helm installed and configured.  
+d) **MQ Broker Considerations:** Our method uses a NodePort for MQ. This can be used either with or without an external load balancer configured. If deploying without a load balancer, you must specify a node to host MQ, and this will not be HA. If using a load balancer, be aware that LB configuration could lead MQ connections to be lost. If this happens, on the client you will see a log like "unable to connect to broker, retrying ..." In either case DNS must be configured to point broker.domain either to the LB or directly to the hosting node. Finally, you can use a special TCPIngressRoute if Traefik is your Ingress provider ([see this repo](https://github.com/geragcp/netmaker-k3s) for an example). This is ideal, but is not a standard k8s object, so we avoid it to make installations possible across an array of k8s configurations.  
+
+Assuming you are prepared for the above, we can begin to deploy Netmaker.  
+
+### 1. Create Namespace
+`kubectl create ns netmaker`  
+`kubectl config set-context --current --namespace=netmaker`  
+
+### 2. Deploy Database
+
+Netmaker can use sqlite, postgres, or rqlite as a backing database. For HA, we recommend using Postgres, as Bitnami provides a reliable helm chart for deploying an HA pqsql cluster.
+  
+Follow these instructions:  
+https://github.com/bitnami/charts/tree/master/bitnami/postgresql-ha  
+  
+`helm install postgres bitnami/postgresql`
+  
+Once completed, retrieve the password to access postgres:
+
+`kubectl get secret --namespace netmaker postgres-postgresql -o jsonpath="{.data.postgres-password}" | base64 -d`  
+
+### 3. Deploy MQTT
+
+Based on the prerequisites, you will have either an external load balancer, traefik + TCPIngressRoute, or neither:  
+a) **External LB:** Configure your LB to load balance 31883/tcp to your nodes. This will be for MQ traffic. **Important:** make sure that latency is not significant between clients and MQ. In testing, we found that some load balancers introduce too much latency, causing MQ to be unuseable.  
+b) **Traefik:** If you have Traefik as your Ingress provider, create a TCPIngressRoute from 443 to mq service on port 8883. [Example Here](https://github.com/geragcp/netmaker-k3s/blob/main/08-ingress.yaml)  
+c) **Neither (default):** Choose a cluster node to house MQTT and then run the following:  
+
+c.a) `kubectl label node <your node name> mqhost=true`  
+
+c.b) `sed -i 's/MQ_NODE_NAME/<your node name>/g' mosquitto.yaml`  
+
+(If you are not using option C, comment out the pod affinity section from mosquitto.yaml.)
+
+You also need an RWX storage class. Run the following to input your RWX storage class:
+
+`sed -i 's/RWX_STORAGE_CLASS/<your storage class name>/g' mosquitto.yaml`
+
+Now, apply the file:
+
+`kubectl apply -f mosquitto.yaml`  
+
+MQ should be in CrashLoopBackoff until Netmaker is deployed. If it's in pending state, check the pvc or the pod status (node selector may be incorrect).  
+
+### 4. Deploy Netmaker Server
+
+Make sure Wildcard DNS is set up for a netmaker subdomain, for instance: nm.mydomain.com. If you do not wish to use wildcard, edit the YAML file directly. Note you will need entries for broker.domain, api.domain, and dashboard.domain.  
+
+`sed -i 's/NETMAKER_SUBDOMAIN/<your subdomain>/g' netmaker-server.yaml`  
+
+Next, enter your postgres info, including the name of your postgres deployment and the password you retrieved above.  
+  
+`sed -i 's/DB_NAME/<postgres helm name>/g' netmaker-server.yaml`  
+  
+`sed -i 's/DB_PASS/<postgres helm password>/g' netmaker-server.yaml`  
+
+Next, choose a secret password for your Netmaker API:
+
+`sed -i 's/REPLACE_MASTER_KEY/<super secret password>/g' netmaker-server.yaml`  
+  
+Finally, you will need to create an Ingress object for your Netmaker API. An example is included in the YAML file for nginx + letsencrypt. You may use/modify this example, or create your own ingress which routes to the netmaker-rest service on port 8081. But make sure to deploy Ingress before moving on!  
+
+Now, apply the file:  
+
+`kubectl apply -f netmaker-server.yaml`  
+
+Most likely, MQ will require a restart at this point:  
+
+`kubectl delete pod mosquitto-<pod name>`  
+
+When successful, all netmaker pods should display the following log:  
+
+`[netmaker] 2022-00-00 00:00:00 successfully connected to mq broker`  
+
+
+### 5. Deploy Netmaker UI
+
+Much like above, you must make sure wildcard DNS is configured and make considerations for Ingress. Once again, add in your subdomain:  
+  
+`sed -i 's/NETMAKER_SUBDOMAIN/<your subdomain>/g' netmaker-ui.yaml`  
+
+Again, Ingress is commented out. If you are using Nginx + LetsEncrypt, you can uncomment and use the yaml. Otherwise, set up Ingress manually.  
+
+`kubectl apply -f netmaker-ui.yaml`  
+
+At this point, you should be able to reach the server at domain.yourdomain and start setting up your networks.  
+  
+### Troubleshooting
+
+Sometimes, the server has a hard time connecting to MQ using the self-generated certs on the first try. If this happens, try the following:
+
+1. restart MQ: `kubectl delete pod <mq pod name>`  
+2. restart netmaker pods:  
+2.a.  `kubectl scale sts netmaker --replicas=0`  
+2.b.  `kubectl delete pods netmaker-0 netmaker-1 netmaker-2`  
+2.c.  `kubectl scale sts netmaker --replicas=3`  
+  
+In addition, try deleting the certs in MQ before running the above:
+
+1. `kubectl exec -it mosquitto-<pod name> /bin/sh`
+2. `rm mosquitto/certs/*`
+3. `exit`
+4. `kubectl delete pod mosquitto-<pod name>`
+2. `kubectl scale sts netmaker --replicas=0` (wait until pods are down) `kubectl scale sts netmaker --replicas=3`

k8s/server/mosquitto.yaml

@@ -0,0 +1,151 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: mosquitto
+spec:
+  progressDeadlineSeconds: 600
+  replicas: 1
+  selector:
+    matchLabels:
+      app.kubernetes.io/instance: mosquitto
+      app.kubernetes.io/name: mosquitto
+  strategy:
+    type: Recreate
+  template:
+    metadata:
+      labels:
+        app.kubernetes.io/instance: mosquitto
+        app.kubernetes.io/name: mosquitto
+    spec:
+      affinity:
+        nodeAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+            nodeSelectorTerms:
+            - matchExpressions:
+              - key: mqhost
+                operator: In
+                values:
+                - "true"
+      containers:
+      - image: eclipse-mosquitto:2.0.11-openssl
+        imagePullPolicy: IfNotPresent
+        livenessProbe:
+          failureThreshold: 3
+          periodSeconds: 10
+          successThreshold: 1
+          tcpSocket:
+            port: 8883
+          timeoutSeconds: 1
+        name: mosquitto
+        ports:
+        - containerPort: 1883        
+          name: mqtt
+          protocol: TCP
+        - containerPort: 8883        
+          name: mqtt2
+          protocol: TCP
+        readinessProbe:
+          failureThreshold: 3
+          periodSeconds: 10
+          successThreshold: 1
+          tcpSocket:
+            port: 8883
+          timeoutSeconds: 1
+        resources: {}
+        startupProbe:
+          failureThreshold: 30
+          periodSeconds: 5
+          successThreshold: 1
+          tcpSocket:
+            port: 8883
+          timeoutSeconds: 1
+        terminationMessagePath: /dev/termination-log
+        terminationMessagePolicy: File
+        volumeMounts:
+        - mountPath: /mosquitto/config/mosquitto.conf
+          name: mosquitto-config
+          subPath: mosquitto.conf
+        - mountPath: /mosquitto/certs
+          name: shared-certs
+      dnsPolicy: ClusterFirst
+      restartPolicy: Always
+      terminationGracePeriodSeconds: 30
+      volumes:
+      - configMap:
+          name: mosquitto-config
+        name: mosquitto-config
+      - name: shared-certs
+        persistentVolumeClaim:
+          claimName: shared-certs-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: mq
+  namespace: netmaker
+spec:
+  ports:
+  - name: mqtt
+    port: 1883
+    protocol: TCP
+    targetPort: mqtt
+  - name: mqtt2
+    port: 8883
+    protocol: TCP
+    targetPort: mqtt2    
+  selector:
+    app.kubernetes.io/instance: mosquitto
+    app.kubernetes.io/name: mosquitto
+  sessionAffinity: None
+---
+apiVersion: v1
+data:
+  mosquitto.conf: |
+    per_listener_settings true
+    listener 8883
+    allow_anonymous false
+    require_certificate true
+    use_identity_as_username true
+    cafile /mosquitto/certs/root.pem
+    certfile /mosquitto/certs/server.pem
+    keyfile /mosquitto/certs/server.key
+    listener 1883
+    allow_anonymous true
+kind: ConfigMap
+metadata:
+  labels:
+    app.kubernetes.io/instance: mosquitto
+    app.kubernetes.io/name: mosquitto
+  name: mosquitto-config
+  namespace: netmaker
+---
+kind: PersistentVolumeClaim
+apiVersion: v1
+metadata:
+  name: shared-certs-pvc
+spec:
+  storageClassName: RWX_STORAGE_CLASS
+  accessModes:
+    - ReadWriteMany
+  resources:
+    requests:
+      storage: 100Mi
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+  name: 'netmaker-mqtt'
+spec:
+  externalTrafficPolicy: Local
+  type: NodePort
+  selector:
+    app.kubernetes.io/instance: mosquitto
+    app.kubernetes.io/name: mosquitto
+  ports:
+  - port: 31883
+    nodePort: 31883
+    protocol: TCP
+    targetPort: 8883
+    name: nm-mqtt

k8s/server/netmaker-server.yaml

@@ -0,0 +1,228 @@
+apiVersion: apps/v1
+kind: StatefulSet
+metadata:
+  labels:
+    app: netmaker
+  name: netmaker
+spec:
+  replicas: 3
+  serviceName: netmaker-headless
+  selector:
+    matchLabels:
+      app: netmaker
+  template:
+    metadata:
+      labels:
+        app: netmaker
+    spec:
+      initContainers:
+      - name: init-sysctl
+        image: busybox
+        imagePullPolicy: IfNotPresent
+        command: ["/bin/sh", "-c"]
+        args: ["sysctl -w net.ipv4.ip_forward=1 && sysctl -w net.ipv4.conf.all.src_valid_mark=1 && sysctl -w net.ipv6.conf.all.disable_ipv6=0 && sysctl -w net.ipv6.conf.all.forwarding=1"]
+        securityContext:
+          privileged: true
+      dnsPolicy: ClusterFirstWithHostNet
+      affinity:
+        podAntiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution:
+          - labelSelector:
+              matchExpressions:
+              - key: app
+                operator: In
+                values:
+                - netmaker
+            topologyKey: "kubernetes.io/hostname"
+      containers:
+      - env:
+        - name: NODE_ID
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: metadata.name
+        - name: SERVER_NAME
+          value: broker.NETMAKER_SUBDOMAIN
+        - name: SERVER_API_CONN_STRING
+          value: api.NETMAKER_SUBDOMAIN:443
+        - name: SERVER_HTTP_HOST
+          value: api.NETMAKER_SUBDOMAIN
+        - name: API_PORT
+          value: "8081"
+        - name: WG_QUICK_USERSPACE_IMPLEMENTATION
+          value: wireguard-go
+        - name: DNS_MODE
+          value: "off"
+        - name: CLIENT_MODE
+          value: "on"
+        - name: DISPLAY_KEYS
+          value: "on"
+        - name: DATABASE
+          value: postgres
+        - name: SQL_HOST
+          value: "DB_NAME-postgresql" 
+        - name: SQL_PORT
+          value: "5432"
+        - name: SQL_DB
+          value: "postgres"
+        - name: SQL_USER
+          value: "postgres"
+        - name: SQL_PASS
+          value: "DB_PASS"
+        - name: MASTER_KEY
+          value: REPLACE_MASTER_KEY
+        - name: CORS_ALLOWED_ORIGIN
+          value: '*'
+        - name: MQ_HOST
+          value: "mq"
+        - name: MQ_PORT
+          value: "31883"
+        - name: MQ_SERVER_PORT
+          value: "1883"
+        - name: PLATFORM
+          value: "Kubernetes"
+        - name: VERBOSITY
+          value: "3"
+        image: gravitl/netmaker:v0.14.5
+        imagePullPolicy: Always
+        name: netmaker
+        ports:
+        - containerPort: 8081
+          protocol: TCP
+        - containerPort: 31821
+          protocol: UDP
+        - containerPort: 31822
+          protocol: UDP
+        - containerPort: 31823
+          protocol: UDP
+        - containerPort: 31824
+          protocol: UDP
+        - containerPort: 31825
+          protocol: UDP
+        - containerPort: 31826
+          protocol: UDP
+        - containerPort: 31827
+          protocol: UDP
+        - containerPort: 31828
+          protocol: UDP
+        - containerPort: 31829
+          protocol: UDP
+        - containerPort: 31830
+          protocol: UDP
+        resources: {}
+        securityContext:
+          capabilities:
+            add:
+            - NET_ADMIN
+            - NET_RAW
+            - SYS_MODULE
+        volumeMounts:
+        - mountPath: /etc/netmaker/
+          name: shared-certs
+      volumes:
+      - name: shared-certs
+        persistentVolumeClaim:
+          claimName: shared-certs-pvc
+---
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+  name: 'netmaker-wireguard'
+spec:
+  externalTrafficPolicy: Local
+  type: NodePort
+  ports:
+  - port: 31821
+    nodePort: 31821
+    protocol: UDP
+    targetPort: 31821
+    name: wg-iface-31821
+  - port: 31822
+    nodePort: 31822
+    protocol: UDP
+    targetPort: 31822
+    name: wg-iface-31822
+  - port: 31823
+    nodePort: 31823
+    protocol: UDP
+    targetPort: 31823
+    name: wg-iface-31823
+  - port: 31824
+    nodePort: 31824
+    protocol: UDP
+    targetPort: 31824
+    name: wg-iface-31824
+  - port: 31825
+    nodePort: 31825
+    protocol: UDP
+    targetPort: 31825
+    name: wg-iface-31825
+  - port: 31826
+    nodePort: 31826
+    protocol: UDP
+    targetPort: 31826
+    name: wg-iface-31826
+  - port: 31827
+    nodePort: 31827
+    protocol: UDP
+    targetPort: 31827
+    name: wg-iface-31827
+  - port: 31828
+    nodePort: 31828
+    protocol: UDP
+    targetPort: 31828
+    name: wg-iface-31828
+  - port: 31829
+    nodePort: 31829
+    protocol: UDP
+    targetPort: 31829
+    name: wg-iface-31829
+  - port: 31830
+    nodePort: 31830
+    protocol: UDP
+    targetPort: 31830
+    name: wg-iface-31830
+  selector:
+    app: 'netmaker'
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: 'netmaker-rest'
+spec:
+  ports:
+  - name: rest
+    port: 8081
+    protocol: TCP
+    targetPort: 8081
+  selector:
+    app: 'netmaker'
+  sessionAffinity: None
+  type: ClusterIP
+# ---
+# apiVersion: networking.k8s.io/v1
+# kind: Ingress
+# metadata:
+#   name: nm-api-ingress-nginx
+#   annotations:
+#     nginx.ingress.kubernetes.io/rewrite-target: /
+#     cert-manager.io/cluster-issuer: "letsencrypt-nginx"
+#     nginx.ingress.kubernetes.io/ssl-redirect: 'true'
+# spec:
+#   ingressClassName: nginx
+#   tls:
+#   - hosts:
+#     - api.NETMAKER_SUBDOMAIN
+#     secretName: nm-api-tls
+#   rules:
+#   - host: api.NETMAKER_SUBDOMAIN
+#     http:
+#       paths:
+#       - path: /
+#         pathType: Prefix
+#         backend:
+#           service:
+#             name: netmaker-rest
+#             port:
+#               number: 8081

k8s/server/netmaker-ui.yaml

@@ -0,0 +1,64 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: netmaker-ui
+spec:
+  replicas: 2
+  selector:
+    matchLabels:
+      app: netmaker-ui
+  template:
+    metadata:
+      labels:
+        app: netmaker-ui
+    spec:
+      containers:
+      - name: netmaker-ui
+        image: gravitl/netmaker-ui:v0.14.5
+        ports:
+        - containerPort: 443
+        env:
+        - name: BACKEND_URL
+          value: 'https://api.NETMAKER_SUBDOMAIN'
+      terminationGracePeriodSeconds: 15
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: 'netmaker-ui'
+spec:
+  ports:
+  - port: 80
+    protocol: TCP
+    targetPort: 80
+  selector:
+    app: 'netmaker-ui'
+  sessionAffinity: None
+  type: 'ClusterIP'
+# ---
+# apiVersion: networking.k8s.io/v1
+# kind: Ingress
+# metadata:
+#   name: nm-ui-ingress-nginx
+#   annotations:
+#     nginx.ingress.kubernetes.io/rewrite-target: /
+#     cert-manager.io/cluster-issuer: "letsencrypt-nginx"
+#     nginx.ingress.kubernetes.io/ssl-redirect: 'true'
+# spec:
+#   ingressClassName: nginx
+#   tls:
+#   - hosts:
+#     - dashboard.NETMAKER_SUBDOMAIN
+#     secretName: nm-ui-tls
+#   rules:
+#   - host: dashboard.NETMAKER_SUBDOMAIN
+#     http:
+#       paths:
+#       - path: /
+#         pathType: Prefix
+#         backend:
+#           service:
+#             name: netmaker-ui
+#             port:
+#               number: 80

kube/components/mongo-statefulset.yaml

@@ -1,59 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: mongo
-  labels:
-    name: mongo
-spec:
-  ports:
-    - port: 27017
-      targetPort: 27017
-  clusterIP: None
-  selector:
-    role: mongo
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mongo
-spec:
-  serviceName: "mongo"
-  replicas: 1
-  selector:
-    matchLabels:
-      role: mongo
-  template:
-    metadata:
-      labels:
-        app: mongo
-        role: mongo
-    spec:
-      containers:
-      - name: mongo
-        image: mongo
-        env:
-          - name: MONGO_INITDB_ROOT_USERNAME
-            value: mongoadmin
-          - name: MONGO_INITDB_ROOT_PASSWORD
-            value: mongopass
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: mongovol
-          mountPath: /data/db
-      volumes:
-      - name: mongovol
-        persistentVolumeClaim:
-          claimName: mongodb-pvc
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: mongodb-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 7Gi
-  storageClassName: microk8s-hostpath

kube/components/netclient-template.yaml.backup

@@ -1,62 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: netclient
-  labels:
-    app: netclient
-spec:
-  selector:
-    matchLabels:
-      app: netclient
-  template:
-    metadata:
-      labels:
-        app: netclient
-    spec:
-      hostNetwork: true
-      containers:
-      - name: netclient
-        image: gravitl/netclient:v0.5.8
-        command: ['bash', '-c', "/root/netclient join -t $ACCESS_TOKEN --daemon off --name $(echo $NODE_NAME| sed -e s/.$NETWORK//); while true; do /root/netclient checkin -n $NETWORK; sleep $SLEEP; done"]
-        env:
-        - name: ACCESS_TOKEN
-          value: "ACCESS_TOKEN_VALUE"
-        - name: NETWORK
-          value: "microk8s"
-        - name: SLEEP
-          value: "30"
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        volumeMounts:
-        - mountPath: /etc/netclient
-          name: etc-netclient
-        - mountPath: /usr/bin/wg
-          name: wg
-        - mountPath: /var/run/dbus/system_bus_socket
-          name: systemd-bus-socket
-        securityContext:
-          privileged: true
-          #capabilities:
-          #  add:
-          #  - ["NET_ADMIN","SYS_ADMIN","SYS_MODULE"]
-      volumes:
-      - hostPath:
-          path: /etc/netclient
-          type: DirectoryOrCreate
-        name: etc-netclient
-      - hostPath:
-          path: /usr/bin/wg
-          type: File
-        name: wg 
-      - hostPath:
-          path: /usr/bin/resolvectl
-          type: File
-        name: resolvectl
-      - hostPath:
-          path: /var/run/dbus/system_bus_socket
-          type: ""
-        name: systemd-bus-socket
-
-

kube/components/netmaker-api.yaml

@@ -1,87 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-api
-  labels:
-    app: netmaker-api
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-api
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-api
-    spec:
-      containers:
-      - name: netmaker-api
-        image: gravitl/netmaker:v0.5.7
-        ports:
-        - containerPort: 8081
-        volumeMounts:
-        - name: nm-pvc
-          mountPath: /root/config/dnsconfig
-        env:
-        - name: SERVER_API_CONN_STRING
-          value: "api.nm.k8s.gravitl.com:443"
-        - name: COREDNS_ADDR
-          value: "netmaker-dns"
-        - name: SERVER_HTTP_HOST
-          value: "api.nm.k8s.gravitl.com"
-        - name: API_PORT
-          value: "8081"
-        - name: AGENT_BACKEND
-          value: "off"
-        - name: CLIENT_MODE
-          value: "off"
-        - name: DNS_MODE
-          value: "on"
-        - name: MASTER_KEY
-          value: "Unkn0wn!"
-        - name: MASTER_KEY
-          value: "secretkey"
-        - name: CORS_ALLOWED_ORIGIN
-          value: "*"
-        - name: DISABLE_REMOTE_IP_CHECK
-          value: "on"
-        - name: MONGO_ADMIN
-          value: "mongoadmin"
-        - name: MONGO_PASS
-          value: "mongopass"
-        - name: MONGO_HOST
-          value: "mongo-0.mongo"
-        - name: MONGO_OPTS
-          value: "/?authSource=admin"
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: nm-pvc
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 128Mi
-  storageClassName: microk8s-hostpath
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-api
-  name: netmaker-api
-spec:
-  ports:
-  - port: 8081
-    protocol: TCP
-    targetPort: 8081
-  selector:
-    app: netmaker-api
-  sessionAffinity: None
-  type: ClusterIP

kube/components/netmaker-backend.yaml

@@ -1,98 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-backend
-  labels:
-    app: netmaker-backend
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-backend
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-backend
-    spec:
-      containers:
-      - name: netmaker-backend
-        image: gravitl/netmaker:v0.5.7
-        ports:
-        - containerPort: 8081
-        volumeMounts:
-        - name: nm-pvc
-          mountPath: /root/config/dnsconfig
-        env:
-        - name: SERVER_API_CONN_STRING
-          value: "api.nm.k8s.gravitl.com:443"
-        - name: COREDNS_ADDR
-          value: "10.152.183.53"
-        - name: SERVER_HTTP_HOST
-          value: "api.k8s.gravitl.com"
-        - name: API_PORT
-          value: "8081"
-        - name: CLIENT_MODE
-          value: "off"
-        - name: MASTER_KEY
-          value: "Unkn0wn!"
-        - name: MASTER_KEY
-          value: "secretkey"
-        - name: CORS_ALLOWED_ORIGIN
-          value: "*"
-        - name: DISABLE_REMOTE_IP_CHECK
-          value: "on"
-        - name: MONGO_ADMIN
-          value: "mongoadmin"
-        - name: MONGO_PASS
-          value: "mongopass"
-        - name: MONGO_HOST
-          value: "mongo-0.mongo"
-        - name: MONGO_OPTS
-          value: "/?authSource=admin"
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: nm-pvc
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 128Mi
-  storageClassName: microk8s-hostpath
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-  name: netmaker-api
-spec:
-  ports:
-  - port: 8081
-    protocol: TCP
-    targetPort: 8081
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-spec:
-  ports:
-  - port: 443
-    protocol: TCP
-    targetPort: 443
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP

kube/components/netmaker-dns.yaml

@@ -1,71 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-dns
-  labels:
-    app: netmaker-dns
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-dns
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-dns
-    spec:
-      containers:
-      - args:
-        - -conf
-        - /root/dnsconfig/Corefile
-        image: coredns/coredns
-        imagePullPolicy: Always
-        name: netmaker-dns
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /root/dnsconfig
-          name: nm-pvc
-          readOnly: true
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - all
-      dnsPolicy: "None"
-      dnsConfig:
-        nameservers:
-          - 127.0.0.1
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-dns
-  name: netmaker-dns
-spec:
-  ports:
-  - port: 53
-    protocol: UDP
-    targetPort: 53
-    name: udp
-  - port: 53
-    protocol: TCP
-    targetPort: 53
-    name: tcp
-  selector:
-    app: netmaker-dns
-  sessionAffinity: None
-  type: ClusterIP
-  clusterIP: 10.152.183.53

kube/components/netmaker-grpc.yaml

@@ -1,82 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-grpc
-  labels:
-    app: netmaker-grpc
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-grpc
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-grpc
-    spec:
-      containers:
-      - name: netmaker-grpc
-        image: gravitl/netmaker:v0.5.7
-        ports:
-        - containerPort: 443
-        volumeMounts:
-        - name: nm-pvc
-          mountPath: /root/dnsconfig
-        env:
-        - name: SERVER_API_CONN_STRING
-          value: "api.nm.k8s.gravitl.com:443"
-        - name: SERVER_GRPC_CONN_STRING
-          value: "grpc.nm.k8s.gravitl.com:443"
-        - name: COREDNS_ADDR
-          value: "netmaker-dns"
-        - name: GRPC_SSL
-          value: "on"
-        - name: CLIENT_MODE
-          value: "off"
-        - name: DNS_MODE
-          value: "on"
-        - name: MASTER_KEY
-          value: "Unkn0wn!"
-        - name: SERVER_GRPC_WIREGUARD
-          value: "off"
-        - name: MASTER_KEY
-          value: "secretkey"
-        - name: CORS_ALLOWED_ORIGIN
-          value: "*"
-        - name: DISABLE_REMOTE_IP_CHECK
-          value: "on"
-        - name: MONGO_ADMIN
-          value: "mongoadmin"
-        - name: MONGO_PASS
-          value: "mongopass"
-        - name: MONGO_HOST
-          value: "mongo-0.mongo"
-        - name: MONGO_OPTS
-          value: "/?authSource=admin"
-        - name: SERVER_GRPC_HOST
-          value: "0.0.0.0"
-        - name: GRPC_PORT
-          value: "443"
-        - name: REST_BACKEND
-          value: "off"
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-grpc
-  name: netmaker-grpc
-spec:
-  ports:
-  - port: 443
-    protocol: TCP
-    targetPort: 443
-  selector:
-    app: netmaker-grpc
-  sessionAffinity: None
-  type: ClusterIP
-

kube/components/netmaker-ingress-api.yaml

@@ -1,25 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  annotations:
-    kubernetes.io/ingress.class: "traefik"
-    kubernetes.io/ingress.allow-http: "false"    
-    traefik.ingress.kubernetes.io/redirect-entry-point: https
-    traefik.ingress.kubernetes.io/redirect-permanent: "true"
-    traefik.ingress.kubernetes.io/rule-type: "PathPrefixStrip"
-    cert-manager.io/cluster-issuer: wildcard-issuer
-  name: nm-api-ingress
-  namespace: netmaker
-spec:
-  rules:
-  - host: api.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        backend:
-          serviceName: netmaker
-          servicePort: 8081
-  tls:
-  - hosts: 
-    - api.NETMAKER_BASE_DOMAIN
-    secretName: cert-nm-api

kube/components/netmaker-ingress-frontend.yaml

@@ -1,25 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  annotations:
-    kubernetes.io/ingress.class: "traefik"
-    kubernetes.io/ingress.allow-http: "false"    
-    traefik.ingress.kubernetes.io/redirect-entry-point: https
-    traefik.ingress.kubernetes.io/redirect-permanent: "true"
-    traefik.ingress.kubernetes.io/rule-type: "PathPrefixStrip"
-    cert-manager.io/cluster-issuer: wildcard-issuer
-  name: nm-ui-ingress
-  namespace: netmaker
-spec:
-  rules:
-  - host: dashboard.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        backend:
-          serviceName: netmaker-ui
-          servicePort: 80
-  tls:
-  - hosts: 
-    - dashboard.NETMAKER_BASE_DOMAIN
-    secretName: cert-nm-ui

kube/components/netmaker-ingress-grpc.yaml

@@ -1,17 +0,0 @@
-apiVersion: extensions/v1beta1
-kind: Ingress
-metadata:
-  annotations:
-    kubernetes.io/ingress.class: "traefik"
-    ingress.kubernetes.io/protocol: "h2c"
-  name: nm-grpc-ingress
-  namespace: netmaker
-spec:
-  rules:
-  - host: grpc.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        backend:
-          serviceName: netmaker-grpc
-          servicePort: 50051

kube/components/netmaker-template.yaml.backup

@@ -1,345 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: mongo
-  labels:
-    name: mongo
-spec:
-  ports:
-    - port: 27017
-      targetPort: 27017
-  clusterIP: None
-  selector:
-    role: mongo
----
-apiVersion: apps/v1
-kind: StatefulSet
-metadata:
-  name: mongo
-spec:
-  serviceName: "mongo"
-  replicas: 1
-  selector:
-    matchLabels:
-      role: mongo
-  template:
-    metadata:
-      labels:
-        app: mongo
-        role: mongo
-    spec:
-      containers:
-      - name: mongo
-        image: mongo
-        env:
-          - name: MONGO_INITDB_ROOT_USERNAME
-            value: mongoadmin
-          - name: MONGO_INITDB_ROOT_PASSWORD
-            value: mongopass
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: mongovol
-          mountPath: /data/db
-      volumes:
-      - name: mongovol
-        persistentVolumeClaim:
-          claimName: mongodb-pvc
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: mongodb-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 7Gi
-  storageClassName: microk8s-hostpath
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-backend
-  labels:
-    app: netmaker-backend
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-backend
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-backend
-    spec:
-      containers:
-      - name: netmaker-backend
-        image: gravitl/netmaker:v0.5.7
-        ports:
-        - containerPort: 8081
-        volumeMounts:
-        - name: nm-pvc
-          mountPath: /root/config/dnsconfig
-        env:
-        - name: SERVER_API_CONN_STRING
-          value: "api.NETMAKER_BASE_DOMAIN:443"
-        - name: COREDNS_ADDR
-          value: "10.152.183.53"
-        - name: SERVER_HTTP_HOST
-          value: "api.NETMAKER_BASE_DOMAIN"
-        - name: API_PORT
-          value: "8081"
-        - name: CLIENT_MODE
-          value: "off"
-        - name: MASTER_KEY
-          value: "Unkn0wn!"
-        - name: MASTER_KEY
-          value: "secretkey"
-        - name: CORS_ALLOWED_ORIGIN
-          value: "*"
-        - name: DISABLE_REMOTE_IP_CHECK
-          value: "on"
-        - name: MONGO_ADMIN
-          value: "mongoadmin"
-        - name: MONGO_PASS
-          value: "mongopass"
-        - name: MONGO_HOST
-          value: "mongo-0.mongo"
-        - name: MONGO_OPTS
-          value: "/?authSource=admin"
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: nm-pvc
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 128Mi
-  storageClassName: microk8s-hostpath
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-  name: netmaker-api
-spec:
-  ports:
-  - port: 8081
-    protocol: TCP
-    targetPort: 8081
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-  name: netmaker-grpc
-spec:
-  ports:
-  - port: 443
-    protocol: TCP
-    targetPort: 443
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-dns
-  labels:
-    app: netmaker-dns
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-dns
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-dns
-    spec:
-      containers:
-      - args:
-        - -conf
-        - /root/dnsconfig/Corefile
-        image: coredns/coredns
-        imagePullPolicy: Always
-        name: netmaker-dns
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /root/dnsconfig
-          name: nm-pvc
-          readOnly: true
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - all
-      dnsPolicy: "None"
-      dnsConfig:
-        nameservers:
-          - 127.0.0.1
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-dns
-  name: netmaker-dns
-spec:
-  ports:
-  - port: 53
-    protocol: UDP
-    targetPort: 53
-    name: udp
-  - port: 53
-    protocol: TCP
-    targetPort: 53
-    name: tcp
-  selector:
-    app: netmaker-dns
-  sessionAffinity: None
-  type: ClusterIP
-  clusterIP: 10.152.183.53
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-ui
-  labels:
-    app: netmaker-ui
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-ui
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-ui
-    spec:
-      containers:
-      - name: netmaker-ui
-        image: gravitl/netmaker-ui:v0.5
-        ports:
-        - containerPort: 80
-        env:
-        - name: BACKEND_URL
-          value: "https://api.NETMAKER_BASE_DOMAIN"
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-ui
-  name: netmaker-ui
-spec:
-  ports:
-  - port: 80
-    protocol: TCP
-    targetPort: 80
-  selector:
-    app: netmaker-ui
-  sessionAffinity: None
-  type: ClusterIP
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-api-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: public
-  tls:
-  - hosts:
-    - api.NETMAKER_BASE_DOMAIN
-    secretName: nm-api-tls
-  rules:
-  - host: api.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-api
-            port:
-              number: 8081
-
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-grpc-ingress-nginx
-  annotations:
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
-spec:
-  ingressClassName: public
-  tls:
-  - hosts:
-    - grpc.NETMAKER_BASE_DOMAIN
-    secretName: nm-grpc-tls
-  rules:
-  - host: grpc.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-grpc
-            port:
-              number: 443
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-ui-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: public
-  tls:
-  - hosts:
-    - dashboard.NETMAKER_BASE_DOMAIN
-    secretName: nm-ui-tls
-  rules:
-  - host: dashboard.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-ui
-            port:
-              number: 80

kube/components/netmaker-ui.yaml

@@ -1,40 +0,0 @@
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-ui
-  labels:
-    app: netmaker-ui
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-ui
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-ui
-    spec:
-      containers:
-      - name: netmaker-ui
-        image: gravitl/netmaker-ui:v0.5
-        ports:
-        - containerPort: 80
-        env:
-        - name: BACKEND_URL
-          value: "https://api.nm.k8s.gravitl.com"
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-ui
-  name: netmaker-ui
-spec:
-  ports:
-  - port: 80
-    protocol: TCP
-    targetPort: 80
-  selector:
-    app: netmaker-ui
-  sessionAffinity: None
-  type: ClusterIP

kube/components/nm-ingress-api-nginx.yaml

@@ -1,26 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-api-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: public
-  tls:
-  - hosts:
-    - api.NETMAKER_BASE_DOMAIN
-    secretName: nm-api-tls
-  rules:
-  - host: api.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-api
-            port:
-              number: 8081
-

kube/components/nm-ingress-grpc-nginx.yaml

@@ -1,25 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-grpc-ingress-nginx
-  annotations:
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
-spec:
-  ingressClassName: public
-  tls:
-  - hosts:
-    - grpc.NETMAKER_BASE_DOMAIN
-    secretName: nm-grpc-tls-2
-  rules:
-  - host: grpc.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-grpc
-            port:
-              number: 443

kube/components/nm-ingress-ui-nginx.yaml

@@ -1,26 +0,0 @@
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-ui-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: public
-  tls:
-  - hosts:
-    - dashboard.NETMAKER_BASE_DOMAIN
-    secretName: nm-ui-tls
-  rules:
-  - host: dashboard.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-ui
-            port:
-              number: 80
-

kube/netclient-daemonset.yaml

@@ -1,58 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: netclient
-  labels:
-    app: netclient
-spec:
-  selector:
-    matchLabels:
-      app: netclient
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netclient
-    spec:
-      hostNetwork: true
-      containers:
-      - name: netclient
-        image: gravitl/netclient:v0.7.2
-        command: ['bash', '-c', "netclient checkin -n $NETWORK; sleep $SLEEP"]
-        env:
-        - name: ACCESS_TOKEN
-          value: "XXXX"
-        - name: NETWORK
-          value: "YYYY"
-        - name: SLEEP
-          value: 30
-        volumeMounts:
-        - mountPath: /etc/netclient
-          name: etc-netclient
-        - mountPath: /usr/bin/wg
-          name: wg
-        securityContext:
-          privileged: true
-      initContainers:
-      - name: netclient-join
-        image: gravitl/netclient:v0.7.2
-        command: ['bash', '-c', "netclient join -t $ACCESS_TOKEN --daemon off"]
-        env:
-        - name: ACCESS_TOKEN
-          value: "XXXX"
-        volumeMounts:
-        - mountPath: /etc/netclient
-          name: etc-netclient
-        - mountPath: /usr/bin/wg
-          name: wg
-        securityContext:
-          privileged: true
-  volumes:
-  - hostPath:
-      path: /etc/netclient
-      type: DirectoryOrCreate
-    name: etc-netclient
-  - hostPath:
-      path: /usr/bin/wg
-      type: File
-    name: wg

kube/netclient-template-doks-uspace.yaml

@@ -1,43 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: netclient-1
-  labels:
-    app: netclient-1
-spec:
-  selector:
-    matchLabels:
-      app: netclient-1
-  template:
-    metadata:
-      labels:
-        app: netclient-1
-    spec:
-      hostNetwork: true
-      containers:
-      - name: netclient-1
-        image: gravitl/netclient:0.9.2-doks-uspace
-        env:
-        - name: NETCLIENT_ROAMING
-          value: "no"
-        - name: NETCLIENT_PORT
-          value: "51821"
-        - name: NETCLIENT_IS_STATIC
-          value: "yes"
-        - name: NETCLIENT_ENDPOINT
-          valueFrom:
-            fieldRef:
-              fieldPath: status.hostIP
-        - name: TOKEN
-          value: "<token>"
-        volumeMounts:
-        - mountPath: /etc/netclient
-          name: etc-netclient
-        securityContext:
-          privileged: true
-      volumes:
-      - hostPath:
-          path: /etc/netclient
-          type: DirectoryOrCreate
-        name: etc-netclient
-

kube/netclient-template-doks.yaml

@@ -1,93 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: netclient-1
-  labels:
-    app: netclient-1
-spec:
-  selector:
-    matchLabels:
-      app: netclient-1
-  template:
-    metadata:
-      labels:
-        app: netclient-1
-    spec:
-      hostNetwork: true
-      containers:
-      - name: netclient-1
-        image: gravitl/netclient:0.9.2-doks
-        env:
-        - name: NETCLIENT_ROAMING
-          value: "no"
-        - name: NETCLIENT_PORT
-          value: "51821"
-        - name: NETCLIENT_IS_STATIC
-          value: "yes"
-        - name: NETCLIENT_ENDPOINT
-          valueFrom:
-            fieldRef:
-              fieldPath: status.hostIP
-        - name: TOKEN
-          value: "<token>"
-        volumeMounts:
-        - mountPath: /etc/netclient
-          name: etc-netclient
-        - mountPath: /usr/bin/wg
-          name: wg
-        securityContext:
-          privileged: true
-      volumes:
-      - hostPath:
-          path: /etc/netclient
-          type: DirectoryOrCreate
-        name: etc-netclient
-      - hostPath:
-          path: /usr/bin/wg
-          type: File
-        name: wg
----
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: wireguard-controller
-  labels:
-    app: wireguard-controller
-spec:
-  selector:
-    matchLabels:
-      app: wireguard-controller
-  template:
-    metadata:
-      labels:
-        app: wireguard-controller
-    spec:
-      hostNetwork: true
-      containers:
-      - image: gravitl/netclient:0.9.2-doks
-        imagePullPolicy: IfNotPresent
-        name: wg-installer
-        command: ['bash', '-c']
-        args:
-          - while [ 1 ];
-            do if ! command -v wg &> /dev/null;
-            then echo "wireguard not installed, installing";
-            echo 'deb http://deb.debian.org/debian buster-backports main contrib non-free' > /etc/apt/sources.list.d/buster-backports.list;
-            apt update;
-            sudo apt -y install linux-headers-$(uname --kernel-release);
-            apt -y install wireguard wireguard-tools;
-            else echo "wireguard installed";
-            sleep 300;
-            fi;
-            done
-        securityContext:
-          privileged: true
-        volumeMounts:
-        - name: rootfolder
-          mountPath: /
-      volumes:
-      - hostPath:
-          path: /
-          type: ""
-        name: rootfolder
-

kube/netclient-template.yaml

@@ -1,57 +0,0 @@
-apiVersion: apps/v1
-kind: DaemonSet
-metadata:
-  name: netclient
-  labels:
-    app: netclient
-spec:
-  selector:
-    matchLabels:
-      app: netclient
-  template:
-    metadata:
-      labels:
-        app: netclient
-    spec:
-      hostNetwork: true
-      containers:
-      - name: netclient
-        image: gravitl/netclient:v0.5.11
-        command: ['bash', '-c', "/root/netclient join -t $ACCESS_TOKEN --daemon off --name $(echo $NODE_NAME| sed -e s/.$NETWORK//); while true; do /root/netclient checkin --dns on -n $NETWORK; sleep $SLEEP; done"]
-        env:
-        - name: ACCESS_TOKEN
-          value: "ACCESS_TOKEN_VALUE"
-        - name: NETWORK
-          value: "microk8s"
-        - name: SLEEP
-          value: "30"
-        - name: NODE_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: spec.nodeName
-        volumeMounts:
-        - mountPath: /etc/netclient
-          name: etc-netclient
-        - mountPath: /usr/bin/wg
-          name: wg
-        - mountPath: /var/run/dbus/system_bus_socket
-          name: systemd-bus-socket
-        securityContext:
-          privileged: true
-      volumes:
-      - hostPath:
-          path: /etc/netclient
-          type: DirectoryOrCreate
-        name: etc-netclient
-      - hostPath:
-          path: /usr/bin/wg
-          type: File
-        name: wg 
-      - hostPath:
-          path: /usr/bin/resolvectl
-          type: File
-        name: resolvectl
-      - hostPath:
-          path: /var/run/dbus/system_bus_socket
-          type: ""
-        name: systemd-bus-socket

kube/netmaker-template-udp.yaml

@@ -1,353 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: rqlite-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-backend
-  labels:
-    app: netmaker-backend
-spec:
-  nodeSelector:
-    netmaker-server: true
-  selector:
-    matchLabels:
-      app: netmaker-backend
-  replicas: 1
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app: netmaker-backend
-    spec:
-      containers:
-      - name: netmaker-backend
-        image: gravitl/netmaker:0.7.2
-        imagePullPolicy: Always
-        ports:
-        - containerPort: 8081
-        volumeMounts:
-        - name: nm-pvc
-          mountPath: /root/config/dnsconfig
-        - mountPath: /etc/netclient
-          name: etc-netclient
-        - mountPath: /usr/bin/wg
-          name: wg
-        - mountPath: /var/run/dbus/system_bus_socket
-          name: systemd-bus-socket
-        - mountPath: /sys/fs/cgroup
-          name: cgroup
-        - mountPath: /run/systemd/system
-          name: run-systemd
-        - mountPath: /etc/systemd/system
-          name: etc-systemd
-        securityContext:
-          privileged: true
-        env:
-        - name: SERVER_API_CONN_STRING
-          value: "api.NETMAKER_BASE_DOMAIN:443"
-        - name: COREDNS_ADDR
-          value: "10.152.183.53"
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: SERVER_HTTP_HOST
-          value: "api.NETMAKER_BASE_DOMAIN:443"
-        - name: API_PORT
-          value: "8081"
-        - name: CLIENT_MODE
-          value: "off"
-        - name: MASTER_KEY
-          value: "Unkn0wn!"
-        - name: PLATFORM
-          value: "Kubernetes"
-        - name: CORS_ALLOWED_ORIGIN
-          value: "*"
-      - name: rqlite
-        image: rqlite/rqlite
-        ports:
-        - containerPort: 4001
-        - containerPort: 4002
-        volumeMounts:
-        - name: rqlitevol
-          mountPath: /rqlite/file/data
-      volumes:
-      - name: rqlitevol
-        persistentVolumeClaim:
-          claimName: rqlite-pvc
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
-      - hostPath:
-          path: /etc/netclient
-          type: DirectoryOrCreate
-        name: etc-netclient
-      - hostPath:
-          path: /usr/bin/wg
-          type: File
-        name: wg
-      - hostPath:
-          path: /usr/bin/resolvectl
-          type: File
-        name: resolvectl
-      - hostPath:
-          path: /var/run/dbus/system_bus_socket
-          type: ""
-        name: systemd-bus-socket
-      - hostPath:
-          path: /etc/systemd/system
-          type: ""
-        name: etc-systemd
-      - hostPath:
-          path: /run/systemd/system
-          type: ""
-        name: run-systemd
-      - hostPath:
-          path: /sys/fs/cgroup
-          type: ""
-        name: cgroup
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: nm-pvc
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 128Mi
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-  name: netmaker-api
-spec:
-  ports:
-  - port: 8081
-    protocol: TCP
-    targetPort: 8081
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-  name: netmaker-grpc
-spec:
-  ports:
-  - port: 443
-    protocol: TCP
-    targetPort: 443
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-dns
-  labels:
-    app: netmaker-dns
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-dns
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-dns
-    spec:
-      containers:
-      - args:
-        - -conf
-        - /root/dnsconfig/Corefile
-        image: coredns/coredns
-        imagePullPolicy: Always
-        name: netmaker-dns
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /root/dnsconfig
-          name: nm-pvc
-          readOnly: true
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - all
-      dnsPolicy: "None"
-      dnsConfig:
-        nameservers:
-          - 127.0.0.1
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-dns
-  name: netmaker-dns
-spec:
-  ports:
-  - port: 53
-    protocol: UDP
-    targetPort: 53
-    name: udp
-  - port: 53
-    protocol: TCP
-    targetPort: 53
-    name: tcp
-  selector:
-    app: netmaker-dns
-  sessionAffinity: None
-  type: ClusterIP
-  clusterIP: 10.152.183.53
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-ui
-  labels:
-    app: netmaker-ui
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-ui
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-ui
-    spec:
-      containers:
-      - name: netmaker-ui
-        image: gravitl/netmaker-ui:v0.7
-        ports:
-        - containerPort: 80
-        env:
-        - name: BACKEND_URL
-          value: "https://api.NETMAKER_BASE_DOMAIN"
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-ui
-  name: netmaker-ui
-spec:
-  ports:
-  - port: 80
-    protocol: TCP
-    targetPort: 80
-  selector:
-    app: netmaker-ui
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-api-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: nginx
-  tls:
-  - hosts:
-    - api.NETMAKER_BASE_DOMAIN
-    secretName: nm-api-tls
-  rules:
-  - host: api.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-api
-            port:
-              number: 8081
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-grpc-ingress-nginx
-  annotations:
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
-spec:
-  ingressClassName: nginx
-  tls:
-  - hosts:
-    - grpc.NETMAKER_BASE_DOMAIN
-    secretName: nm-grpc-tls
-  rules:
-  - host: grpc.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-grpc
-            port:
-              number: 443
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-ui-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: nginx
-  tls:
-  - hosts:
-    - dashboard.NETMAKER_BASE_DOMAIN
-    secretName: nm-ui-tls
-  rules:
-  - host: dashboard.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-ui
-            port:
-              number: 80

kube/netmaker-template.yaml

@@ -1,311 +0,0 @@
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: rqlite-pvc
-spec:
-  accessModes:
-    - ReadWriteOnce
-  resources:
-    requests:
-      storage: 1Gi
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-backend
-  labels:
-    app: netmaker-backend
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-backend
-  replicas: 1
-  strategy:
-    type: Recreate
-  template:
-    metadata:
-      labels:
-        app: netmaker-backend
-    spec:
-      containers:
-      - name: netmaker-backend
-        image: gravitl/netmaker:v0.7
-        imagePullPolicy: Always
-        ports:
-        - containerPort: 8081
-        securityContext:
-          privileged: true
-        env:
-        - name: SERVER_API_CONN_STRING
-          value: "api.NETMAKER_BASE_DOMAIN:443"
-        - name: COREDNS_ADDR
-          value: "10.152.183.53"
-        - name: POD_IP
-          valueFrom:
-            fieldRef:
-              fieldPath: status.podIP
-        - name: SERVER_HTTP_HOST
-          value: "api.NETMAKER_BASE_DOMAIN"
-        - name: API_PORT
-          value: "8081"
-        - name: CLIENT_MODE
-          value: "off"
-        - name: MASTER_KEY
-          value: "Unkn0wn!"
-        - name: PLATFORM
-          value: "Kubernetes"
-        - name: CORS_ALLOWED_ORIGIN
-          value: "*"
-        volumeMounts:
-        - name: nm-pvc
-          mountPath: /root/config/dnsconfig
-      - name: rqlite
-        image: rqlite/rqlite
-        ports:
-        - containerPort: 4001
-        - containerPort: 4002
-        volumeMounts:
-        - name: rqlitevol
-          mountPath: /rqlite/file/data
-      volumes:
-      - name: rqlitevol
-        persistentVolumeClaim:
-          claimName: rqlite-pvc
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: PersistentVolumeClaim
-metadata:
-  name: nm-pvc
-spec:
-  accessModes:
-    - ReadWriteMany
-  resources:
-    requests:
-      storage: 128Mi
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-  name: netmaker-api
-spec:
-  ports:
-  - port: 8081
-    protocol: TCP
-    targetPort: 8081
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-backend
-  name: netmaker-grpc
-spec:
-  ports:
-  - port: 443
-    protocol: TCP
-    targetPort: 443
-  selector:
-    app: netmaker-backend
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-dns
-  labels:
-    app: netmaker-dns
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-dns
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-dns
-    spec:
-      containers:
-      - args:
-        - -conf
-        - /root/dnsconfig/Corefile
-        image: coredns/coredns
-        imagePullPolicy: Always
-        name: netmaker-dns
-        ports:
-        - containerPort: 53
-          name: dns
-          protocol: UDP
-        - containerPort: 53
-          name: dns-tcp
-          protocol: TCP
-        volumeMounts:
-        - mountPath: /root/dnsconfig
-          name: nm-pvc
-          readOnly: true
-        securityContext:
-          allowPrivilegeEscalation: false
-          capabilities:
-            add:
-            - NET_BIND_SERVICE
-            drop:
-            - all
-      dnsPolicy: "None"
-      dnsConfig:
-        nameservers:
-          - 127.0.0.1
-      volumes:
-      - name: nm-pvc
-        persistentVolumeClaim:
-          claimName: nm-pvc
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-dns
-  name: netmaker-dns
-spec:
-  ports:
-  - port: 53
-    protocol: UDP
-    targetPort: 53
-    name: udp
-  - port: 53
-    protocol: TCP
-    targetPort: 53
-    name: tcp
-  selector:
-    app: netmaker-dns
-  sessionAffinity: None
-  type: ClusterIP
-  clusterIP: 10.152.183.53
----
-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: netmaker-ui
-  labels:
-    app: netmaker-ui
-spec:
-  selector:
-    matchLabels:
-      app: netmaker-ui
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: netmaker-ui
-    spec:
-      containers:
-      - name: netmaker-ui
-        image: gravitl/netmaker-ui:v0.7
-        ports:
-        - containerPort: 80
-        env:
-        - name: BACKEND_URL
-          value: "https://api.NETMAKER_BASE_DOMAIN"
----
-apiVersion: v1
-kind: Service
-metadata:
-  labels:
-    app: netmaker-ui
-  name: netmaker-ui
-spec:
-  ports:
-  - port: 80
-    protocol: TCP
-    targetPort: 80
-  selector:
-    app: netmaker-ui
-  sessionAffinity: None
-  type: ClusterIP
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-api-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: nginx
-  tls:
-  - hosts:
-    - api.NETMAKER_BASE_DOMAIN
-    secretName: nm-api-tls
-  rules:
-  - host: api.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-api
-            port:
-              number: 8081
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-grpc-ingress-nginx
-  annotations:
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-    nginx.ingress.kubernetes.io/backend-protocol: "GRPC"
-spec:
-  ingressClassName: nginx
-  tls:
-  - hosts:
-    - grpc.NETMAKER_BASE_DOMAIN
-    secretName: nm-grpc-tls
-  rules:
-  - host: grpc.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-grpc
-            port:
-              number: 443
----
-apiVersion: networking.k8s.io/v1
-kind: Ingress
-metadata:
-  name: nm-ui-ingress-nginx
-  annotations:
-    nginx.ingress.kubernetes.io/rewrite-target: /
-    cert-manager.io/cluster-issuer: "letsencrypt-prod"
-    nginx.ingress.kubernetes.io/ssl-redirect: 'true'
-spec:
-  ingressClassName: nginx
-  tls:
-  - hosts:
-    - dashboard.NETMAKER_BASE_DOMAIN
-    secretName: nm-ui-tls
-  rules:
-  - host: dashboard.NETMAKER_BASE_DOMAIN
-    http:
-      paths:
-      - path: /
-        pathType: Prefix
-        backend:
-          service:
-            name: netmaker-ui
-            port:
-              number: 80

logic/auth.go

@@ -229,6 +229,10 @@ func UpdateUser(userchange models.User, user models.User) (models.User, error) {
 func ValidateUser(user models.User) error {
 
 	v := validator.New()
+	_ = v.RegisterValidation("in_charset", func(fl validator.FieldLevel) bool {
+		isgood := user.NameInCharSet()
+		return isgood
+	})
 	err := v.Struct(user)
 
 	if err != nil {

logic/jwts.go

@@ -37,11 +37,11 @@ func CreateJWT(uuid string, macAddress string, network string) (response string,
 		ID:         uuid,
 		Network:    network,
 		MacAddress: macAddress,
-		StandardClaims: jwt.StandardClaims{
+		RegisteredClaims: jwt.RegisteredClaims{
 			Issuer:    "Netmaker",
 			Subject:   fmt.Sprintf("node|%s", uuid),
-			IssuedAt:  time.Now().Unix(),
-			ExpiresAt: expirationTime.Unix(),
+			IssuedAt:  jwt.NewNumericDate(time.Now()),
+			ExpiresAt: jwt.NewNumericDate(expirationTime),
 		},
 	}
 
@@ -60,11 +60,11 @@ func CreateUserJWT(username string, networks []string, isadmin bool) (response s
 		UserName: username,
 		Networks: networks,
 		IsAdmin:  isadmin,
-		StandardClaims: jwt.StandardClaims{
+		RegisteredClaims: jwt.RegisteredClaims{
 			Issuer:    "Netmaker",
-			IssuedAt:  time.Now().Unix(),
 			Subject:   fmt.Sprintf("user|%s", username),
-			ExpiresAt: expirationTime.Unix(),
+			IssuedAt:  jwt.NewNumericDate(time.Now()),
+			ExpiresAt: jwt.NewNumericDate(expirationTime),
 		},
 	}
 

logic/networks.go

@@ -580,7 +580,7 @@ func GetNetwork(networkname string) (models.Network, error) {
 // NetIDInNetworkCharSet - checks if a netid of a network uses valid characters
 func NetIDInNetworkCharSet(network *models.Network) bool {
 
-	charset := "abcdefghijklmnopqrstuvwxyz1234567890-_."
+	charset := "abcdefghijklmnopqrstuvwxyz1234567890-_"
 
 	for _, char := range network.NetID {
 		if !strings.Contains(charset, string(char)) {
@@ -622,28 +622,6 @@ func ParseNetwork(value string) (models.Network, error) {
 	return network, err
 }
 
-// ValidateNetworkUpdate - checks if network is valid to update
-func ValidateNetworkUpdate(network models.Network) error {
-	v := validator.New()
-
-	_ = v.RegisterValidation("netid_valid", func(fl validator.FieldLevel) bool {
-		if fl.Field().String() == "" {
-			return true
-		}
-		inCharSet := nameInNetworkCharSet(fl.Field().String())
-		return inCharSet
-	})
-
-	err := v.Struct(network)
-
-	if err != nil {
-		for _, e := range err.(validator.ValidationErrors) {
-			logger.Log(1, "validator", e.Error())
-		}
-	}
-	return err
-}
-
 // KeyUpdate - updates keys on network
 func KeyUpdate(netname string) (models.Network, error) {
 	err := networkNodesUpdateAction(netname, models.NODE_UPDATE_KEY)
@@ -699,18 +677,6 @@ func networkNodesUpdateAction(networkName string, action string) error {
 	return nil
 }
 
-func nameInNetworkCharSet(name string) bool {
-
-	charset := "abcdefghijklmnopqrstuvwxyz1234567890-_."
-
-	for _, char := range name {
-		if !strings.Contains(charset, strings.ToLower(string(char))) {
-			return false
-		}
-	}
-	return true
-}
-
 func deleteInterface(ifacename string, postdown string) error {
 	var err error
 	if !ncutils.IsKernel() {

logic/peers.go

@@ -267,30 +267,10 @@ func GetAllowedIPs(node, peer *models.Node) []net.IPNet {
 	// handle egress gateway peers
 	if peer.IsEgressGateway == "yes" {
 		//hasGateway = true
-		ranges := peer.EgressGatewayRanges
-		for _, iprange := range ranges { // go through each cidr for egress gateway
-			_, ipnet, err := net.ParseCIDR(iprange) // confirming it's valid cidr
-			if err != nil {
-				logger.Log(1, "could not parse gateway IP range. Not adding ", iprange)
-				continue // if can't parse CIDR
-			}
-			nodeEndpointArr := strings.Split(peer.Endpoint, ":") // getting the public ip of node
-			if ipnet.Contains(net.ParseIP(nodeEndpointArr[0])) { // ensuring egress gateway range does not contain endpoint of node
-				logger.Log(2, "egress IP range of ", iprange, " overlaps with ", node.Endpoint, ", omitting")
-				continue // skip adding egress range if overlaps with node's ip
-			}
-			// TODO: Could put in a lot of great logic to avoid conflicts / bad routes
-			if ipnet.Contains(net.ParseIP(node.LocalAddress)) { // ensuring egress gateway range does not contain public ip of node
-				logger.Log(2, "egress IP range of ", iprange, " overlaps with ", node.LocalAddress, ", omitting")
-				continue // skip adding egress range if overlaps with node's local ip
-			}
-			if err != nil {
-				logger.Log(1, "error encountered when setting egress range", err.Error())
-			} else {
-				allowedips = append(allowedips, *ipnet)
-			}
-		}
+		egressIPs := getEgressIPs(node, peer)
+		allowedips = append(allowedips, egressIPs...)
 	}
+
 	// handle ingress gateway peers
 	if peer.IsIngressGateway == "yes" {
 		extPeers, err := getExtPeers(peer)
@@ -335,6 +315,15 @@ func GetAllowedIPs(node, peer *models.Node) []net.IPNet {
 				}
 				allowedips = append(allowedips, relayAddr)
 			}
+			relayedNode, err := findNode(ip)
+			if err != nil {
+				logger.Log(1, "unable to find node for relayed address", ip, err.Error())
+				continue
+			}
+			if relayedNode.IsEgressGateway == "yes" {
+				extAllowedIPs := getEgressIPs(node, relayedNode)
+				allowedips = append(allowedips, extAllowedIPs...)
+			}
 		}
 	}
 	return allowedips
@@ -423,7 +412,6 @@ func GetPeerUpdateForRelayedNode(node *models.Node, udppeers map[string]string)
 				allowedips = append(allowedips[:i], allowedips[i+1:]...)
 			}
 		}
-
 	}
 
 	pubkey, err := wgtypes.ParseKey(relay.PublicKey)
@@ -477,3 +465,30 @@ func GetPeerUpdateForRelayedNode(node *models.Node, udppeers map[string]string)
 	peerUpdate.DNS = getPeerDNS(node.Network)
 	return peerUpdate, nil
 }
+
+func getEgressIPs(node, peer *models.Node) []net.IPNet {
+	allowedips := []net.IPNet{}
+	for _, iprange := range peer.EgressGatewayRanges { // go through each cidr for egress gateway
+		_, ipnet, err := net.ParseCIDR(iprange) // confirming it's valid cidr
+		if err != nil {
+			logger.Log(1, "could not parse gateway IP range. Not adding ", iprange)
+			continue // if can't parse CIDR
+		}
+		nodeEndpointArr := strings.Split(peer.Endpoint, ":") // getting the public ip of node
+		if ipnet.Contains(net.ParseIP(nodeEndpointArr[0])) { // ensuring egress gateway range does not contain endpoint of node
+			logger.Log(2, "egress IP range of ", iprange, " overlaps with ", node.Endpoint, ", omitting")
+			continue // skip adding egress range if overlaps with node's ip
+		}
+		// TODO: Could put in a lot of great logic to avoid conflicts / bad routes
+		if ipnet.Contains(net.ParseIP(node.LocalAddress)) { // ensuring egress gateway range does not contain public ip of node
+			logger.Log(2, "egress IP range of ", iprange, " overlaps with ", node.LocalAddress, ", omitting")
+			continue // skip adding egress range if overlaps with node's local ip
+		}
+		if err != nil {
+			logger.Log(1, "error encountered when setting egress range", err.Error())
+		} else {
+			allowedips = append(allowedips, *ipnet)
+		}
+	}
+	return allowedips
+}

logic/server.go

@@ -164,6 +164,11 @@ func ServerJoin(networkSettings *models.Network) (models.Node, error) {
 // ServerUpdate - updates the server
 // replaces legacy Checkin code
 func ServerUpdate(serverNode *models.Node, ifaceDelta bool) error {
+	if !IsLocalServer(serverNode) {
+		logger.Log(1, "skipping server update as not the leader")
+		return nil
+	}
+
 	var err = ServerPull(serverNode, ifaceDelta)
 	if isDeleteError(err) {
 		return DeleteNodeByID(serverNode, true)
@@ -201,7 +206,7 @@ func GetServerPeers(serverNode *models.Node) ([]wgtypes.PeerConfig, bool, []stri
 	nodes, err := GetNetworkNodes(serverNode.Network)
 	if err == nil {
 		for _, node := range nodes {
-			if node.IsEgressGateway == "yes" {
+			if node.IsEgressGateway == "yes" && !IsLocalServer(&node) {
 				gateways = append(gateways, node.EgressGatewayRanges...)
 			}
 		}

main.go

@@ -120,7 +120,9 @@ func initialize() { // Client Mode Prereq Check
 		}
 	}
 
-	genCerts()
+	if err = genCerts(); err != nil {
+		logger.Log(0, "something went wrong when generating broker certs", err.Error())
+	}
 
 	if servercfg.IsMessageQueueBackend() {
 		if err = mq.ServerStartNotify(); err != nil {
@@ -166,7 +168,8 @@ func startControllers() {
 // Should we be using a context vice a waitgroup????????????
 func runMessageQueue(wg *sync.WaitGroup) {
 	defer wg.Done()
-	logger.Log(0, "connecting to mq broker at", servercfg.GetMessageQueueEndpoint())
+	brokerHost, secure := servercfg.GetMessageQueueEndpoint()
+	logger.Log(0, "connecting to mq broker at", brokerHost, "with TLS?", fmt.Sprintf("%v", secure))
 	var client = mq.SetupMQTT(false) // Set up the subscription listener
 	ctx, cancel := context.WithCancel(context.Background())
 	go mq.Keepalive(ctx)
@@ -190,24 +193,31 @@ func genCerts() error {
 	logger.Log(0, "checking keys and certificates")
 	var private *ed25519.PrivateKey
 	var err error
-	private, err = tls.ReadKey(functions.GetNetmakerPath() + "/root.key")
-	if errors.Is(err, os.ErrNotExist) {
+
+	// == ROOT key handling ==
+
+	private, err = serverctl.ReadKeyFromDB(tls.ROOT_KEY_NAME)
+	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) {
 		logger.Log(0, "generating new root key")
 		_, newKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
 			return err
 		}
-		if err := tls.SaveKey(functions.GetNetmakerPath(), "/root.key", newKey); err != nil {
-			return err
-		}
 		private = &newKey
 	} else if err != nil {
 		return err
 	}
-	ca, err := tls.ReadCert(functions.GetNetmakerPath() + ncutils.GetSeparator() + "root.pem")
+	logger.Log(2, "saving root.key")
+	if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.ROOT_KEY_NAME, *private); err != nil {
+		return err
+	}
+
+	// == ROOT cert handling ==
+
+	ca, err := serverctl.ReadCertFromDB(tls.ROOT_PEM_NAME)
 	//if cert doesn't exist or will expire within 10 days --- but can't do this as clients won't be able to connect
 	//if errors.Is(err, os.ErrNotExist) || cert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
-	if errors.Is(err, os.ErrNotExist) {
+	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) || ca.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
 		logger.Log(0, "generating new root CA")
 		caName := tls.NewName("CA Root", "US", "Gravitl")
 		csr, err := tls.NewCSR(*private, caName)
@@ -218,15 +228,19 @@ func genCerts() error {
 		if err != nil {
 			return err
 		}
-		if err := tls.SaveCert(functions.GetNetmakerPath(), ncutils.GetSeparator()+"root.pem", rootCA); err != nil {
-			return err
-		}
 		ca = rootCA
 	} else if err != nil {
 		return err
 	}
-	cert, err := tls.ReadCert(functions.GetNetmakerPath() + "/server.pem")
-	if errors.Is(err, os.ErrNotExist) || cert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
+	logger.Log(2, "saving root.pem")
+	if err := serverctl.SaveCert(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.ROOT_PEM_NAME, ca); err != nil {
+		return err
+	}
+
+	// == SERVER cert handling ==
+
+	cert, err := serverctl.ReadCertFromDB(tls.SERVER_PEM_NAME)
+	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) || cert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
 		//gen new key
 		logger.Log(0, "generating new server key/certificate")
 		_, key, err := ed25519.GenerateKey(rand.Reader)
@@ -238,18 +252,78 @@ func genCerts() error {
 		if err != nil {
 			return err
 		}
-		cert, err := tls.NewEndEntityCert(*private, csr, ca, tls.CERTIFICATE_VALIDITY)
+		newCert, err := tls.NewEndEntityCert(*private, csr, ca, tls.CERTIFICATE_VALIDITY)
+		if err != nil {
+			return err
+		}
+		if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_KEY_NAME, key); err != nil {
+			return err
+		}
+		cert = newCert
+	} else if err != nil {
+		return err
+	} else if err == nil {
+		if serverKey, err := serverctl.ReadKeyFromDB(tls.SERVER_KEY_NAME); err == nil {
+			logger.Log(2, "saving server.key")
+			if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_KEY_NAME, *serverKey); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
+	}
+	logger.Log(2, "saving server.pem")
+	if err := serverctl.SaveCert(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_PEM_NAME, cert); err != nil {
+		return err
+	}
+
+	// == SERVER-CLIENT connection cert handling ==
+
+	serverClientCert, err := serverctl.ReadCertFromDB(tls.SERVER_CLIENT_PEM)
+	if errors.Is(err, os.ErrNotExist) || database.IsEmptyRecord(err) || serverClientCert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
+		//gen new key
+		logger.Log(0, "generating new server client key/certificate")
+		_, key, err := ed25519.GenerateKey(rand.Reader)
+		if err != nil {
+			return err
+		}
+		serverName := tls.NewCName(tls.SERVER_CLIENT_ENTRY)
+		csr, err := tls.NewCSR(key, serverName)
 		if err != nil {
 			return err
 		}
-		if err := tls.SaveKey(functions.GetNetmakerPath(), "/server.key", key); err != nil {
+		newServerClientCert, err := tls.NewEndEntityCert(*private, csr, ca, tls.CERTIFICATE_VALIDITY)
+		if err != nil {
 			return err
 		}
-		if err := tls.SaveCert(functions.GetNetmakerPath(), "/server.pem", cert); err != nil {
+
+		if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_CLIENT_KEY, key); err != nil {
 			return err
 		}
+		serverClientCert = newServerClientCert
 	} else if err != nil {
 		return err
+	} else if err == nil {
+		logger.Log(2, "saving serverclient.key")
+		if serverClientKey, err := serverctl.ReadKeyFromDB(tls.SERVER_CLIENT_KEY); err == nil {
+			if err := serverctl.SaveKey(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_CLIENT_KEY, *serverClientKey); err != nil {
+				return err
+			}
+		} else {
+			return err
+		}
 	}
-	return nil
+
+	logger.Log(2, "saving serverclient.pem")
+	if err := serverctl.SaveCert(functions.GetNetmakerPath()+ncutils.GetSeparator(), tls.SERVER_CLIENT_PEM, serverClientCert); err != nil {
+		return err
+	}
+
+	logger.Log(1, "ensure the root.pem, root.key, server.pem, and server.key files are updated on your broker")
+
+	return serverctl.SetClientTLSConf(
+		functions.GetNetmakerPath()+ncutils.GetSeparator()+tls.SERVER_CLIENT_PEM,
+		functions.GetNetmakerPath()+ncutils.GetSeparator()+tls.SERVER_CLIENT_KEY,
+		ca,
+	)
 }

models/names.go

@@ -5,7 +5,7 @@ import (
 	"time"
 )
 
-// 4-7 chars only
+// NAMES - list of names 4-7 chars in length
 var NAMES = []string{
 	"logic",
 	"warrant",
@@ -145,7 +145,7 @@ var NAMES = []string{
 	"tesla",
 }
 
-// must be 4 chars or less
+// SMALL_NAMES - list of small (4 char or less) names
 var SMALL_NAMES = []string{
 	"ace",
 	"odd",
@@ -231,19 +231,15 @@ var SMALL_NAMES = []string{
 	"cold",
 }
 
+// GenerateNodeName - generates a random node name
 func GenerateNodeName() string {
 	rand.Seed(time.Now().UnixNano())
 	return SMALL_NAMES[rand.Intn(len(SMALL_NAMES))] + "-" + NAMES[seededRand.Intn(len(NAMES))]
 }
 
+// RetrieveLogo - retrieves the ascii art logo for Netmaker
 func RetrieveLogo() string {
-	return `
-    ______     ______     ______     __   __   __     ______   __                        
-   /\  ___\   /\  == \   /\  __ \   /\ \ / /  /\ \   /\__  _\ /\ \                       
-   \ \ \__ \  \ \  __<   \ \  __ \  \ \ \'/   \ \ \  \/_/\ \/ \ \ \____                  
-    \ \_____\  \ \_\ \_\  \ \_\ \_\  \ \__|    \ \_\    \ \_\  \ \_____\                 
-     \/_____/   \/_/ /_/   \/_/\/_/   \/_/      \/_/     \/_/   \/_____/                 
-                                                                                         
+	return `              
  __   __     ______     ______   __    __     ______     __  __     ______     ______    
 /\ "-.\ \   /\  ___\   /\__  _\ /\ "-./  \   /\  __ \   /\ \/ /    /\  ___\   /\  == \   
 \ \ \-.  \  \ \  __\   \/_/\ \/ \ \ \-./\ \  \ \  __ \  \ \  _"-.  \ \  __\   \ \  __<   

models/structs.go

@@ -1,6 +1,8 @@
 package models
 
 import (
+	"strings"
+
 	jwt "github.com/golang-jwt/jwt/v4"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -17,7 +19,7 @@ type AuthParams struct {
 
 // User struct - struct for Users
 type User struct {
-	UserName string   `json:"username" bson:"username" validate:"min=3,max=40,regexp=^(([a-zA-Z,\-,\.]*)|([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4})){3,40}$"`
+	UserName string   `json:"username" bson:"username" validate:"min=3,max=40,in_charset|email"`
 	Password string   `json:"password" bson:"password" validate:"required,min=5"`
 	Networks []string `json:"networks" bson:"networks"`
 	IsAdmin  bool     `json:"isadmin" bson:"isadmin"`
@@ -25,7 +27,7 @@ type User struct {
 
 // ReturnUser - return user struct
 type ReturnUser struct {
-	UserName string   `json:"username" bson:"username" validate:"min=3,max=40,regexp=^(([a-zA-Z,\-,\.]*)|([A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,4})){3,40}$"`
+	UserName string   `json:"username" bson:"username"`
 	Networks []string `json:"networks" bson:"networks"`
 	IsAdmin  bool     `json:"isadmin" bson:"isadmin"`
 }
@@ -41,7 +43,7 @@ type UserClaims struct {
 	IsAdmin  bool
 	UserName string
 	Networks []string
-	jwt.StandardClaims
+	jwt.RegisteredClaims
 }
 
 // SuccessfulUserLoginResponse - successlogin struct
@@ -56,7 +58,7 @@ type Claims struct {
 	ID         string
 	MacAddress string
 	Network    string
-	jwt.StandardClaims
+	jwt.RegisteredClaims
 }
 
 // SuccessfulLoginResponse is struct to send the request response
@@ -206,3 +208,14 @@ type ServerConfig struct {
 	MQPort      string `yaml:"mqport"`
 	Server      string `yaml:"server"`
 }
+
+// User.NameInCharset - returns if name is in charset below or not
+func (user *User) NameInCharSet() bool {
+	charset := "abcdefghijklmnopqrstuvwxyz1234567890-."
+	for _, char := range user.UserName {
+		if !strings.Contains(charset, strings.ToLower(string(char))) {
+			return false
+		}
+	}
+	return true
+}

mq/handlers.go

@@ -132,8 +132,10 @@ func updateNodePeers(currentNode *models.Node) {
 		logger.Log(1, "server node:", currentServerNode.ID, "failed update")
 		return
 	}
-	if err := PublishPeerUpdate(currentNode); err != nil {
-		logger.Log(1, "error publishing peer update ", err.Error())
-		return
+	if logic.IsLeader(&currentServerNode) {
+		if err := PublishPeerUpdate(currentNode, false); err != nil {
+			logger.Log(1, "error publishing peer update ", err.Error())
+			return
+		}
 	}
 }

mq/mq.go

@@ -2,13 +2,13 @@ package mq
 
 import (
 	"context"
-	"log"
 	"time"
 
 	mqtt "github.com/eclipse/paho.mqtt.golang"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
+	"github.com/gravitl/netmaker/serverctl"
 )
 
 // KEEPALIVE_TIMEOUT - time in seconds for timeout
@@ -24,9 +24,13 @@ var peer_force_send = 0
 // SetupMQTT creates a connection to broker and return client
 func SetupMQTT(publish bool) mqtt.Client {
 	opts := mqtt.NewClientOptions()
-	opts.AddBroker(servercfg.GetMessageQueueEndpoint())
+	broker, secure := servercfg.GetMessageQueueEndpoint()
+	opts.AddBroker(broker)
 	id := ncutils.MakeRandomString(23)
 	opts.ClientID = id
+	if secure {
+		opts.SetTLSConfig(&serverctl.TlsConfig)
+	}
 	opts.SetAutoReconnect(true)
 	opts.SetConnectRetry(true)
 	opts.SetConnectRetryInterval(time.Second << 2)
@@ -58,9 +62,9 @@ func SetupMQTT(publish bool) mqtt.Client {
 			logger.Log(2, "unable to connect to broker, retrying ...")
 			if time.Now().After(tperiod) {
 				if token.Error() == nil {
-					log.Fatal(0, "could not connect to broker, token timeout, exiting ...")
+					logger.FatalLog("could not connect to broker, token timeout, exiting ...")
 				} else {
-					log.Fatal(0, "could not connect to broker, exiting ...", token.Error())
+					logger.FatalLog("could not connect to broker, exiting ...", token.Error().Error())
 				}
 			}
 		} else {
@@ -68,6 +72,9 @@ func SetupMQTT(publish bool) mqtt.Client {
 		}
 		time.Sleep(2 * time.Second)
 	}
+	if !publish {
+		logger.Log(0, "successfully connected to mq broker")
+	}
 	return client
 }
 

mq/publishers.go

@@ -12,7 +12,7 @@ import (
 )
 
 // PublishPeerUpdate --- deterines and publishes a peer update to all the peers of a node
-func PublishPeerUpdate(newNode *models.Node) error {
+func PublishPeerUpdate(newNode *models.Node, publishToSelf bool) error {
 	if !servercfg.IsMessageQueueBackend() {
 		return nil
 	}
@@ -26,6 +26,10 @@ func PublishPeerUpdate(newNode *models.Node) error {
 		if node.IsServer == "yes" {
 			continue
 		}
+		if !publishToSelf && newNode.ID == node.ID {
+			//skip self
+			continue
+		}
 		peerUpdate, err := logic.GetPeerUpdate(&node)
 		if err != nil {
 			logger.Log(1, "error getting peer update for node", node.ID, err.Error())
@@ -70,7 +74,7 @@ func PublishExtPeerUpdate(node *models.Node) error {
 	if err = publish(node, fmt.Sprintf("peers/%s/%s", node.Network, node.ID), data); err != nil {
 		return err
 	}
-	go PublishPeerUpdate(node)
+	go PublishPeerUpdate(node, false)
 	return nil
 }
 
@@ -121,13 +125,18 @@ func sendPeers() {
 		serverNode, errN := logic.GetNetworkServerLeader(network.NetID)
 		if errN == nil {
 			serverNode.SetLastCheckIn()
-			logic.UpdateNode(&serverNode, &serverNode)
+			if err := logic.UpdateNode(&serverNode, &serverNode); err != nil {
+				logger.Log(0, "failed checkin for server node", serverNode.Name, "on network", network.NetID, err.Error())
+			}
+		}
+		isLeader := logic.IsLocalServer(&serverNode)
+		if errN == nil && isLeader {
 			if network.DefaultUDPHolePunch == "yes" {
 				if logic.ShouldPublishPeerPorts(&serverNode) || force {
 					if force {
 						logger.Log(2, "sending scheduled peer update (5 min)")
 					}
-					err = PublishPeerUpdate(&serverNode)
+					err = PublishPeerUpdate(&serverNode, false)
 					if err != nil {
 						logger.Log(1, "error publishing udp port updates for network", network.NetID)
 						logger.Log(1, errN.Error())
@@ -135,10 +144,14 @@ func sendPeers() {
 				}
 			}
 		} else {
-			logger.Log(1, "unable to retrieve leader for network ", network.NetID)
+			if isLeader {
+				logger.Log(1, "unable to retrieve leader for network ", network.NetID)
+			}
+			logger.Log(2, "server checkin complete for server", serverNode.Name, "on network", network.NetID)
 			serverctl.SyncServerNetwork(network.NetID)
-			logger.Log(1, errN.Error())
-			continue
+			if errN != nil {
+				logger.Log(1, errN.Error())
+			}
 		}
 	}
 }

netclient/command/commands.go

@@ -94,13 +94,13 @@ func Pull(cfg *config.ClientConfig) error {
 	}
 	//generate new client key if one doesn' exist
 	var private *ed25519.PrivateKey
-	private, err = tls.ReadKey(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
+	private, err = tls.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
 	if err != nil {
 		_, newKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
 			return err
 		}
-		if err := tls.SaveKey(ncutils.GetNetclientPath(), ncutils.GetSeparator()+"client.key", newKey); err != nil {
+		if err := tls.SaveKeyToFile(ncutils.GetNetclientPath(), ncutils.GetSeparator()+"client.key", newKey); err != nil {
 			return err
 		}
 		private = &newKey

netclient/daemon/macos.go

@@ -29,10 +29,6 @@ func SetupMacDaemon() error {
 		return err
 	}
 
-	_, errN := os.Stat("~/Library/LaunchAgents")
-	if os.IsNotExist(errN) {
-		os.Mkdir("~/Library/LaunchAgents", 0755)
-	}
 	err = CreateMacService(MAC_SERVICE_NAME)
 	if err != nil {
 		return err
@@ -64,7 +60,7 @@ func RestartLaunchD() {
 
 // StopLaunchD - stop launch daemon
 func StopLaunchD() {
-	ncutils.RunCmd("launchctl unload  /System/Library/LaunchDaemons/"+MAC_SERVICE_NAME+".plist", true)
+	ncutils.RunCmd("launchctl unload  /Library/LaunchDaemons/"+MAC_SERVICE_NAME+".plist", true)
 }
 
 // CreateMacService - Creates the mac service file for LaunchDaemons

netclient/functions/clientconfig.go

@@ -0,0 +1,74 @@
+package functions
+
+import (
+	"strconv"
+	"strings"
+
+	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/netclient/config"
+	"github.com/gravitl/netmaker/netclient/functions/upgrades"
+	"github.com/gravitl/netmaker/netclient/ncutils"
+)
+
+// UpdateClientConfig - function is called on daemon start to update clientConfig if required
+// Usage :  set update required to true and and update logic to function
+func UpdateClientConfig() {
+	defer upgrades.ReleaseUpgrades()
+
+	networks, _ := ncutils.GetSystemNetworks()
+	if len(networks) == 0 {
+		return
+	}
+	logger.Log(0, "checking for netclient updates...")
+	for _, network := range networks {
+		cfg := config.ClientConfig{}
+		cfg.Network = network
+		cfg.ReadConfig()
+		major, minor, _ := Version(cfg.Node.Version)
+		if major == 0 && minor < 14 {
+			logger.Log(0, "schema of network", cfg.Network, "is out of date and cannot be updated\n Correct behaviour of netclient cannot be guaranteed")
+			continue
+		}
+		configChanged := false
+		for _, u := range upgrades.Upgrades {
+			if ncutils.StringSliceContains(u.RequiredVersions, cfg.Node.Version) {
+				logger.Log(0, "upgrading node", cfg.Node.Name, "on network", cfg.Node.Network, "from", cfg.Node.Version, "to", u.NewVersion)
+				upgrades.UpgradeFunction(u.OP)(&cfg)
+				cfg.Node.Version = u.NewVersion
+				configChanged = true
+			}
+		}
+		if configChanged {
+			//save and publish
+			if err := PublishNodeUpdate(&cfg); err != nil {
+				logger.Log(0, "error publishing node update during schema change", err.Error())
+			}
+		}
+	}
+	logger.Log(0, "finished updates")
+}
+
+// Version - parse version string into component parts
+// version string must be semantic version of form 1.2.3 or v1.2.3
+// otherwise 0, 0, 0 will be returned.
+func Version(version string) (int, int, int) {
+	var major, minor, patch int
+	var errMajor, errMinor, errPatch error
+	parts := strings.Split(version, ".")
+	//ensure semantic version
+	if len(parts) < 3 {
+		return major, minor, patch
+	}
+	if strings.Contains(parts[0], "v") {
+		majors := strings.Split(parts[0], "v")
+		major, errMajor = strconv.Atoi(majors[1])
+	} else {
+		major, errMajor = strconv.Atoi(parts[0])
+	}
+	minor, errMinor = strconv.Atoi(parts[1])
+	patch, errPatch = strconv.Atoi(parts[2])
+	if errMajor != nil || errMinor != nil || errPatch != nil {
+		return 0, 0, 0
+	}
+	return major, minor, patch
+}

netclient/functions/common.go

@@ -10,7 +10,9 @@ import (
 	"net"
 	"net/http"
 	"os"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
@@ -25,6 +27,9 @@ import (
 // LINUX_APP_DATA_PATH - linux path
 const LINUX_APP_DATA_PATH = "/etc/netmaker"
 
+// HTTP_TIMEOUT - timeout in seconds for http requests
+const HTTP_TIMEOUT = 30
+
 // ListPorts - lists ports of WireGuard devices
 func ListPorts() error {
 	wgclient, err := wgctrl.New()
@@ -339,7 +344,9 @@ func API(data any, method, url, authorization string) (*http.Response, error) {
 	if authorization != "" {
 		request.Header.Set("authorization", "Bearer "+authorization)
 	}
-	client := http.Client{}
+	client := http.Client{
+		Timeout: HTTP_TIMEOUT * time.Second,
+	}
 	return client.Do(request)
 }
 
@@ -409,3 +416,11 @@ func SetServerInfo(cfg *config.ClientConfig) error {
 
 	return nil
 }
+
+func informPortChange(node *models.Node) {
+	if node.ListenPort == 0 {
+		logger.Log(0, "UDP hole punching enabled for node", node.Name)
+	} else {
+		logger.Log(0, "node", node.Name, "is using port", strconv.Itoa(int(node.ListenPort)))
+	}
+}

netclient/functions/daemon.go

@@ -42,6 +42,7 @@ type cachedMessage struct {
 
 // Daemon runs netclient daemon from command line
 func Daemon() error {
+	UpdateClientConfig()
 	serverSet := make(map[string]bool)
 	// == initial pull of all networks ==
 	networks, _ := ncutils.GetSystemNetworks()
@@ -268,7 +269,7 @@ func setupMQTT(cfg *config.ClientConfig, publish bool) (mqtt.Client, error) {
 
 func reRegisterWithServer(cfg *config.ClientConfig) {
 	logger.Log(0, "connection issue detected.. attempt connection with new certs and broker information")
-	key, err := ssl.ReadKey(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
+	key, err := ssl.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
 	if err != nil {
 		_, *key, err = ed25519.GenerateKey(rand.Reader)
 		if err != nil {
@@ -328,7 +329,7 @@ func read(network, which string) string {
 		if readMessage.LastSeen.IsZero() {
 			return ""
 		}
-		if time.Now().After(readMessage.LastSeen.Add(time.Minute * 10)) { // check if message has been there over a minute
+		if time.Now().After(readMessage.LastSeen.Add(time.Hour * 24)) { // check if message has been there over a minute
 			messageCache.Delete(fmt.Sprintf("%s%s", network, which)) // remove old message if expired
 			return ""
 		}

netclient/functions/join.go

@@ -170,8 +170,13 @@ func JoinNetwork(cfg *config.ClientConfig, privateKey string) error {
 		}
 	}
 	logger.Log(1, "node created on remote server...updating configs")
-	cfg.Node = node
-	err = config.ModNodeConfig(&cfg.Node)
+	err = ncutils.ModPort(&node)
+	if err != nil {
+		return err
+	}
+	informPortChange(&node)
+
+	err = config.ModNodeConfig(&node)
 	if err != nil {
 		return err
 	}
@@ -188,6 +193,7 @@ func JoinNetwork(cfg *config.ClientConfig, privateKey string) error {
 	if err != nil {
 		return err
 	}
+	cfg.Node = node
 	if err := Register(cfg); err != nil {
 		return err
 	}

netclient/functions/mqhandlers.go

@@ -44,13 +44,13 @@ func NodeUpdate(client mqtt.Client, msg mqtt.Message) {
 		return
 	}
 
-	logger.Log(0, "received message to update node "+newNode.Name)
 	// see if cache hit, if so skip
 	var currentMessage = read(newNode.Network, lastNodeUpdate)
 	if currentMessage == string(data) {
 		return
 	}
 	insert(newNode.Network, lastNodeUpdate, string(data)) // store new message in cache
+	logger.Log(0, "received message to update node "+newNode.Name)
 
 	// ensure that OS never changes
 	newNode.OS = runtime.GOOS
@@ -105,25 +105,31 @@ func NodeUpdate(client mqtt.Client, msg mqtt.Message) {
 	}
 	file := ncutils.GetNetclientPathSpecific() + nodeCfg.Node.Interface + ".conf"
 
-	if err := wireguard.UpdateWgInterface(file, privateKey, nameserver, newNode); err != nil {
-		logger.Log(0, "error updating wireguard config "+err.Error())
-		return
-	}
-	if keepaliveChange {
-		wireguard.UpdateKeepAlive(file, newNode.PersistentKeepalive)
-	}
 	if ifaceDelta { // if a change caused an ifacedelta we need to notify the server to update the peers
+		err = ncutils.ModPort(&newNode)
+		if err != nil {
+			logger.Log(0, "error modifying node port on", newNode.Name, "-", err.Error())
+			return
+		}
+		informPortChange(&newNode)
+		if err := wireguard.UpdateWgInterface(file, privateKey, nameserver, newNode); err != nil {
+			logger.Log(0, "error updating wireguard config "+err.Error())
+			return
+		}
+		if keepaliveChange {
+			wireguard.UpdateKeepAlive(file, newNode.PersistentKeepalive)
+		}
 		logger.Log(0, "applying WG conf to "+file)
 		if ncutils.IsWindows() {
 			wireguard.RemoveConfGraceful(nodeCfg.Node.Interface)
 		}
 		err = wireguard.ApplyConf(&nodeCfg.Node, nodeCfg.Node.Interface, file)
 		if err != nil {
-			logger.Log(0, "error restarting wg after node update "+err.Error())
+			logger.Log(0, "error restarting wg after node update -", err.Error())
 			return
 		}
 
-		time.Sleep(time.Second >> 0)
+		time.Sleep(time.Second)
 		//	if newNode.DNSOn == "yes" {
 		//		for _, server := range newNode.NetworkSettings.DefaultServerAddrs {
 		//			if server.IsLeader {

netclient/functions/mqpublish.go

@@ -136,7 +136,7 @@ func publish(nodeCfg *config.ClientConfig, dest string, msg []byte, qos byte) er
 	}
 
 	if token := client.Publish(dest, qos, false, encrypted); !token.WaitTimeout(30*time.Second) || token.Error() != nil {
-		logger.Log(0, "could not connect to broker at "+nodeCfg.Server.Server+":8883")
+		logger.Log(0, "could not connect to broker at "+nodeCfg.Server.Server+":"+nodeCfg.Server.MQPort)
 		var err error
 		if token.Error() == nil {
 			err = errors.New("connection timeout")
@@ -151,10 +151,10 @@ func publish(nodeCfg *config.ClientConfig, dest string, msg []byte, qos byte) er
 }
 
 func checkCertExpiry(cfg *config.ClientConfig) error {
-	cert, err := tls.ReadCert(ncutils.GetNetclientServerPath(cfg.Server.Server) + ncutils.GetSeparator() + "client.pem")
+	cert, err := tls.ReadCertFromFile(ncutils.GetNetclientServerPath(cfg.Server.Server) + ncutils.GetSeparator() + "client.pem")
 	//if cert doesn't exist or will expire within 10 days
 	if errors.Is(err, os.ErrNotExist) || cert.NotAfter.Before(time.Now().Add(time.Hour*24*10)) {
-		key, err := tls.ReadKey(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
+		key, err := tls.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
 		if err != nil {
 			return err
 		}

netclient/functions/pull.go

@@ -62,10 +62,17 @@ func Pull(network string, iface bool) (*models.Node, error) {
 			logger.Log(0, "unable to update server config: "+err.Error())
 		}
 	}
-	if iface {
-		if err = config.ModNodeConfig(&resNode); err != nil {
+	if nodeGET.Node.ListenPort != cfg.Node.ListenPort {
+		err = ncutils.ModPort(&resNode)
+		if err != nil {
 			return nil, err
 		}
+		informPortChange(&resNode)
+	}
+	if err = config.ModNodeConfig(&resNode); err != nil {
+		return nil, err
+	}
+	if iface {
 		if err = wireguard.SetWGConfig(network, false, nodeGET.Peers[:]); err != nil {
 			return nil, err
 		}

netclient/functions/register.go

@@ -20,19 +20,19 @@ func Register(cfg *config.ClientConfig) error {
 	//generate new key if one doesn' exist
 	var private *ed25519.PrivateKey
 	var err error
-	private, err = tls.ReadKey(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
+	private, err = tls.ReadKeyFromFile(ncutils.GetNetclientPath() + ncutils.GetSeparator() + "client.key")
 	if err != nil {
 		_, newKey, err := ed25519.GenerateKey(rand.Reader)
 		if err != nil {
 			return err
 		}
-		if err := tls.SaveKey(ncutils.GetNetclientPath(), ncutils.GetSeparator()+"client.key", newKey); err != nil {
+		if err := tls.SaveKeyToFile(ncutils.GetNetclientPath(), ncutils.GetSeparator()+"client.key", newKey); err != nil {
 			return err
 		}
 		private = &newKey
 	}
 	//check if cert exists
-	_, err = tls.ReadCert(ncutils.GetNetclientServerPath(cfg.Server.Server) + ncutils.GetSeparator() + "client.pem")
+	_, err = tls.ReadCertFromFile(ncutils.GetNetclientServerPath(cfg.Server.Server) + ncutils.GetSeparator() + "client.pem")
 	if errors.Is(err, os.ErrNotExist) {
 		if err := RegisterWithServer(private, cfg); err != nil {
 			return err
@@ -88,10 +88,10 @@ func RegisterWithServer(private *ed25519.PrivateKey, cfg *config.ClientConfig) e
 	//the pubkeys are included in the response so the values in the certificate can be updated appropriately
 	resp.CA.PublicKey = resp.CAPubKey
 	resp.Cert.PublicKey = resp.CertPubKey
-	if err := tls.SaveCert(ncutils.GetNetclientServerPath(cfg.Server.Server)+ncutils.GetSeparator(), "root.pem", &resp.CA); err != nil {
+	if err := tls.SaveCertToFile(ncutils.GetNetclientServerPath(cfg.Server.Server)+ncutils.GetSeparator(), tls.ROOT_PEM_NAME, &resp.CA); err != nil {
 		return err
 	}
-	if err := tls.SaveCert(ncutils.GetNetclientServerPath(cfg.Server.Server)+ncutils.GetSeparator(), "client.pem", &resp.Cert); err != nil {
+	if err := tls.SaveCertToFile(ncutils.GetNetclientServerPath(cfg.Server.Server)+ncutils.GetSeparator(), "client.pem", &resp.Cert); err != nil {
 		return err
 	}
 	logger.Log(0, "certificates/key saved ")

netclient/functions/upgrades/types.go

@@ -0,0 +1,13 @@
+package upgrades
+
+import "github.com/gravitl/netmaker/netclient/config"
+
+// UpgradeFunction - logic for upgrade
+type UpgradeFunction func(*config.ClientConfig)
+
+// UpgradeInfo - struct for holding upgrade info
+type UpgradeInfo struct {
+	RequiredVersions []string
+	NewVersion       string
+	OP               UpgradeFunction
+}

netclient/functions/upgrades/upgrades.go

@@ -0,0 +1,20 @@
+package upgrades
+
+func init() {
+	addUpgrades([]UpgradeInfo{
+		upgrade0145,
+	})
+}
+
+// Upgrades - holds all upgrade funcs
+var Upgrades = []UpgradeInfo{}
+
+// addUpgrades - Adds upgrades to make to client
+func addUpgrades(upgrades []UpgradeInfo) {
+	Upgrades = append(Upgrades, upgrades...)
+}
+
+// ReleaseUpgrades - releases upgrade funcs from memory
+func ReleaseUpgrades() {
+	Upgrades = nil
+}

netclient/functions/upgrades/v0-14-5.go

@@ -0,0 +1,24 @@
+package upgrades
+
+import (
+	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/netclient/config"
+)
+
+var upgrade0145 = UpgradeInfo{
+	RequiredVersions: []string{
+		"v0.14.0",
+		"v0.14.1",
+		"v0.14.2",
+		"v0.14.3",
+		"v0.14.4",
+	},
+	NewVersion: "v0.14.5",
+	OP:         update0145,
+}
+
+func update0145(cfg *config.ClientConfig) {
+	// do stuff for 14.X -> 14.5
+	// No-op
+	logger.Log(0, "updating schema for 0.14.5")
+}

netclient/ncutils/netclientutils.go

@@ -41,6 +41,9 @@ const NO_DB_RECORDS = "could not find any records"
 // LINUX_APP_DATA_PATH - linux path
 const LINUX_APP_DATA_PATH = "/etc/netclient"
 
+// MAC_APP_DATA_PATH - linux path
+const MAC_APP_DATA_PATH = "/Applications/Netclient"
+
 // WINDOWS_APP_DATA_PATH - windows path
 const WINDOWS_APP_DATA_PATH = "C:\\Program Files (x86)\\Netclient"
 
@@ -265,7 +268,7 @@ func GetNetclientPath() string {
 	if IsWindows() {
 		return WINDOWS_APP_DATA_PATH
 	} else if IsMac() {
-		return "/etc/netclient/"
+		return MAC_APP_DATA_PATH
 	} else {
 		return LINUX_APP_DATA_PATH
 	}
@@ -301,7 +304,7 @@ func GetNetclientServerPath(server string) string {
 	if IsWindows() {
 		return WINDOWS_APP_DATA_PATH + "\\" + server + "\\"
 	} else if IsMac() {
-		return "/etc/netclient/" + server + "/"
+		return MAC_APP_DATA_PATH + "/" + server + "/"
 	} else {
 		return LINUX_APP_DATA_PATH + "/" + server
 	}
@@ -312,7 +315,7 @@ func GetNetclientPathSpecific() string {
 	if IsWindows() {
 		return WINDOWS_APP_DATA_PATH + "\\"
 	} else if IsMac() {
-		return "/etc/netclient/config/"
+		return MAC_APP_DATA_PATH + "/config/"
 	} else {
 		return LINUX_APP_DATA_PATH + "/config/"
 	}
@@ -491,11 +494,9 @@ func CheckUID() {
 
 // CheckWG - Checks if WireGuard is installed. If not, exit
 func CheckWG() {
-	var _, err = exec.LookPath("wg")
 	uspace := GetWireGuard()
-	if err != nil {
+	if !HasWG() {
 		if uspace == "wg" {
-			logger.Log(0, err.Error())
 			log.Fatal("WireGuard not installed. Please install WireGuard (wireguard-tools) and try again.")
 		}
 		logger.Log(0, "running with userspace wireguard: ", uspace)
@@ -504,6 +505,12 @@ func CheckWG() {
 	}
 }
 
+// HasWG - returns true if wg command exists
+func HasWG() bool {
+	var _, err = exec.LookPath("wg")
+	return err == nil
+}
+
 // ConvertKeyToBytes - util to convert a key to bytes to use elsewhere
 func ConvertKeyToBytes(key *[32]byte) ([]byte, error) {
 	var buffer bytes.Buffer
@@ -574,3 +581,14 @@ func GetIPNetFromString(ip string) (net.IPNet, error) {
 	}
 	return *ipnet, err
 }
+
+// ModPort - Change Node Port if UDP Hole Punching or ListenPort is not free
+func ModPort(node *models.Node) error {
+	var err error
+	if node.UDPHolePunch == "yes" {
+		node.ListenPort = 0
+	} else {
+		node.ListenPort, err = GetFreePort(node.ListenPort)
+	}
+	return err
+}

netclient/ncutils/netclientutils_darwin.go

@@ -1,19 +1,28 @@
 package ncutils
 
 import (
-	"github.com/gravitl/netmaker/logger"
 	"os/exec"
 	"strings"
+
+	"github.com/gravitl/netmaker/logger"
 )
 
+// WHITESPACE_PLACEHOLDER - used with RunCMD - if a path has whitespace, use this to avoid running path as 2 args in RunCMD
+const WHITESPACE_PLACEHOLDER = "+-+-+-+"
+
 // RunCmd - runs a local command
 func RunCmd(command string, printerr bool) (string, error) {
+
 	args := strings.Fields(command)
+	// return whitespace after split
+	for i, arg := range args {
+		args[i] = strings.Replace(arg, WHITESPACE_PLACEHOLDER, " ", -1)
+	}
 	cmd := exec.Command(args[0], args[1:]...)
 	cmd.Wait()
 	out, err := cmd.CombinedOutput()
 	if err != nil && printerr {
-		logger.Log(0, "error running command:", command)
+		logger.Log(0, "error running command:", strings.Join(args, " "))
 		logger.Log(0, strings.TrimSuffix(string(out), "\n"))
 	}
 	return string(out), err

netclient/netclient.exe.manifest.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <assembly xmlns="urn:schemas-microsoft-com:asm.v1" manifestVersion="1.0">
     <assemblyIdentity
-            version="0.14.4.0"
+            version="0.14.5.0"
             processorArchitecture="*"
             name="netclient.exe"
             type="win32"

netclient/versioninfo.json

@@ -29,7 +29,7 @@
         "OriginalFilename": "",
         "PrivateBuild": "",
         "ProductName": "Netclient",
-        "ProductVersion": "v0.14.4.0",
+        "ProductVersion": "v0.14.5.0",
         "SpecialBuild": ""
     },
     "VarFileInfo": {

netclient/wireguard/common.go

@@ -138,15 +138,9 @@ func InitWireguard(node *models.Node, privkey string, peers []wgtypes.PeerConfig
 		return err
 	}
 	defer wgclient.Close()
-	cfg, err := config.ReadConfig(node.Network)
-	if err != nil {
-		return err
-	}
 	//nodecfg := modcfg.Node
 	var ifacename string
-	if cfg.Node.Interface != "" {
-		ifacename = cfg.Node.Interface
-	} else if node.Interface != "" {
+	if node.Interface != "" {
 		ifacename = node.Interface
 	} else {
 		return fmt.Errorf("no interface to configure")
@@ -154,14 +148,7 @@ func InitWireguard(node *models.Node, privkey string, peers []wgtypes.PeerConfig
 	if node.PrimaryAddress() == "" {
 		return fmt.Errorf("no address to configure")
 	}
-	logger.Log(1, "turn on UDP hole punching (dynamic port setting)? "+cfg.Node.UDPHolePunch)
-	if node.UDPHolePunch == "yes" {
-		node.ListenPort = 0
-	} else {
-		//get available port based on current default
-		node.ListenPort, err = ncutils.GetFreePort(node.ListenPort)
-	}
-	if err := WriteWgConfig(&cfg.Node, key.String(), peers); err != nil {
+	if err := WriteWgConfig(node, key.String(), peers); err != nil {
 		logger.Log(1, "error writing wg conf file: ", err.Error())
 		return err
 	}
@@ -222,7 +209,7 @@ func InitWireguard(node *models.Node, privkey string, peers []wgtypes.PeerConfig
 
 	//ipv4
 	if node.Address != "" {
-		_, cidr, cidrErr := net.ParseCIDR(cfg.NetworkSettings.AddressRange)
+		_, cidr, cidrErr := net.ParseCIDR(node.NetworkSettings.AddressRange)
 		if cidrErr == nil {
 			local.SetCIDRRoute(ifacename, node.Address, cidr)
 		} else {
@@ -232,13 +219,12 @@ func InitWireguard(node *models.Node, privkey string, peers []wgtypes.PeerConfig
 	}
 	if node.Address6 != "" {
 		//ipv6
-		_, cidr, cidrErr := net.ParseCIDR(cfg.NetworkSettings.AddressRange6)
+		_, cidr, cidrErr := net.ParseCIDR(node.NetworkSettings.AddressRange6)
 		if cidrErr == nil {
 			local.SetCIDRRoute(ifacename, node.Address6, cidr)
 		} else {
 			logger.Log(1, "could not set cidr route properly: ", cidrErr.Error())
 		}
-
 		local.SetCurrentPeerRoutes(ifacename, node.Address6, peers)
 	}
 	return err
@@ -251,27 +237,24 @@ func SetWGConfig(network string, peerupdate bool, peers []wgtypes.PeerConfig) er
 	if err != nil {
 		return err
 	}
-
-	nodecfg := cfg.Node
-
 	privkey, err := RetrievePrivKey(network)
 	if err != nil {
 		return err
 	}
 	if peerupdate && !ncutils.IsFreeBSD() && !(ncutils.IsLinux() && !ncutils.IsKernel()) {
 		var iface string
-		iface = nodecfg.Interface
+		iface = cfg.Node.Interface
 		if ncutils.IsMac() {
-			iface, err = local.GetMacIface(nodecfg.PrimaryAddress())
+			iface, err = local.GetMacIface(cfg.Node.PrimaryAddress())
 			if err != nil {
 				return err
 			}
 		}
-		err = SetPeers(iface, &nodecfg, peers)
+		err = SetPeers(iface, &cfg.Node, peers)
 	} else if peerupdate {
-		err = InitWireguard(&nodecfg, privkey, peers, true)
+		err = InitWireguard(&cfg.Node, privkey, peers, true)
 	} else {
-		err = InitWireguard(&nodecfg, privkey, peers, false)
+		err = InitWireguard(&cfg.Node, privkey, peers, false)
 	}
 
 	return err

netclient/wireguard/mac.go

@@ -208,6 +208,7 @@ func addRoute(addr string, iface string) error {
 // setConfig - sets configuration of the wireguard interface from the config file
 func setConfig(realIface string, confPath string) error {
 	confString := getConfig(confPath)
+	// pathFormatted := strings.Replace(confPath, " ", "\\ ", -1)
 	err := os.WriteFile(confPath+".tmp", []byte(confString), 0600)
 	if err != nil {
 		return err
@@ -219,6 +220,7 @@ func setConfig(realIface string, confPath string) error {
 
 // getConfig - gets config from config file and strips out incompatible fields
 func getConfig(path string) string {
+	// pathFormatted := strings.Replace(path, " ", "\\ ", -1)
 	var confCmd = "grep -v -e Address -e MTU -e PostUp -e PostDown "
 	confRaw, _ := ncutils.RunCmd(confCmd+path, false)
 	return confRaw

scripts/nm-quick.sh

@@ -133,7 +133,7 @@ echo "setting docker-compose..."
 
 mkdir -p /etc/netmaker
 
-wget -q -O /root/docker-compose.yml https://raw.githubusercontent.com/gravitl/netmaker/master/compose/docker-compose.traefik.yml
+wget -q -O /root/docker-compose.yml https://raw.githubusercontent.com/gravitl/netmaker/master/compose/docker-compose.yml
 sed -i "s/NETMAKER_BASE_DOMAIN/$NETMAKER_BASE_DOMAIN/g" /root/docker-compose.yml
 sed -i "s/SERVER_PUBLIC_IP/$SERVER_PUBLIC_IP/g" /root/docker-compose.yml
 sed -i "s/COREDNS_IP/$COREDNS_IP/g" /root/docker-compose.yml

servercfg/serverconf.go

@@ -221,26 +221,16 @@ func GetMQPort() string {
 	return port
 }
 
-// GetMQServerPort - get mq port for server
-func GetMQServerPort() string {
-	port := "1883" //default
-	if os.Getenv("MQ_SERVER_PORT") != "" {
-		port = os.Getenv("MQ_SERVER_PORT")
-	} else if config.Config.Server.MQServerPort != "" {
-		port = config.Config.Server.MQServerPort
-	}
-	return port
-}
-
 // GetMessageQueueEndpoint - gets the message queue endpoint
-func GetMessageQueueEndpoint() string {
+func GetMessageQueueEndpoint() (string, bool) {
 	host, _ := GetPublicIP()
 	if os.Getenv("MQ_HOST") != "" {
 		host = os.Getenv("MQ_HOST")
 	} else if config.Config.Server.MQHOST != "" {
 		host = config.Config.Server.MQHOST
 	}
-	return host + ":" + GetMQServerPort()
+	secure := strings.Contains(host, "mqtts") || strings.Contains(host, "ssl")
+	return host + ":" + GetMQServerPort(), secure
 }
 
 // GetMasterKey - gets the configured master key of server
@@ -542,18 +532,31 @@ func GetServerCheckinInterval() int64 {
 }
 
 // GetAuthProviderInfo = gets the oauth provider info
-func GetAuthProviderInfo() []string {
+func GetAuthProviderInfo() (pi []string) {
 	var authProvider = ""
+
+	defer func() {
+		if authProvider == "oidc" {
+			if os.Getenv("OIDC_ISSUER") != "" {
+				pi = append(pi, os.Getenv("OIDC_ISSUER"))
+			} else if config.Config.Server.OIDCIssuer != "" {
+				pi = append(pi, config.Config.Server.OIDCIssuer)
+			} else {
+				pi = []string{"", "", ""}
+			}
+		}
+	}()
+
 	if os.Getenv("AUTH_PROVIDER") != "" && os.Getenv("CLIENT_ID") != "" && os.Getenv("CLIENT_SECRET") != "" {
 		authProvider = strings.ToLower(os.Getenv("AUTH_PROVIDER"))
-		if authProvider == "google" || authProvider == "azure-ad" || authProvider == "github" {
+		if authProvider == "google" || authProvider == "azure-ad" || authProvider == "github" || authProvider == "oidc" {
 			return []string{authProvider, os.Getenv("CLIENT_ID"), os.Getenv("CLIENT_SECRET")}
 		} else {
 			authProvider = ""
 		}
 	} else if config.Config.Server.AuthProvider != "" && config.Config.Server.ClientID != "" && config.Config.Server.ClientSecret != "" {
 		authProvider = strings.ToLower(config.Config.Server.AuthProvider)
-		if authProvider == "google" || authProvider == "azure-ad" || authProvider == "github" {
+		if authProvider == "google" || authProvider == "azure-ad" || authProvider == "github" || authProvider == "oidc" {
 			return []string{authProvider, config.Config.Server.ClientID, config.Config.Server.ClientSecret}
 		}
 	}
@@ -575,3 +578,14 @@ func GetAzureTenant() string {
 func GetRce() bool {
 	return os.Getenv("RCE") == "on" || config.Config.Server.RCE == "on"
 }
+
+// GetMQServerPort - get mq port for server
+func GetMQServerPort() string {
+	port := "1883" //default
+	if os.Getenv("MQ_SERVER_PORT") != "" {
+		port = os.Getenv("MQ_SERVER_PORT")
+	} else if config.Config.Server.MQServerPort != "" {
+		port = config.Config.Server.MQServerPort
+	}
+	return port
+}

serverctl/tls.go

@@ -0,0 +1,140 @@
+package serverctl
+
+import (
+	"crypto/ed25519"
+	ssl "crypto/tls"
+	"crypto/x509"
+	"encoding/json"
+	"encoding/pem"
+	"errors"
+	"fmt"
+
+	"github.com/gravitl/netmaker/database"
+	"github.com/gravitl/netmaker/tls"
+)
+
+// TlsConfig - holds this servers TLS conf in memory
+var TlsConfig ssl.Config
+
+// SaveCert - save a certificate to file and DB
+func SaveCert(path, name string, cert *x509.Certificate) error {
+	if err := SaveCertToDB(name, cert); err != nil {
+		return err
+	}
+	return tls.SaveCertToFile(path, name, cert)
+}
+
+// SaveCertToDB - save a certificate to the certs database
+func SaveCertToDB(name string, cert *x509.Certificate) error {
+	if certBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: cert.Raw,
+	}); len(certBytes) > 0 {
+		data, err := json.Marshal(&certBytes)
+		if err != nil {
+			return fmt.Errorf("failed to marshal certificate - %v ", err)
+		}
+		return database.Insert(name, string(data), database.CERTS_TABLE_NAME)
+	} else {
+		return fmt.Errorf("failed to write cert to DB - %s ", name)
+	}
+}
+
+// SaveKey - save a private key (ed25519) to file and DB
+func SaveKey(path, name string, key ed25519.PrivateKey) error {
+	if err := SaveKeyToDB(name, key); err != nil {
+		return err
+	}
+	return tls.SaveKeyToFile(path, name, key)
+}
+
+// SaveKeyToDB - save a private key (ed25519) to the specified path
+func SaveKeyToDB(name string, key ed25519.PrivateKey) error {
+	privBytes, err := x509.MarshalPKCS8PrivateKey(key)
+	if err != nil {
+		return fmt.Errorf("failed to marshal key %v ", err)
+	}
+	if pemBytes := pem.EncodeToMemory(&pem.Block{
+		Type:  "PRIVATE KEY",
+		Bytes: privBytes,
+	}); len(pemBytes) > 0 {
+		data, err := json.Marshal(&pemBytes)
+		if err != nil {
+			return fmt.Errorf("failed to marshal key %v ", err)
+		}
+		return database.Insert(name, string(data), database.CERTS_TABLE_NAME)
+	} else {
+		return fmt.Errorf("failed to write key to DB - %v ", err)
+	}
+}
+
+// ReadCertFromDB - reads a certificate from the database
+func ReadCertFromDB(name string) (*x509.Certificate, error) {
+	certString, err := database.FetchRecord(database.CERTS_TABLE_NAME, name)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read file %w", err)
+	}
+	var certBytes []byte
+	if err = json.Unmarshal([]byte(certString), &certBytes); err != nil {
+		return nil, fmt.Errorf("unable to unmarshal db cert %w", err)
+	}
+	block, _ := pem.Decode(certBytes)
+	if block == nil || block.Type != "CERTIFICATE" {
+		return nil, errors.New("not a cert " + block.Type)
+	}
+	cert, err := x509.ParseCertificate(block.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse cert %w", err)
+	}
+	return cert, nil
+}
+
+// ReadKeyFromDB - reads a private key (ed25519) from the database
+func ReadKeyFromDB(name string) (*ed25519.PrivateKey, error) {
+	keyString, err := database.FetchRecord(database.CERTS_TABLE_NAME, name)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read key value from db - %w", err)
+	}
+	var bytes []byte
+	if err = json.Unmarshal([]byte(keyString), &bytes); err != nil {
+		return nil, fmt.Errorf("unable to unmarshal db key - %w", err)
+	}
+	keyBytes, _ := pem.Decode(bytes)
+	key, err := x509.ParsePKCS8PrivateKey(keyBytes.Bytes)
+	if err != nil {
+		return nil, fmt.Errorf("unable to parse key from DB -  %w", err)
+	}
+	private := key.(ed25519.PrivateKey)
+	return &private, nil
+}
+
+// SetClientTLSConf - saves client cert for servers to connect to MQ broker with
+func SetClientTLSConf(serverClientPemPath, serverClientKeyPath string, ca *x509.Certificate) error {
+	certpool := x509.NewCertPool()
+	if caData := pem.EncodeToMemory(&pem.Block{
+		Type:  "CERTIFICATE",
+		Bytes: ca.Raw,
+	}); len(caData) <= 0 {
+		return fmt.Errorf("could not encode CA cert to memory for server client")
+	} else {
+		ok := certpool.AppendCertsFromPEM(caData)
+		if !ok {
+			return fmt.Errorf("failed to append root cert to server client cert")
+		}
+	}
+	clientKeyPair, err := ssl.LoadX509KeyPair(serverClientPemPath, serverClientKeyPath)
+	if err != nil {
+		return err
+	}
+	certs := []ssl.Certificate{clientKeyPair}
+
+	TlsConfig = ssl.Config{
+		RootCAs:            certpool,
+		ClientAuth:         ssl.NoClientCert,
+		ClientCAs:          nil,
+		Certificates:       certs,
+		InsecureSkipVerify: false,
+	}
+
+	return nil
+}

tls/tls.go

@@ -17,8 +17,31 @@ import (
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
-// CERTTIFICAT_VALIDITY duration of certificate validity in days
-const CERTIFICATE_VALIDITY = 365
+const (
+	// CERTTIFICATE_VALIDITY duration of certificate validity in days
+	CERTIFICATE_VALIDITY = 365
+
+	// SERVER_KEY_NAME - name of server cert private key
+	SERVER_KEY_NAME = "server.key"
+
+	// ROOT_KEY_NAME - name of root cert private key
+	ROOT_KEY_NAME = "root.key"
+
+	// SERVER_PEM_NAME - name of server pem
+	SERVER_PEM_NAME = "server.pem"
+
+	// ROOT_PEM_NAME - name of root pem
+	ROOT_PEM_NAME = "root.pem"
+
+	// SERVER_CLIENT_PEM - the name of server client cert
+	SERVER_CLIENT_PEM = "serverclient.pem"
+
+	// SERVER_CLIENT_KEY - the name of server client key
+	SERVER_CLIENT_KEY = "serverclient.key"
+
+	// SERVER_CLIENT_ENTRY - the server client cert key for DB
+	SERVER_CLIENT_ENTRY = "servercliententry"
+)
 
 type (
 	// Key is the struct for an edwards representation point
@@ -189,8 +212,8 @@ func SaveRequest(path, name string, csr *x509.CertificateRequest) error {
 	return nil
 }
 
-// SaveCert save a certificate to the specified path
-func SaveCert(path, name string, cert *x509.Certificate) error {
+// SaveCertToFile save a certificate to the specified path
+func SaveCertToFile(path, name string, cert *x509.Certificate) error {
 	//certbytes, err := x509.ParseCertificate(cert)
 	if err := os.MkdirAll(path, 0600); err != nil {
 		return fmt.Errorf("failed to create dir %s %w", path, err)
@@ -209,8 +232,8 @@ func SaveCert(path, name string, cert *x509.Certificate) error {
 	return nil
 }
 
-// SaveKey save a private key (ed25519) to the specified path
-func SaveKey(path, name string, key ed25519.PrivateKey) error {
+// SaveKeyToFile save a private key (ed25519) to the certs database
+func SaveKeyToFile(path, name string, key ed25519.PrivateKey) error {
 	//func SaveKey(name string, key *ecdsa.PrivateKey) error {
 	if err := os.MkdirAll(path, 0600); err != nil {
 		return fmt.Errorf("failed to create dir %s %w", path, err)
@@ -233,8 +256,8 @@ func SaveKey(path, name string, key ed25519.PrivateKey) error {
 	return nil
 }
 
-// ReadCert reads a certificate from disk
-func ReadCert(name string) (*x509.Certificate, error) {
+// ReadCertFromFile reads a certificate from disk
+func ReadCertFromFile(name string) (*x509.Certificate, error) {
 	contents, err := os.ReadFile(name)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read file %w", err)
@@ -250,8 +273,8 @@ func ReadCert(name string) (*x509.Certificate, error) {
 	return cert, nil
 }
 
-// ReadKey reads a private key (ed25519) from disk
-func ReadKey(name string) (*ed25519.PrivateKey, error) {
+// ReadKeyFromFile reads a private key (ed25519) from disk
+func ReadKeyFromFile(name string) (*ed25519.PrivateKey, error) {
 	bytes, err := os.ReadFile(name)
 	if err != nil {
 		return nil, fmt.Errorf("unable to read file %w", err)