Merge branch 'develop' into dev-nm-quick-resolv

.github/workflows/buildandrelease.yml

@@ -33,20 +33,32 @@ jobs:
 
       - name: Build
         run: |
+          env CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netmaker main.go
           cd netclient
-          env GOOS=linux GOARCH=amd64 go build -o build/netclient main.go
-          env GOOS=linux GOARCH=arm GOARM=5 go build -o build/netclient-arm5/netclient main.go
-          env GOOS=linux GOARCH=arm GOARM=6 go build -o build/netclient-arm6/netclient main.go
-          env GOOS=linux GOARCH=arm GOARM=7 go build -o build/netclient-arm7/netclient main.go
-          env GOOS=linux GOARCH=arm64 go build -o build/netclient-arm64/netclient main.go
-          env GOOS=linux GOARCH=mipsle go build -ldflags "-s -w" -o build/netclient-mipsle/netclient main.go && upx build/netclient-mipsle/netclient
-          env CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -o build/netclient-freebsd/netclient main.go
-          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm GOARM=5 go build -o build/netclient-freebsd-arm5/netclient main.go
-          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm GOARM=6 go build -o build/netclient-freebsd-arm6/netclient main.go
-          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm GOARM=7 go build -o build/netclient-freebsd-arm7/netclient main.go
-          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm64 go build -o build/netclient-freebsd-arm64/netclient main.go
-          env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -o build/netclient-darwin/netclient main.go
+          env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient main.go
+          env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=5 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-arm5/netclient main.go
+          env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=6 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-arm6/netclient main.go
+          env CGO_ENABLED=0 GOOS=linux GOARCH=arm GOARM=7 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-arm7/netclient main.go
+          env CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-arm64/netclient main.go
+          env CGO_ENABLED=0 GOOS=linux GOARCH=mipsle go build -ldflags "-s -w -X 'main.version=$NETMAKER_VERSION'" -o build/netclient-mipsle/netclient main.go && upx build/netclient-mipsle/netclient
+          env CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build -ldflags="-X 'main.Version=${NETMAKER_VERSION}'" -o build/netclient-freebsd/netclient main.go
+          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm GOARM=5 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-freebsd-arm5/netclient main.go
+          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm GOARM=6 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-freebsd-arm6/netclient main.go
+          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm GOARM=7 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-freebsd-arm7/netclient main.go
+          env CGO_ENABLED=0 GOOS=freebsd GOARCH=arm64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-freebsd-arm64/netclient main.go
+          env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-darwin/netclient main.go
+          env CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -ldflags="-X 'main.version=${NETMAKER_VERSION}'" -o build/netclient-darwin-arm64/netclient main.go
           
+      - name: Upload netmaker x86 to Release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: build/netmaker
+          tag: ${{ env.NETMAKER_VERSION }}
+          overwrite: true
+          prerelease: true
+          asset_name: netmaker
+
       - name: Upload x86 to Release
         uses: svenstaro/upload-release-action@v2
         with:
@@ -166,3 +178,13 @@ jobs:
           overwrite: true
           prerelease: true
           asset_name: netclient-darwin
+
+      - name: Upload darwin-arm64 to Release
+        uses: svenstaro/upload-release-action@v2
+        with:
+          repo_token: ${{ secrets.GITHUB_TOKEN }}
+          file: netclient/build/netclient-darwin-arm64/netclient
+          tag: ${{ env.NETMAKER_VERSION }}
+          overwrite: true
+          prerelease: true
+          asset_name: netclient-darwin-arm64

.github/workflows/publish-docker.yml

@@ -18,10 +18,10 @@ jobs:
         run: |
             if [[ -n "${{ github.event.inputs.tag }}" ]]; then
               TAG=${{ github.event.inputs.tag }}
-            elif [[ "${{ github.base_ref }}" == 'master' ]]; then
+            elif [[ "${{ github.ref_name }}" == 'master' ]]; then
               TAG="latest"
             else
-              TAG="${{ github.base_ref }}"
+              TAG="${{ github.ref_name }}"
             fi
             echo "TAG=${TAG}" >> $GITHUB_ENV
       - 
@@ -47,6 +47,7 @@ jobs:
           load: true
           platforms: linux/amd64
           tags: ${{ env.TAG }}
+          build-args: version=${{ env.TAG }}
       -
         name: Test x86
         run: |
@@ -61,6 +62,7 @@ jobs:
           load: true
           platforms: linux/arm64
           tags: ${{ env.TAG }}
+          build-args: version=${{ env.TAG }}
       -
         name: Test arm
         run: |
@@ -75,3 +77,4 @@ jobs:
           platforms: linux/amd64, linux/arm64
           push: true
           tags: ${{ github.repository }}:${{ env.TAG }}
+          build-args: version=${{ env.TAG }}

.github/workflows/publish-netclient-docker.yml

@@ -0,0 +1,83 @@
+name: Publish Netclient Docker
+
+on:
+  workflow_dispatch:
+    inputs:
+      tag:
+        description: 'docker tag'
+        required: true
+  release:
+    types: [published]
+
+jobs:
+  docker:
+    runs-on: ubuntu-latest
+    steps:
+      - 
+        name: Set tag
+        run: |
+            if [[ -n "${{ github.event.inputs.tag }}" ]]; then
+              TAG=${{ github.event.inputs.tag }}
+            elif [[ "${{ github.ref_name }}" == 'master' ]]; then
+              TAG="latest"
+            else
+              TAG="${{ github.ref_name }}"
+            fi
+            echo "TAG=${TAG}" >> $GITHUB_ENV
+      - 
+        name: Checkout
+        uses: actions/checkout@v2
+      - 
+        name: Set up QEMU
+        uses: docker/setup-qemu-action@v1
+      - 
+        name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v1
+      - 
+        name: Login to DockerHub
+        uses: docker/login-action@v1
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - 
+        name: Build x86 and export to Docker
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          load: true
+          platforms: linux/amd64
+          file: ./docker/Dockerfile-netclient-multiarch
+          tags: ${{ env.TAG }}
+          build-args: version=${{ env.TAG }}  
+      -
+        name: Test x86
+        run: |
+            docker run --rm ${{ env.TAG }}&
+            sleep 10
+            kill %1
+      -
+        name: Build arm and export to Docker
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          load: true
+          platforms: linux/arm64
+          file: ./docker/Dockerfile-netclient-multiarch
+          tags: ${{ env.TAG }}
+          build-args: version=${{ env.TAG }}  
+      -
+        name: Test arm
+        run: |
+            docker run --rm ${{ env.TAG }}&
+            sleep 10
+            kill %1
+      -
+        name: Build and push
+        uses: docker/build-push-action@v2
+        with:
+          context: .
+          platforms: linux/amd64, linux/arm64
+          file: ./docker/Dockerfile-netclient-multiarch
+          push: true
+          tags: gravitl/netclient:${{ env.TAG }}
+          build-args: version=${{ env.TAG }}  

.github/workflows/purgeGHCR.yml

@@ -0,0 +1,28 @@
+name: Purge untagged images from GHCR
+
+on: 
+    workflow_dispatch
+
+jobs:
+  purge:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Prune Netmaker
+        uses: vlaurin/action-ghcr-prune@main
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          organization: gravitl
+          container: netmaker
+          dry-run: true # Dry-run first, then change to `false`
+          untagged: true
+      - name: Prune Netclient
+        uses: vlaurin/action-ghcr-prune@main
+        with:
+          token: ${{ secrets.GITHUB_TOKEN }}
+          organization: gravitl
+          container: netclient
+          dry-run: true # Dry-run first, then change to `false`
+          untagged: true
+          

.github/workflows/test-artifacts.yml

@@ -33,23 +33,85 @@ jobs:
                   platforms: linux/amd64
                   push: true
                   tags: ghcr.io/${{ github.repository }}:testing
+                  build-args: version=testing
+    docker-netclient:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout                 
+              uses: actions/checkout@v2
+            - name: Setup Go
+              uses: actions/setup-go@v2
+              with:
+                  go-version: 1.17
+            - name: Set up QEMU
+              uses: docker/setup-qemu-action@v1
+            - name: Set up Docker Buildx
+              uses: docker/setup-buildx-action@v1
+            - name: Login to DockerHub
+              uses: docker/login-action@v1
+              with:
+                  registry: ghcr.io
+                  username: ${{ github.actor }}
+                  password: ${{ secrets.GITHUB_TOKEN }}
+            - name: Build and Push test
+              uses: docker/build-push-action@v2
+              with:
+                  context: .
+                  platforms: linux/amd64, linux/arm64
+                  file: ./docker/Dockerfile-netclient-multiarch
+                  push: true
+                  tags: ghcr.io/gravitl/netclient:testing
+                  build-args: version=testing                
     netclient:
         runs-on: ubuntu-latest
         steps:
             - name: Checkout
               uses: actions/checkout@v2
+            - name: Setup Go
+              uses: actions/setup-go@v2
+              with:
+                go-version: 1.17
             - name: build client
               run: |
                 cd netclient
-                go build -ldflags="-X 'main.version=testing'" .
-                curl -H 'Authorization: Bearer ${{ secrets.NUSAK_MASTERKEY }}' \
-                -H 'Content-Type: multipart/form-data' --form upload='./netclient' \
-                -X POST https://dashboard.nusak.ca/api/file/netclient
-    #deploy:
-         #runs-on: ubuntu-latest
-         #steps:
-            #- name: Deploy Testing Server and Client(s)
-            #  run: |
-            #      curl -X POST https://api.github.com/mattkasun/terraform-test/dispatches \
-            #      -H 'Accept: application/vnd.github.everest-preview+json' \
-            #      -u ${{ secrets.ACCESS_TOKEN }} 
+                env CGO_ENABLED=0 go build -ldflags="-X 'main.version=testing'" -o build/netclient
+            - name: deploy
+              uses: mdallasanta/[email protected]
+              with:
+                local: ./netclient/build/netclient                            # Local file path - REQUIRED false - DEFAULT ./
+                remote: /var/www/files/testing/                               # Remote file path - REQUIRED false - DEFAULT ~/
+                host: fileserver.clustercat.com                               # Remote server address - REQUIRED true
+                #port: ${{secrets.PORT}}                                      # Remote server port - REQUIRED false - DEFAULT 22
+                user: root                                                    # Remote server user - REQUIRED true
+                #password: ${{secrets.PASSWORD}}                              # User password - REQUIRED at least one of "password" or "key" 
+                key: ${{secrets.TESTING_SSH_KEY}}                             # Remote server private key - REQUIRED at least one of "password" or "key" 
+                #pre_upload: echo "This will be executed before the upload!"  # Command to run via ssh before scp upload - REQUIRED false
+                #post_upload: echo "This will be executed after the upload!"  # Command to run via ssh after scp upload - REQUIRED false
+                #ssh_options: -o StrictHostKeyChecking=no                     # A set of ssh_option separated by -o - REQUIRED false - DEFAULT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                #scp_options: -v                                              # Flags to use during scp - REQUIRED false - DEFAULT ''
+    netmaker:
+        runs-on: ubuntu-latest
+        steps:
+            - name: Checkout
+              uses: actions/checkout@v2
+            - name: Setup Go
+              uses: actions/setup-go@v2
+              with:
+                go-version: 1.17
+            - name: build server
+              run:
+                env CGO_ENABLED=1 go build -ldflags="-X 'main.version=testing'" -o build/netmaker
+            - name: deploy
+              uses: mdallasanta/[email protected]
+              with:
+                local: ./build/netmaker                                       # Local file path - REQUIRED false - DEFAULT ./
+                remote: /var/www/files/testing/                               # Remote file path - REQUIRED false - DEFAULT ~/
+                host: fileserver.clustercat.com                               # Remote server address - REQUIRED true
+                #port: ${{secrets.PORT}}                                      # Remote server port - REQUIRED false - DEFAULT 22
+                user: root                                                    # Remote server user - REQUIRED true
+                #password: ${{secrets.PASSWORD}}                              # User password - REQUIRED at least one of "password" or "key" 
+                key: ${{secrets.TESTING_SSH_KEY}}                             # Remote server private key - REQUIRED at least one of "password" or "key" 
+                #pre_upload: echo "This will be executed before the upload!"  # Command to run via ssh before scp upload - REQUIRED false
+                #post_upload: echo "This will be executed after the upload!"  # Command to run via ssh after scp upload - REQUIRED false
+                #ssh_options: -o StrictHostKeyChecking=no                     # A set of ssh_option separated by -o - REQUIRED false - DEFAULT -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
+                #scp_options: -v                                              # Flags to use during scp - REQUIRED false - DEFAULT ''

.github/workflows/test.yml

@@ -4,6 +4,23 @@ on:
   push:
 
 jobs:
+  build:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v2
+      - name: Setup Go
+        uses: actions/setup-go@v2
+        with:
+            go-version: 1.17
+      - name: Build
+        run: |
+         env CGO_ENABLED=1 GOOS=linux GOARCH=amd64 go build main.go
+         cd netclient
+         env CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build main.go
+         env CGO_ENABLED=0 GOOS=freebsd GOARCH=amd64 go build main.go
+         env CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build main.go
+         env CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build main.go
   tests:
     env:
       DATABASE: sqlite

.gitignore

@@ -17,3 +17,4 @@ config/dnsconfig/
 data/
 .vscode/
 .idea/
+

CONTRIBUTING.md

@@ -0,0 +1,71 @@
+# How to contribute to Netmaker
+
+Hey! We're glad you're here. We need your help making Netmaker as great as it can be.
+
+If you haven't already, come chat with us in [Discord](https://discord.gg/zRb9Vfhk8A). We can help you find the right thing to work on.
+
+Before you start contributing, take a moment to check here if it makes sense.
+
+#### **Did you find a bug?**
+
+* Search on on GitHub under [Issues](https://github.com/gravitl/netmaker/issues) to make sure the bug was not already discovered.
+
+* If you don't find an open issue that addresses the problem, you can [open a new one](https://github.com/gravitl/netmaker/issues/new). 
+
+* If you're creating a new issue, include a **title and clear description**, as much relevant information as possible **including logs**, and an explanation/output demonstrating expected behavior vs. actual behavior. Make sure to specify the **version of netmaker/netclient.** If it's a server issue, describe the environment where the server is running. If it's a client issue, give us the operating system and any relevant environment factors (CGNAT, 4g router, etc).
+
+* Respond to team queries in a timely manner, since stale issues will be closed.
+
+#### **Did you write a patch that fixes a bug?**
+
+* Open a new GitHub pull request with the patch **against the 'develop' branch**.
+
+* The PR should clearly describe the problem and solution. Include an issue number if possible.
+
+* Make sure to add comments for any new functions
+
+#### **Did you fix whitespace, format code, or make a purely cosmetic patch?**
+
+Cosmetic changes that do not add substantial useability, stability, functionality, or testability to the code base will not be accepted. The calculation is simple. If it will take more time to merge and test than it took you to make and submit the code, it is likely not worthwhile (execptions exist of course for critical errors with easy fixes).
+
+#### **Do you want to add a new feature to Netmaker?**
+
+* Once again, join the [Discord](https://discord.gg/zRb9Vfhk8A)! Bring it up there and we can discuss. Even if you do not know what you want to build, but you want to build something, we can help you choose something from the roadmap.
+
+#### **Do you want to contribute to Netmaker documentation?**
+
+* Make sure your documentation compiles correctly
+
+* You will need [sphinx](https://www.sphinx-doc.org/en/master/usage/installation.html) and the [material theme](https://github.com/bashtage/sphinx-material/) to run the documentation locally.
+
+* Once the above plugins are installed, you can navigate to the **docs** directory and run **make html**
+
+* View the compiled files (start with index.html under _build) in your browser and make sure your changes look correct before submitting.
+
+
+## Submitting Changes
+
+* Please label your branch using our convention: **purpose_version_thing-you-did**. Purpose is either feature, bugfix, or hotfix.
+
+* Examples: feature_v0.9.5_widget, bugfix_v0.8.2_ipv6-changes
+
+* Please open a [Pull Request](https://github.com/gravitl/netmaker/compare/develop...master?expand=1) against the develop branch with your branch which clearly describes everything you've done and references any related GitHub issues. 
+
+* You will need to sign the CLA in order for us to accept your changes (a bot should appear asking you to sign)
+
+* Please respond to any feedback in a timely manner. Stale PR's will be closed periodically.
+
+## Coding conventions
+
+Take a look around the code to get a feel for how we're doing things.
+
+* Use private functions where possible
+* Use the custom loggers for log messages
+* Comment any new public functions
+
+
+
+
+## Thanks for taking the time to read this! You're awesome, and we look forward to working with you!
+  
+-The Netmaker Team

Dockerfile

@@ -1,17 +1,16 @@
 #first stage - builder
 FROM golang:1.17-alpine as builder
-ARG version
+ARG version 
 RUN apk add build-base
 WORKDIR /app
 COPY . .
 ENV GO111MODULE=auto
 
-# RUN GOOS=linux CGO_ENABLED=1 go build -tags debug -ldflags="-s -X 'main.version=$version'" -o netmaker main.go
-RUN GOOS=linux CGO_ENABLED=1 go build -ldflags="-s -X 'main.version=$version'" -o netmaker main.go
+RUN GOOS=linux CGO_ENABLED=1 go build -ldflags="-s -X 'main.version=${version}'" -o netmaker main.go
 FROM alpine:3.14.3
 
 # add a c lib
-RUN apk add gcompat iptables
+RUN apk add gcompat iptables wireguard-tools
 # set the working directory
 WORKDIR /root/
 RUN mkdir -p /etc/netclient/config

README.md

@@ -8,7 +8,7 @@
 
 <p align="center">
   <a href="https://github.com/gravitl/netmaker/releases">
-    <img src="https://img.shields.io/badge/Version-0.9.3-informational?style=flat-square" />
+    <img src="https://img.shields.io/badge/Version-0.11.1-informational?style=flat-square" />
   </a>
   <a href="https://hub.docker.com/r/gravitl/netmaker/tags">
     <img src="https://img.shields.io/docker/pulls/gravitl/netmaker" />
@@ -43,9 +43,9 @@
 2. Open ports 443, 80, 53, and 51821-51830/udp on the VM firewall and in cloud security settings.
 3. Run the script **(see below for optional configurations)**:
 
-`sudo wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | bash`
+`wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash`
 
-<img src="./docs/images/install-server.gif" width="50%" /><img src="./docs/images/visit-website.gif" width="50%" />
+<img src="./docs/images/visit-website.gif" width="50%" /><img src="./docs/images/graph-readme.gif" width="50%" />
 
 Upon completion, the logs will display the instructions to connect various devices. These can also be retrieved from the UI under "Access Keys."
 
@@ -55,11 +55,11 @@ After installing Netmaker, check out the [Walkthrough](https://itnext.io/getting
 
 **Deploy a "Hub-And-Spoke VPN" on the server**  
 *This will configure a standard VPN (non-meshed) for private internet access, with 10 clients (-c).*  
-`sudo wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | bash -s -- -v true -c 10`  
+`wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash -s -- -v true -c 10`  
 
 **Specify Domain and Email**  
 *Make sure your wildcard domain is pointing towards the server ip.*  
-`sudo wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | bash -s -- -d mynetmaker.domain.com -e [email protected]`  
+`wget -qO - https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/nm-quick.sh | sudo bash -s -- -d mynetmaker.domain.com -e [email protected]`  
 
 **Script Options**  
 ```

SECURITY.md

@@ -0,0 +1,12 @@
+# Security Policy
+
+Netmaker is reliant on secure networking. If you find a vulnerability or bug please report it.
+Depending on complexity or severity, the Gravitl team may compensate (aka. bug bounty) the reporter. 
+However, there is no official bug bounty program up yet for the Netmaker project.
+
+## Supported Versions
+- We currently are only able to support work on the latest version(s)
+
+## Reporting a Vulnerability
+
+Please report security issues to `[email protected]`

compose/docker-compose.contained.yml

@@ -3,15 +3,18 @@ version: "3.4"
 services:
   netmaker:
     container_name: netmaker
-    image: gravitl/netmaker:v0.9.3
+    image: gravitl/netmaker:v0.11.1
     volumes:
       - dnsconfig:/root/config/dnsconfig
-      - /usr/bin/wg:/usr/bin/wg
       - sqldata:/root/data
     cap_add: 
       - NET_ADMIN
+      - NET_RAW
+      - SYS_MODULE
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
     restart: always
-    privileged: true
     environment:
       SERVER_HOST: "SERVER_PUBLIC_IP"
       SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
@@ -25,11 +28,15 @@ services:
       GRPC_PORT: "50051"
       CLIENT_MODE: "on"
       MASTER_KEY: "REPLACE_MASTER_KEY"
-      SERVER_GRPC_WIREGUARD: "off"
       CORS_ALLOWED_ORIGIN: "*"
       DISPLAY_KEYS: "on"
       DATABASE: "sqlite"
       NODE_ID: "netmaker-server-1"
+      MQ_HOST: "mq"
+      HOST_NETWORK: "off"
+      MANAGE_IPTABLES: "on"
+      PORT_FORWARD_SERVICES: "mq,dns,ssh"
+      VERBOSITY: "1"
     ports:
       - "51821-51830:51821-51830/udp"
       - "8081:8081"
@@ -38,7 +45,7 @@ services:
     container_name: netmaker-ui
     depends_on:
       - netmaker
-    image: gravitl/netmaker-ui:v0.9.3
+    image: gravitl/netmaker-ui:v0.11.1
     links:
       - "netmaker:api"
     ports:
@@ -53,9 +60,6 @@ services:
     command: -conf /root/dnsconfig/Corefile
     container_name: coredns
     restart: always
-    ports:
-      - "COREDNS_IP:53:53/udp"
-      - "COREDNS_IP:53:53/tcp"
     volumes:
       - dnsconfig:/root/dnsconfig
   caddy:
@@ -68,9 +72,21 @@ services:
       # - $PWD/site:/srv # you could also serve a static site in site folder
       - caddy_data:/data
       - caddy_conf:/config
+  mq:
+    image: eclipse-mosquitto:2.0.14
+    container_name: mq
+    restart: unless-stopped
+    ports:
+      - "1883:1883"
+    volumes:
+      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - mosquitto_data:/mosquitto/data
+      - mosquitto_logs:/mosquitto/log
 volumes:
   caddy_data: {}
   caddy_conf: {}
   sqldata: {}
   dnsconfig: {}
+  mosquitto_data: {}
+  mosquitto_logs: {}
 

compose/docker-compose.caddy.yml → compose/docker-compose.hostnetwork.yml

@@ -3,21 +3,18 @@ version: "3.4"
 services:
   netmaker:
     container_name: netmaker
-    image: gravitl/netmaker:v0.9.3
+    image: gravitl/netmaker:v0.11.1
     volumes:
-      - /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket
-      - /run/systemd/system:/run/systemd/system
-      - /etc/systemd/system:/etc/systemd/system
-      - /sys/fs/cgroup:/sys/fs/cgroup
-      - /usr/bin/wg:/usr/bin/wg
       - dnsconfig:/root/config/dnsconfig
+      - /usr/bin/wg:/usr/bin/wg
       - sqldata:/root/data
-    cap_add: 
+      - /run/xtables.lock:/run/xtables.lock
+    cap_add:
       - NET_ADMIN
-      - SYS_ADMIN
-    restart: always
+      - NET_RAW
+      - SYS_MODULE
     network_mode: host
-    privileged: true
+    restart: always
     environment:
       SERVER_HOST: "SERVER_PUBLIC_IP"
       SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
@@ -35,12 +32,16 @@ services:
       CORS_ALLOWED_ORIGIN: "*"
       DISPLAY_KEYS: "on"
       DATABASE: "sqlite"
+      HOST_NETWORK: "on"
       NODE_ID: "netmaker-server-1"
+      MANAGE_IPTABLES: "on"
+      PORT_FORWARD_SERVICES: ""
+      VERBOSITY: "1"
   netmaker-ui:
     container_name: netmaker-ui
     depends_on:
       - netmaker
-    image: gravitl/netmaker-ui:v0.9.3
+    image: gravitl/netmaker-ui:0.11.1
     links:
       - "netmaker:api"
     ports:
@@ -56,8 +57,8 @@ services:
     container_name: coredns
     restart: always
     ports:
-      - "COREDNS_IP:53:53/udp"
-      - "COREDNS_IP:53:53/tcp"
+      - "53053:53/udp"
+      - "53053:53/tcp"
     volumes:
       - dnsconfig:/root/dnsconfig
   caddy:
@@ -70,8 +71,20 @@ services:
       # - $PWD/site:/srv # you could also serve a static site in site folder
       - caddy_data:/data
       - caddy_conf:/config
+  mq:
+    image: eclipse-mosquitto:2.0.14
+    container_name: mq
+    restart: unless-stopped
+    ports:
+      - "1883:1883"
+    volumes:
+      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - mosquitto_data:/mosquitto/data
+      - mosquitto_logs:/mosquitto/log
 volumes:
   caddy_data: {}
   caddy_conf: {}
   sqldata: {}
   dnsconfig: {}
+  mosquitto_data: {}
+  mosquitto_logs: {}

compose/docker-compose.nocaddy.yml

@@ -0,0 +1,83 @@
+version: "3.4"
+
+services:
+  netmaker:
+    container_name: netmaker
+    image: gravitl/netmaker:v0.11.1
+    volumes:
+      - dnsconfig:/root/config/dnsconfig
+      - sqldata:/root/data
+    cap_add: 
+      - NET_ADMIN
+      - NET_RAW
+      - SYS_MODULE
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
+    restart: always
+    environment:
+      SERVER_HOST: "SERVER_PUBLIC_IP"
+      SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
+      SERVER_GRPC_CONN_STRING: "grpc.NETMAKER_BASE_DOMAIN:443"
+      COREDNS_ADDR: "SERVER_PUBLIC_IP"
+      GRPC_SSL: "on"
+      DNS_MODE: "on"
+      SERVER_HTTP_HOST: "api.NETMAKER_BASE_DOMAIN"
+      SERVER_GRPC_HOST: "grpc.NETMAKER_BASE_DOMAIN"
+      API_PORT: "8081"
+      GRPC_PORT: "50051"
+      CLIENT_MODE: "on"
+      MASTER_KEY: "REPLACE_MASTER_KEY"
+      CORS_ALLOWED_ORIGIN: "*"
+      DISPLAY_KEYS: "on"
+      DATABASE: "sqlite"
+      NODE_ID: "netmaker-server-1"
+      MQ_HOST: "mq"
+      HOST_NETWORK: "off"
+      MANAGE_IPTABLES: "on"
+      PORT_FORWARD_SERVICES: "mq,dns,ssh"
+      VERBOSITY: "1"
+    ports:
+      - "51821-51830:51821-51830/udp"
+      - "8081:8081"
+      - "50051:50051"
+  netmaker-ui:
+    container_name: netmaker-ui
+    depends_on:
+      - netmaker
+    image: gravitl/netmaker-ui:v0.11.1
+    links:
+      - "netmaker:api"
+    ports:
+      - "8082:80"
+    environment:
+      BACKEND_URL: "https://api.NETMAKER_BASE_DOMAIN"
+    restart: always
+  coredns:
+    depends_on:
+      - netmaker 
+    image: coredns/coredns
+    command: -conf /root/dnsconfig/Corefile
+    container_name: coredns
+    restart: always
+    ports:
+      - "COREDNS_IP:53:53/udp"
+      - "COREDNS_IP:53:53/tcp"
+    volumes:
+      - dnsconfig:/root/dnsconfig
+  mq:
+    image: eclipse-mosquitto:2.0.14
+    container_name: mq
+    restart: unless-stopped
+    ports:
+      - "1883:1883"
+    volumes:
+      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - mosquitto_data:/mosquitto/data
+      - mosquitto_logs:/mosquitto/log
+volumes:
+  sqldata: {}
+  dnsconfig: {}
+  mosquitto_data: {}
+  mosquitto_logs: {}
+

compose/docker-compose.nodns.yml

@@ -3,14 +3,18 @@ version: "3.4"
 services:
   netmaker:
     container_name: netmaker
-    image: gravitl/netmaker:v0.9.3
+    image: gravitl/netmaker:v0.11.1
     volumes:
-      - /usr/bin/wg:/usr/bin/wg
+      - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
     cap_add: 
       - NET_ADMIN
+      - NET_RAW
+      - SYS_MODULE
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
     restart: always
-    privileged: true
     environment:
       SERVER_HOST: "SERVER_PUBLIC_IP"
       SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
@@ -23,11 +27,16 @@ services:
       API_PORT: "8081"
       GRPC_PORT: "50051"
       CLIENT_MODE: "on"
-      DISPLAY_KEYS: "on"
       MASTER_KEY: "REPLACE_MASTER_KEY"
-      SERVER_GRPC_WIREGUARD: "off"
       CORS_ALLOWED_ORIGIN: "*"
+      DISPLAY_KEYS: "on"
       DATABASE: "sqlite"
+      NODE_ID: "netmaker-server-1"
+      MQ_HOST: "mq"
+      HOST_NETWORK: "off"
+      MANAGE_IPTABLES: "on"
+      PORT_FORWARD_SERVICES: "mq,dns,ssh"
+      VERBOSITY: "1"
     ports:
       - "51821-51830:51821-51830/udp"
       - "8081:8081"
@@ -36,7 +45,7 @@ services:
     container_name: netmaker-ui
     depends_on:
       - netmaker
-    image: gravitl/netmaker-ui:v0.9.3
+    image: gravitl/netmaker-ui:v0.11.1
     links:
       - "netmaker:api"
     ports:
@@ -54,7 +63,19 @@ services:
       # - $PWD/site:/srv # you could also serve a static site in site folder
       - caddy_data:/data
       - caddy_conf:/config
+  mq:
+    image: eclipse-mosquitto:2.0.14
+    container_name: mq
+    restart: unless-stopped
+    ports:
+      - "1883:1883"
+    volumes:
+      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - mosquitto_data:/mosquitto/data
+      - mosquitto_logs:/mosquitto/log
 volumes:
   caddy_data: {}
   caddy_conf: {}
   sqldata: {}
+  mosquitto_data: {}
+  mosquitto_logs: {}

compose/docker-compose.reference.yml

@@ -1,24 +1,20 @@
 services:
-  rqlite:
-    container_name: rqlite
-    image: rqlite/rqlite
-    network_mode: host
-    restart: always
-    volumes:
-      - sqldata:/rqlite/file/data
   netmaker: # The Primary Server for running Netmaker
-    privileged: true # Necessary to run sudo/root level commands on host system. Take out if not running with CLIENT_MODE=on
+    privileged: true # Necessary to run sudo/root level commands on host system. Likely using this if running with host networking on.
     container_name: netmaker
-    depends_on:
-      - rqlite
-    image: gravitl/netmaker:v0.9.3
+    image: gravitl/netmaker:v0.11.1
     volumes: # Volume mounts necessary for CLIENT_MODE to control wireguard networking on host (except dnsconfig, which is where dns config files are stored for use by CoreDNS)
       - dnsconfig:/root/config/dnsconfig # Netmaker writes Corefile to this location, which gets mounted by CoreDNS for DNS configuration.
-      - /usr/bin/wg:/usr/bin/wg
-    cap_add: # Necessary for CLIENT_MODE. Should be removed if turned off. 
+      - sqldata:/root/data
+    cap_add: # Necessary capabilities to set iptables when running in container
       - NET_ADMIN
+      - NET_RAW
+      - SYS_MODULE
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
     restart: always
-    network_mode: host # Necessary for CLIENT_MODE. Should be removed if turned off, but then need to add port mappings
+    network_mode: host # Must configure with very particular settngs for host networking to work. Do not just set on!
     environment:
       SERVER_HOST: "" # All the Docker Compose files pre-populate this with HOST_IP, which you replace as part of the install instructions. This will set both HTTP and GRPC host.
       SERVER_HTTP_HOST: "127.0.0.1" # Overrides SERVER_HOST if set. Useful for making HTTP and GRPC available via different interfaces/networks.
@@ -37,28 +33,58 @@ services:
       DISPLAY_KEYS: "on" # Show keys permanently in UI (until deleted) as opposed to 1-time display.
       SERVER_API_CONN_STRING: "" # Changes the api connection string. IP:PORT format. By default is empty and uses SERVER_HOST:API_PORT
       SERVER_GRPC_CONN_STRING: "" # Changes the grpc connection string. IP:PORT format. By default is empty and uses SERVER_HOST:GRPC_PORT
+      RCE: "off" # Enables setting PostUp and PostDown (arbitrary commands) on nodes from the server. Off by default.
+      NODE_ID: "" # Sets the name/id of the nodes that the server creates. Necessary for HA configurations to identify between servers (for instance, netmaker-1, netmaker-2, etc). For non-HA deployments, is not necessary.
+      TELEMETRY: "on" # Whether or not to send telemetry data to help improve Netmaker. Switch to "off" to opt out of sending telemetry.
+      MQ_HOST: "mq" # the address of the mq server. If running from docker compose it will be "mq". Otherwise, need to input address. If using "host networking", it will find and detect the IP of the mq container.
+      HOST_NETWORK: "off" # whether or not host networking is turned on. Only turn on if configured for host networking (see docker-compose.hostnetwork.yml). Will set host-level settings like iptables.
+      MANAGE_IPTABLES: "on" # set iptables on the machine being managed in order to forward properly from wireguard interface to MQ and other services listed in "port forward services"
+      PORT_FORWARD_SERVICES: "mq,dns,ssh" #services for which to configure port forwarding on the machine. 'ssh' forwards port 22 over wireguard, enabling ssh to server over wireguard. dns enables private dns over wireguard. mq enables mq.
   netmaker-ui: # The Netmaker UI Component
     container_name: netmaker-ui
     depends_on:
       - netmaker
-    image: gravitl/netmaker-ui:v0.9.3
+    image: gravitl/netmaker-ui:v0.11.1
     links:
       - "netmaker:api"
     ports:
       - "8082:80"
     environment:
       BACKEND_URL: "http://HOST_IP:8081" # URL where UI will send API requests. Change based on SERVER_HOST, SERVER_HTTP_HOST, and API_PORT
+  restart: always
   coredns: # The DNS Server. Remove this section if DNS_MODE="off"
     depends_on:
       - netmaker 
     image: coredns/coredns
-    command: -conf /root/dnsconfig/Corefile # Config location for Corefile. This is the path of file which is also mounted to Netmaker for modification.
+    command: -conf /root/dnsconfig/Corefile
     container_name: coredns
     restart: always
-    ports:
-      - "53:53/udp" # Likely needs to run at port 53 for adequate nameserver usage.
     volumes:
       - dnsconfig:/root/dnsconfig
+  caddy:
+    image: caddy:latest
+    container_name: caddy
+    restart: unless-stopped
+    network_mode: host # Wants ports 80 and 443!
+    volumes:
+      - /root/Caddyfile:/etc/caddy/Caddyfile
+      # - $PWD/site:/srv # you could also serve a static site in site folder
+      - caddy_data:/data
+      - caddy_conf:/config  
+  mq: # the MQTT broker for netmaker
+    image: eclipse-mosquitto:2.0.14
+    container_name: mq
+    restart: unless-stopped
+    ports:
+      - "1883:1883"
+    volumes:
+      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf # need to pull conf file from github before running (under docker/mosquitto.conf)
+      - mosquitto_data:/mosquitto/data
+      - mosquitto_logs:/mosquitto/log
 volumes:
-  sqldata: {}
-  dnsconfig: {}
+  caddy_data: {} # storage for caddy data
+  caddy_conf: {} # storage for caddy configuration file
+  sqldata: {} # storage for embedded sqlite
+  dnsconfig: {} # storage for coredns
+  mosquitto_data: {} # storage for mqtt data
+  mosquitto_logs: {} # storage for mqtt logs

compose/docker-compose.yml

@@ -3,21 +3,18 @@ version: "3.4"
 services:
   netmaker:
     container_name: netmaker
-    image: gravitl/netmaker:v0.9.3
+    image: gravitl/netmaker:v0.11.1
     volumes:
-      - /var/run/dbus/system_bus_socket:/var/run/dbus/system_bus_socket
-      - /run/systemd/system:/run/systemd/system
-      - /etc/systemd/system:/etc/systemd/system
-      - /sys/fs/cgroup:/sys/fs/cgroup
-      - /usr/bin/wg:/usr/bin/wg
       - dnsconfig:/root/config/dnsconfig
       - sqldata:/root/data
     cap_add: 
       - NET_ADMIN
-      - SYS_ADMIN
+      - NET_RAW
+      - SYS_MODULE
+    sysctls:
+      - net.ipv4.ip_forward=1
+      - net.ipv4.conf.all.src_valid_mark=1
     restart: always
-    network_mode: host
-    privileged: true
     environment:
       SERVER_HOST: "SERVER_PUBLIC_IP"
       SERVER_API_CONN_STRING: "api.NETMAKER_BASE_DOMAIN:443"
@@ -31,16 +28,24 @@ services:
       GRPC_PORT: "50051"
       CLIENT_MODE: "on"
       MASTER_KEY: "REPLACE_MASTER_KEY"
-      SERVER_GRPC_WIREGUARD: "off"
       CORS_ALLOWED_ORIGIN: "*"
-      DATABASE: "sqlite"
       DISPLAY_KEYS: "on"
+      DATABASE: "sqlite"
       NODE_ID: "netmaker-server-1"
+      MQ_HOST: "mq"
+      HOST_NETWORK: "off"
+      MANAGE_IPTABLES: "on"
+      PORT_FORWARD_SERVICES: "mq,dns,ssh"
+      VERBOSITY: "1"
+    ports:
+      - "51821-51830:51821-51830/udp"
+      - "8081:8081"
+      - "50051:50051"
   netmaker-ui:
     container_name: netmaker-ui
     depends_on:
       - netmaker
-    image: gravitl/netmaker-ui:v0.9.3
+    image: gravitl/netmaker-ui:v0.11.1
     links:
       - "netmaker:api"
     ports:
@@ -55,11 +60,33 @@ services:
     command: -conf /root/dnsconfig/Corefile
     container_name: coredns
     restart: always
-    ports:
-      - "COREDNS_IP:53:53/udp"
-      - "COREDNS_IP:53:53/tcp"
     volumes:
       - dnsconfig:/root/dnsconfig
+  caddy:
+    image: caddy:latest
+    container_name: caddy
+    restart: unless-stopped
+    network_mode: host # Wants ports 80 and 443!
+    volumes:
+      - /root/Caddyfile:/etc/caddy/Caddyfile
+      # - $PWD/site:/srv # you could also serve a static site in site folder
+      - caddy_data:/data
+      - caddy_conf:/config
+  mq:
+    image: eclipse-mosquitto:2.0.14
+    container_name: mq
+    restart: unless-stopped
+    ports:
+      - "1883:1883"
+    volumes:
+      - /root/mosquitto.conf:/mosquitto/config/mosquitto.conf
+      - mosquitto_data:/mosquitto/data
+      - mosquitto_logs:/mosquitto/log
 volumes:
+  caddy_data: {}
+  caddy_conf: {}
   sqldata: {}
   dnsconfig: {}
+  mosquitto_data: {}
+  mosquitto_logs: {}
+

config/config.go

@@ -43,23 +43,22 @@ type ServerConfig struct {
 	GRPCHost              string `yaml:"grpchost"`
 	GRPCPort              string `yaml:"grpcport"`
 	GRPCSecure            string `yaml:"grpcsecure"`
+	MQHOST                string `yaml:"mqhost"`
 	MasterKey             string `yaml:"masterkey"`
 	DNSKey                string `yaml:"dnskey"`
 	AllowedOrigin         string `yaml:"allowedorigin"`
 	NodeID                string `yaml:"nodeid"`
 	RestBackend           string `yaml:"restbackend"`
 	AgentBackend          string `yaml:"agentbackend"`
+	MessageQueueBackend   string `yaml:"messagequeuebackend"`
 	ClientMode            string `yaml:"clientmode"`
 	DNSMode               string `yaml:"dnsmode"`
-	SplitDNS              string `yaml:"splitdns"`
 	DisableRemoteIPCheck  string `yaml:"disableremoteipcheck"`
-	DisableDefaultNet     string `yaml:"disabledefaultnet"`
 	GRPCSSL               string `yaml:"grpcssl"`
 	Version               string `yaml:"version"`
 	SQLConn               string `yaml:"sqlconn"`
 	Platform              string `yaml:"platform"`
 	Database              string `yaml:"database"`
-	CheckinInterval       string `yaml:"checkininterval"`
 	DefaultNodeLimit      int32  `yaml:"defaultnodelimit"`
 	Verbosity             int32  `yaml:"verbosity"`
 	ServerCheckinInterval int64  `yaml:"servercheckininterval"`
@@ -70,6 +69,14 @@ type ServerConfig struct {
 	DisplayKeys           string `yaml:"displaykeys"`
 	AzureTenant           string `yaml:"azuretenant"`
 	RCE                   string `yaml:"rce"`
+	Debug                 bool   `yaml:"debug"`
+	Telemetry             string `yaml:"telemetry"`
+	ManageIPTables        string `yaml:"manageiptables"`
+	PortForwardServices   string `yaml:"portforwardservices"`
+	HostNetwork           string `yaml:"hostnetwork"`
+	CommsCIDR             string `yaml:"commscidr"`
+	MQPort                string `yaml:"mqport"`
+	CommsID               string `yaml:"commsid"`
 }
 
 // SQLConfig - Generic SQL Config

config/environments/dev.yaml

@@ -11,4 +11,5 @@ server:
   dnsmode: "" # defaults to "on" or DNS_MODE (if set)
   sqlconn: "" # defaults to "http://" or SQL_CONN (if set)
   disableremoteipcheck: "" # defaults to "false" or DISABLE_REMOTE_IP_CHECK (if set)
-  version: "" # version of server
+  version: "" # version of server
+  rce: "" # defaults to "off"

controllers/auth_grpc.go

@@ -72,7 +72,7 @@ func grpcAuthorize(ctx context.Context) error {
 
 	authToken := authHeader[0]
 
-	mac, network, err := logic.VerifyToken(authToken)
+	nodeID, _, network, err := logic.VerifyToken(authToken)
 	if err != nil {
 		return err
 	}
@@ -82,10 +82,10 @@ func grpcAuthorize(ctx context.Context) error {
 	if err != nil {
 		return status.Errorf(codes.Unauthenticated, "Unauthorized. Network does not exist: "+network)
 	}
-	emptynode := models.Node{}
-	node, err := logic.GetNodeByMacAddress(network, mac)
+	node, err := logic.GetNodeByID(nodeID)
 	if database.IsEmptyRecord(err) {
-		if node, err = logic.GetDeletedNodeByMacAddress(network, mac); err == nil {
+		// == DELETE replace logic after 2 major version updates ==
+		if node, err = logic.GetDeletedNodeByID(node.ID); err == nil {
 			if functions.RemoveDeletedNode(node.ID) {
 				return status.Errorf(codes.Unauthenticated, models.NODE_DELETE)
 			}
@@ -93,7 +93,7 @@ func grpcAuthorize(ctx context.Context) error {
 		}
 		return status.Errorf(codes.Unauthenticated, "Empty record")
 	}
-	if err != nil || node.MacAddress == emptynode.MacAddress {
+	if err != nil || node.ID == "" {
 		return status.Errorf(codes.Unauthenticated, "Node does not exist.")
 	}
 
@@ -106,42 +106,54 @@ func grpcAuthorize(ctx context.Context) error {
 // Login - node authenticates using its password and retrieves a JWT for authorization.
 func (s *NodeServiceServer) Login(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
 
-	//out := new(LoginResponse)
-	var reqNode models.Node
-	if err := json.Unmarshal([]byte(req.Data), &reqNode); err != nil {
+	var reqNode, err = getNodeFromRequestData(req.Data)
+	if err != nil {
 		return nil, err
 	}
 
-	macaddress := reqNode.MacAddress
+	nodeID := reqNode.ID
 	network := reqNode.Network
 	password := reqNode.Password
+	macaddress := reqNode.MacAddress
 
 	var result models.NodeAuth
-	var err error
-	// err := errors.New("generic server error")
 
-	if macaddress == "" {
+	if nodeID == "" {
 		//TODO: Set Error  response
-		err = errors.New("missing mac address")
+		err = errors.New("missing node ID")
 		return nil, err
 	} else if password == "" {
 		err = errors.New("missing password")
 		return nil, err
 	} else {
-		//Search DB for node with Mac Address. Ignore pending nodes (they should not be able to authenticate with API until approved).
+		//Search DB for node with ID. Ignore pending nodes (they should not be able to authenticate with API until approved).
 		collection, err := database.FetchRecords(database.NODES_TABLE_NAME)
 		if err != nil {
 			return nil, err
 		}
+		var found = false
 		for _, value := range collection {
 			if err = json.Unmarshal([]byte(value), &result); err != nil {
 				continue // finish going through nodes
 			}
-			if result.MacAddress == macaddress && result.Network == network {
+			if result.ID == nodeID && result.Network == network {
+				found = true
 				break
 			}
 		}
 
+		if !found {
+			deletedNode, err := database.FetchRecord(database.DELETED_NODES_TABLE_NAME, nodeID)
+			if err != nil {
+				err = errors.New("node not found")
+				return nil, err
+			}
+			if err = json.Unmarshal([]byte(deletedNode), &result); err != nil {
+				err = errors.New("node data corrupted")
+				return nil, err
+			}
+		}
+
 		//compare password from request to stored password in database
 		//might be able to have a common hash (certificates?) and compare those so that a password isn't passed in in plain text...
 		//TODO: Consider a way of hashing the password client side before sending, or using certificates
@@ -150,7 +162,7 @@ func (s *NodeServiceServer) Login(ctx context.Context, req *nodepb.Object) (*nod
 			return nil, err
 		} else {
 			//Create a new JWT for the node
-			tokenString, err := logic.CreateJWT(macaddress, result.Network)
+			tokenString, err := logic.CreateJWT(result.ID, macaddress, result.Network)
 
 			if err != nil {
 				return nil, err

controllers/config/dnsconfig/netmaker.hosts

@@ -1 +1,2 @@
-10.0.0.1         node-4bukt.skynet
+10.0.0.1         testnode.skynet
+10.0.0.2         myhost.skynet

controllers/controller.go

@@ -15,6 +15,7 @@ import (
 	"github.com/gravitl/netmaker/servercfg"
 )
 
+// HttpHandlers - handler functions for REST interactions
 var HttpHandlers = []interface{}{
 	nodeHandlers,
 	userHandlers,
@@ -23,7 +24,6 @@ var HttpHandlers = []interface{}{
 	fileHandlers,
 	serverHandlers,
 	extClientHandlers,
-	loggerHandlers,
 }
 
 // HandleRESTRequests - handles the rest requests
@@ -64,7 +64,7 @@ func HandleRESTRequests(wg *sync.WaitGroup) {
 
 	// After receiving CTRL+C Properly stop the server
 	logger.Log(0, "Stopping the REST server...")
-	srv.Shutdown(context.TODO())
 	logger.Log(0, "REST Server closed.")
 	logger.DumpFile(fmt.Sprintf("data/netmaker.log.%s", time.Now().Format(logger.TimeFormatDay)))
+	srv.Shutdown(context.TODO())
 }

controllers/ext_client.go

@@ -14,6 +14,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 	"github.com/skip2/go-qrcode"
 )
 
@@ -25,11 +26,11 @@ func extClientHandlers(r *mux.Router) {
 	r.HandleFunc("/api/extclients/{network}/{clientid}/{type}", securityCheck(false, http.HandlerFunc(getExtClientConf))).Methods("GET")
 	r.HandleFunc("/api/extclients/{network}/{clientid}", securityCheck(false, http.HandlerFunc(updateExtClient))).Methods("PUT")
 	r.HandleFunc("/api/extclients/{network}/{clientid}", securityCheck(false, http.HandlerFunc(deleteExtClient))).Methods("DELETE")
-	r.HandleFunc("/api/extclients/{network}/{macaddress}", securityCheck(false, http.HandlerFunc(createExtClient))).Methods("POST")
+	r.HandleFunc("/api/extclients/{network}/{nodeid}", securityCheck(false, http.HandlerFunc(createExtClient))).Methods("POST")
 }
 
-func checkIngressExists(network string, macaddress string) bool {
-	node, err := logic.GetNodeByMacAddress(network, macaddress)
+func checkIngressExists(nodeID string) bool {
+	node, err := logic.GetNodeByID(nodeID)
 	if err != nil {
 		return false
 	}
@@ -122,9 +123,9 @@ func getExtClientConf(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	gwnode, err := logic.GetNodeByMacAddress(client.Network, client.IngressGatewayID)
+	gwnode, err := logic.GetNodeByID(client.IngressGatewayID)
 	if err != nil {
-		logger.Log(1, fmt.Sprintf("%s %s %s", r.Header.Get("user"), "Could not retrieve Ingress Gateway Node", client.IngressGatewayID))
+		logger.Log(1, r.Header.Get("user"), "Could not retrieve Ingress Gateway Node", client.IngressGatewayID)
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
@@ -211,8 +212,8 @@ func createExtClient(w http.ResponseWriter, r *http.Request) {
 	var params = mux.Vars(r)
 
 	networkName := params["network"]
-	macaddress := params["macaddress"]
-	ingressExists := checkIngressExists(networkName, macaddress)
+	nodeid := params["nodeid"]
+	ingressExists := checkIngressExists(nodeid)
 	if !ingressExists {
 		returnErrorResponse(w, r, formatError(errors.New("ingress does not exist"), "internal"))
 		return
@@ -220,8 +221,8 @@ func createExtClient(w http.ResponseWriter, r *http.Request) {
 
 	var extclient models.ExtClient
 	extclient.Network = networkName
-	extclient.IngressGatewayID = macaddress
-	node, err := logic.GetNodeByMacAddress(networkName, macaddress)
+	extclient.IngressGatewayID = nodeid
+	node, err := logic.GetNodeByID(nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
@@ -238,6 +239,10 @@ func createExtClient(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	w.WriteHeader(http.StatusOK)
+	err = mq.PublishExtPeerUpdate(&node)
+	if err != nil {
+		logger.Log(1, "error setting ext peers on "+nodeid+": "+err.Error())
+	}
 }
 
 func updateExtClient(w http.ResponseWriter, r *http.Request) {
@@ -282,13 +287,30 @@ func deleteExtClient(w http.ResponseWriter, r *http.Request) {
 	// get params
 	var params = mux.Vars(r)
 
-	err := logic.DeleteExtClient(params["network"], params["clientid"])
+	extclient, err := logic.GetExtClient(params["clientid"], params["network"])
+	if err != nil {
+		err = errors.New("Could not delete extclient " + params["clientid"])
+		returnErrorResponse(w, r, formatError(err, "internal"))
+		return
+	}
+	ingressnode, err := logic.GetNodeByID(extclient.IngressGatewayID)
+	if err != nil {
+		returnErrorResponse(w, r, formatError(err, "internal"))
+		return
+	}
+
+	err = logic.DeleteExtClient(params["network"], params["clientid"])
 
 	if err != nil {
 		err = errors.New("Could not delete extclient " + params["clientid"])
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
+
+	err = mq.PublishExtPeerUpdate(&ingressnode)
+	if err != nil {
+		logger.Log(1, "error setting ext peers on "+ingressnode.ID+": "+err.Error())
+	}
 	logger.Log(1, r.Header.Get("user"),
 		"Deleted extclient client", params["clientid"], "from network", params["network"])
 	returnSuccessResponse(w, r, params["clientid"]+" deleted.")

controllers/logger.go

@@ -1,23 +0,0 @@
-package controller
-
-import (
-	"fmt"
-	"net/http"
-	"time"
-
-	"github.com/gorilla/mux"
-	"github.com/gravitl/netmaker/logger"
-)
-
-func loggerHandlers(r *mux.Router) {
-	r.HandleFunc("/api/logs", securityCheck(true, http.HandlerFunc(getLogs))).Methods("GET")
-}
-
-func getLogs(w http.ResponseWriter, r *http.Request) {
-	var currentTime = time.Now().Format(logger.TimeFormatDay)
-	var currentFilePath = fmt.Sprintf("data/netmaker.log.%s", currentTime)
-	logger.DumpFile(currentFilePath)
-	logger.ResetLogs()
-	w.WriteHeader(http.StatusOK)
-	w.Write([]byte(logger.Retrieve(currentFilePath)))
-}

controllers/network.go

@@ -3,6 +3,7 @@ package controller
 import (
 	"encoding/json"
 	"errors"
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -11,11 +12,15 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/servercfg"
 	"github.com/gravitl/netmaker/serverctl"
 )
 
+// ALL_NETWORK_ACCESS - represents all networks
 const ALL_NETWORK_ACCESS = "THIS_USER_HAS_ALL"
+
+// NO_NETWORKS_PRESENT - represents no networks
 const NO_NETWORKS_PRESENT = "THIS_USER_HAS_NONE"
 
 func networkHandlers(r *mux.Router) {
@@ -25,7 +30,7 @@ func networkHandlers(r *mux.Router) {
 	r.HandleFunc("/api/networks/{networkname}", securityCheck(false, http.HandlerFunc(updateNetwork))).Methods("PUT")
 	r.HandleFunc("/api/networks/{networkname}/nodelimit", securityCheck(true, http.HandlerFunc(updateNetworkNodeLimit))).Methods("PUT")
 	r.HandleFunc("/api/networks/{networkname}", securityCheck(true, http.HandlerFunc(deleteNetwork))).Methods("DELETE")
-	r.HandleFunc("/api/networks/{networkname}/keyupdate", securityCheck(false, http.HandlerFunc(keyUpdate))).Methods("POST")
+	r.HandleFunc("/api/networks/{networkname}/keyupdate", securityCheck(true, http.HandlerFunc(keyUpdate))).Methods("POST")
 	r.HandleFunc("/api/networks/{networkname}/keys", securityCheck(false, http.HandlerFunc(createAccessKey))).Methods("POST")
 	r.HandleFunc("/api/networks/{networkname}/keys", securityCheck(false, http.HandlerFunc(getAccessKeys))).Methods("GET")
 	r.HandleFunc("/api/networks/{networkname}/keys/{name}", securityCheck(false, http.HandlerFunc(deleteAccessKey))).Methods("DELETE")
@@ -42,7 +47,7 @@ func getNetworks(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	allnetworks := []models.Network{}
-	err := errors.New("Networks Error")
+	var err error
 	if networksSlice[0] == ALL_NETWORK_ACCESS {
 		allnetworks, err = logic.GetNetworks()
 		if err != nil && !database.IsEmptyRecord(err) {
@@ -63,6 +68,7 @@ func getNetworks(w http.ResponseWriter, r *http.Request) {
 			allnetworks[i] = net
 		}
 	}
+
 	logger.Log(2, r.Header.Get("user"), "fetched networks.")
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(allnetworks)
@@ -74,6 +80,10 @@ func getNetwork(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	var params = mux.Vars(r)
 	netname := params["networkname"]
+	if isCommsEdit(w, r, netname) {
+		return
+	}
+
 	network, err := logic.GetNetwork(netname)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
@@ -91,6 +101,10 @@ func keyUpdate(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	var params = mux.Vars(r)
 	netname := params["networkname"]
+	if isCommsEdit(w, r, netname) {
+		return
+	}
+
 	network, err := logic.KeyUpdate(netname)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
@@ -99,6 +113,19 @@ func keyUpdate(w http.ResponseWriter, r *http.Request) {
 	logger.Log(2, r.Header.Get("user"), "updated key on network", netname)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(network)
+	nodes, err := logic.GetNetworkNodes(netname)
+	if err != nil {
+		logger.Log(2, "failed to retrieve network nodes for network", netname, err.Error())
+		return
+	}
+	for _, node := range nodes {
+		logger.Log(2, "updating node ", node.Name, " for a key update")
+		if node.IsServer != "yes" {
+			if err = mq.NodeUpdate(&node); err != nil {
+				logger.Log(1, "failed to send update to node during a network wide key update", node.Name, node.ID, err.Error())
+			}
+		}
+	}
 }
 
 // Update a network
@@ -107,6 +134,7 @@ func updateNetwork(w http.ResponseWriter, r *http.Request) {
 	var params = mux.Vars(r)
 	var network models.Network
 	netname := params["networkname"]
+
 	network, err := logic.GetParentNetwork(netname)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
@@ -124,7 +152,7 @@ func updateNetwork(w http.ResponseWriter, r *http.Request) {
 		newNetwork.DefaultPostUp = network.DefaultPostUp
 	}
 
-	rangeupdate, localrangeupdate, err := logic.UpdateNetwork(&network, &newNetwork)
+	rangeupdate, localrangeupdate, holepunchupdate, err := logic.UpdateNetwork(&network, &newNetwork)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "badrequest"))
 		return
@@ -149,6 +177,26 @@ func updateNetwork(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
+	if holepunchupdate {
+		err = logic.UpdateNetworkHolePunching(network.NetID, newNetwork.DefaultUDPHolePunch)
+		if err != nil {
+			returnErrorResponse(w, r, formatError(err, "internal"))
+			return
+		}
+	}
+	if rangeupdate || localrangeupdate || holepunchupdate {
+		nodes, err := logic.GetNetworkNodes(network.NetID)
+		if err != nil {
+			returnErrorResponse(w, r, formatError(err, "internal"))
+			return
+		}
+		for _, node := range nodes {
+			if err = mq.NodeUpdate(&node); err != nil {
+				logger.Log(1, "failed to send update to node during a network wide update", node.Name, node.ID, err.Error())
+			}
+		}
+	}
+
 	logger.Log(1, r.Header.Get("user"), "updated network", netname)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(newNetwork)
@@ -183,16 +231,19 @@ func updateNetworkNodeLimit(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(network)
 }
 
-//Delete a network
-//Will stop you if  there's any nodes associated
+// Delete a network
+// Will stop you if  there's any nodes associated
 func deleteNetwork(w http.ResponseWriter, r *http.Request) {
 	// Set header
 	w.Header().Set("Content-Type", "application/json")
 
 	var params = mux.Vars(r)
 	network := params["networkname"]
-	err := logic.DeleteNetwork(network)
+	if isCommsEdit(w, r, network) {
+		return
+	}
 
+	err := logic.DeleteNetwork(network)
 	if err != nil {
 		errtype := "badrequest"
 		if strings.Contains(err.Error(), "Node check failed") {
@@ -226,16 +277,17 @@ func createNetwork(w http.ResponseWriter, r *http.Request) {
 	}
 
 	if servercfg.IsClientMode() != "off" {
-		var success bool
-		success, err = serverctl.AddNetwork(&network)
-		if err != nil || !success {
+		var node models.Node
+		node, err = logic.ServerJoin(&network)
+		if err != nil {
 			logic.DeleteNetwork(network.NetID)
 			if err == nil {
-				err = errors.New("Failed to add server to network " + network.DisplayName)
+				err = errors.New("Failed to add server to network " + network.NetID)
 			}
 			returnErrorResponse(w, r, formatError(err, "internal"))
 			return
 		}
+		getServerAddrs(&node)
 	}
 
 	logger.Log(1, r.Header.Get("user"), "created network", network.NetID)
@@ -249,6 +301,9 @@ func createAccessKey(w http.ResponseWriter, r *http.Request) {
 	var accesskey models.AccessKey
 	//start here
 	netname := params["networkname"]
+	if isCommsEdit(w, r, netname) {
+		return
+	}
 	network, err := logic.GetParentNetwork(netname)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
@@ -301,3 +356,21 @@ func deleteAccessKey(w http.ResponseWriter, r *http.Request) {
 	logger.Log(1, r.Header.Get("user"), "deleted access key", keyname, "on network,", netname)
 	w.WriteHeader(http.StatusOK)
 }
+
+func isCommsEdit(w http.ResponseWriter, r *http.Request, netname string) bool {
+	if netname == serverctl.COMMS_NETID {
+		returnErrorResponse(w, r, formatError(fmt.Errorf("cannot access comms network"), "internal"))
+		return true
+	}
+	return false
+}
+
+func filterCommsNetwork(networks []models.Network) []models.Network {
+	var filterdNets []models.Network
+	for i := range networks {
+		if networks[i].IsComms != "yes" && networks[i].NetID != servercfg.GetCommsID() {
+			filterdNets = append(filterdNets, networks[i])
+		}
+	}
+	return filterdNets
+}

controllers/network_test.go

@@ -1,12 +1,13 @@
 package controller
 
 import (
+	"os"
 	"testing"
-	"time"
 
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/serverctl"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -23,7 +24,8 @@ func TestCreateNetwork(t *testing.T) {
 	var network models.Network
 	network.NetID = "skynet"
 	network.AddressRange = "10.0.0.1/24"
-	network.DisplayName = "mynetwork"
+	// if tests break - check here (removed displayname)
+	//network.DisplayName = "mynetwork"
 
 	err := logic.CreateNetwork(network)
 	assert.Nil(t, err)
@@ -60,20 +62,6 @@ func TestDeleteNetwork(t *testing.T) {
 	})
 }
 
-func TestKeyUpdate(t *testing.T) {
-	t.Skip() //test is failing on last assert  --- not sure why
-	database.InitializeDatabase()
-	createNet()
-	existing, err := logic.GetNetwork("skynet")
-	assert.Nil(t, err)
-	time.Sleep(time.Second * 1)
-	network, err := logic.KeyUpdate("skynet")
-	assert.Nil(t, err)
-	network, err = logic.GetNetwork("skynet")
-	assert.Nil(t, err)
-	assert.Greater(t, network.KeyUpdateTimeStamp, existing.KeyUpdateTimeStamp)
-}
-
 func TestCreateKey(t *testing.T) {
 	database.InitializeDatabase()
 	createNet()
@@ -193,6 +181,7 @@ func TestSecurityCheck(t *testing.T) {
 	//these seem to work but not sure it the tests are really testing the functionality
 
 	database.InitializeDatabase()
+	os.Setenv("MASTER_KEY", "secretkey")
 	t.Run("NoNetwork", func(t *testing.T) {
 		err, networks, username := SecurityCheck(false, "", "Bearer secretkey")
 		assert.Nil(t, err)
@@ -243,28 +232,6 @@ func TestValidateNetworkUpdate(t *testing.T) {
 			},
 			errMessage: "Field validation for 'AddressRange6' failed on the 'cidr' tag",
 		},
-
-		{
-			testname: "BadDisplayName",
-			network: models.Network{
-				DisplayName: "skynet*",
-			},
-			errMessage: "Field validation for 'DisplayName' failed on the 'alphanum' tag",
-		},
-		{
-			testname: "DisplayNameTooLong",
-			network: models.Network{
-				DisplayName: "Thisisareallylongdisplaynamethatistoolong",
-			},
-			errMessage: "Field validation for 'DisplayName' failed on the 'max' tag",
-		},
-		{
-			testname: "DisplayNameTooShort",
-			network: models.Network{
-				DisplayName: "1",
-			},
-			errMessage: "Field validation for 'DisplayName' failed on the 'min' tag",
-		},
 		{
 			testname: "InvalidNetID",
 			network: models.Network{
@@ -307,20 +274,6 @@ func TestValidateNetworkUpdate(t *testing.T) {
 			},
 			errMessage: "Field validation for 'LocalRange' failed on the 'cidr' tag",
 		},
-		{
-			testname: "CheckInIntervalTooBig",
-			network: models.Network{
-				DefaultCheckInInterval: 100001,
-			},
-			errMessage: "Field validation for 'DefaultCheckInInterval' failed on the 'max' tag",
-		},
-		{
-			testname: "CheckInIntervalTooSmall",
-			network: models.Network{
-				DefaultCheckInInterval: 1,
-			},
-			errMessage: "Field validation for 'DefaultCheckInInterval' failed on the 'min' tag",
-		},
 	}
 	for _, tc := range cases {
 		t.Run(tc.testname, func(t *testing.T) {
@@ -344,9 +297,9 @@ func createNet() {
 	var network models.Network
 	network.NetID = "skynet"
 	network.AddressRange = "10.0.0.1/24"
-	network.DisplayName = "mynetwork"
 	_, err := logic.GetNetwork("skynet")
 	if err != nil {
 		logic.CreateNetwork(network)
 	}
+	serverctl.InitializeCommsNetwork()
 }

controllers/node.go

@@ -2,6 +2,7 @@ package controller
 
 import (
 	"encoding/json"
+	"fmt"
 	"net/http"
 	"strings"
 
@@ -11,6 +12,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/crypto/bcrypt"
 )
@@ -19,16 +21,16 @@ func nodeHandlers(r *mux.Router) {
 
 	r.HandleFunc("/api/nodes", authorize(false, "user", http.HandlerFunc(getAllNodes))).Methods("GET")
 	r.HandleFunc("/api/nodes/{network}", authorize(true, "network", http.HandlerFunc(getNetworkNodes))).Methods("GET")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}", authorize(true, "node", http.HandlerFunc(getNode))).Methods("GET")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}", authorize(true, "node", http.HandlerFunc(updateNode))).Methods("PUT")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}", authorize(true, "node", http.HandlerFunc(deleteNode))).Methods("DELETE")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}/createrelay", authorize(true, "user", http.HandlerFunc(createRelay))).Methods("POST")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}/deleterelay", authorize(true, "user", http.HandlerFunc(deleteRelay))).Methods("DELETE")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}/creategateway", authorize(true, "user", http.HandlerFunc(createEgressGateway))).Methods("POST")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}/deletegateway", authorize(true, "user", http.HandlerFunc(deleteEgressGateway))).Methods("DELETE")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}/createingress", securityCheck(false, http.HandlerFunc(createIngressGateway))).Methods("POST")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}/deleteingress", securityCheck(false, http.HandlerFunc(deleteIngressGateway))).Methods("DELETE")
-	r.HandleFunc("/api/nodes/{network}/{macaddress}/approve", authorize(true, "user", http.HandlerFunc(uncordonNode))).Methods("POST")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}", authorize(true, "node", http.HandlerFunc(getNode))).Methods("GET")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}", authorize(true, "node", http.HandlerFunc(updateNode))).Methods("PUT")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}", authorize(true, "node", http.HandlerFunc(deleteNode))).Methods("DELETE")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}/createrelay", authorize(true, "user", http.HandlerFunc(createRelay))).Methods("POST")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}/deleterelay", authorize(true, "user", http.HandlerFunc(deleteRelay))).Methods("DELETE")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}/creategateway", authorize(true, "user", http.HandlerFunc(createEgressGateway))).Methods("POST")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}/deletegateway", authorize(true, "user", http.HandlerFunc(deleteEgressGateway))).Methods("DELETE")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}/createingress", securityCheck(false, http.HandlerFunc(createIngressGateway))).Methods("POST")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}/deleteingress", securityCheck(false, http.HandlerFunc(deleteIngressGateway))).Methods("DELETE")
+	r.HandleFunc("/api/nodes/{network}/{nodeid}/approve", authorize(true, "user", http.HandlerFunc(uncordonNode))).Methods("POST")
 	r.HandleFunc("/api/nodes/{network}", createNode).Methods("POST")
 	r.HandleFunc("/api/nodes/adm/{network}/lastmodified", authorize(true, "network", http.HandlerFunc(getLastModified))).Methods("GET")
 	r.HandleFunc("/api/nodes/adm/{network}/authenticate", authenticate).Methods("POST")
@@ -78,7 +80,7 @@ func authenticate(response http.ResponseWriter, request *http.Request) {
 				if err := json.Unmarshal([]byte(value), &result); err != nil {
 					continue
 				}
-				if result.MacAddress == authRequest.MacAddress && result.IsPending != "yes" && result.Network == networkname {
+				if (result.ID == authRequest.ID || result.MacAddress == authRequest.MacAddress) && result.IsPending != "yes" && result.Network == networkname {
 					break
 				}
 			}
@@ -97,7 +99,7 @@ func authenticate(response http.ResponseWriter, request *http.Request) {
 				returnErrorResponse(response, request, errorResponse)
 				return
 			} else {
-				tokenString, _ := logic.CreateJWT(authRequest.MacAddress, result.Network)
+				tokenString, _ := logic.CreateJWT(authRequest.ID, authRequest.MacAddress, result.Network)
 
 				if tokenString == "" {
 					errorResponse.Code = http.StatusBadRequest
@@ -177,21 +179,28 @@ func authorize(networkCheck bool, authNetwork string, next http.Handler) http.Ha
 			}
 
 			var isAuthorized = false
-			var macaddress = ""
+			var nodeID = ""
 			username, networks, isadmin, errN := logic.VerifyUserToken(authToken)
+			if errN != nil {
+				errorResponse = models.ErrorResponse{
+					Code: http.StatusUnauthorized, Message: "W1R3: Unauthorized, Invalid Token Processed.",
+				}
+				returnErrorResponse(w, r, errorResponse)
+				return
+			}
 			isnetadmin := isadmin
 			if errN == nil && isadmin {
-				macaddress = "mastermac"
+				nodeID = "mastermac"
 				isAuthorized = true
 				r.Header.Set("ismasterkey", "yes")
 			}
 			if !isadmin && params["network"] != "" {
-				if functions.SliceContains(networks, params["network"]) {
+				if logic.StringSliceContains(networks, params["network"]) {
 					isnetadmin = true
 				}
 			}
 			//The mastermac (login with masterkey from config) can do everything!! May be dangerous.
-			if macaddress == "mastermac" {
+			if nodeID == "mastermac" {
 				isAuthorized = true
 				r.Header.Set("ismasterkey", "yes")
 				//for everyone else, there's poor man's RBAC. The "cases" are defined in the routes in the handlers
@@ -201,12 +210,12 @@ func authorize(networkCheck bool, authNetwork string, next http.Handler) http.Ha
 				case "all":
 					isAuthorized = true
 				case "nodes":
-					isAuthorized = (macaddress != "") || isnetadmin
+					isAuthorized = (nodeID != "") || isnetadmin
 				case "network":
 					if isnetadmin {
 						isAuthorized = true
 					} else {
-						node, err := logic.GetNodeByMacAddress(params["network"], macaddress)
+						node, err := logic.GetNodeByID(nodeID)
 						if err != nil {
 							errorResponse = models.ErrorResponse{
 								Code: http.StatusUnauthorized, Message: "W1R3: Missing Auth Token.",
@@ -220,7 +229,7 @@ func authorize(networkCheck bool, authNetwork string, next http.Handler) http.Ha
 					if isnetadmin {
 						isAuthorized = true
 					} else {
-						isAuthorized = (macaddress == params["macaddress"])
+						isAuthorized = (nodeID == params["netid"])
 					}
 				case "user":
 					isAuthorized = true
@@ -254,6 +263,7 @@ func getNetworkNodes(w http.ResponseWriter, r *http.Request) {
 	var nodes []models.Node
 	var params = mux.Vars(r)
 	networkName := params["network"]
+
 	nodes, err := logic.GetNetworkNodes(networkName)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
@@ -290,9 +300,9 @@ func getAllNodes(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 	//Return all the nodes in JSON format
-	logger.Log(2, r.Header.Get("user"), "fetched nodes")
+	logger.Log(3, r.Header.Get("user"), "fetched all nodes they have access to")
 	w.WriteHeader(http.StatusOK)
-	json.NewEncoder(w).Encode(nodes)
+	json.NewEncoder(w).Encode(filterCommsNodes(nodes))
 }
 
 func getUsersNodes(user models.User) ([]models.Node, error) {
@@ -315,12 +325,16 @@ func getNode(w http.ResponseWriter, r *http.Request) {
 
 	var params = mux.Vars(r)
 
-	node, err := logic.GetNode(params["macaddress"], params["network"])
+	node, err := logic.GetNodeByID(params["nodeid"])
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
-	logger.Log(2, r.Header.Get("user"), "fetched node", params["macaddress"])
+	if logic.IsNodeInComms(&node) {
+		returnErrorResponse(w, r, formatError(err, "internal"))
+		return
+	}
+	logger.Log(2, r.Header.Get("user"), "fetched node", params["nodeid"])
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
 }
@@ -386,8 +400,8 @@ func createNode(w http.ResponseWriter, r *http.Request) {
 	validKey := logic.IsKeyValid(networkName, node.AccessKey)
 
 	if !validKey {
-		//Check to see if network will allow manual sign up
-		//may want to switch this up with the valid key check and avoid a DB call that way.
+		// Check to see if network will allow manual sign up
+		// may want to switch this up with the valid key check and avoid a DB call that way.
 		if network.AllowManualSignUp == "yes" {
 			node.IsPending = "yes"
 		} else {
@@ -404,17 +418,20 @@ func createNode(w http.ResponseWriter, r *http.Request) {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
+
 	logger.Log(1, r.Header.Get("user"), "created new node", node.Name, "on network", node.Network)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
+	runForceServerUpdate(&node)
 }
 
-//Takes node out of pending state
-//TODO: May want to use cordon/uncordon terminology instead of "ispending".
+// Takes node out of pending state
+// TODO: May want to use cordon/uncordon terminology instead of "ispending".
 func uncordonNode(w http.ResponseWriter, r *http.Request) {
 	var params = mux.Vars(r)
 	w.Header().Set("Content-Type", "application/json")
-	node, err := logic.UncordonNode(params["network"], params["macaddress"])
+	var nodeid = params["nodeid"]
+	node, err := logic.UncordonNode(nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
@@ -422,8 +439,12 @@ func uncordonNode(w http.ResponseWriter, r *http.Request) {
 	logger.Log(1, r.Header.Get("user"), "uncordoned node", node.Name)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode("SUCCESS")
+
+	runUpdates(&node, false)
 }
 
+// == EGRESS ==
+
 func createEgressGateway(w http.ResponseWriter, r *http.Request) {
 	var gateway models.EgressGatewayRequest
 	var params = mux.Vars(r)
@@ -434,30 +455,36 @@ func createEgressGateway(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	gateway.NetID = params["network"]
-	gateway.NodeID = params["macaddress"]
+	gateway.NodeID = params["nodeid"]
 	node, err := logic.CreateEgressGateway(gateway)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
+
 	logger.Log(1, r.Header.Get("user"), "created egress gateway on node", gateway.NodeID, "on network", gateway.NetID)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
+
+	runUpdates(&node, true)
 }
 
 func deleteEgressGateway(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	var params = mux.Vars(r)
-	nodeMac := params["macaddress"]
+	nodeid := params["nodeid"]
 	netid := params["network"]
-	node, err := logic.DeleteEgressGateway(netid, nodeMac)
+	node, err := logic.DeleteEgressGateway(netid, nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
-	logger.Log(1, r.Header.Get("user"), "deleted egress gateway", nodeMac, "on network", netid)
+
+	logger.Log(1, r.Header.Get("user"), "deleted egress gateway", nodeid, "on network", netid)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
+
+	runUpdates(&node, true)
 }
 
 // == INGRESS ==
@@ -465,30 +492,36 @@ func deleteEgressGateway(w http.ResponseWriter, r *http.Request) {
 func createIngressGateway(w http.ResponseWriter, r *http.Request) {
 	var params = mux.Vars(r)
 	w.Header().Set("Content-Type", "application/json")
-	nodeMac := params["macaddress"]
+	nodeid := params["nodeid"]
 	netid := params["network"]
-	node, err := logic.CreateIngressGateway(netid, nodeMac)
+	node, err := logic.CreateIngressGateway(netid, nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
-	logger.Log(1, r.Header.Get("user"), "created ingress gateway on node", nodeMac, "on network", netid)
+
+	logger.Log(1, r.Header.Get("user"), "created ingress gateway on node", nodeid, "on network", netid)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
+
+	runUpdates(&node, true)
 }
 
 func deleteIngressGateway(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	var params = mux.Vars(r)
-	nodeMac := params["macaddress"]
-	node, err := logic.DeleteIngressGateway(params["network"], nodeMac)
+	nodeid := params["nodeid"]
+	node, err := logic.DeleteIngressGateway(params["network"], nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
-	logger.Log(1, r.Header.Get("user"), "deleted ingress gateway", nodeMac)
+
+	logger.Log(1, r.Header.Get("user"), "deleted ingress gateway", nodeid)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
+
+	runUpdates(&node, true)
 }
 
 func updateNode(w http.ResponseWriter, r *http.Request) {
@@ -498,7 +531,7 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 
 	var node models.Node
 	//start here
-	node, err := logic.GetNodeByMacAddress(params["network"], params["macaddress"])
+	node, err := logic.GetNodeByID(params["nodeid"])
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
@@ -511,7 +544,6 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 		returnErrorResponse(w, r, formatError(err, "badrequest"))
 		return
 	}
-	newNode.PullChanges = "yes"
 	relayupdate := false
 	if node.IsRelay == "yes" && len(newNode.RelayAddrs) > 0 {
 		if len(newNode.RelayAddrs) != len(node.RelayAddrs) {
@@ -530,28 +562,34 @@ func updateNode(w http.ResponseWriter, r *http.Request) {
 		newNode.PostUp = node.PostUp
 	}
 
+	ifaceDelta := logic.IfaceDelta(&node, &newNode)
+
 	err = logic.UpdateNode(&node, &newNode)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
 	if relayupdate {
-		logic.UpdateRelay(node.Network, node.RelayAddrs, newNode.RelayAddrs)
+		updatenodes := logic.UpdateRelay(node.Network, node.RelayAddrs, newNode.RelayAddrs)
 		if err = logic.NetworkNodesUpdatePullChanges(node.Network); err != nil {
 			logger.Log(1, "error setting relay updates:", err.Error())
 		}
+		if len(updatenodes) > 0 {
+			for _, relayedNode := range updatenodes {
+				runUpdates(&relayedNode, false)
+			}
+		}
 	}
 
 	if servercfg.IsDNSMode() {
-		err = logic.SetDNS()
-	}
-	if err != nil {
-		returnErrorResponse(w, r, formatError(err, "internal"))
-		return
+		logic.SetDNS()
 	}
-	logger.Log(1, r.Header.Get("user"), "updated node", node.MacAddress, "on network", node.Network)
+
+	logger.Log(1, r.Header.Get("user"), "updated node", node.ID, "on network", node.Network)
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(newNode)
+
+	runUpdates(&newNode, ifaceDelta)
 }
 
 func deleteNode(w http.ResponseWriter, r *http.Request) {
@@ -560,17 +598,75 @@ func deleteNode(w http.ResponseWriter, r *http.Request) {
 
 	// get params
 	var params = mux.Vars(r)
-	var node, err = logic.GetNode(params["macaddress"], params["network"])
+	var nodeid = params["nodeid"]
+	var node, err = logic.GetNodeByID(nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "badrequest"))
 		return
 	}
-	err = logic.DeleteNode(&node, false)
+	if isServer(&node) {
+		returnErrorResponse(w, r, formatError(fmt.Errorf("cannot delete server node"), "badrequest"))
+		return
+	}
+	//send update to node to be deleted before deleting on server otherwise message cannot be sent
+	node.Action = models.NODE_DELETE
+
+	err = logic.DeleteNodeByID(&node, false)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
+	returnSuccessResponse(w, r, nodeid+" deleted.")
+
+	logger.Log(1, r.Header.Get("user"), "Deleted node", nodeid, "from network", params["network"])
+	runUpdates(&node, false)
+	runForceServerUpdate(&node)
+}
+
+func runUpdates(node *models.Node, ifaceDelta bool) {
+	go func() { // don't block http response
+		// publish node update if not server
+		if err := mq.NodeUpdate(node); err != nil {
+			logger.Log(1, "error publishing node update to node", node.Name, node.ID, err.Error())
+		}
+
+		if err := runServerUpdate(node, ifaceDelta); err != nil {
+			logger.Log(1, "error running server update", err.Error())
+		}
+	}()
+}
+
+// updates local peers for a server on a given node's network
+func runServerUpdate(node *models.Node, ifaceDelta bool) error {
 
-	logger.Log(1, r.Header.Get("user"), "Deleted node", params["macaddress"], "from network", params["network"])
-	returnSuccessResponse(w, r, params["macaddress"]+" deleted.")
+	if servercfg.IsClientMode() != "on" || !isServer(node) {
+		return nil
+	}
+
+	currentServerNode, err := logic.GetNetworkServerLocal(node.Network)
+	if err != nil {
+		return err
+	}
+
+	if ifaceDelta && logic.IsLeader(&currentServerNode) {
+		if err := mq.PublishPeerUpdate(&currentServerNode); err != nil {
+			logger.Log(1, "failed to publish peer update "+err.Error())
+		}
+	}
+
+	if err := logic.ServerUpdate(&currentServerNode, ifaceDelta); err != nil {
+		logger.Log(1, "server node:", currentServerNode.ID, "failed update")
+		return err
+	}
+	return nil
+}
+
+func filterCommsNodes(nodes []models.Node) []models.Node {
+	var filterdNodes []models.Node
+	for i := range nodes {
+		if !logic.IsNodeInComms(&nodes[i]) {
+			filterdNodes = append(filterdNodes, nodes[i])
+		}
+	}
+	return filterdNodes
 }

controllers/node_grpc.go

@@ -4,13 +4,17 @@ import (
 	"context"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"strings"
+	"time"
 
 	nodepb "github.com/gravitl/netmaker/grpc"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/servercfg"
+	"github.com/gravitl/netmaker/serverctl"
 )
 
 // NodeServiceServer - represents the service server for gRPC
@@ -20,20 +24,17 @@ type NodeServiceServer struct {
 
 // NodeServiceServer.ReadNode - reads node and responds with gRPC
 func (s *NodeServiceServer) ReadNode(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
-	// convert string id (from proto) to mongoDB ObjectId
-	macAndNetwork := strings.Split(req.Data, "###")
-
-	if len(macAndNetwork) != 2 {
-		return nil, errors.New("could not read node, invalid node id given")
-	}
-	node, err := logic.GetNode(macAndNetwork[0], macAndNetwork[1])
+	var node, err = getNodeFromRequestData(req.Data)
 	if err != nil {
 		return nil, err
 	}
+
 	node.NetworkSettings, err = logic.GetNetworkSettings(node.Network)
 	if err != nil {
 		return nil, err
 	}
+	getServerAddrs(&node)
+
 	node.SetLastCheckIn()
 	// Cast to ReadNodeRes type
 	nodeData, errN := json.Marshal(&node)
@@ -62,7 +63,6 @@ func (s *NodeServiceServer) CreateNode(ctx context.Context, req *nodepb.Object)
 	if err != nil {
 		return nil, err
 	}
-
 	if !validKey {
 		if node.NetworkSettings.AllowManualSignUp == "yes" {
 			node.IsPending = "yes"
@@ -70,6 +70,33 @@ func (s *NodeServiceServer) CreateNode(ctx context.Context, req *nodepb.Object)
 			return nil, errors.New("invalid key, and network does not allow no-key signups")
 		}
 	}
+	getServerAddrs(&node)
+
+	key, keyErr := logic.RetrievePublicTrafficKey()
+	if keyErr != nil {
+		logger.Log(0, "error retrieving key: ", keyErr.Error())
+		return nil, keyErr
+	}
+
+	if key == nil {
+		logger.Log(0, "error: server traffic key is nil")
+		return nil, fmt.Errorf("error: server traffic key is nil")
+	}
+	if node.TrafficKeys.Mine == nil {
+		logger.Log(0, "error: node traffic key is nil")
+		return nil, fmt.Errorf("error: node traffic key is nil")
+	}
+
+	node.TrafficKeys = models.TrafficKeys{
+		Mine:   node.TrafficKeys.Mine,
+		Server: key,
+	}
+
+	commID, err := logic.FetchCommsNetID()
+	if err != nil {
+		return nil, err
+	}
+	node.CommID = commID
 
 	err = logic.CreateNode(&node)
 	if err != nil {
@@ -86,25 +113,42 @@ func (s *NodeServiceServer) CreateNode(ctx context.Context, req *nodepb.Object)
 		Type: nodepb.NODE_TYPE,
 	}
 
-	err = logic.SetNetworkNodesLastModified(node.Network)
-	if err != nil {
-		return nil, err
-	}
+	runForceServerUpdate(&node)
+
+	go func(node *models.Node) {
+		if node.UDPHolePunch == "yes" {
+			var currentServerNode, getErr = logic.GetNetworkServerLeader(node.Network)
+			if getErr != nil {
+				return
+			}
+			for i := 0; i < 5; i++ {
+				if logic.HasPeerConnected(node) {
+					if logic.ShouldPublishPeerPorts(&currentServerNode) {
+						err = mq.PublishPeerUpdate(&currentServerNode)
+						if err != nil {
+							logger.Log(1, "error publishing port updates when node", node.Name, "joined")
+						}
+						break
+					}
+				}
+				time.Sleep(time.Second << 1) // allow time for client to startup
+			}
+		}
+	}(&node)
 
 	return response, nil
 }
 
 // NodeServiceServer.UpdateNode updates a node and responds over gRPC
+// DELETE ONE DAY - DEPRECATED
 func (s *NodeServiceServer) UpdateNode(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
-	// Get the node data from the request
+
 	var newnode models.Node
 	if err := json.Unmarshal([]byte(req.GetData()), &newnode); err != nil {
 		return nil, err
 	}
-	macaddress := newnode.MacAddress
-	networkName := newnode.Network
 
-	node, err := logic.GetNodeByMacAddress(networkName, macaddress)
+	node, err := logic.GetNodeByID(newnode.ID)
 	if err != nil {
 		return nil, err
 	}
@@ -122,29 +166,64 @@ func (s *NodeServiceServer) UpdateNode(ctx context.Context, req *nodepb.Object)
 	if err != nil {
 		return nil, err
 	}
+	getServerAddrs(&newnode)
+
 	nodeData, errN := json.Marshal(&newnode)
 	if errN != nil {
 		return nil, err
 	}
+
 	return &nodepb.Object{
 		Data: string(nodeData),
 		Type: nodepb.NODE_TYPE,
 	}, nil
 }
 
+func getServerAddrs(node *models.Node) {
+	serverNodes := logic.GetServerNodes(serverctl.COMMS_NETID)
+	//pubIP, _ := servercfg.GetPublicIP()
+	if len(serverNodes) == 0 {
+		if err := serverctl.SyncServerNetwork(serverctl.COMMS_NETID); err != nil {
+			return
+		}
+	}
+
+	var serverAddrs = make([]models.ServerAddr, 0)
+
+	for _, node := range serverNodes {
+		if node.Address != "" {
+			serverAddrs = append(serverAddrs, models.ServerAddr{
+				IsLeader: logic.IsLeader(&node),
+				Address:  node.Address,
+			})
+		}
+	}
+
+	networkSettings, _ := logic.GetParentNetwork(node.Network)
+	// TODO consolidate functionality around files
+	networkSettings.NodesLastModified = time.Now().Unix()
+	networkSettings.DefaultServerAddrs = serverAddrs
+	if err := logic.SaveNetwork(&networkSettings); err != nil {
+		logger.Log(1, "unable to save network on serverAddr update", err.Error())
+	}
+	node.NetworkSettings.DefaultServerAddrs = networkSettings.DefaultServerAddrs
+}
+
 // NodeServiceServer.DeleteNode - deletes a node and responds over gRPC
 func (s *NodeServiceServer) DeleteNode(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
-	nodeID := req.GetData()
-	var nodeInfo = strings.Split(nodeID, "###")
-	if len(nodeInfo) != 2 {
-		return nil, errors.New("node not found")
+
+	var node, err = getNodeFromRequestData(req.Data)
+	if err != nil {
+		return nil, err
 	}
-	var node, err = logic.GetNode(nodeInfo[0], nodeInfo[1])
-	err = logic.DeleteNode(&node, true)
+
+	err = logic.DeleteNodeByID(&node, true)
 	if err != nil {
 		return nil, err
 	}
 
+	runForceServerUpdate(&node)
+
 	return &nodepb.Object{
 		Data: "success",
 		Type: nodepb.STRING_TYPE,
@@ -153,49 +232,41 @@ func (s *NodeServiceServer) DeleteNode(ctx context.Context, req *nodepb.Object)
 
 // NodeServiceServer.GetPeers - fetches peers over gRPC
 func (s *NodeServiceServer) GetPeers(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
-	macAndNetwork := strings.Split(req.Data, "###")
-	if len(macAndNetwork) == 2 {
-		// TODO: Make constant and new variable for isServer
-		node, err := logic.GetNode(macAndNetwork[0], macAndNetwork[1])
-		if err != nil {
-			return nil, err
-		}
-		if node.IsServer == "yes" && logic.IsLeader(&node) {
-			logic.SetNetworkServerPeers(&node)
-		}
-		excludeIsRelayed := node.IsRelay != "yes"
-		var relayedNode string
-		if node.IsRelayed == "yes" {
-			relayedNode = node.Address
-		}
-		peers, err := logic.GetPeersList(macAndNetwork[1], excludeIsRelayed, relayedNode)
-		if err != nil {
+
+	var node, err = getNodeFromRequestData(req.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	peers, err := logic.GetPeersList(&node)
+	if err != nil {
+		if strings.Contains(err.Error(), logic.RELAY_NODE_ERR) {
+			peers, err = logic.PeerListUnRelay(node.ID, node.Network)
+			if err != nil {
+				return nil, err
+			}
+		} else {
 			return nil, err
 		}
-
-		peersData, err := json.Marshal(&peers)
-		logger.Log(3, node.Address, "checked in successfully")
-		return &nodepb.Object{
-			Data: string(peersData),
-			Type: nodepb.NODE_TYPE,
-		}, err
 	}
+
+	peersData, err := json.Marshal(&peers)
+	logger.Log(3, node.Address, "checked in successfully")
 	return &nodepb.Object{
-		Data: "",
+		Data: string(peersData),
 		Type: nodepb.NODE_TYPE,
-	}, errors.New("could not fetch peers, invalid node id")
+	}, err
 }
 
 // NodeServiceServer.GetExtPeers - returns ext peers for a gateway node
 func (s *NodeServiceServer) GetExtPeers(ctx context.Context, req *nodepb.Object) (*nodepb.Object, error) {
-	// Initiate a NodeItem type to write decoded data to
-	//data := &models.PeersResponse{}
-	// collection.Find returns a cursor for our (empty) query
-	macAndNetwork := strings.Split(req.Data, "###")
-	if len(macAndNetwork) != 2 {
-		return nil, errors.New("did not receive valid node id when fetching ext peers")
-	}
-	peers, err := logic.GetExtPeersList(macAndNetwork[0], macAndNetwork[1])
+
+	var node, err = getNodeFromRequestData(req.Data)
+	if err != nil {
+		return nil, err
+	}
+
+	peers, err := logic.GetExtPeersList(&node)
 	if err != nil {
 		return nil, err
 	}
@@ -222,3 +293,34 @@ func (s *NodeServiceServer) GetExtPeers(ctx context.Context, req *nodepb.Object)
 		Type: nodepb.EXT_PEER,
 	}, nil
 }
+
+// == private methods ==
+
+func getNodeFromRequestData(data string) (models.Node, error) {
+	var reqNode models.Node
+	var err error
+
+	if err = json.Unmarshal([]byte(data), &reqNode); err != nil {
+		return models.Node{}, err
+	}
+	return logic.GetNodeByID(reqNode.ID)
+}
+
+func isServer(node *models.Node) bool {
+	return node.IsServer == "yes"
+}
+
+func runForceServerUpdate(node *models.Node) {
+	go func() {
+		if err := mq.PublishPeerUpdate(node); err != nil {
+			logger.Log(1, "failed a peer update after creation of node", node.Name)
+		}
+
+		var currentServerNode, getErr = logic.GetNetworkServerLeader(node.Network)
+		if getErr == nil {
+			if err := logic.ServerUpdate(&currentServerNode, false); err != nil {
+				logger.Log(1, "server node:", currentServerNode.ID, "failed update")
+			}
+		}
+	}()
+}

controllers/node_test.go

@@ -13,20 +13,31 @@ func TestCreateEgressGateway(t *testing.T) {
 	var gateway models.EgressGatewayRequest
 	gateway.Interface = "eth0"
 	gateway.Ranges = []string{"10.100.100.0/24"}
+	gateway.NetID = "skynet"
 	database.InitializeDatabase()
 	deleteAllNetworks()
 	createNet()
 	t.Run("NoNodes", func(t *testing.T) {
 		node, err := logic.CreateEgressGateway(gateway)
 		assert.Equal(t, models.Node{}, node)
-		assert.EqualError(t, err, "unable to get record key")
+		assert.EqualError(t, err, "could not find any records")
+	})
+	t.Run("Non-linux node", func(t *testing.T) {
+		createnode := models.Node{PublicKey: "DM5qhLAE20PG9BbfBCger+Ac9D2NDOwCtY1rbYDLf34=", Name: "testnode", Endpoint: "10.0.0.1", MacAddress: "01:02:03:04:05:06", Password: "password", Network: "skynet", OS: "freebsd"}
+		err := logic.CreateNode(&createnode)
+		assert.Nil(t, err)
+		gateway.NodeID = createnode.ID
+		node, err := logic.CreateEgressGateway(gateway)
+		assert.Equal(t, models.Node{}, node)
+		assert.EqualError(t, err, "freebsd is unsupported for egress gateways")
 	})
 	t.Run("Success", func(t *testing.T) {
+		deleteAllNodes()
 		testnode := createTestNode()
-		gateway.NetID = "skynet"
-		gateway.NodeID = testnode.MacAddress
+		gateway.NodeID = testnode.ID
 
 		node, err := logic.CreateEgressGateway(gateway)
+		t.Log(node)
 		assert.Nil(t, err)
 		assert.Equal(t, "yes", node.IsEgressGateway)
 		assert.Equal(t, gateway.Ranges, node.EgressGatewayRanges)
@@ -38,12 +49,11 @@ func TestDeleteEgressGateway(t *testing.T) {
 	database.InitializeDatabase()
 	deleteAllNetworks()
 	createNet()
-	createTestNode()
 	testnode := createTestNode()
 	gateway.Interface = "eth0"
 	gateway.Ranges = []string{"10.100.100.0/24"}
 	gateway.NetID = "skynet"
-	gateway.NodeID = testnode.MacAddress
+	gateway.NodeID = testnode.ID
 	t.Run("Success", func(t *testing.T) {
 		node, err := logic.CreateEgressGateway(gateway)
 		assert.Nil(t, err)
@@ -68,13 +78,8 @@ func TestDeleteEgressGateway(t *testing.T) {
 		node, err := logic.DeleteEgressGateway(gateway.NetID, "01:02:03")
 		assert.EqualError(t, err, "no result found")
 		assert.Equal(t, models.Node{}, node)
+		deleteAllNodes()
 	})
-	t.Run("BadNet", func(t *testing.T) {
-		node, err := logic.DeleteEgressGateway("badnet", gateway.NodeID)
-		assert.EqualError(t, err, "no result found")
-		assert.Equal(t, models.Node{}, node)
-	})
-
 }
 
 func TestGetNetworkNodes(t *testing.T) {
@@ -84,13 +89,12 @@ func TestGetNetworkNodes(t *testing.T) {
 	t.Run("BadNet", func(t *testing.T) {
 		node, err := logic.GetNetworkNodes("badnet")
 		assert.Nil(t, err)
-		assert.Equal(t, []models.Node{}, node)
-		//assert.Equal(t, "mongo: no documents in result", err.Error())
+		assert.Nil(t, node)
 	})
 	t.Run("NoNodes", func(t *testing.T) {
 		node, err := logic.GetNetworkNodes("skynet")
 		assert.Nil(t, err)
-		assert.Equal(t, []models.Node{}, node)
+		assert.Nil(t, node)
 	})
 	t.Run("Success", func(t *testing.T) {
 		createTestNode()
@@ -105,18 +109,13 @@ func TestUncordonNode(t *testing.T) {
 	deleteAllNetworks()
 	createNet()
 	node := createTestNode()
-	t.Run("BadNet", func(t *testing.T) {
-		resp, err := logic.UncordonNode("badnet", node.MacAddress)
-		assert.Equal(t, models.Node{}, resp)
-		assert.EqualError(t, err, "no result found")
-	})
-	t.Run("BadMac", func(t *testing.T) {
-		resp, err := logic.UncordonNode("skynet", "01:02:03")
+	t.Run("BadID", func(t *testing.T) {
+		resp, err := logic.UncordonNode("blahblah")
 		assert.Equal(t, models.Node{}, resp)
 		assert.EqualError(t, err, "no result found")
 	})
 	t.Run("Success", func(t *testing.T) {
-		resp, err := logic.UncordonNode("skynet", node.MacAddress)
+		resp, err := logic.UncordonNode(node.ID)
 		assert.Nil(t, err)
 		assert.Equal(t, "no", resp.IsPending)
 	})
@@ -134,7 +133,7 @@ func TestValidateEgressGateway(t *testing.T) {
 		gateway.Interface = ""
 		err := logic.ValidateEgressGateway(gateway)
 		assert.NotNil(t, err)
-		assert.Equal(t, "Interface cannot be empty", err.Error())
+		assert.Equal(t, "interface cannot be empty", err.Error())
 	})
 	t.Run("Success", func(t *testing.T) {
 		gateway.Interface = "eth0"
@@ -145,14 +144,11 @@ func TestValidateEgressGateway(t *testing.T) {
 }
 
 func deleteAllNodes() {
-	nodes, _ := logic.GetAllNodes()
-	for _, node := range nodes {
-		logic.DeleteNode(&node, true)
-	}
+	database.DeleteAllRecords(database.NODES_TABLE_NAME)
 }
 
 func createTestNode() *models.Node {
-	createnode := models.Node{PublicKey: "DM5qhLAE20PG9BbfBCger+Ac9D2NDOwCtY1rbYDLf34=", Name: "testnode", Endpoint: "10.0.0.1", MacAddress: "01:02:03:04:05:06", Password: "password", Network: "skynet"}
+	createnode := models.Node{PublicKey: "DM5qhLAE20PG9BbfBCger+Ac9D2NDOwCtY1rbYDLf34=", Name: "testnode", Endpoint: "10.0.0.1", MacAddress: "01:02:03:04:05:06", Password: "password", Network: "skynet", OS: "linux"}
 	logic.CreateNode(&createnode)
 	return &createnode
 }

controllers/relay.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/mq"
 )
 
 func createRelay(w http.ResponseWriter, r *http.Request) {
@@ -20,28 +21,42 @@ func createRelay(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	relay.NetID = params["network"]
-	relay.NodeID = params["macaddress"]
-	node, err := logic.CreateRelay(relay)
+	relay.NodeID = params["nodeid"]
+	updatenodes, node, err := logic.CreateRelay(relay)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
 	logger.Log(1, r.Header.Get("user"), "created relay on node", relay.NodeID, "on network", relay.NetID)
+	for _, relayedNode := range updatenodes {
+		err = mq.NodeUpdate(&relayedNode)
+		if err != nil {
+			logger.Log(1, "error sending update to relayed node ", relayedNode.Address, "on network", relay.NetID, ": ", err.Error())
+		}
+	}
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
+	runUpdates(&node, true)
 }
 
 func deleteRelay(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	var params = mux.Vars(r)
-	nodeMac := params["macaddress"]
+	nodeid := params["nodeid"]
 	netid := params["network"]
-	node, err := logic.DeleteRelay(netid, nodeMac)
+	updatenodes, node, err := logic.DeleteRelay(netid, nodeid)
 	if err != nil {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
-	logger.Log(1, r.Header.Get("user"), "deleted egress gateway", nodeMac, "on network", netid)
+	logger.Log(1, r.Header.Get("user"), "deleted relay server", nodeid, "on network", netid)
+	for _, relayedNode := range updatenodes {
+		err = mq.NodeUpdate(&relayedNode)
+		if err != nil {
+			logger.Log(1, "error sending update to relayed node ", relayedNode.Address, "on network", netid, ": ", err.Error())
+		}
+	}
 	w.WriteHeader(http.StatusOK)
 	json.NewEncoder(w).Encode(node)
+	runUpdates(&node, true)
 }

controllers/security.go

@@ -98,9 +98,9 @@ func SecurityCheck(reqAdmin bool, netname string, token string) (error, []string
 	return nil, userNetworks, username
 }
 
-//Consider a more secure way of setting master key
+// Consider a more secure way of setting master key
 func authenticateMaster(tokenString string) bool {
-	return tokenString == servercfg.GetMasterKey()
+	return tokenString == servercfg.GetMasterKey() && servercfg.GetMasterKey() != ""
 }
 
 //Consider a more secure way of setting master key

controllers/server.go

@@ -9,7 +9,6 @@ import (
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/servercfg"
-	"github.com/gravitl/netmaker/serverctl"
 )
 
 func serverHandlers(r *mux.Router) {
@@ -50,7 +49,7 @@ func securityCheckServer(adminonly bool, next http.Handler) http.HandlerFunc {
 			returnErrorResponse(w, r, errorResponse)
 			return
 		}
-		if adminonly && !isadmin && !authenticateMasterServer(authToken) {
+		if adminonly && !isadmin && !authenticateMaster(authToken) {
 			returnErrorResponse(w, r, errorResponse)
 			return
 		}
@@ -58,11 +57,6 @@ func securityCheckServer(adminonly bool, next http.Handler) http.HandlerFunc {
 	}
 }
 
-//Consider a more secure way of setting master key
-func authenticateMasterServer(tokenString string) bool {
-	return tokenString == servercfg.GetMasterKey()
-}
-
 func removeNetwork(w http.ResponseWriter, r *http.Request) {
 	// Set header
 	w.Header().Set("Content-Type", "application/json")
@@ -70,9 +64,8 @@ func removeNetwork(w http.ResponseWriter, r *http.Request) {
 	// get params
 	var params = mux.Vars(r)
 
-	success, err := serverctl.RemoveNetwork(params["network"])
-
-	if err != nil || !success {
+	err := logic.DeleteNetwork(params["network"])
+	if err != nil {
 		json.NewEncoder(w).Encode("Could not remove server from network " + params["network"])
 		return
 	}

controllers/user.go

@@ -12,6 +12,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 func userHandlers(r *mux.Router) {
@@ -166,6 +167,11 @@ func createUser(w http.ResponseWriter, r *http.Request) {
 	// get node from body of request
 	_ = json.NewDecoder(r.Body).Decode(&user)
 
+	if !user.IsAdmin && isAddingComms(user.Networks) {
+		returnErrorResponse(w, r, formatError(fmt.Errorf("can not add comms network to non admin"), "badrequest"))
+		return
+	}
+
 	user, err := logic.CreateUser(user)
 
 	if err != nil {
@@ -194,6 +200,10 @@ func updateUserNetworks(w http.ResponseWriter, r *http.Request) {
 		returnErrorResponse(w, r, formatError(err, "internal"))
 		return
 	}
+	if !userchange.IsAdmin && isAddingComms(userchange.Networks) {
+		returnErrorResponse(w, r, formatError(fmt.Errorf("can not add comms network to non admin"), "badrequest"))
+		return
+	}
 
 	err = logic.UpdateUserNetworks(userchange.Networks, userchange.IsAdmin, &user)
 	if err != nil {
@@ -219,6 +229,10 @@ func updateUser(w http.ResponseWriter, r *http.Request) {
 		returnErrorResponse(w, r, formatError(fmt.Errorf("can not update user info for oauth user %s", username), "forbidden"))
 		return
 	}
+	if !user.IsAdmin && isAddingComms(user.Networks) {
+		returnErrorResponse(w, r, formatError(fmt.Errorf("can not add comms network to non admin"), "badrequest"))
+		return
+	}
 	var userchange models.User
 	// we decode our body request params
 	err = json.NewDecoder(r.Body).Decode(&userchange)
@@ -288,3 +302,13 @@ func deleteUser(w http.ResponseWriter, r *http.Request) {
 	logger.Log(1, username, "was deleted")
 	json.NewEncoder(w).Encode(params["username"] + " deleted.")
 }
+
+func isAddingComms(networks []string) bool {
+	commsID := servercfg.GetCommsID()
+	for i := range networks {
+		if networks[i] == commsID {
+			return true
+		}
+	}
+	return false
+}

database/database.go

@@ -1,12 +1,17 @@
 package database
 
 import (
+	"crypto/rand"
 	"encoding/json"
 	"errors"
 	"time"
 
+	"github.com/google/uuid"
 	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
+	"golang.org/x/crypto/nacl/box"
 )
 
 // NETWORKS_TABLE_NAME - networks table
@@ -27,15 +32,18 @@ const DNS_TABLE_NAME = "dns"
 // EXT_CLIENT_TABLE_NAME - ext client table
 const EXT_CLIENT_TABLE_NAME = "extclients"
 
-// INT_CLIENTS_TABLE_NAME - int client table
-const INT_CLIENTS_TABLE_NAME = "intclients"
-
 // PEERS_TABLE_NAME - peers table
 const PEERS_TABLE_NAME = "peers"
 
-// SERVERCONF_TABLE_NAME
+// SERVERCONF_TABLE_NAME - stores server conf
 const SERVERCONF_TABLE_NAME = "serverconf"
 
+// SERVER_UUID_TABLE_NAME - stores unique netmaker server data
+const SERVER_UUID_TABLE_NAME = "serveruuid"
+
+// SERVER_UUID_RECORD_KEY - telemetry thing
+const SERVER_UUID_RECORD_KEY = "serveruuid"
+
 // DATABASE_FILENAME - database file name
 const DATABASE_FILENAME = "netmaker.db"
 
@@ -105,7 +113,7 @@ func InitializeDatabase() error {
 		time.Sleep(2 * time.Second)
 	}
 	createTables()
-	return nil
+	return initializeUUID()
 }
 
 func createTables() {
@@ -115,9 +123,9 @@ func createTables() {
 	createTable(USERS_TABLE_NAME)
 	createTable(DNS_TABLE_NAME)
 	createTable(EXT_CLIENT_TABLE_NAME)
-	createTable(INT_CLIENTS_TABLE_NAME)
 	createTable(PEERS_TABLE_NAME)
 	createTable(SERVERCONF_TABLE_NAME)
+	createTable(SERVER_UUID_TABLE_NAME)
 	createTable(GENERATED_TABLE_NAME)
 }
 
@@ -128,7 +136,8 @@ func createTable(tableName string) error {
 // IsJSONString - checks if valid json
 func IsJSONString(value string) bool {
 	var jsonInt interface{}
-	return json.Unmarshal([]byte(value), &jsonInt) == nil
+	var nodeInt models.Node
+	return json.Unmarshal([]byte(value), &jsonInt) == nil || json.Unmarshal([]byte(value), &nodeInt) == nil
 }
 
 // Insert - inserts object into db
@@ -184,6 +193,44 @@ func FetchRecords(tableName string) (map[string]string, error) {
 	return getCurrentDB()[FETCH_ALL].(func(string) (map[string]string, error))(tableName)
 }
 
+// initializeUUID - create a UUID record for server if none exists
+func initializeUUID() error {
+	records, err := FetchRecords(SERVER_UUID_TABLE_NAME)
+	if err != nil {
+		if !IsEmptyRecord(err) {
+			return err
+		}
+	} else if len(records) > 0 {
+		return nil
+	}
+	// setup encryption keys
+	var trafficPubKey, trafficPrivKey, errT = box.GenerateKey(rand.Reader) // generate traffic keys
+	if errT != nil {
+		return errT
+	}
+	tPriv, err := ncutils.ConvertKeyToBytes(trafficPrivKey)
+	if err != nil {
+		return err
+	}
+
+	tPub, err := ncutils.ConvertKeyToBytes(trafficPubKey)
+	if err != nil {
+		return err
+	}
+
+	telemetry := models.Telemetry{
+		UUID:           uuid.NewString(),
+		TrafficKeyPriv: tPriv,
+		TrafficKeyPub:  tPub,
+	}
+	telJSON, err := json.Marshal(&telemetry)
+	if err != nil {
+		return err
+	}
+
+	return Insert(SERVER_UUID_RECORD_KEY, string(telJSON), SERVER_UUID_TABLE_NAME)
+}
+
 // CloseDB - closes a database gracefully
 func CloseDB() {
 	getCurrentDB()[CLOSE_DB].(func())()

database/sqlite.go

@@ -30,7 +30,7 @@ var SQLITE_FUNCTIONS = map[string]interface{}{
 func initSqliteDB() error {
 	// == create db file if not present ==
 	if _, err := os.Stat("data"); os.IsNotExist(err) {
-		os.Mkdir("data", 0744)
+		os.Mkdir("data", 0700)
 	}
 	dbFilePath := filepath.Join("data", dbFilename)
 	if _, err := os.Stat(dbFilePath); os.IsNotExist(err) {

defaultvalues.sh

@@ -1,14 +0,0 @@
-#!/bin/bash
-#Source this file if using default mongo settings from readme
-# if i've done my work correctly, this file will be defunct
-#  refer to config folder for new method
-export API_PORT=8081
-export GRPC_PORT=50051
-export MONGO_USER=mongoadmin
-export MONGO_PASS=mongopass
-export MONGO_HOST=localhost
-export MASTER_KEY=c4tsRc001
-export MONGO_PORT=27017
-export MONGO_OPTS='/?authSource=admin'
-export MASTER_TOKEN="mastertoken"
-export CREATE_KEY="newnode123"

docker/Caddyfile

@@ -7,6 +7,8 @@
 https://dashboard.NETMAKER_BASE_DOMAIN {
         # Apply basic security headers
         header {
+                # Enable cross origin access to *.NETMAKER_BASE_DOMAIN
+                Access-Control-Allow-Origin *.NETMAKER_BASE_DOMAIN
                 # Enable HTTP Strict Transport Security (HSTS)
                 Strict-Transport-Security "max-age=31536000;"
                 # Enable cross-site filter (XSS) and tell browser to block detected attacks

docker/Dockerfile-netclient-kernel

@@ -0,0 +1,39 @@
+FROM debian:buster as builder
+# add glib support daemon manager
+
+RUN apt update -y && apt install -y wget bash gcc musl-dev openssl golang git build-essential libmnl-dev iptables
+
+RUN wget -O go.tgz https://dl.google.com/go/go1.17.1.linux-amd64.tar.gz
+
+RUN tar -C /usr/local -xzf go.tgz
+
+WORKDIR /usr/local/go/src
+
+RUN chmod +x make.bash
+
+RUN ./make.bash
+
+ENV PATH="/usr/local/go/bin:$PATH"
+
+ENV GOPATH=/opt/go/
+
+ENV PATH=$PATH:$GOPATH/bin
+
+WORKDIR /app
+
+COPY . .
+
+ENV GO111MODULE=auto
+
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 /usr/local/go/bin/go build -ldflags="-w -s" -o netclient-app netclient/main.go
+
+FROM debian:buster
+
+WORKDIR /root/
+
+RUN apt update -y && apt install -y bash curl wget traceroute procps dnsutils iptables openresolv iproute2
+COPY --from=builder /app/netclient-app ./netclient
+COPY --from=builder /app/scripts/netclient.sh .
+RUN chmod 0755 netclient && chmod 0755 netclient.sh
+
+ENTRYPOINT ["/bin/sh", "./netclient.sh"]

docker/Dockerfile-netclient-multiarch

@@ -0,0 +1,22 @@
+FROM golang:latest as builder
+# add glib support daemon manager
+WORKDIR /app
+ARG version
+
+COPY . .
+
+ENV GO111MODULE=auto
+
+RUN GOOS=linux CGO_ENABLED=0 /usr/local/go/bin/go build -ldflags="-w -s -X 'main.version=${TAG}'" -o netclient-app netclient/main.go
+
+FROM alpine:3.13.6
+
+WORKDIR /root/
+
+RUN apk add --no-cache --update bash libmnl gcompat iptables openresolv iproute2 wireguard-tools 
+COPY --from=builder /app/netclient-app ./netclient
+COPY --from=builder /app/scripts/netclient.sh .
+RUN chmod 0755 netclient && chmod 0755 netclient.sh
+
+
+ENTRYPOINT ["/bin/sh", "./netclient.sh"]

docker/Dockerfile-netmaker-slim

@@ -11,7 +11,7 @@ RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=1 /usr/local/go/bin/go build -ldflags="-
 
 FROM alpine:3.13.6
 # add a c lib
-RUN apk add gcompat iptables
+RUN apk add gcompat iptables wireguard-tools
 # set the working directory
 WORKDIR /root/
 

docker/mosquitto.conf

@@ -0,0 +1,4 @@
+persistence true
+per_listener_settings true
+listener 1883
+allow_anonymous true

docs/_build/html/.buildinfo

@@ -1,4 +1,4 @@
 # Sphinx build info version 1
 # This file hashes the configuration used when building these files. When it is not found, a full rebuild will be done.
-config: 7d73735f557dbf90d3004bb9dfde743b
+config: f5642f0efe1bade3b9e81e56c8b29995
 tags: 645f666f9bcd5a90fca523b33c5a78b7

docs/_build/html/_sources/about.rst.txt

@@ -1,6 +1,12 @@
-===============
+==========
 About
-===============
+==========
+
+.. image:: images/netmaker-simple.png
+   :width: 60%
+   :alt: Netmaker Architecture Diagram
+   :align: center
+
 
 What is Netmaker?
 ==================
@@ -25,9 +31,9 @@ Netmaker relies on WireGuard to create tunnels between machines. At its core, Ne
 - the admin server, called Netmaker
 - the agent, called Netclient
 
-As the network manager, you interact with the server to create and manage networks and devices. The server holds configurations for these networks and devices, which are retrieved by the netclients (agent). 
+As the network manager, you interact with the server to create and manage networks and devices. The server holds configurations for these networks and devices, which are retrieved by the netclients (agent). The server runs an MQTT (message queue) broker for client-server communication.
 
-The netclient is installed on any machine you would like to add to a given network, whether that machine is a VM, Server, or IoT device. The netclient reaches out to the server, and the server tells it how it should configure the network. By doing this across many machines simultaneously, we create a dynamic, fully configurable virtual networks.
+The netclient is installed on any machine you would like to add to a given network, whether that machine is a VM, Server, or IoT device. The netclient subscribes to the server's MQ broker, and the server tells it how it should configure WireGuard. The client will let the server know when any local changes should be pushed out to the other clients. By doing this across many machines simultaneously, we create a dynamic, fully configurable virtual networks.
 
 The Netmaker server does not typically route traffic. Otherwise, this would be a hub-and-spoke model, which is very slow. Instead, Netmaker just tells the machines on the network how they can reach each other directly. This is called a *full mesh* network and is much faster. Even if the server goes down, as long as none of the existing machines change substantially, your network will still run just fine.
 

docs/_build/html/_sources/api.rst.txt

@@ -12,13 +12,16 @@ Authentication
 ==============
 API calls must be authenticated via a header of  the format  `-H "Authorization: Bearer <YOUR_SECRET_KEY>"` There are two methods to obtain YOUR_SECRET_KEY:
 1. Using the masterkey. By default, this value is "secret key," but you should change this on your instance and keep it secure. This value can be set via env var at startup or in a config file (config/environments/< env >.yaml). See the [general usage](./USAGE.md) documentation for more details.
-2. Using a JWT recieved for a node. This  can be retrieved by calling the `/api/nodes/<network>/authenticate` endpoint, as documented below.
+2. Using a JWT received for a node. This  can be retrieved by calling the `/api/nodes/<network>/authenticate` endpoint, as documented below.
 
 
 Format of Calls for Curl
 ========================
-Requests take the format of `curl -H "Authorization: Bearer <YOUR_SECRET_KEY>" -H 'Content-Type: application/json' localhost:8081/api/path/to/endpoint`
+Requests take the format of 
 
+.. code-block::
+
+    curl -H "Authorization: Bearer <YOUR_SECRET_KEY>" -H 'Content-Type: application/json' localhost:8081/api/path/to/endpoint
 
 API Documentation
 =================
@@ -41,18 +44,21 @@ Networks API
   
 Networks API Call Examples
 --------------------------  
-  
-**Get All Networks:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks | jq`
 
-**Create Network:** `curl -d '{"addressrange":"10.70.0.0/16","netid":"skynet"}' -H "Authorization: Bearer YOUR_SECRET_KEY" -H 'Content-Type: application/json' localhost:8081/api/networks`
+.. code-block::
+
+    Get All Networks: curl -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks | jq
 
-**Get Network:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet | jq`
+    Create Network: curl -d '{"addressrange":"10.70.0.0/16","netid":"skynet"}' -H "Authorization: Bearer YOUR_SECRET_KEY" -H 'Content-Type: application/json' localhost:8081/api/networks
 
-**Update Network:** `curl -X PUT -d '{"displayname":"my-house"}' -H "Authorization: Bearer YOUR_SECRET_KEY" -H 'Content-Type: application/json' localhost:8081/api/networks/skynet`
+    Get Network: curl -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet | jq
 
-**Delete Network:** `curl -X DELETE -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet`
+    Update Network: curl -X PUT -d '{"displayname":"my-house"}' -H "Authorization: Bearer YOUR_SECRET_KEY" -H 'Content-Type: application/json' localhost:8081/api/networks/skynet
+
+    Delete Network: curl -X DELETE -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet
+
+    Cycle PublicKeys on all Nodes: curl -X POST -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet/keyupdate
 
-**Cycle PublicKeys on all Nodes:** `curl -X POST -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet/keyupdate`
 
 Access Keys API
 ---------------
@@ -66,13 +72,15 @@ Access Keys API
   
 Access Keys API Call Examples
 -----------------------------
-   
-**Get All Keys:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet/keys | jq`
-  
-**Create Key:** `curl -d '{"uses":10,"name":"mykey"}' -H "Authorization: Bearer YOUR_SECRET_KEY" -H 'Content-Type: application/json' localhost:8081/api/networks/skynet/keys`
-  
-**Delete Key:** `curl -X DELETE -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet/keys/mykey`
-  
+
+.. code-block::
+
+    Get All Keys: curl -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet/keys | jq
+    
+    Create Key: curl -d '{"uses":10,"name":"mykey"}' -H "Authorization: Bearer YOUR_SECRET_KEY" -H 'Content-Type: application/json' localhost:8081/api/networks/skynet/keys
+    
+    Delete Key: curl -X DELETE -H "Authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/networks/skynet/keys/mykey
+
     
 Nodes API
 ---------
@@ -104,29 +112,31 @@ Nodes API
   
 Nodes API Call Examples
 ----------------------- 
-  
-**Get All Nodes:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/nodes | jq`
-  
-**Get Network Nodes:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/nodes/skynet | jq`
+
+.. code-block::
+
+    Get All Nodes: curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/nodes | jq
     
-**Create Node:** `curl  -d  '{ "endpoint": 100.200.100.200, "publickey": aorijqalrik3ajflaqrdajhkr,"macaddress": "8c:90:b5:06:f1:d9","password": "reallysecret","localaddress": "172.16.16.1","accesskey": "aA3bVG0rnItIRXDx","listenport": 6400}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet`
+    Get Network Nodes: curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/nodes/skynet | jq
+        
+    Create Node: curl  -d  '{ "endpoint": 100.200.100.200, "publickey": aorijqalrik3ajflaqrdajhkr,"macaddress": "8c:90:b5:06:f1:d9","password": "reallysecret","localaddress": "172.16.16.1","accesskey": "aA3bVG0rnItIRXDx","listenport": 6400}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet
+        
+    Get Node: curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/nodes/skynet/{macaddress} | jq  
     
-**Get Node:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/nodes/skynet/{macaddress} | jq`  
-  
-**Update Node:** `curl -X PUT -d '{"name":"laptop1"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9`
-  
-**Delete Node:** `curl -X DELETE -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/skynet/nodes/8c:90:b5:06:f1:d9`
-  
-**Create a Gateway:** `curl  -d  '{ "rangestring": "172.31.0.0/16", "interface": "eth0"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9/creategateway`
-  
-**Delete a Gateway:** `curl -X DELETE -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9/deletegateway`
-  
-**Approve a Pending Node:** `curl -X POST -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9/approve`
-  
-**Get Last Modified Date (Last Modified Node in Network):** `curl -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/adm/skynet/lastmodified`
+    Update Node: curl -X PUT -d '{"name":"laptop1"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9
+    
+    Delete Node: curl -X DELETE -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/skynet/nodes/8c:90:b5:06:f1:d9
+    
+    Create a Gateway: curl  -d  '{ "rangestring": "172.31.0.0/16", "interface": "eth0"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9/creategateway
+    
+    Delete a Gateway: curl -X DELETE -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9/deletegateway
+    
+    Approve a Pending Node: curl -X POST -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/skynet/8c:90:b5:06:f1:d9/approve
+    
+    Get Last Modified Date (Last Modified Node in Network): curl -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/nodes/adm/skynet/lastmodified
+
+    Authenticate: curl -d  '{"macaddress": "8c:90:b5:06:f1:d9", "password": "YOUR_PASSWORD"}' -H 'Content-Type: application/json' localhost:8081/api/nodes/adm/skynet/authenticate
 
-**Authenticate:** `curl -d  '{"macaddress": "8c:90:b5:06:f1:d9", "password": "YOUR_PASSWORD"}' -H 'Content-Type: application/json' localhost:8081/api/nodes/adm/skynet/authenticate`
-  
 
 Users API
 -----------------------
@@ -148,19 +158,21 @@ Users API
   
 Users API Calls Examples
 ------------------------
-  
-**Get User:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/users/{username} | jq`
 
-**Update User:** `curl -X PUT -d '{"password":"noonewillguessthis"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/users/{username}`
-  
-**Delete User:** `curl -X DELETE -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/users/{username}`
-  
-**Check for Admin User:** `curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/users/adm/hasadmin`
-  
-**Create Admin User:** `curl -d '{ "username": "smartguy", "password": "YOUR_PASS"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/users/adm/createadmin`
-   
-**Authenticate:** `curl -d  '{"username": "smartguy", "password": "YOUR_PASS"}' -H 'Content-Type: application/json' localhost:8081/api/nodes/adm/skynet/authenticate`
-  
+.. code-block::
+
+    Get User: curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/users/{username} | jq
+
+    Update User: curl -X PUT -d '{"password":"noonewillguessthis"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/users/{username}
+    
+    Delete User: curl -X DELETE -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/users/{username}
+    
+    Check for Admin User: curl -H "Authorization: Bearer YOUR_SECRET_KEY" http://localhost:8081/api/users/adm/hasadmin
+    
+    Create Admin User: curl -d '{ "username": "smartguy", "password": "YOUR_PASS"}' -H 'Content-Type: application/json' -H "authorization: Bearer YOUR_SECRET_KEY" localhost:8081/api/users/adm/createadmin
+    
+    Authenticate: curl -d  '{"username": "smartguy", "password": "YOUR_PASS"}' -H 'Content-Type: application/json' localhost:8081/api/nodes/adm/skynet/authenticate
+
 
 Server Management API
 ---------------------

docs/_build/html/_sources/architecture.rst.txt

@@ -2,13 +2,13 @@
 Architecture
 ===============
 
-.. image:: images/nm-diagram-2.jpg
-   :width: 45%
+.. image:: images/nm-diagram-3.png
+   :width: 100%
    :alt: Netmaker Architecture Diagram
    :align: center
     
 
-*Pictured Above: A diagram of Netmaker's Architecture.*
+*Pictured Above: A detailed diagram of Netmaker's Architecture.*
 
 
 Core Concepts
@@ -21,7 +21,7 @@ WireGuard
 
 WireGuard is a relatively new but very important technology which was recently added to the Linux kernel. WireGuard creates very fast but simple encrypted tunnels between devices. From the `WireGuard <https://www.wireguard.com/>`_ website, "it might be regarded as the most secure, easiest to use, and simplest VPN solution in the industry."
 
-Previous solutions like OpenVPN and IPSec are considerably more heavy and complex, while being less performant. All existing VPN tunnelling solutions will cause a significant increase in your network latency. WireGuard is the first to achieve near over-the-line network speeds, meaning you see no signigifant performance impact.  With the release of WireGuard, there is little reason to use any other existing tunnel encryption technology.
+Previous solutions like OpenVPN and IPSec are considerably more heavy and complex, while being less performant. All existing VPN tunneling solutions will cause a significant increase in your network latency. WireGuard is the first to achieve near over-the-line network speeds, meaning you see no significant performance impact.  With the release of WireGuard, there is little reason to use any other existing tunnel encryption technology.
 
 Mesh Network
 -------------
@@ -47,7 +47,7 @@ Netmaker
 
 Netmaker is a platform built off of WireGuard which enables users to create mesh networks between their devices. Netmaker can create both full and partial mesh networks depending on the use case.
 
-When we refer to Netmaker in aggregate, we are typically referring to Netmaker and the netclient, as well as other supporting services such as CoreDNS, rqlite, and UI webserver.
+When we refer to Netmaker in aggregate, we are typically referring to Netmaker and the netclient, as well as other supporting services such as CoreDNS, rqlite, and UI webserver. There is also almost always a proxy server / LB, which is typically Caddy.
 
 From an end user perspective, they typically interact with the Netmaker UI, or even just run the install script for the netclient on their devices. The other components run in the background invisibly. 
 
@@ -56,7 +56,7 @@ Netmaker does a lot of work to set configurations for you, so that you don't hav
 Node
 ------
 
-A machine in a Netmaker network, which is managed by the Netclient, is referred to as a Node, as you will see in the UI. A Node can be a VM, a bare metal server, a desktop computer, an IoT device, or any other number of internet-connected machines on which the netclient is installed. A node is simply an endpoint in the network, which can send traffic to all the other nodes, and recieve traffic from all of the other nodes.
+A machine in a Netmaker network, which is managed by the Netclient, is referred to as a Node, as you will see in the UI. A Node can be a VM, a bare metal server, a desktop computer, an IoT device, or any other number of internet-connected machines on which the netclient is installed. A node is simply an endpoint in the network, which can send traffic to all the other nodes, and receive traffic from all of the other nodes.
 
 SystemD
 -------
@@ -85,6 +85,7 @@ These modes include client mode and dns mode. Either of these can be disabled bu
 
 The Netmaker server interacts with either sqlite (default), postgres, or rqlite, a distributed version of sqlite, as its database. This DB holds information about nodes, networks, users, and other important data. This data is configuration data. For the most part, Netmaker serves configuration data to Nodes, telling them how they should configure themselves. The Netclient is the agent that actually does that configuration.
 
+The components of the server are usually proxied via Caddy, or an alternative like Nginx and Traefik. The proxy handles SSL certificates to secure traffic, and routes to the UI, API, and gRPC server.
 
 Netclient
 ----------------
@@ -99,15 +100,15 @@ The 'join' command attempts to add the machine to the Netmaker network using sen
 
 The netclient then sets up the system daemon (if running in daemon mode), and configures WireGuard. At this point it should be part of the network.
 
-If running in daemon mode, on a periodic basis (systemd timer), the netclient performs a "check in." It will authenticate with the server, and check to see if anything has changed in the network. It will also post changes about its own local configuration if there. If there has been a change, the server will return new configurations and the netclient will reconfigure the network. If not running in daemon mode, it is up to the operator to perform check ins (netclient checkin -n < network name >).
+If running in daemon mode, the node subscribes to the MQTT server running with Netmaker, which will send it periodic updates when the network changes. The node will also detect local changes and send them to the server. Any change in configuration will lead to a network update to keep everything in sync. If the node is not running with the in daemon on, it is up to the operator to keep the netclient up-to-date by running regular "pulls" (netclient pull).
 
-The check in process is what allows Netmaker to create dynamic mesh networks. As nodes are added to, removed from, and modified on the network, other nodes are notified, and make appropriate changes.
+This pub-sub system allows Netmaker to create dynamic mesh networks. As nodes are added to, removed from, and modified on the network, other nodes are notified, and make appropriate changes.
 
 
 Database (sqlite, rqlite, postgres)
 -------------------------------------
 
-As of v0.8, Netmaker uses sqlite by default as a database. It can also use PostgreSQL, or rqlite, a distributed (RAFT consensus) databaseand. Netmaker interacts with this database to store and retrieve information about nodes, networks, and users. 
+As of v0.8, Netmaker uses sqlite by default as a database. It can also use PostgreSQL, or rqlite, a distributed (RAFT consensus) database. Netmaker interacts with this database to store and retrieve information about nodes, networks, and users. 
 
 Additional database support (besides sqlite and rqlite) is very easy to implement for special use cases. Netmaker uses simple key value lookups to run the networks, and the database was designed to be extensible, so support for key-value stores and other SQL-based databases can be achieved by changing a single file.
 
@@ -124,6 +125,20 @@ CoreDNS
 
 Netmaker allows users to provide and manage Private DNS for their nodes. This requires a nameserver, and CoreDNS is the chosen nameserver. CoreDNS is lightweight and extensible. CoreDNS loads dns settings from a simple file, managed by Netmaker, and serves out DNS info for managed nodes. DNS can be tricky, and DNS management is currently only supported on a small set of devices, specifically those running systemd-resolved. However, the Netmaker CoreDNS instance can be added manually as a nameserver to other devices. DNS mode can also be turned off.
 
+Caddy
+-------
+
+Caddy is the default proxy for Netmaker if you set it up via Quick Start. Caddy is an extremely simple and docker-friendly proxy, which can be compared to Nginx, Traefik, or HAProxy. We use Caddy by default because of the ease of management, and integration with gRPC. A typical setup for Nginx might take dozens of lines of code, and we need to request and manage SSL certificates separately.
+
+Caddy handles all these things automatically in very few lines of code. You can see our default "Caddyfile" here, which is fed to the container and has all the configuration necessary to configure the proxy for our app:
+
+https://github.com/gravitl/netmaker/blob/master/docker/Caddyfile
+
+Mosquitto Broker (MQTT)
+-------------------------
+
+The Moquitto broker is the default MQTT broker that ships with Netmaker, though technically, any MQTT broker should work so long as the correct configuration is applied. The broker enables the establishment of a pub-sub messaging system, whereby clients subscribe to recieve updates. When the server recieves a change (via API/UI/gRPC), it will publish that change to the broker that pushes out the change to the appropriate nodes. In Netmaker, the messages are double encrypted. Once by Golang, and again by sending all messages over a WireGuard tunnel. 
+
 External Client
 ----------------
 
@@ -146,20 +161,14 @@ Below is a high level, step-by-step overview of the flow of communications withi
 2. Admin creates an access key for signing up new nodes
 3. Both of the above requests are routed to the server via an API call from the front end
 4. Admin runs the netclient install script on any given node (machine).
-5. Netclient decodes key, which contains the GRPC server location and port
-6. Netclient uses information to register and set up WireGuard tunnel to GRPC server
-7. Netclient retrieves/sets local information, including open ports for WireGuard, public IP, and generating key pairs for peers
-8. Netclient reaches out to GRPC server with this information, authenticating via access key.
-9. Netmaker server verifies information and creates the node, setting default values for any missing information. 
-10. Timestamp is set for the network (see #16). 
-11. Netmaker returns settings as response to netclient. Some settings may be added or modified based on the network.
-12. Netclient recieves response. If successful, it takes any additional info returned from Netmaker and configures the local system/WireGuard
-13. Netclient sends another request to Netmaker's GRPC server, this time to retrieve the peers list (all other clients in the network).
-14. Netmaker sends back peers list, including current known configurations of all nodes in network.
-15. Netclient configures WireGuard with this information. At this point, the node is fully configured as a part of the network and should be able to reach the other nodes via private address.
-16. Netclient begins daemon (system timer) to run check in's with the server. It awaits changes, reporting local changes, and retrieving changes from any other nodes in the network.
-17. Other netclients on the network, upon checking in with the Netmaker server, will see that the timestamp has updated, and they will retrieve a new peers list, completing the update cycle.
-
+5. Netclient decodes key, which contains the server location
+6. Netclient gathers and sets appropriate information to configure itself as a node: it generates key pairs, gets public and local addresses, and sets a port.
+7. Netclient sends this information to the server, authenticating with its access key 
+8. Netmaker server verifies information and creates the node, setting default values for any missing information, and returns a response. 
+9. Upon successful registration, Netclient pulls the latest peers list from the server and set up a WireGuard interface
+10. Netclient configures itself as a daemon (if joining for the first time) and subscribes to MQ using the server's WireGuard address.
+11. Netclient regularly retrieves local information, checking for changes in things like IP and keys. If there is a change, it pushes them to the server.
+12. If a change occurs in any other peer, or peers are added/removed, an update will be sent to the Netclient via MQ, and it will re-configure WireGuard.
 
 Compatible Systems for Netclient
 ==================================
@@ -182,7 +191,7 @@ The following systems should be operable natively with Netclient in daemon mode:
         - Raspian.
         - Arch
         - CentOS
-        - CoreOS
+        - Fedora CoreOS
 
 To manage DNS (optional), the node must have systemd-resolved. Systems that have this enabled include:
         - Arch

docs/_build/html/_sources/client-installation.rst.txt

@@ -1,6 +1,6 @@
-====================
-Client Installation
-====================
+================================
+Advanced Client Installation
+================================
 
 This document tells you how to install the netclient on machines that will be a part of your Netmaker network, as well as non-compatible systems.
 
@@ -30,6 +30,48 @@ Windows will by default have firewall rules that prevent inbound connections. If
 
 If you want to allow all peers access, but do not want to configure firewall rules for all peers, you can configure access for one peer, and set it as a Relay Server.
 
+Running the install script
+----------------------------
+
+Some file locations have issues running the install script, such as running from the root C:/ folder. Users have noted the following locations work well for running the install powershell script:
+
+- `C:/Program Files/wireguard`
+- `C:/Windows/System32`
+
+Running netclient commands
+----------------------------
+
+If running the netclient manually ("netclient join", "netclient checkin", "netclient pull") it should be run from outside of the installed directory, which will be either:
+
+- `C:/Program Files/netclient`
+- `C:/ProgramData/netclient`
+
+It is better to call it from a different directory.
+
+High CPU Utilization
+--------------------------
+
+With some versions of WireGuard on Windows, high CPU utilization has been found with the netclient. This is typically due to interaction with the WireGuard GUI component (app). If you're experiencing high CPU utilization, close the WireGuard app. WireGuard will still be running, but the CPU usage should go back down to normal.
+
+Notes on OpenWRT
+===========================
+
+Deploying on OpenWRT depends a lot on the version of OpenWRT and the hardware being used. If the primary installer does not work, there are two things you can try:
+
+1. This community-run package for OpenWRT: https://github.com/sbilly/netmaker-openwrt
+
+2. Manual installation:
+
+- download (wget) the netclient package for your hardware from the netclient releases: https://github.com/gravitl/netmaker/releases
+- rename to "netclient"
+- Run as root from a bash shell on OpenWRT
+
+3. You may experience an issue with the length of the token, which has limits on some OpenWRT shells. If you run into this problem, you can use the following script to convert your token into a "netclient join" command:
+
+- `wget https://raw.githubusercontent.com/gravitl/netmaker/master/scripts/token-convert.sh`
+- ./token-convert <token value>
+- Run the output on your OpenWRT machine
+
 Modes and System Compatibility
 ==================================
 
@@ -125,18 +167,25 @@ Viewing Logs
   ``netclient list``
 
 **to tail logs**
-  ``journalctl -u netclient@<net name> -f``
-
-**to view all logs**
-  ``journalctl -u netclient@<net name>``
+  ``journalctl -u netclient``
 
 **to get most recent log run**
-  ``systemctl status netclient@<net name>``
+  ``systemctl status netclient``
+
+Re-syncing netclient (basic troubleshooting)
+-----------------------------------------------
+
+If the daemon is not running correctly run, try restarting the daemon, or pulling changes directly (don't do both at once)
+
+  ``systemctl restart netclient``
+
+  ``sudo netclient pull``
+
 
 Making Updates
 ----------------
 
-``vim /etc/netclient/netconfig-<network>``
+``vim /etc/netclient/config/netconfig-<network>``
 
 Change any of the variables in this file, and changes will be pushed to the server and processed locally on the next checkin.
 

docs/_build/html/_sources/egress-gateway.rst.txt

@@ -0,0 +1,96 @@
+=====================================
+Egress Gateway
+=====================================
+
+Introduction
+===============
+
+.. image:: images/egress1.png
+   :width: 80%
+   :alt: Gateway
+   :align: center
+
+Netmaker allows your clients to reach external networks via an Egress Gateway. The Egress Gateway is a netclient which has been deployed to a server or router with access to a given subnet.
+
+In the netmaker UI, that node is set as an "egress gateway." Range(s) are specified which this node has access to. Once created, all clients (and all new ext clients) in the network will be able to reach those ranges via the gateway.
+
+Configuring an Egress Gateway
+==================================
+
+Configuring an Egress Gateway is very straight forward. As a prerequisite, you must know what you are trying to access remotely. For instance:
+
+- a VPC
+- a Kubernetes network
+- a home network
+- an office network
+- a data center
+
+After you have determined this, you must next deploy a netclient in a compatible location where the network is accessible. For instance, a Linux server or router in the office, or a Kubernetes worker node. This machine should be stable and relatively static (not expected to change its IP frequently or shut down unexpectedly).
+
+Next, you must determine which interface to use in order to reach the internal network. As an example, lets say there is a machine in the network at 10.10.10.2, and you have deployed the netclient on a different machine. You can run 
+
+.. code-block::
+
+   ip route get 10.10.10.2
+
+This should return the interface used to reach that address (e.x. "eth2")
+
+Finally, once you have determined the interface, the subnet, and deployed your netclient, you can go to your Netmaker UI and set the node as a gateway.
+
+.. image:: images/egress7.png
+   :width: 80%
+   :alt: Gateway
+   :align: center
+
+At this point simply insert the range(s) into the first field, and the interface name into the second field, and click "create".
+
+.. image:: images/ui-6.jpg
+   :width: 80%
+   :alt: Gateway
+   :align: center
+
+Netmaker will set iptables rules on the node, which will then implement these rules, allowing it to route traffic from the network to the specified range(s).
+
+Use Cases
+============
+
+1) Remote Access
+-------------------
+
+A common scenario would be to combine this with an "Ingress Gateway" to create a simple method for accessing a home or office network. Such a setup would typically have only two nodes: the ingress and egress gateways. The Ingress Gateway should usually be globally accessible, which makes the Netmaker server itself a good candidate. This means you need only the netmaker server as the Ingress, and one additional machine (in the private network you wish to reach), as the Egress.
+
+.. image:: images/egress2.png
+   :width: 80%
+   :alt: Gateway
+   :align: center
+
+In some scenarios, a single node will act as both ingress and egress! For instance, you can enable acess to a VPC using your Netmaker server, deployed with a public IP. Traffic comes in over the public IP (encrypted of course) and then routes to the VPC subnet via the egress gateway.
+
+.. image:: images/egress3.png
+   :width: 50%
+   :alt: Gateway
+   :align: center
+
+2) VPN / NAT Gateway
+-----------------------
+
+Most people think of a VPN as a remote server that keeps your internet traffic secure while you browse the web, or as a tool for accessing internet services in another country,using a VPN server based in that country.
+
+These are not typical use cases for Netmaker, but can be easily enabled.
+
+**The most important note is this: Do not use 0.0.0.0/0 as your egress gateway.** This is how you typically set up a "standard" VPN with WireGuard, however, it will not work with Netmaker. The Netclient specifically ignores gateways that overlap with local ranges (for efficiency ranges). 0.0.0.0 overlaps with everything, so it is always ignored.
+
+Instead, use the following list of ranges:
+
+.. code-block::
+
+   0.0.0.0/5,8.0.0.0/7,11.0.0.0/8,12.0.0.0/6,16.0.0.0/4,32.0.0.0/3,64.0.0.0/2,128.0.0.0/3,160.0.0.0/5,168.0.0.0/6,172.0.0.0/12,172.32.0.0/11,172.64.0.0/10,172.128.0.0/9,173.0.0.0/8,174.0.0.0/7,176.0.0.0/4,192.0.0.0/9,192.128.0.0/11,192.160.0.0/13,192.169.0.0/16,192.170.0.0/15,192.172.0.0/14,192.176.0.0/12,192.192.0.0/10,193.0.0.0/8,194.0.0.0/7,196.0.0.0/6,200.0.0.0/5,208.0.0.0/4
+
+This list encompasses the standard "public" network ranges, and ignores the standard "private" network ranges.
+
+Simply paste this list into your "egress gateway ranges" and your clients should begin routing public-facing traffic over the gateway.
+
+.. image:: images/egress5.png
+   :width: 50%
+   :alt: Gateway
+   :align: center

docs/_build/html/_sources/external-clients.rst.txt

@@ -1,10 +1,15 @@
-================
-External Clients
-================
+=====================================
+Ingress + External Clients
+=====================================
 
 Introduction
 ===============
 
+.. image:: images/ingress1.png
+   :width: 50%
+   :alt: Gateway
+   :align: center
+
 Netmaker allows for "external clients" to reach into a network and access services via an Ingress Gateway. So what is an "external client"? An external client is any machine which cannot or should not be meshed. This can include:
         - Phones
         - Laptops

docs/_build/html/_sources/index.rst.txt

@@ -9,11 +9,6 @@
    :alt: Netmaker WireGuard
    :align: center
 
-.. role:: raw-html(raw)
-    :format: html
-
-:raw-html:`<br />`
-
 =======================================
 Welcome to the Netmaker Documentation
 =======================================
@@ -23,68 +18,71 @@ Netmaker is a platform for creating and managing fast, secure, and dynamic virtu
 
 This documentation covers Netmaker's :doc:`installation <./server-installation>`, :doc:`usage <./usage>`, :doc:`troubleshooting <./support>`, and customization, as well as reference documents for the :doc:`API <./api>`, UI and Agent configuration. All of the `source code <https://github.com/gravitl/netmaker>`_ for Netmaker is on GitHub.
 
-
-.. :raw-html:`<br />`
-
-.. .. raw:: html
-..   :file: youtube-1.html
+**For Kubernetes-specific guidance, please see the** `Netmaker Kubernetes Documentation. <https://k8s.netmaker.org>`_
 
 About
-------
-A quick overview of Netmaker, explaining what it is, how it works, and why you should be using it.
+--------
 
-.. toctree::
-   :maxdepth: 2
-   
-   about
-
-Architecture
----------------
-
-A technical overview of Netmaker, including design decisions and limitations.
+High-level information about what Netmaker is and how it works.
 
 .. toctree::
    :maxdepth: 2
+
+   about
    
    architecture
 
-Install
+Getting Started
 ------------------------------------
 
-Choose the right install method for you.
+How to install Netmaker and set up your first network.
 
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
 
    install
 
-Quick Start
----------------
+   quick-start
 
-A quick start guide to getting up and running with Netmaker and WireGuard as quickly as possible.
+   getting-started
+
+Ingress, Egress, and Relays
+------------------------------
+
+How to give machines outside of the Netmaker network access to network resources via an Ingress Gateway:
 
 .. toctree::
    :maxdepth: 2
+   
+   external-clients
+
+How to give machines inside the Netmaker network access to external network resources via an Egress Gateway:
 
-   quick-start
 
 .. toctree::
    :maxdepth: 2
+   
+   egress-gateway
 
-   getting-started
+How to make machines inside the network reachable if they are blocked by NAT/Firewall:
 
-Quick Start Nginx (depreciated)
-------------------------------------
+.. toctree::
+   :maxdepth: 2
+   
+   relay-server
 
-An older guide to getting up and running with Netmaker using Nginx as quickly as possible.
+Kubernetes Documentation
+---------------------------
 
 .. toctree::
-   :maxdepth: 1
 
-   quick-start-nginx
+   Kubernetes <https://k8s.netmaker.org>
+   
+`Netmaker Kubernetes Documentation <https://k8s.netmaker.org>`_
 
-Server Installation
---------------------
+
+Advanced Server Installation
+-------------------------------
 
 A detailed guide to installing the Netmaker server (API, DB, UI, DNS), and configuration options.
 
@@ -93,59 +91,69 @@ A detailed guide to installing the Netmaker server (API, DB, UI, DNS), and confi
    
    server-installation
 
-Oauth Configuration
---------------------
+Advanced Client Installation
+--------------------------------
 
-A simple guide to configuring OAuth for Netmaker.
+A detailed guide to installing the Netmaker agent (netclient) on devices and configuration options.
 
 .. toctree::
    :maxdepth: 2
    
-   oauth
+   client-installation
 
 
-Client Installation
+Oauth Configuration
 --------------------
 
-A detailed guide to installing the Netmaker agent (netclient) on devices and configuration options.
+A simple guide to configuring OAuth for Netmaker.
 
 .. toctree::
    :maxdepth: 2
    
-   client-installation
+   oauth
 
-External Clients
---------------------
 
-A detailed guide to give clients outside of the Netmaker network access to network resources.
+External Guides
+----------------
+
+A handful of guides for use cases including site-to-site, Kubernetes, private DNS, and more.
 
 .. toctree::
    :maxdepth: 2
    
-   external-clients
+   usage
 
-Guides
-----------------
+UI Reference
+---------------
 
-A handful of guides for use cases including site-to-site, Kubernetes, private DNS, and more.
+A reference document for the Netmaker Server UI, with annotated screenshot detailing each field.
 
 .. toctree::
    :maxdepth: 2
-   
-   usage
+
+   ui-reference
 
 API Reference
 ---------------
 
 A reference document for the Netmaker Server API, and example API calls for various use cases.
 
-**Coming Soon:** Swagger Documentation
-
 .. toctree::
    :maxdepth: 1
 
    api
 
+Upgrades
+----------------
+
+Upgrading the Netmaker server and clients.
+
+.. toctree::
+   :maxdepth: 1
+
+   upgrades
+
+
 Troubleshooting
 ----------------
 

docs/_build/html/_sources/oauth.rst.txt

@@ -35,7 +35,7 @@ Instructions for Microsoft Azure AD: https://oauth2-proxy.github.io/oauth2-proxy
 Configuring Netmaker
 ======================
 
-After you have configured your OAuth provider, take note of the CLIENT_ID and CLIENT_SECRET.
+After you have configured your OAuth provider, take note of the CLIENT_ID and CLIENT_SECRET. If you are using Azure for oauth, you may also want to note down the Azure tenant ID you wish to use.
 
 Next, Configure Netmaker with the following environment variables. If any are left blank, OAuth will fail.
 
@@ -46,7 +46,7 @@ Next, Configure Netmaker with the following environment variables. If any are le
     CLIENT_SECRET: "<client secret of your oauth provider>"
     SERVER_HTTP_HOST: "api.<netmaker base domain>"
     FRONTEND_URL: "https://dashboard.<netmaker base domain>"
-
+    AZURE_TENANT: "<only for azure, you may optionally specify the tenant for the OAuth>"
 
 After restarting your server, the Netmaker logs will indicate if the OAuth provider was successfully initialized:
 

docs/_build/html/_sources/quick-start-nginx.rst.txt

@@ -1,170 +0,0 @@
-==================================
-Install with Nginx (depreciated)
-==================================
-
-This is the old quick start guide, which contains instructions using Nginx and Docker CE. It is recommended to use the new quick start guide with Caddy instead.
-
-0. Introduction
-==================
-
-We assume for this installation that you want all of the Netmaker features enabled, you want your server to be secure, and you want your server to be accessible from anywhere.
-
-This instance will not be HA. However, it should comfortably handle around one hundred concurrent clients and support the most common use cases.
-
-If you are deploying for a business or enterprise use case and this setup will not fit your needs, please contact [email protected], or check out the business subscription plans at https://gravitl.com/plans/business.
-
-By the end of this guide, you will have Netmaker installed on a public VM linked to your custom domain, secured behind an Nginx reverse proxy.
-
-For information about deploying more advanced configurations, see the :doc:`Advanced Installation <./server-installation>` docs. 
-
-
-1. Prerequisites
-==================
--  **Virtual Machine**
-   
-   - Preferably from a cloud provider (e.x: DigitalOcean, Linode, AWS, GCP, etc.)
-      - We do not recommend Oracle Cloud, as VM's here have been known to cause network interference.
-   - Public, static IP 
-   - Min 1GB RAM, 1 CPU (4GB RAM, 2CPU preferred)
-      - Nginx may have performance issues if using a cloud VPS with a single, shared CPU
-   - 2GB+ of storage 
-   - Ubuntu  20.04 Installed
-
-- **Domain**
-
-  - A publicly owned domain (e.x. example.com, mysite.biz) 
-  - Permission and access to modify DNS records via DNS service (e.x: Route53)
-
-2. Install Dependencies
-========================
-
-``ssh root@your-host``
-
-Install Docker
----------------
-Begin by installing the community version of Docker and docker-compose (there are issues with the snap version). You can follow the official `Docker instructions here <https://docs.docker.com/engine/install/>`_. Or, you can use the below series of commands which should work on Ubuntu 20.04.
-
-.. code-block::
-
-  sudo apt-get remove docker docker-engine docker.io containerd runc
-  sudo apt-get update
-  sudo apt-get -y install apt-transport-https ca-certificates curl gnupg lsb-release
-  curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg  
-  echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null
-  sudo apt-get update
-  sudo apt-get -y install docker-ce docker-ce-cli containerd.io
-  sudo curl -L "https://github.com/docker/compose/releases/download/1.29.2/docker-compose-$(uname -s)-$(uname -m)" -o /usr/local/bin/docker-compose
-  sudo chmod +x /usr/local/bin/docker-compose
-  docker --version
-  docker-compose --version
-
-At this point Docker should be installed.
-
-Install Dependencies
------------------------------
-
-In addition to Docker, this installation requires WireGuard, Nginx, and Certbot.
-
-``sudo apt -y install wireguard wireguard-tools nginx certbot python3-certbot-nginx net-tools``
-
- 
-3. Prepare VM
-===============================
-
-Prepare Domain
-----------------------------
-1. Choose a base domain or subdomain for Netmaker. If you own **example.com**, this should be something like **netmaker.example.com**
-
-- You must point your wildcard domain to the public IP of your VM, e.x: *.example.com --> <your public ip>
-
-2. Add an A record pointing to your VM using your DNS service provider for *.netmaker.example.com (inserting your own subdomain of course).
-3. Netmaker will create three subdomains on top of this. For the example above those subdomains would be:
-
-- dashboard.netmaker.example.com
-
-- api.netmaker.example.com
-
-- grpc.netmaker.example.com
-
-Moving forward we will refer to your base domain using **<your base domain>**. Replace these references with your domain (e.g. netmaker.example.com).
-
-4. ``nslookup host.<your base domain>`` (inserting your domain) should now return the IP of your VM.
-
-5. Generate SSL Certificates using certbot:
-
-``sudo certbot certonly --manual --preferred-challenges=dns --email [email protected] --server https://acme-v02.api.letsencrypt.org/directory --agree-tos --manual-public-ip-logging-ok -d "*.<your base domain>"``
-
-The above command (using your domain instead of <your base domain>), will prompt you to enter a TXT record in your DNS service provider. Do this, and **wait one  minute** before clicking enter, or it may fail and you will have to run the command again.
-
-Prepare Firewall
------------------
-
-Make sure firewall settings are appropriate for Netmaker. You need ports 53 and 443. On the server you can run:
-
-
-.. code-block::
-
-  sudo ufw allow proto tcp from any to any port 443 && sudo ufw allow 53/udp && sudo ufw allow 53/tcp
-
-**Based on your cloud provider, you may also need to set inbound security rules for your server. This will be dependent on your cloud provider. Be sure to check before moving on:**
-  - allow 443/tcp from all
-  - allow 53/udp and 53/tcp from all
-
-In addition to the above ports, you will need to make sure that your cloud's firewall or security groups are opened for the range of ports that Netmaker's WireGuard interfaces consume.
-
-Netmaker will create one interface per network, starting from 51821. So, if you plan on having 5 networks, you will want to have at least 51821-51825 open (udp).
-
-Prepare Nginx
------------------
-
-Nginx will serve the SSL certificate with your chosen domain and forward traffic to netmaker.
-
-Get the nginx configuration file:
-
-``wget https://raw.githubusercontent.com/gravitl/netmaker/master/nginx/netmaker-nginx-template.conf``
-
-Insert your domain in the configuration file and add to nginx:
-
-.. code-block::
-
-  sed -i 's/NETMAKER_BASE_DOMAIN/<your base domain>/g' netmaker-nginx-template.conf
-  sudo cp netmaker-nginx-template.conf /etc/nginx/conf.d/<your base domain>.conf
-  nginx -t && nginx -s reload
-  systemctl restart nginx
-
-4. Install Netmaker
-====================
-
-Prepare Templates
-------------------
-
-**Note on COREDNS_IP:** Depending on your cloud provider, the public IP may not be bound directly to the VM on which you are running. In such cases, CoreDNS cannot bind to this IP, and you should use the IP of the default interface on your machine in place of COREDNS_IP. If the public IP **is** bound to the VM, you can simply use the same IP as SERVER_PUBLIC_IP.
-
-.. code-block::
-
-  wget https://raw.githubusercontent.com/gravitl/netmaker/master/compose/docker-compose.yml
-  sed -i 's/NETMAKER_BASE_DOMAIN/<your base domain>/g' docker-compose.yml
-  sed -i 's/SERVER_PUBLIC_IP/<your server ip>/g' docker-compose.yml
-  sed -i 's/COREDNS_IP/<your server ip>/g' docker-compose.yml
-
-Generate a unique master key and insert it:
-
-.. code-block::
-
-  tr -dc A-Za-z0-9 </dev/urandom | head -c 30 ; echo ''
-  sed -i 's/REPLACE_MASTER_KEY/<your generated key>/g' docker-compose.yml
-
-You may want to save this key for future use with the API.
-
-Start Netmaker
-----------------
-
-``sudo docker-compose -f docker-compose.yml up -d``
-
-navigate to dashboard.<your base domain> to log into the UI.
-
-To troubleshoot issues, start with:
-
-``docker logs netmaker``
-
-Or check out the :doc:`troubleshoooting docs <./troubleshoot>`.

docs/_build/html/_sources/quick-start.rst.txt

@@ -44,7 +44,7 @@ For information about deploying more advanced configurations, see the :doc:`Adva
 1. Prepare DNS
 ================
 
-Create a wildcard A record pointing to the public IP of your VM. As an example, *.netmaker.example.com.
+Create a wildcard A record pointing to the public IP of your VM. As an example, \*.netmaker.example.com.
 
 Caddy will create 3 subdomains with this wildcard, EX:
 
@@ -74,7 +74,8 @@ Make sure firewall settings are set for Netmaker both on the VM and with your cl
 Make sure the following ports are open both on the VM and in the cloud security groups:
 
 - **443 (tcp):** for Dashboard, REST API, and gRPC
-- **53 (udp and tcp):** for CoreDNS
+- **80 (tcp):** for LetsEncrypt
+- **53 (udp and tcp):** for CoreDNS - This is no longer necessary as of 0.10.0, as by default DNS queries will run over WireGuard.
 - **51821-518XX (udp):** for WireGuard - Netmaker needs one port per network, starting with 51821, so open up a range depending on the number of networks you plan on having. For instance, 51821-51830.
 
 .. code-block::
@@ -83,7 +84,8 @@ Make sure the following ports are open both on the VM and in the cloud security
 
 **Again, based on your cloud provider, you may additionally need to set inbound security rules for your server (for instance, on AWS). This will be dependent on your cloud provider. Be sure to check before moving on:**
   - allow 443/tcp from all
-  - allow 53/udp and 53/tcp from all
+  - allow 80/tcp from all
+  - (optional) allow 53/udp and 53/tcp from all
   - allow 51821-51830/udp from all
 
 
@@ -93,7 +95,8 @@ Make sure the following ports are open both on the VM and in the cloud security
 Prepare Docker Compose 
 ------------------------
 
-**Note on COREDNS_IP:** Depending on your cloud provider, the public IP may not be bound directly to the VM on which you are running. In such cases, CoreDNS cannot bind to this IP, and you should use the IP of the default interface on your machine in place of COREDNS_IP. This command will get you the correct IP for CoreDNS in many cases:
+**Note 1 on COREDNS_IP:** As of 0.10.0, the default installation does not require COREDNS_IP to be set. Queries will run over WireGuard.
+**Note 2 on COREDNS_IP:** Depending on your cloud provider, the public IP may not be bound directly to the VM on which you are running. In such cases, CoreDNS cannot bind to this IP, and you should use the IP of the default interface on your machine in place of COREDNS_IP. This command will get you the correct IP for CoreDNS in many cases:
 
 .. code-block::
 
@@ -127,6 +130,17 @@ Prepare Caddy
   sed -i 's/NETMAKER_BASE_DOMAIN/<your base domain>/g' /root/Caddyfile
   sed -i 's/YOUR_EMAIL/<your email>/g' /root/Caddyfile
 
+Prepare MQ
+------------------------
+
+
+You must retrieve the MQ configuration file for Mosquitto.
+
+.. code-block::
+
+  wget -O /root/mosquitto.conf https://raw.githubusercontent.com/gravitl/netmaker/master/docker/mosquitto.conf
+
+
 Start Netmaker
 ----------------
 

docs/_build/html/_sources/relay-server.rst.txt

@@ -0,0 +1,36 @@
+=====================================
+Relay Servers
+=====================================
+
+Introduction
+===============
+
+.. image:: images/relay1.png
+   :width: 80%
+   :alt: Relay
+   :align: center
+
+Sometimes nodes are in hard-to-reach places. Typically this will be due to a CGNAT, Double NAT, or restrictive firewall. In such scenarios, a direct peer-to-peer connection with all other nodes might be impossible.
+
+For this reason, Netmaker has a Relay Server functionality. At any time you may designate a publicly reachable node (such as the Netmaker Server) as a Relay, and tell it which machines it should relay. Then, all traffic routing to and from that machine will go through the relay. This allows you to circumvent the above issues and ensure connectivity when direct measures do not work.
+
+Configuring a Relay
+==================================
+
+To create a relay, you can use any node in your network, but it should have a public IP address (not behind a NAT). Your Netmaker server can be a relay server and makes for a good default choice if you are unsure of which node to select.
+
+Simply click the relay button in the nodes list. Then, specify the nodes which it should relay. You can either enter the IP's directly, select from a list, or click "Select All."
+
+.. image:: images/ui-7.jpg
+   :width: 80%
+   :alt: Relay
+   :align: center
+
+If you choose "select all" this essentially turns your network into a hub-and-spoke network. All traffic now routes over the relay node. This can create a bottleneck and slow down your network, but in some scenarios may simplify network operations.
+
+After creation, you can change the list of relayed nodes by clicking "edit node" and editing the list (Field #12 below).
+
+.. image:: images/ui-5.jpg
+   :width: 40%
+   :alt: Relay
+   :align: center

docs/_build/html/_sources/server-installation.rst.txt

@@ -138,8 +138,51 @@ CLIENT_MODE:
 
     **Description:** Specifies if server should deploy itself as a node (client) in each network. May be turned to "off" for more restricted servers.
 
+RCE:  
+    **Default:** "off"
+
+    **Description:** The server enables you to set PostUp and PostDown commands for nodes, which is standard for WireGuard with wg-quick, but is also **Remote Code Execution**, which is a critical vulnerability if the server is exploited. Because of this, it's turned off by default, but if turned on, PostUp and PostDown become editable.
+
+SERVER_GRPC_WIREGUARD
+    **Depreciated:** no longer in use
+
+DISPLAY_KEYS
+    **Default:** "on"
+
+    **Description:** If "on", will allow you to always show the key values of "access keys". This could be considered a vulnerability, so if turned "off", will only display key values once, and it is up to the users to store the key values locally.
+
+NODE_ID
+    **Default:** <system mac addres>
+
+    **Description:** This setting is used for HA configurations of the server, to identify between different servers. Nodes are given ID's like netmaker-1, netmaker-2, and netmaker-3. If the server is not HA, there is no reason to set this field.
+
+TELEMETRY
+    **Default:** "on"
+
+    **Description:** If "on", the server will send anonymous telemetry data once daily, which is used to improve the product. Data sent includes counts (integer values) for the number of nodes, types of nodes, users, and networks. It also sends the version of the server.
+
+MQ_HOST 
+    **Default:** (public IP of server)
+
+    **Description:** The address of the mq server. If running from docker compose it will be "mq". If using "host networking", it will find and detect the IP of the mq container. Otherwise, need to input address. If not set, it will use the public IP of the server. the port 1883 will be appended automatically. This is the expected reachable port for MQ and cannot be changed at this time.
+
+HOST_NETWORK: 
+    **Default:** "off"
+
+    **Description:** Whether or not host networking is turned on. Only turn on if configured for host networking (see docker-compose.hostnetwork.yml). Will set host-level settings like iptables and forwarding for MQ.
+
+MANAGE_IPTABLES: 
+    **Default:** "on"
+
+    **Description:** # Sets iptables on the machine being managed in order to forward properly from wireguard interface to MQ and other services listed in "port forward services." It's better to leave this on unless you know what you're doing.
+
+PORT_FORWARD_SERVICES: 
+    **Default:** ""
+
+    **Description:** Comma-separated list of services for which to configure port forwarding on the machine. Options include "mq,dns,ssh". Typically best to leave mq and dns on. ssh can be removed.'ssh' forwards port 22 over wireguard, enabling ssh to server over wireguard. dns enables private dns over wireguard. mq enables mq.
+
 Config File Reference
-----------------------
+-----------------------
 A config file may be placed under config/environments/<env-name>.yml. To read this file at runtime, provide the environment variable NETMAKER_ENV at runtime. For instance, dev.yml paired with ENV=dev. Netmaker will load the specified Config file. This allows you to store and manage configurations for different environments. Below is a reference Config File you may use.
 
 .. literalinclude:: ../config/environments/dev.yaml
@@ -153,6 +196,20 @@ All environment variables and options are enabled in this file. It is the equiva
 .. literalinclude:: ../compose/docker-compose.reference.yml
   :language: YAML
 
+Available docker-compose files
+---------------------------------
+
+The default options for docker-compose can be found here: https://github.com/gravitl/netmaker/tree/master/compose
+
+The following is a brief description of each:
+
+- `docker-compose.contained.yml <https://github.com/gravitl/netmaker/blob/master/compose/docker-compose.contained.yml>`_ - This is the default docker-compose, used in the quick start and deployment script in the README on GitHub. It deploys Netmaker with all options included (Caddy and CoreDNS) and has "self-contained" netclients, meaning they do not affect host networking.
+- `docker-compose.coredns.yml <https://github.com/gravitl/netmaker/blob/master/compose/docker-compose.coredns.yml>`_ - This is a simple compose used to spin up a standalone CoreDNS server. Can be useful if, for instance, you are unning Netmaker on baremetal but need CoreDNS.
+- `docker-compose.hostnetwork.yml <https://github.com/gravitl/netmaker/blob/master/compose/docker-compose.hostnetwork.yml>`_ - This is similar to the docker-compose.contained.yml but with a key difference: it has advanced permissions and mounts host volumes to control networking on the host level.
+- `docker-compose.nocaddy.yml <https://github.com/gravitl/netmaker/blob/master/compose/docker-compose.nocaddy.yml>`_ -= This is the same as docker-compose.contained.yml but without Caddy, in case you need to use a different proxy like Nginx, Traefik, or HAProxy.
+- `docker-compose.nodns.yml <https://github.com/gravitl/netmaker/blob/master/compose/docker-compose.nodns.yml>`_ - This is the same as docker-compose.contained.yml but without CoreDNS, in which case you will not have the Private DNS feature.
+- `docker-compose.reference.yml <https://github.com/gravitl/netmaker/blob/master/compose/docker-compose.reference.yml>`_ - This is the same as docker-compose.contained.yml but with all variable options on display and annotated (it's what we show right above this section). Use this to determine which variables you should add or change in your configuration.
+- `docker-compose.yml <https://github.com/gravitl/netmaker/blob/master/compose/docker-compose.yml>`_ - This is a renamed docker-compose.contained.yml. It is meant only to act as a placeholder for what we consider the "primary" docker-compose that users should work with.
 
 DNS Mode Setup
 ====================================
@@ -224,18 +281,21 @@ This template is equivalent but omits CoreDNS.
 Linux Install without Docker
 =============================
 
-Most systems support Docker, but some do not. In such environments, there are many options for installing Netmaker. Netmaker is available as a binary file, and there is a zip file of the Netmaker UI static HTML on GitHub. Beyond the UI and Server, you need to install MongoDB and CoreDNS (optional). 
-
-To start, we recommend following the Nginx instructions in the :doc:`Quick Install <./quick-start>` guide to enable SSL for your environment.
+Most systems support Docker, but some do not. In such environments, there are many options for installing Netmaker. Netmaker is available as a binary file, and there is a zip file of the Netmaker UI static HTML on GitHub. Beyond the UI and Server, you may want to optionally install a database (sqlite is embedded, rqlite or postgres are supported) and CoreDNS (also optional). 
 
 Once this is enabled and configured for a domain, you can continue with the below. The recommended server runs Ubuntu 20.04.
 
-rqlite Setup
-----------------
+Database Setup (optional)
+--------------------------
+
+You can run the netmaker binary standalone and it will run an embedded sqlite server. Data goes in the data/ directory. Optionally, you can run PostgreSQL or rqlite. Instructions for rqlite are below.
+
 1. Install rqlite on your server: https://github.com/rqlite/rqlite
 
 2. Run rqlite: rqlited -node-id 1 ~/node.1
 
+If using rqlite or postgres, you must change the DATABASE environment/config variable and enter connection details.
+
 Server Setup
 -------------
 1. **Run the install script:** 
@@ -265,8 +325,18 @@ The following uses Nginx as an http server. You may alternatively use Apache or
   sudo sh -c 'BACKEND_URL=http://<YOUR BACKEND API URL>:PORT /usr/share/nginx/html/generate_config_js.sh >/usr/share/nginx/html/config.js'
   sudo systemctl start nginx
 
-CoreDNS Setup
-----------------
+CoreDNS Setup (optional)
+----------------------------
+
+CoreDNS is only required if you want private DNS features. Once installed, you must set the CoreDNS variables in the env settings of the server.
+
+See https://coredns.io/manual/toc/#installation
+
+Proxy / Load Balancer
+------------------------
+
+You will need to proxy connections to your UI and Server. By default the ports are 8081, 8082, and 50051 (grpc). This proxy should handle SSL certificates. We recommend Caddy or Nginx (you can follow the Nginx guide in these docs). The proxy must be able to handle gRPC connections.
+
 
 .. _KubeInstall:
 

docs/_build/html/_sources/support.rst.txt

@@ -5,25 +5,10 @@ Support
 FAQ
 ======
 
-Does/Will Netmaker Support X Operating System?
---------------------------------------------------
-
-Netmaker is initially available on a limited number of operating systems for good reason: Every operating system is designed differently. With a small team, we can either focus on making Netmaker do a lot on a few number of operating systems, or a little on a bunch of operating systems. We chose the first option. You can view the System Compatibility docs for more info, but in general, you should only be using Netmaker on systemd linux right now.
-
-However, via "external clients", any device that supports WireGuard can be added to the network. 
-
-In future iterations will expand the operating system support for Netclient, and devices that must use the "external client" feature can switch to Netclient.
-
-How do I install the Netclient on X?
----------------------------------------
-
-As per the above, there are many unsupported operating systems. You are still welcome to try, it is just an executable binary file after all. If the system is unix-based and has kernel WireGuard installed, netclient may very well mesh the device into the network. However, the service likely will encounter problems retrieving updates.
-
-
 Is Netmaker a VPN like NordNPN?
 --------------------------------
 
-No. Netmaker makes Virtual Networks, which are technically VPNs, but different. It's more like a corporate VPN, or a VPC (if you're familiar with AWS).
+No. Netmaker makes Virtual Networks, which are technically VPNs, but different. It's more like a corporate VPN, or a VPC (if you're familiar with AWS). Netmaker is often compared to OpenVPN, Tailscale, or Nebula.
 
 If you're looking to achieve self-hosted web browsing, with functionality similar to NordVPN, ExpressVPN, Surfshark, Tunnelbear, or Private Internet Access, this is probably not the project for you. Technically, you can accomplish this with Netmaker, but it would be a little like using a all-terrain vehicle for stock car racing.
 
@@ -34,23 +19,52 @@ https://github.com/pivpn/pivpn
 https://github.com/subspacecloud/subspace
 https://github.com/mullvad/mullvadvpn-app
 
-Do you offer any enterprise support?
---------------------------------------
+Do you have an 'Exit Nodes' feature?
+---------------------------------------
+
+Please see the :doc:`Egress Gateway <./egress-gateway>` documentation.
 
-If you are interested in enterprise support for your project, please contact [email protected].
+Do you offer any business or enterprise support?
+---------------------------------------------------
+
+Yes, please contact [email protected] or visit https://gravitl.com/plans.
 
 
 Why the SSPL License?
 ----------------------
 
-We thought long and hard about the license. Ultimately, we think this is the best way to support and ensure the health of the project long term. The community deserves something that is well-maintained, and in order to do that, eventually we need some financial support. We won't do that by limiting the project, but we will offer some additional support, and hosted options for things people would end up paying for anyway (relay servers, load balancing support, backups). 
+As of now, we think the SSPL is the best way to ensure the long-term viability of the project, but we are regularly evaluating this to see if an OSI-approved license makes more sense.
+
+We believe the SSPL lets most people run the project the way they want, for both for private use and business use, while giving us a path to maintain viability. We are working to make sure the guidelines clear, and do not want the license to impact the community's ability to use and modify the project.
+
+If you believe the SSPL will negatively impact your ability to use the project, please do not hesitate to reach out.
+
+Telemetry
+==============
+
+As of v0.10.0, Netmaker collects "opt-out" telemetry data. To opt out, simply set "TELEMETRY=off" in your docker-compose file.
+
+Please consider participating in telemetry, as it helps us focus on the features and bug fixes which are most useful to users. Netmaker is a broad platform, and without this data, it is difficult to know where the team should spend its limited resources.
 
-While SSPL is not an OSI-approved open source license, it let's people generally run the project however they want, both for private use and business use, without running into the issue of someone else monetizing the project and making it financially untenable. We are working on making the guidelines clear, and will make sure that the license does not impact the communities ability to use and modify the project.
+The following is the full list of telemetry data we collect. Besides "Server Version" all data is simply an integer count:
 
-If you have concerns about the license leading to project restrictions down the road, just know that there are other paid, closed-source/closed-core options out there, so beyond not wanting to follow that path, we also don't think it's a good idea economically either. We firmly believe that having the project open is not only right, but the best option.
+- Randomized server ID
+- Count of nodes
+- Count of "non-server" nodes
+- Count of external clients
+- Count of networks
+- Count of users
+- Count of linux nodes
+- Count of freebsd nodes
+- Count of macos nodes
+- Count of windows nodes
+- Count of docker nodes
+- Count of k8s nodes
+- Server version
 
-All that said, we will re-evaluate the license on a regular basis and determine if an OSI-approved license makes more sense. It's just easier to move from SSPL to another license than vice-versa.
+We use  `PostHog <https://https://posthog.com/>`_, an open source and trusted framework for telemetry data.
 
+To look at exactly we collect telemetry, you can view the source code under serverctl/telemetry.go: https://github.com/gravitl/netmaker/blob/master/serverctl/telemetry.go
 
 Contact
 ===========