Merge branch 'develop' into GRA-1401/host-nattype

.github/workflows/branchtest.yml

@@ -0,0 +1,58 @@
+name: Deploy and Test Branch
+
+on:
+  workflow_dispatch:
+  pull_request:
+      types: [opened, synchronize, reopened]
+      branches: [develop]
+
+jobs:
+  skip-check:
+    runs-on: ubuntu-latest
+    outputs:
+      skip: ${{ steps.check.outputs.skip }}
+    steps:
+      - id: skip
+        uses: fkirc/skip-duplicate-actions@v5
+        with:
+          concurrent_skipping: 'always'
+  getbranch:
+    runs-on: ubuntu-latest
+    needs: skip-check
+    if: ${{ needs.skip-check.outputs.skip != 'true' }}
+    outputs:
+      netclientbranch: ${{ steps.checkbranch.outputs.netclientbranch }}
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+        with:
+          repository: gravitl/netclient
+          ref: develop
+      - name: check if branch exists
+        id: checkbranch
+        run: |
+          if git show-ref ${{ github.head_ref}}; then
+            echo branch exists
+            echo "netclientbranch=${{ github.head_ref }}" >> $GITHUB_OUTPUT
+          else
+            echo branch does not exist
+            echo "netclientbranch=develop" >> $GITHUB_OUTPUT
+          fi
+  
+  terraform:
+    needs: getbranch
+    uses: gravitl/devops/.github/workflows/terraform.yml@master
+    with:
+      netmakerbranch: ${{ github.head_ref }}
+      netclientbranch: ${{ needs.getbranch.outputs.netclientbranch }}
+    secrets: inherit
+
+
+  testbranch:
+    needs: [getbranch, terraform]
+    uses: gravitl/devops/.github/workflows/branchtest.yml@master
+    with:
+      tag: ${{ github.run_id }}-${{ github.run_attempt }}
+      network: terraform
+    secrets: inherit
+    

.github/workflows/deletedroplets.yml

@@ -0,0 +1,38 @@
+name: Delete Droplets
+
+on:
+  workflow_run:
+    workflows: [Deploy and Test Branch]
+    types:
+      - completed
+
+jobs:
+  on-success:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'success' }}
+    steps:
+      - name: delete droplets
+        run: |
+          sleep 15m
+          curl -X GET \
+            -H "Content-Type: application/json" \
+            -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
+            "https://api.digitalocean.com/v2/droplets?tag_name=$TAG"
+        env:
+          DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }}
+          TAG: ${{ github.event.workflow_run.run_id }}-${{ github.event.workflow_run.run_attempt }}
+
+  on-failure:
+    runs-on: ubuntu-latest
+    if: ${{ github.event.workflow_run.conclusion == 'failure' }}
+    steps:
+      - name: delete droplets
+        run: |
+          sleep 6h
+          curl -X GET \
+            -H "Content-Type: application/json" \
+            -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
+            "https://api.digitalocean.com/v2/droplets?tag_name=$TAG"
+        env:
+          DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }}
+          TAG: ${{ github.event.workflow_run.run_id }}-${{ github.event.workflow_run.run_attempt }}

controllers/ext_client.go

@@ -215,7 +215,7 @@ func getExtClientConf(w http.ResponseWriter, r *http.Request) {
 	if network.DefaultKeepalive != 0 {
 		keepalive = "PersistentKeepalive = " + strconv.Itoa(int(network.DefaultKeepalive))
 	}
-	gwendpoint := host.EndpointIP.String() + ":" + strconv.Itoa(logic.GetPeerListenPort(host))
+	gwendpoint := host.EndpointIP.String() + ":" + strconv.Itoa(host.ListenPort)
 	newAllowedIPs := network.AddressRange
 	if newAllowedIPs != "" && network.AddressRange6 != "" {
 		newAllowedIPs += ","

controllers/hosts.go

@@ -1,6 +1,7 @@
 package controller
 
 import (
+	"context"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -26,6 +27,7 @@ func hostHandlers(r *mux.Router) {
 	r.HandleFunc("/api/hosts/{hostid}/relay", logic.SecurityCheck(false, http.HandlerFunc(createHostRelay))).Methods(http.MethodPost)
 	r.HandleFunc("/api/hosts/{hostid}/relay", logic.SecurityCheck(false, http.HandlerFunc(deleteHostRelay))).Methods(http.MethodDelete)
 	r.HandleFunc("/api/hosts/adm/authenticate", authenticateHost).Methods(http.MethodPost)
+	r.HandleFunc("/api/v1/host", authorize(true, false, "host", http.HandlerFunc(pull))).Methods(http.MethodGet)
 }
 
 // swagger:route GET /api/hosts hosts getHosts
@@ -53,6 +55,53 @@ func getHosts(w http.ResponseWriter, r *http.Request) {
 	json.NewEncoder(w).Encode(apiHosts)
 }
 
+// swagger:route GET /api/v1/host pull pullHost
+//
+// Used by clients for "pull" command
+//
+//			Schemes: https
+//
+//			Security:
+//	  		oauth
+//
+//			Responses:
+//				200: pull
+func pull(w http.ResponseWriter, r *http.Request) {
+
+	hostID := r.Header.Get(hostIDHeader) // return JSON/API formatted keys
+	if len(hostID) == 0 {
+		logger.Log(0, "no host authorized to pull")
+		logic.ReturnErrorResponse(w, r, logic.FormatError(fmt.Errorf("no host authorized to pull"), "internal"))
+		return
+	}
+	host, err := logic.GetHost(hostID)
+	if err != nil {
+		logger.Log(0, "no host found during pull", hostID)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	hPU, err := logic.GetPeerUpdateForHost(context.Background(), "", host, nil, nil)
+	if err != nil {
+		logger.Log(0, "could not pull peers for host", hostID)
+		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
+		return
+	}
+	serverConf := servercfg.GetServerInfo()
+	if servercfg.GetBrokerType() == servercfg.EmqxBrokerType {
+		serverConf.MQUserName = hostID
+	}
+	response := models.HostPull{
+		Host:         *host,
+		ServerConfig: serverConf,
+		Peers:        hPU.Peers,
+		PeerIDs:      hPU.PeerIDs,
+	}
+
+	logger.Log(1, hostID, "completed a pull")
+	w.WriteHeader(http.StatusOK)
+	json.NewEncoder(w).Encode(&response)
+}
+
 // swagger:route PUT /api/hosts/{hostid} hosts updateHost
 //
 // Updates a Netclient host on Netmaker server.

controllers/node.go

@@ -19,6 +19,8 @@ import (
 	"golang.org/x/crypto/bcrypt"
 )
 
+var hostIDHeader = "host-id"
+
 func nodeHandlers(r *mux.Router) {
 
 	r.HandleFunc("/api/nodes", authorize(false, false, "user", http.HandlerFunc(getAllNodes))).Methods(http.MethodGet)
@@ -152,7 +154,7 @@ func authenticate(response http.ResponseWriter, request *http.Request) {
 // even if it's technically ok
 // This is kind of a poor man's RBAC. There's probably a better/smarter way.
 // TODO: Consider better RBAC implementations
-func authorize(nodesAllowed, networkCheck bool, authNetwork string, next http.Handler) http.HandlerFunc {
+func authorize(hostAllowed, networkCheck bool, authNetwork string, next http.Handler) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		var errorResponse = models.ErrorResponse{
 			Code: http.StatusUnauthorized, Message: logic.Unauthorized_Msg,
@@ -184,11 +186,11 @@ func authorize(nodesAllowed, networkCheck bool, authNetwork string, next http.Ha
 				logic.ReturnErrorResponse(w, r, errorResponse)
 				return
 			}
-			//check if node instead of user
-			if nodesAllowed {
+			// check if host instead of user
+			if hostAllowed {
 				// TODO --- should ensure that node is only operating on itself
-				if _, _, _, err := logic.VerifyToken(authToken); err == nil {
-
+				if hostID, _, _, err := logic.VerifyHostToken(authToken); err == nil {
+					r.Header.Set(hostIDHeader, hostID)
 					// this indicates request is from a node
 					// used for failover - if a getNode comes from node, this will trigger a metrics wipe
 					next.ServeHTTP(w, r)
@@ -244,6 +246,7 @@ func authorize(nodesAllowed, networkCheck bool, authNetwork string, next http.Ha
 					} else {
 						isAuthorized = (nodeID == params["netid"])
 					}
+				case "host":
 				case "user":
 					isAuthorized = true
 				default:

controllers/relay.go

@@ -115,7 +115,7 @@ func createHostRelay(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	relay.HostID = params["hostid"]
-	relayHost, _, err := logic.CreateHostRelay(relay)
+	relayHost, relayedHosts, err := logic.CreateHostRelay(relay)
 	if err != nil {
 		logger.Log(0, r.Header.Get("user"),
 			fmt.Sprintf("failed to create relay on host [%s]: %v", relay.HostID, err))
@@ -131,15 +131,14 @@ func createHostRelay(w http.ResponseWriter, r *http.Request) {
 	}
 	logger.Log(1, r.Header.Get("user"), "created relay on host", relay.HostID)
 	go func(relayHostID string) {
-		relatedhosts := logic.GetRelatedHosts(relayHostID)
-		for _, relatedHost := range relatedhosts {
-			relatedHost.ProxyEnabled = true
-			logic.UpsertHost(&relatedHost)
+		for _, relayedHost := range relayedHosts {
+			relayedHost.ProxyEnabled = true
+			logic.UpsertHost(&relayedHost)
 			if err := mq.HostUpdate(&models.HostUpdate{
 				Action: models.UpdateHost,
-				Host:   relatedHost,
+				Host:   relayedHost,
 			}); err != nil {
-				logger.Log(0, "failed to send host update: ", relatedHost.ID.String(), err.Error())
+				logger.Log(0, "failed to send host update: ", relayedHost.ID.String(), err.Error())
 			}
 		}
 		if err := mq.PublishPeerUpdate(); err != nil {

logic/extpeers.go

@@ -194,6 +194,23 @@ func UpdateExtClient(newclientid string, network string, enabled bool, client *m
 	if err != nil {
 		return client, err
 	}
+	if newclientid != client.ClientID {
+		//name change only
+		client.ClientID = newclientid
+		client.LastModified = time.Now().Unix()
+		data, err := json.Marshal(&client)
+		if err != nil {
+			return nil, err
+		}
+		key, err := GetRecordKey(client.ClientID, client.Network)
+		if err != nil {
+			return nil, err
+		}
+		if err = database.Insert(key, string(data), database.EXT_CLIENT_TABLE_NAME); err != nil {
+			return client, err
+		}
+		return client, nil
+	}
 	client.ClientID = newclientid
 	client.Enabled = enabled
 	SetClientACLs(client, newACLs)

logic/jwts.go

@@ -129,8 +129,8 @@ func VerifyUserToken(tokenString string) (username string, networks []string, is
 	return "", nil, false, err
 }
 
-// VerifyToken - [nodes] Only
-func VerifyToken(tokenString string) (hostID string, mac string, network string, err error) {
+// VerifyHostToken - [hosts] Only
+func VerifyHostToken(tokenString string) (hostID string, mac string, network string, err error) {
 	claims := &models.Claims{}
 
 	// this may be a stupid way of serving up a master key

logic/peers.go

@@ -219,7 +219,7 @@ func GetPeerUpdateForHost(ctx context.Context, network string, host *models.Host
 				}
 				peerConfig.Endpoint = &net.UDPAddr{
 					IP:   peerHost.EndpointIP,
-					Port: GetPeerListenPort(peerHost),
+					Port: peerHost.ListenPort,
 				}
 
 				if uselocal {

models/structs.go

@@ -198,6 +198,14 @@ type TrafficKeys struct {
 	Server []byte `json:"server" bson:"server" yaml:"server"`
 }
 
+// HostPull - response of a host's pull
+type HostPull struct {
+	Host         Host                 `json:"host" yaml:"host"`
+	Peers        []wgtypes.PeerConfig `json:"peers" yaml:"peers"`
+	ServerConfig ServerConfig         `json:"server_config" yaml:"server_config"`
+	PeerIDs      PeerMap              `json:"peer_ids,omitempty" yaml:"peer_ids,omitempty"`
+}
+
 // NodeGet - struct for a single node get response
 type NodeGet struct {
 	Node         Node                 `json:"node" bson:"node" yaml:"node"`

mq/publishers.go

@@ -88,15 +88,19 @@ func PublishSingleHostPeerUpdate(ctx context.Context, host *models.Host, deleted
 	if len(peerUpdate.Peers) == 0 { // no peers to send
 		return nil
 	}
+	proxyUpdate, err := logic.GetProxyUpdateForHost(ctx, host)
+	if err != nil {
+		return err
+	}
+	proxyUpdate.Server = servercfg.GetServer()
 	if host.ProxyEnabled {
-		proxyUpdate, err := logic.GetProxyUpdateForHost(ctx, host)
-		if err != nil {
-			return err
-		}
 		proxyUpdate.Action = models.ProxyUpdate
-		peerUpdate.ProxyUpdate = proxyUpdate
+	} else {
+		proxyUpdate.Action = models.NoProxy
 	}
 
+	peerUpdate.ProxyUpdate = proxyUpdate
+
 	data, err := json.Marshal(&peerUpdate)
 	if err != nil {
 		return err

scripts/nm-quick.sh

@@ -43,7 +43,7 @@ usage () {
     echo "  -t      tag of build; if buildtype=version, tag=version. If builtype=branch or builtype=local, tag=branch"
     echo "  -a      auto-build; skip prompts and use defaults, if none provided"
     echo "examples:"
-	echo "          nm-quick.sh -e -b version -t v0.18.5"
+	echo "          nm-quick.sh -e -b version -t $LATEST"
 	echo "          nm-quick.sh -e -b local -t feature_v0.17.2_newfeature"	
 	echo "          nm-quick.sh -e -b branch -t develop"
     exit 1
@@ -210,11 +210,8 @@ configure_netclient() {
 # setup_nmctl - pulls nmctl and makes it executable
 setup_nmctl() {
 
-	# DEV_TEMP - Temporary instructions for testing
-	wget -O /usr/bin/nmctl https://fileserver.netmaker.org/testing/nmctl
+	wget -O /usr/bin/nmctl https://github.com/gravitl/netmaker/releases/download/$LATEST/nmctl_linux_amd64
 
-	# RELEASE_REPLACE - Use this once release is ready
-	# wget https://github.com/gravitl/netmaker/releases/download/v0.17.1/nmctl
     chmod +x /usr/bin/nmctl
     echo "using server api.$NETMAKER_BASE_DOMAIN"
     echo "using master key $MASTER_KEY"

scripts/nm-upgrade.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-LATEST="testing"
+LATEST="v0.18.5"
 
 # check_version - make sure current version is 0.17.1 before continuing
 check_version() {
@@ -337,17 +337,11 @@ set_compose() {
   rm /root/wait.sh
   rm /root/mosquitto.conf
 
-  # DEV_TEMP
-  wget -O /root/wait.sh https://raw.githubusercontent.com/gravitl/netmaker/develop/docker/wait.sh 
-  # RELEASE_REPLACE - Use this once release is ready
-  # wget -O /root/wait.sh https://raw.githubusercontent.com/gravitl/netmaker/master/docker/wait.sh
+  wget -O /root/wait.sh https://raw.githubusercontent.com/gravitl/netmaker/master/docker/wait.sh
 
   chmod +x /root/wait.sh
 
-  # DEV_TEMP
-  wget -O /root/mosquitto.conf https://raw.githubusercontent.com/gravitl/netmaker/develop/docker/mosquitto.conf
-  # RELEASE_REPLACE - Use this once release is ready
-  # wget -O /root/wait.sh https://raw.githubusercontent.com/gravitl/netmaker/master/docker/wait.sh
+  wget -O /root/mosquitto.conf https://raw.githubusercontent.com/gravitl/netmaker/master/docker/mosquitto.conf
 
   chmod +x /root/mosquitto.conf
 
@@ -431,7 +425,7 @@ setup_netclient() {
 	netclient uninstall
 	set -e
 
-  wget -O /tmp/netclient https://fileserver.netmaker.org/$LATEST/netclient 
+  wget -O /tmp/netclient https://github.com/gravitl/netclient/releases/download/$LATEST/netclient_linux_amd64 
 
 	chmod +x /tmp/netclient
 	/tmp/netclient install
@@ -446,11 +440,8 @@ setup_netclient() {
 # setup_nmctl - pulls nmctl and makes it executable
 setup_nmctl() {
 
-  # DEV_TEMP - Temporary instructions for testing
-  wget https://fileserver.netmaker.org/testing/nmctl
- 
-  # RELEASE_REPLACE - Use this once release is ready
-  # wget https://github.com/gravitl/netmaker/releases/download/v0.17.1/nmctl
+    wget -O nmctl https://github.com/gravitl/netmaker/releases/download/$LATEST/nmctl_linux_amd64
+  
     chmod +x nmctl
     echo "using server $SERVER_HTTP_HOST"
     echo "using master key $MASTER_KEY"