Merge branch 'develop' into nm-quick-ee-instructions-update

.github/workflows/branchtest.yml

@@ -39,71 +39,12 @@ jobs:
             echo branch does not exist
             echo "netclientbranch=develop" >> $GITHUB_OUTPUT
           fi
-  
-  getserver:
-    runs-on: ubuntu-latest
-    needs: skip-check
-    if: ${{ needs.skip-check.outputs.skip != 'true' }}
-    outputs:
-      netmakerserver: ${{ steps.getserver.outputs.server }}
-    steps:
-      - name: setup ssh
-        run: |
-          mkdir -p ~/.ssh/
-          echo "$SSH_KEY" > ~/.ssh/id_devops
-          chmod 600 ~/.ssh/id_devops
-          cat >>~/.ssh/config <<END
-          Host *.clustercat.com
-            User root
-            IdentityFile ~/.ssh/id_devops
-            StrictHostKeyChecking no
-          END
-        env:
-          SSH_KEY: ${{ secrets.TESTING_SSH_KEY }}
-      - name: getserver
-        id: getserver
-        run: |
-          server=""
-          for arg in "branch1" "branch2" "branch3" "branch4" "branch5"; do
-            echo checking $arg
-            result=$( ssh root@server.${arg}.clustercat.com '~/branchtesting/check.sh') 
-            echo $result
-            if [ "$result" == "pass" ]
-            then
-              server=$arg
-              echo $server >> /tmp/server
-              break
-            fi
-          done
-          echo server is $server
-          if [ "$server" == "" ]
-          then
-            echo server not set
-            exit 1
-          fi
-          echo "netmakerserver=${ server }" >> $GITHUB_OUTPUT
-      - name: save server name
-        uses: actions/upload-artifact@v3
-        with:
-          name: server
-          path: /tmp/ping
-          retention-days: 3
-          
-  terraform:
-    needs: [getbranch, getserver]
-    uses: gravitl/devops/.github/workflows/terraform.yml@master
-    with:
-      netmakerbranch: ${{ github.head_ref }}
-      netclientbranch: ${{ needs.getbranch.outputs.netclientbranch }}
-      server: ${{ needs.getserver.outputs.netmakerserver }}
-    secrets: inherit
-
 
-  testbranch:
-    needs: [getbranch, terraform]
-    uses: gravitl/devops/.github/workflows/branchtest.yml@master
+  branchtest:
+    uses: gravitl/devops/.github/workflows/testdeploybranch.yml@master
+    needs: [getbranch, skip-check]
     with:
+      netclientbranch: ${{ needs.getbranch.outputs.netclientbranch }}
+      netmakerbranch: ${{ github.head_ref }}
       tag: ${{ github.run_id }}-${{ github.run_attempt }}
-      network: netmaker
-    secrets: inherit
-    
+    secrets: inherit          

controllers/hosts.go

@@ -9,7 +9,6 @@ import (
 	"github.com/gorilla/mux"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
-	"github.com/gravitl/netmaker/logic/hostactions"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/mq"
 	"github.com/gravitl/netmaker/servercfg"
@@ -261,18 +260,14 @@ func addHostToNetwork(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	logger.Log(1, "added new node", newNode.ID.String(), "to host", currHost.Name)
-	hostactions.AddAction(models.HostUpdate{
-		Action: models.JoinHostToNetwork,
-		Host:   *currHost,
-		Node:   *newNode,
-	})
-	if servercfg.IsMessageQueueBackend() {
+	go func() {
 		mq.HostUpdate(&models.HostUpdate{
-			Action: models.RequestAck,
+			Action: models.JoinHostToNetwork,
 			Host:   *currHost,
+			Node:   *newNode,
 		})
-	}
-
+		mq.PublishPeerUpdate()
+	}()
 	logger.Log(2, r.Header.Get("user"), fmt.Sprintf("added host %s to network %s", currHost.Name, network))
 	w.WriteHeader(http.StatusOK)
 }
@@ -312,6 +307,25 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
+	if node.IsRelayed {
+		// cleanup node from relayednodes on relay node
+		relayNode, err := logic.GetNodeByID(node.RelayedBy)
+		if err == nil {
+			relayedNodes := []string{}
+			for _, relayedNodeID := range relayNode.RelayedNodes {
+				if relayedNodeID == node.ID.String() {
+					continue
+				}
+				relayedNodes = append(relayedNodes, relayedNodeID)
+			}
+			relayNode.RelayedNodes = relayedNodes
+			logic.UpsertNode(&relayNode)
+		}
+	}
+	if node.IsRelay {
+		// unset all the relayed nodes
+		logic.SetRelayedNodes(false, node.ID.String(), node.RelayedNodes)
+	}
 	node.Action = models.NODE_DELETE
 	node.PendingDelete = true
 	logger.Log(1, "deleting  node", node.ID.String(), "from host", currHost.Name)
@@ -320,10 +334,10 @@ func deleteHostFromNetwork(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	// notify node change
-
 	runUpdates(node, false)
 	go func() { // notify of peer change
-		if err := mq.PublishPeerUpdate(); err != nil {
+		err = mq.PublishDeletedNodePeerUpdate(node)
+		if err != nil {
 			logger.Log(1, "error publishing peer update ", err.Error())
 		}
 		if err := mq.PublishDNSDelete(node, currHost); err != nil {

controllers/node.go

@@ -755,7 +755,6 @@ func deleteNode(w http.ResponseWriter, r *http.Request) {
 			relayNode.RelayedNodes = relayedNodes
 			logic.UpsertNode(&relayNode)
 		}
-
 	}
 	if node.IsRelay {
 		// unset all the relayed nodes
@@ -772,17 +771,12 @@ func deleteNode(w http.ResponseWriter, r *http.Request) {
 	if !fromNode { // notify node change
 		runUpdates(&node, false)
 	}
-	go func(deletedNode *models.Node, fromNode bool) { // notify of peer change
+	go func() { // notify of peer change
 		var err error
-		if fromNode {
-			err = mq.PublishDeletedNodePeerUpdate(deletedNode)
-		} else {
-			err = mq.PublishPeerUpdate()
-		}
+		err = mq.PublishDeletedNodePeerUpdate(&node)
 		if err != nil {
 			logger.Log(1, "error publishing peer update ", err.Error())
 		}
-
 		host, err := logic.GetHost(node.HostID.String())
 		if err != nil {
 			logger.Log(1, "failed to retrieve host for node", node.ID.String(), err.Error())
@@ -790,7 +784,7 @@ func deleteNode(w http.ResponseWriter, r *http.Request) {
 		if err := mq.PublishDNSDelete(&node, host); err != nil {
 			logger.Log(1, "error publishing dns update", err.Error())
 		}
-	}(&node, fromNode)
+	}()
 }
 
 func runUpdates(node *models.Node, ifaceDelta bool) {

go.mod

@@ -3,7 +3,7 @@ module github.com/gravitl/netmaker
 go 1.19
 
 require (
-	github.com/eclipse/paho.mqtt.golang v1.4.2
+	github.com/eclipse/paho.mqtt.golang v1.4.3
 	github.com/go-playground/validator/v10 v10.14.1
 	github.com/golang-jwt/jwt/v4 v4.5.0
 	github.com/google/uuid v1.3.0
@@ -35,13 +35,12 @@ require (
 require (
 	github.com/coreos/go-oidc/v3 v3.6.0
 	github.com/gorilla/websocket v1.5.0
-	github.com/pkg/errors v0.9.1
 	golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1
-	gortc.io/stun v1.23.0
 )
 
 require (
 	github.com/devilcove/httpclient v0.6.0
+	github.com/go-jose/go-jose/v3 v3.0.0
 	github.com/guumaster/tablewriter v0.0.10
 	github.com/matryer/is v1.4.1
 	github.com/olekukonko/tablewriter v0.0.5
@@ -51,7 +50,6 @@ require (
 require (
 	cloud.google.com/go/compute/metadata v0.2.1 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.2 // indirect
-	github.com/go-jose/go-jose/v3 v3.0.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect

go.sum

@@ -16,8 +16,8 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/devilcove/httpclient v0.6.0 h1:M5YAfHeNbu+0QxCiOCo/fKN+Hf0BtF/6aovu3NNgcKk=
 github.com/devilcove/httpclient v0.6.0/go.mod h1:ctrAO2gRgTT+GxtRdWBp2SMQ+vacuxXlbhmlM4oWhs8=
-github.com/eclipse/paho.mqtt.golang v1.4.2 h1:66wOzfUHSSI1zamx7jR6yMEI5EuHnT1G6rNA5PM12m4=
-github.com/eclipse/paho.mqtt.golang v1.4.2/go.mod h1:JGt0RsEwEX+Xa/agj90YJ9d9DH2b7upDZMK9HRbFvCA=
+github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQNCyp70xik=
+github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE=
 github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
@@ -50,7 +50,6 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH
 github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q=
 github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI=
 github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So=
-github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc=
 github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/guumaster/tablewriter v0.0.10 h1:A0HD94yMdt4usgxBjoEceNeE0XMJ027euoHAzsPqBQs=
@@ -83,8 +82,6 @@ github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721 h1:RlZweED6sbSArvlE9
 github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
-github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=
-github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posthog/posthog-go v0.0.0-20211028072449-93c17c49e2b0 h1:Y2hUrkfuM0on62KZOci/VLijlkdF/yeWU262BQgvcjE=
@@ -127,7 +124,6 @@ golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERs
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks=
-golang.org/x/net v0.0.0-20200425230154-ff2c4b7c35a0/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20210928044308-7d9f5e0b762b/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/net v0.0.0-20211111083644-e5c967477495/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
@@ -142,7 +138,6 @@ golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o=
 golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
-golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -185,5 +180,3 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gortc.io/stun v1.23.0 h1:CpRQFjakCZMwVKTwInKbcCzlBklj62LGzD3NPdFyGrE=
-gortc.io/stun v1.23.0/go.mod h1:XD5lpONVyjvV3BgOyJFNo0iv6R2oZB4L+weMqxts+zg=

logic/peers.go

@@ -312,7 +312,6 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 				}
 				hostPeerUpdate.NodePeers = append(hostPeerUpdate.NodePeers, nodePeer)
 			}
-			//}
 		}
 		var extPeers []wgtypes.PeerConfig
 		var extPeerIDAndAddrs []models.IDandAddr
@@ -387,6 +386,18 @@ func GetPeerUpdateForHost(network string, host *models.Host, allNodes []models.N
 		}
 		hostPeerUpdate.Peers[i] = peer
 	}
+	if deletedNode != nil {
+		peerHost, err := GetHost(deletedNode.HostID.String())
+		if err == nil && host.ID != peerHost.ID {
+			if _, ok := peerIndexMap[peerHost.PublicKey.String()]; !ok {
+				hostPeerUpdate.Peers = append(hostPeerUpdate.Peers, wgtypes.PeerConfig{
+					PublicKey: peerHost.PublicKey,
+					Remove:    true,
+				})
+			}
+		}
+
+	}
 
 	for i := range hostPeerUpdate.NodePeers {
 		peer := hostPeerUpdate.NodePeers[i]