Merge pull request #1753 from gravitl/feature_proxy_ext

proxy as option, better peer updates to proxy, fix relay with new updates

compose/docker-compose.yml

@@ -41,6 +41,7 @@ services:
       PORT_FORWARD_SERVICES: "dns"
       MQ_ADMIN_PASSWORD: "REPLACE_MQ_ADMIN_PASSWORD"
       STUN_PORT: "3478"
+      PROXY: "on"
     ports:
       - "51821-51830:51821-51830/udp"
     expose:

config/config.go

@@ -77,6 +77,7 @@ type ServerConfig struct {
 	NetmakerAccountID     string `yaml:"netmaker_account_id"`
 	IsEE                  string `yaml:"is_ee"`
 	StunPort              string `yaml:"stun_port"`
+	Proxy                 string `yaml:"proxy"`
 }
 
 // SQLConfig - Generic SQL Config

controllers/node.go

@@ -481,6 +481,14 @@ func getNode(w http.ResponseWriter, r *http.Request) {
 		ServerConfig: servercfg.GetServerInfo(),
 		PeerIDs:      peerUpdate.PeerIDs,
 	}
+	if node.Proxy {
+		proxyPayload, err := logic.GetPeersForProxy(&node, false)
+		if err == nil {
+			response.ProxyUpdate = proxyPayload
+		} else {
+			logger.Log(0, "failed to get proxy update: ", err.Error())
+		}
+	}
 
 	if servercfg.Is_EE && nodeRequest {
 		if err = logic.EnterpriseResetAllPeersFailovers(node.ID, node.Network); err != nil {
@@ -1121,7 +1129,6 @@ func runUpdates(node *models.Node, ifaceDelta bool) {
 
 // updates local peers for a server on a given node's network
 func runServerUpdate(node *models.Node, ifaceDelta bool) error {
-
 	if servercfg.IsClientMode() != "on" || !isServer(node) {
 		return nil
 	}

go.mod

@@ -41,6 +41,7 @@ require (
 
 require (
 	github.com/coreos/go-oidc/v3 v3.4.0
+	github.com/google/gopacket v1.1.19
 	github.com/gorilla/websocket v1.5.0
 	github.com/pkg/errors v0.9.1
 	github.com/sirupsen/logrus v1.9.0

go.sum

@@ -232,6 +232,8 @@ github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8
 github.com/google/go-cmp v0.5.8 h1:e6P7q2lk1O+qJJb4BtCQXlK8vWEO8V1ZeuEdJNOqZyg=
 github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
+github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8=
+github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=

logic/gateway.go

@@ -230,8 +230,8 @@ func CreateIngressGateway(netid string, nodeid string, failover bool) (models.No
 		}
 	}
 	node.SetLastModified()
-	node.PostUp = postUpCmd
-	node.PostDown = postDownCmd
+	// node.PostUp = postUpCmd
+	// node.PostDown = postDownCmd
 	node.UDPHolePunch = "no"
 	if failover && servercfg.Is_EE {
 		node.Failover = "yes"

logic/peers.go

@@ -13,6 +13,7 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic/acls/nodeacls"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/nm-proxy/manager"
 	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/exp/slices"
@@ -31,12 +32,16 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 	if !onlyPeers {
 		if node.IsRelayed == "yes" {
 			relayNode := FindRelay(node)
-			relayEndpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", relayNode.Endpoint, relayNode.LocalListenPort))
-			if err != nil {
-				logger.Log(1, "failed to resolve relay node endpoint: ", err.Error())
+			if relayNode != nil {
+				relayEndpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", relayNode.Endpoint, relayNode.LocalListenPort))
+				if err != nil {
+					logger.Log(1, "failed to resolve relay node endpoint: ", err.Error())
+				}
+				proxyPayload.IsRelayed = true
+				proxyPayload.RelayedTo = relayEndpoint
+			} else {
+				logger.Log(0, "couldn't find relay node for:  ", node.ID, node.PublicKey)
 			}
-			proxyPayload.IsRelayed = true
-			proxyPayload.RelayedTo = relayEndpoint
 
 		}
 		if node.IsRelay == "yes" {
@@ -45,6 +50,7 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 				logger.Log(1, "failed to relayed nodes: ", node.Name, err.Error())
 				proxyPayload.IsRelay = false
 			} else {
+
 				relayPeersMap := make(map[string]manager.RelayedConf)
 				for _, relayedNode := range relayedNodes {
 					payload, err := GetPeersForProxy(&relayedNode, true)
@@ -64,6 +70,7 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 				proxyPayload.RelayedPeerConf = relayPeersMap
 			}
 		}
+
 	}
 
 	for _, peer := range currentPeers {
@@ -76,17 +83,25 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 			logger.Log(1, "failed to parse node pub key: ", peer.ID)
 			continue
 		}
-		endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", peer.Endpoint, peer.LocalListenPort))
+		listenPort := peer.LocalListenPort
+		if listenPort == 0 {
+			listenPort = peer.ListenPort
+		}
+		endpoint, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", peer.Endpoint, listenPort))
 		if err != nil {
 			logger.Log(1, "failed to resolve udp addr for node: ", peer.ID, peer.Endpoint, err.Error())
 			continue
 		}
-		allowedips := getNodeAllowedIPs(node, &peer)
+		allowedips := getNodeAllowedIPs(&peer, node)
 		var keepalive time.Duration
 		if node.PersistentKeepalive != 0 {
 			// set_keepalive
 			keepalive, _ = time.ParseDuration(strconv.FormatInt(int64(node.PersistentKeepalive), 10) + "s")
 		}
+		proxyStatus := peer.Proxy
+		if peer.IsServer == "yes" {
+			proxyStatus = servercfg.IsProxyEnabled()
+		}
 		peers = append(peers, wgtypes.PeerConfig{
 			PublicKey:                   pubkey,
 			Endpoint:                    endpoint,
@@ -94,14 +109,22 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 			PersistentKeepaliveInterval: &keepalive,
 			ReplaceAllowedIPs:           true,
 		})
+		peerConfMap[peer.PublicKey] = manager.PeerConf{
+			Address: peer.PrimaryAddress(),
+			Proxy:   proxyStatus,
+		}
+
 		if !onlyPeers && peer.IsRelayed == "yes" {
 			relayNode := FindRelay(&peer)
 			if relayNode != nil {
 				relayTo, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", relayNode.Endpoint, relayNode.LocalListenPort))
 				if err == nil {
 					peerConfMap[peer.PublicKey] = manager.PeerConf{
+
 						IsRelayed: true,
 						RelayedTo: relayTo,
+						Address:   peer.PrimaryAddress(),
+						Proxy:     proxyStatus,
 					}
 				}
 
@@ -109,6 +132,15 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 
 		}
 	}
+	var extPeers []wgtypes.PeerConfig
+	extPeers, peerConfMap, err = getExtPeersForProxy(node, peerConfMap)
+	if err == nil {
+		peers = append(peers, extPeers...)
+
+	} else if !database.IsEmptyRecord(err) {
+		logger.Log(1, "error retrieving external clients:", err.Error())
+	}
+	proxyPayload.IsIngress = node.IsIngressGateway == "yes"
 	proxyPayload.Peers = peers
 	proxyPayload.PeerMap = peerConfMap
 	proxyPayload.InterfaceName = node.Interface
@@ -151,9 +183,9 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 		return models.PeerUpdate{}, err
 	}
 
-	// if node.IsRelayed == "yes" {
-	// 	return GetPeerUpdateForRelayedNode(node, udppeers)
-	// }
+	if node.IsRelayed == "yes" {
+		return GetPeerUpdateForRelayedNode(node, udppeers)
+	}
 
 	// #1 Set Keepalive values: set_keepalive
 	// #2 Set local address: set_local - could be a LOT BETTER and fix some bugs with additional logic
@@ -175,15 +207,15 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 		// if the node is not a server, set the endpoint
 		var setEndpoint = !(node.IsServer == "yes")
 
-		// if peer.IsRelayed == "yes" {
-		// 	if !(node.IsRelay == "yes" && ncutils.StringSliceContains(node.RelayAddrs, peer.PrimaryAddress())) {
-		// 		//skip -- will be added to relay
-		// 		continue
-		// 	} else if node.IsRelay == "yes" && ncutils.StringSliceContains(node.RelayAddrs, peer.PrimaryAddress()) {
-		// 		// dont set peer endpoint if it's relayed by node
-		// 		setEndpoint = false
-		// 	}
-		// }
+		if peer.IsRelayed == "yes" {
+			if !(node.IsRelay == "yes" && ncutils.StringSliceContains(node.RelayAddrs, peer.PrimaryAddress())) {
+				//skip -- will be added to relay
+				continue
+			} else if node.IsRelay == "yes" && ncutils.StringSliceContains(node.RelayAddrs, peer.PrimaryAddress()) {
+				// dont set peer endpoint if it's relayed by node
+				setEndpoint = false
+			}
+		}
 		if !nodeacls.AreNodesAllowed(nodeacls.NetworkID(node.Network), nodeacls.NodeID(node.ID), nodeacls.NodeID(peer.ID)) {
 			//skip if not permitted by acl
 			continue
@@ -298,7 +330,7 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 func getExtPeers(node *models.Node) ([]wgtypes.PeerConfig, []models.IDandAddr, error) {
 	var peers []wgtypes.PeerConfig
 	var idsAndAddr []models.IDandAddr
-	extPeers, err := GetExtPeersList(node)
+	extPeers, err := GetNetworkExtClients(node.Network)
 	if err != nil {
 		return peers, idsAndAddr, err
 	}
@@ -355,6 +387,72 @@ func getExtPeers(node *models.Node) ([]wgtypes.PeerConfig, []models.IDandAddr, e
 
 }
 
+func getExtPeersForProxy(node *models.Node, proxyPeerConf map[string]manager.PeerConf) ([]wgtypes.PeerConfig, map[string]manager.PeerConf, error) {
+	var peers []wgtypes.PeerConfig
+
+	extPeers, err := GetNetworkExtClients(node.Network)
+	if err != nil {
+		return peers, proxyPeerConf, err
+	}
+	for _, extPeer := range extPeers {
+		pubkey, err := wgtypes.ParseKey(extPeer.PublicKey)
+		if err != nil {
+			logger.Log(1, "error parsing ext pub key:", err.Error())
+			continue
+		}
+
+		if node.PublicKey == extPeer.PublicKey {
+			continue
+		}
+
+		var allowedips []net.IPNet
+		var peer wgtypes.PeerConfig
+		if extPeer.Address != "" {
+			var peeraddr = net.IPNet{
+				IP:   net.ParseIP(extPeer.Address),
+				Mask: net.CIDRMask(32, 32),
+			}
+			if peeraddr.IP != nil && peeraddr.Mask != nil {
+				allowedips = append(allowedips, peeraddr)
+			}
+		}
+
+		if extPeer.Address6 != "" {
+			var addr6 = net.IPNet{
+				IP:   net.ParseIP(extPeer.Address6),
+				Mask: net.CIDRMask(128, 128),
+			}
+			if addr6.IP != nil && addr6.Mask != nil {
+				allowedips = append(allowedips, addr6)
+			}
+		}
+
+		peer = wgtypes.PeerConfig{
+			PublicKey:         pubkey,
+			ReplaceAllowedIPs: true,
+			AllowedIPs:        allowedips,
+		}
+
+		extConf := manager.PeerConf{
+			IsExtClient: true,
+			Address:     extPeer.Address,
+		}
+		if extPeer.IngressGatewayID == node.ID {
+			extConf.IsAttachedExtClient = true
+		}
+		ingGatewayUdpAddr, err := net.ResolveUDPAddr("udp", extPeer.IngressGatewayEndpoint)
+		if err == nil {
+			extConf.IngressGatewayEndPoint = ingGatewayUdpAddr
+		}
+
+		proxyPeerConf[peer.PublicKey.String()] = extConf
+
+		peers = append(peers, peer)
+	}
+	return peers, proxyPeerConf, nil
+
+}
+
 // GetAllowedIPs - calculates the wireguard allowedip field for a peer of a node based on the peer and node settings
 func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet {
 	var allowedips []net.IPNet
@@ -392,60 +490,60 @@ func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet
 		}
 	}
 	// handle relay gateway peers
-	// if peer.IsRelay == "yes" {
-	// 	for _, ip := range peer.RelayAddrs {
-	// 		//find node ID of relayed peer
-	// 		relayedPeer, err := findNode(ip)
-	// 		if err != nil {
-	// 			logger.Log(0, "failed to find node for ip ", ip, err.Error())
-	// 			continue
-	// 		}
-	// 		if relayedPeer == nil {
-	// 			continue
-	// 		}
-	// 		if relayedPeer.ID == node.ID {
-	// 			//skip self
-	// 			continue
-	// 		}
-	// 		//check if acl permits comms
-	// 		if !nodeacls.AreNodesAllowed(nodeacls.NetworkID(node.Network), nodeacls.NodeID(node.ID), nodeacls.NodeID(relayedPeer.ID)) {
-	// 			continue
-	// 		}
-	// 		if iplib.Version(net.ParseIP(ip)) == 4 {
-	// 			relayAddr := net.IPNet{
-	// 				IP:   net.ParseIP(ip),
-	// 				Mask: net.CIDRMask(32, 32),
-	// 			}
-	// 			allowedips = append(allowedips, relayAddr)
-	// 		}
-	// 		if iplib.Version(net.ParseIP(ip)) == 6 {
-	// 			relayAddr := net.IPNet{
-	// 				IP:   net.ParseIP(ip),
-	// 				Mask: net.CIDRMask(128, 128),
-	// 			}
-	// 			allowedips = append(allowedips, relayAddr)
-	// 		}
-	// 		relayedNode, err := findNode(ip)
-	// 		if err != nil {
-	// 			logger.Log(1, "unable to find node for relayed address", ip, err.Error())
-	// 			continue
-	// 		}
-	// 		if relayedNode.IsEgressGateway == "yes" {
-	// 			extAllowedIPs := getEgressIPs(node, relayedNode)
-	// 			allowedips = append(allowedips, extAllowedIPs...)
-	// 		}
-	// 		if relayedNode.IsIngressGateway == "yes" {
-	// 			extPeers, _, err := getExtPeers(relayedNode)
-	// 			if err == nil {
-	// 				for _, extPeer := range extPeers {
-	// 					allowedips = append(allowedips, extPeer.AllowedIPs...)
-	// 				}
-	// 			} else {
-	// 				logger.Log(0, "failed to retrieve extclients from relayed ingress", err.Error())
-	// 			}
-	// 		}
-	// 	}
-	// }
+	if peer.IsRelay == "yes" {
+		for _, ip := range peer.RelayAddrs {
+			//find node ID of relayed peer
+			relayedPeer, err := findNode(ip)
+			if err != nil {
+				logger.Log(0, "failed to find node for ip ", ip, err.Error())
+				continue
+			}
+			if relayedPeer == nil {
+				continue
+			}
+			if relayedPeer.ID == node.ID {
+				//skip self
+				continue
+			}
+			//check if acl permits comms
+			if !nodeacls.AreNodesAllowed(nodeacls.NetworkID(node.Network), nodeacls.NodeID(node.ID), nodeacls.NodeID(relayedPeer.ID)) {
+				continue
+			}
+			if iplib.Version(net.ParseIP(ip)) == 4 {
+				relayAddr := net.IPNet{
+					IP:   net.ParseIP(ip),
+					Mask: net.CIDRMask(32, 32),
+				}
+				allowedips = append(allowedips, relayAddr)
+			}
+			if iplib.Version(net.ParseIP(ip)) == 6 {
+				relayAddr := net.IPNet{
+					IP:   net.ParseIP(ip),
+					Mask: net.CIDRMask(128, 128),
+				}
+				allowedips = append(allowedips, relayAddr)
+			}
+			relayedNode, err := findNode(ip)
+			if err != nil {
+				logger.Log(1, "unable to find node for relayed address", ip, err.Error())
+				continue
+			}
+			if relayedNode.IsEgressGateway == "yes" {
+				extAllowedIPs := getEgressIPs(node, relayedNode)
+				allowedips = append(allowedips, extAllowedIPs...)
+			}
+			if relayedNode.IsIngressGateway == "yes" {
+				extPeers, _, err := getExtPeers(relayedNode)
+				if err == nil {
+					for _, extPeer := range extPeers {
+						allowedips = append(allowedips, extPeer.AllowedIPs...)
+					}
+				} else {
+					logger.Log(0, "failed to retrieve extclients from relayed ingress", err.Error())
+				}
+			}
+		}
+	}
 	return allowedips
 }
 

logic/server.go

@@ -175,16 +175,19 @@ func ServerJoin(networkSettings *models.Network) (models.Node, error) {
 	if err != nil {
 		return returnNode, err
 	}
-	proxyPayload, err := GetPeersForProxy(node, false)
-	if err != nil && !ncutils.IsEmptyRecord(err) {
-		logger.Log(1, "failed to retrieve peers")
-		return returnNode, err
-	}
+	if servercfg.IsProxyEnabled() {
+		proxyPayload, err := GetPeersForProxy(node, false)
+		if err != nil && !ncutils.IsEmptyRecord(err) {
+			logger.Log(1, "failed to retrieve peers")
+			return returnNode, err
+		}
 
-	ProxyMgmChan <- &manager.ManagerAction{
-		Action:  manager.AddInterface,
-		Payload: proxyPayload,
+		ProxyMgmChan <- &manager.ManagerAction{
+			Action:  manager.AddInterface,
+			Payload: proxyPayload,
+		}
 	}
+
 	return *node, nil
 }
 

logic/wireguard.go

@@ -10,6 +10,7 @@ import (
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/netclient/wireguard"
 	"github.com/gravitl/netmaker/nm-proxy/manager"
+	"github.com/gravitl/netmaker/servercfg"
 	"golang.zx2c4.com/wireguard/wgctrl"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
@@ -160,17 +161,20 @@ func setWGConfig(node *models.Node, peerupdate bool) error {
 		logger.Log(3, "finished setting wg config on server", node.Name)
 
 	}
-	logger.Log(0, "--------> ADD/Update INTERFACE TO PROXY.....")
-	proxyPayload, err := GetPeersForProxy(node, false)
-	if err != nil {
-		logger.Log(0, "failed to get peers for proxy: ", err.Error())
-	} else {
+	if servercfg.IsProxyEnabled() {
+		logger.Log(0, "--------> ADD/Update INTERFACE TO PROXY.....")
+		proxyPayload, err := GetPeersForProxy(node, false)
+		if err != nil {
+			logger.Log(0, "failed to get peers for proxy: ", err.Error())
+		} else {
 
-		ProxyMgmChan <- &manager.ManagerAction{
-			Action:  manager.AddInterface,
-			Payload: proxyPayload,
+			ProxyMgmChan <- &manager.ManagerAction{
+				Action:  manager.AddInterface,
+				Payload: proxyPayload,
+			}
 		}
 	}
+
 	return nil
 }
 

main.go

@@ -172,24 +172,29 @@ func startControllers() {
 	if !servercfg.IsAgentBackend() && !servercfg.IsRestBackend() && !servercfg.IsMessageQueueBackend() {
 		logger.Log(0, "No Server Mode selected, so nothing is being served! Set Agent mode (AGENT_BACKEND) or Rest mode (REST_BACKEND) or MessageQueue (MESSAGEQUEUE_BACKEND) to 'true'.")
 	}
+
 	// starts the stun server
 	waitnetwork.Add(1)
 	go stunserver.Start(&waitnetwork)
-	waitnetwork.Add(1)
-	go func() {
-		defer waitnetwork.Done()
-		ctx, cancel := context.WithCancel(context.Background())
+	if servercfg.IsProxyEnabled() {
+
 		waitnetwork.Add(1)
-		go nmproxy.Start(ctx, logic.ProxyMgmChan, servercfg.GetAPIHost())
-		err := serverctl.SyncServerNetworkWithProxy()
-		if err != nil {
-			logger.Log(0, "failed to sync proxy with server interfaces: ", err.Error())
-		}
-		quit := make(chan os.Signal, 1)
-		signal.Notify(quit, syscall.SIGTERM, os.Interrupt)
-		<-quit
-		cancel()
-	}()
+		go func() {
+			defer waitnetwork.Done()
+			ctx, cancel := context.WithCancel(context.Background())
+			waitnetwork.Add(1)
+
+			go nmproxy.Start(ctx, logic.ProxyMgmChan, servercfg.GetAPIHost())
+			err := serverctl.SyncServerNetworkWithProxy()
+			if err != nil {
+				logger.Log(0, "failed to sync proxy with server interfaces: ", err.Error())
+			}
+			quit := make(chan os.Signal, 1)
+			signal.Notify(quit, syscall.SIGTERM, os.Interrupt)
+			<-quit
+			cancel()
+		}()
+	}
 
 	waitnetwork.Wait()
 }

models/node.go

@@ -106,6 +106,7 @@ type Node struct {
 	DefaultACL string `json:"defaultacl,omitempty" bson:"defaultacl,omitempty" yaml:"defaultacl,omitempty" validate:"checkyesornoorunset"`
 	OwnerID    string `json:"ownerid,omitempty" bson:"ownerid,omitempty" yaml:"ownerid,omitempty"`
 	Failover   string `json:"failover" bson:"failover" yaml:"failover" validate:"checkyesorno"`
+	Proxy      bool   `json:"proxy" bson:"proxy" yaml:"proxy"`
 }
 
 // NodesArray - used for node sorting

models/structs.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	jwt "github.com/golang-jwt/jwt/v4"
+	"github.com/gravitl/netmaker/nm-proxy/manager"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
@@ -202,10 +203,11 @@ type TrafficKeys struct {
 
 // NodeGet - struct for a single node get response
 type NodeGet struct {
-	Node         Node                 `json:"node" bson:"node" yaml:"node"`
-	Peers        []wgtypes.PeerConfig `json:"peers" bson:"peers" yaml:"peers"`
-	ServerConfig ServerConfig         `json:"serverconfig" bson:"serverconfig" yaml:"serverconfig"`
-	PeerIDs      PeerMap              `json:"peerids,omitempty" bson:"peerids,omitempty" yaml:"peerids,omitempty"`
+	Node         Node                   `json:"node" bson:"node" yaml:"node"`
+	Peers        []wgtypes.PeerConfig   `json:"peers" bson:"peers" yaml:"peers"`
+	ServerConfig ServerConfig           `json:"serverconfig" bson:"serverconfig" yaml:"serverconfig"`
+	PeerIDs      PeerMap                `json:"peerids,omitempty" bson:"peerids,omitempty" yaml:"peerids,omitempty"`
+	ProxyUpdate  manager.ManagerPayload `json:"proxy_update,omitempty" bson:"proxy_update,omitempty" yaml:"proxy_update,omitempty"`
 }
 
 // ServerConfig - struct for dealing with the server information for a netclient

mq/dynsec_helper.go

@@ -182,6 +182,12 @@ func FetchNetworkAcls(network string) []Acl {
 			Priority: -1,
 			Allow:    true,
 		},
+		{
+			AclType:  "publishClientReceive",
+			Topic:    fmt.Sprintf("proxy/%s/#", network),
+			Priority: -1,
+			Allow:    true,
+		},
 		{
 			AclType:  "subscribePattern",
 			Topic:    "#",
@@ -206,6 +212,12 @@ func fetchServerAcls() []Acl {
 			Priority: -1,
 			Allow:    true,
 		},
+		{
+			AclType:  "publishClientSend",
+			Topic:    "proxy/#",
+			Priority: -1,
+			Allow:    true,
+		},
 		{
 			AclType:  "publishClientSend",
 			Topic:    "update/#",

mq/publishers.go

@@ -31,6 +31,13 @@ func PublishPeerUpdate(newNode *models.Node, publishToSelf bool) error {
 		// 	logger.Log(1, "failed to publish proxy update to node", node.Name, "on network", node.Network, ":", err.Error())
 		// }
 		if node.IsServer == "yes" {
+			if servercfg.IsProxyEnabled() {
+				err := PublishProxyUpdate(manager.AddInterface, &node)
+				if err != nil {
+					logger.Log(0, "failed to send proxy update for server: ", err.Error())
+				}
+			}
+
 			continue
 		}
 		if !publishToSelf && newNode.ID == node.ID {
@@ -63,18 +70,22 @@ func PublishProxyUpdate(action manager.ProxyAction, node *models.Node) error {
 
 // PublishSinglePeerUpdate --- determines and publishes a peer update to one node
 func PublishSinglePeerUpdate(node *models.Node) error {
+
 	peerUpdate, err := logic.GetPeerUpdate(node)
 	if err != nil {
 		return err
 	}
-	proxyUpdate, err := logic.GetPeersForProxy(node, false)
-	if err != nil {
-		return err
-	}
-	peerUpdate.ProxyUpdate = manager.ManagerAction{
-		Action:  manager.AddInterface,
-		Payload: proxyUpdate,
+	if node.Proxy {
+		proxyUpdate, err := logic.GetPeersForProxy(node, false)
+		if err != nil {
+			return err
+		}
+		peerUpdate.ProxyUpdate = manager.ManagerAction{
+			Action:  manager.AddInterface,
+			Payload: proxyUpdate,
+		}
 	}
+
 	data, err := json.Marshal(&peerUpdate)
 	if err != nil {
 		return err
@@ -113,7 +124,9 @@ func PublishExtPeerUpdate(node *models.Node) error {
 
 // NodeUpdate -- publishes a node update
 func NodeUpdate(node *models.Node) error {
+	var err error
 	if !servercfg.IsMessageQueueBackend() || node.IsServer == "yes" {
+
 		return nil
 	}
 	logger.Log(3, "publishing node update to "+node.Name)
@@ -122,19 +135,23 @@ func NodeUpdate(node *models.Node) error {
 		node.NetworkSettings.AccessKeys = []models.AccessKey{} // not to be sent (don't need to spread access keys around the network; we need to know how to reach other nodes, not become them)
 	}
 
-	data, err := json.Marshal(node)
-	if err != nil {
-		logger.Log(2, "error marshalling node update ", err.Error())
-		return err
-	}
-	if err = publish(node, fmt.Sprintf("update/%s/%s", node.Network, node.ID), data); err != nil {
-		logger.Log(2, "error publishing node update to peer ", node.ID, err.Error())
-		return err
-	}
-	err = PublishProxyUpdate(manager.AddInterface, node)
-	if err != nil {
-		logger.Log(1, "failed to publish proxy update to node", node.Name, "on network", node.Network, ":", err.Error())
+	if node.Proxy {
+		err = PublishProxyUpdate(manager.AddInterface, node)
+		if err != nil {
+			logger.Log(1, "failed to publish proxy update to node", node.Name, "on network", node.Network, ":", err.Error())
+		}
+	} else {
+		data, err := json.Marshal(node)
+		if err != nil {
+			logger.Log(2, "error marshalling node update ", err.Error())
+			return err
+		}
+		if err = publish(node, fmt.Sprintf("update/%s/%s", node.Network, node.ID), data); err != nil {
+			logger.Log(2, "error publishing node update to peer ", node.ID, err.Error())
+			return err
+		}
 	}
+
 	return nil
 }
 
@@ -145,6 +162,7 @@ func ProxyUpdate(proxyPayload *manager.ManagerAction, node *models.Node) error {
 	}
 	if node.IsServer == "yes" {
 		logic.ProxyMgmChan <- proxyPayload
+		return nil
 	}
 	logger.Log(3, "publishing proxy update to "+node.Name)
 
@@ -154,7 +172,7 @@ func ProxyUpdate(proxyPayload *manager.ManagerAction, node *models.Node) error {
 		return err
 	}
 	if err = publish(node, fmt.Sprintf("proxy/%s/%s", node.Network, node.ID), data); err != nil {
-		logger.Log(2, "error publishing node update to peer ", node.ID, err.Error())
+		logger.Log(2, "error publishing proxy update to peer ", node.ID, err.Error())
 		return err
 	}
 	return nil
@@ -219,7 +237,10 @@ func sendPeers() {
 			if errN != nil {
 				logger.Log(1, errN.Error())
 			}
-			serverctl.SyncServerNetworkWithProxy()
+			if servercfg.IsProxyEnabled() {
+				serverctl.SyncServerNetworkWithProxy()
+			}
+
 		}
 	}
 }

netclient/cli_options/flags.go

@@ -12,6 +12,13 @@ func GetFlags(hostname string) []cli.Flag {
 			Value:   "all",
 			Usage:   "Network to perform specified action against.",
 		},
+		&cli.StringFlag{
+			Name: "proxy",
+			// Aliases: []string{"np"},
+			EnvVars: []string{"NETMAKER_PROXY"},
+			Value:   "off",
+			Usage:   "To enable/disable proxy.",
+		},
 		&cli.StringFlag{
 			Name:    "password",
 			Aliases: []string{"p"},

netclient/config/config.go

@@ -227,6 +227,10 @@ func GetCLIConfig(c *cli.Context) (ClientConfig, string, error) {
 		if c.String("key") != "" {
 			cfg.AccessKey = c.String("key")
 		}
+		if c.String("proxy") != "" {
+			cfg.Node.Proxy = c.String("proxy") == "on"
+		}
+		log.Println("_______________> PROXY:   ", cfg.Node.Proxy)
 		if c.String("network") != "all" {
 			cfg.Network = c.String("network")
 			cfg.Node.Network = c.String("network")

netclient/functions/common.go

@@ -22,6 +22,7 @@ import (
 	"github.com/gravitl/netmaker/netclient/local"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/netclient/wireguard"
+	"github.com/gravitl/netmaker/nm-proxy/manager"
 	"golang.zx2c4.com/wireguard/wgctrl"
 )
 
@@ -192,6 +193,12 @@ func LeaveNetwork(network string) error {
 	if err := removeHostDNS(cfg.Node.Interface, ncutils.IsWindows()); err != nil {
 		logger.Log(0, "failed to delete dns entries for", cfg.Node.Interface, err.Error())
 	}
+	ProxyMgmChan <- &manager.ManagerAction{
+		Action: manager.DeleteInterface,
+		Payload: manager.ManagerPayload{
+			InterfaceName: cfg.Node.Interface,
+		},
+	}
 	logger.Log(2, "restarting daemon")
 	return daemon.Restart()
 }

netclient/functions/daemon.go

@@ -6,7 +6,6 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"log"
 	"net"
 	"net/http"
 	"os"
@@ -34,7 +33,6 @@ import (
 
 var ProxyMgmChan = make(chan *manager.ManagerAction, 100)
 var messageCache = new(sync.Map)
-
 var serverSet map[string]bool
 
 var mqclient mqtt.Client
@@ -131,31 +129,35 @@ func startGoRoutines(wg *sync.WaitGroup) context.CancelFunc {
 		cfg.ReadConfig()
 		apiHost, _, err := net.SplitHostPort(cfg.Server.API)
 		if err == nil {
+			wg.Add(1)
 			go nmproxy.Start(ctx, ProxyMgmChan, apiHost)
+			logger.Log(0, "Proxy Shutting down....")
+
 		}
+
 	}
 
 	go func(networks []string) {
 
-		for _, network := range networks {
-			logger.Log(0, "Collecting interface and peers info to configure proxy...")
-			cfg := config.ClientConfig{}
-			cfg.Network = network
-			cfg.ReadConfig()
-			node, err := GetNodeInfo(&cfg)
-			if err != nil {
-				log.Println("Failed to get node info: ", err)
-				continue
-			}
-			ProxyMgmChan <- &manager.ManagerAction{
-				Action: manager.AddInterface,
-				Payload: manager.ManagerPayload{
-					InterfaceName: node.Node.Interface,
-					Peers:         node.Peers,
-				},
-			}
-
-		}
+		// for _, network := range networks {
+		// 	logger.Log(0, "Collecting interface and peers info to configure proxy...")
+		// 	cfg := config.ClientConfig{}
+		// 	cfg.Network = network
+		// 	cfg.ReadConfig()
+		// 	node, err := GetNodeInfo(&cfg)
+		// 	if err != nil {
+		// 		log.Println("Failed to get node info: ", err)
+		// 		continue
+		// 	}
+		// 	ProxyMgmChan <- &manager.ManagerAction{
+		// 		Action: manager.AddInterface,
+		// 		Payload: manager.ManagerPayload{
+		// 			InterfaceName: node.Node.Interface,
+		// 			Peers:         node.Peers,
+		// 		},
+		// 	}
+
+		// }
 
 	}(networks)
 	return cancel
@@ -223,9 +225,9 @@ func setSubscriptions(client mqtt.Client, nodeCfg *config.ClientConfig) {
 	}
 	if token := client.Subscribe(fmt.Sprintf("proxy/%s/%s", nodeCfg.Node.Network, nodeCfg.Node.ID), 0, mqtt.MessageHandler(ProxyUpdate)); token.WaitTimeout(mq.MQ_TIMEOUT*time.Second) && token.Error() != nil {
 		if token.Error() == nil {
-			logger.Log(0, "network:", nodeCfg.Node.Network, "connection timeout")
+			logger.Log(0, "###### network:", nodeCfg.Node.Network, "connection timeout")
 		} else {
-			logger.Log(0, "network:", nodeCfg.Node.Network, token.Error().Error())
+			logger.Log(0, "###### network:", nodeCfg.Node.Network, token.Error().Error())
 		}
 		return
 	}

netclient/functions/mqhandlers.go

@@ -14,7 +14,6 @@ import (
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/netclient/config"
-	"github.com/gravitl/netmaker/netclient/local"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/netclient/wireguard"
 	"github.com/gravitl/netmaker/nm-proxy/manager"
@@ -37,17 +36,17 @@ func ProxyUpdate(client mqtt.Client, msg mqtt.Message) {
 	var network = parseNetworkFromTopic(msg.Topic())
 	nodeCfg.Network = network
 	nodeCfg.ReadConfig()
-
+	logger.Log(0, "---------> Recieved a proxy update")
 	data, dataErr := decryptMsg(&nodeCfg, msg.Payload())
 	if dataErr != nil {
 		return
 	}
 	err := json.Unmarshal([]byte(data), &proxyUpdate)
 	if err != nil {
-		logger.Log(0, "error unmarshalling node update data"+err.Error())
+		logger.Log(0, "error unmarshalling proxy update data"+err.Error())
 		return
 	}
-	logger.Log(0, "---------> recieved a proxy update")
+
 	ProxyMgmChan <- &proxyUpdate
 }
 
@@ -68,6 +67,15 @@ func NodeUpdate(client mqtt.Client, msg mqtt.Message) {
 		logger.Log(0, "error unmarshalling node update data"+err.Error())
 		return
 	}
+	if newNode.Proxy {
+		if newNode.Proxy != nodeCfg.Node.Proxy {
+			if err := config.Write(&nodeCfg, nodeCfg.Network); err != nil {
+				logger.Log(0, nodeCfg.Node.Network, "error updating node configuration: ", err.Error())
+			}
+		}
+		logger.Log(0, "Node is attached with proxy,ignore this node update...")
+		return
+	}
 
 	// see if cache hit, if so skip
 	var currentMessage = read(newNode.Network, lastNodeUpdate)
@@ -227,6 +235,11 @@ func UpdatePeers(client mqtt.Client, msg mqtt.Message) {
 		cfg.Server.Version = peerUpdate.ServerVersion
 		config.Write(&cfg, cfg.Network)
 	}
+
+	if cfg.Node.Proxy {
+		ProxyMgmChan <- &peerUpdate.ProxyUpdate
+		return
+	}
 	file := ncutils.GetNetclientPathSpecific() + cfg.Node.Interface + ".conf"
 	internetGateway, err := wireguard.UpdateWgPeers(file, peerUpdate.Peers)
 	if err != nil {
@@ -252,23 +265,23 @@ func UpdatePeers(client mqtt.Client, msg mqtt.Message) {
 		UpdateLocalListenPort(&cfg)
 		return
 	}
-	queryAddr := cfg.Node.PrimaryAddress()
+	// queryAddr := cfg.Node.PrimaryAddress()
 
 	//err = wireguard.SyncWGQuickConf(cfg.Node.Interface, file)
-	var iface = cfg.Node.Interface
-	if ncutils.IsMac() {
-		iface, err = local.GetMacIface(queryAddr)
-		if err != nil {
-			logger.Log(0, "error retrieving mac iface: "+err.Error())
-			return
-		}
-	}
-	err = wireguard.SetPeers(iface, &cfg.Node, peerUpdate.Peers)
-	if err != nil {
-		logger.Log(0, "error syncing wg after peer update: "+err.Error())
-		return
-	}
-	ProxyMgmChan <- &peerUpdate.ProxyUpdate
+	// var iface = cfg.Node.Interface
+	// if ncutils.IsMac() {
+	// 	iface, err = local.GetMacIface(queryAddr)
+	// 	if err != nil {
+	// 		logger.Log(0, "error retrieving mac iface: "+err.Error())
+	// 		return
+	// 	}
+	// }
+	// err = wireguard.SetPeers(iface, &cfg.Node, peerUpdate.Peers)
+	// if err != nil {
+	// 	logger.Log(0, "error syncing wg after peer update: "+err.Error())
+	// 	return
+	// }
+
 	logger.Log(0, "network:", cfg.Node.Network, "received peer update for node "+cfg.Node.Name+" "+cfg.Node.Network)
 	if cfg.Node.DNSOn == "yes" {
 		if err := setHostDNS(peerUpdate.DNS, cfg.Node.Interface, ncutils.IsWindows()); err != nil {

netclient/functions/pull.go

@@ -15,6 +15,7 @@ import (
 	"github.com/gravitl/netmaker/netclient/local"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/netclient/wireguard"
+	"github.com/gravitl/netmaker/nm-proxy/manager"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 	//homedir "github.com/mitchellh/go-homedir"
 )
@@ -62,32 +63,44 @@ func Pull(network string, iface bool) (*models.Node, error) {
 			logger.Log(0, "unable to update server config: "+err.Error())
 		}
 	}
-	if nodeGET.Node.ListenPort != cfg.Node.LocalListenPort {
-		if err := wireguard.RemoveConf(resNode.Interface, false); err != nil {
-			logger.Log(0, "error remove interface", resNode.Interface, err.Error())
+	if nodeGET.Node.Proxy {
+		ProxyMgmChan <- &manager.ManagerAction{
+			Action:  manager.AddInterface,
+			Payload: nodeGET.ProxyUpdate,
 		}
-		err = ncutils.ModPort(&resNode)
-		if err != nil {
-			return nil, err
+	}
+	if !nodeGET.Node.Proxy {
+		if nodeGET.Node.ListenPort != cfg.Node.LocalListenPort {
+			if err := wireguard.RemoveConf(resNode.Interface, false); err != nil {
+				logger.Log(0, "error remove interface", resNode.Interface, err.Error())
+			}
+			err = ncutils.ModPort(&resNode)
+			if err != nil {
+				return nil, err
+			}
+			informPortChange(&resNode)
 		}
-		informPortChange(&resNode)
 	}
+
 	if err = config.ModNodeConfig(&resNode); err != nil {
 		return nil, err
 	}
-	if iface {
-		if err = wireguard.SetWGConfig(network, false, nodeGET.Peers[:]); err != nil {
-			return nil, err
-		}
-	} else {
-		if err = wireguard.SetWGConfig(network, true, nodeGET.Peers[:]); err != nil {
-			if errors.Is(err, os.ErrNotExist) && !ncutils.IsFreeBSD() {
-				return Pull(network, true)
-			} else {
+	if !nodeGET.Node.Proxy {
+		if iface {
+			if err = wireguard.SetWGConfig(network, false, nodeGET.Peers[:]); err != nil {
 				return nil, err
 			}
+		} else {
+			if err = wireguard.SetWGConfig(network, true, nodeGET.Peers[:]); err != nil {
+				if errors.Is(err, os.ErrNotExist) && !ncutils.IsFreeBSD() {
+					return Pull(network, true)
+				} else {
+					return nil, err
+				}
+			}
 		}
 	}
+
 	var bkupErr = config.SaveBackup(network)
 	if bkupErr != nil {
 		logger.Log(0, "unable to update backup file for", network)

nm-proxy/common/common.go

@@ -13,6 +13,8 @@ import (
 
 var IsHostNetwork bool
 var IsRelay bool
+var IsIngressGateway bool
+var IsRelayed bool
 
 const (
 	NmProxyPort = 51722
@@ -30,25 +32,31 @@ type ConnConfig struct {
 	// Key is a public key of a remote peer
 	Key string
 	// LocalKey is a public key of a local peer
-	LocalKey        string
-	LocalWgPort     int
-	RemoteProxyIP   net.IP
-	RemoteWgPort    int
-	RemoteProxyPort int
+	LocalKey            string
+	LocalWgPort         int
+	RemoteProxyIP       net.IP
+	RemoteWgPort        int
+	RemoteProxyPort     int
+	IsExtClient         bool
+	IsRelayed           bool
+	RelayedEndpoint     *net.UDPAddr
+	IsAttachedExtClient bool
+	IngressGateWay      *net.UDPAddr
 }
+
 type Config struct {
-	Port         int
-	BodySize     int
-	Addr         string
-	RemoteKey    string
-	LocalKey     string
-	WgInterface  *wg.WGIface
-	AllowedIps   []net.IPNet
-	PreSharedKey *wgtypes.Key
+	Port        int
+	BodySize    int
+	Addr        string
+	RemoteKey   string
+	LocalKey    string
+	WgInterface *wg.WGIface
+	PeerConf    *wgtypes.PeerConfig
 }
 
 // Proxy -  WireguardProxy proxies
 type Proxy struct {
+	Status bool
 	Ctx    context.Context
 	Cancel context.CancelFunc
 
@@ -58,19 +66,30 @@ type Proxy struct {
 }
 
 type RemotePeer struct {
-	PeerKey   string
-	Interface string
-	Endpoint  *net.UDPAddr
+	PeerKey             string
+	Interface           string
+	Endpoint            *net.UDPAddr
+	IsExtClient         bool
+	IsAttachedExtClient bool
+}
+
+type WgIfaceConf struct {
+	Iface   *wgtypes.Device
+	PeerMap map[string]*Conn
 }
 
-var WgIFaceMap = make(map[string]map[string]*Conn)
+var WgIFaceMap = make(map[string]WgIfaceConf)
 
 var PeerKeyHashMap = make(map[string]RemotePeer)
 
-var WgIfaceKeyMap = make(map[string]struct{})
+var WgIfaceKeyMap = make(map[string]RemotePeer)
 
 var RelayPeerMap = make(map[string]map[string]RemotePeer)
 
+var ExtClientsWaitTh = make(map[string][]context.CancelFunc)
+
+var PeerAddrMap = make(map[string]map[string]*Conn)
+
 // RunCmd - runs a local command
 func RunCmd(command string, printerr bool) (string, error) {
 	args := strings.Fields(command)

nm-proxy/manager/manager.go

@@ -1,19 +1,33 @@
 package manager
 
 import (
+	"context"
 	"crypto/md5"
 	"errors"
 	"fmt"
 	"log"
 	"net"
+	"reflect"
 	"runtime"
+	"time"
 
 	"github.com/gravitl/netmaker/nm-proxy/common"
 	peerpkg "github.com/gravitl/netmaker/nm-proxy/peer"
+	"github.com/gravitl/netmaker/nm-proxy/proxy"
 	"github.com/gravitl/netmaker/nm-proxy/wg"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
+/*
+TODO:-
+	1. ON Ingress node
+		--> for attached ext clients
+			-> start sniffer (will recieve pkts from ext clients (add ebf filter to listen on only ext traffic) if not intended to the interface forward it.)
+			-> start remote conn after endpoint is updated
+		-->
+*/
+var sent bool
+
 type ProxyAction string
 
 type ManagerPayload struct {
@@ -21,6 +35,7 @@ type ManagerPayload struct {
 	Peers           []wgtypes.PeerConfig   `json:"peers"`
 	PeerMap         map[string]PeerConf    `json:"peer_map"`
 	IsRelayed       bool                   `json:"is_relayed"`
+	IsIngress       bool                   `json:"is_ingress"`
 	RelayedTo       *net.UDPAddr           `json:"relayed_to"`
 	IsRelay         bool                   `json:"is_relay"`
 	RelayedPeerConf map[string]RelayedConf `json:"relayed_conf"`
@@ -31,18 +46,20 @@ type RelayedConf struct {
 	RelayedPeerPubKey   string               `json:"relayed_peer_pub_key"`
 	Peers               []wgtypes.PeerConfig `json:"relayed_peers"`
 }
+
 type PeerConf struct {
-	IsRelayed bool         `json:"is_relayed"`
-	RelayedTo *net.UDPAddr `json:"relayed_to"`
+	IsExtClient            bool         `json:"is_ext_client"`
+	Address                string       `json:"address"`
+	IsAttachedExtClient    bool         `json:"is_attached_ext_client"`
+	IngressGatewayEndPoint *net.UDPAddr `json:"ingress_gateway_endpoint"`
+	IsRelayed              bool         `json:"is_relayed"`
+	RelayedTo              *net.UDPAddr `json:"relayed_to"`
+	Proxy                  bool         `json:"proxy"`
 }
 
 const (
-	AddInterface ProxyAction = "ADD_INTERFACE"
-	DeletePeer   ProxyAction = "DELETE_PEER"
-	UpdatePeer   ProxyAction = "UPDATE_PEER"
-	RelayPeers   ProxyAction = "RELAY_PEERS"
-	RelayUpdate  ProxyAction = "RELAY_UPDATE"
-	RelayTo      ProxyAction = "RELAY_TO"
+	AddInterface    ProxyAction = "ADD_INTERFACE"
+	DeleteInterface ProxyAction = "DELETE_INTERFACE"
 )
 
 type ManagerAction struct {
@@ -58,32 +75,44 @@ func StartProxyManager(manageChan chan *ManagerAction) {
 			log.Printf("-------> PROXY-MANAGER: %+v\n", mI)
 			switch mI.Action {
 			case AddInterface:
-				common.IsRelay = mI.Payload.IsRelay
-				if mI.Payload.IsRelay {
-					mI.RelayPeers()
-				}
+
+				mI.ExtClients()
 				err := mI.AddInterfaceToProxy()
 				if err != nil {
 					log.Printf("failed to add interface: [%s] to proxy: %v\n  ", mI.Payload.InterfaceName, err)
 				}
-			case UpdatePeer:
-				//mI.UpdatePeerProxy()
-			case DeletePeer:
-				mI.DeletePeers()
-			case RelayPeers:
-				mI.RelayPeers()
-			case RelayUpdate:
-				mI.RelayUpdate()
+			case DeleteInterface:
+				mI.DeleteInterface()
 			}
 
 		}
 	}
 }
 
+func (m *ManagerAction) DeleteInterface() {
+	var err error
+	if runtime.GOOS == "darwin" {
+		m.Payload.InterfaceName, err = wg.GetRealIface(m.Payload.InterfaceName)
+		if err != nil {
+			log.Println("failed to get real iface: ", err)
+			return
+		}
+	}
+	if wgProxyConf, ok := common.WgIFaceMap[m.Payload.InterfaceName]; ok {
+		cleanUpInterface(wgProxyConf)
+	}
+
+}
+
 func (m *ManagerAction) RelayUpdate() {
 	common.IsRelay = m.Payload.IsRelay
 }
 
+func (m *ManagerAction) ExtClients() {
+	common.IsIngressGateway = m.Payload.IsIngress
+
+}
+
 func (m *ManagerAction) RelayPeers() {
 	common.IsRelay = true
 	for relayedNodePubKey, relayedNodeConf := range m.Payload.RelayedPeerConf {
@@ -92,11 +121,14 @@ func (m *ManagerAction) RelayPeers() {
 			common.RelayPeerMap[relayedNodePubKeyHash] = make(map[string]common.RemotePeer)
 		}
 		for _, peer := range relayedNodeConf.Peers {
-			peer.Endpoint.Port = common.NmProxyPort
-			remotePeerKeyHash := fmt.Sprintf("%x", md5.Sum([]byte(peer.PublicKey.String())))
-			common.RelayPeerMap[relayedNodePubKeyHash][remotePeerKeyHash] = common.RemotePeer{
-				Endpoint: peer.Endpoint,
+			if peer.Endpoint != nil {
+				peer.Endpoint.Port = common.NmProxyPort
+				remotePeerKeyHash := fmt.Sprintf("%x", md5.Sum([]byte(peer.PublicKey.String())))
+				common.RelayPeerMap[relayedNodePubKeyHash][remotePeerKeyHash] = common.RemotePeer{
+					Endpoint: peer.Endpoint,
+				}
 			}
+
 		}
 		relayedNodeConf.RelayedPeerEndpoint.Port = common.NmProxyPort
 		common.RelayPeerMap[relayedNodePubKeyHash][relayedNodePubKeyHash] = common.RemotePeer{
@@ -106,113 +138,320 @@ func (m *ManagerAction) RelayPeers() {
 	}
 }
 
-func (m *ManagerAction) DeletePeers() {
-	if len(m.Payload.Peers) == 0 {
-		log.Println("No Peers to delete...")
-		return
+func cleanUpInterface(ifaceConf common.WgIfaceConf) {
+	log.Println("########------------>  CLEANING UP: ", ifaceConf.Iface.Name)
+	for _, peerI := range ifaceConf.PeerMap {
+		peerI.Proxy.Cancel()
 	}
-	peersMap, ok := common.WgIFaceMap[m.Payload.InterfaceName]
-	if !ok {
-		log.Println("interface not found: ", m.Payload.InterfaceName)
-		return
+	delete(common.WgIFaceMap, ifaceConf.Iface.Name)
+}
+
+func (m *ManagerAction) processPayload() (*wg.WGIface, error) {
+	var err error
+	var wgIface *wg.WGIface
+	if m.Payload.InterfaceName == "" {
+		return nil, errors.New("interface cannot be empty")
+	}
+	if len(m.Payload.Peers) == 0 {
+		return nil, errors.New("no peers to add")
 	}
 
-	for _, peerI := range m.Payload.Peers {
-		if peerConf, ok := peersMap[peerI.PublicKey.String()]; ok {
-			peerConf.Proxy.Cancel()
-			delete(peersMap, peerI.PublicKey.String())
+	if runtime.GOOS == "darwin" {
+		m.Payload.InterfaceName, err = wg.GetRealIface(m.Payload.InterfaceName)
+		if err != nil {
+			log.Println("failed to get real iface: ", err)
 		}
 	}
-	common.WgIFaceMap[m.Payload.InterfaceName] = peersMap
-}
-
-func (m *ManagerAction) UpdatePeerProxy() {
-	if len(m.Payload.Peers) == 0 {
-		log.Println("No Peers to add...")
-		return
+	wgIface, err = wg.NewWGIFace(m.Payload.InterfaceName, "127.0.0.1/32", wg.DefaultMTU)
+	if err != nil {
+		log.Println("Failed init new interface: ", err)
+		return nil, err
+	}
+	var wgProxyConf common.WgIfaceConf
+	var ok bool
+	if wgProxyConf, ok = common.WgIFaceMap[m.Payload.InterfaceName]; !ok {
+		for i := len(m.Payload.Peers) - 1; i >= 0; i-- {
+			if !m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].Proxy {
+				log.Println("-----------> skipping peer, proxy is off: ", m.Payload.Peers[i].PublicKey)
+				if err := wgIface.Update(m.Payload.Peers[i], false); err != nil {
+					log.Println("falied to update peer: ", err)
+				}
+				m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
+			}
+		}
+		return wgIface, nil
 	}
-	peers, ok := common.WgIFaceMap[m.Payload.InterfaceName]
-	if !ok {
-		log.Println("interface not found: ", m.Payload.InterfaceName)
-		return
+	if m.Payload.IsRelay {
+		m.RelayPeers()
+	}
+	common.IsRelay = m.Payload.IsRelay
+	// check if node is getting relayed
+	if common.IsRelayed != m.Payload.IsRelayed {
+		common.IsRelayed = m.Payload.IsRelayed
+		cleanUpInterface(wgProxyConf)
+		return wgIface, nil
 	}
 
-	for _, peerI := range m.Payload.Peers {
-		if peerI.Endpoint == nil {
-			log.Println("Endpoint nil for peer: ", peerI.PublicKey.String())
-			continue
-		}
+	// sync map with wg device config
+	// check if listen port has changed
+	if wgIface.Device.ListenPort != wgProxyConf.Iface.ListenPort {
+		// reset proxy for this interface
+		cleanUpInterface(wgProxyConf)
+		return wgIface, nil
+	}
+	// check device conf different from proxy
+	//wgProxyConf.Iface = wgIface.Device
+	for i := len(m.Payload.Peers) - 1; i >= 0; i-- {
+
+		if currentPeer, ok := wgProxyConf.PeerMap[m.Payload.Peers[i].PublicKey.String()]; ok {
+			// check if proxy is off for the peer
+			if !m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].Proxy {
+
+				// cleanup proxy connections for the peer
+				currentPeer.Proxy.Cancel()
+				time.Sleep(time.Second * 3)
+				delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+				// update the peer with actual endpoint
+				if err := wgIface.Update(m.Payload.Peers[i], false); err != nil {
+					log.Println("falied to update peer: ", err)
+				}
+				m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
+				continue
+
+			}
+			// check if peer is not connected to proxy
+			devPeer, err := wg.GetPeer(m.Payload.InterfaceName, currentPeer.Config.Key)
+			if err == nil {
+				log.Printf("---------> COMAPRING ENDPOINT: DEV: %s, Proxy: %s", devPeer.Endpoint.String(), currentPeer.Proxy.LocalConn.LocalAddr().String())
+				if devPeer.Endpoint.String() != currentPeer.Proxy.LocalConn.LocalAddr().String() {
+					log.Println("---------> endpoint is not set to proxy: ", currentPeer.Config.Key)
+					currentPeer.Proxy.Cancel()
+					time.Sleep(time.Second * 3)
+					delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+					continue
+				}
+			}
+			//check if peer is being relayed
+			if currentPeer.Config.IsRelayed != m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].IsRelayed {
+				log.Println("---------> peer relay status has been changed: ", currentPeer.Config.Key)
+				currentPeer.Proxy.Cancel()
+				time.Sleep(time.Second * 3)
+				delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+				continue
+			}
+			// check if relay endpoint has been changed
+			if currentPeer.Config.RelayedEndpoint != nil &&
+				m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].RelayedTo != nil &&
+				currentPeer.Config.RelayedEndpoint.String() != m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].RelayedTo.String() {
+				log.Println("---------> peer relay endpoint has been changed: ", currentPeer.Config.Key)
+				currentPeer.Proxy.Cancel()
+				time.Sleep(time.Second * 3)
+				delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+				continue
+			}
+			if !reflect.DeepEqual(m.Payload.Peers[i], *currentPeer.Proxy.Config.PeerConf) {
+				if currentPeer.Proxy.RemoteConn.IP.String() != m.Payload.Peers[i].Endpoint.IP.String() {
+					log.Println("----------> Resetting proxy for Peer: ", currentPeer.Config.Key, m.Payload.InterfaceName)
+					currentPeer.Proxy.Cancel()
+					time.Sleep(time.Second * 3)
+					delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+
+				} else {
 
-		if peerConf, ok := peers[peerI.PublicKey.String()]; ok {
+					log.Println("----->##### Updating Peer on Interface: ", m.Payload.InterfaceName, currentPeer.Config.Key)
+					updatePeerConf := m.Payload.Peers[i]
+					localUdpAddr, err := net.ResolveUDPAddr("udp", currentPeer.Proxy.LocalConn.LocalAddr().String())
+					if err == nil {
+						updatePeerConf.Endpoint = localUdpAddr
+					}
+					if err := wgIface.Update(updatePeerConf, true); err != nil {
+						log.Println("failed to update peer: ", currentPeer.Config.Key, err)
+					}
+					currentPeer.Proxy.Config.PeerConf = &m.Payload.Peers[i]
+					wgProxyConf.PeerMap[currentPeer.Config.Key] = currentPeer
+					// delete the peer from the list
+					log.Println("-----------> deleting peer from list: ", m.Payload.Peers[i].PublicKey)
+					m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
 
-			peerConf.Config.RemoteWgPort = peerI.Endpoint.Port
-			peers[peerI.PublicKey.String()] = peerConf
-			common.WgIFaceMap[m.Payload.InterfaceName] = peers
-			log.Printf("---->####### Updated PEER: %+v\n", peerConf)
+				}
+
+			} else {
+				// delete the peer from the list
+				log.Println("-----------> No updates observed so deleting peer: ", m.Payload.Peers[i].PublicKey)
+				m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
+			}
+
+		} else if !m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].Proxy {
+			log.Println("-----------> skipping peer, proxy is off: ", m.Payload.Peers[i].PublicKey)
+			if err := wgIface.Update(m.Payload.Peers[i], false); err != nil {
+				log.Println("falied to update peer: ", err)
+			}
+			m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
 		}
 	}
+	for _, currPeerI := range wgProxyConf.PeerMap {
+		if _, ok := m.Payload.PeerMap[currPeerI.Config.Key]; !ok {
+			currPeerI.Proxy.Cancel()
+			time.Sleep(time.Second * 3)
+			// delete peer from interface
+			log.Println("CurrPeer Not Found, Deleting Peer from Interface: ", currPeerI.Config.Key)
+			if err := wgIface.RemovePeer(currPeerI.Config.Key); err != nil {
+				log.Println("failed to remove peer: ", currPeerI.Config.Key, err)
+			}
+			delete(wgProxyConf.PeerMap, currPeerI.Config.Key)
 
-}
-
-func cleanUp(iface string) {
-	if peers, ok := common.WgIFaceMap[iface]; ok {
-		log.Println("########------------>  CLEANING UP: ", iface)
-		for _, peerI := range peers {
-			peerI.Proxy.Cancel()
 		}
 	}
-	delete(common.WgIFaceMap, iface)
+	common.WgIFaceMap[m.Payload.InterfaceName] = wgProxyConf
+
+	// if peers, ok := common.WgIFaceMap[iface]; ok {
+	// 	log.Println("########------------>  CLEANING UP: ", iface)
+	// 	for _, peerI := range peers {
+	// 		peerI.Proxy.Cancel()
+	// 	}
+	// }
+	// delete(common.WgIFaceMap, iface)
+	// delete(common.PeerAddrMap, iface)
+	// if waitThs, ok := common.ExtClientsWaitTh[iface]; ok {
+	// 	for _, cancelF := range waitThs {
+	// 		cancelF()
+	// 	}
+	// 	delete(common.ExtClientsWaitTh, iface)
+	// }
+
+	log.Println("CLEANED UP..........")
+	return wgIface, nil
 }
 
 func (m *ManagerAction) AddInterfaceToProxy() error {
 	var err error
-	if m.Payload.InterfaceName == "" {
-		return errors.New("interface cannot be empty")
-	}
-	if len(m.Payload.Peers) == 0 {
-		log.Println("No Peers to add...")
-		return nil
-	}
-	ifaceName := m.Payload.InterfaceName
-	log.Println("--------> IFACE: ", ifaceName)
-	if runtime.GOOS == "darwin" {
-		ifaceName, err = wg.GetRealIface(ifaceName)
-		if err != nil {
-			log.Println("failed to get real iface: ", err)
-		}
-	}
-	cleanUp(ifaceName)
 
-	wgInterface, err := wg.NewWGIFace(ifaceName, "127.0.0.1/32", wg.DefaultMTU)
+	wgInterface, err := m.processPayload()
 	if err != nil {
-		log.Println("Failed init new interface: ", err)
 		return err
 	}
-	log.Printf("wg: %+v\n", wgInterface)
 
+	log.Printf("wg: %+v\n", wgInterface)
+	wgListenAddr, err := proxy.GetInterfaceListenAddr(wgInterface.Port)
+	if err != nil {
+		log.Println("failed to get wg listen addr: ", err)
+		return err
+	}
+	common.WgIfaceKeyMap[fmt.Sprintf("%x", md5.Sum([]byte(wgInterface.Device.PublicKey.String())))] = common.RemotePeer{
+		PeerKey:   wgInterface.Device.PublicKey.String(),
+		Interface: wgInterface.Name,
+		Endpoint:  wgListenAddr,
+	}
 	for _, peerI := range m.Payload.Peers {
-		if peerI.Endpoint == nil {
+
+		peerConf := m.Payload.PeerMap[peerI.PublicKey.String()]
+		if peerI.Endpoint == nil && !(peerConf.IsAttachedExtClient || peerConf.IsExtClient) {
 			log.Println("Endpoint nil for peer: ", peerI.PublicKey.String())
 			continue
 		}
-		common.PeerKeyHashMap[fmt.Sprintf("%x", md5.Sum([]byte(peerI.PublicKey.String())))] = common.RemotePeer{
-			Interface: ifaceName,
-			PeerKey:   peerI.PublicKey.String(),
+		if peerConf.IsExtClient && !common.IsIngressGateway {
+			continue
+		}
+		shouldProceed := false
+		if peerConf.IsExtClient && peerConf.IsAttachedExtClient {
+			// check if ext client got endpoint,otherwise continue
+			for _, devpeerI := range wgInterface.Device.Peers {
+				if devpeerI.PublicKey.String() == peerI.PublicKey.String() && devpeerI.Endpoint != nil {
+					peerI.Endpoint = devpeerI.Endpoint
+					shouldProceed = true
+					break
+				}
+			}
+
+		} else {
+			shouldProceed = true
+		}
+		if peerConf.IsExtClient && peerConf.IsAttachedExtClient && shouldProceed {
+			//ctx, cancel := context.WithCancel(context.Background())
+			//common.ExtClientsWaitTh[wgInterface.Name] = append(common.ExtClientsWaitTh[wgInterface.Name], cancel)
+			//go proxy.StartSniffer(ctx, wgInterface.Name, peerConf.Address, wgInterface.Port)
+		}
+
+		if peerConf.IsExtClient && !peerConf.IsAttachedExtClient {
+			peerI.Endpoint = peerConf.IngressGatewayEndPoint
+		}
+		if shouldProceed {
+			common.PeerKeyHashMap[fmt.Sprintf("%x", md5.Sum([]byte(peerI.PublicKey.String())))] = common.RemotePeer{
+				Interface:           m.Payload.InterfaceName,
+				PeerKey:             peerI.PublicKey.String(),
+				IsExtClient:         peerConf.IsExtClient,
+				Endpoint:            peerI.Endpoint,
+				IsAttachedExtClient: peerConf.IsAttachedExtClient,
+			}
 		}
+
 		var isRelayed bool
 		var relayedTo *net.UDPAddr
 		if m.Payload.IsRelayed {
 			isRelayed = true
 			relayedTo = m.Payload.RelayedTo
 		} else {
-			if val, ok := m.Payload.PeerMap[peerI.PublicKey.String()]; ok {
-				isRelayed = val.IsRelayed
-				relayedTo = val.RelayedTo
-			}
+
+			isRelayed = peerConf.IsRelayed
+			relayedTo = peerConf.RelayedTo
+
+		}
+		if !shouldProceed && peerConf.IsAttachedExtClient {
+			log.Println("Extclient endpoint not updated yet....skipping")
+			// TODO - watch the interface for ext client update
+			go func(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig,
+				isRelayed bool, relayTo *net.UDPAddr, peerConf PeerConf) {
+				addExtClient := false
+				ctx, cancel := context.WithCancel(context.Background())
+				common.ExtClientsWaitTh[wgInterface.Name] = append(common.ExtClientsWaitTh[wgInterface.Name], cancel)
+				defer func() {
+					if addExtClient {
+						log.Println("GOT ENDPOINT for Extclient adding peer...")
+						//go proxy.StartSniffer(ctx, wgInterface.Name, peerConf.Address, wgInterface.Port)
+						common.PeerKeyHashMap[fmt.Sprintf("%x", md5.Sum([]byte(peer.PublicKey.String())))] = common.RemotePeer{
+							Interface:           wgInterface.Name,
+							PeerKey:             peer.PublicKey.String(),
+							IsExtClient:         peerConf.IsExtClient,
+							IsAttachedExtClient: peerConf.IsAttachedExtClient,
+							Endpoint:            peer.Endpoint,
+						}
+
+						peerpkg.AddNewPeer(wgInterface, peer, peerConf.Address, isRelayed,
+							peerConf.IsExtClient, peerConf.IsAttachedExtClient, relayedTo)
+					}
+				}()
+				for {
+					select {
+					case <-ctx.Done():
+						log.Println("Exiting extclient watch Thread for: ", wgInterface.Device.PublicKey.String())
+						return
+					default:
+						wgInterface, err := wg.NewWGIFace(m.Payload.InterfaceName, "127.0.0.1/32", wg.DefaultMTU)
+						if err != nil {
+							log.Println("Failed init new interface: ", err)
+							return
+						}
+						for _, devpeerI := range wgInterface.Device.Peers {
+							if devpeerI.PublicKey.String() == peer.PublicKey.String() && devpeerI.Endpoint != nil {
+								peer.Endpoint = devpeerI.Endpoint
+								addExtClient = true
+								return
+							}
+						}
+						time.Sleep(time.Second * 5)
+					}
+
+				}
+
+			}(wgInterface, &peerI, isRelayed, relayedTo, peerConf)
+			continue
 		}
 
-		peerpkg.AddNewPeer(wgInterface, &peerI, isRelayed, relayedTo)
+		peerpkg.AddNewPeer(wgInterface, &peerI, peerConf.Address, isRelayed,
+			peerConf.IsExtClient, peerConf.IsAttachedExtClient, relayedTo)
 	}
 	log.Printf("------> PEERHASHMAP: %+v\n", common.PeerKeyHashMap)
+	log.Printf("-------> WgKeyHashMap: %+v\n", common.WgIfaceKeyMap)
+	log.Printf("-------> WgIFaceMap: %+v\n", common.WgIFaceMap)
 	return nil
 }

nm-proxy/nm-proxy.go

@@ -21,7 +21,7 @@ import (
 func Start(ctx context.Context, mgmChan chan *manager.ManagerAction, apiServerAddr string) {
 	log.Println("Starting Proxy...")
 	common.IsHostNetwork = (os.Getenv("HOST_NETWORK") == "" || os.Getenv("HOST_NETWORK") == "on")
-	go manager.StartProxyManager(mgmChan)
+
 	hInfo := stun.GetHostInfo(apiServerAddr)
 	stun.Host = hInfo
 	log.Printf("HOSTINFO: %+v", hInfo)
@@ -33,6 +33,7 @@ func Start(ctx context.Context, mgmChan chan *manager.ManagerAction, apiServerAd
 	if err != nil {
 		log.Fatal("failed to create proxy: ", err)
 	}
+	go manager.StartProxyManager(mgmChan)
 	server.NmProxyServer.Listen(ctx)
 
 }

nm-proxy/peer/peer.go

@@ -1,7 +1,7 @@
 package peer
 
 import (
-	"crypto/md5"
+	"errors"
 	"fmt"
 	"log"
 	"net"
@@ -33,32 +33,48 @@ type ConnConfig struct {
 	RemoteProxyPort int
 }
 
-func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, isRelayed bool, relayTo *net.UDPAddr) error {
+func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, peerAddr string,
+	isRelayed, isExtClient, isAttachedExtClient bool, relayTo *net.UDPAddr) error {
 
 	c := proxy.Config{
 		Port:        peer.Endpoint.Port,
 		LocalKey:    wgInterface.Device.PublicKey.String(),
 		RemoteKey:   peer.PublicKey.String(),
 		WgInterface: wgInterface,
-		AllowedIps:  peer.AllowedIPs,
+
+		PeerConf: peer,
 	}
 	p := proxy.NewProxy(c)
+	peerPort := common.NmProxyPort
+	if isExtClient && isAttachedExtClient {
+		peerPort = peer.Endpoint.Port
 
+	}
 	peerEndpoint := peer.Endpoint.IP.String()
 	if isRelayed {
 		//go server.NmProxyServer.KeepAlive(peer.Endpoint.IP.String(), common.NmProxyPort)
+		if relayTo == nil {
+			return errors.New("relay endpoint is nil")
+		}
 		peerEndpoint = relayTo.IP.String()
 	}
-	remoteConn, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", peerEndpoint, common.NmProxyPort))
+
+	remoteConn, err := net.ResolveUDPAddr("udp", fmt.Sprintf("%s:%d", peerEndpoint, peerPort))
 	if err != nil {
 		return err
 	}
 	log.Printf("----> Established Remote Conn with RPeer: %s, ----> RAddr: %s", peer.PublicKey, remoteConn.String())
-	log.Printf("Starting proxy for Peer: %s\n", peer.PublicKey.String())
-	err = p.Start(remoteConn)
-	if err != nil {
-		return err
+
+	if !(isExtClient && isAttachedExtClient) {
+		log.Printf("Starting proxy for Peer: %s\n", peer.PublicKey.String())
+		err = p.Start(remoteConn)
+		if err != nil {
+			return err
+		}
+	} else {
+		log.Println("Not Starting Proxy for Attached ExtClient...")
 	}
+
 	connConf := common.ConnConfig{
 		Key:             peer.PublicKey.String(),
 		LocalKey:        wgInterface.Device.PublicKey.String(),
@@ -66,6 +82,8 @@ func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, isRelayed boo
 		RemoteProxyIP:   net.ParseIP(peer.Endpoint.IP.String()),
 		RemoteWgPort:    peer.Endpoint.Port,
 		RemoteProxyPort: common.NmProxyPort,
+		IsRelayed:       isRelayed,
+		RelayedEndpoint: relayTo,
 	}
 
 	peerProxy := common.Proxy{
@@ -76,7 +94,7 @@ func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, isRelayed boo
 			LocalKey:    wgInterface.Device.PublicKey.String(),
 			RemoteKey:   peer.PublicKey.String(),
 			WgInterface: wgInterface,
-			AllowedIps:  peer.AllowedIPs,
+			PeerConf:    peer,
 		},
 
 		RemoteConn: remoteConn,
@@ -90,11 +108,22 @@ func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, isRelayed boo
 		Proxy:  peerProxy,
 	}
 	if _, ok := common.WgIFaceMap[wgInterface.Name]; ok {
-		common.WgIFaceMap[wgInterface.Name][peer.PublicKey.String()] = &peerConn
+		common.WgIFaceMap[wgInterface.Name].PeerMap[peer.PublicKey.String()] = &peerConn
+	} else {
+		ifaceConf := common.WgIfaceConf{
+			Iface:   wgInterface.Device,
+			PeerMap: make(map[string]*common.Conn),
+		}
+
+		common.WgIFaceMap[wgInterface.Name] = ifaceConf
+		common.WgIFaceMap[wgInterface.Name].PeerMap[peer.PublicKey.String()] = &peerConn
+	}
+	if _, ok := common.PeerAddrMap[wgInterface.Name]; ok {
+		common.PeerAddrMap[wgInterface.Name][peerAddr] = &peerConn
 	} else {
-		common.WgIFaceMap[wgInterface.Name] = make(map[string]*common.Conn)
-		common.WgIFaceMap[wgInterface.Name][peer.PublicKey.String()] = &peerConn
+		common.PeerAddrMap[wgInterface.Name] = make(map[string]*common.Conn)
+		common.PeerAddrMap[wgInterface.Name][peerAddr] = &peerConn
 	}
-	common.WgIfaceKeyMap[fmt.Sprintf("%x", md5.Sum([]byte(wgInterface.Device.PublicKey.String())))] = struct{}{}
+
 	return nil
 }

nm-proxy/proxy/proxy.go

@@ -17,14 +17,13 @@ const (
 )
 
 type Config struct {
-	Port         int
-	BodySize     int
-	Addr         string
-	RemoteKey    string
-	LocalKey     string
-	WgInterface  *wg.WGIface
-	AllowedIps   []net.IPNet
-	PreSharedKey *wgtypes.Key
+	Port        int
+	BodySize    int
+	Addr        string
+	RemoteKey   string
+	LocalKey    string
+	WgInterface *wg.WGIface
+	PeerConf    *wgtypes.PeerConfig
 }
 
 // Proxy -  WireguardProxy proxies
@@ -102,3 +101,133 @@ func getBoardCastAddress() ([]net.Addr, error) {
 	}
 	return nil, errors.New("couldn't obtain the broadcast addr")
 }
+
+// func StartSniffer(ctx context.Context, ifaceName, extClientAddr string, port int) {
+// 	log.Println("Starting Packet Sniffer for iface: ", ifaceName)
+// 	var (
+// 		snapshotLen int32 = 1024
+// 		promiscuous bool  = false
+// 		err         error
+// 		timeout     time.Duration = 1 * time.Microsecond
+// 		handle      *pcap.Handle
+// 	)
+// 	// Open device
+// 	handle, err = pcap.OpenLive(ifaceName, snapshotLen, promiscuous, timeout)
+// 	if err != nil {
+// 		log.Println("failed to start sniffer for iface: ", ifaceName, err)
+// 		return
+// 	}
+// 	// if err := handle.SetBPFFilter(fmt.Sprintf("src %s and port %d", extClientAddr, port)); err != nil {
+// 	// 	log.Println("failed to set bpf filter: ", err)
+// 	// 	return
+// 	// }
+// 	defer handle.Close()
+
+// 	// var tcp layers.TCP
+// 	// var icmp layers.ICMPv4
+// 	// var udp layers.UDP
+// 	// parser := gopacket.NewDecodingLayerParser(layers.LayerTypeIPv4, &udp, &tcp, &icmp)
+
+// 	packetSource := gopacket.NewPacketSource(handle, handle.LinkType())
+// 	for {
+// 		select {
+// 		case <-ctx.Done():
+// 			log.Println("Stopping packet sniffer for iface: ", ifaceName, " port: ", port)
+// 			return
+// 		default:
+// 			packet, err := packetSource.NextPacket()
+// 			if err == nil {
+// 				//processPkt(ifaceName, packet)
+// 				ipLayer := packet.Layer(layers.LayerTypeIPv4)
+// 				if ipLayer != nil {
+// 					fmt.Println("IPv4 layer detected.")
+// 					ip, _ := ipLayer.(*layers.IPv4)
+
+// 					// IP layer variables:
+// 					// Version (Either 4 or 6)
+// 					// IHL (IP Header Length in 32-bit words)
+// 					// TOS, Length, Id, Flags, FragOffset, TTL, Protocol (TCP?),
+// 					// Checksum, SrcIP, DstIP
+// 					fmt.Println("#########################")
+// 					fmt.Printf("From %s to %s\n", ip.SrcIP, ip.DstIP)
+// 					fmt.Println("Protocol: ", ip.Protocol)
+// 					if ip.DstIP.String() == extClientAddr || ip.SrcIP.String() == extClientAddr {
+// 						if ifacePeers, ok := common.PeerAddrMap[ifaceName]; ok {
+// 							if peerConf, ok := ifacePeers[ip.DstIP.String()]; ok {
+// 								log.Println("-----> Fowarding PKT From ExtClient: ", extClientAddr, " to: ", peerConf)
+// 								//server.NmProxyServer.Server.WriteTo(packet.Data(),  )
+// 							}
+
+// 						}
+// 					}
+
+// 					fmt.Println("#########################")
+// 				}
+// 			}
+// 		}
+
+// 	}
+// }
+
+// func processPkt(iface string, packet gopacket.Packet) {
+// 	// Let's see if the packet is an ethernet packet
+// 	// ethernetLayer := packet.Layer(layers.LayerTypeEthernet)
+// 	// if ethernetLayer != nil {
+// 	// 	fmt.Println("Ethernet layer detected.")
+// 	// 	ethernetPacket, _ := ethernetLayer.(*layers.Ethernet)
+// 	// 	fmt.Println("Source MAC: ", ethernetPacket.SrcMAC)
+// 	// 	fmt.Println("Destination MAC: ", ethernetPacket.DstMAC)
+// 	// 	// Ethernet type is typically IPv4 but could be ARP or other
+// 	// 	fmt.Println("Ethernet type: ", ethernetPacket.EthernetType)
+// 	// 	fmt.Println()
+// 	// }
+
+// 	// Let's see if the packet is IP (even though the ether type told us)
+// 	ipLayer := packet.Layer(layers.LayerTypeIPv4)
+// 	if ipLayer != nil {
+// 		fmt.Println("IPv4 layer detected.")
+// 		ip, _ := ipLayer.(*layers.IPv4)
+
+// 		// IP layer variables:
+// 		// Version (Either 4 or 6)
+// 		// IHL (IP Header Length in 32-bit words)
+// 		// TOS, Length, Id, Flags, FragOffset, TTL, Protocol (TCP?),
+// 		// Checksum, SrcIP, DstIP
+// 		fmt.Printf("From %s to %s\n", ip.SrcIP, ip.DstIP)
+// 		fmt.Println("Protocol: ", ip.Protocol)
+// 		fmt.Println()
+
+// 	}
+
+// 	// udpLayer := packet.Layer(layers.LayerTypeUDP)
+// 	// if udpLayer != nil {
+// 	// 	udp, _ := udpLayer.(*layers.UDP)
+// 	// 	fmt.Printf("UDP: From port %d to %d\n", udp.SrcPort, udp.DstPort)
+// 	// 	fmt.Println()
+// 	// }
+
+// 	// // Iterate over all layers, printing out each layer type
+// 	// fmt.Println("All packet layers:")
+// 	// for _, layer := range packet.Layers() {
+// 	// 	fmt.Println("- ", layer.LayerType())
+// 	// }
+
+// 	// When iterating through packet.Layers() above,
+// 	// if it lists Payload layer then that is the same as
+// 	// this applicationLayer. applicationLayer contains the payload
+// 	// applicationLayer := packet.ApplicationLayer()
+// 	// if applicationLayer != nil {
+// 	// 	fmt.Println("Application layer/Payload found.")
+// 	// 	fmt.Printf("%s\n", applicationLayer.Payload())
+
+// 	// 	// Search for a string inside the payload
+// 	// 	if strings.Contains(string(applicationLayer.Payload()), "HTTP") {
+// 	// 		fmt.Println("HTTP found!")
+// 	// 	}
+// 	// }
+
+// 	// Check for errors
+// 	if err := packet.ErrorLayer(); err != nil {
+// 		fmt.Println("Error decoding some part of the packet:", err)
+// 	}
+// }

nm-proxy/proxy/wireguard.go

@@ -24,12 +24,13 @@ func NewProxy(config Config) *Proxy {
 
 // proxyToRemote proxies everything from Wireguard to the RemoteKey peer
 func (p *Proxy) ProxyToRemote() {
-	buf := make([]byte, 1500)
 
 	go func() {
 		<-p.Ctx.Done()
-		defer p.LocalConn.Close()
+		log.Println("Closing connection for: ", p.LocalConn.LocalAddr().String())
+		p.LocalConn.Close()
 	}()
+	buf := make([]byte, 65000)
 	for {
 		select {
 		case <-p.Ctx.Done():
@@ -40,14 +41,16 @@ func (p *Proxy) ProxyToRemote() {
 					log.Println("Failed to split host: ", p.LocalConn.LocalAddr().String(), err)
 					return
 				}
-				if host == "127.0.0.1" {
-					return
-				}
-				_, err = common.RunCmd(fmt.Sprintf("ifconfig lo0 -alias %s 255.255.255.255", host), true)
-				if err != nil {
-					log.Println("Failed to add alias: ", err)
+
+				if host != "127.0.0.1" {
+					_, err = common.RunCmd(fmt.Sprintf("ifconfig lo0 -alias %s 255.255.255.255", host), true)
+					if err != nil {
+						log.Println("Failed to add alias: ", err)
+					}
 				}
+
 			}
+
 			return
 		default:
 
@@ -56,18 +59,20 @@ func (p *Proxy) ProxyToRemote() {
 				log.Println("ERRR READ: ", err)
 				continue
 			}
-			peers := common.WgIFaceMap[p.Config.WgInterface.Name]
-			if peerI, ok := peers[p.Config.RemoteKey]; ok {
+			//go func(buf []byte, n int) {
+			ifaceConf := common.WgIFaceMap[p.Config.WgInterface.Name]
+			if peerI, ok := ifaceConf.PeerMap[p.Config.RemoteKey]; ok {
 				var srcPeerKeyHash, dstPeerKeyHash string
 				buf, n, srcPeerKeyHash, dstPeerKeyHash = packet.ProcessPacketBeforeSending(buf, n, peerI.Config.LocalKey, peerI.Config.Key)
 				if err != nil {
 					log.Println("failed to process pkt before sending: ", err)
 				}
-				log.Printf("PROXING TO REMOTE!!!---> %s >>>>> %s [[ SrcPeerHash: %s, DstPeerHash: %s ]]\n",
-					server.NmProxyServer.Server.LocalAddr().String(), p.RemoteConn.String(), srcPeerKeyHash, dstPeerKeyHash)
+				log.Printf("PROXING TO REMOTE!!!---> %s >>>>> %s >>>>> %s [[ SrcPeerHash: %s, DstPeerHash: %s ]]\n",
+					p.LocalConn.LocalAddr(), server.NmProxyServer.Server.LocalAddr().String(), p.RemoteConn.String(), srcPeerKeyHash, dstPeerKeyHash)
 			} else {
 				log.Printf("Peer: %s not found in config\n", p.Config.RemoteKey)
-				continue
+				p.Cancel()
+				return
 			}
 			//test(n, buf)
 
@@ -75,6 +80,8 @@ func (p *Proxy) ProxyToRemote() {
 			if err != nil {
 				log.Println("Failed to send to remote: ", err)
 			}
+			//}(buf, n)
+
 		}
 	}
 }
@@ -91,9 +98,9 @@ func (p *Proxy) updateEndpoint() error {
 		return err
 	}
 	// add local proxy connection as a Wireguard peer
-	log.Printf("---> ## Updating Peer:  %+v\n", p.Config)
-	err = p.Config.WgInterface.UpdatePeer(p.Config.RemoteKey, p.Config.AllowedIps, wg.DefaultWgKeepAlive,
-		udpAddr, p.Config.PreSharedKey)
+	log.Printf("---> ####### Updating Peer:  %+v\n", p.Config.PeerConf)
+	err = p.Config.WgInterface.UpdatePeer(p.Config.RemoteKey, p.Config.PeerConf.AllowedIPs, wg.DefaultWgKeepAlive,
+		udpAddr, p.Config.PeerConf.PresharedKey)
 	if err != nil {
 		return err
 	}
@@ -105,17 +112,8 @@ func (p *Proxy) Start(remoteConn *net.UDPAddr) error {
 	p.RemoteConn = remoteConn
 
 	var err error
-	// err = p.Config.WgInterface.GetWgIface(p.Config.WgInterface.Name)
-	// if err != nil {
-	// 	log.Println("Failed to get iface: ", p.Config.WgInterface.Name, err)
-	// 	return err
-	// }
-	// wgAddr, err := GetInterfaceIpv4Addr(p.Config.WgInterface.Name)
-	// if err != nil {
-	// 	log.Println("failed to get interface addr: ", err)
-	// 	return err
-	// }
-	log.Printf("----> WGIFACE: %+v\n", p.Config.WgInterface)
+
+	//log.Printf("----> WGIFACE: %+v\n", p.Config.WgInterface)
 	addr, err := GetFreeIp(common.DefaultCIDR, p.Config.WgInterface.Port)
 	if err != nil {
 		log.Println("Failed to get freeIp: ", err)
@@ -129,7 +127,7 @@ func (p *Proxy) Start(remoteConn *net.UDPAddr) error {
 	if runtime.GOOS == "darwin" {
 		wgListenAddr.IP = net.ParseIP(addr)
 	}
-	log.Println("--------->#### Wg Listen Addr: ", wgListenAddr.String())
+	//log.Println("--------->#### Wg Listen Addr: ", wgListenAddr.String())
 	p.LocalConn, err = net.DialUDP("udp", &net.UDPAddr{
 		IP:   net.ParseIP(addr),
 		Port: common.NmProxyPort,

nm-proxy/server/server.go

@@ -36,16 +36,16 @@ type ProxyServer struct {
 func (p *ProxyServer) Listen(ctx context.Context) {
 
 	// Buffer with indicated body size
-	buffer := make([]byte, 1532)
+	buffer := make([]byte, 65032)
 	for {
 
 		select {
 		case <-ctx.Done():
 			log.Println("--------->### Shutting down Proxy.....")
 			// clean up proxy connections
-			for iface, peers := range common.WgIFaceMap {
+			for iface, ifaceConf := range common.WgIFaceMap {
 				log.Println("########------------>  CLEANING UP: ", iface)
-				for _, peerI := range peers {
+				for _, peerI := range ifaceConf.PeerMap {
 					peerI.Proxy.Cancel()
 				}
 			}
@@ -54,16 +54,34 @@ func (p *ProxyServer) Listen(ctx context.Context) {
 			return
 		default:
 			// Read Packet
+
 			n, source, err := p.Server.ReadFromUDP(buffer)
 			if err != nil { // in future log errors?
 				log.Println("RECV ERROR: ", err)
 				continue
 			}
+			//go func(buffer []byte, source *net.UDPAddr, n int) {
+
 			var srcPeerKeyHash, dstPeerKeyHash string
 			n, srcPeerKeyHash, dstPeerKeyHash = packet.ExtractInfo(buffer, n)
-			//log.Printf("--------> RECV PKT [DSTPORT: %d], [SRCKEYHASH: %s], SourceIP: [%s] \n", localWgPort, srcPeerKeyHash, source.IP.String())
-			if common.IsRelay && dstPeerKeyHash != "" && srcPeerKeyHash != "" {
-				if _, ok := common.WgIfaceKeyMap[dstPeerKeyHash]; !ok {
+			//log.Printf("--------> RECV PKT , [SRCKEYHASH: %s], SourceIP: [%s] \n", srcPeerKeyHash, source.IP.String())
+			if _, ok := common.WgIfaceKeyMap[dstPeerKeyHash]; !ok {
+				// if common.IsIngressGateway {
+				// 	log.Println("----> fowarding PKT to EXT client...")
+				// 	if val, ok := common.PeerKeyHashMap[dstPeerKeyHash]; ok && val.IsAttachedExtClient {
+
+				// 		log.Printf("-------->Forwarding the pkt to extClient  [ SourceIP: %s ], [ SourceKeyHash: %s ], [ DstIP: %s ], [ DstHashKey: %s ] \n",
+				// 			source.String(), srcPeerKeyHash, val.Endpoint.String(), dstPeerKeyHash)
+				// 		_, err = NmProxyServer.Server.WriteToUDP(buffer[:n], val.Endpoint)
+				// 		if err != nil {
+				// 			log.Println("Failed to send to remote: ", err)
+				// 		}
+				// 		continue
+
+				// 	}
+				// }
+
+				if common.IsRelay {
 
 					log.Println("----------> Relaying######")
 					// check for routing map and forward to right proxy
@@ -75,6 +93,7 @@ func (p *ProxyServer) Listen(ctx context.Context) {
 							if err != nil {
 								log.Println("Failed to send to remote: ", err)
 							}
+							//continue
 						}
 					} else {
 						if remoteMap, ok := common.RelayPeerMap[dstPeerKeyHash]; ok {
@@ -85,30 +104,46 @@ func (p *ProxyServer) Listen(ctx context.Context) {
 								if err != nil {
 									log.Println("Failed to send to remote: ", err)
 								}
+								//continue
 							}
 						}
+
 					}
 
 				}
+
 			}
 
 			if peerInfo, ok := common.PeerKeyHashMap[srcPeerKeyHash]; ok {
-				if peers, ok := common.WgIFaceMap[peerInfo.Interface]; ok {
-					if peerI, ok := peers[peerInfo.PeerKey]; ok {
+				if ifaceConf, ok := common.WgIFaceMap[peerInfo.Interface]; ok {
+					if peerI, ok := ifaceConf.PeerMap[peerInfo.PeerKey]; ok {
 						log.Printf("PROXING TO LOCAL!!!---> %s <<<< %s <<<<<<<< %s   [[ RECV PKT [SRCKEYHASH: %s], [DSTKEYHASH: %s], SourceIP: [%s] ]]\n",
 							peerI.Proxy.LocalConn.RemoteAddr(), peerI.Proxy.LocalConn.LocalAddr(),
 							fmt.Sprintf("%s:%d", source.IP.String(), source.Port), srcPeerKeyHash, dstPeerKeyHash, source.IP.String())
 						_, err = peerI.Proxy.LocalConn.Write(buffer[:n])
 						if err != nil {
 							log.Println("Failed to proxy to Wg local interface: ", err)
-							continue
+							//continue
 						}
 
 					}
 				}
 
 			}
-
+			// // forward to all interfaces
+			// for _, ifaceCfg := range common.WgIfaceKeyMap {
+			// 	log.Println("###--------> Forwarding Unknown PKT to ", ifaceCfg.Interface)
+			// 	conn, err := net.DialUDP("udp", nil, ifaceCfg.Endpoint)
+			// 	if err == nil {
+			// 		_, err := conn.Write(buffer[:n])
+			// 		if err != nil {
+			// 			log.Println("Failed to forward the unknown pkt to ifcace: ", ifaceCfg.Interface, err)
+			// 		}
+			// 		conn.Close()
+			// 	}
+
+			// }
+			//}(buffer, source, n)
 		}
 
 	}

nm-proxy/wg/wg.go

@@ -74,7 +74,7 @@ func (w *WGIface) GetWgIface(iface string) error {
 		return err
 	}
 
-	log.Printf("----> DEVICE: %+v\n", dev)
+	//log.Printf("----> DEVICE: %+v\n", dev)
 	w.Device = dev
 	w.Port = dev.ListenPort
 	return nil
@@ -124,9 +124,9 @@ func (w *WGIface) UpdatePeer(peerKey string, allowedIps []net.IPNet, keepAlive t
 		return err
 	}
 	peer := wgtypes.PeerConfig{
-		PublicKey: peerKeyParsed,
-		// ReplaceAllowedIPs:           true,
-		// AllowedIPs:                  allowedIps,
+		PublicKey:                   peerKeyParsed,
+		ReplaceAllowedIPs:           true,
+		AllowedIPs:                  allowedIps,
 		PersistentKeepaliveInterval: &keepAlive,
 		PresharedKey:                preSharedKey,
 		Endpoint:                    endpoint,
@@ -234,3 +234,73 @@ func RunCmd(command string, printerr bool) (string, error) {
 	}
 	return string(out), err
 }
+
+// RemovePeer removes a Wireguard Peer from the interface iface
+func (w *WGIface) RemovePeer(peerKey string) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+
+	log.Printf("Removing peer %s from interface %s ", peerKey, w.Name)
+
+	peerKeyParsed, err := wgtypes.ParseKey(peerKey)
+	if err != nil {
+		return err
+	}
+
+	peer := wgtypes.PeerConfig{
+		PublicKey: peerKeyParsed,
+		Remove:    true,
+	}
+
+	config := wgtypes.Config{
+		Peers: []wgtypes.PeerConfig{peer},
+	}
+	err = w.configureDevice(config)
+	if err != nil {
+		return fmt.Errorf("received error \"%v\" while removing peer %s from interface %s", err, peerKey, w.Name)
+	}
+	return nil
+}
+
+// UpdatePeer
+func (w *WGIface) Update(peerConf wgtypes.PeerConfig, updateOnly bool) error {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	var err error
+	log.Printf("---------> NEWWWWWW Updating peer %+v from interface %s ", peerConf, w.Name)
+
+	peerConf.UpdateOnly = updateOnly
+	peerConf.ReplaceAllowedIPs = true
+	config := wgtypes.Config{
+		Peers: []wgtypes.PeerConfig{peerConf},
+	}
+	err = w.configureDevice(config)
+	if err != nil {
+		return fmt.Errorf("received error \"%v\" while Updating peer %s from interface %s", err, peerConf.PublicKey.String(), w.Name)
+	}
+	return nil
+}
+
+func GetPeer(ifaceName, peerPubKey string) (wgtypes.Peer, error) {
+	wg, err := wgctrl.New()
+	if err != nil {
+		return wgtypes.Peer{}, err
+	}
+	defer func() {
+		err = wg.Close()
+		if err != nil {
+			log.Printf("got error while closing wgctl: %v", err)
+		}
+	}()
+
+	wgDevice, err := wg.Device(ifaceName)
+	if err != nil {
+		return wgtypes.Peer{}, err
+	}
+	for _, peer := range wgDevice.Peers {
+		if peer.PublicKey.String() == peerPubKey {
+			return peer, nil
+		}
+	}
+	return wgtypes.Peer{}, fmt.Errorf("peer not found")
+}

servercfg/serverconf.go

@@ -672,3 +672,13 @@ func GetStunPort() string {
 	}
 	return port
 }
+
+func IsProxyEnabled() bool {
+	var enabled = false //default
+	if os.Getenv("PROXY") != "" {
+		enabled = os.Getenv("PROXY") == "on"
+	} else if config.Config.Server.Proxy != "" {
+		enabled = config.Config.Server.Proxy == "on"
+	}
+	return enabled
+}