Merge pull request #1766 from gravitl/feature_proxy_ext

Establish connection to ext client using handshake message

Dockerfile

@@ -6,13 +6,13 @@ WORKDIR /app
 COPY . .
 ENV GO111MODULE=auto
 
-RUN apk add git
+RUN apk add git libpcap-dev
 RUN GOOS=linux CGO_ENABLED=1 go build ${tags} -ldflags="-s -X 'main.version=${version}'" .
 # RUN go build -tags=ee . -o netmaker main.go
 FROM alpine:3.15.2
 
 # add a c lib
-RUN apk add gcompat iptables wireguard-tools
+RUN apk add gcompat iptables wireguard-tools libpcap-dev
 # set the working directory
 WORKDIR /root/
 RUN mkdir -p /etc/netclient/config

controllers/ext_client.go

@@ -332,7 +332,11 @@ func createExtClient(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 		return
 	}
-	extclient.IngressGatewayEndpoint = node.Endpoint + ":" + strconv.FormatInt(int64(node.ListenPort), 10)
+	listenPort := node.ListenPort
+	if node.Proxy {
+		listenPort = 51722
+	}
+	extclient.IngressGatewayEndpoint = node.Endpoint + ":" + strconv.FormatInt(int64(listenPort), 10)
 
 	extclient.Enabled = true
 	parentNetwork, err := logic.GetNetwork(networkName)

logic/gateway.go

@@ -230,8 +230,8 @@ func CreateIngressGateway(netid string, nodeid string, failover bool) (models.No
 		}
 	}
 	node.SetLastModified()
-	// node.PostUp = postUpCmd
-	// node.PostDown = postDownCmd
+	node.PostUp = postUpCmd
+	node.PostDown = postDownCmd
 	node.UDPHolePunch = "no"
 	if failover && servercfg.Is_EE {
 		node.Failover = "yes"

logic/peers.go

@@ -29,6 +29,16 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 	if err != nil {
 		return proxyPayload, err
 	}
+	var metrics *models.Metrics
+	if servercfg.Is_EE {
+		metrics, _ = GetMetrics(node.ID)
+	}
+	if metrics == nil {
+		metrics = &models.Metrics{}
+	}
+	if metrics.FailoverPeers == nil {
+		metrics.FailoverPeers = make(map[string]string)
+	}
 	if !onlyPeers {
 		if node.IsRelayed == "yes" {
 			relayNode := FindRelay(node)
@@ -92,7 +102,7 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 			logger.Log(1, "failed to resolve udp addr for node: ", peer.ID, peer.Endpoint, err.Error())
 			continue
 		}
-		allowedips := getNodeAllowedIPs(&peer, node)
+		allowedips := GetAllowedIPs(node, &peer, metrics, false)
 		var keepalive time.Duration
 		if node.PersistentKeepalive != 0 {
 			// set_keepalive
@@ -132,15 +142,23 @@ func GetPeersForProxy(node *models.Node, onlyPeers bool) (manager.ManagerPayload
 
 		}
 	}
-	var extPeers []wgtypes.PeerConfig
-	extPeers, peerConfMap, err = getExtPeersForProxy(node, peerConfMap)
-	if err == nil {
-		peers = append(peers, extPeers...)
+	if node.IsIngressGateway == "yes" {
+		var extPeers []wgtypes.PeerConfig
+		extPeers, peerConfMap, err = getExtPeersForProxy(node, peerConfMap)
+		if err == nil {
+			peers = append(peers, extPeers...)
 
-	} else if !database.IsEmptyRecord(err) {
-		logger.Log(1, "error retrieving external clients:", err.Error())
+		} else if !database.IsEmptyRecord(err) {
+			logger.Log(1, "error retrieving external clients:", err.Error())
+		}
 	}
+
 	proxyPayload.IsIngress = node.IsIngressGateway == "yes"
+	addr := node.Address
+	if addr == "" {
+		addr = node.Address6
+	}
+	proxyPayload.WgAddr = addr
 	proxyPayload.Peers = peers
 	proxyPayload.PeerMap = peerConfMap
 	proxyPayload.InterfaceName = node.Interface
@@ -280,7 +298,7 @@ func GetPeerUpdate(node *models.Node) (models.PeerUpdate, error) {
 			}
 		}
 
-		allowedips := GetAllowedIPs(node, &peer, metrics)
+		allowedips := GetAllowedIPs(node, &peer, metrics, true)
 		var keepalive time.Duration
 		if node.PersistentKeepalive != 0 {
 			// set_keepalive
@@ -454,7 +472,7 @@ func getExtPeersForProxy(node *models.Node, proxyPeerConf map[string]manager.Pee
 }
 
 // GetAllowedIPs - calculates the wireguard allowedip field for a peer of a node based on the peer and node settings
-func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet {
+func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics, fetchRelayedIps bool) []net.IPNet {
 	var allowedips []net.IPNet
 	allowedips = getNodeAllowedIPs(peer, node)
 
@@ -468,7 +486,7 @@ func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet
 			allowedips = append(allowedips, extPeer.AllowedIPs...)
 		}
 		// if node is a failover node, add allowed ips from nodes it is handling
-		if peer.Failover == "yes" && metrics.FailoverPeers != nil {
+		if metrics != nil && peer.Failover == "yes" && metrics.FailoverPeers != nil {
 			// traverse through nodes that need handling
 			logger.Log(3, "peer", peer.Name, "was found to be failover for", node.Name, "checking failover peers...")
 			for k := range metrics.FailoverPeers {
@@ -490,7 +508,7 @@ func GetAllowedIPs(node, peer *models.Node, metrics *models.Metrics) []net.IPNet
 		}
 	}
 	// handle relay gateway peers
-	if peer.IsRelay == "yes" {
+	if fetchRelayedIps && peer.IsRelay == "yes" {
 		for _, ip := range peer.RelayAddrs {
 			//find node ID of relayed peer
 			relayedPeer, err := findNode(ip)

mq/publishers.go

@@ -26,10 +26,7 @@ func PublishPeerUpdate(newNode *models.Node, publishToSelf bool) error {
 		return err
 	}
 	for _, node := range networkNodes {
-		// err := PublishProxyUpdate(manager.AddInterface, &node)
-		// if err != nil {
-		// 	logger.Log(1, "failed to publish proxy update to node", node.Name, "on network", node.Network, ":", err.Error())
-		// }
+
 		if node.IsServer == "yes" {
 			if servercfg.IsProxyEnabled() {
 				err := PublishProxyUpdate(manager.AddInterface, &node)

nm-proxy/common/common.go

@@ -1,94 +1,31 @@
 package common
 
 import (
-	"context"
 	"log"
-	"net"
 	"os/exec"
 	"strings"
 
-	"github.com/gravitl/netmaker/nm-proxy/wg"
-	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+	"github.com/gravitl/netmaker/nm-proxy/models"
 )
 
 var IsHostNetwork bool
 var IsRelay bool
 var IsIngressGateway bool
 var IsRelayed bool
+var IsServer bool
+var InterfaceName string
 
-const (
-	NmProxyPort = 51722
-	DefaultCIDR = "127.0.0.1/8"
-)
-
-type Conn struct {
-	Config ConnConfig
-	Proxy  Proxy
-}
-
-// ConnConfig is a peer Connection configuration
-type ConnConfig struct {
-
-	// Key is a public key of a remote peer
-	Key string
-	// LocalKey is a public key of a local peer
-	LocalKey            string
-	LocalWgPort         int
-	RemoteProxyIP       net.IP
-	RemoteWgPort        int
-	RemoteProxyPort     int
-	IsExtClient         bool
-	IsRelayed           bool
-	RelayedEndpoint     *net.UDPAddr
-	IsAttachedExtClient bool
-	IngressGateWay      *net.UDPAddr
-}
-
-type Config struct {
-	Port        int
-	BodySize    int
-	Addr        string
-	RemoteKey   string
-	LocalKey    string
-	WgInterface *wg.WGIface
-	PeerConf    *wgtypes.PeerConfig
-}
-
-// Proxy -  WireguardProxy proxies
-type Proxy struct {
-	Status bool
-	Ctx    context.Context
-	Cancel context.CancelFunc
-
-	Config     Config
-	RemoteConn *net.UDPAddr
-	LocalConn  net.Conn
-}
-
-type RemotePeer struct {
-	PeerKey             string
-	Interface           string
-	Endpoint            *net.UDPAddr
-	IsExtClient         bool
-	IsAttachedExtClient bool
-}
-
-type WgIfaceConf struct {
-	Iface   *wgtypes.Device
-	PeerMap map[string]*Conn
-}
-
-var WgIFaceMap = make(map[string]WgIfaceConf)
+var WgIFaceMap = make(map[string]models.WgIfaceConf)
 
-var PeerKeyHashMap = make(map[string]RemotePeer)
+var PeerKeyHashMap = make(map[string]models.RemotePeer)
 
-var WgIfaceKeyMap = make(map[string]RemotePeer)
+var WgIfaceKeyMap = make(map[string]models.RemotePeer)
 
-var RelayPeerMap = make(map[string]map[string]RemotePeer)
+var RelayPeerMap = make(map[string]map[string]models.RemotePeer)
 
-var ExtClientsWaitTh = make(map[string][]context.CancelFunc)
+var ExtClientsWaitTh = make(map[string]models.ExtClientPeer)
 
-var PeerAddrMap = make(map[string]map[string]*Conn)
+var ExtSourceIpMap = make(map[string]models.RemotePeer)
 
 // RunCmd - runs a local command
 func RunCmd(command string, printerr bool) (string, error) {

nm-proxy/manager/manager.go

@@ -9,9 +9,9 @@ import (
 	"net"
 	"reflect"
 	"runtime"
-	"time"
 
 	"github.com/gravitl/netmaker/nm-proxy/common"
+	"github.com/gravitl/netmaker/nm-proxy/models"
 	peerpkg "github.com/gravitl/netmaker/nm-proxy/peer"
 	"github.com/gravitl/netmaker/nm-proxy/proxy"
 	"github.com/gravitl/netmaker/nm-proxy/wg"
@@ -32,6 +32,7 @@ type ProxyAction string
 
 type ManagerPayload struct {
 	InterfaceName   string                 `json:"interface_name"`
+	WgAddr          string                 `json:"wg_addr"`
 	Peers           []wgtypes.PeerConfig   `json:"peers"`
 	PeerMap         map[string]PeerConf    `json:"peer_map"`
 	IsRelayed       bool                   `json:"is_relayed"`
@@ -76,7 +77,7 @@ func StartProxyManager(manageChan chan *ManagerAction) {
 			switch mI.Action {
 			case AddInterface:
 
-				mI.ExtClients()
+				mI.SetIngressGateway()
 				err := mI.AddInterfaceToProxy()
 				if err != nil {
 					log.Printf("failed to add interface: [%s] to proxy: %v\n  ", mI.Payload.InterfaceName, err)
@@ -108,7 +109,7 @@ func (m *ManagerAction) RelayUpdate() {
 	common.IsRelay = m.Payload.IsRelay
 }
 
-func (m *ManagerAction) ExtClients() {
+func (m *ManagerAction) SetIngressGateway() {
 	common.IsIngressGateway = m.Payload.IsIngress
 
 }
@@ -118,30 +119,30 @@ func (m *ManagerAction) RelayPeers() {
 	for relayedNodePubKey, relayedNodeConf := range m.Payload.RelayedPeerConf {
 		relayedNodePubKeyHash := fmt.Sprintf("%x", md5.Sum([]byte(relayedNodePubKey)))
 		if _, ok := common.RelayPeerMap[relayedNodePubKeyHash]; !ok {
-			common.RelayPeerMap[relayedNodePubKeyHash] = make(map[string]common.RemotePeer)
+			common.RelayPeerMap[relayedNodePubKeyHash] = make(map[string]models.RemotePeer)
 		}
 		for _, peer := range relayedNodeConf.Peers {
 			if peer.Endpoint != nil {
-				peer.Endpoint.Port = common.NmProxyPort
+				peer.Endpoint.Port = models.NmProxyPort
 				remotePeerKeyHash := fmt.Sprintf("%x", md5.Sum([]byte(peer.PublicKey.String())))
-				common.RelayPeerMap[relayedNodePubKeyHash][remotePeerKeyHash] = common.RemotePeer{
+				common.RelayPeerMap[relayedNodePubKeyHash][remotePeerKeyHash] = models.RemotePeer{
 					Endpoint: peer.Endpoint,
 				}
 			}
 
 		}
-		relayedNodeConf.RelayedPeerEndpoint.Port = common.NmProxyPort
-		common.RelayPeerMap[relayedNodePubKeyHash][relayedNodePubKeyHash] = common.RemotePeer{
+		relayedNodeConf.RelayedPeerEndpoint.Port = models.NmProxyPort
+		common.RelayPeerMap[relayedNodePubKeyHash][relayedNodePubKeyHash] = models.RemotePeer{
 			Endpoint: relayedNodeConf.RelayedPeerEndpoint,
 		}
 
 	}
 }
 
-func cleanUpInterface(ifaceConf common.WgIfaceConf) {
+func cleanUpInterface(ifaceConf models.WgIfaceConf) {
 	log.Println("########------------>  CLEANING UP: ", ifaceConf.Iface.Name)
 	for _, peerI := range ifaceConf.PeerMap {
-		peerI.Proxy.Cancel()
+		peerI.StopConn()
 	}
 	delete(common.WgIFaceMap, ifaceConf.Iface.Name)
 }
@@ -162,12 +163,13 @@ func (m *ManagerAction) processPayload() (*wg.WGIface, error) {
 			log.Println("failed to get real iface: ", err)
 		}
 	}
+	common.InterfaceName = m.Payload.InterfaceName
 	wgIface, err = wg.NewWGIFace(m.Payload.InterfaceName, "127.0.0.1/32", wg.DefaultMTU)
 	if err != nil {
 		log.Println("Failed init new interface: ", err)
 		return nil, err
 	}
-	var wgProxyConf common.WgIfaceConf
+	var wgProxyConf models.WgIfaceConf
 	var ok bool
 	if wgProxyConf, ok = common.WgIFaceMap[m.Payload.InterfaceName]; !ok {
 		for i := len(m.Payload.Peers) - 1; i >= 0; i-- {
@@ -177,6 +179,7 @@ func (m *ManagerAction) processPayload() (*wg.WGIface, error) {
 					log.Println("falied to update peer: ", err)
 				}
 				m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
+				continue
 			}
 		}
 		return wgIface, nil
@@ -200,17 +203,46 @@ func (m *ManagerAction) processPayload() (*wg.WGIface, error) {
 		return wgIface, nil
 	}
 	// check device conf different from proxy
-	//wgProxyConf.Iface = wgIface.Device
+	wgProxyConf.Iface = wgIface.Device
+	// sync peer map with new update
+	for _, currPeerI := range wgProxyConf.Iface.Peers {
+		if _, ok := m.Payload.PeerMap[currPeerI.PublicKey.String()]; !ok {
+			if val, ok := wgProxyConf.PeerMap[currPeerI.PublicKey.String()]; ok {
+				if val.IsAttachedExtClient {
+					log.Println("------> Deleting ExtClient Watch Thread: ", currPeerI.PublicKey.String())
+					if val, ok := common.ExtClientsWaitTh[currPeerI.PublicKey.String()]; ok {
+						val.CancelFunc()
+						delete(common.ExtClientsWaitTh, currPeerI.PublicKey.String())
+					}
+					log.Println("-----> Deleting Ext Client from Src Ip Map: ", currPeerI.PublicKey.String())
+					delete(common.ExtSourceIpMap, val.PeerConf.Endpoint.String())
+				}
+				val.StopConn()
+			}
+
+			// delete peer from interface
+			log.Println("CurrPeer Not Found, Deleting Peer from Interface: ", currPeerI.PublicKey.String())
+			if err := wgIface.RemovePeer(currPeerI.PublicKey.String()); err != nil {
+				log.Println("failed to remove peer: ", currPeerI.PublicKey.String(), err)
+			}
+			delete(common.PeerKeyHashMap, fmt.Sprintf("%x", md5.Sum([]byte(currPeerI.PublicKey.String()))))
+			delete(wgProxyConf.PeerMap, currPeerI.PublicKey.String())
+
+		}
+	}
 	for i := len(m.Payload.Peers) - 1; i >= 0; i-- {
 
 		if currentPeer, ok := wgProxyConf.PeerMap[m.Payload.Peers[i].PublicKey.String()]; ok {
+			if currentPeer.IsAttachedExtClient {
+				m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
+				continue
+			}
 			// check if proxy is off for the peer
 			if !m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].Proxy {
 
 				// cleanup proxy connections for the peer
-				currentPeer.Proxy.Cancel()
-				time.Sleep(time.Second * 3)
-				delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+				currentPeer.StopConn()
+				delete(wgProxyConf.PeerMap, currentPeer.Key)
 				// update the peer with actual endpoint
 				if err := wgIface.Update(m.Payload.Peers[i], false); err != nil {
 					log.Println("falied to update peer: ", err)
@@ -220,55 +252,51 @@ func (m *ManagerAction) processPayload() (*wg.WGIface, error) {
 
 			}
 			// check if peer is not connected to proxy
-			devPeer, err := wg.GetPeer(m.Payload.InterfaceName, currentPeer.Config.Key)
+			devPeer, err := wg.GetPeer(m.Payload.InterfaceName, currentPeer.Key)
 			if err == nil {
-				log.Printf("---------> COMAPRING ENDPOINT: DEV: %s, Proxy: %s", devPeer.Endpoint.String(), currentPeer.Proxy.LocalConn.LocalAddr().String())
-				if devPeer.Endpoint.String() != currentPeer.Proxy.LocalConn.LocalAddr().String() {
-					log.Println("---------> endpoint is not set to proxy: ", currentPeer.Config.Key)
-					currentPeer.Proxy.Cancel()
-					time.Sleep(time.Second * 3)
-					delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+				log.Printf("---------> COMAPRING ENDPOINT: DEV: %s, Proxy: %s", devPeer.Endpoint.String(), currentPeer.LocalConn.LocalAddr().String())
+				if devPeer.Endpoint.String() != currentPeer.LocalConn.LocalAddr().String() {
+					log.Println("---------> endpoint is not set to proxy: ", currentPeer.Key)
+					currentPeer.StopConn()
+					delete(wgProxyConf.PeerMap, currentPeer.Key)
 					continue
 				}
 			}
 			//check if peer is being relayed
-			if currentPeer.Config.IsRelayed != m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].IsRelayed {
-				log.Println("---------> peer relay status has been changed: ", currentPeer.Config.Key)
-				currentPeer.Proxy.Cancel()
-				time.Sleep(time.Second * 3)
-				delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+			if currentPeer.IsRelayed != m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].IsRelayed {
+				log.Println("---------> peer relay status has been changed: ", currentPeer.Key)
+				currentPeer.StopConn()
+				delete(wgProxyConf.PeerMap, currentPeer.Key)
 				continue
 			}
 			// check if relay endpoint has been changed
-			if currentPeer.Config.RelayedEndpoint != nil &&
+			if currentPeer.RelayedEndpoint != nil &&
 				m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].RelayedTo != nil &&
-				currentPeer.Config.RelayedEndpoint.String() != m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].RelayedTo.String() {
-				log.Println("---------> peer relay endpoint has been changed: ", currentPeer.Config.Key)
-				currentPeer.Proxy.Cancel()
-				time.Sleep(time.Second * 3)
-				delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+				currentPeer.RelayedEndpoint.String() != m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].RelayedTo.String() {
+				log.Println("---------> peer relay endpoint has been changed: ", currentPeer.Key)
+				currentPeer.StopConn()
+				delete(wgProxyConf.PeerMap, currentPeer.Key)
 				continue
 			}
-			if !reflect.DeepEqual(m.Payload.Peers[i], *currentPeer.Proxy.Config.PeerConf) {
-				if currentPeer.Proxy.RemoteConn.IP.String() != m.Payload.Peers[i].Endpoint.IP.String() {
-					log.Println("----------> Resetting proxy for Peer: ", currentPeer.Config.Key, m.Payload.InterfaceName)
-					currentPeer.Proxy.Cancel()
-					time.Sleep(time.Second * 3)
-					delete(wgProxyConf.PeerMap, currentPeer.Config.Key)
+			if !reflect.DeepEqual(m.Payload.Peers[i], *currentPeer.PeerConf) {
+				if currentPeer.RemoteConn.IP.String() != m.Payload.Peers[i].Endpoint.IP.String() {
+					log.Println("----------> Resetting proxy for Peer: ", currentPeer.Key, m.Payload.InterfaceName)
+					currentPeer.StopConn()
+					delete(wgProxyConf.PeerMap, currentPeer.Key)
 
 				} else {
 
-					log.Println("----->##### Updating Peer on Interface: ", m.Payload.InterfaceName, currentPeer.Config.Key)
+					log.Println("----->##### Updating Peer on Interface: ", m.Payload.InterfaceName, currentPeer.Key)
 					updatePeerConf := m.Payload.Peers[i]
-					localUdpAddr, err := net.ResolveUDPAddr("udp", currentPeer.Proxy.LocalConn.LocalAddr().String())
+					localUdpAddr, err := net.ResolveUDPAddr("udp", currentPeer.LocalConn.LocalAddr().String())
 					if err == nil {
 						updatePeerConf.Endpoint = localUdpAddr
 					}
 					if err := wgIface.Update(updatePeerConf, true); err != nil {
-						log.Println("failed to update peer: ", currentPeer.Config.Key, err)
+						log.Println("failed to update peer: ", currentPeer.Key, err)
 					}
-					currentPeer.Proxy.Config.PeerConf = &m.Payload.Peers[i]
-					wgProxyConf.PeerMap[currentPeer.Config.Key] = currentPeer
+					currentPeer.PeerConf = &m.Payload.Peers[i]
+					wgProxyConf.PeerMap[currentPeer.Key] = currentPeer
 					// delete the peer from the list
 					log.Println("-----------> deleting peer from list: ", m.Payload.Peers[i].PublicKey)
 					m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
@@ -281,7 +309,7 @@ func (m *ManagerAction) processPayload() (*wg.WGIface, error) {
 				m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
 			}
 
-		} else if !m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].Proxy {
+		} else if !m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].Proxy && !m.Payload.PeerMap[m.Payload.Peers[i].PublicKey.String()].IsAttachedExtClient {
 			log.Println("-----------> skipping peer, proxy is off: ", m.Payload.Peers[i].PublicKey)
 			if err := wgIface.Update(m.Payload.Peers[i], false); err != nil {
 				log.Println("falied to update peer: ", err)
@@ -289,35 +317,10 @@ func (m *ManagerAction) processPayload() (*wg.WGIface, error) {
 			m.Payload.Peers = append(m.Payload.Peers[:i], m.Payload.Peers[i+1:]...)
 		}
 	}
-	for _, currPeerI := range wgProxyConf.PeerMap {
-		if _, ok := m.Payload.PeerMap[currPeerI.Config.Key]; !ok {
-			currPeerI.Proxy.Cancel()
-			time.Sleep(time.Second * 3)
-			// delete peer from interface
-			log.Println("CurrPeer Not Found, Deleting Peer from Interface: ", currPeerI.Config.Key)
-			if err := wgIface.RemovePeer(currPeerI.Config.Key); err != nil {
-				log.Println("failed to remove peer: ", currPeerI.Config.Key, err)
-			}
-			delete(wgProxyConf.PeerMap, currPeerI.Config.Key)
 
-		}
-	}
-	common.WgIFaceMap[m.Payload.InterfaceName] = wgProxyConf
+	// sync dev peers with new update
 
-	// if peers, ok := common.WgIFaceMap[iface]; ok {
-	// 	log.Println("########------------>  CLEANING UP: ", iface)
-	// 	for _, peerI := range peers {
-	// 		peerI.Proxy.Cancel()
-	// 	}
-	// }
-	// delete(common.WgIFaceMap, iface)
-	// delete(common.PeerAddrMap, iface)
-	// if waitThs, ok := common.ExtClientsWaitTh[iface]; ok {
-	// 	for _, cancelF := range waitThs {
-	// 		cancelF()
-	// 	}
-	// 	delete(common.ExtClientsWaitTh, iface)
-	// }
+	common.WgIFaceMap[m.Payload.InterfaceName] = wgProxyConf
 
 	log.Println("CLEANED UP..........")
 	return wgIface, nil
@@ -337,7 +340,7 @@ func (m *ManagerAction) AddInterfaceToProxy() error {
 		log.Println("failed to get wg listen addr: ", err)
 		return err
 	}
-	common.WgIfaceKeyMap[fmt.Sprintf("%x", md5.Sum([]byte(wgInterface.Device.PublicKey.String())))] = common.RemotePeer{
+	common.WgIfaceKeyMap[fmt.Sprintf("%x", md5.Sum([]byte(wgInterface.Device.PublicKey.String())))] = models.RemotePeer{
 		PeerKey:   wgInterface.Device.PublicKey.String(),
 		Interface: wgInterface.Name,
 		Endpoint:  wgListenAddr,
@@ -349,41 +352,10 @@ func (m *ManagerAction) AddInterfaceToProxy() error {
 			log.Println("Endpoint nil for peer: ", peerI.PublicKey.String())
 			continue
 		}
-		if peerConf.IsExtClient && !common.IsIngressGateway {
-			continue
-		}
-		shouldProceed := false
-		if peerConf.IsExtClient && peerConf.IsAttachedExtClient {
-			// check if ext client got endpoint,otherwise continue
-			for _, devpeerI := range wgInterface.Device.Peers {
-				if devpeerI.PublicKey.String() == peerI.PublicKey.String() && devpeerI.Endpoint != nil {
-					peerI.Endpoint = devpeerI.Endpoint
-					shouldProceed = true
-					break
-				}
-			}
-
-		} else {
-			shouldProceed = true
-		}
-		if peerConf.IsExtClient && peerConf.IsAttachedExtClient && shouldProceed {
-			//ctx, cancel := context.WithCancel(context.Background())
-			//common.ExtClientsWaitTh[wgInterface.Name] = append(common.ExtClientsWaitTh[wgInterface.Name], cancel)
-			//go proxy.StartSniffer(ctx, wgInterface.Name, peerConf.Address, wgInterface.Port)
-		}
 
 		if peerConf.IsExtClient && !peerConf.IsAttachedExtClient {
 			peerI.Endpoint = peerConf.IngressGatewayEndPoint
 		}
-		if shouldProceed {
-			common.PeerKeyHashMap[fmt.Sprintf("%x", md5.Sum([]byte(peerI.PublicKey.String())))] = common.RemotePeer{
-				Interface:           m.Payload.InterfaceName,
-				PeerKey:             peerI.PublicKey.String(),
-				IsExtClient:         peerConf.IsExtClient,
-				Endpoint:            peerI.Endpoint,
-				IsAttachedExtClient: peerConf.IsAttachedExtClient,
-			}
-		}
 
 		var isRelayed bool
 		var relayedTo *net.UDPAddr
@@ -396,19 +368,29 @@ func (m *ManagerAction) AddInterfaceToProxy() error {
 			relayedTo = peerConf.RelayedTo
 
 		}
-		if !shouldProceed && peerConf.IsAttachedExtClient {
-			log.Println("Extclient endpoint not updated yet....skipping")
-			// TODO - watch the interface for ext client update
+		if peerConf.IsAttachedExtClient {
+			log.Println("Extclient Thread...")
 			go func(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig,
-				isRelayed bool, relayTo *net.UDPAddr, peerConf PeerConf) {
+				isRelayed bool, relayTo *net.UDPAddr, peerConf PeerConf, ingGwAddr string) {
 				addExtClient := false
+				commChan := make(chan *net.UDPAddr, 100)
 				ctx, cancel := context.WithCancel(context.Background())
-				common.ExtClientsWaitTh[wgInterface.Name] = append(common.ExtClientsWaitTh[wgInterface.Name], cancel)
+				common.ExtClientsWaitTh[peerI.PublicKey.String()] = models.ExtClientPeer{
+					CancelFunc: cancel,
+					CommChan:   commChan,
+				}
 				defer func() {
 					if addExtClient {
 						log.Println("GOT ENDPOINT for Extclient adding peer...")
-						//go proxy.StartSniffer(ctx, wgInterface.Name, peerConf.Address, wgInterface.Port)
-						common.PeerKeyHashMap[fmt.Sprintf("%x", md5.Sum([]byte(peer.PublicKey.String())))] = common.RemotePeer{
+						common.PeerKeyHashMap[fmt.Sprintf("%x", md5.Sum([]byte(peer.PublicKey.String())))] = models.RemotePeer{
+							Interface:           wgInterface.Name,
+							PeerKey:             peer.PublicKey.String(),
+							IsExtClient:         peerConf.IsExtClient,
+							IsAttachedExtClient: peerConf.IsAttachedExtClient,
+							Endpoint:            peer.Endpoint,
+						}
+
+						common.ExtSourceIpMap[peer.Endpoint.String()] = models.RemotePeer{
 							Interface:           wgInterface.Name,
 							PeerKey:             peer.PublicKey.String(),
 							IsExtClient:         peerConf.IsExtClient,
@@ -418,34 +400,35 @@ func (m *ManagerAction) AddInterfaceToProxy() error {
 
 						peerpkg.AddNewPeer(wgInterface, peer, peerConf.Address, isRelayed,
 							peerConf.IsExtClient, peerConf.IsAttachedExtClient, relayedTo)
+
 					}
+					log.Println("Exiting extclient watch Thread for: ", peer.PublicKey.String())
 				}()
 				for {
 					select {
 					case <-ctx.Done():
-						log.Println("Exiting extclient watch Thread for: ", wgInterface.Device.PublicKey.String())
 						return
-					default:
-						wgInterface, err := wg.NewWGIFace(m.Payload.InterfaceName, "127.0.0.1/32", wg.DefaultMTU)
-						if err != nil {
-							log.Println("Failed init new interface: ", err)
+					case endpoint := <-commChan:
+						if endpoint != nil {
+							addExtClient = true
+							peer.Endpoint = endpoint
+							delete(common.ExtClientsWaitTh, peer.PublicKey.String())
 							return
 						}
-						for _, devpeerI := range wgInterface.Device.Peers {
-							if devpeerI.PublicKey.String() == peer.PublicKey.String() && devpeerI.Endpoint != nil {
-								peer.Endpoint = devpeerI.Endpoint
-								addExtClient = true
-								return
-							}
-						}
-						time.Sleep(time.Second * 5)
 					}
 
 				}
 
-			}(wgInterface, &peerI, isRelayed, relayedTo, peerConf)
+			}(wgInterface, &peerI, isRelayed, relayedTo, peerConf, m.Payload.WgAddr)
 			continue
 		}
+		common.PeerKeyHashMap[fmt.Sprintf("%x", md5.Sum([]byte(peerI.PublicKey.String())))] = models.RemotePeer{
+			Interface:           m.Payload.InterfaceName,
+			PeerKey:             peerI.PublicKey.String(),
+			IsExtClient:         peerConf.IsExtClient,
+			Endpoint:            peerI.Endpoint,
+			IsAttachedExtClient: peerConf.IsAttachedExtClient,
+		}
 
 		peerpkg.AddNewPeer(wgInterface, &peerI, peerConf.Address, isRelayed,
 			peerConf.IsExtClient, peerConf.IsAttachedExtClient, relayedTo)

nm-proxy/models/peer.go

@@ -0,0 +1,46 @@
+package models
+
+import (
+	"context"
+	"net"
+
+	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
+)
+
+const (
+	NmProxyPort = 51722
+	DefaultCIDR = "127.0.0.1/8"
+)
+
+// ConnConfig is a peer Connection configuration
+type ConnConfig struct {
+
+	// Key is a public key of a remote peer
+	Key                 string
+	IsExtClient         bool
+	IsRelayed           bool
+	RelayedEndpoint     *net.UDPAddr
+	IsAttachedExtClient bool
+	PeerConf            *wgtypes.PeerConfig
+	StopConn            context.CancelFunc
+	RemoteConn          *net.UDPAddr
+	LocalConn           net.Conn
+}
+
+type RemotePeer struct {
+	PeerKey             string
+	Interface           string
+	Endpoint            *net.UDPAddr
+	IsExtClient         bool
+	IsAttachedExtClient bool
+}
+
+type ExtClientPeer struct {
+	CancelFunc context.CancelFunc
+	CommChan   chan *net.UDPAddr
+}
+
+type WgIfaceConf struct {
+	Iface   *wgtypes.Device
+	PeerMap map[string]*ConnConfig
+}

nm-proxy/nm-proxy.go

@@ -21,7 +21,6 @@ import (
 func Start(ctx context.Context, mgmChan chan *manager.ManagerAction, apiServerAddr string) {
 	log.Println("Starting Proxy...")
 	common.IsHostNetwork = (os.Getenv("HOST_NETWORK") == "" || os.Getenv("HOST_NETWORK") == "on")
-
 	hInfo := stun.GetHostInfo(apiServerAddr)
 	stun.Host = hInfo
 	log.Printf("HOSTINFO: %+v", hInfo)

nm-proxy/packet/packet_helper.go

@@ -0,0 +1,85 @@
+package packet
+
+import (
+	"bytes"
+	"encoding/base64"
+	"encoding/binary"
+	"errors"
+	"log"
+	"net"
+	"time"
+
+	"github.com/gravitl/netmaker/nm-proxy/common"
+	"golang.org/x/crypto/blake2s"
+	"golang.org/x/crypto/chacha20poly1305"
+	"golang.org/x/crypto/poly1305"
+	"golang.zx2c4.com/wireguard/tai64n"
+)
+
+var (
+	InitialChainKey [blake2s.Size]byte
+	InitialHash     [blake2s.Size]byte
+	ZeroNonce       [chacha20poly1305.NonceSize]byte
+)
+
+func init() {
+	InitialChainKey = blake2s.Sum256([]byte(NoiseConstruction))
+	mixHash(&InitialHash, &InitialChainKey, []byte(WGIdentifier))
+}
+
+type MessageInitiation struct {
+	Type      uint32
+	Sender    uint32
+	Ephemeral NoisePublicKey
+	Static    [NoisePublicKeySize + poly1305.TagSize]byte
+	Timestamp [tai64n.TimestampSize + poly1305.TagSize]byte
+	MAC1      [blake2s.Size128]byte
+	MAC2      [blake2s.Size128]byte
+}
+
+func ConsumeHandshakeInitiationMsg(initiator bool, buf []byte, src *net.UDPAddr, devicePubKey NoisePublicKey, devicePrivKey NoisePrivateKey) error {
+
+	var (
+		hash     [blake2s.Size]byte
+		chainKey [blake2s.Size]byte
+	)
+	var err error
+	var msg MessageInitiation
+	reader := bytes.NewReader(buf[:])
+	err = binary.Read(reader, binary.LittleEndian, &msg)
+	if err != nil {
+		log.Println("Failed to decode initiation message")
+		return err
+	}
+
+	if msg.Type != MessageInitiationType {
+		return errors.New("not handshake initiation message")
+	}
+	log.Println("-----> ConsumeHandshakeInitiationMsg, Intitator:  ", initiator)
+	mixHash(&hash, &InitialHash, devicePubKey[:])
+	mixHash(&hash, &hash, msg.Ephemeral[:])
+	mixKey(&chainKey, &InitialChainKey, msg.Ephemeral[:])
+
+	// decrypt static key
+	var peerPK NoisePublicKey
+	var key [chacha20poly1305.KeySize]byte
+	ss := sharedSecret(&devicePrivKey, msg.Ephemeral)
+	if isZero(ss[:]) {
+		return errors.New("no secret")
+	}
+	KDF2(&chainKey, &key, chainKey[:], ss[:])
+	aead, _ := chacha20poly1305.New(key[:])
+	_, err = aead.Open(peerPK[:0], ZeroNonce[:], msg.Static[:], hash[:])
+	if err != nil {
+		return err
+	}
+	log.Println("--------> Got HandShake from peer: ", base64.StdEncoding.EncodeToString(peerPK[:]), src)
+	if val, ok := common.ExtClientsWaitTh[base64.StdEncoding.EncodeToString(peerPK[:])]; ok {
+		val.CommChan <- src
+		time.Sleep(time.Second * 3)
+	}
+
+	setZero(hash[:])
+	setZero(chainKey[:])
+	return nil
+}

nm-proxy/packet/utils.go

@@ -0,0 +1,98 @@
+package packet
+
+import (
+	"crypto/hmac"
+	"crypto/subtle"
+	"hash"
+
+	"github.com/gravitl/netmaker/nm-proxy/wg"
+	"golang.org/x/crypto/blake2s"
+	"golang.org/x/crypto/curve25519"
+)
+
+const (
+	MessageInitiationType = 1
+
+	NoisePublicKeySize  = 32
+	NoisePrivateKeySize = 32
+
+	NoiseConstruction = "Noise_IKpsk2_25519_ChaChaPoly_BLAKE2s"
+	WGIdentifier      = "WireGuard v1 zx2c4 [email protected]"
+	WGLabelMAC1       = "mac1----"
+	WGLabelCookie     = "cookie--"
+)
+
+func mixKey(dst, c *[blake2s.Size]byte, data []byte) {
+	KDF1(dst, c[:], data)
+}
+
+func mixHash(dst, h *[blake2s.Size]byte, data []byte) {
+	hash, _ := blake2s.New256(nil)
+	hash.Write(h[:])
+	hash.Write(data)
+	hash.Sum(dst[:0])
+	hash.Reset()
+}
+func HMAC1(sum *[blake2s.Size]byte, key, in0 []byte) {
+	mac := hmac.New(func() hash.Hash {
+		h, _ := blake2s.New256(nil)
+		return h
+	}, key)
+	mac.Write(in0)
+	mac.Sum(sum[:0])
+}
+
+func HMAC2(sum *[blake2s.Size]byte, key, in0, in1 []byte) {
+	mac := hmac.New(func() hash.Hash {
+		h, _ := blake2s.New256(nil)
+		return h
+	}, key)
+	mac.Write(in0)
+	mac.Write(in1)
+	mac.Sum(sum[:0])
+}
+
+func KDF1(t0 *[blake2s.Size]byte, key, input []byte) {
+	HMAC1(t0, key, input)
+	HMAC1(t0, t0[:], []byte{0x1})
+}
+
+func KDF2(t0, t1 *[blake2s.Size]byte, key, input []byte) {
+	var prk [blake2s.Size]byte
+	HMAC1(&prk, key, input)
+	HMAC1(t0, prk[:], []byte{0x1})
+	HMAC2(t1, prk[:], t0[:], []byte{0x2})
+	setZero(prk[:])
+}
+
+func setZero(arr []byte) {
+	for i := range arr {
+		arr[i] = 0
+	}
+}
+func isZero(val []byte) bool {
+	acc := 1
+	for _, b := range val {
+		acc &= subtle.ConstantTimeByteEq(b, 0)
+	}
+	return acc == 1
+}
+
+func GetDeviceKeys(ifaceName string) (NoisePrivateKey, NoisePublicKey, error) {
+	wgPrivKey := wg.GetWgIfacePrivKey(ifaceName)
+	wgPubKey := wg.GetWgIfacePubKey(ifaceName)
+
+	return wgPrivKey, wgPubKey, nil
+}
+
+type (
+	NoisePublicKey  [NoisePublicKeySize]byte
+	NoisePrivateKey [NoisePrivateKeySize]byte
+)
+
+func sharedSecret(sk *NoisePrivateKey, pk NoisePublicKey) (ss [NoisePublicKeySize]byte) {
+	apk := (*[NoisePublicKeySize]byte)(&pk)
+	ask := (*[NoisePrivateKeySize]byte)(sk)
+	curve25519.ScalarMult(&ss, ask, apk)
+	return ss
+}

nm-proxy/peer/peer.go

@@ -7,32 +7,12 @@ import (
 	"net"
 
 	"github.com/gravitl/netmaker/nm-proxy/common"
+	"github.com/gravitl/netmaker/nm-proxy/models"
 	"github.com/gravitl/netmaker/nm-proxy/proxy"
 	"github.com/gravitl/netmaker/nm-proxy/wg"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
 )
 
-type Conn struct {
-	Config ConnConfig
-	Proxy  proxy.Proxy
-}
-
-// ConnConfig is a peer Connection configuration
-type ConnConfig struct {
-
-	// Key is a public key of a remote peer
-	Key string
-	// LocalKey is a public key of a local peer
-	LocalKey string
-
-	ProxyConfig     proxy.Config
-	AllowedIPs      string
-	LocalWgPort     int
-	RemoteProxyIP   net.IP
-	RemoteWgPort    int
-	RemoteProxyPort int
-}
-
 func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, peerAddr string,
 	isRelayed, isExtClient, isAttachedExtClient bool, relayTo *net.UDPAddr) error {
 
@@ -41,11 +21,11 @@ func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, peerAddr stri
 		LocalKey:    wgInterface.Device.PublicKey.String(),
 		RemoteKey:   peer.PublicKey.String(),
 		WgInterface: wgInterface,
-
-		PeerConf: peer,
+		IsExtClient: isExtClient,
+		PeerConf:    peer,
 	}
 	p := proxy.NewProxy(c)
-	peerPort := common.NmProxyPort
+	peerPort := models.NmProxyPort
 	if isExtClient && isAttachedExtClient {
 		peerPort = peer.Endpoint.Port
 
@@ -65,64 +45,37 @@ func AddNewPeer(wgInterface *wg.WGIface, peer *wgtypes.PeerConfig, peerAddr stri
 	}
 	log.Printf("----> Established Remote Conn with RPeer: %s, ----> RAddr: %s", peer.PublicKey, remoteConn.String())
 
-	if !(isExtClient && isAttachedExtClient) {
-		log.Printf("Starting proxy for Peer: %s\n", peer.PublicKey.String())
-		err = p.Start(remoteConn)
-		if err != nil {
-			return err
-		}
-	} else {
-		log.Println("Not Starting Proxy for Attached ExtClient...")
+	// if !(isExtClient && isAttachedExtClient) {
+	log.Printf("Starting proxy for Peer: %s\n", peer.PublicKey.String())
+	err = p.Start(remoteConn)
+	if err != nil {
+		return err
 	}
-
-	connConf := common.ConnConfig{
-		Key:             peer.PublicKey.String(),
-		LocalKey:        wgInterface.Device.PublicKey.String(),
-		LocalWgPort:     wgInterface.Device.ListenPort,
-		RemoteProxyIP:   net.ParseIP(peer.Endpoint.IP.String()),
-		RemoteWgPort:    peer.Endpoint.Port,
-		RemoteProxyPort: common.NmProxyPort,
-		IsRelayed:       isRelayed,
-		RelayedEndpoint: relayTo,
+	// } else {
+	// 	log.Println("Not Starting Proxy for Attached ExtClient...")
+	// }
+
+	connConf := models.ConnConfig{
+		Key:                 peer.PublicKey.String(),
+		IsRelayed:           isRelayed,
+		RelayedEndpoint:     relayTo,
+		IsAttachedExtClient: isAttachedExtClient,
+		PeerConf:            peer,
+		StopConn:            p.Cancel,
+		RemoteConn:          remoteConn,
+		LocalConn:           p.LocalConn,
 	}
 
-	peerProxy := common.Proxy{
-		Ctx:    p.Ctx,
-		Cancel: p.Cancel,
-		Config: common.Config{
-			Port:        peer.Endpoint.Port,
-			LocalKey:    wgInterface.Device.PublicKey.String(),
-			RemoteKey:   peer.PublicKey.String(),
-			WgInterface: wgInterface,
-			PeerConf:    peer,
-		},
-
-		RemoteConn: remoteConn,
-		LocalConn:  p.LocalConn,
-	}
-	if isRelayed {
-		connConf.RemoteProxyIP = relayTo.IP
-	}
-	peerConn := common.Conn{
-		Config: connConf,
-		Proxy:  peerProxy,
-	}
 	if _, ok := common.WgIFaceMap[wgInterface.Name]; ok {
-		common.WgIFaceMap[wgInterface.Name].PeerMap[peer.PublicKey.String()] = &peerConn
+		common.WgIFaceMap[wgInterface.Name].PeerMap[peer.PublicKey.String()] = &connConf
 	} else {
-		ifaceConf := common.WgIfaceConf{
+		ifaceConf := models.WgIfaceConf{
 			Iface:   wgInterface.Device,
-			PeerMap: make(map[string]*common.Conn),
+			PeerMap: make(map[string]*models.ConnConfig),
 		}
 
 		common.WgIFaceMap[wgInterface.Name] = ifaceConf
-		common.WgIFaceMap[wgInterface.Name].PeerMap[peer.PublicKey.String()] = &peerConn
-	}
-	if _, ok := common.PeerAddrMap[wgInterface.Name]; ok {
-		common.PeerAddrMap[wgInterface.Name][peerAddr] = &peerConn
-	} else {
-		common.PeerAddrMap[wgInterface.Name] = make(map[string]*common.Conn)
-		common.PeerAddrMap[wgInterface.Name][peerAddr] = &peerConn
+		common.WgIFaceMap[wgInterface.Name].PeerMap[peer.PublicKey.String()] = &connConf
 	}
 
 	return nil

nm-proxy/proxy/proxy.go

@@ -23,14 +23,14 @@ type Config struct {
 	RemoteKey   string
 	LocalKey    string
 	WgInterface *wg.WGIface
+	IsExtClient bool
 	PeerConf    *wgtypes.PeerConfig
 }
 
 // Proxy -  WireguardProxy proxies
 type Proxy struct {
-	Ctx    context.Context
-	Cancel context.CancelFunc
-
+	Ctx        context.Context
+	Cancel     context.CancelFunc
 	Config     Config
 	RemoteConn *net.UDPAddr
 	LocalConn  net.Conn
@@ -102,7 +102,7 @@ func getBoardCastAddress() ([]net.Addr, error) {
 	return nil, errors.New("couldn't obtain the broadcast addr")
 }
 
-// func StartSniffer(ctx context.Context, ifaceName, extClientAddr string, port int) {
+// func StartSniffer(ctx context.Context, ifaceName, ingGwAddr, extClientAddr string, port int) {
 // 	log.Println("Starting Packet Sniffer for iface: ", ifaceName)
 // 	var (
 // 		snapshotLen int32 = 1024
@@ -150,13 +150,17 @@ func getBoardCastAddress() ([]net.Addr, error) {
 // 					// Checksum, SrcIP, DstIP
 // 					fmt.Println("#########################")
 // 					fmt.Printf("From %s to %s\n", ip.SrcIP, ip.DstIP)
-// 					fmt.Println("Protocol: ", ip.Protocol)
-// 					if ip.DstIP.String() == extClientAddr || ip.SrcIP.String() == extClientAddr {
-// 						if ifacePeers, ok := common.PeerAddrMap[ifaceName]; ok {
-// 							if peerConf, ok := ifacePeers[ip.DstIP.String()]; ok {
-// 								log.Println("-----> Fowarding PKT From ExtClient: ", extClientAddr, " to: ", peerConf)
-// 								//server.NmProxyServer.Server.WriteTo(packet.Data(),  )
-// 							}
+// 					fmt.Println("Protocol: ", ip.Protocol.String())
+// 					if (ip.SrcIP.String() == extClientAddr && ip.DstIP.String() != ingGwAddr) ||
+// 						(ip.DstIP.String() == extClientAddr && ip.SrcIP.String() != ingGwAddr) {
+
+// 						log.Println("-----> Fowarding PKT From: ", ip.SrcIP, " to: ", ip.DstIP)
+// 						c, err := net.Dial("ip", ip.DstIP.String())
+// 						if err == nil {
+// 							c.Write(ip.Payload)
+// 							c.Close()
+// 						} else {
+// 							log.Println("------> Failed to forward packet from sniffer: ", err)
 
 // 						}
 // 					}

nm-proxy/proxy/wireguard.go

@@ -11,6 +11,7 @@ import (
 
 	"github.com/c-robinson/iplib"
 	"github.com/gravitl/netmaker/nm-proxy/common"
+	"github.com/gravitl/netmaker/nm-proxy/models"
 	"github.com/gravitl/netmaker/nm-proxy/packet"
 	"github.com/gravitl/netmaker/nm-proxy/server"
 	"github.com/gravitl/netmaker/nm-proxy/wg"
@@ -59,14 +60,46 @@ func (p *Proxy) ProxyToRemote() {
 				log.Println("ERRR READ: ", err)
 				continue
 			}
+
 			//go func(buf []byte, n int) {
 			ifaceConf := common.WgIFaceMap[p.Config.WgInterface.Name]
 			if peerI, ok := ifaceConf.PeerMap[p.Config.RemoteKey]; ok {
 				var srcPeerKeyHash, dstPeerKeyHash string
-				buf, n, srcPeerKeyHash, dstPeerKeyHash = packet.ProcessPacketBeforeSending(buf, n, peerI.Config.LocalKey, peerI.Config.Key)
-				if err != nil {
-					log.Println("failed to process pkt before sending: ", err)
+				if !p.Config.IsExtClient {
+					buf, n, srcPeerKeyHash, dstPeerKeyHash = packet.ProcessPacketBeforeSending(buf, n, ifaceConf.Iface.PublicKey.String(), peerI.Key)
+					if err != nil {
+						log.Println("failed to process pkt before sending: ", err)
+					}
+				} else {
+					// unknown peer to proxy -> check if extclient and handle it
+					// consume handshake message for ext clients
+					// msgType := binary.LittleEndian.Uint32(buf[:n])
+					// switch msgType {
+					// case models.MessageInitiationType:
+
+					// 	devPriv, devPubkey, err := packet.GetDeviceKeys(common.InterfaceName)
+					// 	if err == nil {
+					// 		err := packet.ConsumeHandshakeInitiationMsg(true, buf[:n], p.RemoteConn, devPubkey, devPriv)
+					// 		if err != nil {
+					// 			log.Println("---------> @@@ failed to decode HS: ", err)
+					// 		}
+					// 	} else {
+					// 		log.Println("failed to get device keys: ", err)
+					// 	}
+					// case models.MessageResponseType:
+					// 	devPriv, devPubkey, err := packet.GetDeviceKeys(common.InterfaceName)
+					// 	if err == nil {
+					// 		err := packet.ConsumeMessageResponse(true, buf[:n], p.RemoteConn, devPubkey, devPriv)
+					// 		if err != nil {
+					// 			log.Println("---------> @@@ failed to decode HS: ", err)
+					// 		}
+					// 	} else {
+					// 		log.Println("failed to get device keys: ", err)
+					// 	}
+
+					// }
 				}
+
 				log.Printf("PROXING TO REMOTE!!!---> %s >>>>> %s >>>>> %s [[ SrcPeerHash: %s, DstPeerHash: %s ]]\n",
 					p.LocalConn.LocalAddr(), server.NmProxyServer.Server.LocalAddr().String(), p.RemoteConn.String(), srcPeerKeyHash, dstPeerKeyHash)
 			} else {
@@ -114,7 +147,7 @@ func (p *Proxy) Start(remoteConn *net.UDPAddr) error {
 	var err error
 
 	//log.Printf("----> WGIFACE: %+v\n", p.Config.WgInterface)
-	addr, err := GetFreeIp(common.DefaultCIDR, p.Config.WgInterface.Port)
+	addr, err := GetFreeIp(models.DefaultCIDR, p.Config.WgInterface.Port)
 	if err != nil {
 		log.Println("Failed to get freeIp: ", err)
 		return err
@@ -130,7 +163,7 @@ func (p *Proxy) Start(remoteConn *net.UDPAddr) error {
 	//log.Println("--------->#### Wg Listen Addr: ", wgListenAddr.String())
 	p.LocalConn, err = net.DialUDP("udp", &net.UDPAddr{
 		IP:   net.ParseIP(addr),
-		Port: common.NmProxyPort,
+		Port: models.NmProxyPort,
 	}, wgListenAddr)
 	if err != nil {
 		log.Printf("failed dialing to local Wireguard port,Err: %v\n", err)
@@ -171,7 +204,7 @@ func GetFreeIp(cidrAddr string, dstPort int) (string, error) {
 
 		conn, err := net.DialUDP("udp", &net.UDPAddr{
 			IP:   net.ParseIP(newAddrs.String()),
-			Port: common.NmProxyPort,
+			Port: models.NmProxyPort,
 		}, &net.UDPAddr{
 			IP:   net.ParseIP("127.0.0.1"),
 			Port: dstPort,

nm-proxy/server/server.go

@@ -2,12 +2,14 @@ package server
 
 import (
 	"context"
+	"encoding/binary"
 	"fmt"
 	"log"
 	"net"
 	"time"
 
 	"github.com/gravitl/netmaker/nm-proxy/common"
+	"github.com/gravitl/netmaker/nm-proxy/models"
 	"github.com/gravitl/netmaker/nm-proxy/packet"
 )
 
@@ -17,7 +19,7 @@ var (
 
 const (
 	defaultBodySize = 10000
-	defaultPort     = common.NmProxyPort
+	defaultPort     = models.NmProxyPort
 )
 
 type Config struct {
@@ -46,7 +48,7 @@ func (p *ProxyServer) Listen(ctx context.Context) {
 			for iface, ifaceConf := range common.WgIFaceMap {
 				log.Println("########------------>  CLEANING UP: ", iface)
 				for _, peerI := range ifaceConf.PeerMap {
-					peerI.Proxy.Cancel()
+					peerI.StopConn()
 				}
 			}
 			// close server connection
@@ -61,10 +63,11 @@ func (p *ProxyServer) Listen(ctx context.Context) {
 				continue
 			}
 			//go func(buffer []byte, source *net.UDPAddr, n int) {
-
+			origBufferLen := n
 			var srcPeerKeyHash, dstPeerKeyHash string
 			n, srcPeerKeyHash, dstPeerKeyHash = packet.ExtractInfo(buffer, n)
 			//log.Printf("--------> RECV PKT , [SRCKEYHASH: %s], SourceIP: [%s] \n", srcPeerKeyHash, source.IP.String())
+
 			if _, ok := common.WgIfaceKeyMap[dstPeerKeyHash]; !ok {
 				// if common.IsIngressGateway {
 				// 	log.Println("----> fowarding PKT to EXT client...")
@@ -118,32 +121,52 @@ func (p *ProxyServer) Listen(ctx context.Context) {
 				if ifaceConf, ok := common.WgIFaceMap[peerInfo.Interface]; ok {
 					if peerI, ok := ifaceConf.PeerMap[peerInfo.PeerKey]; ok {
 						log.Printf("PROXING TO LOCAL!!!---> %s <<<< %s <<<<<<<< %s   [[ RECV PKT [SRCKEYHASH: %s], [DSTKEYHASH: %s], SourceIP: [%s] ]]\n",
-							peerI.Proxy.LocalConn.RemoteAddr(), peerI.Proxy.LocalConn.LocalAddr(),
+							peerI.LocalConn.RemoteAddr(), peerI.LocalConn.LocalAddr(),
 							fmt.Sprintf("%s:%d", source.IP.String(), source.Port), srcPeerKeyHash, dstPeerKeyHash, source.IP.String())
-						_, err = peerI.Proxy.LocalConn.Write(buffer[:n])
+						_, err = peerI.LocalConn.Write(buffer[:n])
 						if err != nil {
 							log.Println("Failed to proxy to Wg local interface: ", err)
 							//continue
 						}
+						continue
 
 					}
 				}
 
 			}
-			// // forward to all interfaces
-			// for _, ifaceCfg := range common.WgIfaceKeyMap {
-			// 	log.Println("###--------> Forwarding Unknown PKT to ", ifaceCfg.Interface)
-			// 	conn, err := net.DialUDP("udp", nil, ifaceCfg.Endpoint)
-			// 	if err == nil {
-			// 		_, err := conn.Write(buffer[:n])
-			// 		if err != nil {
-			// 			log.Println("Failed to forward the unknown pkt to ifcace: ", ifaceCfg.Interface, err)
-			// 		}
-			// 		conn.Close()
-			// 	}
-
-			// }
-			//}(buffer, source, n)
+			if peerInfo, ok := common.ExtSourceIpMap[source.String()]; ok {
+				if ifaceConf, ok := common.WgIFaceMap[peerInfo.Interface]; ok {
+					if peerI, ok := ifaceConf.PeerMap[peerInfo.PeerKey]; ok {
+						log.Printf("PROXING TO LOCAL!!!---> %s <<<< %s <<<<<<<< %s   [[ RECV PKT [SRCKEYHASH: %s], [DSTKEYHASH: %s], SourceIP: [%s] ]]\n",
+							peerI.LocalConn.RemoteAddr(), peerI.LocalConn.LocalAddr(),
+							fmt.Sprintf("%s:%d", source.IP.String(), source.Port), srcPeerKeyHash, dstPeerKeyHash, source.IP.String())
+						_, err = peerI.LocalConn.Write(buffer[:origBufferLen])
+						if err != nil {
+							log.Println("Failed to proxy to Wg local interface: ", err)
+							//continue
+						}
+						continue
+
+					}
+				}
+			}
+			// unknown peer to proxy -> check if extclient and handle it
+			// consume handshake message for ext clients
+			msgType := binary.LittleEndian.Uint32(buffer[:4])
+			switch msgType {
+			case packet.MessageInitiationType:
+
+				devPriv, devPubkey, err := packet.GetDeviceKeys(common.InterfaceName)
+				if err == nil {
+					err := packet.ConsumeHandshakeInitiationMsg(false, buffer[:origBufferLen], source, devPubkey, devPriv)
+					if err != nil {
+						log.Println("---------> @@@ failed to decode HS: ", err)
+					}
+				} else {
+					log.Println("failed to get device keys: ", err)
+				}
+			}
+
 		}
 
 	}

nm-proxy/stun/stun.go

@@ -7,7 +7,7 @@ import (
 	"strconv"
 	"strings"
 
-	"github.com/gravitl/netmaker/nm-proxy/common"
+	"github.com/gravitl/netmaker/nm-proxy/models"
 	"gortc.io/stun"
 )
 
@@ -29,7 +29,7 @@ func GetHostInfo(stunHostAddr string) (info HostInfo) {
 	}
 	l := &net.UDPAddr{
 		IP:   net.ParseIP(""),
-		Port: common.NmProxyPort,
+		Port: models.NmProxyPort,
 	}
 	conn, err := net.DialUDP("udp", l, s)
 	if err != nil {

nm-proxy/wg/wg.go

@@ -80,18 +80,33 @@ func (w *WGIface) GetWgIface(iface string) error {
 	return nil
 }
 
-func GetWgIfacePubKey(iface string) string {
+func GetWgIfacePubKey(iface string) [32]byte {
 	wgClient, err := wgctrl.New()
 	if err != nil {
 		log.Println("Error fetching pub key: ", iface, err)
-		return ""
+		return [32]byte{}
 	}
 	dev, err := wgClient.Device(iface)
 	if err != nil {
 		log.Println("Error fetching pub key: ", iface, err)
-		return ""
+		return [32]byte{}
 	}
-	return dev.PublicKey.String()
+
+	return dev.PublicKey
+}
+
+func GetWgIfacePrivKey(iface string) [32]byte {
+	wgClient, err := wgctrl.New()
+	if err != nil {
+		log.Println("Error fetching pub key: ", iface, err)
+		return [32]byte{}
+	}
+	dev, err := wgClient.Device(iface)
+	if err != nil {
+		log.Println("Error fetching pub key: ", iface, err)
+		return [32]byte{}
+	}
+	return dev.PrivateKey
 }
 
 // parseAddress parse a string ("1.2.3.4/24") address to WG Address