Merge pull request #302 from gravitl/feature_v0.8.1_egressingress

Feature v0.8.1 egressingress

config/config.go

@@ -48,6 +48,7 @@ type ServerConfig struct {
 	AgentBackend         string `yaml:"agentbackend"`
 	ClientMode           string `yaml:"clientmode"`
 	DNSMode              string `yaml:"dnsmode"`
+	SplitDNS           string `yaml:"splitdns"`
 	DisableRemoteIPCheck string `yaml:"disableremoteipcheck"`
 	DisableDefaultNet    string `yaml:"disabledefaultnet"`
 	GRPCSSL              string `yaml:"grpcssl"`

controllers/common.go

@@ -14,7 +14,6 @@ import (
 	"golang.org/x/crypto/bcrypt"
 )
 
-
 func GetPeersList(networkName string, excludeRelayed bool, relayedNodeAddr string) ([]models.Node, error) {
 	var peers []models.Node
 	var relayNode models.Node
@@ -218,6 +217,7 @@ func GetNode(macaddress string, network string) (models.Node, error) {
 	if err = json.Unmarshal([]byte(data), &node); err != nil {
 		return node, err
 	}
+	node.SetDefaults()
 
 	return node, err
 }
@@ -253,7 +253,9 @@ func CreateNode(node models.Node, networkName string) (models.Node, error) {
 			node.IsServer = "yes"
 		}
 	}
-
+	if servercfg.IsDNSMode() && node.DNSOn == ""{
+		node.DNSOn = "yes"
+	}
 	node.SetDefaults()
 	node.Address, err = functions.UniqueAddress(networkName)
 	if err != nil {
@@ -269,9 +271,6 @@ func CreateNode(node models.Node, networkName string) (models.Node, error) {
 		//returnErrorResponse(w, r, errorResponse)
 		return node, err
 	}
-	if servercfg.IsDNSMode() {
-		node.DNSOn = "yes"
-	}
 	err = node.Validate(false)
 	if err != nil {
 		return node, err

controllers/dnsHttpController.go

@@ -8,6 +8,7 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/functions"
 	"github.com/gravitl/netmaker/models"
+	"github.com/gravitl/netmaker/servercfg"
 	"github.com/txn2/txeh"
 )
 
@@ -161,8 +162,9 @@ func SetDNS() error {
 	if err != nil {
 		return err
 	}
-	err = functions.SetCorefile(corefilestring)
-
+	if servercfg.IsSplitDNS() {
+		err = functions.SetCorefile(corefilestring)
+	}
 	return err
 }
 

controllers/extClientHttpController.go

@@ -375,7 +375,7 @@ func DeleteExtClient(network string, clientid string) error {
  */
 func DeleteGatewayExtClients(gatewayID string, networkName string) error {
 	currentExtClients, err := GetNetworkExtClients(networkName)
-	if err != nil {
+	if err != nil && !database.IsEmptyRecord(err) {
 		return err
 	}
 	for _, extClient := range currentExtClients {

controllers/networkHttpController.go

@@ -133,7 +133,7 @@ func getNetworks(w http.ResponseWriter, r *http.Request) {
 	err := errors.New("Networks Error")
 	if networksSlice[0] == ALL_NETWORK_ACCESS {
 		allnetworks, err = models.GetNetworks()
-		if err != nil && !database.IsEmptyRecord(err){
+		if err != nil && !database.IsEmptyRecord(err) {
 			returnErrorResponse(w, r, formatError(err, "internal"))
 			return
 		}
@@ -390,7 +390,7 @@ func CreateNetwork(network models.Network) error {
 		return err
 	}
 
-	if servercfg.IsClientMode() {
+	if servercfg.IsClientMode() != "off" {
 		var success bool
 		success, err = serverctl.AddNetwork(network.NetID)
 		if err != nil || !success {

controllers/nodeGrpcController.go

@@ -148,7 +148,7 @@ func (s *NodeServiceServer) GetPeers(ctx context.Context, req *nodepb.Object) (*
 		}
 
 		peersData, err := json.Marshal(&peers)
-		functions.PrintUserLog(node.Address,"checked in successfully",3)
+		functions.PrintUserLog(node.Address, "checked in successfully", 3)
 		return &nodepb.Object{
 			Data: string(peersData),
 			Type: nodepb.NODE_TYPE,

controllers/nodeHttpController.go

@@ -597,6 +597,10 @@ func DeleteEgressGateway(network, macaddress string) (models.Node, error) {
 	node.EgressGatewayRanges = []string{}
 	node.PostUp = ""
 	node.PostDown = ""
+	if node.IsIngressGateway == "yes" { // check if node is still an ingress gateway before completely deleting postdown/up rules
+		node.PostUp = "iptables -A FORWARD -i " + node.Interface + " -j ACCEPT; iptables -t nat -A POSTROUTING -o " + node.Interface + " -j MASQUERADE"
+		node.PostDown = "iptables -D FORWARD -i " + node.Interface + " -j ACCEPT; iptables -t nat -D POSTROUTING -o " + node.Interface + " -j MASQUERADE"
+	}
 	node.SetLastModified()
 	node.PullChanges = "yes"
 	key, err := functions.GetRecordKey(node.MacAddress, node.Network)

docker/Dockerfile-builder

@@ -0,0 +1,23 @@
+FROM alpine:3.13.6
+
+RUN apk add --no-cache --virtual .build-deps bash gcc musl-dev openssl go 
+
+RUN wget -O go.tgz https://dl.google.com/go/go1.17.1.linux-amd64.tar.gz 
+
+RUN tar -C /usr/local -xzf go.tgz 
+
+WORKDIR /usr/local/go/src
+
+RUN chmod +x make.bash
+
+RUN ./make.bash 
+
+ENV PATH="/usr/local/go/bin:$PATH"
+
+ENV GOPATH=/opt/go/ 
+
+ENV PATH=$PATH:$GOPATH/bin 
+
+RUN apk del .build-deps
+
+RUN apk add build-base

docker/Dockerfile-netclient-slim-solo

@@ -0,0 +1,22 @@
+FROM gravitl/builder:latest as builder
+# add glib support daemon manager
+WORKDIR /app
+
+COPY . .
+
+ENV GO111MODULE=auto
+
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 /usr/local/go/bin/go build -ldflags="-w -s" -o netclient-app netclient/main.go
+
+FROM alpine:3.13.6
+
+RUN apk add gcompat iptables && mkdir -p /etc/netclient
+# set the working directory
+WORKDIR /root/
+
+COPY --from=builder /app/netclient-app /etc/netclient/netclient
+COPY --from=builder /app/scripts/netclient.sh .
+
+RUN chmod 0755 /etc/netclient/netclient && chmod 0755 netclient.sh
+
+ENTRYPOINT ["./netclient.sh"]

docker/Dockerfile-netmaker-slim

@@ -0,0 +1,34 @@
+#first stage - builder
+FROM gravitl/builder as builder
+
+WORKDIR /app
+
+COPY . .
+
+ENV GO111MODULE=auto
+
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=1 /usr/local/go/bin/go build -ldflags="-w -s" -o netmaker main.go
+
+WORKDIR /app/netclient
+
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 /usr/local/go/bin/go build -ldflags="-w -s" -o netclient main.go
+#second stage
+
+FROM alpine:3.13.6
+# add a c lib
+RUN apk add gcompat iptables
+# set the working directory
+WORKDIR /root/
+
+RUN mkdir /etc/netclient
+
+COPY --from=builder /app/netmaker .
+COPY --from=builder /app/config config
+COPY --from=builder /app/netclient/netclient  /etc/netclient/netclient
+
+RUN chmod 0755 /etc/netclient/netclient
+
+EXPOSE 8081
+EXPOSE 50051
+
+ENTRYPOINT ["./netmaker"]

docker/Dockerfile-netmaker-slim-solo

@@ -0,0 +1,26 @@
+FROM gravitl/builder:latest as builder
+# add glib support daemon manager
+WORKDIR /app
+
+COPY . .
+
+ENV GO111MODULE=auto
+
+RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=1 /usr/local/go/bin/go build -ldflags="-w -s" -o netmaker main.go
+
+FROM alpine:3.13.6
+
+RUN apk add gcompat
+# set the working directory
+WORKDIR /root/
+
+COPY --from=builder /app/netmaker .
+COPY --from=builder /app/config .
+
+RUN mkdir -p config/dnsconfig
+RUN touch config/dnsconfig/Corefile && touch config/dnsconfig/netmaker.hosts
+
+EXPOSE 8081
+EXPOSE 50051
+
+ENTRYPOINT ["./netmaker"]

functions/helpers.go

@@ -556,6 +556,8 @@ func GetNodeByMacAddress(network string, macaddress string) (models.Node, error)
 		return models.Node{}, err
 	}
 
+	node.SetDefaults()
+
 	return node, nil
 }
 
@@ -577,6 +579,8 @@ func GetDeletedNodeByMacAddress(network string, macaddress string) (models.Node,
 		return models.Node{}, err
 	}
 
+	node.SetDefaults()
+
 	return node, nil
 }
 

functions/local.go

@@ -27,7 +27,7 @@ func SetDNSDir() error {
         }
 		_, err = os.Stat(dir + "/config/dnsconfig/Corefile")
         if os.IsNotExist(err) {
-			err = SetCorefile("example.com")
+			err = SetCorefile(".")
 			if err != nil {
 				PrintUserLog("",err.Error(),0)
 			}

main.go

@@ -11,6 +11,7 @@ import (
 	"os/signal"
 	"strconv"
 	"sync"
+	"time"
 
 	controller "github.com/gravitl/netmaker/controllers"
 	"github.com/gravitl/netmaker/database"
@@ -19,6 +20,7 @@ import (
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
+	"github.com/gravitl/netmaker/serverctl"
 	"google.golang.org/grpc"
 )
 
@@ -37,7 +39,7 @@ func initialize() { // Client Mode Prereq Check
 		log.Fatal(err)
 	}
 	log.Println("database successfully connected.")
-	if servercfg.IsClientMode() {
+	if servercfg.IsClientMode() != "off" {
 		output, err := ncutils.RunCmd("id -u", true)
 		if err != nil {
 			log.Println("Error running 'id -u' for prereq check. Please investigate or disable client mode.")
@@ -51,6 +53,9 @@ func initialize() { // Client Mode Prereq Check
 		if uid != 0 {
 			log.Fatal("To run in client mode requires root privileges. Either disable client mode or run with sudo.")
 		}
+		if err := serverctl.InitServerNetclient(); err != nil {
+			log.Fatal("Did not find netclient to use CLIENT_MODE")
+		}
 	}
 
 	if servercfg.IsDNSMode() {
@@ -75,6 +80,13 @@ func startControllers() {
 		waitnetwork.Add(1)
 		go runGRPC(&waitnetwork)
 	}
+
+	// Run the client in goroutine locally if CLIENT_MODE is "contained"
+	if servercfg.IsClientMode() == "contained" {
+		waitnetwork.Add(1)
+		go runClient(&waitnetwork)
+	}
+
 	if servercfg.IsDNSMode() {
 		err := controller.SetDNS()
 		if err != nil {
@@ -96,8 +108,22 @@ func startControllers() {
 	if !servercfg.IsAgentBackend() && !servercfg.IsRestBackend() {
 		log.Println("No Server Mode selected, so nothing is being served! Set either Agent mode (AGENT_BACKEND) or Rest mode (REST_BACKEND) to 'true'.")
 	}
+
 	waitnetwork.Wait()
-	log.Println("exiting")
+	log.Println("[netmaker] exiting")
+}
+
+func runClient(wg *sync.WaitGroup) {
+	defer wg.Done()
+	log.Println("CLIENT_MODE running as contained")
+	go func() {
+		for {
+			if err := serverctl.HandleContainedClient(); err != nil {
+				// PASS
+			}
+			time.Sleep(time.Second * 15)
+		}
+	}()
 }
 
 func runGRPC(wg *sync.WaitGroup) {

models/network.go

@@ -178,7 +178,7 @@ func (network *Network) SetNetworkLastModified() {
 
 func (network *Network) SetDefaults() {
 	if network.DefaultUDPHolePunch == "" {
-		if servercfg.IsClientMode() {
+		if servercfg.IsClientMode() != "off" {
 			network.DefaultUDPHolePunch = "yes"
 		} else {
 			network.DefaultUDPHolePunch = "no"

models/node.go

@@ -532,25 +532,6 @@ func GetAllNodes() ([]Node, error) {
 	return nodes, nil
 }
 
-func GetNode(macaddress string, network string) (Node, error) {
-
-	var node Node
-
-	key, err := GetID(macaddress, network)
-	if err != nil {
-		return node, err
-	}
-	data, err := database.FetchRecord(database.NODES_TABLE_NAME, key)
-	if err != nil {
-		return node, err
-	}
-	if err = json.Unmarshal([]byte(data), &node); err != nil {
-		return node, err
-	}
-
-	return node, err
-}
-
 func GetID(macaddress string, network string) (string, error) {
 	if macaddress == "" || network == "" {
 		return "", errors.New("unable to get record key")

netclient/config/config.go

@@ -282,7 +282,7 @@ func GetCLIConfig(c *cli.Context) (ClientConfig, string, error) {
 	cfg.Node.Address = c.String("address")
 	cfg.Node.Address6 = c.String("addressIPV6")
 	cfg.Node.Roaming = c.String("roaming")
-	cfg.Node.DNSOn = c.String("dns")
+	cfg.Node.DNSOn = c.String("dnson")
 	cfg.Node.IsLocal = c.String("islocal")
 	cfg.Node.IsDualStack = c.String("isdualstack")
 	cfg.Node.PostUp = c.String("postup")

netclient/daemon/macos.go

@@ -4,18 +4,33 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
-
+	"path/filepath"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 )
 
 const MAC_SERVICE_NAME = "com.gravitl.netclient"
 
 func SetupMacDaemon() error {
+
+	dir, err := filepath.Abs(filepath.Dir(os.Args[0]))
+	if err != nil {
+		return err
+	}
+	binarypath := dir + "/netclient"
+
+	if !ncutils.FileExists("/etc/netclient/netclient") {
+		_, err = ncutils.Copy(binarypath, "/etc/netclient/netclient")
+		if err != nil {
+			log.Println(err)
+			return err
+		}
+	}
+
 	_, errN := os.Stat("~/Library/LaunchAgents")
 	if os.IsNotExist(errN) {
 		os.Mkdir("~/Library/LaunchAgents", 0755)
 	}
-	err := CreateMacService(MAC_SERVICE_NAME)
+	err = CreateMacService(MAC_SERVICE_NAME)
 	if err != nil {
 		return err
 	}

netclient/functions/checkin.go

@@ -6,7 +6,6 @@ import (
 	"os"
 	"runtime"
 	"strings"
-
 	nodepb "github.com/gravitl/netmaker/grpc"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/netclient/auth"

netclient/functions/join.go

@@ -105,6 +105,7 @@ func JoinNetwork(cfg config.ClientConfig, privateKey string) error {
 		LocalAddress:        cfg.Node.LocalAddress,
 		Interface:           cfg.Node.Interface,
 		PublicKey:           cfg.Node.PublicKey,
+		DNSOn:           	 cfg.Node.DNSOn,
 		Name:                cfg.Node.Name,
 		Endpoint:            cfg.Node.Endpoint,
 		SaveConfig:          cfg.Node.SaveConfig,
@@ -118,7 +119,6 @@ func JoinNetwork(cfg config.ClientConfig, privateKey string) error {
 	if err != nil {
 		return err
 	}
-
 	// Create node on server
 	res, err := wcclient.CreateNode(
 		context.TODO(),
@@ -152,6 +152,7 @@ func JoinNetwork(cfg config.ClientConfig, privateKey string) error {
 		}
 		node.Endpoint = node.LocalAddress
 	}
+
 	err = config.ModConfig(&node)
 	if err != nil {
 		return err

scripts/daemon.sh

@@ -0,0 +1,11 @@
+# create a logs file
+sudo touch /etc/netclient/netclient.logs
+echo "[netclient] created logs file in /etc/netclient/netclient.logs"
+echo "[netclient] Starting netclient checkins"
+# loop and call checkin -n all
+while [ 1 ]; do
+    # add logs to netclient.logs
+    sudo /etc/netclient/netclient checkin -n all >> /etc/netclient/netclient.logs 2&1>
+    sleep 15
+done &
+echo "[netclient] exiting"

scripts/netclient.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+echo "[netclient] joining network"
+
+/etc/netclient/netclient join -t $NETCLIENT_ACCESSTOKEN -daemon off -dnson no
+
+echo "[netclient] Starting netclient checkin"
+# loop and call checkin -n all
+while [ 1 ]; do
+    # add logs to netclient.logs
+    /etc/netclient/netclient checkin -n all
+    sleep $SLEEP
+done
+echo "[netclient] exiting"

servercfg/serverconf.go

@@ -39,8 +39,8 @@ func GetServerConfig() config.ServerConfig {
 		cfg.AgentBackend = "on"
 	}
 	cfg.ClientMode = "off"
-	if IsClientMode() {
-		cfg.ClientMode = "on"
+	if IsClientMode() != "off" {
+		cfg.ClientMode = IsClientMode()
 	}
 	cfg.DNSMode = "off"
 	if IsDNSMode() {
@@ -222,15 +222,21 @@ func IsAgentBackend() bool {
 	}
 	return isagent
 }
-func IsClientMode() bool {
-	isclient := true
+func IsClientMode() string {
+	isclient := "on"
 	if os.Getenv("CLIENT_MODE") != "" {
 		if os.Getenv("CLIENT_MODE") == "off" {
-			isclient = false
+			isclient = "off"
+		}
+		if os.Getenv("CLIENT_MODE") == "contained" {
+			isclient = "contained"
 		}
 	} else if config.Config.Server.ClientMode != "" {
 		if config.Config.Server.ClientMode == "off" {
-			isclient = false
+			isclient = "off"
+		}
+		if config.Config.Server.ClientMode == "contained" {
+			isclient = "contained"
 		}
 	}
 	return isclient
@@ -345,3 +351,13 @@ func GetSQLConn() string {
 	}
 	return sqlconn
 }
+
+func IsSplitDNS() bool {
+	issplit := false
+	if os.Getenv("IS_SPLIT_DNS") == "yes" {
+		issplit = true
+	} else if config.Config.Server.SplitDNS == "yes" {
+		issplit = true
+	}
+	return issplit
+}

serverctl/serverctl.go

@@ -100,41 +100,79 @@ func RemoveNetwork(network string) (bool, error) {
 
 }
 
-func AddNetwork(network string) (bool, error) {
-	pubip, err := servercfg.GetPublicIP()
-	if err != nil {
-		log.Println("could not get public IP.")
-		return false, err
-	}
+func InitServerNetclient() error {
 	netclientDir := ncutils.GetNetclientPath()
 	netclientPath := ncutils.GetNetclientPathSpecific()
-	_, err = os.Stat(netclientDir)
+	_, err := os.Stat(netclientDir)
 	if os.IsNotExist(err) {
 		os.Mkdir(netclientDir, 744)
 	} else if err != nil {
 		log.Println("could not find or create", netclientDir)
-		return false, err
-	}
-	token, err := functions.CreateServerToken(network)
-	if err != nil {
-		log.Println("could not create server token for " + network)
-		return false, err
+		return err
 	}
 	_, err = os.Stat(netclientPath + "netclient")
 	if os.IsNotExist(err) {
 		err = InstallNetclient()
 		if err != nil {
-			return false, err
+			return err
 		}
 	}
 	err = os.Chmod(netclientPath+"netclient", 0755)
 	if err != nil {
-		log.Println("could not change netclient directory permissions")
+		log.Println("could not change netclient binary permissions")
+		return err
+	}
+	return nil
+}
+
+func HandleContainedClient() error {
+	log.SetFlags(log.Flags() &^ (log.Llongfile | log.Lshortfile))
+
+	netclientPath := ncutils.GetNetclientPathSpecific()
+	checkinCMD := exec.Command(netclientPath+"netclient", "checkin", "-n", "all")
+	if servercfg.GetVerbose() >= 2 {
+		checkinCMD.Stdout = os.Stdout
+	}
+	checkinCMD.Stderr = os.Stderr
+	err := checkinCMD.Start()
+	if err != nil {
+		if servercfg.GetVerbose() >= 2 {
+			log.Println(err)
+		}
+	}
+	err = checkinCMD.Wait()
+	if err != nil {
+		if servercfg.GetVerbose() >= 2 {
+			log.Println(err)
+		}
+	}
+	if servercfg.GetVerbose() >= 3 {
+		log.Println("[server netclient]", "completed a checkin call")
+	}
+	return nil
+}
+
+func AddNetwork(network string) (bool, error) {
+	pubip, err := servercfg.GetPublicIP()
+	if err != nil {
+		log.Println("could not get public IP.")
 		return false, err
 	}
-	functions.PrintUserLog(models.NODE_SERVER_NAME, "executing network join: "+netclientPath+"netclient "+"join "+"-t "+token+" -name "+models.NODE_SERVER_NAME+" -endpoint "+pubip, 0)
+	netclientPath := ncutils.GetNetclientPathSpecific()
 
-	joinCMD := exec.Command(netclientPath+"netclient", "join", "-t", token, "-name", models.NODE_SERVER_NAME, "-endpoint", pubip)
+	token, err := functions.CreateServerToken(network)
+	if err != nil {
+		log.Println("could not create server token for " + network)
+		return false, err
+	}
+
+	functions.PrintUserLog(models.NODE_SERVER_NAME, "executing network join: "+netclientPath+"netclient "+"join "+"-t "+token+" -name "+models.NODE_SERVER_NAME+" -endpoint "+pubip, 0)
+	var joinCMD *exec.Cmd
+	if servercfg.IsClientMode() == "contained" {
+		joinCMD = exec.Command(netclientPath+"netclient", "join", "-t", token, "-name", models.NODE_SERVER_NAME, "-endpoint", pubip, "-daemon", "off", "-dnson", "no")
+	} else {
+		joinCMD = exec.Command(netclientPath+"netclient", "join", "-t", token, "-name", models.NODE_SERVER_NAME, "-endpoint", pubip)
+	}
 	joinCMD.Stdout = os.Stdout
 	joinCMD.Stderr = os.Stderr
 	err = joinCMD.Start()