resolve merge conflicts

.github/workflows/branchtest.yml

@@ -2,6 +2,11 @@ name: Deploy and Test Branch
 
 on:
   workflow_dispatch:
+    inputs:
+      branches:
+        description: 'Branch to deploy and test'
+        required: true
+        default: 'develop'
   pull_request:
       types: [opened, synchronize, reopened]
       branches: [develop]
@@ -28,7 +33,7 @@ jobs:
         uses: actions/checkout@v4
         with:
           repository: gravitl/netclient
-          ref: develop
+          ref: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || 'develop' }}
       - name: check if branch exists
         id: getbranch 
         run: |
@@ -45,6 +50,6 @@ jobs:
     needs: [getbranch, skip-check]
     with:
       netclientbranch: ${{ needs.getbranch.outputs.netclientbranch }}
-      netmakerbranch: ${{ github.head_ref }}
+      netmakerbranch: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || github.head_ref }}
       tag: ${{ github.run_id }}-${{ github.run_attempt }}
     secrets: inherit          

.github/workflows/deletedroplets.yml

@@ -37,13 +37,28 @@ jobs:
       - name: delete droplets
         if: success() || failure()
         run: |
-          sleep 15m
-          curl -X DELETE \
+          sleep 1m
+          response=$(curl -X DELETE \
             -H "Content-Type: application/json" \
             -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
-            "https://api.digitalocean.com/v2/droplets?tag_name=$TAG"
+            -w "\n%{http_code}" \
+            "https://api.digitalocean.com/v2/droplets?tag_name=$TAG")
+          
+          status_code=$(echo "$response" | tail -n1)
+          body=$(echo "$response" | sed '$d')
+          
+          echo "Response body: $body"
+          echo "Status code: $status_code"
+          
+          if [ "$status_code" -eq 204 ]; then
+            echo "Droplets deleted successfully"
+          else
+            echo "Failed to delete droplets. Status code: $status_code"
+            exit 1
+          fi
+          sleep 1m
         env:
-          DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }}
+          DIGITALOCEAN_TOKEN: ${{ secrets.DO_TEST_TOKEN }}
           TAG: ${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
       - name: mark server as available
         if: success() || failure()
@@ -94,13 +109,28 @@ jobs:
       - name: delete droplets
         if: success() || failure()
         run: |
-          sleep 3h
-          curl -X DELETE \
+          sleep 1m
+          response=$(curl -X DELETE \
             -H "Content-Type: application/json" \
             -H "Authorization: Bearer $DIGITALOCEAN_TOKEN" \
-            "https://api.digitalocean.com/v2/droplets?tag_name=$TAG"
+            -w "\n%{http_code}" \
+            "https://api.digitalocean.com/v2/droplets?tag_name=$TAG")
+          
+          status_code=$(echo "$response" | tail -n1)
+          body=$(echo "$response" | sed '$d')
+          
+          echo "Response body: $body"
+          echo "Status code: $status_code"
+          
+          if [ "$status_code" -eq 204 ]; then
+            echo "Droplets deleted successfully"
+          else
+            echo "Failed to delete droplets. Status code: $status_code"
+            exit 1
+          fi
+          sleep 1m
         env:
-          DIGITALOCEAN_TOKEN: ${{ secrets.DIGITALOCEAN_TOKEN }}
+          DIGITALOCEAN_TOKEN: ${{ secrets.DO_TEST_TOKEN }}
           TAG: ${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }}
       - name: mark server as available
         if: success() || failure()

controllers/user.go

@@ -23,6 +23,8 @@ var (
 	upgrader = websocket.Upgrader{}
 )
 
+var ListRoles = listRoles
+
 func userHandlers(r *mux.Router) {
 	r.HandleFunc("/api/users/adm/hassuperadmin", hasSuperAdmin).Methods(http.MethodGet)
 	r.HandleFunc("/api/users/adm/createsuperadmin", createSuperAdmin).Methods(http.MethodPost)
@@ -35,6 +37,7 @@ func userHandlers(r *mux.Router) {
 	r.HandleFunc("/api/users/{username}", logic.SecurityCheck(false, logic.ContinueIfUserMatch(http.HandlerFunc(getUser)))).Methods(http.MethodGet)
 	r.HandleFunc("/api/v1/users", logic.SecurityCheck(false, logic.ContinueIfUserMatch(http.HandlerFunc(getUserV1)))).Methods(http.MethodGet)
 	r.HandleFunc("/api/users", logic.SecurityCheck(true, http.HandlerFunc(getUsers))).Methods(http.MethodGet)
+	r.HandleFunc("/api/v1/users/roles", logic.SecurityCheck(true, http.HandlerFunc(ListRoles))).Methods(http.MethodGet)
 
 }
 
@@ -407,6 +410,9 @@ func createUser(w http.ResponseWriter, r *http.Request) {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(err, "badrequest"))
 		return
 	}
+	if !servercfg.IsPro {
+		user.PlatformRoleID = models.AdminRole
+	}
 
 	if user.PlatformRoleID == "" {
 		logic.ReturnErrorResponse(w, r, logic.FormatError(errors.New("platform role is missing"), "badrequest"))
@@ -710,3 +716,24 @@ func socketHandler(w http.ResponseWriter, r *http.Request) {
 	// Start handling the session
 	go auth.SessionHandler(conn)
 }
+
+// @Summary     lists all user roles.
+// @Router      /api/v1/user/roles [get]
+// @Tags        Users
+// @Param       role_id param string true "roleid required to get the role details"
+// @Success     200 {object}  []models.UserRolePermissionTemplate
+// @Failure     500 {object} models.ErrorResponse
+func listRoles(w http.ResponseWriter, r *http.Request) {
+	var roles []models.UserRolePermissionTemplate
+	var err error
+	roles, err = logic.ListPlatformRoles()
+	if err != nil {
+		logic.ReturnErrorResponse(w, r, models.ErrorResponse{
+			Code:    http.StatusInternalServerError,
+			Message: err.Error(),
+		})
+		return
+	}
+
+	logic.ReturnSuccessResponseWithJson(w, r, roles, "successfully fetched user roles permission templates")
+}

go.mod

@@ -38,11 +38,9 @@ require (
 )
 
 require (
-	github.com/go-jose/go-jose/v3 v3.0.3
 	github.com/guumaster/tablewriter v0.0.10
 	github.com/matryer/is v1.4.1
 	github.com/olekukonko/tablewriter v0.0.5
-	github.com/resendlabs/resend-go v1.7.0
 	github.com/spf13/cobra v1.8.1
 	gopkg.in/mail.v2 v2.3.1
 )
@@ -50,6 +48,7 @@ require (
 require (
 	cloud.google.com/go/compute/metadata v0.3.0 // indirect
 	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/go-jose/go-jose/v3 v3.0.3 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/rivo/uniseg v0.2.0 // indirect
 	github.com/seancfoley/bintree v1.3.1 // indirect

go.sum

@@ -14,8 +14,6 @@ github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
 github.com/eclipse/paho.mqtt.golang v1.4.3 h1:2kwcUGn8seMUfWndX0hGbvH8r7crgcJguQNCyp70xik=
 github.com/eclipse/paho.mqtt.golang v1.4.3/go.mod h1:CSYvoAlsMkhYOXh/oKyxa8EcBci6dVkLCbo5tTC1RIE=
-github.com/eclipse/paho.mqtt.golang v1.5.0 h1:EH+bUVJNgttidWFkLLVKaQPGmkTUfQQqjOsyvMGvD6o=
-github.com/eclipse/paho.mqtt.golang v1.5.0/go.mod h1:du/2qNQVqJf/Sqs4MEL77kR8QTqANF7XU7Fk0aOTAgk=
 github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk=
 github.com/felixge/httpsnoop v1.0.3/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U=
 github.com/gabriel-vasile/mimetype v1.4.3 h1:in2uUcidCuFcDKtdcBxlR0rJ1+fsokWf+uqxgUFjbI0=
@@ -65,10 +63,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/posthog/posthog-go v0.0.0-20211028072449-93c17c49e2b0 h1:Y2hUrkfuM0on62KZOci/VLijlkdF/yeWU262BQgvcjE=
 github.com/posthog/posthog-go v0.0.0-20211028072449-93c17c49e2b0/go.mod h1:oa2sAs9tGai3VldabTV0eWejt/O4/OOD7azP8GaikqU=
-github.com/posthog/posthog-go v1.2.18 h1:2CBA0LOB0up+gon+xpeXuhFw69gZpjAYxQoBBGwiDWw=
-github.com/posthog/posthog-go v1.2.18/go.mod h1:QjlpryJtfYLrZF2GUkAhejH4E7WlDbdKkvOi5hLmkdg=
-github.com/resendlabs/resend-go v1.7.0 h1:DycOqSXtw2q7aB+Nt9DDJUDtaYcrNPGn1t5RFposas0=
-github.com/resendlabs/resend-go v1.7.0/go.mod h1:yip1STH7Bqfm4fD0So5HgyNbt5taG5Cplc4xXxETyLI=
 github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
 github.com/rivo/uniseg v0.2.0 h1:S1pD9weZBuJdFmowNwbpi7BJ8TNftyUImj/0WQi72jY=
 github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
@@ -102,8 +96,6 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/crypto v0.19.0/go.mod h1:Iy9bg/ha4yyC70EfRS8jz+B6ybOBKMaSxLj6P6oBDfU=
 golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
 golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
-golang.org/x/crypto v0.25.0 h1:ypSNr+bnYL2YhwoMt2zPxHFmbAN1KZs/njMG3hxUp30=
-golang.org/x/crypto v0.25.0/go.mod h1:T+wALwcMOSE0kXgUAnPAHqTLW+XHgcELELW8VaDgm/M=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1 h1:k/i9J1pBpvlfR+9QsetwPyERsqu1GIbi967PQMq3Ivc=
 golang.org/x/exp v0.0.0-20230522175609-2e198f4a06a1/go.mod h1:V1LtkGg67GoY2N1AnLN78QLrzxkLyJw7RJb1gzOOz9w=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
@@ -115,8 +107,6 @@ golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs=
 golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg=
 golang.org/x/net v0.23.0 h1:7EYJ93RZ9vYSZAIb2x3lnuvqO5zneoD6IvWjuhfxjTs=
 golang.org/x/net v0.23.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg=
-golang.org/x/net v0.27.0 h1:5K3Njcw06/l2y9vpGCSdcxWOYHOUk3dVNGDXN+FvAys=
-golang.org/x/net v0.27.0/go.mod h1:dDi0PyhWNoiUOrAS8uXv/vnScO4wnHQO4mj9fn/RytE=
 golang.org/x/oauth2 v0.21.0 h1:tsimM75w1tF/uws5rbeHzIWxEqElMehnc+iW793zsZs=
 golang.org/x/oauth2 v0.21.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -134,8 +124,6 @@ golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws=
 golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.22.0 h1:RI27ohtqKCnwULzJLqkv897zojh5/DwS/ENaMzUOaWI=
-golang.org/x/sys v0.22.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=

logic/auth.go

@@ -278,6 +278,9 @@ func UpdateUser(userchange, user *models.User) (*models.User, error) {
 		user.UserName = userchange.UserName
 	}
 	if userchange.Password != "" {
+		if len(userchange.Password) < 5 {
+			return &models.User{}, errors.New("password requires min 5 characters")
+		}
 		// encrypt that password so we never see it again
 		hash, err := bcrypt.GenerateFromPassword([]byte(userchange.Password), 5)
 
@@ -306,7 +309,6 @@ func UpdateUser(userchange, user *models.User) (*models.User, error) {
 	if err != nil {
 		return &models.User{}, err
 	}
-
 	if err = database.DeleteRecord(database.USERS_TABLE_NAME, queryUser); err != nil {
 		return &models.User{}, err
 	}

logic/networks.go

@@ -42,19 +42,35 @@ func SetAllocatedIpMap() error {
 		pMap := map[string]net.IP{}
 		netName := v.NetID
 
+		//nodes
 		nodes, err := GetNetworkNodes(netName)
 		if err != nil {
 			slog.Error("could not load node for network", netName, "error", err.Error())
-			continue
-		}
-
-		for _, n := range nodes {
+		} else {
+			for _, n := range nodes {
 
-			if n.Address.IP != nil {
-				pMap[n.Address.IP.String()] = n.Address.IP
+				if n.Address.IP != nil {
+					pMap[n.Address.IP.String()] = n.Address.IP
+				}
+				if n.Address6.IP != nil {
+					pMap[n.Address6.IP.String()] = n.Address6.IP
+				}
 			}
-			if n.Address6.IP != nil {
-				pMap[n.Address6.IP.String()] = n.Address6.IP
+
+		}
+
+		//extClients
+		extClients, err := GetNetworkExtClients(netName)
+		if err != nil {
+			slog.Error("could not load extClient for network", netName, "error", err.Error())
+		} else {
+			for _, extClient := range extClients {
+				if extClient.Address != "" {
+					pMap[extClient.Address] = net.ParseIP(extClient.Address)
+				}
+				if extClient.Address6 != "" {
+					pMap[extClient.Address6] = net.ParseIP(extClient.Address6)
+				}
 			}
 		}
 

logic/security.go

@@ -6,7 +6,6 @@ import (
 	"strings"
 
 	"github.com/gorilla/mux"
-	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/servercfg"
 )
@@ -27,12 +26,10 @@ func SecurityCheck(reqAdmin bool, next http.Handler) http.HandlerFunc {
 
 	return func(w http.ResponseWriter, r *http.Request) {
 		r.Header.Set("ismaster", "no")
-		logger.Log(0, "next", r.URL.String())
 		isGlobalAccesss := r.Header.Get("IS_GLOBAL_ACCESS") == "yes"
 		bearerToken := r.Header.Get("Authorization")
 		username, err := GetUserNameFromToken(bearerToken)
 		if err != nil {
-			logger.Log(0, "next 1", r.URL.String(), err.Error())
 			ReturnErrorResponse(w, r, FormatError(err, "unauthorized"))
 			return
 		}
@@ -103,7 +100,6 @@ func ContinueIfUserMatch(next http.Handler) http.HandlerFunc {
 			requestedUser, _ = url.QueryUnescape(r.URL.Query().Get("username"))
 		}
 		if requestedUser != r.Header.Get("user") {
-			logger.Log(0, "next 2", r.URL.String(), errorResponse.Message)
 			ReturnErrorResponse(w, r, errorResponse)
 			return
 		}

logic/user_mgmt.go

@@ -66,6 +66,27 @@ func GetRole(roleID models.UserRoleID) (models.UserRolePermissionTemplate, error
 	return ur, nil
 }
 
+// ListPlatformRoles - lists user platform roles permission templates
+func ListPlatformRoles() ([]models.UserRolePermissionTemplate, error) {
+	data, err := database.FetchRecords(database.USER_PERMISSIONS_TABLE_NAME)
+	if err != nil && !database.IsEmptyRecord(err) {
+		return []models.UserRolePermissionTemplate{}, err
+	}
+	userRoles := []models.UserRolePermissionTemplate{}
+	for _, dataI := range data {
+		userRole := models.UserRolePermissionTemplate{}
+		err := json.Unmarshal([]byte(dataI), &userRole)
+		if err != nil {
+			continue
+		}
+		if userRole.NetworkID != "" {
+			continue
+		}
+		userRoles = append(userRoles, userRole)
+	}
+	return userRoles, nil
+}
+
 func userRolesInit() {
 	d, _ := json.Marshal(SuperAdminPermissionTemplate)
 	database.Insert(SuperAdminPermissionTemplate.ID.String(), string(d), database.USER_PERMISSIONS_TABLE_NAME)

models/user_mgmt.go

@@ -138,16 +138,17 @@ type UserGroup struct {
 
 // User struct - struct for Users
 type User struct {
-	UserName       string                                `json:"username" bson:"username" validate:"min=3,max=40,in_charset|email"`
-	Password       string                                `json:"password" bson:"password" validate:"required,min=5"`
-	IsAdmin        bool                                  `json:"isadmin" bson:"isadmin"` // deprecated
-	IsSuperAdmin   bool                                  `json:"issuperadmin"`           // deprecated
-	RemoteGwIDs    map[string]struct{}                   `json:"remote_gw_ids"`          // deprecated
-	AuthType       AuthType                              `json:"auth_type"`
-	UserGroups     map[UserGroupID]struct{}              `json:"user_group_ids"`
-	PlatformRoleID UserRoleID                            `json:"platform_role_id"`
-	NetworkRoles   map[NetworkID]map[UserRoleID]struct{} `json:"network_roles"`
-	LastLoginTime  time.Time                             `json:"last_login_time"`
+	UserName                   string                                `json:"username" bson:"username" validate:"min=3,max=40,in_charset|email"`
+	ExternalIdentityProviderID string                                `json:"external_identity_provider_id"`
+	Password                   string                                `json:"password" bson:"password" validate:"required,min=5"`
+	IsAdmin                    bool                                  `json:"isadmin" bson:"isadmin"` // deprecated
+	IsSuperAdmin               bool                                  `json:"issuperadmin"`           // deprecated
+	RemoteGwIDs                map[string]struct{}                   `json:"remote_gw_ids"`          // deprecated
+	AuthType                   AuthType                              `json:"auth_type"`
+	UserGroups                 map[UserGroupID]struct{}              `json:"user_group_ids"`
+	PlatformRoleID             UserRoleID                            `json:"platform_role_id"`
+	NetworkRoles               map[NetworkID]map[UserRoleID]struct{} `json:"network_roles"`
+	LastLoginTime              time.Time                             `json:"last_login_time"`
 }
 
 type ReturnUserWithRolesAndGroups struct {

pro/auth/auth.go

@@ -7,6 +7,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/golang-jwt/jwt/v4"
 	"github.com/gorilla/websocket"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
@@ -236,6 +237,17 @@ func getStateAndCode(r *http.Request) (string, string) {
 	return state, code
 }
 
+func getUserEmailFromClaims(token string) string {
+	accessToken, _ := jwt.Parse(token, func(token *jwt.Token) (interface{}, error) {
+		return []byte(""), nil
+	})
+	if accessToken == nil {
+		return ""
+	}
+	claims, _ := accessToken.Claims.(jwt.MapClaims)
+	return claims["email"].(string)
+}
+
 func (user *OAuthUser) getUserName() string {
 	var userName string
 	if user.Email != "" {

pro/auth/azure-ad.go

@@ -3,6 +3,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -33,7 +34,7 @@ func initAzureAD(redirectURL string, clientID string, clientSecret string) {
 		RedirectURL:  redirectURL,
 		ClientID:     clientID,
 		ClientSecret: clientSecret,
-		Scopes:       []string{"User.Read"},
+		Scopes:       []string{"User.Read", "email", "profile", "openid"},
 		Endpoint:     microsoft.AzureADEndpoint(servercfg.GetAzureTenant()),
 	}
 }
@@ -60,27 +61,37 @@ func handleAzureCallback(w http.ResponseWriter, r *http.Request) {
 	var content, err = getAzureUserInfo(rState, rCode)
 	if err != nil {
 		logger.Log(1, "error when getting user info from azure:", err.Error())
-		if strings.Contains(err.Error(), "invalid oauth state") {
+		if strings.Contains(err.Error(), "invalid oauth state") || strings.Contains(err.Error(), "failed to fetch user email from SSO state") {
 			handleOauthNotValid(w)
 			return
 		}
 		handleOauthNotConfigured(w)
 		return
 	}
-
 	var inviteExists bool
 	// check if invite exists for User
-	in, err := logic.GetUserInvite(content.UserPrincipalName)
+	in, err := logic.GetUserInvite(content.Email)
 	if err == nil {
 		inviteExists = true
 	}
 	// check if user approval is already pending
-	if !inviteExists && logic.IsPendingUser(content.UserPrincipalName) {
+	if !inviteExists && logic.IsPendingUser(content.Email) {
 		handleOauthUserSignUpApprovalPending(w)
 		return
 	}
-
-	_, err = logic.GetUser(content.UserPrincipalName)
+	// if user exists with provider ID, convert them into email ID
+	user, err := logic.GetUser(content.UserPrincipalName)
+	if err == nil {
+		_, err := logic.GetUser(content.Email)
+		if err != nil {
+			user.UserName = content.Email
+			user.ExternalIdentityProviderID = content.UserPrincipalName
+			database.DeleteRecord(database.USERS_TABLE_NAME, content.UserPrincipalName)
+			d, _ := json.Marshal(user)
+			database.Insert(user.UserName, string(d), database.USERS_TABLE_NAME)
+		}
+	}
+	_, err = logic.GetUser(content.Email)
 	if err != nil {
 		if database.IsEmptyRecord(err) { // user must not exist, so try to make one
 			if inviteExists {
@@ -90,19 +101,20 @@ func handleAzureCallback(w http.ResponseWriter, r *http.Request) {
 					logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 					return
 				}
+				user.ExternalIdentityProviderID = content.UserPrincipalName
 				if err = logic.CreateUser(&user); err != nil {
 					handleSomethingWentWrong(w)
 					return
 				}
-				logic.DeleteUserInvite(user.UserName)
-				logic.DeletePendingUser(content.UserPrincipalName)
+				logic.DeleteUserInvite(content.Email)
+				logic.DeletePendingUser(content.Email)
 			} else {
-				if !isEmailAllowed(content.UserPrincipalName) {
+				if !isEmailAllowed(content.Email) {
 					handleOauthUserNotAllowedToSignUp(w)
 					return
 				}
 				err = logic.InsertPendingUser(&models.User{
-					UserName: content.UserPrincipalName,
+					UserName: content.Email,
 				})
 				if err != nil {
 					handleSomethingWentWrong(w)
@@ -116,7 +128,7 @@ func handleAzureCallback(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	user, err := logic.GetUser(content.UserPrincipalName)
+	user, err = logic.GetUser(content.Email)
 	if err != nil {
 		handleOauthUserNotFound(w)
 		return
@@ -136,7 +148,7 @@ func handleAzureCallback(w http.ResponseWriter, r *http.Request) {
 	}
 	// send a netmaker jwt token
 	var authRequest = models.UserAuthParams{
-		UserName: content.UserPrincipalName,
+		UserName: content.Email,
 		Password: newPass,
 	}
 
@@ -146,8 +158,8 @@ func handleAzureCallback(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.Log(1, "completed azure OAuth sigin in for", content.UserPrincipalName)
-	http.Redirect(w, r, servercfg.GetFrontendURL()+"/login?login="+jwt+"&user="+content.UserPrincipalName, http.StatusPermanentRedirect)
+	logger.Log(1, "completed azure OAuth sigin in for", content.Email)
+	http.Redirect(w, r, servercfg.GetFrontendURL()+"/login?login="+jwt+"&user="+content.Email, http.StatusPermanentRedirect)
 }
 
 func getAzureUserInfo(state string, code string) (*OAuthUser, error) {
@@ -166,8 +178,9 @@ func getAzureUserInfo(state string, code string) (*OAuthUser, error) {
 	}
 	var httpReq, reqErr = http.NewRequest("GET", "https://graph.microsoft.com/v1.0/me", nil)
 	if reqErr != nil {
-		return nil, fmt.Errorf("failed to create request to GitHub")
+		return nil, fmt.Errorf("failed to create request to microsoft")
 	}
+
 	httpReq.Header.Set("Authorization", "Bearer "+token.AccessToken)
 	response, err := http.DefaultClient.Do(httpReq)
 	if err != nil {
@@ -183,6 +196,13 @@ func getAzureUserInfo(state string, code string) (*OAuthUser, error) {
 		return nil, fmt.Errorf("failed parsing email from response data: %s", err.Error())
 	}
 	userInfo.AccessToken = string(data)
+	if userInfo.Email == "" {
+		userInfo.Email = getUserEmailFromClaims(token.AccessToken)
+	}
+	if userInfo.Email == "" {
+		err = errors.New("failed to fetch user email from SSO state")
+		return userInfo, err
+	}
 	return userInfo, nil
 }
 

pro/auth/error.go

@@ -18,7 +18,7 @@ var htmlBaseTemplate = `<!DOCTYPE html>
 	<script type="text/javascript">
 	function redirect()
     {
-    	window.location.href="` + servercfg.GetFrontendURL() + `";
+    	window.location.href="` + fmt.Sprintf("https://dashboard.%s/login", servercfg.GetNmBaseDomain()) + `";
     }
 	</script>
 	<style>

pro/auth/github.go

@@ -3,6 +3,7 @@ package auth
 import (
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io"
 	"net/http"
@@ -33,7 +34,7 @@ func initGithub(redirectURL string, clientID string, clientSecret string) {
 		RedirectURL:  redirectURL,
 		ClientID:     clientID,
 		ClientSecret: clientSecret,
-		Scopes:       []string{},
+		Scopes:       []string{"read:user", "user:email"},
 		Endpoint:     github.Endpoint,
 	}
 }
@@ -60,26 +61,39 @@ func handleGithubCallback(w http.ResponseWriter, r *http.Request) {
 	var content, err = getGithubUserInfo(rState, rCode)
 	if err != nil {
 		logger.Log(1, "error when getting user info from github:", err.Error())
-		if strings.Contains(err.Error(), "invalid oauth state") {
+		if strings.Contains(err.Error(), "invalid oauth state") || strings.Contains(err.Error(), "failed to fetch user email from SSO state") {
 			handleOauthNotValid(w)
 			return
 		}
 		handleOauthNotConfigured(w)
 		return
 	}
-
 	var inviteExists bool
 	// check if invite exists for User
-	in, err := logic.GetUserInvite(content.Login)
+	in, err := logic.GetUserInvite(content.Email)
 	if err == nil {
 		inviteExists = true
 	}
 	// check if user approval is already pending
-	if !inviteExists && logic.IsPendingUser(content.Login) {
+	if !inviteExists && logic.IsPendingUser(content.Email) {
 		handleOauthUserSignUpApprovalPending(w)
 		return
 	}
-	_, err = logic.GetUser(content.Login)
+	// if user exists with provider ID, convert them into email ID
+	user, err := logic.GetUser(content.Login)
+	if err == nil {
+		// checks if user exists with email
+		_, err := logic.GetUser(content.Email)
+		if err != nil {
+			user.UserName = content.Email
+			user.ExternalIdentityProviderID = content.Login
+			database.DeleteRecord(database.USERS_TABLE_NAME, content.Login)
+			d, _ := json.Marshal(user)
+			database.Insert(user.UserName, string(d), database.USERS_TABLE_NAME)
+		}
+
+	}
+	_, err = logic.GetUser(content.Email)
 	if err != nil {
 		if database.IsEmptyRecord(err) { // user must not exist, so try to make one
 			if inviteExists {
@@ -89,19 +103,20 @@ func handleGithubCallback(w http.ResponseWriter, r *http.Request) {
 					logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 					return
 				}
+				user.ExternalIdentityProviderID = content.Login
 				if err = logic.CreateUser(&user); err != nil {
 					handleSomethingWentWrong(w)
 					return
 				}
-				logic.DeleteUserInvite(user.UserName)
-				logic.DeletePendingUser(content.Login)
+				logic.DeleteUserInvite(content.Email)
+				logic.DeletePendingUser(content.Email)
 			} else {
-				if !isEmailAllowed(content.Login) {
+				if !isEmailAllowed(content.Email) {
 					handleOauthUserNotAllowedToSignUp(w)
 					return
 				}
 				err = logic.InsertPendingUser(&models.User{
-					UserName: content.Login,
+					UserName: content.Email,
 				})
 				if err != nil {
 					handleSomethingWentWrong(w)
@@ -115,7 +130,7 @@ func handleGithubCallback(w http.ResponseWriter, r *http.Request) {
 			return
 		}
 	}
-	user, err := logic.GetUser(content.Login)
+	user, err = logic.GetUser(content.Email)
 	if err != nil {
 		handleOauthUserNotFound(w)
 		return
@@ -135,7 +150,7 @@ func handleGithubCallback(w http.ResponseWriter, r *http.Request) {
 	}
 	// send a netmaker jwt token
 	var authRequest = models.UserAuthParams{
-		UserName: content.Login,
+		UserName: content.Email,
 		Password: newPass,
 	}
 
@@ -145,11 +160,11 @@ func handleGithubCallback(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	logger.Log(1, "completed github OAuth sigin in for", content.Login)
-	http.Redirect(w, r, servercfg.GetFrontendURL()+"/login?login="+jwt+"&user="+content.Login, http.StatusPermanentRedirect)
+	logger.Log(1, "completed github OAuth sigin in for", content.Email)
+	http.Redirect(w, r, servercfg.GetFrontendURL()+"/login?login="+jwt+"&user="+content.Email, http.StatusPermanentRedirect)
 }
 
-func getGithubUserInfo(state string, code string) (*OAuthUser, error) {
+func getGithubUserInfo(state, code string) (*OAuthUser, error) {
 	oauth_state_string, isValid := logic.IsStateValid(state)
 	if (!isValid || state != oauth_state_string) && !isStateCached(state) {
 		return nil, fmt.Errorf("invalid oauth state")
@@ -186,9 +201,55 @@ func getGithubUserInfo(state string, code string) (*OAuthUser, error) {
 		return nil, fmt.Errorf("failed parsing email from response data: %s", err.Error())
 	}
 	userInfo.AccessToken = string(data)
+	if userInfo.Email == "" {
+		// if user's email is not made public, get the info from the github emails api
+		logger.Log(2, "fetching user email from github api")
+		userInfo.Email, err = getGithubEmailsInfo(token.AccessToken)
+		if err != nil {
+			logger.Log(0, "failed to fetch user's email from github: ", err.Error())
+		}
+	}
+	if userInfo.Email == "" {
+		err = errors.New("failed to fetch user email from SSO state")
+		return userInfo, err
+	}
 	return userInfo, nil
 }
 
 func verifyGithubUser(token *oauth2.Token) bool {
 	return token.Valid()
 }
+
+func getGithubEmailsInfo(accessToken string) (string, error) {
+
+	var httpClient = &http.Client{}
+	var httpReq, reqErr = http.NewRequest("GET", "https://api.github.com/user/emails", nil)
+	if reqErr != nil {
+		return "", fmt.Errorf("failed to create request to GitHub")
+	}
+	httpReq.Header.Add("Accept", "application/vnd.github.v3+json")
+	httpReq.Header.Set("Authorization", "token "+accessToken)
+	response, err := httpClient.Do(httpReq)
+	if err != nil {
+		return "", fmt.Errorf("failed getting user info: %s", err.Error())
+	}
+	defer response.Body.Close()
+	contents, err := io.ReadAll(response.Body)
+	if err != nil {
+		return "", fmt.Errorf("failed reading response body: %s", err.Error())
+	}
+
+	emailsInfo := []interface{}{}
+	err = json.Unmarshal(contents, &emailsInfo)
+	if err != nil {
+		return "", err
+	}
+	for _, info := range emailsInfo {
+		emailInfoMap := info.(map[string]interface{})
+		if emailInfoMap["primary"].(bool) {
+			return emailInfoMap["email"].(string), nil
+		}
+
+	}
+	return "", errors.New("email not found")
+}

pro/auth/google.go

@@ -90,7 +90,7 @@ func handleGoogleCallback(w http.ResponseWriter, r *http.Request) {
 					logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 					return
 				}
-
+				user.ExternalIdentityProviderID = content.Email
 				if err = logic.CreateUser(&user); err != nil {
 					handleSomethingWentWrong(w)
 					return

pro/auth/oidc.go

@@ -80,10 +80,9 @@ func handleOIDCCallback(w http.ResponseWriter, r *http.Request) {
 		handleOauthNotConfigured(w)
 		return
 	}
-
 	var inviteExists bool
 	// check if invite exists for User
-	in, err := logic.GetUserInvite(content.Login)
+	in, err := logic.GetUserInvite(content.Email)
 	if err == nil {
 		inviteExists = true
 	}
@@ -102,6 +101,7 @@ func handleOIDCCallback(w http.ResponseWriter, r *http.Request) {
 					logic.ReturnErrorResponse(w, r, logic.FormatError(err, "internal"))
 					return
 				}
+				user.ExternalIdentityProviderID = content.Email
 				if err = logic.CreateUser(&user); err != nil {
 					handleSomethingWentWrong(w)
 					return

pro/controllers/users.go

@@ -30,7 +30,6 @@ func UserHandlers(r *mux.Router) {
 	r.HandleFunc("/api/oauth/register/{regKey}", proAuth.RegisterHostSSO).Methods(http.MethodGet)
 
 	// User Role Handlers
-	r.HandleFunc("/api/v1/users/roles", logic.SecurityCheck(true, http.HandlerFunc(listRoles))).Methods(http.MethodGet)
 	r.HandleFunc("/api/v1/users/role", logic.SecurityCheck(true, http.HandlerFunc(getRole))).Methods(http.MethodGet)
 	r.HandleFunc("/api/v1/users/role", logic.SecurityCheck(true, http.HandlerFunc(createRole))).Methods(http.MethodPost)
 	r.HandleFunc("/api/v1/users/role", logic.SecurityCheck(true, http.HandlerFunc(updateRole))).Methods(http.MethodPut)
@@ -499,12 +498,12 @@ func deleteUserGroup(w http.ResponseWriter, r *http.Request) {
 // @Param       role_id param string true "roleid required to get the role details"
 // @Success     200 {object}  []models.UserRolePermissionTemplate
 // @Failure     500 {object} models.ErrorResponse
-func listRoles(w http.ResponseWriter, r *http.Request) {
+func ListRoles(w http.ResponseWriter, r *http.Request) {
 	platform, _ := url.QueryUnescape(r.URL.Query().Get("platform"))
 	var roles []models.UserRolePermissionTemplate
 	var err error
 	if platform == "true" {
-		roles, err = proLogic.ListPlatformRoles()
+		roles, err = logic.ListPlatformRoles()
 	} else {
 		roles, err = proLogic.ListNetworkRoles()
 	}
@@ -850,7 +849,6 @@ func getUserRemoteAccessGwsV1(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	userGwNodes := proLogic.GetUserRAGNodes(*user)
-	logger.Log(0, fmt.Sprintf("1. User Gw Nodes: %+v", userGwNodes))
 	for _, extClient := range allextClients {
 		node, ok := userGwNodes[extClient.IngressGatewayID]
 		if !ok {
@@ -886,7 +884,6 @@ func getUserRemoteAccessGwsV1(w http.ResponseWriter, r *http.Request) {
 			delete(userGwNodes, node.ID.String())
 		}
 	}
-	logger.Log(0, fmt.Sprintf("2. User Gw Nodes: %+v", userGwNodes))
 	// add remaining gw nodes to resp
 	for gwID := range userGwNodes {
 		node, err := logic.GetNodeByID(gwID)

pro/initialize.go

@@ -34,6 +34,7 @@ func InitPro() {
 		proControllers.FailOverHandlers,
 		proControllers.InetHandlers,
 	)
+	controller.ListRoles = proControllers.ListRoles
 	logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() {
 		// == License Handling ==
 		enableLicenseHook := false

pro/logic/user_mgmt.go

@@ -201,27 +201,6 @@ func ListNetworkRoles() ([]models.UserRolePermissionTemplate, error) {
 	return userRoles, nil
 }
 
-// ListPlatformRoles - lists user platform roles permission templates
-func ListPlatformRoles() ([]models.UserRolePermissionTemplate, error) {
-	data, err := database.FetchRecords(database.USER_PERMISSIONS_TABLE_NAME)
-	if err != nil && !database.IsEmptyRecord(err) {
-		return []models.UserRolePermissionTemplate{}, err
-	}
-	userRoles := []models.UserRolePermissionTemplate{}
-	for _, dataI := range data {
-		userRole := models.UserRolePermissionTemplate{}
-		err := json.Unmarshal([]byte(dataI), &userRole)
-		if err != nil {
-			continue
-		}
-		if userRole.NetworkID != "" {
-			continue
-		}
-		userRoles = append(userRoles, userRole)
-	}
-	return userRoles, nil
-}
-
 func ValidateCreateRoleReq(userRole *models.UserRolePermissionTemplate) error {
 	// check if role exists with this id
 	_, err := logic.GetRole(userRole.ID)

release.md

@@ -1,15 +1,14 @@
 # Netmaker v0.25.0
 
 ## Whats New ✨
-- Validation Checks For Egress Routes
-- Network Change Detection System
-- Removed Creation Of ACLs For EMQX
+- Advanced User Management with Network Roles and Groups
+- User Invitation via Email and Magic Links
 
 ## What's Fixed/Improved 🛠
-- Removed RAG Metadata Length Restriction
+
 - Scalability Improvements
 - Optimised Traffic Flow Over MQ
-- Improved Validation Checks For Internet GWS
+- Improved Peer Updates with Batching
 
 ## Known Issues 🐞
 
@@ -18,5 +17,5 @@
 - IPv6 DNS Entries Are Not Working.
 - Stale Peer On The Interface, When Forced Removed From Multiple Networks At Once.
 - Can Still Ping Domain Name Even When DNS Toggle Is Switched Off.
-- WireGuard DNS issue on most flavors of Ubuntu 24.04 and some other newer Linux distributions. The issue is affecting the Remote Access Client (RAC) and the plain WireGuard external clients. Workaround can be found here https://help.netmaker.io/en/articles/9612016-extclient-rac-dns-issue-on-ubuntu-24-04.
+- WireGuard DNS issue on most flavours of Ubuntu 24.04 and some other newer Linux distributions. The issue is affecting the Remote Access Client (RAC) and the plain WireGuard external clients. Workaround can be found here https://help.netmaker.io/en/articles/9612016-extclient-rac-dns-issue-on-ubuntu-24-04.
 

scripts/nm-quick.sh

@@ -131,7 +131,7 @@ setup_netclient() {
 
 	echo "waiting for netclient to become available"
 	local found=false
-	local file=/etc/netclient/nodes.yml
+	local file=/etc/netclient/nodes.json
 	for ((a = 1; a <= 90; a++)); do
 		if [ -f "$file" ]; then
 			found=true
@@ -149,13 +149,13 @@ setup_netclient() {
 # configure_netclient - configures server's netclient as a default host and an ingress gateway
 configure_netclient() {
 	sleep 2
-	NODE_ID=$(sudo cat /etc/netclient/nodes.yml | yq -r .netmaker.commonnode.id)
+	NODE_ID=$(sudo cat /etc/netclient/nodes.json | jq -r .netmaker.id)
 	if [ "$NODE_ID" = "" ] || [ "$NODE_ID" = "null" ]; then
 		echo "Error obtaining NODE_ID for the new network"
 		exit 1
 	fi
 	echo "register complete. New node ID: $NODE_ID"
-	HOST_ID=$(sudo cat /etc/netclient/netclient.yml | yq -r .host.id)
+	HOST_ID=$(sudo cat /etc/netclient/netclient.json | jq -r .id)
 	if [ "$HOST_ID" = "" ] || [ "$HOST_ID" = "null" ]; then
 		echo "Error obtaining HOST_ID for the new network"
 		exit 1