GRA-1298: License check changes, free tier limits for saas (#2418)

* set free tier limits through config

* add host limit to config

* check for host limit on free tier

* fix license validation, replace node limit with hosts

* add hosts to telemetry data

* debug init

* validate license every 1hr

* hook manager, api to fetch server usage

* hook manager, server usage api

* encode json server usage api

* update ngork url

* update license validation endpoint

* avoid setting limits on eer

* adding hotfix

* correct users limits env var

* add comments to exported funcs

---------

Co-authored-by: afeiszli <[email protected]>

config/config.go

@@ -83,6 +83,11 @@ type ServerConfig struct {
 	TurnUserName               string    `yaml:"turn_username"`
 	TurnPassword               string    `yaml:"turn_password"`
 	UseTurn                    bool      `yaml:"use_turn"`
+	UsersLimit                 int       `yaml:"user_limit"`
+	ClientsLimit               int       `yaml:"client_limit"`
+	NetworksLimit              int       `yaml:"network_limit"`
+	HostsLimit                 int       `yaml:"host_limit"`
+	DeployedByOperator         bool      `yaml:"deployed_by_operator"`
 }
 
 // ProxyMode - default proxy mode for server

controllers/limits.go

@@ -6,7 +6,6 @@ import (
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logic"
 	"github.com/gravitl/netmaker/models"
-	"github.com/gravitl/netmaker/servercfg"
 )
 
 // limit consts
@@ -23,20 +22,13 @@ func checkFreeTierLimits(limit_choice int, next http.Handler) http.HandlerFunc {
 			Code: http.StatusForbidden, Message: "free tier limits exceeded on networks",
 		}
 
-		if logic.Free_Tier && servercfg.Is_EE { // check that free tier limits not exceeded
+		if logic.Free_Tier { // check that free tier limits not exceeded
 			if limit_choice == networks_l {
 				currentNetworks, err := logic.GetNetworks()
 				if (err != nil && !database.IsEmptyRecord(err)) || len(currentNetworks) >= logic.Networks_Limit {
 					logic.ReturnErrorResponse(w, r, errorResponse)
 					return
 				}
-			} else if limit_choice == node_l {
-				nodes, err := logic.GetAllNodes()
-				if (err != nil && !database.IsEmptyRecord(err)) || len(nodes) >= logic.Node_Limit {
-					errorResponse.Message = "free tier limits exceeded on nodes"
-					logic.ReturnErrorResponse(w, r, errorResponse)
-					return
-				}
 			} else if limit_choice == users_l {
 				users, err := logic.GetUsers()
 				if (err != nil && !database.IsEmptyRecord(err)) || len(users) >= logic.Users_Limit {

controllers/server.go

@@ -22,6 +22,38 @@ func serverHandlers(r *mux.Router) {
 	r.HandleFunc("/api/server/getconfig", allowUsers(http.HandlerFunc(getConfig))).Methods(http.MethodGet)
 	r.HandleFunc("/api/server/getserverinfo", Authorize(true, false, "node", http.HandlerFunc(getServerInfo))).Methods(http.MethodGet)
 	r.HandleFunc("/api/server/status", http.HandlerFunc(getStatus)).Methods(http.MethodGet)
+	r.HandleFunc("/api/server/usage", Authorize(true, false, "user", http.HandlerFunc(getUsage))).Methods(http.MethodGet)
+}
+func getUsage(w http.ResponseWriter, r *http.Request) {
+	type usage struct {
+		Hosts    int `json:"hosts"`
+		Clients  int `json:"clients"`
+		Networks int `json:"networks"`
+		Users    int `json:"users"`
+	}
+	var serverUsage usage
+	hosts, err := logic.GetAllHosts()
+	if err == nil {
+		serverUsage.Hosts = len(hosts)
+	}
+	clients, err := logic.GetAllExtClients()
+	if err == nil {
+		serverUsage.Clients = len(clients)
+	}
+	users, err := logic.GetUsers()
+	if err == nil {
+		serverUsage.Users = len(users)
+	}
+	networks, err := logic.GetNetworks()
+	if err == nil {
+		serverUsage.Networks = len(networks)
+	}
+	w.Header().Set("Content-Type", "application/json")
+	json.NewEncoder(w).Encode(models.SuccessResponse{
+		Code:     http.StatusOK,
+		Response: serverUsage,
+	})
+
 }
 
 // swagger:route GET /api/server/status server getStatus
@@ -41,6 +73,12 @@ func getStatus(w http.ResponseWriter, r *http.Request) {
 	type status struct {
 		DB     bool `json:"db_connected"`
 		Broker bool `json:"broker_connected"`
+		Usage  struct {
+			Hosts    int `json:"hosts"`
+			Clients  int `json:"clients"`
+			Networks int `json:"networks"`
+			Users    int `json:"users"`
+		} `json:"usage"`
 	}
 
 	currentServerStatus := status{

ee/initialize.go

@@ -16,6 +16,7 @@ import (
 // InitEE - Initialize EE Logic
 func InitEE() {
 	setIsEnterprise()
+	servercfg.Is_EE = true
 	models.SetLogo(retrieveEELogo())
 	controller.HttpHandlers = append(
 		controller.HttpHandlers,
@@ -27,13 +28,8 @@ func InitEE() {
 	logic.EnterpriseCheckFuncs = append(logic.EnterpriseCheckFuncs, func() {
 		// == License Handling ==
 		ValidateLicense()
-		if Limits.FreeTier {
-			logger.Log(0, "proceeding with Free Tier license")
-			logic.SetFreeTierForTelemetry(true)
-		} else {
-			logger.Log(0, "proceeding with Paid Tier license")
-			logic.SetFreeTierForTelemetry(false)
-		}
+		logger.Log(0, "proceeding with Paid Tier license")
+		logic.SetFreeTierForTelemetry(false)
 		// == End License Handling ==
 		AddLicenseHooks()
 		resetFailover()
@@ -46,17 +42,6 @@ func InitEE() {
 	logic.AllowClientNodeAccess = eelogic.RemoveDeniedNodeFromClient
 }
 
-func setControllerLimits() {
-	logic.Node_Limit = Limits.Nodes
-	logic.Users_Limit = Limits.Users
-	logic.Clients_Limit = Limits.Clients
-	logic.Free_Tier = Limits.FreeTier
-	servercfg.Is_EE = true
-	if logic.Free_Tier {
-		logic.Networks_Limit = 3
-	}
-}
-
 func resetFailover() {
 	nets, err := logic.GetNetworks()
 	if err == nil {

ee/license.go

@@ -9,12 +9,13 @@ import (
 	"encoding/json"
 	"fmt"
 	"io"
-	"math"
 	"net/http"
+	"time"
 
 	"github.com/gravitl/netmaker/database"
 	"github.com/gravitl/netmaker/logger"
 	"github.com/gravitl/netmaker/logic"
+	"github.com/gravitl/netmaker/models"
 	"github.com/gravitl/netmaker/netclient/ncutils"
 	"github.com/gravitl/netmaker/servercfg"
 	"golang.org/x/crypto/nacl/box"
@@ -31,8 +32,14 @@ type apiServerConf struct {
 
 // AddLicenseHooks - adds the validation and cache clear hooks
 func AddLicenseHooks() {
-	logic.AddHook(ValidateLicense)
-	logic.AddHook(ClearLicenseCache)
+	logic.HookManagerCh <- models.HookDetails{
+		Hook:     ValidateLicense,
+		Interval: time.Hour,
+	}
+	logic.HookManagerCh <- models.HookDetails{
+		Hook:     ClearLicenseCache,
+		Interval: time.Hour,
+	}
 }
 
 // ValidateLicense - the initial license check for netmaker server
@@ -58,8 +65,8 @@ func ValidateLicense() error {
 	}
 
 	licenseSecret := LicenseSecret{
-		UserID: netmakerAccountID,
-		Limits: getCurrentServerLimit(),
+		AssociatedID: netmakerAccountID,
+		Limits:       getCurrentServerLimit(),
 	}
 
 	secretData, err := json.Marshal(&licenseSecret)
@@ -92,17 +99,6 @@ func ValidateLicense() error {
 		logger.FatalLog0(errValidation.Error())
 	}
 
-	Limits.Networks = math.MaxInt
-	Limits.FreeTier = license.FreeTier == "yes"
-	Limits.Clients = license.LimitClients
-	Limits.Nodes = license.LimitNodes
-	Limits.Servers = license.LimitServers
-	Limits.Users = license.LimitUsers
-	if Limits.FreeTier {
-		Limits.Networks = 3
-	}
-	setControllerLimits()
-
 	logger.Log(0, "License validation succeeded!")
 	return nil
 }
@@ -167,6 +163,7 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro
 	}
 
 	msg := ValidateLicenseRequest{
+		LicenseKey:     servercfg.GetLicenseKey(),
 		NmServerPubKey: base64encode(publicKeyBytes),
 		EncryptedPart:  base64encode(encryptedData),
 	}
@@ -180,9 +177,6 @@ func validateLicenseKey(encryptedData []byte, publicKey *[32]byte) ([]byte, erro
 	if err != nil {
 		return nil, err
 	}
-	reqParams := req.URL.Query()
-	reqParams.Add("licensevalue", servercfg.GetLicenseKey())
-	req.URL.RawQuery = reqParams.Encode()
 	req.Header.Add("Content-Type", "application/json")
 	req.Header.Add("Accept", "application/json")
 	client := &http.Client{}

ee/types.go

@@ -3,7 +3,7 @@ package ee
 import "fmt"
 
 const (
-	api_endpoint               = "https://api.controller.netmaker.io/api/v1/license/validate"
+	api_endpoint               = "https://api.accounts.netmaker.io/api/v1/license/validate"
 	license_cache_key          = "license_response_cache"
 	license_validation_err_msg = "invalid license"
 	server_id_key              = "nm-server-id"
@@ -11,38 +11,17 @@ const (
 
 var errValidation = fmt.Errorf(license_validation_err_msg)
 
-// Limits - limits to be referenced throughout server
-var Limits = GlobalLimits{
-	Servers:  0,
-	Users:    0,
-	Nodes:    0,
-	Clients:  0,
-	Networks: 0,
-	FreeTier: false,
-}
-
-// GlobalLimits - struct for holding global limits on this netmaker server in memory
-type GlobalLimits struct {
-	Servers  int
-	Users    int
-	Nodes    int
-	Clients  int
-	FreeTier bool
-	Networks int
-}
-
 // LicenseKey - the license key struct representation with associated data
 type LicenseKey struct {
-	LicenseValue   string `json:"license_value"` // actual (public) key and the unique value for the key
-	Expiration     int64  `json:"expiration"`
-	LimitServers   int    `json:"limit_servers"`
-	LimitUsers     int    `json:"limit_users"`
-	LimitNodes     int    `json:"limit_nodes"`
-	LimitClients   int    `json:"limit_clients"`
-	Metadata       string `json:"metadata"`
-	SubscriptionID string `json:"subscription_id"` // for a paid subscription (non-free-tier license)
-	FreeTier       string `json:"free_tier"`       // yes if free tier
-	IsActive       string `json:"is_active"`       // yes if active
+	LicenseValue  string `json:"license_value"` // actual (public) key and the unique value for the key
+	Expiration    int64  `json:"expiration"`
+	LimitServers  int    `json:"limit_servers"`
+	LimitUsers    int    `json:"limit_users"`
+	LimitHosts    int    `json:"limit_hosts"`
+	LimitNetworks int    `json:"limit_networks"`
+	LimitClients  int    `json:"limit_clients"`
+	Metadata      string `json:"metadata"`
+	IsActive      bool   `json:"is_active"` // yes if active
 }
 
 // ValidatedLicense - the validated license struct
@@ -53,28 +32,31 @@ type ValidatedLicense struct {
 
 // LicenseSecret - the encrypted struct for sending user-id
 type LicenseSecret struct {
-	UserID string        `json:"user_id" binding:"required"` // UUID for user foreign key to User table
-	Limits LicenseLimits `json:"limits" binding:"required"`
+	AssociatedID string        `json:"associated_id" binding:"required"` // UUID for user foreign key to User table
+	Limits       LicenseLimits `json:"limits" binding:"required"`
 }
 
 // LicenseLimits - struct license limits
 type LicenseLimits struct {
-	Servers int `json:"servers" binding:"required"`
-	Users   int `json:"users" binding:"required"`
-	Nodes   int `json:"nodes" binding:"required"`
-	Clients int `json:"clients" binding:"required"`
+	Servers  int `json:"servers"`
+	Users    int `json:"users"`
+	Hosts    int `json:"hosts"`
+	Clients  int `json:"clients"`
+	Networks int `json:"networks"`
 }
 
 // LicenseLimits.SetDefaults - sets the default values for limits
 func (l *LicenseLimits) SetDefaults() {
 	l.Clients = 0
 	l.Servers = 1
-	l.Nodes = 0
+	l.Hosts = 0
 	l.Users = 1
+	l.Networks = 0
 }
 
 // ValidateLicenseRequest - used for request to validate license endpoint
 type ValidateLicenseRequest struct {
+	LicenseKey     string `json:"license_key" binding:"required"`
 	NmServerPubKey string `json:"nm_server_pub_key" binding:"required"` // Netmaker server public key used to send data back to Netmaker for the Netmaker server to decrypt (eg output from validating license)
 	EncryptedPart  string `json:"secret" binding:"required"`
 }

ee/util.go

@@ -30,12 +30,11 @@ func base64decode(input string) []byte {
 
 	return bytes
 }
-
 func getCurrentServerLimit() (limits LicenseLimits) {
 	limits.SetDefaults()
-	nodes, err := logic.GetAllNodes()
+	hosts, err := logic.GetAllHosts()
 	if err == nil {
-		limits.Nodes = len(nodes)
+		limits.Hosts = len(hosts)
 	}
 	clients, err := logic.GetAllExtClients()
 	if err == nil {
@@ -45,5 +44,9 @@ func getCurrentServerLimit() (limits LicenseLimits) {
 	if err == nil {
 		limits.Users = len(users)
 	}
+	networks, err := logic.GetNetworks()
+	if err == nil {
+		limits.Networks = len(networks)
+	}
 	return
 }

logic/hosts.go

@@ -93,7 +93,14 @@ func GetHost(hostid string) (*models.Host, error) {
 
 // CreateHost - creates a host if not exist
 func CreateHost(h *models.Host) error {
-	_, err := GetHost(h.ID.String())
+	hosts, err := GetAllHosts()
+	if err != nil && !database.IsEmptyRecord(err) {
+		return err
+	}
+	if len(hosts) >= Hosts_Limit {
+		return errors.New("free tier limits exceeded on hosts")
+	}
+	_, err = GetHost(h.ID.String())
 	if (err != nil && !database.IsEmptyRecord(err)) || (err == nil) {
 		return ErrHostExists
 	}

logic/serverconf.go

@@ -4,17 +4,18 @@ import (
 	"encoding/json"
 
 	"github.com/gravitl/netmaker/database"
+	"github.com/gravitl/netmaker/servercfg"
 )
 
 var (
-	// Node_Limit - dummy var for community
-	Node_Limit = 1000000000
 	// Networks_Limit - dummy var for community
 	Networks_Limit = 1000000000
 	// Users_Limit - dummy var for community
 	Users_Limit = 1000000000
 	// Clients_Limit - dummy var for community
 	Clients_Limit = 1000000000
+	// Hosts_Limit - dummy var for community
+	Hosts_Limit = 1000000000
 	// Free_Tier - specifies if free tier
 	Free_Tier = false
 )
@@ -85,3 +86,11 @@ func StoreJWTSecret(privateKey string) error {
 	}
 	return database.Insert("nm-jwt-secret", string(data), database.SERVERCONF_TABLE_NAME)
 }
+
+func SetFreeTierLimits() {
+	Free_Tier = true
+	Users_Limit = servercfg.GetUserLimit()
+	Clients_Limit = servercfg.GetClientLimit()
+	Networks_Limit = servercfg.GetNetworkLimit()
+	Hosts_Limit = servercfg.GetHostLimit()
+}

logic/telemetry.go

@@ -60,6 +60,7 @@ func sendTelemetry() error {
 		Event:      "daily checkin",
 		Properties: posthog.NewProperties().
 			Set("nodes", d.Nodes).
+			Set("hosts", d.Hosts).
 			Set("servers", d.Servers).
 			Set("non-server nodes", d.Count.NonServer).
 			Set("extclients", d.ExtClients).
@@ -84,6 +85,7 @@ func fetchTelemetryData() (telemetryData, error) {
 	data.ExtClients = getDBLength(database.EXT_CLIENT_TABLE_NAME)
 	data.Users = getDBLength(database.USERS_TABLE_NAME)
 	data.Networks = getDBLength(database.NETWORKS_TABLE_NAME)
+	data.Hosts = getDBLength(database.HOSTS_TABLE_NAME)
 	data.Version = servercfg.GetVersion()
 	data.Servers = getServerCount()
 	nodes, err := GetAllNodes()
@@ -167,6 +169,7 @@ func getDBLength(dbname string) int {
 // telemetryData - What data to send to posthog
 type telemetryData struct {
 	Nodes      int
+	Hosts      int
 	ExtClients int
 	Users      int
 	Count      clientCount

logic/timer.go

@@ -1,10 +1,13 @@
 package logic
 
 import (
+	"context"
 	"fmt"
+	"sync"
 	"time"
 
 	"github.com/gravitl/netmaker/logger"
+	"github.com/gravitl/netmaker/models"
 )
 
 // == Constants ==
@@ -12,6 +15,9 @@ import (
 // How long to wait before sending telemetry to server (24 hours)
 const timer_hours_between_runs = 24
 
+// HookManagerCh - channel to add any new hooks
+var HookManagerCh = make(chan models.HookDetails, 2)
+
 // == Public ==
 
 // TimerCheckpoint - Checks if 24 hours has passed since telemetry was last sent. If so, sends telemetry data to posthog
@@ -40,6 +46,36 @@ func AddHook(ifaceToAdd interface{}) {
 	timeHooks = append(timeHooks, ifaceToAdd)
 }
 
+// StartHookManager - listens on `HookManagerCh` to run any hook
+func StartHookManager(ctx context.Context, wg *sync.WaitGroup) {
+	defer wg.Done()
+	for {
+		select {
+		case <-ctx.Done():
+			logger.Log(0, "## Stopping Hook Manager")
+			return
+		case newhook := <-HookManagerCh:
+			wg.Add(1)
+			go addHookWithInterval(ctx, wg, newhook.Hook, newhook.Interval)
+		}
+	}
+}
+
+func addHookWithInterval(ctx context.Context, wg *sync.WaitGroup, hook func() error, interval time.Duration) {
+	defer wg.Done()
+	ticker := time.NewTicker(interval)
+	defer ticker.Stop()
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-ticker.C:
+			hook()
+		}
+	}
+
+}
+
 // == private ==
 
 // timeHooks - functions to run once a day, functions must take no parameters

main.go

@@ -42,6 +42,9 @@ func main() {
 	initialize()                       // initial db and acls
 	setGarbageCollection()
 	setVerbosity()
+	if servercfg.DeployedByOperator() && !servercfg.Is_EE {
+		logic.SetFreeTierLimits()
+	}
 	defer database.CloseDB()
 	ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGTERM, os.Interrupt)
 	defer stop()
@@ -89,7 +92,6 @@ func initialize() { // Client Mode Prereq Check
 	if err != nil {
 		logger.Log(1, "Timer error occurred: ", err.Error())
 	}
-
 	logic.EnterpriseCheck()
 
 	var authProvider = auth.InitializeAuthProvider()
@@ -150,6 +152,9 @@ func startControllers(wg *sync.WaitGroup, ctx context.Context) {
 	// starts the stun server
 	wg.Add(1)
 	go stunserver.Start(wg, ctx)
+
+	wg.Add(1)
+	go logic.StartHookManager(ctx, wg)
 }
 
 // Should we be using a context vice a waitgroup????????????

models/structs.go

@@ -2,6 +2,7 @@ package models
 
 import (
 	"strings"
+	"time"
 
 	jwt "github.com/golang-jwt/jwt/v4"
 	"golang.zx2c4.com/wireguard/wgctrl/wgtypes"
@@ -274,3 +275,18 @@ type StunServer struct {
 	Domain string `json:"domain" yaml:"domain"`
 	Port   int    `json:"port" yaml:"port"`
 }
+
+// HookDetails - struct to hold hook info
+type HookDetails struct {
+	Hook     func() error
+	Interval time.Duration
+}
+
+// LicenseLimits - struct license limits
+type LicenseLimits struct {
+	Servers  int `json:"servers"`
+	Users    int `json:"users"`
+	Hosts    int `json:"hosts"`
+	Clients  int `json:"clients"`
+	Networks int `json:"networks"`
+}

mq/emqx.go

@@ -6,11 +6,14 @@ import (
 	"fmt"
 	"io"
 	"net/http"
+	"strings"
 	"sync"
 
 	"github.com/gravitl/netmaker/servercfg"
 )
 
+const already_exists = "ALREADY_EXISTS"
+
 type (
 	emqxUser struct {
 		UserID   string `json:"user_id"`
@@ -99,7 +102,9 @@ func CreateEmqxUser(username, password string, admin bool) error {
 		if err != nil {
 			return err
 		}
-		return fmt.Errorf("error creating EMQX user %v", string(msg))
+		if !strings.Contains(string(msg), already_exists) {
+			return fmt.Errorf("error creating EMQX user %v", string(msg))
+		}
 	}
 	return nil
 }

servercfg/serverconf.go

@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/gravitl/netmaker/config"
+
 	"github.com/gravitl/netmaker/models"
 )
 
@@ -741,6 +742,58 @@ func IsProxyEnabled() bool {
 	return enabled
 }
 
+// GetNetworkLimit - fetches free tier limits on users
+func GetUserLimit() int {
+	var userslimit int
+	if os.Getenv("USERS_LIMIT") != "" {
+		userslimit, _ = strconv.Atoi(os.Getenv("USERS_LIMIT"))
+	} else {
+		userslimit = config.Config.Server.UsersLimit
+	}
+	return userslimit
+}
+
+// GetNetworkLimit - fetches free tier limits on networks
+func GetNetworkLimit() int {
+	var networkslimit int
+	if os.Getenv("NETWORKS_LIMIT") != "" {
+		networkslimit, _ = strconv.Atoi(os.Getenv("NETWORKS_LIMIT"))
+	} else {
+		networkslimit = config.Config.Server.NetworksLimit
+	}
+	return networkslimit
+}
+
+// GetClientLimit - fetches free tier limits on ext. clients
+func GetClientLimit() int {
+	var clientsLimit int
+	if os.Getenv("CLIENTS_LIMIT") != "" {
+		clientsLimit, _ = strconv.Atoi(os.Getenv("CLIENTS_LIMIT"))
+	} else {
+		clientsLimit = config.Config.Server.ClientsLimit
+	}
+	return clientsLimit
+}
+
+// GetHostLimit - fetches free tier limits on hosts
+func GetHostLimit() int {
+	var hostsLimit int
+	if os.Getenv("HOSTS_LIMIT") != "" {
+		hostsLimit, _ = strconv.Atoi(os.Getenv("HOSTS_LIMIT"))
+	} else {
+		hostsLimit = config.Config.Server.HostsLimit
+	}
+	return hostsLimit
+}
+
+// DeployedByOperator - returns true if the instance is deployed by netmaker operator
+func DeployedByOperator() bool {
+	if os.Getenv("DEPLOYED_BY_OPERATOR") != "" {
+		return os.Getenv("DEPLOYED_BY_OPERATOR") == "true"
+	}
+	return config.Config.Server.DeployedByOperator
+}
+
 // GetDefaultProxyMode - default proxy mode for a server
 func GetDefaultProxyMode() config.ProxyMode {
 	var (