feat(go): add schema migration facade;

database/database.go

@@ -102,6 +102,35 @@ const (
 
 var dbMutex sync.RWMutex
 
+var Tables = []string{
+	NETWORKS_TABLE_NAME,
+	NODES_TABLE_NAME,
+	CERTS_TABLE_NAME,
+	DELETED_NODES_TABLE_NAME,
+	USERS_TABLE_NAME,
+	DNS_TABLE_NAME,
+	EXT_CLIENT_TABLE_NAME,
+	PEERS_TABLE_NAME,
+	SERVERCONF_TABLE_NAME,
+	SERVER_UUID_TABLE_NAME,
+	GENERATED_TABLE_NAME,
+	NODE_ACLS_TABLE_NAME,
+	SSO_STATE_CACHE,
+	METRICS_TABLE_NAME,
+	NETWORK_USER_TABLE_NAME,
+	USER_GROUPS_TABLE_NAME,
+	CACHE_TABLE_NAME,
+	HOSTS_TABLE_NAME,
+	ENROLLMENT_KEYS_TABLE_NAME,
+	HOST_ACTIONS_TABLE_NAME,
+	PENDING_USERS_TABLE_NAME,
+	USER_PERMISSIONS_TABLE_NAME,
+	USER_INVITES_TABLE_NAME,
+	TAG_TABLE_NAME,
+	ACLS_TABLE_NAME,
+	PEER_ACK_TABLE,
+}
+
 func getCurrentDB() map[string]interface{} {
 	switch servercfg.GetDB() {
 	case "rqlite":
@@ -135,32 +164,9 @@ func InitializeDatabase() error {
 }
 
 func createTables() {
-	CreateTable(NETWORKS_TABLE_NAME)
-	CreateTable(NODES_TABLE_NAME)
-	CreateTable(CERTS_TABLE_NAME)
-	CreateTable(DELETED_NODES_TABLE_NAME)
-	CreateTable(USERS_TABLE_NAME)
-	CreateTable(DNS_TABLE_NAME)
-	CreateTable(EXT_CLIENT_TABLE_NAME)
-	CreateTable(PEERS_TABLE_NAME)
-	CreateTable(SERVERCONF_TABLE_NAME)
-	CreateTable(SERVER_UUID_TABLE_NAME)
-	CreateTable(GENERATED_TABLE_NAME)
-	CreateTable(NODE_ACLS_TABLE_NAME)
-	CreateTable(SSO_STATE_CACHE)
-	CreateTable(METRICS_TABLE_NAME)
-	CreateTable(NETWORK_USER_TABLE_NAME)
-	CreateTable(USER_GROUPS_TABLE_NAME)
-	CreateTable(CACHE_TABLE_NAME)
-	CreateTable(HOSTS_TABLE_NAME)
-	CreateTable(ENROLLMENT_KEYS_TABLE_NAME)
-	CreateTable(HOST_ACTIONS_TABLE_NAME)
-	CreateTable(PENDING_USERS_TABLE_NAME)
-	CreateTable(USER_PERMISSIONS_TABLE_NAME)
-	CreateTable(USER_INVITES_TABLE_NAME)
-	CreateTable(TAG_TABLE_NAME)
-	CreateTable(ACLS_TABLE_NAME)
-	CreateTable(PEER_ACK_TABLE)
+	for _, table := range Tables {
+		_ = CreateTable(table)
+	}
 }
 
 func CreateTable(tableName string) error {

migrate/migrate_schema.go

@@ -0,0 +1,183 @@
+package migrate
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"github.com/gravitl/netmaker/database"
+	"github.com/gravitl/netmaker/db"
+	"github.com/gravitl/netmaker/schema"
+	"github.com/gravitl/netmaker/servercfg"
+	"gorm.io/gorm"
+	"os"
+	"path/filepath"
+)
+
+// ToSQLSchema migrates the data from key-value
+// db to sql db.
+//
+// This function archives the old data and does not
+// delete it.
+//
+// Based on the db server, the archival is done in the
+// following way:
+//
+// 1. Sqlite: Moves the old data to a
+// netmaker_archive.db file.
+//
+// 2. Postgres: Moves the data to a netmaker_archive
+// schema within the same database.
+func ToSQLSchema() error {
+	// initialize sql schema db.
+	err := db.InitializeDB(schema.ListModels()...)
+	if err != nil {
+		return err
+	}
+
+	// migrate, if not done already.
+	err = migrate()
+	if err != nil {
+		return err
+	}
+
+	// archive key-value schema db, if not done already.
+	// ignore errors.
+	_ = archive()
+
+	return nil
+}
+
+func migrate() error {
+	// begin a new transaction.
+	dbctx := db.BeginTx(context.TODO())
+	commit := false
+	defer func() {
+		if commit {
+			db.FromContext(dbctx).Commit()
+		} else {
+			db.FromContext(dbctx).Rollback()
+		}
+	}()
+
+	// check if migrated already.
+	migrationJob := &schema.Job{
+		ID: "migration-v1.0.0",
+	}
+	err := migrationJob.Get(dbctx)
+	if err != nil {
+		if !errors.Is(err, gorm.ErrRecordNotFound) {
+			return err
+		}
+
+		// initialize key-value schema db.
+		err := database.InitializeDatabase()
+		if err != nil {
+			return err
+		}
+		defer database.CloseDB()
+
+		// migrate.
+		// TODO: add migration code.
+
+		// mark migration job completed.
+		err = migrationJob.Create(dbctx)
+		if err != nil {
+			return err
+		}
+
+		commit = true
+	}
+
+	return nil
+}
+
+func archive() error {
+	dbServer := servercfg.GetDB()
+	if dbServer != "sqlite" && dbServer != "postgres" {
+		return nil
+	}
+
+	// begin a new transaction.
+	dbctx := db.BeginTx(context.TODO())
+	commit := false
+	defer func() {
+		if commit {
+			db.FromContext(dbctx).Commit()
+		} else {
+			db.FromContext(dbctx).Rollback()
+		}
+	}()
+
+	// check if key-value schema db archived already.
+	archivalJob := &schema.Job{
+		ID: "archival-v1.0.0",
+	}
+	err := archivalJob.Get(dbctx)
+	if err != nil {
+		if !errors.Is(err, gorm.ErrRecordNotFound) {
+			return err
+		}
+
+		// archive.
+		switch dbServer {
+		case "sqlite":
+			err = sqliteArchiveOldData()
+		default:
+			err = pgArchiveOldData()
+		}
+		if err != nil {
+			return err
+		}
+
+		// mark archival job completed.
+		err = archivalJob.Create(dbctx)
+		if err != nil {
+			return err
+		}
+
+		commit = true
+	} else {
+		// remove the residual
+		if dbServer == "sqlite" {
+			_ = os.Remove(filepath.Join("data", "netmaker.db"))
+		}
+	}
+
+	return nil
+}
+
+func sqliteArchiveOldData() error {
+	oldDBFilePath := filepath.Join("data", "netmaker.db")
+	archiveDBFilePath := filepath.Join("data", "netmaker_archive.db")
+
+	// check if netmaker_archive.db exist.
+	_, err := os.Stat(archiveDBFilePath)
+	if err == nil {
+		return nil
+	} else if !os.IsNotExist(err) {
+		return err
+	}
+
+	// rename old db file to netmaker_archive.db.
+	return os.Rename(oldDBFilePath, archiveDBFilePath)
+}
+
+func pgArchiveOldData() error {
+	_, err := database.PGDB.Exec("CREATE SCHEMA IF NOT EXISTS netmaker_archive")
+	if err != nil {
+		return err
+	}
+
+	for _, table := range database.Tables {
+		_, err := database.PGDB.Exec(
+			fmt.Sprintf(
+				"ALTER TABLE public.%s SET SCHEMA netmaker_archive",
+				table,
+			),
+		)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}