multi: add kvdb to sql dev migration

Author: ViktorTigerstrom

config_dev.go

@@ -3,6 +3,7 @@
 package terminal
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -87,7 +88,9 @@ func defaultDevConfig() *DevConfig {
 }
 
 // NewStores creates a new stores instance based on the chosen database backend.
-func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
+func NewStores(ctx context.Context, cfg *Config,
+	clock clock.Clock) (*stores, error) {
+
 	var (
 		networkDir = filepath.Join(cfg.LitDir, cfg.Network)
 		stores     = &stores{
@@ -114,7 +117,10 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
 
 		if !cfg.Sqlite.SkipMigrations {
 			err = sqldb.ApplyAllMigrations(
-				sqlStore, migrationstreams.LitdMigrationStreams,
+				sqlStore,
+				migrationstreams.MakeMigrationStreams(
+					ctx, cfg.MacaroonPath, clock,
+				),
 			)
 			if err != nil {
 				return stores, fmt.Errorf("error applying "+
@@ -156,7 +162,10 @@ func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
 
 		if !cfg.Postgres.SkipMigrations {
 			err = sqldb.ApplyAllMigrations(
-				sqlStore, migrationstreams.LitdMigrationStreams,
+				sqlStore,
+				migrationstreams.MakeMigrationStreams(
+					ctx, cfg.MacaroonPath, clock,
+				),
 			)
 			if err != nil {
 				return stores, fmt.Errorf("error applying "+

config_prod.go

@@ -3,6 +3,7 @@
 package terminal
 
 import (
+	"context"
 	"fmt"
 	"path/filepath"
 
@@ -29,7 +30,9 @@ func (c *DevConfig) Validate(_, _ string) error {
 
 // NewStores creates a new instance of the stores struct using the default Bolt
 // backend since in production, this is currently the only backend supported.
-func NewStores(cfg *Config, clock clock.Clock) (*stores, error) {
+func NewStores(_ context.Context, cfg *Config,
+	clock clock.Clock) (*stores, error) {
+
 	networkDir := filepath.Join(cfg.LitDir, cfg.Network)
 
 	stores := &stores{

db/migrations.go

@@ -28,7 +28,8 @@ const (
 // environment.
 //
 // NOTE: This function is not located in the migrationstreams package to avoid
-// cyclic dependencies.
+// cyclic dependencies. This test migration stream does not run the kvdb to sql
+// migration, as we already have separate unit tests which tests the migration.
 func MakeTestMigrationStreams() []sqldb.MigrationStream {
 	migStream := sqldb.MigrationStream{
 		MigrateTableName: pgx.DefaultMigrationsTable,

db/migrationstreams/post_migration_callbacks_dev.go

@@ -0,0 +1,107 @@
+//go:build dev
+
+package migrationstreams
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"path/filepath"
+	"time"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/lightninglabs/lightning-terminal/accounts"
+	"github.com/lightninglabs/lightning-terminal/db/sqlcmig6"
+	"github.com/lightninglabs/lightning-terminal/firewalldb"
+	"github.com/lightninglabs/lightning-terminal/session"
+	"github.com/lightningnetwork/lnd/clock"
+	"github.com/lightningnetwork/lnd/sqldb/v2"
+)
+
+// MakePostStepCallbacksMig6 turns the post migration checks into a map of post
+// step callbacks that can be used with the migrate package. The keys of the map
+// are the migration versions, and the values are the callbacks that will be
+// executed after the migration with the corresponding version is applied.
+func MakePostStepCallbacksMig6(ctx context.Context, db *sqldb.BaseDB,
+	macPath string, clock clock.Clock,
+	migVersion uint) migrate.PostStepCallback {
+
+	mig6queries := sqlcmig6.NewForType(db, db.BackendType)
+	mig6executor := sqldb.NewTransactionExecutor(
+		db, func(tx *sql.Tx) *sqlcmig6.Queries {
+			return mig6queries.WithTx(tx)
+		},
+	)
+
+	return func(_ *migrate.Migration, _ database.Driver) error {
+		// We ignore the actual driver that's being returned here, since
+		// we use migrate.NewWithInstance() to create the migration
+		// instance from our already instantiated database backend that
+		// is also passed into this function.
+		return mig6executor.ExecTx(
+			ctx, sqldb.NewWriteTx(),
+			func(q6 *sqlcmig6.Queries) error {
+				log.Infof("Running post migration callback "+
+					"for migration version %d", migVersion)
+
+				return kvdbToSqlMigrationCallback(
+					ctx, macPath, db, clock, q6,
+				)
+			}, sqldb.NoOpReset,
+		)
+	}
+}
+
+func kvdbToSqlMigrationCallback(ctx context.Context, macPath string,
+	_ *sqldb.BaseDB, clock clock.Clock, q *sqlcmig6.Queries) error {
+
+	start := time.Now()
+	log.Infof("Starting KVDB to SQL migration for all stores")
+
+	accountStore, err := accounts.NewBoltStore(
+		filepath.Dir(macPath), accounts.DBFilename, clock,
+	)
+	if err != nil {
+		return err
+	}
+
+	err = accounts.MigrateAccountStoreToSQL(ctx, accountStore.DB, q)
+	if err != nil {
+		return fmt.Errorf("error migrating account store to "+
+			"SQL: %v", err)
+	}
+
+	sessionStore, err := session.NewDB(
+		filepath.Dir(macPath), session.DBFilename,
+		clock, accountStore,
+	)
+	if err != nil {
+		return err
+	}
+
+	err = session.MigrateSessionStoreToSQL(ctx, sessionStore.DB, q)
+	if err != nil {
+		return fmt.Errorf("error migrating session store to "+
+			"SQL: %v", err)
+	}
+
+	firewallStore, err := firewalldb.NewBoltDB(
+		filepath.Dir(macPath), firewalldb.DBFilename,
+		sessionStore, accountStore, clock,
+	)
+	if err != nil {
+		return err
+	}
+
+	err = firewalldb.MigrateFirewallDBToSQL(ctx, firewallStore.DB, q)
+	if err != nil {
+		return fmt.Errorf("error migrating firewalldb store "+
+			"to SQL: %v", err)
+	}
+
+	log.Infof("Succesfully migrated all KVDB stores to SQL in: %v",
+		time.Since(start))
+
+	return nil
+}

db/migrationstreams/sql_migrations.go

@@ -3,14 +3,18 @@
 package migrationstreams
 
 import (
+	"context"
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database/pgx/v5"
 	"github.com/lightninglabs/lightning-terminal/db"
+	"github.com/lightningnetwork/lnd/clock"
 	"github.com/lightningnetwork/lnd/sqldb/v2"
 )
 
-var (
-	LitdMigrationStream = sqldb.MigrationStream{
+func MakeMigrationStreams(_ context.Context, _ string,
+	_ clock.Clock) []sqldb.MigrationStream {
+
+	migStream := sqldb.MigrationStream{
 		MigrateTableName: pgx.DefaultMigrationsTable,
 		SQLFileDirectory: "sqlc/migrations",
 		Schemas:          db.SqlSchemas,
@@ -29,5 +33,6 @@ var (
 			return make(map[uint]migrate.PostStepCallback), nil
 		},
 	}
-	LitdMigrationStreams = []sqldb.MigrationStream{LitdMigrationStream}
-)
+
+	return []sqldb.MigrationStream{migStream}
+}

db/migrationstreams/sql_migrations_dev.go

@@ -3,15 +3,29 @@
 package migrationstreams
 
 import (
+	"context"
 	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database/pgx/v5"
 	"github.com/lightninglabs/lightning-terminal/db"
+	"github.com/lightningnetwork/lnd/clock"
 	"github.com/lightningnetwork/lnd/sqldb/v2"
 )
 
-var (
+const (
+	// KVDBtoSQLMigVersion is the version of the migration that migrates the
+	// kvdb to the sql database.
+	//
+	// TODO: When this the kvdb to sql migration goes live into prod, this
+	// should be moved to non dev db/migrations.go file, and this constant
+	// value should be updated to reflect the real migration number.
+	KVDBtoSQLMigVersion = 1
+)
+
+func MakeMigrationStreams(ctx context.Context, macPath string,
+	clock clock.Clock) []sqldb.MigrationStream {
+
 	// Create the prod migration stream.
-	migStream = sqldb.MigrationStream{
+	migStream := sqldb.MigrationStream{
 		MigrateTableName: pgx.DefaultMigrationsTable,
 		SQLFileDirectory: "sqlc/migrations",
 		Schemas:          db.SqlSchemas,
@@ -32,7 +46,7 @@ var (
 	}
 
 	// Create the dev migration stream.
-	migStreamDev = sqldb.MigrationStream{
+	migStreamDev := sqldb.MigrationStream{
 		MigrateTableName: pgx.DefaultMigrationsTable + "_dev",
 		SQLFileDirectory: "sqlc/migrations_dev",
 		Schemas:          db.SqlSchemas,
@@ -48,8 +62,23 @@ var (
 			db *sqldb.BaseDB) (map[uint]migrate.PostStepCallback,
 			error) {
 
-			return make(map[uint]migrate.PostStepCallback), nil
+			// Any Callbacks added to this map will be executed when
+			// after the dev migration number for the uint key in
+			// the map has been applied. If no entry exists for a
+			// given uint, then no callback will be executed for
+			// that migration number. This is useful for adding a
+			// code migration step as a callback to be run
+			// after a specific migration of a given number has been
+			// applied.
+			res := make(map[uint]migrate.PostStepCallback)
+
+			res[KVDBtoSQLMigVersion] = MakePostStepCallbacksMig6(
+				ctx, db, macPath, clock, KVDBtoSQLMigVersion,
+			)
+
+			return res, nil
 		},
 	}
-	LitdMigrationStreams = []sqldb.MigrationStream{migStream, migStreamDev}
-)
+
+	return []sqldb.MigrationStream{migStream, migStreamDev}
+}

db/sqlc/migrations_dev/000001_dev_test_migration.down.sql renamed to db/sqlc/migrations_dev/000001_code_migration_kvdb_to_sql.down.sql

@@ -447,7 +447,7 @@ func (g *LightningTerminal) start(ctx context.Context) error {
 		return fmt.Errorf("could not create network directory: %v", err)
 	}
 
-	g.stores, err = NewStores(g.cfg, clock.NewDefaultClock())
+	g.stores, err = NewStores(ctx, g.cfg, clock.NewDefaultClock())
 	if err != nil {
 		return fmt.Errorf("could not create stores: %v", err)
 	}