whatcanGOwrong

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/README.md

@@ -0,0 +1,46 @@
+# Google Cloud Spanner
+
+## Usage
+
+See [Google Spanner Documentation](https://cloud.google.com/spanner/docs) for
+more details.
+
+The DSN must be given in the following format.
+
+`spanner://projects/{projectId}/instances/{instanceId}/databases/{databaseName}?param=true`
+
+as described in [README.md#database-urls](../../README.md#database-urls)
+
+| Param | WithInstance Config | Description |
+| ----- | ------------------- | ----------- |
+| `x-migrations-table` | `MigrationsTable` | Name of the migrations table |
+| `x-clean-statements` | `CleanStatements` | Whether to parse and clean DDL statements before running migration towards Spanner (Required for comments and multiple statements) |
+| `url` | `DatabaseName` | The full path to the Spanner database resource. If provided as part of `Config` it must not contain a scheme or query string to match the format `projects/{projectId}/instances/{instanceId}/databases/{databaseName}`|
+| `projectId` || The Google Cloud Platform project id
+| `instanceId` || The id of the instance running Spanner
+| `databaseName` || The name of the Spanner database
+
+> **Note:** Google Cloud Spanner migrations can take a considerable amount of 
+> time. The migrations provided as part of the example take about 6 minutes to 
+> run on a small instance.
+>
+> ```log
+> 1481574547/u create_users_table (21.354507597s)
+> 1496539702/u add_city_to_users (41.647359754s)
+> 1496601752/u add_index_on_user_emails (2m12.155787369s)
+> 1496602638/u create_books_table (2m30.77299181s)
+
+## DDL with comments
+
+At the moment the GCP Spanner backed does not seem to allow for comments (See https://issuetracker.google.com/issues/159730604)
+so in order to be able to use migration with DDL containing comments `x-clean-statements` is required
+
+## Multiple statements
+
+In order to be able to use more than 1 DDL statement in the same migration file, the file has to be parsed and therefore the `x-clean-statements` flag is required
+
+## Testing
+
+To unit test the `spanner` driver, `SPANNER_DATABASE` needs to be set. You'll
+need to sign-up to Google Cloud Platform (GCP) and have a running Spanner
+instance since it is not possible to run Google Spanner outside GCP.

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1481574547_create_users_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE Users

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1481574547_create_users_table.up.sql

@@ -0,0 +1,5 @@
+CREATE TABLE Users (
+  UserId  INT64,
+  Name    STRING(40),
+  Email   STRING(83)
+) PRIMARY KEY(UserId)

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1496539702_add_city_to_users.down.sql

@@ -0,0 +1 @@
+ALTER TABLE Users DROP COLUMN city

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1496539702_add_city_to_users.up.sql

@@ -0,0 +1 @@
+ALTER TABLE Users ADD COLUMN city STRING(100)

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1496601752_add_index_on_user_emails.down.sql

@@ -0,0 +1 @@
+DROP INDEX UsersEmailIndex

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1496601752_add_index_on_user_emails.up.sql

@@ -0,0 +1 @@
+CREATE UNIQUE INDEX UsersEmailIndex ON Users (Email)

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1496602638_create_books_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE Books

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1496602638_create_books_table.up.sql

@@ -0,0 +1,6 @@
+CREATE TABLE Books (
+  UserId INT64,
+  Name    STRING(40),
+  Author  STRING(40)
+) PRIMARY KEY(UserId, Name), 
+INTERLEAVE IN PARENT Users ON DELETE CASCADE

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1621360367_create_transactions_table.down.sql

@@ -0,0 +1 @@
+DROP TABLE Transactions

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/examples/migrations/1621360367_create_transactions_table.up.sql

@@ -0,0 +1,6 @@
+CREATE TABLE Transactions (
+  UserId        INT64,
+  TransactionId STRING(40),
+  Total         NUMERIC
+) PRIMARY KEY(UserId, TransactionId), 
+INTERLEAVE IN PARENT Users ON DELETE CASCADE

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/spanner.go

@@ -0,0 +1,356 @@
+package spanner
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"io"
+	"log"
+	nurl "net/url"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"cloud.google.com/go/spanner"
+	sdb "cloud.google.com/go/spanner/admin/database/apiv1"
+	"cloud.google.com/go/spanner/spansql"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database"
+
+	adminpb "cloud.google.com/go/spanner/admin/database/apiv1/databasepb"
+	"github.com/hashicorp/go-multierror"
+	uatomic "go.uber.org/atomic"
+	"google.golang.org/api/iterator"
+)
+
+func init() {
+	db := Spanner{}
+	database.Register("spanner", &db)
+}
+
+// DefaultMigrationsTable is used if no custom table is specified
+const DefaultMigrationsTable = "SchemaMigrations"
+
+const (
+	unlockedVal = 0
+	lockedVal   = 1
+)
+
+// Driver errors
+var (
+	ErrNilConfig      = errors.New("no config")
+	ErrNoDatabaseName = errors.New("no database name")
+	ErrNoSchema       = errors.New("no schema")
+	ErrDatabaseDirty  = errors.New("database is dirty")
+	ErrLockHeld       = errors.New("unable to obtain lock")
+	ErrLockNotHeld    = errors.New("unable to release already released lock")
+)
+
+// Config used for a Spanner instance
+type Config struct {
+	MigrationsTable string
+	DatabaseName    string
+	// Whether to parse the migration DDL with spansql before
+	// running them towards Spanner.
+	// Parsing outputs clean DDL statements such as reformatted
+	// and void of comments.
+	CleanStatements bool
+}
+
+// Spanner implements database.Driver for Google Cloud Spanner
+type Spanner struct {
+	db *DB
+
+	config *Config
+
+	lock *uatomic.Uint32
+}
+
+type DB struct {
+	admin *sdb.DatabaseAdminClient
+	data  *spanner.Client
+}
+
+func NewDB(admin sdb.DatabaseAdminClient, data spanner.Client) *DB {
+	return &DB{
+		admin: &admin,
+		data:  &data,
+	}
+}
+
+// WithInstance implements database.Driver
+func WithInstance(instance *DB, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	}
+
+	if len(config.DatabaseName) == 0 {
+		return nil, ErrNoDatabaseName
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	sx := &Spanner{
+		db:     instance,
+		config: config,
+		lock:   uatomic.NewUint32(unlockedVal),
+	}
+
+	if err := sx.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+
+	return sx, nil
+}
+
+// Open implements database.Driver
+func (s *Spanner) Open(url string) (database.Driver, error) {
+	purl, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+
+	ctx := context.Background()
+
+	adminClient, err := sdb.NewDatabaseAdminClient(ctx)
+	if err != nil {
+		return nil, err
+	}
+	dbname := strings.Replace(migrate.FilterCustomQuery(purl).String(), "spanner://", "", 1)
+	dataClient, err := spanner.NewClient(ctx, dbname)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	migrationsTable := purl.Query().Get("x-migrations-table")
+
+	cleanQuery := purl.Query().Get("x-clean-statements")
+	clean := false
+	if cleanQuery != "" {
+		clean, err = strconv.ParseBool(cleanQuery)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	db := &DB{admin: adminClient, data: dataClient}
+	return WithInstance(db, &Config{
+		DatabaseName:    dbname,
+		MigrationsTable: migrationsTable,
+		CleanStatements: clean,
+	})
+}
+
+// Close implements database.Driver
+func (s *Spanner) Close() error {
+	s.db.data.Close()
+	return s.db.admin.Close()
+}
+
+// Lock implements database.Driver but doesn't do anything because Spanner only
+// enqueues the UpdateDatabaseDdlRequest.
+func (s *Spanner) Lock() error {
+	if swapped := s.lock.CAS(unlockedVal, lockedVal); swapped {
+		return nil
+	}
+	return ErrLockHeld
+}
+
+// Unlock implements database.Driver but no action required, see Lock.
+func (s *Spanner) Unlock() error {
+	if swapped := s.lock.CAS(lockedVal, unlockedVal); swapped {
+		return nil
+	}
+	return ErrLockNotHeld
+}
+
+// Run implements database.Driver
+func (s *Spanner) Run(migration io.Reader) error {
+	migr, err := io.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+
+	stmts := []string{string(migr)}
+	if s.config.CleanStatements {
+		stmts, err = cleanStatements(migr)
+		if err != nil {
+			return err
+		}
+	}
+
+	ctx := context.Background()
+	op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
+		Database:   s.config.DatabaseName,
+		Statements: stmts,
+	})
+
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}
+	}
+
+	if err := op.Wait(ctx); err != nil {
+		return &database.Error{OrigErr: err, Err: "migration failed", Query: migr}
+	}
+
+	return nil
+}
+
+// SetVersion implements database.Driver
+func (s *Spanner) SetVersion(version int, dirty bool) error {
+	ctx := context.Background()
+
+	_, err := s.db.data.ReadWriteTransaction(ctx,
+		func(ctx context.Context, txn *spanner.ReadWriteTransaction) error {
+			m := []*spanner.Mutation{
+				spanner.Delete(s.config.MigrationsTable, spanner.AllKeys()),
+				spanner.Insert(s.config.MigrationsTable,
+					[]string{"Version", "Dirty"},
+					[]interface{}{version, dirty},
+				)}
+			return txn.BufferWrite(m)
+		})
+	if err != nil {
+		return &database.Error{OrigErr: err}
+	}
+
+	return nil
+}
+
+// Version implements database.Driver
+func (s *Spanner) Version() (version int, dirty bool, err error) {
+	ctx := context.Background()
+
+	stmt := spanner.Statement{
+		SQL: `SELECT Version, Dirty FROM ` + s.config.MigrationsTable + ` LIMIT 1`,
+	}
+	iter := s.db.data.Single().Query(ctx, stmt)
+	defer iter.Stop()
+
+	row, err := iter.Next()
+	switch err {
+	case iterator.Done:
+		return database.NilVersion, false, nil
+	case nil:
+		var v int64
+		if err = row.Columns(&v, &dirty); err != nil {
+			return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)}
+		}
+		version = int(v)
+	default:
+		return 0, false, &database.Error{OrigErr: err, Query: []byte(stmt.SQL)}
+	}
+
+	return version, dirty, nil
+}
+
+var nameMatcher = regexp.MustCompile(`(CREATE TABLE\s(\S+)\s)|(CREATE.+INDEX\s(\S+)\s)`)
+
+// Drop implements database.Driver. Retrieves the database schema first and
+// creates statements to drop the indexes and tables accordingly.
+// Note: The drop statements are created in reverse order to how they're
+// provided in the schema. Assuming the schema describes how the database can
+// be "build up", it seems logical to "unbuild" the database simply by going the
+// opposite direction. More testing
+func (s *Spanner) Drop() error {
+	ctx := context.Background()
+	res, err := s.db.admin.GetDatabaseDdl(ctx, &adminpb.GetDatabaseDdlRequest{
+		Database: s.config.DatabaseName,
+	})
+	if err != nil {
+		return &database.Error{OrigErr: err, Err: "drop failed"}
+	}
+	if len(res.Statements) == 0 {
+		return nil
+	}
+
+	stmts := make([]string, 0)
+	for i := len(res.Statements) - 1; i >= 0; i-- {
+		s := res.Statements[i]
+		m := nameMatcher.FindSubmatch([]byte(s))
+
+		if len(m) == 0 {
+			continue
+		} else if tbl := m[2]; len(tbl) > 0 {
+			stmts = append(stmts, fmt.Sprintf(`DROP TABLE %s`, tbl))
+		} else if idx := m[4]; len(idx) > 0 {
+			stmts = append(stmts, fmt.Sprintf(`DROP INDEX %s`, idx))
+		}
+	}
+
+	op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
+		Database:   s.config.DatabaseName,
+		Statements: stmts,
+	})
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))}
+	}
+	if err := op.Wait(ctx); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(strings.Join(stmts, "; "))}
+	}
+
+	return nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Spanner type.
+func (s *Spanner) ensureVersionTable() (err error) {
+	if err = s.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := s.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	ctx := context.Background()
+	tbl := s.config.MigrationsTable
+	iter := s.db.data.Single().Read(ctx, tbl, spanner.AllKeys(), []string{"Version"})
+	if err := iter.Do(func(r *spanner.Row) error { return nil }); err == nil {
+		return nil
+	}
+
+	stmt := fmt.Sprintf(`CREATE TABLE %s (
+    Version INT64 NOT NULL,
+    Dirty    BOOL NOT NULL
+	) PRIMARY KEY(Version)`, tbl)
+
+	op, err := s.db.admin.UpdateDatabaseDdl(ctx, &adminpb.UpdateDatabaseDdlRequest{
+		Database:   s.config.DatabaseName,
+		Statements: []string{stmt},
+	})
+
+	if err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(stmt)}
+	}
+	if err := op.Wait(ctx); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(stmt)}
+	}
+
+	return nil
+}
+
+func cleanStatements(migration []byte) ([]string, error) {
+	// The Spanner GCP backend does not yet support comments for the UpdateDatabaseDdl RPC
+	// (see https://issuetracker.google.com/issues/159730604) we use
+	// spansql to parse the DDL and output valid stamements without comments
+	ddl, err := spansql.ParseDDL("", string(migration))
+	if err != nil {
+		return nil, err
+	}
+	stmts := make([]string, 0, len(ddl.List))
+	for _, stmt := range ddl.List {
+		stmts = append(stmts, stmt.SQL())
+	}
+	return stmts, nil
+}

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/spanner/spanner_test.go

@@ -0,0 +1,171 @@
+package spanner
+
+import (
+	"fmt"
+	"os"
+	"testing"
+
+	"github.com/golang-migrate/migrate/v4"
+
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+
+	"cloud.google.com/go/spanner/spannertest"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// withSpannerEmulator is not thread-safe and cannot be used with parallel tests since it sets the emulator
+func withSpannerEmulator(t *testing.T, testFunc func(t *testing.T)) {
+	t.Helper()
+	srv, err := spannertest.NewServer("localhost:0")
+	if err != nil {
+		t.Fatal("Failed to create Spanner emulator:", err)
+	}
+	// This is not thread-safe
+	if err := os.Setenv("SPANNER_EMULATOR_HOST", srv.Addr); err != nil {
+		t.Fatal("Failed to set SPANNER_EMULATOR_HOST env var:", err)
+	}
+	defer srv.Close()
+	testFunc(t)
+
+}
+
+const db = "projects/abc/instances/def/databases/testdb"
+
+func Test(t *testing.T) {
+	withSpannerEmulator(t, func(t *testing.T) {
+		uri := fmt.Sprintf("spanner://%s", db)
+		s := &Spanner{}
+		d, err := s.Open(uri)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.Test(t, d, []byte("CREATE TABLE test (id BOOL) PRIMARY KEY (id)"))
+	})
+}
+
+func TestMigrate(t *testing.T) {
+	withSpannerEmulator(t, func(t *testing.T) {
+		s := &Spanner{}
+		uri := fmt.Sprintf("spanner://%s", db)
+		d, err := s.Open(uri)
+		if err != nil {
+			t.Fatal(err)
+		}
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", uri, d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m)
+	})
+}
+
+func TestCleanStatements(t *testing.T) {
+	testCases := []struct {
+		name           string
+		multiStatement string
+		expected       []string
+	}{
+		{
+			name:           "no statement",
+			multiStatement: "",
+			expected:       []string{},
+		},
+		{
+			name:           "single statement, single line, no semicolon, no comment",
+			multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id)",
+			expected:       []string{"CREATE TABLE table_name (\n  id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
+		},
+		{
+			name: "single statement, multi line, no semicolon, no comment",
+			multiStatement: `CREATE TABLE table_name (
+			id STRING(255) NOT NULL,
+		) PRIMARY KEY (id)`,
+			expected: []string{"CREATE TABLE table_name (\n  id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
+		},
+		{
+			name:           "single statement, single line, with semicolon, no comment",
+			multiStatement: "CREATE TABLE table_name (id STRING(255) NOT NULL) PRIMARY KEY (id);",
+			expected:       []string{"CREATE TABLE table_name (\n  id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
+		},
+		{
+			name: "single statement, multi line, with semicolon, no comment",
+			multiStatement: `CREATE TABLE table_name (
+			id STRING(255) NOT NULL,
+		) PRIMARY KEY (id);`,
+			expected: []string{"CREATE TABLE table_name (\n  id STRING(255) NOT NULL,\n) PRIMARY KEY(id)"},
+		},
+		{
+			name: "multi statement, with trailing semicolon. no comment",
+			// From https://github.com/mattes/migrate/pull/281
+			multiStatement: `CREATE TABLE table_name (
+			id STRING(255) NOT NULL,
+		) PRIMARY KEY(id);
+
+		CREATE INDEX table_name_id_idx ON table_name (id);`,
+			expected: []string{`CREATE TABLE table_name (
+  id STRING(255) NOT NULL,
+) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
+		},
+		{
+			name: "multi statement, no trailing semicolon, no comment",
+			// From https://github.com/mattes/migrate/pull/281
+			multiStatement: `CREATE TABLE table_name (
+			id STRING(255) NOT NULL,
+		) PRIMARY KEY(id);
+
+		CREATE INDEX table_name_id_idx ON table_name (id)`,
+			expected: []string{`CREATE TABLE table_name (
+  id STRING(255) NOT NULL,
+) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
+		},
+		{
+			name: "multi statement, no trailing semicolon, standalone comment",
+			// From https://github.com/mattes/migrate/pull/281
+			multiStatement: `CREATE TABLE table_name (
+			-- standalone comment
+			id STRING(255) NOT NULL,
+		) PRIMARY KEY(id);
+
+		CREATE INDEX table_name_id_idx ON table_name (id)`,
+			expected: []string{`CREATE TABLE table_name (
+  id STRING(255) NOT NULL,
+) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
+		},
+		{
+			name: "multi statement, no trailing semicolon, inline comment",
+			// From https://github.com/mattes/migrate/pull/281
+			multiStatement: `CREATE TABLE table_name (
+			id STRING(255) NOT NULL, -- inline comment
+		) PRIMARY KEY(id);
+
+		CREATE INDEX table_name_id_idx ON table_name (id)`,
+			expected: []string{`CREATE TABLE table_name (
+  id STRING(255) NOT NULL,
+) PRIMARY KEY(id)`, "CREATE INDEX table_name_id_idx ON table_name(id)"},
+		},
+		{
+			name: "alter table with SET OPTIONS",
+			multiStatement: `ALTER TABLE users ALTER COLUMN created
+			SET OPTIONS (allow_commit_timestamp=true);`,
+			expected: []string{"ALTER TABLE users ALTER COLUMN created SET OPTIONS (allow_commit_timestamp = true)"},
+		},
+		{
+			name: "column with NUMERIC type",
+			multiStatement: `CREATE TABLE table_name (
+				id STRING(255) NOT NULL,
+				sum NUMERIC,
+			) PRIMARY KEY (id)`,
+			expected: []string{"CREATE TABLE table_name (\n  id STRING(255) NOT NULL,\n  sum NUMERIC,\n) PRIMARY KEY(id)"},
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			stmts, err := cleanStatements([]byte(tc.multiStatement))
+			require.NoError(t, err, "Error cleaning statements")
+			assert.Equal(t, tc.expected, stmts)
+		})
+	}
+}