whatcanGOwrong

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/cassandra/README.md

@@ -0,0 +1,45 @@
+# Cassandra / ScyllaDB
+
+* `Drop()` method will not work on Cassandra 2.X because it rely on
+system_schema table which comes with 3.X
+* Other methods should work properly but are **not tested**
+* The Cassandra driver (gocql) does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
+  * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
+  * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
+
+**ScyllaDB**
+
+* No additional configuration is required since it is a drop-in replacement for Cassandra.
+* The `Drop()` method` works for ScyllaDB 5.1
+
+
+## Usage
+`cassandra://host:port/keyspace?param1=value&param2=value2`
+
+
+| URL Query  | Default value | Description |
+|------------|-------------|-----------|
+| `x-migrations-table` | schema_migrations | Name of the migrations table |
+| `x-multi-statement` | false | Enable multiple statements to be ran in a single migration (See note above) |
+| `port` | 9042 | The port to bind to  |
+| `consistency` | ALL | Migration consistency
+| `protocol` |  | Cassandra protocol version (3 or 4)
+| `timeout` | 1 minute | Migration timeout
+| `connect-timeout` | 600ms | Initial connection timeout to the cluster |
+| `username` | nil | Username to use when authenticating. |
+| `password` | nil | Password to use when authenticating. |
+| `sslcert` | | Cert file location. The file must contain PEM encoded data. |
+| `sslkey` | | Key file location. The file must contain PEM encoded data. |
+| `sslrootcert` | | The location of the root certificate file. The file must contain PEM encoded data. |
+| `sslmode` | | Whether or not to use SSL (disable\|require\|verify-ca\|verify-full) |
+| `disable-host-lookup`| false | Disable initial host lookup. |
+
+`timeout` is parsed using [time.ParseDuration(s string)](https://golang.org/pkg/time/#ParseDuration)
+
+
+## Upgrading from v1
+
+1. Write down the current migration version from schema_migrations
+2. `DROP TABLE schema_migrations`
+4. Download and install the latest migrate version.
+5. Force the current migration version with `migrate force <current_version>`.

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/cassandra/cassandra.go

@@ -0,0 +1,352 @@
+package cassandra
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	nurl "net/url"
+	"strconv"
+	"strings"
+	"time"
+
+	"go.uber.org/atomic"
+
+	"github.com/gocql/gocql"
+	"github.com/golang-migrate/migrate/v4/database"
+	"github.com/golang-migrate/migrate/v4/database/multistmt"
+	"github.com/hashicorp/go-multierror"
+)
+
+func init() {
+	db := new(Cassandra)
+	database.Register("cassandra", db)
+}
+
+var (
+	multiStmtDelimiter = []byte(";")
+
+	DefaultMultiStatementMaxSize = 10 * 1 << 20 // 10 MB
+)
+
+var DefaultMigrationsTable = "schema_migrations"
+
+var (
+	ErrNilConfig     = errors.New("no config")
+	ErrNoKeyspace    = errors.New("no keyspace provided")
+	ErrDatabaseDirty = errors.New("database is dirty")
+	ErrClosedSession = errors.New("session is closed")
+)
+
+type Config struct {
+	MigrationsTable       string
+	KeyspaceName          string
+	MultiStatementEnabled bool
+	MultiStatementMaxSize int
+}
+
+type Cassandra struct {
+	session  *gocql.Session
+	isLocked atomic.Bool
+
+	// Open and WithInstance need to guarantee that config is never nil
+	config *Config
+}
+
+func WithInstance(session *gocql.Session, config *Config) (database.Driver, error) {
+	if config == nil {
+		return nil, ErrNilConfig
+	} else if len(config.KeyspaceName) == 0 {
+		return nil, ErrNoKeyspace
+	}
+
+	if session.Closed() {
+		return nil, ErrClosedSession
+	}
+
+	if len(config.MigrationsTable) == 0 {
+		config.MigrationsTable = DefaultMigrationsTable
+	}
+
+	if config.MultiStatementMaxSize <= 0 {
+		config.MultiStatementMaxSize = DefaultMultiStatementMaxSize
+	}
+
+	c := &Cassandra{
+		session: session,
+		config:  config,
+	}
+
+	if err := c.ensureVersionTable(); err != nil {
+		return nil, err
+	}
+
+	return c, nil
+}
+
+func (c *Cassandra) Open(url string) (database.Driver, error) {
+	u, err := nurl.Parse(url)
+	if err != nil {
+		return nil, err
+	}
+
+	// Check for missing mandatory attributes
+	if len(u.Path) == 0 {
+		return nil, ErrNoKeyspace
+	}
+
+	cluster := gocql.NewCluster(u.Host)
+	cluster.Keyspace = strings.TrimPrefix(u.Path, "/")
+	cluster.Consistency = gocql.All
+	cluster.Timeout = 1 * time.Minute
+
+	if len(u.Query().Get("username")) > 0 && len(u.Query().Get("password")) > 0 {
+		authenticator := gocql.PasswordAuthenticator{
+			Username: u.Query().Get("username"),
+			Password: u.Query().Get("password"),
+		}
+		cluster.Authenticator = authenticator
+	}
+
+	// Retrieve query string configuration
+	if len(u.Query().Get("consistency")) > 0 {
+		var consistency gocql.Consistency
+		consistency, err = parseConsistency(u.Query().Get("consistency"))
+		if err != nil {
+			return nil, err
+		}
+
+		cluster.Consistency = consistency
+	}
+	if len(u.Query().Get("protocol")) > 0 {
+		var protoversion int
+		protoversion, err = strconv.Atoi(u.Query().Get("protocol"))
+		if err != nil {
+			return nil, err
+		}
+		cluster.ProtoVersion = protoversion
+	}
+	if len(u.Query().Get("timeout")) > 0 {
+		var timeout time.Duration
+		timeout, err = time.ParseDuration(u.Query().Get("timeout"))
+		if err != nil {
+			return nil, err
+		}
+		cluster.Timeout = timeout
+	}
+	if len(u.Query().Get("connect-timeout")) > 0 {
+		var connectTimeout time.Duration
+		connectTimeout, err = time.ParseDuration(u.Query().Get("connect-timeout"))
+		if err != nil {
+			return nil, err
+		}
+		cluster.ConnectTimeout = connectTimeout
+	}
+
+	if len(u.Query().Get("sslmode")) > 0 {
+		if u.Query().Get("sslmode") != "disable" {
+			sslOpts := &gocql.SslOptions{}
+
+			if len(u.Query().Get("sslrootcert")) > 0 {
+				sslOpts.CaPath = u.Query().Get("sslrootcert")
+			}
+			if len(u.Query().Get("sslcert")) > 0 {
+				sslOpts.CertPath = u.Query().Get("sslcert")
+			}
+			if len(u.Query().Get("sslkey")) > 0 {
+				sslOpts.KeyPath = u.Query().Get("sslkey")
+			}
+
+			if u.Query().Get("sslmode") == "verify-full" {
+				sslOpts.EnableHostVerification = true
+			}
+
+			cluster.SslOpts = sslOpts
+		}
+	}
+
+	if len(u.Query().Get("disable-host-lookup")) > 0 {
+		if flag, err := strconv.ParseBool(u.Query().Get("disable-host-lookup")); err != nil && flag {
+			cluster.DisableInitialHostLookup = true
+		} else if err != nil {
+			return nil, err
+		}
+	}
+
+	session, err := cluster.CreateSession()
+	if err != nil {
+		return nil, err
+	}
+
+	multiStatementMaxSize := DefaultMultiStatementMaxSize
+	if s := u.Query().Get("x-multi-statement-max-size"); len(s) > 0 {
+		multiStatementMaxSize, err = strconv.Atoi(s)
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return WithInstance(session, &Config{
+		KeyspaceName:          strings.TrimPrefix(u.Path, "/"),
+		MigrationsTable:       u.Query().Get("x-migrations-table"),
+		MultiStatementEnabled: u.Query().Get("x-multi-statement") == "true",
+		MultiStatementMaxSize: multiStatementMaxSize,
+	})
+}
+
+func (c *Cassandra) Close() error {
+	c.session.Close()
+	return nil
+}
+
+func (c *Cassandra) Lock() error {
+	if !c.isLocked.CAS(false, true) {
+		return database.ErrLocked
+	}
+	return nil
+}
+
+func (c *Cassandra) Unlock() error {
+	if !c.isLocked.CAS(true, false) {
+		return database.ErrNotLocked
+	}
+	return nil
+}
+
+func (c *Cassandra) Run(migration io.Reader) error {
+	if c.config.MultiStatementEnabled {
+		var err error
+		if e := multistmt.Parse(migration, multiStmtDelimiter, c.config.MultiStatementMaxSize, func(m []byte) bool {
+			tq := strings.TrimSpace(string(m))
+			if tq == "" {
+				return true
+			}
+			if e := c.session.Query(tq).Exec(); e != nil {
+				err = database.Error{OrigErr: e, Err: "migration failed", Query: m}
+				return false
+			}
+			return true
+		}); e != nil {
+			return e
+		}
+		return err
+	}
+
+	migr, err := io.ReadAll(migration)
+	if err != nil {
+		return err
+	}
+	// run migration
+	if err := c.session.Query(string(migr)).Exec(); err != nil {
+		// TODO: cast to Cassandra error and get line number
+		return database.Error{OrigErr: err, Err: "migration failed", Query: migr}
+	}
+	return nil
+}
+
+func (c *Cassandra) SetVersion(version int, dirty bool) error {
+	// DELETE instead of TRUNCATE because AWS Keyspaces does not support it
+	// see: https://docs.aws.amazon.com/keyspaces/latest/devguide/cassandra-apis.html
+	squery := `SELECT version FROM "` + c.config.MigrationsTable + `"`
+	dquery := `DELETE FROM "` + c.config.MigrationsTable + `" WHERE version = ?`
+	iter := c.session.Query(squery).Iter()
+	var previous int
+	for iter.Scan(&previous) {
+		if err := c.session.Query(dquery, previous).Exec(); err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(dquery)}
+		}
+	}
+	if err := iter.Close(); err != nil {
+		return &database.Error{OrigErr: err, Query: []byte(squery)}
+	}
+
+	// Also re-write the schema version for nil dirty versions to prevent
+	// empty schema version for failed down migration on the first migration
+	// See: https://github.com/golang-migrate/migrate/issues/330
+	if version >= 0 || (version == database.NilVersion && dirty) {
+		query := `INSERT INTO "` + c.config.MigrationsTable + `" (version, dirty) VALUES (?, ?)`
+		if err := c.session.Query(query, version, dirty).Exec(); err != nil {
+			return &database.Error{OrigErr: err, Query: []byte(query)}
+		}
+	}
+
+	return nil
+}
+
+// Return current keyspace version
+func (c *Cassandra) Version() (version int, dirty bool, err error) {
+	query := `SELECT version, dirty FROM "` + c.config.MigrationsTable + `" LIMIT 1`
+	err = c.session.Query(query).Scan(&version, &dirty)
+	switch {
+	case err == gocql.ErrNotFound:
+		return database.NilVersion, false, nil
+
+	case err != nil:
+		if _, ok := err.(*gocql.Error); ok {
+			return database.NilVersion, false, nil
+		}
+		return 0, false, &database.Error{OrigErr: err, Query: []byte(query)}
+
+	default:
+		return version, dirty, nil
+	}
+}
+
+func (c *Cassandra) Drop() error {
+	// select all tables in current schema
+	query := fmt.Sprintf(`SELECT table_name from system_schema.tables WHERE keyspace_name='%s'`, c.config.KeyspaceName)
+	iter := c.session.Query(query).Iter()
+	var tableName string
+	for iter.Scan(&tableName) {
+		err := c.session.Query(fmt.Sprintf(`DROP TABLE %s`, tableName)).Exec()
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// ensureVersionTable checks if versions table exists and, if not, creates it.
+// Note that this function locks the database, which deviates from the usual
+// convention of "caller locks" in the Cassandra type.
+func (c *Cassandra) ensureVersionTable() (err error) {
+	if err = c.Lock(); err != nil {
+		return err
+	}
+
+	defer func() {
+		if e := c.Unlock(); e != nil {
+			if err == nil {
+				err = e
+			} else {
+				err = multierror.Append(err, e)
+			}
+		}
+	}()
+
+	err = c.session.Query(fmt.Sprintf("CREATE TABLE IF NOT EXISTS %s (version bigint, dirty boolean, PRIMARY KEY(version))", c.config.MigrationsTable)).Exec()
+	if err != nil {
+		return err
+	}
+	if _, _, err = c.Version(); err != nil {
+		return err
+	}
+	return nil
+}
+
+// ParseConsistency wraps gocql.ParseConsistency
+// to return an error instead of a panicking.
+func parseConsistency(consistencyStr string) (consistency gocql.Consistency, err error) {
+	defer func() {
+		if r := recover(); r != nil {
+			var ok bool
+			err, ok = r.(error)
+			if !ok {
+				err = fmt.Errorf("Failed to parse consistency \"%s\": %v", consistencyStr, r)
+			}
+		}
+	}()
+	consistency = gocql.ParseConsistency(consistencyStr)
+
+	return consistency, nil
+}

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/cassandra/cassandra_test.go

@@ -0,0 +1,122 @@
+package cassandra
+
+import (
+	"context"
+	"fmt"
+	"github.com/golang-migrate/migrate/v4"
+	"strconv"
+	"testing"
+)
+
+import (
+	"github.com/dhui/dktest"
+	"github.com/gocql/gocql"
+)
+
+import (
+	dt "github.com/golang-migrate/migrate/v4/database/testing"
+	"github.com/golang-migrate/migrate/v4/dktesting"
+	_ "github.com/golang-migrate/migrate/v4/source/file"
+)
+
+var (
+	opts = dktest.Options{PortRequired: true, ReadyFunc: isReady}
+	// Supported versions: http://cassandra.apache.org/download/
+	// Although Cassandra 2.x is supported by the Apache Foundation,
+	// the migrate db driver only supports Cassandra 3.x since it uses
+	// the system_schema keyspace.
+	// last ScyllaDB version tested is 5.1.11
+	specs = []dktesting.ContainerSpec{
+		{ImageName: "cassandra:3.0", Options: opts},
+		{ImageName: "cassandra:3.11", Options: opts},
+		{ImageName: "scylladb/scylla:5.1.11", Options: opts},
+	}
+)
+
+func isReady(ctx context.Context, c dktest.ContainerInfo) bool {
+	// Cassandra exposes 5 ports (7000, 7001, 7199, 9042 & 9160)
+	// We only need the port bound to 9042
+	ip, portStr, err := c.Port(9042)
+	if err != nil {
+		return false
+	}
+	port, err := strconv.Atoi(portStr)
+	if err != nil {
+		return false
+	}
+
+	cluster := gocql.NewCluster(ip)
+	cluster.Port = port
+	cluster.Consistency = gocql.All
+	p, err := cluster.CreateSession()
+	if err != nil {
+		return false
+	}
+	defer p.Close()
+	// Create keyspace for tests
+	if err = p.Query("CREATE KEYSPACE testks WITH REPLICATION = {'class': 'SimpleStrategy', 'replication_factor':1}").Exec(); err != nil {
+		return false
+	}
+	return true
+}
+
+func Test(t *testing.T) {
+	t.Run("test", test)
+	t.Run("testMigrate", testMigrate)
+
+	t.Cleanup(func() {
+		for _, spec := range specs {
+			t.Log("Cleaning up ", spec.ImageName)
+			if err := spec.Cleanup(); err != nil {
+				t.Error("Error removing ", spec.ImageName, "error:", err)
+			}
+		}
+	})
+}
+
+func test(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.Port(9042)
+		if err != nil {
+			t.Fatal("Unable to get mapped port:", err)
+		}
+		addr := fmt.Sprintf("cassandra://%v:%v/testks", ip, port)
+		p := &Cassandra{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+		dt.Test(t, d, []byte("SELECT table_name from system_schema.tables"))
+	})
+}
+
+func testMigrate(t *testing.T) {
+	dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) {
+		ip, port, err := c.Port(9042)
+		if err != nil {
+			t.Fatal("Unable to get mapped port:", err)
+		}
+		addr := fmt.Sprintf("cassandra://%v:%v/testks", ip, port)
+		p := &Cassandra{}
+		d, err := p.Open(addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		defer func() {
+			if err := d.Close(); err != nil {
+				t.Error(err)
+			}
+		}()
+
+		m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "testks", d)
+		if err != nil {
+			t.Fatal(err)
+		}
+		dt.TestMigrate(t, m)
+	})
+}

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/cassandra/examples/migrations/1_simple_select.down.sql

@@ -0,0 +1 @@
+SELECT table_name from system_schema.tables

go/pkg/mod/github.com/golang-migrate/migrate/v4@v4.17.1/database/cassandra/examples/migrations/1_simple_select.up.sql

@@ -0,0 +1 @@
+SELECT table_name from system_schema.tables