From 6192940020eb4a1a369fa61481d4a1cc76b74a6f Mon Sep 17 00:00:00 2001 From: Chi Fujii Date: Tue, 5 Mar 2024 12:51:47 +0900 Subject: [PATCH] use sql directly --- cmd/envtool/envtool.go | 160 +++++++++++++++++----------------- cmd/envtool/envtool_test.go | 24 ++--- cmd/envtool/system_users.json | 84 ++++++++++++++++++ cmd/envtool/user.json | 145 ++++++++++++++++++++++++++++++ 4 files changed, 322 insertions(+), 91 deletions(-) create mode 100644 cmd/envtool/system_users.json create mode 100644 cmd/envtool/user.json diff --git a/cmd/envtool/envtool.go b/cmd/envtool/envtool.go index 829bb963ec3a..681aea1304bb 100644 --- a/cmd/envtool/envtool.go +++ b/cmd/envtool/envtool.go @@ -33,19 +33,15 @@ import ( "time" "github.com/alecthomas/kong" + "github.com/jackc/pgerrcode" + "github.com/jackc/pgx/v5/pgconn" "github.com/prometheus/client_golang/prometheus" - "go.mongodb.org/mongo-driver/bson" - "go.mongodb.org/mongo-driver/mongo" - "go.mongodb.org/mongo-driver/mongo/options" "go.uber.org/zap" "go.uber.org/zap/zapcore" "github.com/FerretDB/FerretDB/build/version" mysqlpool "github.com/FerretDB/FerretDB/internal/backends/mysql/metadata/pool" "github.com/FerretDB/FerretDB/internal/backends/postgresql/metadata/pool" - "github.com/FerretDB/FerretDB/internal/clientconn" - "github.com/FerretDB/FerretDB/internal/clientconn/connmetrics" - "github.com/FerretDB/FerretDB/internal/handler/registry" "github.com/FerretDB/FerretDB/internal/util/ctxutil" "github.com/FerretDB/FerretDB/internal/util/debug" "github.com/FerretDB/FerretDB/internal/util/lazyerrors" @@ -58,6 +54,12 @@ var ( //go:embed error.tmpl errorTemplateB []byte + //go:embed user.json + user string + + //go:embed system_users.json + systemUsers string + // Parsed error template. errorTemplate = template.Must(template.New("error").Option("missingkey=error").Parse(string(errorTemplateB))) ) @@ -138,29 +140,42 @@ func setupAnyPostgres(ctx context.Context, logger *zap.SugaredLogger, uri string ctxutil.SleepWithJitter(ctx, time.Second, retry) } - if ctx.Err() != nil { - return ctx.Err() - } + return ctx.Err() +} - dbName := strings.Trim(u.Path, "/") +// setupPostgres configures `postgres` container. +func setupPostgres(ctx context.Context, logger *zap.SugaredLogger) error { + l := logger.Named("postgres") - err = setupUser(ctx, logger, uint16(port), dbName) + // user `username` must exist, but password may be any, even empty + err := setupAnyPostgres(ctx, l, "postgres://username@127.0.0.1:5432/ferretdb") if err != nil { return err } - return setupUser(ctx, logger, uint16(port), "template1") -} + err = setupPostgresUser(ctx, logger, "postgres://username@127.0.0.1:5432/ferretdb") + if err != nil { + return err + } -// setupPostgres configures `postgres` container. -func setupPostgres(ctx context.Context, logger *zap.SugaredLogger) error { - // user `username` must exist, but password may be any, even empty - return setupAnyPostgres(ctx, logger.Named("postgres"), "postgres://username@127.0.0.1:5432/ferretdb") + return setupPostgresUser(ctx, logger, "postgres://username@127.0.0.1:5432/template1") } // setupPostgresSecured configures `postgres_secured` container. func setupPostgresSecured(ctx context.Context, logger *zap.SugaredLogger) error { - return setupAnyPostgres(ctx, logger.Named("postgres_secured"), "postgres://username:password@127.0.0.1:5433/ferretdb") + l := logger.Named("postgres_secured") + + err := setupAnyPostgres(ctx, l, "postgres://username:password@127.0.0.1:5433/ferretdb") + if err != nil { + return err + } + + err = setupPostgresUser(ctx, logger, "postgres://username:password@127.0.0.1:5433/ferretdb") + if err != nil { + return err + } + + return setupPostgresUser(ctx, logger, "postgres://username:password@127.0.0.1:5433/template1") } // setupMySQL configures `mysql` container. @@ -236,105 +251,88 @@ func setupMongodb(ctx context.Context, logger *zap.SugaredLogger) error { return ctx.Err() } -// setupUser creates a user in admin database with supported mechanisms. -// The user uses username/password credential which is the same as the PostgreSQL credentials. +// setupPostgresUser creates a user with username/password in admin database with supported mechanisms. +// The user uses the same credential as the PostgreSQL credentials. +// It creates admin database (PostgreSQL admin schema), if it does not exist. +// It also creates system.users collection, if it does not exist. // // Without this, once the first user is created, the authentication fails // as username/password does not exist in admin.system.users collection. -func setupUser(ctx context.Context, logger *zap.SugaredLogger, postgreSQLPort uint16, dbName string) error { - if err := waitForPort(ctx, logger.Named("postgreSQL"), postgreSQLPort); err != nil { - return err - } - +func setupPostgresUser(ctx context.Context, logger *zap.SugaredLogger, uri string) error { sp, err := state.NewProvider("") if err != nil { return err } - postgreSQLURL := fmt.Sprintf("postgres://username:password@localhost:%d/%s", postgreSQLPort, dbName) - listenerMetrics := connmetrics.NewListenerMetrics() - handlerOpts := ®istry.NewHandlerOpts{ - Logger: logger.Desugar(), - ConnMetrics: listenerMetrics.ConnMetrics, - StateProvider: sp, - PostgreSQLURL: postgreSQLURL, - TestOpts: registry.TestOpts{ - CappedCleanupPercentage: 20, - EnableNewAuth: true, - }, + u, err := url.Parse(uri) + if err != nil { + return err } - h, closeBackend, err := registry.NewHandler("postgresql", handlerOpts) + p, err := pool.New(uri, logger.Desugar(), sp) if err != nil { return err } - defer closeBackend() + defer p.Close() + + username := u.User.Username() + password, _ := u.User.Password() - listenerOpts := clientconn.NewListenerOpts{ - Mode: clientconn.NormalMode, - Metrics: listenerMetrics, - Handler: h, - Logger: logger.Desugar(), - TCP: "127.0.0.1:0", + dbPool, err := p.Get(username, password) + if err != nil { + return err } - l := clientconn.NewListener(&listenerOpts) + defer dbPool.Close() - runErr := make(chan error) + var pgErr *pgconn.PgError - ctx, cancel := context.WithCancel(ctx) - defer cancel() + q := `CREATE SCHEMA admin` + if _, err = dbPool.Exec(ctx, q); err != nil && (!errors.As(err, &pgErr) || pgErr.Code != pgerrcode.DuplicateSchema) { + return err + } - go func() { - if err = l.Run(ctx); err != nil && !errors.Is(err, context.Canceled) { - runErr <- err + if err == nil { + q = `CREATE TABLE admin._ferretdb_database_metadata (_jsonb jsonb)` + if _, err = dbPool.Exec(ctx, q); err != nil { + return err } - }() - defer close(runErr) + q = `CREATE UNIQUE INDEX _ferretdb_database_metadata_id_idx ON admin._ferretdb_database_metadata (((_jsonb->'_id')))` + if _, err = dbPool.Exec(ctx, q); err != nil { + return err + } - select { - case err = <-runErr: - if err != nil { + q = `CREATE UNIQUE INDEX _ferretdb_database_metadata_table_idx ON admin._ferretdb_database_metadata (((_jsonb->'table')))` + if _, err = dbPool.Exec(ctx, q); err != nil { return err } - case <-time.After(time.Millisecond): } - port := l.TCPAddr().(*net.TCPAddr).Port - uri := fmt.Sprintf("mongodb://username:password@localhost:%d/?authMechanism=PLAIN", port) - clientOpts := options.Client().ApplyURI(uri) - - client, err := mongo.Connect(ctx, clientOpts) - if err != nil { + q = `CREATE TABLE admin.system_users_aff2f7ce (_jsonb jsonb)` + if _, err = dbPool.Exec(ctx, q); err != nil && (!errors.As(err, &pgErr) || pgErr.Code != pgerrcode.DuplicateTable) { return err } - defer func() { - err = client.Disconnect(ctx) - }() - - //nolint:forbidigo // allow usage of bson for setup dev and test environment - if err = client.Database("admin").RunCommand(ctx, bson.D{ - bson.E{Key: "createUser", Value: "username"}, - bson.E{Key: "roles", Value: bson.A{}}, - bson.E{Key: "pwd", Value: "password"}, - bson.E{Key: "mechanisms", Value: bson.A{"PLAIN", "SCRAM-SHA-1", "SCRAM-SHA-256"}}, - }).Err(); err != nil { - var cmdErr mongo.CommandError - if errors.As(err, &cmdErr) && cmdErr.Code == 51003 { - return nil + if err == nil { + q = `CREATE UNIQUE INDEX "system_users_aff2f7ce__id__67399184_idx" ON admin.system_users_aff2f7ce (((_jsonb->'_id')))` + if _, err = dbPool.Exec(ctx, q); err != nil { + return err } - return err + q = `INSERT INTO admin._ferretdb_database_metadata (_jsonb) VALUES ($1)` + if _, err = dbPool.Exec(ctx, q, systemUsers); err != nil { + return err + } } - if ctx.Err() == context.Canceled { - return nil + q = `INSERT INTO admin.system_users_aff2f7ce (_jsonb) VALUES ($1)` + if _, err = dbPool.Exec(ctx, q, user); err != nil && (!errors.As(err, &pgErr) || pgErr.Code != pgerrcode.UniqueViolation) { + return err } - return ctx.Err() + return nil } // setupMongodbSecured configures `mongodb_secured` container. diff --git a/cmd/envtool/envtool_test.go b/cmd/envtool/envtool_test.go index 1ce7b70e17d8..60a99d98c981 100644 --- a/cmd/envtool/envtool_test.go +++ b/cmd/envtool/envtool_test.go @@ -99,19 +99,18 @@ func TestPackageVersion(t *testing.T) { assert.Equal(t, "1.0.0", output.String()) } -func TestSetupUser(t *testing.T) { +func TestSetupPostgresUser(t *testing.T) { if testing.Short() { t.Skip("skipping in -short mode") } t.Parallel() - ctx := testutil.Ctx(t) - baseURI := "postgres://username@127.0.0.1:5432/ferretdb" - l := testutil.Logger(t) + baseURI := "postgres://username@127.0.0.1:5432/ferretdb?search_path=" cfg, err := pgxpool.ParseConfig(baseURI) require.NoError(t, err) + l := testutil.Logger(t) cfg.MinConns = 0 cfg.MaxConns = 1 cfg.ConnConfig.Tracer = &tracelog.TraceLog{ @@ -119,28 +118,33 @@ func TestSetupUser(t *testing.T) { LogLevel: tracelog.LogLevelTrace, } + ctx := testutil.Ctx(t) p, err := pgxpool.NewWithConfig(ctx, cfg) require.NoError(t, err) dbName := testutil.DatabaseName(t) + sanitizedName := pgx.Identifier{dbName}.Sanitize() + + _, err = p.Exec(ctx, fmt.Sprintf("DROP DATABASE IF EXISTS %s", sanitizedName)) + require.NoError(t, err) // use template0 because template1 may already have the user created - q := fmt.Sprintf("CREATE DATABASE %s TEMPLATE template0", pgx.Identifier{dbName}.Sanitize()) - _, err = p.Exec(ctx, q) + _, err = p.Exec(ctx, fmt.Sprintf("CREATE DATABASE %s TEMPLATE template0", sanitizedName)) require.NoError(t, err) t.Cleanup(func() { defer p.Close() - q = fmt.Sprintf("DROP DATABASE %s", pgx.Identifier{dbName}.Sanitize()) - _, err = p.Exec(context.Background(), q) + _, err = p.Exec(context.Background(), fmt.Sprintf("DROP DATABASE %s", sanitizedName)) require.NoError(t, err) }) - err = setupUser(ctx, l.Sugar(), 5432, dbName) + uri := fmt.Sprintf("postgres://username@127.0.0.1:5432/%s", dbName) + + err = setupPostgresUser(ctx, l.Sugar(), uri) require.NoError(t, err) // if the user already exists, it should not fail - err = setupUser(ctx, l.Sugar(), 5432, dbName) + err = setupPostgresUser(ctx, l.Sugar(), uri) require.NoError(t, err) } diff --git a/cmd/envtool/system_users.json b/cmd/envtool/system_users.json new file mode 100644 index 000000000000..616364268838 --- /dev/null +++ b/cmd/envtool/system_users.json @@ -0,0 +1,84 @@ +{ + "$s": { + "p": { + "_id": { + "t": "string" + }, + "uuid": { + "t": "string" + }, + "table": { + "t": "string" + }, + "indexes": { + "t": "array", + "i": [ + { + "t": "object", + "$s": { + "p": { + "pgindex": { + "t": "string" + }, + "name": { + "t": "string" + }, + "key": { + "t": "object", + "$s": { + "p": { + "_id": { + "t": "int" + } + }, + "$k": [ + "_id" + ] + } + }, + "unique": { + "t": "bool" + } + }, + "$k": [ + "pgindex", + "name", + "key", + "unique" + ] + } + } + ] + }, + "cappedSize": { + "t": "long" + }, + "cappedDocs": { + "t": "long" + } + }, + "$k": [ + "_id", + "uuid", + "table", + "indexes", + "cappedSize", + "cappedDocs" + ] + }, + "_id": "system.users", + "uuid": "b34c2094-2086-4184-8761-f5692fadb2f2", + "table": "system_users_aff2f7ce", + "indexes": [ + { + "pgindex": "system_users_aff2f7ce__id__67399184_idx", + "name": "_id_", + "key": { + "_id": 1 + }, + "unique": true + } + ], + "cappedSize": 0, + "cappedDocs": 0 +} diff --git a/cmd/envtool/user.json b/cmd/envtool/user.json new file mode 100644 index 000000000000..7156c405b700 --- /dev/null +++ b/cmd/envtool/user.json @@ -0,0 +1,145 @@ +{ + "$s": { + "p": { + "_id": { + "t": "string" + }, + "credentials": { + "t": "object", + "$s": { + "p": { + "SCRAM-SHA-1": { + "t": "object", + "$s": { + "p": { + "storedKey": { + "t": "string" + }, + "iterationCount": { + "t": "int" + }, + "salt": { + "t": "string" + }, + "serverKey": { + "t": "string" + } + }, + "$k": [ + "storedKey", + "iterationCount", + "salt", + "serverKey" + ] + } + }, + "SCRAM-SHA-256": { + "t": "object", + "$s": { + "p": { + "storedKey": { + "t": "string" + }, + "iterationCount": { + "t": "int" + }, + "salt": { + "t": "string" + }, + "serverKey": { + "t": "string" + } + }, + "$k": [ + "storedKey", + "iterationCount", + "salt", + "serverKey" + ] + } + }, + "PLAIN": { + "t": "object", + "$s": { + "p": { + "algo": { + "t": "string" + }, + "iterationCount": { + "t": "long" + }, + "hash": { + "t": "binData", + "s": 0 + }, + "salt": { + "t": "binData", + "s": 0 + } + }, + "$k": [ + "algo", + "iterationCount", + "hash", + "salt" + ] + } + } + }, + "$k": [ + "SCRAM-SHA-1", + "SCRAM-SHA-256", + "PLAIN" + ] + } + }, + "user": { + "t": "string" + }, + "db": { + "t": "string" + }, + "roles": { + "t": "array", + "i": [] + }, + "userId": { + "t": "binData", + "s": 4 + } + }, + "$k": [ + "_id", + "credentials", + "user", + "db", + "roles", + "userId" + ] + }, + "_id": "admin.username", + "credentials": { + "SCRAM-SHA-1": { + "storedKey": "o+viQ4miKbpHYVuv62ys3qlfcHE=", + "iterationCount": 10000, + "salt": "a8nzn5BcBhXRhScuRy39cQ==", + "serverKey": "xzgG8EYUmN+NQ9vwbQuVLgtyifQ=" + }, + "SCRAM-SHA-256": { + "storedKey": "ADDyKR9ukJMC2wXQztMbaIrTNZh7RdBSNtQ3ehnAv+U=", + "iterationCount": 15000, + "salt": "ZSTAnwxr7HEZ44F5+1qOH06asrPR/30CyhgjYw==", + "serverKey": "nu1KrJQnTXhLMFBPxv6lVqakf1lEtB8gF/NFpd3aXdc=" + }, + "PLAIN": { + "algo": "PBKDF2-HMAC-SHA256", + "iterationCount": 600000, + "hash": "1oHFV6ZG+3eBSMv/HPpfcs50dvdidhArDHmF6pPtU54=", + "salt": "sxK4yjYYSss9JmYhfBRzwA==" + } + }, + "user": "username", + "db": "admin", + "roles": [], + "userId": "Qi0njDnoRT66fs6AKwtfoA==" +}