Skip to content

Commit

Permalink
Update Go client libraries for etcd
Browse files Browse the repository at this point in the history
Signed-off-by: Tero Saarni <tero.saarni@est.tech>
  • Loading branch information
tsaarni committed Jul 8, 2021
1 parent 30ce696 commit 4807e31
Show file tree
Hide file tree
Showing 9 changed files with 205 additions and 195 deletions.
3 changes: 3 additions & 0 deletions changelog/11980.txt
@@ -0,0 +1,3 @@
```release-note:improvement
physical/etcd: Upgrade etcd3 client to v3.5.0 and etcd2 to v2.305.0
```
25 changes: 14 additions & 11 deletions go.mod
Expand Up @@ -33,7 +33,7 @@ require (
github.com/client9/misspell v0.3.4
github.com/cockroachdb/cockroach-go v0.0.0-20181001143604-e0a95dfd547c
github.com/containerd/containerd v1.4.3 // indirect
github.com/coreos/go-semver v0.2.0
github.com/coreos/go-semver v0.3.0
github.com/coreos/go-systemd v0.0.0-20191104093116-d3cd4ed1dbcf
github.com/denisenkom/go-mssqldb v0.0.0-20200428022330-06a60b6afbbc
github.com/docker/docker v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible
Expand All @@ -51,7 +51,7 @@ require (
github.com/go-sql-driver/mysql v1.5.0
github.com/go-test/deep v1.0.7
github.com/gocql/gocql v0.0.0-20210401103645-80ab1e13e309
github.com/golang/protobuf v1.4.2
github.com/golang/protobuf v1.5.2
github.com/google/go-cmp v0.5.5
github.com/google/go-github v17.0.0+incompatible
github.com/google/go-metrics-stackdriver v0.2.0
Expand Down Expand Up @@ -141,15 +141,17 @@ require (
github.com/pkg/errors v0.9.1
github.com/posener/complete v1.2.3
github.com/pquerna/otp v1.2.1-0.20191009055518-468c2dd2b58d
github.com/prometheus/client_golang v1.7.1
github.com/prometheus/common v0.11.1
github.com/prometheus/client_golang v1.11.0
github.com/prometheus/common v0.26.0
github.com/rboyer/safeio v0.2.1
github.com/ryanuber/columnize v2.1.0+incompatible
github.com/ryanuber/go-glob v1.0.0
github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da
github.com/sasha-s/go-deadlock v0.2.0
github.com/sethvargo/go-limiter v0.3.0
github.com/shirou/gopsutil v3.21.5+incompatible
github.com/smartystreets/goconvey v1.6.4 // indirect
github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271 // indirect
github.com/stretchr/testify v1.7.0
github.com/tidwall/pretty v1.0.1 // indirect
github.com/tklauser/go-sysconf v0.3.6 // indirect
Expand All @@ -158,23 +160,24 @@ require (
github.com/xi2/xz v0.0.0-20171230120015-48954b6210f8 // indirect
github.com/yuin/gopher-lua v0.0.0-20200816102855-ee81675732da // indirect
go.etcd.io/bbolt v1.3.5
go.etcd.io/etcd v0.5.0-alpha.5.0.20200425165423-262c93980547
go.etcd.io/etcd/client/pkg/v3 v3.5.0
go.etcd.io/etcd/client/v2 v2.305.0
go.etcd.io/etcd/client/v3 v3.5.0
go.mongodb.org/mongo-driver v1.4.6
go.opentelemetry.io/otel v0.20.0
go.opentelemetry.io/otel/sdk v0.20.0
go.opentelemetry.io/otel/trace v0.20.0
go.uber.org/atomic v1.6.0
go.uber.org/atomic v1.7.0
go.uber.org/goleak v1.1.10
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a
golang.org/x/net v0.0.0-20210510120150-4163338589ed
golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c // indirect
golang.org/x/sys v0.0.0-20210514084401-e8d321eab015
golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1
golang.org/x/tools v0.0.0-20210101214203-2dba1e4ea05c
golang.org/x/tools v0.1.2
google.golang.org/api v0.29.0
google.golang.org/grpc v1.29.1
google.golang.org/protobuf v1.25.0
google.golang.org/grpc v1.38.0
google.golang.org/protobuf v1.26.0
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce
gopkg.in/ory-am/dockertest.v3 v3.3.4
gopkg.in/square/go-jose.v2 v2.5.1
Expand Down
196 changes: 69 additions & 127 deletions go.sum

Large diffs are not rendered by default.

87 changes: 87 additions & 0 deletions helper/testhelpers/etcd/etcdhelper.go
@@ -0,0 +1,87 @@
package etcd

import (
"context"
"fmt"
"net/url"
"os"
"testing"
"time"

"github.com/hashicorp/vault/helper/testhelpers/docker"
clientv3 "go.etcd.io/etcd/client/v3"
)

type Config struct {
docker.ServiceURL
}

// PrepareTestContainer created etcd docker container. If environment variabe
// ETCD_ADDR is set, the tests are executed against specified address and
// container is not launched.
func PrepareTestContainer(t *testing.T, version string) (func(), *Config) {
if addr := os.Getenv("ETCD_ADDR"); addr != "" {
url, err := docker.NewServiceURLParse(addr)
if err != nil {
t.Fatal(err)
}
return func() {}, &Config{ServiceURL: *url}
}

// Check https://github.com/etcd-io/etcd/releases for latest releases.
runner, err := docker.NewServiceRunner(docker.RunOptions{
ContainerName: "etcd",
ImageRepo: "gcr.io/etcd-development/etcd",
ImageTag: version,
Cmd: []string{"/usr/local/bin/etcd",
"--name", "s1",
"--listen-client-urls", "http://0.0.0.0:2379",
"--advertise-client-urls", "http://0.0.0.0:2379",
"--listen-peer-urls", "http://0.0.0.0:2380",
"--initial-advertise-peer-urls", "http://0.0.0.0:2380",
"--initial-cluster", "s1=http://0.0.0.0:2380",
"--initial-cluster-token", "tkn",
"--initial-cluster-state", "new",
"--log-level", "info",
"--logger", "zap",
"--log-outputs", "stderr",
"--enable-v2", // v2 will be deprecated after etcd 3.5
},
Ports: []string{"2379/tcp"},
})
if err != nil {
t.Fatalf("Could not start docker etcd container: %s", err)
}

svc, err := runner.StartService(context.Background(), func(ctx context.Context, host string, port int) (docker.ServiceConfig, error) {
address := fmt.Sprintf("%s:%d", host, port)
s := docker.NewServiceURL(url.URL{
Scheme: "http",
Host: address,
})

client, err := clientv3.New(clientv3.Config{
Endpoints: []string{address},
DialTimeout: 2 * time.Minute,
})
if err != nil {
return nil, fmt.Errorf("could not connect to etcd container: %w", err)
}

// Enable authentication for the tests.
client.RoleAdd(ctx, "root")
client.UserAdd(ctx, "root", "insecure")
client.UserGrantRole(ctx, "root", "root")
client.AuthEnable(ctx)
client.Close()

return &Config{
ServiceURL: *s,
}, nil
})
if err != nil {
t.Fatalf("Could not start docker etcd container: %s", err)
}

return svc.Cleanup, svc.Config.(*Config)
}
2 changes: 1 addition & 1 deletion physical/etcd/etcd.go
Expand Up @@ -11,7 +11,7 @@ import (
"github.com/coreos/go-semver/semver"
log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/sdk/physical"
"go.etcd.io/etcd/client"
"go.etcd.io/etcd/client/v2"
)

var (
Expand Down
4 changes: 2 additions & 2 deletions physical/etcd/etcd2.go
Expand Up @@ -15,8 +15,8 @@ import (
log "github.com/hashicorp/go-hclog"
multierror "github.com/hashicorp/go-multierror"
"github.com/hashicorp/vault/sdk/physical"
"go.etcd.io/etcd/client"
"go.etcd.io/etcd/pkg/transport"
"go.etcd.io/etcd/client/pkg/v3/transport"
"go.etcd.io/etcd/client/v2"
)

const (
Expand Down
6 changes: 3 additions & 3 deletions physical/etcd/etcd3.go
Expand Up @@ -16,9 +16,9 @@ import (
"github.com/hashicorp/vault/sdk/helper/parseutil"
"github.com/hashicorp/vault/sdk/helper/strutil"
"github.com/hashicorp/vault/sdk/physical"
"go.etcd.io/etcd/clientv3"
"go.etcd.io/etcd/clientv3/concurrency"
"go.etcd.io/etcd/pkg/transport"
"go.etcd.io/etcd/client/pkg/v3/transport"
clientv3 "go.etcd.io/etcd/client/v3"
"go.etcd.io/etcd/client/v3/concurrency"
)

// EtcdBackend is a physical backend that stores data at specific
Expand Down
20 changes: 12 additions & 8 deletions physical/etcd/etcd3_test.go
Expand Up @@ -2,33 +2,37 @@ package etcd

import (
"fmt"
"os"
"testing"
"time"

log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/helper/testhelpers/etcd"
"github.com/hashicorp/vault/sdk/helper/logging"
"github.com/hashicorp/vault/sdk/physical"
)

func TestEtcd3Backend(t *testing.T) {
addr := os.Getenv("ETCD_ADDR")
if addr == "" {
t.Skipf("Skipped. No etcd3 server found")
}
cleanup, config := etcd.PrepareTestContainer(t, "v3.5.0")
defer cleanup()

logger := logging.NewVaultLogger(log.Debug)
config := map[string]string{
configMap := map[string]string{
"address": config.URL().String(),
"path": fmt.Sprintf("/vault-%d", time.Now().Unix()),
"etcd_api": "3",
"username": "root",
"password": "insecure",

// Syncing adverticed client urls should be disabled since docker port mapping confuses the client.
"sync": "false",
}

b, err := NewEtcdBackend(config, logger)
b, err := NewEtcdBackend(configMap, logger)
if err != nil {
t.Fatalf("err: %s", err)
}

b2, err := NewEtcdBackend(config, logger)
b2, err := NewEtcdBackend(configMap, logger)
if err != nil {
t.Fatalf("err: %s", err)
}
Expand Down
57 changes: 14 additions & 43 deletions physical/etcd/etcd_test.go
@@ -1,67 +1,38 @@
package etcd

import (
"context"
"fmt"
"os"
"testing"
"time"

log "github.com/hashicorp/go-hclog"
"github.com/hashicorp/vault/helper/testhelpers/etcd"
"github.com/hashicorp/vault/sdk/helper/logging"
"github.com/hashicorp/vault/sdk/physical"

"go.etcd.io/etcd/client"
)

func TestEtcdBackend(t *testing.T) {
addr := os.Getenv("ETCD_ADDR")
if addr == "" {
t.SkipNow()
}

cfg := client.Config{
Endpoints: []string{addr},
Transport: client.DefaultTransport,
}

c, err := client.New(cfg)
if err != nil {
t.Fatalf("err: %s", err)
}

ctx, cancel := context.WithTimeout(context.Background(), client.DefaultRequestTimeout)
syncErr := c.Sync(ctx)
cancel()
if syncErr != nil {
t.Fatalf("err: %v", EtcdSyncClusterError)
}

kAPI := client.NewKeysAPI(c)

randPath := fmt.Sprintf("/vault-%d", time.Now().Unix())
defer func() {
delOpts := &client.DeleteOptions{
Recursive: true,
}
if _, err := kAPI.Delete(context.Background(), randPath, delOpts); err != nil {
t.Fatalf("err: %v", err)
}
}()
cleanup, config := etcd.PrepareTestContainer(t, "v3.5.0")
defer cleanup()

// Generate new etcd backend. The etcd address is read from ETCD_ADDR. No
// need to provide it explicitly.
logger := logging.NewVaultLogger(log.Debug)
config := map[string]string{
"path": randPath,
configMap := map[string]string{
"address": config.URL().String(),
"path": fmt.Sprintf("/vault-%d", time.Now().Unix()),
"etcd_api": "2",
"username": "root",
"password": "insecure",

// Syncing adverticed client urls should be disabled since docker port mapping confuses the client.
"sync": "false",
}

b, err := NewEtcdBackend(config, logger)
b, err := NewEtcdBackend(configMap, logger)
if err != nil {
t.Fatalf("err: %s", err)
}

b2, err := NewEtcdBackend(config, logger)
b2, err := NewEtcdBackend(configMap, logger)
if err != nil {
t.Fatalf("err: %s", err)
}
Expand Down

0 comments on commit 4807e31

Please sign in to comment.