diff --git a/backend_test.go b/backend_test.go index a8f3a7d..f6feba5 100644 --- a/backend_test.go +++ b/backend_test.go @@ -22,7 +22,7 @@ var ( func getBackend(throwsErr bool) (*backend, logical.Storage) { config := &logical.BackendConfig{ - Logger: logging.NewVaultLogger(log.Error), + Logger: logging.NewVaultLogger(log.Debug), System: &logical.StaticSystemView{ DefaultLeaseTTLVal: defaultLeaseTTLVal, @@ -42,7 +42,9 @@ func getBackend(throwsErr bool) (*backend, logical.Storage) { b.cancelQueue = cancel // Load queue and kickoff new periodic ticker - go b.initQueue(ictx, &logical.InitializationRequest{config.StorageView}) + b.initQueue(ictx, &logical.InitializationRequest{ + Storage: config.StorageView, + }) return b, config.StorageView } @@ -66,7 +68,7 @@ func (f *fakeLdapClient) Get(_ *client.Config, _ string) (*client.Entry, error) return client.NewEntry(entry), err } -func (f *fakeLdapClient) UpdatePassword(conf *client.Config, dn string, newPassword string) error { +func (f *fakeLdapClient) UpdatePassword(_ *client.Config, _ string, _ string) error { var err error if f.throwErrs { err = errors.New("forced error") @@ -74,7 +76,7 @@ func (f *fakeLdapClient) UpdatePassword(conf *client.Config, dn string, newPassw return err } -func (f *fakeLdapClient) UpdateRootPassword(conf *client.Config, newPassword string) error { +func (f *fakeLdapClient) UpdateRootPassword(_ *client.Config, _ string) error { var err error if f.throwErrs { err = errors.New("forced error") @@ -90,7 +92,7 @@ func (f *fakeLdapClient) Del(_ *client.Config, _ *ldap.DelRequest) error { return fmt.Errorf("not implemented") } -func (f *fakeLdapClient) Execute(conf *client.Config, entries []*ldif.Entry, continueOnError bool) (err error) { +func (f *fakeLdapClient) Execute(_ *client.Config, _ []*ldif.Entry, _ bool) (err error) { return fmt.Errorf("not implemented") } diff --git a/path_rotate.go b/path_rotate.go index 669feae..56660ac 100644 --- a/path_rotate.go +++ b/path_rotate.go @@ -24,12 +24,12 @@ func (b *backend) pathRotateCredentials() []*framework.Path { Fields: map[string]*framework.FieldSchema{}, Operations: map[logical.Operation]framework.OperationHandler{ logical.UpdateOperation: &framework.PathOperation{ - Callback: b.pathRotateCredentialsUpdate, + Callback: b.pathRotateRootCredentialsUpdate, ForwardPerformanceStandby: true, ForwardPerformanceSecondary: true, }, logical.CreateOperation: &framework.PathOperation{ - Callback: b.pathRotateCredentialsUpdate, + Callback: b.pathRotateRootCredentialsUpdate, ForwardPerformanceStandby: true, ForwardPerformanceSecondary: true, }, @@ -64,7 +64,7 @@ func (b *backend) pathRotateCredentials() []*framework.Path { } } -func (b *backend) pathRotateCredentialsUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { +func (b *backend) pathRotateRootCredentialsUpdate(ctx context.Context, req *logical.Request, data *framework.FieldData) (*logical.Response, error) { if _, hasTimeout := ctx.Deadline(); !hasTimeout { var cancel func() ctx, cancel = context.WithTimeout(ctx, defaultCtxTimeout) @@ -133,10 +133,14 @@ func (b *backend) pathRotateRoleCredentialsUpdate(ctx context.Context, req *logi } } - resp, err := b.setStaticAccountPassword(ctx, req.Storage, &setStaticAccountInput{ + input := &setStaticAccountInput{ RoleName: name, Role: role, - }) + } + if walID, ok := item.Value.(string); ok { + input.WALID = walID + } + resp, err := b.setStaticAccountPassword(ctx, req.Storage, input) if err != nil { b.Logger().Warn("unable to rotate credentials in rotate-role", "error", err) // Update the priority to re-try this rotation and re-add the item to @@ -149,6 +153,8 @@ func (b *backend) pathRotateRoleCredentialsUpdate(ctx context.Context, req *logi } } else { item.Priority = resp.RotationTime.Add(role.StaticAccount.RotationPeriod).Unix() + // Clear any stored WAL ID as we must have successfully deleted our WAL to get here. + item.Value = "" } // Add their rotation to the queue. We use pushErr here to distinguish between @@ -159,9 +165,14 @@ func (b *backend) pathRotateRoleCredentialsUpdate(ctx context.Context, req *logi return nil, pushErr } + if err != nil { + return nil, fmt.Errorf("unable to finish rotating credentials; retries will "+ + "continue in the background but it is also safe to retry manually: %w", err) + } + // We're not returning creds here because we do not know if its been processed // by the queue. - return nil, err + return nil, nil } // rollBackPassword uses naive exponential backoff to retry updating to an old password, diff --git a/path_static_roles.go b/path_static_roles.go index 4ff93ff..e1cf056 100644 --- a/path_static_roles.go +++ b/path_static_roles.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + "github.com/hashicorp/go-multierror" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/locksutil" "github.com/hashicorp/vault/sdk/logical" @@ -146,7 +147,28 @@ func (b *backend) pathStaticRoleDelete(ctx context.Context, req *logical.Request return nil, err } - return nil, nil + walIDs, err := framework.ListWAL(ctx, req.Storage) + if err != nil { + return nil, err + } + var merr *multierror.Error + for _, walID := range walIDs { + wal, err := b.findStaticWAL(ctx, req.Storage, walID) + if err != nil { + merr = multierror.Append(merr, err) + continue + } + if wal != nil && name == wal.RoleName { + b.Logger().Debug("deleting WAL for deleted role", "WAL ID", walID, "role", name) + err = framework.DeleteWAL(ctx, req.Storage, walID) + if err != nil { + b.Logger().Debug("failed to delete WAL for deleted role", "WAL ID", walID, "error", err) + merr = multierror.Append(merr, err) + } + } + } + + return nil, merr.ErrorOrNil() } func (b *backend) pathStaticRoleRead(ctx context.Context, req *logical.Request, d *framework.FieldData) (*logical.Response, error) { @@ -221,6 +243,7 @@ func (b *backend) pathStaticRoleCreateUpdate(ctx context.Context, req *logical.R // Only call setStaticAccountPassword if we're creating the role for the // first time + var item *queue.Item switch req.Operation { case logical.CreateOperation: // setStaticAccountPassword calls Storage.Put and saves the role to storage @@ -229,10 +252,24 @@ func (b *backend) pathStaticRoleCreateUpdate(ctx context.Context, req *logical.R Role: role, }) if err != nil { + if resp != nil && resp.WALID != "" { + b.Logger().Debug("deleting WAL for failed role creation", "WAL ID", resp.WALID, "role", name) + walDeleteErr := framework.DeleteWAL(ctx, req.Storage, resp.WALID) + if walDeleteErr != nil { + b.Logger().Debug("failed to delete WAL for failed role creation", "WAL ID", resp.WALID, "error", walDeleteErr) + var merr *multierror.Error + merr = multierror.Append(merr, err) + merr = multierror.Append(merr, fmt.Errorf("failed to clean up WAL from failed role creation: %w", walDeleteErr)) + err = merr.ErrorOrNil() + } + } return nil, err } // guard against RotationTime not being set or zero-value lvr = resp.RotationTime + item = &queue.Item{ + Key: name, + } case logical.UpdateOperation: // store updated Role entry, err := logical.StorageEntryJSON(staticRolePath+name, role) @@ -244,20 +281,19 @@ func (b *backend) pathStaticRoleCreateUpdate(ctx context.Context, req *logical.R } // In case this is an update, remove any previous version of the item from - // the queue - + // the queue. The existing item could be tracking a WAL ID for this role, + // so it's important to keep the existing item rather than recreate it. //TODO: Add retry logic - _, err = b.popFromRotationQueueByKey(name) + item, err = b.popFromRotationQueueByKey(name) if err != nil { return nil, err } } + item.Priority = lvr.Add(role.StaticAccount.RotationPeriod).Unix() + // Add their rotation to the queue - if err := b.pushItem(&queue.Item{ - Key: name, - Priority: lvr.Add(role.StaticAccount.RotationPeriod).Unix(), - }); err != nil { + if err := b.pushItem(item); err != nil { return nil, err } diff --git a/path_static_roles_test.go b/path_static_roles_test.go index f6db655..7e2437d 100644 --- a/path_static_roles_test.go +++ b/path_static_roles_test.go @@ -2,6 +2,7 @@ package openldap import ( "context" + "strings" "testing" "github.com/hashicorp/vault/sdk/logical" @@ -452,3 +453,150 @@ func TestListRoles(t *testing.T) { } }) } + +func TestWALsStillTrackedAfterUpdate(t *testing.T) { + ctx := context.Background() + b, storage := getBackend(false) + defer b.Cleanup(ctx) + configureOpenLDAPMount(t, b, storage) + + createRole(t, b, storage, "hashicorp") + + generateWALFromFailedRotation(t, b, storage, "hashicorp") + requireWALs(t, storage, 1) + + _, err := b.HandleRequest(ctx, &logical.Request{ + Operation: logical.UpdateOperation, + Path: staticRolePath + "hashicorp", + Storage: storage, + Data: map[string]interface{}{ + "username": "hashicorp", + "dn": "uid=hashicorp,ou=users,dc=hashicorp,dc=com", + "rotation_period": "600s", + }, + }) + if err != nil { + t.Fatal(err) + } + walIDs := requireWALs(t, storage, 1) + + // Now when we trigger a manual rotate, it should use the WAL's new password + // which will tell us that the in-memory structure still kept track of the + // WAL in addition to it still being in storage. + wal, err := b.findStaticWAL(ctx, storage, walIDs[0]) + if err != nil { + t.Fatal(err) + } + _, err = b.HandleRequest(context.Background(), &logical.Request{ + Operation: logical.UpdateOperation, + Path: "rotate-role/hashicorp", + Storage: storage, + }) + if err != nil { + t.Fatal(err) + } + role, err := b.staticRole(ctx, storage, "hashicorp") + if err != nil { + t.Fatal(err) + } + if role.StaticAccount.Password != wal.NewPassword { + t.Fatal() + } + requireWALs(t, storage, 0) +} + +func TestWALsDeletedOnRoleCreationFailed(t *testing.T) { + ctx := context.Background() + b, storage := getBackend(true) + defer b.Cleanup(ctx) + configureOpenLDAPMount(t, b, storage) + + for i := 0; i < 3; i++ { + resp, err := b.HandleRequest(ctx, &logical.Request{ + Operation: logical.CreateOperation, + Path: staticRolePath + "hashicorp", + Storage: storage, + Data: map[string]interface{}{ + "username": "hashicorp", + "dn": "uid=hashicorp,ou=users,dc=hashicorp,dc=com", + "rotation_period": "5s", + }, + }) + if err == nil { + t.Fatal("expected error from OpenLDAP") + } + if !strings.Contains(err.Error(), "forced error") { + t.Fatal("expected forced error message", resp, err) + } + } + + requireWALs(t, storage, 0) +} + +func TestWALsDeletedOnRoleDeletion(t *testing.T) { + ctx := context.Background() + b, storage := getBackend(false) + defer b.Cleanup(ctx) + configureOpenLDAPMount(t, b, storage) + + // Create the roles + roleNames := []string{"hashicorp", "2"} + for _, roleName := range roleNames { + createRole(t, b, storage, roleName) + } + + // Fail to rotate the roles + for _, roleName := range roleNames { + generateWALFromFailedRotation(t, b, storage, roleName) + } + + // Should have 2 WALs hanging around + requireWALs(t, storage, 2) + + // Delete one of the static roles + _, err := b.HandleRequest(ctx, &logical.Request{ + Operation: logical.DeleteOperation, + Path: "static-role/hashicorp", + Storage: storage, + }) + if err != nil { + t.Fatal(err) + } + + // 1 WAL should be cleared by the delete + requireWALs(t, storage, 1) +} + +func configureOpenLDAPMount(t *testing.T, b *backend, storage logical.Storage) { + t.Helper() + resp, err := b.HandleRequest(context.Background(), &logical.Request{ + Operation: logical.CreateOperation, + Path: configPath, + Storage: storage, + Data: map[string]interface{}{ + "binddn": "tester", + "bindpass": "pa$$w0rd", + "url": "ldap://138.91.247.105", + "certificate": validCertificate, + }, + }) + if err != nil || (resp != nil && resp.IsError()) { + t.Fatalf("err:%s resp:%#v\n", err, resp) + } +} + +func createRole(t *testing.T, b *backend, storage logical.Storage, roleName string) { + _, err := b.HandleRequest(context.Background(), &logical.Request{ + Operation: logical.CreateOperation, + Path: "static-role/" + roleName, + Storage: storage, + Data: map[string]interface{}{ + "username": roleName, + "dn": "uid=hashicorp,ou=users,dc=hashicorp,dc=com", + "rotation_period": "86400s", + }, + }) + if err != nil { + t.Fatal(err) + } +} diff --git a/rotation.go b/rotation.go index 3139f82..04268d0 100644 --- a/rotation.go +++ b/rotation.go @@ -7,7 +7,6 @@ import ( "time" "github.com/hashicorp/errwrap" - "github.com/hashicorp/go-multierror" "github.com/hashicorp/vault/sdk/framework" "github.com/hashicorp/vault/sdk/helper/base62" "github.com/hashicorp/vault/sdk/helper/consts" @@ -62,21 +61,34 @@ func (b *backend) populateQueue(ctx context.Context, s logical.Storage) { item := queue.Item{ Key: roleName, - Priority: role.StaticAccount.LastVaultRotation.Add(role.StaticAccount.RotationPeriod).Unix(), + Priority: role.StaticAccount.NextRotationTime().Unix(), } // Check if role name is in map walEntry := walMap[roleName] if walEntry != nil { // Check walEntry last vault time - if !walEntry.LastVaultRotation.IsZero() && walEntry.LastVaultRotation.Before(role.StaticAccount.LastVaultRotation) { + if walEntry.LastVaultRotation.IsZero() { + // A WAL's last Vault rotation can only ever be 0 for a role that + // was never successfully created. So we know this WAL couldn't + // have been created for this role we just retrieved from storage. + // i.e. it must be a hangover from a previous attempt at creating + // a role with the same name + log.Debug("deleting WAL with zero last rotation time", "WAL ID", walEntry.walID, "created", walEntry.walCreatedAt) + if err := framework.DeleteWAL(ctx, s, walEntry.walID); err != nil { + log.Warn("unable to delete zero-time WAL", "error", err, "WAL ID", walEntry.walID) + } + } else if walEntry.LastVaultRotation.Before(role.StaticAccount.LastVaultRotation) { // WAL's last vault rotation record is older than the role's data, so // delete and move on + log.Debug("deleting outdated WAL", "WAL ID", walEntry.walID, "created", walEntry.walCreatedAt) if err := framework.DeleteWAL(ctx, s, walEntry.walID); err != nil { log.Warn("unable to delete WAL", "error", err, "WAL ID", walEntry.walID) } } else { - log.Info("adjusting priority for Role") + log.Info("found WAL for role", + "role", item.Key, + "WAL ID", walEntry.walID) item.Value = walEntry.walID item.Priority = time.Now().Unix() } @@ -110,14 +122,15 @@ func (b *backend) runTicker(ctx context.Context, s logical.Storage) { // credential setting or rotation in the event of partial failure. type setCredentialsWAL struct { NewPassword string `json:"new_password"` - OldPassword string `json:"old_password"` RoleName string `json:"role_name"` Username string `json:"username"` DN string `json:"dn"` LastVaultRotation time.Time `json:"last_vault_rotation"` - walID string + // Private fields which will not be included in json.Marshal/Unmarshal. + walID string + walCreatedAt int64 // Unix time at which the WAL was created. } // rotateCredentials sets a new password for a static account. This method is @@ -191,18 +204,7 @@ func (b *backend) rotateCredential(ctx context.Context, s logical.Storage) bool // If there is a WAL entry related to this Role, the corresponding WAL ID // should be stored in the Item's Value field. if walID, ok := item.Value.(string); ok { - walEntry, err := b.findStaticWAL(ctx, s, walID) - if err != nil { - b.Logger().Error("error finding static WAL", "error", err) - item.Priority = time.Now().Add(10 * time.Second).Unix() - if err := b.pushItem(item); err != nil { - b.Logger().Error("unable to push item on to queue", "error", err) - } - } - if walEntry != nil && walEntry.NewPassword != "" { - input.Password = walEntry.NewPassword - input.WALID = walID - } + input.WALID = walID } resp, err := b.setStaticAccountPassword(ctx, s, input) @@ -223,6 +225,8 @@ func (b *backend) rotateCredential(ctx context.Context, s logical.Storage) bool // Go to next item return true } + // Clear any stored WAL ID as we must have successfully deleted our WAL to get here. + item.Value = "" lvr := resp.RotationTime if lvr.IsZero() { @@ -252,12 +256,12 @@ func (b *backend) findStaticWAL(ctx context.Context, s logical.Storage, id strin data := wal.Data.(map[string]interface{}) walEntry := setCredentialsWAL{ - walID: id, - NewPassword: data["new_password"].(string), - OldPassword: data["old_password"].(string), - RoleName: data["role_name"].(string), - Username: data["username"].(string), - DN: data["dn"].(string), + walID: id, + walCreatedAt: wal.CreatedAt, + NewPassword: data["new_password"].(string), + RoleName: data["role_name"].(string), + Username: data["username"].(string), + DN: data["dn"].(string), } lvr, err := time.Parse(time.RFC3339, data["last_vault_rotation"].(string)) if err != nil { @@ -269,16 +273,13 @@ func (b *backend) findStaticWAL(ctx context.Context, s logical.Storage, id strin } type setStaticAccountInput struct { - RoleName string - Role *roleEntry - Password string - CreateUser bool - WALID string + RoleName string + Role *roleEntry + WALID string } type setStaticAccountOutput struct { RotationTime time.Time - Password string // Optional return field, in the event WAL was created and not destroyed // during the operation WALID string @@ -298,7 +299,6 @@ type setStaticAccountOutput struct { // This method does not perform any operations on the priority queue. Those // tasks must be handled outside of this method. func (b *backend) setStaticAccountPassword(ctx context.Context, s logical.Storage, input *setStaticAccountInput) (*setStaticAccountOutput, error) { - var merr error if input == nil || input.Role == nil || input.RoleName == "" { return nil, errors.New("input was empty when attempting to set credentials for static account") } @@ -312,32 +312,54 @@ func (b *backend) setStaticAccountPassword(ctx context.Context, s logical.Storag // Re-use WAL ID if present, otherwise PUT a new WAL output := &setStaticAccountOutput{WALID: input.WALID} + b.Lock() + defer b.Unlock() + config, err := readConfig(ctx, s) if err != nil { - return nil, err + return output, err } if config == nil { - return nil, errors.New("the config is currently unset") - } - - newPassword, err := b.GeneratePassword(ctx, config) - if err != nil { - return nil, err + return output, errors.New("the config is currently unset") } - oldPassword := input.Role.StaticAccount.Password + var newPassword string + if output.WALID != "" { + wal, err := b.findStaticWAL(ctx, s, output.WALID) + if err != nil { + return output, errwrap.Wrapf("error retrieving WAL entry: {{err}}", err) + } - // Take out the backend lock since we are swapping out the connection - b.Lock() - defer b.Unlock() + switch { + case wal != nil && wal.NewPassword != "": + newPassword = wal.NewPassword + default: + if wal == nil { + b.Logger().Error("expected role to have WAL, but WAL not found in storage", "role", input.RoleName, "WAL ID", output.WALID) + } else { + b.Logger().Error("expected WAL to have a new password set, but empty", "role", input.RoleName, "WAL ID", output.WALID) + err = framework.DeleteWAL(ctx, s, output.WALID) + if err != nil { + b.Logger().Warn("failed to delete WAL with no new password", "error", err, "WAL ID", output.WALID) + } + } + // If there's anything wrong with the WAL in storage, we'll need + // to generate a fresh WAL and password + output.WALID = "" + } + } if output.WALID == "" { + newPassword, err = b.GeneratePassword(ctx, config) + if err != nil { + return output, err + } + b.Logger().Debug("writing WAL", "role", input.RoleName, "WAL ID", output.WALID) output.WALID, err = framework.PutWAL(ctx, s, staticWALKey, &setCredentialsWAL{ RoleName: input.RoleName, Username: input.Role.StaticAccount.Username, DN: input.Role.StaticAccount.DN, NewPassword: newPassword, - OldPassword: oldPassword, LastVaultRotation: input.Role.StaticAccount.LastVaultRotation, }) if err != nil { @@ -345,21 +367,17 @@ func (b *backend) setStaticAccountPassword(ctx context.Context, s logical.Storag } } - // Update the password remotely. - if err := b.client.UpdatePassword(config.LDAP, input.Role.StaticAccount.DN, newPassword); err != nil { - return nil, err + if newPassword == "" { + b.Logger().Error("newPassword was empty, re-generating based on the password policy") + newPassword, err = b.GeneratePassword(ctx, config) + if err != nil { + return output, err + } } - // Update the password locally. - if pwdStoringErr := storePassword(ctx, s, config); pwdStoringErr != nil { - // We were unable to store the new password locally. We can't continue in this state because we won't be able - // to roll any passwords, including our own to get back into a state of working. So, we need to roll back to - // the last password we successfully got into storage. - if rollbackErr := b.rollBackPassword(ctx, config, oldPassword); rollbackErr != nil { - return nil, fmt.Errorf(`unable to store new password due to %s and unable to return to previous password due -to %s, configure a new binddn and bindpass to restore openldap function`, pwdStoringErr, rollbackErr) - } - return nil, fmt.Errorf("unable to update password due to storage err: %s", pwdStoringErr) + // Update the password remotely. + if err := b.client.UpdatePassword(config.LDAP, input.Role.StaticAccount.DN, newPassword); err != nil { + return output, err } // Store updated role information @@ -379,12 +397,13 @@ to %s, configure a new binddn and bindpass to restore openldap function`, pwdSto // Cleanup WAL after successfully rotating and pushing new item on to queue if err := framework.DeleteWAL(ctx, s, output.WALID); err != nil { - merr = multierror.Append(merr, err) - return output, merr + b.Logger().Warn("error deleting WAL", "WAL ID", output.WALID, "error", err) + return output, err } + b.Logger().Debug("deleted WAL", "WAL ID", output.WALID) // The WAL has been deleted, return new setStaticAccountOutput without it - return &setStaticAccountOutput{RotationTime: lvr}, merr + return &setStaticAccountOutput{RotationTime: lvr}, nil } func (b *backend) GeneratePassword(ctx context.Context, cfg *config) (string, error) { @@ -463,13 +482,39 @@ func (b *backend) loadStaticWALs(ctx context.Context, s logical.Storage) (map[st continue } if role == nil || role.StaticAccount == nil { + b.Logger().Debug("deleting WAL with nil role or static account", "WAL ID", walEntry.walID) if err := framework.DeleteWAL(ctx, s, walEntry.walID); err != nil { b.Logger().Warn("unable to delete WAL", "error", err, "WAL ID", walEntry.walID) } continue } - walEntry.walID = walID + if existingWALEntry, exists := walMap[walEntry.RoleName]; exists { + b.Logger().Debug("multiple WALs detected for role", "role", walEntry.RoleName, + "loaded WAL ID", existingWALEntry.walID, "created at", existingWALEntry.walCreatedAt, "last vault rotation", existingWALEntry.LastVaultRotation, + "candidate WAL ID", walEntry.walID, "created at", walEntry.walCreatedAt, "last vault rotation", walEntry.LastVaultRotation) + + if walEntry.walCreatedAt > existingWALEntry.walCreatedAt { + // If the existing WAL is older, delete it from storage and fall + // through to inserting our current WAL into the map. + b.Logger().Debug("deleting stale loaded WAL", "WAL ID", existingWALEntry.walID) + err = framework.DeleteWAL(ctx, s, existingWALEntry.walID) + if err != nil { + b.Logger().Warn("unable to delete loaded WAL", "error", err, "WAL ID", existingWALEntry.walID) + } + } else { + // If we already have a more recent WAL entry in the map, delete + // this one and continue onto the next WAL. + b.Logger().Debug("deleting stale candidate WAL", "WAL ID", walEntry.walID) + err = framework.DeleteWAL(ctx, s, walID) + if err != nil { + b.Logger().Warn("unable to delete candidate WAL", "error", err, "WAL ID", walEntry.walID) + } + continue + } + } + + b.Logger().Debug("loaded WAL", "WAL ID", walID) walMap[walEntry.RoleName] = walEntry } return walMap, nil diff --git a/rotation_test.go b/rotation_test.go index a03121d..9f08f27 100644 --- a/rotation_test.go +++ b/rotation_test.go @@ -5,7 +5,11 @@ import ( "testing" "time" + log "github.com/hashicorp/go-hclog" + "github.com/hashicorp/vault/sdk/framework" + "github.com/hashicorp/vault/sdk/helper/logging" "github.com/hashicorp/vault/sdk/logical" + "github.com/hashicorp/vault/sdk/queue" ) func TestAutoRotate(t *testing.T) { @@ -103,3 +107,253 @@ func TestAutoRotate(t *testing.T) { } }) } + +func TestRollsPasswordForwardsUsingWAL(t *testing.T) { + ctx := context.Background() + b, storage := getBackend(false) + defer b.Cleanup(ctx) + configureOpenLDAPMount(t, b, storage) + createRole(t, b, storage, "hashicorp") + + role, err := b.staticRole(ctx, storage, "hashicorp") + if err != nil { + t.Fatal(err) + } + oldPassword := role.StaticAccount.Password + + generateWALFromFailedRotation(t, b, storage, "hashicorp") + walIDs := requireWALs(t, storage, 1) + wal, err := b.findStaticWAL(ctx, storage, walIDs[0]) + if err != nil { + t.Fatal(err) + } + role, err = b.staticRole(ctx, storage, "hashicorp") + if err != nil { + t.Fatal(err) + } + // Role's password should still be the WAL's old password + if role.StaticAccount.Password != oldPassword { + t.Fatal(role.StaticAccount.Password, oldPassword) + } + + // Trigger a retry on the rotation, it should use WAL's new password + _, err = b.HandleRequest(ctx, &logical.Request{ + Operation: logical.UpdateOperation, + Path: "rotate-role/hashicorp", + Storage: storage, + }) + if err != nil { + t.Fatal(err) + } + + role, err = b.staticRole(ctx, storage, "hashicorp") + if err != nil { + t.Fatal(err) + } + if role.StaticAccount.Password != wal.NewPassword { + t.Fatal(role.StaticAccount.Password, wal.NewPassword) + } + // WAL should be cleared by the successful rotate + requireWALs(t, storage, 0) +} + +func TestStoredWALsCorrectlyProcessed(t *testing.T) { + const walNewPassword = "new-password-from-wal" + for _, tc := range []struct { + name string + shouldRotate bool + wal *setCredentialsWAL + }{ + { + "WAL is kept and used for roll forward", + true, + &setCredentialsWAL{ + RoleName: "hashicorp", + Username: "hashicorp", + DN: "uid=hashicorp,ou=users,dc=hashicorp,dc=com", + NewPassword: walNewPassword, + LastVaultRotation: time.Now().Add(time.Hour), + }, + }, + { + "zero-time WAL is discarded on load", + false, + &setCredentialsWAL{ + RoleName: "hashicorp", + Username: "hashicorp", + DN: "uid=hashicorp,ou=users,dc=hashicorp,dc=com", + NewPassword: walNewPassword, + LastVaultRotation: time.Time{}, + }, + }, + { + "empty-password WAL is kept but a new password is generated", + true, + &setCredentialsWAL{ + RoleName: "hashicorp", + Username: "hashicorp", + DN: "uid=hashicorp,ou=users,dc=hashicorp,dc=com", + NewPassword: "", + LastVaultRotation: time.Now().Add(time.Hour), + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + config := &logical.BackendConfig{ + Logger: logging.NewVaultLogger(log.Debug), + + System: &logical.StaticSystemView{ + DefaultLeaseTTLVal: defaultLeaseTTLVal, + MaxLeaseTTLVal: maxLeaseTTLVal, + }, + StorageView: &logical.InmemStorage{}, + } + + b := Backend(&fakeLdapClient{throwErrs: false}) + b.Setup(context.Background(), config) + + b.credRotationQueue = queue.New() + initCtx := context.Background() + ictx, cancel := context.WithCancel(initCtx) + b.cancelQueue = cancel + + defer b.Cleanup(ctx) + configureOpenLDAPMount(t, b, config.StorageView) + createRole(t, b, config.StorageView, "hashicorp") + role, err := b.staticRole(ctx, config.StorageView, "hashicorp") + if err != nil { + t.Fatal(err) + } + initialPassword := role.StaticAccount.Password + + // Set up a WAL for our test case + framework.PutWAL(ctx, config.StorageView, staticWALKey, tc.wal) + requireWALs(t, config.StorageView, 1) + // Reset the rotation queue to simulate startup memory state + b.credRotationQueue = queue.New() + + // Now finish the startup process by populating the queue, which should discard the WAL + b.initQueue(ictx, &logical.InitializationRequest{ + Storage: config.StorageView, + }) + + if tc.shouldRotate { + requireWALs(t, config.StorageView, 1) + } else { + requireWALs(t, config.StorageView, 0) + } + + // Run one tick + b.rotateCredentials(ctx, config.StorageView) + requireWALs(t, config.StorageView, 0) + + role, err = b.staticRole(ctx, config.StorageView, "hashicorp") + if err != nil { + t.Fatal(err) + } + item, err := b.popFromRotationQueueByKey("hashicorp") + if err != nil { + t.Fatal(err) + } + + if tc.shouldRotate { + if tc.wal.NewPassword != "" { + // Should use WAL's new_password field + if role.StaticAccount.Password != walNewPassword { + t.Fatal() + } + } else { + // Should rotate but ignore WAL's new_password field + if role.StaticAccount.Password == initialPassword { + t.Fatal() + } + if role.StaticAccount.Password == walNewPassword { + t.Fatal() + } + } + } else { + // Ensure the role was not promoted for early rotation + if item.Priority < time.Now().Add(time.Hour).Unix() { + t.Fatal("priority should be for about a week away, but was", item.Priority) + } + if role.StaticAccount.Password != initialPassword { + t.Fatal("password should not have been rotated yet") + } + } + }) + } +} + +func TestDeletesOlderWALsOnLoad(t *testing.T) { + ctx := context.Background() + b, storage := getBackend(false) + defer b.Cleanup(ctx) + configureOpenLDAPMount(t, b, storage) + createRole(t, b, storage, "hashicorp") + + // Create 4 WALs, with a clear winner for most recent. + wal := &setCredentialsWAL{ + RoleName: "hashicorp", + Username: "hashicorp", + DN: "uid=hashicorp,ou=users,dc=hashicorp,dc=com", + NewPassword: "some-new-password", + LastVaultRotation: time.Now(), + } + for i := 0; i < 3; i++ { + _, err := framework.PutWAL(ctx, storage, staticWALKey, wal) + if err != nil { + t.Fatal(err) + } + } + time.Sleep(2 * time.Second) + // We expect this WAL to have the latest createdAt timestamp + walID, err := framework.PutWAL(ctx, storage, staticWALKey, wal) + if err != nil { + t.Fatal(err) + } + requireWALs(t, storage, 4) + + walMap, err := b.loadStaticWALs(ctx, storage) + if err != nil { + t.Fatal(err) + } + if len(walMap) != 1 || walMap["hashicorp"] == nil || walMap["hashicorp"].walID != walID { + t.Fatal() + } + requireWALs(t, storage, 1) +} + +func generateWALFromFailedRotation(t *testing.T, b *backend, storage logical.Storage, roleName string) { + t.Helper() + // Fail to rotate the roles + ldapClient := b.client.(*fakeLdapClient) + originalValue := ldapClient.throwErrs + ldapClient.throwErrs = true + defer func() { + ldapClient.throwErrs = originalValue + }() + + _, err := b.HandleRequest(context.Background(), &logical.Request{ + Operation: logical.UpdateOperation, + Path: "rotate-role/" + roleName, + Storage: storage, + }) + if err == nil { + t.Fatal("expected error") + } +} + +// returns a slice of the WAL IDs in storage +func requireWALs(t *testing.T, storage logical.Storage, expectedCount int) []string { + t.Helper() + wals, err := storage.List(context.Background(), "wal/") + if err != nil { + t.Fatal(err) + } + if len(wals) != expectedCount { + t.Fatal("expected WALs", expectedCount, "got", len(wals)) + } + + return wals +} diff --git a/scripts/build.sh b/scripts/build.sh index e8a8d1e..d93e852 100755 --- a/scripts/build.sh +++ b/scripts/build.sh @@ -21,10 +21,6 @@ SCRATCH="$DIR/tmp" mkdir -p "$SCRATCH/plugins" echo "--> Vault server" -echo " Writing config" -tee "$SCRATCH/vault.hcl" > /dev/null < Cleaning up" kill -INT "$VAULT_PID" + wait $VAULT_PID rm -rf "$SCRATCH" } trap cleanup EXIT