diff --git a/changelog/pending/20221025--cli--allow-globbing-for-resources-that-do-not-yet-exist.yaml b/changelog/pending/20221025--cli--allow-globbing-for-resources-that-do-not-yet-exist.yaml new file mode 100644 index 000000000000..fa6e8a53b5cd --- /dev/null +++ b/changelog/pending/20221025--cli--allow-globbing-for-resources-that-do-not-yet-exist.yaml @@ -0,0 +1,4 @@ +changes: +- type: feat + scope: cli + description: Allow globbing for resources that do not yet exist diff --git a/pkg/cmd/pulumi/destroy.go b/pkg/cmd/pulumi/destroy.go index 0c746cfdae19..b0f821ce9c7c 100644 --- a/pkg/cmd/pulumi/destroy.go +++ b/pkg/cmd/pulumi/destroy.go @@ -24,6 +24,7 @@ import ( "github.com/pulumi/pulumi/pkg/v3/backend" "github.com/pulumi/pulumi/pkg/v3/backend/display" "github.com/pulumi/pulumi/pkg/v3/engine" + "github.com/pulumi/pulumi/pkg/v3/resource/deploy" "github.com/pulumi/pulumi/pkg/v3/resource/graph" "github.com/pulumi/pulumi/sdk/v3/go/common/resource" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" @@ -176,27 +177,17 @@ func newDestroyCmd() *cobra.Command { return result.FromError(fmt.Errorf("validating stack config: %w", configError)) } - targetUrns := []resource.URN{} - for _, t := range *targets { - targetUrns = append(targetUrns, snap.GlobUrn(resource.URN(t))...) - } - if len(targetUrns) == 0 && len(*targets) > 0 { - if !jsonDisplay { - fmt.Printf("There were no resources matching the wildcards provided.\n") - } - return nil - } - refreshOption, err := getRefreshOption(proj, refresh) if err != nil { return result.FromError(err) } - if targets != nil && len(*targets) > 0 && excludeProtected { + if len(*targets) > 0 && excludeProtected { return result.FromError(errors.New("You cannot specify --target and --exclude-protected")) } var protectedCount int + var targetUrns []string = *targets if excludeProtected { contract.Assert(len(targetUrns) == 0) targetUrns, protectedCount, err = handleExcludeProtected(ctx, s) @@ -218,7 +209,7 @@ func newDestroyCmd() *cobra.Command { Parallel: parallel, Debug: debug, Refresh: refreshOption, - DestroyTargets: targetUrns, + DestroyTargets: deploy.NewUrnTargets(targetUrns), TargetDependents: targetDependents, UseLegacyDiff: useLegacyDiff(), DisableProviderPreview: disableProviderPreview(), @@ -375,7 +366,7 @@ func seperateProtected(resources []*resource.State) ( } // Returns the number of protected resources that remain. Appends all unprotected resources to `targetUrns`. -func handleExcludeProtected(ctx context.Context, s backend.Stack) ([]resource.URN, int, error) { +func handleExcludeProtected(ctx context.Context, s backend.Stack) ([]string, int, error) { // Get snapshot snapshot, err := s.Snapshot(ctx) if err != nil { @@ -384,9 +375,9 @@ func handleExcludeProtected(ctx context.Context, s backend.Stack) ([]resource.UR return nil, 0, errors.New("Failed to find the stack snapshot. Are you in a stack?") } unprotected, protected := seperateProtected(snapshot.Resources) - targetUrns := []resource.URN{} - for _, r := range unprotected { - targetUrns = append(targetUrns, r.URN) + targetUrns := make([]string, len(unprotected)) + for i, r := range unprotected { + targetUrns[i] = string(r.URN) } return targetUrns, len(protected), nil } diff --git a/pkg/cmd/pulumi/preview.go b/pkg/cmd/pulumi/preview.go index 96609fafd019..201481da0940 100644 --- a/pkg/cmd/pulumi/preview.go +++ b/pkg/cmd/pulumi/preview.go @@ -24,8 +24,8 @@ import ( "github.com/pulumi/pulumi/pkg/v3/backend" "github.com/pulumi/pulumi/pkg/v3/backend/display" "github.com/pulumi/pulumi/pkg/v3/engine" + "github.com/pulumi/pulumi/pkg/v3/resource/deploy" "github.com/pulumi/pulumi/sdk/v3/go/common/diag" - "github.com/pulumi/pulumi/sdk/v3/go/common/resource" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" "github.com/pulumi/pulumi/sdk/v3/go/common/util/contract" "github.com/pulumi/pulumi/sdk/v3/go/common/util/result" @@ -168,19 +168,19 @@ func newPreviewCmd() *cobra.Command { return result.FromError(fmt.Errorf("getting stack configuration: %w", err)) } - targetURNs := []resource.URN{} + targetURNs := []string{} for _, t := range targets { - targetURNs = append(targetURNs, resource.URN(t)) + targetURNs = append(targetURNs, t) } - replaceURNs := []resource.URN{} + replaceURNs := []string{} for _, r := range replaces { - replaceURNs = append(replaceURNs, resource.URN(r)) + replaceURNs = append(replaceURNs, r) } for _, tr := range targetReplaces { - targetURNs = append(targetURNs, resource.URN(tr)) - replaceURNs = append(replaceURNs, resource.URN(tr)) + targetURNs = append(targetURNs, tr) + replaceURNs = append(replaceURNs, tr) } refreshOption, err := getRefreshOption(proj, refresh) @@ -194,12 +194,12 @@ func newPreviewCmd() *cobra.Command { Parallel: parallel, Debug: debug, Refresh: refreshOption, - ReplaceTargets: replaceURNs, + ReplaceTargets: deploy.NewUrnTargets(replaceURNs), UseLegacyDiff: useLegacyDiff(), DisableProviderPreview: disableProviderPreview(), DisableResourceReferences: disableResourceReferences(), DisableOutputValues: disableOutputValues(), - UpdateTargets: targetURNs, + UpdateTargets: deploy.NewUrnTargets(targetURNs), TargetDependents: targetDependents, // If we're trying to save a plan then we _need_ to generate it. We also turn this on in // experimental mode to just get more testing of it. diff --git a/pkg/cmd/pulumi/refresh.go b/pkg/cmd/pulumi/refresh.go index ac8052089128..cef5662e1c4b 100644 --- a/pkg/cmd/pulumi/refresh.go +++ b/pkg/cmd/pulumi/refresh.go @@ -212,9 +212,9 @@ func newRefreshCmd() *cobra.Command { } } - targetUrns := []resource.URN{} + targetUrns := []string{} for _, t := range *targets { - targetUrns = append(targetUrns, resource.URN(t)) + targetUrns = append(targetUrns, t) } opts.Engine = engine.UpdateOptions{ @@ -224,7 +224,7 @@ func newRefreshCmd() *cobra.Command { DisableProviderPreview: disableProviderPreview(), DisableResourceReferences: disableResourceReferences(), DisableOutputValues: disableOutputValues(), - RefreshTargets: targetUrns, + RefreshTargets: deploy.NewUrnTargets(targetUrns), } changes, res := s.Refresh(ctx, backend.UpdateOperation{ diff --git a/pkg/cmd/pulumi/up.go b/pkg/cmd/pulumi/up.go index 5b573c256f6f..d85083aee0bb 100644 --- a/pkg/cmd/pulumi/up.go +++ b/pkg/cmd/pulumi/up.go @@ -29,7 +29,6 @@ import ( "github.com/pulumi/pulumi/pkg/v3/engine" "github.com/pulumi/pulumi/pkg/v3/resource/deploy" "github.com/pulumi/pulumi/pkg/v3/resource/stack" - "github.com/pulumi/pulumi/sdk/v3/go/common/resource" "github.com/pulumi/pulumi/sdk/v3/go/common/resource/config" "github.com/pulumi/pulumi/sdk/v3/go/common/tokens" "github.com/pulumi/pulumi/sdk/v3/go/common/util/cmdutil" @@ -121,40 +120,19 @@ func newUpCmd() *cobra.Command { return result.FromError(fmt.Errorf("validating stack config: %w", configErr)) } - targetURNs, replaceURNs := []resource.URN{}, []resource.URN{} + targetURNs, replaceURNs := []string{}, []string{} - if len(targets)+len(replaces)+len(targetReplaces) > 0 { - // The s.Snapshot call below adds needless latency as s.Update further below will call - // (*cloudBackend).getSnapshot again, presumably re-retrieving the same result over the network. - // Although s.Snapshot has a cache, it does not get hit by s.Update as of this writing. For now: - // only call s.Snapshot here if targets, replaces, or targetReplaces require it. - snap, err := s.Snapshot(ctx) - if err != nil { - return result.FromError(err) - } - for _, t := range targets { - targetURNs = append(targetURNs, snap.GlobUrn(resource.URN(t))...) - } - - for _, r := range replaces { - replaceURNs = append(replaceURNs, snap.GlobUrn(resource.URN(r))...) - } + for _, t := range targets { + targetURNs = append(targetURNs, t) + } - for _, tr := range targetReplaces { - targetURNs = append(targetURNs, snap.GlobUrn(resource.URN(tr))...) - replaceURNs = append(replaceURNs, snap.GlobUrn(resource.URN(tr))...) - } + for _, r := range replaces { + replaceURNs = append(replaceURNs, r) } - if len(targetURNs) == 0 && len(targets)+len(targetReplaces) > 0 { - // Wildcards were used, but they all evaluated to empty. We don't - // want a targeted update to turn into a general update, so we - // should abort. - if !jsonDisplay { - fmt.Printf("There were no resources matching the wildcards provided.\n") - fmt.Printf("Wildcards can only be used to target resources that already exist.\n") - } - return nil + for _, tr := range targetReplaces { + targetURNs = append(targetURNs, tr) + replaceURNs = append(replaceURNs, tr) } refreshOption, err := getRefreshOption(proj, refresh) @@ -166,13 +144,13 @@ func newUpCmd() *cobra.Command { Parallel: parallel, Debug: debug, Refresh: refreshOption, - RefreshTargets: targetURNs, - ReplaceTargets: replaceURNs, + RefreshTargets: deploy.NewUrnTargets(targetURNs), + ReplaceTargets: deploy.NewUrnTargets(replaceURNs), UseLegacyDiff: useLegacyDiff(), DisableProviderPreview: disableProviderPreview(), DisableResourceReferences: disableResourceReferences(), DisableOutputValues: disableOutputValues(), - UpdateTargets: targetURNs, + UpdateTargets: deploy.NewUrnTargets(targetURNs), TargetDependents: targetDependents, // If we're in experimental mode then we trigger a plan to be generated during the preview phase // which will be constrained to during the update phase. @@ -527,7 +505,7 @@ func newUpCmd() *cobra.Command { " Wildcards (*, **) are also supported") cmd.PersistentFlags().StringArrayVar( &replaces, "replace", []string{}, - "Specify resources to replace. Multiple resources can be specified using --replace urn1 --replace urn2."+ + "Specify a single resource URN to replace. Multiple resources can be specified using --replace urn1 --replace urn2."+ " Wildcards (*, **) are also supported") cmd.PersistentFlags().StringArrayVar( &targetReplaces, "target-replace", []string{}, diff --git a/pkg/engine/lifecycletest/refresh_test.go b/pkg/engine/lifecycletest/refresh_test.go index df2ac504136e..85504bcc7c62 100644 --- a/pkg/engine/lifecycletest/refresh_test.go +++ b/pkg/engine/lifecycletest/refresh_test.go @@ -336,7 +336,7 @@ func validateRefreshDeleteCombination(t *testing.T, names []string, targets []st refreshTargets = append(refreshTargets, pickURN(t, urns, names, target)) } - p.Options.RefreshTargets = refreshTargets + p.Options.RefreshTargets = deploy.NewUrnTargetsFromUrns(refreshTargets) newResource := func(urn resource.URN, id resource.ID, delete bool, dependencies ...resource.URN) *resource.State { return &resource.State{ @@ -496,10 +496,10 @@ func validateRefreshBasicsCombination(t *testing.T, names []string, targets []st refreshTargets := []resource.URN{} for _, target := range targets { - refreshTargets = append(p.Options.RefreshTargets, pickURN(t, urns, names, target)) + refreshTargets = append(p.Options.RefreshTargets.Literals(), pickURN(t, urns, names, target)) } - p.Options.RefreshTargets = refreshTargets + p.Options.RefreshTargets = deploy.NewUrnTargetsFromUrns(refreshTargets) newResource := func(urn resource.URN, id resource.ID, delete bool, dependencies ...resource.URN) *resource.State { return &resource.State{ diff --git a/pkg/engine/lifecycletest/target_test.go b/pkg/engine/lifecycletest/target_test.go index 751ef0b9906c..6fa46455a59c 100644 --- a/pkg/engine/lifecycletest/target_test.go +++ b/pkg/engine/lifecycletest/target_test.go @@ -118,7 +118,7 @@ func destroySpecificTargets( destroyTargets = append(destroyTargets, pickURN(t, urns, complexTestDependencyGraphNames, target)) } - p.Options.DestroyTargets = destroyTargets + p.Options.DestroyTargets = deploy.NewUrnTargetsFromUrns(destroyTargets) t.Logf("Destroying targets: %v", destroyTargets) // If we're not forcing the targets to be destroyed, then expect to get a failure here as @@ -138,7 +138,7 @@ func destroySpecificTargets( deleted[entry.Step.URN()] = true } - for _, target := range p.Options.DestroyTargets { + for _, target := range p.Options.DestroyTargets.Literals() { assert.Contains(t, deleted, target) } @@ -165,21 +165,24 @@ func TestUpdateTarget(t *testing.T) { t.Run(fmt.Sprintf("update %v", subset), func(t *testing.T) { t.Parallel() - updateSpecificTargets(t, subset, false /*targetDependents*/) + updateSpecificTargets(t, subset, nil, false /*targetDependents*/, -1) }) } } - updateSpecificTargets(t, []string{"A"}, false /*targetDependents*/) + updateSpecificTargets(t, []string{"A"}, nil, false /*targetDependents*/, -1) // Also update a target that doesn't exist to make sure we don't crash or otherwise go off the rails. updateInvalidTarget(t) // We want to check that targetDependents is respected - updateSpecificTargets(t, []string{"C"}, true /*targetDependents*/) + updateSpecificTargets(t, []string{"C"}, nil, true /*targetDependents*/, -1) + + updateSpecificTargets(t, nil, []string{"**C**"}, false, 1) + updateSpecificTargets(t, nil, []string{"**providers:pkgA**"}, false, 3) } -func updateSpecificTargets(t *testing.T, targets []string, targetDependents bool) { +func updateSpecificTargets(t *testing.T, targets, globTargets []string, targetDependents bool, expectedUpdates int) { // A // _________|_________ // B C D @@ -219,13 +222,13 @@ func updateSpecificTargets(t *testing.T, targets []string, targetDependents bool p.Options.Host = deploytest.NewPluginHost(nil, nil, program, loaders...) p.Options.TargetDependents = targetDependents - updateTargets := []resource.URN{} + updateTargets := globTargets for _, target := range targets { updateTargets = append(updateTargets, - pickURN(t, urns, complexTestDependencyGraphNames, target)) + string(pickURN(t, urns, complexTestDependencyGraphNames, target))) } - p.Options.UpdateTargets = updateTargets + p.Options.UpdateTargets = deploy.NewUrnTargets(updateTargets) t.Logf("Updating targets: %v", updateTargets) p.Steps = []TestStep{{ @@ -249,13 +252,13 @@ func updateSpecificTargets(t *testing.T, targets []string, targetDependents bool } } - for _, target := range p.Options.UpdateTargets { + for _, target := range p.Options.UpdateTargets.Literals() { assert.Contains(t, updated, target) } if !targetDependents { // We should only perform updates on the entries we have targeted. - for _, target := range p.Options.UpdateTargets { + for _, target := range p.Options.UpdateTargets.Literals() { assert.Contains(t, targets, target.Name().String()) } } else { @@ -275,10 +278,12 @@ func updateSpecificTargets(t *testing.T, targets []string, targetDependents bool assert.True(t, found, "Updates: %v", updateList) } - for _, target := range p.Options.UpdateTargets { + for _, target := range p.Options.UpdateTargets.Literals() { assert.NotContains(t, sames, target) } - + if expectedUpdates > -1 { + assert.Equal(t, expectedUpdates, len(updated), "Updates = %#v", updated) + } return res }, }} @@ -325,7 +330,7 @@ func updateInvalidTarget(t *testing.T) { p.Options.Host = deploytest.NewPluginHost(nil, nil, program, loaders...) - p.Options.UpdateTargets = []resource.URN{"foo"} + p.Options.UpdateTargets = deploy.NewUrnTargetsFromUrns([]resource.URN{"foo"}) t.Logf("Updating invalid targets: %v", p.Options.UpdateTargets) p.Steps = []TestStep{{ @@ -374,7 +379,7 @@ func TestCreateDuringTargetedUpdate_CreateMentionedAsTarget(t *testing.T) { resA := p.NewURN("pkgA:m:typA", "resA", "") resB := p.NewURN("pkgA:m:typA", "resB", "") p.Options.Host = host2 - p.Options.UpdateTargets = []resource.URN{resA, resB} + p.Options.UpdateTargets = deploy.NewUrnTargetsFromUrns([]resource.URN{resA, resB}) p.Steps = []TestStep{{ Op: Update, ExpectFailure: false, @@ -436,7 +441,7 @@ func TestCreateDuringTargetedUpdate_UntargetedCreateNotReferenced(t *testing.T) resA := p.NewURN("pkgA:m:typA", "resA", "") p.Options.Host = host2 - p.Options.UpdateTargets = []resource.URN{resA} + p.Options.UpdateTargets = deploy.NewUrnTargetsFromUrns([]resource.URN{resA}) p.Steps = []TestStep{{ Op: Update, ExpectFailure: false, @@ -500,7 +505,7 @@ func TestCreateDuringTargetedUpdate_UntargetedCreateReferencedByTarget(t *testin host2 := deploytest.NewPluginHost(nil, nil, program2, loaders...) p.Options.Host = host2 - p.Options.UpdateTargets = []resource.URN{resA} + p.Options.UpdateTargets = deploy.NewUrnTargetsFromUrns([]resource.URN{resA}) p.Steps = []TestStep{{ Op: Update, ExpectFailure: true, @@ -554,7 +559,7 @@ func TestCreateDuringTargetedUpdate_UntargetedCreateReferencedByUntargetedCreate host2 := deploytest.NewPluginHost(nil, nil, program2, loaders...) p.Options.Host = host2 - p.Options.UpdateTargets = []resource.URN{resA} + p.Options.UpdateTargets = deploy.NewUrnTargetsFromUrns([]resource.URN{resA}) p.Steps = []TestStep{{ Op: Update, ExpectFailure: false, @@ -614,11 +619,11 @@ func TestReplaceSpecificTargets(t *testing.T) { return pickURN(t, urns, complexTestDependencyGraphNames, name) } - p.Options.ReplaceTargets = []resource.URN{ + p.Options.ReplaceTargets = deploy.NewUrnTargetsFromUrns([]resource.URN{ getURN("F"), getURN("B"), getURN("G"), - } + }) p.Steps = []TestStep{{ Op: Update, @@ -639,11 +644,11 @@ func TestReplaceSpecificTargets(t *testing.T) { } } - for _, target := range p.Options.ReplaceTargets { + for _, target := range p.Options.ReplaceTargets.Literals() { assert.Contains(t, replaced, target) } - for _, target := range p.Options.ReplaceTargets { + for _, target := range p.Options.ReplaceTargets.Literals() { assert.NotContains(t, sames, target) } @@ -849,7 +854,7 @@ func destroySpecificTargetsWithChildren( destroyTargets = append(destroyTargets, pickURN(t, urns, componentBasedTestDependencyGraphNames, target)) } - p.Options.DestroyTargets = destroyTargets + p.Options.DestroyTargets = deploy.NewUrnTargetsFromUrns(destroyTargets) t.Logf("Destroying targets: %v", destroyTargets) // If we're not forcing the targets to be destroyed, then expect to get a failure here as @@ -869,7 +874,7 @@ func destroySpecificTargetsWithChildren( deleted[entry.Step.URN()] = true } - for _, target := range p.Options.DestroyTargets { + for _, target := range p.Options.DestroyTargets.Literals() { assert.Contains(t, deleted, target) } diff --git a/pkg/engine/update.go b/pkg/engine/update.go index 0abcca55b72b..bc7fa1fbabd2 100644 --- a/pkg/engine/update.go +++ b/pkg/engine/update.go @@ -113,16 +113,16 @@ type UpdateOptions struct { Refresh bool // Specific resources to refresh during a refresh operation. - RefreshTargets []resource.URN + RefreshTargets deploy.UrnTargets // Specific resources to replace during an update operation. - ReplaceTargets []resource.URN + ReplaceTargets deploy.UrnTargets // Specific resources to destroy during a destroy operation. - DestroyTargets []resource.URN + DestroyTargets deploy.UrnTargets // Specific resources to update during an update operation. - UpdateTargets []resource.URN + UpdateTargets deploy.UrnTargets // true if we're allowing dependent targets to change, even if not specified in one of the above // XXXTargets lists. diff --git a/pkg/resource/deploy/deployment.go b/pkg/resource/deploy/deployment.go index 3129066f098d..5cdf10ba68dc 100644 --- a/pkg/resource/deploy/deployment.go +++ b/pkg/resource/deploy/deployment.go @@ -18,6 +18,8 @@ import ( "context" "fmt" "math" + "regexp" + "strings" "sync" uuid "github.com/gofrs/uuid" @@ -50,20 +52,20 @@ type BackendClient interface { // Options controls the deployment process. type Options struct { - Events Events // an optional events callback interface. - Parallel int // the degree of parallelism for resource operations (<=1 for serial). - Refresh bool // whether or not to refresh before executing the deployment. - RefreshOnly bool // whether or not to exit after refreshing. - RefreshTargets []resource.URN // The specific resources to refresh during a refresh op. - ReplaceTargets []resource.URN // Specific resources to replace. - DestroyTargets []resource.URN // Specific resources to destroy. - UpdateTargets []resource.URN // Specific resources to update. - TargetDependents bool // true if we're allowing things to proceed, even with unspecified targets - TrustDependencies bool // whether or not to trust the resource dependency graph. - UseLegacyDiff bool // whether or not to use legacy diffing behavior. - DisableResourceReferences bool // true to disable resource reference support. - DisableOutputValues bool // true to disable output value support. - GeneratePlan bool // true to enable plan generation. + Events Events // an optional events callback interface. + Parallel int // the degree of parallelism for resource operations (<=1 for serial). + Refresh bool // whether or not to refresh before executing the deployment. + RefreshOnly bool // whether or not to exit after refreshing. + RefreshTargets UrnTargets // The specific resources to refresh during a refresh op. + ReplaceTargets UrnTargets // Specific resources to replace. + DestroyTargets UrnTargets // Specific resources to destroy. + UpdateTargets UrnTargets // Specific resources to update. + TargetDependents bool // true if we're allowing things to proceed, even with unspecified targets + TrustDependencies bool // whether or not to trust the resource dependency graph. + UseLegacyDiff bool // whether or not to use legacy diffing behavior. + DisableResourceReferences bool // true to disable resource reference support. + DisableOutputValues bool // true to disable output value support. + GeneratePlan bool // true to enable plan generation. } // DegreeOfParallelism returns the degree of parallelism that should be used during the @@ -80,6 +82,102 @@ func (o Options) InfiniteParallelism() bool { return o.Parallel == math.MaxInt32 } +// An immutable set of urns to target with an operation. +// +// The zero value of UrnTargets is the set of all URNs. +type UrnTargets struct { + // UrnTargets is internally made up of two components: literals, which are fully + // specified URNs and globs, which are partially specified URNs. + + literals []resource.URN + globs map[string]*regexp.Regexp +} + +// Create a new set of targets. +// +// Each element is considered a glob if it contains any '*' and an URN otherwise. No other +// URN validation is performed. +// +// If len(urnOrGlobs) == 0, an unconstrained set will be created. +func NewUrnTargets(urnOrGlobs []string) UrnTargets { + literals, globs := []resource.URN{}, map[string]*regexp.Regexp{} + for _, urn := range urnOrGlobs { + if strings.ContainsRune(urn, '*') { + globs[urn] = nil + } else { + literals = append(literals, resource.URN(urn)) + } + } + return UrnTargets{literals, globs} +} + +// Create a new set of targets from fully resolved URNs. +func NewUrnTargetsFromUrns(urns []resource.URN) UrnTargets { + return UrnTargets{urns, nil} +} + +// Return if the target set constrains the set of acceptable URNs. +func (t UrnTargets) IsConstrained() bool { + return len(t.literals) > 0 || len(t.globs) > 0 +} + +// Get a regexp that can match on the glob. This function caches regexp generation. +func (t UrnTargets) getMatcher(glob string) *regexp.Regexp { + if r := t.globs[glob]; r != nil { + return r + } + segmentGlob := strings.Split(glob, "**") + for i, v := range segmentGlob { + part := strings.Split(v, "*") + for i, v := range part { + part[i] = regexp.QuoteMeta(v) + } + segmentGlob[i] = strings.Join(part, "[^:]*") + } + + // Because we have quoted all input, this is safe to compile. + r := regexp.MustCompile("^" + strings.Join(segmentGlob, ".*") + "$") + + // We cache and return the matcher + t.globs[glob] = r + return r +} + +// Check if Targets contains the URN. +// +// If method receiver is not initialized, `true` is always returned. +func (t UrnTargets) Contains(urn resource.URN) bool { + if !t.IsConstrained() { + return true + } + for _, literal := range t.literals { + if literal == urn { + return true + } + } + for glob := range t.globs { + if t.getMatcher(glob).MatchString(string(urn)) { + return true + } + } + return false +} + +// URN literals specified as targets. +// +// It doesn't make sense to iterate over all targets, since the list of targets may be +// infinite. +func (t UrnTargets) Literals() []resource.URN { + return t.literals +} + +// Adds a literal iff t is already initialized. +func (t *UrnTargets) addLiteral(urn resource.URN) { + if t.IsConstrained() { + t.literals = append(t.literals, urn) + } +} + // StepExecutorEvents is an interface that can be used to hook resource lifecycle events. type StepExecutorEvents interface { OnResourceStepPre(step Step) (interface{}, error) diff --git a/pkg/resource/deploy/deployment_executor.go b/pkg/resource/deploy/deployment_executor.go index 6d5b81513700..583b80568299 100644 --- a/pkg/resource/deploy/deployment_executor.go +++ b/pkg/resource/deploy/deployment_executor.go @@ -40,28 +40,11 @@ type deploymentExecutor struct { stepExec *stepExecutor // step executor owned by this deployment } -// A set is returned of all the target URNs to facilitate later callers. The set can be 'nil' -// indicating no targets, or will be non-nil and non-empty if there are targets. Only URNs in the -// original array are in the set. i.e. it's only checked for containment. The value of the map is -// unused. -func createTargetMap(targets []resource.URN) map[resource.URN]bool { - if len(targets) == 0 { - return nil - } - - targetMap := make(map[resource.URN]bool) - for _, target := range targets { - targetMap[target] = true - } - - return targetMap -} - // checkTargets validates that all the targets passed in refer to existing resources. Diagnostics // are generated for any target that cannot be found. The target must either have existed in the stack // prior to running the operation, or it must be the urn for a resource that was created. -func (ex *deploymentExecutor) checkTargets(targets []resource.URN, op display.StepOp) result.Result { - if len(targets) == 0 { +func (ex *deploymentExecutor) checkTargets(targets UrnTargets, op display.StepOp) result.Result { + if !targets.IsConstrained() { return nil } @@ -72,12 +55,8 @@ func (ex *deploymentExecutor) checkTargets(targets []resource.URN, op display.St } hasUnknownTarget := false - for _, target := range targets { - hasOld := false - if _, has := olds[target]; has { - hasOld = true - } - + for _, target := range targets.Literals() { + hasOld := olds != nil && olds[target] != nil hasNew := news != nil && news[target] if !hasOld && !hasNew { hasUnknownTarget = true @@ -184,9 +163,9 @@ func (ex *deploymentExecutor) Execute(callerCtx context.Context, opts Options, p // Non-nil means 'update only in this set'. We don't error if the user specifies a target // during `update` that we don't know about because it might be the urn for a resource they // want to create. - updateTargetsOpt := createTargetMap(opts.UpdateTargets) - replaceTargetsOpt := createTargetMap(opts.ReplaceTargets) - destroyTargetsOpt := createTargetMap(opts.DestroyTargets) + updateTargetsOpt := opts.UpdateTargets + replaceTargetsOpt := opts.ReplaceTargets + destroyTargetsOpt := opts.DestroyTargets if res := ex.checkTargets(opts.ReplaceTargets, OpReplace); res != nil { return nil, res } @@ -194,7 +173,7 @@ func (ex *deploymentExecutor) Execute(callerCtx context.Context, opts Options, p return nil, res } - if (updateTargetsOpt != nil || replaceTargetsOpt != nil) && destroyTargetsOpt != nil { + if (updateTargetsOpt.IsConstrained() || replaceTargetsOpt.IsConstrained()) && destroyTargetsOpt.IsConstrained() { contract.Failf("Should not be possible to have both .DestroyTargets and .UpdateTargets or .ReplaceTargets") } @@ -302,7 +281,7 @@ func (ex *deploymentExecutor) Execute(callerCtx context.Context, opts Options, p // Now that we've performed all steps in the deployment, ensure that the list of targets to update was // valid. We have to do this *after* performing the steps as the target list may have referred - // to a resource that was created in one of hte steps. + // to a resource that was created in one of the steps. if res == nil { res = ex.checkTargets(opts.UpdateTargets, OpUpdate) } @@ -372,7 +351,7 @@ func (ex *deploymentExecutor) Execute(callerCtx context.Context, opts Options, p } func (ex *deploymentExecutor) performDeletes( - ctx context.Context, updateTargetsOpt, destroyTargetsOpt map[resource.URN]bool) result.Result { + ctx context.Context, updateTargetsOpt, destroyTargetsOpt UrnTargets) result.Result { defer func() { // We're done here - signal completion so that the step executor knows to terminate. @@ -389,10 +368,10 @@ func (ex *deploymentExecutor) performDeletes( // At this point we have generated the set of resources above that we would normally want to // delete. However, if the user provided -target's we will only actually delete the specific // resources that are in the set explicitly asked for. - var targetsOpt map[resource.URN]bool - if updateTargetsOpt != nil { + var targetsOpt UrnTargets + if updateTargetsOpt.IsConstrained() { targetsOpt = updateTargetsOpt - } else if destroyTargetsOpt != nil { + } else if destroyTargetsOpt.IsConstrained() { targetsOpt = destroyTargetsOpt } @@ -420,7 +399,7 @@ func (ex *deploymentExecutor) performDeletes( // After executing targeted deletes, we may now have resources that depend on the resource that // were deleted. Go through and clean things up accordingly for them. - if targetsOpt != nil { + if targetsOpt.IsConstrained() { resourceToStep := make(map[*resource.State]Step) for _, step := range deleteSteps { resourceToStep[ex.deployment.olds[step.URN()]] = step @@ -562,9 +541,8 @@ func (ex *deploymentExecutor) refresh(callerCtx context.Context, opts Options, p // specific targets. steps := []Step{} resourceToStep := map[*resource.State]Step{} - targetMapOpt := createTargetMap(opts.RefreshTargets) for _, res := range prev.Resources { - if targetMapOpt == nil || targetMapOpt[res.URN] { + if opts.RefreshTargets.Contains(res.URN) { step := NewRefreshStep(ex.deployment, res, nil) steps = append(steps, step) resourceToStep[res] = step diff --git a/pkg/resource/deploy/deployment_test.go b/pkg/resource/deploy/deployment_test.go index 1a77aee0db03..4a4b3c929083 100644 --- a/pkg/resource/deploy/deployment_test.go +++ b/pkg/resource/deploy/deployment_test.go @@ -47,3 +47,66 @@ func TestPendingOperationsDeployment(t *testing.T) { _, err := NewDeployment(&plugin.Context{}, &Target{}, snap, nil, &fixedSource{}, nil, false, nil) assert.NoError(t, err) } + +func TestGlobUrn(t *testing.T) { + t.Parallel() + + globs := []struct { + input string + expected []resource.URN + unexpected []resource.URN + }{ + { + input: "**", + expected: []resource.URN{ + "urn:pulumi:stack::test::typ$aws:resource::aname", + "urn:pulumi:stack::test::typ$aws:resource::bar", + "urn:pulumi:stack::test::typ$azure:resource::bar", + }, + }, + { + input: "urn:pulumi:stack::test::typ*:resource::bar", + expected: []resource.URN{ + "urn:pulumi:stack::test::typ$aws:resource::bar", + "urn:pulumi:stack::test::typ$azure:resource::bar", + }, + unexpected: []resource.URN{ + "urn:pulumi:stack::test::ty:resource::bar", + "urn:pulumi:stack::test::type:resource::foobar", + }, + }, + { + input: "**:aname", + expected: []resource.URN{"urn:pulumi:stack::test::typ$aws:resource::aname"}, + unexpected: []resource.URN{"urn:pulumi:stack::test::typ$aws:resource::somename"}, + }, + { + input: "*:*:stack::test::typ$aws:resource::*", + expected: []resource.URN{ + "urn:pulumi:stack::test::typ$aws:resource::aname", + "urn:pulumi:stack::test::typ$aws:resource::bar", + }, + unexpected: []resource.URN{ + "urn:pulumi:stack::test::typ$azure:resource::aname", + }, + }, + { + input: "stack::test::typ$aws:resource::none", + expected: []resource.URN{"stack::test::typ$aws:resource::none"}, + unexpected: []resource.URN{ + "stack::test::typ$aws:resource::nonee", + }, + }, + } + for _, tt := range globs { + tt := tt + t.Run(tt.input, func(t *testing.T) { + t.Parallel() + + targets := NewUrnTargets([]string{tt.input}) + for _, urn := range tt.expected { + assert.True(t, targets.Contains(urn)) + } + }) + } +} diff --git a/pkg/resource/deploy/snapshot.go b/pkg/resource/deploy/snapshot.go index c47d320a7495..e352a56c94ed 100644 --- a/pkg/resource/deploy/snapshot.go +++ b/pkg/resource/deploy/snapshot.go @@ -16,9 +16,6 @@ package deploy import ( "fmt" - "regexp" - "sort" - "strings" "github.com/pulumi/pulumi/pkg/v3/resource/deploy/providers" "github.com/pulumi/pulumi/pkg/v3/secrets" @@ -184,48 +181,6 @@ func (snap *Snapshot) VerifyIntegrity() error { return nil } -// Performs glob style expansion on urns that contain '*'. Each urn can be -// expanded into 0-n actual urns, depending on what underlying resources exist -// in the snapshot. URNs are returned in sorted order. All returned urns are unique. -func (snap *Snapshot) GlobUrn(urn resource.URN) []resource.URN { - if !strings.Contains(string(urn), "*") { - return []resource.URN{urn} - } - segmentGlob := strings.Split(string(urn), "**") - for i, v := range segmentGlob { - part := strings.Split(v, "*") - for i, v := range part { - part[i] = regexp.QuoteMeta(v) - } - segmentGlob[i] = strings.Join(part, "[^:]*") - } - - // Because we have quoted all input, this is safe to compile. - glob := regexp.MustCompile("^" + strings.Join(segmentGlob, ".*") + "$") - - results := make(map[string]struct{}) - for _, r := range snap.Resources { - name := string(r.URN) - if glob.Match([]byte(name)) { - results[name] = struct{}{} - } - } - - // cleanup - result := make([]string, len(results)) - i := 0 - for k := range results { - result[i] = k - i++ - } - urns := make([]resource.URN, len(result)) - sort.Strings(result) - for i, u := range result { - urns[i] = resource.URN(u) - } - return urns -} - // Applies a non-mutating modification for every resource.State in the // Snapshot, returns the edited Snapshot. func (snap *Snapshot) withUpdatedResources(update func(*resource.State) *resource.State) *Snapshot { diff --git a/pkg/resource/deploy/snapshot_test.go b/pkg/resource/deploy/snapshot_test.go index ca2e31f6cc0d..66d2d6ee3255 100644 --- a/pkg/resource/deploy/snapshot_test.go +++ b/pkg/resource/deploy/snapshot_test.go @@ -25,57 +25,6 @@ func createSnapshotPtr() *Snapshot { return &s } -func TestGlobUrn(t *testing.T) { - t.Parallel() - - snap := createSnapshot() - - globs := []struct { - input string - expected []resource.URN - }{ - { - input: "**", - expected: []resource.URN{ - "urn:pulumi:stack::test::typ$aws:resource::aname", - "urn:pulumi:stack::test::typ$aws:resource::bar", - "urn:pulumi:stack::test::typ$azure:resource::bar", - }, - }, - { - input: "urn:pulumi:stack::test::typ*:resource::bar", - expected: []resource.URN{ - "urn:pulumi:stack::test::typ$aws:resource::bar", - "urn:pulumi:stack::test::typ$azure:resource::bar", - }, - }, - { - input: "**:aname", - expected: []resource.URN{"urn:pulumi:stack::test::typ$aws:resource::aname"}, - }, - { - input: "*:*:stack::test::typ$aws:resource::*", - expected: []resource.URN{ - "urn:pulumi:stack::test::typ$aws:resource::aname", - "urn:pulumi:stack::test::typ$aws:resource::bar", - }, - }, - { - input: "stack::test::typ$aws:resource::none", - expected: []resource.URN{"stack::test::typ$aws:resource::none"}, - }, - } - for _, tt := range globs { - tt := tt - t.Run(tt.input, func(t *testing.T) { - t.Parallel() - - actual := snap.GlobUrn(resource.URN(tt.input)) - assert.Equal(t, tt.expected, actual) - }) - } -} - func TestSnapshotNormalizeURNReferences(t *testing.T) { t.Parallel() s1 := createSnapshotPtr() diff --git a/pkg/resource/deploy/step_generator.go b/pkg/resource/deploy/step_generator.go index 746d071dc9f8..2b84add8ae3f 100644 --- a/pkg/resource/deploy/step_generator.go +++ b/pkg/resource/deploy/step_generator.go @@ -38,8 +38,8 @@ type stepGenerator struct { deployment *Deployment // the deployment to which this step generator belongs opts Options // options for this step generator - updateTargetsOpt map[resource.URN]bool // the set of resources to update; resources not in this set will be same'd - replaceTargetsOpt map[resource.URN]bool // the set of resoures to replace + updateTargetsOpt UrnTargets // the set of resources to update; resources not in this set will be same'd + replaceTargetsOpt UrnTargets // the set of resoures to replace // signals that one or more errors have been reported to the user, and the deployment should terminate // in error. This primarily allows `preview` to aggregate many policy violation events and @@ -72,32 +72,33 @@ type stepGenerator struct { } func (sg *stepGenerator) isTargetedUpdate() bool { - return sg.updateTargetsOpt != nil || sg.replaceTargetsOpt != nil + return sg.updateTargetsOpt.IsConstrained() || sg.replaceTargetsOpt.IsConstrained() } // isTargetedForUpdate returns if `res` is targeted for update. The function accommodates // `--target-dependents`. `targetDependentsForUpdate` should probably be called if this function // returns true. func (sg *stepGenerator) isTargetedForUpdate(res *resource.State) bool { - if sg.updateTargetsOpt == nil || sg.updateTargetsOpt[res.URN] { + if sg.updateTargetsOpt.Contains(res.URN) { return true } else if !sg.opts.TargetDependents { return false } + if res.Provider != "" { res, err := providers.ParseReference(res.Provider) contract.AssertNoError(err) - if sg.updateTargetsOpt[res.URN()] { + if sg.updateTargetsOpt.Contains(res.URN()) { return true } } if res.Parent != "" { - if sg.updateTargetsOpt[res.Parent] { + if sg.updateTargetsOpt.Contains(res.Parent) { return true } } for _, dep := range res.Dependencies { - if dep != "" && sg.updateTargetsOpt[dep] { + if dep != "" && sg.updateTargetsOpt.Contains(dep) { return true } } @@ -105,7 +106,7 @@ func (sg *stepGenerator) isTargetedForUpdate(res *resource.State) bool { } func (sg *stepGenerator) isTargetedReplace(urn resource.URN) bool { - return sg.replaceTargetsOpt != nil && sg.replaceTargetsOpt[urn] + return sg.replaceTargetsOpt.IsConstrained() && sg.replaceTargetsOpt.Contains(urn) } func (sg *stepGenerator) Errored() bool { @@ -724,8 +725,8 @@ func (sg *stepGenerator) generateSteps(event RegisterResourceEvent) ([]Step, res } isTargeted := sg.isTargetedForUpdate(new) - if isTargeted && sg.updateTargetsOpt != nil { - sg.updateTargetsOpt[urn] = true + if isTargeted { + sg.updateTargetsOpt.addLiteral(urn) } // Case 3: hasOld @@ -996,7 +997,7 @@ func (sg *stepGenerator) generateStepsFromDiff( return nil, nil } -func (sg *stepGenerator) GenerateDeletes(targetsOpt map[resource.URN]bool) ([]Step, result.Result) { +func (sg *stepGenerator) GenerateDeletes(targetsOpt UrnTargets) ([]Step, result.Result) { // To compute the deletion list, we must walk the list of old resources *backwards*. This is because the list is // stored in dependency order, and earlier elements are possibly leaf nodes for later elements. We must not delete // dependencies prior to their dependent nodes. @@ -1109,7 +1110,7 @@ func (sg *stepGenerator) GenerateDeletes(targetsOpt map[resource.URN]bool) ([]St deletingUnspecifiedTarget := false for _, step := range dels { urn := step.URN() - if targetsOpt != nil && !targetsOpt[urn] && !sg.opts.TargetDependents { + if !targetsOpt.Contains(urn) && !sg.opts.TargetDependents { d := diag.GetResourceWillBeDestroyedButWasNotSpecifiedInTargetList(urn) // Targets were specified, but didn't include this resource to create. Report all the @@ -1139,11 +1140,11 @@ func (sg *stepGenerator) GenerateDeletes(targetsOpt map[resource.URN]bool) ([]St // getTargetDependents returns the (transitive) set of dependents on the target resources. // This includes both implicit and explicit dependents in the DAG itself, as well as children. -func (sg *stepGenerator) getTargetDependents(targetsOpt map[resource.URN]bool) map[resource.URN]bool { +func (sg *stepGenerator) getTargetDependents(targetsOpt UrnTargets) map[resource.URN]bool { // Seed the list with the initial set of targets. var frontier []*resource.State for _, res := range sg.deployment.prev.Resources { - if _, has := targetsOpt[res.URN]; has { + if targetsOpt.Contains(res.URN) { frontier = append(frontier, res) } } @@ -1176,9 +1177,9 @@ func (sg *stepGenerator) getTargetDependents(targetsOpt map[resource.URN]bool) m // will include the targetsOpt resources, but may contain more than just that, if there are dependent // or child resources that require the targets to exist (and so are implicated in the deletion). func (sg *stepGenerator) determineAllowedResourcesToDeleteFromTargets( - targetsOpt map[resource.URN]bool) (map[resource.URN]bool, result.Result) { + targetsOpt UrnTargets) (map[resource.URN]bool, result.Result) { - if targetsOpt == nil { + if !targetsOpt.IsConstrained() { // no specific targets, so we won't filter down anything return nil, nil } @@ -1874,7 +1875,7 @@ func (sg *stepGenerator) AnalyzeResources() result.Result { // newStepGenerator creates a new step generator that operates on the given deployment. func newStepGenerator( - deployment *Deployment, opts Options, updateTargetsOpt, replaceTargetsOpt map[resource.URN]bool) *stepGenerator { + deployment *Deployment, opts Options, updateTargetsOpt, replaceTargetsOpt UrnTargets) *stepGenerator { return &stepGenerator{ deployment: deployment,