diff --git a/internal/command/init.go b/internal/command/init.go index 415cbe06d96a..625f5d39fdb8 100644 --- a/internal/command/init.go +++ b/internal/command/init.go @@ -4,6 +4,8 @@ import ( "context" "fmt" "log" + "reflect" + "sort" "strings" "github.com/hashicorp/hcl/v2" @@ -543,6 +545,11 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, ctx, done := c.InterruptibleContext() defer done() + // We want to print out a nice warning if we don't manage to pull + // checksums for all our providers. This is tracked via callbacks + // and incomplete providers are stored here for later analysis. + var incompleteProviders []string + // Because we're currently just streaming a series of events sequentially // into the terminal, we're showing only a subset of the events to keep // things relatively concise. Later it'd be nice to have a progress UI @@ -784,6 +791,41 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, c.Ui.Info(fmt.Sprintf("- Installed %s v%s (%s%s)", provider.ForDisplay(), version, authResult, keyID)) }, + ProvidersLockUpdated: func(provider addrs.Provider, version getproviders.Version, localHashes []getproviders.Hash, signedHashes []getproviders.Hash, priorHashes []getproviders.Hash) { + // We're going to use this opportunity to track if we have any + // "incomplete" installs of providers. An incomplete install is + // when we are only going to write the local hashes into our lock + // file which means a `terraform init` command will fail in future + // when used on machines of a different architecture. + // + // We want to print a warning about this. + + if len(signedHashes) > 0 { + // If we have any signedHashes hashes then we don't worry - as + // we know we retrieved all available hashes for this version + // anyway. + return + } + + // If local hashes and prior hashes are exactly the same then + // it means we didn't record any signed hashes previously, and + // we know we're not adding any extra in now (because we already + // checked the signedHashes), so that's a problem. + // + // In the actual check here, if we have any priorHashes and those + // hashes are not the same as the local hashes then we're going to + // accept that this provider has been configured correctly. + if len(priorHashes) > 0 && !reflect.DeepEqual(localHashes, priorHashes) { + return + } + + // Now, either signedHashes is empty, or priorHashes is exactly the + // same as our localHashes which means we never retrieved the + // signedHashes previously. + // + // Either way, this is bad. Let's complain/warn. + incompleteProviders = append(incompleteProviders, provider.ForDisplay()) + }, ProvidersFetched: func(authResults map[addrs.Provider]*getproviders.PackageAuthenticationResult) { thirdPartySigned := false for _, authResult := range authResults { @@ -798,18 +840,6 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, "https://www.terraform.io/docs/cli/plugins/signing.html")) } }, - HashPackageFailure: func(provider addrs.Provider, version getproviders.Version, err error) { - diags = diags.Append(tfdiags.Sourceless( - tfdiags.Error, - "Failed to validate installed provider", - fmt.Sprintf( - "Validating provider %s v%s failed: %s", - provider.ForDisplay(), - version, - err, - ), - )) - }, } ctx = evts.OnContext(ctx) @@ -869,6 +899,22 @@ func (c *InitCommand) getProviders(config *configs.Config, state *states.State, return true, false, diags } + // Jump in here and add a warning if any of the providers are incomplete. + if len(incompleteProviders) > 0 { + // We don't really care about the order here, we just want the + // output to be deterministic. + sort.Slice(incompleteProviders, func(i, j int) bool { + return incompleteProviders[i] < incompleteProviders[j] + }) + diags = diags.Append(tfdiags.Sourceless( + tfdiags.Warning, + incompleteLockFileInformationHeader, + fmt.Sprintf( + incompleteLockFileInformationBody, + strings.Join(incompleteProviders, "\n - "), + getproviders.CurrentPlatform.String()))) + } + if previousLocks.Empty() { // A change from empty to non-empty is special because it suggests // we're running "terraform init" for the first time against a @@ -1190,3 +1236,18 @@ Alternatively, upgrade to the latest version of Terraform for compatibility with // No version of the provider is compatible. const errProviderVersionIncompatible = `No compatible versions of provider %s were found.` + +// incompleteLockFileInformationHeader is the summary displayed to users when +// the lock file has only recorded local hashes. +const incompleteLockFileInformationHeader = `Incomplete lock file information for providers` + +// incompleteLockFileInformationBody is the body of text displayed to users when +// the lock file has only recorded local hashes. +const incompleteLockFileInformationBody = `Due to your customized provider installation methods, Terraform was forced to calculate lock file checksums locally for the following providers: + - %s + +The current .terraform.lock.hcl file only includes checksums for %s, so Terraform running on another platform will fail to install these providers. + +To calculate additional checksums for another platform, run: + terraform providers lock -platform=linux_amd64 +(where linux_amd64 is the platform to generate)` diff --git a/internal/command/init_test.go b/internal/command/init_test.go index c97be55faa18..2609677bdf1c 100644 --- a/internal/command/init_test.go +++ b/internal/command/init_test.go @@ -1644,7 +1644,7 @@ func TestInit_providerSource(t *testing.T) { }) defer close() - ui := new(cli.MockUi) + ui := cli.NewMockUi() view, _ := testView(t) m := Meta{ testingOverrides: metaOverridesForProvider(testProvider()), @@ -1726,13 +1726,16 @@ func TestInit_providerSource(t *testing.T) { }, ), } + if diff := cmp.Diff(gotProviderLocks, wantProviderLocks, depsfile.ProviderLockComparer); diff != "" { t.Errorf("wrong version selections after upgrade\n%s", diff) } - outputStr := ui.OutputWriter.String() - if want := "Installed hashicorp/test v1.2.3 (verified checksum)"; !strings.Contains(outputStr, want) { - t.Fatalf("unexpected output: %s\nexpected to include %q", outputStr, want) + if got, want := ui.OutputWriter.String(), "Installed hashicorp/test v1.2.3 (verified checksum)"; !strings.Contains(got, want) { + t.Fatalf("unexpected output: %s\nexpected to include %q", got, want) + } + if got, want := ui.ErrorWriter.String(), "\n - hashicorp/source\n - hashicorp/test\n - hashicorp/test-beta"; !strings.Contains(got, want) { + t.Fatalf("wrong error message\nshould contain: %s\ngot:\n%s", want, got) } } diff --git a/internal/providercache/installer.go b/internal/providercache/installer.go index e753a73c04af..129478868e7c 100644 --- a/internal/providercache/installer.go +++ b/internal/providercache/installer.go @@ -385,14 +385,14 @@ NeedProvider: // calculated from the package we just linked, which allows // the lock file to gradually transition to recording newer hash // schemes when they become available. - var newHashes []getproviders.Hash + var priorHashes []getproviders.Hash if lock != nil && lock.Version() == version { // If the version we're installing is identical to the // one we previously locked then we'll keep all of the // hashes we saved previously and add to it. Otherwise // we'll be starting fresh, because each version has its // own set of packages and thus its own hashes. - newHashes = append(newHashes, preferredHashes...) + priorHashes = append(priorHashes, preferredHashes...) // NOTE: The behavior here is unfortunate when a particular // provider version was already cached on the first time @@ -423,8 +423,17 @@ NeedProvider: // The hashes slice gets deduplicated in the lock file // implementation, so we don't worry about potentially // creating a duplicate here. + var newHashes []getproviders.Hash + newHashes = append(newHashes, priorHashes...) newHashes = append(newHashes, newHash) locks.SetProvider(provider, version, reqs[provider], newHashes) + if cb := evts.ProvidersLockUpdated; cb != nil { + // We want to ensure that newHash and priorHashes are + // sorted. newHash is a single value, so it's definitely + // sorted. priorHashes are pulled from the lock file, so + // are also already sorted. + cb(provider, version, []getproviders.Hash{newHash}, nil, priorHashes) + } if cb := evts.LinkFromCacheSuccess; cb != nil { cb(provider, version, new.PackageDir) @@ -524,14 +533,14 @@ NeedProvider: // The hashes slice gets deduplicated in the lock file // implementation, so we don't worry about potentially // creating duplicates here. - var newHashes []getproviders.Hash + var priorHashes []getproviders.Hash if lock != nil && lock.Version() == version { // If the version we're installing is identical to the // one we previously locked then we'll keep all of the // hashes we saved previously and add to it. Otherwise // we'll be starting fresh, because each version has its // own set of packages and thus its own hashes. - newHashes = append(newHashes, preferredHashes...) + priorHashes = append(priorHashes, preferredHashes...) } newHash, err := new.Hash() if err != nil { @@ -542,15 +551,32 @@ NeedProvider: } continue } - newHashes = append(newHashes, newHash) + + var signedHashes []getproviders.Hash if authResult.SignedByAnyParty() { // We'll trust new hashes from upstream only if they were verified // as signed by a suitable key. Otherwise, we'd record only // a new hash we just calculated ourselves from the bytes on disk, // and so the hashes would cover only the current platform. - newHashes = append(newHashes, meta.AcceptableHashes()...) + signedHashes = append(signedHashes, meta.AcceptableHashes()...) } + + var newHashes []getproviders.Hash + newHashes = append(newHashes, newHash) + newHashes = append(newHashes, priorHashes...) + newHashes = append(newHashes, signedHashes...) + locks.SetProvider(provider, version, reqs[provider], newHashes) + if cb := evts.ProvidersLockUpdated; cb != nil { + // newHash and priorHashes are already sorted. + // But we do need to sort signedHashes so we can reason about it + // sensibly. + sort.Slice(signedHashes, func(i, j int) bool { + return string(signedHashes[i]) < string(signedHashes[j]) + }) + + cb(provider, version, []getproviders.Hash{newHash}, signedHashes, priorHashes) + } if cb := evts.FetchPackageSuccess; cb != nil { cb(provider, version, new.PackageDir, authResult) diff --git a/internal/providercache/installer_events.go b/internal/providercache/installer_events.go index 8c27cc91421f..8fc579af2666 100644 --- a/internal/providercache/installer_events.go +++ b/internal/providercache/installer_events.go @@ -106,15 +106,28 @@ type InstallerEvents struct { FetchPackageSuccess func(provider addrs.Provider, version getproviders.Version, localDir string, authResult *getproviders.PackageAuthenticationResult) FetchPackageFailure func(provider addrs.Provider, version getproviders.Version, err error) + // The ProvidersLockUpdated event is called whenever the lock file will be + // updated. It provides the following information: + // + // - `localHashes`: Hashes computed on the local system by analyzing + // files on disk. + // - `signedHashes`: Hashes signed by the private key that the origin + // registry claims is the owner of this provider. + // - `priorHashes`: Hashes already present in the lock file before we + // made any changes. + // + // The final lock file will be updated with a union of all the provided + // hashes. It not just likely, but expected that there will be duplicates + // shared between all three collections of hashes i.e. the local hash and + // remote hashes could already be in the cached hashes. + // + // In addition, we place a guarantee that the hash slices will be ordered + // in the same manner enforced by the lock file within NewProviderLock. + ProvidersLockUpdated func(provider addrs.Provider, version getproviders.Version, localHashes []getproviders.Hash, signedHashes []getproviders.Hash, priorHashes []getproviders.Hash) + // The ProvidersFetched event is called after all fetch operations if at // least one provider was fetched successfully. ProvidersFetched func(authResults map[addrs.Provider]*getproviders.PackageAuthenticationResult) - - // HashPackageFailure is called if the installer is unable to determine - // the hash of the contents of an installed package after installation. - // In that case, the selection will not be recorded in the target cache - // directory's lock file. - HashPackageFailure func(provider addrs.Provider, version getproviders.Version, err error) } // OnContext produces a context with all of the same behaviors as the given diff --git a/internal/providercache/installer_events_test.go b/internal/providercache/installer_events_test.go index 8879fb68a734..cde5b7f0abf5 100644 --- a/internal/providercache/installer_events_test.go +++ b/internal/providercache/installer_events_test.go @@ -164,20 +164,22 @@ func installerLogEventsForTests(into chan<- *testInstallerEventLogItem) *Install }{version.String(), err.Error()}, } }, - ProvidersFetched: func(authResults map[addrs.Provider]*getproviders.PackageAuthenticationResult) { + ProvidersLockUpdated: func(provider addrs.Provider, version getproviders.Version, localHashes []getproviders.Hash, signedHashes []getproviders.Hash, priorHashes []getproviders.Hash) { into <- &testInstallerEventLogItem{ - Event: "ProvidersFetched", - Args: authResults, - } - }, - HashPackageFailure: func(provider addrs.Provider, version getproviders.Version, err error) { - into <- &testInstallerEventLogItem{ - Event: "HashPackageFailure", + Event: "ProvidersLockUpdated", Provider: provider, Args: struct { Version string - Error string - }{version.String(), err.Error()}, + Local []getproviders.Hash + Signed []getproviders.Hash + Prior []getproviders.Hash + }{version.String(), localHashes, signedHashes, priorHashes}, + } + }, + ProvidersFetched: func(authResults map[addrs.Provider]*getproviders.PackageAuthenticationResult) { + into <- &testInstallerEventLogItem{ + Event: "ProvidersFetched", + Args: authResults, } }, } diff --git a/internal/providercache/installer_test.go b/internal/providercache/installer_test.go index 0ca70388833d..2284397047d8 100644 --- a/internal/providercache/installer_test.go +++ b/internal/providercache/installer_test.go @@ -171,6 +171,21 @@ func TestEnsureProviderVersions(t *testing.T) { Location getproviders.PackageLocation }{"2.1.0", beepProviderDir}, }, + { + Event: "ProvidersLockUpdated", + Provider: beepProvider, + Args: struct { + Version string + Local []getproviders.Hash + Signed []getproviders.Hash + Prior []getproviders.Hash + }{ + "2.1.0", + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + nil, + nil, + }, + }, { Event: "FetchPackageSuccess", Provider: beepProvider, @@ -287,6 +302,21 @@ func TestEnsureProviderVersions(t *testing.T) { Location getproviders.PackageLocation }{"2.1.0", beepProviderDir}, }, + { + Event: "ProvidersLockUpdated", + Provider: beepProvider, + Args: struct { + Version string + Local []getproviders.Hash + Signed []getproviders.Hash + Prior []getproviders.Hash + }{ + "2.1.0", + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + nil, + nil, + }, + }, { Event: "FetchPackageSuccess", Provider: beepProvider, @@ -411,6 +441,21 @@ func TestEnsureProviderVersions(t *testing.T) { inst.globalCacheDir.BasePath(), }, }, + { + Event: "ProvidersLockUpdated", + Provider: beepProvider, + Args: struct { + Version string + Local []getproviders.Hash + Signed []getproviders.Hash + Prior []getproviders.Hash + }{ + "2.1.0", + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + nil, + nil, + }, + }, { Event: "LinkFromCacheSuccess", Provider: beepProvider, @@ -535,6 +580,21 @@ func TestEnsureProviderVersions(t *testing.T) { Location getproviders.PackageLocation }{"2.0.0", beepProviderDir}, }, + { + Event: "ProvidersLockUpdated", + Provider: beepProvider, + Args: struct { + Version string + Local []getproviders.Hash + Signed []getproviders.Hash + Prior []getproviders.Hash + }{ + "2.0.0", + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + nil, + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + }, + }, { Event: "FetchPackageSuccess", Provider: beepProvider, @@ -763,6 +823,21 @@ func TestEnsureProviderVersions(t *testing.T) { Location getproviders.PackageLocation }{"2.1.0", beepProviderDir}, }, + { + Event: "ProvidersLockUpdated", + Provider: beepProvider, + Args: struct { + Version string + Local []getproviders.Hash + Signed []getproviders.Hash + Prior []getproviders.Hash + }{ + "2.1.0", + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + nil, + nil, + }, + }, { Event: "FetchPackageSuccess", Provider: beepProvider, @@ -929,6 +1004,21 @@ func TestEnsureProviderVersions(t *testing.T) { Location getproviders.PackageLocation }{"1.0.0", beepProviderDir}, }, + { + Event: "ProvidersLockUpdated", + Provider: beepProvider, + Args: struct { + Version string + Local []getproviders.Hash + Signed []getproviders.Hash + Prior []getproviders.Hash + }{ + "1.0.0", + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + nil, + []getproviders.Hash{"h1:2y06Ykj0FRneZfGCTxI9wRTori8iB7ZL5kQ6YyEnh84="}, + }, + }, { Event: "FetchPackageSuccess", Provider: beepProvider,