Skip to content
Permalink

Comparing changes

Choose two branches to see what’s changed or to start a new pull request. If you need to, you can also or learn more about diff comparisons.

Open a pull request

Create a new pull request by comparing changes across two branches. If you need to, you can also . Learn more about diff comparisons here.
base repository: gruntwork-io/terragrunt
Failed to load repositories. Confirm that selected base ref is valid, then try again.
Loading
base: v0.38.12
Choose a base ref
...
head repository: gruntwork-io/terragrunt
Failed to load repositories. Confirm that selected head ref is valid, then try again.
Loading
compare: v0.39.0
Choose a head ref
  • 2 commits
  • 10 files changed
  • 2 contributors

Commits on Sep 28, 2022

  1. Copy the full SHA
    5dd6ea5 View commit details

Commits on Sep 29, 2022

  1. #2259 Improve handling of failures in outputs reading (#2288)

    * Update dependency handling to return mock outputs if outputs reading fails
    
    * Add test to check render json with mock outputs
    
    * Tests cleanup
    denis256 authored Sep 29, 2022
    Copy the full SHA
    b93f6ef View commit details
18 changes: 16 additions & 2 deletions config/dependency.go
Original file line number Diff line number Diff line change
@@ -31,6 +31,8 @@ import (
"github.com/gruntwork-io/terragrunt/util"
)

const renderJsonCommand = "render-json"

type Dependency struct {
Name string `hcl:",label" cty:"name"`
ConfigPath string `hcl:"config_path,attr" cty:"config_path"`
@@ -379,7 +381,7 @@ func (dependencyConfig Dependency) shouldReturnMockOutputs(terragruntOptions *op
dependencyConfig.MockOutputsAllowedTerraformCommands == nil ||
len(*dependencyConfig.MockOutputsAllowedTerraformCommands) == 0 ||
util.ListContainsElement(*dependencyConfig.MockOutputsAllowedTerraformCommands, terragruntOptions.OriginalTerraformCommand)
return defaultOutputsSet && allowedCommand
return defaultOutputsSet && allowedCommand || isRenderJsonCommand(terragruntOptions)
}

// Return the output from the state of another module, managed by terragrunt. This function will parse the provided
@@ -395,7 +397,14 @@ func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options

jsonBytes, err := getOutputJsonWithCaching(targetConfig, terragruntOptions)
if err != nil {
return nil, true, err
if !isRenderJsonCommand(terragruntOptions) {
return nil, true, err
}
terragruntOptions.Logger.Warnf("Failed to read outputs from %s referenced in %s as %s, fallback to mock outputs. Error: %v", targetConfig, terragruntOptions.TerragruntConfigPath, dependencyConfig.Name, err)
jsonBytes, err = json.Marshal(dependencyConfig.MockOutputs)
if err != nil {
return nil, true, err
}
}
isEmpty := string(jsonBytes) == "{}"

@@ -412,6 +421,11 @@ func getTerragruntOutput(dependencyConfig Dependency, terragruntOptions *options
return &convertedOutput, isEmpty, errors.WithStackTrace(err)
}

// This function will true if terragrunt was invoked with renderJsonCommand
func isRenderJsonCommand(terragruntOptions *options.TerragruntOptions) bool {
return util.ListContainsElement(terragruntOptions.TerraformCliArgs, renderJsonCommand)
}

// getOutputJsonWithCaching will run terragrunt output on the target config if it is not already cached.
func getOutputJsonWithCaching(targetConfig string, terragruntOptions *options.TerragruntOptions) ([]byte, error) {
// Acquire synchronization lock to ensure only one instance of output is called per config.
2 changes: 1 addition & 1 deletion configstack/stack.go
Original file line number Diff line number Diff line change
@@ -123,7 +123,7 @@ func (stack *Stack) summarizePlanAllErrors(terragruntOptions *options.Terragrunt
dependenciesMsg = fmt.Sprintf(" contains dependencies to %v and", stack.Modules[i].Config.Dependencies.Paths)
}
terragruntOptions.Logger.Infof("%v%v refers to remote state "+
"you may have to apply your changes in the dependencies prior running terragrunt plan-all.\n",
"you may have to apply your changes in the dependencies prior running terragrunt run-all plan.\n",
stack.Modules[i].Path,
dependenciesMsg,
)
Original file line number Diff line number Diff line change
@@ -86,7 +86,7 @@ Finally, if you make some changes to your project, you could evaluate the impact
Note: It is important to realize that you could get errors running `run-all plan` if you have dependencies between your
projects and some of those dependencies haven’t been applied yet.

*Ex: If module A depends on module B and module B hasn’t been applied yet, then plan-all will show the plan for B, but exit with an error when trying to show the plan for A.*
*Ex: If module A depends on module B and module B hasn’t been applied yet, then run-all plan will show the plan for B, but exit with an error when trying to show the plan for A.*

cd root
terragrunt run-all plan
@@ -160,11 +160,11 @@ If any of the modules failed to deploy, then Terragrunt will not attempt to depl

Terragrunt will return an error indicating the dependency hasn’t been applied yet if the terraform module managed by the terragrunt config referenced in a `dependency` block has not been applied yet. This is because you cannot actually fetch outputs out of an unapplied Terraform module, even if there are no resources being created in the module.

This is most problematic when running commands that do not modify state (e.g `plan-all` and `validate-all`) on a completely new setup where no infrastructure has been deployed. You won’t be able to `plan` or `validate` a module if you can’t determine the `inputs`. If the module depends on the outputs of another module that hasn’t been applied yet, you won’t be able to compute the `inputs` unless the dependencies are all applied. However, in real life usage, you would want to run `validate-all` or `plan-all` on a completely new set of infrastructure.
This is most problematic when running commands that do not modify state (e.g `run-all plan` and `run-all validate`) on a completely new setup where no infrastructure has been deployed. You won’t be able to `plan` or `validate` a module if you can’t determine the `inputs`. If the module depends on the outputs of another module that hasn’t been applied yet, you won’t be able to compute the `inputs` unless the dependencies are all applied. However, in real life usage, you would want to run `run-all validate` or `run-all plan` on a completely new set of infrastructure.

To address this, you can provide mock outputs to use when a module hasn’t been applied yet. This is configured using the `mock_outputs` attribute on the `dependency` block and it corresponds to a map that will be injected in place of the actual dependency outputs if the target config hasn’t been applied yet.

For example, in the previous example with a `mysql` module and `vpc` module, suppose you wanted to place in a temporary, dummy value for the `vpc_id` during a `validate-all` for the `mysql` module. You can specify in `mysql/terragrunt.hcl`:
For example, in the previous example with a `mysql` module and `vpc` module, suppose you wanted to place in a temporary, dummy value for the `vpc_id` during a `run-all validate` for the `mysql` module. You can specify in `mysql/terragrunt.hcl`:

dependency "vpc" {
config_path = "../vpc"
@@ -197,7 +197,7 @@ You can use the `mock_outputs_allowed_terraform_commands` attribute to indicate
vpc_id = dependency.vpc.outputs.vpc_id
}

Note that indicating `validate` means that the `mock_outputs` will be used either with `validate` or with `validate-all`.
Note that indicating `validate` means that the `mock_outputs` will be used either with `validate` or with `run-all validate`.

You can also use `skip_outputs` on the `dependency` block to specify the dependency without pulling in the outputs:

@@ -292,7 +292,7 @@ Once you’ve specified the dependencies in each `terragrunt.hcl` file, when you

If any of the modules fail to deploy, then Terragrunt will not attempt to deploy the modules that depend on them. Once you’ve fixed the error, it’s usually safe to re-run the `run-all apply` or `run-all destroy` command again, since it’ll be a no-op for the modules that already deployed successfully, and should only affect the ones that had an error the last time around.

To check all of your dependencies and validate the code in them, you can use the `validate-all` command.
To check all of your dependencies and validate the code in them, you can use the `run-all validate` command.

To check the dependency graph you can use the `graph-dependencies` command (similar to the `terraform graph` command),
the graph is output in DOT format The typical program that can read this format is GraphViz, but many web services are also available to read this format.
31 changes: 15 additions & 16 deletions docs/_docs/04_reference/cli-options.md
Original file line number Diff line number Diff line change
@@ -79,10 +79,10 @@ This will recursively search the current working directory for any folders that
[`dependency`](/docs/reference/config-blocks-and-attributes/#dependency) and
[`dependencies`](/docs/reference/config-blocks-and-attributes/#dependencies) blocks.

**[WARNING] Using `run-all` with `plan` is currently broken for certain use cases**. If you have a stack of Terragrunt modules with
dependencies between them—either via `dependency` blocks or `terraform_remote_state` data sources—and you've never
deployed them, then `plan-all` will fail as it will not be possible to resolve the `dependency` blocks or
`terraform_remote_state` data sources! Please [see here for more
**[WARNING] Using `run-all` with `plan` is currently broken for certain use cases**. If you have a stack of Terragrunt
modules with dependencies between them—either via `dependency` blocks or `terraform_remote_state` data sources—and
you've never deployed them, then `run-all plan` will fail as it will not be possible to resolve the `dependency` blocks
or `terraform_remote_state` data sources! Please [see here for more
information](https://github.com/gruntwork-io/terragrunt/issues/720#issuecomment-497888756).

**[NOTE]** Using `run-all` with `apply` or `destroy` silently adds the `-auto-approve` flag to the command line
@@ -103,17 +103,17 @@ context.
Example:

```bash
terragrunt plan-all
terragrunt run-all plan
```

This will recursively search the current working directory for any folders that contain Terragrunt modules and run
`plan` in each one, concurrently, while respecting ordering defined via
[`dependency`](/docs/reference/config-blocks-and-attributes/#dependency) and
[`dependencies`](/docs/reference/config-blocks-and-attributes/#dependencies) blocks.

**[WARNING] `plan-all` is currently broken for certain use cases**. If you have a stack of Terragrunt modules with
**[WARNING] `run-all plan` is currently broken for certain use cases**. If you have a stack of Terragrunt modules with
dependencies between them—either via `dependency` blocks or `terraform_remote_state` data sources—and you've never
deployed them, then `plan-all` will fail as it will not be possible to resolve the `dependency` blocks or
deployed them, then `run-all plan` will fail as it will not be possible to resolve the `dependency` blocks or
`terraform_remote_state` data sources! Please [see here for more
information](https://github.com/gruntwork-io/terragrunt/issues/720#issuecomment-497888756).

@@ -511,8 +511,7 @@ prefix `--terragrunt-` (e.g., `--terragrunt-config`). The currently available op
A custom path to the `terragrunt.hcl` or `terragrunt.hcl.json` file. The
default path is `terragrunt.hcl` (preferred) or `terragrunt.hcl.json` in the current directory (see
[Configuration]({{site.baseurl}}/docs/getting-started/configuration/#configuration) for a slightly more nuanced
explanation). This argument is not used with the `apply-all`, `destroy-all`, `output-all`, `validate-all`, and
`plan-all` commands.
explanation). This argument is not used with the `run-all` commands.


### terragrunt-tfpath
@@ -584,9 +583,9 @@ This setting will default to `no` for the following cases:
**Requires an argument**: `--terragrunt-working-dir /path/to/working-directory`

Set the directory where Terragrunt should execute the `terraform` command. Default is the current working directory.
Note that for the `apply-all`, `destroy-all`, `output-all`, `validate-all`, and `plan-all` commands, this parameter has
a different meaning: Terragrunt will apply or destroy all the Terraform modules in the subfolders of the
`terragrunt-working-dir`, running `terraform` in the root of each module it finds.
Note that for the `run-all` commands, this parameter has a different meaning: Terragrunt will apply or destroy all the
Terraform modules in the subfolders of the `terragrunt-working-dir`, running `terraform` in the root of each module it
finds.


### terragrunt-download-dir
@@ -608,10 +607,10 @@ Default is `.terragrunt-cache` in the working directory. We recommend adding thi

Download Terraform configurations from the specified source into a temporary folder, and run Terraform in that temporary
folder. The source should use the same syntax as the [Terraform module
source](https://www.terraform.io/docs/modules/sources.html) parameter. If you specify this argument for the `apply-all`,
`destroy-all`, `output-all`, `validate-all`, or `plan-all` commands, Terragrunt will assume this is the local file path
for all of your Terraform modules, and for each module processed by the `xxx-all` command, Terragrunt will automatically
append the path of `source` parameter in each module to the `--terragrunt-source` parameter you passed in.
source](https://www.terraform.io/docs/modules/sources.html) parameter. If you specify this argument for the `run-all`
commands, Terragrunt will assume this is the local file path for all of your Terraform modules, and for each module
processed by the `run-all` command, Terragrunt will automatically append the path of `source` parameter in each module
to the `--terragrunt-source` parameter you passed in.


### terragrunt-source-map
Empty file.
13 changes: 13 additions & 0 deletions test/fixture-render-json-mock-outputs/app/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
include "root" {
path = find_in_parent_folders()
}

dependency "module" {
config_path = "../dependency"

mock_outputs = {
security_group_id = "sg-abcd1234"
bastion_host_security_group_id = "123"
}
mock_outputs_allowed_terraform_commands = ["validate" ]
}
Empty file.
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
include "root" {
path = find_in_parent_folders()
}
14 changes: 14 additions & 0 deletions test/fixture-render-json-mock-outputs/terragrunt.hcl
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
remote_state {
backend = "s3"
config = {
encrypt = true
bucket = "test-tf-state"
key = "${path_relative_to_include()}/terraform.tfstate"
dynamodb_table = "test-terraform-locks"
accesslogging_bucket_name = "test-tf-logs"
}
generate = {
path = "backend.tf"
if_exists = "overwrite_terragrunt"
}
}
50 changes: 50 additions & 0 deletions test/integration_test.go
Original file line number Diff line number Diff line change
@@ -130,6 +130,7 @@ const (
TEST_FIXTURE_BROKEN_LOCALS = "fixture-broken-locals"
TEST_FIXTURE_BROKEN_DEPENDENCY = "fixture-broken-dependency"
TEST_FIXTURE_RENDER_JSON_METADATA = "fixture-render-json-metadata"
TEST_FIXTURE_RENDER_JSON_MOCK_OUTPUTS = "fixture-render-json-mock-outputs"
TERRAFORM_BINARY = "terraform"
TERRAFORM_FOLDER = ".terraform"
TERRAFORM_STATE = "terraform.tfstate"
@@ -4699,6 +4700,55 @@ func TestRenderJsonMetadataDependencies(t *testing.T) {
assert.True(t, reflect.DeepEqual(expectedDependencies, dependencies))
}

func TestRenderJsonWithMockOutputs(t *testing.T) {
t.Parallel()

tmpEnvPath := copyEnvironment(t, TEST_FIXTURE_RENDER_JSON_MOCK_OUTPUTS)
cleanupTerraformFolder(t, tmpEnvPath)
tmpDir := util.JoinPath(tmpEnvPath, TEST_FIXTURE_RENDER_JSON_MOCK_OUTPUTS, "app")

var expectedMetadata = map[string]interface{}{
"found_in_file": util.JoinPath(tmpDir, "terragrunt.hcl"),
}

jsonOut := filepath.Join(tmpDir, "terragrunt_rendered.json")

runTerragrunt(t, fmt.Sprintf("terragrunt render-json --with-metadata --terragrunt-non-interactive --terragrunt-log-level debug --terragrunt-working-dir %s --terragrunt-json-out %s", tmpDir, jsonOut))

jsonBytes, err := ioutil.ReadFile(jsonOut)
require.NoError(t, err)

var renderedJson = map[string]interface{}{}
require.NoError(t, json.Unmarshal(jsonBytes, &renderedJson))

dependency := renderedJson[config.MetadataDependency]

var expectedDependency = map[string]interface{}{
"module": map[string]interface{}{
"metadata": expectedMetadata,
"value": map[string]interface{}{
"config_path": "../dependency",
"mock_outputs": map[string]interface{}{
"bastion_host_security_group_id": "123",
"security_group_id": "sg-abcd1234",
},
"mock_outputs_allowed_terraform_commands": [1]string{"validate"},
"mock_outputs_merge_strategy_with_state": nil,
"mock_outputs_merge_with_state": nil,
"name": "module",
"outputs": nil,
"skip": nil,
},
},
}
serializedDependency, err := json.Marshal(dependency)
assert.NoError(t, err)

serializedExpectedDependency, err := json.Marshal(expectedDependency)
assert.NoError(t, err)
assert.Equal(t, string(serializedExpectedDependency), string(serializedDependency))
}

func TestRenderJsonMetadataIncludes(t *testing.T) {
t.Parallel()