From c51cee84466f7c200c614bac22d9e8663d9b41dc Mon Sep 17 00:00:00 2001 From: Nathan Hammond Date: Tue, 11 Apr 2023 21:17:56 +0800 Subject: [PATCH] Add data to run summary. --- cli/integration_tests/dry_json/monorepo.t | 15 ++++- .../dry_json/single_package.t | 6 +- .../dry_json/single_package_no_config.t | 6 +- .../dry_json/single_package_with_deps.t | 11 +++- .../single_package/run-summary.t | 1 + .../strict_env_vars/dry_json.t | 22 ++++++++ cli/internal/env/env.go | 4 ++ cli/internal/graph/graph.go | 22 +++++++- cli/internal/nodes/packagetask.go | 2 + cli/internal/run/dry_run.go | 10 +++- cli/internal/run/global_hash.go | 52 ++++++++--------- cli/internal/run/real_run.go | 23 +++----- cli/internal/run/run.go | 10 ++++ cli/internal/runsummary/format_json.go | 3 + cli/internal/runsummary/globalhash_summary.go | 5 +- cli/internal/runsummary/run_summary.go | 2 + cli/internal/runsummary/task_summary.go | 9 ++- cli/internal/taskhash/taskhash.go | 56 ++++++++++++++++++- 18 files changed, 202 insertions(+), 57 deletions(-) create mode 100644 cli/integration_tests/strict_env_vars/dry_json.t diff --git a/cli/integration_tests/dry_json/monorepo.t b/cli/integration_tests/dry_json/monorepo.t index 372bcbbcfe79c..c4fb1006db946 100644 --- a/cli/integration_tests/dry_json/monorepo.t +++ b/cli/integration_tests/dry_json/monorepo.t @@ -72,6 +72,7 @@ Setup $ cat tmpjson.log | jq 'keys' [ + "envMode", "globalCacheInputs", "id", "packages", @@ -120,13 +121,16 @@ Setup }, "expandedOutputs": [], "framework": "", + "envMode": "Infer", "environmentVariables": { "configured": [], "inferred": [], "global": [ "SOME_ENV_VAR=", "VERCEL_ANALYTICS_ID=" - ] + ], + "passthrough": null, + "globalPassthrough": null } } @@ -166,6 +170,7 @@ Setup }, "expandedOutputs": [], "framework": "", + "envMode": "Infer", "environmentVariables": { "configured": [ "NODE_ENV=" @@ -174,7 +179,9 @@ Setup "global": [ "SOME_ENV_VAR=", "VERCEL_ANALYTICS_ID=" - ] + ], + "passthrough": null, + "globalPassthrough": null } } @@ -188,7 +195,9 @@ Run again with NODE_ENV set and see the value in the summary. --filter=util work "global": [ "SOME_ENV_VAR=", "VERCEL_ANALYTICS_ID=" - ] + ], + "passthrough": null, + "globalPassthrough": null } Tasks that don't exist throw an error diff --git a/cli/integration_tests/dry_json/single_package.t b/cli/integration_tests/dry_json/single_package.t index d401aa8533243..46d9a0580a151 100644 --- a/cli/integration_tests/dry_json/single_package.t +++ b/cli/integration_tests/dry_json/single_package.t @@ -29,6 +29,7 @@ Setup } } }, + "envMode": "Infer", "tasks": [ { "task": "build", @@ -67,12 +68,15 @@ Setup }, "expandedOutputs": [], "framework": "\u003cNO FRAMEWORK DETECTED\u003e", + "envMode": "Infer", "environmentVariables": { "configured": [], "inferred": [], "global": [ "VERCEL_ANALYTICS_ID=" - ] + ], + "passthrough": null, + "globalPassthrough": null } } ] diff --git a/cli/integration_tests/dry_json/single_package_no_config.t b/cli/integration_tests/dry_json/single_package_no_config.t index 37adf3e883142..50f399e144bda 100644 --- a/cli/integration_tests/dry_json/single_package_no_config.t +++ b/cli/integration_tests/dry_json/single_package_no_config.t @@ -26,6 +26,7 @@ Setup } } }, + "envMode": "Infer", "tasks": [ { "task": "build", @@ -58,12 +59,15 @@ Setup }, "expandedOutputs": [], "framework": "\u003cNO FRAMEWORK DETECTED\u003e", + "envMode": "Infer", "environmentVariables": { "configured": [], "inferred": [], "global": [ "VERCEL_ANALYTICS_ID=" - ] + ], + "passthrough": null, + "globalPassthrough": null } } ] diff --git a/cli/integration_tests/dry_json/single_package_with_deps.t b/cli/integration_tests/dry_json/single_package_with_deps.t index 72909803c1e12..a0d0ccadd8d1e 100644 --- a/cli/integration_tests/dry_json/single_package_with_deps.t +++ b/cli/integration_tests/dry_json/single_package_with_deps.t @@ -39,6 +39,7 @@ Setup } } }, + "envMode": "Infer", "tasks": [ { "task": "build", @@ -78,12 +79,15 @@ Setup }, "expandedOutputs": [], "framework": "\u003cNO FRAMEWORK DETECTED\u003e", + "envMode": "Infer", "environmentVariables": { "configured": [], "inferred": [], "global": [ "VERCEL_ANALYTICS_ID=" - ] + ], + "passthrough": null, + "globalPassthrough": null } }, { @@ -122,12 +126,15 @@ Setup }, "expandedOutputs": [], "framework": "\u003cNO FRAMEWORK DETECTED\u003e", + "envMode": "Infer", "environmentVariables": { "configured": [], "inferred": [], "global": [ "VERCEL_ANALYTICS_ID=" - ] + ], + "passthrough": null, + "globalPassthrough": null } } ] diff --git a/cli/integration_tests/single_package/run-summary.t b/cli/integration_tests/single_package/run-summary.t index e941c6f6ad096..03bb3e3a8b744 100644 --- a/cli/integration_tests/single_package/run-summary.t +++ b/cli/integration_tests/single_package/run-summary.t @@ -49,6 +49,7 @@ Check "command", "dependencies", "dependents", + "envMode", "environmentVariables", "excludedOutputs", "execution", diff --git a/cli/integration_tests/strict_env_vars/dry_json.t b/cli/integration_tests/strict_env_vars/dry_json.t new file mode 100644 index 0000000000000..9ee26e93b5a5c --- /dev/null +++ b/cli/integration_tests/strict_env_vars/dry_json.t @@ -0,0 +1,22 @@ +Setup + $ . ${TESTDIR}/../_helpers/setup.sh + $ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) strict_env_vars + +Empty passthroughs are null + $ ${TURBO} build --dry=json | jq -r '.tasks[0].environmentVariables | { passthrough, globalPassthrough }' + { + "passthrough": null, + "globalPassthrough": null + } + +Make sure that we populate the JSON output + $ cp "$TESTDIR/fixture-configs/all.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet + $ ${TURBO} build --dry=json | jq -r '.tasks[0].environmentVariables | { passthrough, globalPassthrough }' + { + "passthrough": [ + "LOCAL_VAR_PT=" + ], + "globalPassthrough": [ + "GLOBAL_VAR_PT=" + ] + } diff --git a/cli/internal/env/env.go b/cli/internal/env/env.go index 31ca69775d65d..1546b3da7a936 100644 --- a/cli/internal/env/env.go +++ b/cli/internal/env/env.go @@ -56,6 +56,10 @@ type EnvironmentVariablePairs []string // mapToPair returns a deterministically sorted set of EnvironmentVariablePairs from an EnvironmentVariableMap // It takes a transformer value to operate on each key-value pair and return a string func (evm EnvironmentVariableMap) mapToPair(transformer func(k string, v string) string) EnvironmentVariablePairs { + if evm == nil { + return nil + } + // convert to set to eliminate duplicates pairs := make([]string, 0, len(evm)) for k, v := range evm { diff --git a/cli/internal/graph/graph.go b/cli/internal/graph/graph.go index 214df0f50f89d..1c0d10b63482f 100644 --- a/cli/internal/graph/graph.go +++ b/cli/internal/graph/graph.go @@ -11,6 +11,7 @@ import ( "github.com/hashicorp/go-hclog" "github.com/pyr-sh/dag" + "github.com/vercel/turbo/cli/internal/env" "github.com/vercel/turbo/cli/internal/fs" "github.com/vercel/turbo/cli/internal/nodes" "github.com/vercel/turbo/cli/internal/runsummary" @@ -50,6 +51,7 @@ type CompleteGraph struct { func (g *CompleteGraph) GetPackageTaskVisitor( ctx gocontext.Context, taskGraph *dag.AcyclicGraph, + globalEnvMode util.EnvMode, getArgs func(taskID string) []string, logger hclog.Logger, visitor func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error, @@ -76,12 +78,19 @@ func (g *CompleteGraph) GetPackageTaskVisitor( return fmt.Errorf("Could not find definition for task") } + // Task env mode is only independent when global env mode is `infer`. + taskEnvMode := globalEnvMode + if taskEnvMode == util.Infer && taskDefinition.PassthroughEnv != nil { + taskEnvMode = util.Strict + } + // TODO: maybe we can remove this PackageTask struct at some point packageTask := &nodes.PackageTask{ TaskID: taskID, Task: taskName, PackageName: packageName, Pkg: pkg, + EnvMode: taskEnvMode, Dir: pkg.Dir.ToString(), TaskDefinition: taskDefinition, Outputs: taskDefinition.Outputs.Inclusions, @@ -111,6 +120,13 @@ func (g *CompleteGraph) GetPackageTaskVisitor( packageTask.LogFile = logFile packageTask.Command = command + var envVarPassthroughMap env.EnvironmentVariableMap + if taskDefinition.PassthroughEnv != nil { + if envVarPassthroughDetailedMap, err := env.GetHashableEnvVars(taskDefinition.PassthroughEnv, nil, ""); err == nil { + envVarPassthroughMap = envVarPassthroughDetailedMap.BySource.Explicit + } + } + summary := &runsummary.TaskSummary{ TaskID: taskID, Task: taskName, @@ -126,9 +142,11 @@ func (g *CompleteGraph) GetPackageTaskVisitor( Command: command, CommandArguments: passThruArgs, Framework: framework, + EnvMode: taskEnvMode, EnvVars: runsummary.TaskEnvVarSummary{ - Configured: envVars.BySource.Explicit.ToSecretHashable(), - Inferred: envVars.BySource.Matching.ToSecretHashable(), + Configured: envVars.BySource.Explicit.ToSecretHashable(), + Inferred: envVars.BySource.Matching.ToSecretHashable(), + Passthrough: envVarPassthroughMap.ToSecretHashable(), }, ExternalDepsHash: pkg.ExternalDepsHash, } diff --git a/cli/internal/nodes/packagetask.go b/cli/internal/nodes/packagetask.go index ef2a29350a7d1..e2dcb27dabda1 100644 --- a/cli/internal/nodes/packagetask.go +++ b/cli/internal/nodes/packagetask.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/vercel/turbo/cli/internal/fs" + "github.com/vercel/turbo/cli/internal/util" ) // PackageTask represents running a particular task in a particular package @@ -13,6 +14,7 @@ type PackageTask struct { Task string PackageName string Pkg *fs.PackageJSON + EnvMode util.EnvMode TaskDefinition *fs.TaskDefinition Dir string Command string diff --git a/cli/internal/run/dry_run.go b/cli/internal/run/dry_run.go index c535e89696f96..679153a57bc7a 100644 --- a/cli/internal/run/dry_run.go +++ b/cli/internal/run/dry_run.go @@ -10,10 +10,12 @@ import ( "github.com/vercel/turbo/cli/internal/cache" "github.com/vercel/turbo/cli/internal/cmdutil" "github.com/vercel/turbo/cli/internal/core" + "github.com/vercel/turbo/cli/internal/fs" "github.com/vercel/turbo/cli/internal/graph" "github.com/vercel/turbo/cli/internal/nodes" "github.com/vercel/turbo/cli/internal/runsummary" "github.com/vercel/turbo/cli/internal/taskhash" + "github.com/vercel/turbo/cli/internal/util" ) // DryRun gets all the info needed from tasks and prints out a summary, but doesn't actually @@ -25,6 +27,7 @@ func DryRun( engine *core.Engine, _ *taskhash.Tracker, // unused, but keep here for parity with RealRun method signature turboCache cache.Cache, + turboJSON *fs.TurboJSON, base *cmdutil.CmdBase, summary runsummary.Meta, ) error { @@ -32,6 +35,11 @@ func DryRun( taskSummaries := []*runsummary.TaskSummary{} + globalEnvMode := rs.Opts.runOpts.EnvMode + if globalEnvMode == util.Infer && turboJSON.GlobalPassthroughEnv != nil { + globalEnvMode = util.Strict + } + mu := sync.Mutex{} execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error { // Assign some fallbacks if they were missing @@ -59,7 +67,7 @@ func DryRun( return rs.ArgsForTask(taskID) } - visitorFn := g.GetPackageTaskVisitor(ctx, engine.TaskGraph, getArgs, base.Logger, execFunc) + visitorFn := g.GetPackageTaskVisitor(ctx, engine.TaskGraph, globalEnvMode, getArgs, base.Logger, execFunc) execOpts := core.EngineExecutionOptions{ Concurrency: 1, Parallel: false, diff --git a/cli/internal/run/global_hash.go b/cli/internal/run/global_hash.go index 6167ca25f67e1..bd4ed06c13b93 100644 --- a/cli/internal/run/global_hash.go +++ b/cli/internal/run/global_hash.go @@ -45,28 +45,35 @@ type oldGlobalHashable struct { } // calculateGlobalHashFromHashable returns a hash string from the globalHashable -func calculateGlobalHashFromHashable(named GlobalHashable) (string, error) { - // When we aren't in infer mode, we can hash the whole object - if named.envMode != util.Infer { - return fs.HashObject(named) - } +func calculateGlobalHashFromHashable(full GlobalHashable) (string, error) { + switch full.envMode { + case util.Infer: + if full.envVarPassthroughs != nil { + // In infer mode, if there is any passThru config (even if it is an empty array) + // we'll hash the whole object, so we can detect changes to that config + // Further, resolve the envMode to the concrete value. + full.envMode = util.Strict + return fs.HashObject(full) + } - // In infer mode, if there is any passThru config (even if it is an empty array) - // we'll hash the whole object, so we can detect changes to that config - if named.envVarPassthroughs != nil { - return fs.HashObject(named) + // If we're in infer mode, and there is no global pass through config, + // we use the old struct layout. this will be true for everyone not using the strict env + // feature, and we don't want to break their cache. + return fs.HashObject(oldGlobalHashable{ + globalFileHashMap: full.globalFileHashMap, + rootExternalDepsHash: full.rootExternalDepsHash, + envVars: full.envVars.All.ToHashable(), + globalCacheKey: full.globalCacheKey, + pipeline: full.pipeline, + }) + case util.Loose: + // Remove the passthroughs from hash consideration if we're explicitly loose. + full.envVarPassthroughs = nil + return fs.HashObject(full) + default: + // When we aren't in infer or loose mode we can hash the whole object as is. + return fs.HashObject(full) } - - // If we're in infer mode, and there is no global pass through config, - // we need to use the old struct layout. This will be true for everyone - // not using the strict env feature and we don't want to break their cache. - return fs.HashObject(oldGlobalHashable{ - globalFileHashMap: named.globalFileHashMap, - rootExternalDepsHash: named.rootExternalDepsHash, - envVars: named.envVars.All.ToHashable(), - globalCacheKey: named.globalCacheKey, - pipeline: named.pipeline, - }) } func calculateGlobalHash( @@ -128,11 +135,6 @@ func calculateGlobalHash( return GlobalHashable{}, fmt.Errorf("error hashing files: %w", err) } - // Remove the passthroughs from hash consideration if we're explicitly loose. - if envMode == util.Loose { - envVarPassthroughs = nil - } - return GlobalHashable{ globalFileHashMap: globalFileHashMap, rootExternalDepsHash: rootPackageJSON.ExternalDepsHash, diff --git a/cli/internal/run/real_run.go b/cli/internal/run/real_run.go index b238d754676ff..f04ce220314e6 100644 --- a/cli/internal/run/real_run.go +++ b/cli/internal/run/real_run.go @@ -72,6 +72,11 @@ func RealRun( runCache := runcache.New(turboCache, base.RepoRoot, rs.Opts.runcacheOpts, colorCache) + globalEnvMode := rs.Opts.runOpts.EnvMode + if globalEnvMode == util.Infer && turboJSON.GlobalPassthroughEnv != nil { + globalEnvMode = util.Strict + } + ec := &execContext{ colorCache: colorCache, runSummary: runSummary, @@ -124,7 +129,7 @@ func RealRun( return rs.ArgsForTask(taskID) } - visitorFn := g.GetPackageTaskVisitor(ctx, engine.TaskGraph, getArgs, base.Logger, execFunc) + visitorFn := g.GetPackageTaskVisitor(ctx, engine.TaskGraph, globalEnvMode, getArgs, base.Logger, execFunc) errs := engine.Execute(visitorFn, execOpts) // Track if we saw any child with a non-zero exit code @@ -217,20 +222,6 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas progressLogger := ec.logger.Named("") progressLogger.Debug("start") - strictEnv := false - switch ec.rs.Opts.runOpts.EnvMode { - case util.Infer: - globalStrict := ec.passthroughEnv != nil - taskStrict := packageTask.TaskDefinition.PassthroughEnv != nil - inferredStrict := taskStrict || globalStrict - - strictEnv = inferredStrict - case util.Loose: - strictEnv = false - case util.Strict: - strictEnv = true - } - passThroughArgs := ec.rs.ArgsForTask(packageTask.Task) hash := packageTask.Hash ec.logger.Debug("task hash", "value", hash) @@ -296,7 +287,7 @@ func (ec *execContext) exec(ctx gocontext.Context, packageTask *nodes.PackageTas currentState := env.GetEnvMap() passthroughEnv := env.EnvironmentVariableMap{} - if strictEnv { + if packageTask.EnvMode == util.Strict { defaultPassthrough := []string{ "PATH", "SHELL", diff --git a/cli/internal/run/run.go b/cli/internal/run/run.go index bcae022e8dc3f..d8b3f26aba16d 100644 --- a/cli/internal/run/run.go +++ b/cli/internal/run/run.go @@ -15,6 +15,7 @@ import ( "github.com/vercel/turbo/cli/internal/core" "github.com/vercel/turbo/cli/internal/daemon" "github.com/vercel/turbo/cli/internal/daemonclient" + "github.com/vercel/turbo/cli/internal/env" "github.com/vercel/turbo/cli/internal/fs" "github.com/vercel/turbo/cli/internal/graph" "github.com/vercel/turbo/cli/internal/process" @@ -340,6 +341,13 @@ func (r *run) run(ctx gocontext.Context, targets []string) error { } } + var envVarPassthroughMap env.EnvironmentVariableMap + if globalHashable.envVarPassthroughs != nil { + if envVarPassthroughDetailedMap, err := env.GetHashableEnvVars(globalHashable.envVarPassthroughs, nil, ""); err == nil { + envVarPassthroughMap = envVarPassthroughDetailedMap.BySource.Explicit + } + } + // RunSummary contains information that is statically analyzable about // the tasks that we expect to run based on the user command. summary := runsummary.NewRunSummary( @@ -354,6 +362,7 @@ func (r *run) run(ctx gocontext.Context, targets []string) error { globalHashable.globalFileHashMap, globalHashable.rootExternalDepsHash, globalHashable.envVars, + envVarPassthroughMap, globalHashable.globalCacheKey, globalHashable.pipeline, ), @@ -368,6 +377,7 @@ func (r *run) run(ctx gocontext.Context, targets []string) error { engine, taskHashTracker, turboCache, + turboJSON, r.base, summary, ) diff --git a/cli/internal/runsummary/format_json.go b/cli/internal/runsummary/format_json.go index 0508f38010d11..76a0a40f7fdcb 100644 --- a/cli/internal/runsummary/format_json.go +++ b/cli/internal/runsummary/format_json.go @@ -5,6 +5,7 @@ import ( "github.com/pkg/errors" "github.com/segmentio/ksuid" + "github.com/vercel/turbo/cli/internal/util" ) // FormatJSON returns a json string representing a RunSummary @@ -29,6 +30,7 @@ func (rsm *Meta) FormatJSON() ([]byte, error) { func (rsm *Meta) normalize() { for _, t := range rsm.RunSummary.Tasks { t.EnvVars.Global = rsm.RunSummary.GlobalHashSummary.envVars + t.EnvVars.GlobalPassthrough = rsm.RunSummary.GlobalHashSummary.passthroughEnvVars } // Remove execution summary for dry runs @@ -58,6 +60,7 @@ type nonMonorepoRunSummary struct { TurboVersion string `json:"turboVersion"` GlobalHashSummary *GlobalHashSummary `json:"globalCacheInputs"` Packages []string `json:"-"` + EnvMode util.EnvMode `json:"envMode"` ExecutionSummary *executionSummary `json:"execution,omitempty"` Tasks []*TaskSummary `json:"tasks"` } diff --git a/cli/internal/runsummary/globalhash_summary.go b/cli/internal/runsummary/globalhash_summary.go index 09467df59b7b1..e24976d5f99cb 100644 --- a/cli/internal/runsummary/globalhash_summary.go +++ b/cli/internal/runsummary/globalhash_summary.go @@ -14,7 +14,8 @@ type GlobalHashSummary struct { Pipeline fs.PristinePipeline `json:"rootPipeline"` // This is a private field because and not in JSON, because we'll add it to each task - envVars env.EnvironmentVariablePairs + envVars env.EnvironmentVariablePairs + passthroughEnvVars env.EnvironmentVariablePairs } // NewGlobalHashSummary creates a GlobalHashSummary struct from a set of fields. @@ -22,11 +23,13 @@ func NewGlobalHashSummary( fileHashMap map[turbopath.AnchoredUnixPath]string, rootExternalDepsHash string, envVars env.DetailedMap, + passthroughEnvVars env.EnvironmentVariableMap, globalCacheKey string, pipeline fs.PristinePipeline, ) *GlobalHashSummary { return &GlobalHashSummary{ envVars: envVars.All.ToSecretHashable(), + passthroughEnvVars: passthroughEnvVars.ToSecretHashable(), GlobalFileHashMap: fileHashMap, RootExternalDepsHash: rootExternalDepsHash, GlobalCacheKey: globalCacheKey, diff --git a/cli/internal/runsummary/run_summary.go b/cli/internal/runsummary/run_summary.go index e5c5fc28cc539..47d3da613418d 100644 --- a/cli/internal/runsummary/run_summary.go +++ b/cli/internal/runsummary/run_summary.go @@ -56,6 +56,7 @@ type RunSummary struct { TurboVersion string `json:"turboVersion"` GlobalHashSummary *GlobalHashSummary `json:"globalCacheInputs"` Packages []string `json:"packages"` + EnvMode util.EnvMode `json:"envMode"` ExecutionSummary *executionSummary `json:"execution,omitempty"` Tasks []*TaskSummary `json:"tasks"` } @@ -93,6 +94,7 @@ func NewRunSummary( ExecutionSummary: executionSummary, TurboVersion: turboVersion, Packages: packages, + EnvMode: runOpts.EnvMode, Tasks: []*TaskSummary{}, GlobalHashSummary: globalHashSummary, }, diff --git a/cli/internal/runsummary/task_summary.go b/cli/internal/runsummary/task_summary.go index bb08dfaea0022..b5afdd0a5ffeb 100644 --- a/cli/internal/runsummary/task_summary.go +++ b/cli/internal/runsummary/task_summary.go @@ -30,15 +30,18 @@ type TaskSummary struct { ResolvedTaskDefinition *fs.TaskDefinition `json:"resolvedTaskDefinition"` ExpandedOutputs []turbopath.AnchoredSystemPath `json:"expandedOutputs"` Framework string `json:"framework"` + EnvMode util.EnvMode `json:"envMode"` EnvVars TaskEnvVarSummary `json:"environmentVariables"` Execution *TaskExecutionSummary `json:"execution,omitempty"` // omit when it's not set } // TaskEnvVarSummary contains the environment variables that impacted a task's hash type TaskEnvVarSummary struct { - Configured []string `json:"configured"` - Inferred []string `json:"inferred"` - Global []string `json:"global"` + Configured []string `json:"configured"` + Inferred []string `json:"inferred"` + Global []string `json:"global"` + Passthrough []string `json:"passthrough"` + GlobalPassthrough []string `json:"globalPassthrough"` } // cleanForSinglePackage converts a TaskSummary to remove references to workspaces diff --git a/cli/internal/taskhash/taskhash.go b/cli/internal/taskhash/taskhash.go index 7c4423efa9f85..0082eb468dd03 100644 --- a/cli/internal/taskhash/taskhash.go +++ b/cli/internal/taskhash/taskhash.go @@ -277,18 +277,68 @@ func (th *Tracker) CalculateFileHashes( return nil } -type taskHashInputs struct { +type taskHashable struct { packageDir turbopath.AnchoredUnixPath hashOfFiles string externalDepsHash string task string outputs fs.TaskOutputs passThruArgs []string + envMode util.EnvMode + passthroughEnv []string hashableEnvPairs []string globalHash string taskDependencyHashes []string } +type oldTaskHashable struct { + packageDir turbopath.AnchoredUnixPath + hashOfFiles string + externalDepsHash string + task string + outputs fs.TaskOutputs + passThruArgs []string + hashableEnvPairs []string + globalHash string + taskDependencyHashes []string +} + +// calculateTaskHashFromHashable returns a hash string from the taskHashable +func calculateTaskHashFromHashable(full *taskHashable) (string, error) { + switch full.envMode { + case util.Infer: + if full.passthroughEnv != nil { + // In infer mode, if there is any passThru config (even if it is an empty array) + // we'll hash the whole object, so we can detect changes to that config + // Further, resolve the envMode to the concrete value. + full.envMode = util.Strict + return fs.HashObject(full) + } + + // If we're in infer mode, and there is no global pass through config, + // we can use the old anonymous struct. this will be true for everyone not using the strict env + // feature, and we don't want to break their cache. + return fs.HashObject(&oldTaskHashable{ + packageDir: full.packageDir, + hashOfFiles: full.hashOfFiles, + externalDepsHash: full.externalDepsHash, + task: full.task, + outputs: full.outputs, + passThruArgs: full.passThruArgs, + hashableEnvPairs: full.hashableEnvPairs, + globalHash: full.globalHash, + taskDependencyHashes: full.taskDependencyHashes, + }) + case util.Loose: + // Remove the passthroughs from hash consideration if we're explicitly loose. + full.passthroughEnv = nil + return fs.HashObject(full) + default: + // When we aren't in infer or loose mode we can hash the whole object as is. + return fs.HashObject(full) + } +} + func (th *Tracker) calculateDependencyHashes(dependencySet dag.Set) ([]string, error) { dependencyHashSet := make(util.Set) @@ -354,13 +404,15 @@ func (th *Tracker) CalculateTaskHash(packageTask *nodes.PackageTask, dependencyS // log any auto detected env vars logger.Debug(fmt.Sprintf("task hash env vars for %s:%s", packageTask.PackageName, packageTask.Task), "vars", hashableEnvPairs) - hash, err := fs.HashObject(&taskHashInputs{ + hash, err := calculateTaskHashFromHashable(&taskHashable{ packageDir: packageTask.Pkg.Dir.ToUnixPath(), hashOfFiles: hashOfFiles, externalDepsHash: packageTask.Pkg.ExternalDepsHash, task: packageTask.Task, outputs: outputs.Sort(), passThruArgs: args, + envMode: packageTask.EnvMode, + passthroughEnv: packageTask.TaskDefinition.PassthroughEnv, hashableEnvPairs: hashableEnvPairs, globalHash: th.globalHash, taskDependencyHashes: taskDependencyHashes,