Skip to content

Commit

Permalink
Add data to run summary.
Browse files Browse the repository at this point in the history
  • Loading branch information
nathanhammond committed Apr 11, 2023
1 parent ecced88 commit c51cee8
Show file tree
Hide file tree
Showing 18 changed files with 202 additions and 57 deletions.
15 changes: 12 additions & 3 deletions cli/integration_tests/dry_json/monorepo.t
Expand Up @@ -72,6 +72,7 @@ Setup

$ cat tmpjson.log | jq 'keys'
[
"envMode",
"globalCacheInputs",
"id",
"packages",
Expand Down Expand Up @@ -120,13 +121,16 @@ Setup
},
"expandedOutputs": [],
"framework": "<NO FRAMEWORK DETECTED>",
"envMode": "Infer",
"environmentVariables": {
"configured": [],
"inferred": [],
"global": [
"SOME_ENV_VAR=",
"VERCEL_ANALYTICS_ID="
]
],
"passthrough": null,
"globalPassthrough": null
}
}

Expand Down Expand Up @@ -166,6 +170,7 @@ Setup
},
"expandedOutputs": [],
"framework": "<NO FRAMEWORK DETECTED>",
"envMode": "Infer",
"environmentVariables": {
"configured": [
"NODE_ENV="
Expand All @@ -174,7 +179,9 @@ Setup
"global": [
"SOME_ENV_VAR=",
"VERCEL_ANALYTICS_ID="
]
],
"passthrough": null,
"globalPassthrough": null
}
}

Expand All @@ -188,7 +195,9 @@ Run again with NODE_ENV set and see the value in the summary. --filter=util work
"global": [
"SOME_ENV_VAR=",
"VERCEL_ANALYTICS_ID="
]
],
"passthrough": null,
"globalPassthrough": null
}

Tasks that don't exist throw an error
Expand Down
6 changes: 5 additions & 1 deletion cli/integration_tests/dry_json/single_package.t
Expand Up @@ -29,6 +29,7 @@ Setup
}
}
},
"envMode": "Infer",
"tasks": [
{
"task": "build",
Expand Down Expand Up @@ -67,12 +68,15 @@ Setup
},
"expandedOutputs": [],
"framework": "\u003cNO FRAMEWORK DETECTED\u003e",
"envMode": "Infer",
"environmentVariables": {
"configured": [],
"inferred": [],
"global": [
"VERCEL_ANALYTICS_ID="
]
],
"passthrough": null,
"globalPassthrough": null
}
}
]
Expand Down
6 changes: 5 additions & 1 deletion cli/integration_tests/dry_json/single_package_no_config.t
Expand Up @@ -26,6 +26,7 @@ Setup
}
}
},
"envMode": "Infer",
"tasks": [
{
"task": "build",
Expand Down Expand Up @@ -58,12 +59,15 @@ Setup
},
"expandedOutputs": [],
"framework": "\u003cNO FRAMEWORK DETECTED\u003e",
"envMode": "Infer",
"environmentVariables": {
"configured": [],
"inferred": [],
"global": [
"VERCEL_ANALYTICS_ID="
]
],
"passthrough": null,
"globalPassthrough": null
}
}
]
Expand Down
11 changes: 9 additions & 2 deletions cli/integration_tests/dry_json/single_package_with_deps.t
Expand Up @@ -39,6 +39,7 @@ Setup
}
}
},
"envMode": "Infer",
"tasks": [
{
"task": "build",
Expand Down Expand Up @@ -78,12 +79,15 @@ Setup
},
"expandedOutputs": [],
"framework": "\u003cNO FRAMEWORK DETECTED\u003e",
"envMode": "Infer",
"environmentVariables": {
"configured": [],
"inferred": [],
"global": [
"VERCEL_ANALYTICS_ID="
]
],
"passthrough": null,
"globalPassthrough": null
}
},
{
Expand Down Expand Up @@ -122,12 +126,15 @@ Setup
},
"expandedOutputs": [],
"framework": "\u003cNO FRAMEWORK DETECTED\u003e",
"envMode": "Infer",
"environmentVariables": {
"configured": [],
"inferred": [],
"global": [
"VERCEL_ANALYTICS_ID="
]
],
"passthrough": null,
"globalPassthrough": null
}
}
]
Expand Down
1 change: 1 addition & 0 deletions cli/integration_tests/single_package/run-summary.t
Expand Up @@ -49,6 +49,7 @@ Check
"command",
"dependencies",
"dependents",
"envMode",
"environmentVariables",
"excludedOutputs",
"execution",
Expand Down
22 changes: 22 additions & 0 deletions cli/integration_tests/strict_env_vars/dry_json.t
@@ -0,0 +1,22 @@
Setup
$ . ${TESTDIR}/../_helpers/setup.sh
$ . ${TESTDIR}/../_helpers/setup_monorepo.sh $(pwd) strict_env_vars

Empty passthroughs are null
$ ${TURBO} build --dry=json | jq -r '.tasks[0].environmentVariables | { passthrough, globalPassthrough }'
{
"passthrough": null,
"globalPassthrough": null
}

Make sure that we populate the JSON output
$ cp "$TESTDIR/fixture-configs/all.json" "$(pwd)/turbo.json" && git commit -am "no comment" --quiet
$ ${TURBO} build --dry=json | jq -r '.tasks[0].environmentVariables | { passthrough, globalPassthrough }'
{
"passthrough": [
"LOCAL_VAR_PT="
],
"globalPassthrough": [
"GLOBAL_VAR_PT="
]
}
4 changes: 4 additions & 0 deletions cli/internal/env/env.go
Expand Up @@ -56,6 +56,10 @@ type EnvironmentVariablePairs []string
// mapToPair returns a deterministically sorted set of EnvironmentVariablePairs from an EnvironmentVariableMap
// It takes a transformer value to operate on each key-value pair and return a string
func (evm EnvironmentVariableMap) mapToPair(transformer func(k string, v string) string) EnvironmentVariablePairs {
if evm == nil {
return nil
}

// convert to set to eliminate duplicates
pairs := make([]string, 0, len(evm))
for k, v := range evm {
Expand Down
22 changes: 20 additions & 2 deletions cli/internal/graph/graph.go
Expand Up @@ -11,6 +11,7 @@ import (

"github.com/hashicorp/go-hclog"
"github.com/pyr-sh/dag"
"github.com/vercel/turbo/cli/internal/env"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/nodes"
"github.com/vercel/turbo/cli/internal/runsummary"
Expand Down Expand Up @@ -50,6 +51,7 @@ type CompleteGraph struct {
func (g *CompleteGraph) GetPackageTaskVisitor(
ctx gocontext.Context,
taskGraph *dag.AcyclicGraph,
globalEnvMode util.EnvMode,
getArgs func(taskID string) []string,
logger hclog.Logger,
visitor func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error,
Expand All @@ -76,12 +78,19 @@ func (g *CompleteGraph) GetPackageTaskVisitor(
return fmt.Errorf("Could not find definition for task")
}

// Task env mode is only independent when global env mode is `infer`.
taskEnvMode := globalEnvMode
if taskEnvMode == util.Infer && taskDefinition.PassthroughEnv != nil {
taskEnvMode = util.Strict
}

// TODO: maybe we can remove this PackageTask struct at some point
packageTask := &nodes.PackageTask{
TaskID: taskID,
Task: taskName,
PackageName: packageName,
Pkg: pkg,
EnvMode: taskEnvMode,
Dir: pkg.Dir.ToString(),
TaskDefinition: taskDefinition,
Outputs: taskDefinition.Outputs.Inclusions,
Expand Down Expand Up @@ -111,6 +120,13 @@ func (g *CompleteGraph) GetPackageTaskVisitor(
packageTask.LogFile = logFile
packageTask.Command = command

var envVarPassthroughMap env.EnvironmentVariableMap
if taskDefinition.PassthroughEnv != nil {
if envVarPassthroughDetailedMap, err := env.GetHashableEnvVars(taskDefinition.PassthroughEnv, nil, ""); err == nil {
envVarPassthroughMap = envVarPassthroughDetailedMap.BySource.Explicit
}
}

summary := &runsummary.TaskSummary{
TaskID: taskID,
Task: taskName,
Expand All @@ -126,9 +142,11 @@ func (g *CompleteGraph) GetPackageTaskVisitor(
Command: command,
CommandArguments: passThruArgs,
Framework: framework,
EnvMode: taskEnvMode,
EnvVars: runsummary.TaskEnvVarSummary{
Configured: envVars.BySource.Explicit.ToSecretHashable(),
Inferred: envVars.BySource.Matching.ToSecretHashable(),
Configured: envVars.BySource.Explicit.ToSecretHashable(),
Inferred: envVars.BySource.Matching.ToSecretHashable(),
Passthrough: envVarPassthroughMap.ToSecretHashable(),
},
ExternalDepsHash: pkg.ExternalDepsHash,
}
Expand Down
2 changes: 2 additions & 0 deletions cli/internal/nodes/packagetask.go
Expand Up @@ -5,6 +5,7 @@ import (
"fmt"

"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/util"
)

// PackageTask represents running a particular task in a particular package
Expand All @@ -13,6 +14,7 @@ type PackageTask struct {
Task string
PackageName string
Pkg *fs.PackageJSON
EnvMode util.EnvMode
TaskDefinition *fs.TaskDefinition
Dir string
Command string
Expand Down
10 changes: 9 additions & 1 deletion cli/internal/run/dry_run.go
Expand Up @@ -10,10 +10,12 @@ import (
"github.com/vercel/turbo/cli/internal/cache"
"github.com/vercel/turbo/cli/internal/cmdutil"
"github.com/vercel/turbo/cli/internal/core"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/graph"
"github.com/vercel/turbo/cli/internal/nodes"
"github.com/vercel/turbo/cli/internal/runsummary"
"github.com/vercel/turbo/cli/internal/taskhash"
"github.com/vercel/turbo/cli/internal/util"
)

// DryRun gets all the info needed from tasks and prints out a summary, but doesn't actually
Expand All @@ -25,13 +27,19 @@ func DryRun(
engine *core.Engine,
_ *taskhash.Tracker, // unused, but keep here for parity with RealRun method signature
turboCache cache.Cache,
turboJSON *fs.TurboJSON,
base *cmdutil.CmdBase,
summary runsummary.Meta,
) error {
defer turboCache.Shutdown()

taskSummaries := []*runsummary.TaskSummary{}

globalEnvMode := rs.Opts.runOpts.EnvMode
if globalEnvMode == util.Infer && turboJSON.GlobalPassthroughEnv != nil {
globalEnvMode = util.Strict
}

mu := sync.Mutex{}
execFunc := func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error {
// Assign some fallbacks if they were missing
Expand Down Expand Up @@ -59,7 +67,7 @@ func DryRun(
return rs.ArgsForTask(taskID)
}

visitorFn := g.GetPackageTaskVisitor(ctx, engine.TaskGraph, getArgs, base.Logger, execFunc)
visitorFn := g.GetPackageTaskVisitor(ctx, engine.TaskGraph, globalEnvMode, getArgs, base.Logger, execFunc)
execOpts := core.EngineExecutionOptions{
Concurrency: 1,
Parallel: false,
Expand Down
52 changes: 27 additions & 25 deletions cli/internal/run/global_hash.go
Expand Up @@ -45,28 +45,35 @@ type oldGlobalHashable struct {
}

// calculateGlobalHashFromHashable returns a hash string from the globalHashable
func calculateGlobalHashFromHashable(named GlobalHashable) (string, error) {
// When we aren't in infer mode, we can hash the whole object
if named.envMode != util.Infer {
return fs.HashObject(named)
}
func calculateGlobalHashFromHashable(full GlobalHashable) (string, error) {
switch full.envMode {
case util.Infer:
if full.envVarPassthroughs != nil {
// In infer mode, if there is any passThru config (even if it is an empty array)
// we'll hash the whole object, so we can detect changes to that config
// Further, resolve the envMode to the concrete value.
full.envMode = util.Strict
return fs.HashObject(full)
}

// In infer mode, if there is any passThru config (even if it is an empty array)
// we'll hash the whole object, so we can detect changes to that config
if named.envVarPassthroughs != nil {
return fs.HashObject(named)
// If we're in infer mode, and there is no global pass through config,
// we use the old struct layout. this will be true for everyone not using the strict env
// feature, and we don't want to break their cache.
return fs.HashObject(oldGlobalHashable{
globalFileHashMap: full.globalFileHashMap,
rootExternalDepsHash: full.rootExternalDepsHash,
envVars: full.envVars.All.ToHashable(),
globalCacheKey: full.globalCacheKey,
pipeline: full.pipeline,
})
case util.Loose:
// Remove the passthroughs from hash consideration if we're explicitly loose.
full.envVarPassthroughs = nil
return fs.HashObject(full)
default:
// When we aren't in infer or loose mode we can hash the whole object as is.
return fs.HashObject(full)
}

// If we're in infer mode, and there is no global pass through config,
// we need to use the old struct layout. This will be true for everyone
// not using the strict env feature and we don't want to break their cache.
return fs.HashObject(oldGlobalHashable{
globalFileHashMap: named.globalFileHashMap,
rootExternalDepsHash: named.rootExternalDepsHash,
envVars: named.envVars.All.ToHashable(),
globalCacheKey: named.globalCacheKey,
pipeline: named.pipeline,
})
}

func calculateGlobalHash(
Expand Down Expand Up @@ -128,11 +135,6 @@ func calculateGlobalHash(
return GlobalHashable{}, fmt.Errorf("error hashing files: %w", err)
}

// Remove the passthroughs from hash consideration if we're explicitly loose.
if envMode == util.Loose {
envVarPassthroughs = nil
}

return GlobalHashable{
globalFileHashMap: globalFileHashMap,
rootExternalDepsHash: rootPackageJSON.ExternalDepsHash,
Expand Down

0 comments on commit c51cee8

Please sign in to comment.