Exclude Pipeline Definition from Global Hash (#4545)

The global hash does not need to include the pipeline so long as all
material elements to the pipeline are present in the task hash. After
#4529 that invariant became true.

This PR does two things:
- Removes `pipeline` from the global hash key inputs.
- Adjusts the dry run JSON schema to move `rootPipeline` from
`globalHashInputs` and instead attach it to a new root key, `pipeline`.
This is a possibly-breaking change so I incremented the
`runSummarySchemaVersion`.

🚨🚨🚨
This PR triggers a universal cache invalidation! The payoff is no longer
breaking the cache any time `turbo.json` is updated.

---------

Co-authored-by: Nathan Hammond <Nathan Hammond>

cli/internal/fs/turbo_json.go

@@ -286,14 +286,9 @@ type TaskOutputs struct {
 }
 
 // Sort contents of task outputs
-func (to TaskOutputs) Sort() TaskOutputs {
-	var inclusions []string
-	var exclusions []string
-	copy(inclusions, to.Inclusions)
-	copy(exclusions, to.Exclusions)
-	sort.Strings(inclusions)
-	sort.Strings(exclusions)
-	return TaskOutputs{Inclusions: inclusions, Exclusions: exclusions}
+func (to *TaskOutputs) Sort() {
+	sort.Strings(to.Inclusions)
+	sort.Strings(to.Exclusions)
 }
 
 // readTurboConfig reads turbo.json from a provided path

cli/internal/fs/turbo_json_test.go

@@ -212,10 +212,11 @@ func Test_TaskOutputsSort(t *testing.T) {
 	inclusions := []string{"foo/**", "bar"}
 	exclusions := []string{"special-file", ".hidden/**"}
 	taskOutputs := TaskOutputs{Inclusions: inclusions, Exclusions: exclusions}
-	sortedOutputs := taskOutputs.Sort()
-	assertIsSorted(t, sortedOutputs.Inclusions, "Inclusions")
-	assertIsSorted(t, sortedOutputs.Exclusions, "Exclusions")
-	assert.False(t, cmp.DeepEqual(taskOutputs, sortedOutputs)().Success())
+	taskOutputs.Sort()
+	assertIsSorted(t, taskOutputs.Inclusions, "Inclusions")
+	assertIsSorted(t, taskOutputs.Exclusions, "Exclusions")
+
+	assert.True(t, cmp.DeepEqual(taskOutputs, TaskOutputs{Inclusions: []string{"bar", "foo/**"}, Exclusions: []string{".hidden/**", "special-file"}})().Success())
 }
 
 // Helpers

cli/internal/nodes/packagetask.go

@@ -38,8 +38,10 @@ func (pt *PackageTask) HashableOutputs() fs.TaskOutputs {
 	inclusionOutputs := []string{fmt.Sprintf(".turbo/turbo-%v.log", pt.Task)}
 	inclusionOutputs = append(inclusionOutputs, pt.TaskDefinition.Outputs.Inclusions...)
 
-	return fs.TaskOutputs{
+	hashable := fs.TaskOutputs{
 		Inclusions: inclusionOutputs,
 		Exclusions: pt.TaskDefinition.Outputs.Exclusions,
 	}
+	hashable.Sort()
+	return hashable
 }

cli/internal/run/global_hash.go

@@ -28,7 +28,6 @@ type GlobalHashableInputs struct {
 	rootExternalDepsHash string
 	envVars              env.DetailedMap
 	globalCacheKey       string
-	pipeline             fs.PristinePipeline
 	envVarPassthroughs   []string
 	envMode              util.EnvMode
 	frameworkInference   bool
@@ -39,7 +38,6 @@ type newGlobalHashable struct {
 	rootExternalDepsHash string
 	envVars              env.EnvironmentVariablePairs
 	globalCacheKey       string
-	pipeline             fs.PristinePipeline
 	envVarPassthroughs   []string
 	envMode              util.EnvMode
 	frameworkInference   bool
@@ -54,7 +52,6 @@ func newGlobalHash(full GlobalHashableInputs) (string, error) {
 		rootExternalDepsHash: full.rootExternalDepsHash,
 		envVars:              full.envVars.All.ToHashable(),
 		globalCacheKey:       full.globalCacheKey,
-		pipeline:             full.pipeline,
 		envVarPassthroughs:   full.envVarPassthroughs,
 		envMode:              full.envMode,
 		frameworkInference:   full.frameworkInference,
@@ -66,7 +63,6 @@ type oldGlobalHashable struct {
 	rootExternalDepsHash string
 	envVars              env.EnvironmentVariablePairs
 	globalCacheKey       string
-	pipeline             fs.PristinePipeline
 }
 
 // oldGlobalHash is a transformation of GlobalHashableInputs.
@@ -79,7 +75,6 @@ func oldGlobalHash(full GlobalHashableInputs) (string, error) {
 		rootExternalDepsHash: full.rootExternalDepsHash,
 		envVars:              full.envVars.All.ToHashable(),
 		globalCacheKey:       full.globalCacheKey,
-		pipeline:             full.pipeline,
 	})
 }
 
@@ -122,7 +117,6 @@ func calculateGlobalHashFromHashableInputs(full GlobalHashableInputs) (string, e
 func getGlobalHashInputs(
 	rootpath turbopath.AbsoluteSystemPath,
 	rootPackageJSON *fs.PackageJSON,
-	pipeline fs.Pipeline,
 	envVarDependencies []string,
 	globalFileDependencies []string,
 	packageManager *packagemanager.PackageManager,
@@ -190,7 +184,6 @@ func getGlobalHashInputs(
 		rootExternalDepsHash: rootPackageJSON.ExternalDepsHash,
 		envVars:              globalHashableEnvVars,
 		globalCacheKey:       _globalCacheKey,
-		pipeline:             pipeline.Pristine(),
 		envVarPassthroughs:   envVarPassthroughs,
 		envMode:              envMode,
 		frameworkInference:   frameworkInference,

cli/internal/run/run.go

@@ -240,7 +240,6 @@ func (r *run) run(ctx gocontext.Context, targets []string, executionState *turbo
 	globalHashInputs, err := getGlobalHashInputs(
 		r.base.RepoRoot,
 		rootPackageJSON,
-		pipeline,
 		turboJSON.GlobalEnv,
 		turboJSON.GlobalDeps,
 		pkgDepGraph.PackageManager,
@@ -372,7 +371,6 @@ func (r *run) run(ctx gocontext.Context, targets []string, executionState *turbo
 			globalHashInputs.envVars,
 			envVarPassthroughMap,
 			globalHashInputs.globalCacheKey,
-			globalHashInputs.pipeline,
 		),
 		rs.Opts.SynthesizeCommand(rs.Targets),
 	)

cli/internal/runsummary/format_text.go

@@ -42,9 +42,6 @@ func (rsm Meta) FormatAndPrintText(workspaceInfos workspace.Catalog) error {
 	fmt.Fprintln(w1, util.Sprintf("  ${GREY}Global Files\t=\t%d${RESET}", fileCount))
 	fmt.Fprintln(w1, util.Sprintf("  ${GREY}External Dependencies Hash\t=\t%s${RESET}", summary.GlobalHashSummary.RootExternalDepsHash))
 	fmt.Fprintln(w1, util.Sprintf("  ${GREY}Global Cache Key\t=\t%s${RESET}", summary.GlobalHashSummary.GlobalCacheKey))
-	if bytes, err := json.Marshal(summary.GlobalHashSummary.Pipeline); err == nil {
-		fmt.Fprintln(w1, util.Sprintf("  ${GREY}Root pipeline\t=\t%s${RESET}", bytes))
-	}
 	if err := w1.Flush(); err != nil {
 		return err
 	}

cli/internal/runsummary/globalhash_summary.go

@@ -2,7 +2,6 @@ package runsummary
 
 import (
 	"github.com/vercel/turbo/cli/internal/env"
-	"github.com/vercel/turbo/cli/internal/fs"
 	"github.com/vercel/turbo/cli/internal/turbopath"
 )
 
@@ -11,7 +10,6 @@ type GlobalHashSummary struct {
 	GlobalCacheKey       string                                `json:"rootKey"`
 	GlobalFileHashMap    map[turbopath.AnchoredUnixPath]string `json:"files"`
 	RootExternalDepsHash string                                `json:"hashOfExternalDependencies"`
-	Pipeline             fs.PristinePipeline                   `json:"rootPipeline"`
 
 	// This is a private field because and not in JSON, because we'll add it to each task
 	envVars            env.EnvironmentVariablePairs
@@ -25,14 +23,12 @@ func NewGlobalHashSummary(
 	envVars env.DetailedMap,
 	passthroughEnvVars env.EnvironmentVariableMap,
 	globalCacheKey string,
-	pipeline fs.PristinePipeline,
 ) *GlobalHashSummary {
 	return &GlobalHashSummary{
 		envVars:              envVars.All.ToSecretHashable(),
 		passthroughEnvVars:   passthroughEnvVars.ToSecretHashable(),
 		GlobalFileHashMap:    fileHashMap,
 		RootExternalDepsHash: rootExternalDepsHash,
 		GlobalCacheKey:       globalCacheKey,
-		Pipeline:             pipeline,
 	}
 }

cli/internal/taskhash/taskhash.go

@@ -289,7 +289,7 @@ func (th *Tracker) CalculateTaskHash(logger hclog.Logger, packageTask *nodes.Pac
 		hashOfFiles:          hashOfFiles,
 		externalDepsHash:     packageTask.Pkg.ExternalDepsHash,
 		task:                 packageTask.Task,
-		outputs:              outputs.Sort(),
+		outputs:              outputs,
 		passThruArgs:         args,
 		envMode:              packageTask.EnvMode,
 		passthroughEnv:       packageTask.TaskDefinition.PassthroughEnv,

turborepo-tests/integration/tests/composable_config/composing-add-keys.t

@@ -15,13 +15,13 @@ Setup
   \xe2\x80\xa2 Packages in scope: add-keys (esc)
   \xe2\x80\xa2 Running add-keys-task in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  add-keys:add-keys-underlying-task: cache miss, executing 78349dfba4d58116
+  add-keys:add-keys-underlying-task: cache miss, executing 0c509035f4552a61
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: > add-keys-underlying-task
   add-keys:add-keys-underlying-task: > echo "running add-keys-underlying-task"
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: running add-keys-underlying-task
-  add-keys:add-keys-task: cache miss, executing c3b35d8aaef32d5b
+  add-keys:add-keys-task: cache miss, executing efedacb4cb107531
   add-keys:add-keys-task: 
   add-keys:add-keys-task: > add-keys-task
   add-keys:add-keys-task: > echo "running add-keys-task" > out/foo.min.txt
@@ -43,13 +43,13 @@ Setup
   \xe2\x80\xa2 Packages in scope: add-keys (esc)
   \xe2\x80\xa2 Running add-keys-task in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  add-keys:add-keys-underlying-task: cache hit, replaying output 78349dfba4d58116
+  add-keys:add-keys-underlying-task: cache hit, replaying output 0c509035f4552a61
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: > add-keys-underlying-task
   add-keys:add-keys-underlying-task: > echo "running add-keys-underlying-task"
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: running add-keys-underlying-task
-  add-keys:add-keys-task: cache hit, suppressing output c3b35d8aaef32d5b
+  add-keys:add-keys-task: cache hit, suppressing output efedacb4cb107531
 
    Tasks:    2 successful, 2 total
   Cached:    2 cached, 2 total
@@ -61,13 +61,13 @@ Setup
   \xe2\x80\xa2 Packages in scope: add-keys (esc)
   \xe2\x80\xa2 Running add-keys-task in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  add-keys:add-keys-underlying-task: cache miss, executing 1626b8684b31ff83
+  add-keys:add-keys-underlying-task: cache miss, executing 7dd0b2907b19e446
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: > add-keys-underlying-task
   add-keys:add-keys-underlying-task: > echo "running add-keys-underlying-task"
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: running add-keys-underlying-task
-  add-keys:add-keys-task: cache miss, executing da7c8429d1c793ba
+  add-keys:add-keys-task: cache miss, executing e2563a8d7a344f42
   add-keys:add-keys-task: 
   add-keys:add-keys-task: > add-keys-task
   add-keys:add-keys-task: > echo "running add-keys-task" > out/foo.min.txt
@@ -82,13 +82,13 @@ Setup
   \xe2\x80\xa2 Packages in scope: add-keys (esc)
   \xe2\x80\xa2 Running add-keys-task in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  add-keys:add-keys-underlying-task: cache hit, replaying output 1626b8684b31ff83
+  add-keys:add-keys-underlying-task: cache hit, replaying output 7dd0b2907b19e446
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: > add-keys-underlying-task
   add-keys:add-keys-underlying-task: > echo "running add-keys-underlying-task"
   add-keys:add-keys-underlying-task: 
   add-keys:add-keys-underlying-task: running add-keys-underlying-task
-  add-keys:add-keys-task: cache miss, executing 72e64a10a351980c
+  add-keys:add-keys-task: cache miss, executing 3ab6557cb6441554
   add-keys:add-keys-task: 
   add-keys:add-keys-task: > add-keys-task
   add-keys:add-keys-task: > echo "running add-keys-task" > out/foo.min.txt

turborepo-tests/integration/tests/composable_config/composing-add-tasks.t

@@ -6,7 +6,7 @@ Setup
   \xe2\x80\xa2 Packages in scope: add-tasks (esc)
   \xe2\x80\xa2 Running added-task in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  add-tasks:added-task: cache miss, executing 301e333fc0e306e9
+  add-tasks:added-task: cache miss, executing 17ece6abfb90db13
   add-tasks:added-task: 
   add-tasks:added-task: > added-task
   add-tasks:added-task: > echo "running added-task" > out/foo.min.txt

turborepo-tests/integration/tests/composable_config/composing-cache.t

@@ -14,7 +14,7 @@ This test covers:
   \xe2\x80\xa2 Packages in scope: cached (esc)
   \xe2\x80\xa2 Running cached-task-1 in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  cached:cached-task-1: cache miss, executing 26af2215a5ceec12
+  cached:cached-task-1: cache miss, executing 502618642595f819
   cached:cached-task-1: 
   cached:cached-task-1: > cached-task-1
   cached:cached-task-1: > echo 'cached-task-1' > out/foo.min.txt
@@ -39,7 +39,7 @@ This test covers:
   \xe2\x80\xa2 Packages in scope: cached (esc)
   \xe2\x80\xa2 Running cached-task-2 in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  cached:cached-task-2: cache bypass, force executing 90e566e56bf1dc12
+  cached:cached-task-2: cache bypass, force executing 8ea803dc604a3dcb
   cached:cached-task-2: 
   cached:cached-task-2: > cached-task-2
   cached:cached-task-2: > echo 'cached-task-2' > out/foo.min.txt
@@ -61,7 +61,7 @@ no `cache` config in root, cache:false in workspace
   \xe2\x80\xa2 Packages in scope: cached (esc)
   \xe2\x80\xa2 Running cached-task-3 in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  cached:cached-task-3: cache bypass, force executing a54ad1f951b3f194
+  cached:cached-task-3: cache bypass, force executing a1b24869531dd8f2
   cached:cached-task-3: 
   cached:cached-task-3: > cached-task-3
   cached:cached-task-3: > echo 'cached-task-3' > out/foo.min.txt
@@ -85,7 +85,7 @@ we already have a workspace that doesn't have a config
   \xe2\x80\xa2 Packages in scope: missing-workspace-config (esc)
   \xe2\x80\xa2 Running cached-task-4 in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  missing-workspace-config:cached-task-4: cache bypass, force executing b200610f021ed0b8
+  missing-workspace-config:cached-task-4: cache bypass, force executing a98fe7d15234488d
   missing-workspace-config:cached-task-4: 
   missing-workspace-config:cached-task-4: > cached-task-4
   missing-workspace-config:cached-task-4: > echo 'cached-task-4' > out/foo.min.txt

turborepo-tests/integration/tests/composable_config/composing-config-change.t

@@ -4,13 +4,13 @@ Setup
 
 # 1. First run, check the hash
   $ ${TURBO} run config-change-task --filter=config-change --dry=json | jq .tasks[0].hash
-  "bbd88577648790db"
+  "ac8ef319babcdfdc"
 
 2. Run again and assert task hash stays the same
   $ ${TURBO} run config-change-task --filter=config-change --dry=json | jq .tasks[0].hash
-  "bbd88577648790db"
+  "ac8ef319babcdfdc"
 
 3. Change turbo.json and assert that hash changes
   $ cp $TARGET_DIR/apps/config-change/turbo-changed.json $TARGET_DIR/apps/config-change/turbo.json
   $ ${TURBO} run config-change-task --filter=config-change --dry=json | jq .tasks[0].hash
-  "f0e74a964afd751f"
+  "b94c5e8c556d70cb"

turborepo-tests/integration/tests/composable_config/composing-cross-workspace.t

@@ -5,13 +5,13 @@ Setup
   \xe2\x80\xa2 Packages in scope: cross-workspace (esc)
   \xe2\x80\xa2 Running cross-workspace-task in 1 packages (esc)
   \xe2\x80\xa2 Remote caching disabled (esc)
-  blank-pkg:cross-workspace-underlying-task: cache miss, executing 8489d29bbcbbad66
+  blank-pkg:cross-workspace-underlying-task: cache miss, executing 7e12345313b2d76f
   blank-pkg:cross-workspace-underlying-task: 
   blank-pkg:cross-workspace-underlying-task: > cross-workspace-underlying-task
   blank-pkg:cross-workspace-underlying-task: > echo "cross-workspace-underlying-task from blank-pkg"
   blank-pkg:cross-workspace-underlying-task: 
   blank-pkg:cross-workspace-underlying-task: cross-workspace-underlying-task from blank-pkg
-  cross-workspace:cross-workspace-task: cache miss, executing b5f0e030cae7046a
+  cross-workspace:cross-workspace-task: cache miss, executing 8a2b15ea16852f5c
   cross-workspace:cross-workspace-task: 
   cross-workspace:cross-workspace-task: > cross-workspace-task
   cross-workspace:cross-workspace-task: > echo "cross-workspace-task"

turborepo-tests/integration/tests/composable_config/composing-missing-workspace-config-deps.t

@@ -16,22 +16,22 @@ Setup
   \xe2\x80\xa2 Remote caching disabled (esc)
 
   $ cat tmp.log | grep "missing-workspace-config:missing-workspace-config-task-with-deps"
-  missing-workspace-config:missing-workspace-config-task-with-deps: cache miss, executing 663ecc932e855517
+  missing-workspace-config:missing-workspace-config-task-with-deps: cache miss, executing 3834b61108827b4d
   missing-workspace-config:missing-workspace-config-task-with-deps: 
   missing-workspace-config:missing-workspace-config-task-with-deps: > missing-workspace-config-task-with-deps
   missing-workspace-config:missing-workspace-config-task-with-deps: > echo "running missing-workspace-config-task-with-deps" > out/foo.min.txt
   missing-workspace-config:missing-workspace-config-task-with-deps: 
 
   $ cat tmp.log | grep "missing-workspace-config:missing-workspace-config-underlying-task"
-  missing-workspace-config:missing-workspace-config-underlying-task: cache miss, executing f5b6890c769fbfc0
+  missing-workspace-config:missing-workspace-config-underlying-task: cache miss, executing 90a40cdc23446a61
   missing-workspace-config:missing-workspace-config-underlying-task: 
   missing-workspace-config:missing-workspace-config-underlying-task: > missing-workspace-config-underlying-task
   missing-workspace-config:missing-workspace-config-underlying-task: > echo "running missing-workspace-config-underlying-task"
   missing-workspace-config:missing-workspace-config-underlying-task: 
   missing-workspace-config:missing-workspace-config-underlying-task: running missing-workspace-config-underlying-task
 
   $ cat tmp.log | grep "blank-pkg:missing-workspace-config-underlying-topo-task"
-  blank-pkg:missing-workspace-config-underlying-topo-task: cache miss, executing 9ed2e168d7105985
+  blank-pkg:missing-workspace-config-underlying-topo-task: cache miss, executing e8425e7d05ed24f2
   blank-pkg:missing-workspace-config-underlying-topo-task: 
   blank-pkg:missing-workspace-config-underlying-topo-task: > missing-workspace-config-underlying-topo-task
   blank-pkg:missing-workspace-config-underlying-topo-task: > echo "missing-workspace-config-underlying-topo-task from blank-pkg"