forked from vercel/turbo
-
Notifications
You must be signed in to change notification settings - Fork 1
/
graph.go
276 lines (234 loc) · 9.34 KB
/
graph.go
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
// Package graph contains the CompleteGraph struct and some methods around it
package graph
import (
gocontext "context"
"fmt"
"path/filepath"
"regexp"
"sort"
"strings"
"github.com/hashicorp/go-hclog"
"github.com/pyr-sh/dag"
"github.com/vercel/turbo/cli/internal/env"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/nodes"
"github.com/vercel/turbo/cli/internal/runsummary"
"github.com/vercel/turbo/cli/internal/taskhash"
"github.com/vercel/turbo/cli/internal/turbopath"
"github.com/vercel/turbo/cli/internal/util"
"github.com/vercel/turbo/cli/internal/workspace"
)
// CompleteGraph represents the common state inferred from the filesystem and pipeline.
// It is not intended to include information specific to a particular run.
type CompleteGraph struct {
// WorkspaceGraph expresses the dependencies between packages
WorkspaceGraph dag.AcyclicGraph
// Pipeline is config from turbo.json
Pipeline fs.Pipeline
// WorkspaceInfos stores the package.json contents by package name
WorkspaceInfos workspace.Catalog
// GlobalHash is the hash of all global dependencies
GlobalHash string
RootNode string
// Map of TaskDefinitions by taskID
TaskDefinitions map[string]*fs.TaskDefinition
RepoRoot turbopath.AbsoluteSystemPath
TaskHashTracker *taskhash.Tracker
}
// GetPackageTaskVisitor wraps a `visitor` function that is used for walking the TaskGraph
// during execution (or dry-runs). The function returned here does not execute any tasks itself,
// but it helps curry some data from the Complete Graph and pass it into the visitor function.
func (g *CompleteGraph) GetPackageTaskVisitor(
ctx gocontext.Context,
taskGraph *dag.AcyclicGraph,
frameworkInference bool,
globalEnvMode util.EnvMode,
getArgs func(taskID string) []string,
logger hclog.Logger,
execFunc func(ctx gocontext.Context, packageTask *nodes.PackageTask, taskSummary *runsummary.TaskSummary) error,
) func(taskID string) error {
return func(taskID string) error {
packageName, taskName := util.GetPackageTaskFromId(taskID)
pkg, ok := g.WorkspaceInfos.PackageJSONs[packageName]
if !ok {
return fmt.Errorf("cannot find package %v for task %v", packageName, taskID)
}
// Check for root task
var command string
if cmd, ok := pkg.Scripts[taskName]; ok {
command = cmd
}
if packageName == util.RootPkgName && commandLooksLikeTurbo(command) {
return fmt.Errorf("root task %v (%v) looks like it invokes turbo and might cause a loop", taskName, command)
}
taskDefinition, ok := g.TaskDefinitions[taskID]
if !ok {
return fmt.Errorf("Could not find definition for task")
}
// Task env mode is only independent when global env mode is `infer`.
taskEnvMode := globalEnvMode
useOldTaskHashable := false
if taskEnvMode == util.Infer {
if taskDefinition.PassthroughEnv != nil {
taskEnvMode = util.Strict
} else {
// If we're in infer mode we have just detected non-usage of strict env vars.
// Since we haven't stabilized this we don't want to break their cache.
useOldTaskHashable = true
// But our old behavior's actual meaning of this state is `loose`.
taskEnvMode = util.Loose
}
}
// TODO: maybe we can remove this PackageTask struct at some point
packageTask := &nodes.PackageTask{
TaskID: taskID,
Task: taskName,
PackageName: packageName,
Pkg: pkg,
EnvMode: taskEnvMode,
Dir: pkg.Dir.ToString(),
TaskDefinition: taskDefinition,
Outputs: taskDefinition.Outputs.Inclusions,
ExcludedOutputs: taskDefinition.Outputs.Exclusions,
}
passThruArgs := getArgs(taskName)
hash, err := g.TaskHashTracker.CalculateTaskHash(
logger,
packageTask,
taskGraph.DownEdges(taskID),
frameworkInference,
passThruArgs,
useOldTaskHashable,
)
// Not being able to construct the task hash is a hard error
if err != nil {
return fmt.Errorf("Hashing error: %v", err)
}
pkgDir := pkg.Dir
packageTask.Hash = hash
envVars := g.TaskHashTracker.GetEnvVars(taskID)
expandedInputs := g.TaskHashTracker.GetExpandedInputs(packageTask)
framework := g.TaskHashTracker.GetFramework(taskID)
logFile := repoRelativeLogFile(pkgDir, taskName)
packageTask.LogFile = logFile
packageTask.Command = command
var envVarPassthroughMap env.EnvironmentVariableMap
if taskDefinition.PassthroughEnv != nil {
if envVarPassthroughDetailedMap, err := env.GetHashableEnvVars(taskDefinition.PassthroughEnv, nil, ""); err == nil {
envVarPassthroughMap = envVarPassthroughDetailedMap.BySource.Explicit
}
}
summary := &runsummary.TaskSummary{
TaskID: taskID,
Task: taskName,
Hash: hash,
Package: packageName,
Dir: pkgDir.ToString(),
Outputs: taskDefinition.Outputs.Inclusions,
ExcludedOutputs: taskDefinition.Outputs.Exclusions,
LogFile: logFile,
ResolvedTaskDefinition: taskDefinition,
ExpandedInputs: expandedInputs,
ExpandedOutputs: []turbopath.AnchoredSystemPath{},
Command: command,
CommandArguments: passThruArgs,
Framework: framework,
EnvMode: taskEnvMode,
EnvVars: runsummary.TaskEnvVarSummary{
Configured: envVars.BySource.Explicit.ToSecretHashable(),
Inferred: envVars.BySource.Matching.ToSecretHashable(),
Passthrough: envVarPassthroughMap.ToSecretHashable(),
},
ExternalDepsHash: pkg.ExternalDepsHash,
}
if ancestors, err := g.getTaskGraphAncestors(taskGraph, packageTask.TaskID); err == nil {
summary.Dependencies = ancestors
}
if descendents, err := g.getTaskGraphDescendants(taskGraph, packageTask.TaskID); err == nil {
summary.Dependents = descendents
}
return execFunc(ctx, packageTask, summary)
}
}
// GetPipelineFromWorkspace returns the Unmarshaled fs.Pipeline struct from turbo.json in the given workspace.
func (g *CompleteGraph) GetPipelineFromWorkspace(workspaceName string, isSinglePackage bool) (fs.Pipeline, error) {
turboConfig, err := g.GetTurboConfigFromWorkspace(workspaceName, isSinglePackage)
if err != nil {
return nil, err
}
return turboConfig.Pipeline, nil
}
// GetTurboConfigFromWorkspace returns the Unmarshaled fs.TurboJSON from turbo.json in the given workspace.
func (g *CompleteGraph) GetTurboConfigFromWorkspace(workspaceName string, isSinglePackage bool) (*fs.TurboJSON, error) {
cachedTurboConfig, ok := g.WorkspaceInfos.TurboConfigs[workspaceName]
if ok {
return cachedTurboConfig, nil
}
var workspacePackageJSON *fs.PackageJSON
if pkgJSON, err := g.GetPackageJSONFromWorkspace(workspaceName); err == nil {
workspacePackageJSON = pkgJSON
} else {
return nil, err
}
// Note: pkgJSON.Dir for the root workspace will be an empty string, and for
// other workspaces, it will be a relative path.
workspaceAbsolutePath := workspacePackageJSON.Dir.RestoreAnchor(g.RepoRoot)
turboConfig, err := fs.LoadTurboConfig(workspaceAbsolutePath, workspacePackageJSON, isSinglePackage)
// If we failed to load a TurboConfig, bubble up the error
if err != nil {
return nil, err
}
// add to cache
g.WorkspaceInfos.TurboConfigs[workspaceName] = turboConfig
return g.WorkspaceInfos.TurboConfigs[workspaceName], nil
}
// GetPackageJSONFromWorkspace returns an Unmarshaled struct of the package.json in the given workspace
func (g *CompleteGraph) GetPackageJSONFromWorkspace(workspaceName string) (*fs.PackageJSON, error) {
if pkgJSON, ok := g.WorkspaceInfos.PackageJSONs[workspaceName]; ok {
return pkgJSON, nil
}
return nil, fmt.Errorf("No package.json for %s", workspaceName)
}
// repoRelativeLogFile returns the path to the log file for this task execution as a
// relative path from the root of the monorepo.
func repoRelativeLogFile(dir turbopath.AnchoredSystemPath, taskName string) string {
return filepath.Join(dir.ToStringDuringMigration(), ".turbo", fmt.Sprintf("turbo-%v.log", taskName))
}
// getTaskGraphAncestors gets all the ancestors for a given task in the graph.
// "ancestors" are all tasks that the given task depends on.
func (g *CompleteGraph) getTaskGraphAncestors(taskGraph *dag.AcyclicGraph, taskID string) ([]string, error) {
ancestors, err := taskGraph.Ancestors(taskID)
if err != nil {
return nil, err
}
stringAncestors := []string{}
for _, dep := range ancestors {
// Don't leak out internal root node name, which are just placeholders
if !strings.Contains(dep.(string), g.RootNode) {
stringAncestors = append(stringAncestors, dep.(string))
}
}
sort.Strings(stringAncestors)
return stringAncestors, nil
}
// getTaskGraphDescendants gets all the descendants for a given task in the graph.
// "descendants" are all tasks that depend on the given taskID.
func (g *CompleteGraph) getTaskGraphDescendants(taskGraph *dag.AcyclicGraph, taskID string) ([]string, error) {
descendents, err := taskGraph.Descendents(taskID)
if err != nil {
return nil, err
}
stringDescendents := []string{}
for _, dep := range descendents {
// Don't leak out internal root node name, which are just placeholders
if !strings.Contains(dep.(string), g.RootNode) {
stringDescendents = append(stringDescendents, dep.(string))
}
}
sort.Strings(stringDescendents)
return stringDescendents, nil
}
var _isTurbo = regexp.MustCompile(`(?:^|\s)turbo(?:$|\s)`)
func commandLooksLikeTurbo(command string) bool {
return _isTurbo.MatchString(command)
}