Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Synthesize a command for a Run #4499

Merged
merged 3 commits into from
Apr 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
2 changes: 2 additions & 0 deletions cli/integration_tests/single_package/run-summary.t
Original file line number Diff line number Diff line change
Expand Up @@ -20,9 +20,11 @@ Check
[
"attempted",
"cached",
"command",
"endTime",
"exitCode",
"failed",
"repoPath",
"startTime",
"success"
]
Expand Down
18 changes: 5 additions & 13 deletions cli/internal/run/run.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,18 +30,6 @@ import (
"github.com/pkg/errors"
)

var _cmdLong = `
Run tasks across projects in your monorepo.
By default, turbo executes tasks in topological order (i.e.
dependencies first) and then caches the results. Re-running commands for
tasks already in the cache will skip re-execution and immediately move
artifacts from the cache into the correct output folders (as if the task
occurred again).
Arguments passed after '--' will be passed through to the named tasks.
`

// ExecuteRun executes the run command
func ExecuteRun(ctx gocontext.Context, helper *cmdutil.Helper, signalWatcher *signals.Watcher, args *turbostate.ParsedArgsFromRust) error {
base, err := helper.GetCmdBase(args)
Expand Down Expand Up @@ -73,7 +61,9 @@ func optsFromArgs(args *turbostate.ParsedArgsFromRust) (*Opts, error) {

opts := getDefaultOptions()
// aliases := make(map[string]string)
scope.OptsFromArgs(&opts.scopeOpts, args)
if err := scope.OptsFromArgs(&opts.scopeOpts, args); err != nil {
return nil, err
}

// Cache flags
opts.clientOpts.Timeout = args.RemoteCacheTimeout
Expand Down Expand Up @@ -358,6 +348,7 @@ func (r *run) run(ctx gocontext.Context, targets []string) error {
startAt,
r.base.UI,
r.base.RepoRoot,
rs.Opts.scopeOpts.PackageInferenceRoot,
r.base.TurboVersion,
r.base.APIClient,
rs.Opts.runOpts,
Expand All @@ -369,6 +360,7 @@ func (r *run) run(ctx gocontext.Context, targets []string) error {
globalHashable.globalCacheKey,
globalHashable.pipeline,
),
rs.Opts.SynthesizeCommand(rs.Targets),
)

// Dry Run
Expand Down
32 changes: 32 additions & 0 deletions cli/internal/run/run_spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
package run

import (
"strings"

"github.com/vercel/turbo/cli/internal/cache"
"github.com/vercel/turbo/cli/internal/client"
"github.com/vercel/turbo/cli/internal/runcache"
Expand Down Expand Up @@ -45,6 +47,36 @@ type Opts struct {
scopeOpts scope.Opts
}

// SynthesizeCommand produces a command that produces an equivalent set of packages, tasks,
// and task arguments to what the current set of opts selects.
func (o *Opts) SynthesizeCommand(tasks []string) string {
cmd := "turbo run"
cmd += " " + strings.Join(tasks, " ")
for _, filterPattern := range o.scopeOpts.FilterPatterns {
cmd += " --filter=" + filterPattern
}
for _, filterPattern := range o.scopeOpts.LegacyFilter.AsFilterPatterns() {
cmd += " --filter=" + filterPattern
}
if o.runOpts.Parallel {
cmd += " --parallel"
}
if o.runOpts.ContinueOnError {
cmd += " --continue"
}
if o.runOpts.DryRun {
if o.runOpts.DryRunJSON {
cmd += " --dry=json"
} else {
cmd += " --dry"
}
}
if len(o.runOpts.PassThroughArgs) > 0 {
cmd += " -- " + strings.Join(o.runOpts.PassThroughArgs, " ")
}
return cmd
}

// getDefaultOptions returns the default set of Opts for every run
func getDefaultOptions() *Opts {
return &Opts{
Expand Down
107 changes: 107 additions & 0 deletions cli/internal/run/run_spec_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
package run

import (
"testing"

"github.com/vercel/turbo/cli/internal/scope"
"github.com/vercel/turbo/cli/internal/util"
)

func TestSynthesizeCommand(t *testing.T) {
testCases := []struct {
filterPatterns []string
legacyFilter scope.LegacyFilter
passThroughArgs []string
parallel bool
continueOnError bool
dryRun bool
dryRunJSON bool
tasks []string
expected string
}{
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
expected: "turbo run build --filter=my-app",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
passThroughArgs: []string{"-v", "--foo=bar"},
expected: "turbo run build --filter=my-app -- -v --foo=bar",
},
{
legacyFilter: scope.LegacyFilter{
Entrypoints: []string{"my-app"},
SkipDependents: true,
},
tasks: []string{"build"},
passThroughArgs: []string{"-v", "--foo=bar"},
expected: "turbo run build --filter=my-app -- -v --foo=bar",
},
{
legacyFilter: scope.LegacyFilter{
Entrypoints: []string{"my-app"},
SkipDependents: true,
},
filterPatterns: []string{"other-app"},
tasks: []string{"build"},
passThroughArgs: []string{"-v", "--foo=bar"},
expected: "turbo run build --filter=other-app --filter=my-app -- -v --foo=bar",
},
{
legacyFilter: scope.LegacyFilter{
Entrypoints: []string{"my-app"},
IncludeDependencies: true,
Since: "some-ref",
},
filterPatterns: []string{"other-app"},
tasks: []string{"build"},
expected: "turbo run build --filter=other-app --filter=...my-app...[some-ref]...",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
parallel: true,
continueOnError: true,
expected: "turbo run build --filter=my-app --parallel --continue",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
dryRun: true,
expected: "turbo run build --filter=my-app --dry",
},
{
filterPatterns: []string{"my-app"},
tasks: []string{"build"},
dryRun: true,
dryRunJSON: true,
expected: "turbo run build --filter=my-app --dry=json",
},
}

for _, testCase := range testCases {
testCase := testCase
t.Run(testCase.expected, func(t *testing.T) {
o := Opts{
scopeOpts: scope.Opts{
FilterPatterns: testCase.filterPatterns,
LegacyFilter: testCase.legacyFilter,
},
runOpts: util.RunOpts{
PassThroughArgs: testCase.passThroughArgs,
Parallel: testCase.parallel,
ContinueOnError: testCase.continueOnError,
DryRun: testCase.dryRun,
DryRunJSON: testCase.dryRunJSON,
},
}
cmd := o.SynthesizeCommand(testCase.tasks)
if cmd != testCase.expected {
t.Errorf("SynthesizeCommand() got %v, want %v", cmd, testCase.expected)
}
})
}

}
33 changes: 21 additions & 12 deletions cli/internal/runsummary/execution_summary.go
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ import (

"github.com/vercel/turbo/cli/internal/chrometracing"
"github.com/vercel/turbo/cli/internal/fs"
"github.com/vercel/turbo/cli/internal/turbopath"

"github.com/mitchellh/cli"
)
Expand Down Expand Up @@ -112,10 +113,12 @@ type executionSummary struct {
profileFilename string

// These get serialized to JSON
success int // number of tasks that exited successfully (does not include cache hits)
failure int // number of tasks that exited with failure
cached int // number of tasks that had a cache hit
attempted int // number of tasks that started
command string // a synthesized turbo command to produce this invocation
repoPath turbopath.RelativeSystemPath // the (possibly empty) path from the turborepo root to where the command was run
success int // number of tasks that exited successfully (does not include cache hits)
failure int // number of tasks that exited with failure
cached int // number of tasks that had a cache hit
attempted int // number of tasks that started
startedAt time.Time
endedAt time.Time
exitCode int
Expand All @@ -125,14 +128,18 @@ type executionSummary struct {
// We'll use an anonmyous, private struct for this, so it's not confusingly duplicated.
func (es *executionSummary) MarshalJSON() ([]byte, error) {
serializable := struct {
Success int `json:"success"`
Failure int `json:"failed"`
Cached int `json:"cached"`
Attempted int `json:"attempted"`
StartTime int64 `json:"startTime"`
EndTime int64 `json:"endTime"`
ExitCode int `json:"exitCode"`
Command string `json:"command"`
RepoPath string `json:"repoPath"`
Success int `json:"success"`
Failure int `json:"failed"`
Cached int `json:"cached"`
Attempted int `json:"attempted"`
StartTime int64 `json:"startTime"`
EndTime int64 `json:"endTime"`
ExitCode int `json:"exitCode"`
}{
Command: es.command,
RepoPath: es.repoPath.ToString(),
StartTime: es.startedAt.UnixMilli(),
EndTime: es.endedAt.UnixMilli(),
Success: es.success,
Expand All @@ -146,12 +153,14 @@ func (es *executionSummary) MarshalJSON() ([]byte, error) {
}

// newExecutionSummary creates a executionSummary instance to track events in a `turbo run`.`
func newExecutionSummary(start time.Time, tracingProfile string) *executionSummary {
func newExecutionSummary(command string, repoPath turbopath.RelativeSystemPath, start time.Time, tracingProfile string) *executionSummary {
if tracingProfile != "" {
chrometracing.EnableTracing()
}

return &executionSummary{
command: command,
repoPath: repoPath,
success: 0,
failure: 0,
cached: 0,
Expand Down
50 changes: 22 additions & 28 deletions cli/internal/runsummary/run_summary.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,7 +5,6 @@ import (
"encoding/json"
"fmt"
"path/filepath"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -41,14 +40,16 @@ const (
// Meta is a wrapper around the serializable RunSummary, with some extra information
// about the Run and references to other things that we need.
type Meta struct {
RunSummary *RunSummary
ui cli.Ui
repoRoot turbopath.AbsoluteSystemPath // used to write run summary
singlePackage bool
shouldSave bool
apiClient *client.APIClient
spaceID string
runType runType
RunSummary *RunSummary
ui cli.Ui
repoRoot turbopath.AbsoluteSystemPath // used to write run summary
repoPath turbopath.RelativeSystemPath
singlePackage bool
shouldSave bool
apiClient *client.APIClient
spaceID string
runType runType
synthesizedCommand string
}

// RunSummary contains a summary of what happens in the `turbo run` command and why.
Expand All @@ -67,11 +68,13 @@ func NewRunSummary(
startAt time.Time,
ui cli.Ui,
repoRoot turbopath.AbsoluteSystemPath,
repoPath turbopath.RelativeSystemPath,
turboVersion string,
apiClient *client.APIClient,
runOpts util.RunOpts,
packages []string,
globalHashSummary *GlobalHashSummary,
synthesizedCommand string,
) Meta {
singlePackage := runOpts.SinglePackage
profile := runOpts.Profile
Expand All @@ -86,7 +89,7 @@ func NewRunSummary(
}
}

executionSummary := newExecutionSummary(startAt, profile)
executionSummary := newExecutionSummary(synthesizedCommand, repoPath, startAt, profile)

return Meta{
RunSummary: &RunSummary{
Expand All @@ -98,13 +101,14 @@ func NewRunSummary(
Tasks: []*TaskSummary{},
GlobalHashSummary: globalHashSummary,
},
ui: ui,
runType: runType,
repoRoot: repoRoot,
singlePackage: singlePackage,
shouldSave: shouldSave,
apiClient: apiClient,
spaceID: spaceID,
ui: ui,
runType: runType,
repoRoot: repoRoot,
singlePackage: singlePackage,
shouldSave: shouldSave,
apiClient: apiClient,
spaceID: spaceID,
synthesizedCommand: synthesizedCommand,
}
}

Expand Down Expand Up @@ -181,16 +185,6 @@ func (summary *RunSummary) TrackTask(taskID string) (func(outcome executionEvent
return summary.ExecutionSummary.run(taskID)
}

// command returns a best guess command for the entire Run.
// TODO: we should thread this through from the entry point rather than make it up
func (summary *RunSummary) command() string {
taskNames := make(util.Set, len(summary.Tasks))
for _, task := range summary.Tasks {
taskNames.Add(task.Task)
}
return fmt.Sprintf("turbo run %s", strings.Join(taskNames.UnsafeListOfStrings(), " "))
}

// Save saves the run summary to a file
func (rsm *Meta) save() error {
json, err := rsm.FormatJSON()
Expand Down Expand Up @@ -219,7 +213,7 @@ func (rsm *Meta) record() []error {
// can happen when the Run actually starts, so we can send updates to Vercel as the tasks progress.
runsURL := fmt.Sprintf(runsEndpoint, rsm.spaceID)
var runID string
payload := newVercelRunCreatePayload(rsm.RunSummary)
payload := rsm.newVercelRunCreatePayload()
if startPayload, err := json.Marshal(payload); err == nil {
if resp, err := rsm.apiClient.JSONPost(runsURL, startPayload); err != nil {
errs = append(errs, err)
Expand Down