From 53460849a277b518096536fe870efe564118601a Mon Sep 17 00:00:00 2001 From: Victor Savkin Date: Thu, 5 May 2022 11:17:01 -0400 Subject: [PATCH] feat(core): introduce an explicit variable for deciding on how output is printed --- docs/generated/cli/affected.md | 8 ++ docs/generated/cli/run-many.md | 8 ++ packages/nx/bin/run-executor.ts | 10 +-- packages/nx/src/command-line/affected.ts | 1 - packages/nx/src/command-line/nx-commands.ts | 29 +++++-- packages/nx/src/command-line/run-many.ts | 10 +-- packages/nx/src/command-line/run-one.ts | 1 - .../forked-process-task-runner.ts | 52 ++++++----- packages/nx/src/tasks-runner/run-command.ts | 86 +++++++++---------- .../nx/src/tasks-runner/task-orchestrator.ts | 8 +- packages/nx/src/tasks-runner/utils.ts | 11 ++- packages/nx/src/utils/command-line-utils.ts | 6 +- 12 files changed, 130 insertions(+), 100 deletions(-) diff --git a/docs/generated/cli/affected.md b/docs/generated/cli/affected.md index 9aa0acdd9e09c..02755c9356380 100644 --- a/docs/generated/cli/affected.md +++ b/docs/generated/cli/affected.md @@ -109,6 +109,14 @@ Default: false Isolate projects which previously failed +### output-style + +Type: string + +Choices: [dynamic, static, stream] + +Defines how Nx emits outputs tasks logs + ### parallel Type: string diff --git a/docs/generated/cli/run-many.md b/docs/generated/cli/run-many.md index b3264ae2623ec..36f1fff743b01 100644 --- a/docs/generated/cli/run-many.md +++ b/docs/generated/cli/run-many.md @@ -73,6 +73,14 @@ Default: false Only run the target on projects which previously failed +### output-style + +Type: string + +Choices: [dynamic, static, stream] + +Defines how Nx emits outputs tasks logs + ### parallel Type: string diff --git a/packages/nx/bin/run-executor.ts b/packages/nx/bin/run-executor.ts index 852805cd2cf9f..6eaf0732e1507 100644 --- a/packages/nx/bin/run-executor.ts +++ b/packages/nx/bin/run-executor.ts @@ -3,7 +3,7 @@ import { appendFileSync, openSync, writeFileSync } from 'fs'; if (process.env.NX_TERMINAL_OUTPUT_PATH) { setUpOutputWatching( process.env.NX_TERMINAL_CAPTURE_STDERR === 'true', - process.env.NX_FORWARD_OUTPUT === 'true' + process.env.NX_STREAM_OUTPUT === 'true' ); } @@ -45,13 +45,13 @@ function requireCli() { * We need to collect all stdout and stderr and store it, so the caching mechanism * could store it. * - * Writing stdout and stderr into different stream is too risky when using TTY. + * Writing stdout and stderr into different streams is too risky when using TTY. * * So we are simply monkey-patching the Javascript object. In this case the actual output will always be correct. * And the cached output should be correct unless the CLI bypasses process.stdout or console.log and uses some * C-binary to write to stdout. */ -function setUpOutputWatching(captureStderr: boolean, forwardOutput: boolean) { +function setUpOutputWatching(captureStderr: boolean, streamOutput: boolean) { const stdoutWrite = process.stdout._write; const stderrWrite = process.stderr._write; @@ -67,7 +67,7 @@ function setUpOutputWatching(captureStderr: boolean, forwardOutput: boolean) { ) => { onlyStdout.push(chunk); appendFileSync(stdoutAndStderrLogFileHandle, chunk); - if (forwardOutput) { + if (streamOutput) { stdoutWrite.apply(process.stdout, [chunk, encoding, callback]); } else { callback(); @@ -80,7 +80,7 @@ function setUpOutputWatching(captureStderr: boolean, forwardOutput: boolean) { callback: Function ) => { appendFileSync(stdoutAndStderrLogFileHandle, chunk); - if (forwardOutput) { + if (streamOutput) { stderrWrite.apply(process.stderr, [chunk, encoding, callback]); } else { callback(); diff --git a/packages/nx/src/command-line/affected.ts b/packages/nx/src/command-line/affected.ts index 2c03482a2584f..51f543970043d 100644 --- a/packages/nx/src/command-line/affected.ts +++ b/packages/nx/src/command-line/affected.ts @@ -107,7 +107,6 @@ export async function affected( env, nxArgs, overrides, - nxArgs.hideCachedOutput ? 'hide-cached-output' : 'default', null ); break; diff --git a/packages/nx/src/command-line/nx-commands.ts b/packages/nx/src/command-line/nx-commands.ts index 65c2495b1084f..3a49661e26736 100644 --- a/packages/nx/src/command-line/nx-commands.ts +++ b/packages/nx/src/command-line/nx-commands.ts @@ -72,7 +72,9 @@ ${daemonHelpOutput} describe: 'Run target for multiple listed projects', builder: (yargs) => linkToNxDevAndExamples( - withRunManyOptions(withParallelOption(withTargetOption(yargs))), + withRunManyOptions( + withOutputStyleOption(withParallelOption(withTargetOption(yargs))) + ), 'run-many' ), handler: async (args) => (await import('./run-many')).runMany({ ...args }), @@ -82,7 +84,9 @@ ${daemonHelpOutput} describe: 'Run target for affected projects', builder: (yargs) => linkToNxDevAndExamples( - withAffectedOptions(withParallelOption(withTargetOption(yargs))), + withAffectedOptions( + withOutputStyleOption(withParallelOption(withTargetOption(yargs))) + ), 'affected' ), handler: async (args) => @@ -93,7 +97,7 @@ ${daemonHelpOutput} describe: false, builder: (yargs) => linkToNxDevAndExamples( - withAffectedOptions(withParallelOption(yargs)), + withAffectedOptions(withOutputStyleOption(withParallelOption(yargs))), 'affected' ), handler: async (args) => @@ -107,7 +111,7 @@ ${daemonHelpOutput} describe: false, builder: (yargs) => linkToNxDevAndExamples( - withAffectedOptions(withParallelOption(yargs)), + withAffectedOptions(withOutputStyleOption(withParallelOption(yargs))), 'affected' ), handler: async (args) => @@ -121,7 +125,7 @@ ${daemonHelpOutput} describe: false, builder: (yargs) => linkToNxDevAndExamples( - withAffectedOptions(withParallelOption(yargs)), + withAffectedOptions(withOutputStyleOption(withParallelOption(yargs))), 'affected' ), handler: async (args) => @@ -135,7 +139,7 @@ ${daemonHelpOutput} describe: false, builder: (yargs) => linkToNxDevAndExamples( - withAffectedOptions(withParallelOption(yargs)), + withAffectedOptions(withOutputStyleOption(withParallelOption(yargs))), 'affected' ), handler: async (args) => @@ -524,6 +528,14 @@ function withParallelOption(yargs: yargs.Argv): yargs.Argv { }); } +function withOutputStyleOption(yargs: yargs.Argv): yargs.Argv { + return yargs.option('output-style', { + describe: 'Defines how Nx emits outputs tasks logs', + type: 'string', + choices: ['dynamic', 'static', 'stream'], + }); +} + function withTargetOption(yargs: yargs.Argv): yargs.Argv { return yargs.option('target', { describe: 'Task to run for affected projects', @@ -596,6 +608,11 @@ function withRunOneOptions(yargs: yargs.Argv) { .option('project', { describe: 'Target project', type: 'string', + }) + .option('output-style', { + describe: 'Defines how Nx emits outputs tasks logs', + type: 'string', + choices: ['dynamic', 'static', 'stream', 'compact'], }); if (executorShouldShowHelp) { diff --git a/packages/nx/src/command-line/run-many.ts b/packages/nx/src/command-line/run-many.ts index 1d3b39528026f..0fda9fadcd1e7 100644 --- a/packages/nx/src/command-line/run-many.ts +++ b/packages/nx/src/command-line/run-many.ts @@ -23,15 +23,7 @@ export async function runMany(parsedArgs: yargs.Arguments & RawNxArgs) { const projects = projectsToRun(nxArgs, projectGraph); const env = readEnvironment(); - await runCommand( - projects, - projectGraph, - env, - nxArgs, - overrides, - nxArgs.hideCachedOutput ? 'hide-cached-output' : 'default', - null - ); + await runCommand(projects, projectGraph, env, nxArgs, overrides, null); } function projectsToRun( diff --git a/packages/nx/src/command-line/run-one.ts b/packages/nx/src/command-line/run-one.ts index 11c2cf4231dae..c6f0c1db921a2 100644 --- a/packages/nx/src/command-line/run-one.ts +++ b/packages/nx/src/command-line/run-one.ts @@ -49,7 +49,6 @@ export async function runOne( env, nxArgs, overrides, - 'run-one', opts.project ); } diff --git a/packages/nx/src/tasks-runner/forked-process-task-runner.ts b/packages/nx/src/tasks-runner/forked-process-task-runner.ts index a585458d03501..c85ad49f29ef2 100644 --- a/packages/nx/src/tasks-runner/forked-process-task-runner.ts +++ b/packages/nx/src/tasks-runner/forked-process-task-runner.ts @@ -96,9 +96,9 @@ export class ForkedProcessTaskRunner { public forkProcessPipeOutputCapture( task: Task, { - forwardOutput, + streamOutput, temporaryOutputPath, - }: { forwardOutput: boolean; temporaryOutputPath: string } + }: { streamOutput: boolean; temporaryOutputPath: string } ) { return new Promise<{ code: number; terminalOutput: string }>((res, rej) => { try { @@ -107,11 +107,11 @@ export class ForkedProcessTaskRunner { task, task.overrides['verbose'] === true ); - - if (forwardOutput) { + if (streamOutput) { output.logCommand(args.join(' ')); output.addNewline(); } + const p = fork(this.cliPath, serializedArgs, { stdio: ['inherit', 'pipe', 'pipe', 'ipc'], env: this.getEnvVariablesForTask( @@ -119,22 +119,22 @@ export class ForkedProcessTaskRunner { process.env.FORCE_COLOR === undefined ? 'true' : process.env.FORCE_COLOR, - undefined, - forwardOutput + null, + null ), }); this.processes.add(p); let out = []; let outWithErr = []; p.stdout.on('data', (chunk) => { - if (forwardOutput) { + if (streamOutput) { process.stdout.write(chunk); } out.push(chunk.toString()); outWithErr.push(chunk.toString()); }); p.stderr.on('data', (chunk) => { - if (forwardOutput) { + if (streamOutput) { process.stderr.write(chunk); } outWithErr.push(chunk.toString()); @@ -146,7 +146,7 @@ export class ForkedProcessTaskRunner { // print all the collected output| const terminalOutput = outWithErr.join(''); - if (!forwardOutput) { + if (!streamOutput) { this.options.lifeCycle.printTaskTerminalOutput( task, code === 0 ? 'success' : 'failure', @@ -166,9 +166,9 @@ export class ForkedProcessTaskRunner { public forkProcessDirectOutputCapture( task: Task, { - forwardOutput, + streamOutput, temporaryOutputPath, - }: { forwardOutput: boolean; temporaryOutputPath: string } + }: { streamOutput: boolean; temporaryOutputPath: string } ) { return new Promise<{ code: number; terminalOutput: string }>((res, rej) => { try { @@ -177,7 +177,7 @@ export class ForkedProcessTaskRunner { task, task.overrides['verbose'] === true ); - if (forwardOutput) { + if (streamOutput) { output.logCommand(args.join(' ')); output.addNewline(); } @@ -187,7 +187,7 @@ export class ForkedProcessTaskRunner { task, undefined, temporaryOutputPath, - forwardOutput + streamOutput ), }); this.processes.add(p); @@ -198,7 +198,7 @@ export class ForkedProcessTaskRunner { let terminalOutput = ''; try { terminalOutput = this.readTerminalOutput(temporaryOutputPath); - if (!forwardOutput) { + if (!streamOutput) { this.options.lifeCycle.printTaskTerminalOutput( task, code === 0 ? 'success' : 'failure', @@ -252,9 +252,9 @@ export class ForkedProcessTaskRunner { task: Task, forceColor: string, outputPath: string, - forwardOutput: boolean + streamOutput: boolean ) { - return { + const res = { // Start With Dotenv Variables ...this.getDotenvVariablesForTask(task), // User Process Env Variables override Dotenv Variables @@ -264,15 +264,23 @@ export class ForkedProcessTaskRunner { task, forceColor, outputPath, - forwardOutput + streamOutput ), }; + + // we have to delete it because if we invoke Nx from within Nx, we need to reset those values + if (!outputPath) { + delete res.NX_TERMINAL_OUTPUT_PATH; + delete res.NX_STREAM_OUTPUT; + } + delete res.NX_SET_CLI; + return res; } private getNxEnvVariablesForForkedProcess( forceColor: string, outputPath?: string, - forwardOutput?: boolean + streamOutput?: boolean ) { const env: NodeJS.ProcessEnv = { FORCE_COLOR: forceColor, @@ -285,8 +293,8 @@ export class ForkedProcessTaskRunner { if (this.options.captureStderr) { env.NX_TERMINAL_CAPTURE_STDERR = 'true'; } - if (forwardOutput) { - env.NX_FORWARD_OUTPUT = 'true'; + if (streamOutput) { + env.NX_STREAM_OUTPUT = 'true'; } } return env; @@ -296,7 +304,7 @@ export class ForkedProcessTaskRunner { task: Task, forceColor: string, outputPath: string, - forwardOutput: boolean + streamOutput: boolean ) { const env: NodeJS.ProcessEnv = { NX_TASK_TARGET_PROJECT: task.target.project, @@ -312,7 +320,7 @@ export class ForkedProcessTaskRunner { ...this.getNxEnvVariablesForForkedProcess( forceColor, outputPath, - forwardOutput + streamOutput ), ...env, }; diff --git a/packages/nx/src/tasks-runner/run-command.ts b/packages/nx/src/tasks-runner/run-command.ts index fc916ba5387ff..453612335c42b 100644 --- a/packages/nx/src/tasks-runner/run-command.ts +++ b/packages/nx/src/tasks-runner/run-command.ts @@ -1,7 +1,6 @@ import { TasksRunner, TaskStatus } from './tasks-runner'; import { join } from 'path'; import { workspaceRoot } from '../utils/app-root'; -import { logger, stripIndent } from '../utils/logger'; import { NxArgs } from '../utils/command-line-utils'; import { isRelativePath } from '../utils/fileutils'; import { @@ -9,11 +8,10 @@ import { projectHasTargetAndConfiguration, } from '../utils/project-graph-utils'; import { output } from '../utils/output'; -import { getDependencyConfigs, shouldForwardOutput } from './utils'; +import { getDependencyConfigs, shouldStreamOutput } from './utils'; import { CompositeLifeCycle, LifeCycle } from './life-cycle'; import { StaticRunManyTerminalOutputLifeCycle } from './life-cycles/static-run-many-terminal-output-life-cycle'; import { StaticRunOneTerminalOutputLifeCycle } from './life-cycles/static-run-one-terminal-output-life-cycle'; -import { EmptyTerminalOutputLifeCycle } from './life-cycles/empty-terminal-output-life-cycle'; import { TaskTimingsLifeCycle } from './life-cycles/task-timings-life-cycle'; import { createRunManyDynamicOutputRenderer } from './life-cycles/dynamic-run-many-terminal-output-life-cycle'; import { TaskProfilingLifeCycle } from './life-cycles/task-profiling-life-cycle'; @@ -29,22 +27,20 @@ import { async function getTerminalOutputLifeCycle( initiatingProject: string, - terminalOutputStrategy: 'default' | 'hide-cached-output' | 'run-one', projectNames: string[], tasks: Task[], nxArgs: NxArgs, overrides: Record, runnerOptions: any ): Promise<{ lifeCycle: LifeCycle; renderIsDone: Promise }> { - const showVerboseOutput = - !!overrides.verbose || process.env.NX_VERBOSE_LOGGING === 'true'; - - if (terminalOutputStrategy === 'run-one') { - if ( - shouldUseDynamicLifeCycle(tasks, runnerOptions) && - !showVerboseOutput && - process.env.NX_TASKS_RUNNER_DYNAMIC_OUTPUT !== 'false' - ) { + const isRunOne = initiatingProject != null; + const useDynamicOutput = + shouldUseDynamicLifeCycle(tasks, runnerOptions, nxArgs.outputStyle) && + process.env.NX_VERBOSE_LOGGING !== 'true' && + process.env.NX_TASKS_RUNNER_DYNAMIC_OUTPUT !== 'false'; + + if (isRunOne) { + if (useDynamicOutput) { return await createRunOneDynamicOutputRenderer({ initiatingProject, tasks, @@ -61,32 +57,25 @@ async function getTerminalOutputLifeCycle( ), renderIsDone: Promise.resolve(), }; - } else if (terminalOutputStrategy === 'hide-cached-output') { - return { - lifeCycle: new EmptyTerminalOutputLifeCycle(), - renderIsDone: Promise.resolve(), - }; - } else if ( - shouldUseDynamicLifeCycle(tasks, runnerOptions) && - !showVerboseOutput && - process.env.NX_TASKS_RUNNER_DYNAMIC_OUTPUT !== 'false' - ) { - return await createRunManyDynamicOutputRenderer({ - projectNames, - tasks, - args: nxArgs, - overrides, - }); } else { - return { - lifeCycle: new StaticRunManyTerminalOutputLifeCycle( + if (useDynamicOutput) { + return await createRunManyDynamicOutputRenderer({ projectNames, tasks, - nxArgs, - overrides - ), - renderIsDone: Promise.resolve(), - }; + args: nxArgs, + overrides, + }); + } else { + return { + lifeCycle: new StaticRunManyTerminalOutputLifeCycle( + projectNames, + tasks, + nxArgs, + overrides + ), + renderIsDone: Promise.resolve(), + }; + } } } @@ -96,7 +85,6 @@ export async function runCommand( { nxJson }: { nxJson: NxJsonConfiguration }, nxArgs: NxArgs, overrides: any, - terminalOutputStrategy: 'default' | 'hide-cached-output' | 'run-one', initiatingProject: string | null ) { const { tasksRunner, runnerOptions } = getRunner(nxArgs, nxJson); @@ -115,9 +103,11 @@ export async function runCommand( ); const projectNames = projectsToRun.map((t) => t.name); + if (nxArgs.outputStyle == 'stream') { + process.env.NX_STREAM_OUTPUT = 'true'; + } const { lifeCycle, renderIsDone } = await getTerminalOutputLifeCycle( initiatingProject, - terminalOutputStrategy, projectNames, tasks, nxArgs, @@ -138,7 +128,8 @@ export async function runCommand( tasks, { ...runnerOptions, lifeCycle: new CompositeLifeCycle(lifeCycles) }, { - initiatingProject, + initiatingProject: + nxArgs.outputStyle === 'compact' ? null : initiatingProject, target: nxArgs.target, projectGraph, nxJson, @@ -238,12 +229,17 @@ export function createTasksForProjectToRun( return Array.from(tasksMap.values()); } -function shouldUseDynamicLifeCycle(tasks: Task[], options: any) { - const isTTY = !!process.stdout.isTTY; - const noForwarding = !tasks.find((t) => - shouldForwardOutput(t, null, options) - ); - return isTTY && noForwarding && !isCI(); +function shouldUseDynamicLifeCycle( + tasks: Task[], + options: any, + outputStyle: string +) { + if (!process.stdout.isTTY) return false; + if (isCI()) return false; + if (outputStyle === 'static' || outputStyle === 'stream') return false; + + const noForwarding = !tasks.find((t) => shouldStreamOutput(t, null, options)); + return noForwarding; } function addTasksForProjectTarget( diff --git a/packages/nx/src/tasks-runner/task-orchestrator.ts b/packages/nx/src/tasks-runner/task-orchestrator.ts index a9a198da37699..627c2cb88650a 100644 --- a/packages/nx/src/tasks-runner/task-orchestrator.ts +++ b/packages/nx/src/tasks-runner/task-orchestrator.ts @@ -12,7 +12,7 @@ import { getOutputs, isCacheableTask, removeTasksFromTaskGraph, - shouldForwardOutput, + shouldStreamOutput, } from './utils'; import { Batch, TasksSchedule } from './tasks-schedule'; import { TaskMetadata } from './life-cycle'; @@ -280,7 +280,7 @@ export class TaskOrchestrator { try { // obtain metadata const temporaryOutputPath = this.cache.temporaryOutputPath(task); - const forwardOutput = shouldForwardOutput( + const streamOutput = shouldStreamOutput( task, this.initiatingProject, this.options @@ -293,14 +293,14 @@ export class TaskOrchestrator { task, { temporaryOutputPath, - forwardOutput, + streamOutput, } ) : await this.forkedProcessTaskRunner.forkProcessDirectOutputCapture( task, { temporaryOutputPath, - forwardOutput, + streamOutput, } ); diff --git a/packages/nx/src/tasks-runner/utils.ts b/packages/nx/src/tasks-runner/utils.ts index 83cd5ab1a2907..44e6022a81d96 100644 --- a/packages/nx/src/tasks-runner/utils.ts +++ b/packages/nx/src/tasks-runner/utils.ts @@ -277,7 +277,7 @@ export function getSerializedArgsForTask(task: Task, isVerbose: boolean) { ]; } -export function shouldForwardOutput( +export function shouldStreamOutput( task: Task, initiatingProject: string | null, options: { @@ -285,8 +285,8 @@ export function shouldForwardOutput( cacheableTargets?: string[] | null; } ): boolean { - if (process.env.NX_FORWARD_OUTPUT === 'true') return true; - if (!isCacheableTask(task, options)) return true; + if (process.env.NX_STREAM_OUTPUT === 'true') return true; + if (longRunningTask(task)) return true; if (task.target.project === initiatingProject) return true; return false; } @@ -307,5 +307,8 @@ export function isCacheableTask( } function longRunningTask(task: Task) { - return !!task.overrides['watch']; + const t = task.target.target; + return ( + !!task.overrides['watch'] || t === 'serve' || t === 'dev' || t === 'start' + ); } diff --git a/packages/nx/src/utils/command-line-utils.ts b/packages/nx/src/utils/command-line-utils.ts index 8a46cf32061b2..e3a44a1db5ed4 100644 --- a/packages/nx/src/utils/command-line-utils.ts +++ b/packages/nx/src/utils/command-line-utils.ts @@ -81,7 +81,7 @@ const runOne: string[] = [ 'with-deps', 'skip-nx-cache', 'scan', - 'hide-cached-output', + 'output-style', ]; const runMany: string[] = [...runOne, 'projects', 'all']; @@ -126,8 +126,8 @@ export interface NxArgs { select?: string; skipNxCache?: boolean; 'skip-nx-cache'?: boolean; - 'hide-cached-output'?: boolean; - hideCachedOutput?: boolean; + 'output-style'?: string; + outputStyle?: string; scan?: boolean; }