From a32d46c5a36246b2349d72f7b50f7f0e532744bb Mon Sep 17 00:00:00 2001 From: Victor Savkin Date: Thu, 10 Mar 2022 16:33:26 -0500 Subject: [PATCH] feat(core): provide an experimental hashing mode for jest and cyrpess --- packages/cypress/executors.json | 1 + .../cypress/src/executors/cypress/hasher.ts | 27 ++++ packages/jest/executors.json | 1 + packages/jest/src/executors/jest/hasher.ts | 27 ++++ .../linter/src/executors/eslint/hasher.ts | 58 ++++---- packages/tao/src/shared/workspace.ts | 7 +- .../workspace/src/core/hasher/hasher.spec.ts | 130 ++++++++++++++++-- packages/workspace/src/core/hasher/hasher.ts | 69 ++++++++-- .../src/tasks-runner/task-orchestrator.ts | 1 + .../src/tasks-runner/tasks-schedule.spec.ts | 3 + .../src/tasks-runner/tasks-schedule.ts | 29 ++-- 11 files changed, 285 insertions(+), 68 deletions(-) create mode 100644 packages/cypress/src/executors/cypress/hasher.ts create mode 100644 packages/jest/src/executors/jest/hasher.ts diff --git a/packages/cypress/executors.json b/packages/cypress/executors.json index e15fbb8ffd2c7..dbebfe23bf3c7 100644 --- a/packages/cypress/executors.json +++ b/packages/cypress/executors.json @@ -10,6 +10,7 @@ "cypress": { "implementation": "./src/executors/cypress/cypress.impl", "schema": "./src/executors/cypress/schema.json", + "hasher": "./src/executors/cypress/hasher", "description": "Run Cypress e2e tests" } } diff --git a/packages/cypress/src/executors/cypress/hasher.ts b/packages/cypress/src/executors/cypress/hasher.ts new file mode 100644 index 0000000000000..60aeeaa377ee6 --- /dev/null +++ b/packages/cypress/src/executors/cypress/hasher.ts @@ -0,0 +1,27 @@ +import { + NxJsonConfiguration, + ProjectGraph, + Task, + TaskGraph, + WorkspaceJsonConfiguration, +} from '@nrwl/devkit'; +import { Hash, Hasher } from '@nrwl/workspace/src/core/hasher/hasher'; + +export default async function run( + task: Task, + context: { + hasher: Hasher; + projectGraph: ProjectGraph; + taskGraph: TaskGraph; + workspaceConfig: WorkspaceJsonConfiguration & NxJsonConfiguration; + } +): Promise { + const cypressPluginConfig = context.workspaceConfig.pluginsConfig + ? (context.workspaceConfig.pluginsConfig['@nrwl/cypress'] as any) + : undefined; + const filter = + cypressPluginConfig && cypressPluginConfig.hashingExcludesTestsOfDeps + ? 'exclude-tests-of-deps' + : 'all-files'; + return context.hasher.hashTaskWithDepsAndContext(task, filter); +} diff --git a/packages/jest/executors.json b/packages/jest/executors.json index 170432ffefdbd..fd10463840d1e 100644 --- a/packages/jest/executors.json +++ b/packages/jest/executors.json @@ -11,6 +11,7 @@ "implementation": "./src/executors/jest/jest.impl", "batchImplementation": "./src/executors/jest/jest.impl#batchJest", "schema": "./src/executors/jest/schema.json", + "hasher": "./src/executors/jest/hasher", "description": "Run Jest unit tests" } } diff --git a/packages/jest/src/executors/jest/hasher.ts b/packages/jest/src/executors/jest/hasher.ts new file mode 100644 index 0000000000000..10ea47ef7713b --- /dev/null +++ b/packages/jest/src/executors/jest/hasher.ts @@ -0,0 +1,27 @@ +import { + NxJsonConfiguration, + ProjectGraph, + Task, + TaskGraph, + WorkspaceJsonConfiguration, +} from '@nrwl/devkit'; +import { Hash, Hasher } from '@nrwl/workspace/src/core/hasher/hasher'; + +export default async function run( + task: Task, + context: { + hasher: Hasher; + projectGraph: ProjectGraph; + taskGraph: TaskGraph; + workspaceConfig: WorkspaceJsonConfiguration & NxJsonConfiguration; + } +): Promise { + const jestPluginConfig = context.workspaceConfig.pluginsConfig + ? (context.workspaceConfig.pluginsConfig['@nrwl/jest'] as any) + : undefined; + const filter = + jestPluginConfig && jestPluginConfig.hashingExcludesTestsOfDeps + ? 'exclude-tests-of-deps' + : 'all-files'; + return context.hasher.hashTaskWithDepsAndContext(task, filter); +} diff --git a/packages/linter/src/executors/eslint/hasher.ts b/packages/linter/src/executors/eslint/hasher.ts index 685d613462772..d03b1905c59ca 100644 --- a/packages/linter/src/executors/eslint/hasher.ts +++ b/packages/linter/src/executors/eslint/hasher.ts @@ -1,48 +1,44 @@ -import { ProjectGraph, Task, TaskGraph } from '@nrwl/devkit'; +import { + ProjectGraph, + Task, + TaskGraph, + WorkspaceJsonConfiguration, +} from '@nrwl/devkit'; import { Hash, Hasher } from '@nrwl/workspace/src/core/hasher/hasher'; -import { appRootPath } from '@nrwl/tao/src/utils/app-root'; -import { Workspaces } from '@nrwl/tao/src/shared/workspace'; -import { readCachedProjectGraph } from '@nrwl/workspace/src/core/project-graph'; export default async function run( task: Task, - taskGraph: TaskGraph, - hasher: Hasher + context: { + hasher: Hasher; + projectGraph: ProjectGraph; + taskGraph: TaskGraph; + workspaceConfig: WorkspaceJsonConfiguration; + } ): Promise { if (task.overrides['hasTypeAwareRules'] === true) { - return hasher.hashTaskWithDepsAndContext(task); + return context.hasher.hashTaskWithDepsAndContext(task); } - if (!(global as any).projectGraph) { - try { - (global as any).projectGraph = readCachedProjectGraph(); - } catch { - // do nothing, if project graph is unavailable we fallback to using all projects - } - } - const projectGraph = (global as any).projectGraph; - const command = hasher.hashCommand(task); - const sources = await hasher.hashSource(task); - const workspace = new Workspaces(appRootPath).readWorkspaceConfiguration(); - const deps = projectGraph - ? allDeps(task.id, taskGraph, projectGraph) - : Object.keys(workspace.projects); - const tags = hasher.hashArray( - deps.map((d) => (workspace.projects[d].tags || []).join('|')) + + const command = context.hasher.hashCommand(task); + const source = await context.hasher.hashSource(task); + const deps = allDeps(task.id, context.taskGraph, context.projectGraph); + const tags = context.hasher.hashArray( + deps.map((d) => (context.workspaceConfig.projects[d].tags || []).join('|')) ); - const context = await hasher.hashContext(); + const taskContext = await context.hasher.hashContext(); return { - value: hasher.hashArray([ + value: context.hasher.hashArray([ command, - sources, + source, tags, - context.implicitDeps.value, - context.runtime.value, + taskContext.implicitDeps.value, + taskContext.runtime.value, ]), details: { command, - nodes: { [task.target.project]: sources, tags }, - implicitDeps: context.implicitDeps.files, - runtime: context.runtime.runtime, + nodes: { [task.target.project]: source, tags }, + implicitDeps: taskContext.implicitDeps.files, + runtime: taskContext.runtime.runtime, }, }; } diff --git a/packages/tao/src/shared/workspace.ts b/packages/tao/src/shared/workspace.ts index eb629887663be..7ceb7a869d11a 100644 --- a/packages/tao/src/shared/workspace.ts +++ b/packages/tao/src/shared/workspace.ts @@ -263,6 +263,9 @@ export interface ExecutorContext { } export class Workspaces { + private cachedWorkspaceConfig: WorkspaceJsonConfiguration & + NxJsonConfiguration; + constructor(private root: string) {} relativeCwd(cwd: string) { @@ -289,6 +292,7 @@ export class Workspaces { readWorkspaceConfiguration(): WorkspaceJsonConfiguration & NxJsonConfiguration { + if (this.cachedWorkspaceConfig) return this.cachedWorkspaceConfig; const nxJsonPath = path.join(this.root, 'nx.json'); const nxJson = readNxJson(nxJsonPath); const workspaceFile = workspaceConfigName(this.root); @@ -305,7 +309,8 @@ export class Workspaces { ); assertValidWorkspaceConfiguration(nxJson); - return { ...workspace, ...nxJson }; + this.cachedWorkspaceConfig = { ...workspace, ...nxJson }; + return this.cachedWorkspaceConfig; } isNxExecutor(nodeModule: string, executor: string) { diff --git a/packages/workspace/src/core/hasher/hasher.spec.ts b/packages/workspace/src/core/hasher/hasher.spec.ts index de1efe2c16d58..17d384ec76b59 100644 --- a/packages/workspace/src/core/hasher/hasher.spec.ts +++ b/packages/workspace/src/core/hasher/hasher.spec.ts @@ -72,7 +72,6 @@ describe('Hasher', () => { }); it('should create project hash', async () => { - hashes['/file'] = 'file.hash'; const hasher = new Hasher( { nodes: { @@ -128,7 +127,6 @@ describe('Hasher', () => { }); it('should create project hash with tsconfig.base.json cache', async () => { - hashes['/file'] = 'file.hash'; const hasher = new Hasher( { nodes: { @@ -227,8 +225,6 @@ describe('Hasher', () => { }); it('should hash projects with dependencies', async () => { - hashes['/filea'] = 'a.hash'; - hashes['/fileb'] = 'b.hash'; const hasher = new Hasher( { nodes: { @@ -237,7 +233,10 @@ describe('Hasher', () => { type: 'lib', data: { root: '', - files: [{ file: '/filea.ts', hash: 'a.hash' }], + files: [ + { file: '/filea.ts', hash: 'a.hash' }, + { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + ], }, }, child: { @@ -245,7 +244,10 @@ describe('Hasher', () => { type: 'lib', data: { root: '', - files: [{ file: '/fileb.ts', hash: 'b.hash' }], + files: [ + { file: '/fileb.ts', hash: 'b.hash' }, + { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + ], }, }, }, @@ -264,6 +266,114 @@ describe('Hasher', () => { overrides: { prop: 'prop-value' }, }); + // note that the parent hash is based on parent source files only! + expect(hash.details.nodes).toEqual({ + child: + '/fileb.ts|/fileb.spec.ts|b.hash|b.spec.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + parent: + '/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + }); + + it('should hash projects with dependencies (exclude spec files of dependencies)', async () => { + const hasher = new Hasher( + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: '', + files: [ + { file: '/filea.ts', hash: 'a.hash' }, + { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + ], + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: '', + files: [ + { file: '/fileb.ts', hash: 'b.hash' }, + { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + ], + }, + }, + }, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + }, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTaskWithDepsAndContext( + { + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }, + 'exclude-tests-of-deps' + ); + + // note that the parent hash is based on parent source files only! + expect(hash.details.nodes).toEqual({ + child: + '/fileb.ts|b.hash|{"root":"libs/child"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + parent: + '/filea.ts|/filea.spec.ts|a.hash|a.spec.hash|{"root":"libs/parent"}|{"compilerOptions":{"paths":{"@nrwl/parent":["libs/parent/src/index.ts"],"@nrwl/child":["libs/child/src/index.ts"]}}}', + }); + }); + + it('should hash projects with dependencies (exclude spec files of all projects)', async () => { + const hasher = new Hasher( + { + nodes: { + parent: { + name: 'parent', + type: 'lib', + data: { + root: '', + files: [ + { file: '/filea.ts', hash: 'a.hash' }, + { file: '/filea.spec.ts', hash: 'a.spec.hash' }, + ], + }, + }, + child: { + name: 'child', + type: 'lib', + data: { + root: '', + files: [ + { file: '/fileb.ts', hash: 'b.hash' }, + { file: '/fileb.spec.ts', hash: 'b.spec.hash' }, + ], + }, + }, + }, + dependencies: { + parent: [{ source: 'parent', target: 'child', type: 'static' }], + }, + }, + {} as any, + {}, + createHashing() + ); + + const hash = await hasher.hashTaskWithDepsAndContext( + { + target: { project: 'parent', target: 'build' }, + id: 'parent-build', + overrides: { prop: 'prop-value' }, + }, + 'exclude-tests-of-all' + ); + // note that the parent hash is based on parent source files only! expect(hash.details.nodes).toEqual({ child: @@ -274,8 +384,6 @@ describe('Hasher', () => { }); it('should hash dependent npm project versions', async () => { - hashes['/filea'] = 'a.hash'; - hashes['/fileb'] = 'b.hash'; const hasher = new Hasher( { nodes: { @@ -324,8 +432,6 @@ describe('Hasher', () => { }); it('should hash when circular dependencies', async () => { - hashes['/filea'] = 'a.hash'; - hashes['/fileb'] = 'b.hash'; const hasher = new Hasher( { nodes: { @@ -396,8 +502,6 @@ describe('Hasher', () => { }); it('should hash implicit deps', async () => { - hashes['/filea'] = 'a.hash'; - hashes['/fileb'] = 'b.hash'; const hasher = new Hasher( { nodes: { @@ -442,8 +546,6 @@ describe('Hasher', () => { }); it('should hash missing dependent npm project versions', async () => { - hashes['/filea'] = 'a.hash'; - hashes['/fileb'] = 'b.hash'; const hasher = new Hasher( { nodes: { diff --git a/packages/workspace/src/core/hasher/hasher.ts b/packages/workspace/src/core/hasher/hasher.ts index 06682197d42e0..a7d3a77bdbb15 100644 --- a/packages/workspace/src/core/hasher/hasher.ts +++ b/packages/workspace/src/core/hasher/hasher.ts @@ -73,13 +73,21 @@ export class Hasher { }); } - async hashTaskWithDepsAndContext(task: Task): Promise { + async hashTaskWithDepsAndContext( + task: Task, + filter: + | 'all-files' + | 'exclude-tests-of-all' + | 'exclude-tests-of-deps' = 'all-files' + ): Promise { const command = this.hashCommand(task); const values = (await Promise.all([ - this.projectHashes.hashProject(task.target.project, [ + this.projectHashes.hashProject( task.target.project, - ]), + [task.target.project], + filter + ), this.implicitDepsHash(), this.runtimeInputsHash(), ])) as [ @@ -131,7 +139,10 @@ export class Hasher { } async hashSource(task: Task): Promise { - return this.projectHashes.hashProjectNodeSource(task.target.project); + return this.projectHashes.hashProjectNodeSource( + task.target.project, + 'all-files' + ); } hashArray(values: string[]): string { @@ -306,7 +317,8 @@ class ProjectHasher { async hashProject( projectName: string, - visited: string[] + visited: string[], + filter: 'all-files' | 'exclude-tests-of-all' | 'exclude-tests-of-deps' ): Promise { return Promise.resolve().then(async () => { const deps = this.projectGraph.dependencies[projectName] ?? []; @@ -317,12 +329,21 @@ class ProjectHasher { return null; } else { visited.push(d.target); - return await this.hashProject(d.target, visited); + return await this.hashProject(d.target, visited, filter); } }) ) ).filter((r) => !!r); - const projectHash = await this.hashProjectNodeSource(projectName); + const filterForProject = + filter === 'all-files' + ? 'all-files' + : filter === 'exclude-tests-of-deps' && visited[0] === projectName + ? 'all-files' + : 'exclude-tests'; + const projectHash = await this.hashProjectNodeSource( + projectName, + filterForProject + ); const nodes = depHashes.reduce( (m, c) => { return { ...m, ...c.nodes }; @@ -337,9 +358,13 @@ class ProjectHasher { }); } - async hashProjectNodeSource(projectName: string) { - if (!this.sourceHashes[projectName]) { - this.sourceHashes[projectName] = new Promise(async (res) => { + async hashProjectNodeSource( + projectName: string, + filter: 'all-files' | 'exclude-tests' + ) { + const mapKey = `${projectName}-${filter}`; + if (!this.sourceHashes[mapKey]) { + this.sourceHashes[mapKey] = new Promise(async (res) => { const p = this.projectGraph.nodes[projectName]; if (!p) { @@ -366,8 +391,13 @@ class ProjectHasher { return; } - const fileNames = p.data.files.map((f) => f.file); - const values = p.data.files.map((f) => f.hash); + const filteredFiles = + filter === 'all-files' + ? p.data.files + : p.data.files.filter((f) => !this.isSpec(f.file)); + + const fileNames = filteredFiles.map((f) => f.file); + const values = filteredFiles.map((f) => f.hash); const workspaceJson = JSON.stringify( this.workspaceJson.projects[projectName] ?? '' @@ -391,7 +421,20 @@ class ProjectHasher { ); }); } - return this.sourceHashes[projectName]; + return this.sourceHashes[mapKey]; + } + + private isSpec(file: string) { + return ( + file.endsWith('.spec.ts') || + file.endsWith('.test.ts') || + file.endsWith('-test.ts') || + file.endsWith('-spec.ts') || + file.endsWith('.spec.js') || + file.endsWith('.test.js') || + file.endsWith('-test.js') || + file.endsWith('-spec.js') + ); } private removeOtherProjectsPathRecords(projectName: string) { diff --git a/packages/workspace/src/tasks-runner/task-orchestrator.ts b/packages/workspace/src/tasks-runner/task-orchestrator.ts index 24ea9c2d7183b..4021812a1821b 100644 --- a/packages/workspace/src/tasks-runner/task-orchestrator.ts +++ b/packages/workspace/src/tasks-runner/task-orchestrator.ts @@ -24,6 +24,7 @@ export class TaskOrchestrator { private forkedProcessTaskRunner = new ForkedProcessTaskRunner(this.options); private tasksSchedule = new TasksSchedule( this.hasher, + this.projectGraph, this.taskGraph, this.workspace, this.options diff --git a/packages/workspace/src/tasks-runner/tasks-schedule.spec.ts b/packages/workspace/src/tasks-runner/tasks-schedule.spec.ts index dc56bd193b739..d522d9c7a4ab7 100644 --- a/packages/workspace/src/tasks-runner/tasks-schedule.spec.ts +++ b/packages/workspace/src/tasks-runner/tasks-schedule.spec.ts @@ -82,12 +82,15 @@ describe('TasksSchedule', () => { }, }; + const projectGraph = {} as any; + const hasher = { hashTaskWithDepsAndContext: () => 'hash', } as any; taskSchedule = new TasksSchedule( hasher, + projectGraph, taskGraph, workspace as Workspaces, { diff --git a/packages/workspace/src/tasks-runner/tasks-schedule.ts b/packages/workspace/src/tasks-runner/tasks-schedule.ts index bd1ef2c96ded0..375cfc462bab9 100644 --- a/packages/workspace/src/tasks-runner/tasks-schedule.ts +++ b/packages/workspace/src/tasks-runner/tasks-schedule.ts @@ -1,4 +1,9 @@ -import { Task, TaskGraph } from '@nrwl/devkit'; +import { + ProjectGraph, + Task, + TaskGraph, + WorkspaceConfiguration, +} from '@nrwl/devkit'; import { Workspaces } from '@nrwl/tao/src/shared/workspace'; @@ -29,9 +34,10 @@ export class TasksSchedule { constructor( private readonly hasher: Hasher, - private taskGraph: TaskGraph, - private workspace: Workspaces, - private options: DefaultTasksRunnerOptions + private readonly projectGraph: ProjectGraph, + private readonly taskGraph: TaskGraph, + private readonly workspaces: Workspaces, + private readonly options: DefaultTasksRunnerOptions ) {} public async scheduleNextTasks() { @@ -97,7 +103,7 @@ export class TasksSchedule { const batchMap: Record = {}; for (const root of this.notScheduledTaskGraph.roots) { const rootTask = this.notScheduledTaskGraph.tasks[root]; - const executorName = getExecutorNameForTask(rootTask, this.workspace); + const executorName = getExecutorNameForTask(rootTask, this.workspaces); this.processTaskForBatches(batchMap, rootTask, executorName, true); } for (const [executorName, taskGraph] of Object.entries(batchMap)) { @@ -123,9 +129,9 @@ export class TasksSchedule { ) { const { batchImplementationFactory } = getExecutorForTask( task, - this.workspace + this.workspaces ); - const executorName = getExecutorNameForTask(task, this.workspace); + const executorName = getExecutorNameForTask(task, this.workspaces); if (rootExecutorName !== executorName) { return; } @@ -161,9 +167,14 @@ export class TasksSchedule { } private async hashTask(task: Task) { - const customHasher = getCustomHasher(task, this.workspace); + const customHasher = getCustomHasher(task, this.workspaces); const { value, details } = await (customHasher - ? customHasher(task, this.taskGraph, this.hasher) + ? customHasher(task, { + hasher: this.hasher, + projectGraph: this.projectGraph, + taskGraph: this.taskGraph, + workspaceConfig: this.workspaces.readWorkspaceConfiguration(), + }) : this.hasher.hashTaskWithDepsAndContext(task)); task.hash = value; task.hashDetails = details;