feat(core): support nested ignore files

This change adds support for ignore files (`.gitignore` and `.nxignore`) at any level, not just the workspace root. Previously those files were not taken into account. Code to evaluate ignore files was repeated multiple times throughout the codebase and has been refactor into a utility module. Closes #10442
nrwl · May 24, 2022 · 924f44c · 924f44c
1 parent e2a4ce8
commit 924f44c
Show file tree

Hide file tree

Showing 13 changed files with 335 additions and 223 deletions.
diff --git a/packages/angular/src/executors/file-server/file-server.impl.ts b/packages/angular/src/executors/file-server/file-server.impl.ts
@@ -1,16 +1,15 @@
-import { execFileSync, fork } from 'child_process';
 import {
   ExecutorContext,
   joinPathFragments,
   readJsonFile,
   workspaceLayout,
 } from '@nrwl/devkit';
-import ignore from 'ignore';
-import { readFileSync } from 'fs';
-import { Schema } from './schema';
+import { execFileSync, fork } from 'child_process';
 import { watch } from 'chokidar';
+import { createWorkspaceIgnore } from 'nx/src/utils/ignore';
 import { platform } from 'os';
 import { resolve } from 'path';
+import { Schema } from './schema';
 
 // platform specific command name
 const pmCmd = platform() === 'win32' ? `npx.cmd` : 'npx';
@@ -84,22 +83,11 @@ function getBuildTargetOutputPath(options: Schema, context: ExecutorContext) {
   return outputPath;
 }
 
-function getIgnoredGlobs(root: string) {
-  const ig = ignore();
-  try {
-    ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
-  } catch {}
-  try {
-    ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
-  } catch {}
-  return ig;
-}
-
 function createFileWatcher(
   root: string,
   changeHandler: () => void
 ): () => void {
-  const ignoredGlobs = getIgnoredGlobs(root);
+  const ignore = createWorkspaceIgnore(root);
   const layout = workspaceLayout();
 
   const watcher = watch(
@@ -113,7 +101,7 @@ function createFileWatcher(
     }
   );
   watcher.on('all', (_event: string, path: string) => {
-    if (ignoredGlobs.ignores(path)) return;
+    if (ignore.ignores(path)) return;
     changeHandler();
   });
   return () => watcher.close();

diff --git a/packages/angular/src/generators/stories/lib/tree-utilities.ts b/packages/angular/src/generators/stories/lib/tree-utilities.ts
@@ -1,36 +1,31 @@
 import type { Tree } from '@nrwl/devkit';
 import { joinPathFragments } from '@nrwl/devkit';
-import type { Ignore } from 'ignore';
-import ignore from 'ignore';
+import { createGitignoreFromTree } from 'nx/src/utils/ignore';
 
 export function getAllFilesRecursivelyFromDir(
   tree: Tree,
   dir: string
 ): string[] {
-  if (isPathIgnored(tree, dir)) {
-    return [];
-  }
+  const ignore = createGitignoreFromTree(tree);
 
-  const files: string[] = [];
-  const children = tree.children(dir);
-  children.forEach((child) => {
-    const childPath = joinPathFragments(dir, child);
-    if (tree.isFile(childPath)) {
-      files.push(childPath);
-    } else {
-      files.push(...getAllFilesRecursivelyFromDir(tree, childPath));
+  function inner(dir: string) {
+    if (ignore.ignores(dir)) {
+      return [];
     }
-  });
 
-  return files;
-}
+    const files: string[] = [];
+    const children = tree.children(dir);
+    children.forEach((child) => {
+      const childPath = joinPathFragments(dir, child);
+      if (tree.isFile(childPath)) {
+        files.push(childPath);
+      } else {
+        files.push(...inner(childPath));
+      }
+    });
 
-function isPathIgnored(tree: Tree, path: string): boolean {
-  let ig: Ignore;
-  if (tree.exists('.gitignore')) {
-    ig = ignore();
-    ig.add(tree.read('.gitignore', 'utf-8'));
+    return files;
   }
 
-  return path !== '' && ig?.ignores(path);
+  return inner(dir);
 }
diff --git a/packages/devkit/src/generators/visit-not-ignored-files.ts b/packages/devkit/src/generators/visit-not-ignored-files.ts
@@ -1,5 +1,5 @@
 import type { Tree } from 'nx/src/generators/tree';
-import ignore, { Ignore } from 'ignore';
+import { createGitignoreFromTree } from 'nx/src/utils/ignore';
 import { join, relative, sep } from 'path';
 
 /**
@@ -10,26 +10,27 @@ export function visitNotIgnoredFiles(
   dirPath: string = tree.root,
   visitor: (path: string) => void
 ): void {
-  let ig: Ignore;
-  if (tree.exists('.gitignore')) {
-    ig = ignore();
-    ig.add(tree.read('.gitignore', 'utf-8'));
-  }
-  dirPath = normalizePathRelativeToRoot(dirPath, tree.root);
-  if (dirPath !== '' && ig?.ignores(dirPath)) {
-    return;
-  }
-  for (const child of tree.children(dirPath)) {
-    const fullPath = join(dirPath, child);
-    if (ig?.ignores(fullPath)) {
-      continue;
+  const ignore = createGitignoreFromTree(tree);
+
+  function inner(dirPath: string) {
+    dirPath = normalizePathRelativeToRoot(dirPath, tree.root);
+    if (dirPath !== '' && ignore.ignores(dirPath)) {
+      return;
     }
-    if (tree.isFile(fullPath)) {
-      visitor(fullPath);
-    } else {
-      visitNotIgnoredFiles(tree, fullPath, visitor);
+    for (const child of tree.children(dirPath)) {
+      const fullPath = join(dirPath, child);
+      if (ignore.ignores(fullPath)) {
+        continue;
+      }
+      if (tree.isFile(fullPath)) {
+        visitor(fullPath);
+      } else {
+        inner(fullPath);
+      }
     }
   }
+
+  inner(dirPath);
 }
 
 function normalizePathRelativeToRoot(path: string, root: string): string {

diff --git a/packages/js/src/utils/copy-assets-handler.ts b/packages/js/src/utils/copy-assets-handler.ts
@@ -1,11 +1,12 @@
+import { logger } from '@nrwl/devkit';
+import { AssetGlob } from '@nrwl/workspace/src/utilities/assets';
 import type { Event } from '@parcel/watcher';
+import * as fg from 'fast-glob';
+import * as fse from 'fs-extra';
+import { Ignore } from 'ignore';
 import * as minimatch from 'minimatch';
+import { createWorkspaceIgnore } from 'nx/src/utils/ignore';
 import * as path from 'path';
-import * as fse from 'fs-extra';
-import ignore, { Ignore } from 'ignore';
-import * as fg from 'fast-glob';
-import { AssetGlob } from '@nrwl/workspace/src/utilities/assets';
-import { logger } from '@nrwl/devkit';
 
 export type FileEventType = 'create' | 'update' | 'delete';
 
@@ -58,15 +59,7 @@ export class CopyAssetsHandler {
     this.projectDir = opts.projectDir;
     this.outputDir = opts.outputDir;
     this.callback = opts.callback ?? defaultFileEventHandler;
-
-    // TODO(jack): Should handle nested .gitignore files
-    this.ignore = ignore();
-    const gitignore = path.join(opts.rootDir, '.gitignore');
-    const nxignore = path.join(opts.rootDir, '.nxignore');
-    if (fse.existsSync(gitignore))
-      this.ignore.add(fse.readFileSync(gitignore).toString());
-    if (fse.existsSync(nxignore))
-      this.ignore.add(fse.readFileSync(nxignore).toString());
+    this.ignore = createWorkspaceIgnore(opts.rootDir);
 
     this.assetGlobs = opts.assets.map((f) => {
       let isGlob = false;

diff --git a/packages/nx/src/command-line/dep-graph.ts b/packages/nx/src/command-line/dep-graph.ts
@@ -1,26 +1,26 @@
-import { workspaceRoot } from 'nx/src/utils/app-root';
 import { watch } from 'chokidar';
 import { createHash } from 'crypto';
 import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { copySync, ensureDirSync } from 'fs-extra';
 import * as http from 'http';
-import ignore from 'ignore';
 import * as open from 'open';
 import { basename, dirname, extname, isAbsolute, join, parse } from 'path';
 import { performance } from 'perf_hooks';
 import { URL, URLSearchParams } from 'url';
-import { workspaceLayout } from '../project-graph/file-utils';
-import { defaultFileHasher } from '../hasher/file-hasher';
-import { output } from '../utils/output';
-import { writeJsonFile } from '../utils/fileutils';
-import { joinPathFragments } from '../utils/path';
 import {
   ProjectGraph,
   ProjectGraphDependency,
   ProjectGraphProjectNode,
 } from '../config/project-graph';
+import { defaultFileHasher } from '../hasher/file-hasher';
+import { workspaceLayout } from '../project-graph/file-utils';
 import { pruneExternalNodes } from '../project-graph/operators';
 import { createProjectGraphAsync } from '../project-graph/project-graph';
+import { workspaceRoot } from '../utils/app-root';
+import { writeJsonFile } from '../utils/fileutils';
+import { createWorkspaceIgnore } from '../utils/ignore';
+import { output } from '../utils/output';
+import { joinPathFragments } from '../utils/path';
 
 export interface DepGraphClientResponse {
   hash: string;
@@ -408,17 +408,6 @@ let currentDepGraphClientResponse: DepGraphClientResponse = {
   exclude: [],
 };
 
-function getIgnoredGlobs(root: string) {
-  const ig = ignore();
-  try {
-    ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
-  } catch {}
-  try {
-    ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
-  } catch {}
-  return ig;
-}
-
 function startWatcher() {
   createFileWatcher(workspaceRoot, async () => {
     output.note({ title: 'Recalculating project graph...' });
@@ -448,7 +437,7 @@ function debounce(fn: (...args) => void, time: number) {
 }
 
 function createFileWatcher(root: string, changeHandler: () => Promise<void>) {
-  const ignoredGlobs = getIgnoredGlobs(root);
+  const ignore = createWorkspaceIgnore(root);
   const layout = workspaceLayout();
 
   const watcher = watch(
@@ -464,7 +453,7 @@ function createFileWatcher(root: string, changeHandler: () => Promise<void>) {
   watcher.on(
     'all',
     debounce(async (event: string, path: string) => {
-      if (ignoredGlobs.ignores(path)) return;
+      if (ignore.ignores(path)) return;
       await changeHandler();
     }, 500)
   );

diff --git a/packages/nx/src/config/workspaces.ts b/packages/nx/src/config/workspaces.ts
@@ -1,31 +1,30 @@
 import { sync as globSync } from 'fast-glob';
-import { existsSync, readFileSync } from 'fs';
-import ignore, { Ignore } from 'ignore';
+import { existsSync } from 'fs';
+import { Ignore } from 'ignore';
 import * as path from 'path';
 import { basename, dirname, join } from 'path';
 import { performance } from 'perf_hooks';
-
 import { workspaceRoot } from '../utils/app-root';
 import { readJsonFile } from '../utils/fileutils';
+import { createWorkspaceIgnore } from '../utils/ignore';
 import { logger } from '../utils/logger';
 import { loadNxPlugins, readPluginPackageJson } from '../utils/nx-plugin';
-
-import type { NxJsonConfiguration } from './nx-json';
-import {
-  ProjectConfiguration,
-  WorkspaceJsonConfiguration,
-} from './workspace-json-project-json';
+import { sortObjectByKeys } from '../utils/object-sort';
+import { PackageJson } from '../utils/package-json';
 import {
+  CustomHasher,
   Executor,
   ExecutorConfig,
-  TaskGraphExecutor,
+  ExecutorsJson,
   Generator,
   GeneratorsJson,
-  ExecutorsJson,
-  CustomHasher,
+  TaskGraphExecutor,
 } from './misc-interfaces';
-import { PackageJson } from '../utils/package-json';
-import { sortObjectByKeys } from 'nx/src/utils/object-sort';
+import type { NxJsonConfiguration } from './nx-json';
+import {
+  ProjectConfiguration,
+  WorkspaceJsonConfiguration,
+} from './workspace-json-project-json';
 
 export function workspaceConfigName(root: string) {
   if (existsSync(path.join(root, 'angular.json'))) {
@@ -509,7 +508,7 @@ function getGlobPatternsFromPackageManagerWorkspaces(root: string): string[] {
 }
 
 export function globForProjectFiles(
-  root,
+  root: string,
   nxJson?: NxJsonConfiguration,
   ignorePluginInference = false
 ) {
@@ -551,24 +550,14 @@ export function globForProjectFiles(
     join(root, '.git'),
   ];
 
-  /**
-   * TODO: This utility has been implemented multiple times across the Nx codebase,
-   * discuss whether it should be moved to a shared location.
-   */
-  const ig = ignore();
-  try {
-    ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
-  } catch {}
-  try {
-    ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
-  } catch {}
+  const ignore = createWorkspaceIgnore(root);
 
   const globResults = globSync(combinedProjectGlobPattern, {
     ignore: ALWAYS_IGNORE,
     absolute: false,
     cwd: root,
   });
-  projectGlobCache = deduplicateProjectFiles(globResults, ig);
+  projectGlobCache = deduplicateProjectFiles(globResults, ignore);
   performance.mark('finish-glob-for-projects');
   performance.measure(
     'glob-for-project-files',
@@ -578,15 +567,15 @@ export function globForProjectFiles(
   return projectGlobCache;
 }
 
-export function deduplicateProjectFiles(files: string[], ig?: Ignore) {
+export function deduplicateProjectFiles(files: string[], ignore?: Ignore) {
   const filtered = new Map();
   files.forEach((file) => {
     const projectFolder = dirname(file);
     const projectFile = basename(file);
     if (
-      ig?.ignores(file) || // file is in .gitignore or .nxignore
+      ignore?.ignores(file) || // file is in .gitignore or .nxignore
       file === 'package.json' || // file is workspace root package json
-      // project.json or equivallent inferred project file has been found
+      // project.json or equivalent inferred project file has been found
       (filtered.has(projectFolder) && projectFile !== 'project.json')
     ) {
       return;