feat(core): support nested ignore files

This change adds support for ignore files (`.gitignore` and `.nxignore`) at any level, not just the workspace root. Previously those files were not taken into account. Code to evaluate ignore files was repeated multiple times throughout the codebase and has been refactor into a utility module. Closes #10442
nrwl · Jun 7, 2022 · d7c9657 · d7c9657
1 parent 838170e
commit d7c9657
Show file tree

Hide file tree

Showing 18 changed files with 417 additions and 245 deletions.
diff --git a/docs/generated/devkit/index.md b/docs/generated/devkit/index.md
@@ -132,6 +132,8 @@ It only uses language primitives and immutable objects
 - [applyChangesToString](../../devkit/index#applychangestostring)
 - [convertNxExecutor](../../devkit/index#convertnxexecutor)
 - [convertNxGenerator](../../devkit/index#convertnxgenerator)
+- [createIgnoreFromFS](../../devkit/index#createignorefromfs)
+- [createIgnoreFromTree](../../devkit/index#createignorefromtree)
 - [createProjectGraphAsync](../../devkit/index#createprojectgraphasync)
 - [defaultTasksRunner](../../devkit/index#defaulttasksrunner)
 - [detectPackageManager](../../devkit/index#detectpackagemanager)
@@ -897,6 +899,46 @@ Convert an Nx Generator into an Angular Devkit Schematic.
 
 ---
 
+### createIgnoreFromFS
+
+▸ **createIgnoreFromFS**(`rootDir`, `ignoreFiles`): `Ignore`
+
+Reads ignore files from the file system and returns an object which can be
+used to check whether a file should be ignored.
+
+#### Parameters
+
+| Name          | Type       | Description                                                                                                                            |
+| :------------ | :--------- | :------------------------------------------------------------------------------------------------------------------------------------- |
+| `rootDir`     | `string`   | The directory in which to start searching for ignore files. Paths evaluated by the returned object must be relative to this directory. |
+| `ignoreFiles` | `string`[] | The filename of ignore files to include, e.g. ".gitignore"                                                                             |
+
+#### Returns
+
+`Ignore`
+
+---
+
+### createIgnoreFromTree
+
+▸ **createIgnoreFromTree**(`tree`, `ignoreFiles`): `Ignore`
+
+Reads ignore files from a Tree and returns an object which can be
+used to check whether a file should be ignored.
+
+#### Parameters
+
+| Name          | Type                              | Description                                                                                                                |
+| :------------ | :-------------------------------- | :------------------------------------------------------------------------------------------------------------------------- |
+| `tree`        | [`Tree`](../../devkit/index#tree) | The tree in which to searching for ignore files. Paths evaluated by the returned object must be relative to the tree root. |
+| `ignoreFiles` | `string`[]                        | The filename of ignore files to include, e.g. ".gitignore"                                                                 |
+
+#### Returns
+
+`Ignore`
+
+---
+
 ### createProjectGraphAsync
 
 ▸ **createProjectGraphAsync**(): `Promise`<[`ProjectGraph`](../../devkit/index#projectgraph)\>

diff --git a/docs/generated/packages/devkit.json b/docs/generated/packages/devkit.json
diff --git a/packages/add-nx-to-monorepo/src/add-nx-to-monorepo.ts b/packages/add-nx-to-monorepo/src/add-nx-to-monorepo.ts
@@ -1,13 +1,17 @@
 #!/usr/bin/env node
 
-import * as path from 'path';
-import * as fs from 'fs';
+import {
+  createIgnoreFromFS,
+  output,
+  readJsonFile,
+  writeJsonFile,
+} from '@nrwl/devkit';
 import * as cp from 'child_process';
 import { execSync } from 'child_process';
 import * as enquirer from 'enquirer';
+import * as fs from 'fs';
+import * as path from 'path';
 import * as yargsParser from 'yargs-parser';
-import { output, readJsonFile, writeJsonFile } from '@nrwl/devkit';
-import ignore from 'ignore';
 
 const parsedArgs = yargsParser(process.argv, {
   boolean: ['nxCloud'],
@@ -89,12 +93,11 @@ function allProjectPackageJsonFiles(repoRoot: string) {
 }
 
 function allPackageJsonFiles(repoRoot: string, dirName: string) {
-  const ignoredGlobs = getIgnoredGlobs(repoRoot);
+  const ignore = createIgnoreFromFS(repoRoot, ['.gitignore']);
   const relDirName = path.relative(repoRoot, dirName);
   if (
     relDirName &&
-    (ignoredGlobs.ignores(relDirName) ||
-      relDirName.indexOf(`node_modules`) > -1)
+    (ignore.ignores(relDirName) || relDirName.indexOf(`node_modules`) > -1)
   ) {
     return [];
   }
@@ -103,7 +106,7 @@ function allPackageJsonFiles(repoRoot: string, dirName: string) {
   try {
     fs.readdirSync(dirName).forEach((c) => {
       const child = path.join(dirName, c);
-      if (ignoredGlobs.ignores(path.relative(repoRoot, child))) {
+      if (ignore.ignores(path.relative(repoRoot, child))) {
         return;
       }
       try {
@@ -121,15 +124,6 @@ function allPackageJsonFiles(repoRoot: string, dirName: string) {
   return res;
 }
 
-function getIgnoredGlobs(repoRoot: string) {
-  const ig = ignore();
-  try {
-    ig.add(fs.readFileSync(`${repoRoot}/.gitignore`).toString());
-    // eslint-disable-next-line no-empty
-  } catch (e) {}
-  return ig;
-}
-
 // creating project descs
 interface ProjectDesc {
   name: string;

diff --git a/packages/angular/src/executors/file-server/file-server.impl.ts b/packages/angular/src/executors/file-server/file-server.impl.ts
@@ -1,17 +1,15 @@
-import { execFileSync, fork } from 'child_process';
 import {
   ExecutorContext,
   joinPathFragments,
-  readJsonFile,
   workspaceLayout,
 } from '@nrwl/devkit';
-import ignore from 'ignore';
-import { readFileSync } from 'fs';
-import { Schema } from './schema';
+import { execFileSync, fork } from 'child_process';
 import { watch } from 'chokidar';
+import { createWorkspaceIgnore } from 'nx/src/utils/ignore';
+import { readModulePackageJson } from 'nx/src/utils/package-json';
 import { platform } from 'os';
 import { resolve } from 'path';
-import { readModulePackageJson } from 'nx/src/utils/package-json';
+import { Schema } from './schema';
 
 // platform specific command name
 const pmCmd = platform() === 'win32' ? `npx.cmd` : 'npx';
@@ -85,22 +83,11 @@ function getBuildTargetOutputPath(options: Schema, context: ExecutorContext) {
   return outputPath;
 }
 
-function getIgnoredGlobs(root: string) {
-  const ig = ignore();
-  try {
-    ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
-  } catch {}
-  try {
-    ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
-  } catch {}
-  return ig;
-}
-
 function createFileWatcher(
   root: string,
   changeHandler: () => void
 ): () => void {
-  const ignoredGlobs = getIgnoredGlobs(root);
+  const ignore = createWorkspaceIgnore(root);
   const layout = workspaceLayout();
 
   const watcher = watch(
@@ -114,7 +101,7 @@ function createFileWatcher(
     }
   );
   watcher.on('all', (_event: string, path: string) => {
-    if (ignoredGlobs.ignores(path)) return;
+    if (ignore.ignores(path)) return;
     changeHandler();
   });
   return () => watcher.close();

diff --git a/packages/angular/src/generators/stories/lib/tree-utilities.ts b/packages/angular/src/generators/stories/lib/tree-utilities.ts
@@ -1,36 +1,31 @@
 import type { Tree } from '@nrwl/devkit';
 import { joinPathFragments } from '@nrwl/devkit';
-import type { Ignore } from 'ignore';
-import ignore from 'ignore';
+import { createGitignoreFromTree } from 'nx/src/utils/ignore';
 
 export function getAllFilesRecursivelyFromDir(
   tree: Tree,
   dir: string
 ): string[] {
-  if (isPathIgnored(tree, dir)) {
-    return [];
-  }
+  const ignore = createGitignoreFromTree(tree);
 
-  const files: string[] = [];
-  const children = tree.children(dir);
-  children.forEach((child) => {
-    const childPath = joinPathFragments(dir, child);
-    if (tree.isFile(childPath)) {
-      files.push(childPath);
-    } else {
-      files.push(...getAllFilesRecursivelyFromDir(tree, childPath));
+  function inner(dir: string) {
+    if (ignore.ignores(dir)) {
+      return [];
     }
-  });
 
-  return files;
-}
+    const files: string[] = [];
+    const children = tree.children(dir);
+    children.forEach((child) => {
+      const childPath = joinPathFragments(dir, child);
+      if (tree.isFile(childPath)) {
+        files.push(childPath);
+      } else {
+        files.push(...inner(childPath));
+      }
+    });
 
-function isPathIgnored(tree: Tree, path: string): boolean {
-  let ig: Ignore;
-  if (tree.exists('.gitignore')) {
-    ig = ignore();
-    ig.add(tree.read('.gitignore', 'utf-8'));
+    return files;
   }
 
-  return path !== '' && ig?.ignores(path);
+  return inner(dir);
 }
diff --git a/packages/devkit/index.ts b/packages/devkit/index.ts
@@ -319,3 +319,8 @@ export { Hash, Hasher } from 'nx/src/hasher/hasher';
  * @category Utils
  */
 export { cacheDir } from 'nx/src/utils/cache-directory';
+
+/**
+ * @category Utils
+ */
+export { createIgnoreFromFS, createIgnoreFromTree } from 'nx/src/utils/ignore';
diff --git a/packages/devkit/src/generators/visit-not-ignored-files.ts b/packages/devkit/src/generators/visit-not-ignored-files.ts
@@ -1,5 +1,5 @@
 import type { Tree } from 'nx/src/generators/tree';
-import ignore, { Ignore } from 'ignore';
+import { createGitignoreFromTree } from 'nx/src/utils/ignore';
 import { join, relative, sep } from 'path';
 
 /**
@@ -10,26 +10,27 @@ export function visitNotIgnoredFiles(
   dirPath: string = tree.root,
   visitor: (path: string) => void
 ): void {
-  let ig: Ignore;
-  if (tree.exists('.gitignore')) {
-    ig = ignore();
-    ig.add(tree.read('.gitignore', 'utf-8'));
-  }
-  dirPath = normalizePathRelativeToRoot(dirPath, tree.root);
-  if (dirPath !== '' && ig?.ignores(dirPath)) {
-    return;
-  }
-  for (const child of tree.children(dirPath)) {
-    const fullPath = join(dirPath, child);
-    if (ig?.ignores(fullPath)) {
-      continue;
+  const ignore = createGitignoreFromTree(tree);
+
+  function inner(dirPath: string) {
+    dirPath = normalizePathRelativeToRoot(dirPath, tree.root);
+    if (dirPath !== '' && ignore.ignores(dirPath)) {
+      return;
     }
-    if (tree.isFile(fullPath)) {
-      visitor(fullPath);
-    } else {
-      visitNotIgnoredFiles(tree, fullPath, visitor);
+    for (const child of tree.children(dirPath)) {
+      const fullPath = join(dirPath, child);
+      if (ignore.ignores(fullPath)) {
+        continue;
+      }
+      if (tree.isFile(fullPath)) {
+        visitor(fullPath);
+      } else {
+        inner(fullPath);
+      }
     }
   }
+
+  inner(dirPath);
 }
 
 function normalizePathRelativeToRoot(path: string, root: string): string {

diff --git a/packages/js/src/utils/copy-assets-handler.ts b/packages/js/src/utils/copy-assets-handler.ts
@@ -1,11 +1,12 @@
+import { logger } from '@nrwl/devkit';
+import { AssetGlob } from '@nrwl/workspace/src/utilities/assets';
 import type { Event } from '@parcel/watcher';
+import * as fg from 'fast-glob';
+import * as fse from 'fs-extra';
+import { Ignore } from 'ignore';
 import * as minimatch from 'minimatch';
+import { createWorkspaceIgnore } from 'nx/src/utils/ignore';
 import * as path from 'path';
-import * as fse from 'fs-extra';
-import ignore, { Ignore } from 'ignore';
-import * as fg from 'fast-glob';
-import { AssetGlob } from '@nrwl/workspace/src/utilities/assets';
-import { logger } from '@nrwl/devkit';
 
 export type FileEventType = 'create' | 'update' | 'delete';
 
@@ -58,15 +59,7 @@ export class CopyAssetsHandler {
     this.projectDir = opts.projectDir;
     this.outputDir = opts.outputDir;
     this.callback = opts.callback ?? defaultFileEventHandler;
-
-    // TODO(jack): Should handle nested .gitignore files
-    this.ignore = ignore();
-    const gitignore = path.join(opts.rootDir, '.gitignore');
-    const nxignore = path.join(opts.rootDir, '.nxignore');
-    if (fse.existsSync(gitignore))
-      this.ignore.add(fse.readFileSync(gitignore).toString());
-    if (fse.existsSync(nxignore))
-      this.ignore.add(fse.readFileSync(nxignore).toString());
+    this.ignore = createWorkspaceIgnore(opts.rootDir);
 
     this.assetGlobs = opts.assets.map((f) => {
       let isGlob = false;

diff --git a/packages/nx/src/command-line/dep-graph.ts b/packages/nx/src/command-line/dep-graph.ts
@@ -1,26 +1,26 @@
-import { workspaceRoot } from 'nx/src/utils/workspace-root';
 import { watch } from 'chokidar';
 import { createHash } from 'crypto';
 import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { copySync, ensureDirSync } from 'fs-extra';
 import * as http from 'http';
-import ignore from 'ignore';
 import * as open from 'open';
 import { basename, dirname, extname, isAbsolute, join, parse } from 'path';
 import { performance } from 'perf_hooks';
 import { URL, URLSearchParams } from 'url';
 import { workspaceLayout } from '../config/configuration';
-import { defaultFileHasher } from '../hasher/file-hasher';
-import { output } from '../utils/output';
-import { writeJsonFile } from '../utils/fileutils';
-import { joinPathFragments } from '../utils/path';
 import {
   ProjectGraph,
   ProjectGraphDependency,
   ProjectGraphProjectNode,
 } from '../config/project-graph';
+import { defaultFileHasher } from '../hasher/file-hasher';
 import { pruneExternalNodes } from '../project-graph/operators';
 import { createProjectGraphAsync } from '../project-graph/project-graph';
+import { writeJsonFile } from '../utils/fileutils';
+import { createWorkspaceIgnore } from '../utils/ignore';
+import { output } from '../utils/output';
+import { joinPathFragments } from '../utils/path';
+import { workspaceRoot } from '../utils/workspace-root';
 
 export interface DepGraphClientResponse {
   hash: string;
@@ -408,17 +408,6 @@ let currentDepGraphClientResponse: DepGraphClientResponse = {
   exclude: [],
 };
 
-function getIgnoredGlobs(root: string) {
-  const ig = ignore();
-  try {
-    ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
-  } catch {}
-  try {
-    ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
-  } catch {}
-  return ig;
-}
-
 function startWatcher() {
   createFileWatcher(workspaceRoot, async () => {
     output.note({ title: 'Recalculating project graph...' });
@@ -448,7 +437,7 @@ function debounce(fn: (...args) => void, time: number) {
 }
 
 function createFileWatcher(root: string, changeHandler: () => Promise<void>) {
-  const ignoredGlobs = getIgnoredGlobs(root);
+  const ignore = createWorkspaceIgnore(root);
   const layout = workspaceLayout();
 
   const watcher = watch(
@@ -464,7 +453,7 @@ function createFileWatcher(root: string, changeHandler: () => Promise<void>) {
   watcher.on(
     'all',
     debounce(async (event: string, path: string) => {
-      if (ignoredGlobs.ignores(path)) return;
+      if (ignore.ignores(path)) return;
       await changeHandler();
     }, 500)
   );