feat(core): support nested ignore files

docs/generated/devkit/index.md

@@ -132,6 +132,8 @@ It only uses language primitives and immutable objects
 - [applyChangesToString](../../devkit/index#applychangestostring)
 - [convertNxExecutor](../../devkit/index#convertnxexecutor)
 - [convertNxGenerator](../../devkit/index#convertnxgenerator)
+- [createIgnore](../../devkit/index#createignore)
+- [createIgnoreFromTree](../../devkit/index#createignorefromtree)
 - [createProjectGraphAsync](../../devkit/index#createprojectgraphasync)
 - [defaultTasksRunner](../../devkit/index#defaulttasksrunner)
 - [detectPackageManager](../../devkit/index#detectpackagemanager)
@@ -897,6 +899,46 @@ Convert an Nx Generator into an Angular Devkit Schematic.
 
 ---
 
+### createIgnore
+
+▸ **createIgnore**(`rootDir?`, `ignoreFiles?`): `Ignore`
+
+Reads ignore files from the file system and returns an object which can be
+used to check whether a file should be ignored.
+
+#### Parameters
+
+| Name          | Type       | Default value          | Description                                                                                                                                                            |
+| :------------ | :--------- | :--------------------- | :--------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
+| `rootDir`     | `string`   | `workspaceRoot`        | The directory in which to start searching for ignore files. Paths evaluated by the returned object must be relative to this directory. Defaults to the workspace root. |
+| `ignoreFiles` | `string`[] | `workspaceIgnoreFiles` | The filename of ignore files to include, e.g. ".gitignore". Defaults to [".gitignore", ".nxignore"].                                                                   |
+
+#### Returns
+
+`Ignore`
+
+---
+
+### createIgnoreFromTree
+
+▸ **createIgnoreFromTree**(`tree`, `ignoreFiles?`): `Ignore`
+
+Reads ignore files from a Tree and returns an object which can be
+used to check whether a file should be ignored.
+
+#### Parameters
+
+| Name          | Type                              | Default value          | Description                                                                                                                |
+| :------------ | :-------------------------------- | :--------------------- | :------------------------------------------------------------------------------------------------------------------------- |
+| `tree`        | [`Tree`](../../devkit/index#tree) | `undefined`            | The tree in which to searching for ignore files. Paths evaluated by the returned object must be relative to the tree root. |
+| `ignoreFiles` | `string`[]                        | `workspaceIgnoreFiles` | The filename of ignore files to include, e.g. ".gitignore". Defaults to [".gitignore", ".nxignore"].                       |
+
+#### Returns
+
+`Ignore`
+
+---
+
 ### createProjectGraphAsync
 
 ▸ **createProjectGraphAsync**(): `Promise`<[`ProjectGraph`](../../devkit/index#projectgraph)\>

packages/add-nx-to-monorepo/src/add-nx-to-monorepo.ts

@@ -1,13 +1,17 @@
 #!/usr/bin/env node
 
-import * as path from 'path';
-import * as fs from 'fs';
+import {
+  createIgnore,
+  output,
+  readJsonFile,
+  writeJsonFile,
+} from '@nrwl/devkit';
 import * as cp from 'child_process';
 import { execSync } from 'child_process';
 import * as enquirer from 'enquirer';
+import * as fs from 'fs';
+import * as path from 'path';
 import * as yargsParser from 'yargs-parser';
-import { output, readJsonFile, writeJsonFile } from '@nrwl/devkit';
-import ignore from 'ignore';
 import { directoryExists } from 'nx/src/utils/fileutils';
 
 const parsedArgs = yargsParser(process.argv, {
@@ -90,12 +94,11 @@ function allProjectPackageJsonFiles(repoRoot: string) {
 }
 
 function allPackageJsonFiles(repoRoot: string, dirName: string) {
-  const ignoredGlobs = getIgnoredGlobs(repoRoot);
+  const ignore = createIgnore(repoRoot, ['.gitignore']);
   const relDirName = path.relative(repoRoot, dirName);
   if (
     relDirName &&
-    (ignoredGlobs.ignores(relDirName) ||
-      relDirName.indexOf(`node_modules`) > -1)
+    (ignore.ignores(relDirName) || relDirName.indexOf(`node_modules`) > -1)
   ) {
     return [];
   }
@@ -104,7 +107,7 @@ function allPackageJsonFiles(repoRoot: string, dirName: string) {
   try {
     fs.readdirSync(dirName).forEach((c) => {
       const child = path.join(dirName, c);
-      if (ignoredGlobs.ignores(path.relative(repoRoot, child))) {
+      if (ignore.ignores(path.relative(repoRoot, child))) {
         return;
       }
       try {
@@ -122,15 +125,6 @@ function allPackageJsonFiles(repoRoot: string, dirName: string) {
   return res;
 }
 
-function getIgnoredGlobs(repoRoot: string) {
-  const ig = ignore();
-  try {
-    ig.add(fs.readFileSync(`${repoRoot}/.gitignore`).toString());
-    // eslint-disable-next-line no-empty
-  } catch (e) {}
-  return ig;
-}
-
 // creating project descs
 interface ProjectDesc {
   name: string;

packages/angular/src/executors/file-server/file-server.impl.ts

@@ -1,17 +1,15 @@
-import { execFileSync, fork } from 'child_process';
 import {
   ExecutorContext,
   joinPathFragments,
-  readJsonFile,
   workspaceLayout,
 } from '@nrwl/devkit';
-import ignore from 'ignore';
-import { readFileSync } from 'fs';
-import { Schema } from './schema';
+import { execFileSync, fork } from 'child_process';
 import { watch } from 'chokidar';
+import { createIgnore } from 'nx/src/utils/ignore';
+import { readModulePackageJson } from 'nx/src/utils/package-json';
 import { platform } from 'os';
 import { resolve } from 'path';
-import { readModulePackageJson } from 'nx/src/utils/package-json';
+import { Schema } from './schema';
 
 // platform specific command name
 const pmCmd = platform() === 'win32' ? `npx.cmd` : 'npx';
@@ -85,22 +83,11 @@ function getBuildTargetOutputPath(options: Schema, context: ExecutorContext) {
   return outputPath;
 }
 
-function getIgnoredGlobs(root: string) {
-  const ig = ignore();
-  try {
-    ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
-  } catch {}
-  try {
-    ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
-  } catch {}
-  return ig;
-}
-
 function createFileWatcher(
   root: string,
   changeHandler: () => void
 ): () => void {
-  const ignoredGlobs = getIgnoredGlobs(root);
+  const ignore = createIgnore(root);
   const layout = workspaceLayout();
 
   const watcher = watch(
@@ -114,7 +101,7 @@ function createFileWatcher(
     }
   );
   watcher.on('all', (_event: string, path: string) => {
-    if (ignoredGlobs.ignores(path)) return;
+    if (ignore.ignores(path)) return;
     changeHandler();
   });
   return () => watcher.close();

packages/angular/src/generators/stories/lib/tree-utilities.ts

@@ -1,36 +1,31 @@
 import type { Tree } from '@nrwl/devkit';
 import { joinPathFragments } from '@nrwl/devkit';
-import type { Ignore } from 'ignore';
-import ignore from 'ignore';
+import { createGitIgnoreFromTree } from 'nx/src/utils/ignore';
 
 export function getAllFilesRecursivelyFromDir(
   tree: Tree,
   dir: string
 ): string[] {
-  if (isPathIgnored(tree, dir)) {
-    return [];
-  }
+  const ignore = createGitIgnoreFromTree(tree);
 
-  const files: string[] = [];
-  const children = tree.children(dir);
-  children.forEach((child) => {
-    const childPath = joinPathFragments(dir, child);
-    if (tree.isFile(childPath)) {
-      files.push(childPath);
-    } else {
-      files.push(...getAllFilesRecursivelyFromDir(tree, childPath));
+  function inner(dir: string) {
+    if (ignore.ignores(dir)) {
+      return [];
     }
-  });
 
-  return files;
-}
+    const files: string[] = [];
+    const children = tree.children(dir);
+    children.forEach((child) => {
+      const childPath = joinPathFragments(dir, child);
+      if (tree.isFile(childPath)) {
+        files.push(childPath);
+      } else {
+        files.push(...inner(childPath));
+      }
+    });
 
-function isPathIgnored(tree: Tree, path: string): boolean {
-  let ig: Ignore;
-  if (tree.exists('.gitignore')) {
-    ig = ignore();
-    ig.add(tree.read('.gitignore', 'utf-8'));
+    return files;
   }
 
-  return path !== '' && ig?.ignores(path);
+  return inner(dir);
 }

packages/devkit/index.ts

@@ -319,3 +319,8 @@ export { Hash, Hasher } from 'nx/src/hasher/hasher';
  * @category Utils
  */
 export { cacheDir } from 'nx/src/utils/cache-directory';
+
+/**
+ * @category Utils
+ */
+export { createIgnore, createIgnoreFromTree } from 'nx/src/utils/ignore';

packages/devkit/src/generators/visit-not-ignored-files.ts

@@ -1,5 +1,5 @@
 import type { Tree } from 'nx/src/generators/tree';
-import ignore, { Ignore } from 'ignore';
+import { createGitIgnoreFromTree } from 'nx/src/utils/ignore';
 import { join, relative, sep } from 'path';
 
 /**
@@ -10,26 +10,27 @@ export function visitNotIgnoredFiles(
   dirPath: string = tree.root,
   visitor: (path: string) => void
 ): void {
-  let ig: Ignore;
-  if (tree.exists('.gitignore')) {
-    ig = ignore();
-    ig.add(tree.read('.gitignore', 'utf-8'));
-  }
-  dirPath = normalizePathRelativeToRoot(dirPath, tree.root);
-  if (dirPath !== '' && ig?.ignores(dirPath)) {
-    return;
-  }
-  for (const child of tree.children(dirPath)) {
-    const fullPath = join(dirPath, child);
-    if (ig?.ignores(fullPath)) {
-      continue;
+  const ignore = createGitIgnoreFromTree(tree);
+
+  function inner(dirPath: string) {
+    dirPath = normalizePathRelativeToRoot(dirPath, tree.root);
+    if (dirPath !== '' && ignore.ignores(dirPath)) {
+      return;
     }
-    if (tree.isFile(fullPath)) {
-      visitor(fullPath);
-    } else {
-      visitNotIgnoredFiles(tree, fullPath, visitor);
+    for (const child of tree.children(dirPath)) {
+      const fullPath = join(dirPath, child);
+      if (ignore.ignores(fullPath)) {
+        continue;
+      }
+      if (tree.isFile(fullPath)) {
+        visitor(fullPath);
+      } else {
+        inner(fullPath);
+      }
     }
   }
+
+  inner(dirPath);
 }
 
 function normalizePathRelativeToRoot(path: string, root: string): string {

packages/js/src/utils/copy-assets-handler.ts

@@ -1,11 +1,12 @@
+import { logger } from '@nrwl/devkit';
+import { AssetGlob } from '@nrwl/workspace/src/utilities/assets';
 import type { Event } from '@parcel/watcher';
+import * as fg from 'fast-glob';
+import * as fse from 'fs-extra';
+import { Ignore } from 'ignore';
 import * as minimatch from 'minimatch';
+import { createIgnore } from 'nx/src/utils/ignore';
 import * as path from 'path';
-import * as fse from 'fs-extra';
-import ignore, { Ignore } from 'ignore';
-import * as fg from 'fast-glob';
-import { AssetGlob } from '@nrwl/workspace/src/utilities/assets';
-import { logger } from '@nrwl/devkit';
 
 export type FileEventType = 'create' | 'update' | 'delete';
 
@@ -58,15 +59,7 @@ export class CopyAssetsHandler {
     this.projectDir = opts.projectDir;
     this.outputDir = opts.outputDir;
     this.callback = opts.callback ?? defaultFileEventHandler;
-
-    // TODO(jack): Should handle nested .gitignore files
-    this.ignore = ignore();
-    const gitignore = path.join(opts.rootDir, '.gitignore');
-    const nxignore = path.join(opts.rootDir, '.nxignore');
-    if (fse.existsSync(gitignore))
-      this.ignore.add(fse.readFileSync(gitignore).toString());
-    if (fse.existsSync(nxignore))
-      this.ignore.add(fse.readFileSync(nxignore).toString());
+    this.ignore = createIgnore(opts.rootDir);
 
     this.assetGlobs = opts.assets.map((f) => {
       let isGlob = false;

packages/nx/src/command-line/dep-graph.ts

@@ -1,26 +1,26 @@
-import { workspaceRoot } from 'nx/src/utils/workspace-root';
 import { watch } from 'chokidar';
 import { createHash } from 'crypto';
 import { existsSync, readFileSync, statSync, writeFileSync } from 'fs';
 import { copySync, ensureDirSync } from 'fs-extra';
 import * as http from 'http';
-import ignore from 'ignore';
 import * as open from 'open';
 import { basename, dirname, extname, isAbsolute, join, parse } from 'path';
 import { performance } from 'perf_hooks';
 import { URL, URLSearchParams } from 'url';
 import { workspaceLayout } from '../config/configuration';
-import { defaultFileHasher } from '../hasher/file-hasher';
-import { output } from '../utils/output';
-import { writeJsonFile } from '../utils/fileutils';
-import { joinPathFragments } from '../utils/path';
 import {
   ProjectGraph,
   ProjectGraphDependency,
   ProjectGraphProjectNode,
 } from '../config/project-graph';
+import { defaultFileHasher } from '../hasher/file-hasher';
 import { pruneExternalNodes } from '../project-graph/operators';
 import { createProjectGraphAsync } from '../project-graph/project-graph';
+import { writeJsonFile } from '../utils/fileutils';
+import { createIgnore } from '../utils/ignore';
+import { output } from '../utils/output';
+import { joinPathFragments } from '../utils/path';
+import { workspaceRoot } from '../utils/workspace-root';
 
 export interface DepGraphClientResponse {
   hash: string;
@@ -408,17 +408,6 @@ let currentDepGraphClientResponse: DepGraphClientResponse = {
   exclude: [],
 };
 
-function getIgnoredGlobs(root: string) {
-  const ig = ignore();
-  try {
-    ig.add(readFileSync(`${root}/.gitignore`, 'utf-8'));
-  } catch {}
-  try {
-    ig.add(readFileSync(`${root}/.nxignore`, 'utf-8'));
-  } catch {}
-  return ig;
-}
-
 function startWatcher() {
   createFileWatcher(workspaceRoot, async () => {
     output.note({ title: 'Recalculating project graph...' });
@@ -448,7 +437,7 @@ function debounce(fn: (...args) => void, time: number) {
 }
 
 function createFileWatcher(root: string, changeHandler: () => Promise<void>) {
-  const ignoredGlobs = getIgnoredGlobs(root);
+  const ignore = createIgnore(root);
   const layout = workspaceLayout();
 
   const watcher = watch(
@@ -464,7 +453,7 @@ function createFileWatcher(root: string, changeHandler: () => Promise<void>) {
   watcher.on(
     'all',
     debounce(async (event: string, path: string) => {
-      if (ignoredGlobs.ignores(path)) return;
+      if (ignore.ignores(path)) return;
       await changeHandler();
     }, 500)
   );