From cdc9349b74c8cff45d8f63b43c2b9072f5803e7c Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Wed, 19 Oct 2022 16:00:33 -0400 Subject: [PATCH 01/38] fs: add recursive watch to linux --- doc/api/fs.md | 4 - lib/fs.js | 27 ++- lib/internal/fs/linux_watcher.js | 207 ++++++++++++++++++ .../parallel/test-fs-watch-recursive-linux.js | 138 ++++++++++++ test/parallel/test-fs-watch-recursive.js | 8 +- 5 files changed, 364 insertions(+), 20 deletions(-) create mode 100644 lib/internal/fs/linux_watcher.js create mode 100644 test/parallel/test-fs-watch-recursive-linux.js diff --git a/doc/api/fs.md b/doc/api/fs.md index fa3368da281da4..c7f7b9b16a5eae 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -4377,10 +4377,6 @@ the returned {fs.FSWatcher}. The `fs.watch` API is not 100% consistent across platforms, and is unavailable in some situations. -The recursive option is only supported on macOS and Windows. -An `ERR_FEATURE_UNAVAILABLE_ON_PLATFORM` exception will be thrown -when the option is used on a platform that does not support it. - On Windows, no events will be emitted if the watched directory is moved or renamed. An `EPERM` error is reported when the watched directory is deleted. diff --git a/lib/fs.js b/lib/fs.js index 5e110eef17dd8b..5bd709874c2eaf 100644 --- a/lib/fs.js +++ b/lib/fs.js @@ -57,6 +57,7 @@ const { const pathModule = require('path'); const { isArrayBufferView } = require('internal/util/types'); +const linuxWatcher = require('internal/fs/linux_watcher'); // We need to get the statValues from the binding at the callsite since // it's re-initialized after deserialization. @@ -68,7 +69,6 @@ const { codes: { ERR_FS_FILE_TOO_LARGE, ERR_INVALID_ARG_VALUE, - ERR_FEATURE_UNAVAILABLE_ON_PLATFORM, }, AbortError, uvErrmapGet, @@ -161,7 +161,7 @@ let FileReadStream; let FileWriteStream; const isWindows = process.platform === 'win32'; -const isOSX = process.platform === 'darwin'; +const isLinux = process.platform === 'linux'; function showTruncateDeprecation() { @@ -2297,13 +2297,22 @@ function watch(filename, options, listener) { if (options.persistent === undefined) options.persistent = true; if (options.recursive === undefined) options.recursive = false; - if (options.recursive && !(isOSX || isWindows)) - throw new ERR_FEATURE_UNAVAILABLE_ON_PLATFORM('watch recursively'); - const watcher = new watchers.FSWatcher(); - watcher[watchers.kFSWatchStart](filename, - options.persistent, - options.recursive, - options.encoding); + + let watcher; + + // TODO(anonrig): Remove this when/if libuv supports it. + // libuv does not support recursive file watch on Linux due to + // the limitations of inotify. + if (options.recursive && isLinux) { + watcher = new linuxWatcher.FSWatcher(options); + watcher[linuxWatcher.kFSWatchStart](filename); + } else { + watcher = new watchers.FSWatcher(); + watcher[watchers.kFSWatchStart](filename, + options.persistent, + options.recursive, + options.encoding); + } if (listener) { watcher.addListener('change', listener); diff --git a/lib/internal/fs/linux_watcher.js b/lib/internal/fs/linux_watcher.js new file mode 100644 index 00000000000000..2ee732b5f9d563 --- /dev/null +++ b/lib/internal/fs/linux_watcher.js @@ -0,0 +1,207 @@ +'use strict'; + +const { EventEmitter } = require('events'); +const path = require('path'); +const { SafeMap, Symbol, StringPrototypeStartsWith } = primordials; +const { validateObject } = require('internal/validators'); +const { kEmptyObject } = require('internal/util'); +const { ERR_FEATURE_UNAVAILABLE_ON_PLATFORM } = require('internal/errors'); + +const kFSWatchStart = Symbol('kFSWatchStart'); + +let internalSync; +let internalPromises; + +function lazyLoadFsPromises() { + internalPromises ??= require('fs/promises'); + return internalPromises; +} + +function lazyLoadFsSync() { + internalSync ??= require('fs'); + return internalSync; +} + +async function traverse(dir, files = new SafeMap()) { + const { stat, opendir } = lazyLoadFsPromises(); + + files.set(dir, await stat(dir)); + + try { + const directory = await opendir(dir); + + for await (const file of directory) { + const f = path.join(dir, file.name); + + try { + const stats = await stat(f); + + files.set(f, stats); + + if (stats.isDirectory()) { + await traverse(f, files); + } + } catch (error) { + if (error.code !== 'ENOENT' || error.code !== 'EPERM') { + this.emit('error', error); + } + } + + } + } catch (error) { + if (error.code !== 'EACCES') { + this.emit('error', error); + } + } + + return files; +} + +class FSWatcher extends EventEmitter { + #options = null; + #closed = false; + #files = new SafeMap(); + #rootPath = path.resolve(); + + /** + * @param {{ + * persistent?: boolean; + * recursive?: boolean; + * encoding?: string; + * signal?: AbortSignal; + * }} [options] + */ + constructor(options = kEmptyObject) { + super(); + + validateObject(options, 'options'); + this.#options = options; + } + + close() { + const { unwatchFile } = lazyLoadFsSync(); + this.#closed = true; + + for (const file of this.#files.keys()) { + unwatchFile(file); + } + + this.emit('close'); + } + + #getPath(file) { + if (file === this.#rootPath) { + return this.#rootPath; + } + + return path.relative(this.#rootPath, file); + } + + #unwatchFolder(file) { + const { unwatchFile } = lazyLoadFsSync(); + + for (const filename of this.#files.keys()) { + if (StringPrototypeStartsWith(filename, file)) { + unwatchFile(filename); + } + } + } + + async #watchFolder(folder) { + const { opendir, stat } = lazyLoadFsPromises(); + + try { + const files = await opendir(folder); + + for await (const file of files) { + const f = path.join(folder, file.name); + + if (this.#closed) { + break; + } + + if (!this.#files.has(f)) { + const fileStats = await stat(f); + + this.#files.set(f, fileStats); + this.emit('change', 'rename', this.#getPath(f)); + + if (fileStats.isDirectory()) { + await this.#watchFolder(f); + } else { + this.#watchFile(f); + } + } + } + } catch (error) { + this.emit('error', error); + } + } + + /** + * @param {string} file + */ + #watchFile(file) { + const { watchFile } = lazyLoadFsSync(); + + if (this.#closed) { + return; + } + + const existingStat = this.#files.get(file); + + watchFile(file, { + persistent: this.#options.persistent, + }, (statWatcher, previousStatWatcher) => { + if (existingStat && !existingStat.isDirectory() && + statWatcher.nlink !== 0 && existingStat.mtime.getTime() === statWatcher.mtime.getTime()) { + return; + } + + this.#files.set(file, statWatcher); + + if (statWatcher.birthtimeMs === 0 && previousStatWatcher.birthtimeMs !== 0) { + // The file is now deleted + this.#files.delete(file); + this.emit('change', 'rename', this.#getPath(file)); + + if (statWatcher.isDirectory()) { + this.#unwatchFolder(file); + } + } else if (statWatcher.isDirectory()) { + this.#watchFolder(file); + this.emit('change', 'change', this.#getPath(file)); + } else { + this.emit('change', 'change', this.#getPath(file)); + } + }); + } + + /** + * @param {string | Buffer | URL} filename + */ + async [kFSWatchStart](filename) { + this.#rootPath = filename; + this.#closed = false; + this.#files = await traverse(filename); + + for (const f of this.#files.keys()) { + this.#watchFile(f); + } + } + + ref() { + // This is kept to have the same API with FSWatcher + throw new ERR_FEATURE_UNAVAILABLE_ON_PLATFORM('ref'); + } + + unref() { + // This is kept to have the same API with FSWatcher + throw new ERR_FEATURE_UNAVAILABLE_ON_PLATFORM('unref'); + } +} + +module.exports = { + FSWatcher, + kFSWatchStart, +}; diff --git a/test/parallel/test-fs-watch-recursive-linux.js b/test/parallel/test-fs-watch-recursive-linux.js new file mode 100644 index 00000000000000..e1b019b04c47a1 --- /dev/null +++ b/test/parallel/test-fs-watch-recursive-linux.js @@ -0,0 +1,138 @@ +'use strict'; + +const common = require('../common'); +const { setTimeout } = require('timers/promises'); + +if (!common.hasCrypto) + common.skip('missing crypto'); + +const { randomUUID } = require('crypto'); +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); + +const tmpdir = require('../common/tmpdir'); +const testDir = tmpdir.path; +tmpdir.refresh(); + +(async () => { + { + // Add a file to already watching folder + + const testsubdir = fs.mkdtempSync(testDir + path.sep); + const file = `${randomUUID()}.txt`; + const filePath = path.join(testsubdir, file); + const watcher = fs.watch(testsubdir, { recursive: true }); + + let watcherClosed = false; + watcher.on('change', function(event, filename) { + assert.ok(event === 'change' || event === 'rename'); + + if (filename === file) { + watcher.close(); + watcherClosed = true; + } + }); + + await setTimeout(100); + fs.writeFileSync(filePath, 'world'); + + process.on('exit', function() { + assert(watcherClosed, 'watcher Object was not closed'); + }); + } + + { + // Add a folder to already watching folder + + const testsubdir = fs.mkdtempSync(testDir + path.sep); + const file = `folder-${randomUUID()}`; + const filePath = path.join(testsubdir, file); + const watcher = fs.watch(testsubdir, { recursive: true }); + + let watcherClosed = false; + watcher.on('change', function(event, filename) { + assert.ok(event === 'change' || event === 'rename'); + + if (filename === file) { + watcher.close(); + watcherClosed = true; + } + }); + + await setTimeout(100); + fs.mkdirSync(filePath); + + process.on('exit', function() { + assert(watcherClosed, 'watcher Object was not closed'); + }); + } + + { + // Add a file to newly created folder to already watching folder + + const testsubdir = fs.mkdtempSync(testDir + path.sep); + const file = `folder-${randomUUID()}`; + const filePath = path.join(testsubdir, file); + const watcher = fs.watch(testsubdir, { recursive: true }); + const childrenFile = `file-${randomUUID()}.txt`; + const childrenAbsolutePath = path.join(filePath, childrenFile); + + let watcherClosed = false; + + watcher.on('change', function(event, filename) { + assert.ok(event === 'change' || event === 'rename'); + + if (filename === path.join(file, childrenFile)) { + watcher.close(); + watcherClosed = true; + } + }); + + await setTimeout(100); + fs.mkdirSync(filePath); + await setTimeout(100); + fs.writeFileSync(childrenAbsolutePath, 'world'); + + process.on('exit', function() { + assert(watcherClosed, 'watcher Object was not closed'); + }); + } + + { + // Add a file to subfolder of a watching folder + + const testsubdir = fs.mkdtempSync(testDir + path.sep); + const file = `folder-${randomUUID()}`; + const filePath = path.join(testsubdir, file); + fs.mkdirSync(filePath); + + const subFolder = `subfolder-${randomUUID()}`; + const subfolderPath = path.join(filePath, subFolder); + + fs.mkdirSync(subfolderPath); + + const watcher = fs.watch(testsubdir, { recursive: true }); + const childrenFile = `file-${randomUUID()}.txt`; + const childrenAbsolutePath = path.join(subfolderPath, childrenFile); + const relativePath = path.join(file, subFolder, childrenFile); + + let watcherClosed = false; + + watcher.on('change', function(event, filename) { + assert.ok(event === 'change' || event === 'rename'); + + if (filename === relativePath) { + watcher.close(); + watcherClosed = true; + } + }); + + await setTimeout(100); + fs.writeFileSync(childrenAbsolutePath, 'world'); + + process.on('exit', function() { + assert(watcherClosed, 'watcher Object was not closed'); + }); + } +})().then(common.mustCall()); diff --git a/test/parallel/test-fs-watch-recursive.js b/test/parallel/test-fs-watch-recursive.js index 70b413814e78f4..a8fc6e59cb836f 100644 --- a/test/parallel/test-fs-watch-recursive.js +++ b/test/parallel/test-fs-watch-recursive.js @@ -17,12 +17,6 @@ tmpdir.refresh(); const testsubdir = fs.mkdtempSync(testDir + path.sep); const relativePathOne = path.join(path.basename(testsubdir), filenameOne); const filepathOne = path.join(testsubdir, filenameOne); - -if (!common.isOSX && !common.isWindows) { - assert.throws(() => { fs.watch(testDir, { recursive: true }); }, - { code: 'ERR_FEATURE_UNAVAILABLE_ON_PLATFORM' }); - return; -} const watcher = fs.watch(testDir, { recursive: true }); let watcherClosed = false; @@ -41,7 +35,7 @@ watcher.on('change', function(event, filename) { }); let interval; -if (common.isOSX) { +if (common.isOSX || common.isLinux) { interval = setInterval(function() { fs.writeFileSync(filepathOne, 'world'); }, 10); From 296646b0d7e1bc44013164a00176a132c3062b29 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Fri, 21 Oct 2022 19:46:13 -0400 Subject: [PATCH 02/38] fs: replace traverse with readdir for performance --- lib/internal/fs/linux_watcher.js | 40 +++++++++++--------------------- 1 file changed, 13 insertions(+), 27 deletions(-) diff --git a/lib/internal/fs/linux_watcher.js b/lib/internal/fs/linux_watcher.js index 2ee732b5f9d563..1e08dd859135b3 100644 --- a/lib/internal/fs/linux_watcher.js +++ b/lib/internal/fs/linux_watcher.js @@ -6,6 +6,7 @@ const { SafeMap, Symbol, StringPrototypeStartsWith } = primordials; const { validateObject } = require('internal/validators'); const { kEmptyObject } = require('internal/util'); const { ERR_FEATURE_UNAVAILABLE_ON_PLATFORM } = require('internal/errors'); +const { Dirent, UV_DIRENT_DIR } = require('internal/fs/utils'); const kFSWatchStart = Symbol('kFSWatchStart'); @@ -22,35 +23,20 @@ function lazyLoadFsSync() { return internalSync; } -async function traverse(dir, files = new SafeMap()) { - const { stat, opendir } = lazyLoadFsPromises(); +function traverse(dir, files = new SafeMap()) { + const { readdirSync } = lazyLoadFsSync() - files.set(dir, await stat(dir)); + const filenames = readdirSync(dir, { withFileTypes: true }); - try { - const directory = await opendir(dir); + files.set(dir, new Dirent(dir, UV_DIRENT_DIR)); - for await (const file of directory) { - const f = path.join(dir, file.name); + for (let file of filenames) { + const f = path.join(dir, file.name); - try { - const stats = await stat(f); + files.set(f, file); - files.set(f, stats); - - if (stats.isDirectory()) { - await traverse(f, files); - } - } catch (error) { - if (error.code !== 'ENOENT' || error.code !== 'EPERM') { - this.emit('error', error); - } - } - - } - } catch (error) { - if (error.code !== 'EACCES') { - this.emit('error', error); + if (file.isDirectory()) { + traverse(f, files); } } @@ -154,7 +140,7 @@ class FSWatcher extends EventEmitter { persistent: this.#options.persistent, }, (statWatcher, previousStatWatcher) => { if (existingStat && !existingStat.isDirectory() && - statWatcher.nlink !== 0 && existingStat.mtime.getTime() === statWatcher.mtime.getTime()) { + statWatcher.nlink !== 0 && existingStat.mtime?.getTime() === statWatcher.mtime?.getTime()) { return; } @@ -180,10 +166,10 @@ class FSWatcher extends EventEmitter { /** * @param {string | Buffer | URL} filename */ - async [kFSWatchStart](filename) { + [kFSWatchStart](filename) { this.#rootPath = filename; this.#closed = false; - this.#files = await traverse(filename); + this.#files = traverse(filename); for (const f of this.#files.keys()) { this.#watchFile(f); From 77cb9a0cb639702862f8dbbed620f2f760af61a9 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Sat, 22 Oct 2022 13:02:47 -0400 Subject: [PATCH 03/38] fs: fix linter issues --- lib/internal/fs/linux_watcher.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/internal/fs/linux_watcher.js b/lib/internal/fs/linux_watcher.js index 1e08dd859135b3..9a2b0faa323a0d 100644 --- a/lib/internal/fs/linux_watcher.js +++ b/lib/internal/fs/linux_watcher.js @@ -24,13 +24,13 @@ function lazyLoadFsSync() { } function traverse(dir, files = new SafeMap()) { - const { readdirSync } = lazyLoadFsSync() + const { readdirSync } = lazyLoadFsSync(); const filenames = readdirSync(dir, { withFileTypes: true }); files.set(dir, new Dirent(dir, UV_DIRENT_DIR)); - for (let file of filenames) { + for (const file of filenames) { const f = path.join(dir, file.name); files.set(f, file); From 5b72449fe22075c5c92286cf1733446b5c50a463 Mon Sep 17 00:00:00 2001 From: Yagiz Nizipli Date: Sat, 22 Oct 2022 13:30:23 -0400 Subject: [PATCH 04/38] fs: move linux watcher to internal/fs/watch --- doc/api/fs.md | 8 +++ lib/fs.js | 2 +- .../fs/{linux_watcher.js => watch/linux.js} | 60 +++++++++---------- 3 files changed, 39 insertions(+), 31 deletions(-) rename lib/internal/fs/{linux_watcher.js => watch/linux.js} (80%) diff --git a/doc/api/fs.md b/doc/api/fs.md index c7f7b9b16a5eae..9c0204cd284e1d 100644 --- a/doc/api/fs.md +++ b/doc/api/fs.md @@ -6403,6 +6403,10 @@ By default, all {fs.FSWatcher} objects are "ref'ed", making it normally unnecessary to call `watcher.ref()` unless `watcher.unref()` had been called previously. +`watcher.ref()` is not available on Linux. An `ERR_FEATURE_UNAVAILABLE_ON_PLATFORM` +exception will be thrown when the function is used on a platform that +does not support it. + #### `watcher.unref()`