diff --git a/.github/funding.yml b/.github/funding.yml deleted file mode 100644 index c1b652f..0000000 --- a/.github/funding.yml +++ /dev/null @@ -1,4 +0,0 @@ -github: sindresorhus -open_collective: sindresorhus -tidelift: npm/globby -custom: https://sindresorhus.com/donate diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5609829..087eb91 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -10,16 +10,15 @@ jobs: fail-fast: false matrix: node-version: + - 20 - 18 - - 16 - - 14 os: - ubuntu-latest - macos-latest - windows-latest steps: - - uses: actions/checkout@v3 - - uses: actions/setup-node@v3 + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 with: node-version: ${{ matrix.node-version }} - run: npm install diff --git a/bench.js b/bench.js index a07a1cd..c9204c0 100644 --- a/bench.js +++ b/bench.js @@ -3,7 +3,6 @@ import fs from 'node:fs'; import path from 'node:path'; import {fileURLToPath} from 'node:url'; import Benchmark from 'benchmark'; -import rimraf from 'rimraf'; import * as globbyMainBranch from '@globby/main-branch'; import gs from 'glob-stream'; import fastGlob from 'fast-glob'; @@ -83,7 +82,7 @@ const benchs = [ const before = () => { process.chdir(__dirname); - rimraf.sync(BENCH_DIR); + fs.rmdirSync(BENCH_DIR, {recursive: true}); fs.mkdirSync(BENCH_DIR); process.chdir(BENCH_DIR); @@ -100,7 +99,7 @@ const before = () => { const after = () => { process.chdir(__dirname); - rimraf.sync(BENCH_DIR); + fs.rmdirSync(BENCH_DIR, {recursive: true}); }; const suites = []; diff --git a/ignore.js b/ignore.js index 3d8e1a8..5d9c5e2 100644 --- a/ignore.js +++ b/ignore.js @@ -1,10 +1,12 @@ import process from 'node:process'; import fs from 'node:fs'; +import fsPromises from 'node:fs/promises'; import path from 'node:path'; import fastGlob from 'fast-glob'; import gitIgnore from 'ignore'; import slash from 'slash'; -import {toPath, isNegativePattern} from './utilities.js'; +import {toPath} from 'unicorn-magic'; +import {isNegativePattern} from './utilities.js'; const ignoreFilesGlobOptions = { ignore: [ @@ -57,7 +59,7 @@ const getIsIgnoredPredicate = (files, cwd) => { }; const normalizeOptions = (options = {}) => ({ - cwd: toPath(options.cwd) || process.cwd(), + cwd: toPath(options.cwd) ?? process.cwd(), suppressErrors: Boolean(options.suppressErrors), deep: typeof options.deep === 'number' ? options.deep : Number.POSITIVE_INFINITY, }); @@ -70,7 +72,7 @@ export const isIgnoredByIgnoreFiles = async (patterns, options) => { const files = await Promise.all( paths.map(async filePath => ({ filePath, - content: await fs.promises.readFile(filePath, 'utf8'), + content: await fsPromises.readFile(filePath, 'utf8'), })), ); diff --git a/index.d.ts b/index.d.ts index 7e3c981..e88d6d4 100644 --- a/index.d.ts +++ b/index.d.ts @@ -1,11 +1,11 @@ -import {Options as FastGlobOptions, Entry} from 'fast-glob'; +import {type Options as FastGlobOptions, type Entry} from 'fast-glob'; export type GlobEntry = Entry; -export interface GlobTask { +export type GlobTask = { readonly patterns: string[]; readonly options: Options; -} +}; export type ExpandDirectoriesOption = | boolean @@ -14,7 +14,7 @@ export type ExpandDirectoriesOption = type FastGlobOptionsWithoutCwd = Omit; -export interface Options extends FastGlobOptionsWithoutCwd { +export type Options = { /** If set to `true`, `globby` will automatically glob directories for you. If you define an `Array` it will only glob files that matches the patterns inside the `Array`. You can also define an `Object` with `files` and `extensions` like in the example below. @@ -61,11 +61,11 @@ export interface Options extends FastGlobOptionsWithoutCwd { @default process.cwd() */ readonly cwd?: URL | string; -} +} & FastGlobOptionsWithoutCwd; -export interface GitignoreOptions { +export type GitignoreOptions = { readonly cwd?: URL | string; -} +}; export type GlobbyFilterFunction = (path: URL | string) => boolean; diff --git a/index.js b/index.js index ee570f0..d0837cf 100644 --- a/index.js +++ b/index.js @@ -1,14 +1,16 @@ +import process from 'node:process'; import fs from 'node:fs'; import nodePath from 'node:path'; -import merge2 from 'merge2'; +import mergeStreams from '@sindresorhus/merge-streams'; import fastGlob from 'fast-glob'; -import dirGlob from 'dir-glob'; +import {isDirectory, isDirectorySync} from 'path-type'; +import {toPath} from 'unicorn-magic'; import { GITIGNORE_FILES_PATTERN, isIgnoredByIgnoreFiles, isIgnoredByIgnoreFilesSync, } from './ignore.js'; -import {FilterStream, toPath, isNegativePattern} from './utilities.js'; +import {isNegativePattern} from './utilities.js'; const assertPatternsInput = patterns => { if (patterns.some(pattern => typeof pattern !== 'string')) { @@ -16,20 +18,50 @@ const assertPatternsInput = patterns => { } }; +const normalizePathForDirectoryGlob = (filePath, cwd) => { + const path = isNegativePattern(filePath) ? filePath.slice(1) : filePath; + return nodePath.isAbsolute(path) ? path : nodePath.join(cwd, path); +}; + +const getDirectoryGlob = ({directoryPath, files, extensions}) => { + const extensionGlob = extensions?.length > 0 ? `.${extensions.length > 1 ? `{${extensions.join(',')}}` : extensions[0]}` : ''; + return files + ? files.map(file => nodePath.posix.join(directoryPath, `**/${nodePath.extname(file) ? file : `${file}${extensionGlob}`}`)) + : [nodePath.posix.join(directoryPath, `**${extensionGlob ? `/${extensionGlob}` : ''}`)]; +}; + +const directoryToGlob = async (directoryPaths, { + cwd = process.cwd(), + files, + extensions, +} = {}) => { + const globs = await Promise.all(directoryPaths.map(async directoryPath => + (await isDirectory(normalizePathForDirectoryGlob(directoryPath, cwd))) ? getDirectoryGlob({directoryPath, files, extensions}) : directoryPath), + ); + + return globs.flat(); +}; + +const directoryToGlobSync = (directoryPaths, { + cwd = process.cwd(), + files, + extensions, +} = {}) => directoryPaths.flatMap(directoryPath => isDirectorySync(normalizePathForDirectoryGlob(directoryPath, cwd)) ? getDirectoryGlob({directoryPath, files, extensions}) : directoryPath); + const toPatternsArray = patterns => { patterns = [...new Set([patterns].flat())]; assertPatternsInput(patterns); return patterns; }; -const checkCwdOption = options => { - if (!options.cwd) { +const checkCwdOption = cwd => { + if (!cwd) { return; } let stat; try { - stat = fs.statSync(options.cwd); + stat = fs.statSync(cwd); } catch { return; } @@ -42,20 +74,18 @@ const checkCwdOption = options => { const normalizeOptions = (options = {}) => { options = { ...options, - ignore: options.ignore || [], - expandDirectories: options.expandDirectories === undefined - ? true - : options.expandDirectories, + ignore: options.ignore ?? [], + expandDirectories: options.expandDirectories ?? true, cwd: toPath(options.cwd), }; - checkCwdOption(options); + checkCwdOption(options.cwd); return options; }; -const normalizeArguments = fn => async (patterns, options) => fn(toPatternsArray(patterns), normalizeOptions(options)); -const normalizeArgumentsSync = fn => (patterns, options) => fn(toPatternsArray(patterns), normalizeOptions(options)); +const normalizeArguments = function_ => async (patterns, options) => function_(toPatternsArray(patterns), normalizeOptions(options)); +const normalizeArgumentsSync = function_ => (patterns, options) => function_(toPatternsArray(patterns), normalizeOptions(options)); const getIgnoreFilesPatterns = options => { const {ignoreFiles, gitignore} = options; @@ -86,16 +116,19 @@ const createFilterFunction = isIgnored => { const seen = new Set(); return fastGlobResult => { - const path = fastGlobResult.path || fastGlobResult; - const pathKey = nodePath.normalize(path); - const seenOrIgnored = seen.has(pathKey) || (isIgnored && isIgnored(path)); + const pathKey = nodePath.normalize(fastGlobResult.path ?? fastGlobResult); + + if (seen.has(pathKey) || (isIgnored && isIgnored(pathKey))) { + return false; + } + seen.add(pathKey); - return !seenOrIgnored; + + return true; }; }; const unionFastGlobResults = (results, filter) => results.flat().filter(fastGlobResult => filter(fastGlobResult)); -const unionFastGlobStreams = (streams, filter) => merge2(streams).pipe(new FilterStream(fastGlobResult => filter(fastGlobResult))); const convertNegativePatterns = (patterns, options) => { const tasks = []; @@ -133,7 +166,7 @@ const convertNegativePatterns = (patterns, options) => { return tasks; }; -const getDirGlobOptions = (options, cwd) => ({ +const normalizeExpandDirectoriesOption = (options, cwd) => ({ ...(cwd ? {cwd} : {}), ...(Array.isArray(options) ? {files: options} : options), }); @@ -147,8 +180,7 @@ const generateTasks = async (patterns, options) => { return globTasks; } - const patternExpandOptions = getDirGlobOptions(expandDirectories, cwd); - const ignoreExpandOptions = cwd ? {cwd} : undefined; + const directoryToGlobOptions = normalizeExpandDirectoriesOption(expandDirectories, cwd); return Promise.all( globTasks.map(async task => { @@ -158,8 +190,8 @@ const generateTasks = async (patterns, options) => { patterns, options.ignore, ] = await Promise.all([ - dirGlob(patterns, patternExpandOptions), - dirGlob(options.ignore, ignoreExpandOptions), + directoryToGlob(patterns, directoryToGlobOptions), + directoryToGlob(options.ignore, {cwd}), ]); return {patterns, options}; @@ -169,20 +201,18 @@ const generateTasks = async (patterns, options) => { const generateTasksSync = (patterns, options) => { const globTasks = convertNegativePatterns(patterns, options); - const {cwd, expandDirectories} = options; if (!expandDirectories) { return globTasks; } - const patternExpandOptions = getDirGlobOptions(expandDirectories, cwd); - const ignoreExpandOptions = cwd ? {cwd} : undefined; + const directoryToGlobSyncOptions = normalizeExpandDirectoriesOption(expandDirectories, cwd); return globTasks.map(task => { let {patterns, options} = task; - patterns = dirGlob.sync(patterns, patternExpandOptions); - options.ignore = dirGlob.sync(options.ignore, ignoreExpandOptions); + patterns = directoryToGlobSync(patterns, directoryToGlobSyncOptions); + options.ignore = directoryToGlobSync(options.ignore, {cwd}); return {patterns, options}; }); }; @@ -195,8 +225,8 @@ export const globby = normalizeArguments(async (patterns, options) => { generateTasks(patterns, options), getFilter(options), ]); - const results = await Promise.all(tasks.map(task => fastGlob(task.patterns, task.options))); + const results = await Promise.all(tasks.map(task => fastGlob(task.patterns, task.options))); return unionFastGlobResults(results, filter); }); @@ -204,7 +234,6 @@ export const globbySync = normalizeArgumentsSync((patterns, options) => { const tasks = generateTasksSync(patterns, options); const filter = getFilterSync(options); const results = tasks.map(task => fastGlob.sync(task.patterns, task.options)); - return unionFastGlobResults(results, filter); }); @@ -212,8 +241,12 @@ export const globbyStream = normalizeArgumentsSync((patterns, options) => { const tasks = generateTasksSync(patterns, options); const filter = getFilterSync(options); const streams = tasks.map(task => fastGlob.stream(task.patterns, task.options)); + const stream = mergeStreams(streams).filter(fastGlobResult => filter(fastGlobResult)); + + // TODO: Make it return a web stream at some point. + // return Readable.toWeb(stream); - return unionFastGlobStreams(streams, filter); + return stream; }); export const isDynamicPattern = normalizeArgumentsSync( diff --git a/index.test-d.ts b/index.test-d.ts index f09fcd9..7fe7839 100644 --- a/index.test-d.ts +++ b/index.test-d.ts @@ -1,10 +1,9 @@ -import {Buffer} from 'node:buffer'; -import {URL} from 'node:url'; +import {type Buffer} from 'node:buffer'; import {expectType} from 'tsd'; import { - GlobTask, - GlobEntry, - GlobbyFilterFunction, + type GlobTask, + type GlobEntry, + type GlobbyFilterFunction, globby, globbySync, globbyStream, diff --git a/package.json b/package.json index e2bdb72..4bcccd7 100644 --- a/package.json +++ b/package.json @@ -11,9 +11,13 @@ "url": "https://sindresorhus.com" }, "type": "module", - "exports": "./index.js", + "exports": { + "types": "./index.d.ts", + "default": "./index.js" + }, + "sideEffects": false, "engines": { - "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + "node": ">=18" }, "scripts": { "bench": "npm update @globby/main-branch glob-stream fast-glob && node bench.js", @@ -59,33 +63,27 @@ "git" ], "dependencies": { - "dir-glob": "^3.0.1", - "fast-glob": "^3.3.0", + "@sindresorhus/merge-streams": "^1.0.0", + "fast-glob": "^3.3.2", "ignore": "^5.2.4", - "merge2": "^1.4.1", - "slash": "^4.0.0" + "path-type": "^5.0.0", + "slash": "^5.1.0", + "unicorn-magic": "^0.1.0" }, "devDependencies": { "@globby/main-branch": "sindresorhus/globby#main", - "@types/node": "^20.3.3", + "@types/node": "^20.9.0", "ava": "^5.3.1", "benchmark": "2.1.4", "glob-stream": "^8.0.0", - "rimraf": "^5.0.1", - "tempy": "^3.0.0", - "tsd": "^0.28.1", - "typescript": "^5.1.6", - "xo": "^0.54.2" + "tempy": "^3.1.0", + "tsd": "^0.29.0", + "xo": "^0.56.0" }, "xo": { "ignores": [ "fixtures" - ], - "rules": { - "@typescript-eslint/consistent-type-definitions": "off", - "n/prefer-global/url": "off", - "@typescript-eslint/consistent-type-imports": "off" - } + ] }, "ava": { "files": [ diff --git a/readme.md b/readme.md index ff36479..72108ef 100644 --- a/readme.md +++ b/readme.md @@ -15,8 +15,8 @@ Based on [`fast-glob`](https://github.com/mrmlnc/fast-glob) but adds a bunch of ## Install -``` -$ npm install globby +```sh +npm install globby ``` ## Usage @@ -66,17 +66,15 @@ If set to `true`, `globby` will automatically glob directories for you. If you d ```js import {globby} from 'globby'; -(async () => { - const paths = await globby('images', { - expandDirectories: { - files: ['cat', 'unicorn', '*.jpg'], - extensions: ['png'] - } - }); - - console.log(paths); - //=> ['cat.png', 'unicorn.png', 'cow.jpg', 'rainbow.jpg'] -})(); +const paths = await globby('images', { + expandDirectories: { + files: ['cat', 'unicorn', '*.jpg'], + extensions: ['png'] + } +}); + +console.log(paths); +//=> ['cat.png', 'unicorn.png', 'cow.jpg', 'rainbow.jpg'] ``` Note that if you set this option to `false`, you won't get back matched directories unless you set `onlyFiles: false`. @@ -105,16 +103,14 @@ Returns `string[]` of matching paths. Returns a [`stream.Readable`](https://nodejs.org/api/stream.html#stream_readable_streams) of matching paths. -Since Node.js 10, [readable streams are iterable](https://nodejs.org/api/stream.html#stream_readable_symbol_asynciterator), so you can loop over glob matches in a [`for await...of` loop](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) like this: +For example, loop over glob matches in a [`for await...of` loop](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) like this: ```js import {globbyStream} from 'globby'; -(async () => { - for await (const path of globbyStream('*.tmp')) { - console.log(path); - } -})(); +for await (const path of globbyStream('*.tmp')) { + console.log(path); +} ``` ### generateGlobTasks(patterns, options?) @@ -169,12 +165,6 @@ Just a quick overview. [Various patterns and expected matches.](https://github.com/sindresorhus/multimatch/blob/main/test/test.js) -## globby for enterprise - -Available as part of the Tidelift Subscription. - -The maintainers of globby and thousands of other packages are working with Tidelift to deliver commercial support and maintenance for the open source dependencies you use to build your applications. Save time, reduce risk, and improve code health, while paying the maintainers of the exact dependencies you use. [Learn more.](https://tidelift.com/subscription/pkg/npm-globby?utm_source=npm-globby&utm_medium=referral&utm_campaign=enterprise&utm_term=repo) - ## Related - [multimatch](https://github.com/sindresorhus/multimatch) - Match against a list instead of the filesystem diff --git a/tests/generate-glob-tasks.js b/tests/generate-glob-tasks.js index d7aa020..b2c7c2a 100644 --- a/tests/generate-glob-tasks.js +++ b/tests/generate-glob-tasks.js @@ -1,7 +1,9 @@ import util from 'node:util'; import process from 'node:process'; import path from 'node:path'; +import fs from 'node:fs'; import test from 'ava'; +import {temporaryDirectory} from 'tempy'; import { generateGlobTasks, generateGlobTasksSync, @@ -220,3 +222,13 @@ test('random patterns', async t => { ); } }); + +// Test for https://github.com/sindresorhus/globby/issues/147 +test.failing('expandDirectories should work with globstar prefix', async t => { + const cwd = temporaryDirectory(); + const filePath = path.join(cwd, 'a', 'b'); + fs.mkdirSync(filePath, {recursive: true}); + const tasks = await runGenerateGlobTasks(t, ['**/b'], {cwd}); + t.is(tasks.length, 1); + t.deepEqual(tasks[0].patterns, ['**/b/**']); +}); diff --git a/tests/globby.js b/tests/globby.js index 18b296e..127b0f0 100644 --- a/tests/globby.js +++ b/tests/globby.js @@ -39,22 +39,12 @@ const stabilizeResult = result => result return fastGlobResult; }) - .sort((a, b) => (a.path || a).localeCompare(b.path || b)); - -const streamToArray = async stream => { - const result = []; - for await (const chunk of stream) { - result.push(chunk); - } - - return result; -}; + .sort((a, b) => (a.path ?? a).localeCompare(b.path ?? b)); const runGlobby = async (t, patterns, options) => { const syncResult = globbySync(patterns, options); const promiseResult = await globby(patterns, options); - // TODO: Use `stream.toArray()` when targeting Node.js 16. - const streamResult = await streamToArray(globbyStream(patterns, options)); + const streamResult = await globbyStream(patterns, options).toArray(); const result = stabilizeResult(promiseResult); t.deepEqual( diff --git a/utilities.js b/utilities.js index c09c4dc..1ed1f27 100644 --- a/utilities.js +++ b/utilities.js @@ -1,17 +1 @@ -import {fileURLToPath} from 'node:url'; -import {Transform} from 'node:stream'; - -export const toPath = urlOrPath => urlOrPath instanceof URL ? fileURLToPath(urlOrPath) : urlOrPath; - -export class FilterStream extends Transform { - constructor(filter) { - super({ - objectMode: true, - transform(data, encoding, callback) { - callback(undefined, filter(data) ? data : undefined); - }, - }); - } -} - export const isNegativePattern = pattern => pattern[0] === '!';