/
index.js
452 lines (396 loc) · 15.4 KB
/
index.js
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
'use strict'
const { Walker: IgnoreWalker } = require('ignore-walk')
const { lstatSync: lstat, readFileSync: readFile } = require('fs')
const { basename, dirname, extname, join, relative, resolve, sep } = require('path')
// symbols used to represent synthetic rule sets
const defaultRules = Symbol('npm-packlist.rules.default')
const strictRules = Symbol('npm-packlist.rules.strict')
// There may be others, but :?|<> are handled by node-tar
const nameIsBadForWindows = file => /\*/.test(file)
// these are the default rules that are applied to everything except for non-link bundled deps
const defaults = [
'.npmignore',
'.gitignore',
'**/.git',
'**/.svn',
'**/.hg',
'**/CVS',
'**/.git/**',
'**/.svn/**',
'**/.hg/**',
'**/CVS/**',
'/.lock-wscript',
'/.wafpickle-*',
'/build/config.gypi',
'npm-debug.log',
'**/.npmrc',
'.*.swp',
'.DS_Store',
'**/.DS_Store/**',
'._*',
'**/._*/**',
'*.orig',
'/archived-packages/**',
]
const strictDefaults = [
// these are forcibly included at all levels
'!/readme{,.*[^~$]}',
'!/copying{,.*[^~$]}',
'!/license{,.*[^~$]}',
'!/licence{,.*[^~$]}',
// these are forcibly excluded
'/.git',
]
const normalizePath = (path) => path.split('\\').join('/')
const readOutOfTreeIgnoreFiles = (root, rel, result = []) => {
for (const file of ['.npmignore', '.gitignore']) {
try {
const ignoreContent = readFile(join(root, file), { encoding: 'utf8' })
result.push(ignoreContent)
// break the loop immediately after reading, this allows us to prioritize
// the .npmignore and discard the .gitignore if one is present
break
} catch (err) {
// we ignore ENOENT errors completely because we don't care if the file doesn't exist
// but we throw everything else because failing to read a file that does exist is
// something that the user likely wants to know about
// istanbul ignore next -- we do not need to test a thrown error
if (err.code !== 'ENOENT') {
throw err
}
}
}
if (!rel) {
return result
}
const firstRel = rel.split(sep, 1)[0]
const newRoot = join(root, firstRel)
const newRel = relative(newRoot, join(root, rel))
return readOutOfTreeIgnoreFiles(newRoot, newRel, result)
}
class PackWalker extends IgnoreWalker {
constructor (tree, opts) {
const options = {
...opts,
includeEmpty: false,
follow: false,
// we path.resolve() here because ignore-walk doesn't do it and we want full paths
path: resolve(opts?.path || tree.path).replace(/\\/g, '/'),
ignoreFiles: opts?.ignoreFiles || [
defaultRules,
'package.json',
'.npmignore',
'.gitignore',
strictRules,
],
}
super(options)
this.isPackage = options.isPackage
this.seen = options.seen || new Set()
this.tree = tree
this.requiredFiles = options.requiredFiles || []
const additionalDefaults = []
if (options.prefix && options.workspaces) {
const path = normalizePath(options.path)
const prefix = normalizePath(options.prefix)
const workspaces = options.workspaces.map((ws) => normalizePath(ws))
// istanbul ignore else - this does nothing unless we need it to
if (path !== prefix && workspaces.includes(path)) {
// if path and prefix are not the same directory, and workspaces has path in it
// then we know path is a workspace directory. in order to not drop ignore rules
// from directories between the workspaces root (prefix) and the workspace itself
// (path) we need to find and read those now
const relpath = relative(options.prefix, dirname(options.path))
additionalDefaults.push(...readOutOfTreeIgnoreFiles(options.prefix, relpath))
} else if (path === prefix) {
// on the other hand, if the path and prefix are the same, then we ignore workspaces
// so that we don't pack a workspace as part of the root project. append them as
// normalized relative paths from the root
additionalDefaults.push(...workspaces.map((w) => normalizePath(relative(options.path, w))))
}
}
// go ahead and inject the default rules now
this.injectRules(defaultRules, [...defaults, ...additionalDefaults])
if (!this.isPackage) {
// if this instance is not a package, then place some strict default rules, and append
// known required files for this directory
this.injectRules(strictRules, [
...strictDefaults,
...this.requiredFiles.map((file) => `!${file}`),
])
}
}
// overridden method: we intercept the reading of the package.json file here so that we can
// process it into both the package.json file rules as well as the strictRules synthetic rule set
addIgnoreFile (file, callback) {
// if we're adding anything other than package.json, then let ignore-walk handle it
if (file !== 'package.json' || !this.isPackage) {
return super.addIgnoreFile(file, callback)
}
return this.processPackage(callback)
}
// overridden method: if we're done, but we're a package, then we also need to evaluate bundles
// before we actually emit our done event
emit (ev, data) {
if (ev !== 'done' || !this.isPackage) {
return super.emit(ev, data)
}
// we intentionally delay the done event while keeping the function sync here
// eslint-disable-next-line promise/catch-or-return, promise/always-return
this.gatherBundles().then(() => {
super.emit('done', this.result)
})
return true
}
// overridden method: before actually filtering, we make sure that we've removed the rules for
// files that should no longer take effect due to our order of precedence
filterEntries () {
if (this.ignoreRules['package.json']) {
// package.json means no .npmignore or .gitignore
this.ignoreRules['.npmignore'] = null
this.ignoreRules['.gitignore'] = null
} else if (this.ignoreRules['.npmignore']) {
// .npmignore means no .gitignore
this.ignoreRules['.gitignore'] = null
}
return super.filterEntries()
}
// overridden method: we never want to include anything that isn't a file or directory
onstat (opts, callback) {
if (!opts.st.isFile() && !opts.st.isDirectory()) {
return callback()
}
return super.onstat(opts, callback)
}
// overridden method: we want to refuse to pack files that are invalid, node-tar protects us from
// a lot of them but not all
stat (opts, callback) {
if (nameIsBadForWindows(opts.entry)) {
return callback()
}
return super.stat(opts, callback)
}
// overridden method: this is called to create options for a child walker when we step
// in to a normal child directory (this will never be a bundle). the default method here
// copies the root's `ignoreFiles` value, but we don't want to respect package.json for
// subdirectories, so we override it with a list that intentionally omits package.json
walkerOpt (entry, opts) {
let ignoreFiles = null
// however, if we have a tree, and we have workspaces, and the directory we're about
// to step into is a workspace, then we _do_ want to respect its package.json
if (this.tree.workspaces) {
const workspaceDirs = [...this.tree.workspaces.values()]
.map((dir) => dir.replace(/\\/g, '/'))
const entryPath = join(this.path, entry).replace(/\\/g, '/')
if (workspaceDirs.includes(entryPath)) {
ignoreFiles = [
defaultRules,
'package.json',
'.npmignore',
'.gitignore',
strictRules,
]
}
} else {
ignoreFiles = [
defaultRules,
'.npmignore',
'.gitignore',
strictRules,
]
}
return {
...super.walkerOpt(entry, opts),
ignoreFiles,
// we map over our own requiredFiles and pass ones that are within this entry
requiredFiles: this.requiredFiles
.map((file) => {
if (relative(file, entry) === '..') {
return relative(entry, file).replace(/\\/g, '/')
}
return false
})
.filter(Boolean),
}
}
// overridden method: we want child walkers to be instances of this class, not ignore-walk
walker (entry, opts, callback) {
new PackWalker(this.tree, this.walkerOpt(entry, opts)).on('done', callback).start()
}
// overridden method: we use a custom sort method to help compressibility
sort (a, b) {
// optimize for compressibility
// extname, then basename, then locale alphabetically
// https://twitter.com/isntitvacant/status/1131094910923231232
const exta = extname(a).toLowerCase()
const extb = extname(b).toLowerCase()
const basea = basename(a).toLowerCase()
const baseb = basename(b).toLowerCase()
return exta.localeCompare(extb, 'en') ||
basea.localeCompare(baseb, 'en') ||
a.localeCompare(b, 'en')
}
// convenience method: this joins the given rules with newlines, appends a trailing newline,
// and calls the internal onReadIgnoreFile method
injectRules (filename, rules, callback = () => {}) {
this.onReadIgnoreFile(filename, `${rules.join('\n')}\n`, callback)
}
// custom method: this is called by addIgnoreFile when we find a package.json, it uses the
// arborist tree to pull both default rules and strict rules for the package
processPackage (callback) {
const {
bin,
browser,
files,
main,
} = this.tree.package
// rules in these arrays are inverted since they are patterns we want to _not_ ignore
const ignores = []
const strict = [
...strictDefaults,
'!/package.json',
'/.git',
'/node_modules',
'/package-lock.json',
'/yarn.lock',
'/pnpm-lock.yaml',
]
// if we have a files array in our package, we need to pull rules from it
if (files) {
for (let file of files) {
// invert the rule because these are things we want to include
if (file.startsWith('/')) {
file = file.slice(1)
} else if (file.startsWith('./')) {
file = file.slice(2)
} else if (file.endsWith('/*')) {
file = file.slice(0, -2)
}
const inverse = `!${file}`
try {
// if an entry in the files array is a specific file, then we need to include it as a
// strict requirement for this package. if it's a directory or a pattern, it's a default
// pattern instead. this is ugly, but we have to stat to find out if it's a file
const stat = lstat(join(this.path, file.replace(/^!+/, '')).replace(/\\/g, '/'))
// if we have a file and we know that, it's strictly required
if (stat.isFile()) {
strict.unshift(inverse)
this.requiredFiles.push(file)
} else if (stat.isDirectory()) {
// otherwise, it's a default ignore, and since we got here we know it's not a pattern
// so we include the directory contents
ignores.push(inverse)
ignores.push(`${inverse}/**`)
}
// if the thing exists, but is neither a file or a directory, we don't want it at all
} catch (err) {
// if lstat throws, then we assume we're looking at a pattern and treat it as a default
ignores.push(inverse)
}
}
// we prepend a '*' to exclude everything, followed by our inverted file rules
// which now mean to include those
this.injectRules('package.json', ['*', ...ignores])
}
// browser is required
if (browser) {
strict.push(`!/${browser}`)
}
// main is required
if (main) {
strict.push(`!/${main}`)
}
// each bin is required
if (bin) {
for (const key in bin) {
strict.push(`!/${bin[key]}`)
}
}
// and now we add all of the strict rules to our synthetic file
this.injectRules(strictRules, strict, callback)
}
// custom method: after we've finished gathering the files for the root package, we call this
// before emitting the 'done' event in order to gather all of the files for bundled deps
async gatherBundles () {
if (this.seen.has(this.tree)) {
return
}
// add this node to our seen tracker
this.seen.add(this.tree)
// if we're the project root, then we look at our bundleDependencies, otherwise we got here
// because we're a bundled dependency of the root, which means we need to include all prod
// and optional dependencies in the bundle
let toBundle
if (this.tree.isProjectRoot) {
const { bundleDependencies } = this.tree.package
toBundle = bundleDependencies || []
} else {
const { dependencies, optionalDependencies } = this.tree.package
toBundle = Object.keys(dependencies || {}).concat(Object.keys(optionalDependencies || {}))
}
for (const dep of toBundle) {
const edge = this.tree.edgesOut.get(dep)
// no edgeOut = missing node, so skip it. we can't pack it if it's not here
// we also refuse to pack peer dependencies and dev dependencies
if (!edge || edge.peer || edge.dev) {
continue
}
// get a reference to the node we're bundling
const node = this.tree.edgesOut.get(dep).to
// we use node.path for the path because we want the location the node was linked to,
// not where it actually lives on disk
const path = node.path
// but link nodes don't have edgesOut, so we need to pass in the target of the node
// in order to make sure we correctly traverse its dependencies
const tree = node.target
// and start building options to be passed to the walker for this package
const walkerOpts = {
path,
isPackage: true,
ignoreFiles: [],
seen: this.seen, // pass through seen so we can prevent infinite circular loops
}
// if our node is a link, we apply defaultRules. we don't do this for regular bundled
// deps because their .npmignore and .gitignore files are excluded by default and may
// override defaults
if (node.isLink) {
walkerOpts.ignoreFiles.push(defaultRules)
}
// _all_ nodes will follow package.json rules from their package root
walkerOpts.ignoreFiles.push('package.json')
// only link nodes will obey .npmignore or .gitignore
if (node.isLink) {
walkerOpts.ignoreFiles.push('.npmignore')
walkerOpts.ignoreFiles.push('.gitignore')
}
// _all_ nodes follow strict rules
walkerOpts.ignoreFiles.push(strictRules)
// create a walker for this dependency and gather its results
const walker = new PackWalker(tree, walkerOpts)
const bundled = await new Promise((pResolve, pReject) => {
walker.on('error', pReject)
walker.on('done', pResolve)
walker.start()
})
// now we make sure we have our paths correct from the root, and accumulate everything into
// our own result set to deduplicate
const relativeFrom = relative(this.root, walker.path)
for (const file of bundled) {
this.result.add(join(relativeFrom, file).replace(/\\/g, '/'))
}
}
}
}
const walk = (tree, options, callback) => {
if (typeof options === 'function') {
callback = options
options = {}
}
const p = new Promise((pResolve, pReject) => {
new PackWalker(tree, { ...options, isPackage: true })
.on('done', pResolve).on('error', pReject).start()
})
return callback ? p.then(res => callback(null, res), callback) : p
}
module.exports = walk
walk.Walker = PackWalker