From ff34d6cd6f2077962cba1ef9c893a958ac7174f8 Mon Sep 17 00:00:00 2001
From: Nathan Fritz <fritzy@github.com>
Date: Wed, 28 Jul 2021 22:01:35 -0700
Subject: [PATCH] feat(cache): initial implementation of ls and rm

PR-URL: https://github.com/npm/cli/pull/3592
Credit: @fritzy
Close: #3592
Reviewed-by: @nlf
---
 lib/cache.js                                  | 128 ++++++--
 .../test/lib/load-all-commands.js.test.cjs    |   3 +-
 .../test/lib/utils/npm-usage.js.test.cjs      |   3 +-
 test/lib/cache.js                             | 279 +++++++++++++++++-
 4 files changed, 382 insertions(+), 31 deletions(-)

diff --git a/lib/cache.js b/lib/cache.js
index 55fb3e863631c..aed2cce31e63f 100644
--- a/lib/cache.js
+++ b/lib/cache.js
@@ -4,7 +4,59 @@ const log = require('npmlog')
 const pacote = require('pacote')
 const path = require('path')
 const rimraf = promisify(require('rimraf'))
+const semver = require('semver')
 const BaseCommand = require('./base-command.js')
+const npa = require('npm-package-arg')
+const jsonParse = require('json-parse-even-better-errors')
+
+const searchCachePackage = async (path, spec, cacheKeys) => {
+  const parsed = npa(spec)
+  if (parsed.rawSpec !== '' && parsed.type === 'tag')
+    throw new Error(`Cannot list cache keys for a tagged package.`)
+  const searchMFH = new RegExp(`^make-fetch-happen:request-cache:.*(?<!/[@a-zA-Z]+)/${parsed.name}/-/(${parsed.name}[^/]+.tgz)$`)
+  const searchPack = new RegExp(`^make-fetch-happen:request-cache:.*/${parsed.escapedName}$`)
+  const results = new Set()
+  cacheKeys = new Set(cacheKeys)
+  for (const key of cacheKeys) {
+    // match on the public key registry url format
+    if (searchMFH.test(key)) {
+      // extract the version from the filename
+      const filename = key.match(searchMFH)[1]
+      const noExt = filename.slice(0, -4)
+      const noScope = `${parsed.name.split('/').pop()}-`
+      const ver = noExt.slice(noScope.length)
+      if (semver.satisfies(ver, parsed.rawSpec))
+        results.add(key)
+      continue
+    }
+    // is this key a packument?
+    if (!searchPack.test(key))
+      continue
+
+    results.add(key)
+    let packument, details
+    try {
+      details = await cacache.get(path, key)
+      packument = jsonParse(details.data)
+    } catch (_) {
+      // if we couldn't parse the packument, abort
+      continue
+    }
+    if (!packument.versions || typeof packument.versions !== 'object')
+      continue
+    // assuming this is a packument
+    for (const ver of Object.keys(packument.versions)) {
+      if (semver.satisfies(ver, parsed.rawSpec)) {
+        if (packument.versions[ver].dist
+          && typeof packument.versions[ver].dist === 'object'
+          && packument.versions[ver].dist.tarball !== undefined
+          && cacheKeys.has(`make-fetch-happen:request-cache:${packument.versions[ver].dist.tarball}`))
+          results.add(`make-fetch-happen:request-cache:${packument.versions[ver].dist.tarball}`)
+      }
+    }
+  }
+  return results
+}
 
 class Cache extends BaseCommand {
   static get description () {
@@ -29,7 +81,8 @@ class Cache extends BaseCommand {
       'add <tarball url>',
       'add <git url>',
       'add <name>@<version>',
-      'clean',
+      'clean [<key>]',
+      'ls [<name>@<version>]',
       'verify',
     ]
   }
@@ -37,13 +90,15 @@ class Cache extends BaseCommand {
   async completion (opts) {
     const argv = opts.conf.argv.remain
     if (argv.length === 2)
-      return ['add', 'clean', 'verify']
+      return ['add', 'clean', 'verify', 'ls', 'delete']
 
     // TODO - eventually...
     switch (argv[2]) {
       case 'verify':
       case 'clean':
       case 'add':
+      case 'ls':
+      case 'delete':
         return []
     }
   }
@@ -61,6 +116,8 @@ class Cache extends BaseCommand {
         return await this.add(args)
       case 'verify': case 'check':
         return await this.verify()
+      case 'ls':
+        return await this.ls(args)
       default:
         throw Object.assign(new Error(this.usage), { code: 'EUSAGE' })
     }
@@ -68,27 +125,38 @@ class Cache extends BaseCommand {
 
   // npm cache clean [pkg]*
   async clean (args) {
-    if (args.length)
-      throw new Error('npm cache clear does not accept arguments')
-
     const cachePath = path.join(this.npm.cache, '_cacache')
-    if (!this.npm.config.get('force')) {
-      throw new Error(`As of npm@5, the npm cache self-heals from corruption issues
-by treating integrity mismatches as cache misses.  As a result,
-data extracted from the cache is guaranteed to be valid.  If you
-want to make sure everything is consistent, use \`npm cache verify\`
-instead.  Deleting the cache can only make npm go slower, and is
-not likely to correct any problems you may be encountering!
-
-On the other hand, if you're debugging an issue with the installer,
-or race conditions that depend on the timing of writing to an empty
-cache, you can use \`npm install --cache /tmp/empty-cache\` to use a
-temporary cache instead of nuking the actual one.
-
-If you're sure you want to delete the entire cache, rerun this command
-with --force.`)
+    if (args.length === 0) {
+      if (!this.npm.config.get('force')) {
+        throw new Error(`As of npm@5, the npm cache self-heals from corruption issues
+  by treating integrity mismatches as cache misses.  As a result,
+  data extracted from the cache is guaranteed to be valid.  If you
+  want to make sure everything is consistent, use \`npm cache verify\`
+  instead.  Deleting the cache can only make npm go slower, and is
+  not likely to correct any problems you may be encountering!
+
+  On the other hand, if you're debugging an issue with the installer,
+  or race conditions that depend on the timing of writing to an empty
+  cache, you can use \`npm install --cache /tmp/empty-cache\` to use a
+  temporary cache instead of nuking the actual one.
+
+  If you're sure you want to delete the entire cache, rerun this command
+  with --force.`)
+      }
+      return rimraf(cachePath)
+    }
+    for (const key of args) {
+      let entry
+      try {
+        entry = await cacache.get(cachePath, key)
+      } catch (err) {
+        this.npm.log.warn(`Not Found: ${key}`)
+        break
+      }
+      this.npm.output(`Deleted: ${key}`)
+      await cacache.rm.entry(cachePath, key)
+      await cacache.rm.content(cachePath, entry.integrity)
     }
-    return rimraf(cachePath)
   }
 
   // npm cache add <tarball-url>...
@@ -131,6 +199,24 @@ with --force.`)
     this.npm.output(`Index entries: ${stats.totalEntries}`)
     this.npm.output(`Finished in ${stats.runTime.total / 1000}s`)
   }
+
+  // npm cache ls [--package <spec> ...]
+  async ls (specs) {
+    const cachePath = path.join(this.npm.cache, '_cacache')
+    const cacheKeys = Object.keys(await cacache.ls(cachePath))
+    if (specs.length > 0) {
+      // get results for each package spec specified
+      const results = new Set()
+      for (const spec of specs) {
+        const keySet = await searchCachePackage(cachePath, spec, cacheKeys)
+        for (const key of keySet)
+          results.add(key)
+      }
+      [...results].sort((a, b) => a.localeCompare(b, 'en')).forEach(key => this.npm.output(key))
+      return
+    }
+    cacheKeys.sort((a, b) => a.localeCompare(b, 'en')).forEach(key => this.npm.output(key))
+  }
 }
 
 module.exports = Cache
diff --git a/tap-snapshots/test/lib/load-all-commands.js.test.cjs b/tap-snapshots/test/lib/load-all-commands.js.test.cjs
index 8cf2e2837e295..9f811a0058fd1 100644
--- a/tap-snapshots/test/lib/load-all-commands.js.test.cjs
+++ b/tap-snapshots/test/lib/load-all-commands.js.test.cjs
@@ -102,7 +102,8 @@ npm cache add <folder>
 npm cache add <tarball url>
 npm cache add <git url>
 npm cache add <name>@<version>
-npm cache clean
+npm cache clean [<key>]
+npm cache ls [<name>@<version>]
 npm cache verify
 
 Options:
diff --git a/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs b/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs
index 50f6481f6e848..0fd36c7c1d372 100644
--- a/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs
+++ b/tap-snapshots/test/lib/utils/npm-usage.js.test.cjs
@@ -251,7 +251,8 @@ All commands:
                     npm cache add <tarball url>
                     npm cache add <git url>
                     npm cache add <name>@<version>
-                    npm cache clean
+                    npm cache clean [<key>]
+                    npm cache ls [<name>@<version>]
                     npm cache verify
                     
                     Options:
diff --git a/test/lib/cache.js b/test/lib/cache.js
index d3d6f5b8845de..c6405303202b8 100644
--- a/test/lib/cache.js
+++ b/test/lib/cache.js
@@ -1,6 +1,7 @@
 const t = require('tap')
 const { fake: mockNpm } = require('../fixtures/mock-npm.js')
 const path = require('path')
+const npa = require('npm-package-arg')
 
 const usageUtil = () => 'usage instructions'
 
@@ -34,16 +35,104 @@ const pacote = {
   },
 }
 
+let cacacheEntries = {}
+let cacacheContent = {}
+
+const setupCacacheFixture = () => {
+  cacacheEntries = {}
+  cacacheContent = {}
+  const pkgs = [
+    ['webpack@4.44.1', 'https://registry.npmjs.org', true],
+    ['npm@1.2.0', 'https://registry.npmjs.org', true],
+    ['webpack@4.47.0', 'https://registry.npmjs.org', true],
+    ['foo@1.2.3-beta', 'https://registry.npmjs.org', true],
+    ['ape-ecs@2.1.7', 'https://registry.npmjs.org', true],
+    ['@fritzy/staydown@3.1.1', 'https://registry.npmjs.org', true],
+    ['@gar/npm-expansion@2.1.0', 'https://registry.npmjs.org', true],
+    ['@gar/npm-expansion@3.0.0-beta', 'https://registry.npmjs.org', true],
+    ['extemporaneously@44.2.2', 'https://somerepo.github.org', false],
+    ['corrupted@3.1.0', 'https://registry.npmjs.org', true],
+    ['missing-dist@23.0.0', 'https://registry.npmjs.org', true],
+    ['missing-version@16.2.0', 'https://registry.npmjs.org', true],
+  ]
+  pkgs.forEach(pkg => addCacachePkg(...pkg))
+  // corrupt the packument
+  cacacheContent[
+    [cacacheEntries['make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted'].integrity]
+  ].data = Buffer.from('<>>>}"')
+  // nuke the version dist
+  cacacheContent[
+    [cacacheEntries['make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist'].integrity]
+  ].data = Buffer.from(JSON.stringify({ versions: { '23.0.0': {} } }))
+  // make the version a non-object
+  cacacheContent[
+    [cacacheEntries['make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version'].integrity]
+  ].data = Buffer.from(JSON.stringify({ versions: 'hello' }))
+}
+
+const packuments = {}
+
+let contentId = 0
 const cacacheVerifyStats = {
   keptSize: 100,
   verifiedContent: 1,
   totalEntries: 1,
   runTime: { total: 2000 },
 }
+
+const addCacacheKey = (key, content) => {
+  contentId++
+  cacacheEntries[key] = { integrity: `${contentId}` }
+  cacacheContent[`${contentId}`] = {}
+}
+const addCacachePkg = (spec, registry, publicURL) => {
+  const parts = npa(spec)
+  const ver = parts.rawSpec || '1.0.0'
+  let url = `${registry}/${parts.name}/-/${parts.name}-${ver}.tgz`
+  if (!publicURL)
+    url = `${registry}/aabbcc/${contentId}`
+  const key = `make-fetch-happen:request-cache:${url}`
+  const pkey = `make-fetch-happen:request-cache:${registry}/${parts.escapedName}`
+  if (!packuments[parts.escapedName]) {
+    packuments[parts.escapedName] = {
+      versions: {},
+    }
+    addCacacheKey(pkey)
+  }
+  packuments[parts.escapedName].versions[ver] = {
+    dist: {
+      tarball: url,
+    },
+  }
+  addCacacheKey(key)
+  cacacheContent[cacacheEntries[pkey].integrity] = {
+    data: Buffer.from(JSON.stringify(packuments[parts.escapedName])),
+  }
+}
+
 const cacache = {
   verify: (path) => {
     return cacacheVerifyStats
   },
+  get: (path, key) => {
+    if (cacacheEntries[key] === undefined
+      || cacacheContent[cacacheEntries[key].integrity] === undefined)
+      throw new Error()
+    return cacacheContent[cacacheEntries[key].integrity]
+  },
+  rm: {
+    entry: (path, key) => {
+      if (cacacheEntries[key] === undefined)
+        throw new Error()
+      delete cacacheEntries[key]
+    },
+    content: (path, sha) => {
+      delete cacacheContent[sha]
+    },
+  },
+  ls: (path) => {
+    return cacacheEntries
+  },
 }
 
 const Cache = t.mock('../../lib/cache.js', {
@@ -61,6 +150,11 @@ const npm = mockNpm({
   output: (msg) => {
     outputOutput.push(msg)
   },
+  log: {
+    warn: (...args) => {
+      logOutput.push(['warn', ...args])
+    },
+  },
 })
 const cache = new Cache(npm)
 
@@ -94,13 +188,6 @@ t.test('cache clean (force)', t => {
   })
 })
 
-t.test('cache clean with arg', t => {
-  cache.exec(['rm', 'pkg'], err => {
-    t.match(err.message, 'does not accept arguments', 'should throw error')
-    t.end()
-  })
-})
-
 t.test('cache add no arg', t => {
   t.teardown(() => {
     logOutput = []
@@ -136,7 +223,7 @@ t.test('cache add pkg only', t => {
 
 t.test('cache add multiple pkgs', t => {
   t.teardown(() => {
-    logOutput = []
+    outputOutput = []
     tarballStreamSpec = ''
     tarballStreamOpts = {}
   })
@@ -154,6 +241,182 @@ t.test('cache add multiple pkgs', t => {
   })
 })
 
+t.test('cache ls', t => {
+  t.teardown(() => {
+    outputOutput = []
+    logOutput = []
+  })
+  setupCacacheFixture()
+  cache.exec(['ls'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy/staydown/-/@fritzy/staydown-3.1.1.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy%2fstaydown',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-2.1.0.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-3.0.0-beta.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar%2fnpm-expansion',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/ape-ecs',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/ape-ecs/-/ape-ecs-2.1.7.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted/-/corrupted-3.1.0.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/foo',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist/-/missing-dist-23.0.0.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version/-/missing-version-16.2.0.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/npm',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/npm/-/npm-1.2.0.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.47.0.tgz',
+      'make-fetch-happen:request-cache:https://somerepo.github.org/aabbcc/14',
+      'make-fetch-happen:request-cache:https://somerepo.github.org/extemporaneously',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache ls pkgs', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', 'webpack@>4.44.1', 'npm'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/npm',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/npm/-/npm-1.2.0.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.47.0.tgz',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache ls special', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', 'foo@1.2.3-beta'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/foo',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/foo/-/foo-1.2.3-beta.tgz',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache ls nonpublic registry', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', 'extemporaneously'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://somerepo.github.org/aabbcc/14',
+      'make-fetch-happen:request-cache:https://somerepo.github.org/extemporaneously',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache ls tagged', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', 'webpack@latest'], err => {
+    t.match(err.message, 'tagged package', 'should throw warning')
+    t.end()
+  })
+})
+
+t.test('cache ls scoped and scoped slash', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', '@fritzy/staydown', '@gar/npm-expansion'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy/staydown/-/@fritzy/staydown-3.1.1.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@fritzy%2fstaydown',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar/npm-expansion/-/@gar/npm-expansion-2.1.0.tgz',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/@gar%2fnpm-expansion',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache ls corrupted', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', 'corrupted'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/corrupted/-/corrupted-3.1.0.tgz',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache ls missing packument dist', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', 'missing-dist'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-dist/-/missing-dist-23.0.0.tgz',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache ls missing packument version not an object', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['ls', 'missing-version'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version',
+      'make-fetch-happen:request-cache:https://registry.npmjs.org/missing-version/-/missing-version-16.2.0.tgz',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache rm', t => {
+  t.teardown(() => {
+    outputOutput = []
+  })
+  cache.exec(['rm',
+    'make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz'], err => {
+    t.error(err)
+    t.strictSame(outputOutput, [
+      'Deleted: make-fetch-happen:request-cache:https://registry.npmjs.org/webpack/-/webpack-4.44.1.tgz',
+    ])
+    t.end()
+  })
+})
+
+t.test('cache rm unfound', t => {
+  t.teardown(() => {
+    outputOutput = []
+    logOutput = []
+  })
+  cache.exec(['rm', 'made-up-key'], err => {
+    t.error(err)
+    t.strictSame(logOutput, [
+      ['warn', 'Not Found: made-up-key'],
+    ], 'logs correctly')
+    t.end()
+  })
+})
+
 t.test('cache verify', t => {
   t.teardown(() => {
     outputOutput = []