feat: add tarball details for published packages (#4600)

* feat: add tarball details for published packages (part 1)

* pnpm-lock

* Update pnpm-lock

* switch to tar-stream

* fix api test (no dist)

.changeset/wicked-worms-wash.md

@@ -0,0 +1,6 @@
+---
+'@verdaccio/store': patch
+'@verdaccio/tarball': patch
+---
+
+feat: add tarball details for published packages

packages/core/tarball/package.json

@@ -24,7 +24,7 @@
   "repository": {
     "type": "https",
     "url": "https://github.com/verdaccio/verdaccio",
-    "directory": "packages/core/url-resolver"
+    "directory": "packages/core/tarball"
   },
   "bugs": {
     "url": "https://github.com/verdaccio/verdaccio/issues"
@@ -33,11 +33,13 @@
     "access": "public"
   },
   "dependencies": {
-    "debug": "4.3.4",
     "@verdaccio/core": "workspace:7.0.0-next-7.13",
     "@verdaccio/url": "workspace:12.0.0-next-7.13",
     "@verdaccio/utils": "workspace:7.0.0-next-7.13",
-    "lodash": "4.17.21"
+    "debug": "4.3.4",
+    "gunzip-maybe": "^1.4.2",
+    "lodash": "4.17.21",
+    "tar-stream": "^3.1.7"
   },
   "devDependencies": {
     "@verdaccio/types": "workspace:12.0.0-next.2",

packages/core/tarball/src/getTarballDetails.ts

@@ -0,0 +1,34 @@
+import gunzipMaybe from 'gunzip-maybe';
+import { Readable } from 'stream';
+import * as tarStream from 'tar-stream';
+
+export type TarballDetails = {
+  fileCount: number;
+  unpackedSize: number; // in bytes
+};
+
+export async function getTarballDetails(readable: Readable): Promise<TarballDetails> {
+  let fileCount = 0;
+  let unpackedSize = 0;
+
+  const unpack = tarStream.extract();
+
+  return new Promise((resolve, reject) => {
+    readable
+      .pipe(gunzipMaybe())
+      .pipe(unpack)
+      .on('entry', (header, stream, next) => {
+        fileCount++;
+        unpackedSize += Number(header.size);
+        stream.resume(); // important to ensure that "entry" events keep firing
+        next();
+      })
+      .on('finish', () => {
+        resolve({
+          fileCount,
+          unpackedSize,
+        });
+      })
+      .on('error', reject);
+  });
+}

packages/core/tarball/src/index.ts

@@ -5,5 +5,6 @@ export {
   convertDistVersionToLocalTarballsUrl,
 } from './convertDistRemoteToLocalTarballUrls';
 export { extractTarballFromUrl, getLocalRegistryTarballUri } from './getLocalRegistryTarballUri';
+export { TarballDetails, getTarballDetails } from './getTarballDetails';
 
 export { RequestOptions };

packages/core/tarball/tests/getTarballDetails.spec.ts

@@ -0,0 +1,37 @@
+import fs from 'fs';
+import path from 'path';
+import { Readable } from 'stream';
+
+import { getTarballDetails } from '../src/getTarballDetails.ts';
+
+const getFilePath = (filename: string): string => {
+  return path.resolve(__dirname, `assets/${filename}`);
+};
+
+const getFileBuffer = async (filename: string): Promise<Buffer> => {
+  return fs.promises.readFile(getFilePath(filename));
+};
+
+describe('getTarballDetails', () => {
+  test('should return stats of tarball (gzipped)', async () => {
+    const buffer = await getFileBuffer('tarball.tgz');
+    const readable = Readable.from(buffer);
+    const details = await getTarballDetails(readable);
+    expect(details.fileCount).toBe(2);
+    expect(details.unpackedSize).toBe(1280);
+  });
+
+  test('should return stats of tarball (uncompressed)', async () => {
+    const buffer = await getFileBuffer('tarball.tar');
+    const readable = Readable.from(buffer);
+    const details = await getTarballDetails(readable);
+    expect(details.fileCount).toBe(2);
+    expect(details.unpackedSize).toBe(1280);
+  });
+
+  test('should throw an error if the buffer is corrupt', async () => {
+    const corruptBuffer = Buffer.from('this is not a tarball');
+    const readable = Readable.from(corruptBuffer);
+    await expect(getTarballDetails(readable)).rejects.toThrow();
+  });
+});

packages/store/src/storage.ts

@@ -34,9 +34,11 @@ import {
 } from '@verdaccio/proxy';
 import Search from '@verdaccio/search';
 import {
+  TarballDetails,
   convertDistRemoteToLocalTarballUrls,
   convertDistVersionToLocalTarballsUrl,
   extractTarballFromUrl,
+  getTarballDetails,
 } from '@verdaccio/tarball';
 import {
   AbbreviatedManifest,
@@ -1044,6 +1046,8 @@ class Storage {
     // at this point document is either created or existed before
     const [firstAttachmentKey] = Object.keys(_attachments);
     const buffer = this.getBufferManifest(body._attachments[firstAttachmentKey].data as string);
+    const readable = Readable.from(buffer);
+    const tarballStats = await this.getTarballStats(versions[versionToPublish], readable);
 
     try {
       // we check if package exist already locally
@@ -1082,7 +1086,7 @@ class Storage {
           _.isNil(manifest.readme) === false ? String(manifest.readme) : '';
       }
       // addVersion will move the readme from the the published version to the root level
-      await this.addVersion(name, versionToPublish, versions[versionToPublish], null);
+      await this.addVersion(name, versionToPublish, versions[versionToPublish], null, tarballStats);
     } catch (err: any) {
       logger.error({ err: err.message }, 'updated version has failed: @{err}');
       debug('error on create a version for %o with error %o', name, err.message);
@@ -1110,7 +1114,6 @@ class Storage {
 
     // 3. upload the tarball to the storage
     try {
-      const readable = Readable.from(buffer);
       await this.uploadTarball(name, basename(firstAttachmentKey), readable, {
         signal: options.signal,
       });
@@ -1283,7 +1286,8 @@ class Storage {
     name: string,
     version: string,
     metadata: Version,
-    tag: StringValue
+    tag: StringValue,
+    tarballStats: TarballDetails
   ): Promise<void> {
     debug(`add version %s package for %s`, version, name);
     await this.updatePackage(name, async (data: Manifest): Promise<Manifest> => {
@@ -1295,6 +1299,12 @@ class Storage {
       metadata.contributors = normalizeContributors(metadata.contributors as Author[]);
       debug('%s` contributors normalized', name);
 
+      // Update tarball stats
+      if (metadata.dist) {
+        metadata.dist.fileCount = tarballStats.fileCount;
+        metadata.dist.unpackedSize = tarballStats.unpackedSize;
+      }
+
       // if uploaded tarball has a different shasum, it's very likely that we
       // have some kind of error
       if (validatioUtils.isObject(metadata.dist) && _.isString(metadata.dist.tarball)) {
@@ -1905,6 +1915,25 @@ class Storage {
       return cacheManifest;
     }
   }
+
+  private async getTarballStats(version: Version, readable: Readable): Promise<TarballDetails> {
+    if (
+      version.dist == undefined ||
+      version.dist?.fileCount == undefined ||
+      version.dist?.unpackedSize == undefined
+    ) {
+      debug('tarball stats not found, calculating');
+      try {
+        return await getTarballDetails(readable);
+      } catch (err: any) {
+        logger.error({ err: err.message }, 'getting tarball details has failed: @{err}');
+        throw err;
+      }
+    } else {
+      debug('tarball stats found');
+      return { fileCount: version.dist.fileCount, unpackedSize: version.dist.unpackedSize };
+    }
+  }
 }
 
 export { Storage };

packages/store/test/storage.spec.ts

@@ -262,10 +262,12 @@ describe('storage', () => {
         expect(manifestVersion._id).toEqual(`${pkgName}@1.0.1`);
         expect(manifestVersion.description).toEqual('package generated');
         expect(manifestVersion.dist).toEqual({
+          fileCount: 4,
           integrity:
             'sha512-6gHiERpiDgtb3hjqpQH5/i7zRmvYi9pmCjQf2ZMy3QEa9wVk9RgdZaPWUt7ZOnWUPFjcr9cmE6dUBf+XoPoH4g==',
           shasum: '2c03764f651a9f016ca0b7620421457b619151b9',
           tarball: 'http://localhost:5555/upstream/-/upstream-1.0.1.tgz',
+          unpackedSize: 543,
         });
 
         expect(manifestVersion.contributors).toEqual([]);