Improve processing resources with Brotli encoding (close #2743)

package.json

@@ -18,7 +18,6 @@
     "acorn-hammerhead": "0.5.0",
     "asar": "^2.0.1",
     "bowser": "1.6.0",
-    "brotli": "^1.3.1",
     "crypto-md5": "^1.0.0",
     "css": "2.2.3",
     "debug": "4.3.1",

src/processing/encoding/index.ts

@@ -1,12 +1,45 @@
 import zlib from 'zlib';
-import { gzip, deflate, gunzip, inflate, inflateRaw } from '../../utils/promisified-functions';
-import { brotliCompress, brotliDecompress } from './brotli';
+
+import {
+    gzip,
+    deflate,
+    gunzip,
+    inflate,
+    inflateRaw,
+    brotliCompress,
+    brotliDecompress,
+} from '../../utils/promisified-functions';
+
 import charsetEncoder from 'iconv-lite';
 import Charset from './charset';
 
-const GZIP_CONTENT_ENCODING    = 'gzip';
-const DEFLATE_CONTENT_ENCODING = 'deflate';
-const BROTLI_CONTENT_ENCODING  = 'br';
+const enum CONTENT_ENCODING {
+    GZIP = 'gzip',
+    DEFLATE = 'deflate',
+    BROTLI = 'br'
+}
+
+// NOTE: https://github.com/request/request/pull/2492/files
+// Be more lenient with decoding compressed responses, since (very rarely)
+// servers send slightly invalid gzip responses that are still accepted
+// by common browsers.
+// Always using Z_SYNC_FLUSH is what cURL does.
+// GH-1915
+const GZIP_DECODING_OPTIONS = {
+    flush:       zlib.Z_SYNC_FLUSH,
+    finishFlush: zlib.Z_SYNC_FLUSH
+};
+
+// NOTE: https://github.com/DevExpress/testcafe-hammerhead/issues/2743
+// The default compression level (11) causes the strong performance degradation.
+// This is why, we decrease the compression level same as other frameworks
+// (https://github.com/dotnet/runtime/issues/26097, https://github.com/koajs/compress/issues/121)
+
+const BROTLI_DECODING_OPTIONS = {
+    params: {
+        [zlib.constants.BROTLI_PARAM_QUALITY]: 5
+    }
+};
 
 // NOTE: IIS has a bug when it sends 'raw deflate' compressed data for the 'Deflate' Accept-Encoding header.
 // (see: http://zoompf.com/2012/02/lose-the-wait-http-compression)
@@ -23,21 +56,14 @@ async function inflateWithFallback (data: Buffer): Promise<Buffer> {
 }
 
 export async function decodeContent (content: Buffer, encoding: string, charset: Charset): Promise<string> {
-    if (encoding === GZIP_CONTENT_ENCODING) {
-        // NOTE: https://github.com/request/request/pull/2492/files
-        // Be more lenient with decoding compressed responses, since (very rarely)
-        // servers send slightly invalid gzip responses that are still accepted
-        // by common browsers.
-        // Always using Z_SYNC_FLUSH is what cURL does.
-        // GH-1915
-        content = await gunzip(content, { flush: zlib.Z_SYNC_FLUSH, finishFlush: zlib.Z_SYNC_FLUSH });
-    }
+    if (encoding === CONTENT_ENCODING.GZIP)
+        content = await gunzip(content, GZIP_DECODING_OPTIONS);
 
-    else if (encoding === DEFLATE_CONTENT_ENCODING)
+    else if (encoding === CONTENT_ENCODING.DEFLATE)
         content = await inflateWithFallback(content);
 
-    else if (encoding === BROTLI_CONTENT_ENCODING)
-        content = await brotliDecompress(content);
+    else if (encoding === CONTENT_ENCODING.BROTLI)
+        content = await brotliDecompress(content, BROTLI_DECODING_OPTIONS);
 
     charset.fromBOM(content);
 
@@ -47,14 +73,14 @@ export async function decodeContent (content: Buffer, encoding: string, charset:
 export async function encodeContent (content: string, encoding: string, charset: Charset): Promise<Buffer> {
     const encodedContent = charsetEncoder.encode(content, charset.get(), { addBOM: charset.isFromBOM() });
 
-    if (encoding === GZIP_CONTENT_ENCODING)
+    if (encoding === CONTENT_ENCODING.GZIP)
         return gzip(encodedContent);
 
-    if (encoding === DEFLATE_CONTENT_ENCODING)
+    if (encoding === CONTENT_ENCODING.DEFLATE)
         return deflate(encodedContent);
 
-    if (encoding === BROTLI_CONTENT_ENCODING)
-        return brotliCompress(encodedContent);
+    if (encoding === CONTENT_ENCODING.BROTLI)
+        return brotliCompress(encodedContent, BROTLI_DECODING_OPTIONS);
 
     return encodedContent;
 }

src/utils/promisified-functions.ts

@@ -1,20 +1,22 @@
-import zlib from 'zlib';
+import zlib, { BrotliOptions, InputType, ZlibOptions } from 'zlib';
 import { promisify } from 'util';
 import fs from 'fs';
 import childProcess from 'child_process';
 
-export const gzip: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer>       = promisify(zlib.gzip);
-export const deflate: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer>    = promisify(zlib.deflate);
-export const gunzip: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer>     = promisify(zlib.gunzip);
-export const inflate: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer>    = promisify(zlib.inflate);
-export const inflateRaw: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer> = promisify(zlib.inflateRaw);
-export const readDir: (path: string) => Promise<string[]>                                     = promisify(fs.readdir);
+export const gzip: (buf: InputType, options?: ZlibOptions) => Promise<Buffer>               = promisify(zlib.gzip);
+export const deflate: (buf: InputType, options?: ZlibOptions) => Promise<Buffer>            = promisify(zlib.deflate);
+export const gunzip: (buf: InputType, options?: ZlibOptions) => Promise<Buffer>             = promisify(zlib.gunzip);
+export const inflate: (buf: InputType, options?: ZlibOptions) => Promise<Buffer>            = promisify(zlib.inflate);
+export const inflateRaw: (buf: InputType, options?: ZlibOptions) => Promise<Buffer>         = promisify(zlib.inflateRaw);
+export const brotliCompress: (buf: InputType, options?: BrotliOptions) => Promise<Buffer>   = promisify(zlib.brotliCompress);
+export const brotliDecompress: (buf: InputType, options?: BrotliOptions) => Promise<Buffer> = promisify(zlib.brotliDecompress);
 
-export const readFile       = promisify(fs.readFile);
-export const stat           = promisify(fs.stat);
-export const access         = promisify(fs.access);
-export const makeDir        = promisify(fs.mkdir);
-export const writeFile      = promisify(fs.writeFile);
-export const fsObjectExists = (fsPath: string) => stat(fsPath).then(() => true, () => false);
+export const readDir: (path: string) => Promise<string[]> = promisify(fs.readdir);
+export const readFile                                     = promisify(fs.readFile);
+export const stat                                         = promisify(fs.stat);
+export const access                                       = promisify(fs.access);
+export const makeDir                                      = promisify(fs.mkdir);
+export const writeFile                                    = promisify(fs.writeFile);
+export const fsObjectExists                               = (fsPath: string) => stat(fsPath).then(() => true, () => false);
 
 export const exec = promisify(childProcess.exec);

test/server/encoding-test.js

@@ -1,7 +1,7 @@
-const expect        = require('chai').expect;
-const encodeContent = require('../../lib/processing/encoding').encodeContent;
-const decodeContent = require('../../lib/processing/encoding').decodeContent;
-const Charset       = require('../../lib/processing/encoding/charset');
+const { expect }                       = require('chai');
+const { encodeContent, decodeContent } = require('../../lib/processing/encoding');
+const Charset                          = require('../../lib/processing/encoding/charset');
+const crypto                           = require('crypto');
 
 describe('Content encoding', () => {
     const src = Buffer.from('Answer to the Ultimate Question of Life, the Universe, and Everything.');
@@ -37,4 +37,22 @@ describe('Content encoding', () => {
                 expect(err).to.be.an('object');
             });
     });
+
+    it('Brotli decoding performance (GH-2743)', async () => {
+        const charset = new Charset();
+
+        charset.set('utf8', 2);
+
+        const content = crypto.randomBytes(10 * 1000 * 1000).toString('hex');
+
+        const start = Date.now();
+
+        const encoded = await encodeContent(content, 'br', charset);
+
+        await decodeContent(encoded, 'br', charset);
+
+        const executionTime = Date.now() - start;
+
+        expect(executionTime).below(5000);
+    });
 });