Skip to content

Commit

Permalink
initial (#2752)
Browse files Browse the repository at this point in the history
  • Loading branch information
@miherlosev
miherlosev committed Apr 5, 2022
1 parent cae8a7c commit c2b0ab0
Show file tree
Hide file tree
Showing 5 changed files with 84 additions and 62 deletions.
1 change: 0 additions & 1 deletion package.json
View file
Expand Up @@ -18,7 +18,6 @@
"acorn-hammerhead": "0.6.1",
"asar": "^2.0.1",
"bowser": "1.6.0",
"brotli": "^1.3.1",
"crypto-md5": "^1.0.0",
"css": "2.2.3",
"debug": "4.3.1",
Expand Down
23 changes: 0 additions & 23 deletions src/processing/encoding/brotli.ts
View file

This file was deleted.

68 changes: 47 additions & 21 deletions src/processing/encoding/index.ts
View file
@@ -1,12 +1,45 @@
import zlib from 'zlib';
import { gzip, deflate, gunzip, inflate, inflateRaw } from '../../utils/promisified-functions';
import { brotliCompress, brotliDecompress } from './brotli';

import {
gzip,
deflate,
gunzip,
inflate,
inflateRaw,
brotliCompress,
brotliDecompress,
} from '../../utils/promisified-functions';

import charsetEncoder from 'iconv-lite';
import Charset from './charset';

const GZIP_CONTENT_ENCODING = 'gzip';
const DEFLATE_CONTENT_ENCODING = 'deflate';
const BROTLI_CONTENT_ENCODING = 'br';
const enum CONTENT_ENCODING {
GZIP = 'gzip',
DEFLATE = 'deflate',
BROTLI = 'br'
}

// NOTE: https://github.com/request/request/pull/2492/files
// Be more lenient with decoding compressed responses, since (very rarely)
// servers send slightly invalid gzip responses that are still accepted
// by common browsers.
// Always using Z_SYNC_FLUSH is what cURL does.
// GH-1915
const GZIP_DECODING_OPTIONS = {
flush: zlib.Z_SYNC_FLUSH,
finishFlush: zlib.Z_SYNC_FLUSH
};

// NOTE: https://github.com/DevExpress/testcafe-hammerhead/issues/2743
// The default compression level (11) causes the strong performance degradation.
// This is why, we decrease the compression level same as other frameworks
// (https://github.com/dotnet/runtime/issues/26097, https://github.com/koajs/compress/issues/121)

const BROTLI_DECODING_OPTIONS = {
params: {
[zlib.constants.BROTLI_PARAM_QUALITY]: 5
}
};

// NOTE: IIS has a bug when it sends 'raw deflate' compressed data for the 'Deflate' Accept-Encoding header.
// (see: http://zoompf.com/2012/02/lose-the-wait-http-compression)
Expand All @@ -23,21 +56,14 @@ async function inflateWithFallback (data: Buffer): Promise<Buffer> {
}

export async function decodeContent (content: Buffer, encoding: string, charset: Charset): Promise<string> {
if (encoding === GZIP_CONTENT_ENCODING) {
// NOTE: https://github.com/request/request/pull/2492/files
// Be more lenient with decoding compressed responses, since (very rarely)
// servers send slightly invalid gzip responses that are still accepted
// by common browsers.
// Always using Z_SYNC_FLUSH is what cURL does.
// GH-1915
content = await gunzip(content, { flush: zlib.Z_SYNC_FLUSH, finishFlush: zlib.Z_SYNC_FLUSH });
}
if (encoding === CONTENT_ENCODING.GZIP)
content = await gunzip(content, GZIP_DECODING_OPTIONS);

else if (encoding === DEFLATE_CONTENT_ENCODING)
else if (encoding === CONTENT_ENCODING.DEFLATE)
content = await inflateWithFallback(content);

else if (encoding === BROTLI_CONTENT_ENCODING)
content = await brotliDecompress(content);
else if (encoding === CONTENT_ENCODING.BROTLI)
content = await brotliDecompress(content, BROTLI_DECODING_OPTIONS);

charset.fromBOM(content);

Expand All @@ -47,14 +73,14 @@ export async function decodeContent (content: Buffer, encoding: string, charset:
export async function encodeContent (content: string, encoding: string, charset: Charset): Promise<Buffer> {
const encodedContent = charsetEncoder.encode(content, charset.get(), { addBOM: charset.isFromBOM() });

if (encoding === GZIP_CONTENT_ENCODING)
if (encoding === CONTENT_ENCODING.GZIP)
return gzip(encodedContent);

if (encoding === DEFLATE_CONTENT_ENCODING)
if (encoding === CONTENT_ENCODING.DEFLATE)
return deflate(encodedContent);

if (encoding === BROTLI_CONTENT_ENCODING)
return brotliCompress(encodedContent);
if (encoding === CONTENT_ENCODING.BROTLI)
return brotliCompress(encodedContent, BROTLI_DECODING_OPTIONS);

return encodedContent;
}
28 changes: 15 additions & 13 deletions src/utils/promisified-functions.ts
View file
@@ -1,20 +1,22 @@
import zlib from 'zlib';
import zlib, { BrotliOptions, InputType, ZlibOptions } from 'zlib';
import { promisify } from 'util';
import fs from 'fs';
import childProcess from 'child_process';

export const gzip: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer> = promisify(zlib.gzip);
export const deflate: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer> = promisify(zlib.deflate);
export const gunzip: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer> = promisify(zlib.gunzip);
export const inflate: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer> = promisify(zlib.inflate);
export const inflateRaw: (buf: zlib.InputType, options?: zlib.ZlibOptions) => Promise<Buffer> = promisify(zlib.inflateRaw);
export const readDir: (path: string) => Promise<string[]> = promisify(fs.readdir);
export const gzip: (buf: InputType, options?: ZlibOptions) => Promise<Buffer> = promisify(zlib.gzip);
export const deflate: (buf: InputType, options?: ZlibOptions) => Promise<Buffer> = promisify(zlib.deflate);
export const gunzip: (buf: InputType, options?: ZlibOptions) => Promise<Buffer> = promisify(zlib.gunzip);
export const inflate: (buf: InputType, options?: ZlibOptions) => Promise<Buffer> = promisify(zlib.inflate);
export const inflateRaw: (buf: InputType, options?: ZlibOptions) => Promise<Buffer> = promisify(zlib.inflateRaw);
export const brotliCompress: (buf: InputType, options?: BrotliOptions) => Promise<Buffer> = promisify(zlib.brotliCompress);
export const brotliDecompress: (buf: InputType, options?: BrotliOptions) => Promise<Buffer> = promisify(zlib.brotliDecompress);

export const readFile = promisify(fs.readFile);
export const stat = promisify(fs.stat);
export const access = promisify(fs.access);
export const makeDir = promisify(fs.mkdir);
export const writeFile = promisify(fs.writeFile);
export const fsObjectExists = (fsPath: string) => stat(fsPath).then(() => true, () => false);
export const readDir: (path: string) => Promise<string[]> = promisify(fs.readdir);
export const readFile = promisify(fs.readFile);
export const stat = promisify(fs.stat);
export const access = promisify(fs.access);
export const makeDir = promisify(fs.mkdir);
export const writeFile = promisify(fs.writeFile);
export const fsObjectExists = (fsPath: string) => stat(fsPath).then(() => true, () => false);

export const exec = promisify(childProcess.exec);
26 changes: 22 additions & 4 deletions test/server/encoding-test.js
View file
@@ -1,7 +1,7 @@
const expect = require('chai').expect;
const encodeContent = require('../../lib/processing/encoding').encodeContent;
const decodeContent = require('../../lib/processing/encoding').decodeContent;
const Charset = require('../../lib/processing/encoding/charset');
const { expect } = require('chai');
const { encodeContent, decodeContent } = require('../../lib/processing/encoding');
const Charset = require('../../lib/processing/encoding/charset');
const crypto = require('crypto');

describe('Content encoding', () => {
const src = Buffer.from('Answer to the Ultimate Question of Life, the Universe, and Everything.');
Expand Down Expand Up @@ -37,4 +37,22 @@ describe('Content encoding', () => {
expect(err).to.be.an('object');
});
});

it('Brotli decoding performance (GH-2743)', async () => {
const charset = new Charset();

charset.set('utf8', 2);

const content = crypto.randomBytes(10 * 1000 * 1000).toString('hex');

const start = Date.now();

const encoded = await encodeContent(content, 'br', charset);

await decodeContent(encoded, 'br', charset);

const executionTime = Date.now() - start;

expect(executionTime).below(5000);
});
});

0 comments on commit c2b0ab0

Please sign in to comment.