From 12614320d34478693ad3821e75f7b15da44c2230 Mon Sep 17 00:00:00 2001 From: Durran Jordan Date: Wed, 18 May 2022 23:19:43 +0200 Subject: [PATCH] feat(NODE-1837): add zstd compression option (#3237) --- .evergreen/config.in.yml | 21 +++ .evergreen/config.yml | 52 +++++++ .evergreen/generate_evergreen_tasks.js | 20 +++ .evergreen/run-tests.sh | 12 ++ package-lock.json | 145 ++++++++++++++++++ package.json | 1 + src/cmap/wire_protocol/compression.ts | 30 +++- src/deps.ts | 24 +++ test/tools/runner/hooks/configuration.js | 3 +- .../types/community/changes_from_36.test-d.ts | 2 +- .../cmap/wire_protocol/compression.test.ts | 56 +++++++ 11 files changed, 361 insertions(+), 5 deletions(-) create mode 100644 test/unit/cmap/wire_protocol/compression.test.ts diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml index de48ac71b2..86864e3739 100644 --- a/.evergreen/config.in.yml +++ b/.evergreen/config.in.yml @@ -101,6 +101,7 @@ functions: ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ LOAD_BALANCER=${LOAD_BALANCER} \ + COMPRESSOR=${COMPRESSOR} \ bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh # run-orchestration generates expansion file with the MONGODB_URI for the cluster - command: expansions.update @@ -239,6 +240,26 @@ functions: LOAD_BALANCER="${LOAD_BALANCER}" \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + "run-compression-tests": + - command: shell.exec + type: test + params: + working_dir: src + timeout_secs: 300 + script: | + ${PREPARE_SHELL} + + MONGODB_URI="${MONGODB_URI}" \ + AUTH=${AUTH} \ + SSL=${SSL} \ + MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ + NODE_VERSION=${NODE_VERSION} \ + TOPOLOGY="${TOPOLOGY}" \ + COMPRESSOR="${COMPRESSOR}" \ + SKIP_DEPS=${SKIP_DEPS|1} \ + NO_EXIT=${NO_EXIT|1} \ + bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + "run lint checks": - command: subprocess.exec type: test diff --git a/.evergreen/config.yml b/.evergreen/config.yml index d9ab425556..bbf80529f9 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -84,6 +84,7 @@ functions: ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \ REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \ LOAD_BALANCER=${LOAD_BALANCER} \ + COMPRESSOR=${COMPRESSOR} \ bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh - command: expansions.update params: @@ -214,6 +215,25 @@ functions: NO_EXIT=${NO_EXIT|1} \ LOAD_BALANCER="${LOAD_BALANCER}" \ bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh + run-compression-tests: + - command: shell.exec + type: test + params: + working_dir: src + timeout_secs: 300 + script: | + ${PREPARE_SHELL} + + MONGODB_URI="${MONGODB_URI}" \ + AUTH=${AUTH} \ + SSL=${SSL} \ + MONGODB_API_VERSION="${MONGODB_API_VERSION}" \ + NODE_VERSION=${NODE_VERSION} \ + TOPOLOGY="${TOPOLOGY}" \ + COMPRESSOR="${COMPRESSOR}" \ + SKIP_DEPS=${SKIP_DEPS|1} \ + NO_EXIT=${NO_EXIT|1} \ + bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh run lint checks: - command: subprocess.exec type: test @@ -1154,6 +1174,32 @@ tasks: - func: run socks5 tests vars: SSL: ssl + - name: test-zstd-compression + tags: + - latest + - zstd + commands: + - func: install dependencies + - func: bootstrap mongo-orchestration + vars: + VERSION: latest + TOPOLOGY: replica_set + AUTH: auth + COMPRESSOR: zstd + - func: run-compression-tests + - name: test-snappy-compression + tags: + - latest + - snappy + commands: + - func: install dependencies + - func: bootstrap mongo-orchestration + vars: + VERSION: latest + TOPOLOGY: replica_set + AUTH: auth + COMPRESSOR: snappy + - func: run-compression-tests - name: test-tls-support-latest tags: - tls-support @@ -1964,6 +2010,8 @@ buildvariants: - test-auth-ldap - test-socks5 - test-socks5-tls + - test-zstd-compression + - test-snappy-compression - test-tls-support-latest - test-tls-support-6.0 - test-tls-support-5.0 @@ -2019,6 +2067,8 @@ buildvariants: - test-auth-ldap - test-socks5 - test-socks5-tls + - test-zstd-compression + - test-snappy-compression - test-tls-support-latest - test-tls-support-6.0 - test-tls-support-5.0 @@ -2070,6 +2120,8 @@ buildvariants: - test-atlas-data-lake - test-socks5 - test-socks5-tls + - test-zstd-compression + - test-snappy-compression - test-tls-support-latest - test-tls-support-6.0 - test-tls-support-5.0 diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index c819c38f9e..6d6a5e8daa 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -189,6 +189,26 @@ TASKS.push( ] ); +['zstd', 'snappy'].forEach(compressor => { + TASKS.push({ + name: `test-${compressor}-compression`, + tags: ['latest', compressor], + commands: [ + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + VERSION: 'latest', + TOPOLOGY: 'replica_set', + AUTH: 'auth', + COMPRESSOR: compressor + } + }, + { func: 'run-compression-tests' } + ] + }); +}); + TLS_VERSIONS.forEach(VERSION => { TASKS.push({ name: `test-tls-support-${VERSION}`, diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh index cae34a6769..94f92d5704 100755 --- a/.evergreen/run-tests.sh +++ b/.evergreen/run-tests.sh @@ -15,6 +15,7 @@ set -o errexit # Exit the script with error if any of the commands fail AUTH=${AUTH:-noauth} MONGODB_URI=${MONGODB_URI:-} TEST_NPM_SCRIPT=${TEST_NPM_SCRIPT:-check:integration-coverage} +COMPRESSOR=${COMPRESSOR:-} if [[ -z "${NO_EXIT}" ]]; then TEST_NPM_SCRIPT="$TEST_NPM_SCRIPT -- --exit" fi @@ -35,6 +36,14 @@ else source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" fi +if [ "$COMPRESSOR" != "" ]; then + if [[ "$MONGODB_URI" == *"?"* ]]; then + export MONGODB_URI="${MONGODB_URI}&compressors=${COMPRESSOR}" + else + export MONGODB_URI="${MONGODB_URI}/?compressors=${COMPRESSOR}" + fi +fi + # only run FLE tets on hosts we explicitly choose to test on if [[ -z "${CLIENT_ENCRYPTION}" ]]; then unset AWS_ACCESS_KEY_ID; @@ -48,6 +57,8 @@ else fi npm install mongodb-client-encryption@">=2.2.0-alpha.0" +npm install @mongodb-js/zstd +npm install snappy export AUTH=$AUTH export SINGLE_MONGOS_LB_URI=${SINGLE_MONGOS_LB_URI} @@ -56,5 +67,6 @@ export MONGODB_API_VERSION=${MONGODB_API_VERSION} export MONGODB_URI=${MONGODB_URI} export LOAD_BALANCER=${LOAD_BALANCER} export TEST_CSFLE=${TEST_CSFLE} +export COMPRESSOR=${COMPRESSOR} # Do not add quotes, due to the way NO_EXIT is handled npm run ${TEST_NPM_SCRIPT} diff --git a/package-lock.json b/package-lock.json index 18702c4830..08400b8161 100644 --- a/package-lock.json +++ b/package-lock.json @@ -19,6 +19,7 @@ "@istanbuljs/nyc-config-typescript": "^1.0.2", "@microsoft/api-extractor": "^7.20.0", "@microsoft/tsdoc-config": "^0.15.2", + "@mongodb-js/zstd": "^1.0.0", "@types/chai": "^4.3.0", "@types/chai-subset": "^1.3.3", "@types/express": "^4.17.13", @@ -792,6 +793,102 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/@mongodb-js/zstd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.0.0.tgz", + "integrity": "sha512-vApdTgi6gxRjgKpKm8z5wtZ5dGNTfh4AewomjMhHanDvCOkv3HQHjJhoodtecdqZnaJm3WcvTYgcL53rQAsOaA==", + "dev": true, + "engines": { + "node": ">= 10" + }, + "optionalDependencies": { + "@mongodb-js/zstd-darwin-arm64": "1.0.0", + "@mongodb-js/zstd-darwin-x64": "1.0.0", + "@mongodb-js/zstd-linux-arm64-gnu": "1.0.0", + "@mongodb-js/zstd-linux-x64-gnu": "1.0.0", + "@mongodb-js/zstd-win32-x64-msvc": "1.0.0" + } + }, + "node_modules/@mongodb-js/zstd-darwin-arm64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.0.0.tgz", + "integrity": "sha512-WRVWJ5BZOcxmbHRPFhUPAIAb8BVLDvH1dJzL3+Wl+xPDevu6eoFKZ6ZPsyyZe4aBFnVgS4bqlpwdQ4ShzrY+HQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-darwin-x64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.0.0.tgz", + "integrity": "sha512-7cInPU8m3by4EawyaDe3GD0WytrTeS9+B3JX2e+XeDChf9RXzFs+zXmVu51lIq8aqS2J+LwYoBPV/xxPdxTS1g==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "darwin" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-arm64-gnu": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.0.0.tgz", + "integrity": "sha512-qGL8mp2UyJNqGObgnh3NBzAxScVSQxn7bZsLrkSWy1xTytXgAgPmmnmIPhIH5q7Aa/sk797bkIprO8G0bRrNTQ==", + "cpu": [ + "arm64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-linux-x64-gnu": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.0.0.tgz", + "integrity": "sha512-5oBwpDjA3pY0XMo6uqr59ZOJjdkY4Yay6szzIRBzk//z1XpWjPs2Lf2zfRlnlzH7Q3EZEoHqFkk+kH9ZvHBC4Q==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "linux" + ], + "engines": { + "node": ">= 10" + } + }, + "node_modules/@mongodb-js/zstd-win32-x64-msvc": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.0.0.tgz", + "integrity": "sha512-c/7D7l0pw9yIsCmuUDxRaJxT9cmxn3A9u1GeslHuz/DRSMVHY6Sb5srMJlBviUF1Q7YGK5teM03JoHso4+vLlw==", + "cpu": [ + "x64" + ], + "dev": true, + "optional": true, + "os": [ + "win32" + ], + "engines": { + "node": ">= 10" + } + }, "node_modules/@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", @@ -8647,6 +8744,54 @@ } } }, + "@mongodb-js/zstd": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd/-/zstd-1.0.0.tgz", + "integrity": "sha512-vApdTgi6gxRjgKpKm8z5wtZ5dGNTfh4AewomjMhHanDvCOkv3HQHjJhoodtecdqZnaJm3WcvTYgcL53rQAsOaA==", + "dev": true, + "requires": { + "@mongodb-js/zstd-darwin-arm64": "1.0.0", + "@mongodb-js/zstd-darwin-x64": "1.0.0", + "@mongodb-js/zstd-linux-arm64-gnu": "1.0.0", + "@mongodb-js/zstd-linux-x64-gnu": "1.0.0", + "@mongodb-js/zstd-win32-x64-msvc": "1.0.0" + } + }, + "@mongodb-js/zstd-darwin-arm64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-arm64/-/zstd-darwin-arm64-1.0.0.tgz", + "integrity": "sha512-WRVWJ5BZOcxmbHRPFhUPAIAb8BVLDvH1dJzL3+Wl+xPDevu6eoFKZ6ZPsyyZe4aBFnVgS4bqlpwdQ4ShzrY+HQ==", + "dev": true, + "optional": true + }, + "@mongodb-js/zstd-darwin-x64": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-darwin-x64/-/zstd-darwin-x64-1.0.0.tgz", + "integrity": "sha512-7cInPU8m3by4EawyaDe3GD0WytrTeS9+B3JX2e+XeDChf9RXzFs+zXmVu51lIq8aqS2J+LwYoBPV/xxPdxTS1g==", + "dev": true, + "optional": true + }, + "@mongodb-js/zstd-linux-arm64-gnu": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-arm64-gnu/-/zstd-linux-arm64-gnu-1.0.0.tgz", + "integrity": "sha512-qGL8mp2UyJNqGObgnh3NBzAxScVSQxn7bZsLrkSWy1xTytXgAgPmmnmIPhIH5q7Aa/sk797bkIprO8G0bRrNTQ==", + "dev": true, + "optional": true + }, + "@mongodb-js/zstd-linux-x64-gnu": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-linux-x64-gnu/-/zstd-linux-x64-gnu-1.0.0.tgz", + "integrity": "sha512-5oBwpDjA3pY0XMo6uqr59ZOJjdkY4Yay6szzIRBzk//z1XpWjPs2Lf2zfRlnlzH7Q3EZEoHqFkk+kH9ZvHBC4Q==", + "dev": true, + "optional": true + }, + "@mongodb-js/zstd-win32-x64-msvc": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@mongodb-js/zstd-win32-x64-msvc/-/zstd-win32-x64-msvc-1.0.0.tgz", + "integrity": "sha512-c/7D7l0pw9yIsCmuUDxRaJxT9cmxn3A9u1GeslHuz/DRSMVHY6Sb5srMJlBviUF1Q7YGK5teM03JoHso4+vLlw==", + "dev": true, + "optional": true + }, "@nodelib/fs.scandir": { "version": "2.1.5", "resolved": "https://registry.npmjs.org/@nodelib/fs.scandir/-/fs.scandir-2.1.5.tgz", diff --git a/package.json b/package.json index 3b6a9bc8c7..3666aa34f7 100644 --- a/package.json +++ b/package.json @@ -35,6 +35,7 @@ "@istanbuljs/nyc-config-typescript": "^1.0.2", "@microsoft/api-extractor": "^7.20.0", "@microsoft/tsdoc-config": "^0.15.2", + "@mongodb-js/zstd": "^1.0.0", "@types/chai": "^4.3.0", "@types/chai-subset": "^1.3.3", "@types/express": "^4.17.13", diff --git a/src/cmap/wire_protocol/compression.ts b/src/cmap/wire_protocol/compression.ts index 556f7412f3..9f98051fcb 100644 --- a/src/cmap/wire_protocol/compression.ts +++ b/src/cmap/wire_protocol/compression.ts @@ -1,7 +1,7 @@ import * as zlib from 'zlib'; import { LEGACY_HELLO_COMMAND } from '../../constants'; -import { PKG_VERSION, Snappy } from '../../deps'; +import { PKG_VERSION, Snappy, ZStandard } from '../../deps'; import { MongoDecompressionError, MongoInvalidArgumentError } from '../../error'; import type { Callback } from '../../utils'; import type { OperationDescription } from '../message_stream'; @@ -10,7 +10,8 @@ import type { OperationDescription } from '../message_stream'; export const Compressor = Object.freeze({ none: 0, snappy: 1, - zlib: 2 + zlib: 2, + zstd: 3 } as const); /** @public */ @@ -32,6 +33,9 @@ export const uncompressibleCommands = new Set([ 'copydb' ]); +const MAX_COMPRESSOR_ID = 3; +const ZSTD_COMPRESSION_LEVEL = 3; + // Facilitate compressing a message using an agreed compressor export function compress( self: { options: OperationDescription & zlib.ZlibOptions }, @@ -61,6 +65,15 @@ export function compress( } zlib.deflate(dataToBeCompressed, zlibOptions, callback as zlib.CompressCallback); break; + case 'zstd': + if ('kModuleError' in ZStandard) { + return callback(ZStandard['kModuleError']); + } + ZStandard.compress(dataToBeCompressed, ZSTD_COMPRESSION_LEVEL).then( + buffer => callback(undefined, buffer), + error => callback(error) + ); + break; default: throw new MongoInvalidArgumentError( `Unknown compressor ${self.options.agreedCompressor} failed to compress` @@ -74,7 +87,7 @@ export function decompress( compressedData: Buffer, callback: Callback ): void { - if (compressorID < 0 || compressorID > Math.max(2)) { + if (compressorID < 0 || compressorID > MAX_COMPRESSOR_ID) { throw new MongoDecompressionError( `Server sent message compressed using an unsupported compressor. (Received compressor ID ${compressorID})` ); @@ -95,6 +108,17 @@ export function decompress( } break; } + case Compressor.zstd: { + if ('kModuleError' in ZStandard) { + return callback(ZStandard['kModuleError']); + } + + ZStandard.decompress(compressedData).then( + buffer => callback(undefined, buffer), + error => callback(error) + ); + break; + } case Compressor.zlib: zlib.inflate(compressedData, callback as zlib.CompressCallback); break; diff --git a/src/deps.ts b/src/deps.ts index 36f4e20e4a..4c6ce98bef 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -44,6 +44,30 @@ export interface KerberosClient { unwrap: (challenge: string, callback?: Callback) => Promise | void; } +type ZStandardLib = { + /** + * Compress using zstd. + * @param buf - Buffer to be compressed. + */ + compress(buf: Buffer, level?: number): Promise; + + /** + * Decompress using zstd. + */ + decompress(buf: Buffer): Promise; +}; + +export let ZStandard: ZStandardLib | { kModuleError: MongoMissingDependencyError } = + makeErrorModule( + new MongoMissingDependencyError( + 'Optional module `@mongodb-js/zstd` not found. Please install it to enable zstd compression' + ) + ); + +try { + ZStandard = require('@mongodb-js/zstd'); +} catch {} // eslint-disable-line + type SnappyLib = { [PKG_VERSION]: { major: number; minor: number; patch: number }; diff --git a/test/tools/runner/hooks/configuration.js b/test/tools/runner/hooks/configuration.js index f7ad51d89d..107068ae7c 100644 --- a/test/tools/runner/hooks/configuration.js +++ b/test/tools/runner/hooks/configuration.js @@ -140,7 +140,8 @@ const testConfigBeforeHook = async function () { kerberos: process.env.KRB5_PRINCIPAL != null, ldap: MONGODB_URI.includes('authMechanism=PLAIN'), ocsp: process.env.OCSP_TLS_SHOULD_SUCCEED != null && process.env.CA_FILE != null, - socks5: MONGODB_URI.includes('proxyHost=') + socks5: MONGODB_URI.includes('proxyHost='), + compressor: process.env.COMPRESSOR }; console.error(inspect(currentEnv, { colors: true })); diff --git a/test/types/community/changes_from_36.test-d.ts b/test/types/community/changes_from_36.test-d.ts index 74e9c14a1b..d43c313097 100644 --- a/test/types/community/changes_from_36.test-d.ts +++ b/test/types/community/changes_from_36.test-d.ts @@ -70,7 +70,7 @@ expectAssignable<((host: string, cert: PeerCertificate) => Error | undefined) | // compression options have simpler specification: // old way: {compression: { compressors: ['zlib', 'snappy'] }} expectType>(false); -expectType<('none' | 'snappy' | 'zlib')[] | string | undefined>(options.compressors); +expectType<('none' | 'snappy' | 'zlib' | 'zstd')[] | string | undefined>(options.compressors); // Removed cursor API const cursor = new MongoClient('').db().aggregate(); diff --git a/test/unit/cmap/wire_protocol/compression.test.ts b/test/unit/cmap/wire_protocol/compression.test.ts new file mode 100644 index 0000000000..b5f82dafed --- /dev/null +++ b/test/unit/cmap/wire_protocol/compression.test.ts @@ -0,0 +1,56 @@ +import { expect } from 'chai'; + +import { compress, Compressor, decompress } from '../../../../src/cmap/wire_protocol/compression'; + +describe('compression', function () { + describe('.compress()', function () { + context('when the compression library is zstd', function () { + const buffer = Buffer.from('test'); + + context('when a level is not provided', function () { + const options = { options: { agreedCompressor: 'zstd' } }; + + it('compresses the data', function (done) { + compress(options, buffer, (error, data) => { + expect(error).to.not.exist; + const zstdMagicNumber = data.reverse().toString('hex').substring(16, 26); + // Zstd magic number first set of bytes is is 0xFD2FB528 + expect(zstdMagicNumber).to.equal('00fd2fb528'); + done(); + }); + }); + }); + + context('when a level is provided', function () { + const options = { options: { agreedCompressor: 'zstd', zstdCompressionLevel: 2 } }; + + it('compresses the data', function (done) { + compress(options, buffer, (error, data) => { + expect(error).to.not.exist; + const zstdMagicNumber = data.reverse().toString('hex').substring(16, 26); + // Zstd magic number first set of bytes is is 0xFD2FB528 + expect(zstdMagicNumber).to.equal('00fd2fb528'); + done(); + }); + }); + }); + }); + }); + + describe('.decompress()', function () { + context('when the compression library is zstd', function () { + const buffer = Buffer.from('test'); + const options = { options: { agreedCompressor: 'zstd' } }; + + it('decompresses the data', function (done) { + compress(options, buffer, (error, data) => { + expect(error).to.not.exist; + decompress(Compressor.zstd, data, (err, decompressed) => { + expect(decompressed).to.deep.equal(buffer); + done(); + }); + }); + }); + }); + }); +});