feat(NODE-1837): add zstd compression option (#3237)

mongodb · May 18, 2022 · 1261432 · 1261432
1 parent 981465c
commit 1261432
Show file tree

Hide file tree

Showing 11 changed files with 361 additions and 5 deletions.
diff --git a/.evergreen/config.in.yml b/.evergreen/config.in.yml
@@ -101,6 +101,7 @@ functions:
           ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
           REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
           LOAD_BALANCER=${LOAD_BALANCER} \
+          COMPRESSOR=${COMPRESSOR} \
           bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     # run-orchestration generates expansion file with the MONGODB_URI for the cluster
     - command: expansions.update
@@ -239,6 +240,26 @@ functions:
           LOAD_BALANCER="${LOAD_BALANCER}" \
             bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
 
+  "run-compression-tests":
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        timeout_secs: 300
+        script: |
+          ${PREPARE_SHELL}
+
+          MONGODB_URI="${MONGODB_URI}" \
+          AUTH=${AUTH} \
+          SSL=${SSL} \
+          MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
+          NODE_VERSION=${NODE_VERSION} \
+          TOPOLOGY="${TOPOLOGY}" \
+          COMPRESSOR="${COMPRESSOR}" \
+          SKIP_DEPS=${SKIP_DEPS|1} \
+          NO_EXIT=${NO_EXIT|1} \
+            bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
+
   "run lint checks":
     - command: subprocess.exec
       type: test

diff --git a/.evergreen/config.yml b/.evergreen/config.yml
@@ -84,6 +84,7 @@ functions:
           ORCHESTRATION_FILE=${ORCHESTRATION_FILE} \
           REQUIRE_API_VERSION=${REQUIRE_API_VERSION} \
           LOAD_BALANCER=${LOAD_BALANCER} \
+          COMPRESSOR=${COMPRESSOR} \
           bash ${DRIVERS_TOOLS}/.evergreen/run-orchestration.sh
     - command: expansions.update
       params:
@@ -214,6 +215,25 @@ functions:
           NO_EXIT=${NO_EXIT|1} \
           LOAD_BALANCER="${LOAD_BALANCER}" \
             bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
+  run-compression-tests:
+    - command: shell.exec
+      type: test
+      params:
+        working_dir: src
+        timeout_secs: 300
+        script: |
+          ${PREPARE_SHELL}
+
+          MONGODB_URI="${MONGODB_URI}" \
+          AUTH=${AUTH} \
+          SSL=${SSL} \
+          MONGODB_API_VERSION="${MONGODB_API_VERSION}" \
+          NODE_VERSION=${NODE_VERSION} \
+          TOPOLOGY="${TOPOLOGY}" \
+          COMPRESSOR="${COMPRESSOR}" \
+          SKIP_DEPS=${SKIP_DEPS|1} \
+          NO_EXIT=${NO_EXIT|1} \
+            bash ${PROJECT_DIRECTORY}/.evergreen/run-tests.sh
   run lint checks:
     - command: subprocess.exec
       type: test
@@ -1154,6 +1174,32 @@ tasks:
       - func: run socks5 tests
         vars:
           SSL: ssl
+  - name: test-zstd-compression
+    tags:
+      - latest
+      - zstd
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: latest
+          TOPOLOGY: replica_set
+          AUTH: auth
+          COMPRESSOR: zstd
+      - func: run-compression-tests
+  - name: test-snappy-compression
+    tags:
+      - latest
+      - snappy
+    commands:
+      - func: install dependencies
+      - func: bootstrap mongo-orchestration
+        vars:
+          VERSION: latest
+          TOPOLOGY: replica_set
+          AUTH: auth
+          COMPRESSOR: snappy
+      - func: run-compression-tests
   - name: test-tls-support-latest
     tags:
       - tls-support
@@ -1964,6 +2010,8 @@ buildvariants:
       - test-auth-ldap
       - test-socks5
       - test-socks5-tls
+      - test-zstd-compression
+      - test-snappy-compression
       - test-tls-support-latest
       - test-tls-support-6.0
       - test-tls-support-5.0
@@ -2019,6 +2067,8 @@ buildvariants:
       - test-auth-ldap
       - test-socks5
       - test-socks5-tls
+      - test-zstd-compression
+      - test-snappy-compression
       - test-tls-support-latest
       - test-tls-support-6.0
       - test-tls-support-5.0
@@ -2070,6 +2120,8 @@ buildvariants:
       - test-atlas-data-lake
       - test-socks5
       - test-socks5-tls
+      - test-zstd-compression
+      - test-snappy-compression
       - test-tls-support-latest
       - test-tls-support-6.0
       - test-tls-support-5.0

diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js
@@ -189,6 +189,26 @@ TASKS.push(
   ]
 );
 
+['zstd', 'snappy'].forEach(compressor => {
+  TASKS.push({
+    name: `test-${compressor}-compression`,
+    tags: ['latest', compressor],
+    commands: [
+      { func: 'install dependencies' },
+      {
+        func: 'bootstrap mongo-orchestration',
+        vars: {
+          VERSION: 'latest',
+          TOPOLOGY: 'replica_set',
+          AUTH: 'auth',
+          COMPRESSOR: compressor
+        }
+      },
+      { func: 'run-compression-tests' }
+    ]
+  });
+});
+
 TLS_VERSIONS.forEach(VERSION => {
   TASKS.push({
     name: `test-tls-support-${VERSION}`,

diff --git a/.evergreen/run-tests.sh b/.evergreen/run-tests.sh
@@ -15,6 +15,7 @@ set -o errexit  # Exit the script with error if any of the commands fail
 AUTH=${AUTH:-noauth}
 MONGODB_URI=${MONGODB_URI:-}
 TEST_NPM_SCRIPT=${TEST_NPM_SCRIPT:-check:integration-coverage}
+COMPRESSOR=${COMPRESSOR:-}
 if [[ -z "${NO_EXIT}" ]]; then
   TEST_NPM_SCRIPT="$TEST_NPM_SCRIPT -- --exit"
 fi
@@ -35,6 +36,14 @@ else
   source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh"
 fi
 
+if [ "$COMPRESSOR" != "" ]; then
+  if [[ "$MONGODB_URI" == *"?"* ]]; then
+    export MONGODB_URI="${MONGODB_URI}&compressors=${COMPRESSOR}"
+  else
+    export MONGODB_URI="${MONGODB_URI}/?compressors=${COMPRESSOR}"
+  fi
+fi
+
 # only run FLE tets on hosts we explicitly choose to test on
 if [[ -z "${CLIENT_ENCRYPTION}" ]]; then
   unset AWS_ACCESS_KEY_ID;
@@ -48,6 +57,8 @@ else
 fi
 
 npm install mongodb-client-encryption@">=2.2.0-alpha.0"
+npm install @mongodb-js/zstd
+npm install snappy
 
 export AUTH=$AUTH
 export SINGLE_MONGOS_LB_URI=${SINGLE_MONGOS_LB_URI}
@@ -56,5 +67,6 @@ export MONGODB_API_VERSION=${MONGODB_API_VERSION}
 export MONGODB_URI=${MONGODB_URI}
 export LOAD_BALANCER=${LOAD_BALANCER}
 export TEST_CSFLE=${TEST_CSFLE}
+export COMPRESSOR=${COMPRESSOR}
 # Do not add quotes, due to the way NO_EXIT is handled
 npm run ${TEST_NPM_SCRIPT}
diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -35,6 +35,7 @@
     "@istanbuljs/nyc-config-typescript": "^1.0.2",
     "@microsoft/api-extractor": "^7.20.0",
     "@microsoft/tsdoc-config": "^0.15.2",
+    "@mongodb-js/zstd": "^1.0.0",
     "@types/chai": "^4.3.0",
     "@types/chai-subset": "^1.3.3",
     "@types/express": "^4.17.13",