feat(NODE-3777): add csfle kmip support (#3070)

.evergreen/config.yml.in

@@ -119,6 +119,16 @@ functions:
           ${PREPARE_SHELL}
           DRIVERS_TOOLS="${DRIVERS_TOOLS}" bash ${DRIVERS_TOOLS}/.evergreen/atlas_data_lake/run-mongohouse-local.sh
 
+  "bootstrap kms servers":
+    - command: subprocess.exec
+      params:
+        background: true
+        working_dir: src
+        binary: bash
+        args:
+          - .evergreen/run-kms-servers.sh
+        env:
+          DRIVERS_TOOLS: ${DRIVERS_TOOLS}
   "run tests":
     - command: shell.exec
       type: test
@@ -133,6 +143,8 @@ functions:
             export AWS_ACCESS_KEY_ID='${AWS_ACCESS_KEY_ID}'
             export AWS_SECRET_ACCESS_KEY='${AWS_SECRET_ACCESS_KEY}'
             export AWS_DEFAULT_REGION='us-east-1'
+            export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
+            export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
           EOT
           fi
     - command: shell.exec
@@ -193,8 +205,7 @@ functions:
     - command: shell.exec
       params:
         script: |
-          DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} \
-            bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start
+          MONGODB_URI=${MONGODB_URI} bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh start
     - command: expansions.update
       params:
         file: lb-expansion.yml
@@ -203,7 +214,6 @@ functions:
     - command: shell.exec
       params:
         script: |
-          DRIVERS_TOOLS=${DRIVERS_TOOLS} MONGODB_URI=${MONGODB_URI} \
           bash ${DRIVERS_TOOLS}/.evergreen/run-load-balancer.sh stop
 
   "run-lb-tests":

.evergreen/generate_evergreen_tasks.js

@@ -62,6 +62,7 @@ function makeTask({ mongoVersion, topology, tags = [], auth = 'auth' }) {
           AUTH: auth
         }
       },
+      { func: 'bootstrap kms servers' },
       { func: 'run tests' }
     ]
   };
@@ -96,6 +97,7 @@ BASE_TASKS.push({
         AUTH: 'auth'
       }
     },
+    { func: 'bootstrap kms servers' },
     {
       func: 'run tests',
       vars: {
@@ -163,6 +165,7 @@ TASKS.push(
             TOPOLOGY: 'replica_set'
           }
         },
+        { func: 'bootstrap kms servers' },
         { func: 'run socks5 tests' }
       ]
     },
@@ -637,7 +640,6 @@ BUILD_VARIANTS.push({
 });
 
 const oneOffFuncs = [
-  { func: 'run custom csfle tests' },
   { func: 'run custom snappy tests' },
   {
     func: 'run bson-ext test',
@@ -669,6 +671,28 @@ const oneOffFuncAsTasks = oneOffFuncs.map(oneOffFunc => ({
   ]
 }));
 
+oneOffFuncAsTasks.push({
+  name: 'run-custom-csfle-tests',
+  tags: ['run-custom-dependency-tests'],
+  commands: [
+    {
+      func: 'install dependencies',
+      vars: {
+        NODE_LTS_NAME: LOWEST_LTS
+      }
+    },
+    {
+      func: 'bootstrap mongo-orchestration',
+      vars: {
+        VERSION: 'latest',
+        TOPOLOGY: 'server'
+      }
+    },
+    { func: 'bootstrap kms servers' },
+    { func: 'run custom csfle tests' }
+  ]
+});
+
 // TODO NODE-3897 - generate combined coverage report
 const coverageTask = {
   name: 'download and merge coverage'.split(' ').join('-'),

.evergreen/run-bson-ext-test.sh

@@ -23,6 +23,7 @@ fi
 # run tests
 echo "Running $AUTH tests over $SSL, connecting to $MONGODB_URI"
 
+npm install mongodb-client-encryption@">=2.0.0-beta.3"
 npm install bson-ext
 
 export MONGODB_API_VERSION=${MONGODB_API_VERSION}

.evergreen/run-custom-csfle-tests.sh

@@ -63,6 +63,8 @@ popd # ../csfle-deps-tmp
 cp -R ../csfle-deps-tmp/libmongocrypt/bindings/node node_modules/mongodb-client-encryption
 
 export MONGODB_URI=${MONGODB_URI}
+export KMIP_TLS_CA_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/ca.pem"
+export KMIP_TLS_CERT_FILE="${DRIVERS_TOOLS}/.evergreen/x509gen/client.pem"
 set +o errexit # We want to run both test suites even if the first fails
 npm run check:csfle
 DRIVER_CSFLE_TEST_RESULT=$?
@@ -103,8 +105,3 @@ if [ $DRIVER_CSFLE_TEST_RESULT -ne 0 ]; then
   echo "Driver tests failed, look above for results"
   exit 1
 fi
-
-echo "Test legacy version of FLE bindings"
-rm -rf node_modules/mongodb-client-encryption
-npm install mongodb-client-encryption@"^1.2.7"
-npm run check:csfle

.evergreen/run-kms-servers.sh

@@ -0,0 +1,7 @@
+cd ${DRIVERS_TOOLS}/.evergreen/csfle
+. ./activate_venv.sh
+# by default it always runs on port 5698
+./kmstlsvenv/bin/python3 -u kms_kmip_server.py &
+./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/expired.pem --port 8000 &
+./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/wrong-host.pem --port 8001 &
+./kmstlsvenv/bin/python3 -u kms_http_server.py --ca_file ../x509gen/ca.pem --cert_file ../x509gen/server.pem --port 8002 --require_client_cert &

.evergreen/run-tests.sh

@@ -41,13 +41,13 @@ if [[ -z "${CLIENT_ENCRYPTION}" ]]; then
   unset AWS_ACCESS_KEY_ID;
   unset AWS_SECRET_ACCESS_KEY;
 else
-  npm install mongodb-client-encryption@">=2.0.0-beta.0"
   pip install --upgrade boto3
-
   # Get access to the AWS temporary credentials:
   echo "adding temporary AWS credentials to environment"
   # CSFLE_AWS_TEMP_ACCESS_KEY_ID, CSFLE_AWS_TEMP_SECRET_ACCESS_KEY, CSFLE_AWS_TEMP_SESSION_TOKEN
   . $DRIVERS_TOOLS/.evergreen/csfle/set-temp-creds.sh
 fi
 
+npm install mongodb-client-encryption@">=2.0.0-beta.3"
+
 AUTH=$AUTH SINGLE_MONGOS_LB_URI=${SINGLE_MONGOS_LB_URI} MULTI_MONGOS_LB_URI=${MULTI_MONGOS_LB_URI} MONGODB_API_VERSION=${MONGODB_API_VERSION} MONGODB_UNIFIED_TOPOLOGY=${UNIFIED} MONGODB_URI=${MONGODB_URI} LOAD_BALANCER=${LOAD_BALANCER} npm run ${TEST_NPM_SCRIPT}

src/deps.ts

@@ -174,6 +174,28 @@ export const AutoEncryptionLoggerLevel = Object.freeze({
 export type AutoEncryptionLoggerLevel =
   typeof AutoEncryptionLoggerLevel[keyof typeof AutoEncryptionLoggerLevel];
 
+/** @public */
+export interface AutoEncryptionTlsOptions {
+  /**
+   * Specifies the location of a local .pem file that contains
+   * either the client's TLS/SSL certificate and key or only the
+   * client's TLS/SSL key when tlsCertificateFile is used to
+   * provide the certificate.
+   */
+  tlsCertificateKeyFile?: string;
+  /**
+   * Specifies the password to de-crypt the tlsCertificateKeyFile.
+   */
+  tlsCertificateKeyFilePassword?: string;
+  /**
+   * Specifies the location of a local .pem file that contains the
+   * root certificate chain from the Certificate Authority.
+   * This file is used to validate the certificate presented by the
+   * KMS provider.
+   */
+  tlsCAFile?: string;
+}
+
 /** @public */
 export interface AutoEncryptionOptions {
   /** @internal */
@@ -234,6 +256,17 @@ export interface AutoEncryptionOptions {
        */
       endpoint?: string | undefined;
     };
+    /**
+     * Configuration options for using 'kmip' as your KMS provider
+     */
+    kmip?: {
+      /**
+       * The output endpoint string.
+       * The endpoint consists of a hostname and port separated by a colon.
+       * E.g. "example.com:123". A port is always present.
+       */
+      endpoint?: string;
+    };
   };
   /**
    * A map of namespaces to a local JSON schema for encryption
@@ -264,6 +297,14 @@ export interface AutoEncryptionOptions {
     mongocryptdSpawnArgs?: string[];
   };
   proxyOptions?: ProxyOptions;
+  /** The TLS options to use connecting to the KMS provider */
+  tlsOptions?: {
+    aws?: AutoEncryptionTlsOptions;
+    local?: AutoEncryptionTlsOptions;
+    azure?: AutoEncryptionTlsOptions;
+    gcp?: AutoEncryptionTlsOptions;
+    kmip?: AutoEncryptionTlsOptions;
+  };
 }
 
 /** @public */

src/index.ts

@@ -229,7 +229,7 @@ export type {
 export type { InternalAbstractCursorOptions } from './cursor/abstract_cursor';
 export type { AggregationCursorOptions } from './cursor/aggregation_cursor';
 export type { DbOptions, DbPrivate } from './db';
-export type { AutoEncrypter, AutoEncryptionOptions } from './deps';
+export type { AutoEncrypter, AutoEncryptionOptions, AutoEncryptionTlsOptions } from './deps';
 export type { Encrypter, EncrypterOptions } from './encrypter';
 export type { AnyError, ErrorDescription, MongoNetworkErrorOptions } from './error';
 export type { Explain, ExplainOptions, ExplainVerbosityLike } from './explain';

test/integration/client-side-encryption/client_side_encryption.corpus.spec.test.js

@@ -29,6 +29,9 @@ describe('Client Side Encryption Corpus', function () {
       'base64'
     )
   };
+  kmsProviders.kmip = {
+    endpoint: 'localhost:5698'
+  };
 
   // TODO: build this into EJSON
   // TODO: make a custom chai assertion for this
@@ -55,6 +58,7 @@ describe('Client Side Encryption Corpus', function () {
   const corpusKeyLocal = loadCorpusData('corpus-key-local.json');
   const corpusKeyAws = loadCorpusData('corpus-key-aws.json');
   const corpusKeyAzure = loadCorpusData('corpus-key-azure.json');
+  const corpusKeyKmip = loadCorpusData('corpus-key-kmip.json');
   const corpusKeyGcp = loadCorpusData('corpus-key-gcp.json');
   const corpusAll = filterImportedObject(loadCorpusData('corpus.json'));
   const corpusEncryptedExpectedAll = filterImportedObject(loadCorpusData('corpus-encrypted.json'));
@@ -74,20 +78,23 @@ describe('Client Side Encryption Corpus', function () {
     ['local', corpusKeyLocal._id],
     ['aws', corpusKeyAws._id],
     ['azure', corpusKeyAzure._id],
-    ['gcp', corpusKeyGcp._id]
+    ['gcp', corpusKeyGcp._id],
+    ['kmip', corpusKeyKmip._id]
   ]);
   const keyAltNameMap = new Map([
     ['local', 'local'],
     ['aws', 'aws'],
     ['azure', 'azure'],
-    ['gcp', 'gcp']
+    ['gcp', 'gcp'],
+    ['kmip', 'kmip']
   ]);
   const copyOverValues = new Set([
     '_id',
     'altname_aws',
     'altname_local',
     'altname_azure',
-    'altname_gcp'
+    'altname_gcp',
+    'altname_kmip'
   ]);
 
   let client;
@@ -153,7 +160,13 @@ describe('Client Side Encryption Corpus', function () {
           .catch(() => {})
           .then(() => keyDb.collection(keyVaultCollName))
           .then(keyColl =>
-            keyColl.insertMany([corpusKeyLocal, corpusKeyAws, corpusKeyAzure, corpusKeyGcp])
+            keyColl.insertMany([
+              corpusKeyLocal,
+              corpusKeyAws,
+              corpusKeyAzure,
+              corpusKeyGcp,
+              corpusKeyKmip
+            ])
           );
       });
   });
@@ -195,9 +208,16 @@ describe('Client Side Encryption Corpus', function () {
           //    .. code:: javascript
           //       Mng0NCt4ZHVUYUJCa1kxNkVyNUR1QURhZ2h2UzR2d2RrZzh0cFBwM3R6NmdWMDFBMUN3YkQ5aXRRMkhGRGdQV09wOGVNYUMxT2k3NjZKelhaQmRCZGJkTXVyZG9uSjFk
           //    Configure both objects with ``keyVaultNamespace`` set to ``keyvault.datakeys``.
+          const tlsOptions = {
+            kmip: {
+              tlsCAFile: process.env.KMIP_TLS_CA_FILE,
+              tlsCertificateKeyFile: process.env.KMIP_TLS_CERT_FILE
+            }
+          };
           const autoEncryption = {
             keyVaultNamespace,
-            kmsProviders
+            kmsProviders,
+            tlsOptions
           };
           if (useClientSideSchema) {
             autoEncryption.schemaMap = {
@@ -210,7 +230,8 @@ describe('Client Side Encryption Corpus', function () {
             clientEncryption = new mongodbClientEncryption.ClientEncryption(client, {
               bson: BSON,
               keyVaultNamespace,
-              kmsProviders
+              kmsProviders,
+              tlsOptions
             });
           });
         });

test/integration/client-side-encryption/client_side_encryption.prose.deadlock.js

@@ -2,7 +2,7 @@
 
 const BSON = require('bson');
 const { expect } = require('chai');
-const { dropCollection } = require('../shared');
+const { dropCollection } = require('../../integration/shared');
 const util = require('util');
 const fs = require('fs');
 const path = require('path');