From 7a507f08905d2e30f4c4b53adf0bf506e65e357f Mon Sep 17 00:00:00 2001 From: gjchong25 Date: Tue, 30 Nov 2021 11:57:14 -0600 Subject: [PATCH] fix(NODE-3627): Enable flexible BSON validation for server error key containing invalid utf-8 (#3054) --- package-lock.json | 14 ++-- package.json | 2 +- src/bson.ts | 1 + src/cmap/commands.ts | 19 +++--- test/unit/commands.test.ts | 136 +++++++++++++++++++++++++++++++++++++ 5 files changed, 156 insertions(+), 16 deletions(-) create mode 100644 test/unit/commands.test.ts diff --git a/package-lock.json b/package-lock.json index ffaebefc09..ae93725502 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,7 +9,7 @@ "version": "4.2.0", "license": "Apache-2.0", "dependencies": { - "bson": "^4.5.4", + "bson": "^4.6.0", "denque": "^2.0.1", "mongodb-connection-string-url": "^2.2.0" }, @@ -1557,9 +1557,9 @@ } }, "node_modules/bson": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/bson/-/bson-4.5.4.tgz", - "integrity": "sha512-wIt0bPACnx8Ju9r6IsS2wVtGDHBr9Dxb+U29A1YED2pu8XOhS8aKjOnLZ8sxyXkPwanoK7iWWVhS1+coxde6xA==", + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.6.0.tgz", + "integrity": "sha512-8jw1NU1hglS+Da1jDOUYuNcBJ4cNHCFIqzlwoFNnsTOg2R/ox0aTYcTiBN4dzRa9q7Cvy6XErh3L8ReTEb9AQQ==", "dependencies": { "buffer": "^5.6.0" }, @@ -8466,9 +8466,9 @@ } }, "bson": { - "version": "4.5.4", - "resolved": "https://registry.npmjs.org/bson/-/bson-4.5.4.tgz", - "integrity": "sha512-wIt0bPACnx8Ju9r6IsS2wVtGDHBr9Dxb+U29A1YED2pu8XOhS8aKjOnLZ8sxyXkPwanoK7iWWVhS1+coxde6xA==", + "version": "4.6.0", + "resolved": "https://registry.npmjs.org/bson/-/bson-4.6.0.tgz", + "integrity": "sha512-8jw1NU1hglS+Da1jDOUYuNcBJ4cNHCFIqzlwoFNnsTOg2R/ox0aTYcTiBN4dzRa9q7Cvy6XErh3L8ReTEb9AQQ==", "requires": { "buffer": "^5.6.0" } diff --git a/package.json b/package.json index 1ac940b95d..47067d70ca 100644 --- a/package.json +++ b/package.json @@ -32,7 +32,7 @@ "email": "dbx-node@mongodb.com" }, "dependencies": { - "bson": "^4.5.4", + "bson": "^4.6.0", "denque": "^2.0.1", "mongodb-connection-string-url": "^2.2.0" }, diff --git a/src/bson.ts b/src/bson.ts index 3b445109da..356c158c16 100644 --- a/src/bson.ts +++ b/src/bson.ts @@ -51,6 +51,7 @@ export interface BSONSerializeOptions | 'cacheFunctionsCrc32' | 'allowObjectSmallerThanBufferSize' | 'index' + | 'validation' > { /** Return BSON filled buffers from operations */ raw?: boolean; diff --git a/src/cmap/commands.ts b/src/cmap/commands.ts index 3fa3a71411..322ca736a3 100644 --- a/src/cmap/commands.ts +++ b/src/cmap/commands.ts @@ -469,6 +469,8 @@ export interface MessageHeader { export interface OpResponseOptions extends BSONSerializeOptions { raw?: boolean; documentsReturnedIn?: string | null; + // For now we use this internally to only prevent writeErrors from crashing the driver + validation?: { utf8: { writeErrors: boolean } }; } /** @internal */ @@ -837,22 +839,24 @@ export class BinMsg { const promoteValues = options.promoteValues ?? this.opts.promoteValues; const promoteBuffers = options.promoteBuffers ?? this.opts.promoteBuffers; const bsonRegExp = options.bsonRegExp ?? this.opts.bsonRegExp; + const validation = options.validation ?? { utf8: { writeErrors: false } }; // Set up the options - const _options: BSONSerializeOptions = { + const bsonOptions: BSONSerializeOptions = { promoteLongs, promoteValues, promoteBuffers, - bsonRegExp - }; + bsonRegExp, + validation + // Due to the strictness of the BSON libraries validation option we need this cast + } as BSONSerializeOptions & { validation: { utf8: { writeErrors: boolean } } }; while (this.index < this.data.length) { const payloadType = this.data.readUInt8(this.index++); if (payloadType === 0) { const bsonSize = this.data.readUInt32LE(this.index); const bin = this.data.slice(this.index, this.index + bsonSize); - this.documents.push(raw ? bin : BSON.deserialize(bin, _options)); - + this.documents.push(raw ? bin : BSON.deserialize(bin, bsonOptions)); this.index += bsonSize; } else if (payloadType === 1) { // It was decided that no driver makes use of payload type 1 @@ -865,9 +869,8 @@ export class BinMsg { if (this.documents.length === 1 && documentsReturnedIn != null && raw) { const fieldsAsRaw: Document = {}; fieldsAsRaw[documentsReturnedIn] = true; - _options.fieldsAsRaw = fieldsAsRaw; - - const doc = BSON.deserialize(this.documents[0] as Buffer, _options); + bsonOptions.fieldsAsRaw = fieldsAsRaw; + const doc = BSON.deserialize(this.documents[0] as Buffer, bsonOptions); this.documents = [doc]; } diff --git a/test/unit/commands.test.ts b/test/unit/commands.test.ts new file mode 100644 index 0000000000..a8156abc73 --- /dev/null +++ b/test/unit/commands.test.ts @@ -0,0 +1,136 @@ +import { expect } from 'chai'; +import { BinMsg, MessageHeader } from '../../src/cmap/commands'; +import { BSONError } from 'bson'; +import * as BSON from '../../src/bson'; + +const msgHeader: MessageHeader = { + length: 735, + requestId: 14704565, + responseTo: 4, + opCode: 2013 +}; + +// when top-level key writeErrors contains an error message that has invalid utf8 +const invalidUtf8ErrorMsg = + '0000000000ca020000106e00000000000477726974654572726f727300a50200000330009d02000010696e646578000000000010636f646500f82a0000036b65795061747465726e000f0000001074657874000100000000036b657956616c756500610100000274657874005201000064e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e298830000026572726d736700f1000000453131303030206475706c6963617465206b6579206572726f7220636f6c6c656374696f6e3a20626967646174612e7465737420696e6465783a20746578745f3120647570206b65793a207b20746578743a202264e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e2982e2e2e22207d000000016f6b00000000000000f03f00'; +const msgBodyInvalidUtf8WriteErrors = Buffer.from(invalidUtf8ErrorMsg, 'hex'); +const invalidUtf8ErrorMsgDeserializeInput = Buffer.from(invalidUtf8ErrorMsg.substring(10), 'hex'); +const invalidUtf8InWriteErrorsJSON = { + n: 0, + writeErrors: [ + { + index: 0, + code: 11000, + keyPattern: { + text: 1 + }, + keyValue: { + text: 'd☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃' + }, + errmsg: + 'E11000 duplicate key error collection: bigdata.test index: text_1 dup key: { text: "d☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃�..." }' + } + ], + ok: 1 +}; + +// when another top-level key besides writeErrors has invalid utf8 +const nKeyWithInvalidUtf8 = + '0000000000cc020000026e0005000000f09f98ff000477726974654572726f727300a60200000330009e02000010696e646578000000000010636f646500f82a0000036b65795061747465726e000f0000001074657874000100000000036b657956616c756500610100000274657874005201000064e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e298830000026572726d736700f2000000453131303030206475706c6963617465206b6579206572726f7220636f6c6c656374696f6e3a20626967646174612e7465737420696e6465783a20746578745f3120647570206b65793a207b20746578743a202264e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883e29883efbfbd2e2e2e22207d000000106f6b000100000000'; +const nKeyWithInvalidUtf8DeserializeInput = Buffer.from(nKeyWithInvalidUtf8.substring(10), 'hex'); +const msgBodyNKeyWithInvalidUtf8 = Buffer.from(nKeyWithInvalidUtf8, 'hex'); +const invalidUtf8InNKeyJSON = { + n: '��', + writeErrors: [ + { + index: 0, + code: 11000, + keyPattern: { + text: 1 + }, + keyValue: { + text: 'd☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃' + }, + errmsg: + 'E11000 duplicate key error collection: bigdata.test index: text_1 dup key: { text: "d☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃☃�..." }' + } + ], + ok: 1 +}; + +describe('BinMsg BSON utf8 validation', () => { + context('when validation is disabled for writeErrors', () => { + const binMsgInvalidUtf8ErrorMsg = new BinMsg( + Buffer.alloc(0), + msgHeader, + msgBodyInvalidUtf8WriteErrors + ); + const options = { validation: { utf8: { writeErrors: false } as const } }; + + it('contains replacement characters for invalid utf8 in writeError object', () => { + expect(BSON.deserialize(invalidUtf8ErrorMsgDeserializeInput, options)).to.deep.equals( + invalidUtf8InWriteErrorsJSON + ); + }); + + it('should not throw invalid utf8 error', () => { + expect(() => binMsgInvalidUtf8ErrorMsg.parse(options)).to.not.throw(); + }); + }); + + it('should by default disable validation for writeErrors if no validation specified', () => { + const binMsgInvalidUtf8ErrorMsg = new BinMsg( + Buffer.alloc(0), + msgHeader, + msgBodyInvalidUtf8WriteErrors + ); + const options = { + bsonRegExp: false, + promoteBuffers: false, + promoteLongs: true, + promoteValues: true + }; + expect(() => binMsgInvalidUtf8ErrorMsg.parse(options)).to.not.throw(); + }); + + context('when another key has invalid utf8 and validation is enabled for writeErrors', () => { + const binMsgAnotherKeyWithInvalidUtf8 = new BinMsg( + Buffer.alloc(0), + msgHeader, + msgBodyNKeyWithInvalidUtf8 + ); + const options = { validation: { utf8: { writeErrors: true } as const } }; + + it('should not throw invalid utf8 error', () => { + expect(() => binMsgAnotherKeyWithInvalidUtf8.parse(options)).to.not.throw(); + }); + + it('contains replacement characters for invalid utf8 key', () => { + expect(BSON.deserialize(nKeyWithInvalidUtf8DeserializeInput, options)).to.deep.equals( + invalidUtf8InNKeyJSON + ); + }); + }); + + it('should throw invalid utf8 error when validation enabled for writeErrors', () => { + const binMsgInvalidUtf8ErrorMsg = new BinMsg( + Buffer.alloc(0), + msgHeader, + msgBodyInvalidUtf8WriteErrors + ); + expect(() => + binMsgInvalidUtf8ErrorMsg.parse({ validation: { utf8: { writeErrors: true } } }) + ).to.throw(BSONError, 'Invalid UTF-8 string in BSON document'); + }); + + it('should throw error when another key has invalid utf8 and writeErrors is not validated', () => { + const binMsgAnotherKeyWithInvalidUtf8 = new BinMsg( + Buffer.alloc(0), + msgHeader, + msgBodyNKeyWithInvalidUtf8 + ); + expect(() => + binMsgAnotherKeyWithInvalidUtf8.parse({ validation: { utf8: { writeErrors: false } } }) + ).to.throw(BSONError, 'Invalid UTF-8 string in BSON document'); + }); +});