From fd65f7712e17f0f20b5f6b341fb8824100ec4f0f Mon Sep 17 00:00:00 2001 From: Anna Henningsen Date: Tue, 12 Oct 2021 14:03:18 +0200 Subject: [PATCH] feat(NODE-3633): add Socks5 support --- .evergreen/config.yml | 48 ++++ .evergreen/config.yml.in | 15 ++ .evergreen/generate_evergreen_tasks.js | 31 +++ .evergreen/run-socks5-tests.sh | 42 ++++ .evergreen/socks5srv.py | 237 +++++++++++++++++++ package-lock.json | 49 +++- package.json | 4 +- src/cmap/connect.ts | 149 +++++++++++- src/cmap/connection.ts | 19 +- src/connection_string.ts | 28 +++ src/deps.ts | 6 + src/encrypter.ts | 4 + src/index.ts | 3 +- src/mongo_client.ts | 5 +- test/manual/socks5.test.js | 307 +++++++++++++++++++++++++ 15 files changed, 934 insertions(+), 13 deletions(-) create mode 100644 .evergreen/run-socks5-tests.sh create mode 100755 .evergreen/socks5srv.py create mode 100644 test/manual/socks5.test.js diff --git a/.evergreen/config.yml b/.evergreen/config.yml index 074824c2ba5..7d53a77f99f 100644 --- a/.evergreen/config.yml +++ b/.evergreen/config.yml @@ -293,6 +293,27 @@ functions: export NODE_LTS_NAME='${NODE_LTS_NAME}' bash ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh + run socks5 tests: + - command: shell.exec + type: test + params: + working_dir: src + script: > + export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo + "/opt/mongodbtoolchain/v3/bin/python3") + + export PROJECT_DIRECTORY="$(pwd)" + + export DRIVERS_TOOLS="${DRIVERS_TOOLS}" + + export NODE_LTS_NAME='${NODE_LTS_NAME}' + + export MONGODB_URI="${MONGODB_URI}" + + export SSL="${SSL}" + + + bash ${PROJECT_DIRECTORY}/.evergreen/run-socks5-tests.sh run kerberos tests: - command: shell.exec type: test @@ -907,6 +928,27 @@ tasks: commands: - func: install dependencies - func: run ldap tests + - name: test-socks5 + tags: [] + commands: + - func: install dependencies + - func: bootstrap mongo-orchestration + vars: + VERSION: latest + TOPOLOGY: replica_set + - func: run socks5 tests + - name: test-socks5-tls + tags: [] + commands: + - func: install dependencies + - func: bootstrap mongo-orchestration + vars: + SSL: ssl + VERSION: latest + TOPOLOGY: replica_set + - func: run socks5 tests + vars: + SSL: ssl - name: test-ocsp-valid-cert-server-staples tags: - ocsp @@ -1683,6 +1725,8 @@ buildvariants: - test-atlas-data-lake - test-auth-kerberos - test-auth-ldap + - test-socks5 + - test-socks5-tls - test-ocsp-valid-cert-server-staples - test-ocsp-invalid-cert-server-staples - test-ocsp-valid-cert-server-does-not-staple @@ -1753,6 +1797,8 @@ buildvariants: - test-load-balancer - test-auth-kerberos - test-auth-ldap + - test-socks5 + - test-socks5-tls - test-ocsp-valid-cert-server-staples - test-ocsp-invalid-cert-server-staples - test-ocsp-valid-cert-server-does-not-staple @@ -1819,6 +1865,8 @@ buildvariants: - test-3.6-sharded_cluster - test-latest-server-v1-api - test-atlas-data-lake + - test-socks5 + - test-socks5-tls - test-ocsp-valid-cert-server-staples - test-ocsp-invalid-cert-server-staples - test-ocsp-valid-cert-server-does-not-staple diff --git a/.evergreen/config.yml.in b/.evergreen/config.yml.in index 4932d90de77..a79486c086c 100644 --- a/.evergreen/config.yml.in +++ b/.evergreen/config.yml.in @@ -326,6 +326,21 @@ functions: bash ${PROJECT_DIRECTORY}/.evergreen/run-atlas-tests.sh + "run socks5 tests": + - command: shell.exec + type: test + params: + working_dir: "src" + script: | + export PYTHON_BINARY=$([ "Windows_NT" = "$OS" ] && echo "/cygdrive/c/python/python38/python.exe" || echo "/opt/mongodbtoolchain/v3/bin/python3") + export PROJECT_DIRECTORY="$(pwd)" + export DRIVERS_TOOLS="${DRIVERS_TOOLS}" + export NODE_LTS_NAME='${NODE_LTS_NAME}' + export MONGODB_URI="${MONGODB_URI}" + export SSL="${SSL}" + + bash ${PROJECT_DIRECTORY}/.evergreen/run-socks5-tests.sh + "run kerberos tests": - command: shell.exec type: test diff --git a/.evergreen/generate_evergreen_tasks.js b/.evergreen/generate_evergreen_tasks.js index 1693b368266..21d842a7835 100644 --- a/.evergreen/generate_evergreen_tasks.js +++ b/.evergreen/generate_evergreen_tasks.js @@ -136,6 +136,37 @@ TASKS.push( tags: ['auth', 'ldap'], commands: [{ func: 'install dependencies' }, { func: 'run ldap tests' }] }, + { + name: 'test-socks5', + tags: [], + commands: [ + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + VERSION: 'latest', + TOPOLOGY: 'replica_set' + } + }, + { func: 'run socks5 tests' } + ] + }, + { + name: 'test-socks5-tls', + tags: [], + commands: [ + { func: 'install dependencies' }, + { + func: 'bootstrap mongo-orchestration', + vars: { + SSL: 'ssl', + VERSION: 'latest', + TOPOLOGY: 'replica_set' + } + }, + { func: 'run socks5 tests', vars: { SSL: 'ssl' } } + ] + }, { name: 'test-ocsp-valid-cert-server-staples', tags: ['ocsp'], diff --git a/.evergreen/run-socks5-tests.sh b/.evergreen/run-socks5-tests.sh new file mode 100644 index 00000000000..2c6c118ca1c --- /dev/null +++ b/.evergreen/run-socks5-tests.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +set -o errexit # Exit the script with error if any of the commands fail +set -o xtrace # For debuggability, no external credentials are used here + +source "${PROJECT_DIRECTORY}/.evergreen/init-nvm.sh" + +PYTHON_BINARY=${PYTHON_BINARY:-python3} + +# ssl setup +SSL=${SSL:-nossl} +if [ "$SSL" != "nossl" ]; then + export SSL_KEY_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/client.pem" + export SSL_CA_FILE="$DRIVERS_TOOLS/.evergreen/x509gen/ca.pem" +fi + +# Grab a connection string that only refers to *one* of the hosts in MONGODB_URI +FIRST_HOST=$(node -p 'new (require("mongodb-connection-string-url").default)(process.env.MONGODB_URI).hosts[0]') +# Use localhost:12345 as the URL for the single host that we connect to, +# we configure the Socks5 proxy server script to redirect from this to FIRST_HOST +export MONGODB_URI_SINGLEHOST="mongodb://localhost:12345/" + +# Compute path to socks5 fake server script in a way that works on Windows +SOCKS5_SERVER_SCRIPT="${PROJECT_DIRECTORY}/.evergreen/socks5srv.py" +if [ "Windows_NT" = "$OS" ]; then + SOCKS5_SERVER_SCRIPT=$(cygpath -w "$SOCKS5_SERVER_SCRIPT") +fi + +# First, test with Socks5 + authentication required +"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1080 --auth username:p4ssw0rd --map "localhost:12345 to $FIRST_HOST" & +PID=$! +env SOCKS5_CONFIG='["localhost",1080,"username","p4ssw0rd"]' npm run check:socks5 +kill $PID + +# Second, test with Socks5 + no authentication +"$PYTHON_BINARY" "$SOCKS5_SERVER_SCRIPT" --port 1081 --map "localhost:12345 to $FIRST_HOST" & +PID=$! +env SOCKS5_CONFIG='["localhost",1081]' npm run check:socks5 +kill $PID + +# TODO: It might be worth using something more robust to control +# the Socks5 proxy server script's lifetime diff --git a/.evergreen/socks5srv.py b/.evergreen/socks5srv.py new file mode 100755 index 00000000000..4505d1e4789 --- /dev/null +++ b/.evergreen/socks5srv.py @@ -0,0 +1,237 @@ +#!/usr/bin/env python3 +import socketserver +import socket +import select +import re +import argparse + +# Usage: python3 socks5srv.py --port port [--auth username:password] [--map 'host:port to host:port' ...] +# TODO: Move this script into the shared drivers-evergreen-tools repository + +class AddressRemapper: + """A helper for remapping (host, port) tuples to new (host, port) tuples + + This is useful for Socks5 servers used in testing environments, + because the succesful use of the Socks5 proxy can be demonstrated + by being able to 'connect' to a redirected port, which would always + fail without the proxy, even on localhost-only environments + """ + + def __init__(self, mappings): + self.mappings = [AddressRemapper.parse_single_mapping(string) for string in mappings] + self.add_dns_remappings() + + @staticmethod + def parse_single_mapping(string): + """Parse a single mapping of the for '{host}:{port} to {host}:{port}'""" + + # Accept either [ipv6]:port or host:port + host_re = r"(\[(?P<{0}_ipv6>[^[\]]+)\]|(?P<{0}_host>[^\[]+))" + port_re = r"(?P<{0}_port>\d+)" + + src_re = host_re.format('src') + ':' + port_re.format('src') + dst_re = host_re.format('dst') + ':' + port_re.format('dst') + full_re = '^' + src_re + ' to ' + dst_re + '$' + + match = re.match(full_re, string) + if match is None: + raise Exception("Mapping {} does not match format '{{host}}:{{port}} to {{host}}:{{port}}'".format(string)) + + src = ((match.group('src_ipv6') or match.group('src_host')).encode('utf8'), int(match.group('src_port'))) + dst = ((match.group('dst_ipv6') or match.group('dst_host')).encode('utf8'), int(match.group('dst_port'))) + return (src, dst) + + def add_dns_remappings(self): + """Add mappings for the IP addresses corresponding to hostnames + + For example, if there is a mapping (localhost, 1000) to (localhost, 2000), + then this also adds (127.0.0.1, 1000) to (localhost, 2000).""" + + for src, dst in self.mappings: + host, port = src + try: + addrs = socket.getaddrinfo(host, port, socket.AF_UNSPEC, socket.SOCK_STREAM) + except socket.gaierror: + continue + + existing_src_entries = [src for src, dst in self.mappings] + for af, socktype, proto, canonname, sa in addrs: + if af == socket.AF_INET and sa not in existing_src_entries: + self.mappings.append((sa, dst)) + elif af == socket.AF_INET6 and sa[:2] not in existing_src_entries: + self.mappings.append((sa[:2], dst)) + + def remap(self, hostport): + """Re-map a (host, port) tuple to a new (host, port) tuple if that was requested""" + + for src, dst in self.mappings: + if hostport == src: + return dst + return hostport + +class Socks5Server(socketserver.ThreadingTCPServer): + """A simple Socks5 proxy server""" + + def __init__(self, server_address, RequestHandlerClass, args): + socketserver.ThreadingTCPServer.__init__(self, + server_address, + RequestHandlerClass) + self.args = args + self.address_remapper = AddressRemapper(args.map) + +class Socks5Handler(socketserver.BaseRequestHandler): + """Request handler for Socks5 connections""" + + def finish(self): + """Called after handle(), always just closes the connection""" + + self.request.close() + + def read_exact(self, n): + """Read n bytes from a socket + + In Socks5, strings are prefixed with a single byte containing + their length. This method reads a bytes string containing n bytes + (where n can be a number or a bytes object containing that + single byte). + + If reading from the client ends prematurely, this returns None. + """ + + if type(n) is bytes: + if len(n) == 0: + return None + assert len(n) == 1 + n = n[0] + result = b'' + while len(result) < n: + buf = self.request.recv(n - len(result)) + if buf == b'': + return None + result += buf + return result + + def create_outgoing_tcp_connection(self, dst, port): + """Create an outgoing TCP connection to dst:port""" + + outgoing = None + for res in socket.getaddrinfo(dst, port, socket.AF_UNSPEC, socket.SOCK_STREAM): + af, socktype, proto, canonname, sa = res + try: + outgoing = socket.socket(af, socktype, proto) + except OSError as msg: + continue + try: + outgoing.connect(sa) + except OSError as msg: + outgoing.close() + continue + break + return outgoing + + def handle(self): + """Handle the Socks5 communication with a freshly connected client""" + + # Client greeting + if self.request.recv(1) != b'\x05': # Socks5 only + return + n_auth = self.request.recv(1) + client_auth_methods = self.read_exact(n_auth) + if client_auth_methods is None: + return + + # choose either no-auth or username/password + required_auth_method = b'\x00' if self.server.args.auth is None else b'\x02' + if required_auth_method not in client_auth_methods: + self.request.sendall(b'\x05\xff') + return + + self.request.sendall(b'\x05' + required_auth_method) + if required_auth_method == b'\x02': + auth_version = self.request.recv(1) + if auth_version != b'\x01': # Only username/password auth v1 + return + username_len = self.request.recv(1) + username = self.read_exact(username_len) + password_len = self.request.recv(1) + password = self.read_exact(password_len) + if username is None or password is None: + return + if username.decode('utf8') + ':' + password.decode('utf8') != self.server.args.auth: + return + self.request.sendall(b'\x01\x00') # auth success + + if self.request.recv(1) != b'\x05': # Socks5 only + return + if self.request.recv(1) != b'\x01': # Outgoing TCP only + return + if self.request.recv(1) != b'\x00': # Reserved, must be 0 + return + + addrtype = self.request.recv(1) + dst = None + if addrtype == b'\x01': # IPv4 + ipv4raw = self.read_exact(4) + if ipv4raw is not None: + dst = '.'.join(['{}'] * 4).format(*ipv4raw) + elif addrtype == b'\x03': # Domain + domain_len = self.request.recv(1) + dst = self.read_exact(domain_len) + elif addrtype == b'\x04': # IPv6 + ipv6raw = self.read_exact(16) + if ipv6raw is not None: + dst = ':'.join(['{:0>2x}{:0>2x}'] * 8).format(*ipv6raw) + else: + return + + if dst is None: + return + + portraw = self.read_exact(2) + port = portraw[0] * 256 + portraw[1] + + (dst, port) = self.server.address_remapper.remap((dst, port)) + + outgoing = self.create_outgoing_tcp_connection(dst, port) + if outgoing is None: + self.request.sendall(b'\x05\x01\x00') # just report a general failure + return + # success response, do not bother actually stating the locally bound + # host/port address and instead always say 127.0.0.1:4096. + # for our use case, the client will not be making meaningful use + # of this anyway + self.request.sendall(b'\x05\x00\x00\x01\x7f\x00\x00\x01\x10\x00') + + self.raw_proxy(self.request, outgoing) + + def raw_proxy(self, a, b): + """Proxy data between sockets a and b as-is""" + + with a, b: + while True: + try: + (readable, _, _) = select.select([a, b], [], []) + except (select.error, ValueError): + return + + if not readable: + continue + for sock in readable: + buf = sock.recv(4096) + if buf == b'': + return + if sock is a: + b.sendall(buf) + else: + a.sendall(buf) + +if __name__ == '__main__': + parser = argparse.ArgumentParser(description='Start a Socks5 proxy server.') + parser.add_argument('--port', type=int, required=True) + parser.add_argument('--auth', type=str) + parser.add_argument('--map', type=str, action='append', default=[]) + args = parser.parse_args() + + socketserver.TCPServer.allow_reuse_address = True + with Socks5Server(('localhost', args.port), Socks5Handler, args) as server: + server.serve_forever() diff --git a/package-lock.json b/package-lock.json index 9a00832cc58..9ad7c93daff 100644 --- a/package-lock.json +++ b/package-lock.json @@ -11,7 +11,8 @@ "dependencies": { "bson": "^4.5.4", "denque": "^2.0.1", - "mongodb-connection-string-url": "^2.2.0" + "mongodb-connection-string-url": "^2.2.0", + "socks": "^2.6.1" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.1", @@ -3688,6 +3689,11 @@ "node": ">= 0.10" } }, + "node_modules/ip": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + }, "node_modules/irregular-plurals": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.3.0.tgz", @@ -6101,6 +6107,15 @@ "node": ">=8" } }, + "node_modules/smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==", + "engines": { + "node": ">= 6.0.0", + "npm": ">= 3.0.0" + } + }, "node_modules/snappy": { "version": "6.3.5", "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", @@ -6113,6 +6128,19 @@ "prebuild-install": "5.3.0" } }, + "node_modules/socks": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.6.1.tgz", + "integrity": "sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA==", + "dependencies": { + "ip": "^1.1.5", + "smart-buffer": "^4.1.0" + }, + "engines": { + "node": ">= 10.13.0", + "npm": ">= 3.0.0" + } + }, "node_modules/source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", @@ -10317,6 +10345,11 @@ "integrity": "sha512-agE4QfB2Lkp9uICn7BAqoscw4SZP9kTE2hxiFI3jBPmXJfdqiahTbUuKGsMoN2GtqL9AxhYioAcVvgsb1HvRbA==", "dev": true }, + "ip": { + "version": "1.1.5", + "resolved": "https://registry.npmjs.org/ip/-/ip-1.1.5.tgz", + "integrity": "sha1-vd7XARQpCCjAoDnnLvJfWq7ENUo=" + }, "irregular-plurals": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/irregular-plurals/-/irregular-plurals-3.3.0.tgz", @@ -12173,6 +12206,11 @@ } } }, + "smart-buffer": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/smart-buffer/-/smart-buffer-4.2.0.tgz", + "integrity": "sha512-94hK0Hh8rPqQl2xXc3HsaBoOXKV20MToPkcXvwbISWLEs+64sBq5kFgn2kJDHb1Pry9yrP0dxrCI9RRci7RXKg==" + }, "snappy": { "version": "6.3.5", "resolved": "https://registry.npmjs.org/snappy/-/snappy-6.3.5.tgz", @@ -12184,6 +12222,15 @@ "prebuild-install": "5.3.0" } }, + "socks": { + "version": "2.6.1", + "resolved": "https://registry.npmjs.org/socks/-/socks-2.6.1.tgz", + "integrity": "sha512-kLQ9N5ucj8uIcxrDwjm0Jsqk06xdpBjGNQtpXy4Q8/QY2k+fY7nZH8CARy+hkbG+SGAovmzzuauCpBlb8FrnBA==", + "requires": { + "ip": "^1.1.5", + "smart-buffer": "^4.1.0" + } + }, "source-map": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/source-map/-/source-map-0.6.1.tgz", diff --git a/package.json b/package.json index 7c0c60aa6b7..23ad487db26 100644 --- a/package.json +++ b/package.json @@ -34,7 +34,8 @@ "dependencies": { "bson": "^4.5.4", "denque": "^2.0.1", - "mongodb-connection-string-url": "^2.2.0" + "mongodb-connection-string-url": "^2.2.0", + "socks": "^2.6.1" }, "devDependencies": { "@istanbuljs/nyc-config-typescript": "^1.0.1", @@ -114,6 +115,7 @@ "check:kerberos": "mocha --config \"test/manual/mocharc.json\" test/manual/kerberos.test.js", "check:tls": "mocha --config \"test/manual/mocharc.json\" test/manual/tls_support.test.js", "check:ldap": "mocha --config \"test/manual/mocharc.json\" test/manual/ldap.test.js", + "check:socks5": "mocha --config \"test/manual/mocharc.json\" test/manual/socks5.test.js", "check:csfle": "mocha --file test/tools/runner test/functional/client_side_encryption", "check:snappy": "mocha --file test/tools/runner test/functional/unit_snappy.test.js", "prepare": "node etc/prepare.js", diff --git a/src/cmap/connect.ts b/src/cmap/connect.ts index 07917c80e6f..55ba2f7a341 100644 --- a/src/cmap/connect.ts +++ b/src/cmap/connect.ts @@ -1,6 +1,7 @@ import * as net from 'net'; import * as tls from 'tls'; -import { Connection, ConnectionOptions, CryptoConnection } from './connection'; +import { SocksClient } from 'socks'; +import { Connection, ConnectionOptions, CryptoConnection, ProxyOptions } from './connection'; import { MongoNetworkError, MongoNetworkTimeoutError, @@ -12,7 +13,14 @@ import { } from '../error'; import { AUTH_PROVIDERS, AuthMechanism } from './auth/defaultAuthProviders'; import { AuthContext } from './auth/auth_provider'; -import { makeClientMetadata, ClientMetadata, Callback, CallbackWithType, ns } from '../utils'; +import { + makeClientMetadata, + ClientMetadata, + Callback, + CallbackWithType, + HostAddress, + ns +} from '../utils'; import { MAX_SUPPORTED_WIRE_VERSION, MAX_SUPPORTED_SERVER_VERSION, @@ -33,7 +41,7 @@ const FAKE_MONGODB_SERVICE_ID = export type Stream = Socket | TLSSocket; export function connect(options: ConnectionOptions, callback: Callback): void { - makeConnection(options, (err, socket) => { + makeConnection({ ...options, existingSocket: undefined }, (err, socket) => { if (err || !socket) { return callback(err); } @@ -289,7 +297,9 @@ function parseConnectOptions(options: ConnectionOptions): SocketConnectOpts { } } -function parseSslOptions(options: ConnectionOptions): TLSConnectionOpts { +type MakeConnectionOptions = ConnectionOptions & { existingSocket?: Stream }; + +function parseSslOptions(options: MakeConnectionOptions): TLSConnectionOpts { const result: TLSConnectionOpts = parseConnectOptions(options); // Merge in valid SSL options for (const name of LEGAL_TLS_SOCKET_OPTIONS) { @@ -298,6 +308,10 @@ function parseSslOptions(options: ConnectionOptions): TLSConnectionOpts { } } + if (options.existingSocket) { + result.socket = options.existingSocket; + } + // Set default sni servername to be the same as host if (result.servername == null && result.host && !net.isIP(result.host)) { result.servername = result.host; @@ -310,7 +324,10 @@ const SOCKET_ERROR_EVENT_LIST = ['error', 'close', 'timeout', 'parseError'] as c type ErrorHandlerEventName = typeof SOCKET_ERROR_EVENT_LIST[number] | 'cancel'; const SOCKET_ERROR_EVENTS = new Set(SOCKET_ERROR_EVENT_LIST); -function makeConnection(options: ConnectionOptions, _callback: CallbackWithType) { +function makeConnection( + options: MakeConnectionOptions, + _callback: CallbackWithType +) { const useTLS = options.tls ?? false; const keepAlive = options.keepAlive ?? true; const socketTimeoutMS = options.socketTimeoutMS ?? Reflect.get(options, 'socketTimeout') ?? 0; @@ -321,6 +338,8 @@ function makeConnection(options: ConnectionOptions, _callback: CallbackWithType< ((options.keepAliveInitialDelay ?? 120000) > socketTimeoutMS ? Math.round(socketTimeoutMS / 2) : options.keepAliveInitialDelay) ?? 120000; + const proxyOptions = options.proxyOptions; + const existingSocket = options.existingSocket; let socket: Stream; const callback: Callback = function (err, ret) { @@ -331,12 +350,27 @@ function makeConnection(options: ConnectionOptions, _callback: CallbackWithType< _callback(err, ret); }; + if (proxyOptions?.host != null) { + // Currently, only Socks5 is supported. + return makeSocks5Connection( + { + ...options, + connectTimeoutMS: connectionTimeout, // Should always be present for Socks5 + proxyOptions: undefined + }, + proxyOptions, + callback + ); + } + if (useTLS) { const tlsSocket = tls.connect(parseSslOptions(options)); if (typeof tlsSocket.disableRenegotiation === 'function') { tlsSocket.disableRenegotiation(); } socket = tlsSocket; + } else if (existingSocket) { + socket = existingSocket; } else { socket = net.createConnection(parseConnectOptions(options)); } @@ -381,10 +415,111 @@ function makeConnection(options: ConnectionOptions, _callback: CallbackWithType< options.cancellationToken.once('cancel', cancellationHandler); } - socket.once(connectEvent, connectHandler); + if (existingSocket) { + process.nextTick(connectHandler); + } else { + socket.once(connectEvent, connectHandler); + } +} + +function makeSocks5Connection( + options: MakeConnectionOptions & { proxyOptions: undefined }, + proxyOptions: ProxyOptions, + callback: Callback +) { + if (typeof proxyOptions.host !== 'string' || !proxyOptions.host) { + return process.nextTick( + callback, + new MongoInvalidArgumentError('The hostname is required when specifying proxy options') + ); + } + if (typeof proxyOptions.port !== 'number' && proxyOptions.port != null) { + return process.nextTick( + callback, + new MongoInvalidArgumentError('The port must be a number when specifying proxy options') + ); + } + if (typeof proxyOptions.username !== 'string' && proxyOptions.username != null) { + return process.nextTick( + callback, + new MongoInvalidArgumentError('The username must be a string when specifying proxy options') + ); + } + if (typeof proxyOptions.password !== 'string' && proxyOptions.password != null) { + return process.nextTick( + callback, + new MongoInvalidArgumentError('The username must be a string when specifying proxy options') + ); + } + if ((proxyOptions.username && !proxyOptions.password) || (!proxyOptions.username && proxyOptions.password)) { + return process.nextTick( + callback, + new MongoInvalidArgumentError('Can only specify both of proxy username/password or neither') + ); + } + const proxyHost = proxyOptions.host.includes(':') ? `[${proxyOptions.host}]` : proxyOptions.host; + const proxyPort = proxyOptions.port ?? 1080; + const hostAddress = HostAddress.fromString(`${proxyHost}:${proxyPort}`); + + // First, connect to the proxy server itself: + makeConnection( + { + ...options, + hostAddress, + tls: false + }, + (err, rawSocket) => { + if (err) { + return callback(err); + } + + const destination = parseConnectOptions(options) as net.TcpNetConnectOpts; + if (typeof destination.host !== 'string' || typeof destination.port !== 'number') { + return callback( + new MongoInvalidArgumentError('Can only make Socks5 connections to TCP hosts') + ); + } + + // Then, establish the Socks5 proxy connection: + SocksClient.createConnection( + { + existing_socket: rawSocket, + timeout: options.connectTimeoutMS, + command: 'connect', + destination: { + host: destination.host, + port: destination.port + }, + proxy: { + // host and port are ignored because we pass existing_socket + host: 'localhost', + port: 0, + type: 5, + userId: proxyOptions.username || undefined, + password: proxyOptions.password || undefined + } + }, + (err: AnyError, info: { socket: Stream }) => { + if (err) { + return callback(connectionFailureError('error', err)); + } + + // Finally, now treat the resulting duplex stream as the + // socket over which we send and receive wire protocol messages: + makeConnection( + { + ...options, + existingSocket: info.socket + }, + callback + ); + } + ); + } + ); } -function connectionFailureError(type: string, err: Error) { +function connectionFailureError(type: ErrorHandlerEventName, err: Error) { switch (type) { case 'error': return new MongoNetworkError(err); diff --git a/src/cmap/connection.ts b/src/cmap/connection.ts index ccee670fc05..ae80b166e73 100644 --- a/src/cmap/connection.ts +++ b/src/cmap/connection.ts @@ -114,6 +114,14 @@ export interface GetMoreOptions extends CommandOptions { comment?: Document | string; } +/** @public */ +export interface ProxyOptions { + host: string; + port?: number; + username?: string; + password?: string; +} + /** @public */ export interface ConnectionOptions extends SupportedNodeConnectionOptions, @@ -136,6 +144,7 @@ export interface ConnectionOptions noDelay?: boolean; socketTimeoutMS?: number; cancellationToken?: CancellationToken; + proxyOptions?: ProxyOptions; metadata: ClientMetadata; } @@ -206,7 +215,7 @@ export class Connection extends TypedEventEmitter { constructor(stream: Stream, options: ConnectionOptions) { super(); this.id = options.id; - this.address = streamIdentifier(stream); + this.address = streamIdentifier(stream, options); this.socketTimeoutMS = options.socketTimeoutMS ?? 0; this.monitorCommands = options.monitorCommands; this.serverApi = options.serverApi; @@ -754,7 +763,13 @@ function messageHandler(conn: Connection) { }; } -function streamIdentifier(stream: Stream) { +function streamIdentifier(stream: Stream, options: ConnectionOptions): string { + if (options.proxyOptions) { + // If proxy options are specified, the properties of `stream` itself + // will not accurately reflect what endpoint this is connected to. + return options.hostAddress.toString(); + } + if (typeof stream.address === 'function') { return `${stream.remoteAddress}:${stream.remotePort}`; } diff --git a/src/connection_string.ts b/src/connection_string.ts index ed897000dca..5c8ec4f87e5 100644 --- a/src/connection_string.ts +++ b/src/connection_string.ts @@ -27,6 +27,7 @@ import { ServerApi, ServerApiVersion } from './mongo_client'; +import type { ProxyOptions } from './cmap/connection'; import { MongoCredentials } from './cmap/auth/mongo_credentials'; import type { TagSet } from './sdam/server_description'; import { Logger, LoggerLevel } from './logger'; @@ -860,6 +861,33 @@ export const OPTIONS = { promoteValues: { type: 'boolean' }, + proxyHost: { + target: 'proxyOptions', + transform({ values: [value], options }): ProxyOptions { + return { ...options.proxyOptions, host: String(value) }; + } + } as OptionDescriptor, + proxyOptions: { + type: 'record' + }, + proxyPassword: { + target: 'proxyOptions', + transform({ values: [value], options }): ProxyOptions { + return { host: '', ...options.proxyOptions, password: String(value) || undefined }; + } + } as OptionDescriptor, + proxyPort: { + target: 'proxyOptions', + transform({ values: [value], options }): ProxyOptions { + return { host: '', ...options.proxyOptions, port: Number(value) }; + } + } as OptionDescriptor, + proxyUsername: { + target: 'proxyOptions', + transform({ values: [value], options }): ProxyOptions { + return { host: '', ...options.proxyOptions, username: String(value) || undefined }; + } + } as OptionDescriptor, raw: { default: false, type: 'boolean' diff --git a/src/deps.ts b/src/deps.ts index 35bd4ebf44e..d132114c6ce 100644 --- a/src/deps.ts +++ b/src/deps.ts @@ -262,6 +262,12 @@ export interface AutoEncryptionOptions { /** Command line arguments to use when auto-spawning a mongocryptd */ mongocryptdSpawnArgs?: string[]; }; + proxyOptions?: { + host: string; + port?: number; + username?: string; + password?: string; + }; } /** @public */ diff --git a/src/encrypter.ts b/src/encrypter.ts index 18cd7a1279d..63a36754033 100644 --- a/src/encrypter.ts +++ b/src/encrypter.ts @@ -46,6 +46,10 @@ export class Encrypter { options.autoEncryption.metadataClient = this.getInternalClient(client, uri, options); } + if (options.proxyOptions) { + options.autoEncryption.proxyOptions = options.proxyOptions; + } + options.autoEncryption.bson = Object.create(null); // eslint-disable-next-line @typescript-eslint/no-non-null-assertion options.autoEncryption.bson!.serialize = serialize; diff --git a/src/index.ts b/src/index.ts index 4a80f643bdd..cae4dbfab7f 100644 --- a/src/index.ts +++ b/src/index.ts @@ -189,7 +189,8 @@ export type { CommandOptions, QueryOptions, GetMoreOptions, - ConnectionEvents + ConnectionEvents, + ProxyOptions } from './cmap/connection'; export type { ConnectionPoolMetrics } from './cmap/metrics'; export type { diff --git a/src/mongo_client.ts b/src/mongo_client.ts index 7e4ccfef9b4..1e99089651b 100644 --- a/src/mongo_client.ts +++ b/src/mongo_client.ts @@ -33,7 +33,7 @@ import type { CompressorName } from './cmap/wire_protocol/compression'; import type { TLSSocketOptions, ConnectionOptions as TLSConnectionOptions } from 'tls'; import type { TcpNetConnectOpts } from 'net'; import type { SrvPoller } from './sdam/srv_polling'; -import type { Connection } from './cmap/connection'; +import type { Connection, ProxyOptions } from './cmap/connection'; import type { LEGAL_TLS_SOCKET_OPTIONS, LEGAL_TCP_SOCKET_OPTIONS } from './cmap/connect'; import type { Encrypter } from './encrypter'; import { TypedEventEmitter } from './mongo_types'; @@ -246,6 +246,8 @@ export interface MongoClientOptions extends BSONSerializeOptions, SupportedNodeC autoEncryption?: AutoEncryptionOptions; /** Allows a wrapping driver to amend the client metadata generated by the driver to include information about the wrapping driver */ driverInfo?: DriverInfo; + /** Configures a Socks5 proxy used for creating TCP connections. */ + proxyOptions?: ProxyOptions; /** @internal */ srvPoller?: SrvPoller; @@ -675,6 +677,7 @@ export interface MongoOptions dbName: string; metadata: ClientMetadata; autoEncrypter?: AutoEncrypter; + proxyOptions?: ProxyOptions; /** @internal */ connectionType?: typeof Connection; diff --git a/test/manual/socks5.test.js b/test/manual/socks5.test.js new file mode 100644 index 00000000000..dc36de7cfd4 --- /dev/null +++ b/test/manual/socks5.test.js @@ -0,0 +1,307 @@ +'use strict'; +const { MongoClient } = require('../../src'); +const { default: ConnectionString } = require('mongodb-connection-string-url'); +const { expect } = require('chai'); + +/** + * The SOCKS5_CONFIG environment variable is either a JSON 4-tuple + * [host, port, username, password] or just [host, port]. + */ + +describe('Socks5 Connectivity', function () { + if (!process.env.SOCKS5_CONFIG == null) { + console.error('skipping Socks5 tests, SOCKS5_CONFIG environment variable is not defined'); + + return; + } + + this.timeout(10000); + + const [proxyHost, proxyPort, proxyUsername, proxyPassword] = JSON.parse( + process.env.SOCKS5_CONFIG + ); + const rsConnectionString = new ConnectionString(process.env.MONGODB_URI); + const singleConnectionString = new ConnectionString(process.env.MONGODB_URI_SINGLEHOST); + + if (process.env.SSL === 'ssl') { + rsConnectionString.searchParams.set('tls', 'true'); + rsConnectionString.searchParams.set('tlsCAFile', process.env.SSL_CA_FILE); + singleConnectionString.searchParams.set('tls', 'true'); + singleConnectionString.searchParams.set('tlsCAFile', process.env.SSL_CA_FILE); + } + rsConnectionString.searchParams.set('serverSelectionTimeoutMS', '2000'); + singleConnectionString.searchParams.set('serverSelectionTimeoutMS', '2000'); + + context((proxyUsername ? 'with' : 'without') + ' Socks5 auth required', function () { + context('with missing required Socks5 auth configuration', function () { + if (!proxyUsername) { + beforeEach(function () { + this.skip(); + }); + } + + it('fails to connect to a single host (connection string)', async function () { + const cs = singleConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + cs.searchParams.set('directConnection', 'true'); + try { + await testConnection(cs.toString(), {}); + } catch (err) { + expect(err.name).to.equal('MongoServerSelectionError'); + expect(err.message).to.match(/Received invalid Socks5 initial handshake/); + return; + } + expect.fail('missed exception'); + }); + + it('fails to connect to a single host (config options)', async function () { + try { + await testConnection(singleConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort + }, + directConnection: true + }); + } catch (err) { + expect(err.name).to.equal('MongoServerSelectionError'); + expect(err.message).to.match(/Received invalid Socks5 initial handshake/); + return; + } + expect.fail('missed exception'); + }); + + it('fails to connect to a replica set (connection string)', async function () { + const cs = rsConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + try { + await testConnection(cs.toString(), {}); + } catch (err) { + expect(err.name).to.equal('MongoServerSelectionError'); + expect(err.message).to.match(/Received invalid Socks5 initial handshake/); + return; + } + expect.fail('missed exception'); + }); + + it('fails to connect to a replica set (config options)', async function () { + try { + await testConnection(rsConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort + } + }); + } catch (err) { + expect(err.name).to.equal('MongoServerSelectionError'); + expect(err.message).to.match(/Received invalid Socks5 initial handshake/); + return; + } + expect.fail('missed exception'); + }); + + it('fails to connect to a single host (connection string) if auth is present but wrong', async function () { + const cs = singleConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + cs.searchParams.set('proxyUsername', 'nonexistentuser'); + cs.searchParams.set('proxyPassword', 'badauth'); + cs.searchParams.set('directConnection', 'true'); + try { + await testConnection(cs.toString(), {}); + } catch (err) { + expect(err.name).to.equal('MongoServerSelectionError'); + expect(err.message).to.match(/Socket closed/); + return; + } + expect.fail('missed exception'); + }); + }); + + context('with extraneous Socks5 auth configuration', function () { + if (proxyUsername) { + beforeEach(function () { + this.skip(); + }); + } + + it('can connect to a single host (connection string)', async function () { + const cs = singleConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + cs.searchParams.set('proxyUsername', 'nonexistentuser'); + cs.searchParams.set('proxyPassword', 'badauth'); + cs.searchParams.set('directConnection', 'true'); + await testConnection(cs.toString(), {}); + }); + + it('can connect to a single host (config options)', async function () { + await testConnection(singleConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort, + ...(proxyUsername + ? {} + : { + username: 'nonexistentuser', + password: 'badauth' + }) + }, + directConnection: true + }); + }); + + it('can connect to a replica set (connection string)', async function () { + const cs = rsConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + cs.searchParams.set('proxyUsername', 'nonexistentuser'); + cs.searchParams.set('proxyPassword', 'badauth'); + await testConnection(cs.toString(), {}); + }); + + it('can connect to a replica set (config options)', async function () { + await testConnection(rsConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort, + ...(proxyUsername + ? {} + : { + username: 'nonexistentuser', + password: 'badauth' + }) + } + }); + }); + }); + + context('with matching socks5 authentication', () => { + it('can connect to a single host (connection string, with directConnection)', async function () { + const cs = singleConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + if (proxyUsername) { + cs.searchParams.set('proxyUsername', proxyUsername); + cs.searchParams.set('proxyPassword', proxyPassword); + } + cs.searchParams.set('directConnection', 'true'); + expect(await testConnection(cs.toString(), {})).to.equal('Single'); + }); + + it('can connect to a single host (config options, with directConnection)', async function () { + expect( + await testConnection(singleConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort, + ...(proxyUsername + ? { + username: proxyUsername, + password: proxyPassword + } + : {}) + }, + directConnection: true + }) + ).to.equal('Single'); + }); + + it('can connect to a single host (connection string, without directConnection)', async function () { + const cs = singleConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + if (proxyUsername) { + cs.searchParams.set('proxyUsername', proxyUsername); + cs.searchParams.set('proxyPassword', proxyPassword); + } + cs.searchParams.set('directConnection', 'false'); + expect(await testConnection(cs.toString(), {})).to.equal('ReplicaSetWithPrimary'); + }); + + it('can connect to a single host (config options, without directConnection)', async function () { + expect( + await testConnection(singleConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort, + ...(proxyUsername + ? { + username: proxyUsername, + password: proxyPassword + } + : {}) + }, + directConnection: false + }) + ).to.equal('ReplicaSetWithPrimary'); + }); + + it('can connect to a replica set (connection string)', async function () { + const cs = rsConnectionString.clone(); + cs.searchParams.set('proxyHost', proxyHost); + cs.searchParams.set('proxyPort', String(proxyPort)); + if (proxyUsername) { + cs.searchParams.set('proxyUsername', proxyUsername); + cs.searchParams.set('proxyPassword', proxyPassword); + } + expect(await testConnection(cs.toString(), {})).to.equal('ReplicaSetWithPrimary'); + }); + + it('can connect to a replica set (config options)', async function () { + expect( + await testConnection(rsConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort, + ...(proxyUsername + ? { + username: proxyUsername, + password: proxyPassword + } + : {}) + } + }) + ).to.equal('ReplicaSetWithPrimary'); + }); + + it('does not mention the proxy in command monitoring events', async function () { + const client = new MongoClient(singleConnectionString.toString(), { + proxyOptions: { + host: proxyHost, + port: proxyPort, + ...(proxyUsername + ? { + username: proxyUsername, + password: proxyPassword + } + : {}) + }, + directConnection: true, + monitorCommands: true + }); + const seenCommandAddresses = new Set(); + client.on('commandSucceeded', ev => seenCommandAddresses.add(ev.address)); + + await client.connect(); + await client.db('admin').command({ ismaster: 1 }); + await client.close(); + expect([...seenCommandAddresses]).to.deep.equal(singleConnectionString.hosts); + }); + }); + }); +}); + +async function testConnection(connectionString, clientOptions) { + const client = new MongoClient(connectionString, clientOptions); + let topologyType; + client.on('topologyDescriptionChanged', ev => (topologyType = ev.newDescription.type)); + + await client.connect(); + await client.db('admin').command({ ismaster: 1 }); + await client.db('test').collection('test').findOne({}); + await client.close(); + return topologyType; +}