Merge branch 'master' into godriver2904

.evergreen/config.yml

@@ -437,37 +437,18 @@ functions:
           make -s evg-test-enterprise-auth
 
   run-atlas-test:
+    - command: ec2.assume_role
+      params:
+        role_arn: "${aws_test_secrets_role}"
     - command: shell.exec
       type: test
       params:
         shell: "bash"
         working_dir: src/go.mongodb.org/mongo-driver
+        include_expansions_in_env: ["AWS_ACCESS_KEY_ID", "AWS_SECRET_ACCESS_KEY", "AWS_SESSION_TOKEN"]
         script: |
-          # DO NOT ECHO WITH XTRACE
-          if [ "Windows_NT" = "$OS" ]; then
-            export GOPATH=$(cygpath -w $(dirname $(dirname $(dirname `pwd`))))
-            export GOCACHE=$(cygpath -w "$(pwd)/.cache")
-          else
-            export GOPATH=$(dirname $(dirname $(dirname `pwd`)))
-            export GOCACHE="$(pwd)/.cache"
-          fi;
-          export GOPATH="$GOPATH"
-          export GOROOT="${GO_DIST}"
-          export GOCACHE="$GOCACHE"
-          export PATH="${GCC_PATH}:${GO_DIST}/bin:$PATH"
-          export ATLAS_FREE="${atlas_free_tier_uri}"
-          export ATLAS_REPLSET="${atlas_replica_set_uri}"
-          export ATLAS_SHARD="${atlas_sharded_uri}"
-          export ATLAS_TLS11="${atlas_tls_v11_uri}"
-          export ATLAS_TLS12="${atlas_tls_v12_uri}"
-          export ATLAS_FREE_SRV="${atlas_free_tier_uri_srv}"
-          export ATLAS_REPLSET_SRV="${atlas_replica_set_uri_srv}"
-          export ATLAS_SHARD_SRV="${atlas_sharded_uri_srv}"
-          export ATLAS_TLS11_SRV="${atlas_tls_v11_uri_srv}"
-          export ATLAS_TLS12_SRV="${atlas_tls_v12_uri_srv}"
-          export ATLAS_SERVERLESS="${atlas_serverless_uri}"
-          export ATLAS_SERVERLESS_SRV="${atlas_serverless_uri_srv}"
-          make -s evg-test-atlas
+          ${PREPARE_SHELL}
+          bash etc/run-atlas-test.sh
 
   run-ocsp-test:
     - command: shell.exec
@@ -2228,7 +2209,7 @@ tasks:
           export AZUREKMS_VMNAME=${AZUREKMS_VMNAME}
           echo '${testazurekms_privatekey}' > /tmp/testazurekms.prikey
           export AZUREKMS_PRIVATEKEYPATH=/tmp/testazurekms.prikey
-          AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
+          AZUREKMS_CMD="LD_LIBRARY_PATH=./install/libmongocrypt/lib MONGODB_URI='mongodb://localhost:27017' PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' ./testkms" $DRIVERS_TOOLS/.evergreen/csfle/azurekms/run-command.sh
 
   - name: "testazurekms-fail-task"
     # testazurekms-fail-task runs without environment variables.
@@ -2250,7 +2231,7 @@ tasks:
           LD_LIBRARY_PATH=./install/libmongocrypt/lib \
           MONGODB_URI='mongodb://localhost:27017' \
           EXPECT_ERROR='unable to retrieve azure credentials' \
-          PROVIDER='azure' \
+          PROVIDER='azure' AZUREKMS_KEY_NAME='${AZUREKMS_KEY_NAME}' AZUREKMS_KEY_VAULT_ENDPOINT='${AZUREKMS_KEY_VAULT_ENDPOINT}' \
             ./testkms
 
   - name: "test-fuzz"

.gitignore

@@ -24,4 +24,8 @@ internal/test/compilecheck/compilecheck.so
 
 # Ignore api report files
 api-report.md
-api-report.txt
+api-report.txt
+
+# Ignore secrets files
+secrets-expansion.yml
+secrets-export.sh

Makefile

@@ -1,4 +1,3 @@
-ATLAS_URIS = "$(ATLAS_FREE)" "$(ATLAS_REPLSET)" "$(ATLAS_SHARD)" "$(ATLAS_TLS11)" "$(ATLAS_TLS12)" "$(ATLAS_FREE_SRV)" "$(ATLAS_REPLSET_SRV)" "$(ATLAS_SHARD_SRV)" "$(ATLAS_TLS11_SRV)" "$(ATLAS_TLS12_SRV)" "$(ATLAS_SERVERLESS)" "$(ATLAS_SERVERLESS_SRV)"
 TEST_TIMEOUT = 1800
 
 ### Utility targets. ###
@@ -128,10 +127,6 @@ build-aws-ecs-test:
 evg-test:
 	go test -exec "env PKG_CONFIG_PATH=$(PKG_CONFIG_PATH) LD_LIBRARY_PATH=$(LD_LIBRARY_PATH)" $(BUILD_TAGS) -v -timeout $(TEST_TIMEOUT)s -p 1 ./... >> test.suite
 
-.PHONY: evg-test-atlas
-evg-test-atlas:
-	go run ./cmd/testatlas/main.go $(ATLAS_URIS)
-
 .PHONY: evg-test-atlas-data-lake
 evg-test-atlas-data-lake:
 	ATLAS_DATA_LAKE_INTEGRATION_TEST=true go test -v ./mongo/integration -run TestUnifiedSpecs/atlas-data-lake-testing >> spec_test.suite

cmd/testatlas/main.go

@@ -23,7 +23,11 @@ func main() {
 	uris := flag.Args()
 	ctx := context.Background()
 
+	fmt.Printf("Running atlas tests for %d uris\n", len(uris))
+
 	for idx, uri := range uris {
+		fmt.Printf("Running test %d\n", idx)
+
 		// Set a low server selection timeout so we fail fast if there are errors.
 		clientOpts := options.Client().
 			ApplyURI(uri).
@@ -41,6 +45,8 @@ func main() {
 			panic(fmt.Sprintf("error running test with tlsInsecure at index %d: %v", idx, err))
 		}
 	}
+
+	fmt.Println("Finished!")
 }
 
 func runTest(ctx context.Context, clientOpts *options.ClientOptions) error {

cmd/testkms/main.go

@@ -24,8 +24,8 @@ var datakeyopts = map[string]primitive.M{
 		"key":    "arn:aws:kms:us-east-1:579766882180:key/89fcc2c4-08b0-4bd9-9f25-e30687b580d0",
 	},
 	"azure": bson.M{
-		"keyVaultEndpoint": "https://keyvault-drivers-2411.vault.azure.net/keys/",
-		"keyName":          "KEY-NAME",
+		"keyVaultEndpoint": "",
+		"keyName":          "",
 	},
 	"gcp": bson.M{
 		"projectId": "devprod-drivers",
@@ -53,6 +53,20 @@ func main() {
 	default:
 		ok = true
 	}
+	if provider == "azure" {
+		azureKmsKeyName := os.Getenv("AZUREKMS_KEY_NAME")
+		azureKmsKeyVaultEndpoint := os.Getenv("AZUREKMS_KEY_VAULT_ENDPOINT")
+		if azureKmsKeyName == "" {
+			fmt.Println("ERROR: Please set required AZUREKMS_KEY_NAME environment variable.")
+			ok = false
+		}
+		if azureKmsKeyVaultEndpoint == "" {
+			fmt.Println("ERROR: Please set required AZUREKMS_KEY_VAULT_ENDPOINT environment variable.")
+			ok = false
+		}
+		datakeyopts["azure"]["keyName"] = azureKmsKeyName
+		datakeyopts["azure"]["keyVaultEndpoint"] = azureKmsKeyVaultEndpoint
+	}
 	if !ok {
 		providers := make([]string, 0, len(datakeyopts))
 		for p := range datakeyopts {
@@ -63,6 +77,8 @@ func main() {
 		fmt.Println("- MONGODB_URI as a MongoDB URI. Example: 'mongodb://localhost:27017'")
 		fmt.Println("- EXPECT_ERROR as an optional expected error substring.")
 		fmt.Println("- PROVIDER as a KMS provider, which supports:", strings.Join(providers, ", "))
+		fmt.Println("- AZUREKMS_KEY_NAME as the Azure key name. Required if PROVIDER=azure.")
+		fmt.Println("- AZUREKMS_KEY_VAULT_ENDPOINT as the Azure key name. Required if PROVIDER=azure.")
 		os.Exit(1)
 	}
 

etc/get_aws_secrets.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+# get-aws-secrets
+# Gets AWS secrets from the vault
+set -eu
+
+if [ -z "$DRIVERS_TOOLS" ]; then
+    echo "Please define DRIVERS_TOOLS variable"
+    exit 1
+fi
+
+bash $DRIVERS_TOOLS/.evergreen/auth_aws/setup_secrets.sh $@
+. ./secrets-export.sh

etc/run-atlas-test.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+# run-atlas-test
+# Run atlas connectivity tests.
+set -eu
+set +x
+
+# Get the atlas secrets.
+. etc/get_aws_secrets.sh drivers/atlas_connect
+
+echo "Running cmd/testatlas/main.go"
+go run ./cmd/testatlas/main.go "$ATLAS_REPL" "$ATLAS_SHRD" "$ATLAS_FREE" "$ATLAS_TLS11" "$ATLAS_TLS12" "$ATLAS_SERVERLESS" "$ATLAS_SRV_REPL" "$ATLAS_SRV_SHRD" "$ATLAS_SRV_FREE" "$ATLAS_SRV_TLS11" "$ATLAS_SRV_TLS12" "$ATLAS_SRV_SERVERLESS"

mongo/errors.go

@@ -102,50 +102,56 @@ func replaceErrors(err error) error {
 	return err
 }
 
-// IsDuplicateKeyError returns true if err is a duplicate key error
+// IsDuplicateKeyError returns true if err is a duplicate key error.
 func IsDuplicateKeyError(err error) bool {
-	// handles SERVER-7164 and SERVER-11493
-	for ; err != nil; err = unwrap(err) {
-		if e, ok := err.(ServerError); ok {
-			return e.HasErrorCode(11000) || e.HasErrorCode(11001) || e.HasErrorCode(12582) ||
-				e.HasErrorCodeWithMessage(16460, " E11000 ")
-		}
+	if se := ServerError(nil); errors.As(err, &se) {
+		return se.HasErrorCode(11000) || // Duplicate key error.
+			se.HasErrorCode(11001) || // Duplicate key error on update.
+			// Duplicate key error in a capped collection. See SERVER-7164.
+			se.HasErrorCode(12582) ||
+			// Mongos insert error caused by a duplicate key error. See
+			// SERVER-11493.
+			se.HasErrorCodeWithMessage(16460, " E11000 ")
 	}
 	return false
 }
 
-// IsTimeout returns true if err is from a timeout
+// timeoutErrs is a list of error values that indicate a timeout happened.
+var timeoutErrs = [...]error{
+	context.DeadlineExceeded,
+	driver.ErrDeadlineWouldBeExceeded,
+	topology.ErrServerSelectionTimeout,
+}
+
+// IsTimeout returns true if err was caused by a timeout. For error chains,
+// IsTimeout returns true if any error in the chain was caused by a timeout.
 func IsTimeout(err error) bool {
-	for ; err != nil; err = unwrap(err) {
-		// check unwrappable errors together
-		if err == context.DeadlineExceeded {
-			return true
-		}
-		if err == driver.ErrDeadlineWouldBeExceeded {
-			return true
-		}
-		if err == topology.ErrServerSelectionTimeout {
-			return true
-		}
-		if _, ok := err.(topology.WaitQueueTimeoutError); ok {
-			return true
-		}
-		if ce, ok := err.(CommandError); ok && ce.IsMaxTimeMSExpiredError() {
+	// Check if the error chain contains any of the timeout error values.
+	for _, target := range timeoutErrs {
+		if errors.Is(err, target) {
 			return true
 		}
-		if we, ok := err.(WriteException); ok && we.WriteConcernError != nil &&
-			we.WriteConcernError.IsMaxTimeMSExpiredError() {
+	}
+
+	// Check if the error chain contains any error types that can indicate
+	// timeout.
+	if errors.As(err, &topology.WaitQueueTimeoutError{}) {
+		return true
+	}
+	if ce := (CommandError{}); errors.As(err, &ce) && ce.IsMaxTimeMSExpiredError() {
+		return true
+	}
+	if we := (WriteException{}); errors.As(err, &we) && we.WriteConcernError != nil && we.WriteConcernError.IsMaxTimeMSExpiredError() {
+		return true
+	}
+	if ne := net.Error(nil); errors.As(err, &ne) {
+		return ne.Timeout()
+	}
+	// Check timeout error labels.
+	if le := LabeledError(nil); errors.As(err, &le) {
+		if le.HasErrorLabel("NetworkTimeoutError") || le.HasErrorLabel("ExceededTimeLimitError") {
 			return true
 		}
-		if ne, ok := err.(net.Error); ok {
-			return ne.Timeout()
-		}
-		//timeout error labels
-		if le, ok := err.(LabeledError); ok {
-			if le.HasErrorLabel("NetworkTimeoutError") || le.HasErrorLabel("ExceededTimeLimitError") {
-				return true
-			}
-		}
 	}
 
 	return false

mongo/integration/client_side_encryption_prose_test.go

@@ -378,9 +378,6 @@ func TestClientSideEncryptionProse(t *testing.T) {
 		}
 	})
 	mt.Run("4. bson size limits", func(mt *mtest.T) {
-		// TODO(GODRIVER-2872): Fix and unskip this test case.
-		mt.Skip("Test fails frequently, skipping. See GODRIVER-2872")
-
 		kmsProviders := map[string]map[string]interface{}{
 			"local": {
 				"key": localMasterKey,

mongo/integration/client_test.go

@@ -126,8 +126,6 @@ func TestClient(t *testing.T) {
 			"expected security field to be type %v, got %v", bson.TypeMaxKey, security.Type)
 		_, found := security.Document().LookupErr("SSLServerSubjectName")
 		assert.Nil(mt, found, "SSLServerSubjectName not found in result")
-		_, found = security.Document().LookupErr("SSLServerHasCertificateAuthority")
-		assert.Nil(mt, found, "SSLServerHasCertificateAuthority not found in result")
 	})
 	mt.RunOpts("x509", mtest.NewOptions().Auth(true).SSL(true), func(mt *mtest.T) {
 		testCases := []struct {
@@ -711,7 +709,7 @@ func TestClient(t *testing.T) {
 			err := mt.Client.Ping(ctx, nil)
 			cancel()
 			assert.NotNil(mt, err, "expected Ping to return an error")
-			assert.True(mt, mongo.IsTimeout(err), "expected a timeout error: got %v", err)
+			assert.True(mt, mongo.IsTimeout(err), "expected a timeout error, got: %v", err)
 		}
 
 		// Assert that the Ping timeouts result in no connections being closed.
@@ -733,8 +731,8 @@ func TestClient(t *testing.T) {
 		pair := msgPairs[0]
 		assert.Equal(mt, handshake.LegacyHello, pair.CommandName, "expected command name %s at index 0, got %s",
 			handshake.LegacyHello, pair.CommandName)
-		assert.Equal(mt, wiremessage.OpMsg, pair.Sent.OpCode,
-			"expected 'OP_MSG' OpCode in wire message, got %q", pair.Sent.OpCode.String())
+		assert.Equal(mt, wiremessage.OpQuery, pair.Sent.OpCode,
+			"expected 'OP_QUERY' OpCode in wire message, got %q", pair.Sent.OpCode.String())
 
 		// Look for a saslContinue in the remaining proxied messages and assert that it uses the OP_MSG OpCode, as wire
 		// version is now known to be >= 6.

mongo/integration/mtest/sent_message.go

@@ -37,6 +37,8 @@ type sentMsgParseFn func([]byte) (*SentMessage, error)
 
 func getSentMessageParser(opcode wiremessage.OpCode) (sentMsgParseFn, bool) {
 	switch opcode {
+	case wiremessage.OpQuery:
+		return parseOpQuery, true
 	case wiremessage.OpMsg:
 		return parseSentOpMsg, true
 	case wiremessage.OpCompressed:
@@ -46,6 +48,69 @@ func getSentMessageParser(opcode wiremessage.OpCode) (sentMsgParseFn, bool) {
 	}
 }
 
+func parseOpQuery(wm []byte) (*SentMessage, error) {
+	var ok bool
+
+	if _, wm, ok = wiremessage.ReadQueryFlags(wm); !ok {
+		return nil, errors.New("failed to read query flags")
+	}
+	if _, wm, ok = wiremessage.ReadQueryFullCollectionName(wm); !ok {
+		return nil, errors.New("failed to read full collection name")
+	}
+	if _, wm, ok = wiremessage.ReadQueryNumberToSkip(wm); !ok {
+		return nil, errors.New("failed to read number to skip")
+	}
+	if _, wm, ok = wiremessage.ReadQueryNumberToReturn(wm); !ok {
+		return nil, errors.New("failed to read number to return")
+	}
+
+	query, wm, ok := wiremessage.ReadQueryQuery(wm)
+	if !ok {
+		return nil, errors.New("failed to read query")
+	}
+
+	// If there is no read preference document, the command document is query.
+	// Otherwise, query is in the format {$query: <command document>, $readPreference: <read preference document>}.
+	commandDoc := query
+	var rpDoc bsoncore.Document
+
+	dollarQueryVal, err := query.LookupErr("$query")
+	if err == nil {
+		commandDoc = dollarQueryVal.Document()
+
+		rpVal, err := query.LookupErr("$readPreference")
+		if err != nil {
+			return nil, fmt.Errorf("query %s contains $query but not $readPreference fields", query)
+		}
+		rpDoc = rpVal.Document()
+	}
+
+	// For OP_QUERY, inserts, updates, and deletes are sent as a BSON array of documents inside the main command
+	// document. Pull these sequences out into an ArrayStyle DocumentSequence.
+	var docSequence *bsoncore.DocumentSequence
+	cmdElems, _ := commandDoc.Elements()
+	for _, elem := range cmdElems {
+		switch elem.Key() {
+		case "documents", "updates", "deletes":
+			docSequence = &bsoncore.DocumentSequence{
+				Style: bsoncore.ArrayStyle,
+				Data:  elem.Value().Array(),
+			}
+		}
+		if docSequence != nil {
+			// There can only be one of these arrays in a well-formed command, so we exit the loop once one is found.
+			break
+		}
+	}
+
+	sm := &SentMessage{
+		Command:          commandDoc,
+		ReadPreference:   rpDoc,
+		DocumentSequence: docSequence,
+	}
+	return sm, nil
+}
+
 func parseSentMessage(wm []byte) (*SentMessage, error) {
 	// Re-assign the wire message to "remaining" so "wm" continues to point to the entire message after parsing.
 	_, requestID, _, opcode, remaining, ok := wiremessage.ReadHeader(wm)