Merge branch 'main' into batch_sizes

Taskfile.yml

@@ -501,8 +501,13 @@ tasks:
     desc: "Run security scanners"
     cmds:
       # don't run them in parallel via `deps` because that breaks terminal output
-      - task: security-trivy
       - task: security-govulncheck
+      - task: security-trivy
+
+  security-govulncheck:
+    cmds:
+      - bin/govulncheck{{exeExt}} -test ./...
+      - bin/task{{exeExt}} -d integration integration-security
 
   security-trivy:
     cmds:
@@ -513,11 +518,6 @@ tasks:
         --cache-dir=./tmp/trivy
         --exit-code=1
 
-  security-govulncheck:
-    cmds:
-      - bin/govulncheck{{exeExt}} -test ./...
-      - bin/task{{exeExt}} -d integration integration-security
-
   godocs:
     desc: "Serve Go code documentation"
     cmds:

go.mod

@@ -14,7 +14,7 @@ require (
 	github.com/google/uuid v1.6.0
 	github.com/jackc/pgerrcode v0.0.0-20220416144525-469b46aa5efa
 	github.com/jackc/pgx-zap v0.0.0-20221202020421-94b1cb2f889f
-	github.com/jackc/pgx/v5 v5.5.3
+	github.com/jackc/pgx/v5 v5.5.5
 	github.com/neilotoole/slogt v1.1.0
 	github.com/pmezard/go-difflib v1.0.0
 	github.com/prometheus/client_golang v1.18.0

go.sum

@@ -56,8 +56,8 @@ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgx-zap v0.0.0-20221202020421-94b1cb2f889f h1:ahoGnXfh4wiCisojvzq1PzgxzFwJEUHMI26pUY6oluk=
 github.com/jackc/pgx-zap v0.0.0-20221202020421-94b1cb2f889f/go.mod h1:m9tCxmy1PSUQa5o0aL4rQTowmJD1BK2Zc7dgnK/IrXc=
-github.com/jackc/pgx/v5 v5.5.3 h1:Ces6/M3wbDXYpM8JyyPD57ivTtJACFZJd885pdIaV2s=
-github.com/jackc/pgx/v5 v5.5.3/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
+github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
 github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
 github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=

integration/commands_administration_test.go

@@ -1711,47 +1711,67 @@ func TestCommandsAdministrationCompactForce(t *testing.T) {
 func TestCommandsAdministrationCompactCapped(t *testing.T) {
 	t.Parallel()
 
-	ctx, coll := setup.Setup(t)
-
 	for name, tc := range map[string]struct { //nolint:vet // for readability
 		force             bool
+		cleanupPercentage uint8 // optional, default value is 20
 		maxDocuments      int64
 		sizeInBytes       int64
 		insertDocuments   int32
-		expectedDocuments int64 // insertDocuments - insertDocuments*0.2 (cleanup 20%) + 1 (extra insert after compact)
+		expectedDocuments int64
 
 		skipForMongoDB string // optional, skip test for MongoDB backend with a specific reason
 	}{
 		"OverflowDocs": {
-			force:             true,
-			maxDocuments:      10,
-			sizeInBytes:       100000,
-			insertDocuments:   12, // overflows capped collection max documents
+			force:           true,
+			maxDocuments:    10,
+			sizeInBytes:     100000,
+			insertDocuments: 12,
+			// cleanup will be based on max documents
+			// maxDocuments + 1 (extra insert after compact)
 			expectedDocuments: 11,
 		},
 		"OverflowSize": {
-			force:             true,
-			maxDocuments:      1000,
-			sizeInBytes:       256,
-			insertDocuments:   20, // overflows capped collection size
+			force:           true,
+			sizeInBytes:     256,
+			insertDocuments: 20,
+			// cleanup will be based on size
+			// [insertDocuments * 0.2 (cleanup 20%)] + 1 (extra insert after compact)
 			expectedDocuments: 17,
 		},
+		"Cleanup10Percent": {
+			force:             true,
+			cleanupPercentage: 10,
+			sizeInBytes:       50,
+			insertDocuments:   5,
+			// cleanup will be based on size
+			// [insertDocuments * 0.1 (cleanup 10%) ≈ 0] + 1 (extra insert after compact)
+			expectedDocuments: 6,
+			skipForMongoDB:    "MongoDB cleans up collection precisely close to sizeInBytes, not based on percentage",
+		},
 		"ForceFalse": {
 			force:             false,
 			maxDocuments:      10,
 			sizeInBytes:       100000,
 			insertDocuments:   12, // overflows capped collection max documents
 			expectedDocuments: 11,
-			skipForMongoDB:    "Only {force:true} can be run on active replica set primary",
+			skipForMongoDB:    "Compact command with {force:false} cannot be executed on active replica set primary",
 		},
 	} {
 		name, tc := name, tc
 		t.Run(name, func(t *testing.T) {
+			t.Parallel()
+
 			if tc.skipForMongoDB != "" {
 				setup.SkipForMongoDB(t, tc.skipForMongoDB)
 			}
 
-			t.Parallel()
+			beOpts := setup.NewBackendOpts()
+			if tc.cleanupPercentage != 0 {
+				beOpts.CappedCleanupPercentage = tc.cleanupPercentage
+			}
+
+			s := setup.SetupWithOpts(t, &setup.SetupOpts{BackendOptions: beOpts})
+			ctx, coll := s.Ctx, s.Collection
 
 			collName := testutil.CollectionName(t) + name
 

integration/go.mod

@@ -39,7 +39,7 @@ require (
 	github.com/jackc/pgpassfile v1.0.0 // indirect
 	github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a // indirect
 	github.com/jackc/pgx-zap v0.0.0-20221202020421-94b1cb2f889f // indirect
-	github.com/jackc/pgx/v5 v5.5.3 // indirect
+	github.com/jackc/pgx/v5 v5.5.5 // indirect
 	github.com/jackc/puddle/v2 v2.2.1 // indirect
 	github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect
 	github.com/klauspost/compress v1.13.6 // indirect

integration/go.sum

@@ -48,8 +48,8 @@ github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a h1:bbPeKD0xmW/
 github.com/jackc/pgservicefile v0.0.0-20221227161230-091c0ba34f0a/go.mod h1:5TJZWKEWniPve33vlWYSoGYefn3gLQRzjfDlhSJ9ZKM=
 github.com/jackc/pgx-zap v0.0.0-20221202020421-94b1cb2f889f h1:ahoGnXfh4wiCisojvzq1PzgxzFwJEUHMI26pUY6oluk=
 github.com/jackc/pgx-zap v0.0.0-20221202020421-94b1cb2f889f/go.mod h1:m9tCxmy1PSUQa5o0aL4rQTowmJD1BK2Zc7dgnK/IrXc=
-github.com/jackc/pgx/v5 v5.5.3 h1:Ces6/M3wbDXYpM8JyyPD57ivTtJACFZJd885pdIaV2s=
-github.com/jackc/pgx/v5 v5.5.3/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
+github.com/jackc/pgx/v5 v5.5.5 h1:amBjrZVmksIdNjxGW/IiIMzxMKZFelXbUoPNb+8sjQw=
+github.com/jackc/pgx/v5 v5.5.5/go.mod h1:ez9gk+OAat140fv9ErkZDYFWmXLfV+++K0uAOiwgm1A=
 github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk=
 github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
 github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs=

integration/setup/listener.go

@@ -95,7 +95,7 @@ func listenerMongoDBURI(tb testtb.TB, hostPort, unixSocketPath string, tlsAndAut
 
 // setupListener starts in-process FerretDB server that runs until ctx is canceled.
 // It returns basic MongoDB URI for that listener.
-func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string {
+func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger, opts *BackendOpts) string {
 	tb.Helper()
 
 	_, span := otel.Tracer("").Start(ctx, "setupListener")
@@ -168,6 +168,8 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string
 	sp, err := state.NewProvider("")
 	require.NoError(tb, err)
 
+	require.NotNil(tb, opts)
+
 	handlerOpts := &registry.NewHandlerOpts{
 		Logger:        logger,
 		ConnMetrics:   listenerMetrics.ConnMetrics,
@@ -180,8 +182,8 @@ func setupListener(tb testtb.TB, ctx context.Context, logger *zap.Logger) string
 
 		TestOpts: registry.TestOpts{
 			DisablePushdown:         *disablePushdownF,
-			CappedCleanupPercentage: 20,
-			CappedCleanupInterval:   0,
+			CappedCleanupPercentage: opts.CappedCleanupPercentage,
+			CappedCleanupInterval:   opts.CappedCleanupInterval,
 			EnableNewAuth:           true,
 			BatchSize:               *batchSizeF,
 		},

integration/setup/setup.go

@@ -24,6 +24,7 @@ import (
 	"runtime/trace"
 	"slices"
 	"strings"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -96,6 +97,18 @@ type SetupOpts struct {
 
 	// SetupUser true creates a user and returns an authenticated client.
 	SetupUser bool
+
+	// Options to override default backend configuration.
+	BackendOptions *BackendOpts
+}
+
+// BackendOpts represents backend configuration used for test setup.
+type BackendOpts struct {
+	// Capped collections cleanup interval.
+	CappedCleanupInterval time.Duration
+
+	// Percentage of documents to cleanup for capped collections. If not set, defaults to 20.
+	CappedCleanupPercentage uint8
 }
 
 // SetupResult represents setup results.
@@ -105,6 +118,14 @@ type SetupResult struct {
 	MongoDBURI string
 }
 
+// NewBackendOpts returns BackendOpts with default values set.
+func NewBackendOpts() *BackendOpts {
+	return &BackendOpts{
+		CappedCleanupInterval:   time.Duration(0),
+		CappedCleanupPercentage: uint8(20),
+	}
+}
+
 // IsUnixSocket returns true if MongoDB URI is a Unix domain socket.
 func (s *SetupResult) IsUnixSocket(tb testtb.TB) bool {
 	tb.Helper()
@@ -144,7 +165,11 @@ func SetupWithOpts(tb testtb.TB, opts *SetupOpts) *SetupResult {
 
 	uri := *targetURLF
 	if uri == "" {
-		uri = setupListener(tb, setupCtx, logger)
+		if opts.BackendOptions == nil {
+			opts.BackendOptions = NewBackendOpts()
+		}
+
+		uri = setupListener(tb, setupCtx, logger, opts.BackendOptions)
 	} else {
 		uri = toAbsolutePathURI(tb, *targetURLF)
 	}

integration/setup/setup_compat.go

@@ -103,7 +103,7 @@ func SetupCompatWithOpts(tb testtb.TB, opts *SetupCompatOpts) *SetupCompatResult
 
 	var targetClient *mongo.Client
 	if *targetURLF == "" {
-		uri := setupListener(tb, setupCtx, logger)
+		uri := setupListener(tb, setupCtx, logger, NewBackendOpts())
 		targetClient = setupClient(tb, setupCtx, uri)
 	} else {
 		targetClient = setupClient(tb, setupCtx, *targetURLF)