Merge branch 'main' into chore/options-error

* main: (21 commits)
  feat: optimizes file copies to and from containers (testcontainers#2450)
  fix(exec): updates the `Multiplexed` opt to combine stdout and stderr (testcontainers#2452)
  Upgrade neo4j module to use features from v0.29.1 of testcontainers-go (testcontainers#2463)
  bug:Fix AMQPS url (testcontainers#2462)
  chore: more compose updates in comments
  chore: use "docker compose" (v2) instead of "docker-compose" (v1) (testcontainers#2464)
  chore(deps): bump github/codeql-action from 2.22.12 to 3.24.9 (testcontainers#2459)
  refactor: Add Weaviate modules tests (testcontainers#2447)
  feat(exitcode): Add exit code sugar method (testcontainers#2342)
  feat: add module to support InfluxDB v1.x (testcontainers#1703)
  feat: authenticate docker on PullImage (testcontainers#2446)
  feat: add distribution-registry module (testcontainers#2341)
  chore(deps): Bumping ChromaGo client version (testcontainers#2402)
  chore(deps): bump github.com/docker/docker from 25.0.3+incompatible to 25.0.5+incompatible (testcontainers#2444)
  feat: support passing io.Reader as ContainerFile (testcontainers#2401)
  chore: bump ryuk to latest (testcontainers#2395)
  feat(MustConn): Add MustConnectionString on (some) dbs (testcontainers#2343)
  fix: typo in ci-test-go.yml (testcontainers#2394)
  feat: support for waiting for response headers (testcontainers#2349)
  chore(deps): bump google.golang.org/protobuf from 1.32.0 to 1.33.0 (testcontainers#2392)
  ...

.github/workflows/ci-test-go.yml

@@ -55,7 +55,7 @@ jobs:
         if: ${{ inputs.rootless-docker }}
         uses: ScribeMD/rootless-docker@6bd157a512c2fafa4e0243a8aa87d964eb890886  # v0.2.2
 
-      - name: Remove Docket root socket
+      - name: Remove Docker root socket
         if: ${{ inputs.rootless-docker }}
         run: sudo rm -rf /var/run/docker.sock
 

.github/workflows/ci.yml

@@ -94,7 +94,7 @@ jobs:
       matrix:
         go-version: [1.21.x, 1.x]
         platform: [ubuntu-latest]
-        module: [artemis, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, elasticsearch, gcloud, inbucket, k3s, k6, kafka, localstack, mariadb, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, surrealdb, vault, weaviate]
+        module: [artemis, cassandra, chroma, clickhouse, cockroachdb, compose, consul, couchbase, elasticsearch, gcloud, inbucket, influxdb, k3s, k6, kafka, localstack, mariadb, milvus, minio, mockserver, mongodb, mssql, mysql, nats, neo4j, ollama, openfga, openldap, opensearch, postgres, pulsar, qdrant, rabbitmq, redis, redpanda, registry, surrealdb, vault, weaviate]
     uses: ./.github/workflows/ci-test-go.yml
     with:
       go-version: ${{ matrix.go-version }}

.github/workflows/codeql.yml

@@ -53,7 +53,7 @@ jobs:
 
     # Initializes the CodeQL tools for scanning.
     - name: Initialize CodeQL
-      uses: github/codeql-action/init@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # v3.23.0
+      uses: github/codeql-action/init@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9
       with:
         languages: ${{ matrix.language }}
         # If you wish to specify custom queries, you can do so here or in a config file.
@@ -67,7 +67,7 @@ jobs:
     # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
     # If this step fails, then you should remove it and run the build manually (see below)
     - name: Autobuild
-      uses: github/codeql-action/autobuild@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # v3.23.0
+      uses: github/codeql-action/autobuild@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9
 
     # ℹ️ Command-line programs to run using the OS shell.
     # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -80,6 +80,6 @@ jobs:
     #     ./location_of_script_within_repo/buildscript.sh
 
     - name: Perform CodeQL Analysis
-      uses: github/codeql-action/analyze@e5f05b81d5b6ff8cfa111c80c22c5fd02a384118 # v3.23.0
+      uses: github/codeql-action/analyze@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9
       with:
         category: "/language:${{matrix.language}}"

.github/workflows/scorecards.yml

@@ -51,6 +51,6 @@ jobs:
 
       # required for Code scanning alerts
       - name: "Upload SARIF results to code scanning"
-        uses: github/codeql-action/upload-sarif@1500a131381b66de0c52ac28abb13cd79f4b7ecc # v2.22.12
+        uses: github/codeql-action/upload-sarif@1b1aada464948af03b950897e5eb522f92603cc2 # v3.24.9
         with:
           sarif_file: results.sarif

.gitignore

@@ -11,3 +11,5 @@ src/pip-delete-this-directory.txt
 .DS_Store
 
 TEST-*.xml
+
+**/go.work

.vscode/.testcontainers-go.code-workspace

@@ -57,6 +57,10 @@
             "name": "module / inbucket",
             "path": "../modules/inbucket"
         },
+        {
+            "name": "module / influxdb",
+            "path": "../modules/influxdb"
+        },
         {
             "name": "module / k3s",
             "path": "../modules/k3s"
@@ -149,6 +153,10 @@
             "name": "module / redpanda",
             "path": "../modules/redpanda"
         },
+        {
+            "name": "module / registry",
+            "path": "../modules/registry"
+        },
         {
             "name": "module / surrealdb",
             "path": "../modules/surrealdb"

container.go

@@ -99,11 +99,25 @@ type FromDockerfile struct {
 }
 
 type ContainerFile struct {
-	HostFilePath      string
+	HostFilePath      string    // If Reader is present, HostFilePath is ignored
+	Reader            io.Reader // If Reader is present, HostFilePath is ignored
 	ContainerFilePath string
 	FileMode          int64
 }
 
+// validate validates the ContainerFile
+func (c *ContainerFile) validate() error {
+	if c.HostFilePath == "" && c.Reader == nil {
+		return errors.New("either HostFilePath or Reader must be specified")
+	}
+
+	if c.ContainerFilePath == "" {
+		return errors.New("ContainerFilePath must be specified")
+	}
+
+	return nil
+}
+
 // ContainerRequest represents the parameters used to get a running container
 type ContainerRequest struct {
 	FromDockerfile

container_file_test.go

@@ -0,0 +1,73 @@
+// This test is testing very internal logic that should not be exported away from this package. We'll
+// leave it in the main testcontainers package. Do not use for user facing examples.
+package testcontainers
+
+import (
+	"errors"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+func TestContainerFileValidation(t *testing.T) {
+	type ContainerFileValidationTestCase struct {
+		Name          string
+		ExpectedError error
+		File          ContainerFile
+	}
+
+	f, err := os.Open(filepath.Join(".", "testdata", "hello.sh"))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	testTable := []ContainerFileValidationTestCase{
+		{
+			Name: "valid container file: has hostfilepath",
+			File: ContainerFile{
+				HostFilePath:      "/path/to/host",
+				ContainerFilePath: "/path/to/container",
+			},
+		},
+		{
+			Name: "valid container file: has reader",
+			File: ContainerFile{
+				Reader:            f,
+				ContainerFilePath: "/path/to/container",
+			},
+		},
+		{
+			Name:          "invalid container file",
+			ExpectedError: errors.New("either HostFilePath or Reader must be specified"),
+			File: ContainerFile{
+				HostFilePath:      "",
+				Reader:            nil,
+				ContainerFilePath: "/path/to/container",
+			},
+		},
+		{
+			Name:          "invalid container file",
+			ExpectedError: errors.New("ContainerFilePath must be specified"),
+			File: ContainerFile{
+				HostFilePath:      "/path/to/host",
+				ContainerFilePath: "",
+			},
+		},
+	}
+
+	for _, testCase := range testTable {
+		t.Run(testCase.Name, func(t *testing.T) {
+			err := testCase.File.validate()
+			switch {
+			case err == nil && testCase.ExpectedError == nil:
+				return
+			case err == nil && testCase.ExpectedError != nil:
+				t.Errorf("did not receive expected error: %s", testCase.ExpectedError.Error())
+			case err != nil && testCase.ExpectedError == nil:
+				t.Errorf("received unexpected error: %s", err.Error())
+			case err.Error() != testCase.ExpectedError.Error():
+				t.Errorf("errors mismatch: %s != %s", err.Error(), testCase.ExpectedError.Error())
+			}
+		})
+	}
+}

docker.go

@@ -11,6 +11,7 @@ import (
 	"errors"
 	"fmt"
 	"io"
+	"io/fs"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -602,19 +603,41 @@ func (c *DockerContainer) CopyFileToContainer(ctx context.Context, hostFilePath
 		return c.CopyDirToContainer(ctx, hostFilePath, containerFilePath, fileMode)
 	}
 
-	fileContent, err := os.ReadFile(hostFilePath)
+	f, err := os.Open(hostFilePath)
 	if err != nil {
 		return err
 	}
-	return c.CopyToContainer(ctx, fileContent, containerFilePath, fileMode)
+	defer f.Close()
+
+	info, err := f.Stat()
+	if err != nil {
+		return err
+	}
+
+	// In Go 1.22 os.File is always an io.WriterTo. However, testcontainers
+	// currently allows Go 1.21, so we need to trick the compiler a little.
+	var file fs.File = f
+	return c.copyToContainer(ctx, func(tw io.Writer) error {
+		// Attempt optimized writeTo, implemented in linux
+		if wt, ok := file.(io.WriterTo); ok {
+			_, err := wt.WriteTo(tw)
+			return err
+		}
+		_, err := io.Copy(tw, f)
+		return err
+	}, info.Size(), containerFilePath, fileMode)
 }
 
 // CopyToContainer copies fileContent data to a file in container
 func (c *DockerContainer) CopyToContainer(ctx context.Context, fileContent []byte, containerFilePath string, fileMode int64) error {
-	buffer, err := tarFile(fileContent, containerFilePath, fileMode)
-	if err != nil {
+	return c.copyToContainer(ctx, func(tw io.Writer) error {
+		_, err := tw.Write(fileContent)
 		return err
-	}
+	}, int64(len(fileContent)), containerFilePath, fileMode)
+}
+
+func (c *DockerContainer) copyToContainer(ctx context.Context, fileContent func(tw io.Writer) error, fileContentSize int64, containerFilePath string, fileMode int64) error {
+	buffer, err := tarFile(containerFilePath, fileContent, fileContentSize, fileMode)
 
 	err = c.provider.client.CopyToContainer(ctx, c.ID, "/", buffer, types.CopyToContainerOptions{})
 	if err != nil {
@@ -1027,20 +1050,6 @@ func (p *DockerProvider) CreateContainer(ctx context.Context, req ContainerReque
 			pullOpt := types.ImagePullOptions{
 				Platform: req.ImagePlatform, // may be empty
 			}
-
-			registry, imageAuth, err := DockerImageAuth(ctx, imageName)
-			if err != nil {
-				p.Logger.Printf("Failed to get image auth for %s. Setting empty credentials for the image: %s. Error is:%s", registry, imageName, err)
-			} else {
-				// see https://github.com/docker/docs/blob/e8e1204f914767128814dca0ea008644709c117f/engine/api/sdk/examples.md?plain=1#L649-L657
-				encodedJSON, err := json.Marshal(imageAuth)
-				if err != nil {
-					p.Logger.Printf("Failed to marshal image auth. Setting empty credentials for the image: %s. Error is:%s", imageName, err)
-				} else {
-					pullOpt.RegistryAuth = base64.URLEncoding.EncodeToString(encodedJSON)
-				}
-			}
-
 			if err := p.attemptToPullImage(ctx, imageName, pullOpt); err != nil {
 				return nil, err
 			}
@@ -1245,10 +1254,20 @@ func (p *DockerProvider) ReuseOrCreateContainer(ctx context.Context, req Contain
 // attemptToPullImage tries to pull the image while respecting the ctx cancellations.
 // Besides, if the image cannot be pulled due to ErrorNotFound then no need to retry but terminate immediately.
 func (p *DockerProvider) attemptToPullImage(ctx context.Context, tag string, pullOpt types.ImagePullOptions) error {
-	var (
-		err  error
-		pull io.ReadCloser
-	)
+	registry, imageAuth, err := DockerImageAuth(ctx, tag)
+	if err != nil {
+		p.Logger.Printf("Failed to get image auth for %s. Setting empty credentials for the image: %s. Error is:%s", registry, tag, err)
+	} else {
+		// see https://github.com/docker/docs/blob/e8e1204f914767128814dca0ea008644709c117f/engine/api/sdk/examples.md?plain=1#L649-L657
+		encodedJSON, err := json.Marshal(imageAuth)
+		if err != nil {
+			p.Logger.Printf("Failed to marshal image auth. Setting empty credentials for the image: %s. Error is:%s", tag, err)
+		} else {
+			pullOpt.RegistryAuth = base64.URLEncoding.EncodeToString(encodedJSON)
+		}
+	}
+
+	var pull io.ReadCloser
 	err = backoff.Retry(func() error {
 		pull, err = p.client.ImagePull(ctx, tag, pullOpt)
 		if err != nil {
@@ -1577,7 +1596,8 @@ func (p *DockerProvider) SaveImages(ctx context.Context, output string, images .
 		_ = imageReader.Close()
 	}()
 
-	_, err = io.Copy(outputFile, imageReader)
+	// Attempt optimized readFrom, implemented in linux
+	_, err = outputFile.ReadFrom(imageReader)
 	if err != nil {
 		return fmt.Errorf("writing images to output %w", err)
 	}