Fix race condition when looking up reaper (ryuk) container

docker.go

@@ -86,6 +86,8 @@ type DockerContainer struct {
 	logProductionTimeout *time.Duration
 	logger               Logging
 	lifecycleHooks       []ContainerLifecycleHooks
+
+	healthStatus string // container health status, will default to healthStatusNone if no healthcheck is present
 }
 
 // SetLogger sets the logger for the container
@@ -1590,6 +1592,11 @@ func containerFromDockerResponse(ctx context.Context, response types.Container)
 		return nil, err
 	}
 
+	// the health status of the container, if any
+	if health := container.raw.State.Health; health != nil {
+		container.healthStatus = health.Status
+	}
+
 	return &container, nil
 }
 

reaper.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"context"
 	"fmt"
+	"github.com/docker/docker/api/types"
 	"math/rand"
 	"net"
 	"strings"
@@ -117,6 +118,13 @@ func lookUpReaperContainer(ctx context.Context, sessionID string) (*DockerContai
 			return err
 		}
 
+		// if a health status is present on the container, and the container is not healthy, error
+		if r.healthStatus != "" {
+			if r.healthStatus != types.Healthy && r.healthStatus != types.NoHealthcheck {
+				return fmt.Errorf("container %s is not healthy, wanted status=%s, got status=%s", resp[0].ID[:8], types.Healthy, r.healthStatus)
+			}
+		}
+
 		reaperContainer = r
 
 		return nil
@@ -162,6 +170,27 @@ func reuseOrCreateReaper(ctx context.Context, sessionID string, provider ReaperP
 		if err != nil {
 			return nil, err
 		}
+
+		Logger.Printf("⏳ Waiting for Reaper port to be ready")
+
+		var containerJson *types.ContainerJSON
+
+		if containerJson, err = reaperContainer.Inspect(ctx); err != nil {
+			return nil, fmt.Errorf("failed to inspect reaper container %s: %w", reaperContainer.ID[:8], err)
+		}
+
+		if containerJson != nil && containerJson.NetworkSettings != nil {
+			for port := range containerJson.NetworkSettings.Ports {
+				err := wait.ForListeningPort(port).
+					WithPollInterval(100*time.Millisecond).
+					WaitUntilReady(ctx, reaperContainer)
+				if err != nil {
+					return nil, fmt.Errorf("failed waiting for reaper container %s port %s/%s to be ready: %w",
+						reaperContainer.ID[:8], port.Proto(), port.Port(), err)
+				}
+			}
+		}
+
 		return reaperInstance, nil
 	}