|
| 1 | +import subprocess |
| 2 | +from dataclasses import dataclass, field, fields |
| 3 | +from functools import cached_property |
| 4 | +from json import loads |
| 5 | +from os import PathLike |
| 6 | +from re import split |
| 7 | +from typing import Callable, Literal, Optional, TypeVar, Union |
| 8 | +from urllib.error import HTTPError, URLError |
| 9 | +from urllib.request import urlopen |
| 10 | + |
| 11 | +from testcontainers.core.exceptions import ContainerIsNotRunning, NoSuchPortExposed |
| 12 | +from testcontainers.core.waiting_utils import wait_container_is_ready |
| 13 | + |
| 14 | +_IPT = TypeVar("_IPT") |
| 15 | + |
| 16 | + |
| 17 | +def _ignore_properties(cls: type[_IPT], dict_: any) -> _IPT: |
| 18 | + """omits extra fields like @JsonIgnoreProperties(ignoreUnknown = true) |
| 19 | +
|
| 20 | + https://gist.github.com/alexanderankin/2a4549ac03554a31bef6eaaf2eaf7fd5""" |
| 21 | + if isinstance(dict_, cls): |
| 22 | + return dict_ |
| 23 | + class_fields = {f.name for f in fields(cls)} |
| 24 | + filtered = {k: v for k, v in dict_.items() if k in class_fields} |
| 25 | + return cls(**filtered) |
| 26 | + |
| 27 | + |
| 28 | +@dataclass |
| 29 | +class PublishedPort: |
| 30 | + """ |
| 31 | + Class that represents the response we get from compose when inquiring status |
| 32 | + via `DockerCompose.get_running_containers()`. |
| 33 | + """ |
| 34 | + |
| 35 | + URL: Optional[str] = None |
| 36 | + TargetPort: Optional[str] = None |
| 37 | + PublishedPort: Optional[str] = None |
| 38 | + Protocol: Optional[str] = None |
| 39 | + |
| 40 | + |
| 41 | +OT = TypeVar("OT") |
| 42 | + |
| 43 | + |
| 44 | +def get_only_element_or_raise(array: list[OT], exception: Callable[[], Exception]) -> OT: |
| 45 | + if len(array) != 1: |
| 46 | + e = exception() |
| 47 | + raise e |
| 48 | + return array[0] |
| 49 | + |
| 50 | + |
| 51 | +@dataclass |
| 52 | +class ComposeContainer: |
| 53 | + """ |
| 54 | + A container class that represents a container managed by compose. |
| 55 | + It is not a true testcontainers.core.container.DockerContainer, |
| 56 | + but you can use the id with DockerClient to get that one too. |
| 57 | + """ |
| 58 | + |
| 59 | + ID: Optional[str] = None |
| 60 | + Name: Optional[str] = None |
| 61 | + Command: Optional[str] = None |
| 62 | + Project: Optional[str] = None |
| 63 | + Service: Optional[str] = None |
| 64 | + State: Optional[str] = None |
| 65 | + Health: Optional[str] = None |
| 66 | + ExitCode: Optional[str] = None |
| 67 | + Publishers: list[PublishedPort] = field(default_factory=list) |
| 68 | + |
| 69 | + def __post_init__(self): |
| 70 | + if self.Publishers: |
| 71 | + self.Publishers = [_ignore_properties(PublishedPort, p) for p in self.Publishers] |
| 72 | + |
| 73 | + def get_publisher( |
| 74 | + self, |
| 75 | + by_port: Optional[int] = None, |
| 76 | + by_host: Optional[str] = None, |
| 77 | + prefer_ip_version: Literal["IPV4", "IPv6"] = "IPv4", |
| 78 | + ) -> PublishedPort: |
| 79 | + remaining_publishers = self.Publishers |
| 80 | + |
| 81 | + remaining_publishers = [r for r in remaining_publishers if self._matches_protocol(prefer_ip_version, r)] |
| 82 | + |
| 83 | + if by_port: |
| 84 | + remaining_publishers = [item for item in remaining_publishers if by_port == item.TargetPort] |
| 85 | + if by_host: |
| 86 | + remaining_publishers = [item for item in remaining_publishers if by_host == item.URL] |
| 87 | + if len(remaining_publishers) == 0: |
| 88 | + raise NoSuchPortExposed(f"Could not find publisher for for service {self.Service}") |
| 89 | + return get_only_element_or_raise( |
| 90 | + remaining_publishers, |
| 91 | + lambda: NoSuchPortExposed( |
| 92 | + "get_publisher failed because there is " |
| 93 | + f"not exactly 1 publisher for service {self.Service}" |
| 94 | + f" when filtering by_port={by_port}, by_host={by_host}" |
| 95 | + f" (but {len(remaining_publishers)})" |
| 96 | + ), |
| 97 | + ) |
| 98 | + |
| 99 | + @staticmethod |
| 100 | + def _matches_protocol(prefer_ip_version, r): |
| 101 | + return (":" in r.URL) is (prefer_ip_version == "IPv6") |
| 102 | + |
| 103 | + |
| 104 | +@dataclass |
| 105 | +class DockerCompose: |
| 106 | + """ |
| 107 | + Manage docker compose environments. |
| 108 | +
|
| 109 | + Args: |
| 110 | + context: |
| 111 | + The docker context. It corresponds to the directory containing |
| 112 | + the docker compose configuration file. |
| 113 | + compose_file_name: |
| 114 | + Optional. File name of the docker compose configuration file. |
| 115 | + If specified, you need to also specify the overrides if any. |
| 116 | + pull: |
| 117 | + Pull images before launching environment. |
| 118 | + build: |
| 119 | + Run `docker compose build` before running the environment. |
| 120 | + wait: |
| 121 | + Wait for the services to be healthy |
| 122 | + (as per healthcheck definitions in the docker compose configuration) |
| 123 | + env_file: |
| 124 | + Path to an '.env' file containing environment variables |
| 125 | + to pass to docker compose. |
| 126 | + services: |
| 127 | + The list of services to use from this DockerCompose. |
| 128 | + client_args: |
| 129 | + arguments to pass to docker.from_env() |
| 130 | +
|
| 131 | + Example: |
| 132 | +
|
| 133 | + This example spins up chrome and firefox containers using docker compose. |
| 134 | +
|
| 135 | + .. doctest:: |
| 136 | +
|
| 137 | + >>> from testcontainers.compose import DockerCompose |
| 138 | +
|
| 139 | + >>> compose = DockerCompose("compose/tests", compose_file_name="docker-compose-4.yml", |
| 140 | + ... pull=True) |
| 141 | + >>> with compose: |
| 142 | + ... stdout, stderr = compose.get_logs() |
| 143 | + >>> b"Hello from Docker!" in stdout |
| 144 | + True |
| 145 | +
|
| 146 | + .. code-block:: yaml |
| 147 | +
|
| 148 | + services: |
| 149 | + hello-world: |
| 150 | + image: "hello-world" |
| 151 | + """ |
| 152 | + |
| 153 | + context: Union[str, PathLike] |
| 154 | + compose_file_name: Optional[Union[str, list[str]]] = None |
| 155 | + pull: bool = False |
| 156 | + build: bool = False |
| 157 | + wait: bool = True |
| 158 | + env_file: Optional[str] = None |
| 159 | + services: Optional[list[str]] = None |
| 160 | + |
| 161 | + def __post_init__(self): |
| 162 | + if isinstance(self.compose_file_name, str): |
| 163 | + self.compose_file_name = [self.compose_file_name] |
| 164 | + |
| 165 | + def __enter__(self) -> "DockerCompose": |
| 166 | + self.start() |
| 167 | + return self |
| 168 | + |
| 169 | + def __exit__(self, exc_type, exc_val, exc_tb) -> None: |
| 170 | + self.stop() |
| 171 | + |
| 172 | + def docker_compose_command(self) -> list[str]: |
| 173 | + """ |
| 174 | + Returns command parts used for the docker compose commands |
| 175 | +
|
| 176 | + Returns: |
| 177 | + cmd: Docker compose command parts. |
| 178 | + """ |
| 179 | + return self.compose_command_property |
| 180 | + |
| 181 | + @cached_property |
| 182 | + def compose_command_property(self) -> list[str]: |
| 183 | + docker_compose_cmd = ["docker", "compose"] |
| 184 | + if self.compose_file_name: |
| 185 | + for file in self.compose_file_name: |
| 186 | + docker_compose_cmd += ["-f", file] |
| 187 | + if self.env_file: |
| 188 | + docker_compose_cmd += ["--env-file", self.env_file] |
| 189 | + return docker_compose_cmd |
| 190 | + |
| 191 | + def start(self) -> None: |
| 192 | + """ |
| 193 | + Starts the docker compose environment. |
| 194 | + """ |
| 195 | + base_cmd = self.compose_command_property or [] |
| 196 | + |
| 197 | + # pull means running a separate command before starting |
| 198 | + if self.pull: |
| 199 | + pull_cmd = [*base_cmd, "pull"] |
| 200 | + self._call_command(cmd=pull_cmd) |
| 201 | + |
| 202 | + up_cmd = [*base_cmd, "up"] |
| 203 | + |
| 204 | + # build means modifying the up command |
| 205 | + if self.build: |
| 206 | + up_cmd.append("--build") |
| 207 | + |
| 208 | + if self.wait: |
| 209 | + up_cmd.append("--wait") |
| 210 | + else: |
| 211 | + # we run in detached mode instead of blocking |
| 212 | + up_cmd.append("--detach") |
| 213 | + |
| 214 | + if self.services: |
| 215 | + up_cmd.extend(self.services) |
| 216 | + |
| 217 | + self._call_command(cmd=up_cmd) |
| 218 | + |
| 219 | + def stop(self, down=True) -> None: |
| 220 | + """ |
| 221 | + Stops the docker compose environment. |
| 222 | + """ |
| 223 | + down_cmd = self.compose_command_property[:] |
| 224 | + if down: |
| 225 | + down_cmd += ["down", "--volumes"] |
| 226 | + else: |
| 227 | + down_cmd += ["stop"] |
| 228 | + self._call_command(cmd=down_cmd) |
| 229 | + |
| 230 | + def get_logs(self, *services: str) -> tuple[str, str]: |
| 231 | + """ |
| 232 | + Returns all log output from stdout and stderr of a specific container. |
| 233 | +
|
| 234 | + :param services: which services to get the logs for (or omit, for all) |
| 235 | +
|
| 236 | + Returns: |
| 237 | + stdout: Standard output stream. |
| 238 | + stderr: Standard error stream. |
| 239 | + """ |
| 240 | + logs_cmd = [*self.compose_command_property, "logs", *services] |
| 241 | + |
| 242 | + result = subprocess.run( |
| 243 | + logs_cmd, |
| 244 | + cwd=self.context, |
| 245 | + capture_output=True, |
| 246 | + ) |
| 247 | + return result.stdout.decode("utf-8"), result.stderr.decode("utf-8") |
| 248 | + |
| 249 | + def get_containers(self, include_all=False) -> list[ComposeContainer]: |
| 250 | + """ |
| 251 | + Fetch information about running containers via `docker compose ps --format json`. |
| 252 | + Available only in V2 of compose. |
| 253 | +
|
| 254 | + Returns: |
| 255 | + The list of running containers. |
| 256 | +
|
| 257 | + """ |
| 258 | + |
| 259 | + cmd = [*self.compose_command_property, "ps", "--format", "json"] |
| 260 | + if include_all: |
| 261 | + cmd = [*cmd, "-a"] |
| 262 | + result = subprocess.run(cmd, cwd=self.context, check=True, stdout=subprocess.PIPE) |
| 263 | + stdout = split(r"\r?\n", result.stdout.decode("utf-8")) |
| 264 | + |
| 265 | + containers = [] |
| 266 | + # one line per service in docker 25, single array for docker 24.0.2 |
| 267 | + for line in stdout: |
| 268 | + if not line: |
| 269 | + continue |
| 270 | + data = loads(line) |
| 271 | + if isinstance(data, list): |
| 272 | + containers += [_ignore_properties(ComposeContainer, d) for d in data] |
| 273 | + else: |
| 274 | + containers.append(_ignore_properties(ComposeContainer, data)) |
| 275 | + |
| 276 | + return containers |
| 277 | + |
| 278 | + def get_container( |
| 279 | + self, |
| 280 | + service_name: Optional[str] = None, |
| 281 | + include_all: bool = False, |
| 282 | + ) -> ComposeContainer: |
| 283 | + if not service_name: |
| 284 | + containers = self.get_containers(include_all=include_all) |
| 285 | + return get_only_element_or_raise( |
| 286 | + containers, |
| 287 | + lambda: ContainerIsNotRunning( |
| 288 | + "get_container failed because no service_name given " |
| 289 | + f"and there is not exactly 1 container (but {len(containers)})" |
| 290 | + ), |
| 291 | + ) |
| 292 | + |
| 293 | + matching_containers = [ |
| 294 | + item for item in self.get_containers(include_all=include_all) if item.Service == service_name |
| 295 | + ] |
| 296 | + |
| 297 | + if not matching_containers: |
| 298 | + raise ContainerIsNotRunning(f"{service_name} is not running in the compose context") |
| 299 | + |
| 300 | + return matching_containers[0] |
| 301 | + |
| 302 | + def exec_in_container( |
| 303 | + self, |
| 304 | + command: list[str], |
| 305 | + service_name: Optional[str] = None, |
| 306 | + ) -> tuple[str, str, int]: |
| 307 | + """ |
| 308 | + Executes a command in the container of one of the services. |
| 309 | +
|
| 310 | + Args: |
| 311 | + service_name: Name of the docker compose service to run the command in. |
| 312 | + command: Command to execute. |
| 313 | +
|
| 314 | + :param service_name: specify the service name |
| 315 | + :param command: the command to run in the container |
| 316 | +
|
| 317 | + Returns: |
| 318 | + stdout: Standard output stream. |
| 319 | + stderr: Standard error stream. |
| 320 | + exit_code: The command's exit code. |
| 321 | + """ |
| 322 | + if not service_name: |
| 323 | + service_name = self.get_container().Service |
| 324 | + exec_cmd = [*self.compose_command_property, "exec", "-T", service_name, *command] |
| 325 | + result = subprocess.run( |
| 326 | + exec_cmd, |
| 327 | + cwd=self.context, |
| 328 | + capture_output=True, |
| 329 | + check=True, |
| 330 | + ) |
| 331 | + |
| 332 | + return (result.stdout.decode("utf-8"), result.stderr.decode("utf-8"), result.returncode) |
| 333 | + |
| 334 | + def _call_command( |
| 335 | + self, |
| 336 | + cmd: Union[str, list[str]], |
| 337 | + context: Optional[str] = None, |
| 338 | + ) -> None: |
| 339 | + context = context or self.context |
| 340 | + subprocess.call(cmd, cwd=context) |
| 341 | + |
| 342 | + def get_service_port( |
| 343 | + self, |
| 344 | + service_name: Optional[str] = None, |
| 345 | + port: Optional[int] = None, |
| 346 | + ): |
| 347 | + """ |
| 348 | + Returns the mapped port for one of the services. |
| 349 | +
|
| 350 | + Parameters |
| 351 | + ---------- |
| 352 | + service_name: str |
| 353 | + Name of the docker compose service |
| 354 | + port: int |
| 355 | + The internal port to get the mapping for |
| 356 | +
|
| 357 | + Returns |
| 358 | + ------- |
| 359 | + str: |
| 360 | + The mapped port on the host |
| 361 | + """ |
| 362 | + return self.get_container(service_name).get_publisher(by_port=port).PublishedPort |
| 363 | + |
| 364 | + def get_service_host( |
| 365 | + self, |
| 366 | + service_name: Optional[str] = None, |
| 367 | + port: Optional[int] = None, |
| 368 | + ): |
| 369 | + """ |
| 370 | + Returns the host for one of the services. |
| 371 | +
|
| 372 | + Parameters |
| 373 | + ---------- |
| 374 | + service_name: str |
| 375 | + Name of the docker compose service |
| 376 | + port: int |
| 377 | + The internal port to get the host for |
| 378 | +
|
| 379 | + Returns |
| 380 | + ------- |
| 381 | + str: |
| 382 | + The hostname for the service |
| 383 | + """ |
| 384 | + return self.get_container(service_name).get_publisher(by_port=port).URL |
| 385 | + |
| 386 | + def get_service_host_and_port( |
| 387 | + self, |
| 388 | + service_name: Optional[str] = None, |
| 389 | + port: Optional[int] = None, |
| 390 | + ): |
| 391 | + publisher = self.get_container(service_name).get_publisher(by_port=port) |
| 392 | + return publisher.URL, publisher.PublishedPort |
| 393 | + |
| 394 | + @wait_container_is_ready(HTTPError, URLError) |
| 395 | + def wait_for(self, url: str) -> "DockerCompose": |
| 396 | + """ |
| 397 | + Waits for a response from a given URL. This is typically used to block until a service in |
| 398 | + the environment has started and is responding. Note that it does not assert any sort of |
| 399 | + return code, only check that the connection was successful. |
| 400 | +
|
| 401 | + Args: |
| 402 | + url: URL from one of the services in the environment to use to wait on. |
| 403 | + """ |
| 404 | + with urlopen(url) as response: |
| 405 | + response.read() |
| 406 | + return self |
0 commit comments