From 297e4a609990b9237e276102b170e62cdaa1fde0 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Aug 2023 19:37:29 +0100 Subject: [PATCH 1/4] Add args param to build-frontend option --- cibuildwheel/linux.py | 11 ++--- cibuildwheel/macos.py | 19 +++++---- cibuildwheel/oci_container.py | 4 +- cibuildwheel/options.py | 24 +++++------ cibuildwheel/util.py | 49 ++++++++++++++++++----- cibuildwheel/windows.py | 19 +++++---- test/test_build_frontend_args.py | 34 ++++++++++++++++ unit_test/main_tests/main_options_test.py | 2 +- 8 files changed, 115 insertions(+), 47 deletions(-) create mode 100644 test/test_build_frontend_args.py diff --git a/cibuildwheel/linux.py b/cibuildwheel/linux.py index a4d7c184d..0c6184ef0 100644 --- a/cibuildwheel/linux.py +++ b/cibuildwheel/linux.py @@ -16,9 +16,9 @@ from .typing import PathOrStr from .util import ( AlreadyBuiltWheelError, + BuildFrontendConfig, BuildSelector, NonPlatformWheelError, - build_frontend_or_default, find_compatible_wheel, get_build_verbosity_extra_flags, prepare_command, @@ -177,7 +177,7 @@ def build_in_container( for config in platform_configs: log.build_start(config.identifier) build_options = options.build_options(config.identifier) - build_frontend = build_frontend_or_default(build_options.build_frontend) + build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") dependency_constraint_flags: list[PathOrStr] = [] @@ -243,9 +243,10 @@ def build_in_container( container.call(["rm", "-rf", built_wheel_dir]) container.call(["mkdir", "-p", built_wheel_dir]) - extra_flags = split_config_settings(build_options.config_settings, build_frontend) + extra_flags = split_config_settings(build_options.config_settings, build_frontend.name) + extra_flags += build_frontend.args - if build_frontend == "pip": + if build_frontend.name == "pip": extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity) container.call( [ @@ -260,7 +261,7 @@ def build_in_container( ], env=env, ) - elif build_frontend == "build": + elif build_frontend.name == "build": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) diff --git a/cibuildwheel/macos.py b/cibuildwheel/macos.py index ddc7c4697..77ae492a6 100644 --- a/cibuildwheel/macos.py +++ b/cibuildwheel/macos.py @@ -25,10 +25,10 @@ from .util import ( CIBW_CACHE_PATH, AlreadyBuiltWheelError, - BuildFrontend, + BuildFrontendConfig, + BuildFrontendName, BuildSelector, NonPlatformWheelError, - build_frontend_or_default, call, detect_ci_provider, download, @@ -165,7 +165,7 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, - build_frontend: BuildFrontend, + build_frontend: BuildFrontendName, ) -> dict[str, str]: tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] @@ -334,7 +334,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_frontend_or_default(build_options.build_frontend) + build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") log.build_start(config.identifier) identifier_tmp_dir = tmp_path / config.identifier @@ -357,7 +357,7 @@ def build(options: Options, tmp_path: Path) -> None: config, dependency_constraint_flags, build_options.environment, - build_frontend, + build_frontend.name, ) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) @@ -378,9 +378,12 @@ def build(options: Options, tmp_path: Path) -> None: log.step("Building wheel...") built_wheel_dir.mkdir() - extra_flags = split_config_settings(build_options.config_settings, build_frontend) + extra_flags = split_config_settings( + build_options.config_settings, build_frontend.name + ) + extra_flags += build_frontend.args - if build_frontend == "pip": + if build_frontend.name == "pip": extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity) # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/pypa/cibuildwheel/pull/369 @@ -395,7 +398,7 @@ def build(options: Options, tmp_path: Path) -> None: *extra_flags, env=env, ) - elif build_frontend == "build": + elif build_frontend.name == "build": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) diff --git a/cibuildwheel/oci_container.py b/cibuildwheel/oci_container.py index 1ea22edd9..966ab9686 100644 --- a/cibuildwheel/oci_container.py +++ b/cibuildwheel/oci_container.py @@ -29,7 +29,9 @@ class OCIContainerEngineConfig: @staticmethod def from_config_string(config_string: str) -> OCIContainerEngineConfig: - config_dict = parse_key_value_string(config_string, ["name"]) + config_dict = parse_key_value_string( + config_string, ["name"], ["create_args", "create-args"] + ) name = " ".join(config_dict["name"]) if name not in {"docker", "podman"}: msg = f"unknown container engine {name}" diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index 581e576b8..d85eb02b5 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -27,7 +27,7 @@ from .util import ( MANYLINUX_ARCHS, MUSLLINUX_ARCHS, - BuildFrontend, + BuildFrontendConfig, BuildSelector, DependencyConstraints, TestSelector, @@ -92,7 +92,7 @@ class BuildOptions: test_requires: list[str] test_extras: str build_verbosity: int - build_frontend: BuildFrontend | Literal["default"] + build_frontend: BuildFrontendConfig | None config_settings: str @property @@ -488,7 +488,6 @@ def build_options(self, identifier: str | None) -> BuildOptions: with self.reader.identifier(identifier): before_all = self.reader.get("before-all", sep=" && ") - build_frontend_str = self.reader.get("build-frontend", env_plat=False) environment_config = self.reader.get( "environment", table={"item": '{k}="{v}"', "sep": " "} ) @@ -506,17 +505,16 @@ def build_options(self, identifier: str | None) -> BuildOptions: test_extras = self.reader.get("test-extras", sep=",") build_verbosity_str = self.reader.get("build-verbosity") - build_frontend: BuildFrontend | Literal["default"] - if build_frontend_str == "build": - build_frontend = "build" - elif build_frontend_str == "pip": - build_frontend = "pip" - elif build_frontend_str == "default": - build_frontend = "default" + build_frontend_str = self.reader.get("build-frontend", env_plat=False) + build_frontend: BuildFrontendConfig | None + if not build_frontend_str or build_frontend_str == "default": + build_frontend = None else: - msg = f"cibuildwheel: Unrecognised build frontend {build_frontend_str!r}, only 'pip' and 'build' are supported" - print(msg, file=sys.stderr) - sys.exit(2) + try: + build_frontend = BuildFrontendConfig.from_config_string(build_frontend_str) + except ValueError as e: + print(f"cibuildwheel: {e}", file=sys.stderr) + sys.exit(2) try: environment = parse_environment(environment_config) diff --git a/cibuildwheel/util.py b/cibuildwheel/util.py index 7875ea404..793e5cd95 100644 --- a/cibuildwheel/util.py +++ b/cibuildwheel/util.py @@ -57,16 +57,6 @@ test_fail_cwd_file: Final[Path] = resources_dir / "testing_temp_dir_file.py" -BuildFrontend = Literal["pip", "build"] - - -def build_frontend_or_default( - setting: BuildFrontend | Literal["default"], default: BuildFrontend = "pip" -) -> BuildFrontend: - if setting == "default": - return default - return setting - MANYLINUX_ARCHS: Final[tuple[str, ...]] = ( "x86_64", @@ -376,6 +366,34 @@ def options_summary(self) -> Any: return self.base_file_path.name +BuildFrontendName = Literal["pip", "build"] + + +@dataclass(frozen=True) +class BuildFrontendConfig: + name: BuildFrontendName + args: Sequence[str] = () + + @staticmethod + def from_config_string(config_string: str) -> BuildFrontendConfig: + config_dict = parse_key_value_string(config_string, ["name"], ["args"]) + name = " ".join(config_dict["name"]) + if name not in {"pip", "build"}: + msg = f"Unrecognised build frontend {name}, only 'pip' and 'build' are supported" + raise ValueError(msg) + + name = typing.cast(BuildFrontendName, name) + + args = config_dict.get("args") or [] + return BuildFrontendConfig(name=name, args=args) + + def options_summary(self) -> str | dict[str, str]: + if not self.args: + return self.name + else: + return {"name": self.name, "args": repr(self.args)} + + class NonPlatformWheelError(Exception): def __init__(self) -> None: message = textwrap.dedent( @@ -699,13 +717,19 @@ def fix_ansi_codes_for_github_actions(text: str) -> str: def parse_key_value_string( - key_value_string: str, positional_arg_names: list[str] | None = None + key_value_string: str, + positional_arg_names: Sequence[str] | None = None, + kw_arg_names: Sequence[str] | None = None, ) -> dict[str, list[str]]: """ Parses a string like "docker; create_args: --some-option=value another-option" """ if positional_arg_names is None: positional_arg_names = [] + if kw_arg_names is None: + kw_arg_names = [] + + all_field_names = [*positional_arg_names, *kw_arg_names] shlexer = shlex.shlex(key_value_string, posix=True, punctuation_chars=";:") shlexer.commenters = "" @@ -721,6 +745,9 @@ def parse_key_value_string( if len(field) > 1 and field[1] == ":": field_name = field[0] values = field[2:] + if field_name not in all_field_names: + msg = f"Failed to parse {key_value_string!r}. Unknown field name {field_name!r}" + raise ValueError(msg) else: try: field_name = positional_arg_names[field_i] diff --git a/cibuildwheel/windows.py b/cibuildwheel/windows.py index c291d96da..881821bdd 100644 --- a/cibuildwheel/windows.py +++ b/cibuildwheel/windows.py @@ -25,10 +25,10 @@ from .util import ( CIBW_CACHE_PATH, AlreadyBuiltWheelError, - BuildFrontend, + BuildFrontendConfig, + BuildFrontendName, BuildSelector, NonPlatformWheelError, - build_frontend_or_default, call, download, find_compatible_wheel, @@ -216,7 +216,7 @@ def setup_python( python_configuration: PythonConfiguration, dependency_constraint_flags: Sequence[PathOrStr], environment: ParsedEnvironment, - build_frontend: BuildFrontend, + build_frontend: BuildFrontendName, ) -> dict[str, str]: tmp.mkdir() implementation_id = python_configuration.identifier.split("-")[0] @@ -369,7 +369,7 @@ def build(options: Options, tmp_path: Path) -> None: for config in python_configurations: build_options = options.build_options(config.identifier) - build_frontend = build_frontend_or_default(build_options.build_frontend) + build_frontend = build_options.build_frontend or BuildFrontendConfig("pip") log.build_start(config.identifier) identifier_tmp_dir = tmp_path / config.identifier @@ -390,7 +390,7 @@ def build(options: Options, tmp_path: Path) -> None: config, dependency_constraint_flags, build_options.environment, - build_frontend, + build_frontend.name, ) compatible_wheel = find_compatible_wheel(built_wheels, config.identifier) @@ -414,9 +414,12 @@ def build(options: Options, tmp_path: Path) -> None: log.step("Building wheel...") built_wheel_dir.mkdir() - extra_flags = split_config_settings(build_options.config_settings, build_frontend) + extra_flags = split_config_settings( + build_options.config_settings, build_frontend.name + ) + extra_flags += build_frontend.args - if build_frontend == "pip": + if build_frontend.name == "pip": extra_flags += get_build_verbosity_extra_flags(build_options.build_verbosity) # Path.resolve() is needed. Without it pip wheel may try to fetch package from pypi.org # see https://github.com/pypa/cibuildwheel/pull/369 @@ -431,7 +434,7 @@ def build(options: Options, tmp_path: Path) -> None: *extra_flags, env=env, ) - elif build_frontend == "build": + elif build_frontend.name == "build": if not 0 <= build_options.build_verbosity < 2: msg = f"build_verbosity {build_options.build_verbosity} is not supported for build frontend. Ignoring." log.warning(msg) diff --git a/test/test_build_frontend_args.py b/test/test_build_frontend_args.py new file mode 100644 index 000000000..c2b0fc37c --- /dev/null +++ b/test/test_build_frontend_args.py @@ -0,0 +1,34 @@ +import subprocess + +import pytest + +from . import utils +from .test_projects.c import new_c_project + + +@pytest.mark.parametrize("frontend_name", ["pip", "build"]) +def test_build_frontend_args(tmp_path, capfd, frontend_name): + project = new_c_project() + project_dir = tmp_path / "project" + project.generate(project_dir) + + # the build will fail because the frontend is called with '-h' - it prints the help message + with pytest.raises(subprocess.CalledProcessError): + utils.cibuildwheel_run( + project_dir, + add_env={ + "CIBW_BUILD": "cp311-*", + "CIBW_BUILD_FRONTEND": f"{frontend_name}; args: -h", + }, + ) + + captured = capfd.readouterr() + print(captured.out) + + # check that the help message was printed + if frontend_name == "pip": + assert "Usage:" in captured.out + assert "Wheel Options:" in captured.out + else: + assert "usage:" in captured.out + assert "A simple, correct Python build frontend." in captured.out diff --git a/unit_test/main_tests/main_options_test.py b/unit_test/main_tests/main_options_test.py index 8d2010ee3..5b85da73b 100644 --- a/unit_test/main_tests/main_options_test.py +++ b/unit_test/main_tests/main_options_test.py @@ -365,7 +365,7 @@ def test_defaults(platform, intercepted_build_args): if isinstance(repair_wheel_default, list): repair_wheel_default = " && ".join(repair_wheel_default) assert build_options.repair_command == repair_wheel_default - assert build_options.build_frontend == defaults["build-frontend"] + assert build_options.build_frontend is None if platform == "linux": assert build_options.manylinux_images From 061ea627e2fe9870f547a4c8a56ff7b8887ab8c2 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Sat, 26 Aug 2023 19:47:49 +0100 Subject: [PATCH 2/4] Add docs --- docs/options.md | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/docs/options.md b/docs/options.md index 8df4e8f25..b597684ce 100644 --- a/docs/options.md +++ b/docs/options.md @@ -504,9 +504,19 @@ This option can also be set using the [command-line option](#command-line) `--pr ### `CIBW_BUILD_FRONTEND` {: #build-frontend} > Set the tool to use to build, either "pip" (default for now) or "build" -Choose which build backend to use. Can either be "pip", which will run +Options: + +- `pip[;args: ...]` +- `build[;args: ...]` + +Default: `pip` + +Choose which build frontend to use. Can either be "pip", which will run `python -m pip wheel`, or "build", which will run `python -m build --wheel`. +You can specify extra arguments to pass to `pip wheel` or `build` using the +optional `args` option. + !!! tip Until v2.0.0, [pip] was the only way to build wheels, and is still the default. However, we expect that at some point in the future, cibuildwheel @@ -526,6 +536,9 @@ Choose which build backend to use. Can either be "pip", which will run # Ensure pip is used even if the default changes in the future CIBW_BUILD_FRONTEND: "pip" + + # supply an extra argument to 'pip wheel' + CIBW_BUILD_FRONTEND: "pip; args: --no-build-isolation" ``` !!! tab examples "pyproject.toml" @@ -537,6 +550,9 @@ Choose which build backend to use. Can either be "pip", which will run # Ensure pip is used even if the default changes in the future build-frontend = "pip" + + # supply an extra argument to 'pip wheel' + build-frontend = { name = "pip", args = ["--no-build-isolation"] } ``` ### `CIBW_CONFIG_SETTINGS` {: #config-settings} From 81ef5fac4304fda24f0a96e5b04eb986b4435b6b Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 30 Aug 2023 09:28:34 +0100 Subject: [PATCH 3/4] Add a unit test for the TOML form of the option --- unit_test/options_test.py | 54 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/unit_test/options_test.py b/unit_test/options_test.py index 134db012e..8818ba7e1 100644 --- a/unit_test/options_test.py +++ b/unit_test/options_test.py @@ -253,3 +253,57 @@ def test_container_engine_option(tmp_path: Path, toml_assignment, result_name, r assert parsed_container_engine.name == result_name assert parsed_container_engine.create_args == result_create_args + + +@pytest.mark.parametrize( + ("toml_assignment", "result_name", "result_args"), + [ + ( + "", + None, + None, + ), + ( + 'build-frontend = "build"', + "build", + [], + ), + ( + 'build-frontend = {name = "build"}', + "build", + [], + ), + ( + 'build-frontend = "pip; args: --some-option"', + "pip", + ["--some-option"], + ), + ( + 'build-frontend = {name = "pip", args = ["--some-option"]}', + "pip", + ["--some-option"], + ), + ], +) +def test_build_frontend_option(tmp_path: Path, toml_assignment, result_name, result_args): + args = CommandLineArguments.defaults() + args.package_dir = tmp_path + + tmp_path.joinpath("pyproject.toml").write_text( + textwrap.dedent( + f"""\ + [tool.cibuildwheel] + {toml_assignment} + """ + ) + ) + + options = Options(platform="linux", command_line_arguments=args, env={}) + parsed_build_frontend = options.build_options(identifier=None).build_frontend + + if toml_assignment: + assert parsed_build_frontend is not None + assert parsed_build_frontend.name == result_name + assert parsed_build_frontend.args == result_args + else: + assert parsed_build_frontend is None From 5311f8868456ac4f49832c2b55c5507d31db4dd9 Mon Sep 17 00:00:00 2001 From: Joe Rickerby Date: Wed, 30 Aug 2023 09:28:47 +0100 Subject: [PATCH 4/4] Fix for missing table in option parsing --- cibuildwheel/options.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/cibuildwheel/options.py b/cibuildwheel/options.py index d85eb02b5..d4306a7a6 100644 --- a/cibuildwheel/options.py +++ b/cibuildwheel/options.py @@ -505,7 +505,11 @@ def build_options(self, identifier: str | None) -> BuildOptions: test_extras = self.reader.get("test-extras", sep=",") build_verbosity_str = self.reader.get("build-verbosity") - build_frontend_str = self.reader.get("build-frontend", env_plat=False) + build_frontend_str = self.reader.get( + "build-frontend", + env_plat=False, + table={"item": "{k}:{v}", "sep": "; ", "quote": shlex.quote}, + ) build_frontend: BuildFrontendConfig | None if not build_frontend_str or build_frontend_str == "default": build_frontend = None