From 0906af98708ebd8d6cb059111abcb524209c9ff1 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Fri, 12 Nov 2021 10:35:03 +0100 Subject: [PATCH 1/4] pin multibuild scripts to get manylinux1 wheels back --- kokoro/release/python/linux/build_artifacts.sh | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/kokoro/release/python/linux/build_artifacts.sh b/kokoro/release/python/linux/build_artifacts.sh index 31da5628144..363af709376 100755 --- a/kokoro/release/python/linux/build_artifacts.sh +++ b/kokoro/release/python/linux/build_artifacts.sh @@ -25,7 +25,13 @@ rm -rf multibuild/ mkdir artifacts export ARTIFACT_DIR=$(pwd)/artifacts +# Pin multibuild script to a version just before the default +# manylinux image has switched from manylinux1 to manylinux2014. +# Also, pinning version avoid potentially unwanted future changes from +# silently creeping in. +# See https://github.com/protocolbuffers/protobuf/issues/9180 git clone https://github.com/matthew-brett/multibuild.git +(cd multibuild; git checkout 13a01725b0f0aa551ab34aa2311cdc1c77be4337) cp kokoro/release/python/linux/config.sh config.sh build_artifact_version() { From 7857b15afcf21a409b0cf991a8b30e16f9522ab2 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 16 Nov 2021 11:54:09 +0100 Subject: [PATCH 2/4] pin multibuild, set MB_ML_VER explicitly --- .../release/python/linux/build_artifacts.sh | 34 +++++++++++++------ .../release/python/macos/build_artifacts.sh | 9 ++++- .../python/windows/build_artifacts.bat | 9 +++++ 3 files changed, 40 insertions(+), 12 deletions(-) diff --git a/kokoro/release/python/linux/build_artifacts.sh b/kokoro/release/python/linux/build_artifacts.sh index 363af709376..e1fe98c7787 100755 --- a/kokoro/release/python/linux/build_artifacts.sh +++ b/kokoro/release/python/linux/build_artifacts.sh @@ -25,13 +25,14 @@ rm -rf multibuild/ mkdir artifacts export ARTIFACT_DIR=$(pwd)/artifacts -# Pin multibuild script to a version just before the default -# manylinux image has switched from manylinux1 to manylinux2014. -# Also, pinning version avoid potentially unwanted future changes from -# silently creeping in. -# See https://github.com/protocolbuffers/protobuf/issues/9180 git clone https://github.com/matthew-brett/multibuild.git -(cd multibuild; git checkout 13a01725b0f0aa551ab34aa2311cdc1c77be4337) +# Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from +# silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180). +# IMPORTANT: always pin multibuild at the same commit for: +# - linux/build_artifacts.sh +# - linux/build_artifacts.sh +# - windows/build_artifacts.bat +(cd multibuild; git checkout b89bb903e94308be79abefa4f436bf123ebb1313) cp kokoro/release/python/linux/config.sh config.sh build_artifact_version() { @@ -53,9 +54,20 @@ build_artifact_version() { sudo rm -rf $REPO_DIR } +build_x86_64_artifact_version() { + # Explicitly request building manylinux1 wheels, which is no longer the default. + # https://github.com/protocolbuffers/protobuf/issues/9180 + MB_ML_VER=1 + + # TODO(jtatermusch): currently when crosscompiling, "auditwheel repair" will be disabled + # since auditwheel doesn't work for crosscomiled wheels. + build_artifact_version $@ +} + build_crosscompiled_aarch64_artifact_version() { # crosscompilation is only supported with the dockcross manylinux2014 image DOCKER_IMAGE=dockcross/manylinux2014-aarch64:20210706-65bf2dd + MB_ML_VER=2014 PLAT=aarch64 # TODO(jtatermusch): currently when crosscompiling, "auditwheel repair" will be disabled @@ -63,11 +75,11 @@ build_crosscompiled_aarch64_artifact_version() { build_artifact_version $@ } -build_artifact_version 3.6 -build_artifact_version 3.7 -build_artifact_version 3.8 -build_artifact_version 3.9 -build_artifact_version 3.10 +build_x86_64_artifact_version 3.6 +build_x86_64_artifact_version 3.7 +build_x86_64_artifact_version 3.8 +build_x86_64_artifact_version 3.9 +build_x86_64_artifact_version 3.10 build_crosscompiled_aarch64_artifact_version 3.7 build_crosscompiled_aarch64_artifact_version 3.8 diff --git a/kokoro/release/python/macos/build_artifacts.sh b/kokoro/release/python/macos/build_artifacts.sh index 75644539fa0..5b6d0354314 100755 --- a/kokoro/release/python/macos/build_artifacts.sh +++ b/kokoro/release/python/macos/build_artifacts.sh @@ -25,7 +25,14 @@ rm -rf multibuild/ mkdir artifacts export ARTIFACT_DIR=$(pwd)/artifacts -git clone https://github.com/matthew-brett/multibuild.git +git clone https://github.com/matthew-brett/ +# Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from +# silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180). +# IMPORTANT: always pin multibuild at the same commit for: +# - linux/build_artifacts.sh +# - linux/build_artifacts.sh +# - windows/build_artifacts.bat +(cd multibuild; git checkout b89bb903e94308be79abefa4f436bf123ebb1313) cp kokoro/release/python/macos/config.sh config.sh OLD_PATH=$PATH diff --git a/kokoro/release/python/windows/build_artifacts.bat b/kokoro/release/python/windows/build_artifacts.bat index ac466a901d3..a8f8f783142 100644 --- a/kokoro/release/python/windows/build_artifacts.bat +++ b/kokoro/release/python/windows/build_artifacts.bat @@ -14,6 +14,15 @@ set OLD_PATH=C:\Program Files (x86)\MSBuild\14.0\bin\;%PATH% REM Fetch multibuild git clone https://github.com/matthew-brett/multibuild.git +REM Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from +REM silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180). +REM IMPORTANT: always pin multibuild at the same commit for: +REM - linux/build_artifacts.sh +REM - linux/build_artifacts.sh +REM - windows/build_artifacts.bat +cd multibuild +git checkout b89bb903e94308be79abefa4f436bf123ebb1313 +cd .. REM Install zlib mkdir zlib From ea60c9cb08bb6f7c967d0ace9714aed311fda255 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 3 Jan 2022 10:54:28 +0100 Subject: [PATCH 3/4] use manylinux2010 for python3.10 wheel on x86_64 --- .../release/python/linux/build_artifacts.sh | 29 ++++++++++--------- 1 file changed, 16 insertions(+), 13 deletions(-) diff --git a/kokoro/release/python/linux/build_artifacts.sh b/kokoro/release/python/linux/build_artifacts.sh index e1fe98c7787..9a3fc5841f4 100755 --- a/kokoro/release/python/linux/build_artifacts.sh +++ b/kokoro/release/python/linux/build_artifacts.sh @@ -54,17 +54,20 @@ build_artifact_version() { sudo rm -rf $REPO_DIR } -build_x86_64_artifact_version() { +build_x86_64_manylinux1_artifact_version() { # Explicitly request building manylinux1 wheels, which is no longer the default. # https://github.com/protocolbuffers/protobuf/issues/9180 MB_ML_VER=1 + build_artifact_version $@ +} - # TODO(jtatermusch): currently when crosscompiling, "auditwheel repair" will be disabled - # since auditwheel doesn't work for crosscomiled wheels. +build_x86_64_manylinux2010_artifact_version() { + # Explicitly request building manylinux2010 wheels + MB_ML_VER=2010 build_artifact_version $@ } -build_crosscompiled_aarch64_artifact_version() { +build_crosscompiled_aarch64_manylinux2014_artifact_version() { # crosscompilation is only supported with the dockcross manylinux2014 image DOCKER_IMAGE=dockcross/manylinux2014-aarch64:20210706-65bf2dd MB_ML_VER=2014 @@ -75,13 +78,13 @@ build_crosscompiled_aarch64_artifact_version() { build_artifact_version $@ } -build_x86_64_artifact_version 3.6 -build_x86_64_artifact_version 3.7 -build_x86_64_artifact_version 3.8 -build_x86_64_artifact_version 3.9 -build_x86_64_artifact_version 3.10 +build_x86_64_manylinux1_artifact_version 3.6 +build_x86_64_manylinux1_artifact_version 3.7 +build_x86_64_manylinux1_artifact_version 3.8 +build_x86_64_manylinux1_artifact_version 3.9 +build_x86_64_manylinux2010_artifact_version 3.10 -build_crosscompiled_aarch64_artifact_version 3.7 -build_crosscompiled_aarch64_artifact_version 3.8 -build_crosscompiled_aarch64_artifact_version 3.9 -build_crosscompiled_aarch64_artifact_version 3.10 +build_crosscompiled_aarch64_manylinux2014_artifact_version 3.7 +build_crosscompiled_aarch64_manylinux2014_artifact_version 3.8 +build_crosscompiled_aarch64_manylinux2014_artifact_version 3.9 +build_crosscompiled_aarch64_manylinux2014_artifact_version 3.10 From 6f8d82c2b1f2d391033d7babf20bb2b8857ef688 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 3 Jan 2022 14:59:51 +0100 Subject: [PATCH 4/4] fixup linux python artifact build --- kokoro/release/python/macos/build_artifacts.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kokoro/release/python/macos/build_artifacts.sh b/kokoro/release/python/macos/build_artifacts.sh index 5b6d0354314..aeb4242a6b9 100755 --- a/kokoro/release/python/macos/build_artifacts.sh +++ b/kokoro/release/python/macos/build_artifacts.sh @@ -25,7 +25,7 @@ rm -rf multibuild/ mkdir artifacts export ARTIFACT_DIR=$(pwd)/artifacts -git clone https://github.com/matthew-brett/ +git clone https://github.com/matthew-brett/multibuild.git # Pin multibuild scripts at a known commit to avoid potentially unwanted future changes from # silently creeping in (see https://github.com/protocolbuffers/protobuf/issues/9180). # IMPORTANT: always pin multibuild at the same commit for: