Enable crosscompiling aarch64 python wheels under dockcross manylinux…

… docker image (#8280)

This uses the dockcross manylinux docker image to enable crosscompiling aarch64 python wheels. The wheels built for aarch64 linux are excluded from the release for now, pending testing (pending, in PR#8392).

kokoro/release/python/linux/build_artifacts.sh

@@ -49,9 +49,31 @@ build_artifact_version() {
   sudo rm -rf $REPO_DIR
 }
 
+build_crosscompiled_aarch64_artifact_version() {
+  # crosscompilation is only supported with the dockcross manylinux2014 image
+  DOCKER_IMAGE=dockcross/manylinux2014-aarch64
+  PLAT=aarch64
+
+  # TODO(jtatermusch): currently when crosscompiling, "auditwheel repair" will be disabled
+  # since auditwheel doesn't work for crosscomiled wheels.
+  build_artifact_version $@
+}
+
 build_artifact_version 2.7
 build_artifact_version 3.5
 build_artifact_version 3.6
 build_artifact_version 3.7
 build_artifact_version 3.8
 build_artifact_version 3.9
+
+build_crosscompiled_aarch64_artifact_version 3.7
+build_crosscompiled_aarch64_artifact_version 3.8
+build_crosscompiled_aarch64_artifact_version 3.9
+
+# Put the aarch64 manylinux wheels under the "unofficial" subdirectory.
+# Only wheels directly under the artifacts/ directory will be published
+# to PyPI as part of the protobuf release process.
+# TODO(jtattermusch): include aarch64 wheels in the release
+# once they are sufficiently tested.
+mkdir -p $ARTIFACT_DIR/unofficial
+mv $ARTIFACT_DIR/protobuf-*-manylinux*_aarch64.whl $ARTIFACT_DIR/unofficial

kokoro/release/python/linux/config.sh

@@ -6,15 +6,38 @@ function pre_build {
     # Runs in the root directory of this repository.
     pushd protobuf
 
-    yum install -y devtoolset-2-libatomic-devel
+    if [ "$PLAT" == "aarch64" ]
+    then
+      local configure_host_flag="--host=aarch64"
+    else
+      yum install -y devtoolset-2-libatomic-devel
+    fi
 
-    # Build protoc
+    # Build protoc and libprotobuf
     ./autogen.sh
-    ./configure
-
-    CXXFLAGS="-fPIC -g -O2" ./configure
+    CXXFLAGS="-fPIC -g -O2" ./configure $configure_host_flag
     make -j8
 
+    if [ "$PLAT" == "aarch64" ]
+    then
+      # we are crosscompiling for aarch64 while running on x64
+      # the simplest way for build_py command to be able to generate
+      # the protos is by running the protoc process under
+      # an emulator. That way we don't have to build a x64 version
+      # of protoc. The qemu-arm emulator is already included
+      # in the dockcross docker image.
+      # Running protoc under an emulator is fast as protoc doesn't
+      # really do much.
+
+      # create a simple shell wrapper that runs crosscompiled protoc under qemu
+      echo '#!/bin/bash' >protoc_qemu_wrapper.sh
+      echo 'exec qemu-aarch64 "../src/protoc" "$@"' >>protoc_qemu_wrapper.sh
+      chmod ugo+x protoc_qemu_wrapper.sh
+
+      # PROTOC variable is by build_py step that runs under ./python directory
+      export PROTOC=../protoc_qemu_wrapper.sh
+    fi
+
     # Generate python dependencies.
     pushd python
     python setup.py build_py
@@ -35,7 +58,20 @@ function bdist_wheel_cmd {
     # Modify build version
     pwd
     ls
-    python setup.py bdist_wheel --cpp_implementation --compile_static_extension
+
+    if [ "$PLAT" == "aarch64" ]
+    then
+      # when crosscompiling for aarch64, --plat-name needs to be set explicitly
+      # to end up with correctly named wheel file
+      # the value should be manylinuxABC_ARCH and dockcross docker image
+      # conveniently provides the value in the AUDITWHEEL_PLAT env
+      local plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
+
+      # override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
+      export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(python -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
+    fi
+
+    python setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag
     cp dist/*.whl $abs_wheelhouse
 }
 
@@ -48,3 +84,12 @@ function run_tests {
     python --version
     python -c "from google.protobuf.pyext import _message;"
 }
+
+if [ "$PLAT" == "aarch64" ]
+then
+  # when crosscompiling for aarch64, override the default multibuild's repair_wheelhouse logic
+  # since "auditwheel repair" doesn't work for crosscompiled wheels
+  function repair_wheelhouse {
+      echo "Skipping repair_wheelhouse since auditwheel requires build architecture to match wheel architecture."
+  }
+fi

python/setup.py

@@ -18,6 +18,7 @@
 
 from distutils.command.build_py import build_py as _build_py
 from distutils.command.clean import clean as _clean
+from distutils.command.build_ext import build_ext as _build_ext
 from distutils.spawn import find_executable
 
 # Find the Protocol Compiler.
@@ -157,6 +158,22 @@ def find_package_modules(self, package, package_dir):
             if not any(fnmatch.fnmatchcase(fil, pat=pat) for pat in exclude)]
 
 
+class build_ext(_build_ext):
+  def get_ext_filename(self, ext_name):
+      # since python3.5, python extensions' shared libraries use a suffix that corresponds to the value
+      # of sysconfig.get_config_var('EXT_SUFFIX') and contains info about the architecture the library targets.
+      # E.g. on x64 linux the suffix is ".cpython-XYZ-x86_64-linux-gnu.so"
+      # When crosscompiling python wheels, we need to be able to override this suffix
+      # so that the resulting file name matches the target architecture and we end up with a well-formed
+      # wheel.
+      filename = _build_ext.get_ext_filename(self, ext_name)
+      orig_ext_suffix = sysconfig.get_config_var("EXT_SUFFIX")
+      new_ext_suffix = os.getenv("PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX")
+      if new_ext_suffix and filename.endswith(orig_ext_suffix):
+        filename = filename[:-len(orig_ext_suffix)] + new_ext_suffix
+      return filename
+
+
 class test_conformance(_build_py):
   target = 'test_python'
   def run(self):
@@ -291,6 +308,7 @@ def get_option_from_sys_argv(option_str):
       cmdclass={
           'clean': clean,
           'build_py': build_py,
+          'build_ext': build_ext,
           'test_conformance': test_conformance,
       },
       install_requires=install_requires,