run basic python aarch64 tests under qemu emulator

kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../../..
+
+# running dockcross image without any arguments generates a wrapper
+# scripts that can be used to run commands under the dockcross image
+# easily.
+# See https://github.com/dockcross/dockcross#usage for details
+docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh
+chmod +x dockcross-manylinux2014-aarch64.sh
+
+# the wrapper script has CRLF line endings and bash doesn't like that
+# so we change CRLF line endings into LF.
+sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh
+
+# The dockcross wrapper script runs arbitrary commands under the selected dockcross
+# image with the following properties which make its use very convenient:
+# * the current working directory is mounted under /work so the container can easily
+#   access the curent workspace
+# * the processes in the container run under the same UID and GID as the host process so unlike
+#   vanilla "docker run" invocations, the workspace doesn't get polluted with files
+#   owned by root.
+./dockcross-manylinux2014-aarch64.sh "$@"

kokoro/linux/aarch64/python_crosscompile_aarch64.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+#
+# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator.
+# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image.
+
+set -ex
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+
+./autogen.sh
+CXXFLAGS="-fPIC -g -O2" ./configure
+make -j8
+
+pushd python
+
+# TODO: currently this step relies on qemu being registered with binfmt_misc so that
+# aarch64 binaries are automatically run with an emulator. This works well once
+# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine.
+${PYTHON} setup.py build_py
+
+# when crosscompiling for aarch64, --plat-name needs to be set explicitly
+# to end up with correctly named wheel file
+# the value should be manylinuxABC_ARCH and dockcross docker image
+# conveniently provides the value in the AUDITWHEEL_PLAT env
+plat_name_flag="--plat-name=$AUDITWHEEL_PLAT"
+
+# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix
+export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')"
+
+# Build the python extension inplace to be able to python unittests later
+${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace
+
+# Build the binary wheel (to check it with auditwheel)
+${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag

kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set -ex
+
+# go to the repo root
+cd $(dirname $0)/../../..
+cd python
+
+PYTHON="/opt/python/cp38-cp38/bin/python"
+${PYTHON} -m pip install --user six pytest auditwheel
+
+# check that we are really using aarch64 python
+(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1)
+
+# step 1: run all python unittests
+# we've built the python extension previously with --inplace option
+# so we can just discover all the unittests and run them directly under 
+# the python/ directory.
+LD_LIBRARY_PATH=../src/.libs PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf
+
+# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible.
+# auditwheel needs to run on wheel's target platform (or under an emulator)
+${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl
+
+# step 3: smoketest that the wheel can be installed and run a smokecheck
+${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl
+# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension
+PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfuly loaded the python cpp extension!")'

kokoro/linux/aarch64/test_python_aarch64.sh

@@ -0,0 +1,19 @@
+#!/bin/bash
+
+set -e
+
+# go to the repo root
+cd $(dirname $0)/../../..
+
+# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image
+kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh
+
+# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator
+# * mount the protobuf root as /work to be able to access the crosscompiled files
+# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution
+#   (manylinux_2_24 is debian-based while manylinux2014 is centos-based)
+# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force
+#   running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user
+#   otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity,
+#   we just run map the user's home to a throwaway temporary directory
+docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh