diff --git a/.github/workflows/codespell.yml b/.github/workflows/codespell.yml index 5d252faa068a..46c5515e223e 100644 --- a/.github/workflows/codespell.yml +++ b/.github/workflows/codespell.yml @@ -12,5 +12,5 @@ jobs: - uses: codespell-project/actions-codespell@master with: check_filenames: true - skip: ./.git,./conformance/third_party,*.snk,*.pb,*.pb.cc,*.pb.h,./src/google/protobuf/testdata,./objectivec/Tests,./python/compatibility_tests/v2.5.0/tests/google/protobuf/internal + skip: ./.git,./conformance/third_party,*.snk,*.pb,*.pb.cc,*.pb.h,./src/google/protobuf/testdata,./objectivec/Tests,./python/compatibility_tests/v2.5.0/tests/google/protobuf/internal,./.github/workflows/codespell.yml ignore_words_list: "alow,alse,ba,cleare,copyable,cloneable,dedup,dur,errorprone,files',fo,fundementals,hel,importd,inout,leapyear,nd,nin,ois,ons,parseable,process',te,testof,ue,unparseable,wasn,wee,gae,keyserver,objext,od,OptIn" diff --git a/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh b/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh new file mode 100755 index 000000000000..806c90f25a46 --- /dev/null +++ b/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +set -e + +# go to the repo root +cd $(dirname $0)/../../../.. + +# running dockcross image without any arguments generates a wrapper +# scripts that can be used to run commands under the dockcross image +# easily. +# See https://github.com/dockcross/dockcross#usage for details +docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh +chmod +x dockcross-manylinux2014-aarch64.sh + +# the wrapper script has CRLF line endings and bash doesn't like that +# so we change CRLF line endings into LF. +sed -i 's/\r//g' dockcross-manylinux2014-aarch64.sh + +# The dockcross wrapper script runs arbitrary commands under the selected dockcross +# image with the following properties which make its use very convenient: +# * the current working directory is mounted under /work so the container can easily +# access the current workspace +# * the processes in the container run under the same UID and GID as the host process so unlike +# vanilla "docker run" invocations, the workspace doesn't get polluted with files +# owned by root. +./dockcross-manylinux2014-aarch64.sh "$@" diff --git a/kokoro/linux/aarch64/python_crosscompile_aarch64.sh b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh new file mode 100755 index 000000000000..7933be40d392 --- /dev/null +++ b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh @@ -0,0 +1,34 @@ +#!/bin/bash +# +# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator. +# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image. + +set -ex + +PYTHON="/opt/python/cp38-cp38/bin/python" + +./autogen.sh +CXXFLAGS="-fPIC -g -O2" ./configure +make -j8 + +pushd python + +# TODO: currently this step relies on qemu being registered with binfmt_misc so that +# aarch64 binaries are automatically run with an emulator. This works well once +# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine. +${PYTHON} setup.py build_py + +# when crosscompiling for aarch64, --plat-name needs to be set explicitly +# to end up with correctly named wheel file +# the value should be manylinuxABC_ARCH and dockcross docker image +# conveniently provides the value in the AUDITWHEEL_PLAT env +plat_name_flag="--plat-name=$AUDITWHEEL_PLAT" + +# override the value of EXT_SUFFIX to make sure the crosscompiled .so files in the wheel have the correct filename suffix +export PROTOCOL_BUFFERS_OVERRIDE_EXT_SUFFIX="$(${PYTHON} -c 'import sysconfig; print(sysconfig.get_config_var("EXT_SUFFIX").replace("-x86_64-linux-gnu.so", "-aarch64-linux-gnu.so"))')" + +# Build the python extension inplace to be able to python unittests later +${PYTHON} setup.py build_ext --cpp_implementation --compile_static_extension --inplace + +# Build the binary wheel (to check it with auditwheel) +${PYTHON} setup.py bdist_wheel --cpp_implementation --compile_static_extension $plat_name_flag diff --git a/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh b/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh new file mode 100755 index 000000000000..527fc4849d3e --- /dev/null +++ b/kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -ex + +# go to the repo root +cd $(dirname $0)/../../.. +cd python + +PYTHON="/opt/python/cp38-cp38/bin/python" +${PYTHON} -m pip install --user six pytest auditwheel + +# check that we are really using aarch64 python +(${PYTHON} -c 'import sysconfig; print(sysconfig.get_platform())' | grep -q "linux-aarch64") || (echo "Wrong python platform, needs to be aarch64 python."; exit 1) + +# step 1: run all python unittests +# we've built the python extension previously with --inplace option +# so we can just discover all the unittests and run them directly under +# the python/ directory. +LD_LIBRARY_PATH=../src/.libs PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -m pytest google/protobuf + +# step 2: run auditwheel show to check that the wheel is manylinux2014 compatible. +# auditwheel needs to run on wheel's target platform (or under an emulator) +${PYTHON} -m auditwheel show dist/protobuf-*-manylinux2014_aarch64.whl + +# step 3: smoketest that the wheel can be installed and run a smokecheck +${PYTHON} -m pip install dist/protobuf-*-manylinux2014_aarch64.whl +# when python cpp extension is on, simply importing a message type will trigger loading the cpp extension +PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=cpp ${PYTHON} -c 'import google.protobuf.timestamp_pb2; print("Successfully loaded the python cpp extension!")' diff --git a/kokoro/linux/aarch64/test_python_aarch64.sh b/kokoro/linux/aarch64/test_python_aarch64.sh new file mode 100755 index 000000000000..d5af6d618660 --- /dev/null +++ b/kokoro/linux/aarch64/test_python_aarch64.sh @@ -0,0 +1,19 @@ +#!/bin/bash + +set -e + +# go to the repo root +cd $(dirname $0)/../../.. + +# crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image +kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh + +# once crosscompilation is done, use an actual aarch64 docker image (with a real aarch64 python) to run all the tests under an emulator +# * mount the protobuf root as /work to be able to access the crosscompiled files +# * intentionally use a different image than manylinux2014 so that we don't build and test on the same linux distribution +# (manylinux_2_24 is debian-based while manylinux2014 is centos-based) +# * to avoid running the process inside docker as root (which can pollute the workspace with files owned by root), we force +# running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user +# otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, +# we just run map the user's home to a throwaway temporary directory +docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh