From 6aa19d9145ed031ad1b01618a817999ed02f3224 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 12 Apr 2021 12:10:32 +0200 Subject: [PATCH 1/6] be clever about docker "-it" args --- .../run_dockcross_manylinux2014_aarch64.sh | 9 ++++++++- kokoro/linux/aarch64/test_python_aarch64.sh | 9 ++++++++- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh b/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh index 806c90f25a46..115478b8fc7d 100755 --- a/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh +++ b/kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh @@ -5,11 +5,18 @@ set -e # go to the repo root cd $(dirname $0)/../../../.. +if [[ -t 0 ]]; then + DOCKER_TTY_ARGS="-it" +else + # The input device on kokoro is not a TTY, so -it does not work. + DOCKER_TTY_ARGS= +fi + # running dockcross image without any arguments generates a wrapper # scripts that can be used to run commands under the dockcross image # easily. # See https://github.com/dockcross/dockcross#usage for details -docker run --rm -it dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh +docker run $DOCKER_TTY_ARGS --rm dockcross/manylinux2014-aarch64 >dockcross-manylinux2014-aarch64.sh chmod +x dockcross-manylinux2014-aarch64.sh # the wrapper script has CRLF line endings and bash doesn't like that diff --git a/kokoro/linux/aarch64/test_python_aarch64.sh b/kokoro/linux/aarch64/test_python_aarch64.sh index d5af6d618660..1e2d57021f51 100755 --- a/kokoro/linux/aarch64/test_python_aarch64.sh +++ b/kokoro/linux/aarch64/test_python_aarch64.sh @@ -5,6 +5,13 @@ set -e # go to the repo root cd $(dirname $0)/../../.. +if [[ -t 0 ]]; then + DOCKER_TTY_ARGS="-it" +else + # The input device on kokoro is not a TTY, so -it does not work. + DOCKER_TTY_ARGS= +fi + # crosscompile python extension and the binary wheel under dockcross/manylinux2014-aarch64 image kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh kokoro/linux/aarch64/python_crosscompile_aarch64.sh @@ -16,4 +23,4 @@ kokoro/linux/aarch64/dockcross_helpers/run_dockcross_manylinux2014_aarch64.sh ko # running under current user's UID and GID. To be able to do that, we need to provide a home directory for the user # otherwise the UID would be homeless under the docker container and pip install wouldn't work. For simplicity, # we just run map the user's home to a throwaway temporary directory -docker run -it --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh +docker run $DOCKER_TTY_ARGS --rm --user "$(id -u):$(id -g)" -e "HOME=/home/fake-user" -v "$(mktemp -d):/home/fake-user" -v "$(pwd)":/work -w /work quay.io/pypa/manylinux_2_24_aarch64 kokoro/linux/aarch64/python_run_tests_with_qemu_aarch64.sh From a0b1b70819ffe9fe92b3b74d64d0b49647cd0f8e Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 12 Apr 2021 12:25:26 +0200 Subject: [PATCH 2/6] fixups in python_crosscompile_aarch64.sh add --host=aarch64 to ./configure invocation use protoc_qemu_wrapper.sh --- .../linux/aarch64/python_crosscompile_aarch64.sh | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/kokoro/linux/aarch64/python_crosscompile_aarch64.sh b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh index 7933be40d392..08db37cd8841 100755 --- a/kokoro/linux/aarch64/python_crosscompile_aarch64.sh +++ b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh @@ -8,14 +8,20 @@ set -ex PYTHON="/opt/python/cp38-cp38/bin/python" ./autogen.sh -CXXFLAGS="-fPIC -g -O2" ./configure +CXXFLAGS="-fPIC -g -O2" ./configure --host=aarch64 make -j8 +# create a simple shell wrapper that runs crosscompiled protoc under qemu +echo '#!/bin/bash' >protoc_qemu_wrapper.sh +echo 'exec qemu-aarch64 "../src/protoc" "$@"' >>protoc_qemu_wrapper.sh +chmod ugo+x protoc_qemu_wrapper.sh + +# PROTOC variable is by build_py step that runs under ./python directory +export PROTOC=../protoc_qemu_wrapper.sh + pushd python -# TODO: currently this step relies on qemu being registered with binfmt_misc so that -# aarch64 binaries are automatically run with an emulator. This works well once -# "sudo apt install qemu-user-static binfmt-support" is installed on the host machine. +# NOTE: this step will use protoc_qemu_wrapper.sh to generate protobuf files. ${PYTHON} setup.py build_py # when crosscompiling for aarch64, --plat-name needs to be set explicitly From b4531870b4f1ff9752b8e8e839bc8560e71e3e23 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 13 Apr 2021 11:56:56 +0200 Subject: [PATCH 3/6] add prepare_qemu.sh script --- .../aarch64/qemu_helpers/prepare_qemu.sh | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100755 kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh diff --git a/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh b/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh new file mode 100755 index 000000000000..aa78e994cdc0 --- /dev/null +++ b/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh @@ -0,0 +1,30 @@ +#!/bin/bash +# +# Setup and configure qemu userspace emulator on kokoro worker so that we can seamlessly emulate processes running +# inside docker containers. + +set -ex + +# show pre-existing qemu registration +cat /proc/sys/fs/binfmt_misc/qemu-aarch64 + +# Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that: +# * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278") +# * doesn't register with binfmt_misc with the "F" flag we need (see below) +# To avoid the "unsupported syscall" warnings, we download a recent version of qemu-aarch64-static. +# On newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient. +curl -sSL --output qemu-aarch64-static https://github.com/multiarch/qemu-user-static/releases/download/v5.2.0-2/qemu-aarch64-static +chmod ugo+x qemu-aarch64-static +sudo mv qemu-aarch64-static /usr/local/bin +/usr/local/bin/qemu-aarch64-static --version + +# register qemu-aarch64-static with binfmt_misc to use it as emulator for aarch64 binaries (based on their magic and mask) +# The format of the configuration string is ":name:type:offset:magic:mask:interpreter:flags" +# The most important flag is "F", which allows the qemu-aarch64-static binary to be loaded regardless of chroot and namespaces. +# That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator +# to be accessible from the docker image we're emulating. +echo -1 | sudo tee /proc/sys/fs/binfmt_misc/qemu-aarch64 # delete the existing registration +echo ":qemu-aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-aarch64-static:OCF" | sudo tee /proc/sys/fs/binfmt_misc/register + +# Print current qemu reqistration to make sure everything is setup correctly. +cat /proc/sys/fs/binfmt_misc/qemu-aarch64 From c8239319e1cc2e8adc2bbe28347c0902836676b6 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Mon, 12 Apr 2021 11:26:26 +0200 Subject: [PATCH 4/6] add cfg for python aarch64 tests --- kokoro/linux/python_aarch64/build.sh | 13 +++++++++++++ kokoro/linux/python_aarch64/continuous.cfg | 11 +++++++++++ kokoro/linux/python_aarch64/presubmit.cfg | 11 +++++++++++ 3 files changed, 35 insertions(+) create mode 100755 kokoro/linux/python_aarch64/build.sh create mode 100644 kokoro/linux/python_aarch64/continuous.cfg create mode 100644 kokoro/linux/python_aarch64/presubmit.cfg diff --git a/kokoro/linux/python_aarch64/build.sh b/kokoro/linux/python_aarch64/build.sh new file mode 100755 index 000000000000..2c67396462a6 --- /dev/null +++ b/kokoro/linux/python_aarch64/build.sh @@ -0,0 +1,13 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "continuous" and "presubmit" jobs. + +set -ex + +# Change to repo root +cd $(dirname $0)/../../.. + +kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh + +kokoro/linux/aarch64/test_python_aarch64.sh diff --git a/kokoro/linux/python_aarch64/continuous.cfg b/kokoro/linux/python_aarch64/continuous.cfg new file mode 100644 index 000000000000..dee4a47367be --- /dev/null +++ b/kokoro/linux/python_aarch64/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python_aarch64/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/python_aarch64/presubmit.cfg b/kokoro/linux/python_aarch64/presubmit.cfg new file mode 100644 index 000000000000..dee4a47367be --- /dev/null +++ b/kokoro/linux/python_aarch64/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/python_aarch64/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} From 5d9a5b411872ec7749fe0a9505c5a8cea4be1f40 Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Tue, 13 Apr 2021 15:01:51 +0200 Subject: [PATCH 5/6] fix incorrectly copypasted comment --- kokoro/linux/aarch64/python_crosscompile_aarch64.sh | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/kokoro/linux/aarch64/python_crosscompile_aarch64.sh b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh index 08db37cd8841..db2e7763aff4 100755 --- a/kokoro/linux/aarch64/python_crosscompile_aarch64.sh +++ b/kokoro/linux/aarch64/python_crosscompile_aarch64.sh @@ -1,7 +1,8 @@ #!/bin/bash # -# Builds protobuf C++ with aarch64 crosscompiler and runs a basic set of tests under an emulator. -# NOTE: This script is expected to run under the dockcross/linux-arm64 docker image. +# Builds protobuf python including the C++ extension with aarch64 crosscompiler. +# The outputs of this script are laid out so that we can later test them under an aarch64 emulator. +# NOTE: This script is expected to run under the dockcross/manylinux2014-aarch64 docker image. set -ex From c3ce10e49c44ae2bef540a169d4cf718b1554d0c Mon Sep 17 00:00:00 2001 From: Jan Tattermusch Date: Wed, 14 Apr 2021 14:00:57 +0200 Subject: [PATCH 6/6] use multiarch/qemu-user-static docker image for registering qemu --- .../aarch64/qemu_helpers/prepare_qemu.sh | 24 ++++++++----------- 1 file changed, 10 insertions(+), 14 deletions(-) diff --git a/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh b/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh index aa78e994cdc0..f61320222a84 100755 --- a/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh +++ b/kokoro/linux/aarch64/qemu_helpers/prepare_qemu.sh @@ -10,21 +10,17 @@ cat /proc/sys/fs/binfmt_misc/qemu-aarch64 # Kokoro ubuntu1604 workers have already qemu-user and qemu-user-static packages installed, but it's and old version that: # * prints warning about some syscalls (e.g "qemu: Unsupported syscall: 278") -# * doesn't register with binfmt_misc with the "F" flag we need (see below) -# To avoid the "unsupported syscall" warnings, we download a recent version of qemu-aarch64-static. -# On newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient. -curl -sSL --output qemu-aarch64-static https://github.com/multiarch/qemu-user-static/releases/download/v5.2.0-2/qemu-aarch64-static -chmod ugo+x qemu-aarch64-static -sudo mv qemu-aarch64-static /usr/local/bin -/usr/local/bin/qemu-aarch64-static --version - -# register qemu-aarch64-static with binfmt_misc to use it as emulator for aarch64 binaries (based on their magic and mask) -# The format of the configuration string is ":name:type:offset:magic:mask:interpreter:flags" -# The most important flag is "F", which allows the qemu-aarch64-static binary to be loaded regardless of chroot and namespaces. +# * doesn't register with binfmt_misc with the persistent ("F") flag we need (see below) +# +# To overcome the above limitations, we use the https://github.com/multiarch/qemu-user-static +# docker image to provide a new enough version of qemu-user-static and register it with +# the desired binfmt_misc flags. The most important flag we need is "F" (set by "--persistent yes"), +# which allows the qemu-aarch64-static binary to be loaded eagerly at the time of registration with binfmt_misc. # That way, we can emulate aarch64 binaries running inside docker containers transparently, without needing the emulator -# to be accessible from the docker image we're emulating. -echo -1 | sudo tee /proc/sys/fs/binfmt_misc/qemu-aarch64 # delete the existing registration -echo ":qemu-aarch64:M::\x7fELF\x02\x01\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x02\x00\xb7\x00:\xff\xff\xff\xff\xff\xff\xff\x00\xff\xff\xff\xff\xff\xff\xff\xff\xfe\xff\xff\xff:/usr/local/bin/qemu-aarch64-static:OCF" | sudo tee /proc/sys/fs/binfmt_misc/register +# binary to be accessible from the docker image we're emulating. +# Note that on newer distributions (such as glinux), simply "apt install qemu-user-static" is sufficient +# to install qemu-user-static with the right flags. +docker run --rm --privileged multiarch/qemu-user-static:5.2.0-2 --reset --credential yes --persistent yes # Print current qemu reqistration to make sure everything is setup correctly. cat /proc/sys/fs/binfmt_misc/qemu-aarch64