From e14c3e343ead88d9d3d911e3fddacb19081abf68 Mon Sep 17 00:00:00 2001 From: Marco Concetto Rudilosso Date: Thu, 17 Mar 2022 16:38:36 +0000 Subject: [PATCH] Allow pre-compiled binaries for ruby 3.1.0 (#9566) * Allow pre-compiled binaries for ruby 3.1.1 * add comment * fix build and use ruby 3.1.0 * add ruby31 to build CI for tests and release * trying to fix ci * install ruby 3.1.0 in ruby_build_environment.sh * use head for rvm to install 3.1.0 * just install master version of rvm in prepare_build_macos_rc * force install of master rvm in ruby_build_environment.sh * Use coroutine=universal when compiling ruby31 * use ucontext * fix filename * fix coroutine name * use git head for rake-compiler-dock * use newest rake-compiler-dock version --- kokoro/linux/dockerfile/test/ruby/Dockerfile | 1 + kokoro/linux/ruby31/build.sh | 18 +++++++++ kokoro/linux/ruby31/continuous.cfg | 11 +++++ kokoro/linux/ruby31/presubmit.cfg | 11 +++++ kokoro/macos/prepare_build_macos_rc | 2 +- kokoro/macos/ruby31/build.sh | 12 ++++++ kokoro/macos/ruby31/continuous.cfg | 5 +++ kokoro/macos/ruby31/presubmit.cfg | 5 +++ .../ruby/macos/ruby/ruby_build_environment.sh | 40 +++++++++++++++++++ ruby/Rakefile | 4 +- ruby/google-protobuf.gemspec | 3 +- tests.sh | 5 +++ 12 files changed, 112 insertions(+), 5 deletions(-) create mode 100644 kokoro/linux/ruby31/build.sh create mode 100644 kokoro/linux/ruby31/continuous.cfg create mode 100644 kokoro/linux/ruby31/presubmit.cfg create mode 100644 kokoro/macos/ruby31/build.sh create mode 100644 kokoro/macos/ruby31/continuous.cfg create mode 100644 kokoro/macos/ruby31/presubmit.cfg diff --git a/kokoro/linux/dockerfile/test/ruby/Dockerfile b/kokoro/linux/dockerfile/test/ruby/Dockerfile index cfdc5ba03162..914cd4b4b907 100644 --- a/kokoro/linux/dockerfile/test/ruby/Dockerfile +++ b/kokoro/linux/dockerfile/test/ruby/Dockerfile @@ -35,6 +35,7 @@ RUN /bin/bash -l -c "rvm install 2.5.1" RUN /bin/bash -l -c "rvm install 2.6.0" RUN /bin/bash -l -c "rvm install 2.7.0" RUN /bin/bash -l -c "rvm install 3.0.0" +RUN /bin/bash -l -c "rvm install 3.1.0" RUN /bin/bash -l -c "rvm install jruby-9.2.20.1" RUN /bin/bash -l -c "rvm install jruby-9.3.3.0" diff --git a/kokoro/linux/ruby31/build.sh b/kokoro/linux/ruby31/build.sh new file mode 100644 index 000000000000..c22bdae2efd6 --- /dev/null +++ b/kokoro/linux/ruby31/build.sh @@ -0,0 +1,18 @@ +#!/bin/bash +# +# This is the top-level script we give to Kokoro as the entry point for +# running the "pull request" project: +# +# This script selects a specific Dockerfile (for building a Docker image) and +# a script to run inside that image. Then we delegate to the general +# build_and_run_docker.sh script. + +# Change to repo root +cd $(dirname $0)/../../.. + +export DOCKERHUB_ORGANIZATION=protobuftesting +export DOCKERFILE_DIR=kokoro/linux/dockerfile/test/ruby +export DOCKER_RUN_SCRIPT=kokoro/linux/pull_request_in_docker.sh +export OUTPUT_DIR=testoutput +export TEST_SET="ruby31" +./kokoro/linux/build_and_run_docker.sh diff --git a/kokoro/linux/ruby31/continuous.cfg b/kokoro/linux/ruby31/continuous.cfg new file mode 100644 index 000000000000..0477912b8cac --- /dev/null +++ b/kokoro/linux/ruby31/continuous.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/ruby31/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/linux/ruby31/presubmit.cfg b/kokoro/linux/ruby31/presubmit.cfg new file mode 100644 index 000000000000..0477912b8cac --- /dev/null +++ b/kokoro/linux/ruby31/presubmit.cfg @@ -0,0 +1,11 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/linux/ruby31/build.sh" +timeout_mins: 120 + +action { + define_artifacts { + regex: "**/sponge_log.xml" + } +} diff --git a/kokoro/macos/prepare_build_macos_rc b/kokoro/macos/prepare_build_macos_rc index c0017b64ac82..8e0a87edbbf8 100755 --- a/kokoro/macos/prepare_build_macos_rc +++ b/kokoro/macos/prepare_build_macos_rc @@ -36,5 +36,5 @@ if [[ "${KOKORO_INSTALL_RVM:-}" == "yes" ]] ; then # Old OpenSSL versions cannot handle the SSL certificate used by # https://get.rvm.io, so as a workaround we download RVM directly from # GitHub. See this issue for details: https://github.com/rvm/rvm/issues/5133 - curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s stable --ruby + curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s master --ruby fi diff --git a/kokoro/macos/ruby31/build.sh b/kokoro/macos/ruby31/build.sh new file mode 100644 index 000000000000..1b5a5a5a60eb --- /dev/null +++ b/kokoro/macos/ruby31/build.sh @@ -0,0 +1,12 @@ +#!/bin/bash +# +# Build file to set up and run tests + +# Change to repo root +cd $(dirname $0)/../../.. + +# Prepare worker environment to run tests +KOKORO_INSTALL_RVM=yes +source kokoro/macos/prepare_build_macos_rc + +./tests.sh ruby31 diff --git a/kokoro/macos/ruby31/continuous.cfg b/kokoro/macos/ruby31/continuous.cfg new file mode 100644 index 000000000000..19e16b3eaf5c --- /dev/null +++ b/kokoro/macos/ruby31/continuous.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/macos/ruby31/build.sh" +timeout_mins: 1440 diff --git a/kokoro/macos/ruby31/presubmit.cfg b/kokoro/macos/ruby31/presubmit.cfg new file mode 100644 index 000000000000..19e16b3eaf5c --- /dev/null +++ b/kokoro/macos/ruby31/presubmit.cfg @@ -0,0 +1,5 @@ +# Config file for running tests in Kokoro + +# Location of the build script in repository +build_file: "protobuf/kokoro/macos/ruby31/build.sh" +timeout_mins: 1440 diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index 98270d138338..c3fcd3c584b1 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -3,12 +3,25 @@ set -ex set +ex # rvm script is very verbose and exits with errorcode + +curl -sSL https://rvm.io/mpapis.asc | gpg --import - +curl -sSL https://rvm.io/pkuczynski.asc | gpg --import - + +# Old OpenSSL versions cannot handle the SSL certificate used by +# https://get.rvm.io, so as a workaround we download RVM directly from +# GitHub. See this issue for details: https://github.com/rvm/rvm/issues/5133 +curl -sSL https://raw.githubusercontent.com/rvm/rvm/master/binscripts/rvm-installer | bash -s master --ruby + source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose time rvm install 2.5.0 rvm use 2.5.0 gem install rake-compiler --no-document gem install bundler --no-document +time rvm install 3.1.0 +rvm use 3.1.0 +gem install rake-compiler --no-document +gem install bundler --no-document time rvm install 2.7.0 rvm use 2.7.0 --default gem install rake-compiler --no-document @@ -20,6 +33,8 @@ set -ex rm -rf ~/.rake-compiler CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) +CROSS_RUBY31=$(mktemp tmpfile.XXXXXXXX) + curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/72184e51779b6a3b9b8580b036a052fdc3181ced/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" @@ -52,8 +67,33 @@ patch "$CROSS_RUBY" << EOF end EOF +cp $CROSS_RUBY $CROSS_RUBY31 + +patch "$CROSS_RUBY31" << EOF +--- cross-ruby.rake 2022-03-04 11:49:52.000000000 +0000 ++++ patched 2022-03-04 11:58:22.000000000 +0000 +@@ -114,6 +114,7 @@ + '--enable-static', + '--disable-shared', + '--disable-install-doc', ++ '--with-coroutine=ucontext', + '--without-gmp', + '--with-ext=', + 'LDFLAGS=-pipe', +EOF + MAKE="make -j8" +set +x # rvm commands are very verbose +rvm use 3.1.0 +set -x +ruby --version | grep 'ruby 3.1.0' +for v in 3.1.0 ; do + ccache -c + rake -f "$CROSS_RUBY31" cross-ruby VERSION="$v" HOST=x86_64-darwin MAKE="$MAKE" + rake -f "$CROSS_RUBY31" cross-ruby VERSION="$v" HOST=aarch64-darwin MAKE="$MAKE" +done + set +x # rvm commands are very verbose rvm use 2.7.0 set -x diff --git a/ruby/Rakefile b/ruby/Rakefile index 6f71a2a7214c..1892e6209fe3 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -127,7 +127,7 @@ else ['x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux'].each do |plat| RakeCompilerDock.sh <<-"EOT", platform: plat bundle && \ - IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.0.0:2.7.0:2.6.0:2.5.0 + IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0:2.5.0 EOT end end @@ -135,7 +135,7 @@ else if RUBY_PLATFORM =~ /darwin/ task 'gem:native' do system "rake genproto" - system "rake cross native gem RUBY_CC_VERSION=3.0.0:2.7.0:2.6.0:2.5.1" + system "rake cross native gem RUBY_CC_VERSION=3.1.0:3.0.0:2.7.0:2.6.0:2.5.1" end else task 'gem:native' => [:genproto, 'gem:windows', 'gem:java'] diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 4612ab74fba7..402a6a52f9b5 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -17,8 +17,7 @@ Gem::Specification.new do |s| else s.files += Dir.glob('ext/**/*') s.extensions= ["ext/google/protobuf_c/extconf.rb"] - s.add_development_dependency "rake-compiler-dock", "= 1.1.0" - end + s.add_development_dependency "rake-compiler-dock", "= 1.2.1" end s.test_files = ["tests/basic.rb", "tests/stress.rb", "tests/generated_code_test.rb"] diff --git a/tests.sh b/tests.sh index ce66731e08e4..73460fda3da0 100755 --- a/tests.sh +++ b/tests.sh @@ -422,6 +422,10 @@ build_ruby30() { internal_build_cpp # For conformance tests. cd ruby && bash travis-test.sh ruby-3.0.2 && cd .. } +build_ruby31() { + internal_build_cpp # For conformance tests. + cd ruby && bash travis-test.sh ruby-3.1.0 && cd .. +} build_jruby92() { internal_build_cpp # For conformance tests. @@ -593,6 +597,7 @@ Usage: $0 { cpp | ruby26 | ruby27 | ruby30 | + ruby31 | jruby92 | jruby93 | ruby_all |