From be163c3a117c9e55a9e388e90c23d1eaacf3a2ed Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Wed, 18 Dec 2019 14:31:54 +0900 Subject: [PATCH 1/9] Build extensions for Ruby 2.7 --- kokoro/linux/dockerfile/test/ruby/Dockerfile | 1 + .../ruby/macos/ruby/ruby_build_environment.sh | 14 +++++++------- ruby/Rakefile | 4 ++-- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/kokoro/linux/dockerfile/test/ruby/Dockerfile b/kokoro/linux/dockerfile/test/ruby/Dockerfile index 41bfedeb330f..9037da715f09 100644 --- a/kokoro/linux/dockerfile/test/ruby/Dockerfile +++ b/kokoro/linux/dockerfile/test/ruby/Dockerfile @@ -32,6 +32,7 @@ RUN /bin/bash -l -c "rvm install 2.3.8" RUN /bin/bash -l -c "rvm install 2.4.5" RUN /bin/bash -l -c "rvm install 2.5.1" RUN /bin/bash -l -c "rvm install 2.6.0" +RUN /bin/bash -l -c "rvm install 2.7.0" RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc" diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index 5776e3c3496e..81d377eb39f7 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -5,8 +5,8 @@ set -ex set +ex # rvm script is very verbose and exits with errorcode source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose -time rvm install 2.5.0 -rvm use 2.5.0 --default +time rvm install 2.7.0 +rvm use 2.7.0 --default gem install rake-compiler --no-document gem install bundler --no-document rvm osx-ssl-certs status all @@ -17,13 +17,13 @@ rm -rf ~/.rake-compiler CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) -curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.3/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" +curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.9/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" # See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details patch "$CROSS_RUBY" << EOF --- cross-ruby.rake 2018-04-10 11:32:16.000000000 -0700 +++ patched 2018-04-10 11:40:25.000000000 -0700 -@@ -133,8 +133,10 @@ +@@ -141,8 +141,10 @@ "--host=#{MINGW_HOST}", "--target=#{MINGW_TARGET}", "--build=#{RUBY_BUILD}", @@ -35,9 +35,9 @@ patch "$CROSS_RUBY" << EOF '--with-ext=' ] -@@ -151,6 +153,7 @@ +@@ -159,6 +161,7 @@ # make - file "#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/ruby.exe" => ["#{USER_HOME}/builds/#{MINGW_HOST}/#{RUBY_CC_VERSION}/Makefile"] do |t| + file "#{build_dir}/ruby.exe" => ["#{build_dir}/Makefile"] do |t| chdir File.dirname(t.prerequisites.first) do + sh "test -s verconf.h || rm -f verconf.h" # if verconf.h has size 0, make sure it gets re-built by make sh MAKE @@ -47,7 +47,7 @@ EOF MAKE="make -j8" -for v in 2.6.0 2.5.1 2.4.0 2.3.0 ; do +for v in 2.7.0 2.6.0 2.5.1 2.4.0 2.3.0 ; do ccache -c rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" done diff --git a/ruby/Rakefile b/ruby/Rakefile index ad70e3107ccb..5f9494fa5dc2 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -70,13 +70,13 @@ else task 'gem:windows' do require 'rake_compiler_dock' - RakeCompilerDock.sh "bundle && IN_DOCKER=true rake cross native gem RUBY_CC_VERSION=2.6.0:2.5.0:2.4.0:2.3.0" + RakeCompilerDock.sh "bundle && IN_DOCKER=true rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0" end if RUBY_PLATFORM =~ /darwin/ task 'gem:native' do system "rake genproto" - system "rake cross native gem RUBY_CC_VERSION=2.6.0:2.5.1:2.4.0:2.3.0" + system "rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.1:2.4.0:2.3.0" end else task 'gem:native' => [:genproto, 'gem:windows'] From db23ac26cf42cd696f813de1a25cb55814a10e43 Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Wed, 8 Jan 2020 11:39:07 +0900 Subject: [PATCH 2/9] Try installing bundler 2.x --- kokoro/release/ruby/linux/ruby/ruby_build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/kokoro/release/ruby/linux/ruby/ruby_build.sh b/kokoro/release/ruby/linux/ruby/ruby_build.sh index 9fc42b13eb48..95f1dea906b3 100755 --- a/kokoro/release/ruby/linux/ruby/ruby_build.sh +++ b/kokoro/release/ruby/linux/ruby/ruby_build.sh @@ -11,6 +11,7 @@ fi umask 0022 pushd ruby +gem install bundler -v 2.1.4 bundle install && bundle exec rake gem:native ls pkg mv pkg/* $ARTIFACT_DIR From a32bf1bab86e9eda9bb553eac542c0ac355d745e Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Thu, 9 Jan 2020 19:15:48 +0900 Subject: [PATCH 3/9] Try bumping rake-compiler-dock --- ruby/google-protobuf.gemspec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index bf60b6ae9e34..74f1604757e7 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -17,7 +17,7 @@ Gem::Specification.new do |s| else s.files += Dir.glob('ext/**/*') s.extensions= ["ext/google/protobuf_c/extconf.rb"] - s.add_development_dependency "rake-compiler-dock", "~> 0.6.0" + s.add_development_dependency "rake-compiler-dock", ">= 1.0.1", "< 2.0" end s.test_files = ["tests/basic.rb", "tests/stress.rb", From 855b9904c482afc710fdf149cb0ef24a37504471 Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Fri, 10 Apr 2020 16:30:43 +0900 Subject: [PATCH 4/9] Use standard RCD images --- .../linux/dockerfile/release/ruby_rake_compiler/Dockerfile | 3 --- kokoro/release/ruby/linux/prepare_build.sh | 6 ------ 2 files changed, 9 deletions(-) delete mode 100644 kokoro/linux/dockerfile/release/ruby_rake_compiler/Dockerfile diff --git a/kokoro/linux/dockerfile/release/ruby_rake_compiler/Dockerfile b/kokoro/linux/dockerfile/release/ruby_rake_compiler/Dockerfile deleted file mode 100644 index 866b289343d7..000000000000 --- a/kokoro/linux/dockerfile/release/ruby_rake_compiler/Dockerfile +++ /dev/null @@ -1,3 +0,0 @@ -FROM grpctesting/rake-compiler-dock_53c22085d091183c528303791e7771359f699bcf - -RUN /bin/bash -l -c "gem update --system '2.7.9' && gem install bundler" diff --git a/kokoro/release/ruby/linux/prepare_build.sh b/kokoro/release/ruby/linux/prepare_build.sh index f2257c3c7530..91d186898cbd 100755 --- a/kokoro/release/ruby/linux/prepare_build.sh +++ b/kokoro/release/ruby/linux/prepare_build.sh @@ -7,12 +7,6 @@ echo 'DOCKER_OPTS="${DOCKER_OPTS} --graph=/tmpfs/docker"' | sudo tee --append /e echo 'DOCKER_OPTS="${DOCKER_OPTS} --registry-mirror=https://mirror.gcr.io"' | sudo tee --append /etc/default/docker sudo service docker restart -# Download Docker images from DockerHub -DOCKERHUB_ORGANIZATION=protobuftesting -DOCKERFILE_DIR=kokoro/linux/dockerfile/release/ruby_rake_compiler -DOCKERFILE_PREFIX=$(basename $DOCKERFILE_DIR) -export RAKE_COMPILER_DOCK_IMAGE=${DOCKERHUB_ORGANIZATION}/${DOCKERFILE_PREFIX}_$(sha1sum $DOCKERFILE_DIR/Dockerfile | cut -f1 -d\ ) - # All artifacts come here mkdir artifacts export ARTIFACT_DIR=$(pwd)/artifacts From af7bf3dad0c571a74761b19e0f04bcb58723c32b Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Mon, 13 Apr 2020 14:05:52 +0900 Subject: [PATCH 5/9] Avoid 'rake cross native' with rake-compiler-dock --- ruby/Rakefile | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/ruby/Rakefile b/ruby/Rakefile index 5f9494fa5dc2..905d2c868136 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -70,7 +70,11 @@ else task 'gem:windows' do require 'rake_compiler_dock' - RakeCompilerDock.sh "bundle && IN_DOCKER=true rake cross native gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0" + ['x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux'].each do |plat| + RakeCompilerDock.sh <<-"EOT", platform: plat + bundle && IN_DOCKER=true rake native:#{plat} RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0 + EOT + end end if RUBY_PLATFORM =~ /darwin/ From 3631382c8dee7f65bc2a11b2851611c75407408d Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Tue, 14 Apr 2020 14:42:02 +0900 Subject: [PATCH 6/9] Use Ruby 2.5 for building Ruby <= 2.6 --- .../ruby/macos/ruby/ruby_build_environment.sh | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index 81d377eb39f7..ccb33051034c 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -5,6 +5,10 @@ set -ex set +ex # rvm script is very verbose and exits with errorcode source $HOME/.rvm/scripts/rvm set -e # rvm commands are very verbose +time rvm install 2.5.0 +rvm use 2.5.0 +gem install rake-compiler --no-document +gem install bundler --no-document time rvm install 2.7.0 rvm use 2.7.0 --default gem install rake-compiler --no-document @@ -47,10 +51,25 @@ EOF MAKE="make -j8" -for v in 2.7.0 2.6.0 2.5.1 2.4.0 2.3.0 ; do +set +x # rvm commands are very verbose +rvm use 2.7.0 +set -x +ruby --version | grep 'ruby 2.7.0' +for v in 2.7.0 ; do + ccache -c + rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" +done +set +x +rvm use 2.5.0 +set -x +ruby --version | grep 'ruby 2.5.0' +for v in 2.6.0 2.5.1 2.4.0 2.3.0 ; do ccache -c rake -f "$CROSS_RUBY" cross-ruby VERSION="$v" HOST=x86_64-darwin11 MAKE="$MAKE" done +set +x +rvm use 2.7.0 +set -x sed 's/x86_64-darwin-11/universal-darwin/' ~/.rake-compiler/config.yml > "$CROSS_RUBY" mv "$CROSS_RUBY" ~/.rake-compiler/config.yml From d9e24d9f3c0f46e9e727ac2696c37c554f2ca0b0 Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Tue, 14 Apr 2020 14:42:52 +0900 Subject: [PATCH 7/9] Use rake-compiler 1.1.0 --- kokoro/release/ruby/macos/ruby/ruby_build_environment.sh | 2 +- ruby/google-protobuf.gemspec | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh index ccb33051034c..a4bf9e051eea 100755 --- a/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh +++ b/kokoro/release/ruby/macos/ruby/ruby_build_environment.sh @@ -21,7 +21,7 @@ rm -rf ~/.rake-compiler CROSS_RUBY=$(mktemp tmpfile.XXXXXXXX) -curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.0.9/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" +curl https://raw.githubusercontent.com/rake-compiler/rake-compiler/v1.1.0/tasks/bin/cross-ruby.rake > "$CROSS_RUBY" # See https://github.com/grpc/grpc/issues/12161 for verconf.h patch details patch "$CROSS_RUBY" << EOF diff --git a/ruby/google-protobuf.gemspec b/ruby/google-protobuf.gemspec index 74f1604757e7..cd9af71b6e96 100644 --- a/ruby/google-protobuf.gemspec +++ b/ruby/google-protobuf.gemspec @@ -23,7 +23,7 @@ Gem::Specification.new do |s| "tests/stress.rb", "tests/generated_code_test.rb"] s.required_ruby_version = '>= 2.3' - s.add_development_dependency "rake-compiler", "~> 0.9.5" + s.add_development_dependency "rake-compiler", "~> 1.1.0" s.add_development_dependency "test-unit", '~> 3.0', '>= 3.0.9' s.add_development_dependency "rubygems-tasks", "~> 0.2.4" end From 0fa4840dee7285f04b5092891174d1a858485b52 Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Wed, 15 Apr 2020 09:58:58 +0900 Subject: [PATCH 8/9] Specify target --- ruby/Rakefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ruby/Rakefile b/ruby/Rakefile index 905d2c868136..b0b803dd65bb 100644 --- a/ruby/Rakefile +++ b/ruby/Rakefile @@ -72,7 +72,8 @@ else require 'rake_compiler_dock' ['x86-mingw32', 'x64-mingw32', 'x86_64-linux', 'x86-linux'].each do |plat| RakeCompilerDock.sh <<-"EOT", platform: plat - bundle && IN_DOCKER=true rake native:#{plat} RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0 + bundle && \ + IN_DOCKER=true rake native:#{plat} pkg/#{spec.full_name}-#{plat}.gem RUBY_CC_VERSION=2.7.0:2.6.0:2.5.0:2.4.0:2.3.0 EOT end end From 4cc4273b95fd78110c948223515fa8f6dd82ab74 Mon Sep 17 00:00:00 2001 From: Masaki Hara Date: Thu, 16 Apr 2020 11:02:40 +0900 Subject: [PATCH 9/9] Don't update Ruby test image for now --- kokoro/linux/dockerfile/test/ruby/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/kokoro/linux/dockerfile/test/ruby/Dockerfile b/kokoro/linux/dockerfile/test/ruby/Dockerfile index 9037da715f09..41bfedeb330f 100644 --- a/kokoro/linux/dockerfile/test/ruby/Dockerfile +++ b/kokoro/linux/dockerfile/test/ruby/Dockerfile @@ -32,7 +32,6 @@ RUN /bin/bash -l -c "rvm install 2.3.8" RUN /bin/bash -l -c "rvm install 2.4.5" RUN /bin/bash -l -c "rvm install 2.5.1" RUN /bin/bash -l -c "rvm install 2.6.0" -RUN /bin/bash -l -c "rvm install 2.7.0" RUN /bin/bash -l -c "echo 'gem: --no-ri --no-rdoc' > ~/.gemrc" RUN /bin/bash -l -c "echo 'export PATH=/usr/local/rvm/bin:$PATH' >> ~/.bashrc"