From f7c65d4beddd8630b621acf48fa7ad4dee911e57 Mon Sep 17 00:00:00 2001
From: Cheng Pan <chengpan@apache.org>
Date: Fri, 8 Jul 2022 15:07:54 +0800
Subject: [PATCH] Upgrade gRPC to support Apple Silicon (#13)

What changes were proposed in this pull request?
Upgrade gRPC to support Apple Silicon.

Why are the changes needed?
To support build on Apple M1 chips.

Does this PR introduce any user-facing change?
Yes, gRPC, protobuf related dependencies are updated, and developers who use Apple M1 chips can build now.

How was this patch tested?
local test
---
 .../org/apache/spark/shuffle/TestUtils.java   |  6 ++++++
 .../shuffle/writer/RssShuffleWriterTest.java  | 18 ++++++++++++++++--
 pom.xml                                       | 19 +++++++++++++------
 proto/pom.xml                                 |  2 +-
 4 files changed, 36 insertions(+), 9 deletions(-)

diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java
index 94170830d6..f0a6e1f609 100644
--- a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java
+++ b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java
@@ -20,6 +20,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.SystemUtils;
+
 import org.apache.spark.SparkConf;
 import org.apache.spark.util.EventLoop;
 
@@ -36,4 +38,8 @@ public static RssShuffleManager createShuffleManager(
       Map<String, Set<Long>> failBlockIds) {
     return new RssShuffleManager(conf, isDriver, loop, successBlockIds, failBlockIds);
   }
+
+  public static boolean isMacOnAppleSilicon() {
+    return SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64");
+  }
 }
diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
index 81d0eec453..a943a453b5 100644
--- a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
+++ b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
@@ -229,11 +229,25 @@ public void onError(Throwable e) {
     assertTrue(shuffleWriteMetrics.writeTime() > 0);
     assertEquals(6, shuffleWriteMetrics.recordsWritten());
     // Spark3 and Spark2 use different version lz4, their length is different
-    assertEquals(120, shuffleWriteMetrics.bytesWritten());
+    // it can happen that 2 different platforms compress the same data differently,
+    // yet the decoded outcome remains identical to original.
+    // https://github.com/lz4/lz4/issues/812
+    if (TestUtils.isMacOnAppleSilicon()) {
+      assertEquals(144, shuffleWriteMetrics.bytesWritten());
+    } else {
+      assertEquals(120, shuffleWriteMetrics.bytesWritten());
+    }
 
     assertEquals(6, shuffleBlockInfos.size());
     for (ShuffleBlockInfo shuffleBlockInfo : shuffleBlockInfos) {
-      assertEquals(20, shuffleBlockInfo.getLength());
+      // it can happen that 2 different platforms compress the same data differently,
+      // yet the decoded outcome remains identical to original.
+      // https://github.com/lz4/lz4/issues/812
+      if (TestUtils.isMacOnAppleSilicon()) {
+        assertEquals(24, shuffleBlockInfo.getLength());
+      } else {
+        assertEquals(20, shuffleBlockInfo.getLength());
+      }
       assertEquals(22, shuffleBlockInfo.getUncompressLength());
       assertEquals(0, shuffleBlockInfo.getShuffleId());
       if (shuffleBlockInfo.getPartitionId() == 0) {
diff --git a/pom.xml b/pom.xml
index f2b247a601..99c7849c32 100644
--- a/pom.xml
+++ b/pom.xml
@@ -40,11 +40,12 @@
     <commons-lang3.version>3.10</commons-lang3.version>
     <commons-codec.version>1.9</commons-codec.version>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
-    <error_prone_annotations.version>2.3.4</error_prone_annotations.version>
+    <error_prone_annotations.version>2.10.0</error_prone_annotations.version>
     <execution.root>${user.dir}</execution.root>
     <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
-    <grpc.version>1.33.0</grpc.version>
-    <guava.version>30.0-jre</guava.version>
+    <grpc.version>1.47.0</grpc.version>
+    <gson.version>2.9.0</gson.version>
+    <guava.version>31.0.1-jre</guava.version>
     <hadoop.scope>provided</hadoop.scope>
     <hadoop.version>2.8.5</hadoop.version>
     <httpclient.version>4.5.3</httpclient.version>
@@ -64,8 +65,7 @@
     <picocli.version>4.5.2</picocli.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <prometheus.simpleclient.version>0.9.0</prometheus.simpleclient.version>
-    <protobuf.version>3.12.0</protobuf.version>
-    <protoc.version>3.12.0</protoc.version>
+    <protobuf.version>3.19.2</protobuf.version>
     <roaring.bitmap.version>0.9.15</roaring.bitmap.version>
     <rss.shade.packageName>org.apache.uniffle</rss.shade.packageName>
     <skipDeploy>false</skipDeploy>
@@ -184,6 +184,13 @@
           </exclusion>
         </exclusions>
       </dependency>
+
+      <dependency>
+        <groupId>com.google.code.gson</groupId>
+        <artifactId>gson</artifactId>
+        <version>${gson.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>
@@ -588,7 +595,7 @@
           <version>0.6.1</version>
           <configuration>
             <protocArtifact>
-              com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}
+              com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
             </protocArtifact>
             <pluginId>grpc-java</pluginId>
             <pluginArtifact>
diff --git a/proto/pom.xml b/proto/pom.xml
index 5f474bd996..bdccb11962 100644
--- a/proto/pom.xml
+++ b/proto/pom.xml
@@ -98,7 +98,7 @@
           <artifactId>protobuf-maven-plugin</artifactId>
           <configuration>
             <protocArtifact>
-                com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier}
+                com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
             </protocArtifact>
             <!-- Place these in a location that compiler-plugin is already looking -->
             <outputDirectory>${project.build.directory}/generated-sources</outputDirectory>