Upgrade gRPC to support Apple Silicon (#13)

What changes were proposed in this pull request? Upgrade gRPC to support Apple Silicon. Why are the changes needed? To support build on Apple M1 chips. Does this PR introduce any user-facing change? Yes, gRPC, protobuf related dependencies are updated, and developers who use Apple M1 chips can build now. How was this patch tested? local test
apache · Jul 8, 2022 · f7c65d4 · f7c65d4
1 parent 37beb1f
commit f7c65d4
Show file tree

Hide file tree

Showing 4 changed files with 36 additions and 9 deletions.
diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java
@@ -20,6 +20,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.SystemUtils;
+
 import org.apache.spark.SparkConf;
 import org.apache.spark.util.EventLoop;
 
@@ -36,4 +38,8 @@ public static RssShuffleManager createShuffleManager(
       Map<String, Set<Long>> failBlockIds) {
     return new RssShuffleManager(conf, isDriver, loop, successBlockIds, failBlockIds);
   }
+
+  public static boolean isMacOnAppleSilicon() {
+    return SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64");
+  }
 }
diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
@@ -229,11 +229,25 @@ public void onError(Throwable e) {
     assertTrue(shuffleWriteMetrics.writeTime() > 0);
     assertEquals(6, shuffleWriteMetrics.recordsWritten());
     // Spark3 and Spark2 use different version lz4, their length is different
-    assertEquals(120, shuffleWriteMetrics.bytesWritten());
+    // it can happen that 2 different platforms compress the same data differently,
+    // yet the decoded outcome remains identical to original.
+    // https://github.com/lz4/lz4/issues/812
+    if (TestUtils.isMacOnAppleSilicon()) {
+      assertEquals(144, shuffleWriteMetrics.bytesWritten());
+    } else {
+      assertEquals(120, shuffleWriteMetrics.bytesWritten());
+    }
 
     assertEquals(6, shuffleBlockInfos.size());
     for (ShuffleBlockInfo shuffleBlockInfo : shuffleBlockInfos) {
-      assertEquals(20, shuffleBlockInfo.getLength());
+      // it can happen that 2 different platforms compress the same data differently,
+      // yet the decoded outcome remains identical to original.
+      // https://github.com/lz4/lz4/issues/812
+      if (TestUtils.isMacOnAppleSilicon()) {
+        assertEquals(24, shuffleBlockInfo.getLength());
+      } else {
+        assertEquals(20, shuffleBlockInfo.getLength());
+      }
       assertEquals(22, shuffleBlockInfo.getUncompressLength());
       assertEquals(0, shuffleBlockInfo.getShuffleId());
       if (shuffleBlockInfo.getPartitionId() == 0) {

diff --git a/pom.xml b/pom.xml
@@ -40,11 +40,12 @@
     <commons-lang3.version>3.10</commons-lang3.version>
     <commons-codec.version>1.9</commons-codec.version>
     <codehaus.jackson.version>1.9.13</codehaus.jackson.version>
-    <error_prone_annotations.version>2.3.4</error_prone_annotations.version>
+    <error_prone_annotations.version>2.10.0</error_prone_annotations.version>
     <execution.root>${user.dir}</execution.root>
     <fasterxml.jackson.version>2.10.0</fasterxml.jackson.version>
-    <grpc.version>1.33.0</grpc.version>
-    <guava.version>30.0-jre</guava.version>
+    <grpc.version>1.47.0</grpc.version>
+    <gson.version>2.9.0</gson.version>
+    <guava.version>31.0.1-jre</guava.version>
     <hadoop.scope>provided</hadoop.scope>
     <hadoop.version>2.8.5</hadoop.version>
     <httpclient.version>4.5.3</httpclient.version>
@@ -64,8 +65,7 @@
     <picocli.version>4.5.2</picocli.version>
     <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
     <prometheus.simpleclient.version>0.9.0</prometheus.simpleclient.version>
-    <protobuf.version>3.12.0</protobuf.version>
-    <protoc.version>3.12.0</protoc.version>
+    <protobuf.version>3.19.2</protobuf.version>
     <roaring.bitmap.version>0.9.15</roaring.bitmap.version>
     <rss.shade.packageName>org.apache.uniffle</rss.shade.packageName>
     <skipDeploy>false</skipDeploy>
@@ -184,6 +184,13 @@
           </exclusion>
         </exclusions>
       </dependency>
+
+      <dependency>
+        <groupId>com.google.code.gson</groupId>
+        <artifactId>gson</artifactId>
+        <version>${gson.version}</version>
+      </dependency>
+
       <dependency>
         <groupId>io.netty</groupId>
         <artifactId>netty-all</artifactId>
@@ -588,7 +595,7 @@
           <version>0.6.1</version>
           <configuration>
             <protocArtifact>
-              com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier}
+              com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
             </protocArtifact>
             <pluginId>grpc-java</pluginId>
             <pluginArtifact>

diff --git a/proto/pom.xml b/proto/pom.xml
@@ -98,7 +98,7 @@
           <artifactId>protobuf-maven-plugin</artifactId>
           <configuration>
             <protocArtifact>
-                com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier}
+                com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier}
             </protocArtifact>
             <!-- Place these in a location that compiler-plugin is already looking -->
             <outputDirectory>${project.build.directory}/generated-sources</outputDirectory>