diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java index 94170830d6..f0a6e1f609 100644 --- a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java +++ b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java @@ -20,6 +20,8 @@ import java.util.Map; import java.util.Set; +import org.apache.commons.lang3.SystemUtils; + import org.apache.spark.SparkConf; import org.apache.spark.util.EventLoop; @@ -36,4 +38,8 @@ public static RssShuffleManager createShuffleManager( Map> failBlockIds) { return new RssShuffleManager(conf, isDriver, loop, successBlockIds, failBlockIds); } + + public static boolean isMacOnAppleSilicon() { + return SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64"); + } } diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java index 81d0eec453..a943a453b5 100644 --- a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java +++ b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java @@ -229,11 +229,25 @@ public void onError(Throwable e) { assertTrue(shuffleWriteMetrics.writeTime() > 0); assertEquals(6, shuffleWriteMetrics.recordsWritten()); // Spark3 and Spark2 use different version lz4, their length is different - assertEquals(120, shuffleWriteMetrics.bytesWritten()); + // it can happen that 2 different platforms compress the same data differently, + // yet the decoded outcome remains identical to original. + // https://github.com/lz4/lz4/issues/812 + if (TestUtils.isMacOnAppleSilicon()) { + assertEquals(144, shuffleWriteMetrics.bytesWritten()); + } else { + assertEquals(120, shuffleWriteMetrics.bytesWritten()); + } assertEquals(6, shuffleBlockInfos.size()); for (ShuffleBlockInfo shuffleBlockInfo : shuffleBlockInfos) { - assertEquals(20, shuffleBlockInfo.getLength()); + // it can happen that 2 different platforms compress the same data differently, + // yet the decoded outcome remains identical to original. + // https://github.com/lz4/lz4/issues/812 + if (TestUtils.isMacOnAppleSilicon()) { + assertEquals(24, shuffleBlockInfo.getLength()); + } else { + assertEquals(20, shuffleBlockInfo.getLength()); + } assertEquals(22, shuffleBlockInfo.getUncompressLength()); assertEquals(0, shuffleBlockInfo.getShuffleId()); if (shuffleBlockInfo.getPartitionId() == 0) { diff --git a/pom.xml b/pom.xml index f2b247a601..99c7849c32 100644 --- a/pom.xml +++ b/pom.xml @@ -40,11 +40,12 @@ 3.10 1.9 1.9.13 - 2.3.4 + 2.10.0 ${user.dir} 2.10.0 - 1.33.0 - 30.0-jre + 1.47.0 + 2.9.0 + 31.0.1-jre provided 2.8.5 4.5.3 @@ -64,8 +65,7 @@ 4.5.2 UTF-8 0.9.0 - 3.12.0 - 3.12.0 + 3.19.2 0.9.15 org.apache.uniffle false @@ -184,6 +184,13 @@ + + + com.google.code.gson + gson + ${gson.version} + + io.netty netty-all @@ -588,7 +595,7 @@ 0.6.1 - com.google.protobuf:protoc:${protoc.version}:exe:${os.detected.classifier} + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} grpc-java diff --git a/proto/pom.xml b/proto/pom.xml index 5f474bd996..bdccb11962 100644 --- a/proto/pom.xml +++ b/proto/pom.xml @@ -98,7 +98,7 @@ protobuf-maven-plugin - com.google.protobuf:protoc:3.12.0:exe:${os.detected.classifier} + com.google.protobuf:protoc:${protobuf.version}:exe:${os.detected.classifier} ${project.build.directory}/generated-sources