Fix spark3 test

apache · Jul 4, 2022 · e83d15b · e83d15b
1 parent 8880544
commit e83d15b
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 2 deletions.
diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/TestUtils.java
@@ -20,6 +20,8 @@
 import java.util.Map;
 import java.util.Set;
 
+import org.apache.commons.lang3.SystemUtils;
+
 import org.apache.spark.SparkConf;
 import org.apache.spark.util.EventLoop;
 
@@ -36,4 +38,8 @@ public static RssShuffleManager createShuffleManager(
       Map<String, Set<Long>> failBlockIds) {
     return new RssShuffleManager(conf, isDriver, loop, successBlockIds, failBlockIds);
   }
+
+  public static boolean isMacOnAppleSilicon() {
+    return SystemUtils.IS_OS_MAC_OSX && SystemUtils.OS_ARCH.equals("aarch64");
+  }
 }
diff --git a/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java b/client-spark/spark3/src/test/java/org/apache/spark/shuffle/writer/RssShuffleWriterTest.java
@@ -229,11 +229,25 @@ public void onError(Throwable e) {
     assertTrue(shuffleWriteMetrics.writeTime() > 0);
     assertEquals(6, shuffleWriteMetrics.recordsWritten());
     // Spark3 and Spark2 use different version lz4, their length is different
-    assertEquals(120, shuffleWriteMetrics.bytesWritten());
+    // it can happen that 2 different platforms compress the same data differently,
+    // yet the decoded outcome remains identical to original.
+    // https://github.com/lz4/lz4/issues/812
+    if (TestUtils.isMacOnAppleSilicon()) {
+      assertEquals(144, shuffleWriteMetrics.bytesWritten());
+    } else {
+      assertEquals(120, shuffleWriteMetrics.bytesWritten());
+    }
 
     assertEquals(6, shuffleBlockInfos.size());
     for (ShuffleBlockInfo shuffleBlockInfo : shuffleBlockInfos) {
-      assertEquals(20, shuffleBlockInfo.getLength());
+      // it can happen that 2 different platforms compress the same data differently,
+      // yet the decoded outcome remains identical to original.
+      // https://github.com/lz4/lz4/issues/812
+      if (TestUtils.isMacOnAppleSilicon()) {
+        assertEquals(24, shuffleBlockInfo.getLength());
+      } else {
+        assertEquals(20, shuffleBlockInfo.getLength());
+      }
       assertEquals(22, shuffleBlockInfo.getUncompressLength());
       assertEquals(0, shuffleBlockInfo.getShuffleId());
       if (shuffleBlockInfo.getPartitionId() == 0) {