on 12-30-2022 04:06 AM - edited on 01-19-2023 09:10 AM by FadiB
INC_03070101: Transformation error [Exception in thread "map-output-dispatcher-0" java.lang.UnsatisfiedLinkError: no zstd-jni in java.library.path: [/usr/java/packages/lib, /usr/lib64, /lib64, /lib, /usr/lib]
INC_03070101: Transformation error [Exception in thread "map-output-dispatcher-0" java.lang.UnsatisfiedLinkError: no zstd-jni in java.library.path: [/usr/java/packages/lib, /usr/lib64, /lib64, /lib, /usr/lib] at com.github.luben.zstd.util.Native.load(Native.java:108) at com.github.luben.zstd.util.Native.load(Native.java:60) at org.apache.spark.io.ZStdCompressionCodec.compressedOutputStream(CompressionCodec.scala:224) at org.apache.spark.MapOutputTracker$.serializeMapStatuses(MapOutputTracker.scala:920) at org.apache.spark.ShuffleStatus.$anonfun$serializedMapStatus$2(MapOutputTracker.scala:210) at org.apache.spark.ShuffleStatus.withWriteLock(MapOutputTracker.scala:72) at org.apache.spark.ShuffleStatus.serializedMapStatus(MapOutputTracker.scala:207) at org.apache.spark.MapOutputTrackerMaster$MessageLoop.run(MapOutputTracker.scala:457) 2022-11-02 10:45:14,249 WARN scheduler.TaskSetManager: Lost task 0.0 in stage 1.0 (TID 1, 129.147.132.60, executor 0): org.apache.spark.SparkException: Error communicating with MapOutputTracker at org.apache.spark.MapOutputTracker.askTracker(MapOutputTracker.scala:309) at org.apache.spark.MapOutputTrackerWorker.$anonfun$getStatuses$2(MapOutputTracker.scala:855) at org.apache.spark.util.KeyLock.withLock(KeyLock.scala:64) at org.apache.spark.MapOutputTrackerWorker.getStatuses(MapOutputTracker.scala:851) at org.apache.spark.MapOutputTrackerWorker.getMapSizesByExecutorId(MapOutputTracker.scala:808) at org.apache.spark.shuffle.sort.SortShuffleManager.getReader(SortShuffleManager.scala:128) at org.apache.spark.sql.execution.ShuffledRowRDD.compute(ShuffledRowRDD.scala:185) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.sql.execution.SQLExecutionRDD.compute(SQLExecutionRDD.scala:55) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.$anonfun$getOrCompute$1(RDD.scala:362) at org.apache.spark.storage.BlockManager.$anonfun$doPutIterator$1(BlockManager.scala:1388) at org.apache.spark.storage.BlockManager.org$apache$spark$storage$BlockManager$$doPut(BlockManager.scala:1298) at org.apache.spark.storage.BlockManager.doPutIterator(BlockManager.scala:1362) at org.apache.spark.storage.BlockManager.getOrElseUpdate(BlockManager.scala:1186) at org.apache.spark.rdd.RDD.getOrCompute(RDD.scala:360) at org.apache.spark.rdd.RDD.iterator(RDD.scala:311) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala:313) at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:52) at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:349) at org.apache.spark.rdd.RDD.iterator(RDD.scala
libzstd.x86_64
with yum install libzstd.x86_64
/usr/lib64
as follows:
<incorta_home>/IncortaNode/spark/jars
zstd-jni-1.4.4-3.jar
to the temp folder, then extract it with jar xvf zstd-jni-1.4.4-3.jar
libzstd-jni.so
file that is under linux/amd64
to /usr/lib64