如何为Spark Application指定不同的JDK版本详解

$SPARK_HOME/bin/spark-submit \
 --conf "spark.executorEnv.JAVA_HOME=/usr/java/jdk1.8" \
 ...

$SPARK_HOME/bin/spark-submit \
 --conf "spark.executorEnv.JAVA_HOME=/usr/java/jdk1.8" \
 --conf "spark.yarn.appMasterEnv.JAVA_HOME=/usr/java/jdk1.8" \
 ...

$SPARK_HOME/bin/spark-submit \
 --conf "spark.yarn.dist.archives=jdk-8u141-linux-x64.tar.gz" \
 --conf "spark.executorEnv.JAVA_HOME=./jdk-8u141-linux-x64.tar.gz/jdk1.8.0_141" \
 --conf "spark.yarn.appMasterEnv.JAVA_HOME=./jdk-8u141-linux-x64.tar.gz/jdk1.8.0_141" \
 ...

stan  590751 590745 0 20:45 ?  00:00:14 ./jdk-8u141-linux-x64.tar.gz/jdk1.8.0_141/bin/java -server -Xmx512m -XX:+UseG1GC -XX:+UnlockDiagnosticVMOptions -XX:+G1SummarizeConcMark -XX:InitiatingHeapOccupancyPercent=35 -XX:PermSize=256M -XX:+PrintGCDetails -XX:+PrintGCDateStamps -Xloggc:./gc.log -verbose:gc -Djava.io.tmpdir=/home/stan/tmp/hadoop-stan/nm-local-dir/usercache/stan/appcache/application_1508397483453_0095/container_1508397483453_0095_01_000004/tmp -Dspark.driver.port=52986 -Dspark.yarn.app.container.log.dir=/home/stan//hadoop-2.6.4/logs/userlogs/application_1508397483453_0095/container_1508397483453_0095_01_000004 -XX:OnOutOfMemoryError=kill %p org.apache.spark.executor.CoarseGrainedExecutorBackend --driver-url spark://CoarseGrainedScheduler@10.0.0.110:52986 --executor-id 3 --hostname stan --cores 1 --app-id application_1508397483453_0095 --user-class-path file:/home/stan/tmp/hadoop-stan/nm-local-dir/usercache/stan/appcache/application_1508397483453_0095/container_1508397483453_0095_01_000004/__app__.jar

17/06/27 14:34:41 INFO deprecation: mapred.map.tasks is deprecated. Instead, use mapreduce.job.maps 
17/06/27 14:34:41 INFO MemoryStore: Block broadcast_0 stored as values in memory (estimated size 788.8 KB, free 1246.5 MB) 
17/06/27 14:34:41 INFO MemoryStore: Block broadcast_0_piece0 stored as bytes in memory (estimated size 54.0 KB, free 1246.4 MB) 
17/06/27 14:34:41 INFO BlockManagerInfo: Added broadcast_0_piece0 in memory on 10.50.70.121:37335 (size: 54.0 KB, free: 1247.2 MB) 
17/06/27 14:34:41 INFO SparkContext: Created broadcast 0 from rdd at TradeInfoOutlier.scala:30 
Exception in thread "main" java.lang.NoSuchMethodError: scala.reflect.api.JavaUniverse.runtimeMirror(Ljava/lang/ClassLoader;)Lscala/reflect/api/JavaUniverse$JavaMirror; 
at com.fangdd.data.profile.outlier.TradeInfoOutlier$.main(TradeInfoOutlier.scala:30) 
at com.fangdd.data.profile.outlier.TradeInfoOutlier.main(TradeInfoOutlier.scala) 
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) 
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) 
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) 
at java.lang.reflect.Method.invoke(Method.java:498) 
at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:745) 
at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:181) 
at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:206) 
at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:121) 
at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) 
17/06/27 14:34:42 INFO SparkContext: Invoking stop() from shutdown hook