01
启动代码
bin/spark-submit \--class org.apache.spark.examples.SparkPi \--master yarn \--deploy-mode cluster \./examples/jars/spark-examples_2.11-2.1.1.jar \100
yarn 会按照下面的顺序依次启动了 3 个进程:
SparkSubmit
ApplicationMaster
CoarseGrainedExecutorBackend
1bin/spark-submit 启动脚本分析
if [ -z "${SPARK_HOME}" ]; thensource "$(dirname "$0")"/find-spark-homefi# disable randomized hash for string in Python 3.3+export PYTHONHASHSEED=0//启动类exec "${SPARK_HOME}"/bin/spark-class org.apache.spark.deploy.SparkSubmit "$@"
启动类org.apache.spark.deploy.SparkSubmit
/bin/spark-class
exec "${CMD[@]}"
最终启动类
/test/jdk1.8.0_172/bin/java-cp/test/spark-yarn/conf/:/test/spark-yarn/jars/*:/test/hadoop-2.7.2/etc/hadoop/org.apache.spark.deploy.SparkSubmit--master yarn--deploy-mode cluster--class org.apache.spark.examples.SparkPi./examples/jars/spark-examples_2.11-2.1.1.jar 100
2org.apache.spark.deploy.SparkSubmit 源码
SparkSubmit伴生对象
def main(args: Array[String]): Unit = {/*参数--master yarn--deploy-mode cluster--class org.apache.spark.examples.SparkPi./examples/jars/spark-examples_2.11-2.1.1.jar 100*/val appArgs = new SparkSubmitArguments(args)appArgs.action match {// 如果没有指定 action, 则 action 的默认值是: action = Option(action).getOrElse(SUBMIT)case SparkSubmitAction.SUBMIT => submit(appArgs)case SparkSubmitAction.KILL => kill(appArgs)case SparkSubmitAction.REQUEST_STATUS => requestStatus(appArgs)}}
submit 方法
/*** 使用提供的参数提交应用程序* 1. 准备启动环境.* 根据集群管理器和部署模式为 child main class 设置正确的 classpath, 系统属性,应用参数* 2. 使用启动环境调用 child main class 的 main 方法*/@tailrecprivate def submit(args: SparkSubmitArguments): Unit = {// 准备提交环境 childMainClass = "org.apache.spark.deploy.yarn.Client"val (childArgs, childClasspath, sysProps, childMainClass) = prepareSubmitEnvironment(args)def doRunMain(): Unit = {if (args.proxyUser != null) {} else {runMain(childArgs, childClasspath, sysProps, childMainClass, args.verbose)}}if (args.isStandaloneCluster && args.useRest) {// 在其他任何模式, 仅仅运行准备好的主类} else {doRunMain()}
prepareSubmitEnvironment 方法
/*** Prepare the environment for submitting an application.* This returns a 4-tuple:* (1) the arguments for the child process,* (2) a list of classpath entries for the child,* (3) a map of system properties, and* (4) the main class for the child* Exposed for testing.*/private[deploy] def prepareSubmitEnvironment(args: SparkSubmitArguments): (Seq[String], Seq[String], Map[String, String], String) = {// Return values// 需要返回的值val childArgs = new ArrayBuffer[String]()val childClasspath = new ArrayBuffer[String]()val sysProps = new HashMap[String, String]()var childMainClass = ""// Set the cluster manager// 设置集群管理器val clusterManager: Int = args.master match {case "yarn" => YARN//yarn-client 是老版本case "yarn-client" | "yarn-cluster" =>printWarning(s"Master ${args.master} is deprecated since 2.0." +" Please use master \"yarn\" with specified deploy mode instead.")YARNcase m if m.startsWith("spark") => STANDALONEcase m if m.startsWith("mesos") => MESOScase m if m.startsWith("local") => LOCALcase _ =>printErrorAndExit("Master must either be yarn or start with spark, mesos, local")-1}// Set the deploy mode; default is client mode// 设置发布模式, 默认是 client 模式var deployMode: Int = args.deployMode match {case "client" | null => CLIENT // 默认是 client 模式case "cluster" => CLUSTERcase _ => printErrorAndExit("Deploy mode must be either client or cluster"); -1}// Because the deprecated way of specifying "yarn-cluster" and "yarn-client" encapsulate both// the master and deploy mode, we have some logic to infer the master and deploy mode// from each other if only one is specified, or exit early if they are at odds.// 兼容旧模式: yarn-cluster 和 yarn-clientif (clusterManager == YARN) {(args.master, args.deployMode) match {case ("yarn-cluster", null) =>deployMode = CLUSTERargs.master = "yarn"case ("yarn-cluster", "client") =>printErrorAndExit("Client deploy mode is not compatible with master \"yarn-cluster\"")case ("yarn-client", "cluster") =>printErrorAndExit("Cluster deploy mode is not compatible with master \"yarn-client\"")case (_, mode) =>args.master = "yarn"}// Update args.deployMode if it is null. It will be passed down as a Spark property later.(args.deployMode, deployMode) match {case (null, CLIENT) => args.deployMode = "client"case (null, CLUSTER) => args.deployMode = "cluster"case _ =>}}
doRunMain 方法
def doRunMain(): Unit = { if (args.proxyUser != null) { 。。。 } else { runMain(childArgs, childClasspath, sysProps, childMainClass, args.verbose) }}
runMain 方法
/*** Run the main method of the child class using the provided launch environment.* 使用给定启动环境运行 child class 的 main 方法* 注意: 如果使用了cluster deploy mode, 主类并不是用户提供* Note that this main class will not be the one provided by the user if we're* running cluster deploy mode or python applications.*/private def runMain(childArgs: Seq[String],childClasspath: Seq[String],sysProps: Map[String, String],childMainClass: String,verbose: Boolean): Unit = {...var mainClass: Class[_] = nulltry {// 使用反射的方式加载 childMainClass = "org.apache.spark.deploy.yarn.Client"mainClass = Utils.classForName(childMainClass) // 在 client模式下, 直接指向用户的类} catch {...}// 反射出来 Client 的 main 方法// 如果是 yarn-client 就是反射的用户类的mainval mainMethod = mainClass.getMethod("main", new Array[String](0).getClass)// main 方法必须是静态的if (!Modifier.isStatic(mainMethod.getModifiers)) {throw new IllegalStateException("The main method in the given main class must be static")}...try {// 调用 main 方法.mainMethod.invoke(null, childArgs.toArray)} catch {case t: Throwable =>...}}
3org.apache.spark.deploy.yarn.Client
main方法
def main(argStrings: Array[String]) {// 设置环境变量 SPARK_YARN_MODE 表示运行在 YARN mode// 注意: 任何带有 SPARK_ 前缀的环境变量都会分发到所有的进程, 也包括远程进程System.setProperty("SPARK_YARN_MODE", "true")val sparkConf = new SparkConf// 对传递来的参数进一步封装val args = new ClientArguments(argStrings)new Client(args, sparkConf).run()}
Client(args, sparkConf).run()
/*** Submit an application to the ResourceManager.* If set spark.yarn.submit.waitAppCompletion to true, it will stay alive* reporting the application's status until the application has exited for any reason.* Otherwise, the client process will exit after submission.* If the application finishes with a failed, killed, or undefined status,* throw an appropriate SparkException.** 向 RM 提交应用** 如果设置了spark.yarn.submit.waitAppCompletion 为 true,* 则 Client 会一直保持 alive 来报告应用的状态, 直到应用退出* 否则, 应用提交之后, 客户端会自动退出.** 如果应用由于失败, 被杀或者其他未知转态, 则会抛出异常*/def run(): Unit = {// 提交应用, 返回应用的 idthis.appId = submitApplication()if (!launcherBackend.isConnected() && fireAndForget) {val report = getApplicationReport(appId)...} else {...}}
submitApplication（）
/*** Submit an application running our ApplicationMaster to the ResourceManager.** The stable Yarn API provides a convenience method (YarnClient#createApplication) for* * creating applications and setting up the application submission context. This was not* * available in the alpha API.** 向 ResourceManager 提交运行 ApplicationMaster 的应用程序。**/def submitApplication(): ApplicationId = {var appId: ApplicationId = nulltry {launcherBackend.connect()// Setup the credentials before doing anything else,// so we have don't have issues at any point.setupCredentials()// 初始化 yarn 客户端yarnClient.init(yarnConf)// 启动 yarn 客户端yarnClient.start()logInfo("Requesting a new application from cluster with %d NodeManagers".format(yarnClient.getYarnClusterMetrics.getNumNodeManagers))// Get a new application from our RM// 从 RM 创建一个应用程序val newApp = yarnClient.createApplication()val newAppResponse = newApp.getNewApplicationResponse()// 获取到 applicationIDappId = newAppResponse.getApplicationId()reportLauncherState(SparkAppHandle.State.SUBMITTED)launcherBackend.setAppId(appId.toString)new CallerContext("CLIENT", Option(appId.toString)).setCurrentContext()// Verify whether the cluster has enough resources for our AMverifyClusterResources(newAppResponse)// Set up the appropriate contexts to launch our AM// 设置正确的上下文对象来启动 ApplicationMaster org.apache.spark.deploy.yarn.ApplicationMasterval containerContext = createContainerLaunchContext(newAppResponse)// 创建应用程序提交任务上下文val appContext = createApplicationSubmissionContext(newApp, containerContext)// Finally, submit and monitor the applicationlogInfo(s"Submitting application $appId to ResourceManager")// 提交应用给 ResourceManager 启动 ApplicationMaster// org.apache.spark.deploy.yarn.ApplicationMasteryarnClient.submitApplication(appContext)appId} catch {}}
createContainerLaunchContext（）
private def createContainerLaunchContext(newAppResponse: GetNewApplicationResponse): ContainerLaunchContext = {val amClass =if (isClusterMode) {// 如果是 Cluster 模式Utils.classForName("org.apache.spark.deploy.yarn.ApplicationMaster").getName} else {// 如果是 Client 模式Utils.classForName("org.apache.spark.deploy.yarn.ExecutorLauncher").getName}amContainer}
SparkSubmit 进程启动完毕
4org.apache.spark.deploy.yarn.ApplicationMaster
ApplicationMaster伴生对象的 main方法
def main(args: Array[String]): Unit = {SignalUtils.registerLogger(log)// 构建 ApplicationMasterArguments 对象, 对传来的参数做封装val amArgs: ApplicationMasterArguments = new ApplicationMasterArguments(args)// Load the properties file with the Spark configuration and set entries as system properties,// so that user code run inside the AM also has access to them.// Note: we must do this before SparkHadoopUtil instantiatedif (amArgs.propertiesFile != null) {Utils.getPropertiesFromFile(amArgs.propertiesFile).foreach { case (k, v) =>sys.props(k) = v}}SparkHadoopUtil.get.runAsSparkUser { () =>// 构建 ApplicationMaster 实例, ApplicationMaster 需要与 RM 通讯master = new ApplicationMaster(amArgs, new YarnRMClient)// 运行 ApplicationMaster 的 run 方法, run 方法结束之后, 结束 ApplicationMaster 进程System.exit(master.run())}}
ApplicationMaster 伴生类的 run方法
final def run(): Int = {// 关键核心代码try {val fs = FileSystem.get(yarnConf)if (isClusterMode) {//如果是集群模式runDriver(securityMgr)} else {//client模式runExecutorLauncher(securityMgr)}} catch {}exitCode}
runDriver（）
private def runDriver(securityMgr: SecurityManager): Unit = {addAmIpFilter()// 开始执行用户类. 启动一个子线程来执行用户类的 main 方法. 返回值就是运行用户类的子线程.// 线程名就叫 "Driver"userClassThread = startUserApplication()// This a bit hacky, but we need to wait until the spark.driver.port property has// been set by the Thread executing the user class.logInfo("Waiting for spark context initialization...")val totalWaitTime = sparkConf.get(AM_MAX_WAIT_TIME)try {val sc = ThreadUtils.awaitResult(sparkContextPromise.future,Duration(totalWaitTime, TimeUnit.MILLISECONDS))if (sc != null) {rpcEnv = sc.env.rpcEnvval driverRef = runAMEndpoint(sc.getConf.get("spark.driver.host"),sc.getConf.get("spark.driver.port"),isClusterMode = true)// 注册 ApplicationMaster , 其实就是请求资源registerAM(sc.getConf, rpcEnv, driverRef, sc.ui.map(_.appUIAddress).getOrElse(""),securityMgr)} else {// Sanity check; should never happen in normal operation, since sc should only be null// if the user app did not create a SparkContext.if (!finished) {throw new IllegalStateException("SparkContext is null but app is still running!")}}// 线程 join: 把 userClassThread 线程执行完毕之后再继续执行当前线程.userClassThread.join() // yield 礼让} catch {}}
startUserApplication 方法
private def startUserApplication(): Thread = {// 得到用户类的 main 方法val mainMethod = userClassLoader.loadClass(args.userClass).getMethod("main", classOf[Array[String]])// 创建及线程val userThread = new Thread {override def run() {try {// 调用用户类的主函数mainMethod.invoke(null, userArgs.toArray)} catch {...} finally {...}}}userThread.setContextClassLoader(userClassLoader)userThread.setName("Driver")userThread.start()userThread}
registerAM 方法
private def registerAM(_sparkConf: SparkConf,_rpcEnv: RpcEnv,driverRef: RpcEndpointRef,uiAddress: String,securityMgr: SecurityManager) = {...// 向 RM 注册, 得到 YarnAllocatorallocator = client.register(driverUrl,driverRef,yarnConf,_sparkConf,uiAddress,historyAddress,securityMgr,localResources)// 请求分配资源allocator.allocateResources()}
allocator.allocateResources() 方法
/**请求资源，如果 Yarn 满足了我们的所有要求，我们就会得到一些容器(数量: maxExecutors)。通过在这些容器中启动 Executor 来处理 YARN 授予我们的任何容器。必须同步，因为在此方法中读取的变量会被其他方法更改。*/def allocateResources(): Unit = synchronized {...if (allocatedContainers.size > 0) {...handleAllocatedContainers(allocatedContainers.asScala)}...}
handleAllocatedContainers方法
/**处理 RM 授权给我们的容器*/def handleAllocatedContainers(allocatedContainers: Seq[Container]): Unit = {val containersToUse = new ArrayBuffer[Container](allocatedContainers.size)runAllocatedContainers(containersToUse)}
runAllocatedContainers 方法
/*** Launches executors in the allocated containers.在已经分配的容器中启动 Executors*/private def runAllocatedContainers(containersToUse: ArrayBuffer[Container]): Unit = {// 每个容器上启动一个 Executorfor (container <- containersToUse) {if (numExecutorsRunning < targetNumExecutors) {if (launchContainers) {launcherPool.execute(new Runnable {override def run(): Unit = {try {new ExecutorRunnable(Some(container),conf,sparkConf,driverUrl,executorId,executorHostname,executorMemory,executorCores,appAttemptId.getApplicationId.toString,securityMgr,localResources).run() // 启动 executorupdateInternalState()} catch {...}}})} else {...}} else {...}}}
ExecutorRunnable.run方法
def run(): Unit = {logDebug("Starting Executor Container")// 创建 NodeManager 客户端nmClient = NMClient.createNMClient()// 初始化 NodeManager 客户端nmClient.init(conf)// 启动 NodeManager 客户端nmClient.start()// 启动容器startContainer()}
ExecutorRunnable.startContainer()
def startContainer(): java.util.Map[String, ByteBuffer] = {val ctx = Records.newRecord(classOf[ContainerLaunchContext]).asInstanceOf[ContainerLaunchContext]// 准备要执行的命令val commands = prepareCommand()ctx.setCommands(commands.asJava)// Send the start request to the ContainerManagertry {// 启动容器nmClient.startContainer(container.get, ctx)} catch {...}}
ExecutorRunnable.prepareCommand 方法
private def prepareCommand(): List[String] = {val commands = prefixEnv ++ Seq(YarnSparkHadoopUtil.expandEnvironment(Environment.JAVA_HOME) + "/bin/java","-server") ++javaOpts ++// 要执行的类Seq("org.apache.spark.executor.CoarseGrainedExecutorBackend","--driver-url", masterAddress,"--executor-id", executorId,"--hostname", hostname,"--cores", executorCores.toString,"--app-id", appId) ++userClassPath ++Seq(s"1>${ApplicationConstants.LOG_DIR_EXPANSION_VAR}/stdout",s"2>${ApplicationConstants.LOG_DIR_EXPANSION_VAR}/stderr")commands.map(s => if (s == null) "null" else s).toList}
ApplicationMaster 进程启动完毕
5org.apache.spark.executor.CoarseGrainedExecutorBackend
CoarseGrainedExecutorBackend 伴生类
main方法
def main(args: Array[String]) {// 启动 CoarseGrainedExecutorBackendrun(driverUrl, executorId, hostname, cores, appId, workerUrl, userClassPath)// 运行结束之后退出进程System.exit(0)}
run 方法
/**准备 RpcEnv*/private def run(driverUrl: String,executorId: String,hostname: String,cores: Int,appId: String,workerUrl: Option[String],userClassPath: Seq[URL]) {SparkHadoopUtil.get.runAsSparkUser { () =>val env = SparkEnv.createExecutorEnv(driverConf, executorId, hostname, port, cores, cfg.ioEncryptionKey, isLocal = false)env.rpcEnv.setupEndpoint("Executor", new CoarseGrainedExecutorBackend(env.rpcEnv, driverUrl, executorId, hostname, cores, userClassPath, env))}}
CoarseGrainedExecutorBackend 伴生类
继承自: ThreadSafeRpcEndpoint 是一个RpcEndpoint
查看声明周期：constructor -> onStart -> receive* -> onStop
/*** An end point for the RPC that defines what functions to trigger given a message.** It is guaranteed that <code data-enlighter-language="generic" class="EnlighterJSRAW">onStart</code>, <code data-enlighter-language="generic" class="EnlighterJSRAW">receive</code> and <code data-enlighter-language="generic" class="EnlighterJSRAW">onStop</code> will be called in sequence.** The life-cycle of an endpoint is:** constructor -> onStart -> receive* -> onStop** Note: <code data-enlighter-language="generic" class="EnlighterJSRAW">receive</code> can be called concurrently. If you want <code data-enlighter-language="generic" class="EnlighterJSRAW">receive</code> to be thread-safe, please use* [[ThreadSafeRpcEndpoint]]** If any error is thrown from one of [[RpcEndpoint]] methods except <code data-enlighter-language="generic" class="EnlighterJSRAW">onError</code>, <code data-enlighter-language="generic" class="EnlighterJSRAW">onError</code> will be* invoked with the cause. If <code data-enlighter-language="generic" class="EnlighterJSRAW">onError</code> throws an error, [[RpcEnv]] will ignore it.*/private[spark] trait RpcEndpoint {...}
onStart 方法连接到 Driver, 并向 Driver注册Executor
override def onStart() {rpcEnv.asyncSetupEndpointRefByURI(driverUrl).flatMap { ref =>// This is a very fast action so we can use "ThreadUtils.sameThread"driver = Some(ref)// 向驱动注册 Executor 关键方法ref.ask[Boolean](RegisterExecutor(executorId, self, hostname, cores, extractLogUrls))}(ThreadUtils.sameThread).onComplete {case Success(msg) =>case Failure(e) =>// 注册失败, 退出 executorexitExecutor(1, s"Cannot register with driver: $driverUrl", e, notifyDriver = false)}(ThreadUtils.sameThread)}
Driver端的CoarseGrainedSchedulerBackend.DriverEndPoint 的 receiveAndReply 方法
override def receiveAndReply(context: RpcCallContext): PartialFunction[Any, Unit] = {// 接收注册 Executorcase RegisterExecutor(executorId, executorRef, hostname, cores, logUrls) =>if (executorDataMap.contains(executorId)) { // 已经注册过了...} else {// 给 Executor 发送注册成功的信息executorRef.send(RegisteredExecutor)...}}
Eexcutor端的CoarseGrainedExecutorBackend的receive方法
override def receive: PartialFunction[Any, Unit] = {// 向 Driver 注册成功case RegisteredExecutor =>logInfo("Successfully registered with driver")try {...// 创建 Executor 对象 注意: Executor 其实是一个对象executor = new Executor(executorId, hostname, env, userClassPath, isLocal = false)} catch {...}}
6总结
