大資料分享Spark任務和叢集啟動流程

 

大資料分享Spark任務和叢集啟動流程，Spark叢集啟動流程

1.呼叫start-all.sh指令碼,開始啟動Master

2.Master啟動以後,preStart方法呼叫了一個定時器,定時檢查超時的Worker後刪除

3.啟動指令碼會解析slaves配置檔案,找到啟動Worker的相應節點.開始啟動Worker

4.Worker服務啟動後開始呼叫preStart方法開始向所有的Master進行註冊

5.Master接收到Worker傳送過來的註冊資訊,Master開始儲存註冊資訊並把自己的URL響應給Worker

6.Worker接收到Master的URL後並更新,開始呼叫一個定時器,定時的向Master傳送心跳資訊

任務提交流程

1.Driver端會透過spark-submit指令碼啟動SaparkSubmit程式,此時建立了一個非常重要的物件(SparkContext),開始向Master傳送訊息

2.Master接收到傳送過來的資訊後開始生成任務資訊,並把任務資訊放到一個對列裡

3.Master把所有有效的Worker過濾出來,按照空閒的資源進行排序

4.Master開始向有效的Worker通知拿取任務資訊並啟動相應的Executor

5.Worker啟動Executor並向Driver反向註冊

6.Driver開始把生成的task傳送給相應的Executor,Executor開始執行任務

叢集啟動流程

1.首先建立Master類

import akka.actor.{Actor, ActorSystem, Props}

import com.typesafe.config.{Config, ConfigFactory}

import scala.collection.mutable

import scala.concurrent.duration._

class Master(val masterHost: String, val masterPort: Int) extends Actor{

// 用來儲存Worker的註冊資訊

val idToWorker = new mutable.HashMap[String, WorkerInfo]()

// 用來儲存Worker的資訊

val workers = new mutable.HashSet[WorkerInfo]()

// Worker的超時時間間隔

val checkInterval: Long = 15000

// 生命週期方法，在構造器之後，receive方法之前只呼叫一次

override def preStart(): Unit = {

// 啟動一個定時器，用來定時檢查超時的Worker

import context.dispatcher

context.system.scheduler.schedule(0 millis, checkInterval millis, self, CheckTimeOutWorker)

}

// 在preStart方法之後，不斷的重複呼叫

override def receive: Receive = {

// Worker -> Master

case RegisterWorker(id, host, port, memory, cores) => {

if (!idToWorker.contains(id)){

val workerInfo = new WorkerInfo(id, host, port, memory, cores)

idToWorker += (id -> workerInfo)

workers += workerInfo

println("a worker registered")

sender ! RegisteredWorker(s"akka.tcp://${Master.MASTER_SYSTEM}" +

s"@${masterHost}:${masterPort}/user/${Master.MASTER_ACTOR}")

}

}

case HeartBeat(workerId) => {

// 透過傳過來的workerId獲取對應的WorkerInfo

val workerInfo: WorkerInfo = idToWorker(workerId)

// 獲取當前時間

val currentTime = System.currentTimeMillis()

// 更新最後一次心跳時間

workerInfo.lastHeartbeatTime = currentTime

}

case CheckTimeOutWorker => {

val currentTime = System.currentTimeMillis()

val toRemove: mutable.HashSet[WorkerInfo] =

workers.filter(w => currentTime - w.lastHeartbeatTime > checkInterval)

// 將超時的Worker從idToWorker和workers中移除

toRemove.foreach(deadWorker => {

idToWorker -= deadWorker.id

workers -= deadWorker

})

println(s"num of workers: ${workers.size}")

}

}

}

object Master{

val MASTER_SYSTEM = "MasterSystem"

val MASTER_ACTOR = "Master"

def main(args: Array[String]): Unit = {

val host = args(0)

val port = args(1).toInt

val configStr =

s"""

|akka.actor.provider = "akka.remote.RemoteActorRefProvider"

|akka.remote.netty.tcp.hostname = "$host"

|akka.remote.netty.tcp.port = "$port"

""".stripMargin

// 配置建立Actor需要的配置資訊

val config: Config = ConfigFactory.parseString(configStr)

// 建立ActorSystem

val actorSystem: ActorSystem = ActorSystem(MASTER_SYSTEM, config)

// 用actorSystem例項建立Actor

actorSystem.actorOf(Props(new Master(host, port)), MASTER_ACTOR)

actorSystem.awaitTermination()

}

}

2.建立RemoteMsg特質

trait RemoteMsg extends Serializable{

}

// Master -> self(Master)

case object CheckTimeOutWorker

// Worker -> Master

case class RegisterWorker(id: String, host: String,

port: Int, memory: Int, cores: Int) extends RemoteMsg

// Master -> Worker

case class RegisteredWorker(masterUrl: String) extends RemoteMsg

// Worker -> self

case object SendHeartBeat

// Worker -> Master(HeartBeat)

case class HeartBeat(workerId: String) extends RemoteMsg

3.建立Worker類

import java.util.UUID

import akka.actor.{Actor, ActorRef, ActorSelection, ActorSystem, Props}

import com.typesafe.config.{Config, ConfigFactory}

import scala.concurrent.duration._

class Worker(val host: String, val port: Int, val masterHost: String,

val masterPort: Int, val memory: Int, val cores: Int) extends Actor{

// 生成一個Worker ID

val workerId = UUID.randomUUID().toString

// 用來儲存MasterURL

var masterUrl: String = _

// 心跳時間間隔

val heartBeat_interval: Long = 10000

// master的Actor

var master: ActorSelection = _

override def preStart(){

// 獲取Master的Actor

master = context.actorSelection(s"akka.tcp://${Master.MASTER_SYSTEM}" +

s"@${masterHost}:${masterPort}/user/${Master.MASTER_ACTOR}")

master ! RegisterWorker(workerId, host, port, memory, cores)

}

override def receive: Receive = {

// Worker接收到Master傳送過來的註冊成功的資訊（masterUrl）

case RegisteredWorker(masterUrl) => {

this.masterUrl = masterUrl

// 啟動一個定時器，定時給Master傳送心跳

import context.dispatcher

context.system.scheduler.schedule(0 millis, heartBeat_interval millis, self, SendHeartBeat)

}

case SendHeartBeat => {

// 向Master傳送心跳

master ! HeartBeat(workerId)

}

}

}

object Worker{

val WORKER_SYSTEM = "WorkerSystem"

val WORKER_ACTOR = "Worker"

def main(args: Array[String]): Unit = {

val host = args(0)

val port = args(1).toInt

val masterHost = args(2)

val masterPort = args(3).toInt

val memory = args(4).toInt

val cores = args(5).toInt

val configStr =

s"""

|akka.actor.provider = "akka.remote.RemoteActorRefProvider"

|akka.remote.netty.tcp.hostname = "$host"

|akka.remote.netty.tcp.port = "$port"

""".stripMargin

// 配置建立Actor需要的配置資訊

val config: Config = ConfigFactory.parseString(configStr)

// 建立ActorSystem

val actorSystem: ActorSystem = ActorSystem(WORKER_SYSTEM, config)

// 用actorSystem例項建立Actor

val worker: ActorRef = actorSystem.actorOf(

Props(new Worker(host, port, masterHost, masterPort, memory, cores)), WORKER_ACTOR)

actorSystem.awaitTermination()

}

}

4.建立初始化類

class WorkerInfo(val id: String, val host: String, val port: Int,

val memory: Int, val cores: Int) {

// 初始化最後一次心跳的時間

var lastHeartbeatTime: Long = _

}

5.本地測試需要傳入引數: