好程式設計師大資料分享Spark任務和叢集啟動流程

import akka.actor.{Actor, ActorSystem, Props}

import com.typesafe.config.{Config, ConfigFactory}

import scala.collection.mutable

import scala.concurrent.duration._

class Master(val masterHost: String, val masterPort: Int) extends Actor{

  // 用來儲存Worker的註冊資訊

  val idToWorker = new mutable.HashMap[String, WorkerInfo]()

  // 用來儲存Worker的資訊

  val workers = new mutable.HashSet[WorkerInfo]()

  // Worker的超時時間間隔

  val checkInterval: Long = 15000

  // 生命週期方法，在構造器之後，receive方法之前只呼叫一次

  override def preStart(): Unit = {

    // 啟動一個定時器，用來定時檢查超時的Worker

    import context.dispatcher

    context.system.scheduler.schedule(0 millis, checkInterval millis, self, CheckTimeOutWorker)

  }

  // 在preStart方法之後，不斷的重複呼叫

  override def receive: Receive = {

    // Worker -> Master

    case RegisterWorker(id, host, port, memory, cores) => {

      if (!idToWorker.contains(id)){

        val workerInfo = new WorkerInfo(id, host, port, memory, cores)

        idToWorker += (id -> workerInfo)

        workers += workerInfo

        println("a worker registered")

        sender ! RegisteredWorker(s"akka.tcp://${Master.MASTER_SYSTEM}" +

          s"@${masterHost}:${masterPort}/user/${Master.MASTER_ACTOR}")

      }

    }

    case HeartBeat(workerId) => {

      // 透過傳過來的workerId獲取對應的WorkerInfo

      val workerInfo: WorkerInfo = idToWorker(workerId)

      // 獲取當前時間

      val currentTime = System.currentTimeMillis()

      // 更新最後一次心跳時間

      workerInfo.lastHeartbeatTime = currentTime

    }

    case CheckTimeOutWorker => {

      val currentTime = System.currentTimeMillis()

      val toRemove: mutable.HashSet[WorkerInfo] =

        workers.filter(w => currentTime - w.lastHeartbeatTime > checkInterval)

      // 將超時的Worker從idToWorker和workers中移除

      toRemove.foreach(deadWorker => {

        idToWorker -= deadWorker.id

        workers -= deadWorker

      })

      println(s"num of workers: ${workers.size}")

    }

  }

}

object Master{

  val MASTER_SYSTEM = "MasterSystem"

  val MASTER_ACTOR = "Master"

  def main(args: Array[String]): Unit = {

    val host = args(0)

    val port = args(1).toInt

    val configStr =

      s"""

         |akka.actor.provider = "akka.remote.RemoteActorRefProvider"

         |akka.remote.netty.tcp.hostname = "$host"

         |akka.remote.netty.tcp.port = "$port"

      """.stripMargin

    // 配置建立Actor需要的配置資訊

    val config: Config = ConfigFactory.parseString(configStr)

    // 建立ActorSystem

    val actorSystem: ActorSystem = ActorSystem(MASTER_SYSTEM, config)

    // 用actorSystem例項建立Actor

    actorSystem.actorOf(Props(new Master(host, port)), MASTER_ACTOR)

    actorSystem.awaitTermination()

  }

}

trait RemoteMsg extends Serializable{

}

// Master -> self(Master)

case object CheckTimeOutWorker

// Worker -> Master

case class RegisterWorker(id: String, host: String,

                          port: Int, memory: Int, cores: Int) extends RemoteMsg

// Master -> Worker

case class RegisteredWorker(masterUrl: String) extends RemoteMsg

// Worker -> self

case object SendHeartBeat

// Worker -> Master(HeartBeat)

case class HeartBeat(workerId: String) extends RemoteMsg

import java.util.UUID

import akka.actor.{Actor, ActorRef, ActorSelection, ActorSystem, Props}

import com.typesafe.config.{Config, ConfigFactory}

import scala.concurrent.duration._

class Worker(val host: String, val port: Int, val masterHost: String,

             val masterPort: Int, val memory: Int, val cores: Int) extends Actor{

  // 生成一個Worker ID

  val workerId = UUID.randomUUID().toString

  // 用來儲存MasterURL

  var masterUrl: String = _

  // 心跳時間間隔

  val heartBeat_interval: Long = 10000

  // master的Actor

  var master: ActorSelection = _

  override def preStart(){

    // 獲取Master的Actor

    master = context.actorSelection(s"akka.tcp://${Master.MASTER_SYSTEM}" +

      s"@${masterHost}:${masterPort}/user/${Master.MASTER_ACTOR}")

    master ! RegisterWorker(workerId, host, port, memory, cores)

  }

  override def receive: Receive = {

    // Worker接收到Master傳送過來的註冊成功的資訊（masterUrl）

    case RegisteredWorker(masterUrl) => {

      this.masterUrl = masterUrl

      // 啟動一個定時器，定時給Master傳送心跳

      import context.dispatcher

      context.system.scheduler.schedule(0 millis, heartBeat_interval millis, self, SendHeartBeat)

    }

    case SendHeartBeat => {

      // 向Master傳送心跳

      master ! HeartBeat(workerId)

    }

  }

}

object Worker{

  val WORKER_SYSTEM = "WorkerSystem"

  val WORKER_ACTOR = "Worker"

  def main(args: Array[String]): Unit = {

    val host = args(0)

    val port = args(1).toInt

    val masterHost = args(2)

    val masterPort = args(3).toInt

    val memory = args(4).toInt

    val cores = args(5).toInt

    val configStr =

      s"""

         |akka.actor.provider = "akka.remote.RemoteActorRefProvider"

         |akka.remote.netty.tcp.hostname = "$host"

         |akka.remote.netty.tcp.port = "$port"

      """.stripMargin

    // 配置建立Actor需要的配置資訊

    val config: Config = ConfigFactory.parseString(configStr)

    // 建立ActorSystem

    val actorSystem: ActorSystem = ActorSystem(WORKER_SYSTEM, config)

    // 用actorSystem例項建立Actor

    val worker: ActorRef = actorSystem.actorOf(

      Props(new Worker(host, port, masterHost, masterPort, memory, cores)), WORKER_ACTOR)

    actorSystem.awaitTermination()

  }

}

class WorkerInfo(val id: String, val host: String, val port: Int,

                 val memory: Int, val cores: Int) {

  // 初始化最後一次心跳的時間

  var lastHeartbeatTime: Long = _

}