spark direct kafka 將offset推到zookeeper

破棉襖發表於2016-09-30
SparkKafka


1、spark streaming direct方式讀取kafka效能要好很多，缺點是它不會去zookeeper更新offset，這將導致
    基於zk的KafkaOffsetMonitor監控失效。由於我們流處理的資料量大並且健壯性要求高，我們需要
     透過KafkaOffsetMonitor來實時監控資料消費情況，故自己實現offset推送到zk，如下：


2、spark程式碼：

  1. object KSConvertStreaming{

  3.   val savaLocal = "/xxx/parquet/%s/year=%s/month=%s/day=%s"

  5.   def jsonConvert(jsonStrs: Iterator[String]): Iterator[(String, ArrayBuffer[String])] = {
  6.     StreamingUtils.init
  7.     val typeMap = scala.collection.mutable.Map[String, ArrayBuffer[String]]()
  8.     jsonStrs.foreach(
  9.       x => {
  10.         val res = StreamingUtils.mapToStr(x)
  11.         if (null != res) {
  12.           val msgType = res.get(0)
  13.           if (!typeMap.contains(msgType)) {
  14.             typeMap += (msgType -> new ArrayBuffer[String]())
  15.           }
  16.           typeMap(msgType) += res.get(1)
  17.         }
  18.       }
  19.     )
  20.     typeMap.iterator
  21.   }

  23.   def main(args: Array[String]): Unit = {

  25.     val Array(maxPartition, maxNumber,windownsS,groupName,maxRatePerPartition) = args
  26.     val topicName = "xxx"
  27.     val kafkaAddr = "xxx:9092,xxx:9092,xxx:9092"
  28.     val kafkaParams = Map[String, String](
  29.       "metadata.broker.list" -> kafkaAddr,
  30.       "group.id" -> groupName,
  31.       "auto.offset.reset" -> "largest"
  32.     )
  33.     val topics = Set(topicName)

  35.     println(s"maxPartition -------- $maxPartition")
  36.     println(s"maxNumber -------- $maxNumber")
  37.     println(s"windownsS -------- $windownsS")
  38.     println(s"groupName -------- $groupName")

  40.     val sparkConf = new SparkConf().setAppName("Streaming_Convert")
  41.       .set("spark.yarn.executor.memoryOverhead","1024")
  42.       .set("spark.streaming.kafka.maxRatePerPartition",maxRatePerPartition)  //此處為每秒每個partition的條數
  43.       .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer")
  44.       .set("spark.reducer.maxSizeInFlight", "1m")
  45.     val sc = new SparkContext(sparkConf)
  46.     val sqlContext = new org.apache.spark.sql.SQLContext(sc)
  47.     val ssc = new StreamingContext(sc, Seconds(windownsS.toInt)) //秒單位

  49.     val topicDirs = new ZKGroupTopicDirs(groupName,topicName)

  51.     val zkClient = new ZkClient("xxx:2181,xxx:2181,xxx:2181",Integer.MAX_VALUE,100000,ZKStringSerializer)
  52.     val children = zkClient.countChildren(topicDirs.consumerOffsetDir)

  54.     var kafkaStream: InputDStream[(String, String)] = null
  55.     var fromOffsets: Map[TopicAndPartition, Long] = Map()
  56.     val messageHandler = (mmd: MessageAndMetadata[String, String]) => (mmd.topic, mmd.message())
  57.     if (children > 0) {
  58.       for (i <- 0 until children) {
  59.         val partitionOffset = zkClient.readData[String](s"${topicDirs.consumerOffsetDir}/$i")
  60.         val tp = TopicAndPartition(topicName, i)
  61.         fromOffsets += (tp -> partitionOffset.toLong)
  62.       }
  63.       kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder, (String, String)](ssc, kafkaParams, fromOffsets, messageHandler)
  64.     }
  65.     else {
  66.       kafkaStream = KafkaUtils.createDirectStream[String, String, StringDecoder, StringDecoder](ssc, kafkaParams, topics)
  67.     }

  69.     var offsetRanges = Array[OffsetRange]()

  71.     kafkaStream.transform { rdd =>
  72.       offsetRanges = rdd.asInstanceOf[HasOffsetRanges].offsetRanges
  73.       rdd
  74.     }.map(_._2).foreachRDD {
  75.       rdd =>
  76.         val xRDD = rdd.flatMap(_.split("\n")).mapPartitions(x => jsonConvert(x))
  77.         xRDD.persist(StorageLevel.MEMORY_ONLY)
  78.         val typeCountMap = xRDD.map(x => (x._1, x._2.size)).collect()
  79.         val typeMap = scala.collection.mutable.Map[String, Long]()
  80.         typeCountMap.foreach(x => {
  81.           if (!typeMap.contains(x._1)) {
  82.             typeMap(x._1) = x._2
  83.           } else {
  84.             val tmpCount = typeMap(x._1)
  85.             typeMap(x._1) = x._2 + tmpCount
  86.           }
  87.         })
  88.         var totalCount: Long = 0
  89.         typeMap.foreach(x => {
  90.           println(s"${x._1}:${x._2}")
  91.           totalCount += x._2
  92.         })
  93.         println(s"total : $totalCount" )
  94.         val sortedMap = collection.mutable.LinkedHashMap(typeMap.toSeq.sortWith(_._2 > _._2): _*)

  96.         sortedMap.foreach {
  97.           x => {
  98.             val pointType = x._1
  99.             val count = x._2
  100.             println(s"save type( $pointType ) count( $count )")
  101.             val jsonRDD = xRDD.filter(x => x._1 == pointType).
  102.               flatMap(x => x._2)
  103.             var partitionNum = count / maxNumber.toLong
  104.             if (partitionNum == 0) {
  105.               partitionNum = 1
  106.             }
  107.             if (partitionNum > maxPartition.toLong) {
  108.               partitionNum = maxPartition.toLong
  109.             }

  111.             println(s"\trepartition ( $partitionNum )")
  112.             val arrType = pointType.split('-')
  113.             sqlContext.read.json(jsonRDD).
  114.               repartition(partitionNum.toInt).
  115.               write.mode(SaveMode.Append).
  116.               parquet(savaLocal.format(arrType(0),arrType(1),arrType(2),arrType(3)))

  118.           }
  119.         }
  120.         xRDD.unpersist()

  122.         for (o <- offsetRanges) {
  123.           val zkPath = s"${topicDirs.consumerOffsetDir}/${o.partition}"
  124.           ZkUtils.updatePersistentPath(zkClient, zkPath, o.untilOffset.toString)
  125.         }
  126.     }

  128.     ssc.start()
  129.     ssc.awaitTermination()
  130.   }


  133. }


來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/29754888/viewspace-2125804/，如需轉載，請註明出處，否則將追究法律責任。

相關文章