Spark-stream基礎---sparkStreaming和Kafka整合wordCount單詞計數

專案

sprak-stream與kafak整合wordCount
在IDEA上接收kafka傳來的資料，並進行單詞統計

linux端開啟kafka

//1.先開啟zookeeper（3臺）
zkServer.sh start 
//2.在開啟kafka(3臺)
 bin/kafka-server-start.sh config/server.properties &
//3.建立生產者
bin/kafka-console-producer.sh --broker-list hou-01:9092 --topic wc
//4.控制檯輸入任意單詞

IDEA新增依賴

    <dependency>
        <groupId>org.apache.spark</groupId>
        <artifactId>spark-streaming-kafka-0-8_2.11</artifactId>
        <version>${spark.version}</version>
    </dependency>

1.0版本單詞計數

package day08
import org.apache.spark.SparkConf
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Milliseconds, StreamingContext}

/*
需求：kafka消費資料到sparkStreaming計算
 */
object KafkaWordCount {
  def main(args: Array[String]): Unit = {
    //1.建立StreamingContext
    val conf: SparkConf = new SparkConf().setAppName("kafkaWordCount").setMaster("local[2]")
    val ssc: StreamingContext = new StreamingContext(conf,Milliseconds(2000))


    //2.接入kafka資料來源(如何訪問kafka叢集？zookeeper)
    val zkQuorm: String = "192.168.64.111,192.168.64.112,192.168.64.113"
    //訪問組
    val groupID = "g1"
    //訪問主題
    val topic: Map[String, Int] = Map[String,Int]("wc"->1)
    //建立Dstream
    val kafkaStream: ReceiverInputDStream[(String, String)] = KafkaUtils
      .createStream(ssc,zkQuorm,groupID,topic)

    //3.處理資料
    val data: DStream[String] = kafkaStream.map(_._2)

    //4.啟動streaming程式
    val r: DStream[(String, Int)] = data.flatMap(_.split(" ")).map((_,1)).reduceByKey(_+_)
    r.print()
    ssc.start()

    //5.關閉資源
    ssc.awaitTermination()

  }
}

結果

2.0版本單詞計數

將歷史記錄儲存下來，顯示出來，主要使用dataFunc

package day08

import org.apache.spark.{HashPartitioner, SparkConf}
import org.apache.spark.streaming.dstream.{DStream, ReceiverInputDStream}
import org.apache.spark.streaming.kafka.KafkaUtils
import org.apache.spark.streaming.{Milliseconds, StreamingContext}

object StatusKafkaWordCount {
  //保持歷史狀態 wc 單詞，次數 聚合的key
  //第一個型別：單詞，第二個型別：在每一個分割槽中出現的次數累加的結果
  //第三個型別：是以前的結果
  val updateFunc = (iter:Iterator[(String,Seq[Int],Option[Int])]) => {
    //總的次數= 當前出現的次數 + 以前返回的結果
    iter.map(t => (t._1, t._2.sum + t._3.getOrElse(0)))
  }

  def main(args: Array[String]): Unit = {

    //1.建立程式入口
    val conf: SparkConf = new SparkConf().setAppName("StateKafkaWC").setMaster("local[2]")
    val ssc: StreamingContext = new StreamingContext(conf,Milliseconds(2000))

    //2.需要累加歷史資料 checkpoints
    ssc.checkpoint("hdfs://192.168.64.111:9000/ck")

    //3.接入kafka資料來源
    val zkQuorm: String = "192.168.64.111,192.168.64.112,192.168.64.113"
    //訪問組
    val groupID = "g1"
    //訪問主題
    val topic: Map[String, Int] = Map[String,Int]("wc"->1)
    //建立Dstream
    val kafkaStream: ReceiverInputDStream[(String, String)] = KafkaUtils
      .createStream(ssc,zkQuorm,groupID,topic)

    //4.處理資料
    val data: DStream[String] = kafkaStream.map(_._2)

    //5.加入歷史資料計算

    val r: DStream[(String, Int)] = data.flatMap(_.split(" ")).map((_, 1))
      //引數1：自定義業務函式 引數2：分割槽器設定 引數3：是否使用
      .updateStateByKey(updateFunc, new HashPartitioner(ssc.sparkContext.defaultParallelism), true)

    //6.列印
    r.print()
    //7.啟動程式
    ssc.start()
    //8.關閉資源
    ssc.awaitTermination()

  }
}

結果