Flink-電商使用者行為分析(實時對賬)

連結：https://pan.baidu.com/s/1_DJmEPtNxsCiDnw8KNwmoA 
提取碼：exq9

對於訂單支付事件，使用者支付完成其實並不算完，我們還得確認平臺賬戶上是否到賬了。而往往這會來自不同的日誌資訊，所以我們要同時讀入兩條流的資料來做合併處理。這裡我們利用connect將兩條流進行連線，然後用collect進行處理或者使用join。
接下來我將使用兩種方法(1.collect，2.使用join操作)
collect程式碼實現

import org.apache.flink.api.common.state.{ValueState, ValueStateDescriptor}
import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.co.CoProcessFunction
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.util.Collector


object OrderPayTxMatch {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)

    //從檔案中讀取資料，並轉換成樣例類
    val resource1 = getClass.getResource("/OrderLog.csv")
    val orderEventStream = env.readTextFile(resource1.getPath)
      .map(data=>{
        val dataArray = data.split(",")
        OrderEvent(dataArray(0).toLong,dataArray(1),dataArray(2),dataArray(3).toLong)
      })
      .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor[OrderEvent](Time.seconds(3)) {
        override def extractTimestamp(element: OrderEvent): Long = element.eventTime * 1000L
      })
      .filter(_.txId != "")
      .keyBy(_.txId)

    val resource2 = getClass.getResource("/ReceiptLog.csv")
    val receiptStream = env.readTextFile(resource2.getPath)
      .map(data=>{
        val dataArray = data.split(",")
        ReceiptEvent(dataArray(0),dataArray(1),dataArray(2).toLong)
      })
      .assignAscendingTimestamps(_.timestamp * 1000L)
      .keyBy(_.txId)

    //connect連線兩條流，匹配事件進行處理
    val resultStream = orderEventStream.connect(receiptStream)
      .process(new OrderPayTxDetect())

    //定義側輸出流
    val unmatchedPays = new OutputTag[OrderEvent]("unmatched-pays")
    val unmatchedReceipts = new OutputTag[ReceiptEvent]("unmatched-receipts")

    resultStream.print()
    resultStream.getSideOutput(unmatchedPays).print("unmatched-pays")
    resultStream.getSideOutput(unmatchedReceipts).print("unmatched-receipts")
    env.execute("order pay tx match job")
  }

}
//定義CoProcessFunction，實現兩條流資料的匹配檢測
class OrderPayTxDetect() extends CoProcessFunction[OrderEvent,ReceiptEvent,(OrderEvent,ReceiptEvent)]{
  //定義兩個ValueState，儲存當前交易對應的支付事件和到賬事件
  lazy val payState:ValueState[OrderEvent] = getRuntimeContext.getState(new ValueStateDescriptor[OrderEvent]("pay",
    classOf[OrderEvent]))
  lazy val receiptState:ValueState[ReceiptEvent] = getRuntimeContext.getState(new ValueStateDescriptor[ReceiptEvent]
  ("receipt",classOf[ReceiptEvent]))
  val unmatchedPays = new OutputTag[OrderEvent]("unmatched-pays")
  val unmatchedReceipts = new OutputTag[ReceiptEvent]("unmatched-receipts")
  override def processElement1(pay: OrderEvent, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent,
    ReceiptEvent)]
    #Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
    //pay來了，考察有沒有對應的receipt來過
    val receipt = receiptState.value()
    if (receipt != null){
      //如果已經又receipt，正常輸出到主流
      out.collect((pay,receipt))
    }else{
      //如果receipt還沒來，那麼把pay存入莊濤，註冊一個定時器等待5秒
      payState.update(pay)
      ctx.timerService().registerEventTimeTimer(pay.eventTime * 1000L + 5000L)
    }
  }

  override def processElement2(receipt: ReceiptEvent, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent,
    ReceiptEvent)]
    #Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
    //receipt來了，考察有沒有對應的pay來過
    val pay = payState.value()
    if (pay != null){
      //如果已經有pay，那麼正常匹配，輸出到主流
      out.collect((pay,receipt))
    }else{
      //如果pay還沒來，那麼把receipt存入狀態，註冊一個定時器等待3秒
      receiptState.update(receipt)
      ctx.timerService().registerEventTimeTimer(receipt.timestamp * 1000L + 3000L)
    }
  }

  override def onTimer(timestamp: Long, ctx: CoProcessFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]
    #OnTimerContext, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
    //如果pay不為空，說明receipt沒來，輸出unmatchedPays
    if(payState.value() != null)
      ctx.output(unmatchedPays,payState.value())
    if (receiptState.value() != null)
      ctx.output(unmatchedReceipts,receiptState.value())
    //情況狀態
    payState.clear()
    receiptState.clear()
  }
}

join程式碼實現

import org.apache.flink.streaming.api.TimeCharacteristic
import org.apache.flink.streaming.api.functions.ProcessFunction
import org.apache.flink.streaming.api.functions.co.ProcessJoinFunction
import org.apache.flink.streaming.api.functions.timestamps.BoundedOutOfOrdernessTimestampExtractor
import org.apache.flink.streaming.api.scala._
import org.apache.flink.streaming.api.windowing.time.Time
import org.apache.flink.util.Collector

case class OrderEvent(orderId:Long,eventType:String,txId:String,eventTime:Long)
case class ReceiptEvent(txId:String,payChannel:String,timestamp:Long)

object OrderPayTxMatchWithJoin {
  def main(args: Array[String]): Unit = {
    val env = StreamExecutionEnvironment.getExecutionEnvironment
    env.setParallelism(1)
    env.setStreamTimeCharacteristic(TimeCharacteristic.EventTime)

    //從檔案中讀取資料，並轉換成樣例類
    val resource1 = getClass.getResource("/OrderLog.csv")
    val orderEventStream = env.readTextFile(resource1.getPath)
      .map(data=>{
        val dataArray = data.split(",")
        OrderEvent(dataArray(0).toLong,dataArray(1),dataArray(2),dataArray(3).toLong)
      })
      .assignTimestampsAndWatermarks(new BoundedOutOfOrdernessTimestampExtractor[OrderEvent](Time.seconds(3)) {
        override def extractTimestamp(element: OrderEvent): Long = element.eventTime * 1000L
      })
      .filter(_.txId != "")
      .keyBy(_.txId)

    val resource2 = getClass.getResource("/ReceiptLog.csv")
    val receiptEventStream = env.readTextFile(resource2.getPath)
      .map(data=>{
        val dataArray = data.split(",")
        ReceiptEvent(dataArray(0),dataArray(1),dataArray(2).toLong)
      })
      .assignAscendingTimestamps(_.timestamp * 1000L)
      .keyBy(_.txId)

    //使用join連線兩條流
    val resultStream = orderEventStream
      .intervalJoin(receiptEventStream)
      .between(Time.seconds(-3), Time.seconds(5))
      .process(new OrderPayTxDetectWithJoin())

    resultStream.print()
    env.execute("order pay tx match with join job")
  }

}
//自定義ProcessJoinFunction
class OrderPayTxDetectWithJoin() extends ProcessJoinFunction[OrderEvent,ReceiptEvent,(OrderEvent,ReceiptEvent)]{
  override def processElement(left: OrderEvent, right: ReceiptEvent, ctx: ProcessJoinFunction[OrderEvent, ReceiptEvent, (OrderEvent, ReceiptEvent)]
    #Context, out: Collector[(OrderEvent, ReceiptEvent)]): Unit = {
    out.collect((left,right))
  }
}

總結

雖然join很方便，簡單，但是有侷限性，只能匹配對應上的，不能輸出沒有匹配上的！