Saprk distinct

Saprk core Transformation 轉換運算元

RDD整體上分為Value型別、雙Value型別和Key-Value型別

Value型別

distinct

package com.xcu.bigdata.spark.core.pg02_rdd.pg022_rdd_transform

import org.apache.spark.rdd.RDD
import org.apache.spark.{SparkConf, SparkContext}

/**
 * @Package : com.xcu.bigdata.spark.core.pg02_rdd.pg022_rdd_transform
 * @Author :
 * @Date : 2020 11月 星期五
 * @Desc : 全域性去重
 */
object Spark08_Distinct {
  def main(args: Array[String]): Unit = {
    //建立配置檔案
    val conf: SparkConf = new SparkConf().setAppName("").setMaster("local[*]")
    //建立SparkContext,該物件是提交的入口
    val sc = new SparkContext(conf)
    //建立RDD
    val rdd: RDD[Int] = sc.makeRDD(List(1, 2, 3, 4, 5, 5, 4, 3, 3), 3)
    //用mapPartitionsWithIndex的目的是為了證明全域性去重，和shuffle
    rdd.mapPartitionsWithIndex {
      (index, datas) => {
        println(index + ":" + datas.mkString(","))
        datas
      }
    }.collect()
    println("***********************************")
    val newRDD: RDD[Int] = rdd.distinct()
    newRDD.mapPartitionsWithIndex {
      (index, datas) => {
        println(index + ":" + datas.mkString(","))
        datas
      }
    }.collect()
    //釋放資源
    sc.stop()
  }
}