mapPartitions

mapPartitions與map轉換操作類似，只不過對映函式的輸入引數由RDD中的每一個元素變成了RDD中每一個分割槽的迭代器
def mapPartitions[U](f: Iterator[Double] => Iterator[U],preservesPartitioning: Boolean)(implicit evidence$6: scala.reflect.ClassTag[U]): org.apache.spark.rdd.RDD[U]

• 傳入的引數是一個迭代器
• perservesPartitioning指明mapPartitions是否保留父RDD的partitions的分割槽資訊

scala版本

將每個分割槽內元素乘2

// 分為三個區
 val rdd1: RDD[Int] = sc.parallelize(List(1,2,3,4,5,6),3)
 // mapPartitions需要傳遞
 val mapPartitionsRDD: RDD[Int] = rdd1.mapPartitions((f: Iterator[Int]) => {
   println(f)
   f.map(_ * 2)
 })
 mapPartitionsRDD.collect().foreach(println)

java版本

JavaRDD<Integer> rdd1 = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6));
JavaRDD<Integer> mapPartitionsRDD = rdd1.mapPartitions(new FlatMapFunction<Iterator<Integer>, Integer>() {
    @Override
    public Iterator<Integer> call(Iterator<Integer> i) throws Exception {
        List<Integer> list = new ArrayList<>();
        while (i.hasNext()) {
            Integer x = i.next();
            list.add(x);
        }
        return list.iterator();
            }
    });
List<Integer> collect = mapPartitionsRDD.collect();
for (Integer i : collect) {
    System.out.println(i);
}

mapPartitionsWithIndex

mapPartitionWithIndex和mapPartitions功能類似，只是輸入引數時多了一個分割槽的ID
def mapPartitionsWithIndex[U](f: (Int, Iterator[Double]) => Iterator[U],preservesPartitioning: Boolean)(implicit evidence$9: scala.reflect.ClassTag[U]): org.apache.spark.rdd.RDD[U]

scala版本

val rdd1: RDD[Int] = sc.parallelize(List(1,2,3,4,5,6),3)
val f =(x:Int,y:Iterator[Int])=>{
  println(x+"號分割槽")
  y.map(_*2)
}
val  mapPartitionsWithIndexRDD: RDD[Int] = rdd1.mapPartitionsWithIndex(f)
mapPartitionsWithIndexRDD.collect.foreach(println)

java版本

JavaRDD<Integer> rdd1 = sc.parallelize(Arrays.asList(1, 2, 3, 4, 5, 6),3);
Function2<Integer, Iterator<Integer>, Iterator<String>> function2 = new Function2<Integer, Iterator<Integer>, Iterator<String>>() {
    @Override
    public Iterator<String> call(Integer v1, Iterator<Integer> v2) throws Exception {
        System.out.println(v1 + "號分割槽");
        List<String> list = new ArrayList<>();
        while (v2.hasNext()) {
            String s = "hello" + v2.next();
            list.add(s);
        }
        return list.iterator();
    }
};
// 第二個引數用於指明是否保留父RDD的partitions的分割槽資訊
JavaRDD<String> mapPartitionsWithIndexRDD = rdd1.mapPartitionsWithIndex(function2, false);
List<String> collect = mapPartitionsWithIndexRDD.collect();
for (String s : collect) {
    System.out.println(s);
}