Spark連線MongoDB之Scala

MongoDB Connector for Spark
　　Spark Connector Scala Guide

 

spark-shell --jars "mongo-spark-connector_2.11-2.0.0.jar,mongo-hadoop-core-2.0.2.jar,mongo-java-driver-3.4.2.jar"

 

import org.apache.spark.sql.SparkSession
import com.mongodb.spark._
import com.mongodb.spark.config._
import org.bson.Document

    val spark = SparkSession.builder()
      .master("local")
      .appName("MongoSparkConnector")
      .config("spark.some.config.option", "some-value")
      .getOrCreate()

val uri = "mongodb://172.1.1.1:27017"

    val userDF = spark.sql("""
select
       uid,
       name,
       current_date() version
  from test_table
  limit 100
      """).repartition(8)

// Write to MongoDB
    userDF.write.mode("overwrite").format("com.mongodb.spark.sql").options(
      Map(
        "uri" -> uri,
        "database" -> "test",
        "collection" -> "test_table")).save()

// Read From MongoDB
    val df = spark.read.format("com.mongodb.spark.sql").options(
      Map(
        "uri" -> uri,
        "database" -> "test",
        "collection" -> "test_table")).load()

// 其他方式
    userDF.write.mode("overwrite").format("com.mongodb.spark.sql").options(
      Map(
        "spark.mongodb.input.uri" -> uri,
        "spark.mongodb.output.uri" -> uri,
        "spark.mongodb.output.database" -> "test",
        "spark.mongodb.output.collection" -> "test_table")).save()

    MongoSpark.save(
      userDF.write.mode("overwrite").options(
        Map(
          "spark.mongodb.input.uri" -> uri,
          "spark.mongodb.output.uri" -> uri,
          "spark.mongodb.output.database" -> "test",
          "spark.mongodb.output.collection" -> "test_table")))

    MongoSpark.save(
      userDF.write.mode("overwrite").options(
        Map(
          "uri" -> uri,
          "database" -> "test",
          "collection" -> "test_table")))

    spark.stop()