Spark SQL 最簡單例子

21ca發表於2017-03-30
SparkSQL單例
pom.xml
<dependencies>
  1.     <dependency>
  2.         <groupId>org.apache.spark</groupId>
  3.         <artifactId>spark-core_2.10</artifactId>
  4.         <version>2.1.0</version>
  5.     </dependency>
  6.     <dependency>
  7.         <groupId>org.apache.spark</groupId>
  8.         <artifactId>spark-sql_2.10</artifactId>
  9.         <version>2.1.0</version>
  10.     </dependency>
  11. </dependencies>
Java:
  1. import java.io.Serializable;
  2. import java.util.Arrays;

  4. import org.apache.spark.SparkConf;
  5. import org.apache.spark.api.java.JavaRDD;
  6. import org.apache.spark.api.java.JavaSparkContext;
  7. import org.apache.spark.sql.Dataset;
  8. import org.apache.spark.sql.Row;
  9. import org.apache.spark.sql.SQLContext;
  10. import org.apache.spark.sql.SparkSession;

  12. public class SparkSqlTest {
  13.     public static class Person implements Serializable {
  14.         private static final long serialVersionUID = -6259413972682177507L;
  15.         private String name;
  16.         private int age;
  17.         
  18.         public Person(String name, int age) {
  19.             this.name = name;
  20.             this.age = age;
  21.         }
  22.         public String toString() {
  23.             return name + ": " + age;
  24.         }
  25.         public String getName() {
  26.             return name;
  27.         }
  28.         public void setName(String name) {
  29.             this.name = name;
  30.         }
  31.         public int getAge() {
  32.             return age;
  33.         }
  34.         public void setAge(int age) {
  35.             this.age = age;
  36.         }
  37.     }
  38.     
  39.     public static void main(String[] args) {
  40.         SparkConf conf = new SparkConf().setAppName("Test").setMaster("local");
  41.         JavaSparkContext sc = new JavaSparkContext(conf);
  42.         
  43.         SparkSession spark = SparkSession.builder().appName("Test").getOrCreate();
  44.         JavaRDD<String> input = sc.parallelize(Arrays.asList("abc,1", "test,2"));
  45.         JavaRDD<Person> persons = input.map(s -> s.split(",")).map(s -> new Person(s[0], Integer.parseInt(s[1])));
  46.         
  47.         //[abc: 1, test: 2]
  48.         System.out.println(persons.collect());
  49.         
  50.         Dataset<Row> df = spark.createDataFrame(persons, Person.class);
  51.         
  52.         /*
  53.         +---+----+
  54.         |age|name|
  55.         +---+----+
  56.         | 1| abc|
  57.         | 2|test|
  58.         +---+----+
  59.          */
  60.         df.show();
  61.         
  62.         /*
  63.          root
  64.           |-- age: integer (nullable = false)
  65.           |-- name: string (nullable = true)
  66.          */
  67.         df.printSchema();

  69.         SQLContext sql = new SQLContext(spark);
  70.         sql.registerDataFrameAsTable(df, "person");
  71.         
  72.         /*
  73.         +---+----+
  74.         |age|name|
  75.         +---+----+
  76.         | 2|test|
  77.         +---+----+
  78.          */
  79.         sql.sql("SELECT * FROM person WHERE age>1").show();
  80.         
  81.         sc.close();
  82.     }

  84. }

來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/10742815/viewspace-2136337/，如需轉載，請註明出處，否則將追究法律責任。

相關文章