Spark example

21ca發表於2017-03-07
Spark
pom.xml
  1. <project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
  2.     xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
  3.     <modelVersion>4.0.0</modelVersion>
  4.     <groupId>active </groupId>
  5.     <artifactId>spark-test</artifactId>
  6.     <version>0.0.1-SNAPSHOT</version>

  8.     <dependencies>
  9.         <dependency>
  10.             <groupId>org.apache.spark</groupId>
  11.             <artifactId>spark-core_2.10</artifactId>
  12.             <version>2.1.0</version>
  13.         </dependency>
  14.     </dependencies>
  15. </project>
SparkTest.java
  1. import java.util.Arrays;

  3. import org.apache.spark.SparkConf;
  4. import org.apache.spark.api.java.JavaPairRDD;
  5. import org.apache.spark.api.java.JavaRDD;
  6. import org.apache.spark.api.java.JavaSparkContext;

  8. import scala.Tuple2;

  10. public class SparkTest {

  12.     public static void main(String[] args) {
  13.         SparkConf conf = new SparkConf().setAppName("Test").setMaster("local");
  14.         JavaSparkContext sc = new JavaSparkContext(conf);
  15.         JavaRDD<String> file = sc.parallelize(Arrays.asList("Hello test", "Hello test2", "dds"));
  16.         JavaRDD<String> words = file.flatMap(s -> Arrays.asList(s.split(" |\t|\n|\r")).iterator());
  17.         JavaPairRDD<String, Integer> counts = words.mapToPair(s -> new Tuple2<String, Integer>(s, 1));        
  18.         counts = counts.reduceByKey((x, y) -> x + y);
  19.         
  20.         System.out.println(counts.collect());
  21.         sc.close();
  22.     }

  24. }
可以單獨執行，也可以提交到spark叢集： spark-submit.cmd --class SparkTest D:\workspace\spark-test\target\spark-test-0.0.1-SNAPSHOT.jar

來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/10742815/viewspace-2134860/，如需轉載，請註明出處，否則將追究法律責任。

相關文章