Spark Stream 簡單例子

21ca發表於2017-03-31
Spark單例
pom.xml
<dependency>
  1.     <groupId>org.apache.spark</groupId>
  2.     <artifactId>spark-streaming_2.11</artifactId>
  3.     <version>2.1.0</version>
  4. </dependency>

  1. import java.io.OutputStream;
  2. import java.net.ServerSocket;
  3. import java.net.Socket;
  4. import java.util.Arrays;
  5. import java.util.UUID;

  7. import org.apache.spark.SparkConf;
  8. import org.apache.spark.streaming.Durations;
  9. import org.apache.spark.streaming.api.java.JavaDStream;
  10. import org.apache.spark.streaming.api.java.JavaPairDStream;
  11. import org.apache.spark.streaming.api.java.JavaStreamingContext;

  13. import scala.Tuple2;

  15. public class SparkStreamTest {

  17.     public static void main(String[] args) throws Exception {
  18.         startSockerServer(9999);
  19.         
  20.         SparkConf conf = new SparkConf().setMaster("local[2]").setAppName("NetworkWordCount");
  21.         JavaStreamingContext jssc = new JavaStreamingContext(conf, Durations.seconds(1));
  22.         JavaDStream<String> lines = jssc.socketTextStream("localhost", 9999);

  24.         JavaDStream<String> words = lines.flatMap(x -> Arrays.asList(x.split(" ")).iterator());
  25.         JavaPairDStream<String, Integer> pairs = words.mapToPair(s -> new Tuple2<>(s, 1));
  26.         JavaPairDStream<String, Integer> counts = pairs.reduceByKey((x, y) -> x + y);
  27.         counts.print();
  28.         
  29.         jssc.start();
  30.         jssc.awaitTermination();
  31.         jssc.close();
  32.     }

  34.     private static void startSockerServer(int port) throws Exception {
  35.         new Thread(){
  36.             public void run() {
  37.                 try {
  38.                     ServerSocket server = new ServerSocket(port);
  39.                     Socket socket = server.accept();
  40.                     OutputStream os = socket.getOutputStream();
  41.                     for (int i = 0; i < 1000; i++) {
  42.                         String txt = UUID.randomUUID().toString().replaceAll("-", " ");
  43.                         txt = "Hello test\n";
  44.                         os.write(txt.getBytes());
  45.                         Thread.sleep(10);
  46.                     }
  47.                     socket.close();
  48.                     server.close();
  49.                 } catch (Exception e) {
  50.                     e.printStackTrace();
  51.                 }
  52.             }
  53.         }.start();
  54.     }
  55. }
會不斷地列印出類似的結果：
-------------------------------------------
Time: 1490947248000 ms
-------------------------------------------
(Hello,99)
(test,99)



來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/10742815/viewspace-2136411/，如需轉載，請註明出處，否則將追究法律責任。

相關文章