storm實時計算例項（socket實時接入)
介紹
實現了一個簡單的從實時日誌檔案監聽，寫入socket伺服器，再接入Storm計算的一個流程。
原始碼
日誌監聽實時寫入socket伺服器
package socket;import java.io.BufferedReader;import java.io.File;
import java.io.IOException;
import java.io.InputStreamReader;import java.io.PrintWriter;import java.io.RandomAccessFile;
import java.net.Socket;import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
/*
* 監測資料，通過socket遠端傳送到另外伺服器 ，見MyServerMulti
* ClientRead再通過伺服器從socket裡讀
*
*/
public class LogViewToSocket {
private long lastTimeFileSize = 0; //上次檔案大小
/**
* 實時輸出日誌資訊
* @param logFile 日誌檔案
* @throws IOException
*/
public String getNewFile(File file) { File[] fs=file.listFiles(); long maxtime=0; String newfilename=""; for (int i=0;i<fs.length;i++) { if (fs[i].lastModified()>maxtime) { maxtime=fs[i].lastModified(); newfilename=fs[i].getAbsolutePath(); } } return newfilename; } RandomAccessFile randomFile=null; String newfile=null; String thisfile=null; public void realtimeShowLog(final File logFile,final PrintWriter out) throws IOException{
newfile=getNewFile(logFile); //指定檔案可讀可寫
randomFile = new RandomAccessFile(new File(newfile),"r");
//啟動一個執行緒每1秒鐘讀取新增的日誌資訊
ScheduledExecutorService exec =
Executors.newScheduledThreadPool(1);
exec.scheduleWithFixedDelay(new Runnable(){
public void run() {
try {
//獲得變化部分的
randomFile.seek(lastTimeFileSize);
String tmp = "";
while( (tmp = randomFile.readLine())!= null) {
System.out.println(new String(tmp.getBytes("ISO8859-1")));
out.println(new String(tmp.getBytes("ISO8859-1")));
out.flush();
}
thisfile=getNewFile(logFile); if(!thisfile.equals(newfile))
{
randomFile = new RandomAccessFile(new File(newfile),"r");
lastTimeFileSize=0;
} else
lastTimeFileSize = randomFile.length();
} catch (IOException e) {
throw new RuntimeException(e);
}
}
}, 0, 1, TimeUnit.SECONDS);
}
public static void main(String[] args) throws Exception {
LogViewToSocket view = new LogViewToSocket();
Socket socket=new Socket("192.168.27.100",5678);
PrintWriter out=new PrintWriter(socket.getOutputStream());
final File tmpLogFile = new File("/home/hadoop/test");
view.realtimeShowLog(tmpLogFile,out);
// socket.close();
}
}
socket伺服器處理
import java.io.BufferedReader;
import java.io.IOException;
import java.io.InputStreamReader;
import java.io.PrintWriter;
import java.net.ServerSocket;
import java.net.Socket;
import java.net.SocketAddress;
import java.util.*;
public class MyServerMulti {
private static Socket socket1; public static void main(String[] args) throws IOException {
ServerSocket server = new ServerSocket(5678);
int i=0;
ArrayList<PrintWriter> outs=new ArrayList<PrintWriter>();
/*
* 一個client socket傳送資料過來， server端再發到其他client socket端
*
*/
Socket socket1=null; while (true) {
Socket socket = server.accept();
i++;
System.out.println(i);
System.out.println(socket.getInetAddress());
PrintWriter out= new PrintWriter(socket.getOutputStream());
outs.add(out); if(i==1)
socket1=socket; if(i==2)
invoke(socket1,outs);
}
}
private static void invoke(final Socket client, final ArrayList<PrintWriter> outs) throws IOException {
new Thread(new Runnable() {
public void run() {
BufferedReader in = null;
PrintWriter out = null;
PrintWriter out1 = null; try {
in = new BufferedReader(new InputStreamReader(client.getInputStream()));
out = new PrintWriter(client.getOutputStream());
while (true) {
String msg = in.readLine();
System.out.println(msg);
out.println("Server received " + msg);
out.flush();
/*資料轉傳送到多個client*/
for(int i=0;i<outs.size();i++)
{
out1=outs.get(i);
System.out.println(i);
System.out.println("send msg:"+msg);
out1.println(msg);
out1.flush();
}
System.out.println(client.getInetAddress()); if (msg.equals("bye")) {
break;
}
}
} catch(IOException ex) {
ex.printStackTrace();
} finally {
try {
in.close();
} catch (Exception e) {}
try {
out.close();
} catch (Exception e) {}
try {
client.close();
} catch (Exception e) {}
}
}
}).start();
}
}
storm topology
import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.File;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.FileReader;import java.io.FileWriter;import java.io.IOException;import java.io.InputStreamReader;import java.io.OutputStreamWriter;import java.io.PrintWriter;import java.io.RandomAccessFile;import java.net.Socket;import java.net.UnknownHostException;import java.util.Map;
//import mytest.ThroughputTest.GenSpout;
import backtype.storm.Config;import backtype.storm.LocalCluster;import backtype.storm.StormSubmitter;import backtype.storm.generated.AlreadyAliveException;import backtype.storm.generated.InvalidTopologyException;import backtype.storm.spout.SpoutOutputCollector;import backtype.storm.task.OutputCollector;import backtype.storm.task.TopologyContext;import backtype.storm.topology.BasicOutputCollector;import backtype.storm.topology.OutputFieldsDeclarer;import backtype.storm.topology.TopologyBuilder;import backtype.storm.topology.base.BaseBasicBolt;import backtype.storm.topology.base.BaseRichBolt;import backtype.storm.topology.base.BaseRichSpout;import backtype.storm.tuple.Fields;import backtype.storm.tuple.Tuple;import backtype.storm.tuple.Values;import backtype.storm.utils.Utils;/*
*
*
* storm jar stormtest.jar socket.SocketProcess /home/hadoop/out_socket.txt true
*
*/
public class SocketProcess {
public static class SocketSpout extends BaseRichSpout {
/**
*/
static Socket sock=null; static BufferedReader in=null;
String str=null; private static final long serialVersionUID = 1L; private SpoutOutputCollector _collector; private BufferedReader br; private String dataFile; private BufferedWriter bw2;
RandomAccessFile randomFile; private long lastTimeFileSize = 0;
int cnt=0; //定義spout檔案
SocketSpout(){
}
//定義如何讀取spout檔案
@Override
public void open(Map conf, TopologyContext context,
SpoutOutputCollector collector) { // TODO Auto-generated method stub
_collector = collector; try { sock=new Socket("192.168.27.100",5678); in=
new BufferedReader(new InputStreamReader(sock.getInputStream()));
} catch (UnknownHostException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }
}
//獲取下一個tuple的方法
@Override
public void nextTuple() { // TODO Auto-generated method stub
if(sock==null){ try { sock=new Socket("192.168.27.100",5678); in=
new BufferedReader(new InputStreamReader(sock.getInputStream()));
} catch (UnknownHostException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }
}
while(true){
try { str = in.readLine(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); }
System.out.println(str);
_collector.emit(new Values(str)); if(str.equals("end")){
break;
}
}
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) { // TODO Auto-generated method stub
declarer.declare(new Fields("line"));
}
}
public static class Process extends BaseRichBolt{
private String _seperator; private String _outFile;
PrintWriter pw; private OutputCollector _collector; private BufferedWriter bw;
public Process(String outFile) {
this._outFile = outFile;
}
//把輸出結果儲存到外部檔案裡面。
@Override
public void prepare(Map stormConf, TopologyContext context,
OutputCollector collector) { // TODO Auto-generated method stub
this._collector = collector;
File out = new File(_outFile); try {// br = new BufferedWriter(new FileWriter(out));
bw = new BufferedWriter(new OutputStreamWriter(
new FileOutputStream(out, true)));
} catch (IOException e1) { // TODO Auto-generated catch block
e1.printStackTrace();
}
}
//blot計算單元，把tuple中的資料新增一個bkeep和回車。然後儲存到outfile指定的檔案中。
@Override
public void execute(Tuple input) { // TODO Auto-generated method stub
String line = input.getString(0);// System.out.println(line);
// String[] str = line.split(_seperator);
// System.out.println(str[2]);
try {
bw.write(line+",bkeep"+"
");
bw.flush();
} catch (IOException e) { // TODO Auto-generated catch block
e.printStackTrace();
}
_collector.emit(new Values(line));
}
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) { // TODO Auto-generated method stub
declarer.declare(new Fields("line"));
}
}
public static void main(String[] argv) throws AlreadyAliveException, InvalidTopologyException{
String outFile = argv[0]; //輸出檔案
boolean distribute = Boolean.valueOf(argv[1]); //本地模式還是叢集模式
TopologyBuilder builder = new TopologyBuilder(); //build一個topology
builder.setSpout("spout", new SocketSpout(), 1); //指定spout
builder.setBolt("bolt", new Process(outFile),1).shuffleGrouping("spout"); //指定bolt，包括bolt、process和grouping
Config conf = new Config(); if(distribute){
StormSubmitter.submitTopology("SocketProcess", conf, builder.createTopology());
}else{
LocalCluster cluster = new LocalCluster();
cluster.submitTopology("SocketProcess", conf, builder.createTopology());
}
}
}
最後執行
storm jar stormtest.jar socket.SocketProcess /home/hadoop/out_socket.txt true
spout接受從socket伺服器實時傳送過來的資料，經過topology處理，最終將資料寫入out_socket.txt檔案
本文轉自ljianbing51CTO部落格，原文連結：http://blog.51cto.com/ljianbing/1733313 ，如需轉載請自行聯絡原作者
相關文章
- Storm 實戰：構建大資料實時計算ORM大資料
- 《Storm分散式實時計算模式》——1.7總結ORM分散式模式
- JStorm-Alibaba——Storm的實時流式計算框架JSORM框架
- 從Storm到Flink，有贊五年實時計算效率提升實踐ORM
- 兩個例子(來自Storm實戰 構建大資料實時計算)ORM大資料
- javascript實時顯示時間程式碼例項JavaScript
- 實時計算框架特點及對比：Flink、Spark Streaming、Storm框架SparkORM
- 實時計算小括
- 實時獲取當前時間程式碼例項
- 實時變化的時鐘效果程式碼例項
- 實時計算如何幫助淘寶實現線上「實時選品」？
- 說說實時流式計算
- 實時計算神器：binlog
- Spark 實時計算整合案例Spark
- 實時計算助力1688打造「實時挑貨」系統
- js計算兩個時間點時間間隔的程式碼例項JS
- 你瞭解實時計算嗎？
- 【Twitter Storm系列】flume-ng+Kafka+Storm+HDFS 實時系統搭建ORMKafka
- Kafka實時流資料經Storm至HdfsKafkaORM
- 大資料6.1 - 實時分析（storm和kafka）大資料ORMKafka
- Flink 在有贊實時計算的實踐
- vivo 實時計算平臺建設實踐
- MySQL實時計算QPS，TPS的SHELL指令碼兩例MySql指令碼
- 基於Redis、Storm的實時資料查詢實踐RedisORM
- 計算兩個時間點相差天數例項程式碼
- 用Spark進行實時流計算Spark
- 實時計算無線資料分析
- Flink實時計算topN熱榜
- 實時計算Flink——產品安全
- 實時計算Flink效能調優
- 【MySQL】利用binlog完成實時計算MySql
- 《Storm實時資料處理》一1.1 簡介ORM
- flume-ng+Kafka+Storm+HDFS 實時系統搭建KafkaORM
- Qt 中Socket程式設計例項QT程式設計
- android socket程式設計例項Android程式設計
- 聊一聊實時計算系統設計
- JavaScript倒數計時程式碼例項JavaScript
- Hive實戰—時間滑動視窗計算Hive