storm實時計算例項(socket實時接入)

技術小甜發表於2017-11-14

介紹

實現了一個簡單的從實時日誌檔案監聽,寫入socket伺服器,再接入Storm計算的一個流程。

原始碼

日誌監聽實時寫入socket伺服器

package socket;import java.io.BufferedReader;import java.io.File;     
import java.io.IOException;     
import java.io.InputStreamReader;import java.io.PrintWriter;import java.io.RandomAccessFile;     
import java.net.Socket;import java.util.concurrent.Executors;     
import java.util.concurrent.ScheduledExecutorService;     
import java.util.concurrent.TimeUnit;     
/*
 * 監測資料,通過socket遠端傳送到另外伺服器 ,見MyServerMulti
 * ClientRead再通過伺服器從socket裡讀
 * 
 */
    public class LogViewToSocket {     
    private long lastTimeFileSize = 0;  //上次檔案大小     
    /**   
     * 實時輸出日誌資訊   
     * @param logFile 日誌檔案   
     * @throws IOException   
     */    
      public String getNewFile(File file)  {    File[] fs=file.listFiles();    long maxtime=0;    String newfilename="";    for (int i=0;i<fs.length;i++)    {      if (fs[i].lastModified()>maxtime)      {        maxtime=fs[i].lastModified();        newfilename=fs[i].getAbsolutePath();              }    }    return newfilename;  }   RandomAccessFile randomFile=null;   String newfile=null;   String thisfile=null;    public void realtimeShowLog(final File logFile,final PrintWriter out) throws IOException{     
    	   newfile=getNewFile(logFile);        //指定檔案可讀可寫     
            randomFile = new RandomAccessFile(new File(newfile),"r");     
        //啟動一個執行緒每1秒鐘讀取新增的日誌資訊     
        ScheduledExecutorService exec =      
            Executors.newScheduledThreadPool(1);     
        exec.scheduleWithFixedDelay(new Runnable(){     
            public void run() {     
                try {     
                    //獲得變化部分的     
                    randomFile.seek(lastTimeFileSize);     
                    String tmp = "";     
                    while( (tmp = randomFile.readLine())!= null) {     
                        System.out.println(new String(tmp.getBytes("ISO8859-1"))); 
                        out.println(new String(tmp.getBytes("ISO8859-1")));
                        out.flush(); 
                    }   
                   thisfile=getNewFile(logFile);                   if(!thisfile.equals(newfile))
                   
                   {
                	   randomFile = new RandomAccessFile(new File(newfile),"r");
                	   lastTimeFileSize=0;
                   }                   else
                	   
                    lastTimeFileSize = randomFile.length();     
                   
                } catch (IOException e) {     
                    throw new RuntimeException(e);     
                }     
            }     
        }, 0, 1, TimeUnit.SECONDS);     
    }     
         
    public static void main(String[] args) throws Exception {     
    	LogViewToSocket view = new LogViewToSocket();     

    		Socket socket=new Socket("192.168.27.100",5678); 
   
    	PrintWriter out=new PrintWriter(socket.getOutputStream());    
    	 
    	  

        final File tmpLogFile = new File("/home/hadoop/test");     
        view.realtimeShowLog(tmpLogFile,out); 
       // socket.close();
        
    }     
    
}

socket伺服器處理

import java.io.BufferedReader;  
import java.io.IOException;  
import java.io.InputStreamReader;  
import java.io.PrintWriter;  
import java.net.ServerSocket;  
import java.net.Socket;  
import java.net.SocketAddress;
import java.util.*;  
public class MyServerMulti {  
    private static Socket socket1;  public static void main(String[] args) throws IOException {  
        ServerSocket server = new ServerSocket(5678);  
          int i=0;
          ArrayList<PrintWriter> outs=new ArrayList<PrintWriter>();          
          /*
           * 一個client socket傳送資料過來, server端再發到其他client socket端
           * 
           */
          Socket socket1=null;        while (true) {
        	
            Socket socket = server.accept();  
             i++;
             System.out.println(i);
             System.out.println(socket.getInetAddress());
            	 PrintWriter out= new PrintWriter(socket.getOutputStream());
            	 outs.add(out);            	 if(i==1)
            		  socket1=socket;            	 if(i==2)
            		 
                 invoke(socket1,outs);
            	 
            
        }  
    }  
      
    private static void invoke(final Socket client, final ArrayList<PrintWriter> outs) throws IOException {  
        new Thread(new Runnable() {  
            public void run() {  
                BufferedReader in = null;  
                PrintWriter out = null;  
                PrintWriter out1 = null;                try {  
                    in = new BufferedReader(new InputStreamReader(client.getInputStream()));  
                    out = new PrintWriter(client.getOutputStream());  
  
                    while (true) {  
                        String msg = in.readLine();  
                        System.out.println(msg);  
                        out.println("Server received " + msg);  
                        out.flush();  
                        
                        /*資料轉傳送到多個client*/
                        for(int i=0;i<outs.size();i++)
                        {
                        	out1=outs.get(i);
                        	System.out.println(i);
                        	System.out.println("send msg:"+msg);
                        	 out1.println(msg);
                        	out1.flush();
                        }
                        
                        System.out.println(client.getInetAddress());                        if (msg.equals("bye")) {  
                            break;  
                        }  
                    }  
                } catch(IOException ex) {  
                    ex.printStackTrace();  
                } finally {  
                    try {  
                        in.close();  
                    } catch (Exception e) {}  
                    try {  
                        out.close();  
                    } catch (Exception e) {}  
                    try {  
                        client.close();  
                    } catch (Exception e) {}  
                }  
            }  
        }).start();  
    }  
}

storm topology

import java.io.BufferedReader;import java.io.BufferedWriter;import java.io.File;import java.io.FileNotFoundException;import java.io.FileOutputStream;import java.io.FileReader;import java.io.FileWriter;import java.io.IOException;import java.io.InputStreamReader;import java.io.OutputStreamWriter;import java.io.PrintWriter;import java.io.RandomAccessFile;import java.net.Socket;import java.net.UnknownHostException;import java.util.Map; 
//import mytest.ThroughputTest.GenSpout;
 import backtype.storm.Config;import backtype.storm.LocalCluster;import backtype.storm.StormSubmitter;import backtype.storm.generated.AlreadyAliveException;import backtype.storm.generated.InvalidTopologyException;import backtype.storm.spout.SpoutOutputCollector;import backtype.storm.task.OutputCollector;import backtype.storm.task.TopologyContext;import backtype.storm.topology.BasicOutputCollector;import backtype.storm.topology.OutputFieldsDeclarer;import backtype.storm.topology.TopologyBuilder;import backtype.storm.topology.base.BaseBasicBolt;import backtype.storm.topology.base.BaseRichBolt;import backtype.storm.topology.base.BaseRichSpout;import backtype.storm.tuple.Fields;import backtype.storm.tuple.Tuple;import backtype.storm.tuple.Values;import backtype.storm.utils.Utils;/*
 * 
 *
 *  storm jar stormtest.jar socket.SocketProcess /home/hadoop/out_socket.txt true
 * 
 */
 public class SocketProcess {
         public static class  SocketSpout extends BaseRichSpout {
 
                   /**
                    */
        	  static Socket sock=null;        	  static BufferedReader in=null;
        	  String str=null;                   private static final long serialVersionUID = 1L;                   private SpoutOutputCollector _collector;                   private BufferedReader br;                   private String dataFile;                   private BufferedWriter bw2;
                    RandomAccessFile randomFile;                    private long lastTimeFileSize = 0; 
                    int cnt=0;                   //定義spout檔案
                    SocketSpout(){
                     
                   } 
                   //定義如何讀取spout檔案
                   @Override
                   public void open(Map conf, TopologyContext context,
                                     SpoutOutputCollector collector) {                            // TODO Auto-generated method stub
                            _collector = collector;                            try {                sock=new Socket("192.168.27.100",5678);                 in=   
                  new BufferedReader(new InputStreamReader(sock.getInputStream()));   
              } catch (UnknownHostException e) {                // TODO Auto-generated catch block                e.printStackTrace();              } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();              }
                       
                   } 
                   //獲取下一個tuple的方法
                   @Override
                   public void nextTuple() {                            // TODO Auto-generated method stub
                	   if(sock==null){                		     try {                sock=new Socket("192.168.27.100",5678);                 in=   
                    new BufferedReader(new InputStreamReader(sock.getInputStream()));  
              } catch (UnknownHostException e) {                // TODO Auto-generated catch block                e.printStackTrace();              } catch (IOException e) {                // TODO Auto-generated catch block                e.printStackTrace();              } 
                	   }                	   
                	   
                	   while(true){    
                		  
            try {              str = in.readLine();            } catch (IOException e) {              // TODO Auto-generated catch block              e.printStackTrace();            }
                		System.out.println(str);  
                		_collector.emit(new Values(str));                		if(str.equals("end")){    
                			break;    
                			} 
                		}
                	   
                	   
                	   
                	   
                	   
                	   
                	   
                            
                            
                   } 
 
                   @Override
                   public void declareOutputFields(OutputFieldsDeclarer declarer) {                            // TODO Auto-generated method stub
                            declarer.declare(new Fields("line"));
                   }
                  
         }        
 
         public static class Process extends BaseRichBolt{
 
                   private String _seperator;                   private String _outFile;
                   PrintWriter pw;                   private OutputCollector _collector;                   private BufferedWriter bw;                  
                   public Process(String outFile) {                           
                            this._outFile   = outFile;
                           
                   }                  
                   //把輸出結果儲存到外部檔案裡面。
                   @Override
                   public void prepare(Map stormConf, TopologyContext context,
                                     OutputCollector collector) {                            // TODO Auto-generated method stub
                            this._collector = collector;
                            File out = new File(_outFile);                            try {//                                  br = new BufferedWriter(new FileWriter(out));
                                     bw = new BufferedWriter(new OutputStreamWriter( 
                             new FileOutputStream(out, true))); 
                            } catch (IOException e1) {                                     // TODO Auto-generated catch block
                                     e1.printStackTrace();
                            }                
                   }                  
                   //blot計算單元,把tuple中的資料新增一個bkeep和回車。然後儲存到outfile指定的檔案中。
                   @Override
                   public void execute(Tuple input) {                            // TODO Auto-generated method stub
                            String line = input.getString(0);//                         System.out.println(line);
                       //     String[] str = line.split(_seperator);
                         //   System.out.println(str[2]);
                            try {
                                     bw.write(line+",bkeep"+"
");
                                     bw.flush();
                            } catch (IOException e) {                                     // TODO Auto-generated catch block
                                     e.printStackTrace();
                            }
                           
                            _collector.emit(new Values(line));
                   } 
                   @Override
                   public void declareOutputFields(OutputFieldsDeclarer declarer) {                            // TODO Auto-generated method stub
                            declarer.declare(new Fields("line"));
                   }
                  
         }        
         public static void main(String[] argv) throws AlreadyAliveException, InvalidTopologyException{
                
                   String outFile   = argv[0]; //輸出檔案
                   boolean distribute = Boolean.valueOf(argv[1]);       //本地模式還是叢集模式
                   TopologyBuilder builder = new TopologyBuilder();  //build一個topology
        builder.setSpout("spout", new  SocketSpout(), 1);   //指定spout
        builder.setBolt("bolt", new Process(outFile),1).shuffleGrouping("spout");  //指定bolt,包括bolt、process和grouping
        Config conf = new Config();        if(distribute){
            StormSubmitter.submitTopology("SocketProcess", conf, builder.createTopology());
        }else{
                 LocalCluster cluster = new LocalCluster();
                 cluster.submitTopology("SocketProcess", conf, builder.createTopology());
        }
         }       
}

最後執行 

storm jar stormtest.jar socket.SocketProcess /home/hadoop/out_socket.txt true

spout接受從socket伺服器實時傳送過來的資料,經過topology處理,最終將資料寫入out_socket.txt檔案

本文轉自ljianbing51CTO部落格,原文連結:http://blog.51cto.com/ljianbing/1733313 ,如需轉載請自行聯絡原作者


相關文章