網頁主動探測工具

壹頁書發表於2014-11-24
網頁
單位的專案是IBatis做的,每個查詢的SQL裡面都有很多判斷
上次優化SQL之後,其中的一個分支報錯,但是作為dba,不可能排查每一個分支.
所以,乾脆用爬蟲爬過所有的網頁，主動探測程式的異常.
這樣有兩個好處
1.可以主動檢視網頁是否異常 (500錯誤，404錯誤)
2.可以篩查速度較慢的網頁,從這個方向也可以定位慢SQL吧.（也有伺服器資源不足,造成網路超時的情況）

前提,
必須是網際網路公司,大多數網頁不用登入也可以瀏覽

首先,建表
CREATE SEQUENCE seq_probe_id INCREMENT BY 1 START WITH 1 NOMAXvalue NOCYCLE CACHE 2000;

create table probe( 
    id int primary key,
    host varchar(40) not null, 
    path varchar(500) not null,
    state int not null,
    taskTime int not null, 
    type varchar(10) not null,
    createtime date default sysdate not null 
) ;
其中host是域名,path是網頁的相對路徑,state是HTTP狀態碼,taskTime是網頁獲取時間,單位是毫秒,type是型別(html,htm,jpg等)

程式結構


程式分三個主要步驟,再分別用三個佇列實現生產者消費者模式.
1.連線.根據連線佇列的目標,使用Socket獲取網頁,然後放入解析佇列
2.解析.根據解析佇列的內容,使用正規表示式獲取該網頁的合法連線,將其再放入連線佇列.然後將解析的網頁放入持久化佇列
3.持久化.將持久化佇列的內容存入資料庫,以便查詢。

程式使用三個步驟並行,每個步驟可以併發的方式.
但是通常來說,解析和持久化可以分別用單執行緒的方式執行.


  1. import java.io.BufferedReader;
  2. import java.io.BufferedWriter;
  3. import java.io.InputStreamReader;
  4. import java.io.OutputStreamWriter;
  5. import java.net.InetAddress;
  6. import java.net.Socket;
  7. import java.sql.Connection;
  8. import java.sql.DriverManager;
  9. import java.sql.PreparedStatement;
  10. import java.sql.SQLException;
  11. import java.util.ArrayList;
  12. import java.util.Iterator;
  13. import java.util.List;
  14. import java.util.Set;
  15. import java.util.concurrent.BlockingQueue;
  16. import java.util.concurrent.ConcurrentSkipListSet;
  17. import java.util.concurrent.CopyOnWriteArrayList;
  18. import java.util.concurrent.ExecutorService;
  19. import java.util.concurrent.Executors;
  20. import java.util.concurrent.LinkedBlockingQueue;
  21. import java.util.concurrent.atomic.AtomicInteger;
  22. import java.util.regex.Matcher;
  23. import java.util.regex.Pattern;

  25. public class Probe {

  27.     private static final BlockingQueue<Task> CONNECTLIST = new LinkedBlockingQueue<Task>();
  28.     private static final BlockingQueue<Task> PARSELIST = new LinkedBlockingQueue<Task>();
  29.     private static final BlockingQueue<Task> PERSISTENCELIST = new LinkedBlockingQueue<Task>();
  30.     private static ExecutorService CONNECTTHREADPOOL;
  31.     private static ExecutorService PARSETHREADPOOL;
  32.     private static ExecutorService PERSISTENCETHREADPOOL;
  33.     private static final List<String> DOMAINLIST = new CopyOnWriteArrayList<>();

  35.     static {
  36.         CONNECTTHREADPOOL = Executors.newFixedThreadPool(200);
  37.         PARSETHREADPOOL = Executors.newSingleThreadExecutor();
  38.         PERSISTENCETHREADPOOL = Executors.newFixedThreadPool(1);
  39.         DOMAINLIST.add("域名");
  40.     }

  42.     public static void main(String args[]) throws Exception {
  43.         long start = System.currentTimeMillis();
  44.         CONNECTLIST.put(new Task("域名", 80, "/static/index.html"));

  46.         for (int i = 0; i < 600; i++) {
  47.             CONNECTTHREADPOOL.submit(new ConnectHandler(CONNECTLIST, PARSELIST));
  48.         }
  49.         PARSETHREADPOOL.submit(new ParseHandler(CONNECTLIST, PARSELIST, PERSISTENCELIST, DOMAINLIST));
  50.         PERSISTENCETHREADPOOL.submit(new PersistenceHandler(PERSISTENCELIST));

  52.         while (true) {
  53.             Thread.sleep(1000);
  54.             long end = System.currentTimeMillis();
  55.             float interval = ((end - start) / 1000);
  56.             int connectTotal = ConnectHandler.GETCOUNT();
  57.             int parseTotal = ParseHandler.GETCOUNT();
  58.             int persistenceTotal = PersistenceHandler.GETCOUNT();

  60.             int connectps = Math.round(connectTotal / interval);
  61.             int parseps = Math.round(parseTotal / interval);
  62.             int persistenceps = Math.round(persistenceTotal / interval);
  63.             System.out.print("\r連線總數:" + connectTotal + " \t每秒連線:" + connectps + "\t連線佇列剩餘:" + CONNECTLIST.size()
  64.                     + " \t解析總數:" + parseTotal + " \t每秒解析:" + parseps + "\t解析佇列剩餘:" + PARSELIST.size() + " \t持久化總數:"
  65.                     + persistenceTotal + " \t每秒持久化:" + persistenceps + "\t持久化佇列剩餘:" + PERSISTENCELIST.size());
  66.         }
  67.     }
  68. }

  70. class Task {
  71.     public Task() {
  72.     }

  74.     public void init(String host, int port, String path) {
  75.         this.setCurrentPath(path);
  76.         this.host = host;
  77.         this.port = port;
  78.     }

  80.     public Task(String host, int port, String path) {
  81.         init(host, port, path);
  82.     }

  84.     private String host;
  85.     private int port;
  86.     private String currentPath;
  87.     private long taskTime;
  88.     private String type;
  89.     private String content;
  90.     private int state;

  92.     public int getState() {
  93.         return state;
  94.     }

  96.     public void setState(int state) {
  97.         this.state = state;
  98.     }

  100.     public String getCurrentPath() {
  101.         return currentPath;
  102.     }

  104.     public void setCurrentPath(String currentPath) {
  105.         this.currentPath = currentPath;
  106.         this.type = currentPath.substring(currentPath.indexOf(".") + 1,
  107.                 currentPath.indexOf("?") != -1 ? currentPath.indexOf("?") : currentPath.length());

  109.     }

  111.     public long getTaskTime() {
  112.         return taskTime;
  113.     }

  115.     public void setTaskTime(long taskTime) {
  116.         this.taskTime = taskTime;
  117.     }

  119.     public String getType() {
  120.         return type;
  121.     }

  123.     public void setType(String type) {
  124.         this.type = type;
  125.     }

  127.     public String getHost() {
  128.         return host;
  129.     }

  131.     public int getPort() {
  132.         return port;
  133.     }

  135.     public String getContent() {
  136.         return content;
  137.     }

  139.     public void setContent(String content) {
  140.         this.content = content;

  142.     }
  143. }

  145. class ParseHandler implements Runnable {
  146.     private static Set<String> SET = new ConcurrentSkipListSet<String>();

  148.     public static int GETCOUNT() {
  149.         return COUNT.get();
  150.     }

  152.     private static final AtomicInteger COUNT = new AtomicInteger();
  153.     private BlockingQueue<Task> connectlist;
  154.     private BlockingQueue<Task> parselist;
  155.     private BlockingQueue<Task> persistencelist;
  156.     List<String> domainlist;

  158.     private interface Filter {
  159.         void doFilter(Task fatherTask, Task newTask, String path, Filter chain);
  160.     }

  162.     private class FilterChain implements Filter {
  163.         private List<Filter> list = new ArrayList<Filter>();

  165.         {
  166.             addFilter(new TwoLevel());
  167.             addFilter(new OneLevel());
  168.             addFilter(new FullPath());
  169.             addFilter(new Root());
  170.             addFilter(new Default());
  171.         }

  173.         private void addFilter(Filter filter) {
  174.             list.add(filter);
  175.         }

  177.         private Iterator<Filter> it = list.iterator();

  179.         @Override
  180.         public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
  181.             if (it.hasNext()) {
  182.                 it.next().doFilter(fatherTask, newTask, path, chain);
  183.             }
  184.         }

  186.     }

  188.     private class TwoLevel implements Filter {

  190.         @Override
  191.         public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
  192.             if (path.startsWith("../../")) {
  193.                 String prefix = getPrefix(fatherTask.getCurrentPath(), 3);
  194.                 newTask.init(fatherTask.getHost(), fatherTask.getPort(), path.replace("../../", prefix));
  195.             } else {
  196.                 chain.doFilter(fatherTask, newTask, path, chain);
  197.             }

  199.         }
  200.     }

  202.     private class OneLevel implements Filter {

  204.         @Override
  205.         public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
  206.             if (path.startsWith("../")) {
  207.                 String prefix = getPrefix(fatherTask.getCurrentPath(), 2);
  208.                 newTask.init(fatherTask.getHost(), fatherTask.getPort(), path.replace("../", prefix));
  209.             } else {
  210.                 chain.doFilter(fatherTask, newTask, path, chain);
  211.             }

  213.         }

  215.     }

  217.     private class FullPath implements Filter {

  219.         @Override
  220.         public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
  221.             if (path.startsWith("http://")) {
  222.                 Iterator<String> it = domainlist.iterator();
  223.                 boolean flag = false;
  224.                 while (it.hasNext()) {
  225.                     String domain = it.next();
  226.                     if (path.startsWith("http://" + domain + "/")) {
  227.                         newTask.init(domain, fatherTask.getPort(), path.replace("http://" + domain + "/", "/"));
  228.                         flag = true;
  229.                         break;
  230.                     }
  231.                 }
  232.                 if (!flag) {
  233.                     newTask = null;
  234.                 }
  235.             } else {
  236.                 chain.doFilter(fatherTask, newTask, path, chain);
  237.             }
  238.         }

  240.     }

  242.     private class Root implements Filter {

  244.         @Override
  245.         public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
  246.             if (path.startsWith("/")) {
  247.                 newTask.init(fatherTask.getHost(), fatherTask.getPort(), path);
  248.             } else {
  249.                 chain.doFilter(fatherTask, newTask, path, chain);
  250.             }
  251.         }

  253.     }

  255.     private class Default implements Filter {

  257.         @Override
  258.         public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
  259.             String prefix = getPrefix(fatherTask.getCurrentPath(), 1);
  260.             newTask.init(fatherTask.getHost(), fatherTask.getPort(), prefix + "/" + path);
  261.         }

  263.     }

  265.     public ParseHandler(BlockingQueue<Task> connectlist, BlockingQueue<Task> parselist,
  266.             BlockingQueue<Task> persistencelist, List<String> domainlist) {
  267.         this.connectlist = connectlist;
  268.         this.parselist = parselist;
  269.         this.persistencelist = persistencelist;
  270.         this.domainlist = domainlist;
  271.     }

  273.     private Pattern pattern = Pattern.compile("\"[^\"]+\\.htm[^\"]*\"");

  275.     private void handler() {
  276.         try {
  277.             Task task = parselist.take();
  278.             parseTaskState(task);
  279.             if (200 == task.getState()) {
  280.                 Matcher matcher = pattern.matcher(task.getContent());
  281.                 while (matcher.find()) {
  282.                     String path = matcher.group();
  283.                     if (!path.contains(" ") && !path.contains("\t") && !path.contains("(") && !path.contains(")")
  284.                             && !path.contains(":")) {
  285.                         path = path.substring(1, path.length() - 1);
  286.                         if (!SET.contains(path)) {
  287.                             SET.add(path);
  288.                             createNewTask(task, path);
  289.                         }
  290.                     }

  292.                 }
  293.             }
  294.             task.setContent(null);
  295.             persistencelist.put(task);
  296.         } catch (Exception e) {
  297.             // TODO Auto-generated catch block
  298.             e.printStackTrace();
  299.         }
  300.     }

  302.     private void parseTaskState(Task task) {
  303.         if (task.getContent().startsWith("HTTP/1.1")) {
  304.             task.setState(Integer.parseInt(task.getContent().substring(9, 12)));
  305.         } else {
  306.             task.setState(Integer.parseInt(task.getContent().substring(19, 22)));
  307.         }
  308.     }

  310.     /**
  311.      * @param fatherTask
  312.      * @param path
  313.      * @throws Exception
  314.      */
  315.     private void createNewTask(Task fatherTask, String path) throws Exception {
  316.         Task newTask = new Task();
  317.         FilterChain filterchain = new FilterChain();
  318.         filterchain.doFilter(fatherTask, newTask, path, filterchain);
  319.         if (newTask != null) {
  320.             connectlist.put(newTask);
  321.         }
  322.     }

  324.     private String getPrefix(String s, int count) {
  325.         String prefix = s;
  326.         while (count > 0) {
  327.             prefix = prefix.substring(0, prefix.lastIndexOf("/"));
  328.             count--;
  329.         }
  330.         return "".equals(prefix) ? "/" : prefix;
  331.     }

  333.     @Override
  334.     public void run() {
  335.         while (true) {
  336.             this.handler();
  337.             COUNT.addAndGet(1);
  338.         }
  339.     }

  341. }

  343. class ConnectHandler implements Runnable {
  344.     public static int GETCOUNT() {
  345.         return COUNT.get();
  346.     }

  348.     private static final AtomicInteger COUNT = new AtomicInteger();
  349.     private BlockingQueue<Task> connectlist;
  350.     private BlockingQueue<Task> parselist;

  352.     public ConnectHandler(BlockingQueue<Task> connectlist, BlockingQueue<Task> parselist) {
  353.         this.connectlist = connectlist;
  354.         this.parselist = parselist;
  355.     }

  357.     private void handler() {
  358.         try {
  359.             Task task = connectlist.take();
  360.             long start = System.currentTimeMillis();
  361.             getHtml(task);
  362.             long end = System.currentTimeMillis();
  363.             task.setTaskTime(end - start);
  364.             parselist.put(task);
  365.         } catch (Exception e) {
  366.             // TODO Auto-generated catch block
  367.             e.printStackTrace();
  368.         }
  369.     }

  371.     private void getHtml(Task task) throws Exception {
  372.         StringBuilder sb = new StringBuilder(2048);
  373.         InetAddress addr = InetAddress.getByName(task.getHost());

  375.         // 建立一個Socket

  377.         Socket socket = new Socket(addr, task.getPort());

  379.         // 傳送命令,無非就是在Socket傳送流的基礎上加多一些握手資訊，詳情請了解HTTP協議
  380.         BufferedWriter wr = new BufferedWriter(new OutputStreamWriter(socket.getOutputStream(), "UTF-8"));
  381.         wr.write("GET " + task.getCurrentPath() + " HTTP/1.0\r\n");
  382.         wr.write("HOST:" + task.getHost() + "\r\n");
  383.         wr.write("Accept:*/*\r\n");
  384.         wr.write("\r\n");
  385.         wr.flush();

  387.         // 接收Socket返回的結果,並列印出來
  388.         BufferedReader rd = new BufferedReader(new InputStreamReader(socket.getInputStream()));
  389.         String line;
  390.         while ((line = rd.readLine()) != null) {
  391.             sb.append(line);
  392.         }
  393.         wr.close();
  394.         rd.close();
  395.         task.setContent(sb.toString());
  396.         socket.close();
  397.     }

  399.     @Override
  400.     public void run() {
  401.         while (true) {
  402.             this.handler();
  403.             COUNT.addAndGet(1);
  404.         }
  405.     }
  406. }


  409. class PersistenceHandler implements Runnable {
  410.     static {
  411.         try {
  412.             Class.forName("oracle.jdbc.OracleDriver");
  413.         } catch (ClassNotFoundException e) {
  414.             // TODO Auto-generated catch block
  415.             e.printStackTrace();
  416.         }
  417.     }

  419.     public static int GETCOUNT() {
  420.         return COUNT.get();
  421.     }

  423.     private static final AtomicInteger COUNT = new AtomicInteger();
  424.     private BlockingQueue<Task> persistencelist;

  426.     public PersistenceHandler(BlockingQueue<Task> persistencelist) {
  427.         this.persistencelist = persistencelist;
  428.         try {
  429.             conn = DriverManager.getConnection("jdbc:oracle:thin:127.0.0.1:1521:orcl", "edmond", "edmond");
  430.             ps = conn
  431.                     .prepareStatement("insert into probe(id,host,path,state,tasktime,type) values(seq_probe_id.nextval,?,?,?,?,?)");
  432.         } catch (SQLException e) {
  433.             // TODO Auto-generated catch block
  434.             e.printStackTrace();
  435.         }
  436.     }

  438.     private Connection conn;
  439.     private PreparedStatement ps;

  441.     @Override
  442.     public void run() {
  443.         while (true) {
  444.             this.handler();
  445.             COUNT.addAndGet(1);
  446.         }
  447.     }

  449.     private void handler() {
  450.         try {
  451.             Task task = persistencelist.take();
  452.             ps.setString(1, task.getHost());
  453.             ps.setString(2, task.getCurrentPath());
  454.             ps.setInt(3, task.getState());
  455.             ps.setLong(4, task.getTaskTime());
  456.             ps.setString(5, task.getType());

  458.             ps.executeUpdate();
  459.             conn.commit();
  460.         } catch (InterruptedException e) {
  461.             e.printStackTrace();
  462.         } catch (SQLException e) {
  463.             e.printStackTrace();
  464.         }
  465.     }
  466. }

ParseHandler 使用了一個職責鏈模式,
TwoLevel 處理../../開頭的連線(../../sucai/sucai.htm)
OneLevel 處理../開頭的連線(../sucai/sucai.htm)
FullPath 處理絕對路徑的連線(http://域名/sucai/sucai.htm)
Root 處理/開頭的連線(/sucai/sucai.htm)
Default 處理常規的連線(sucai.htm)


ParseHandler FullPath 過濾需要一個白名單.
這樣可以使程式在固定的域名爬行

ParseHandler parseTaskState 解析狀態碼 可能需要根據實際情況進行調整
比如網頁404,伺服器可能會返回一個錯誤頁，而不是通常的HTTP狀態碼。

第一版僅僅實現了功能,錯誤處理不完整,
所以僅僅在定製的域名下生效,其實並不通用,後續會逐步完善.

來自 “ ITPUB部落格 ” ，連結：http://blog.itpub.net/29254281/viewspace-1344706/，如需轉載，請註明出處，否則將追究法律責任。

相關文章