網頁主動探測工具-使用Reactor模式
接前文
http://blog.itpub.net/29254281/viewspace-1344706/
http://blog.itpub.net/29254281/viewspace-1347985/
http://blog.itpub.net/29254281/viewspace-2134876/
之前的程式碼被大神懟了..被怒批 雜亂無章,誤人子弟
我估計主要是因為物件導向的程度不夠.過程化太明顯了
在網上找了一個Reactor模式的例子,又改了改自己的程式.
因為Oracle太笨重了,這回乾脆換了MySQL好了
改寫之後的程式,在我的電腦上,使用 2執行緒,最大500連線的配置,效能最好。
http://blog.itpub.net/29254281/viewspace-1344706/
http://blog.itpub.net/29254281/viewspace-1347985/
http://blog.itpub.net/29254281/viewspace-2134876/
之前的程式碼被大神懟了..被怒批 雜亂無章,誤人子弟
我估計主要是因為物件導向的程度不夠.過程化太明顯了
在網上找了一個Reactor模式的例子,又改了改自己的程式.
因為Oracle太笨重了,這回乾脆換了MySQL好了
改寫之後的程式,在我的電腦上,使用 2執行緒,最大500連線的配置,效能最好。
- import java.io.IOException;
- import java.net.InetSocketAddress;
- import java.net.SocketAddress;
- import java.nio.ByteBuffer;
- import java.nio.channels.SelectionKey;
- import java.nio.channels.Selector;
- import java.nio.channels.SocketChannel;
- import java.nio.charset.Charset;
- import java.sql.Connection;
- import java.sql.DriverManager;
- import java.sql.PreparedStatement;
- import java.sql.SQLException;
- import java.sql.Timestamp;
- import java.util.ArrayList;
- import java.util.HashSet;
- import java.util.Iterator;
- import java.util.List;
- import java.util.Set;
- import java.util.concurrent.BlockingQueue;
- import java.util.concurrent.LinkedBlockingQueue;
- import java.util.concurrent.atomic.AtomicInteger;
- import java.util.regex.Matcher;
- import java.util.regex.Pattern;
- class Reactor implements Runnable {
- public static int GETCOUNT() {
- return COUNT.get();
- }
- public static int getQueueSize() {
- return QUEUE.size();
- }
- private static final AtomicInteger COUNT = new AtomicInteger();
- private static final AtomicInteger TASKCOUNT = new AtomicInteger();
- public int startTask() {
- return TASKCOUNT.incrementAndGet();
- }
- public int finishTask() {
- return TASKCOUNT.decrementAndGet();
- }
- public int incrementAndGet() {
- return COUNT.incrementAndGet();
- }
- public final Selector selector;
- private static BlockingQueue<Task> QUEUE = new LinkedBlockingQueue<Task>();
- public void addTask(Task task) {
- try {
- QUEUE.put(task);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- }
- public Reactor() throws IOException {
- selector = Selector.open();
- }
- @Override
- public void run() {
- try {
- while (!Thread.interrupted()) {
- int maxClient = 500;
- Task task = null;
- if (TASKCOUNT.get() < maxClient) {
- while ((task = (Task) QUEUE.poll()) != null) {
- new Connector(this, task).run();
- if (TASKCOUNT.get() > maxClient) {
- break;
- }
- }
- }
- selector.select();
- Set<SelectionKey> selectionKeys = selector.selectedKeys();
- Iterator<SelectionKey> it = selectionKeys.iterator();
- // Selector如果發現channel有OP_ACCEPT或READ事件發生,下列遍歷就會進行。
- while (it.hasNext()) {
- // 來一個事件 第一次觸發一個accepter執行緒
- // 以後觸發SocketReadHandler
- SelectionKey selectionKey = it.next();
- dispatch(selectionKey);
- }
- selectionKeys.clear();
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- /**
- * 執行Acceptor或SocketReadHandler
- *
- * @param key
- */
- void dispatch(SelectionKey key) {
- Runnable r = (Runnable) (key.attachment());
- if (r != null) {
- r.run();
- }
- }
- }
- class Connector implements Runnable {
- private Reactor reactor;
- private Task task;
- public Connector(Reactor reactor, Task task) {
- this.reactor = reactor;
- this.task = task;
- }
- @Override
- public void run() {
- try {
- reactor.startTask();
- task.setStarttime(System.currentTimeMillis());
- SocketAddress addr = new InetSocketAddress(task.getHost(), 80);
- SocketChannel socketChannel = SocketChannel.open();
- socketChannel.configureBlocking(false);
- socketChannel.connect(addr);
- BaseHandler base = new BaseHandler();
- base.setTask(task);
- base.setSelector(reactor.selector);
- base.setSocketChannel(socketChannel);
- base.setReactor(reactor);
- if (socketChannel != null)// 呼叫Handler來處理channel
- socketChannel.register(reactor.selector, SelectionKey.OP_CONNECT, new SocketWriteHandler(base));
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- class BaseHandler {
- private Selector selector;
- private SocketChannel socketChannel;
- private Task task;
- private ByteBuffer byteBuffer = ByteBuffer.allocate(2400);
- private Reactor reactor;
- public Reactor getReactor() {
- return reactor;
- }
- public void setReactor(Reactor reactor) {
- this.reactor = reactor;
- }
- public Selector getSelector() {
- return selector;
- }
- public void setSelector(Selector selector) {
- this.selector = selector;
- }
- public SocketChannel getSocketChannel() {
- return socketChannel;
- }
- public void setSocketChannel(SocketChannel socketChannel) {
- this.socketChannel = socketChannel;
- }
- public Task getTask() {
- return task;
- }
- public void setTask(Task task) {
- this.task = task;
- }
- public ByteBuffer getByteBuffer() {
- return byteBuffer;
- }
- }
- class SocketWriteHandler implements Runnable {
- BaseHandler baseHandler;
- public SocketWriteHandler(BaseHandler baseHandler) {
- this.baseHandler = baseHandler;
- ByteBuffer byteBuffer = baseHandler.getByteBuffer();
- Task task = baseHandler.getTask();
- try {
- byteBuffer.put(("GET " + task.getCurrentPath() + " HTTP/1.0\r\n").getBytes("utf8"));
- byteBuffer.put(("HOST:" + task.getHost() + "\r\n").getBytes("utf8"));
- byteBuffer.put(("Accept:*/*\r\n").getBytes("utf8"));
- byteBuffer.put(("\r\n").getBytes("utf8"));
- byteBuffer.flip();
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- @Override
- public void run() {
- try {
- while (!baseHandler.getSocketChannel().finishConnect()) {
- System.out.println("Waiting Connected");
- }
- baseHandler.getSocketChannel().write(baseHandler.getByteBuffer());
- if (baseHandler.getByteBuffer().hasRemaining()) {
- baseHandler.getByteBuffer().compact();
- baseHandler.getSocketChannel().register(baseHandler.getSelector(), SelectionKey.OP_WRITE, this);
- System.out.println("Continue Write");
- } else {
- baseHandler.getSocketChannel().register(baseHandler.getSelector(), SelectionKey.OP_READ,
- new SocketReadHandler(baseHandler));
- baseHandler.getByteBuffer().clear();
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- class SocketReadHandler implements Runnable {
- Charset charset = Charset.forName("utf8");
- Charset gbkcharset = Charset.forName("gbk");
- BaseHandler baseHandler;
- public SocketReadHandler(BaseHandler baseHandler) {
- this.baseHandler = baseHandler;
- }
- @Override
- public void run() {
- try {
- SocketChannel channel = baseHandler.getSocketChannel();
- ByteBuffer byteBuffer = baseHandler.getByteBuffer();
- Task task = baseHandler.getTask();
- int length;
- while ((length = channel.read(byteBuffer)) > 0) {
- byteBuffer.flip();
- task.getContent().append(charset.decode(charset.encode(gbkcharset.decode(byteBuffer))).toString());
- byteBuffer.compact();
- }
- if (length == -1) {
- channel.close();
- task.setEndtime(System.currentTimeMillis());
- baseHandler.getReactor().incrementAndGet();
- baseHandler.getReactor().finishTask();
- new ParseHandler(task, baseHandler.getReactor()).run();
- } else {
- baseHandler.getSocketChannel().register(baseHandler.getSelector(), SelectionKey.OP_READ, this);
- }
- } catch (IOException e) {
- e.printStackTrace();
- }
- }
- }
- public class Probe {
- public static void main(String[] args) throws IOException, InterruptedException {
- for (int i = 0; i <2; i++) {
- Reactor reactor = new Reactor();
- reactor.addTask(new Task("news.163.com", 80, "/index.html"));
- new Thread(reactor, "ReactorThread_" + i).start();
- }
- long start = System.currentTimeMillis();
- while (true) {
- Thread.sleep(1000);
- long end = System.currentTimeMillis();
- float interval = ((end - start) / 1000);
- int connectTotal = Reactor.GETCOUNT();
- int persistenceTotal = PersistenceHandler.GETCOUNT();
- int connectps = Math.round(connectTotal / interval);
- int persistenceps = Math.round(persistenceTotal / interval);
- System.out.print("\r連線總數:" + connectTotal + " \t每秒連線:" + connectps + "\t連線佇列剩餘:" + Reactor.getQueueSize()
- + " \t持久化總數:" + persistenceTotal + " \t每秒持久化:" + persistenceps + "\t持久化佇列剩餘:"
- + PersistenceHandler.getInstance().getSize());
- }
- }
- }
- class Task {
- private String host;
- private int port;
- private String currentPath;
- private long starttime;
- private long endtime;
- private String type;
- private StringBuilder content = new StringBuilder(2400);
- private int state;
- private boolean isValid = true;
- public Task() {
- }
- public Task(String host, int port, String path) {
- init(host, port, path);
- }
- public void init(String host, int port, String path) {
- this.setCurrentPath(path);
- this.host = host;
- this.port = port;
- }
- public long getStarttime() {
- return starttime;
- }
- public void setStarttime(long starttime) {
- this.starttime = starttime;
- }
- public long getEndtime() {
- return endtime;
- }
- public void setEndtime(long endtime) {
- this.endtime = endtime;
- }
- public boolean isValid() {
- return isValid;
- }
- public void setValid(boolean isValid) {
- this.isValid = isValid;
- }
- public int getState() {
- return state;
- }
- public void setState(int state) {
- this.state = state;
- }
- public String getCurrentPath() {
- return currentPath;
- }
- public void setCurrentPath(String currentPath) {
- this.currentPath = currentPath;
- int i = 0;
- if (currentPath.indexOf("?") != -1) {
- i = currentPath.indexOf("?");
- } else {
- if (currentPath.indexOf("#") != -1) {
- i = currentPath.indexOf("#");
- } else {
- i = currentPath.length();
- }
- }
- this.type = currentPath.substring(currentPath.indexOf(".") + 1, i);
- }
- public long getTaskTime() {
- return getEndtime() - getStarttime();
- }
- public String getType() {
- return type;
- }
- public void setType(String type) {
- this.type = type;
- }
- public String getHost() {
- return host;
- }
- public int getPort() {
- return port;
- }
- public StringBuilder getContent() {
- return content;
- }
- public void setContent(StringBuilder content) {
- this.content = content;
- }
- }
- class ParseHandler implements Runnable {
- private static final Set SET = new HashSet();
- PersistenceHandler persistencehandler = PersistenceHandler.getInstance();
- List domainlist = new ArrayList();
- Task task;
- private interface Filter {
- void doFilter(Task fatherTask, Task newTask, String path, Filter chain);
- }
- private class FilterChain implements Filter {
- private List list = new ArrayList();
- {
- addFilter(new TwoLevel());
- addFilter(new OneLevel());
- addFilter(new FullPath());
- addFilter(new Root());
- addFilter(new Default());
- }
- private void addFilter(Filter filter) {
- list.add(filter);
- }
- private Iterator it = list.iterator();
- @Override
- public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
- if (it.hasNext()) {
- ((Filter) it.next()).doFilter(fatherTask, newTask, path, chain);
- }
- }
- }
- private class TwoLevel implements Filter {
- @Override
- public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
- if (path.startsWith("../../")) {
- String prefix = getPrefix(fatherTask.getCurrentPath(), 3);
- newTask.init(fatherTask.getHost(), fatherTask.getPort(), path.replace("../../", prefix));
- } else {
- chain.doFilter(fatherTask, newTask, path, chain);
- }
- }
- }
- private class OneLevel implements Filter {
- @Override
- public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
- if (path.startsWith("../")) {
- String prefix = getPrefix(fatherTask.getCurrentPath(), 2);
- newTask.init(fatherTask.getHost(), fatherTask.getPort(), path.replace("../", prefix));
- } else {
- chain.doFilter(fatherTask, newTask, path, chain);
- }
- }
- }
- private class FullPath implements Filter {
- @Override
- public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
- if (path.startsWith("http://")) {
- Iterator it = domainlist.iterator();
- boolean flag = false;
- while (it.hasNext()) {
- String domain = (String) it.next();
- if (path.startsWith("http://" + domain + "/")) {
- newTask.init(domain, fatherTask.getPort(), path.replace("http://" + domain + "/", "/"));
- flag = true;
- break;
- }
- }
- if (!flag) {
- newTask.setValid(false);
- }
- } else {
- chain.doFilter(fatherTask, newTask, path, chain);
- }
- }
- }
- private class Root implements Filter {
- @Override
- public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
- if (path.startsWith("/")) {
- newTask.init(fatherTask.getHost(), fatherTask.getPort(), path);
- } else {
- chain.doFilter(fatherTask, newTask, path, chain);
- }
- }
- }
- private class Default implements Filter {
- @Override
- public void doFilter(Task fatherTask, Task newTask, String path, Filter chain) {
- if (path.contains(":")) {
- newTask.setValid(false);
- return;
- }
- String prefix = getPrefix(fatherTask.getCurrentPath(), 1);
- newTask.init(fatherTask.getHost(), fatherTask.getPort(), prefix + "/" + path);
- }
- }
- public ParseHandler(Task task, Reactor reactor) {
- this.task = task;
- this.reactor = reactor;
- // 增加白名單
- this.domainlist.add("news.163.com");
- }
- private Reactor reactor;
- private Pattern pattern = Pattern.compile("\"[^\"]+\\.htm[^\"]*\"");
- private void parseTaskState(Task task) {
- if (task.getContent().toString().startsWith("HTTP/1.1")) {
- task.setState(Integer.parseInt(task.getContent().substring(9, 12)));
- } else {
- task.setState(Integer.parseInt(task.getContent().substring(9, 12)));
- }
- }
- /**
- * @param fatherTask
- * @param path
- * @throws Exception
- */
- private void createNewTask(Task fatherTask, String path) throws Exception {
- Task newTask = new Task();
- FilterChain filterchain = new FilterChain();
- filterchain.doFilter(fatherTask, newTask, path, filterchain);
- if (newTask.isValid()) {
- synchronized (SET) {
- if (SET.contains(newTask.getHost() + newTask.getCurrentPath())) {
- return;
- }
- SET.add(newTask.getHost() + newTask.getCurrentPath());
- }
- reactor.addTask(newTask);
- }
- }
- private String getPrefix(String s, int count) {
- String prefix = s;
- while (count > 0) {
- prefix = prefix.substring(0, prefix.lastIndexOf("/"));
- count--;
- }
- return "".equals(prefix) ? "/" : prefix;
- }
- @Override
- public void run() {
- try {
- parseTaskState(task);
- if (200 == task.getState()) {
- Matcher matcher = pattern.matcher(task.getContent());
- while (matcher.find()) {
- String path = matcher.group();
- if (!path.contains(" ") && !path.contains("\t") && !path.contains("(") && !path.contains(")")) {
- path = path.substring(1, path.length() - 1);
- createNewTask(task, path);
- }
- }
- }
- persistencehandler.addTask(task);
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }
- class PersistenceHandler implements Runnable {
- private static class SingletonHandler {
- private static PersistenceHandler obj = new PersistenceHandler();
- }
- public static PersistenceHandler getInstance() {
- return SingletonHandler.obj;
- }
- static {
- try {
- Class.forName("com.mysql.jdbc.Driver");
- } catch (ClassNotFoundException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- public static int GETCOUNT() {
- return COUNT.get();
- }
- private static final AtomicInteger COUNT = new AtomicInteger();
- private BlockingQueue persistencelist;
- public PersistenceHandler() {
- this.persistencelist = new LinkedBlockingQueue();
- new Thread(this, "PersistenceThread").start();
- }
- public void addTask(Task task) {
- try {
- this.persistencelist.put(task);
- } catch (InterruptedException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- }
- public int getSize() {
- return persistencelist.size();
- }
- private Connection conn;
- private PreparedStatement ps;
- @Override
- public void run() {
- try {
- conn = DriverManager.getConnection("jdbc:mysql://127.0.0.1:3306/mvbox", "xx", "xx");
- conn.setAutoCommit(false);
- ps = conn.prepareStatement(
- "insert into probe(host,path,state,tasktime,type,length,createtime) values(?,?,?,?,?,?,?)");
- } catch (SQLException e) {
- // TODO Auto-generated catch block
- e.printStackTrace();
- }
- while (true) {
- this.handler();
- COUNT.addAndGet(1);
- }
- }
- private void handler() {
- try {
- Task task = (Task) persistencelist.take();
- ps.setString(1, task.getHost());
- ps.setString(2, task.getCurrentPath());
- ps.setInt(3, task.getState());
- ps.setLong(4, task.getTaskTime());
- ps.setString(5, task.getType());
- ps.setInt(6, task.getContent().toString().length());
- ps.setTimestamp(7, new Timestamp(task.getEndtime()));
- ps.addBatch();
- if (GETCOUNT() % 500 == 0) {
- ps.executeBatch();
- conn.commit();
- }
- } catch (InterruptedException e) {
- e.printStackTrace();
- } catch (SQLException e) {
- e.printStackTrace();
- }
- }
- }
來自 “ ITPUB部落格 ” ,連結:http://blog.itpub.net/29254281/viewspace-2135131/,如需轉載,請註明出處,否則將追究法律責任。
相關文章
- 網頁主動探測工具網頁
- 網頁主動探測工具-增加Netty模式網頁Netty模式
- 網頁主動探測工具(修改Bug)網頁
- 網頁主動探測工具-NIO優化網頁優化
- Reactor 模式React模式
- Reactor模式React模式
- 網站robots.txt探測工具Parsero網站
- Reactor模式理解React模式
- Reactor模式理解(二)React模式
- 【Linux網路程式設計】Reactor模式與Proactor模式Linux程式設計React模式
- Web應用主動偵測工具SkipfishWeb
- 使用Python編寫一個滲透測試探測工具Python
- Kali路由策略探測工具————firewalk路由
- 核心探測工具systemtap簡介
- 使用 Python 和 Selenium 自動化網頁測試Python網頁
- Web網站敏感目錄/內容探測工具 – CansinaWeb網站
- FTP主動模式和被動模式FTP模式
- 使用 spring reactor 非同步事件驅動SpringReact非同步事件
- 多執行緒Reactor模式執行緒React模式
- SimpleRpc-網路事件響應Reactor設計模式RPC事件React設計模式
- 讓IE8自動使用相容模式顯示網頁的方法模式網頁
- 藍芽裝置探測工具blueranger藍芽Ranger
- zabbix的主動模式和被動模式模式
- zabbix 預設item採集使用被動模式 需要改為主動模式模式
- 網路防嗅探工具SniffJoke
- 網頁程式碼(主頁)(初始版):網頁
- Selenium自動化測試網頁網頁
- Chrome 76 將防止紐約時報等網站探測隱身模式Chrome網站模式
- Zabbix設定主動模式與被動模式模式
- 暴力探測藍芽裝置工具redfang藍芽
- 網路資料嗅探工具HexInject
- zabbix-agent被動模式變為主動模式模式
- Zabbix——zabbix-agent被動模式變主動模式模式
- zabbix被動模式和主動模式的區別模式
- FTP主動模式和被動模式的區別FTP模式
- 軟體分享:網頁監測及IIS重啟工具網頁
- 設計模式 in real world - 反應器模式(Reactor Pattern)設計模式React
- 如何自動重新整理網頁?Auto Refresh Page網頁自動重新整理工具分享網頁