HtmlUnit 爬蟲簡單案例——模擬登陸CSDN

最近要弄一個爬蟲程式，想著先來個簡單的模擬登陸， 在權衡JxBrowser和HtmlUnit 兩種技術，  JxBowser有介面呈現效果，但是對於某些js跳轉之後的效果獲取比較繁瑣。

隨後考慮用HtmlUnit， 想著借用我們們CSND的登陸練練手。誰知道CSDN的登陸，js載入時間超長，不設定長一點的載入時間，按鈕提交根本沒效果，js沒生效。 具體看程式碼註釋吧。 奉勸做爬蟲的同志們，千萬別用CSDN登陸練手，坑死我了。。。

maven配置如下：

	<dependencies>
		<!-- https://mvnrepository.com/artifact/net.sourceforge.htmlunit/htmlunit -->
		<dependency>
			<groupId>net.sourceforge.htmlunit</groupId>
			<artifactId>htmlunit</artifactId>
			<version>2.18</version>
		</dependency>
		<!-- https://mvnrepository.com/artifact/org.jsoup/jsoup -->
		<dependency>
			<groupId>org.jsoup</groupId>
			<artifactId>jsoup</artifactId>
			<version>1.9.2</version>
		</dependency>
	</dependencies>

程式碼如下：

/*
 * Copyright (c) 2017 Create By Shijing All Rights Reserved.
 */
package com.test;

import java.io.IOException;
import java.net.MalformedURLException;
import java.util.HashMap;
import java.util.Map;
import java.util.Set;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.NicelyResynchronizingAjaxController;
import com.gargoylesoftware.htmlunit.SilentCssErrorHandler;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlButtonInput;
import com.gargoylesoftware.htmlunit.html.HtmlForm;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.html.HtmlPasswordInput;
import com.gargoylesoftware.htmlunit.html.HtmlTextInput;
import com.gargoylesoftware.htmlunit.util.Cookie;

public class SimulateLogin
{
    //訪問的目標網址（CSDN）
    private static String TARGET_URL = "https://passport.csdn.net/account/login?from=http://www.csdn.net";

    public static void main(String[] args) throws FailingHttpStatusCodeException, MalformedURLException, IOException
    {
        // 模擬一個瀏覽器
        WebClient webClient = new WebClient(BrowserVersion.CHROME);
        // 設定webClient的相關引數
        webClient.setCssErrorHandler(new SilentCssErrorHandler());  
        //設定ajax
        webClient.setAjaxController(new NicelyResynchronizingAjaxController());
        //設定支援js
        webClient.getOptions().setJavaScriptEnabled(true);
        //CSS渲染禁止
        webClient.getOptions().setCssEnabled(false);
        //超時時間
        webClient.getOptions().setTimeout(50000);
        //設定js丟擲異常:false
        webClient.getOptions().setThrowExceptionOnScriptError(false);
        //允許重定向
        webClient.getOptions().setRedirectEnabled(true);  
        //允許cookie
        webClient.getCookieManager().setCookiesEnabled(true);  
        // 模擬瀏覽器開啟一個目標網址
        HtmlPage page = webClient.getPage(TARGET_URL);
        /**等待js載入完全，CSDN這點 特別坑，js載入時間超長！！！！！！！ 後人切記不要用CSDN模擬登陸！！！！！！！**/
        webClient.waitForBackgroundJavaScript(10000*3);
     // 根據form的名字獲取頁面表單，也可以通過索引來獲取：page.getForms().get(0)     
        HtmlForm form = (HtmlForm) page.getElementById("fm1");         
        HtmlTextInput username = (HtmlTextInput) form.getInputByName("username");  
        HtmlPasswordInput password = (HtmlPasswordInput) form.getInputByName("password");  
        username.setValueAttribute("********");  //使用者名稱
        password.setValueAttribute("********");  //密碼
        HtmlButtonInput button  = (HtmlButtonInput) page.getByXPath("//input[contains(@class, 'logging')]").get(0);
//        ScriptResult result = page.executeJavaScript("javascript:document.getElementsByClassName('logging')[0].click()");
//        HtmlPage retPage = (HtmlPage) result.getNewPage();
        HtmlPage retPage = button.click();
        // 等待JS驅動dom完成獲得還原後的網頁  
        webClient.waitForBackgroundJavaScript(1000);  
        //輸出跳轉網頁的地址
        System.out.println(retPage.getUrl().toString());   
        //輸出跳轉網頁的內容
        System.out.println(retPage.asXml());

        //獲取cookie  
        Set<Cookie> cookies = webClient.getCookieManager().getCookies();
        Map<String, String> responseCookies = new HashMap<String, String>();  
        for (Cookie c : cookies) {  
            responseCookies.put(c.getName(), c.getValue());  
            System.out.print(c.getName()+":"+c.getValue());  
        }  
        webClient.close();
        System.out.println("Success!");
    }
}

另外，CSDN的JS總是莫名其妙的報一堆錯，如果不想看，想忽略的話，在建立WebClient前加上如下程式碼：

        //設定日誌級別，原頁面js異常不列印
        LogFactory.getFactory().setAttribute("org.apache.commons.logging.Log",    "org.apache.commons.logging.impl.NoOpLog");  
        
        java.util.logging.Logger.getLogger("com.gargoylesoftware.htmlunit")  
            .setLevel(Level.OFF);  
  
        java.util.logging.Logger.getLogger("org.apache.commons.httpclient")  
            .setLevel(Level.OFF);