Web自動化之Headless Chrome編碼實戰

API 概覽 && 編碼Tips

文件地址

• github Chrome DevTools Protocol 協議本身的倉庫 有問題可以在這裡提issue
• 協議API文件的倉庫
• API 文件地址 API展示的地方，這個經常用

常用API

• Network 網路請求、Cookie、快取、證照等相關內容
• Page 頁面的載入、資源內容、彈層、截圖、列印等相關內容
• DOM 文件DOM的獲取、修改、刪除、查詢等相關內容
• Runtime JavaScript程式碼的執行，這裡面我們可以搞事情~~

編碼Tips

• 我們這裡不會直接呼叫Websocket相關的內容來呼叫chrome的除錯命令，而是用 這個封裝的庫來做，它是基於Promise風格的
• 每一個功能塊成為一個單獨的domain,像Network，Page，DOM等都是不同的domain
• 幾乎每一個個頭大的domain都有enable方法，需要先呼叫這個方法啟用之後再使用
• 各個domain的介面方法引數都是第一個物件或者說一個Map，不用考慮引數的位置了
• 各個domain的介面返回值也是一個物件，取對應的key就行
• 引數值和返回值經常是meta資訊，經常是各種物件的id資訊，而不是具體的物件內容（這裡可能需要切一下風格）

編碼例項

首先做一個簡單的封裝，準備API的執行環境，具體可參考前一篇關於工具庫的。

const chromeLauncher = require('chrome-launcher'); const chromeRemoteInterface = require('chrome-remote-interface'); const prepareAPI = (config = {}) => { const {host = 'localhost', port = 9222, autoSelectChrome = true, headless = true} = config; const wrapperEntry = chromeLauncher.launch({
        host,
        port,
        autoSelectChrome,
        additionalFlags: [ '--disable-gpu',
            headless ? '--headless' : '' ]
    }).then(chromeInstance => { const remoteInterface = chromeRemoteInterface(config).then(chromeAPI => chromeAPI).catch(err => { throw err;
        }); return Promise.all([chromeInstance, remoteInterface])
    }).catch(err => { throw err
    }); return wrapperEntry
}; 

開啟百度，獲取頁面效能資料，參考

const wrapper = require('the-wrapper-module'); const performanceParser = (perforceTiming) => { let timingGather = {};
    perforceTiming = perforceTiming || {};
    timingGather.redirect = perforceTiming.redirectEnd - perforceTiming.redirectEnd-perforceTiming.redirectStart;
    timingGather.dns = perforceTiming.domainLookupEnd - perforceTiming.domainLookupStart;
    timingGather.tcp = perforceTiming.connectEnd - perforceTiming.connectStart;
    timingGather.request = perforceTiming.responseStart - perforceTiming.requestStart;
    timingGather.response = perforceTiming.responseEnd - perforceTiming.responseStart;
    timingGather.domReady = perforceTiming.domContentLoadedEventStart - perforceTiming.navigationStart;
    timingGather.load = perforceTiming.loadEventStart - perforceTiming.navigationStart; return timingGather;
}; const showPerformanceInfo = (performanceInfo) => {
    performanceInfo = performanceInfo || {}; console.log(`頁面重定向耗時:${performanceInfo.redirect}`); console.log(`DNS查詢耗時:${performanceInfo.dns}`); console.log(`TCP連線耗時:${performanceInfo.tcp}`); console.log(`請求傳送耗時:${performanceInfo.request}`); console.log(`響應接收耗時:${performanceInfo.response}`); console.log(`DOMReady耗時:${performanceInfo.domReady}`); console.log(`頁面載入耗時:${performanceInfo.load}`);
};

wrapper.prepareAPI().then(([chromeInstance, remoteInterface]) => { const {Runtime,Page} = remoteInterface;

    Page.loadEventFired(() => {
        Runtime.evaluate({
            expression:'window.performance.timing.toJSON()',
            returnByValue:true //不加這個引數，拿到的是一個物件的meta資訊,還需要getProperties }).then((resultObj) => { let {result,exceptionDetails} = resultObj; if(!exceptionDetails){
                showPerformanceInfo(performanceParser(result.value))
            }else{ throw exceptionDetails;
            }
        });
    });

    Page.enable().then(() => {
        Page.navigate({
            url:'' })
    });
}); 

開啟百度 搜尋Web自動化 headless chrome，並爬取首屏結果連結

const wrapper = require('the-wrapper-module'); //有this的地方寫成箭頭函式要注意，這裡會有問題 const buttonClick = function () { this.click();
}; const setInputValue = () => { var input = document.getElementById('kw');
    input.value = 'Web自動化 headless chrome';
}; const parseSearchResult = () => { let resultList = []; const linkBlocks = document.querySelectorAll('div.result.c-container'); for (let block of Array.from(linkBlocks)) { let targetObj = block.querySelector('h3');
        resultList.push({
            title: targetObj.textContent,
            link: targetObj.querySelector('a').getAttribute('href')
        });
    } return resultList;
};

wrapper.prepareAPI({ // headless: false  //加上這行程式碼可以檢視瀏覽器的變化 }).then(([chromeInstance, remoteInterface]) => { const {Runtime, DOM, Page, Network} = remoteInterface; let framePointer;
    Promise.all([Page.enable(), Network.enable(), DOM.enable(),Page.setAutoAttachToCreatedPages({autoAttach:true})]).then(() => {
        Page.domContentEventFired(() => { console.log('Page.domContentEventFired')
            Runtime.evaluate({
                expression:`window.location.href`,
                returnByValue:true }).then(result => { console.log(result)
            })
        });
        Page.frameNavigated(() => { console.log('Page.frameNavigated')
            Runtime.evaluate({
                expression:`window.location.href`,
                returnByValue:true }).then(result => { console.log(result)
            })
        })
        Page.loadEventFired(() => { console.log('Page.loadEventFired')
            Runtime.evaluate({
                expression:`window.location.href`,
                returnByValue:true }).then(result => { console.log(result)
            })
            DOM.getDocument().then(({root}) => { //百度首頁表單 DOM.querySelector({
                    nodeId: root.nodeId,
                    selector: '#form' }).then(({nodeId}) => {
                    Promise.all([ //找到 搜尋框填入值 DOM.querySelector({
                            nodeId: nodeId,
                            selector: '#kw' }).then((inputNode) => {

                            Runtime.evaluate({ // 兩種寫法 // expression:'document.getElementById("kw").value = "Web自動化 headless chrome"', expression: `(${setInputValue})()`
                            }); //這段程式碼不起作用 日狗 // DOM.setNodeValue({ //     nodeId:inputNode.nodeId, //     value:'Web自動化 headless chrome' // }); //上面的程式碼需求要這麼寫 // DOM.setAttributeValue({ //     nodeId:inputNode.nodeId, //     name:'value', //     value:'headless chrome' // }); }) //找到 提交按鈕setInputValue , DOM.querySelector({
                            nodeId,
                            selector: '#su' })
                    ]).then(([inputNode, buttonNode]) => {

                        Runtime.evaluate({
                            expression: 'document.getElementById("kw").value',
                        }).then(({result}) => { console.log(result)
                        }); return DOM.resolveNode({
                            nodeId: buttonNode.nodeId
                        }).then(({object}) => { const {objectId} = object; return Runtime.callFunctionOn({
                                objectId,
                                functionDeclaration: `${buttonClick}`
                            })
                        });
                    }).then(() => {
                        setTimeout(() => {
                            Runtime.evaluate({
                                expression: `(${parseSearchResult})()`,
                                returnByValue: true }).then(({result}) => { console.log(result.value) //百度的URL有加密，需要再請求一次拿到真實URL })
                        },3e3)
                    });
                })

            });
        });
        Page.navigate({
            url: '' }).then((frameObj) => {
            framePointer = frameObj
        });
    })

});