基於bs4+requests的python爬蟲偽裝

要匯入fake-useragent庫，需要先用pip安裝，安裝命令：pip install fake-useragent
params是爬蟲偽裝的引數，資料型別為字典dict，裡面有2個鍵值對，2個鍵：headers、proxies。
headers的資料型別是字典，裡面有1個鍵值對，鍵User-Agent對應的值資料型別為字串，User-Agent中文翻譯是使用者代理。
proxies的資料型別是字典，裡面有1個鍵值對，鍵http對應的值資料型別為字串，是代理伺服器的url。
匿名ip主要是從66ip.cn網站獲取。

import requests
from bs4 import BeautifulSoup as bs
from fake_useragent import UserAgent
import random

def getSoup(url,encoding="utf-8",**params):
    print(params)
    reponse = requests.get(url,**params)
    reponse.encoding = encoding
    soup = bs(reponse.text,`lxml`)
    return soup

def cssFind(movie,cssSelector,nth=1):
    if len(movie.select(cssSelector)) >= nth:
        return movie.select(cssSelector)[nth-1].text.strip()
    else:
        return ``

def getProxyList():
    proxies_url_before = "http://www.66ip.cn/areaindex_2/{}.html"
    proxies_url = proxies_url_before.format(random.randint(1,10))
    soup = getSoup(proxies_url)
    item_list = soup.select("table tr")[2:]
    proxies_list = []
    for item in item_list:
        ipAddress = cssFind(item, "td")
        ipPort = cssFind(item, "td", 2)
        proxies_list.append("http://{}:{}".format(ipAddress, ipPort))
    return proxies_list

def getParams():
    ua = UserAgent()
    ip_list = getProxyList()
    params = dict(
        headers = {`User-Agent`: ua.random},
        proxies = {`http`: random.choice(ip_list)}
    )
    return params

if __name__ == "__main__":   
    params = getParams()
    soup = getSoup("https://movie.douban.com/top250?start=50",**params)