Python3爬蟲實戰（urllib模組）

import urllib.request
import os
import re
import time

def url_open(url):
    # 建立一個 Request物件 req
    req = urllib.request.Request(url)

    # 通過 add_header( )方法新增請求頭，防止基本的網站反爬策略
    req.add_header('User-Agent', "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/\
                    537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36")

    # 將獲取的網頁資訊通過read()方法讀取出來
    response = urllib.request.urlopen(req).read()
    return response

# 另一種方法獲取網頁
'''
def url_open(url):
    req = urllib.request.Request(url)
    header = ('User-Agent', "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/\
                    537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36"
    )
    # 建立opner物件
    opener = urllib.request.build_opener()

    # 給該物件新增請求頭
    opener.addheaders = [header]

    # 用open方法獲取網頁並讀取
    response = opener.open(url).read()
    return response
'''

def find_imgs(url):
    # 將網頁內容進行解碼，網頁編碼是GBK，就換成gbk
    html = url_open(url).decode('utf-8')

    # 使用正規表示式獲取目標資料
    p = r'<img src="([^"]+\.jpg)"'
    img_addrs = re.findall(p, html)

    return img_addrs

def download_mm(folder='OOXX'):
    os.mkdir(folder)
    os.chdir(folder)

    page_num = 1  # 設定為從第一頁開始爬取，可以自己改
    x = 0  # 自命名圖片
    img_addrs = []  # 防止圖片重複

    # 只爬取前兩頁的圖片，可改，同時給圖片重新命名
    while page_num <= 2:
        page_url = url + 'a/more_' + str(page_num) + '.html'
        addrs = find_imgs(page_url)
        print(len(addrs))
        # img_addrs = []
        for i in addrs:
            if i in img_addrs:
                continue
            else:
                img_addrs.append(i)
        print(len(img_addrs))
        for each in img_addrs:
            print(each)
        page_num += 1
        time.sleep()
        # x = (len(2img_addrs)+1)*(page_num-1)
    for each in img_addrs:
        filename = str(x) + '.' + each.split('.')[-1]
        x += 1
        with open(filename, 'wb') as f:
            img = url_open(each)
            f.write(img)
        # page_num += 1

if __name__ == '__main__':
    url = 'http://www.meizitu.com/'
    download_mm()

# -*-coding:utf-8 -*-

import urllib.request
import re
import os

def open_url(url):
    req = urllib.request.Request(url)
    req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64)AppleWebKit/537.36 (KHTML, like Gecko)\
     Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0")
    response = urllib.request.urlopen(req).read()

    return response

def find_img(url):
    html = open_url(url).decode('utf-8')
    p = r'<img class="BDE_Image" src="([^"]+\.jpg)"'
    img_addrs = re.findall(p, html)

    for each in img_addrs:
        print(each)
    for each in img_addrs:
        file = each.split("/")[-1]
        with open(file, "wb") as f:
            img = open_url(each)
            f.write(img)

def get_img():
    os.mkdir("TieBaTu")
    os.chdir("TieBaTu")
    find_img(url)

if __name__ == "__main__":
    url = 'https://tieba.baidu.com/p/5085123197'
    get_img()

urllib.parse.urlencode(query, doseq=False, safe='', encoding=None, errors=None)

urlencode（）主要作用就是將url附上要提交的資料。 

通過下面這個爬取有道詞典的例子大家就會清楚urllib.parse.urlencode()的用處了。

import urllib.request
import  urllib.parse
import json

while True:
    content = input("請輸入：")
    if content == "":
        print("歡迎下次使用")
        break
    data = {
         'i': content
        , 'from': 'AUTO'
        , 'to': 'AUTO'
        , 'smartresult': 'dict'
        , 'client': 'fanyideskweb'
        , 'salt': '1514345577426'
        , 'sign': '8a12c3bae1619e0d60247aa90a4d945e'
        , 'doctype': 'json'
        , 'version': '2.1'
        , 'keyfrom': 'fanyi.web'
        , 'action': 'FY_BY_REALTIME'
        , 'typoResult': 'false'
        }
    data = urllib.parse.urlencode(data).encode('utf-8')
    url = 'http://fanyi.youdao.com/translate?smartresult=dict&smartresult=rule&sessionFrom=null'
    req = urllib.request.Request(url,data)
    req.add_header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0")
    response = urllib.request.urlopen(req)
    html = response.read().decode('utf-8')
    
    # 因為翻譯後的資料是通過json格式返回的，所以要新增json模組進行解析
    target = json.loads(html)
    # print(html)
    print("翻譯結果：")
    print(target['translateResult'][0][0]['tgt'])
    print()