小白學 Python 爬蟲（25）：爬取股票資訊

def get_stock_list(stockListURL):
    r =requests.get(stockListURL, headers = headers)
    doc = PyQuery(r.text)
    list = []
    # 獲取所有 section 中 a 節點，並進行迭代
    for i in doc('.stockTable a').items():
        try:
            href = i.attr.href
            list.append(re.findall(r"\d{6}", href)[0])
        except:
            continue
    list = [item.lower() for item in list]  # 將爬取資訊轉換小寫
    return list

將上面的連結當做引數傳入，大家可以自己執行下看下結果，小編這裡就不貼結果了，有點長。。。

獲取詳情資料

詳情的資料看起來好像是在頁面上的，但是，實際上並不在，實際最終獲取資料的地方並不是頁面，而是一個資料介面。

http://qd.10jqka.com.cn/quote.php?cate=real&type=stock&callback=showStockDate&return=json&code=000001

至於是怎麼找出來，小編這次就不說，還是希望各位想學爬蟲的同學能自己動動手，去尋找一下，多找幾次，自然就摸到門路了。

現在資料介面有了，我們先看下返回的資料吧：

showStockDate({"info":{"000001":{"name":"\u5e73\u5b89\u94f6\u884c"}},"data":{"000001":{"10":"16.13","8":"16.14","9":"15.87","13":"78795234.00","19":"1262802470.00","7":"16.12","15":"40225508.00","14":"37528826.00","69":"17.73","70":"14.51","12":"5","17":"945400.00","264648":"0.010","199112":"0.062","1968584":"0.406","2034120":"9.939","1378761":"16.026","526792":"1.675","395720":"-948073.000","461256":"-39.763","3475914":"313014790000.000","1771976":"1.100","6":"16.12","11":""}}})

很明顯，這個結果並不是標準的 json 資料，但這個是 JSONP 返回的標準格式的資料，這裡我們先處理下頭尾，將它變成一個標準的 json 資料，再對照這頁面的資料進行解析，最後將分析好的值寫入資料庫中。

def getStockInfo(list, stockInfoURL):
    count = 0
    for stock in list:
        try:
            url = stockInfoURL   stock
            r = requests.get(url, headers=headers)
            # 將獲取到的資料封裝進字典
            dict1 = json.loads(r.text[14: int(len(r.text)) - 1])
            print(dict1)

            # 獲取字典中的資料構建寫入資料模版
            insert_data = {
                "code": stock,
                "name": dict1['info'][stock]['name'],
                "jinkai": dict1['data'][stock]['7'],
                "chengjiaoliang": dict1['data'][stock]['13'],
                "zhenfu": dict1['data'][stock]['526792'],
                "zuigao": dict1['data'][stock]['8'],
                "chengjiaoe": dict1['data'][stock]['19'],
                "huanshou": dict1['data'][stock]['1968584'],
                "zuidi": dict1['data'][stock]['9'],
                "zuoshou": dict1['data'][stock]['6'],
                "liutongshizhi": dict1['data'][stock]['3475914']
            }
            cursor.execute(sql_insert, insert_data)
            conn.commit()
            print(stock, '：寫入完成')
        except:
            print('寫入異常')
            # 遇到錯誤繼續迴圈
            continue

這裡我們加入異常處理，因為本次爬取的資料有些多，很有可能由於某些原因丟擲異常，我們當然不希望有異常的時候中斷資料抓取，所以這裡新增異常處理繼續抓取資料。

完整程式碼

我們將程式碼稍作封裝，完成本次的實戰。

import requests
import re
import json
from pyquery import PyQuery
import pymysql

# 資料庫連線
def connect():
    conn = pymysql.connect(host='localhost',
                           port=3306,
                           user='root',
                           password='password',
                           database='test',
                           charset='utf8mb4')

    # 獲取操作遊標
    cursor = conn.cursor()
    return {"conn": conn, "cursor": cursor}

connection = connect()
conn, cursor = connection['conn'], connection['cursor']

sql_insert = "insert into stock(code, name, jinkai, chengjiaoliang, zhenfu, zuigao, chengjiaoe, huanshou, zuidi, zuoshou, liutongshizhi, create_date) values (%(code)s, %(name)s, %(jinkai)s, %(chengjiaoliang)s, %(zhenfu)s, %(zuigao)s, %(chengjiaoe)s, %(huanshou)s, %(zuidi)s, %(zuoshou)s, %(liutongshizhi)s, now())"

headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36'
}

def get_stock_list(stockListURL):
    r =requests.get(stockListURL, headers = headers)
    doc = PyQuery(r.text)
    list = []
    # 獲取所有 section 中 a 節點，並進行迭代
    for i in doc('.stockTable a').items():
        try:
            href = i.attr.href
            list.append(re.findall(r"\d{6}", href)[0])
        except:
            continue
    list = [item.lower() for item in list]  # 將爬取資訊轉換小寫
    return list


def getStockInfo(list, stockInfoURL):
    count = 0
    for stock in list:
        try:
            url = stockInfoURL   stock
            r = requests.get(url, headers=headers)
            # 將獲取到的資料封裝進字典
            dict1 = json.loads(r.text[14: int(len(r.text)) - 1])
            print(dict1)

            # 獲取字典中的資料構建寫入資料模版
            insert_data = {
                "code": stock,
                "name": dict1['info'][stock]['name'],
                "jinkai": dict1['data'][stock]['7'],
                "chengjiaoliang": dict1['data'][stock]['13'],
                "zhenfu": dict1['data'][stock]['526792'],
                "zuigao": dict1['data'][stock]['8'],
                "chengjiaoe": dict1['data'][stock]['19'],
                "huanshou": dict1['data'][stock]['1968584'],
                "zuidi": dict1['data'][stock]['9'],
                "zuoshou": dict1['data'][stock]['6'],
                "liutongshizhi": dict1['data'][stock]['3475914']
            }
            cursor.execute(sql_insert, insert_data)
            conn.commit()
            print(stock, '：寫入完成')
        except:
            print('寫入異常')
            # 遇到錯誤繼續迴圈
            continue
def main():
    stock_list_url = 'https://hq.gucheng.com/gpdmylb.html'
    stock_info_url = 'http://qd.10jqka.com.cn/quote.php?cate=real&type=stock&callback=showStockDate&return=json&code='
    list = get_stock_list(stock_list_url)
    # list = ['601766']
    getStockInfo(list, stock_info_url)

if __name__ == '__main__':
    main()