python爬蟲-抓取騰訊招聘資訊頁面

本爬蟲主要使用了requests、json、bs4(BeautifulSoup)等相關模組，不完善之處請大家不吝賜教！:)
出處：https://github.com/jingsupo/python-spider/blob/master/day04/04tencent_hr.py

# -*- coding:utf-8 -*-

import requests, json, time
from bs4 import BeautifulSoup


class tencent_hr(object):
    def __init__(self):
        self.base_url = "http://hr.tencent.com/position.php?"
        self.headers = {"User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64; Trident/7.0; rv:11.0) like Gecko"}
        self.item_list = []
        self.page = 0

    # 傳送請求
    def send_request(self, url, params={}):
        time.sleep(2)
        try:
            response = requests.get(url, params=params, headers=self.headers)
            return response.content
        except Exception as e:
            print e

    # 解析資料
    def parse_data(self, data):
        # 初始化
        bs = BeautifulSoup(data, `lxml`)

        # 獲取標籤-結果為列表
        data_list = bs.select(`.even, .odd`)

        # 將結果中的每一行資料提取出來
        for data in data_list:
            data_dict = {}
            data_dict[`work_name`] = data.select(`td a`)[0].get_text()
            data_dict[`work_type`] = data.select(`td`)[1].get_text()
            data_dict[`work_count`] = data.select(`td`)[2].get_text()
            data_dict[`work_place`] = data.select(`td`)[3].get_text()
            data_dict[`work_time`] = data.select(`td`)[4].get_text()

            # 將每條字典資料新增進列表
            self.item_list.append(data_dict)

        # 判斷是否是最後一頁，條件：是否有noactive值
        # 先找到下一頁的標籤
        next_label = bs.select(`#next`)
        # 根據標籤獲取屬性class的值-返回結果為列表
        judge = next_label[0].get(`class`)

        return judge

    # 寫入檔案
    def write_file(self):
        # 將列表轉換成字串
        data_str = json.dumps(self.item_list)

        with open(`04tencent_hr.json`, `w`) as f:
            f.write(data_str)

    # 排程執行
    def run(self):
        while True:
            # 拼接引數
            params = {
                "keywords": "python",
                "tid": "0",
                "lid": "2156",
                "start": self.page,
            }

            # 傳送請求
            data = self.send_request(self.base_url, params=params)

            # 解析資料
            judge = self.parse_data(data)

            self.page += 10
            print self.page

            # 如果到了最後一頁，出現noactive，跳出迴圈
            if judge:
                break

        self.write_file()


if __name__ == `__main__`:
    spider = tencent_hr()
    spider.run()