Python爬資料之全國中小學資訊

爬取網站：http://www.xuexiaodaquan.com/ 學校大全

技術路線： requests + BeautifulSoup

貌似這個網站反爬蟲還挺牛的，經常就返回自動跳入的139網站，隨意得換著IP試試

需要準備中國市名稱拼音存在EXCEL中，顯示是第一列：市民；第二列：拼音；到市級就可以。

需要挖掘哪些城市就放哪些，如果挖全國，就要放所有市名。

如：

輸出是一個EXCEL，包括：

如：

直接上程式碼：

from bs4 import BeautifulSoup
import requests
import re
import sys
import xlwt
import xlrd
from xlutils.copy import copy

#獲取html
def getHtmlText(url, code="GBK"):
    try:
        headers = {'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/51.0.2704.63 Safari/537.36'}
        r = requests.get(url,headers = headers,timeout = 30)
        r.raise_for_status()
        r.encoding = code
        return r.text
    except:
        return "獲取html異常"
#解析地區，返回地區清單
'''
def getGroundList(htext):
    try:
        grounddict = {}
        soup = BeautifulSoup(htext, "html.parser")
        gdname = soup.find('dl', attrs={'class':'nobackground'})
        keyList = gdname.find_all('a')
        for i in range(1,len(keyList)):
            key = keyList[i].text
            val = keyList[i].get('href')
            grounddict[key] = val
        return grounddict
    except:
        print("getGroundList異常")
'''
#解析頁碼
def getPageCode(htext,typeitem):   
    try:
        soup = BeautifulSoup(htext, "html.parser")
        s1 = soup.find('a', attrs={'class':'last'})
        if (s1):
            pat = re.compile(typeitem + r'pn([0-9]+).html')
            if(s1.get('href')):
               code = pat.search(s1.get('href'))
               if(code):
                   return code.group(1)
        else:
            return 0
            
    except:
        print("getPageCode異常")
    

#解析學校資訊，返回學校名稱、地址、電話、網址
def getSchoolList(htext,fileAddress,cityitem,typeitem):
    try:
        schoolDict = {}
        soup = BeautifulSoup(htext, "html.parser")
        sclist1 = soup.find_all('dl',attrs={'class':'left'})
        sclist2 = soup.find_all('dl',attrs={'class':'right'})
        sclist = sclist1 + sclist2
        for item in sclist:
            schoolDict['城市'] = cityitem
            schoolDict['型別'] = typeitem
            schoolDict['學習名稱'] = item.find('p').text
            sl = item.find_all('li')
            schoolDict['地址'] = sl[0].text
            schoolDict['電話'] = sl[1].text
            schoolDict['網址'] = sl[2].text
            #f = open(fileAddress, 'a', encoding='utf-8')
            #f.write(str(schoolDict)  + '\n' )
            savefile(schoolDict,fileAddress)
    except:
        print("getSchoolList異常")

#儲存到excel
def savefile(schoolDict,fileAddress):
    workbook = xlrd.open_workbook(fileAddress,'w+b')
    sheet = workbook.sheet_by_index(0)
    wb = copy(workbook)
    ws = wb.get_sheet(0)
    rowNum = sheet.nrows
    ws.write(rowNum,0,schoolDict['城市'])
    ws.write(rowNum,1,schoolDict['型別'])
    ws.write(rowNum,2,schoolDict['學習名稱'])
    ws.write(rowNum,3,schoolDict['地址'])
    ws.write(rowNum,4,schoolDict['電話'])
    ws.write(rowNum,5,schoolDict['網址'])
    wb.save(fileAddress)
        
#獲取城市列表,城市由EXCEL檔案儲存
def getCityList():
    try:
        cityFileAddress = r'D:\中國省市名稱拼音.xls'
        file = xlrd.open_workbook(cityFileAddress)
        sheet = file.sheet_by_name('city')
        cityDic = {}
        for i in range(sheet.nrows):
            key = sheet.col_values(0)[i]
            value = sheet.col_values(1)[i].lower()
            cityDic[key] = value
        return cityDic
    except:
        print("getCityList失敗")
            
def main():
    cityList = getCityList()
    typeList = {'小學':'/xiaoxue/','初中':'/chuzhong/','高中':'/gaozhong/'}
    for cityitem in cityList:
        for typeitem in typeList:
            searchUrl = 'http://'+ cityList[cityitem] + '.xuexiaodaquan.com'
            fileAddress = 'D:/school.xls'
            htext = getHtmlText(searchUrl+typeList[typeitem])
            getSchoolList(htext,fileAddress,cityitem,typeitem)
            pagecode = int(getPageCode(htext,typeList[typeitem]))
            if pagecode != 0:
                for i in range(2,pagecode+1):
                    h1text = getHtmlText(searchUrl+typeList[typeitem]+'pn'+str(i)+'.html')
                    getSchoolList(h1text,fileAddress,cityitem,typeitem)
       
main()