# -*- codeing = utf-8 -*- from bs4 import BeautifulSoup # 網頁解析，獲取資料 import re # 正規表示式，進行文字匹配` import urllib.request, urllib.error # 制定URL，獲取網頁資料 import xlwt # 進行excel操作 #import sqlite3 # 進行SQLite資料庫操作 findLink = re.compile(r'<a href="(.*?)">') # 建立正規表示式物件，標售規則 影片詳情連結的規則 findImgSrc = re.compile(r'<img.*src="(.*?)"', re.S) findTitle = re.compile(r'<span class="title">(.*)</span>') findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>') findJudge = re.compile(r'<span>(\d*)人評價</span>') findInq = re.compile(r'<span class="inq">(.*)</span>') findBd = re.compile(r'<p class="">(.*?)</p>', re.S) def main(): baseurl = "https://movie.douban.com/top250?start=" #要爬取的網頁連結 # 1.爬取網頁 datalist = getData(baseurl) savepath = "豆瓣電影Top250.xls" #當前目錄新建XLS，儲存進去 # dbpath = "movie.db" #當前目錄新建資料庫，儲存進去 # 3.儲存資料 saveData(datalist,savepath) #2種儲存方式可以只選擇一種 # saveData2DB(datalist,dbpath) # 爬取網頁 def getData(baseurl): datalist = [] #用來儲存爬取的網頁資訊 for i in range(0, 10): # 呼叫獲取頁面資訊的函式，10次 url = baseurl + str(i * 25) html = askURL(url) # 儲存獲取到的網頁原始碼 # 2.逐一解析資料 soup = BeautifulSoup(html, "html.parser") for item in soup.find_all('div', class_="item"): # 查詢符合要求的字串 data = [] # 儲存一部電影所有資訊 item = str(item) link = re.findall(findLink, item)[0] # 透過正規表示式查詢 data.append(link) imgSrc = re.findall(findImgSrc, item)[0] data.append(imgSrc) titles = re.findall(findTitle, item) if (len(titles) == 2): ctitle = titles[0] data.append(ctitle) otitle = titles[1].replace("/", "") #消除跳脫字元 data.append(otitle) else: data.append(titles[0]) data.append(' ') rating = re.findall(findRating, item)[0] data.append(rating) judgeNum = re.findall(findJudge, item)[0] data.append(judgeNum) inq = re.findall(findInq, item) if len(inq) != 0: inq = inq[0].replace("。", "") data.append(inq) else: data.append(" ") bd = re.findall(findBd, item)[0] bd = re.sub('<br(\s+)?/>(\s+)?', "", bd) bd = re.sub('/', "", bd) data.append(bd.strip()) datalist.append(data) return datalist # 得到指定一個URL的網頁內容 def askURL(url): head = { # 模擬瀏覽器頭部資訊，向豆瓣伺服器傳送訊息 "User-Agent": "Mozilla / 5.0(Windows NT 10.0; Win64; x64) AppleWebKit / 537.36(KHTML, like Gecko) Chrome / 80.0.3987.122 Safari / 537.36" } # 使用者代理，表示告訴豆瓣伺服器，我們是什麼型別的機器、瀏覽器（本質上是告訴瀏覽器，我們可以接收什麼水平的檔案內容） request = urllib.request.Request(url, headers=head) html = "" try: response = urllib.request.urlopen(request) html = response.read().decode("utf-8") except urllib.error.URLError as e: if hasattr(e, "code"): print(e.code) if hasattr(e, "reason"): print(e.reason) return html # 儲存資料到表格 def saveData(datalist,savepath): print("save.......") book = xlwt.Workbook(encoding="utf-8",style_compression=0) #建立workbook物件 sheet = book.add_sheet('豆瓣電影Top250', cell_overwrite_ok=True) #建立工作表 col = ("電影詳情連結","圖片連結","影片中文名","影片外國名","評分","評價數","概況","相關資訊") for i in range(0,8): sheet.write(0,i,col[i]) #列名 for i in range(0,250): # print("第%d條" %(i+1)) #輸出語句，用來測試 data = datalist[i] for j in range(0,8): sheet.write(i+1,j,data[j]) #資料 book.save(savepath) #儲存 # def saveData2DB(datalist,dbpath): # init_db(dbpath) # conn = sqlite3.connect(dbpath) # cur = conn.cursor() # for data in datalist: # for index in range(len(data)): # if index == 4 or index == 5: # continue # data[index] = '"'+data[index]+'"' # sql = ''' # insert into movie250( # info_link,pic_link,cname,ename,score,rated,instroduction,info) # values (%s)'''%",".join(data) # # print(sql) #輸出查詢語句，用來測試 # cur.execute(sql) # conn.commit() # cur.close # conn.close() # def init_db(dbpath): # sql = ''' # create table movie250( # id integer primary key autoincrement, # info_link text, # pic_link text, # cname varchar, # ename varchar , # score numeric, # rated numeric, # instroduction text, # info text # ) # # # ''' #建立資料表 # conn = sqlite3.connect(dbpath) # cursor = conn.cursor() # cursor.execute(sql) # conn.commit() # conn.close() # 儲存資料到資料庫 if __name__ == "__main__": # 當程式執行時 # 呼叫函式 main() # init_db("movietest.db") print("爬取完畢！")