增補部落格 第十三篇 python大作業小說閱讀器（2）爬取

财神给你送元宝發表於2024-06-14
Python
import os
import re
from lxml import etree
import random
import requests
from 視覺化查詢書籍.查詢書籍資訊 import searchbook

# 代理IP池
proxy_list = ["192.168.72.241"]

def get_random_proxy():
return random.choice(proxy_list)

def search(user_leixing, user_title, user_author):
# 呼叫 searchbook 函式並獲取匹配的書籍資訊
book_info = searchbook(user_leixing, user_title, user_author)

# 檢查目錄是否存在
directory = "..\\小說\\" + book_info[0]

if not os.path.exists(directory):
print("目錄不存在")
# 初始化 book 變數
book = [None, None, None]

# 呼叫 searchbook 函式並獲取匹配的書籍資訊
book = searchbook(user_leixing, user_title, user_author)

headers = {
"User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3"
}

index_url = 'https://www.bqg88.cc/' + book[2]

# Using proxy
proxy = {"http": "http://" + get_random_proxy()}
response = requests.get(index_url, headers=headers, proxies=proxy)
response.raise_for_status()

info_list = re.findall('<dd><a href\s*=\s*"(.*?)">(.*?)</a></dd>', response.text)

# Create directory if it doesn't exist
if not os.path.exists(directory):
os.makedirs(directory)

for info in info_list:
url = 'https://www.bqg88.cc/' + info[0]
response = requests.get(url, headers=headers, proxies=proxy)
response.raise_for_status()

print("正在下載" + info[1])

html_data = etree.HTML(response.text)
text_list = html_data.xpath('//div[@id="chaptercontent"]/text()')
text = ''.join(text_list)

book_text = '\n\n' + info[1] + '\n\n'
book_text += text.replace('請收藏本站：https://www.bqg88.cc。筆趣閣手機版：https://m.bqg88.cc', '')

# Save each chapter to a separate text file
chapter_filename = os.path.join(directory, info[1] + '.txt')
with open(chapter_filename, "w", encoding='utf-8') as file:
file.write(book_text)

print("下載完成！")
else:
print("目錄已存在")

　　

相關文章