資料採集與融合技術第四次實踐作業

菜鸟bird發表於2024-11-12

gitee連結：
https://gitee.com/zxbaixuexi/2024scrapy/tree/master/第四次實踐

作業①:

1）

使用 Selenium 框架+ MySQL 資料庫儲存技術路線爬取“滬深 A 股”、“上證 A 股”、“深證 A 股”3 個板塊的股票資料資訊。

候選網站：東方財富網：
http://quote.eastmoney.com/center/gridlist.html#hs_a_board

程式碼：

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.action_chains import ActionChains
import time
from selenium.webdriver.chrome.service import Service as ChromeService
import pymysql

# 連線自己的資料庫，根據自己情況修改
conn = pymysql.connect(host="127.0.0.1", port=3306, user='root', passwd='123456', charset='utf8', db='homework1')
cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)

# 顯式指定Chrome二進位制檔案路徑（根據實際安裝路徑進行修改）
chrome_binary_path = r"D:\noneprogramsoft\Google\Chrome\Application\chrome.exe"
#ChromeDriver
CHROMEDRIVER_PATH = r"C:\Users\supermejane\Desktop\爬蟲實踐\test4\chromedriver.exe"

# 配置Chrome選項
options = webdriver.ChromeOptions()
options.binary_location = chrome_binary_path
#Service
service = ChromeService(executable_path=CHROMEDRIVER_PATH)

# 建立一個WebDriver物件，並指定驅動程式路徑
driver = webdriver.Chrome(service=service, options=options)


urls = ["https://quote.eastmoney.com/center/gridlist.html#hs_a_board","https://quote.eastmoney.com/center/gridlist.html#sh_a_board","https://quote.eastmoney.com/center/gridlist.html#sz_a_board"]
tables = ["husheng","shangzheng","shengzheng"]
# 爬取前n頁股票資訊
def parseData(driver,n,tableName):
    for page in range(1, n + 1):  # 爬取n頁
        print(f"正在爬取第 {page} 頁股票資訊")

        # 定位所有股票資訊的元素
        stocks_list = driver.find_elements(By.XPATH, "//div[@class='listview full']//tbody//tr")

        # 列印股票資訊
        for stock in stocks_list:
            stock_id = stock.find_element(By.XPATH, './/td[1]').text
            stock_code = stock.find_element(By.XPATH, './/td[2]/a').text
            stock_name = stock.find_element(By.XPATH, './/td[3]/a').text
            stock_latest_price = stock.find_element(By.XPATH, './/td[5]//span').text
            stock_change_percent = stock.find_element(By.XPATH, './/td[6]//span').text
            stock_change_amount = stock.find_element(By.XPATH, './/td[7]//span').text
            stock_volume = stock.find_element(By.XPATH, './/td[8]').text
            stock_turnover = stock.find_element(By.XPATH, './/td[9]').text
            stock_amplitude = stock.find_element(By.XPATH, './/td[10]').text
            stock_highest = stock.find_element(By.XPATH, './/td[11]//span').text
            stock_lowest = stock.find_element(By.XPATH, './/td[12]//span').text
            stock_open_price = stock.find_element(By.XPATH, './/td[13]//span').text
            stock_close_price = stock.find_element(By.XPATH, './/td[14]').text

            sql = f"INSERT INTO {tableName} VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
            cursor.execute(sql,
                           [stock_id, stock_code, stock_name, stock_latest_price, stock_change_percent.strip("%"),
                            stock_change_amount,
                            stock_volume[:-1], stock_turnover[:-1], stock_amplitude.strip("%"), stock_highest,
                            stock_lowest, stock_open_price,
                            stock_close_price])
        conn.commit()
        print("資料已存入資料庫")

        # 點選下一頁按鈕
        next_page_button = driver.find_element(By.XPATH, '//*[@id="main-table_paginate"]/a[2]')
        action = ActionChains(driver)
        time.sleep(5)
        action.move_to_element(next_page_button).perform()
        next_page_button.click()

        time.sleep(5)

table_No=0
for url in urls:
    driver.get(url)
    parseData(driver,2,tables[table_No])
    table_No += 1

conn.close()
# 關閉瀏覽器
driver.quit()

執行結果
husheng(滬深)

shangzheng(上證)

shengzheng(深證)

2）心得體會

學會了使用selenium爬取網頁資訊1，使用get獲取網頁資訊，並模擬使用者翻頁，並可以結合Xpath等方式實現解析

作業②:

1）

使用 Selenium 框架+MySQL 爬取中國 mooc 網課程資源資訊（課程號、課程名
稱、學校名稱、主講教師、團隊成員、參加人數、課程進度、課程簡介）

候選網站：中國 mooc 網：https://www.icourse163.org

程式碼

from time import sleep

from selenium import webdriver
from selenium.webdriver.common.by import By
import time
from selenium.webdriver.chrome.service import Service as ChromeService
import pymysql
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from scrapy.selector import Selector
from selenium.webdriver.common.keys import Keys
"""
連結資料庫
"""
conn = pymysql.connect(host="127.0.0.1", port=3306, user='root', passwd='123456', charset='utf8', db='homework1')
cursor = conn.cursor(cursor=pymysql.cursors.DictCursor)

"""
建立ChromeDriver
"""
# 顯式指定Chrome二進位制檔案路徑（根據實際安裝路徑進行修改）
chrome_binary_path = r"D:\noneprogramsoft\Google\Chrome\Application\chrome.exe"
#ChromeDriver
CHROMEDRIVER_PATH = r"C:\Users\supermejane\Desktop\爬蟲實踐\test4\chromedriver.exe"

# 配置Chrome選項
options = webdriver.ChromeOptions()
options.binary_location = chrome_binary_path
#Service
service = ChromeService(executable_path=CHROMEDRIVER_PATH)

# 建立一個WebDriver物件，並指定驅動程式路徑
driver = webdriver.Chrome(service=service, options=options)


"""
請求首頁
"""
#請求
driver.get("https://www.icourse163.org/")

"""
登陸
"""
#等待登入元素載入好再執行操作
WebDriverWait(driver, 10, 0.5).until(
    EC.presence_of_element_located((By.XPATH, '//a[@class="f-f0 navLoginBtn"]'))).click()

iframe = WebDriverWait(driver, 10, 0.5).until(EC.presence_of_element_located((By.XPATH, '//*[@frameborder="0"]')))
#切換到登入介面中
driver.switch_to.frame(iframe)
# 輸入賬號密碼
driver.find_element(By.XPATH, '//*[@id="phoneipt"]').send_keys("19906912753")
time.sleep(3)
driver.find_element(By.XPATH, '//*[@class="j-inputtext dlemail"]').send_keys("Woshi2gouzi@")
time.sleep(3)
#點選登入按鈕
driver.find_element(By.ID, 'submitBtn').click()

time.sleep(5)
"""
搜尋
"""
driver.switch_to.default_content()

#等待輸入框並輸入’python‘
wait = WebDriverWait(driver, 10)
select_course = wait.until(EC.presence_of_element_located((By.XPATH,"//input[@class='ant-input']")))

select_course.send_keys("python")
#輸入回車
select_course.send_keys(Keys.ENTER)


"""
解析資料
"""

time.sleep(5)

content = driver.page_source
print(content)
driver.quit()
selector = Selector(text=content)
rows = selector.xpath("//div[@class='m-course-list']/div/div")

xuhao=0
#清空資料庫
#cursor.execute("delete from course")

# 爬取網站資訊
for row in rows:
    # 課程名稱
    course1= row.xpath(".//span[@class=' u-course-name f-thide']//text()").extract()
    course="".join(course1)
    # 學校
    college=row.xpath(".//a[@class='t21 f-fc9']/text()").extract_first()
    # 主講教師
    teacher=row.xpath(".//a[@class='f-fc9']//text()").extract_first()
    # 團隊成員
    team1 = row.xpath(".//a[@class='f-fc9']//text()").extract()
    team=",".join(team1)
    # 參加人數
    count = row.xpath(".//span[@class='hot']/text()").extract_first()
    # 課程進度
    process = row.xpath(".//span[@class='txt']/text()").extract_first()
    # 課程簡介
    brief1=row.xpath(".//span[@class='p5 brief f-ib f-f0 f-cb']//text()").extract()
    brief=",".join(brief1)

    xuhao+=1

    #sql插入資料
    sql = "INSERT INTO course VALUES (%s, %s, %s, %s, %s, %s, %s, %s)"
    cursor.execute(sql,
                   [xuhao,course,college,teacher,team,count,process,brief])

#提交
conn.commit()
print("資料已存入資料庫")

執行結果

2）心得體會

學會了使用selenium的send_key填寫表單實現登陸以及搜素，另外需要注意登陸時要查詢Ifarme並使用selenium的driver.switch_to.frame(iframe)切換到對應的iframe
還有就是使用sleep配合wait.until(EC.presence_of_element_located())可以有效避免元素未載入的情況