資料採集實踐第四次作業

FungusTanion發表於2024-11-26

Gitee作業連結：
https://gitee.com/FunkTank/crawl_project/tree/master/作業4

作業①

要求：熟練掌握 Selenium 查詢HTML元素、爬取Ajax網頁資料、等待HTML元素等內容。
使用Selenium框架+ MySQL資料庫儲存技術路線爬取“滬深A股”、“上證A股”、“深證A股”3個板塊的股票資料資訊。

1.作業內容

點選檢視程式碼

from selenium import webdriver
from lxml import etree
from selenium.webdriver.common.by import By
import time
import pymysql

def get_data(url):
    html = driver.page_source
    bs = etree.HTML(html)
    lis = bs.xpath('//*[@id="table_wrapper-table"]/tbody/tr')
    time.sleep(2)
    for link in lis:
        a = link.xpath('./td[1]/text()')[0]  # 序號
        b = link.xpath('./td[2]/a/text()')[0]  # 程式碼
        c = link.xpath('./td[3]/a/text()')[0]  # 名稱
        d = link.xpath('./td[5]/span/text()')[0]  # 最新價
        e = link.xpath('./td[6]/span/text()')[0]  # 漲跌幅
        f = link.xpath('./td[7]/span/text()')[0]  # 漲跌額
        g = link.xpath('./td[8]/text()')[0]  # 成交量
        h = link.xpath('./td[9]/text()')[0]  # 成交額
        i = link.xpath('./td[10]/text()')[0]  # 振幅
        j = link.xpath('./td[11]/span/text()')[0]  # 最高
        k = link.xpath('./td[12]/span/text()')[0]  # 最低
        l = link.xpath('./td[13]/span/text()')[0]  # 今開
        m = link.xpath('./td[14]/text()')[0]  # 昨收

        # 使用 pymysql 連線資料庫
        connection = pymysql.connect(
            host='localhost',
            user='root',
            password='',
            database='pachoong',
            charset='utf8mb4'
        )
        try:
            with connection.cursor() as cursor:
                sql = "INSERT INTO stock4 (序號, 程式碼, 名稱, 最新價, 漲跌幅, 漲跌額, 成交量, 成交額, 振幅, 最高, 最低, 今開, 昨收) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s)"
                val = (a, b, c, d, e, f, g, h, i, j, k, l, m)
                cursor.execute(sql, val)
            connection.commit()
            print(a, b, c, d, e, f, g, h, i, j, k, l, m)
        except Exception as e:
            print(f"Error: {e}")
        finally:
            connection.close()

    next_button = driver.find_element(By.XPATH, url)
    next_button.click()
    time.sleep(2)

chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')

driver = webdriver.Chrome(options=chrome_options)

driver.get("https://quote.eastmoney.com/center/gridlist.html#hs_a_board")
get_data('//*[@id="nav_sh_a_board"]/a')
get_data('//*[@id="nav_sz_a_board"]/a')
get_data('//*[@id="nav_bj_a_board"]/a')

輸出資訊：

2.心得體會

在完成這個股票資料爬取任務的過程中，我深刻體會到了自動化資料處理的重要性以及Selenium工具在實際應用中的強大功能。這次實踐不僅提升了我對Selenium和MySQL的掌握程度，也讓我認識到在處理實際專案時，問題分析和解決能力的重要性。

作業②

要求：熟練掌握 Selenium 查詢HTML元素、實現使用者模擬登入、爬取Ajax網頁資料、等待HTML元素等內容。
使用Selenium框架+MySQL爬取中國mooc網課程資源資訊（課程號、課程名稱、學校名稱、主講教師、團隊成員、參加人數、課程進度、課程簡介）

1.作業內容

點選檢視程式碼

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.chrome.options import Options
from selenium.webdriver.support.ui import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from lxml import etree
import pymysql

# 資料庫配置
DB_CONFIG = {
    'host': 'localhost',
    'user': 'root',
    'password': '',
    'database': 'pachoong',
    'charset': 'utf8mb4'
}

# 目標網址
LOGIN_URL = 'https://www.icourse163.org/'
COURSE_URL = 'https://www.icourse163.org/search.htm?search=%20#/'

# 初始化Chrome選項
chrome_options = Options()
chrome_options.add_argument('--headless')  # 無頭模式
chrome_options.add_argument('--disable-gpu')

# 初始化WebDriver
driver = webdriver.Chrome(options=chrome_options)

def wait_for_element(driver, by, value, timeout=10):
    """
    等待元素載入完成
    """
    return WebDriverWait(driver, timeout).until(
        EC.presence_of_element_located((by, value))
    )

def login(driver, login_url, account, password):
    """
    登入到網站
    """
    driver.get(login_url)
    wait_for_element(driver, By.XPATH, '//*[@id="app"]/div/div/div[1]/div[3]/div[3]/div').click()
    
    # 切換到iframe
    frame = wait_for_element(driver, By.XPATH, '/html/body/div[13]/div[2]/div/div/div/div/div/div[1]/div/div[1]/div[2]/div[2]/div[1]/div/iframe')
    driver.switch_to.frame(frame)
    
    # 輸入賬號和密碼
    wait_for_element(driver, By.ID, 'phoneipt').send_keys(account)
    wait_for_element(driver, By.XPATH, '/html/body/div[2]/div[2]/div[2]/form/div/div[4]/div[2]/input[2]').send_keys(password)
    
    # 點選登入按鈕
    wait_for_element(driver, By.XPATH, '/html/body/div[2]/div[2]/div[2]/form/div/div[6]/a').click()
    
    # 等待登入完成，可以根據實際情況調整
    WebDriverWait(driver, 10).until(
        EC.presence_of_element_located((By.XPATH, '//*[@id="app"]/div/div/div[1]/div[3]/div[3]/div'))
    )

def fetch_course_data(driver, course_url):
    """
    獲取課程資訊並儲存到資料庫
    """
    driver.get(course_url)
    wait_for_element(driver, By.XPATH, '/html/body/div[4]/div[2]/div[2]/div[2]/div/div[6]/div[1]/div[1]/label/div').click()
    
    # 滾動頁面以載入所有課程
    scroll_script = """
    var timer = setInterval(function(){
        var scrollTop = document.documentElement.scrollTop || document.body.scrollTop;
        var ispeed = Math.floor(document.body.scrollHeight / 100);
        if(scrollTop > document.body.scrollHeight * 90 / 100){
            clearInterval(timer);
        }
        console.log('scrollTop:' + scrollTop);
        console.log('scrollHeight:' + document.body.scrollHeight);
        window.scrollTo(0, scrollTop + ispeed);
    }, 20);
    """
    driver.execute_script(scroll_script)
    time.sleep(4)  # 等待滾動完成
    
    # 獲取頁面原始碼並解析
    html = driver.page_source
    bs = etree.HTML(html)
    courses = bs.xpath('/html/body/div[4]/div[2]/div[2]/div[2]/div/div[6]/div[2]/div[1]/div/div/div')
    
    # 連線到資料庫
    mydb = pymysql.connect(**DB_CONFIG)
    cursor = mydb.cursor()
    
    try:
        for course in courses:
            # 提取課程資訊
            a = course.xpath('./div[2]/div/div/div[1]/a[2]/span/text()')
            if len(a) != 0 and a[0] == '國家精品':
                b = course.xpath('./div[2]/div/div/div[1]/a[1]/span/text()')
                c = course.xpath('./div[2]/div/div/div[2]/a[1]/text()')
                d = course.xpath('./div[2]/div/div/div[2]/a[2]/text()')
                e = course.xpath('./div[2]/div/div/div[2]/a[2]/text()')
                f = course.xpath('./div[2]/div/div/div[3]/span[2]/text()')
                g = course.xpath('./div[2]/div/div/div[3]/div/span[2]/text()')
                h = course.xpath('./div[2]/div/div/a/span/text()')
                
                # 資料清洗
                course_name = b[0] if b else ''
                college = c[0] if c else ''
                teacher = d[0] if d else ''
                team = e[0] if e else ''
                count = f[0] if f else ''
                process = g[0] if g else '已結束'
                brief = h[0] if h else '空'
                
                # 插入資料庫
                sql = "INSERT INTO mooc (cCourse, cCollege, cTeacher, cTeam, cCount, cProcess, cBrief) VALUES (%s, %s, %s, %s, %s, %s, %s)"
                val = (course_name, college, teacher, team, count, process, brief)
                cursor.execute(sql, val)
                mydb.commit()
                print(course_name, college, teacher, team, count, process, brief)
    except Exception as e:
        print(f"Error: {e}")
    finally:
        cursor.close()
        mydb.close()

def main():
    # 登入
    login(driver, LOGIN_URL, '', '')  # 實際的賬號和密碼
    
    # 獲取課程資料
    fetch_course_data(driver, COURSE_URL)
    
    # 關閉瀏覽器
    driver.quit()

if __name__ == "__main__":
    main()

輸出資訊：

2.心得體會

在熟練掌握Selenium框架後，我成功實現了對中國大學MOOC網課程資源的自動化爬取。首先，我透過Selenium模擬使用者登入，透過顯式等待（WebDriverWait）處理動態載入的元素，確保登入流程的穩定性。接著，我使用Selenium的XPath定位功能，精準抓取課程資訊，包括課程號、課程名稱、學校名稱、主講教師、團隊成員、參加人數、課程進度和課程簡介等關鍵資料。