1 from lxml import etree 2 from selenium import webdriver 3 import pymysql 4 5 def Geturl(fullurl):#獲取每個招聘網頁的連結 6 browser.get(fullurl) 7 shouye_html_text = browser.page_source 8 shouye_ele = etree.HTML(shouye_html_text) 9 zf_list = shouye_ele.xpath(`/html/body/div[4]/div[1]/div[2]/ul/li/div/h3/a/@href`)#連結url 10 zf_url_list = [] 11 for zf_url_lost in zf_list: 12 zf_url = `https://bj.5i5j.com`+zf_url_lost 13 zf_url_list.append(zf_url) 14 return zf_url_list 15 def Getinfo(zp_url_list): 16 for zp_url in zp_url_list: 17 browser.get(zp_url) 18 zp_info_html = browser.page_source 19 zp_ele = etree.HTML(zp_info_html) 20 zp_info_title = str(zp_ele.xpath(`//html/body/div[3]/div[1]/div[1]/h1/text()`)[0]) 21 zp_info_num = str(zp_ele.xpath(`/html/body/div[3]/div[2]/div[2]/div[1]/div[1]/div/p[1]/text()`)[0])+`元/月`#價格 22 zp_info_type = str(zp_ele.xpath(`/html/body/div[3]/div[2]/div[2]/div[1]/div[2]/div/p[1]/text()`)[0])#戶型 23 zp_info_zone = str(zp_ele.xpath(`/html/body/div[3]/div[2]/div[2]/div[1]/div[3]/div/p[1]/text()`)[0])+`平米`#房屋大小 24 zp_info_need_1 = str(zp_ele.xpath(`/html/body/div[3]/div[2]/div[2]/div[2]/ul/li[1]/span/text()`)[0])#房屋資訊 25 zp_info_need_2 = str(zp_ele.xpath(`/html/body/div[3]/div[2]/div[2]/div[2]/ul/li[1]/a/text()`)[0])#房屋資訊 26 zp_info_need = zp_info_need_1+zp_info_need_2 27 connection = pymysql.connect(host=`localhost`, user=`root`, password=`1234`, db=`5i5j`, ) 28 try: 29 with connection.cursor() as cursor: 30 sql = "INSERT INTO `5i5j_info` (`title`,`num`,`type`, `zone`,`need`) VALUES (%s,%s,%s,%s, %s)" 31 cursor.execute(sql, (zp_info_title,zp_info_num,zp_info_type,zp_info_zone,zp_info_need)) 32 connection.commit() 33 finally: 34 connection.close() 35 print(zp_info_title,zp_info_num,zp_info_type,zp_info_zone,zp_info_need) 36 if __name__ == `__main__`: 37 browser = webdriver.Chrome() 38 pags = int(input(`需要幾頁?`)) 39 for i in range(1,pags+1): 40 url = `https://bj.5i5j.com/zufang/huilongguan/n{}/` 41 fullurl = url.format(str(i)) 42 zf_url_list = Geturl(fullurl) 43 print(fullurl) 44 # print(zf_url_list) 45 Getinfo(zf_url_list) 46 browser.close()