# -*- coding: utf-8 -*-# Define here the models for your scraped items## See documentation in:# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class FirproItem(scrapy.Item):
# define the fields for your item here like:# name = scrapy.Field()#定義儲存崗位的名稱的欄位
name = scrapy.Field()
#反饋概率
percent = scrapy.Field()
#釋出公司
company = scrapy.Field()
#崗位月薪
salary = scrapy.Field()
#工作地點
position = scrapy.Field()
複製程式碼
####2.在spiders建立fir_spider.py檔案
# -*- coding: utf-8 -*-
import scrapy
#自定義的爬蟲程式處理類，要繼承scrapy模組的spider型別
class Firspider(scrapy.Spider):
#定義爬蟲程式的名稱，用於程式的啟動使用
name = 'firspider'#定義爬蟲程式執行的作用域--域名
allow_domains = 'http://sou.zhaopin.com'#定義爬蟲程式真實爬取url地址的列表/原組
start_urls = ('http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E4%B8%8A%E6%B5%B7&kw=python&sm=0&p=1&source=0',)
#定義爬蟲獲取到的響應資料處理類#response就是爬取程式獲取的資料
def parse(self,response):
with open(u'智聯.html','w') as f:
f.write(response.body)
複製程式碼
# -*- coding: utf-8 -*-# Define here the models for your scraped items## See documentation in:# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class FirproItem(scrapy.Item):
# define the fields for your item here like:# name = scrapy.Field()#定義儲存崗位的名稱的欄位
name = scrapy.Field()
#反饋概率
percent = scrapy.Field()
#釋出公司
company = scrapy.Field()
#崗位月薪
salary = scrapy.Field()
#工作地點
position = scrapy.Field()
複製程式碼
pipelines.py
# -*- coding: utf-8 -*-# Define your item pipelines here## Don't forget to add your pipeline to the ITEM_PIPELINES setting# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
class FirproPipeline(object):
def __init__(self):
self.file=open('zhilian.json','w')
def process_item(self, item, spider):
text = json.dumps(dict(item),ensure_ascii=False)
self.file.write(text.encode('utf-8'))
print'-----------------'
def close_spider(self,spider):
self.file.close()
#return item複製程式碼
fir_spider.py
# -*- coding: utf-8 -*-
import scrapy
from firPro.items import FirproItem
import re
#自定義的爬蟲程式處理類，要繼承scrapy模組的spider型別
class Firspider(scrapy.Spider):
#定義正則匹配，把匹配到的資料進行替換
reg = re.compile('\s*')
#定義爬蟲程式的名稱，用於程式的啟動使用
name = 'firspider'#定義爬蟲程式執行的作用域--域名
allow_domains = 'http://sou.zhaopin.com'#定義爬蟲程式真實爬取url地址的列表/原組
url = 'http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%E4%B8%8A%E6%B5%B7&kw=python&sm=0&source=0&sg=b8e8fb4080fa47afa69cd683dfbfccf9&p='
p = 1
start_urls = [url + str(p)]
def parse(self, response):
# print (response.body)#獲取所匹配的崗位
job_list= response.xpath('//div[@id="newlist_list_div"]//table')[2:]
for job in job_list:
#建立一個Item物件，用於存放匹配的目標資料
item = FirproItem()
name =job.xpath(".//tr[1]//td[1]//a")
# name = self.reg.sub('', job.xpath(".//td[1]//a/text()[1]").extract())
item["name"] = self.reg.sub('',name.xpath("string(.)").extract()[0])
item["percent"] = job.xpath(".//td[2]//span[1]/text()").extract()
item["company"] = job.xpath(".//td[3]//a/text()").extract()
item["salary"] = job.xpath(".//td[4]/text()").extract()
item["position"] = job.xpath(".//td[5]/text()").extract()
# 將資料提交給模組pipelines處理
yield item
if self.p<=10:
self.p+=1
yield scrapy.Request(self.url + str(self.p),callback=self.parse)
複製程式碼
# -*- coding: utf-8 -*-# Define here the models for your scraped items## See documentation in:# http://doc.scrapy.org/en/latest/topics/items.html
import scrapy
class ZhycItem(scrapy.Item):
# define the fields for your item here like:# name = scrapy.Field()# 定義需要封裝的欄位
name = scrapy.Field()
publish = scrapy.Field()
company = scrapy.Field()
require = scrapy.Field()
salary = scrapy.Field()
desc = scrapy.Field()
複製程式碼
pipelines.py
# -*- coding: utf-8 -*-# Define your item pipelines here## Don't forget to add your pipeline to the ITEM_PIPELINES setting# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
class ZhycPipeline(object):
def __init__(self):
self.file = open("zhonghuayingcai.json", "w")
def process_item(self, item, spider):
text = json.dumps(dict(item), ensure_ascii=False)
self.file.write(text.encode("utf-8"))
print"*****************************************"#return item
def close_spider(self, spider):
self.file.close()
複製程式碼
# -*- coding: utf-8 -*-
import scrapy
class ZlItem(scrapy.Item):
# define the fields for your item here like:# name = scrapy.Field()#崗位名稱
name = scrapy.Field()
#反饋率
percent = scrapy.Field()
#公司名稱
company = scrapy.Field()
#職位月薪
salary = scrapy.Field()
#工作地點
position = scrapy.Field()
複製程式碼
pipelines.py
# -*- coding: utf-8 -*-# Define your item pipelines here## Don't forget to add your pipeline to the ITEM_PIPELINES setting# See: http://doc.scrapy.org/en/latest/topics/item-pipeline.html
import json
class ZlPipeline(object):
def __init__(self):
self.file = open("sdzp.json", "w")
def process_item(self, item, spider):
text = json.dumps(dict(item), ensure_ascii=False)
self.file.write(text.encode("utf-8"))
#return item
def close_spider(self, spider):
self.file.close()
複製程式碼
zlzp.py
# -*- coding: utf-8 -*-
from scrapy.spiders import CrawlSpider,Rule
from scrapy.linkextractors import LinkExtractor
from zl.items import ZlItem
class ZlzpSpider(CrawlSpider):
name = 'sdzpspider'
allowed_domains = ['zhaopin.com']
start_urls = ['http://sou.zhaopin.com/jobs/searchresult.ashx?jl=%e4%b8%8a%e6%b5%b7&kw=python&sm=0&source=0&sg=936e2219abfb4f07a17009a930d54a37&p=1']
#定義超連結的提取規則
page_link = LinkExtractor(allow=('&sg=936e2219abfb4f07a17009a930d54a37&p=\d+'))
#定義爬蟲爬取資料的規則
rules=[
Rule(page_link,callback='parse_content',follow=True)
]
#定義處理函式
def parse_content(self, response):
#獲取整個我們需要的資料區域
job_list = response.xpath('//div[@id="newlist_list_content_table"]//table//tr[1]')
for job in job_list:
#定義一個item,用於存放目標資料
item = ZlItem()
name = job.xpath(".//td[1]//a")
if len(name)>0:
item['name'] = name.xpath('string(.)').extract()[0]
percent = job.xpath('.//td[2]//span/text()')
if len(percent)>0:
item['percent']=percent.extract()[0]
company = job.xpath(".//td[3]//a[1]/text()")
if len(company) > 0:
item["company"] = company.extract()[0]
salary = job.xpath(".//td[4]/text()")
if len(salary) > 0:
item["salary"] = salary.extract()[0]
position = job.xpath(".//td[5]/text()")
if len(position) > 0:
item["position"] = position.extract()[0]
yield item
複製程式碼
