百度地圖POI抓取——python

之前網上有看到用javascript寫的抓取百度地圖POI的程式，效果還不錯。作為python初學者，嘗試用python寫了下面程式碼，如有錯誤，還望大神們提供建議。

# -*- coding: utf-8 -*-

'''
Created on 2014年12月18日

@author: LiXiang
'''
import math
import time
import urllib
import json
import sys
import urllib2
from threading import Thread
from Queue import Queue

class BaiduSpider:
def __init__(self,city='武漢',bounds=[113.6,115.1,29.9,31.4],keyword='ATM',maxRequire=1000):
self.ak="***"
self.baseUrl="http://api.map.baidu.com/place/v2/search";
self.file=open(u'%s.txt' % keyword,'w')
self.resultCount=0 #實際返回poi數量
self.requestTimes=0 #請求次數，即頁數
self.MaxRequire=int(maxRequire)
self.maxPageNum=0
self.city=city
self.keyword=keyword
self.bounds=bounds;


def _start(self):
bounds=self.bounds
delta=0.1;
xmin=float(bounds[0])
xmax=float(bounds[1])
ymin=float(bounds[2])
ymax=float(bounds[3])
xNum=(xmax-xmin)/delta;
yNum=(ymax-ymin)/delta;
rows=int(xNum)
cols=int(yNum)
print u'分割方塊數%d*%d' %(rows,cols)
"""
# 使用單一執行緒
# """
# for i in range(rows):
# for j in range(cols):
# aa=ymin+j*delta
# bb=xmin+i*delta
# cc=aa+delta
# dd=bb+delta
# bound=str(aa)+','+str(bb)+','+str(cc)+','+str(dd)
# print '--%d,%d---------------' %(i,j)
# self._downPatch(bound)
# print '-----------------\n'

"""
使用多執行緒
"""
#q是任務佇列
#NUM是併發執行緒總數
#JOBS是有多少任務
q = Queue()
NUM = 5
JOBS = rows*cols
#具體的處理函式，負責處理單個任務
def do_somthing_using(arguments):
i=arguments/rows
j=arguments%rows
aa=ymin+j*delta
bb=xmin+i*delta
cc=aa+delta
dd=bb+delta
bound=str(aa)+','+str(bb)+','+str(cc)+','+str(dd)
print '--%d,%d-----%s--------------------' %(i,j,bound)
self._downPatch(bound)
print '-----------------------------------\n'
#這個是工作程式，負責不斷從佇列取資料並處理
def working():
while True:
arguments = q.get()
do_somthing_using(arguments)
time.sleep(0.1)
q.task_done()
#fork NUM個執行緒等待佇列
for i in range(NUM):
t = Thread(target=working)
t.setDaemon(True)
t.start()
#把JOBS排入佇列
for i in range(JOBS):
q.put(i)
#等待所有JOBS完成
q.join()
self.file.close()

def _downPatch(self,coord):
query={
'ak':self.ak,
'query':self.keyword,
# 'region':self.city,
'bounds':coord,
'page_size':'20',
'page_num':'0',
'output':'json',
'scope':'2'
}
js=self._fetch(query)
total=int(js['total'])
pageNum=int(math.ceil(total/20))
print u"---該區域總記錄條數-->"+str(total)
print u"---該區域分頁數"+str(pageNum)
poiList=js['results']
if pageNum>self.maxPageNum:
self.maxPageNum=pageNum
if pageNum>=76:
pageNum=75
is_break=False
for i in range(1,pageNum):
query['page_num']=str(i)
self.requestTimes+=1
if self.requestTimes<=self.MaxRequire:
poiList.extend(self._fetch(query)['results'])
else:
is_break=True
break
print u"---該區域查詢到的記錄條數-->"+str(len(poiList))
for poi in poiList:
loc=poi['location']
poi['lat']=loc['lat']
poi['lng']=loc['lng']
poi['type']=''
poi['tag']=''
if poi.has_key('detail_info'):
details=poi["detail_info"]
if details.has_key('type'):
poi['type']=details['type']
if details.has_key('tag'):
poi['tag']=details['tag']
self._save(poi)
self.resultCount+=1
self.file.flush()
if is_break:
sys.exit(0) #退出程式
print u"---該區域處理完畢"

# def _fetch(self,query=None,json=True):
# param = urllib.urlencode(query)
# url = self.baseUrl + '?' + param
# opener = urllib.FancyURLopener()
# data = opener.open(url).read()
# if json:
# return self._tojson(data)
# else:
# return data

def _fetch(self,query=None,json=True):
param = urllib.urlencode(query)
url = self.baseUrl + '?' + param
headers = {
'User-Agent':'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36' \
'(KHTML, like Gecko) Chrome/38.0.2125.111 Safari/537.36'
}
try:
req = urllib2.Request(url, param, headers)
urllib2.socket.timeout=2
res = urllib2.urlopen(req)
data=res.read()
except urllib2.HTTPError,e:
print e.code
if json:
return self._tojson(data)
else:
return data


def _tojson(self, data):
try:
js = json.loads(data, 'utf-8')
except:
js = None

return js
def _save(self, c):
_data = '%f\t%f\t%s\t%s\t%s\t%s\t%s\n' % (c['lat'],c['lng'],c['uid'],c['name'],c['address'],c['type'],c['tag'])
self.file.write(_data)

if __name__ == '__main__':
reload(sys)
sys.setdefaultencoding('utf-8')
print u"請輸入搜尋關鍵字："
keyword=raw_input()
print u"請輸入最大請求次數："
maxRequire=raw_input()
print u"請輸入矩形框，以逗號分開(113.6,115.1,29.9,31.4)："
bounds=raw_input()
bounds=bounds.split(',')
baiduSpider=BaiduSpider(None,bounds,keyword,maxRequire)
baiduSpider._start()
print u'---返回的結果數-->'+str(baiduSpider.resultCount)
print u'---請求次數-->'+str(baiduSpider.requestTimes)
print u'---請求的最大頁面數(不能超過76)'+str(baiduSpider.maxPageNum)

百度地圖POI抓取——python

相關文章