關於非同步爬蟲排序的困惑

lingyuncelia發表於2020-12-26

在這裡插入圖片描述

import asyncio
from asyncio import tasks
import aiohttp #pip install aiohttp
from lxml import etree
for x in range(498232,498242):
    async def fetch(session,url):
        async with session.get(url) as response:
            html=await response.text()
            return html
    async def parser_content(html):
        selector = etree.HTML(html)
        title=str(selector.xpath("//div[@class='read_title']//h1[1]/text()")[0])
        print(x,title)
    async def download_content(url):
        async with aiohttp.ClientSession() as session:
            html=await fetch(session,url)        
            await parser_content(html)
    tasks=[
        asyncio.ensure_future(download_content('https://www.xinshuhaige.com/34953/{}.html'.format(x)))
    ]
loop=asyncio.get_event_loop()
loop.run_until_complete(asyncio.gather(*tasks))

相關文章