python爬虫之aiohttp多任务异步爬虫
python爬虫之aiohttp多任务异步爬虫
爬取的flash服务如下：
from flask import Flask
import timeapp = Flask(__name__)@app.route('/bobo')
def index_bobo():time.sleep(2)return 'Hello bobo'@app.route('/jay')
def index_jay():time.sleep(2)return 'Hello jay'@app.route('/tom')
def index_tom():time.sleep(2)return 'Hello tom'if __name__ == '__main__':app.run(threaded=True)
运行启动flask服务后，多任务爬取代码如下：
#环境安装：pip install aiohttp
#使用该模块中的ClientSession
import requests
import asyncio
import time
import aiohttpstart = time.time()
urls = ['http://127.0.0.1:5000/bobo','http://127.0.0.1:5000/jay','http://127.0.0.1:5000/tom'
]async def get_page(url):async with aiohttp.ClientSession() as session:#get()、post():#headers,params/data,proxy='http://ip:port'async with await session.get(url) as response:#text()返回字符串形式的响应数据#read()返回二进制形式的响应数据#json()返回的就是json对象#注意：获取响应数据操作之前一定要使用await进行手动挂起page_text = await response.text()print(page_text)tasks = []for url in urls:c = get_page(url)task = asyncio.ensure_future(c)tasks.append(task)loop = asyncio.get_event_loop()
loop.run_until_complete(asyncio.wait(tasks))end = time.time()print('总耗时：',end-start)