异步爬虫测试

#!/usr/bin/env python
# -*- coding:utf-8 -*-
import aiofiles
import aiohttp
import asyncio
async def fetch(session, url):
    print("发送请求：", url)
    #异步发送请求，aiohttp.ClientSession().get(url)相当于requests.get(url)
    async with session.get(url, ssl=False) as response:
        #异步下载，response.content.read()相当于requests.content
        content = await response.content.read()
        file_name = url.rsplit('_')[-1]
        # 异步写入文件（下载），aiofiles.open相当于open
        async with aiofiles.open(file_name, mode='wb') as file_object:
            await file_object.write(content)
async def main():
    #异步发送请求，aiohttp.ClientSession()相当于requests
    async with aiohttp.ClientSession() as session:
        #需下载的url列表
        url_list = [
            'https://www3.autoimg.cn/newsdfs/g26/M02/35/A9/120x90_0_autohomecar__ChsEe12AXQ6AOOH_AAFocMs8nzU621.jpg',
            'https://www2.autoimg.cn/newsdfs/g30/M01/3C/E2/120x90_0_autohomecar__ChcCSV2BBICAUntfAADjJFd6800429.jpg',
            'https://www3.autoimg.cn/newsdfs/g26/M0B/3C/65/120x90_0_autohomecar__ChcCP12BFCmAIO83AAGq7vK0sGY193.jpg'
        ]
        #循环生成任务tasks
        tasks = [asyncio.create_task(fetch(session, url)) for url in url_list]
        await asyncio.wait(tasks)
if __name__ == '__main__':
    asyncio.run(main())




此文参考武沛齐老师的文章https://www.cnblogs.com/wupeiqi/p/12834355.html

标签：__,异步,url,爬虫,content,session,测试,async
来源： https://www.cnblogs.com/meng-xiaoyi/p/16521046.html