Python使用asyncio+aiohttp实现高效异步并发下载

pip install aiohttp

pip install aiofiles

import asyncio
import aiohttp
import os

async def download_file(session, url, filename):
    """异步下载单个文件"""
    try:
        async with session.get(url) as response:
            if response.status == 200:
                content = await response.read()
                with open(filename, 'wb') as f:
                    f.write(content)
                print(f"✅ 下载成功: {filename}")
            else:
                print(f"❌ 下载失败 ({response.status}): {url}")
    except Exception as e:
        print(f"⚠️  下载出错: {url} | 错误: {e}")

async def download_all(urls, folder="downloads"):
    """并发下载多个文件"""
    os.makedirs(folder, exist_ok=True)

    # 创建 aiohttp 客户端会话（支持连接复用）
    async with aiohttp.ClientSession() as session:
        tasks = []
        for i, url in enumerate(urls):
            # 生成文件名（可根据需要自定义）
            ext = os.path.splitext(url.split('?')[0])[-1] or '.bin'
            filename = os.path.join(folder, f"file_{i+1}{ext}")
            task = asyncio.create_task(download_file(session, url, filename))
            tasks.append(task)

        # 并发执行所有下载任务
        await asyncio.gather(*tasks)

# 使用示例
if __name__ == "__main__":
    urls = [
        "https://example.com/file1.zip",
        "https://example.com/file2.jpg",
        "https://example.com/file3.pdf",
    ]

    print("🚀 开始异步下载...")
    asyncio.run(download_all(urls))
    print("🎉 所有文件下载完成！")

import aiofiles  # 需要额外安装

async def download_file_stream(session, url, filename):
    try:
        async with session.get(url) as response:
            if response.status == 200:
                async with aiofiles.open(filename, 'wb') as f:
                    async for chunk in response.content.iter_chunked(8192):
                        await f.write(chunk)
                print(f"✅ 流式下载成功: {filename}")
            else:
                print(f"❌ 下载失败 ({response.status}): {url}")
    except Exception as e:
        print(f"⚠️  流式下载出错: {url} | 错误: {e}")

# 在 download_all 中替换 download_file 为 download_file_stream 即可

semaphore = asyncio.Semaphore(10)  # 最多 10 个并发

async def download_file_limited(session, url, filename):
    async with semaphore:
        await download_file(session, url, filename)

timeout = aiohttp.ClientTimeout(total=30)
async with aiohttp.ClientSession(
    timeout=timeout,
    headers={"User-Agent": "Mozilla/5.0 (Python aiohttp)"}
) as session:

from urllib.parse import urlparse
import os

def get_filename_from_url(url):
    path = urlparse(url).path
    name = os.path.basename(path)
    return name if name else "downloaded_file"