Python中多线程发送HTTP请求的三种方案对比

import threading
import queue
import requests
import time

urls = [f"http://httpbin.org/delay/1?id={i}" for i in range(20)]
result_queue = queue.Queue()

def worker(q):
    while not q.empty():
        url = q.get()
        try:
            resp = requests.get(url, timeout=5)
            result_queue.put((url, resp.status_code))
        except Exception as e:
            result_queue.put((url, str(e)))
        finally:
            q.task_done()

# 启动 5 个线程
threads = []
for _ in range(5):
    t = threading.Thread(target=worker, args=(queue.Queue(),))
    t.start()
    threads.append(t)

# 填入任务
task_queue = queue.Queue()
for url in urls:
    task_queue.put(url)

# 重新分配任务给 worker（简化写法，实际应把 task_queue 传进去）
for t in threads:
    t.join()

while not result_queue.empty():
    print(result_queue.get())

import requests
from concurrent.futures import ThreadPoolExecutor, as_completed
import time

urls = [f"http://httpbin.org/delay/1?id={i}" for i in range(20)]

def fetch(url):
    try:
        resp = requests.get(url, timeout=5)
        return url, resp.status_code
    except Exception as e:
        return url, str(e)

t1 = time.time()

with ThreadPoolExecutor(max_workers=5) as executor:
    futures = {executor.submit(fetch, url): url for url in urls}
    
    for future in as_completed(futures):
        url, result = future.result()
        print(f"{url} -> {result}")

print(f"耗时：{time.time() - t1:.2f}s")

import asyncio
import aiohttp
import time

urls = [f"http://httpbin.org/delay/1?id={i}" for i in range(20)]

async def fetch(session, url):
    try:
        async with session.get(url, timeout=5) as resp:
            return url, resp.status
    except Exception as e:
        return url, str(e)

async def main():
    async with aiohttp.ClientSession() as session:
        tasks = [fetch(session, url) for url in urls]
        results = await asyncio.gather(*tasks)
        for url, result in results:
            print(f"{url} -> {result}")

t1 = time.time()
asyncio.run(main())
print(f"耗时：{time.time() - t1:.2f}s")

workers = 100  # ❌ 大概率更慢，还可能被封 IP
workers = 10   # ✅ 大部分场景够用

requests.get(url)  # ❌ 对方不响应，你的线程就永远挂着
requests.get(url, timeout=5)  # ✅ 5秒没响应就放弃

session = requests.Session()  # ❌ 多线程共享同一个 session 会出问题

# 正确做法：每个线程自己创建 session
def fetch(url):
    with requests.Session() as s:  # ✅
        return s.get(url).text