Python爬虫HTTP代理使用教程详解

import requests
 
proxies = {
    "http": "http://123.123.123.123:8080",
    "https": "http://123.123.123.123:8080"
}
 
response = requests.get("https://example.com", proxies=proxies)
print(response.text)

# settings.py
DOWNLOADER_MIDDLEWARES = {
    'scrapy.downloadermiddlewares.httpproxy.HttpProxyMiddleware': 110,
    'myproject.middlewares.ProxyMiddleware': 100,
}
 
# middlewares.py
class ProxyMiddleware:
    def process_request(self, request, spider):
        request.meta['proxy'] = 'http://123.123.123.123:8080'

def check_proxy(proxy):
    try:
        response = requests.get("http://httpbin.org/ip", proxies={"http": proxy}, timeout=5)
        return response.status_code == 200
    except:
        return False

proxy_pool = [
    "http://ip1:port",
    "http://ip2:port",
    "http://ip3:port"
]
 
current_proxy = random.choice(proxy_pool)

def retry(max_retries=3):
    def decorator(func):
        @functools.wraps(func)
        def wrapper(*args, **kwargs):
            for _ in range(max_retries):
                try:
                    return func(*args, **kwargs)
                except requests.exceptions.ProxyError:
                    continue
            return None
        return wrapper
    return decorator

headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36",
    "Referer": "https://www.zdaye.com/"
}

import time
import random
 
time.sleep(random.uniform(1, 3))  # 随机等待1-3秒

session = requests.Session()
response = session.get(url, proxies=proxies)
# 后续请求自动携带cookie

from concurrent.futures import ThreadPoolExecutor
 
with ThreadPoolExecutor(max_workers=10) as executor:
    valid_proxies = list(executor.map(check_proxy, proxy_list))

import redis
 
r = redis.Redis(host='localhost', port=6379, db=0)
r.set("valid_proxy", current_proxy, ex=300)  # 缓存5分钟

def get_best_proxy(target_url):
    # 根据目标网站地域选择同省份代理
    # 优先使用最近验证成功的代理
    pass