从入门到精通详解Python中User-Agent的使用终极指南

import requests

# 基础设置
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36'
}
response = requests.get('https://httpbin.org/user-agent', headers=headers)
print(response.json())

from fake_useragent import UserAgent

ua = UserAgent()
print(ua.chrome)  # 随机Chrome User-Agent
print(ua.firefox) # 随机Firefox User-Agent
print(ua.random)  # 完全随机

# Python2 的痛苦经历
import urllib2
headers = {'User-Agent': u'中文User-Agent'.encode('utf-8')}  # 必须手动编码
response = urllib2.urlopen(req)

# Python3 的优雅解决方案
import urllib.request
headers = {'User-Agent': '中文User-Agent'}  # 直接使用字符串，无需编码
response = urllib.request.urlopen(req)

import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry

# 配置会话，设置User-Agent和重试策略
session = requests.Session()
session.headers.update({
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36',
    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    'Accept-Language': 'en-US,en;q=0.5',
    'Accept-Encoding': 'gzip, deflate',
    'Connection': 'keep-alive',
})

# 配置重试策略
retry_strategy = Retry(
    total=3,
    backoff_factor=1,
    status_forcelist=[429, 500, 502, 503, 504],
)
session.mount('https://', HTTPAdapter(max_retries=retry_strategy))

# 发送请求
response = session.get('https://api.example.com/data')

import aiohttp
import asyncio
from fake_useragent import UserAgent

async def fetch_url(session, url):
    """异步获取页面内容"""
    headers = {
        'User-Agent': UserAgent().random,
        'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
    }
    
    async with session.get(url, headers=headers, timeout=10) as response:
        return await response.text()

async def main():
    urls = ['https://httpbin.org/user-agent'] * 10
    
    async with aiohttp.ClientSession() as session:
        tasks = [fetch_url(session, url) for url in urls]
        results = await asyncio.gather(*tasks)
        
        for i, result in enumerate(results):
            print(f"请求 {i+1}: {result}")

# 运行异步任务
if __name__ == '__main__':
    asyncio.run(main())

import random
import json
from typing import List, Dict, Optional
from dataclasses import dataclass
from fake_useragent import UserAgent

@dataclass
class DeviceProfile:
    """设备配置对象"""
    browser: str
    os: str
    user_agent: str
    weight: int = 1  # 权重，用于控制随机选择概率

class UserAgentManager:
    """
    User-Agent管理器，提供统一的User-Agent管理接口
    """
    
    def __init__(self, use_fallback: bool = True):
        self.ua = UserAgent(use_cache_server=use_fallback)
        self.custom_agents: List[DeviceProfile] = []
        self._load_default_profiles()
    
    def _load_default_profiles(self):
        """加载默认的高质量User-Agent"""
        default_profiles = [
            DeviceProfile("Chrome", "Windows", self.ua.chrome, 3),
            DeviceProfile("Firefox", "Windows", self.ua.firefox, 2),
            DeviceProfile("Safari", "macOS", self.ua.safari, 2),
            DeviceProfile("Edge", "Windows", self.ua.edge, 1),
        ]
        self.custom_agents.extend(default_profiles)
    
    def add_custom_agent(self, profile: DeviceProfile):
        """添加自定义User-Agent"""
        self.custom_agents.append(profile)
    
    def get_random_agent(self, browser: Optional[str] = None) -> str:
        """
        获取随机User-Agent
        
        Args:
            browser: 指定浏览器类型，如'chrome', 'firefox'，None表示随机
            
        Returns:
            User-Agent字符串
        """
        if browser:
            filtered = [p for p in self.custom_agents if p.browser.lower() == browser.lower()]
            if not filtered:
                raise ValueError(f"未找到指定浏览器: {browser}")
            return random.choice(filtered).user_agent
        
        # 基于权重的随机选择
        total_weight = sum(p.weight for p in self.custom_agents)
        pick = random.uniform(0, total_weight)
        
        current = 0
        for profile in self.custom_agents:
            current += profile.weight
            if pick <= current:
                return profile.user_agent
        
        return self.custom_agents[0].user_agent
    
    def get_device_profile(self, user_agent: str) -> Optional[DeviceProfile]:
        """从User-Agent解析设备信息"""
        for profile in self.custom_agents:
            if profile.user_agent == user_agent:
                return profile
        return None
    
    def export_profiles(self, filepath: str):
        """导出配置到JSON文件"""
        data = {
            "profiles": [
                {
                    "browser": p.browser,
                    "os": p.os,
                    "user_agent": p.user_agent,
                    "weight": p.weight
                }
                for p in self.custom_agents
            ]
        }
        with open(filepath, 'w', encoding='utf-8') as f:
            json.dump(data, f, ensure_ascii=False, indent=2)
    
    def import_profiles(self, filepath: str):
        """从JSON文件导入配置"""
        with open(filepath, 'r', encoding='utf-8') as f:
            data = json.load(f)
        
        self.custom_agents = [
            DeviceProfile(**profile) for profile in data["profiles"]
        ]

# 使用示例
if __name__ == '__main__':
    # 创建管理器
    manager = UserAgentManager()
    
    # 添加自定义配置
    manager.add_custom_agent(
        DeviceProfile("Chrome", "Android", 
                     "Mozilla/5.0 (Linux; Android 10; SM-G975F) AppleWebKit/537.36", 
                     weight=5)
    )
    
    # 获取随机User-Agent
    print("随机Chrome:", manager.get_random_agent("chrome"))
    print("完全随机:", manager.get_random_agent())
    
    # 导出配置
    manager.export_profiles("user_agents.json")

class SmartUserAgentManager(UserAgentManager):
    """智能User-Agent管理器"""
    
    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self.usage_count = {}  # 记录每个User-Agent的使用次数
    
    def get_next_agent(self) -> str:
        """轮换算法：优先选择使用次数最少的"""
        min_usage = min(self.usage_count.values(), default=0)
        candidates = [
            p for p in self.custom_agents 
            if self.usage_count.get(p.user_agent, 0) == min_usage
        ]
        
        selected = random.choice(candidates) if candidates else self.custom_agents[0]
        self.usage_count[selected.user_agent] = self.usage_count.get(selected.user_agent, 0) + 1
        
        return selected.user_agent

class SessionWithUA:
    """集成了User-Agent管理的请求会话"""
    
    def __init__(self, manager: UserAgentManager):
        self.manager = manager
        self.session = requests.Session()
        self._update_headers()
    
    def _update_headers(self):
        """更新会话头部"""
        self.session.headers.update({
            'User-Agent': self.manager.get_random_agent(),
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
            'Accept-Language': 'en-US,en;q=0.5',
        })
    
    def get(self, url: str, **kwargs):
        """发送GET请求"""
        # 每10次请求轮换一次User-Agent
        if not hasattr(self, '_request_count'):
            self._request_count = 0
        
        if self._request_count % 10 == 0:
            self._update_headers()
        
        self._request_count += 1
        return self.session.get(url, **kwargs)
    
    def post(self, url: str, **kwargs):
        """发送POST请求"""
        return self.session.post(url, **kwargs)