Python爬虫动态IP代理使用及防止被封的方法

import requests
from bs4 import BeautifulSoup

def get_free_proxies():
    url = 'https://www.xicidaili.com/'
    headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'}
    
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    
    proxies = []
    for row in soup.select('#ip_list tr')[1:]:
        tds = row.select('td')
        ip = tds[1].text
        port = tds[2].text
        protocol = tds[5].text.lower()
        proxies.append({protocol: f"{protocol}://{ip}:{port}"})
        
    return proxies

# Example usage
proxies = get_free_proxies()
print(proxies)

import requests

# 切换代理IP
def get_proxy():
    # 从代理池中获取代理IP
    # 在这里实现获取代理IP的逻辑，可以从付费代理服务商或免费代理网站获取
    proxy = 'http://your_proxy_ip:your_proxy_port'
    return {'http': proxy, 'https': proxy}

# 爬虫请求
def crawl_page(url):
    proxy = get_proxy()
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }

    try:
        response = requests.get(url, headers=headers, proxies=proxy, timeout=5)
        # 处理爬取到的页面
        # ...
        return response.text
    except requests.exceptions.RequestException as e:
       

 print(f"Error: {e}")
        return None

# Example usage
url = 'http://example.com'
html_content = crawl_page(url)
if html_content:
    print(html_content)

import random

def get_random_user_agent():
    user_agents = [
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36',
        'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/91.0.864.54 Safari/537.36',
        # 添加更多的用户代理
    ]
    return random.choice(user_agents)

# 在请求头中添加随机用户代理
headers = {
    'User-Agent': get_random_user_agent()
}

import time

# 设置请求间隔时间
def crawl_with_interval(url, interval=2):
    html_content = crawl_page(url)
    time.sleep(interval)
    return html_content

# Example usage
url = 'http://example.com'
html_content = crawl_with_interval(url)
if html_content:
    print(html_content)