基于Python实现IP代理池

import requests
from bs4 import BeautifulSoup

def get_proxies():
    url = 'https://www.xicidaili.com/nn/'
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/58.0.3029.110 Safari/537.3'}
    response = requests.get(url, headers=headers)
    soup = BeautifulSoup(response.text, 'html.parser')
    proxies = []
    for row in soup.find_all('tr')[1:]:
        tds = row.find_all('td')
        ip = tds[1].text
        port = tds[2].text
        protocol = tds[5].text.lower()
        if protocol == 'http' or protocol == 'https':
            proxies.append(f'{protocol}://{ip}:{port}')
    return proxies

print(get_proxies())

def check_proxy(proxy):
    try:
        response = requests.get('https://httpbin.org/ip', proxies={'http': proxy, 'https': proxy}, timeout=5)
        if response.status_code == 200:
            return True
    except:
        return False
    return False

# 示例：验证代理IP
proxies = get_proxies()
valid_proxies = [proxy for proxy in proxies if check_proxy(proxy)]
print(valid_proxies)

import random

class ProxyPool:
    def __init__(self):
        self.proxies = []
        self.update_proxies()

    def update_proxies(self):
        self.proxies = [proxy for proxy in get_proxies() if check_proxy(proxy)]
        print(f'Updated proxies: {self.proxies}')

    def get_proxy(self):
        if not self.proxies:
            self.update_proxies()
        return random.choice(self.proxies)

# 示例：使用代理池
proxy_pool = ProxyPool()
for _ in range(5):
    proxy = proxy_pool.get_proxy()
    print(f'Using proxy: {proxy}')

import requests
from bs4 import BeautifulSoup
from proxy_pool import ProxyPool  # 假设我们已经定义了ProxyPool类

# 初始化代理池
proxy_pool = ProxyPool()

def fetch_with_proxy(url):
    # 从代理池中获取一个代理
    proxy = proxy_pool.get_proxy()
    print(f'Using proxy: {proxy}')
    
    # 设置代理
    proxies = {
        'http': proxy,
        'https': proxy
    }
    
    try:
        # 使用代理发送请求
        response = requests.get(url, proxies=proxies, timeout=10)
        response.raise_for_status()  # 如果请求返回了一个错误状态码，抛出异常
        return response.text
    except requests.RequestException as e:
        print(f'Request failed: {e}')
        return None

# 要请求的网页
url = 'https://www.example.com'

# 使用代理池发送请求
html_content = fetch_with_proxy(url)

# 解析网页内容
if html_content:
    soup = BeautifulSoup(html_content, 'html.parser')
    title = soup.title.string if soup.title else 'No title found'
    print(f'Title of the page: {title}')